@contractspec/lib.voice 0.2.0

package/dist/conversational/response-orchestrator.js

@@ -0,0 +1,63 @@
+// @bun
+// src/conversational/response-orchestrator.ts
+class ResponseOrchestrator {
+  stt;
+  llm;
+  tts;
+  conversationHistory = [];
+  constructor(stt, llm, tts) {
+    this.stt = stt;
+    this.llm = llm;
+    this.tts = tts;
+  }
+  async* processUserTurn(userAudio, config) {
+    const transcription = await this.stt.transcribe({
+      audio: userAudio,
+      language: config.language,
+      wordTimestamps: false
+    });
+    const userText = transcription.text;
+    yield { type: "user_speech_ended", transcript: userText };
+    yield {
+      type: "transcript",
+      role: "user",
+      text: userText,
+      timestamp: Date.now()
+    };
+    this.conversationHistory.push({ role: "user", content: userText });
+    const llmResponse = await this.llm.chat([
+      {
+        role: "system",
+        content: [{ type: "text", text: config.systemPrompt }]
+      },
+      ...this.conversationHistory.map((msg) => ({
+        role: msg.role,
+        content: [{ type: "text", text: msg.content }]
+      }))
+    ], { model: config.llmModel });
+    const responseText = llmResponse.message.content.find((p) => p.type === "text");
+    const agentText = responseText && responseText.type === "text" ? responseText.text : "I apologize, I could not generate a response.";
+    this.conversationHistory.push({ role: "assistant", content: agentText });
+    yield { type: "agent_speech_started", text: agentText };
+    const synthesis = await this.tts.synthesize({
+      text: agentText,
+      voiceId: config.voiceId,
+      language: config.language,
+      format: config.outputFormat
+    });
+    yield { type: "agent_audio", audio: synthesis.audio.data };
+    yield { type: "agent_speech_ended" };
+    yield {
+      type: "transcript",
+      role: "agent",
+      text: agentText,
+      timestamp: Date.now()
+    };
+  }
+  reset() {
+    this.conversationHistory.length = 0;
+  }
+}
+export {
+  ResponseOrchestrator
+};

package/dist/conversational/transcript-builder.d.ts

@@ -0,0 +1,25 @@
+import type { ConversationalEvent } from '../types';
+import type { ConversationTurn } from './types';
+/**
+ * Build a conversation transcript in real-time from event stream.
+ *
+ * Accumulates turns as events arrive and provides access to the
+ * full transcript at any point during the conversation.
+ */
+export declare class TranscriptBuilder {
+    private readonly turns;
+    private currentTurn;
+    private sessionStartMs;
+    /** Get the current transcript */
+    getTranscript(): ConversationTurn[];
+    /** Get the full transcript as plain text */
+    toText(): string;
+    /** Get the total number of turns */
+    getTurnCount(): number;
+    /**
+     * Process a conversational event and update the transcript.
+     */
+    processEvent(event: ConversationalEvent): void;
+    /** Reset the transcript */
+    reset(): void;
+}

package/dist/conversational/transcript-builder.js

@@ -0,0 +1,64 @@
+// @bun
+// src/conversational/transcript-builder.ts
+class TranscriptBuilder {
+  turns = [];
+  currentTurn = null;
+  sessionStartMs = Date.now();
+  getTranscript() {
+    return [...this.turns];
+  }
+  toText() {
+    return this.turns.map((t) => `[${t.role}] ${t.text}`).join(`
+`);
+  }
+  getTurnCount() {
+    return this.turns.length;
+  }
+  processEvent(event) {
+    switch (event.type) {
+      case "session_started":
+        this.sessionStartMs = Date.now();
+        break;
+      case "user_speech_started":
+        this.currentTurn = {
+          role: "user",
+          startMs: Date.now() - this.sessionStartMs
+        };
+        break;
+      case "user_speech_ended":
+        if (this.currentTurn && this.currentTurn.role === "user") {
+          this.currentTurn.text = event.transcript;
+          this.currentTurn.endMs = Date.now() - this.sessionStartMs;
+          this.turns.push(this.currentTurn);
+          this.currentTurn = null;
+        }
+        break;
+      case "agent_speech_started":
+        this.currentTurn = {
+          role: "agent",
+          text: event.text,
+          startMs: Date.now() - this.sessionStartMs
+        };
+        break;
+      case "agent_speech_ended":
+        if (this.currentTurn && this.currentTurn.role === "agent") {
+          this.currentTurn.endMs = Date.now() - this.sessionStartMs;
+          this.turns.push(this.currentTurn);
+          this.currentTurn = null;
+        }
+        break;
+      case "transcript":
+        break;
+      default:
+        break;
+    }
+  }
+  reset() {
+    this.turns.length = 0;
+    this.currentTurn = null;
+    this.sessionStartMs = Date.now();
+  }
+}
+export {
+  TranscriptBuilder
+};

package/dist/conversational/turn-detector.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Voice Activity Detection (VAD) and silence detection for turn-taking.
+ *
+ * Determines when the user has finished speaking based on audio energy
+ * and silence duration thresholds.
+ */
+export declare class TurnDetector {
+    private readonly silenceThresholdMs;
+    private readonly energyThreshold;
+    private silenceStartMs;
+    /**
+     * @param silenceThresholdMs - Silence duration to trigger end of turn (default 800ms)
+     * @param energyThreshold - Minimum audio energy to consider as speech (default 0.01)
+     */
+    constructor(silenceThresholdMs?: number, energyThreshold?: number);
+    /**
+     * Process an audio chunk and determine if it contains speech.
+     *
+     * @param chunk - Raw audio data
+     * @param timestampMs - Current timestamp in ms
+     * @returns Whether end-of-turn was detected
+     */
+    processChunk(chunk: Uint8Array, timestampMs: number): boolean;
+    /** Reset the detector state */
+    reset(): void;
+    /**
+     * Calculate RMS energy of an audio chunk.
+     * Assumes 16-bit PCM audio.
+     */
+    private calculateEnergy;
+}

package/dist/conversational/turn-detector.js

@@ -0,0 +1,44 @@
+// @bun
+// src/conversational/turn-detector.ts
+class TurnDetector {
+  silenceThresholdMs;
+  energyThreshold;
+  silenceStartMs = null;
+  constructor(silenceThresholdMs = 800, energyThreshold = 0.01) {
+    this.silenceThresholdMs = silenceThresholdMs;
+    this.energyThreshold = energyThreshold;
+  }
+  processChunk(chunk, timestampMs) {
+    const energy = this.calculateEnergy(chunk);
+    const isSpeech = energy > this.energyThreshold;
+    if (isSpeech) {
+      this.silenceStartMs = null;
+      return false;
+    }
+    if (this.silenceStartMs === null) {
+      this.silenceStartMs = timestampMs;
+    }
+    const silenceDurationMs = timestampMs - this.silenceStartMs;
+    return silenceDurationMs >= this.silenceThresholdMs;
+  }
+  reset() {
+    this.silenceStartMs = null;
+  }
+  calculateEnergy(chunk) {
+    if (chunk.length < 2)
+      return 0;
+    let sum = 0;
+    const sampleCount = Math.floor(chunk.length / 2);
+    for (let i = 0;i < chunk.length - 1; i += 2) {
+      const low = chunk[i] ?? 0;
+      const high = chunk[i + 1] ?? 0;
+      const sample = (low | high << 8) << 16 >> 16;
+      const normalized = sample / 32768;
+      sum += normalized * normalized;
+    }
+    return Math.sqrt(sum / sampleCount);
+  }
+}
+export {
+  TurnDetector
+};

package/dist/conversational/types.d.ts

@@ -0,0 +1,55 @@
+import type { ConversationalProvider, ConversationalEvent, ConversationalSessionSummary, AudioFormat, STTProvider, TTSProvider } from '../types';
+import type { LLMProvider } from '@contractspec/lib.contracts-integrations/integrations/providers/llm';
+export interface ConversationConfig {
+    voiceId: string;
+    language?: string;
+    systemPrompt: string;
+    llmModel?: string;
+    inputFormat?: AudioFormat;
+    outputFormat?: AudioFormat;
+    turnDetection?: 'server_vad' | 'push_to_talk';
+    silenceThresholdMs?: number;
+    maxDurationSeconds?: number;
+    /** Tools the agent can invoke mid-conversation */
+    tools?: ConversationalTool[];
+}
+export interface ConversationalTool {
+    name: string;
+    description: string;
+    inputSchema: Record<string, unknown>;
+    handler: (args: Record<string, unknown>) => Promise<string>;
+}
+export interface ConversationState {
+    sessionId: string;
+    status: 'connecting' | 'active' | 'paused' | 'ended';
+    currentTurn: 'user' | 'agent' | 'idle';
+    turnCount: number;
+    durationMs: number;
+    transcript: ConversationTurn[];
+}
+export interface ConversationTurn {
+    role: 'user' | 'agent';
+    text: string;
+    startMs: number;
+    endMs: number;
+    toolCalls?: {
+        name: string;
+        result: string;
+    }[];
+}
+export interface ConversationalOptions {
+    conversational: ConversationalProvider;
+    /** Optional fallback: use separate STT + LLM + TTS if provider doesn't support native conversational */
+    fallbackSTT?: STTProvider;
+    fallbackTTS?: TTSProvider;
+    fallbackLLM?: LLMProvider;
+}
+export interface ManagedSession {
+    state: ConversationState;
+    sendAudio(chunk: Uint8Array): void;
+    sendText(text: string): void;
+    interrupt(): void;
+    close(): Promise<ConversationalSessionSummary>;
+    events: AsyncIterable<ConversationalEvent>;
+}
+export type { ConversationalEvent, ConversationalSessionSummary };

package/dist/conversational/types.js

@@ -0,0 +1 @@
+// @bun

package/dist/conversational/voice-session-manager.d.ts

@@ -0,0 +1,17 @@
+import type { ConversationConfig, ConversationalOptions, ManagedSession } from './types';
+/**
+ * Manage voice conversation sessions.
+ *
+ * Two strategies:
+ * 1. Native: Delegate to ConversationalProvider if it supports full bidirectional.
+ * 2. Composed: Chain STT + LLM + TTS via ResponseOrchestrator (fallback).
+ */
+export declare class VoiceSessionManager {
+    private readonly provider;
+    constructor(options: ConversationalOptions);
+    /**
+     * Start a new voice conversation session.
+     */
+    startSession(config: ConversationConfig): Promise<ManagedSession>;
+    private wrapEvents;
+}

package/dist/conversational/voice-session-manager.js

@@ -0,0 +1,138 @@
+// @bun
+// src/conversational/transcript-builder.ts
+class TranscriptBuilder {
+  turns = [];
+  currentTurn = null;
+  sessionStartMs = Date.now();
+  getTranscript() {
+    return [...this.turns];
+  }
+  toText() {
+    return this.turns.map((t) => `[${t.role}] ${t.text}`).join(`
+`);
+  }
+  getTurnCount() {
+    return this.turns.length;
+  }
+  processEvent(event) {
+    switch (event.type) {
+      case "session_started":
+        this.sessionStartMs = Date.now();
+        break;
+      case "user_speech_started":
+        this.currentTurn = {
+          role: "user",
+          startMs: Date.now() - this.sessionStartMs
+        };
+        break;
+      case "user_speech_ended":
+        if (this.currentTurn && this.currentTurn.role === "user") {
+          this.currentTurn.text = event.transcript;
+          this.currentTurn.endMs = Date.now() - this.sessionStartMs;
+          this.turns.push(this.currentTurn);
+          this.currentTurn = null;
+        }
+        break;
+      case "agent_speech_started":
+        this.currentTurn = {
+          role: "agent",
+          text: event.text,
+          startMs: Date.now() - this.sessionStartMs
+        };
+        break;
+      case "agent_speech_ended":
+        if (this.currentTurn && this.currentTurn.role === "agent") {
+          this.currentTurn.endMs = Date.now() - this.sessionStartMs;
+          this.turns.push(this.currentTurn);
+          this.currentTurn = null;
+        }
+        break;
+      case "transcript":
+        break;
+      default:
+        break;
+    }
+  }
+  reset() {
+    this.turns.length = 0;
+    this.currentTurn = null;
+    this.sessionStartMs = Date.now();
+  }
+}
+
+// src/conversational/voice-session-manager.ts
+class VoiceSessionManager {
+  provider;
+  constructor(options) {
+    this.provider = options.conversational;
+  }
+  async startSession(config) {
+    const transcriptBuilder = new TranscriptBuilder;
+    const session = await this.provider.startSession({
+      voiceId: config.voiceId,
+      language: config.language,
+      systemPrompt: config.systemPrompt,
+      llmModel: config.llmModel,
+      inputFormat: config.inputFormat,
+      outputFormat: config.outputFormat,
+      turnDetection: config.turnDetection,
+      silenceThresholdMs: config.silenceThresholdMs,
+      maxDurationSeconds: config.maxDurationSeconds
+    });
+    const state = {
+      sessionId: "",
+      status: "connecting",
+      currentTurn: "idle",
+      turnCount: 0,
+      durationMs: 0,
+      transcript: []
+    };
+    const wrappedEvents = this.wrapEvents(session.events, state, transcriptBuilder);
+    return {
+      state,
+      sendAudio: (chunk) => session.sendAudio(chunk),
+      sendText: (text) => session.sendText(text),
+      interrupt: () => session.interrupt(),
+      close: async () => {
+        const summary = await session.close();
+        state.status = "ended";
+        return summary;
+      },
+      events: wrappedEvents
+    };
+  }
+  async* wrapEvents(events, state, transcriptBuilder) {
+    for await (const event of events) {
+      transcriptBuilder.processEvent(event);
+      switch (event.type) {
+        case "session_started":
+          state.sessionId = event.sessionId;
+          state.status = "active";
+          break;
+        case "user_speech_started":
+          state.currentTurn = "user";
+          break;
+        case "user_speech_ended":
+          state.currentTurn = "idle";
+          state.turnCount += 1;
+          break;
+        case "agent_speech_started":
+          state.currentTurn = "agent";
+          break;
+        case "agent_speech_ended":
+          state.currentTurn = "idle";
+          state.turnCount += 1;
+          break;
+        case "session_ended":
+          state.status = "ended";
+          state.durationMs = event.durationMs;
+          break;
+      }
+      state.transcript = transcriptBuilder.getTranscript();
+      yield event;
+    }
+  }
+}
+export {
+  VoiceSessionManager
+};

package/dist/docs/conversational.docblock.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @docblock
+ * @title Conversational Sub-domain
+ * @domain voice.conversational
+ * @description
+ * Real-time bidirectional voice conversations.
+ *
+ * Supports two strategies:
+ * 1. Native: Delegate to ConversationalProvider
+ * 2. Composed: Chain STT + LLM + TTS via ResponseOrchestrator
+ *
+ * The VoiceSessionManager class manages session lifecycle.
+ */
+export declare const conversationalDocblock = true;

package/dist/docs/conversational.docblock.js

@@ -0,0 +1,6 @@
+// @bun
+// src/docs/conversational.docblock.ts
+var conversationalDocblock = true;
+export {
+  conversationalDocblock
+};

package/dist/docs/stt.docblock.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @docblock
+ * @title STT Sub-domain
+ * @domain voice.stt
+ * @description
+ * Speech-to-text transcription with diarization and subtitle generation.
+ *
+ * Pipeline: Audio -> SegmentSplitter -> STTProvider -> DiarizationMapper -> SubtitleFormatter -> TranscriptionProject
+ *
+ * The Transcriber class orchestrates the full pipeline.
+ */
+export declare const sttDocblock = true;

package/dist/docs/stt.docblock.js

@@ -0,0 +1,6 @@
+// @bun
+// src/docs/stt.docblock.ts
+var sttDocblock = true;
+export {
+  sttDocblock
+};

package/dist/docs/sync.docblock.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @docblock
+ * @title Sync Layer
+ * @domain voice.sync
+ * @description
+ * Video-gen integration layer for timing synchronization.
+ *
+ * - TimingCalculator: SynthesizedSegments -> VoiceTimingMap
+ * - SceneAdapter: video-gen ScenePlan -> TTSScript
+ * - DurationNegotiator: Balance voice vs scene durations
+ */
+export declare const syncDocblock = true;

package/dist/docs/sync.docblock.js

@@ -0,0 +1,6 @@
+// @bun
+// src/docs/sync.docblock.ts
+var syncDocblock = true;
+export {
+  syncDocblock
+};

package/dist/docs/tts.docblock.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @docblock
+ * @title TTS Sub-domain
+ * @domain voice.tts
+ * @description
+ * Text-to-speech pipeline with scene-aware narration.
+ *
+ * Pipeline: ContentBrief -> PaceAnalyzer -> EmphasisPlanner -> SegmentSynthesizer -> AudioAssembler -> TTSProject
+ *
+ * The VoiceSynthesizer class orchestrates the full pipeline.
+ */
+export declare const ttsDocblock = true;

package/dist/docs/tts.docblock.js

@@ -0,0 +1,6 @@
+// @bun
+// src/docs/tts.docblock.ts
+var ttsDocblock = true;
+export {
+  ttsDocblock
+};

package/dist/docs/voice.docblock.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @docblock
+ * @title Voice Library
+ * @domain voice
+ * @description
+ * Umbrella library for voice capabilities: TTS, STT, and Conversational AI.
+ *
+ * ## Sub-domains
+ * - **TTS**: Text-to-speech synthesis with pacing, emphasis, and video-gen integration
+ * - **STT**: Speech-to-text transcription with diarization and subtitle generation
+ * - **Conversational**: Real-time bidirectional voice conversations
+ * - **Sync**: Video-gen timing integration layer
+ *
+ * ## Import paths
+ * ```ts
+ * import { VoiceSynthesizer } from "@contractspec/lib.voice/tts";
+ * import { Transcriber } from "@contractspec/lib.voice/stt";
+ * import { VoiceSessionManager } from "@contractspec/lib.voice/conversational";
+ * import { TimingCalculator } from "@contractspec/lib.voice/sync";
+ * ```
+ */
+export declare const voiceDocblock = true;

package/dist/docs/voice.docblock.js

@@ -0,0 +1,6 @@
+// @bun
+// src/docs/voice.docblock.ts
+var voiceDocblock = true;
+export {
+  voiceDocblock
+};

package/dist/i18n/catalogs/en.d.ts

@@ -0,0 +1,6 @@
+/**
+ * English (en) translation catalog for @contractspec/lib.voice.
+ *
+ * This is the primary / reference locale. All message keys must be present here.
+ */
+export declare const enMessages: import("@contractspec/lib.contracts-spec/translations").TranslationSpec;

package/dist/i18n/catalogs/en.js

@@ -0,0 +1,92 @@
+// @bun
+// src/i18n/catalogs/en.ts
+import { defineTranslation } from "@contractspec/lib.contracts-spec/translations";
+var enMessages = defineTranslation({
+  meta: {
+    key: "voice.messages",
+    version: "1.0.0",
+    domain: "voice",
+    description: "All user-facing, LLM-facing, and developer-facing strings for the voice package",
+    owners: ["platform"],
+    stability: "experimental"
+  },
+  locale: "en",
+  fallback: "en",
+  messages: {
+    "prompt.tts.system": {
+      value: `You are a voice narration script writer.
+Analyze the content and produce a narration script with pacing directives.
+Return JSON with segments, each having sceneId, text, and contentType.`,
+      description: "TTS system prompt for LLM-enhanced script generation"
+    },
+    "prompt.pace.sceneMatched": {
+      value: "Match voice pacing to scene durations. Adjust rate and emphasis per segment to fit the video timeline.",
+      description: "Prompt for scene-matched pacing strategy"
+    },
+    "prompt.emphasis.system": {
+      value: "You are a voice director. For each segment, determine the optimal emphasis, tone, and speaking rate.",
+      description: "Emphasis planner LLM system prompt"
+    },
+    "pace.intro.description": {
+      value: "Authoritative opening at a measured pace",
+      description: "Description for intro pacing"
+    },
+    "pace.problem.description": {
+      value: "Urgent emphasis on the challenge",
+      description: "Description for problem pacing"
+    },
+    "pace.solution.description": {
+      value: "Calm, clear delivery of the solution",
+      description: "Description for solution pacing"
+    },
+    "pace.metric.description": {
+      value: "Excited emphasis on key results",
+      description: "Description for metric pacing"
+    },
+    "pace.cta.description": {
+      value: "Authoritative call to action",
+      description: "Description for CTA pacing"
+    },
+    "pace.transition.description": {
+      value: "Quick neutral transition",
+      description: "Description for transition pacing"
+    },
+    "stt.transcribing": {
+      value: "Transcribing audio...",
+      description: "Status message during transcription"
+    },
+    "stt.diarization.speaker": {
+      value: "Speaker {index}",
+      description: "Default speaker label",
+      placeholders: [{ name: "index", type: "number" }]
+    },
+    "stt.subtitle.timestamp": {
+      value: "{start} --> {end}",
+      description: "Subtitle timestamp format",
+      placeholders: [
+        { name: "start", type: "string" },
+        { name: "end", type: "string" }
+      ]
+    },
+    "conv.session.started": {
+      value: "Voice session started",
+      description: "Session start notification"
+    },
+    "conv.turn.user": {
+      value: "User is speaking",
+      description: "User turn indicator"
+    },
+    "conv.turn.agent": {
+      value: "Agent is responding",
+      description: "Agent turn indicator"
+    },
+    "conv.session.ended": {
+      value: "Voice session ended. Duration: {durationMs}ms",
+      description: "Session end notification",
+      placeholders: [{ name: "durationMs", type: "number" }]
+    }
+  }
+});
+export {
+  enMessages
+};

package/dist/i18n/catalogs/es.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Spanish (es) translation catalog for @contractspec/lib.voice.
+ */
+export declare const esMessages: import("@contractspec/lib.contracts-spec/translations").TranslationSpec;