@ai-ecoverse/pi-audio 1.12.0 → 1.14.0

package/README.md

@@ -23,6 +23,9 @@ pi-audio does not apply that patch. Hosts such as slicc-agent record monkeypatch
 
 `@ai-ecoverse/pi-audio/durable` appends a `voice.truncate` entry for a `VoiceCut`. The entry has no model messages. Its `edits` replace the target assistant message with the text heard so far and keep tool calls. The stored entry stays intact, and pi-durable derives the shorter message for the next model request. `@earendil-works/pi-durable` is an optional peer (`^1.1.0`); the root package does not import it. `VoiceCut.entryId` is the decimal string of pi-durable's numeric entry id.
 
+`@ai-ecoverse/pi-audio/elevenlabs` registers ElevenLabs speech adapters when `registerElevenLabs()` is called. `scribe_v2` posts multipart audio to `/speech-to-text`. `scribe_v2_realtime` streams base64 audio on `wss://…/speech-to-text/realtime` and commits turns with `turn_prediction` (`turnDetection: 'semantic'`). `eleven_flash_v2_5` and `eleven_multilingual_v2` post JSON to `/text-to-speech/{voice}/with-timestamps` for a string and stream text on `stream-input` for an async iterable, with character alignment. `eleven_v3` and `eleven_v4` stay on the HTTP path (`textStreaming: false`). Scribe prices are $0.22 per hour for batch and $0.39 per hour for realtime.
+
+`voiceTurns` captions include `agent_spoken`. `upTo` follows `AudioSink` playback: per character when the chunk has alignment, and evenly across the sentence when it does not. `captionsFromHistory(entries)` rebuilds user captions from entries whose `data.source` is `voice`, and agent captions from the assistant text. A `voice.truncate` entry adds `agent_spoken` and `agent_cut` at `heardUpTo`, so the rest of the original text stays available as unheard.
 `@ai-ecoverse/pi-audio/xai` registers xAI speech adapters when `registerXAI()` is called. `grok-voice-transcribe-2.0` posts multipart audio to `/stt` (fields before `file`) and streams raw binary frames on `wss://…/stt` after `transcript.created`, with Smart Turn (`turnDetection: 'semantic'`). The catalog id is shared, so `audioCost` uses the streaming rate of $0.20 per hour. `grok-voice-tts-1.0` posts JSON to `/tts` for a string and streams `text.delta` / `audio.delta` on `wss://…/tts` for an async iterable (`textStreaming: true`). Voices are eve, ara, rex, sal, and leo. Codecs are mp3, pcm, wav, mulaw, and alaw.
 
 `@ai-ecoverse/pi-audio/openai` registers OpenAI and OpenRouter speech adapters when `registerOpenAI()` is called. Batch transcription posts multipart audio to `/audio/transcriptions`. OpenRouter (`provider` `openrouter`, or a base URL on `openrouter.ai`) posts JSON `input_audio` instead. Speech posts JSON to `/audio/speech` and reads the raw audio response. OpenAI can return pcm, mp3, wav, opus, or flac. OpenRouter returns pcm or mp3. Realtime transcription uses a WebSocket at `/realtime?intent=transcription` for `gpt-realtime-whisper` and `gpt-live-transcribe`. The catalog key is type, provider, and id, so the batch model keeps the id `gpt-4o-transcribe`.

package/dist/captions.js

@@ -0,0 +1,150 @@
+function spokenOffset(clock, offsetMs) {
+  const length = clock.end - clock.start;
+  if (length <= 0) return clock.start;
+  const alignment = clock.alignment;
+  if (alignment && alignment.startMs.length === length && alignment.durationMs.length === length) {
+    return clock.start + alignedHeard(alignment, offsetMs, length);
+  }
+  if (clock.durationMs <= 0) {
+    if (offsetMs <= 0) return clock.start;
+    return clock.end;
+  }
+  const ratio = offsetMs <= 0 ? 0 : offsetMs / clock.durationMs;
+  const fraction = ratio > 1 ? 1 : ratio;
+  return clock.start + Math.round(fraction * length);
+}
+function alignedHeard(alignment, offsetMs, length) {
+  const first = alignment.startMs[0];
+  let origin = 0;
+  if (first !== void 0) origin = first;
+  const at = origin + offsetMs;
+  let heard = 0;
+  for (let index = 0; index < length; index += 1) {
+    const start = alignment.startMs[index];
+    if (start !== void 0 && start <= at) heard = index + 1;
+  }
+  return heard;
+}
+function utteranceIndex(feedPos, maps) {
+  let upTo = 0;
+  for (const map of maps) {
+    if (map.kind !== "answer") continue;
+    if (feedPos <= map.spokenStart) break;
+    const end = feedPos < map.spokenEnd ? feedPos : map.spokenEnd;
+    upTo += end - map.spokenStart;
+  }
+  return upTo;
+}
+function captionsFromHistory(entries) {
+  const captions = [];
+  const agents = /* @__PURE__ */ new Map();
+  for (const entry of entries) {
+    if (entry.kind === "voice.truncate") {
+      noteCut(captions, agents, entry.data);
+      continue;
+    }
+    const message = firstMessage(entry);
+    if (!message) continue;
+    const voice = voiceMeta(entry.data);
+    if (message.role === "user" && voice) {
+      pushUser(captions, entry, message, voice);
+      continue;
+    }
+    if (message.role !== "assistant") continue;
+    const blocks = textBlocks(message.content);
+    const turnId = String(entry.id);
+    const utteranceId = `${turnId}:agent`;
+    agents.set(turnId, { turnId, utteranceId, blocks });
+    captions.push({
+      type: "agent_generated",
+      turnId,
+      utteranceId,
+      text: blocks.join("")
+    });
+  }
+  return captions;
+}
+function firstMessage(entry) {
+  const model = entry.model;
+  if (!model) return void 0;
+  return model[0];
+}
+function pushUser(captions, entry, message, voice) {
+  const turnId = String(entry.id);
+  const text = textBlocks(message.content).join("");
+  if (voice.confidence === void 0) {
+    captions.push({ type: "user_final", turnId, segmentId: "turn", text });
+    return;
+  }
+  captions.push({
+    type: "user_final",
+    turnId,
+    segmentId: "turn",
+    text,
+    confidence: voice.confidence
+  });
+}
+function noteCut(captions, agents, data) {
+  const cut = cutData(data);
+  if (!cut) return;
+  const agent = agents.get(cut.entryId);
+  if (!agent) return;
+  const heard = heardInBlocks(agent.blocks, cut.contentIndex, cut.heardUpTo);
+  captions.push({
+    type: "agent_spoken",
+    turnId: agent.turnId,
+    utteranceId: agent.utteranceId,
+    upTo: heard
+  });
+  captions.push({
+    type: "agent_cut",
+    turnId: agent.turnId,
+    utteranceId: agent.utteranceId,
+    heardUpTo: heard
+  });
+}
+function heardInBlocks(blocks, contentIndex, heardUpTo) {
+  let prior = 0;
+  let index = 0;
+  for (const block of blocks) {
+    if (index === contentIndex) return prior + clampHeard(block, heardUpTo);
+    prior += block.length;
+    index += 1;
+  }
+  return prior;
+}
+function clampHeard(block, heardUpTo) {
+  if (heardUpTo <= 0) return 0;
+  if (heardUpTo >= block.length) return block.length;
+  return heardUpTo;
+}
+function textBlocks(content) {
+  if (typeof content === "string") return [content];
+  if (!content) return [];
+  const blocks = [];
+  for (const block of content) {
+    if (block.type === "text" && typeof block.text === "string") blocks.push(block.text);
+  }
+  return blocks;
+}
+function voiceMeta(data) {
+  if (typeof data !== "object" || data === null) return void 0;
+  if (!("source" in data) || data.source !== "voice") return void 0;
+  if ("confidence" in data && typeof data.confidence === "number") {
+    return { confidence: data.confidence };
+  }
+  return {};
+}
+function cutData(data) {
+  if (typeof data !== "object" || data === null) return void 0;
+  if (!("entryId" in data) || typeof data.entryId !== "string") return void 0;
+  if (!("contentIndex" in data) || typeof data.contentIndex !== "number") return void 0;
+  if (!("heardUpTo" in data) || typeof data.heardUpTo !== "number") return void 0;
+  return { entryId: data.entryId, contentIndex: data.contentIndex, heardUpTo: data.heardUpTo };
+}
+export {
+  captionsFromHistory,
+  spokenOffset,
+  utteranceIndex
+};
+//# sourceMappingURL=captions.js.map

package/dist/captions.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/captions.ts"],
+  "sourcesContent": ["import type { CharAlignment } from './audio.ts';\nimport type { CaptionEvent } from './turns.ts';\n\nexport interface SpokenClock {\n  start: number;\n  end: number;\n  durationMs: number;\n  alignment?: CharAlignment;\n}\n\nexport interface UtteranceMap {\n  spokenStart: number;\n  spokenEnd: number;\n  kind: string;\n}\n\nexport interface CaptionHistoryEntry {\n  id: string | number;\n  kind: string;\n  model?: readonly HistoryMessage[];\n  data?: unknown;\n}\n\ninterface HistoryMessage {\n  role?: string;\n  content?: string | readonly HistoryBlock[];\n}\n\ninterface HistoryBlock {\n  type?: string;\n  text?: string;\n}\n\ninterface StoredAgent {\n  turnId: string;\n  utteranceId: string;\n  blocks: string[];\n}\n\nexport function spokenOffset(clock: SpokenClock, offsetMs: number): number {\n  const length = clock.end - clock.start;\n  if (length <= 0) return clock.start;\n  const alignment = clock.alignment;\n  if (alignment && alignment.startMs.length === length && alignment.durationMs.length === length) {\n    return clock.start + alignedHeard(alignment, offsetMs, length);\n  }\n  if (clock.durationMs <= 0) {\n    if (offsetMs <= 0) return clock.start;\n    return clock.end;\n  }\n  const ratio = offsetMs <= 0 ? 0 : offsetMs / clock.durationMs;\n  const fraction = ratio > 1 ? 1 : ratio;\n  return clock.start + Math.round(fraction * length);\n}\n\nfunction alignedHeard(alignment: CharAlignment, offsetMs: number, length: number): number {\n  const first = alignment.startMs[0];\n  let origin = 0;\n  if (first !== undefined) origin = first;\n  const at = origin + offsetMs;\n  let heard = 0;\n  for (let index = 0; index < length; index += 1) {\n    const start = alignment.startMs[index];\n    if (start !== undefined && start <= at) heard = index + 1;\n  }\n  return heard;\n}\n\nexport function utteranceIndex(feedPos: number, maps: readonly UtteranceMap[]): number {\n  let upTo = 0;\n  for (const map of maps) {\n    if (map.kind !== 'answer') continue;\n    if (feedPos <= map.spokenStart) break;\n    const end = feedPos < map.spokenEnd ? feedPos : map.spokenEnd;\n    upTo += end - map.spokenStart;\n  }\n  return upTo;\n}\n\nexport function captionsFromHistory(entries: readonly CaptionHistoryEntry[]): CaptionEvent[] {\n  const captions: CaptionEvent[] = [];\n  const agents = new Map<string, StoredAgent>();\n  for (const entry of entries) {\n    if (entry.kind === 'voice.truncate') {\n      noteCut(captions, agents, entry.data);\n      continue;\n    }\n    const message = firstMessage(entry);\n    if (!message) continue;\n    const voice = voiceMeta(entry.data);\n    if (message.role === 'user' && voice) {\n      pushUser(captions, entry, message, voice);\n      continue;\n    }\n    if (message.role !== 'assistant') continue;\n    const blocks = textBlocks(message.content);\n    const turnId = String(entry.id);\n    const utteranceId = `${turnId}:agent`;\n    agents.set(turnId, { turnId, utteranceId, blocks });\n    captions.push({\n      type: 'agent_generated',\n      turnId,\n      utteranceId,\n      text: blocks.join(''),\n    });\n  }\n  return captions;\n}\n\nfunction firstMessage(entry: CaptionHistoryEntry): HistoryMessage | undefined {\n  const model = entry.model;\n  if (!model) return undefined;\n  return model[0];\n}\n\nfunction pushUser(\n  captions: CaptionEvent[],\n  entry: CaptionHistoryEntry,\n  message: HistoryMessage,\n  voice: { confidence?: number }\n): void {\n  const turnId = String(entry.id);\n  const text = textBlocks(message.content).join('');\n  if (voice.confidence === undefined) {\n    captions.push({ type: 'user_final', turnId, segmentId: 'turn', text });\n    return;\n  }\n  captions.push({\n    type: 'user_final',\n    turnId,\n    segmentId: 'turn',\n    text,\n    confidence: voice.confidence,\n  });\n}\n\nfunction noteCut(captions: CaptionEvent[], agents: Map<string, StoredAgent>, data: unknown): void {\n  const cut = cutData(data);\n  if (!cut) return;\n  const agent = agents.get(cut.entryId);\n  if (!agent) return;\n  const heard = heardInBlocks(agent.blocks, cut.contentIndex, cut.heardUpTo);\n  captions.push({\n    type: 'agent_spoken',\n    turnId: agent.turnId,\n    utteranceId: agent.utteranceId,\n    upTo: heard,\n  });\n  captions.push({\n    type: 'agent_cut',\n    turnId: agent.turnId,\n    utteranceId: agent.utteranceId,\n    heardUpTo: heard,\n  });\n}\n\nfunction heardInBlocks(blocks: readonly string[], contentIndex: number, heardUpTo: number): number {\n  let prior = 0;\n  let index = 0;\n  for (const block of blocks) {\n    if (index === contentIndex) return prior + clampHeard(block, heardUpTo);\n    prior += block.length;\n    index += 1;\n  }\n  return prior;\n}\n\nfunction clampHeard(block: string, heardUpTo: number): number {\n  if (heardUpTo <= 0) return 0;\n  if (heardUpTo >= block.length) return block.length;\n  return heardUpTo;\n}\n\nfunction textBlocks(content: HistoryMessage['content']): string[] {\n  if (typeof content === 'string') return [content];\n  if (!content) return [];\n  const blocks: string[] = [];\n  for (const block of content) {\n    if (block.type === 'text' && typeof block.text === 'string') blocks.push(block.text);\n  }\n  return blocks;\n}\n\nfunction voiceMeta(data: unknown): { confidence?: number } | undefined {\n  if (typeof data !== 'object' || data === null) return undefined;\n  if (!('source' in data) || data.source !== 'voice') return undefined;\n  if ('confidence' in data && typeof data.confidence === 'number') {\n    return { confidence: data.confidence };\n  }\n  return {};\n}\n\nfunction cutData(\n  data: unknown\n): { entryId: string; contentIndex: number; heardUpTo: number } | undefined {\n  if (typeof data !== 'object' || data === null) return undefined;\n  if (!('entryId' in data) || typeof data.entryId !== 'string') return undefined;\n  if (!('contentIndex' in data) || typeof data.contentIndex !== 'number') return undefined;\n  if (!('heardUpTo' in data) || typeof data.heardUpTo !== 'number') return undefined;\n  return { entryId: data.entryId, contentIndex: data.contentIndex, heardUpTo: data.heardUpTo };\n}\n"],
+  "mappings": "AAuCO,SAAS,aAAa,OAAoB,UAA0B;AACzE,QAAM,SAAS,MAAM,MAAM,MAAM;AACjC,MAAI,UAAU,EAAG,QAAO,MAAM;AAC9B,QAAM,YAAY,MAAM;AACxB,MAAI,aAAa,UAAU,QAAQ,WAAW,UAAU,UAAU,WAAW,WAAW,QAAQ;AAC9F,WAAO,MAAM,QAAQ,aAAa,WAAW,UAAU,MAAM;AAAA,EAC/D;AACA,MAAI,MAAM,cAAc,GAAG;AACzB,QAAI,YAAY,EAAG,QAAO,MAAM;AAChC,WAAO,MAAM;AAAA,EACf;AACA,QAAM,QAAQ,YAAY,IAAI,IAAI,WAAW,MAAM;AACnD,QAAM,WAAW,QAAQ,IAAI,IAAI;AACjC,SAAO,MAAM,QAAQ,KAAK,MAAM,WAAW,MAAM;AACnD;AAEA,SAAS,aAAa,WAA0B,UAAkB,QAAwB;AACxF,QAAM,QAAQ,UAAU,QAAQ,CAAC;AACjC,MAAI,SAAS;AACb,MAAI,UAAU,OAAW,UAAS;AAClC,QAAM,KAAK,SAAS;AACpB,MAAI,QAAQ;AACZ,WAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS,GAAG;AAC9C,UAAM,QAAQ,UAAU,QAAQ,KAAK;AACrC,QAAI,UAAU,UAAa,SAAS,GAAI,SAAQ,QAAQ;AAAA,EAC1D;AACA,SAAO;AACT;AAEO,SAAS,eAAe,SAAiB,MAAuC;AACrF,MAAI,OAAO;AACX,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,SAAU;AAC3B,QAAI,WAAW,IAAI,YAAa;AAChC,UAAM,MAAM,UAAU,IAAI,YAAY,UAAU,IAAI;AACpD,YAAQ,MAAM,IAAI;AAAA,EACpB;AACA,SAAO;AACT;AAEO,SAAS,oBAAoB,SAAyD;AAC3F,QAAM,WAA2B,CAAC;AAClC,QAAM,SAAS,oBAAI,IAAyB;AAC5C,aAAW,SAAS,SAAS;AAC3B,QAAI,MAAM,SAAS,kBAAkB;AACnC,cAAQ,UAAU,QAAQ,MAAM,IAAI;AACpC;AAAA,IACF;AACA,UAAM,UAAU,aAAa,KAAK;AAClC,QAAI,CAAC,QAAS;AACd,UAAM,QAAQ,UAAU,MAAM,IAAI;AAClC,QAAI,QAAQ,SAAS,UAAU,OAAO;AACpC,eAAS,UAAU,OAAO,SAAS,KAAK;AACxC;AAAA,IACF;AACA,QAAI,QAAQ,SAAS,YAAa;AAClC,UAAM,SAAS,WAAW,QAAQ,OAAO;AACzC,UAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,UAAM,cAAc,GAAG,MAAM;AAC7B,WAAO,IAAI,QAAQ,EAAE,QAAQ,aAAa,OAAO,CAAC;AAClD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,MAAM,OAAO,KAAK,EAAE;AAAA,IACtB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAAwD;AAC5E,QAAM,QAAQ,MAAM;AACpB,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,MAAM,CAAC;AAChB;AAEA,SAAS,SACP,UACA,OACA,SACA,OACM;AACN,QAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,QAAM,OAAO,WAAW,QAAQ,OAAO,EAAE,KAAK,EAAE;AAChD,MAAI,MAAM,eAAe,QAAW;AAClC,aAAS,KAAK,EAAE,MAAM,cAAc,QAAQ,WAAW,QAAQ,KAAK,CAAC;AACrE;AAAA,EACF;AACA,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN;AAAA,IACA,WAAW;AAAA,IACX;AAAA,IACA,YAAY,MAAM;AAAA,EACpB,CAAC;AACH;AAEA,SAAS,QAAQ,UAA0B,QAAkC,MAAqB;AAChG,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK;AACV,QAAM,QAAQ,OAAO,IAAI,IAAI,OAAO;AACpC,MAAI,CAAC,MAAO;AACZ,QAAM,QAAQ,cAAc,MAAM,QAAQ,IAAI,cAAc,IAAI,SAAS;AACzE,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN,QAAQ,MAAM;AAAA,IACd,aAAa,MAAM;AAAA,IACnB,MAAM;AAAA,EACR,CAAC;AACD,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN,QAAQ,MAAM;AAAA,IACd,aAAa,MAAM;AAAA,IACnB,WAAW;AAAA,EACb,CAAC;AACH;AAEA,SAAS,cAAc,QAA2B,cAAsB,WAA2B;AACjG,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,aAAW,SAAS,QAAQ;AAC1B,QAAI,UAAU,aAAc,QAAO,QAAQ,WAAW,OAAO,SAAS;AACtE,aAAS,MAAM;AACf,aAAS;AAAA,EACX;AACA,SAAO;AACT;AAEA,SAAS,WAAW,OAAe,WAA2B;AAC5D,MAAI,aAAa,EAAG,QAAO;AAC3B,MAAI,aAAa,MAAM,OAAQ,QAAO,MAAM;AAC5C,SAAO;AACT;AAEA,SAAS,WAAW,SAA8C;AAChE,MAAI,OAAO,YAAY,SAAU,QAAO,CAAC,OAAO;AAChD,MAAI,CAAC,QAAS,QAAO,CAAC;AACtB,QAAM,SAAmB,CAAC;AAC1B,aAAW,SAAS,SAAS;AAC3B,QAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,SAAU,QAAO,KAAK,MAAM,IAAI;AAAA,EACrF;AACA,SAAO;AACT;AAEA,SAAS,UAAU,MAAoD;AACrE,MAAI,OAAO,SAAS,YAAY,SAAS,KAAM,QAAO;AACtD,MAAI,EAAE,YAAY,SAAS,KAAK,WAAW,QAAS,QAAO;AAC3D,MAAI,gBAAgB,QAAQ,OAAO,KAAK,eAAe,UAAU;AAC/D,WAAO,EAAE,YAAY,KAAK,WAAW;AAAA,EACvC;AACA,SAAO,CAAC;AACV;AAEA,SAAS,QACP,MAC0E;AAC1E,MAAI,OAAO,SAAS,YAAY,SAAS,KAAM,QAAO;AACtD,MAAI,EAAE,aAAa,SAAS,OAAO,KAAK,YAAY,SAAU,QAAO;AACrE,MAAI,EAAE,kBAAkB,SAAS,OAAO,KAAK,iBAAiB,SAAU,QAAO;AAC/E,MAAI,EAAE,eAAe,SAAS,OAAO,KAAK,cAAc,SAAU,QAAO;AACzE,SAAO,EAAE,SAAS,KAAK,SAAS,cAAc,KAAK,cAAc,WAAW,KAAK,UAAU;AAC7F;",
+  "names": []
+}

package/dist/elevenlabs/catalog.js

@@ -0,0 +1,136 @@
+import {
+  registerVoiceModel
+} from "../models.js";
+import { registerSpeechApiProvider, registerTranscriptionApiProvider } from "../registry.js";
+import { listenElevenLabs } from "./listen.js";
+import { speakElevenLabs } from "./speak.js";
+import { transcribeElevenLabs } from "./transcribe.js";
+const tokenCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+const eleven = "https://api.elevenlabs.io/v1";
+const pcm16 = (sampleRate) => ({
+  encoding: "pcm16",
+  sampleRate,
+  channels: 1
+});
+const law = (encoding) => ({
+  encoding,
+  sampleRate: 8e3,
+  channels: 1
+});
+const container = (encoding, sampleRate) => ({
+  encoding,
+  sampleRate,
+  channels: 1
+});
+const voices = [
+  { id: "21m00Tcm4TlvDq8ikWAM", name: "Rachel" },
+  { id: "pNInz6obpgDQGcFmaJgB", name: "Adam" },
+  { id: "EXAVITQu4vr4xnSDxMaL", name: "Bella" },
+  { id: "ErXwobaYiN019PkySvjV", name: "Antoni" },
+  { id: "TxGEqnHWrfWFTfGW9XjX", name: "Josh" }
+];
+const speechFormats = [
+  container("mp3", 44100),
+  container("mp3", 24e3),
+  container("mp3", 22050),
+  pcm16(16e3),
+  pcm16(24e3),
+  pcm16(8e3),
+  container("wav", 16e3),
+  law("mulaw"),
+  law("alaw"),
+  container("opus", 48e3)
+];
+function speech(id, textStreaming, perThousandChars) {
+  return {
+    id,
+    name: id,
+    api: "elevenlabs-tts",
+    provider: "elevenlabs",
+    baseUrl: eleven,
+    type: "speech",
+    textStreaming,
+    outputFormats: speechFormats,
+    voices,
+    cloning: true,
+    cost: tokenCost,
+    audioCost: { perThousandChars }
+  };
+}
+function elevenCatalog() {
+  const batch = {
+    id: "scribe_v2",
+    name: "scribe_v2",
+    api: "elevenlabs-scribe",
+    provider: "elevenlabs",
+    baseUrl: eleven,
+    type: "transcription",
+    streaming: false,
+    turnDetection: "none",
+    inputFormats: [
+      container("wav", 16e3),
+      container("mp3", 16e3),
+      container("flac", 16e3),
+      pcm16(16e3),
+      law("mulaw")
+    ],
+    diarization: true,
+    cost: tokenCost,
+    audioCost: { perInputMinute: 0.22 / 60 }
+  };
+  const realtime = {
+    id: "scribe_v2_realtime",
+    name: "scribe_v2_realtime",
+    api: "elevenlabs-scribe",
+    provider: "elevenlabs",
+    baseUrl: eleven,
+    type: "transcription",
+    streaming: true,
+    turnDetection: "semantic",
+    inputFormats: [
+      pcm16(8e3),
+      pcm16(16e3),
+      pcm16(22050),
+      pcm16(24e3),
+      pcm16(44100),
+      pcm16(48e3),
+      law("mulaw")
+    ],
+    diarization: true,
+    cost: tokenCost,
+    audioCost: { perInputMinute: 0.39 / 60 }
+  };
+  return [
+    batch,
+    realtime,
+    speech("eleven_flash_v2_5", true, 0.05),
+    speech("eleven_multilingual_v2", true, 0.1),
+    speech("eleven_v3", false, 0.1),
+    speech("eleven_v4", false, 0.1)
+  ];
+}
+function registerElevenLabs() {
+  const models = elevenCatalog();
+  const unregisterModels = models.map((model) => registerVoiceModel(model));
+  const unregisterScribe = registerTranscriptionApiProvider({
+    api: "elevenlabs-scribe",
+    transcribe: transcribeElevenLabs,
+    listen: listenElevenLabs
+  });
+  const unregisterSpeech = registerSpeechApiProvider({
+    api: "elevenlabs-tts",
+    speak: speakElevenLabs
+  });
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    for (const unregister of unregisterModels) unregister();
+    unregisterScribe();
+    unregisterSpeech();
+  };
+}
+export {
+  registerElevenLabs
+};
+//# sourceMappingURL=catalog.js.map

package/dist/elevenlabs/catalog.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/elevenlabs/catalog.ts"],
+  "sourcesContent": ["import type { AudioFormat } from '../audio.ts';\nimport {\n  registerVoiceModel,\n  type SpeechModel,\n  type TranscriptionModel,\n  type VoiceInfo,\n} from '../models.ts';\nimport { registerSpeechApiProvider, registerTranscriptionApiProvider } from '../registry.ts';\nimport { listenElevenLabs } from './listen.ts';\nimport { speakElevenLabs } from './speak.ts';\nimport { transcribeElevenLabs } from './transcribe.ts';\n\nconst tokenCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };\nconst eleven = 'https://api.elevenlabs.io/v1';\n\nconst pcm16 = (sampleRate: number): AudioFormat => ({\n  encoding: 'pcm16',\n  sampleRate,\n  channels: 1,\n});\n\nconst law = (encoding: 'mulaw' | 'alaw'): AudioFormat => ({\n  encoding,\n  sampleRate: 8000,\n  channels: 1,\n});\n\nconst container = (encoding: 'wav' | 'mp3' | 'flac' | 'opus', sampleRate: number): AudioFormat => ({\n  encoding,\n  sampleRate,\n  channels: 1,\n});\n\nconst voices: VoiceInfo[] = [\n  { id: '21m00Tcm4TlvDq8ikWAM', name: 'Rachel' },\n  { id: 'pNInz6obpgDQGcFmaJgB', name: 'Adam' },\n  { id: 'EXAVITQu4vr4xnSDxMaL', name: 'Bella' },\n  { id: 'ErXwobaYiN019PkySvjV', name: 'Antoni' },\n  { id: 'TxGEqnHWrfWFTfGW9XjX', name: 'Josh' },\n];\n\nconst speechFormats: AudioFormat[] = [\n  container('mp3', 44100),\n  container('mp3', 24000),\n  container('mp3', 22050),\n  pcm16(16000),\n  pcm16(24000),\n  pcm16(8000),\n  container('wav', 16000),\n  law('mulaw'),\n  law('alaw'),\n  container('opus', 48000),\n];\n\nfunction speech(id: string, textStreaming: boolean, perThousandChars: number): SpeechModel<string> {\n  return {\n    id,\n    name: id,\n    api: 'elevenlabs-tts',\n    provider: 'elevenlabs',\n    baseUrl: eleven,\n    type: 'speech',\n    textStreaming,\n    outputFormats: speechFormats,\n    voices,\n    cloning: true,\n    cost: tokenCost,\n    audioCost: { perThousandChars },\n  };\n}\n\nfunction elevenCatalog(): Array<TranscriptionModel<string> | SpeechModel<string>> {\n  const batch: TranscriptionModel<string> = {\n    id: 'scribe_v2',\n    name: 'scribe_v2',\n    api: 'elevenlabs-scribe',\n    provider: 'elevenlabs',\n    baseUrl: eleven,\n    type: 'transcription',\n    streaming: false,\n    turnDetection: 'none',\n    inputFormats: [\n      container('wav', 16000),\n      container('mp3', 16000),\n      container('flac', 16000),\n      pcm16(16000),\n      law('mulaw'),\n    ],\n    diarization: true,\n    cost: tokenCost,\n    audioCost: { perInputMinute: 0.22 / 60 },\n  };\n  const realtime: TranscriptionModel<string> = {\n    id: 'scribe_v2_realtime',\n    name: 'scribe_v2_realtime',\n    api: 'elevenlabs-scribe',\n    provider: 'elevenlabs',\n    baseUrl: eleven,\n    type: 'transcription',\n    streaming: true,\n    turnDetection: 'semantic',\n    inputFormats: [\n      pcm16(8000),\n      pcm16(16000),\n      pcm16(22050),\n      pcm16(24000),\n      pcm16(44100),\n      pcm16(48000),\n      law('mulaw'),\n    ],\n    diarization: true,\n    cost: tokenCost,\n    audioCost: { perInputMinute: 0.39 / 60 },\n  };\n  return [\n    batch,\n    realtime,\n    speech('eleven_flash_v2_5', true, 0.05),\n    speech('eleven_multilingual_v2', true, 0.1),\n    speech('eleven_v3', false, 0.1),\n    speech('eleven_v4', false, 0.1),\n  ];\n}\n\nexport function registerElevenLabs(): () => void {\n  const models = elevenCatalog();\n  const unregisterModels = models.map((model) => registerVoiceModel(model));\n  const unregisterScribe = registerTranscriptionApiProvider({\n    api: 'elevenlabs-scribe',\n    transcribe: transcribeElevenLabs,\n    listen: listenElevenLabs,\n  });\n  const unregisterSpeech = registerSpeechApiProvider({\n    api: 'elevenlabs-tts',\n    speak: speakElevenLabs,\n  });\n  let released = false;\n  return () => {\n    if (released) return;\n    released = true;\n    for (const unregister of unregisterModels) unregister();\n    unregisterScribe();\n    unregisterSpeech();\n  };\n}\n"],
+  "mappings": "AACA;AAAA,EACE;AAAA,OAIK;AACP,SAAS,2BAA2B,wCAAwC;AAC5E,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAChC,SAAS,4BAA4B;AAErC,MAAM,YAAY,EAAE,OAAO,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,EAAE;AACrE,MAAM,SAAS;AAEf,MAAM,QAAQ,CAAC,gBAAqC;AAAA,EAClD,UAAU;AAAA,EACV;AAAA,EACA,UAAU;AACZ;AAEA,MAAM,MAAM,CAAC,cAA6C;AAAA,EACxD;AAAA,EACA,YAAY;AAAA,EACZ,UAAU;AACZ;AAEA,MAAM,YAAY,CAAC,UAA2C,gBAAqC;AAAA,EACjG;AAAA,EACA;AAAA,EACA,UAAU;AACZ;AAEA,MAAM,SAAsB;AAAA,EAC1B,EAAE,IAAI,wBAAwB,MAAM,SAAS;AAAA,EAC7C,EAAE,IAAI,wBAAwB,MAAM,OAAO;AAAA,EAC3C,EAAE,IAAI,wBAAwB,MAAM,QAAQ;AAAA,EAC5C,EAAE,IAAI,wBAAwB,MAAM,SAAS;AAAA,EAC7C,EAAE,IAAI,wBAAwB,MAAM,OAAO;AAC7C;AAEA,MAAM,gBAA+B;AAAA,EACnC,UAAU,OAAO,KAAK;AAAA,EACtB,UAAU,OAAO,IAAK;AAAA,EACtB,UAAU,OAAO,KAAK;AAAA,EACtB,MAAM,IAAK;AAAA,EACX,MAAM,IAAK;AAAA,EACX,MAAM,GAAI;AAAA,EACV,UAAU,OAAO,IAAK;AAAA,EACtB,IAAI,OAAO;AAAA,EACX,IAAI,MAAM;AAAA,EACV,UAAU,QAAQ,IAAK;AACzB;AAEA,SAAS,OAAO,IAAY,eAAwB,kBAA+C;AACjG,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,KAAK;AAAA,IACL,UAAU;AAAA,IACV,SAAS;AAAA,IACT,MAAM;AAAA,IACN;AAAA,IACA,eAAe;AAAA,IACf;AAAA,IACA,SAAS;AAAA,IACT,MAAM;AAAA,IACN,WAAW,EAAE,iBAAiB;AAAA,EAChC;AACF;AAEA,SAAS,gBAAyE;AAChF,QAAM,QAAoC;AAAA,IACxC,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,KAAK;AAAA,IACL,UAAU;AAAA,IACV,SAAS;AAAA,IACT,MAAM;AAAA,IACN,WAAW;AAAA,IACX,eAAe;AAAA,IACf,cAAc;AAAA,MACZ,UAAU,OAAO,IAAK;AAAA,MACtB,UAAU,OAAO,IAAK;AAAA,MACtB,UAAU,QAAQ,IAAK;AAAA,MACvB,MAAM,IAAK;AAAA,MACX,IAAI,OAAO;AAAA,IACb;AAAA,IACA,aAAa;AAAA,IACb,MAAM;AAAA,IACN,WAAW,EAAE,gBAAgB,OAAO,GAAG;AAAA,EACzC;AACA,QAAM,WAAuC;AAAA,IAC3C,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,KAAK;AAAA,IACL,UAAU;AAAA,IACV,SAAS;AAAA,IACT,MAAM;AAAA,IACN,WAAW;AAAA,IACX,eAAe;AAAA,IACf,cAAc;AAAA,MACZ,MAAM,GAAI;AAAA,MACV,MAAM,IAAK;AAAA,MACX,MAAM,KAAK;AAAA,MACX,MAAM,IAAK;AAAA,MACX,MAAM,KAAK;AAAA,MACX,MAAM,IAAK;AAAA,MACX,IAAI,OAAO;AAAA,IACb;AAAA,IACA,aAAa;AAAA,IACb,MAAM;AAAA,IACN,WAAW,EAAE,gBAAgB,OAAO,GAAG;AAAA,EACzC;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,qBAAqB,MAAM,IAAI;AAAA,IACtC,OAAO,0BAA0B,MAAM,GAAG;AAAA,IAC1C,OAAO,aAAa,OAAO,GAAG;AAAA,IAC9B,OAAO,aAAa,OAAO,GAAG;AAAA,EAChC;AACF;AAEO,SAAS,qBAAiC;AAC/C,QAAM,SAAS,cAAc;AAC7B,QAAM,mBAAmB,OAAO,IAAI,CAAC,UAAU,mBAAmB,KAAK,CAAC;AACxE,QAAM,mBAAmB,iCAAiC;AAAA,IACxD,KAAK;AAAA,IACL,YAAY;AAAA,IACZ,QAAQ;AAAA,EACV,CAAC;AACD,QAAM,mBAAmB,0BAA0B;AAAA,IACjD,KAAK;AAAA,IACL,OAAO;AAAA,EACT,CAAC;AACD,MAAI,WAAW;AACf,SAAO,MAAM;AACX,QAAI,SAAU;AACd,eAAW;AACX,eAAW,cAAc,iBAAkB,YAAW;AACtD,qBAAiB;AACjB,qBAAiB;AAAA,EACnB;AACF;",
+  "names": []
+}

package/dist/elevenlabs/index.js

@@ -0,0 +1,5 @@
+import { registerElevenLabs } from "./catalog.js";
+export {
+  registerElevenLabs
+};
+//# sourceMappingURL=index.js.map

package/dist/elevenlabs/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/elevenlabs/index.ts"],
+  "sourcesContent": ["export { registerElevenLabs } from './catalog.ts';\n"],
+  "mappings": "AAAA,SAAS,0BAA0B;",
+  "names": []
+}

package/dist/elevenlabs/listen.js

@@ -0,0 +1,268 @@
+import { chunkSeconds } from "../audio.js";
+import { encodeAudioBase64 } from "../convert.js";
+import { asVoiceError, isRecord, numberField, stringField } from "../openai/wire.js";
+import {
+  TranscriptionEventStream
+} from "../transcription.js";
+import { priceAudio } from "../usage.js";
+import {
+  awaitOpen,
+  connectSocket,
+  messageText,
+  sameFormat,
+  socketUrl,
+  sttAudioFormat,
+  xiHeaders
+} from "./wire.js";
+const failures = /* @__PURE__ */ new Set([
+  "error",
+  "auth_error",
+  "quota_exceeded",
+  "commit_throttled",
+  "unaccepted_terms",
+  "rate_limited",
+  "queue_overflow",
+  "resource_exhausted",
+  "session_time_limit_exceeded",
+  "input_error",
+  "invalid_request",
+  "chunk_size_exceeded",
+  "insufficient_audio_activity",
+  "transcriber_error"
+]);
+function terms(options) {
+  const vocabulary = options?.vocabulary;
+  if (!vocabulary) return [];
+  return vocabulary;
+}
+function commitStrategy(model) {
+  if (model.turnDetection === "semantic") return "turn_prediction";
+  if (model.turnDetection === "vad") return "vad";
+  return "manual";
+}
+function sttUrl(model, format, options) {
+  const url = socketUrl(model.baseUrl, "speech-to-text/realtime");
+  const params = new URLSearchParams();
+  params.set("model_id", model.id);
+  params.set("audio_format", sttAudioFormat(format));
+  params.set("commit_strategy", commitStrategy(model));
+  params.set("include_timestamps", "true");
+  if (options?.language) params.set("language_code", options.language);
+  for (const term of terms(options)) params.append("keyterms", term);
+  url.search = params.toString();
+  return url.toString();
+}
+function chosenFormat(model, chunk) {
+  const format = chunk === void 0 ? model.inputFormats[0] : chunk.format;
+  if (!format) throw new Error("No input format");
+  sttAudioFormat(format);
+  const known = model.inputFormats.some((item) => sameFormat(item, format));
+  if (!known) throw new Error(`Unsupported audio format ${format.encoding}`);
+  return format;
+}
+function textOr(event) {
+  const text = stringField(event, "text");
+  if (text === void 0) return "";
+  return text;
+}
+function secondsOf(word, key) {
+  const value = numberField(word, key);
+  if (value === void 0) return 0;
+  return value;
+}
+function wordFrom(word) {
+  if (stringField(word, "type") === "spacing") return void 0;
+  const speaker = stringField(word, "speaker_id");
+  const segment = {
+    text: textOr(word),
+    start: secondsOf(word, "start"),
+    end: secondsOf(word, "end")
+  };
+  if (speaker !== void 0) segment.speaker = speaker;
+  return segment;
+}
+function wordsOf(event, state) {
+  if (!Array.isArray(event.words)) return;
+  for (const word of event.words) {
+    if (!isRecord(word)) continue;
+    const segment = wordFrom(word);
+    if (segment) state.segments.push(segment);
+  }
+}
+function emitCommit(spoken, state, push) {
+  if (!state.started) {
+    state.started = true;
+    push({ type: "speech_start", at: 0 });
+  }
+  const segmentId = String(state.segment);
+  state.segment += 1;
+  state.turns.push(spoken);
+  push({ type: "final", segmentId, text: spoken });
+  state.started = false;
+  push({ type: "turn_end", text: spoken, at: 0 });
+}
+function applyListen(event, state, push) {
+  const type = stringField(event, "message_type");
+  if (type === "session_started") {
+    state.ready = true;
+    return;
+  }
+  if (type === "partial_transcript") {
+    if (!state.started) {
+      state.started = true;
+      push({ type: "speech_start", at: 0 });
+    }
+    push({ type: "partial", segmentId: String(state.segment), text: textOr(event) });
+    return;
+  }
+  if (type === "committed_transcript") {
+    const spoken = textOr(event);
+    state.lastCommit = spoken;
+    emitCommit(spoken, state, push);
+    return;
+  }
+  if (type === "committed_transcript_with_timestamps") {
+    const spoken = textOr(event);
+    wordsOf(event, state);
+    if (state.lastCommit !== spoken) emitCommit(spoken, state, push);
+    state.lastCommit = void 0;
+    return;
+  }
+  if (type !== void 0 && failures.has(type)) {
+    const message = stringField(event, "error");
+    state.failed = true;
+    push({
+      type: "error",
+      reason: "error",
+      error: message === void 0 ? "ElevenLabs transcription failed" : message
+    });
+  }
+}
+function finish(model, format, state, push) {
+  if (state.failed || state.completed) return;
+  state.completed = true;
+  const heard = chunkSeconds({ data: new Uint8Array(state.bytes), format, timestamp: 0 });
+  push({
+    type: "done",
+    transcript: {
+      text: state.turns.join(" "),
+      ...state.segments.length > 0 ? { segments: state.segments } : {},
+      usage: priceAudio(model.audioCost, { inputSeconds: heard })
+    }
+  });
+}
+function listenElevenLabs(model, audio, options) {
+  const stream = new TranscriptionEventStream();
+  void pumpListen(model, audio, options, stream);
+  return stream;
+}
+async function pumpListen(model, audio, options, stream) {
+  const state = {
+    turns: [],
+    segments: [],
+    segment: 0,
+    started: false,
+    bytes: 0,
+    opened: false,
+    ready: false,
+    clientDone: false,
+    failed: false,
+    completed: false,
+    lastCommit: void 0
+  };
+  let settled = false;
+  const push = (event) => {
+    if (settled) return;
+    if (event.type === "done" || event.type === "error") settled = true;
+    stream.push(event);
+  };
+  let socket;
+  const readyWait = Promise.withResolvers();
+  const doneWait = Promise.withResolvers();
+  const wake = (kind) => {
+    if (state.ready || state.failed) readyWait.resolve();
+    if (state.failed) doneWait.resolve();
+    if (!state.clientDone) return;
+    if (kind === "committed_transcript" || kind === "committed_transcript_with_timestamps") {
+      doneWait.resolve();
+    }
+    if (kind === "close") doneWait.resolve();
+  };
+  try {
+    if (options?.signal?.aborted) throw new DOMException("aborted", "AbortError");
+    const iterator = audio[Symbol.asyncIterator]();
+    const first = await iterator.next();
+    const format = chosenFormat(model, first.done ? void 0 : first.value);
+    socket = connectSocket(sttUrl(model, format, options), xiHeaders(model, options));
+    let incoming = Promise.resolve();
+    socket.addEventListener("message", (event) => {
+      incoming = incoming.then(() => onMessage(event.data, state, push, wake));
+    });
+    socket.addEventListener("close", () => {
+      if (state.opened && !state.failed && !state.completed && !state.clientDone) {
+        state.failed = true;
+        push({ type: "error", reason: "error", error: "ElevenLabs transcription socket closed" });
+      }
+      wake("close");
+    });
+    await awaitOpen(socket, "ElevenLabs transcription", (message) => {
+      state.failed = true;
+      push({ type: "error", reason: "error", error: message });
+      wake("fail");
+    });
+    if (state.failed) return;
+    state.opened = true;
+    if (!state.ready) await readyWait.promise;
+    if (state.failed) return;
+    if (!first.done) sendChunk(socket, first.value, format, state, false);
+    while (!state.failed) {
+      if (options?.signal?.aborted) throw new DOMException("aborted", "AbortError");
+      const next = await iterator.next();
+      if (next.done) break;
+      sendChunk(socket, next.value, format, state, false);
+    }
+    if (state.failed) return;
+    sendChunk(socket, void 0, format, state, true);
+    state.clientDone = true;
+    await doneWait.promise;
+    finish(model, format, state, push);
+  } catch (error) {
+    const voice = asVoiceError(error);
+    state.failed = true;
+    push({ type: "error", reason: voice.reason, error: voice.error });
+  } finally {
+    socket?.close();
+  }
+}
+function sendChunk(socket, chunk, format, state, commit) {
+  if (chunk && !sameFormat(chunk.format, format)) {
+    throw new Error(`Unsupported audio format ${chunk.format.encoding}`);
+  }
+  const data = chunk ? chunk.data : new Uint8Array();
+  state.bytes += data.byteLength;
+  socket.send(
+    JSON.stringify({
+      message_type: "input_audio_chunk",
+      audio_base_64: encodeAudioBase64(data),
+      commit,
+      sample_rate: format.sampleRate
+    })
+  );
+}
+async function onMessage(data, state, push, wake) {
+  try {
+    const parsed = JSON.parse(await messageText(data));
+    if (!isRecord(parsed)) throw new Error("ElevenLabs transcription event was not JSON");
+    applyListen(parsed, state, push);
+    wake(stringField(parsed, "message_type"));
+  } catch (error) {
+    const voice = asVoiceError(error);
+    state.failed = true;
+    push({ type: "error", reason: voice.reason, error: voice.error });
+    wake("fail");
+  }
+}
+export {
+  listenElevenLabs
+};
+//# sourceMappingURL=listen.js.map