@ai-ecoverse/pi-audio 1.5.0 → 1.6.0

package/README.md

@@ -25,6 +25,12 @@ pi-audio does not apply that patch. Hosts such as slicc-agent record monkeypatch
 
 `convertAudio` and `convertAudioSource` convert between `AudioFormat`s: PCM16, float32, G.711 μ-law and A-law, mono and stereo, sample rate, and PCM16 WAV. Opus, MP3 and FLAC pass through only when the format does not change. `encodeAudioBase64` and `decodeAudioBase64` are the wire helpers for JSON protocols. `AudioChunk.data` stays binary. The same converter runs in an AudioWorklet from `@ai-ecoverse/pi-audio/worklet` (`AUDIO_CONVERT_PROCESSOR`, `registerAudioConvertWorklet`). Adapters declare `inputFormats` and `outputFormats` and do not resample on their own.
 
+`connectRealtime` rejects a transport the model does not list, and a `resumeHandle` when `resumable` is false, with `VoiceRequestError`. `realtimeSocket(url, options, protocols)` keeps that url and copies `protocols`. When `options.gateway` is set it uses `gateway.url` and puts the subprotocol `cf-aig-authorization.<token>` first, which is how a browser sends a Cloudflare AI Gateway credential on the same realtime protocol.
+
+`planDelivery` reads `RealtimeModel.delivery`. An `unsupported` mode throws `VoiceRequestError`. `when_idle` with `approximate` support waits while the model is producing output and runs on `response_done`, `interrupt`, or `close`. Native `when_idle` is sent immediately so the server can schedule it. `planTruncation` adds `{ itemId, audioEndMs }` only when `truncation` is `client` and `interrupt` was given what the listener heard.
+
+`RealtimeLedger` meters one session. Input seconds come from `appendAudio`. Output seconds come from each `output_audio` event: PCM, mulaw, and alaw from the byte length, and a compressed encoding from character alignment. `sendText` and each delivered `inject` add one text message. Tool results do not. `close()` prices those meters with `priceAudio`. Sum that usage once. `response_done.usage` is the adapter's per-response figure, not a second copy of the session total.
+
 ## Development
 
 ```sh

package/dist/audio.js

@@ -1 +1,26 @@
+const sampleWidth = {
+  pcm16: 2,
+  pcmf32: 4,
+  mulaw: 1,
+  alaw: 1,
+  opus: 0,
+  mp3: 0,
+  wav: 0,
+  flac: 0
+};
+function chunkSeconds(chunk) {
+  const width = sampleWidth[chunk.format.encoding];
+  if (width === 0 || chunk.format.sampleRate === 0) return 0;
+  return chunk.data.byteLength / (width * chunk.format.channels * chunk.format.sampleRate);
+}
+function alignmentSeconds(alignment) {
+  if (alignment === void 0) return 0;
+  let ms = 0;
+  for (const duration of alignment.durationMs) ms += duration;
+  return ms / 1e3;
+}
+export {
+  alignmentSeconds,
+  chunkSeconds
+};
 //# sourceMappingURL=audio.js.map

package/dist/audio.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
+  "sources": ["../src/audio.ts"],
+  "sourcesContent": ["export type AudioEncoding = 'pcm16' | 'pcmf32' | 'mulaw' | 'alaw' | 'opus' | 'mp3' | 'wav' | 'flac';\n\nexport interface AudioFormat {\n  encoding: AudioEncoding;\n  sampleRate: number;\n  channels: 1 | 2;\n}\n\nexport interface AudioChunk {\n  data: Uint8Array;\n  format: AudioFormat;\n  timestamp: number;\n}\n\nexport type AudioSource = AsyncIterable<AudioChunk>;\n\nexport type TextSpan =\n  | { entryId: string; contentIndex: number; start: number; end: number }\n  | { itemId: string; start: number; end: number };\n\nexport interface CharAlignment {\n  startMs: number[];\n  durationMs: number[];\n}\n\nexport interface AudioTag {\n  span: TextSpan;\n  alignment?: CharAlignment;\n}\n\nexport interface AudioSink {\n  write(chunk: AudioChunk, tag?: AudioTag): void;\n  flush(): void;\n  readonly playedMs: number;\n  onPlayback(listener: (tag: AudioTag, offsetMs: number) => void): () => void;\n}\n\nconst sampleWidth: Record<AudioEncoding, number> = {\n  pcm16: 2,\n  pcmf32: 4,\n  mulaw: 1,\n  alaw: 1,\n  opus: 0,\n  mp3: 0,\n  wav: 0,\n  flac: 0,\n};\n\nexport function chunkSeconds(chunk: AudioChunk): number {\n  const width = sampleWidth[chunk.format.encoding];\n  if (width === 0 || chunk.format.sampleRate === 0) return 0;\n  return chunk.data.byteLength / (width * chunk.format.channels * chunk.format.sampleRate);\n}\n\nexport function alignmentSeconds(alignment: CharAlignment | undefined): number {\n  if (alignment === undefined) return 0;\n  let ms = 0;\n  for (const duration of alignment.durationMs) ms += duration;\n  return ms / 1000;\n}\n"],
+  "mappings": "AAqCA,MAAM,cAA6C;AAAA,EACjD,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,KAAK;AAAA,EACL,KAAK;AAAA,EACL,MAAM;AACR;AAEO,SAAS,aAAa,OAA2B;AACtD,QAAM,QAAQ,YAAY,MAAM,OAAO,QAAQ;AAC/C,MAAI,UAAU,KAAK,MAAM,OAAO,eAAe,EAAG,QAAO;AACzD,SAAO,MAAM,KAAK,cAAc,QAAQ,MAAM,OAAO,WAAW,MAAM,OAAO;AAC/E;AAEO,SAAS,iBAAiB,WAA8C;AAC7E,MAAI,cAAc,OAAW,QAAO;AACpC,MAAI,KAAK;AACT,aAAW,YAAY,UAAU,WAAY,OAAM;AACnD,SAAO,KAAK;AACd;",
   "names": []
 }

package/dist/faux/realtime.js

@@ -1,12 +1,18 @@
+import { VoiceRequestError } from "../errors.js";
 import {
   registerVoiceModel
 } from "../models.js";
+import {
+  planTruncation,
+  RealtimeLedger,
+  realtimeSocket
+} from "../realtime.js";
 import { registerRealtimeApiProvider } from "../registry.js";
-import { priceAudio } from "../usage.js";
 import { modelKey, pcm, shareApi, zeroCost } from "./shared.js";
 class FauxSession {
   model;
   #calls;
+  #ledger = new RealtimeLedger();
   #queue = [];
   #waiters = [];
   #closed = false;
@@ -17,44 +23,79 @@ class FauxSession {
   get closed() {
     return this.#closed;
   }
+  #assertOpen() {
+    if (this.#closed) throw new VoiceRequestError("error", "session is closed");
+  }
   push(event) {
     if (this.#closed) return;
+    this.#ledger.observe(event);
     const waiter = this.#waiters.shift();
     if (waiter) waiter({ value: event, done: false });
     else this.#queue.push(event);
   }
   appendAudio(chunk) {
+    this.#assertOpen();
+    this.#ledger.addInput(chunk);
     this.#calls.push({ method: "appendAudio", chunk });
   }
   appendVideoFrame(jpeg) {
+    this.#assertOpen();
+    if (this.model.video === void 0)
+      throw new VoiceRequestError("error", "video is unsupported");
     this.#calls.push({ method: "appendVideoFrame", jpeg });
   }
   sendText(text) {
+    this.#assertOpen();
+    this.#ledger.addText();
     this.#calls.push({ method: "sendText", text });
   }
   inject(text, delivery) {
-    this.#calls.push({ method: "inject", text, delivery });
+    this.#assertOpen();
+    const table = this.model.delivery;
+    this.#ledger.submit(table, delivery, () => {
+      this.#assertOpen();
+      this.#ledger.addText();
+      this.#calls.push({ method: "inject", text, delivery, support: table[delivery] });
+    });
   }
   respond(callId, result, delivery) {
-    this.#calls.push({ method: "respond", callId, result, delivery });
+    this.#assertOpen();
+    const send = (support) => {
+      this.#calls.push({ method: "respond", callId, result, delivery, support });
+    };
+    if (delivery === void 0) {
+      send();
+      return;
+    }
+    const table = this.model.delivery;
+    this.#ledger.submit(table, delivery, () => {
+      this.#assertOpen();
+      send(table[delivery]);
+    });
   }
   interrupt(heard) {
-    this.#calls.push({ method: "interrupt", heard });
+    this.#assertOpen();
+    this.#calls.push({
+      method: "interrupt",
+      heard,
+      plan: planTruncation(this.model.truncation, heard)
+    });
+    this.#ledger.settle();
   }
   update(patch) {
+    this.#assertOpen();
+    if (patch.tools !== void 0 && !this.model.liveToolUpdate) {
+      throw new VoiceRequestError("error", "live tool update is unsupported");
+    }
     this.#calls.push({ method: "update", patch });
   }
   async close() {
+    if (this.#closed) return { usage: this.#ledger.usage(this.model.audioCost) };
+    this.#ledger.settle();
     this.#calls.push({ method: "close" });
     this.#closed = true;
     while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });
-    return {
-      usage: priceAudio(this.model.audioCost, {
-        inputSeconds: 0,
-        outputSeconds: 0,
-        textMessages: 0
-      })
-    };
+    return { usage: this.#ledger.usage(this.model.audioCost) };
   }
   async *[Symbol.asyncIterator]() {
     while (true) {
@@ -111,6 +152,9 @@ function registerFauxRealtime(options = {}) {
   return {
     model,
     calls: state.calls,
+    get socket() {
+      return state.socket;
+    },
     emit(event) {
       if (state.session) state.session.push(event);
       if (!state.session || state.session.closed) state.pending.push(event);
@@ -127,9 +171,10 @@ function registerFauxRealtime(options = {}) {
     }
   };
 }
-async function connectFaux(model, _context, _options) {
+async function connectFaux(model, _context, options) {
   const state = states.get(modelKey(model));
   if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);
+  state.socket = realtimeSocket(model.baseUrl, options, ["realtime"]);
   const session = new FauxSession(state.model, state.calls);
   state.session = session;
   for (const event of state.pending) session.push(event);

package/dist/faux/realtime.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/faux/realtime.ts"],
-  "sourcesContent": ["import type { Usage } from '@earendil-works/pi-ai';\nimport type { AudioChunk } from '../audio.ts';\nimport {\n  type Delivery,\n  type DeliverySupport,\n  type RealtimeModel,\n  registerVoiceModel,\n} from '../models.ts';\nimport type {\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n} from '../realtime.ts';\nimport { registerRealtimeApiProvider } from '../registry.ts';\nimport { priceAudio, type RealtimeAudioCost } from '../usage.ts';\nimport { modelKey, pcm, shareApi, zeroCost } from './shared.ts';\n\nexport type FauxRealtimeCall =\n  | { method: 'appendAudio'; chunk: AudioChunk }\n  | { method: 'appendVideoFrame'; jpeg: Uint8Array }\n  | { method: 'sendText'; text: string }\n  | { method: 'inject'; text: string; delivery: Delivery }\n  | {\n      method: 'respond';\n      callId: string;\n      result: Parameters<RealtimeSession['respond']>[1];\n      delivery?: Delivery;\n    }\n  | { method: 'interrupt'; heard?: { itemId: string; heardMs: number } }\n  | { method: 'update'; patch: Parameters<RealtimeSession['update']>[0] }\n  | { method: 'close' };\n\nexport interface FauxRealtimeFlags {\n  toolCalls?: RealtimeModel<string>['toolCalls'];\n  delivery?: Record<Delivery, DeliverySupport>;\n  truncation?: RealtimeModel<string>['truncation'];\n}\n\nexport interface FauxRealtimeOptions extends FauxRealtimeFlags {\n  api?: string;\n  provider?: string;\n  id?: string;\n  name?: string;\n  events?: RealtimeEvent[];\n  audioCost?: RealtimeAudioCost;\n}\n\nexport interface FauxRealtime {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  emit(event: RealtimeEvent): void;\n  setFlags(flags: FauxRealtimeFlags): void;\n  unregister(): void;\n}\n\ninterface IteratorResultEvent {\n  value?: RealtimeEvent;\n  done: boolean;\n}\n\nclass FauxSession implements RealtimeSession {\n  readonly model: RealtimeModel<string>;\n  readonly #calls: FauxRealtimeCall[];\n  readonly #queue: RealtimeEvent[] = [];\n  readonly #waiters: Array<(result: IteratorResultEvent) => void> = [];\n  #closed = false;\n\n  constructor(model: RealtimeModel<string>, calls: FauxRealtimeCall[]) {\n    this.model = model;\n    this.#calls = calls;\n  }\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  push(event: RealtimeEvent): void {\n    if (this.#closed) return;\n    const waiter = this.#waiters.shift();\n    if (waiter) waiter({ value: event, done: false });\n    else this.#queue.push(event);\n  }\n\n  appendAudio(chunk: AudioChunk): void {\n    this.#calls.push({ method: 'appendAudio', chunk });\n  }\n\n  appendVideoFrame(jpeg: Uint8Array): void {\n    this.#calls.push({ method: 'appendVideoFrame', jpeg });\n  }\n\n  sendText(text: string): void {\n    this.#calls.push({ method: 'sendText', text });\n  }\n\n  inject(text: string, delivery: Delivery): void {\n    this.#calls.push({ method: 'inject', text, delivery });\n  }\n\n  respond(\n    callId: string,\n    result: Parameters<RealtimeSession['respond']>[1],\n    delivery?: Delivery\n  ): void {\n    this.#calls.push({ method: 'respond', callId, result, delivery });\n  }\n\n  interrupt(heard?: { itemId: string; heardMs: number }): void {\n    this.#calls.push({ method: 'interrupt', heard });\n  }\n\n  update(patch: Parameters<RealtimeSession['update']>[0]): void {\n    this.#calls.push({ method: 'update', patch });\n  }\n\n  async close(): Promise<{ usage: Usage }> {\n    this.#calls.push({ method: 'close' });\n    this.#closed = true;\n    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });\n    return {\n      usage: priceAudio(this.model.audioCost, {\n        inputSeconds: 0,\n        outputSeconds: 0,\n        textMessages: 0,\n      }),\n    };\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<RealtimeEvent> {\n    while (true) {\n      const queued = this.#queue.shift();\n      if (queued) {\n        yield queued;\n        continue;\n      }\n      if (this.#closed) return;\n      const next = await new Promise<IteratorResultEvent>((resolve) => this.#waiters.push(resolve));\n      if (next.done || next.value === undefined) return;\n      yield next.value;\n    }\n  }\n}\n\ninterface RealtimeState {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  events: RealtimeEvent[];\n  pending: RealtimeEvent[];\n  session?: FauxSession;\n}\n\nconst states = new Map<string, RealtimeState>();\nconst apis = new Map<string, () => void>();\n\nconst nativeDelivery: Record<Delivery, DeliverySupport> = {\n  interrupt: 'native',\n  when_idle: 'native',\n  silent: 'native',\n};\n\nexport function registerFauxRealtime(options: FauxRealtimeOptions = {}): FauxRealtime {\n  const model: RealtimeModel<string> = {\n    id: options.id ?? 'faux-realtime',\n    name: options.name ?? 'Faux realtime',\n    api: options.api ?? 'faux-realtime',\n    provider: options.provider ?? 'faux',\n    baseUrl: 'http://localhost:0',\n    type: 'realtime',\n    transports: ['websocket'],\n    toolCalls: options.toolCalls ?? 'blocking',\n    inputFormat: pcm(24000),\n    outputFormat: pcm(24000),\n    resumable: false,\n    ephemeralAuth: false,\n    voices: [{ id: 'faux', name: 'Faux' }],\n    provisioning: 'none',\n    delivery: options.delivery ?? nativeDelivery,\n    truncation: options.truncation ?? 'client',\n    liveToolUpdate: true,\n    cost: zeroCost,\n    audioCost: options.audioCost ?? {},\n  };\n  const state: RealtimeState = { model, calls: [], events: options.events ?? [], pending: [] };\n  states.set(modelKey(model), state);\n  const unregisterModel = registerVoiceModel(model);\n  const release = shareApi(\n    apis,\n    model.api,\n    () => registerRealtimeApiProvider({ api: model.api, connect: connectFaux }),\n    () => [...states.values()].some((entry) => entry.model.api === model.api)\n  );\n  return {\n    model,\n    calls: state.calls,\n    emit(event) {\n      if (state.session) state.session.push(event);\n      if (!state.session || state.session.closed) state.pending.push(event);\n    },\n    setFlags(flags) {\n      if (flags.toolCalls) state.model.toolCalls = flags.toolCalls;\n      if (flags.delivery) state.model.delivery = flags.delivery;\n      if (flags.truncation) state.model.truncation = flags.truncation;\n    },\n    unregister() {\n      if (states.get(modelKey(model)) === state) states.delete(modelKey(model));\n      unregisterModel();\n      release();\n    },\n  };\n}\n\nasync function connectFaux(\n  model: RealtimeModel<string>,\n  _context: RealtimeContext,\n  _options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const state = states.get(modelKey(model));\n  if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);\n  const session = new FauxSession(state.model, state.calls);\n  state.session = session;\n  for (const event of state.pending) session.push(event);\n  state.pending = [];\n  for (const event of state.events) session.push(event);\n  return session;\n}\n"],
-  "mappings": "AAEA;AAAA,EAIE;AAAA,OACK;AAOP,SAAS,mCAAmC;AAC5C,SAAS,kBAA0C;AACnD,SAAS,UAAU,KAAK,UAAU,gBAAgB;AA6ClD,MAAM,YAAuC;AAAA,EAClC;AAAA,EACA;AAAA,EACA,SAA0B,CAAC;AAAA,EAC3B,WAAyD,CAAC;AAAA,EACnE,UAAU;AAAA,EAEV,YAAY,OAA8B,OAA2B;AACnE,SAAK,QAAQ;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,KAAK,OAA4B;AAC/B,QAAI,KAAK,QAAS;AAClB,UAAM,SAAS,KAAK,SAAS,MAAM;AACnC,QAAI,OAAQ,QAAO,EAAE,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,QAC3C,MAAK,OAAO,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,YAAY,OAAyB;AACnC,SAAK,OAAO,KAAK,EAAE,QAAQ,eAAe,MAAM,CAAC;AAAA,EACnD;AAAA,EAEA,iBAAiB,MAAwB;AACvC,SAAK,OAAO,KAAK,EAAE,QAAQ,oBAAoB,KAAK,CAAC;AAAA,EACvD;AAAA,EAEA,SAAS,MAAoB;AAC3B,SAAK,OAAO,KAAK,EAAE,QAAQ,YAAY,KAAK,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,MAAc,UAA0B;AAC7C,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,SAAS,CAAC;AAAA,EACvD;AAAA,EAEA,QACE,QACA,QACA,UACM;AACN,SAAK,OAAO,KAAK,EAAE,QAAQ,WAAW,QAAQ,QAAQ,SAAS,CAAC;AAAA,EAClE;AAAA,EAEA,UAAU,OAAmD;AAC3D,SAAK,OAAO,KAAK,EAAE,QAAQ,aAAa,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,OAAO,OAAuD;AAC5D,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,QAAmC;AACvC,SAAK,OAAO,KAAK,EAAE,QAAQ,QAAQ,CAAC;AACpC,SAAK,UAAU;AACf,WAAO,KAAK,SAAS,SAAS,EAAG,MAAK,SAAS,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;AACvE,WAAO;AAAA,MACL,OAAO,WAAW,KAAK,MAAM,WAAW;AAAA,QACtC,cAAc;AAAA,QACd,eAAe;AAAA,QACf,cAAc;AAAA,MAChB,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAkC;AAC5D,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,OAAO,MAAM;AACjC,UAAI,QAAQ;AACV,cAAM;AACN;AAAA,MACF;AACA,UAAI,KAAK,QAAS;AAClB,YAAM,OAAO,MAAM,IAAI,QAA6B,CAAC,YAAY,KAAK,SAAS,KAAK,OAAO,CAAC;AAC5F,UAAI,KAAK,QAAQ,KAAK,UAAU,OAAW;AAC3C,YAAM,KAAK;AAAA,IACb;AAAA,EACF;AACF;AAUA,MAAM,SAAS,oBAAI,IAA2B;AAC9C,MAAM,OAAO,oBAAI,IAAwB;AAEzC,MAAM,iBAAoD;AAAA,EACxD,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AACV;AAEO,SAAS,qBAAqB,UAA+B,CAAC,GAAiB;AACpF,QAAM,QAA+B;AAAA,IACnC,IAAI,QAAQ,MAAM;AAAA,IAClB,MAAM,QAAQ,QAAQ;AAAA,IACtB,KAAK,QAAQ,OAAO;AAAA,IACpB,UAAU,QAAQ,YAAY;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM;AAAA,IACN,YAAY,CAAC,WAAW;AAAA,IACxB,WAAW,QAAQ,aAAa;AAAA,IAChC,aAAa,IAAI,IAAK;AAAA,IACtB,cAAc,IAAI,IAAK;AAAA,IACvB,WAAW;AAAA,IACX,eAAe;AAAA,IACf,QAAQ,CAAC,EAAE,IAAI,QAAQ,MAAM,OAAO,CAAC;AAAA,IACrC,cAAc;AAAA,IACd,UAAU,QAAQ,YAAY;AAAA,IAC9B,YAAY,QAAQ,cAAc;AAAA,IAClC,gBAAgB;AAAA,IAChB,MAAM;AAAA,IACN,WAAW,QAAQ,aAAa,CAAC;AAAA,EACnC;AACA,QAAM,QAAuB,EAAE,OAAO,OAAO,CAAC,GAAG,QAAQ,QAAQ,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAC3F,SAAO,IAAI,SAAS,KAAK,GAAG,KAAK;AACjC,QAAM,kBAAkB,mBAAmB,KAAK;AAChD,QAAM,UAAU;AAAA,IACd;AAAA,IACA,MAAM;AAAA,IACN,MAAM,4BAA4B,EAAE,KAAK,MAAM,KAAK,SAAS,YAAY,CAAC;AAAA,IAC1E,MAAM,CAAC,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,MAAM,MAAM,QAAQ,MAAM,GAAG;AAAA,EAC1E;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb,KAAK,OAAO;AACV,UAAI,MAAM,QAAS,OAAM,QAAQ,KAAK,KAAK;AAC3C,UAAI,CAAC,MAAM,WAAW,MAAM,QAAQ,OAAQ,OAAM,QAAQ,KAAK,KAAK;AAAA,IACtE;AAAA,IACA,SAAS,OAAO;AACd,UAAI,MAAM,UAAW,OAAM,MAAM,YAAY,MAAM;AACnD,UAAI,MAAM,SAAU,OAAM,MAAM,WAAW,MAAM;AACjD,UAAI,MAAM,WAAY,OAAM,MAAM,aAAa,MAAM;AAAA,IACvD;AAAA,IACA,aAAa;AACX,UAAI,OAAO,IAAI,SAAS,KAAK,CAAC,MAAM,MAAO,QAAO,OAAO,SAAS,KAAK,CAAC;AACxE,sBAAgB;AAChB,cAAQ;AAAA,IACV;AAAA,EACF;AACF;AAEA,eAAe,YACb,OACA,UACA,UAC0B;AAC1B,QAAM,QAAQ,OAAO,IAAI,SAAS,KAAK,CAAC;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,+BAA+B,MAAM,QAAQ,IAAI,MAAM,EAAE,EAAE;AACvF,QAAM,UAAU,IAAI,YAAY,MAAM,OAAO,MAAM,KAAK;AACxD,QAAM,UAAU;AAChB,aAAW,SAAS,MAAM,QAAS,SAAQ,KAAK,KAAK;AACrD,QAAM,UAAU,CAAC;AACjB,aAAW,SAAS,MAAM,OAAQ,SAAQ,KAAK,KAAK;AACpD,SAAO;AACT;",
+  "sourcesContent": ["import type { Usage } from '@earendil-works/pi-ai';\nimport type { AudioChunk } from '../audio.ts';\nimport { VoiceRequestError } from '../errors.ts';\nimport {\n  type Delivery,\n  type DeliverySupport,\n  type RealtimeModel,\n  registerVoiceModel,\n} from '../models.ts';\nimport {\n  planTruncation,\n  type RealtimeContext,\n  type RealtimeEvent,\n  RealtimeLedger,\n  type RealtimeOptions,\n  type RealtimeSession,\n  type RealtimeSocket,\n  realtimeSocket,\n  type TruncationPlan,\n} from '../realtime.ts';\nimport { registerRealtimeApiProvider } from '../registry.ts';\nimport type { RealtimeAudioCost } from '../usage.ts';\nimport { modelKey, pcm, shareApi, zeroCost } from './shared.ts';\n\nexport type FauxRealtimeCall =\n  | { method: 'appendAudio'; chunk: AudioChunk }\n  | { method: 'appendVideoFrame'; jpeg: Uint8Array }\n  | { method: 'sendText'; text: string }\n  | { method: 'inject'; text: string; delivery: Delivery; support: DeliverySupport }\n  | {\n      method: 'respond';\n      callId: string;\n      result: Parameters<RealtimeSession['respond']>[1];\n      delivery?: Delivery;\n      support?: DeliverySupport;\n    }\n  | {\n      method: 'interrupt';\n      heard?: { itemId: string; heardMs: number };\n      plan: TruncationPlan;\n    }\n  | { method: 'update'; patch: Parameters<RealtimeSession['update']>[0] }\n  | { method: 'close' };\n\nexport interface FauxRealtimeFlags {\n  toolCalls?: RealtimeModel<string>['toolCalls'];\n  delivery?: Record<Delivery, DeliverySupport>;\n  truncation?: RealtimeModel<string>['truncation'];\n}\n\nexport interface FauxRealtimeOptions extends FauxRealtimeFlags {\n  api?: string;\n  provider?: string;\n  id?: string;\n  name?: string;\n  events?: RealtimeEvent[];\n  audioCost?: RealtimeAudioCost;\n}\n\nexport interface FauxRealtime {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  readonly socket: RealtimeSocket | undefined;\n  emit(event: RealtimeEvent): void;\n  setFlags(flags: FauxRealtimeFlags): void;\n  unregister(): void;\n}\n\ninterface IteratorResultEvent {\n  value?: RealtimeEvent;\n  done: boolean;\n}\n\nclass FauxSession implements RealtimeSession {\n  readonly model: RealtimeModel<string>;\n  readonly #calls: FauxRealtimeCall[];\n  readonly #ledger = new RealtimeLedger();\n  readonly #queue: RealtimeEvent[] = [];\n  readonly #waiters: Array<(result: IteratorResultEvent) => void> = [];\n  #closed = false;\n\n  constructor(model: RealtimeModel<string>, calls: FauxRealtimeCall[]) {\n    this.model = model;\n    this.#calls = calls;\n  }\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  #assertOpen(): void {\n    if (this.#closed) throw new VoiceRequestError('error', 'session is closed');\n  }\n\n  push(event: RealtimeEvent): void {\n    if (this.#closed) return;\n    this.#ledger.observe(event);\n    const waiter = this.#waiters.shift();\n    if (waiter) waiter({ value: event, done: false });\n    else this.#queue.push(event);\n  }\n\n  appendAudio(chunk: AudioChunk): void {\n    this.#assertOpen();\n    this.#ledger.addInput(chunk);\n    this.#calls.push({ method: 'appendAudio', chunk });\n  }\n\n  appendVideoFrame(jpeg: Uint8Array): void {\n    this.#assertOpen();\n    if (this.model.video === undefined)\n      throw new VoiceRequestError('error', 'video is unsupported');\n    this.#calls.push({ method: 'appendVideoFrame', jpeg });\n  }\n\n  sendText(text: string): void {\n    this.#assertOpen();\n    this.#ledger.addText();\n    this.#calls.push({ method: 'sendText', text });\n  }\n\n  inject(text: string, delivery: Delivery): void {\n    this.#assertOpen();\n    const table = this.model.delivery;\n    this.#ledger.submit(table, delivery, () => {\n      this.#assertOpen();\n      this.#ledger.addText();\n      this.#calls.push({ method: 'inject', text, delivery, support: table[delivery] });\n    });\n  }\n\n  respond(\n    callId: string,\n    result: Parameters<RealtimeSession['respond']>[1],\n    delivery?: Delivery\n  ): void {\n    this.#assertOpen();\n    const send = (support?: DeliverySupport) => {\n      this.#calls.push({ method: 'respond', callId, result, delivery, support });\n    };\n    if (delivery === undefined) {\n      send();\n      return;\n    }\n    const table = this.model.delivery;\n    this.#ledger.submit(table, delivery, () => {\n      this.#assertOpen();\n      send(table[delivery]);\n    });\n  }\n\n  interrupt(heard?: { itemId: string; heardMs: number }): void {\n    this.#assertOpen();\n    this.#calls.push({\n      method: 'interrupt',\n      heard,\n      plan: planTruncation(this.model.truncation, heard),\n    });\n    this.#ledger.settle();\n  }\n\n  update(patch: Parameters<RealtimeSession['update']>[0]): void {\n    this.#assertOpen();\n    if (patch.tools !== undefined && !this.model.liveToolUpdate) {\n      throw new VoiceRequestError('error', 'live tool update is unsupported');\n    }\n    this.#calls.push({ method: 'update', patch });\n  }\n\n  async close(): Promise<{ usage: Usage }> {\n    if (this.#closed) return { usage: this.#ledger.usage(this.model.audioCost) };\n    this.#ledger.settle();\n    this.#calls.push({ method: 'close' });\n    this.#closed = true;\n    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });\n    return { usage: this.#ledger.usage(this.model.audioCost) };\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<RealtimeEvent> {\n    while (true) {\n      const queued = this.#queue.shift();\n      if (queued) {\n        yield queued;\n        continue;\n      }\n      if (this.#closed) return;\n      const next = await new Promise<IteratorResultEvent>((resolve) => this.#waiters.push(resolve));\n      if (next.done || next.value === undefined) return;\n      yield next.value;\n    }\n  }\n}\n\ninterface RealtimeState {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  events: RealtimeEvent[];\n  pending: RealtimeEvent[];\n  socket?: RealtimeSocket;\n  session?: FauxSession;\n}\n\nconst states = new Map<string, RealtimeState>();\nconst apis = new Map<string, () => void>();\n\nconst nativeDelivery: Record<Delivery, DeliverySupport> = {\n  interrupt: 'native',\n  when_idle: 'native',\n  silent: 'native',\n};\n\nexport function registerFauxRealtime(options: FauxRealtimeOptions = {}): FauxRealtime {\n  const model: RealtimeModel<string> = {\n    id: options.id ?? 'faux-realtime',\n    name: options.name ?? 'Faux realtime',\n    api: options.api ?? 'faux-realtime',\n    provider: options.provider ?? 'faux',\n    baseUrl: 'http://localhost:0',\n    type: 'realtime',\n    transports: ['websocket'],\n    toolCalls: options.toolCalls ?? 'blocking',\n    inputFormat: pcm(24000),\n    outputFormat: pcm(24000),\n    resumable: false,\n    ephemeralAuth: false,\n    voices: [{ id: 'faux', name: 'Faux' }],\n    provisioning: 'none',\n    delivery: options.delivery ?? nativeDelivery,\n    truncation: options.truncation ?? 'client',\n    liveToolUpdate: true,\n    cost: zeroCost,\n    audioCost: options.audioCost ?? {},\n  };\n  const state: RealtimeState = { model, calls: [], events: options.events ?? [], pending: [] };\n  states.set(modelKey(model), state);\n  const unregisterModel = registerVoiceModel(model);\n  const release = shareApi(\n    apis,\n    model.api,\n    () => registerRealtimeApiProvider({ api: model.api, connect: connectFaux }),\n    () => [...states.values()].some((entry) => entry.model.api === model.api)\n  );\n  return {\n    model,\n    calls: state.calls,\n    get socket() {\n      return state.socket;\n    },\n    emit(event) {\n      if (state.session) state.session.push(event);\n      if (!state.session || state.session.closed) state.pending.push(event);\n    },\n    setFlags(flags) {\n      if (flags.toolCalls) state.model.toolCalls = flags.toolCalls;\n      if (flags.delivery) state.model.delivery = flags.delivery;\n      if (flags.truncation) state.model.truncation = flags.truncation;\n    },\n    unregister() {\n      if (states.get(modelKey(model)) === state) states.delete(modelKey(model));\n      unregisterModel();\n      release();\n    },\n  };\n}\n\nasync function connectFaux(\n  model: RealtimeModel<string>,\n  _context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const state = states.get(modelKey(model));\n  if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);\n  state.socket = realtimeSocket(model.baseUrl, options, ['realtime']);\n  const session = new FauxSession(state.model, state.calls);\n  state.session = session;\n  for (const event of state.pending) session.push(event);\n  state.pending = [];\n  for (const event of state.events) session.push(event);\n  return session;\n}\n"],
+  "mappings": "AAEA,SAAS,yBAAyB;AAClC;AAAA,EAIE;AAAA,OACK;AACP;AAAA,EACE;AAAA,EAGA;AAAA,EAIA;AAAA,OAEK;AACP,SAAS,mCAAmC;AAE5C,SAAS,UAAU,KAAK,UAAU,gBAAgB;AAmDlD,MAAM,YAAuC;AAAA,EAClC;AAAA,EACA;AAAA,EACA,UAAU,IAAI,eAAe;AAAA,EAC7B,SAA0B,CAAC;AAAA,EAC3B,WAAyD,CAAC;AAAA,EACnE,UAAU;AAAA,EAEV,YAAY,OAA8B,OAA2B;AACnE,SAAK,QAAQ;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAoB;AAClB,QAAI,KAAK,QAAS,OAAM,IAAI,kBAAkB,SAAS,mBAAmB;AAAA,EAC5E;AAAA,EAEA,KAAK,OAA4B;AAC/B,QAAI,KAAK,QAAS;AAClB,SAAK,QAAQ,QAAQ,KAAK;AAC1B,UAAM,SAAS,KAAK,SAAS,MAAM;AACnC,QAAI,OAAQ,QAAO,EAAE,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,QAC3C,MAAK,OAAO,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,YAAY,OAAyB;AACnC,SAAK,YAAY;AACjB,SAAK,QAAQ,SAAS,KAAK;AAC3B,SAAK,OAAO,KAAK,EAAE,QAAQ,eAAe,MAAM,CAAC;AAAA,EACnD;AAAA,EAEA,iBAAiB,MAAwB;AACvC,SAAK,YAAY;AACjB,QAAI,KAAK,MAAM,UAAU;AACvB,YAAM,IAAI,kBAAkB,SAAS,sBAAsB;AAC7D,SAAK,OAAO,KAAK,EAAE,QAAQ,oBAAoB,KAAK,CAAC;AAAA,EACvD;AAAA,EAEA,SAAS,MAAoB;AAC3B,SAAK,YAAY;AACjB,SAAK,QAAQ,QAAQ;AACrB,SAAK,OAAO,KAAK,EAAE,QAAQ,YAAY,KAAK,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,MAAc,UAA0B;AAC7C,SAAK,YAAY;AACjB,UAAM,QAAQ,KAAK,MAAM;AACzB,SAAK,QAAQ,OAAO,OAAO,UAAU,MAAM;AACzC,WAAK,YAAY;AACjB,WAAK,QAAQ,QAAQ;AACrB,WAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,UAAU,SAAS,MAAM,QAAQ,EAAE,CAAC;AAAA,IACjF,CAAC;AAAA,EACH;AAAA,EAEA,QACE,QACA,QACA,UACM;AACN,SAAK,YAAY;AACjB,UAAM,OAAO,CAAC,YAA8B;AAC1C,WAAK,OAAO,KAAK,EAAE,QAAQ,WAAW,QAAQ,QAAQ,UAAU,QAAQ,CAAC;AAAA,IAC3E;AACA,QAAI,aAAa,QAAW;AAC1B,WAAK;AACL;AAAA,IACF;AACA,UAAM,QAAQ,KAAK,MAAM;AACzB,SAAK,QAAQ,OAAO,OAAO,UAAU,MAAM;AACzC,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ,CAAC;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAmD;AAC3D,SAAK,YAAY;AACjB,SAAK,OAAO,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA,MAAM,eAAe,KAAK,MAAM,YAAY,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAEA,OAAO,OAAuD;AAC5D,SAAK,YAAY;AACjB,QAAI,MAAM,UAAU,UAAa,CAAC,KAAK,MAAM,gBAAgB;AAC3D,YAAM,IAAI,kBAAkB,SAAS,iCAAiC;AAAA,IACxE;AACA,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,QAAmC;AACvC,QAAI,KAAK,QAAS,QAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,KAAK,MAAM,SAAS,EAAE;AAC3E,SAAK,QAAQ,OAAO;AACpB,SAAK,OAAO,KAAK,EAAE,QAAQ,QAAQ,CAAC;AACpC,SAAK,UAAU;AACf,WAAO,KAAK,SAAS,SAAS,EAAG,MAAK,SAAS,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;AACvE,WAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,KAAK,MAAM,SAAS,EAAE;AAAA,EAC3D;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAkC;AAC5D,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,OAAO,MAAM;AACjC,UAAI,QAAQ;AACV,cAAM;AACN;AAAA,MACF;AACA,UAAI,KAAK,QAAS;AAClB,YAAM,OAAO,MAAM,IAAI,QAA6B,CAAC,YAAY,KAAK,SAAS,KAAK,OAAO,CAAC;AAC5F,UAAI,KAAK,QAAQ,KAAK,UAAU,OAAW;AAC3C,YAAM,KAAK;AAAA,IACb;AAAA,EACF;AACF;AAWA,MAAM,SAAS,oBAAI,IAA2B;AAC9C,MAAM,OAAO,oBAAI,IAAwB;AAEzC,MAAM,iBAAoD;AAAA,EACxD,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AACV;AAEO,SAAS,qBAAqB,UAA+B,CAAC,GAAiB;AACpF,QAAM,QAA+B;AAAA,IACnC,IAAI,QAAQ,MAAM;AAAA,IAClB,MAAM,QAAQ,QAAQ;AAAA,IACtB,KAAK,QAAQ,OAAO;AAAA,IACpB,UAAU,QAAQ,YAAY;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM;AAAA,IACN,YAAY,CAAC,WAAW;AAAA,IACxB,WAAW,QAAQ,aAAa;AAAA,IAChC,aAAa,IAAI,IAAK;AAAA,IACtB,cAAc,IAAI,IAAK;AAAA,IACvB,WAAW;AAAA,IACX,eAAe;AAAA,IACf,QAAQ,CAAC,EAAE,IAAI,QAAQ,MAAM,OAAO,CAAC;AAAA,IACrC,cAAc;AAAA,IACd,UAAU,QAAQ,YAAY;AAAA,IAC9B,YAAY,QAAQ,cAAc;AAAA,IAClC,gBAAgB;AAAA,IAChB,MAAM;AAAA,IACN,WAAW,QAAQ,aAAa,CAAC;AAAA,EACnC;AACA,QAAM,QAAuB,EAAE,OAAO,OAAO,CAAC,GAAG,QAAQ,QAAQ,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAC3F,SAAO,IAAI,SAAS,KAAK,GAAG,KAAK;AACjC,QAAM,kBAAkB,mBAAmB,KAAK;AAChD,QAAM,UAAU;AAAA,IACd;AAAA,IACA,MAAM;AAAA,IACN,MAAM,4BAA4B,EAAE,KAAK,MAAM,KAAK,SAAS,YAAY,CAAC;AAAA,IAC1E,MAAM,CAAC,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,MAAM,MAAM,QAAQ,MAAM,GAAG;AAAA,EAC1E;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb,IAAI,SAAS;AACX,aAAO,MAAM;AAAA,IACf;AAAA,IACA,KAAK,OAAO;AACV,UAAI,MAAM,QAAS,OAAM,QAAQ,KAAK,KAAK;AAC3C,UAAI,CAAC,MAAM,WAAW,MAAM,QAAQ,OAAQ,OAAM,QAAQ,KAAK,KAAK;AAAA,IACtE;AAAA,IACA,SAAS,OAAO;AACd,UAAI,MAAM,UAAW,OAAM,MAAM,YAAY,MAAM;AACnD,UAAI,MAAM,SAAU,OAAM,MAAM,WAAW,MAAM;AACjD,UAAI,MAAM,WAAY,OAAM,MAAM,aAAa,MAAM;AAAA,IACvD;AAAA,IACA,aAAa;AACX,UAAI,OAAO,IAAI,SAAS,KAAK,CAAC,MAAM,MAAO,QAAO,OAAO,SAAS,KAAK,CAAC;AACxE,sBAAgB;AAChB,cAAQ;AAAA,IACV;AAAA,EACF;AACF;AAEA,eAAe,YACb,OACA,UACA,SAC0B;AAC1B,QAAM,QAAQ,OAAO,IAAI,SAAS,KAAK,CAAC;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,+BAA+B,MAAM,QAAQ,IAAI,MAAM,EAAE,EAAE;AACvF,QAAM,SAAS,eAAe,MAAM,SAAS,SAAS,CAAC,UAAU,CAAC;AAClE,QAAM,UAAU,IAAI,YAAY,MAAM,OAAO,MAAM,KAAK;AACxD,QAAM,UAAU;AAChB,aAAW,SAAS,MAAM,QAAS,SAAQ,KAAK,KAAK;AACrD,QAAM,UAAU,CAAC;AACjB,aAAW,SAAS,MAAM,OAAQ,SAAQ,KAAK,KAAK;AACpD,SAAO;AACT;",
   "names": []
 }

package/dist/index.js

@@ -12,6 +12,7 @@ import {
   listVoiceModels,
   registerVoiceModel
 } from "./models.js";
+import { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from "./realtime.js";
 import {
   connectRealtime,
   listen,
@@ -26,6 +27,7 @@ import { TranscriptionEventStream } from "./transcription.js";
 import { addAudioCounters, priceAudio } from "./usage.js";
 import { energyVad } from "./vad.js";
 export {
+  RealtimeLedger,
   SpeechEventStream,
   TranscriptionEventStream,
   VoiceRequestError,
@@ -40,7 +42,10 @@ export {
   getVoiceModel,
   listVoiceModels,
   listen,
+  planDelivery,
+  planTruncation,
   priceAudio,
+  realtimeSocket,
   registerRealtimeApiProvider,
   registerSpeechApiProvider,
   registerTranscriptionApiProvider,

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export type {\n  AudioChunk,\n  AudioEncoding,\n  AudioFormat,\n  AudioSink,\n  AudioSource,\n  AudioTag,\n  CharAlignment,\n  TextSpan,\n} from './audio.ts';\nexport { resolveVoiceAuth } from './auth.ts';\nexport { blankUsage } from './blank-usage.ts';\nexport {\n  convertAudio,\n  convertAudioSource,\n  decodeAudioBase64,\n  encodeAudioBase64,\n} from './convert.ts';\nexport { VoiceRequestError } from './errors.ts';\nexport type {\n  Delivery,\n  DeliverySupport,\n  KnownRealtimeApi,\n  KnownSpeechApi,\n  KnownTranscriptionApi,\n  RealtimeApi,\n  RealtimeModel,\n  SpeechApi,\n  SpeechModel,\n  TranscriptionApi,\n  TranscriptionModel,\n  VoiceInfo,\n  VoiceModel,\n  VoiceModelType,\n} from './models.ts';\nexport {\n  getVoiceModel,\n  listVoiceModels,\n  registerVoiceModel,\n} from './models.ts';\nexport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n  RealtimeToken,\n} from './realtime.ts';\nexport {\n  connectRealtime,\n  listen,\n  registerRealtimeApiProvider,\n  registerSpeechApiProvider,\n  registerTranscriptionApiProvider,\n  speak,\n  transcribe,\n} from './registry.ts';\nexport type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';\nexport { SpeechEventStream } from './speech.ts';\nexport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEvent,\n  TranscriptSegment,\n} from './transcription.ts';\nexport { TranscriptionEventStream } from './transcription.ts';\nexport type {\n  AudioCost,\n  AudioCounters,\n  AudioUsage,\n  RealtimeAudioCost,\n  SpeechAudioCost,\n  TranscriptionAudioCost,\n} from './usage.ts';\nexport { addAudioCounters, priceAudio } from './usage.ts';\nexport type { VadHook, VadUpdate } from './vad.ts';\nexport { energyVad } from './vad.ts';\n"],
-  "mappings": "AAUA,SAAS,wBAAwB;AACjC,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAiBlC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AASP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,yBAAyB;AASlC,SAAS,gCAAgC;AASzC,SAAS,kBAAkB,kBAAkB;AAE7C,SAAS,iBAAiB;",
+  "sourcesContent": ["export type {\n  AudioChunk,\n  AudioEncoding,\n  AudioFormat,\n  AudioSink,\n  AudioSource,\n  AudioTag,\n  CharAlignment,\n  TextSpan,\n} from './audio.ts';\nexport { resolveVoiceAuth } from './auth.ts';\nexport { blankUsage } from './blank-usage.ts';\nexport {\n  convertAudio,\n  convertAudioSource,\n  decodeAudioBase64,\n  encodeAudioBase64,\n} from './convert.ts';\nexport { VoiceRequestError } from './errors.ts';\nexport type {\n  Delivery,\n  DeliverySupport,\n  KnownRealtimeApi,\n  KnownSpeechApi,\n  KnownTranscriptionApi,\n  RealtimeApi,\n  RealtimeModel,\n  SpeechApi,\n  SpeechModel,\n  TranscriptionApi,\n  TranscriptionModel,\n  VoiceInfo,\n  VoiceModel,\n  VoiceModelType,\n} from './models.ts';\nexport {\n  getVoiceModel,\n  listVoiceModels,\n  registerVoiceModel,\n} from './models.ts';\nexport type {\n  DeliveryPlan,\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n  RealtimeSocket,\n  RealtimeToken,\n  TruncationPlan,\n} from './realtime.ts';\nexport { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from './realtime.ts';\nexport {\n  connectRealtime,\n  listen,\n  registerRealtimeApiProvider,\n  registerSpeechApiProvider,\n  registerTranscriptionApiProvider,\n  speak,\n  transcribe,\n} from './registry.ts';\nexport type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';\nexport { SpeechEventStream } from './speech.ts';\nexport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEvent,\n  TranscriptSegment,\n} from './transcription.ts';\nexport { TranscriptionEventStream } from './transcription.ts';\nexport type {\n  AudioCost,\n  AudioCounters,\n  AudioUsage,\n  RealtimeAudioCost,\n  SpeechAudioCost,\n  TranscriptionAudioCost,\n} from './usage.ts';\nexport { addAudioCounters, priceAudio } from './usage.ts';\nexport type { VadHook, VadUpdate } from './vad.ts';\nexport { energyVad } from './vad.ts';\n"],
+  "mappings": "AAUA,SAAS,wBAAwB;AACjC,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAiBlC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAYP,SAAS,cAAc,gBAAgB,gBAAgB,sBAAsB;AAC7E;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,yBAAyB;AASlC,SAAS,gCAAgC;AASzC,SAAS,kBAAkB,kBAAkB;AAE7C,SAAS,iBAAiB;",
   "names": []
 }

package/dist/realtime.js

@@ -1 +1,94 @@
+import {
+  alignmentSeconds,
+  chunkSeconds
+} from "./audio.js";
+import { VoiceRequestError } from "./errors.js";
+import { priceAudio } from "./usage.js";
+function planDelivery(delivery, requested) {
+  const support = delivery[requested];
+  if (support !== "native" && support !== "approximate") {
+    throw new VoiceRequestError("error", `unsupported delivery: ${requested}`);
+  }
+  return {
+    delivery: requested,
+    support,
+    waitForIdle: requested === "when_idle" && support === "approximate"
+  };
+}
+function planTruncation(truncation, heard) {
+  if (truncation === "client" && heard !== void 0) {
+    return { item: { itemId: heard.itemId, audioEndMs: heard.heardMs } };
+  }
+  return {};
+}
+function realtimeSocket(url, options, protocols = []) {
+  const gateway = options?.gateway;
+  if (gateway === void 0) return { url, protocols: [...protocols] };
+  return {
+    url: gateway.url,
+    protocols: [`cf-aig-authorization.${gateway.token}`, ...protocols]
+  };
+}
+class RealtimeLedger {
+  #inputSeconds = 0;
+  #outputSeconds = 0;
+  #textMessages = 0;
+  #busy = false;
+  #held = [];
+  get inputSeconds() {
+    return this.#inputSeconds;
+  }
+  get outputSeconds() {
+    return this.#outputSeconds;
+  }
+  get textMessages() {
+    return this.#textMessages;
+  }
+  get busy() {
+    return this.#busy;
+  }
+  addInput(chunk) {
+    this.#inputSeconds += chunkSeconds(chunk);
+  }
+  addOutput(chunk, alignment) {
+    const seconds = chunkSeconds(chunk);
+    this.#outputSeconds += seconds > 0 ? seconds : alignmentSeconds(alignment);
+  }
+  addText() {
+    this.#textMessages += 1;
+  }
+  observe(event) {
+    if (event.type === "output_audio") this.addOutput(event.chunk, event.alignment);
+    if (event.type === "output_audio" || event.type === "output_transcript") this.#busy = true;
+    if (event.type === "response_done") this.#release();
+  }
+  submit(delivery, requested, run) {
+    const plan = planDelivery(delivery, requested);
+    if (plan.waitForIdle && this.#busy) this.#held.push(run);
+    else run();
+    return plan;
+  }
+  settle() {
+    this.#release();
+  }
+  usage(rates) {
+    return priceAudio(rates, {
+      inputSeconds: this.#inputSeconds,
+      outputSeconds: this.#outputSeconds,
+      textMessages: this.#textMessages
+    });
+  }
+  #release() {
+    this.#busy = false;
+    const held = this.#held;
+    this.#held = [];
+    for (const run of held) run();
+  }
+}
+export {
+  RealtimeLedger,
+  planDelivery,
+  planTruncation,
+  realtimeSocket
+};
 //# sourceMappingURL=realtime.js.map

package/dist/realtime.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
+  "sources": ["../src/realtime.ts"],
+  "sourcesContent": ["import type {\n  Message,\n  ProviderRequestOptions,\n  Tool,\n  ToolCall,\n  ToolResultMessage,\n  Usage,\n} from '@earendil-works/pi-ai';\nimport {\n  type AudioChunk,\n  alignmentSeconds,\n  type CharAlignment,\n  chunkSeconds,\n  type TextSpan,\n} from './audio.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { Delivery, DeliverySupport, RealtimeModel } from './models.ts';\nimport { priceAudio, type RealtimeAudioCost } from './usage.ts';\n\nexport interface RealtimeContext {\n  instructions: string;\n  tools: Tool[];\n  voice: string;\n  history?: Message[];\n}\n\nexport interface RealtimeToken {\n  value: string;\n  expiresAt: number;\n  model: string;\n}\n\nexport interface RealtimeOptions extends ProviderRequestOptions {\n  token?: RealtimeToken;\n  transport?: 'websocket' | 'webrtc';\n  gateway?: { url: string; token: string };\n  resumeHandle?: string;\n}\n\nexport type RealtimeEvent =\n  | { type: 'session_start'; sessionId: string }\n  | { type: 'input_speech_start' }\n  | { type: 'input_transcript'; itemId: string; text: string; final: boolean }\n  | {\n      type: 'output_audio';\n      itemId: string;\n      chunk: AudioChunk;\n      span?: TextSpan;\n      alignment?: CharAlignment;\n    }\n  | { type: 'output_transcript'; itemId: string; text: string; final: boolean }\n  | { type: 'tool_call'; call: ToolCall; blocking: boolean }\n  | { type: 'tool_call_cancelled'; callId: string }\n  | { type: 'response_done'; itemId: string; interrupted: boolean; usage: Usage }\n  | { type: 'go_away'; timeLeftMs: number }\n  | { type: 'resumption'; handle: string }\n  | { type: 'error'; error: string; fatal: boolean };\n\nexport interface RealtimeSession extends AsyncIterable<RealtimeEvent> {\n  readonly model: RealtimeModel<string>;\n  appendAudio(chunk: AudioChunk): void;\n  appendVideoFrame?(jpeg: Uint8Array): void;\n  sendText(text: string): void;\n  inject(text: string, delivery: Delivery): void;\n  respond(callId: string, result: ToolResultMessage, delivery?: Delivery): void;\n  interrupt(heard?: { itemId: string; heardMs: number }): void;\n  update(patch: Partial<Pick<RealtimeContext, 'instructions' | 'tools' | 'voice'>>): void;\n  close(): Promise<{ usage: Usage }>;\n}\n\nexport interface RealtimeApiProvider<TApi extends string = string> {\n  api: TApi;\n  connect: (\n    model: RealtimeModel<TApi>,\n    context: RealtimeContext,\n    options?: RealtimeOptions\n  ) => Promise<RealtimeSession>;\n}\n\nexport interface DeliveryPlan {\n  delivery: Delivery;\n  support: DeliverySupport;\n  waitForIdle: boolean;\n}\n\nexport interface TruncationPlan {\n  item?: { itemId: string; audioEndMs: number };\n}\n\nexport interface RealtimeSocket {\n  url: string;\n  protocols: string[];\n}\n\nexport function planDelivery(\n  delivery: Record<Delivery, DeliverySupport>,\n  requested: Delivery\n): DeliveryPlan {\n  const support = delivery[requested];\n  if (support !== 'native' && support !== 'approximate') {\n    throw new VoiceRequestError('error', `unsupported delivery: ${requested}`);\n  }\n  return {\n    delivery: requested,\n    support,\n    waitForIdle: requested === 'when_idle' && support === 'approximate',\n  };\n}\n\nexport function planTruncation(\n  truncation: RealtimeModel<string>['truncation'],\n  heard?: { itemId: string; heardMs: number }\n): TruncationPlan {\n  if (truncation === 'client' && heard !== undefined) {\n    return { item: { itemId: heard.itemId, audioEndMs: heard.heardMs } };\n  }\n  return {};\n}\n\nexport function realtimeSocket(\n  url: string,\n  options?: Pick<RealtimeOptions, 'gateway'>,\n  protocols: readonly string[] = []\n): RealtimeSocket {\n  const gateway = options?.gateway;\n  if (gateway === undefined) return { url, protocols: [...protocols] };\n  return {\n    url: gateway.url,\n    protocols: [`cf-aig-authorization.${gateway.token}`, ...protocols],\n  };\n}\n\nexport class RealtimeLedger {\n  #inputSeconds = 0;\n  #outputSeconds = 0;\n  #textMessages = 0;\n  #busy = false;\n  #held: Array<() => void> = [];\n\n  get inputSeconds(): number {\n    return this.#inputSeconds;\n  }\n\n  get outputSeconds(): number {\n    return this.#outputSeconds;\n  }\n\n  get textMessages(): number {\n    return this.#textMessages;\n  }\n\n  get busy(): boolean {\n    return this.#busy;\n  }\n\n  addInput(chunk: AudioChunk): void {\n    this.#inputSeconds += chunkSeconds(chunk);\n  }\n\n  addOutput(chunk: AudioChunk, alignment?: CharAlignment): void {\n    const seconds = chunkSeconds(chunk);\n    this.#outputSeconds += seconds > 0 ? seconds : alignmentSeconds(alignment);\n  }\n\n  addText(): void {\n    this.#textMessages += 1;\n  }\n\n  observe(event: RealtimeEvent): void {\n    if (event.type === 'output_audio') this.addOutput(event.chunk, event.alignment);\n    if (event.type === 'output_audio' || event.type === 'output_transcript') this.#busy = true;\n    if (event.type === 'response_done') this.#release();\n  }\n\n  submit(\n    delivery: Record<Delivery, DeliverySupport>,\n    requested: Delivery,\n    run: () => void\n  ): DeliveryPlan {\n    const plan = planDelivery(delivery, requested);\n    if (plan.waitForIdle && this.#busy) this.#held.push(run);\n    else run();\n    return plan;\n  }\n\n  settle(): void {\n    this.#release();\n  }\n\n  usage(rates: RealtimeAudioCost): Usage {\n    return priceAudio(rates, {\n      inputSeconds: this.#inputSeconds,\n      outputSeconds: this.#outputSeconds,\n      textMessages: this.#textMessages,\n    });\n  }\n\n  #release(): void {\n    this.#busy = false;\n    const held = this.#held;\n    this.#held = [];\n    for (const run of held) run();\n  }\n}\n"],
+  "mappings": "AAQA;AAAA,EAEE;AAAA,EAEA;AAAA,OAEK;AACP,SAAS,yBAAyB;AAElC,SAAS,kBAA0C;AA6E5C,SAAS,aACd,UACA,WACc;AACd,QAAM,UAAU,SAAS,SAAS;AAClC,MAAI,YAAY,YAAY,YAAY,eAAe;AACrD,UAAM,IAAI,kBAAkB,SAAS,yBAAyB,SAAS,EAAE;AAAA,EAC3E;AACA,SAAO;AAAA,IACL,UAAU;AAAA,IACV;AAAA,IACA,aAAa,cAAc,eAAe,YAAY;AAAA,EACxD;AACF;AAEO,SAAS,eACd,YACA,OACgB;AAChB,MAAI,eAAe,YAAY,UAAU,QAAW;AAClD,WAAO,EAAE,MAAM,EAAE,QAAQ,MAAM,QAAQ,YAAY,MAAM,QAAQ,EAAE;AAAA,EACrE;AACA,SAAO,CAAC;AACV;AAEO,SAAS,eACd,KACA,SACA,YAA+B,CAAC,GAChB;AAChB,QAAM,UAAU,SAAS;AACzB,MAAI,YAAY,OAAW,QAAO,EAAE,KAAK,WAAW,CAAC,GAAG,SAAS,EAAE;AACnE,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,WAAW,CAAC,wBAAwB,QAAQ,KAAK,IAAI,GAAG,SAAS;AAAA,EACnE;AACF;AAEO,MAAM,eAAe;AAAA,EAC1B,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACjB,gBAAgB;AAAA,EAChB,QAAQ;AAAA,EACR,QAA2B,CAAC;AAAA,EAE5B,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,SAAS,OAAyB;AAChC,SAAK,iBAAiB,aAAa,KAAK;AAAA,EAC1C;AAAA,EAEA,UAAU,OAAmB,WAAiC;AAC5D,UAAM,UAAU,aAAa,KAAK;AAClC,SAAK,kBAAkB,UAAU,IAAI,UAAU,iBAAiB,SAAS;AAAA,EAC3E;AAAA,EAEA,UAAgB;AACd,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,QAAQ,OAA4B;AAClC,QAAI,MAAM,SAAS,eAAgB,MAAK,UAAU,MAAM,OAAO,MAAM,SAAS;AAC9E,QAAI,MAAM,SAAS,kBAAkB,MAAM,SAAS,oBAAqB,MAAK,QAAQ;AACtF,QAAI,MAAM,SAAS,gBAAiB,MAAK,SAAS;AAAA,EACpD;AAAA,EAEA,OACE,UACA,WACA,KACc;AACd,UAAM,OAAO,aAAa,UAAU,SAAS;AAC7C,QAAI,KAAK,eAAe,KAAK,MAAO,MAAK,MAAM,KAAK,GAAG;AAAA,QAClD,KAAI;AACT,WAAO;AAAA,EACT;AAAA,EAEA,SAAe;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,OAAiC;AACrC,WAAO,WAAW,OAAO;AAAA,MACvB,cAAc,KAAK;AAAA,MACnB,eAAe,KAAK;AAAA,MACpB,cAAc,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,WAAiB;AACf,SAAK,QAAQ;AACb,UAAM,OAAO,KAAK;AAClB,SAAK,QAAQ,CAAC;AACd,eAAW,OAAO,KAAM,KAAI;AAAA,EAC9B;AACF;",
   "names": []
 }

package/dist/registry.js

@@ -1,4 +1,5 @@
 import { speakInSentences } from "./chunk-speech.js";
+import { VoiceRequestError } from "./errors.js";
 import { listenWithVad } from "./vad-listen.js";
 const transcriptionApis = /* @__PURE__ */ new Map();
 const speechApis = /* @__PURE__ */ new Map();
@@ -45,6 +46,14 @@ function speak(model, input, options) {
   return speakModel(model, input, options);
 }
 function connectRealtime(model, context, options) {
+  if (options?.transport !== void 0 && !model.transports.includes(options.transport)) {
+    return Promise.reject(
+      new VoiceRequestError("error", `unsupported transport: ${options.transport}`)
+    );
+  }
+  if (options?.resumeHandle !== void 0 && !model.resumable) {
+    return Promise.reject(new VoiceRequestError("error", "session is not resumable"));
+  }
   const connect = realtimeApis.get(model.api)?.provider.connect;
   if (!connect) return Promise.reject(missing(model.api));
   return connect(model, context, options);

package/dist/registry.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/registry.ts"],
-  "sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioSource } from './audio.ts';\nimport { speakInSentences } from './chunk-speech.ts';\nimport type { RealtimeModel, SpeechModel, TranscriptionModel } from './models.ts';\nimport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeOptions,\n  RealtimeSession,\n} from './realtime.ts';\nimport type { SpeechApiProvider, SpeechEventStream, SpeechInput } from './speech.ts';\nimport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEventStream,\n} from './transcription.ts';\nimport { listenWithVad } from './vad-listen.ts';\n\ninterface TranscriptionEntry {\n  provider: TranscriptionApiProvider;\n}\n\ninterface SpeechEntry {\n  provider: SpeechApiProvider;\n}\n\ninterface RealtimeEntry {\n  provider: RealtimeApiProvider;\n}\n\nconst transcriptionApis = new Map<string, TranscriptionEntry>();\nconst speechApis = new Map<string, SpeechEntry>();\nconst realtimeApis = new Map<string, RealtimeEntry>();\n\nfunction missing(api: string): Error {\n  return new Error(`No API provider registered for api: ${api}`);\n}\n\nexport function registerTranscriptionApiProvider(provider: TranscriptionApiProvider): () => void {\n  const entry = { provider };\n  transcriptionApis.set(provider.api, entry);\n  return () => {\n    if (transcriptionApis.get(provider.api) === entry) transcriptionApis.delete(provider.api);\n  };\n}\n\nexport function registerSpeechApiProvider(provider: SpeechApiProvider): () => void {\n  const entry = { provider };\n  speechApis.set(provider.api, entry);\n  return () => {\n    if (speechApis.get(provider.api) === entry) speechApis.delete(provider.api);\n  };\n}\n\nexport function registerRealtimeApiProvider(provider: RealtimeApiProvider): () => void {\n  const entry = { provider };\n  realtimeApis.set(provider.api, entry);\n  return () => {\n    if (realtimeApis.get(provider.api) === entry) realtimeApis.delete(provider.api);\n  };\n}\n\nexport function transcribe(\n  model: TranscriptionModel<string>,\n  input: AudioInput,\n  options?: ProviderRequestOptions\n): Promise<Transcript> {\n  const transcribeModel = transcriptionApis.get(model.api)?.provider.transcribe;\n  if (!transcribeModel) return Promise.reject(missing(model.api));\n  return transcribeModel(model, input, options);\n}\n\nexport function listen(\n  model: TranscriptionModel<string>,\n  audio: AudioSource,\n  options?: ListenOptions\n): TranscriptionEventStream {\n  const listenModel = transcriptionApis.get(model.api)?.provider.listen;\n  if (!listenModel) throw missing(model.api);\n  if (model.turnDetection === 'none') return listenWithVad(listenModel, model, audio, options);\n  return listenModel(model, audio, options);\n}\n\nexport function speak(\n  model: SpeechModel<string>,\n  input: SpeechInput,\n  options?: ProviderRequestOptions\n): SpeechEventStream {\n  const speakModel = speechApis.get(model.api)?.provider.speak;\n  if (!speakModel) throw missing(model.api);\n  if (!model.textStreaming) return speakInSentences(speakModel, model, input, options);\n  return speakModel(model, input, options);\n}\n\nexport function connectRealtime(\n  model: RealtimeModel<string>,\n  context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const connect = realtimeApis.get(model.api)?.provider.connect;\n  if (!connect) return Promise.reject(missing(model.api));\n  return connect(model, context, options);\n}\n"],
-  "mappings": "AAEA,SAAS,wBAAwB;AAgBjC,SAAS,qBAAqB;AAc9B,MAAM,oBAAoB,oBAAI,IAAgC;AAC9D,MAAM,aAAa,oBAAI,IAAyB;AAChD,MAAM,eAAe,oBAAI,IAA2B;AAEpD,SAAS,QAAQ,KAAoB;AACnC,SAAO,IAAI,MAAM,uCAAuC,GAAG,EAAE;AAC/D;AAEO,SAAS,iCAAiC,UAAgD;AAC/F,QAAM,QAAQ,EAAE,SAAS;AACzB,oBAAkB,IAAI,SAAS,KAAK,KAAK;AACzC,SAAO,MAAM;AACX,QAAI,kBAAkB,IAAI,SAAS,GAAG,MAAM,MAAO,mBAAkB,OAAO,SAAS,GAAG;AAAA,EAC1F;AACF;AAEO,SAAS,0BAA0B,UAAyC;AACjF,QAAM,QAAQ,EAAE,SAAS;AACzB,aAAW,IAAI,SAAS,KAAK,KAAK;AAClC,SAAO,MAAM;AACX,QAAI,WAAW,IAAI,SAAS,GAAG,MAAM,MAAO,YAAW,OAAO,SAAS,GAAG;AAAA,EAC5E;AACF;AAEO,SAAS,4BAA4B,UAA2C;AACrF,QAAM,QAAQ,EAAE,SAAS;AACzB,eAAa,IAAI,SAAS,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,aAAa,IAAI,SAAS,GAAG,MAAM,MAAO,cAAa,OAAO,SAAS,GAAG;AAAA,EAChF;AACF;AAEO,SAAS,WACd,OACA,OACA,SACqB;AACrB,QAAM,kBAAkB,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AACnE,MAAI,CAAC,gBAAiB,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AAC9D,SAAO,gBAAgB,OAAO,OAAO,OAAO;AAC9C;AAEO,SAAS,OACd,OACA,OACA,SAC0B;AAC1B,QAAM,cAAc,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AAC/D,MAAI,CAAC,YAAa,OAAM,QAAQ,MAAM,GAAG;AACzC,MAAI,MAAM,kBAAkB,OAAQ,QAAO,cAAc,aAAa,OAAO,OAAO,OAAO;AAC3F,SAAO,YAAY,OAAO,OAAO,OAAO;AAC1C;AAEO,SAAS,MACd,OACA,OACA,SACmB;AACnB,QAAM,aAAa,WAAW,IAAI,MAAM,GAAG,GAAG,SAAS;AACvD,MAAI,CAAC,WAAY,OAAM,QAAQ,MAAM,GAAG;AACxC,MAAI,CAAC,MAAM,cAAe,QAAO,iBAAiB,YAAY,OAAO,OAAO,OAAO;AACnF,SAAO,WAAW,OAAO,OAAO,OAAO;AACzC;AAEO,SAAS,gBACd,OACA,SACA,SAC0B;AAC1B,QAAM,UAAU,aAAa,IAAI,MAAM,GAAG,GAAG,SAAS;AACtD,MAAI,CAAC,QAAS,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AACtD,SAAO,QAAQ,OAAO,SAAS,OAAO;AACxC;",
+  "sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioSource } from './audio.ts';\nimport { speakInSentences } from './chunk-speech.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { RealtimeModel, SpeechModel, TranscriptionModel } from './models.ts';\nimport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeOptions,\n  RealtimeSession,\n} from './realtime.ts';\nimport type { SpeechApiProvider, SpeechEventStream, SpeechInput } from './speech.ts';\nimport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEventStream,\n} from './transcription.ts';\nimport { listenWithVad } from './vad-listen.ts';\n\ninterface TranscriptionEntry {\n  provider: TranscriptionApiProvider;\n}\n\ninterface SpeechEntry {\n  provider: SpeechApiProvider;\n}\n\ninterface RealtimeEntry {\n  provider: RealtimeApiProvider;\n}\n\nconst transcriptionApis = new Map<string, TranscriptionEntry>();\nconst speechApis = new Map<string, SpeechEntry>();\nconst realtimeApis = new Map<string, RealtimeEntry>();\n\nfunction missing(api: string): Error {\n  return new Error(`No API provider registered for api: ${api}`);\n}\n\nexport function registerTranscriptionApiProvider(provider: TranscriptionApiProvider): () => void {\n  const entry = { provider };\n  transcriptionApis.set(provider.api, entry);\n  return () => {\n    if (transcriptionApis.get(provider.api) === entry) transcriptionApis.delete(provider.api);\n  };\n}\n\nexport function registerSpeechApiProvider(provider: SpeechApiProvider): () => void {\n  const entry = { provider };\n  speechApis.set(provider.api, entry);\n  return () => {\n    if (speechApis.get(provider.api) === entry) speechApis.delete(provider.api);\n  };\n}\n\nexport function registerRealtimeApiProvider(provider: RealtimeApiProvider): () => void {\n  const entry = { provider };\n  realtimeApis.set(provider.api, entry);\n  return () => {\n    if (realtimeApis.get(provider.api) === entry) realtimeApis.delete(provider.api);\n  };\n}\n\nexport function transcribe(\n  model: TranscriptionModel<string>,\n  input: AudioInput,\n  options?: ProviderRequestOptions\n): Promise<Transcript> {\n  const transcribeModel = transcriptionApis.get(model.api)?.provider.transcribe;\n  if (!transcribeModel) return Promise.reject(missing(model.api));\n  return transcribeModel(model, input, options);\n}\n\nexport function listen(\n  model: TranscriptionModel<string>,\n  audio: AudioSource,\n  options?: ListenOptions\n): TranscriptionEventStream {\n  const listenModel = transcriptionApis.get(model.api)?.provider.listen;\n  if (!listenModel) throw missing(model.api);\n  if (model.turnDetection === 'none') return listenWithVad(listenModel, model, audio, options);\n  return listenModel(model, audio, options);\n}\n\nexport function speak(\n  model: SpeechModel<string>,\n  input: SpeechInput,\n  options?: ProviderRequestOptions\n): SpeechEventStream {\n  const speakModel = speechApis.get(model.api)?.provider.speak;\n  if (!speakModel) throw missing(model.api);\n  if (!model.textStreaming) return speakInSentences(speakModel, model, input, options);\n  return speakModel(model, input, options);\n}\n\nexport function connectRealtime(\n  model: RealtimeModel<string>,\n  context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  if (options?.transport !== undefined && !model.transports.includes(options.transport)) {\n    return Promise.reject(\n      new VoiceRequestError('error', `unsupported transport: ${options.transport}`)\n    );\n  }\n  if (options?.resumeHandle !== undefined && !model.resumable) {\n    return Promise.reject(new VoiceRequestError('error', 'session is not resumable'));\n  }\n  const connect = realtimeApis.get(model.api)?.provider.connect;\n  if (!connect) return Promise.reject(missing(model.api));\n  return connect(model, context, options);\n}\n"],
+  "mappings": "AAEA,SAAS,wBAAwB;AACjC,SAAS,yBAAyB;AAgBlC,SAAS,qBAAqB;AAc9B,MAAM,oBAAoB,oBAAI,IAAgC;AAC9D,MAAM,aAAa,oBAAI,IAAyB;AAChD,MAAM,eAAe,oBAAI,IAA2B;AAEpD,SAAS,QAAQ,KAAoB;AACnC,SAAO,IAAI,MAAM,uCAAuC,GAAG,EAAE;AAC/D;AAEO,SAAS,iCAAiC,UAAgD;AAC/F,QAAM,QAAQ,EAAE,SAAS;AACzB,oBAAkB,IAAI,SAAS,KAAK,KAAK;AACzC,SAAO,MAAM;AACX,QAAI,kBAAkB,IAAI,SAAS,GAAG,MAAM,MAAO,mBAAkB,OAAO,SAAS,GAAG;AAAA,EAC1F;AACF;AAEO,SAAS,0BAA0B,UAAyC;AACjF,QAAM,QAAQ,EAAE,SAAS;AACzB,aAAW,IAAI,SAAS,KAAK,KAAK;AAClC,SAAO,MAAM;AACX,QAAI,WAAW,IAAI,SAAS,GAAG,MAAM,MAAO,YAAW,OAAO,SAAS,GAAG;AAAA,EAC5E;AACF;AAEO,SAAS,4BAA4B,UAA2C;AACrF,QAAM,QAAQ,EAAE,SAAS;AACzB,eAAa,IAAI,SAAS,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,aAAa,IAAI,SAAS,GAAG,MAAM,MAAO,cAAa,OAAO,SAAS,GAAG;AAAA,EAChF;AACF;AAEO,SAAS,WACd,OACA,OACA,SACqB;AACrB,QAAM,kBAAkB,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AACnE,MAAI,CAAC,gBAAiB,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AAC9D,SAAO,gBAAgB,OAAO,OAAO,OAAO;AAC9C;AAEO,SAAS,OACd,OACA,OACA,SAC0B;AAC1B,QAAM,cAAc,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AAC/D,MAAI,CAAC,YAAa,OAAM,QAAQ,MAAM,GAAG;AACzC,MAAI,MAAM,kBAAkB,OAAQ,QAAO,cAAc,aAAa,OAAO,OAAO,OAAO;AAC3F,SAAO,YAAY,OAAO,OAAO,OAAO;AAC1C;AAEO,SAAS,MACd,OACA,OACA,SACmB;AACnB,QAAM,aAAa,WAAW,IAAI,MAAM,GAAG,GAAG,SAAS;AACvD,MAAI,CAAC,WAAY,OAAM,QAAQ,MAAM,GAAG;AACxC,MAAI,CAAC,MAAM,cAAe,QAAO,iBAAiB,YAAY,OAAO,OAAO,OAAO;AACnF,SAAO,WAAW,OAAO,OAAO,OAAO;AACzC;AAEO,SAAS,gBACd,OACA,SACA,SAC0B;AAC1B,MAAI,SAAS,cAAc,UAAa,CAAC,MAAM,WAAW,SAAS,QAAQ,SAAS,GAAG;AACrF,WAAO,QAAQ;AAAA,MACb,IAAI,kBAAkB,SAAS,0BAA0B,QAAQ,SAAS,EAAE;AAAA,IAC9E;AAAA,EACF;AACA,MAAI,SAAS,iBAAiB,UAAa,CAAC,MAAM,WAAW;AAC3D,WAAO,QAAQ,OAAO,IAAI,kBAAkB,SAAS,0BAA0B,CAAC;AAAA,EAClF;AACA,QAAM,UAAU,aAAa,IAAI,MAAM,GAAG,GAAG,SAAS;AACtD,MAAI,CAAC,QAAS,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AACtD,SAAO,QAAQ,OAAO,SAAS,OAAO;AACxC;",
   "names": []
 }

package/dist/types/audio.d.ts

@@ -34,3 +34,5 @@ export interface AudioSink {
     readonly playedMs: number;
     onPlayback(listener: (tag: AudioTag, offsetMs: number) => void): () => void;
 }
+export declare function chunkSeconds(chunk: AudioChunk): number;
+export declare function alignmentSeconds(alignment: CharAlignment | undefined): number;

package/dist/types/faux/realtime.d.ts

@@ -1,7 +1,7 @@
 import type { AudioChunk } from '../audio.ts';
 import { type Delivery, type DeliverySupport, type RealtimeModel } from '../models.ts';
-import type { RealtimeEvent, RealtimeSession } from '../realtime.ts';
-import { type RealtimeAudioCost } from '../usage.ts';
+import { type RealtimeEvent, type RealtimeSession, type RealtimeSocket, type TruncationPlan } from '../realtime.ts';
+import type { RealtimeAudioCost } from '../usage.ts';
 export type FauxRealtimeCall = {
     method: 'appendAudio';
     chunk: AudioChunk;
@@ -15,17 +15,20 @@ export type FauxRealtimeCall = {
     method: 'inject';
     text: string;
     delivery: Delivery;
+    support: DeliverySupport;
 } | {
     method: 'respond';
     callId: string;
     result: Parameters<RealtimeSession['respond']>[1];
     delivery?: Delivery;
+    support?: DeliverySupport;
 } | {
     method: 'interrupt';
     heard?: {
         itemId: string;
         heardMs: number;
     };
+    plan: TruncationPlan;
 } | {
     method: 'update';
     patch: Parameters<RealtimeSession['update']>[0];
@@ -48,6 +51,7 @@ export interface FauxRealtimeOptions extends FauxRealtimeFlags {
 export interface FauxRealtime {
     model: RealtimeModel<string>;
     calls: FauxRealtimeCall[];
+    readonly socket: RealtimeSocket | undefined;
     emit(event: RealtimeEvent): void;
     setFlags(flags: FauxRealtimeFlags): void;
     unregister(): void;

package/dist/types/index.d.ts

@@ -5,7 +5,8 @@ export { convertAudio, convertAudioSource, decodeAudioBase64, encodeAudioBase64,
 export { VoiceRequestError } from './errors.ts';
 export type { Delivery, DeliverySupport, KnownRealtimeApi, KnownSpeechApi, KnownTranscriptionApi, RealtimeApi, RealtimeModel, SpeechApi, SpeechModel, TranscriptionApi, TranscriptionModel, VoiceInfo, VoiceModel, VoiceModelType, } from './models.ts';
 export { getVoiceModel, listVoiceModels, registerVoiceModel, } from './models.ts';
-export type { RealtimeApiProvider, RealtimeContext, RealtimeEvent, RealtimeOptions, RealtimeSession, RealtimeToken, } from './realtime.ts';
+export type { DeliveryPlan, RealtimeApiProvider, RealtimeContext, RealtimeEvent, RealtimeOptions, RealtimeSession, RealtimeSocket, RealtimeToken, TruncationPlan, } from './realtime.ts';
+export { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from './realtime.ts';
 export { connectRealtime, listen, registerRealtimeApiProvider, registerSpeechApiProvider, registerTranscriptionApiProvider, speak, transcribe, } from './registry.ts';
 export type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';
 export { SpeechEventStream } from './speech.ts';

package/dist/types/realtime.d.ts

@@ -1,6 +1,7 @@
 import type { Message, ProviderRequestOptions, Tool, ToolCall, ToolResultMessage, Usage } from '@earendil-works/pi-ai';
-import type { AudioChunk, CharAlignment, TextSpan } from './audio.ts';
-import type { Delivery, RealtimeModel } from './models.ts';
+import { type AudioChunk, type CharAlignment, type TextSpan } from './audio.ts';
+import type { Delivery, DeliverySupport, RealtimeModel } from './models.ts';
+import { type RealtimeAudioCost } from './usage.ts';
 export interface RealtimeContext {
     instructions: string;
     tools: Tool[];
@@ -85,3 +86,38 @@ export interface RealtimeApiProvider<TApi extends string = string> {
     api: TApi;
     connect: (model: RealtimeModel<TApi>, context: RealtimeContext, options?: RealtimeOptions) => Promise<RealtimeSession>;
 }
+export interface DeliveryPlan {
+    delivery: Delivery;
+    support: DeliverySupport;
+    waitForIdle: boolean;
+}
+export interface TruncationPlan {
+    item?: {
+        itemId: string;
+        audioEndMs: number;
+    };
+}
+export interface RealtimeSocket {
+    url: string;
+    protocols: string[];
+}
+export declare function planDelivery(delivery: Record<Delivery, DeliverySupport>, requested: Delivery): DeliveryPlan;
+export declare function planTruncation(truncation: RealtimeModel<string>['truncation'], heard?: {
+    itemId: string;
+    heardMs: number;
+}): TruncationPlan;
+export declare function realtimeSocket(url: string, options?: Pick<RealtimeOptions, 'gateway'>, protocols?: readonly string[]): RealtimeSocket;
+export declare class RealtimeLedger {
+    #private;
+    get inputSeconds(): number;
+    get outputSeconds(): number;
+    get textMessages(): number;
+    get busy(): boolean;
+    addInput(chunk: AudioChunk): void;
+    addOutput(chunk: AudioChunk, alignment?: CharAlignment): void;
+    addText(): void;
+    observe(event: RealtimeEvent): void;
+    submit(delivery: Record<Delivery, DeliverySupport>, requested: Delivery, run: () => void): DeliveryPlan;
+    settle(): void;
+    usage(rates: RealtimeAudioCost): Usage;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-ecoverse/pi-audio",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Voice bindings for pi: speech-to-text, text-to-speech and realtime speech for pi-coding-agent plugins and slicc-agent",
   "type": "module",
   "license": "Apache-2.0",