@ai-ecoverse/pi-audio 1.5.0 → 1.7.0

package/README.md

@@ -25,6 +25,14 @@ pi-audio does not apply that patch. Hosts such as slicc-agent record monkeypatch
 
 `convertAudio` and `convertAudioSource` convert between `AudioFormat`s: PCM16, float32, G.711 μ-law and A-law, mono and stereo, sample rate, and PCM16 WAV. Opus, MP3 and FLAC pass through only when the format does not change. `encodeAudioBase64` and `decodeAudioBase64` are the wire helpers for JSON protocols. `AudioChunk.data` stays binary. The same converter runs in an AudioWorklet from `@ai-ecoverse/pi-audio/worklet` (`AUDIO_CONVERT_PROCESSOR`, `registerAudioConvertWorklet`). Adapters declare `inputFormats` and `outputFormats` and do not resample on their own.
 
+`connectRealtime` rejects a transport the model does not list, and a `resumeHandle` when `resumable` is false, with `VoiceRequestError`. `realtimeSocket(url, options, protocols)` keeps that url and copies `protocols`. When `options.gateway` is set it uses `gateway.url` and puts the subprotocol `cf-aig-authorization.<token>` first, which is how a browser sends a Cloudflare AI Gateway credential on the same realtime protocol.
+
+`planDelivery` reads `RealtimeModel.delivery`. An `unsupported` mode throws `VoiceRequestError`. `when_idle` with `approximate` support waits while the model is producing output and runs on `response_done`, `interrupt`, or `close`. Native `when_idle` is sent immediately so the server can schedule it. `planTruncation` adds `{ itemId, audioEndMs }` only when `truncation` is `client` and `interrupt` was given what the listener heard.
+
+`RealtimeLedger` meters one session. Input seconds come from `appendAudio`. Output seconds come from each `output_audio` event: PCM, mulaw, and alaw from the byte length, and a compressed encoding from character alignment. `sendText` and each delivered `inject` add one text message. Tool results do not. `close()` prices those meters with `priceAudio`. Sum that usage once. `response_done.usage` is the adapter's per-response figure, not a second copy of the session total.
+
+`talkerTools` is `ask_agent` and `read`. pi-audio defines `askAgent` (a pi `Tool` whose parameters are `request` and optional `context`) and the `Thinker` interface. The host implements `Thinker` and supplies `read`. `bindThinker` runs direct tools with `directTimeoutMs` and truncates their text to `directMaxChars`. On a blocking `ask_agent` it answers with `ack` immediately and injects the result later. On an async call it `respond`s with the result and `options.result`. Progress is injected with `options.progress`. `tool_call_cancelled` aborts the thinker. The binding also yields `thinker_progress` / `thinker_result` captions and `thinking` / `tool` / `idle` phases.
+
 ## Development
 
 ```sh

package/dist/audio.js

@@ -1 +1,26 @@
+const sampleWidth = {
+  pcm16: 2,
+  pcmf32: 4,
+  mulaw: 1,
+  alaw: 1,
+  opus: 0,
+  mp3: 0,
+  wav: 0,
+  flac: 0
+};
+function chunkSeconds(chunk) {
+  const width = sampleWidth[chunk.format.encoding];
+  if (width === 0 || chunk.format.sampleRate === 0) return 0;
+  return chunk.data.byteLength / (width * chunk.format.channels * chunk.format.sampleRate);
+}
+function alignmentSeconds(alignment) {
+  if (alignment === void 0) return 0;
+  let ms = 0;
+  for (const duration of alignment.durationMs) ms += duration;
+  return ms / 1e3;
+}
+export {
+  alignmentSeconds,
+  chunkSeconds
+};
 //# sourceMappingURL=audio.js.map

package/dist/audio.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
+  "sources": ["../src/audio.ts"],
+  "sourcesContent": ["export type AudioEncoding = 'pcm16' | 'pcmf32' | 'mulaw' | 'alaw' | 'opus' | 'mp3' | 'wav' | 'flac';\n\nexport interface AudioFormat {\n  encoding: AudioEncoding;\n  sampleRate: number;\n  channels: 1 | 2;\n}\n\nexport interface AudioChunk {\n  data: Uint8Array;\n  format: AudioFormat;\n  timestamp: number;\n}\n\nexport type AudioSource = AsyncIterable<AudioChunk>;\n\nexport type TextSpan =\n  | { entryId: string; contentIndex: number; start: number; end: number }\n  | { itemId: string; start: number; end: number };\n\nexport interface CharAlignment {\n  startMs: number[];\n  durationMs: number[];\n}\n\nexport interface AudioTag {\n  span: TextSpan;\n  alignment?: CharAlignment;\n}\n\nexport interface AudioSink {\n  write(chunk: AudioChunk, tag?: AudioTag): void;\n  flush(): void;\n  readonly playedMs: number;\n  onPlayback(listener: (tag: AudioTag, offsetMs: number) => void): () => void;\n}\n\nconst sampleWidth: Record<AudioEncoding, number> = {\n  pcm16: 2,\n  pcmf32: 4,\n  mulaw: 1,\n  alaw: 1,\n  opus: 0,\n  mp3: 0,\n  wav: 0,\n  flac: 0,\n};\n\nexport function chunkSeconds(chunk: AudioChunk): number {\n  const width = sampleWidth[chunk.format.encoding];\n  if (width === 0 || chunk.format.sampleRate === 0) return 0;\n  return chunk.data.byteLength / (width * chunk.format.channels * chunk.format.sampleRate);\n}\n\nexport function alignmentSeconds(alignment: CharAlignment | undefined): number {\n  if (alignment === undefined) return 0;\n  let ms = 0;\n  for (const duration of alignment.durationMs) ms += duration;\n  return ms / 1000;\n}\n"],
+  "mappings": "AAqCA,MAAM,cAA6C;AAAA,EACjD,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,KAAK;AAAA,EACL,KAAK;AAAA,EACL,MAAM;AACR;AAEO,SAAS,aAAa,OAA2B;AACtD,QAAM,QAAQ,YAAY,MAAM,OAAO,QAAQ;AAC/C,MAAI,UAAU,KAAK,MAAM,OAAO,eAAe,EAAG,QAAO;AACzD,SAAO,MAAM,KAAK,cAAc,QAAQ,MAAM,OAAO,WAAW,MAAM,OAAO;AAC/E;AAEO,SAAS,iBAAiB,WAA8C;AAC7E,MAAI,cAAc,OAAW,QAAO;AACpC,MAAI,KAAK;AACT,aAAW,YAAY,UAAU,WAAY,OAAM;AACnD,SAAO,KAAK;AACd;",
   "names": []
 }

package/dist/faux/realtime.js

@@ -1,12 +1,18 @@
+import { VoiceRequestError } from "../errors.js";
 import {
   registerVoiceModel
 } from "../models.js";
+import {
+  planTruncation,
+  RealtimeLedger,
+  realtimeSocket
+} from "../realtime.js";
 import { registerRealtimeApiProvider } from "../registry.js";
-import { priceAudio } from "../usage.js";
 import { modelKey, pcm, shareApi, zeroCost } from "./shared.js";
 class FauxSession {
   model;
   #calls;
+  #ledger = new RealtimeLedger();
   #queue = [];
   #waiters = [];
   #closed = false;
@@ -17,44 +23,79 @@ class FauxSession {
   get closed() {
     return this.#closed;
   }
+  #assertOpen() {
+    if (this.#closed) throw new VoiceRequestError("error", "session is closed");
+  }
   push(event) {
     if (this.#closed) return;
+    this.#ledger.observe(event);
     const waiter = this.#waiters.shift();
     if (waiter) waiter({ value: event, done: false });
     else this.#queue.push(event);
   }
   appendAudio(chunk) {
+    this.#assertOpen();
+    this.#ledger.addInput(chunk);
     this.#calls.push({ method: "appendAudio", chunk });
   }
   appendVideoFrame(jpeg) {
+    this.#assertOpen();
+    if (this.model.video === void 0)
+      throw new VoiceRequestError("error", "video is unsupported");
     this.#calls.push({ method: "appendVideoFrame", jpeg });
   }
   sendText(text) {
+    this.#assertOpen();
+    this.#ledger.addText();
     this.#calls.push({ method: "sendText", text });
   }
   inject(text, delivery) {
-    this.#calls.push({ method: "inject", text, delivery });
+    this.#assertOpen();
+    const table = this.model.delivery;
+    this.#ledger.submit(table, delivery, () => {
+      this.#assertOpen();
+      this.#ledger.addText();
+      this.#calls.push({ method: "inject", text, delivery, support: table[delivery] });
+    });
   }
   respond(callId, result, delivery) {
-    this.#calls.push({ method: "respond", callId, result, delivery });
+    this.#assertOpen();
+    const send = (support) => {
+      this.#calls.push({ method: "respond", callId, result, delivery, support });
+    };
+    if (delivery === void 0) {
+      send();
+      return;
+    }
+    const table = this.model.delivery;
+    this.#ledger.submit(table, delivery, () => {
+      this.#assertOpen();
+      send(table[delivery]);
+    });
   }
   interrupt(heard) {
-    this.#calls.push({ method: "interrupt", heard });
+    this.#assertOpen();
+    this.#calls.push({
+      method: "interrupt",
+      heard,
+      plan: planTruncation(this.model.truncation, heard)
+    });
+    this.#ledger.settle();
   }
   update(patch) {
+    this.#assertOpen();
+    if (patch.tools !== void 0 && !this.model.liveToolUpdate) {
+      throw new VoiceRequestError("error", "live tool update is unsupported");
+    }
     this.#calls.push({ method: "update", patch });
   }
   async close() {
+    if (this.#closed) return { usage: this.#ledger.usage(this.model.audioCost) };
+    this.#ledger.settle();
     this.#calls.push({ method: "close" });
     this.#closed = true;
     while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });
-    return {
-      usage: priceAudio(this.model.audioCost, {
-        inputSeconds: 0,
-        outputSeconds: 0,
-        textMessages: 0
-      })
-    };
+    return { usage: this.#ledger.usage(this.model.audioCost) };
   }
   async *[Symbol.asyncIterator]() {
     while (true) {
@@ -111,6 +152,9 @@ function registerFauxRealtime(options = {}) {
   return {
     model,
     calls: state.calls,
+    get socket() {
+      return state.socket;
+    },
     emit(event) {
       if (state.session) state.session.push(event);
       if (!state.session || state.session.closed) state.pending.push(event);
@@ -127,9 +171,10 @@ function registerFauxRealtime(options = {}) {
     }
   };
 }
-async function connectFaux(model, _context, _options) {
+async function connectFaux(model, _context, options) {
   const state = states.get(modelKey(model));
   if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);
+  state.socket = realtimeSocket(model.baseUrl, options, ["realtime"]);
   const session = new FauxSession(state.model, state.calls);
   state.session = session;
   for (const event of state.pending) session.push(event);

package/dist/faux/realtime.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/faux/realtime.ts"],
-  "sourcesContent": ["import type { Usage } from '@earendil-works/pi-ai';\nimport type { AudioChunk } from '../audio.ts';\nimport {\n  type Delivery,\n  type DeliverySupport,\n  type RealtimeModel,\n  registerVoiceModel,\n} from '../models.ts';\nimport type {\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n} from '../realtime.ts';\nimport { registerRealtimeApiProvider } from '../registry.ts';\nimport { priceAudio, type RealtimeAudioCost } from '../usage.ts';\nimport { modelKey, pcm, shareApi, zeroCost } from './shared.ts';\n\nexport type FauxRealtimeCall =\n  | { method: 'appendAudio'; chunk: AudioChunk }\n  | { method: 'appendVideoFrame'; jpeg: Uint8Array }\n  | { method: 'sendText'; text: string }\n  | { method: 'inject'; text: string; delivery: Delivery }\n  | {\n      method: 'respond';\n      callId: string;\n      result: Parameters<RealtimeSession['respond']>[1];\n      delivery?: Delivery;\n    }\n  | { method: 'interrupt'; heard?: { itemId: string; heardMs: number } }\n  | { method: 'update'; patch: Parameters<RealtimeSession['update']>[0] }\n  | { method: 'close' };\n\nexport interface FauxRealtimeFlags {\n  toolCalls?: RealtimeModel<string>['toolCalls'];\n  delivery?: Record<Delivery, DeliverySupport>;\n  truncation?: RealtimeModel<string>['truncation'];\n}\n\nexport interface FauxRealtimeOptions extends FauxRealtimeFlags {\n  api?: string;\n  provider?: string;\n  id?: string;\n  name?: string;\n  events?: RealtimeEvent[];\n  audioCost?: RealtimeAudioCost;\n}\n\nexport interface FauxRealtime {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  emit(event: RealtimeEvent): void;\n  setFlags(flags: FauxRealtimeFlags): void;\n  unregister(): void;\n}\n\ninterface IteratorResultEvent {\n  value?: RealtimeEvent;\n  done: boolean;\n}\n\nclass FauxSession implements RealtimeSession {\n  readonly model: RealtimeModel<string>;\n  readonly #calls: FauxRealtimeCall[];\n  readonly #queue: RealtimeEvent[] = [];\n  readonly #waiters: Array<(result: IteratorResultEvent) => void> = [];\n  #closed = false;\n\n  constructor(model: RealtimeModel<string>, calls: FauxRealtimeCall[]) {\n    this.model = model;\n    this.#calls = calls;\n  }\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  push(event: RealtimeEvent): void {\n    if (this.#closed) return;\n    const waiter = this.#waiters.shift();\n    if (waiter) waiter({ value: event, done: false });\n    else this.#queue.push(event);\n  }\n\n  appendAudio(chunk: AudioChunk): void {\n    this.#calls.push({ method: 'appendAudio', chunk });\n  }\n\n  appendVideoFrame(jpeg: Uint8Array): void {\n    this.#calls.push({ method: 'appendVideoFrame', jpeg });\n  }\n\n  sendText(text: string): void {\n    this.#calls.push({ method: 'sendText', text });\n  }\n\n  inject(text: string, delivery: Delivery): void {\n    this.#calls.push({ method: 'inject', text, delivery });\n  }\n\n  respond(\n    callId: string,\n    result: Parameters<RealtimeSession['respond']>[1],\n    delivery?: Delivery\n  ): void {\n    this.#calls.push({ method: 'respond', callId, result, delivery });\n  }\n\n  interrupt(heard?: { itemId: string; heardMs: number }): void {\n    this.#calls.push({ method: 'interrupt', heard });\n  }\n\n  update(patch: Parameters<RealtimeSession['update']>[0]): void {\n    this.#calls.push({ method: 'update', patch });\n  }\n\n  async close(): Promise<{ usage: Usage }> {\n    this.#calls.push({ method: 'close' });\n    this.#closed = true;\n    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });\n    return {\n      usage: priceAudio(this.model.audioCost, {\n        inputSeconds: 0,\n        outputSeconds: 0,\n        textMessages: 0,\n      }),\n    };\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<RealtimeEvent> {\n    while (true) {\n      const queued = this.#queue.shift();\n      if (queued) {\n        yield queued;\n        continue;\n      }\n      if (this.#closed) return;\n      const next = await new Promise<IteratorResultEvent>((resolve) => this.#waiters.push(resolve));\n      if (next.done || next.value === undefined) return;\n      yield next.value;\n    }\n  }\n}\n\ninterface RealtimeState {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  events: RealtimeEvent[];\n  pending: RealtimeEvent[];\n  session?: FauxSession;\n}\n\nconst states = new Map<string, RealtimeState>();\nconst apis = new Map<string, () => void>();\n\nconst nativeDelivery: Record<Delivery, DeliverySupport> = {\n  interrupt: 'native',\n  when_idle: 'native',\n  silent: 'native',\n};\n\nexport function registerFauxRealtime(options: FauxRealtimeOptions = {}): FauxRealtime {\n  const model: RealtimeModel<string> = {\n    id: options.id ?? 'faux-realtime',\n    name: options.name ?? 'Faux realtime',\n    api: options.api ?? 'faux-realtime',\n    provider: options.provider ?? 'faux',\n    baseUrl: 'http://localhost:0',\n    type: 'realtime',\n    transports: ['websocket'],\n    toolCalls: options.toolCalls ?? 'blocking',\n    inputFormat: pcm(24000),\n    outputFormat: pcm(24000),\n    resumable: false,\n    ephemeralAuth: false,\n    voices: [{ id: 'faux', name: 'Faux' }],\n    provisioning: 'none',\n    delivery: options.delivery ?? nativeDelivery,\n    truncation: options.truncation ?? 'client',\n    liveToolUpdate: true,\n    cost: zeroCost,\n    audioCost: options.audioCost ?? {},\n  };\n  const state: RealtimeState = { model, calls: [], events: options.events ?? [], pending: [] };\n  states.set(modelKey(model), state);\n  const unregisterModel = registerVoiceModel(model);\n  const release = shareApi(\n    apis,\n    model.api,\n    () => registerRealtimeApiProvider({ api: model.api, connect: connectFaux }),\n    () => [...states.values()].some((entry) => entry.model.api === model.api)\n  );\n  return {\n    model,\n    calls: state.calls,\n    emit(event) {\n      if (state.session) state.session.push(event);\n      if (!state.session || state.session.closed) state.pending.push(event);\n    },\n    setFlags(flags) {\n      if (flags.toolCalls) state.model.toolCalls = flags.toolCalls;\n      if (flags.delivery) state.model.delivery = flags.delivery;\n      if (flags.truncation) state.model.truncation = flags.truncation;\n    },\n    unregister() {\n      if (states.get(modelKey(model)) === state) states.delete(modelKey(model));\n      unregisterModel();\n      release();\n    },\n  };\n}\n\nasync function connectFaux(\n  model: RealtimeModel<string>,\n  _context: RealtimeContext,\n  _options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const state = states.get(modelKey(model));\n  if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);\n  const session = new FauxSession(state.model, state.calls);\n  state.session = session;\n  for (const event of state.pending) session.push(event);\n  state.pending = [];\n  for (const event of state.events) session.push(event);\n  return session;\n}\n"],
-  "mappings": "AAEA;AAAA,EAIE;AAAA,OACK;AAOP,SAAS,mCAAmC;AAC5C,SAAS,kBAA0C;AACnD,SAAS,UAAU,KAAK,UAAU,gBAAgB;AA6ClD,MAAM,YAAuC;AAAA,EAClC;AAAA,EACA;AAAA,EACA,SAA0B,CAAC;AAAA,EAC3B,WAAyD,CAAC;AAAA,EACnE,UAAU;AAAA,EAEV,YAAY,OAA8B,OAA2B;AACnE,SAAK,QAAQ;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,KAAK,OAA4B;AAC/B,QAAI,KAAK,QAAS;AAClB,UAAM,SAAS,KAAK,SAAS,MAAM;AACnC,QAAI,OAAQ,QAAO,EAAE,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,QAC3C,MAAK,OAAO,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,YAAY,OAAyB;AACnC,SAAK,OAAO,KAAK,EAAE,QAAQ,eAAe,MAAM,CAAC;AAAA,EACnD;AAAA,EAEA,iBAAiB,MAAwB;AACvC,SAAK,OAAO,KAAK,EAAE,QAAQ,oBAAoB,KAAK,CAAC;AAAA,EACvD;AAAA,EAEA,SAAS,MAAoB;AAC3B,SAAK,OAAO,KAAK,EAAE,QAAQ,YAAY,KAAK,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,MAAc,UAA0B;AAC7C,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,SAAS,CAAC;AAAA,EACvD;AAAA,EAEA,QACE,QACA,QACA,UACM;AACN,SAAK,OAAO,KAAK,EAAE,QAAQ,WAAW,QAAQ,QAAQ,SAAS,CAAC;AAAA,EAClE;AAAA,EAEA,UAAU,OAAmD;AAC3D,SAAK,OAAO,KAAK,EAAE,QAAQ,aAAa,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,OAAO,OAAuD;AAC5D,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,QAAmC;AACvC,SAAK,OAAO,KAAK,EAAE,QAAQ,QAAQ,CAAC;AACpC,SAAK,UAAU;AACf,WAAO,KAAK,SAAS,SAAS,EAAG,MAAK,SAAS,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;AACvE,WAAO;AAAA,MACL,OAAO,WAAW,KAAK,MAAM,WAAW;AAAA,QACtC,cAAc;AAAA,QACd,eAAe;AAAA,QACf,cAAc;AAAA,MAChB,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAkC;AAC5D,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,OAAO,MAAM;AACjC,UAAI,QAAQ;AACV,cAAM;AACN;AAAA,MACF;AACA,UAAI,KAAK,QAAS;AAClB,YAAM,OAAO,MAAM,IAAI,QAA6B,CAAC,YAAY,KAAK,SAAS,KAAK,OAAO,CAAC;AAC5F,UAAI,KAAK,QAAQ,KAAK,UAAU,OAAW;AAC3C,YAAM,KAAK;AAAA,IACb;AAAA,EACF;AACF;AAUA,MAAM,SAAS,oBAAI,IAA2B;AAC9C,MAAM,OAAO,oBAAI,IAAwB;AAEzC,MAAM,iBAAoD;AAAA,EACxD,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AACV;AAEO,SAAS,qBAAqB,UAA+B,CAAC,GAAiB;AACpF,QAAM,QAA+B;AAAA,IACnC,IAAI,QAAQ,MAAM;AAAA,IAClB,MAAM,QAAQ,QAAQ;AAAA,IACtB,KAAK,QAAQ,OAAO;AAAA,IACpB,UAAU,QAAQ,YAAY;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM;AAAA,IACN,YAAY,CAAC,WAAW;AAAA,IACxB,WAAW,QAAQ,aAAa;AAAA,IAChC,aAAa,IAAI,IAAK;AAAA,IACtB,cAAc,IAAI,IAAK;AAAA,IACvB,WAAW;AAAA,IACX,eAAe;AAAA,IACf,QAAQ,CAAC,EAAE,IAAI,QAAQ,MAAM,OAAO,CAAC;AAAA,IACrC,cAAc;AAAA,IACd,UAAU,QAAQ,YAAY;AAAA,IAC9B,YAAY,QAAQ,cAAc;AAAA,IAClC,gBAAgB;AAAA,IAChB,MAAM;AAAA,IACN,WAAW,QAAQ,aAAa,CAAC;AAAA,EACnC;AACA,QAAM,QAAuB,EAAE,OAAO,OAAO,CAAC,GAAG,QAAQ,QAAQ,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAC3F,SAAO,IAAI,SAAS,KAAK,GAAG,KAAK;AACjC,QAAM,kBAAkB,mBAAmB,KAAK;AAChD,QAAM,UAAU;AAAA,IACd;AAAA,IACA,MAAM;AAAA,IACN,MAAM,4BAA4B,EAAE,KAAK,MAAM,KAAK,SAAS,YAAY,CAAC;AAAA,IAC1E,MAAM,CAAC,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,MAAM,MAAM,QAAQ,MAAM,GAAG;AAAA,EAC1E;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb,KAAK,OAAO;AACV,UAAI,MAAM,QAAS,OAAM,QAAQ,KAAK,KAAK;AAC3C,UAAI,CAAC,MAAM,WAAW,MAAM,QAAQ,OAAQ,OAAM,QAAQ,KAAK,KAAK;AAAA,IACtE;AAAA,IACA,SAAS,OAAO;AACd,UAAI,MAAM,UAAW,OAAM,MAAM,YAAY,MAAM;AACnD,UAAI,MAAM,SAAU,OAAM,MAAM,WAAW,MAAM;AACjD,UAAI,MAAM,WAAY,OAAM,MAAM,aAAa,MAAM;AAAA,IACvD;AAAA,IACA,aAAa;AACX,UAAI,OAAO,IAAI,SAAS,KAAK,CAAC,MAAM,MAAO,QAAO,OAAO,SAAS,KAAK,CAAC;AACxE,sBAAgB;AAChB,cAAQ;AAAA,IACV;AAAA,EACF;AACF;AAEA,eAAe,YACb,OACA,UACA,UAC0B;AAC1B,QAAM,QAAQ,OAAO,IAAI,SAAS,KAAK,CAAC;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,+BAA+B,MAAM,QAAQ,IAAI,MAAM,EAAE,EAAE;AACvF,QAAM,UAAU,IAAI,YAAY,MAAM,OAAO,MAAM,KAAK;AACxD,QAAM,UAAU;AAChB,aAAW,SAAS,MAAM,QAAS,SAAQ,KAAK,KAAK;AACrD,QAAM,UAAU,CAAC;AACjB,aAAW,SAAS,MAAM,OAAQ,SAAQ,KAAK,KAAK;AACpD,SAAO;AACT;",
+  "sourcesContent": ["import type { Usage } from '@earendil-works/pi-ai';\nimport type { AudioChunk } from '../audio.ts';\nimport { VoiceRequestError } from '../errors.ts';\nimport {\n  type Delivery,\n  type DeliverySupport,\n  type RealtimeModel,\n  registerVoiceModel,\n} from '../models.ts';\nimport {\n  planTruncation,\n  type RealtimeContext,\n  type RealtimeEvent,\n  RealtimeLedger,\n  type RealtimeOptions,\n  type RealtimeSession,\n  type RealtimeSocket,\n  realtimeSocket,\n  type TruncationPlan,\n} from '../realtime.ts';\nimport { registerRealtimeApiProvider } from '../registry.ts';\nimport type { RealtimeAudioCost } from '../usage.ts';\nimport { modelKey, pcm, shareApi, zeroCost } from './shared.ts';\n\nexport type FauxRealtimeCall =\n  | { method: 'appendAudio'; chunk: AudioChunk }\n  | { method: 'appendVideoFrame'; jpeg: Uint8Array }\n  | { method: 'sendText'; text: string }\n  | { method: 'inject'; text: string; delivery: Delivery; support: DeliverySupport }\n  | {\n      method: 'respond';\n      callId: string;\n      result: Parameters<RealtimeSession['respond']>[1];\n      delivery?: Delivery;\n      support?: DeliverySupport;\n    }\n  | {\n      method: 'interrupt';\n      heard?: { itemId: string; heardMs: number };\n      plan: TruncationPlan;\n    }\n  | { method: 'update'; patch: Parameters<RealtimeSession['update']>[0] }\n  | { method: 'close' };\n\nexport interface FauxRealtimeFlags {\n  toolCalls?: RealtimeModel<string>['toolCalls'];\n  delivery?: Record<Delivery, DeliverySupport>;\n  truncation?: RealtimeModel<string>['truncation'];\n}\n\nexport interface FauxRealtimeOptions extends FauxRealtimeFlags {\n  api?: string;\n  provider?: string;\n  id?: string;\n  name?: string;\n  events?: RealtimeEvent[];\n  audioCost?: RealtimeAudioCost;\n}\n\nexport interface FauxRealtime {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  readonly socket: RealtimeSocket | undefined;\n  emit(event: RealtimeEvent): void;\n  setFlags(flags: FauxRealtimeFlags): void;\n  unregister(): void;\n}\n\ninterface IteratorResultEvent {\n  value?: RealtimeEvent;\n  done: boolean;\n}\n\nclass FauxSession implements RealtimeSession {\n  readonly model: RealtimeModel<string>;\n  readonly #calls: FauxRealtimeCall[];\n  readonly #ledger = new RealtimeLedger();\n  readonly #queue: RealtimeEvent[] = [];\n  readonly #waiters: Array<(result: IteratorResultEvent) => void> = [];\n  #closed = false;\n\n  constructor(model: RealtimeModel<string>, calls: FauxRealtimeCall[]) {\n    this.model = model;\n    this.#calls = calls;\n  }\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  #assertOpen(): void {\n    if (this.#closed) throw new VoiceRequestError('error', 'session is closed');\n  }\n\n  push(event: RealtimeEvent): void {\n    if (this.#closed) return;\n    this.#ledger.observe(event);\n    const waiter = this.#waiters.shift();\n    if (waiter) waiter({ value: event, done: false });\n    else this.#queue.push(event);\n  }\n\n  appendAudio(chunk: AudioChunk): void {\n    this.#assertOpen();\n    this.#ledger.addInput(chunk);\n    this.#calls.push({ method: 'appendAudio', chunk });\n  }\n\n  appendVideoFrame(jpeg: Uint8Array): void {\n    this.#assertOpen();\n    if (this.model.video === undefined)\n      throw new VoiceRequestError('error', 'video is unsupported');\n    this.#calls.push({ method: 'appendVideoFrame', jpeg });\n  }\n\n  sendText(text: string): void {\n    this.#assertOpen();\n    this.#ledger.addText();\n    this.#calls.push({ method: 'sendText', text });\n  }\n\n  inject(text: string, delivery: Delivery): void {\n    this.#assertOpen();\n    const table = this.model.delivery;\n    this.#ledger.submit(table, delivery, () => {\n      this.#assertOpen();\n      this.#ledger.addText();\n      this.#calls.push({ method: 'inject', text, delivery, support: table[delivery] });\n    });\n  }\n\n  respond(\n    callId: string,\n    result: Parameters<RealtimeSession['respond']>[1],\n    delivery?: Delivery\n  ): void {\n    this.#assertOpen();\n    const send = (support?: DeliverySupport) => {\n      this.#calls.push({ method: 'respond', callId, result, delivery, support });\n    };\n    if (delivery === undefined) {\n      send();\n      return;\n    }\n    const table = this.model.delivery;\n    this.#ledger.submit(table, delivery, () => {\n      this.#assertOpen();\n      send(table[delivery]);\n    });\n  }\n\n  interrupt(heard?: { itemId: string; heardMs: number }): void {\n    this.#assertOpen();\n    this.#calls.push({\n      method: 'interrupt',\n      heard,\n      plan: planTruncation(this.model.truncation, heard),\n    });\n    this.#ledger.settle();\n  }\n\n  update(patch: Parameters<RealtimeSession['update']>[0]): void {\n    this.#assertOpen();\n    if (patch.tools !== undefined && !this.model.liveToolUpdate) {\n      throw new VoiceRequestError('error', 'live tool update is unsupported');\n    }\n    this.#calls.push({ method: 'update', patch });\n  }\n\n  async close(): Promise<{ usage: Usage }> {\n    if (this.#closed) return { usage: this.#ledger.usage(this.model.audioCost) };\n    this.#ledger.settle();\n    this.#calls.push({ method: 'close' });\n    this.#closed = true;\n    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });\n    return { usage: this.#ledger.usage(this.model.audioCost) };\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<RealtimeEvent> {\n    while (true) {\n      const queued = this.#queue.shift();\n      if (queued) {\n        yield queued;\n        continue;\n      }\n      if (this.#closed) return;\n      const next = await new Promise<IteratorResultEvent>((resolve) => this.#waiters.push(resolve));\n      if (next.done || next.value === undefined) return;\n      yield next.value;\n    }\n  }\n}\n\ninterface RealtimeState {\n  model: RealtimeModel<string>;\n  calls: FauxRealtimeCall[];\n  events: RealtimeEvent[];\n  pending: RealtimeEvent[];\n  socket?: RealtimeSocket;\n  session?: FauxSession;\n}\n\nconst states = new Map<string, RealtimeState>();\nconst apis = new Map<string, () => void>();\n\nconst nativeDelivery: Record<Delivery, DeliverySupport> = {\n  interrupt: 'native',\n  when_idle: 'native',\n  silent: 'native',\n};\n\nexport function registerFauxRealtime(options: FauxRealtimeOptions = {}): FauxRealtime {\n  const model: RealtimeModel<string> = {\n    id: options.id ?? 'faux-realtime',\n    name: options.name ?? 'Faux realtime',\n    api: options.api ?? 'faux-realtime',\n    provider: options.provider ?? 'faux',\n    baseUrl: 'http://localhost:0',\n    type: 'realtime',\n    transports: ['websocket'],\n    toolCalls: options.toolCalls ?? 'blocking',\n    inputFormat: pcm(24000),\n    outputFormat: pcm(24000),\n    resumable: false,\n    ephemeralAuth: false,\n    voices: [{ id: 'faux', name: 'Faux' }],\n    provisioning: 'none',\n    delivery: options.delivery ?? nativeDelivery,\n    truncation: options.truncation ?? 'client',\n    liveToolUpdate: true,\n    cost: zeroCost,\n    audioCost: options.audioCost ?? {},\n  };\n  const state: RealtimeState = { model, calls: [], events: options.events ?? [], pending: [] };\n  states.set(modelKey(model), state);\n  const unregisterModel = registerVoiceModel(model);\n  const release = shareApi(\n    apis,\n    model.api,\n    () => registerRealtimeApiProvider({ api: model.api, connect: connectFaux }),\n    () => [...states.values()].some((entry) => entry.model.api === model.api)\n  );\n  return {\n    model,\n    calls: state.calls,\n    get socket() {\n      return state.socket;\n    },\n    emit(event) {\n      if (state.session) state.session.push(event);\n      if (!state.session || state.session.closed) state.pending.push(event);\n    },\n    setFlags(flags) {\n      if (flags.toolCalls) state.model.toolCalls = flags.toolCalls;\n      if (flags.delivery) state.model.delivery = flags.delivery;\n      if (flags.truncation) state.model.truncation = flags.truncation;\n    },\n    unregister() {\n      if (states.get(modelKey(model)) === state) states.delete(modelKey(model));\n      unregisterModel();\n      release();\n    },\n  };\n}\n\nasync function connectFaux(\n  model: RealtimeModel<string>,\n  _context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const state = states.get(modelKey(model));\n  if (!state) throw new Error(`Unknown faux realtime model ${model.provider}/${model.id}`);\n  state.socket = realtimeSocket(model.baseUrl, options, ['realtime']);\n  const session = new FauxSession(state.model, state.calls);\n  state.session = session;\n  for (const event of state.pending) session.push(event);\n  state.pending = [];\n  for (const event of state.events) session.push(event);\n  return session;\n}\n"],
+  "mappings": "AAEA,SAAS,yBAAyB;AAClC;AAAA,EAIE;AAAA,OACK;AACP;AAAA,EACE;AAAA,EAGA;AAAA,EAIA;AAAA,OAEK;AACP,SAAS,mCAAmC;AAE5C,SAAS,UAAU,KAAK,UAAU,gBAAgB;AAmDlD,MAAM,YAAuC;AAAA,EAClC;AAAA,EACA;AAAA,EACA,UAAU,IAAI,eAAe;AAAA,EAC7B,SAA0B,CAAC;AAAA,EAC3B,WAAyD,CAAC;AAAA,EACnE,UAAU;AAAA,EAEV,YAAY,OAA8B,OAA2B;AACnE,SAAK,QAAQ;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAoB;AAClB,QAAI,KAAK,QAAS,OAAM,IAAI,kBAAkB,SAAS,mBAAmB;AAAA,EAC5E;AAAA,EAEA,KAAK,OAA4B;AAC/B,QAAI,KAAK,QAAS;AAClB,SAAK,QAAQ,QAAQ,KAAK;AAC1B,UAAM,SAAS,KAAK,SAAS,MAAM;AACnC,QAAI,OAAQ,QAAO,EAAE,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,QAC3C,MAAK,OAAO,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,YAAY,OAAyB;AACnC,SAAK,YAAY;AACjB,SAAK,QAAQ,SAAS,KAAK;AAC3B,SAAK,OAAO,KAAK,EAAE,QAAQ,eAAe,MAAM,CAAC;AAAA,EACnD;AAAA,EAEA,iBAAiB,MAAwB;AACvC,SAAK,YAAY;AACjB,QAAI,KAAK,MAAM,UAAU;AACvB,YAAM,IAAI,kBAAkB,SAAS,sBAAsB;AAC7D,SAAK,OAAO,KAAK,EAAE,QAAQ,oBAAoB,KAAK,CAAC;AAAA,EACvD;AAAA,EAEA,SAAS,MAAoB;AAC3B,SAAK,YAAY;AACjB,SAAK,QAAQ,QAAQ;AACrB,SAAK,OAAO,KAAK,EAAE,QAAQ,YAAY,KAAK,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,MAAc,UAA0B;AAC7C,SAAK,YAAY;AACjB,UAAM,QAAQ,KAAK,MAAM;AACzB,SAAK,QAAQ,OAAO,OAAO,UAAU,MAAM;AACzC,WAAK,YAAY;AACjB,WAAK,QAAQ,QAAQ;AACrB,WAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,UAAU,SAAS,MAAM,QAAQ,EAAE,CAAC;AAAA,IACjF,CAAC;AAAA,EACH;AAAA,EAEA,QACE,QACA,QACA,UACM;AACN,SAAK,YAAY;AACjB,UAAM,OAAO,CAAC,YAA8B;AAC1C,WAAK,OAAO,KAAK,EAAE,QAAQ,WAAW,QAAQ,QAAQ,UAAU,QAAQ,CAAC;AAAA,IAC3E;AACA,QAAI,aAAa,QAAW;AAC1B,WAAK;AACL;AAAA,IACF;AACA,UAAM,QAAQ,KAAK,MAAM;AACzB,SAAK,QAAQ,OAAO,OAAO,UAAU,MAAM;AACzC,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ,CAAC;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAmD;AAC3D,SAAK,YAAY;AACjB,SAAK,OAAO,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA,MAAM,eAAe,KAAK,MAAM,YAAY,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAEA,OAAO,OAAuD;AAC5D,SAAK,YAAY;AACjB,QAAI,MAAM,UAAU,UAAa,CAAC,KAAK,MAAM,gBAAgB;AAC3D,YAAM,IAAI,kBAAkB,SAAS,iCAAiC;AAAA,IACxE;AACA,SAAK,OAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,QAAmC;AACvC,QAAI,KAAK,QAAS,QAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,KAAK,MAAM,SAAS,EAAE;AAC3E,SAAK,QAAQ,OAAO;AACpB,SAAK,OAAO,KAAK,EAAE,QAAQ,QAAQ,CAAC;AACpC,SAAK,UAAU;AACf,WAAO,KAAK,SAAS,SAAS,EAAG,MAAK,SAAS,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;AACvE,WAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,KAAK,MAAM,SAAS,EAAE;AAAA,EAC3D;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAkC;AAC5D,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,OAAO,MAAM;AACjC,UAAI,QAAQ;AACV,cAAM;AACN;AAAA,MACF;AACA,UAAI,KAAK,QAAS;AAClB,YAAM,OAAO,MAAM,IAAI,QAA6B,CAAC,YAAY,KAAK,SAAS,KAAK,OAAO,CAAC;AAC5F,UAAI,KAAK,QAAQ,KAAK,UAAU,OAAW;AAC3C,YAAM,KAAK;AAAA,IACb;AAAA,EACF;AACF;AAWA,MAAM,SAAS,oBAAI,IAA2B;AAC9C,MAAM,OAAO,oBAAI,IAAwB;AAEzC,MAAM,iBAAoD;AAAA,EACxD,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AACV;AAEO,SAAS,qBAAqB,UAA+B,CAAC,GAAiB;AACpF,QAAM,QAA+B;AAAA,IACnC,IAAI,QAAQ,MAAM;AAAA,IAClB,MAAM,QAAQ,QAAQ;AAAA,IACtB,KAAK,QAAQ,OAAO;AAAA,IACpB,UAAU,QAAQ,YAAY;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM;AAAA,IACN,YAAY,CAAC,WAAW;AAAA,IACxB,WAAW,QAAQ,aAAa;AAAA,IAChC,aAAa,IAAI,IAAK;AAAA,IACtB,cAAc,IAAI,IAAK;AAAA,IACvB,WAAW;AAAA,IACX,eAAe;AAAA,IACf,QAAQ,CAAC,EAAE,IAAI,QAAQ,MAAM,OAAO,CAAC;AAAA,IACrC,cAAc;AAAA,IACd,UAAU,QAAQ,YAAY;AAAA,IAC9B,YAAY,QAAQ,cAAc;AAAA,IAClC,gBAAgB;AAAA,IAChB,MAAM;AAAA,IACN,WAAW,QAAQ,aAAa,CAAC;AAAA,EACnC;AACA,QAAM,QAAuB,EAAE,OAAO,OAAO,CAAC,GAAG,QAAQ,QAAQ,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAC3F,SAAO,IAAI,SAAS,KAAK,GAAG,KAAK;AACjC,QAAM,kBAAkB,mBAAmB,KAAK;AAChD,QAAM,UAAU;AAAA,IACd;AAAA,IACA,MAAM;AAAA,IACN,MAAM,4BAA4B,EAAE,KAAK,MAAM,KAAK,SAAS,YAAY,CAAC;AAAA,IAC1E,MAAM,CAAC,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,CAAC,UAAU,MAAM,MAAM,QAAQ,MAAM,GAAG;AAAA,EAC1E;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb,IAAI,SAAS;AACX,aAAO,MAAM;AAAA,IACf;AAAA,IACA,KAAK,OAAO;AACV,UAAI,MAAM,QAAS,OAAM,QAAQ,KAAK,KAAK;AAC3C,UAAI,CAAC,MAAM,WAAW,MAAM,QAAQ,OAAQ,OAAM,QAAQ,KAAK,KAAK;AAAA,IACtE;AAAA,IACA,SAAS,OAAO;AACd,UAAI,MAAM,UAAW,OAAM,MAAM,YAAY,MAAM;AACnD,UAAI,MAAM,SAAU,OAAM,MAAM,WAAW,MAAM;AACjD,UAAI,MAAM,WAAY,OAAM,MAAM,aAAa,MAAM;AAAA,IACvD;AAAA,IACA,aAAa;AACX,UAAI,OAAO,IAAI,SAAS,KAAK,CAAC,MAAM,MAAO,QAAO,OAAO,SAAS,KAAK,CAAC;AACxE,sBAAgB;AAChB,cAAQ;AAAA,IACV;AAAA,EACF;AACF;AAEA,eAAe,YACb,OACA,UACA,SAC0B;AAC1B,QAAM,QAAQ,OAAO,IAAI,SAAS,KAAK,CAAC;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,+BAA+B,MAAM,QAAQ,IAAI,MAAM,EAAE,EAAE;AACvF,QAAM,SAAS,eAAe,MAAM,SAAS,SAAS,CAAC,UAAU,CAAC;AAClE,QAAM,UAAU,IAAI,YAAY,MAAM,OAAO,MAAM,KAAK;AACxD,QAAM,UAAU;AAChB,aAAW,SAAS,MAAM,QAAS,SAAQ,KAAK,KAAK;AACrD,QAAM,UAAU,CAAC;AACjB,aAAW,SAAS,MAAM,OAAQ,SAAQ,KAAK,KAAK;AACpD,SAAO;AACT;",
   "names": []
 }

package/dist/index.js

@@ -12,6 +12,7 @@ import {
   listVoiceModels,
   registerVoiceModel
 } from "./models.js";
+import { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from "./realtime.js";
 import {
   connectRealtime,
   listen,
@@ -22,14 +23,18 @@ import {
   transcribe
 } from "./registry.js";
 import { SpeechEventStream } from "./speech.js";
+import { askAgent, bindThinker, talkerTools } from "./talker.js";
 import { TranscriptionEventStream } from "./transcription.js";
 import { addAudioCounters, priceAudio } from "./usage.js";
 import { energyVad } from "./vad.js";
 export {
+  RealtimeLedger,
   SpeechEventStream,
   TranscriptionEventStream,
   VoiceRequestError,
   addAudioCounters,
+  askAgent,
+  bindThinker,
   blankUsage,
   connectRealtime,
   convertAudio,
@@ -40,13 +45,17 @@ export {
   getVoiceModel,
   listVoiceModels,
   listen,
+  planDelivery,
+  planTruncation,
   priceAudio,
+  realtimeSocket,
   registerRealtimeApiProvider,
   registerSpeechApiProvider,
   registerTranscriptionApiProvider,
   registerVoiceModel,
   resolveVoiceAuth,
   speak,
+  talkerTools,
   transcribe
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export type {\n  AudioChunk,\n  AudioEncoding,\n  AudioFormat,\n  AudioSink,\n  AudioSource,\n  AudioTag,\n  CharAlignment,\n  TextSpan,\n} from './audio.ts';\nexport { resolveVoiceAuth } from './auth.ts';\nexport { blankUsage } from './blank-usage.ts';\nexport {\n  convertAudio,\n  convertAudioSource,\n  decodeAudioBase64,\n  encodeAudioBase64,\n} from './convert.ts';\nexport { VoiceRequestError } from './errors.ts';\nexport type {\n  Delivery,\n  DeliverySupport,\n  KnownRealtimeApi,\n  KnownSpeechApi,\n  KnownTranscriptionApi,\n  RealtimeApi,\n  RealtimeModel,\n  SpeechApi,\n  SpeechModel,\n  TranscriptionApi,\n  TranscriptionModel,\n  VoiceInfo,\n  VoiceModel,\n  VoiceModelType,\n} from './models.ts';\nexport {\n  getVoiceModel,\n  listVoiceModels,\n  registerVoiceModel,\n} from './models.ts';\nexport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n  RealtimeToken,\n} from './realtime.ts';\nexport {\n  connectRealtime,\n  listen,\n  registerRealtimeApiProvider,\n  registerSpeechApiProvider,\n  registerTranscriptionApiProvider,\n  speak,\n  transcribe,\n} from './registry.ts';\nexport type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';\nexport { SpeechEventStream } from './speech.ts';\nexport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEvent,\n  TranscriptSegment,\n} from './transcription.ts';\nexport { TranscriptionEventStream } from './transcription.ts';\nexport type {\n  AudioCost,\n  AudioCounters,\n  AudioUsage,\n  RealtimeAudioCost,\n  SpeechAudioCost,\n  TranscriptionAudioCost,\n} from './usage.ts';\nexport { addAudioCounters, priceAudio } from './usage.ts';\nexport type { VadHook, VadUpdate } from './vad.ts';\nexport { energyVad } from './vad.ts';\n"],
-  "mappings": "AAUA,SAAS,wBAAwB;AACjC,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAiBlC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AASP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,yBAAyB;AASlC,SAAS,gCAAgC;AASzC,SAAS,kBAAkB,kBAAkB;AAE7C,SAAS,iBAAiB;",
+  "sourcesContent": ["export type {\n  AudioChunk,\n  AudioEncoding,\n  AudioFormat,\n  AudioSink,\n  AudioSource,\n  AudioTag,\n  CharAlignment,\n  TextSpan,\n} from './audio.ts';\nexport { resolveVoiceAuth } from './auth.ts';\nexport { blankUsage } from './blank-usage.ts';\nexport {\n  convertAudio,\n  convertAudioSource,\n  decodeAudioBase64,\n  encodeAudioBase64,\n} from './convert.ts';\nexport { VoiceRequestError } from './errors.ts';\nexport type {\n  Delivery,\n  DeliverySupport,\n  KnownRealtimeApi,\n  KnownSpeechApi,\n  KnownTranscriptionApi,\n  RealtimeApi,\n  RealtimeModel,\n  SpeechApi,\n  SpeechModel,\n  TranscriptionApi,\n  TranscriptionModel,\n  VoiceInfo,\n  VoiceModel,\n  VoiceModelType,\n} from './models.ts';\nexport {\n  getVoiceModel,\n  listVoiceModels,\n  registerVoiceModel,\n} from './models.ts';\nexport type {\n  DeliveryPlan,\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeEvent,\n  RealtimeOptions,\n  RealtimeSession,\n  RealtimeSocket,\n  RealtimeToken,\n  TruncationPlan,\n} from './realtime.ts';\nexport { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from './realtime.ts';\nexport {\n  connectRealtime,\n  listen,\n  registerRealtimeApiProvider,\n  registerSpeechApiProvider,\n  registerTranscriptionApiProvider,\n  speak,\n  transcribe,\n} from './registry.ts';\nexport type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';\nexport { SpeechEventStream } from './speech.ts';\nexport type {\n  BindThinkerOptions,\n  BoundThinker,\n  Thinker,\n  ThinkerCaption,\n  ThinkerPhase,\n  ThinkerUpdate,\n} from './talker.ts';\nexport { askAgent, bindThinker, talkerTools } from './talker.ts';\nexport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEvent,\n  TranscriptSegment,\n} from './transcription.ts';\nexport { TranscriptionEventStream } from './transcription.ts';\nexport type {\n  AudioCost,\n  AudioCounters,\n  AudioUsage,\n  RealtimeAudioCost,\n  SpeechAudioCost,\n  TranscriptionAudioCost,\n} from './usage.ts';\nexport { addAudioCounters, priceAudio } from './usage.ts';\nexport type { VadHook, VadUpdate } from './vad.ts';\nexport { energyVad } from './vad.ts';\n"],
+  "mappings": "AAUA,SAAS,wBAAwB;AACjC,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAiBlC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAYP,SAAS,cAAc,gBAAgB,gBAAgB,sBAAsB;AAC7E;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,yBAAyB;AASlC,SAAS,UAAU,aAAa,mBAAmB;AASnD,SAAS,gCAAgC;AASzC,SAAS,kBAAkB,kBAAkB;AAE7C,SAAS,iBAAiB;",
   "names": []
 }

package/dist/realtime.js

@@ -1 +1,94 @@
+import {
+  alignmentSeconds,
+  chunkSeconds
+} from "./audio.js";
+import { VoiceRequestError } from "./errors.js";
+import { priceAudio } from "./usage.js";
+function planDelivery(delivery, requested) {
+  const support = delivery[requested];
+  if (support !== "native" && support !== "approximate") {
+    throw new VoiceRequestError("error", `unsupported delivery: ${requested}`);
+  }
+  return {
+    delivery: requested,
+    support,
+    waitForIdle: requested === "when_idle" && support === "approximate"
+  };
+}
+function planTruncation(truncation, heard) {
+  if (truncation === "client" && heard !== void 0) {
+    return { item: { itemId: heard.itemId, audioEndMs: heard.heardMs } };
+  }
+  return {};
+}
+function realtimeSocket(url, options, protocols = []) {
+  const gateway = options?.gateway;
+  if (gateway === void 0) return { url, protocols: [...protocols] };
+  return {
+    url: gateway.url,
+    protocols: [`cf-aig-authorization.${gateway.token}`, ...protocols]
+  };
+}
+class RealtimeLedger {
+  #inputSeconds = 0;
+  #outputSeconds = 0;
+  #textMessages = 0;
+  #busy = false;
+  #held = [];
+  get inputSeconds() {
+    return this.#inputSeconds;
+  }
+  get outputSeconds() {
+    return this.#outputSeconds;
+  }
+  get textMessages() {
+    return this.#textMessages;
+  }
+  get busy() {
+    return this.#busy;
+  }
+  addInput(chunk) {
+    this.#inputSeconds += chunkSeconds(chunk);
+  }
+  addOutput(chunk, alignment) {
+    const seconds = chunkSeconds(chunk);
+    this.#outputSeconds += seconds > 0 ? seconds : alignmentSeconds(alignment);
+  }
+  addText() {
+    this.#textMessages += 1;
+  }
+  observe(event) {
+    if (event.type === "output_audio") this.addOutput(event.chunk, event.alignment);
+    if (event.type === "output_audio" || event.type === "output_transcript") this.#busy = true;
+    if (event.type === "response_done") this.#release();
+  }
+  submit(delivery, requested, run) {
+    const plan = planDelivery(delivery, requested);
+    if (plan.waitForIdle && this.#busy) this.#held.push(run);
+    else run();
+    return plan;
+  }
+  settle() {
+    this.#release();
+  }
+  usage(rates) {
+    return priceAudio(rates, {
+      inputSeconds: this.#inputSeconds,
+      outputSeconds: this.#outputSeconds,
+      textMessages: this.#textMessages
+    });
+  }
+  #release() {
+    this.#busy = false;
+    const held = this.#held;
+    this.#held = [];
+    for (const run of held) run();
+  }
+}
+export {
+  RealtimeLedger,
+  planDelivery,
+  planTruncation,
+  realtimeSocket
+};
 //# sourceMappingURL=realtime.js.map

package/dist/realtime.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
+  "sources": ["../src/realtime.ts"],
+  "sourcesContent": ["import type {\n  Message,\n  ProviderRequestOptions,\n  Tool,\n  ToolCall,\n  ToolResultMessage,\n  Usage,\n} from '@earendil-works/pi-ai';\nimport {\n  type AudioChunk,\n  alignmentSeconds,\n  type CharAlignment,\n  chunkSeconds,\n  type TextSpan,\n} from './audio.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { Delivery, DeliverySupport, RealtimeModel } from './models.ts';\nimport { priceAudio, type RealtimeAudioCost } from './usage.ts';\n\nexport interface RealtimeContext {\n  instructions: string;\n  tools: Tool[];\n  voice: string;\n  history?: Message[];\n}\n\nexport interface RealtimeToken {\n  value: string;\n  expiresAt: number;\n  model: string;\n}\n\nexport interface RealtimeOptions extends ProviderRequestOptions {\n  token?: RealtimeToken;\n  transport?: 'websocket' | 'webrtc';\n  gateway?: { url: string; token: string };\n  resumeHandle?: string;\n}\n\nexport type RealtimeEvent =\n  | { type: 'session_start'; sessionId: string }\n  | { type: 'input_speech_start' }\n  | { type: 'input_transcript'; itemId: string; text: string; final: boolean }\n  | {\n      type: 'output_audio';\n      itemId: string;\n      chunk: AudioChunk;\n      span?: TextSpan;\n      alignment?: CharAlignment;\n    }\n  | { type: 'output_transcript'; itemId: string; text: string; final: boolean }\n  | { type: 'tool_call'; call: ToolCall; blocking: boolean }\n  | { type: 'tool_call_cancelled'; callId: string }\n  | { type: 'response_done'; itemId: string; interrupted: boolean; usage: Usage }\n  | { type: 'go_away'; timeLeftMs: number }\n  | { type: 'resumption'; handle: string }\n  | { type: 'error'; error: string; fatal: boolean };\n\nexport interface RealtimeSession extends AsyncIterable<RealtimeEvent> {\n  readonly model: RealtimeModel<string>;\n  appendAudio(chunk: AudioChunk): void;\n  appendVideoFrame?(jpeg: Uint8Array): void;\n  sendText(text: string): void;\n  inject(text: string, delivery: Delivery): void;\n  respond(callId: string, result: ToolResultMessage, delivery?: Delivery): void;\n  interrupt(heard?: { itemId: string; heardMs: number }): void;\n  update(patch: Partial<Pick<RealtimeContext, 'instructions' | 'tools' | 'voice'>>): void;\n  close(): Promise<{ usage: Usage }>;\n}\n\nexport interface RealtimeApiProvider<TApi extends string = string> {\n  api: TApi;\n  connect: (\n    model: RealtimeModel<TApi>,\n    context: RealtimeContext,\n    options?: RealtimeOptions\n  ) => Promise<RealtimeSession>;\n}\n\nexport interface DeliveryPlan {\n  delivery: Delivery;\n  support: DeliverySupport;\n  waitForIdle: boolean;\n}\n\nexport interface TruncationPlan {\n  item?: { itemId: string; audioEndMs: number };\n}\n\nexport interface RealtimeSocket {\n  url: string;\n  protocols: string[];\n}\n\nexport function planDelivery(\n  delivery: Record<Delivery, DeliverySupport>,\n  requested: Delivery\n): DeliveryPlan {\n  const support = delivery[requested];\n  if (support !== 'native' && support !== 'approximate') {\n    throw new VoiceRequestError('error', `unsupported delivery: ${requested}`);\n  }\n  return {\n    delivery: requested,\n    support,\n    waitForIdle: requested === 'when_idle' && support === 'approximate',\n  };\n}\n\nexport function planTruncation(\n  truncation: RealtimeModel<string>['truncation'],\n  heard?: { itemId: string; heardMs: number }\n): TruncationPlan {\n  if (truncation === 'client' && heard !== undefined) {\n    return { item: { itemId: heard.itemId, audioEndMs: heard.heardMs } };\n  }\n  return {};\n}\n\nexport function realtimeSocket(\n  url: string,\n  options?: Pick<RealtimeOptions, 'gateway'>,\n  protocols: readonly string[] = []\n): RealtimeSocket {\n  const gateway = options?.gateway;\n  if (gateway === undefined) return { url, protocols: [...protocols] };\n  return {\n    url: gateway.url,\n    protocols: [`cf-aig-authorization.${gateway.token}`, ...protocols],\n  };\n}\n\nexport class RealtimeLedger {\n  #inputSeconds = 0;\n  #outputSeconds = 0;\n  #textMessages = 0;\n  #busy = false;\n  #held: Array<() => void> = [];\n\n  get inputSeconds(): number {\n    return this.#inputSeconds;\n  }\n\n  get outputSeconds(): number {\n    return this.#outputSeconds;\n  }\n\n  get textMessages(): number {\n    return this.#textMessages;\n  }\n\n  get busy(): boolean {\n    return this.#busy;\n  }\n\n  addInput(chunk: AudioChunk): void {\n    this.#inputSeconds += chunkSeconds(chunk);\n  }\n\n  addOutput(chunk: AudioChunk, alignment?: CharAlignment): void {\n    const seconds = chunkSeconds(chunk);\n    this.#outputSeconds += seconds > 0 ? seconds : alignmentSeconds(alignment);\n  }\n\n  addText(): void {\n    this.#textMessages += 1;\n  }\n\n  observe(event: RealtimeEvent): void {\n    if (event.type === 'output_audio') this.addOutput(event.chunk, event.alignment);\n    if (event.type === 'output_audio' || event.type === 'output_transcript') this.#busy = true;\n    if (event.type === 'response_done') this.#release();\n  }\n\n  submit(\n    delivery: Record<Delivery, DeliverySupport>,\n    requested: Delivery,\n    run: () => void\n  ): DeliveryPlan {\n    const plan = planDelivery(delivery, requested);\n    if (plan.waitForIdle && this.#busy) this.#held.push(run);\n    else run();\n    return plan;\n  }\n\n  settle(): void {\n    this.#release();\n  }\n\n  usage(rates: RealtimeAudioCost): Usage {\n    return priceAudio(rates, {\n      inputSeconds: this.#inputSeconds,\n      outputSeconds: this.#outputSeconds,\n      textMessages: this.#textMessages,\n    });\n  }\n\n  #release(): void {\n    this.#busy = false;\n    const held = this.#held;\n    this.#held = [];\n    for (const run of held) run();\n  }\n}\n"],
+  "mappings": "AAQA;AAAA,EAEE;AAAA,EAEA;AAAA,OAEK;AACP,SAAS,yBAAyB;AAElC,SAAS,kBAA0C;AA6E5C,SAAS,aACd,UACA,WACc;AACd,QAAM,UAAU,SAAS,SAAS;AAClC,MAAI,YAAY,YAAY,YAAY,eAAe;AACrD,UAAM,IAAI,kBAAkB,SAAS,yBAAyB,SAAS,EAAE;AAAA,EAC3E;AACA,SAAO;AAAA,IACL,UAAU;AAAA,IACV;AAAA,IACA,aAAa,cAAc,eAAe,YAAY;AAAA,EACxD;AACF;AAEO,SAAS,eACd,YACA,OACgB;AAChB,MAAI,eAAe,YAAY,UAAU,QAAW;AAClD,WAAO,EAAE,MAAM,EAAE,QAAQ,MAAM,QAAQ,YAAY,MAAM,QAAQ,EAAE;AAAA,EACrE;AACA,SAAO,CAAC;AACV;AAEO,SAAS,eACd,KACA,SACA,YAA+B,CAAC,GAChB;AAChB,QAAM,UAAU,SAAS;AACzB,MAAI,YAAY,OAAW,QAAO,EAAE,KAAK,WAAW,CAAC,GAAG,SAAS,EAAE;AACnE,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,WAAW,CAAC,wBAAwB,QAAQ,KAAK,IAAI,GAAG,SAAS;AAAA,EACnE;AACF;AAEO,MAAM,eAAe;AAAA,EAC1B,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACjB,gBAAgB;AAAA,EAChB,QAAQ;AAAA,EACR,QAA2B,CAAC;AAAA,EAE5B,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,SAAS,OAAyB;AAChC,SAAK,iBAAiB,aAAa,KAAK;AAAA,EAC1C;AAAA,EAEA,UAAU,OAAmB,WAAiC;AAC5D,UAAM,UAAU,aAAa,KAAK;AAClC,SAAK,kBAAkB,UAAU,IAAI,UAAU,iBAAiB,SAAS;AAAA,EAC3E;AAAA,EAEA,UAAgB;AACd,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,QAAQ,OAA4B;AAClC,QAAI,MAAM,SAAS,eAAgB,MAAK,UAAU,MAAM,OAAO,MAAM,SAAS;AAC9E,QAAI,MAAM,SAAS,kBAAkB,MAAM,SAAS,oBAAqB,MAAK,QAAQ;AACtF,QAAI,MAAM,SAAS,gBAAiB,MAAK,SAAS;AAAA,EACpD;AAAA,EAEA,OACE,UACA,WACA,KACc;AACd,UAAM,OAAO,aAAa,UAAU,SAAS;AAC7C,QAAI,KAAK,eAAe,KAAK,MAAO,MAAK,MAAM,KAAK,GAAG;AAAA,QAClD,KAAI;AACT,WAAO;AAAA,EACT;AAAA,EAEA,SAAe;AACb,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,OAAiC;AACrC,WAAO,WAAW,OAAO;AAAA,MACvB,cAAc,KAAK;AAAA,MACnB,eAAe,KAAK;AAAA,MACpB,cAAc,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,WAAiB;AACf,SAAK,QAAQ;AACb,UAAM,OAAO,KAAK;AAClB,SAAK,QAAQ,CAAC;AACd,eAAW,OAAO,KAAM,KAAI;AAAA,EAC9B;AACF;",
   "names": []
 }

package/dist/registry.js

@@ -1,4 +1,5 @@
 import { speakInSentences } from "./chunk-speech.js";
+import { VoiceRequestError } from "./errors.js";
 import { listenWithVad } from "./vad-listen.js";
 const transcriptionApis = /* @__PURE__ */ new Map();
 const speechApis = /* @__PURE__ */ new Map();
@@ -45,6 +46,14 @@ function speak(model, input, options) {
   return speakModel(model, input, options);
 }
 function connectRealtime(model, context, options) {
+  if (options?.transport !== void 0 && !model.transports.includes(options.transport)) {
+    return Promise.reject(
+      new VoiceRequestError("error", `unsupported transport: ${options.transport}`)
+    );
+  }
+  if (options?.resumeHandle !== void 0 && !model.resumable) {
+    return Promise.reject(new VoiceRequestError("error", "session is not resumable"));
+  }
   const connect = realtimeApis.get(model.api)?.provider.connect;
   if (!connect) return Promise.reject(missing(model.api));
   return connect(model, context, options);

package/dist/registry.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/registry.ts"],
-  "sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioSource } from './audio.ts';\nimport { speakInSentences } from './chunk-speech.ts';\nimport type { RealtimeModel, SpeechModel, TranscriptionModel } from './models.ts';\nimport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeOptions,\n  RealtimeSession,\n} from './realtime.ts';\nimport type { SpeechApiProvider, SpeechEventStream, SpeechInput } from './speech.ts';\nimport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEventStream,\n} from './transcription.ts';\nimport { listenWithVad } from './vad-listen.ts';\n\ninterface TranscriptionEntry {\n  provider: TranscriptionApiProvider;\n}\n\ninterface SpeechEntry {\n  provider: SpeechApiProvider;\n}\n\ninterface RealtimeEntry {\n  provider: RealtimeApiProvider;\n}\n\nconst transcriptionApis = new Map<string, TranscriptionEntry>();\nconst speechApis = new Map<string, SpeechEntry>();\nconst realtimeApis = new Map<string, RealtimeEntry>();\n\nfunction missing(api: string): Error {\n  return new Error(`No API provider registered for api: ${api}`);\n}\n\nexport function registerTranscriptionApiProvider(provider: TranscriptionApiProvider): () => void {\n  const entry = { provider };\n  transcriptionApis.set(provider.api, entry);\n  return () => {\n    if (transcriptionApis.get(provider.api) === entry) transcriptionApis.delete(provider.api);\n  };\n}\n\nexport function registerSpeechApiProvider(provider: SpeechApiProvider): () => void {\n  const entry = { provider };\n  speechApis.set(provider.api, entry);\n  return () => {\n    if (speechApis.get(provider.api) === entry) speechApis.delete(provider.api);\n  };\n}\n\nexport function registerRealtimeApiProvider(provider: RealtimeApiProvider): () => void {\n  const entry = { provider };\n  realtimeApis.set(provider.api, entry);\n  return () => {\n    if (realtimeApis.get(provider.api) === entry) realtimeApis.delete(provider.api);\n  };\n}\n\nexport function transcribe(\n  model: TranscriptionModel<string>,\n  input: AudioInput,\n  options?: ProviderRequestOptions\n): Promise<Transcript> {\n  const transcribeModel = transcriptionApis.get(model.api)?.provider.transcribe;\n  if (!transcribeModel) return Promise.reject(missing(model.api));\n  return transcribeModel(model, input, options);\n}\n\nexport function listen(\n  model: TranscriptionModel<string>,\n  audio: AudioSource,\n  options?: ListenOptions\n): TranscriptionEventStream {\n  const listenModel = transcriptionApis.get(model.api)?.provider.listen;\n  if (!listenModel) throw missing(model.api);\n  if (model.turnDetection === 'none') return listenWithVad(listenModel, model, audio, options);\n  return listenModel(model, audio, options);\n}\n\nexport function speak(\n  model: SpeechModel<string>,\n  input: SpeechInput,\n  options?: ProviderRequestOptions\n): SpeechEventStream {\n  const speakModel = speechApis.get(model.api)?.provider.speak;\n  if (!speakModel) throw missing(model.api);\n  if (!model.textStreaming) return speakInSentences(speakModel, model, input, options);\n  return speakModel(model, input, options);\n}\n\nexport function connectRealtime(\n  model: RealtimeModel<string>,\n  context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  const connect = realtimeApis.get(model.api)?.provider.connect;\n  if (!connect) return Promise.reject(missing(model.api));\n  return connect(model, context, options);\n}\n"],
-  "mappings": "AAEA,SAAS,wBAAwB;AAgBjC,SAAS,qBAAqB;AAc9B,MAAM,oBAAoB,oBAAI,IAAgC;AAC9D,MAAM,aAAa,oBAAI,IAAyB;AAChD,MAAM,eAAe,oBAAI,IAA2B;AAEpD,SAAS,QAAQ,KAAoB;AACnC,SAAO,IAAI,MAAM,uCAAuC,GAAG,EAAE;AAC/D;AAEO,SAAS,iCAAiC,UAAgD;AAC/F,QAAM,QAAQ,EAAE,SAAS;AACzB,oBAAkB,IAAI,SAAS,KAAK,KAAK;AACzC,SAAO,MAAM;AACX,QAAI,kBAAkB,IAAI,SAAS,GAAG,MAAM,MAAO,mBAAkB,OAAO,SAAS,GAAG;AAAA,EAC1F;AACF;AAEO,SAAS,0BAA0B,UAAyC;AACjF,QAAM,QAAQ,EAAE,SAAS;AACzB,aAAW,IAAI,SAAS,KAAK,KAAK;AAClC,SAAO,MAAM;AACX,QAAI,WAAW,IAAI,SAAS,GAAG,MAAM,MAAO,YAAW,OAAO,SAAS,GAAG;AAAA,EAC5E;AACF;AAEO,SAAS,4BAA4B,UAA2C;AACrF,QAAM,QAAQ,EAAE,SAAS;AACzB,eAAa,IAAI,SAAS,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,aAAa,IAAI,SAAS,GAAG,MAAM,MAAO,cAAa,OAAO,SAAS,GAAG;AAAA,EAChF;AACF;AAEO,SAAS,WACd,OACA,OACA,SACqB;AACrB,QAAM,kBAAkB,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AACnE,MAAI,CAAC,gBAAiB,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AAC9D,SAAO,gBAAgB,OAAO,OAAO,OAAO;AAC9C;AAEO,SAAS,OACd,OACA,OACA,SAC0B;AAC1B,QAAM,cAAc,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AAC/D,MAAI,CAAC,YAAa,OAAM,QAAQ,MAAM,GAAG;AACzC,MAAI,MAAM,kBAAkB,OAAQ,QAAO,cAAc,aAAa,OAAO,OAAO,OAAO;AAC3F,SAAO,YAAY,OAAO,OAAO,OAAO;AAC1C;AAEO,SAAS,MACd,OACA,OACA,SACmB;AACnB,QAAM,aAAa,WAAW,IAAI,MAAM,GAAG,GAAG,SAAS;AACvD,MAAI,CAAC,WAAY,OAAM,QAAQ,MAAM,GAAG;AACxC,MAAI,CAAC,MAAM,cAAe,QAAO,iBAAiB,YAAY,OAAO,OAAO,OAAO;AACnF,SAAO,WAAW,OAAO,OAAO,OAAO;AACzC;AAEO,SAAS,gBACd,OACA,SACA,SAC0B;AAC1B,QAAM,UAAU,aAAa,IAAI,MAAM,GAAG,GAAG,SAAS;AACtD,MAAI,CAAC,QAAS,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AACtD,SAAO,QAAQ,OAAO,SAAS,OAAO;AACxC;",
+  "sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioSource } from './audio.ts';\nimport { speakInSentences } from './chunk-speech.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { RealtimeModel, SpeechModel, TranscriptionModel } from './models.ts';\nimport type {\n  RealtimeApiProvider,\n  RealtimeContext,\n  RealtimeOptions,\n  RealtimeSession,\n} from './realtime.ts';\nimport type { SpeechApiProvider, SpeechEventStream, SpeechInput } from './speech.ts';\nimport type {\n  AudioInput,\n  ListenOptions,\n  Transcript,\n  TranscriptionApiProvider,\n  TranscriptionEventStream,\n} from './transcription.ts';\nimport { listenWithVad } from './vad-listen.ts';\n\ninterface TranscriptionEntry {\n  provider: TranscriptionApiProvider;\n}\n\ninterface SpeechEntry {\n  provider: SpeechApiProvider;\n}\n\ninterface RealtimeEntry {\n  provider: RealtimeApiProvider;\n}\n\nconst transcriptionApis = new Map<string, TranscriptionEntry>();\nconst speechApis = new Map<string, SpeechEntry>();\nconst realtimeApis = new Map<string, RealtimeEntry>();\n\nfunction missing(api: string): Error {\n  return new Error(`No API provider registered for api: ${api}`);\n}\n\nexport function registerTranscriptionApiProvider(provider: TranscriptionApiProvider): () => void {\n  const entry = { provider };\n  transcriptionApis.set(provider.api, entry);\n  return () => {\n    if (transcriptionApis.get(provider.api) === entry) transcriptionApis.delete(provider.api);\n  };\n}\n\nexport function registerSpeechApiProvider(provider: SpeechApiProvider): () => void {\n  const entry = { provider };\n  speechApis.set(provider.api, entry);\n  return () => {\n    if (speechApis.get(provider.api) === entry) speechApis.delete(provider.api);\n  };\n}\n\nexport function registerRealtimeApiProvider(provider: RealtimeApiProvider): () => void {\n  const entry = { provider };\n  realtimeApis.set(provider.api, entry);\n  return () => {\n    if (realtimeApis.get(provider.api) === entry) realtimeApis.delete(provider.api);\n  };\n}\n\nexport function transcribe(\n  model: TranscriptionModel<string>,\n  input: AudioInput,\n  options?: ProviderRequestOptions\n): Promise<Transcript> {\n  const transcribeModel = transcriptionApis.get(model.api)?.provider.transcribe;\n  if (!transcribeModel) return Promise.reject(missing(model.api));\n  return transcribeModel(model, input, options);\n}\n\nexport function listen(\n  model: TranscriptionModel<string>,\n  audio: AudioSource,\n  options?: ListenOptions\n): TranscriptionEventStream {\n  const listenModel = transcriptionApis.get(model.api)?.provider.listen;\n  if (!listenModel) throw missing(model.api);\n  if (model.turnDetection === 'none') return listenWithVad(listenModel, model, audio, options);\n  return listenModel(model, audio, options);\n}\n\nexport function speak(\n  model: SpeechModel<string>,\n  input: SpeechInput,\n  options?: ProviderRequestOptions\n): SpeechEventStream {\n  const speakModel = speechApis.get(model.api)?.provider.speak;\n  if (!speakModel) throw missing(model.api);\n  if (!model.textStreaming) return speakInSentences(speakModel, model, input, options);\n  return speakModel(model, input, options);\n}\n\nexport function connectRealtime(\n  model: RealtimeModel<string>,\n  context: RealtimeContext,\n  options?: RealtimeOptions\n): Promise<RealtimeSession> {\n  if (options?.transport !== undefined && !model.transports.includes(options.transport)) {\n    return Promise.reject(\n      new VoiceRequestError('error', `unsupported transport: ${options.transport}`)\n    );\n  }\n  if (options?.resumeHandle !== undefined && !model.resumable) {\n    return Promise.reject(new VoiceRequestError('error', 'session is not resumable'));\n  }\n  const connect = realtimeApis.get(model.api)?.provider.connect;\n  if (!connect) return Promise.reject(missing(model.api));\n  return connect(model, context, options);\n}\n"],
+  "mappings": "AAEA,SAAS,wBAAwB;AACjC,SAAS,yBAAyB;AAgBlC,SAAS,qBAAqB;AAc9B,MAAM,oBAAoB,oBAAI,IAAgC;AAC9D,MAAM,aAAa,oBAAI,IAAyB;AAChD,MAAM,eAAe,oBAAI,IAA2B;AAEpD,SAAS,QAAQ,KAAoB;AACnC,SAAO,IAAI,MAAM,uCAAuC,GAAG,EAAE;AAC/D;AAEO,SAAS,iCAAiC,UAAgD;AAC/F,QAAM,QAAQ,EAAE,SAAS;AACzB,oBAAkB,IAAI,SAAS,KAAK,KAAK;AACzC,SAAO,MAAM;AACX,QAAI,kBAAkB,IAAI,SAAS,GAAG,MAAM,MAAO,mBAAkB,OAAO,SAAS,GAAG;AAAA,EAC1F;AACF;AAEO,SAAS,0BAA0B,UAAyC;AACjF,QAAM,QAAQ,EAAE,SAAS;AACzB,aAAW,IAAI,SAAS,KAAK,KAAK;AAClC,SAAO,MAAM;AACX,QAAI,WAAW,IAAI,SAAS,GAAG,MAAM,MAAO,YAAW,OAAO,SAAS,GAAG;AAAA,EAC5E;AACF;AAEO,SAAS,4BAA4B,UAA2C;AACrF,QAAM,QAAQ,EAAE,SAAS;AACzB,eAAa,IAAI,SAAS,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,aAAa,IAAI,SAAS,GAAG,MAAM,MAAO,cAAa,OAAO,SAAS,GAAG;AAAA,EAChF;AACF;AAEO,SAAS,WACd,OACA,OACA,SACqB;AACrB,QAAM,kBAAkB,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AACnE,MAAI,CAAC,gBAAiB,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AAC9D,SAAO,gBAAgB,OAAO,OAAO,OAAO;AAC9C;AAEO,SAAS,OACd,OACA,OACA,SAC0B;AAC1B,QAAM,cAAc,kBAAkB,IAAI,MAAM,GAAG,GAAG,SAAS;AAC/D,MAAI,CAAC,YAAa,OAAM,QAAQ,MAAM,GAAG;AACzC,MAAI,MAAM,kBAAkB,OAAQ,QAAO,cAAc,aAAa,OAAO,OAAO,OAAO;AAC3F,SAAO,YAAY,OAAO,OAAO,OAAO;AAC1C;AAEO,SAAS,MACd,OACA,OACA,SACmB;AACnB,QAAM,aAAa,WAAW,IAAI,MAAM,GAAG,GAAG,SAAS;AACvD,MAAI,CAAC,WAAY,OAAM,QAAQ,MAAM,GAAG;AACxC,MAAI,CAAC,MAAM,cAAe,QAAO,iBAAiB,YAAY,OAAO,OAAO,OAAO;AACnF,SAAO,WAAW,OAAO,OAAO,OAAO;AACzC;AAEO,SAAS,gBACd,OACA,SACA,SAC0B;AAC1B,MAAI,SAAS,cAAc,UAAa,CAAC,MAAM,WAAW,SAAS,QAAQ,SAAS,GAAG;AACrF,WAAO,QAAQ;AAAA,MACb,IAAI,kBAAkB,SAAS,0BAA0B,QAAQ,SAAS,EAAE;AAAA,IAC9E;AAAA,EACF;AACA,MAAI,SAAS,iBAAiB,UAAa,CAAC,MAAM,WAAW;AAC3D,WAAO,QAAQ,OAAO,IAAI,kBAAkB,SAAS,0BAA0B,CAAC;AAAA,EAClF;AACA,QAAM,UAAU,aAAa,IAAI,MAAM,GAAG,GAAG,SAAS;AACtD,MAAI,CAAC,QAAS,QAAO,QAAQ,OAAO,QAAQ,MAAM,GAAG,CAAC;AACtD,SAAO,QAAQ,OAAO,SAAS,OAAO;AACxC;",
   "names": []
 }

package/dist/talker.js

@@ -0,0 +1,192 @@
+import {
+  Type
+} from "@earendil-works/pi-ai";
+import { VoiceRequestError } from "./errors.js";
+const heardLimit = 20;
+const talkerTools = ["ask_agent", "read"];
+const askAgent = {
+  name: "ask_agent",
+  description: "Hand a request to the coding agent. Use for anything that needs files, the browser, tools or more than a few seconds of thought.",
+  parameters: Type.Object({
+    request: Type.String(),
+    context: Type.Optional(Type.String())
+  })
+};
+class PushStream {
+  #queue = [];
+  #waiters = [];
+  #closed = false;
+  push(value) {
+    if (this.#closed) return;
+    const waiter = this.#waiters.shift();
+    if (waiter) waiter({ value, done: false });
+    else this.#queue.push(value);
+  }
+  close() {
+    if (this.#closed) return;
+    this.#closed = true;
+    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });
+  }
+  async *[Symbol.asyncIterator]() {
+    while (true) {
+      const queued = this.#queue.shift();
+      if (queued !== void 0) {
+        yield queued;
+        continue;
+      }
+      if (this.#closed) return;
+      const next = await new Promise(
+        (resolve) => this.#waiters.push(resolve)
+      );
+      if (next.done || next.value === void 0) return;
+      yield next.value;
+    }
+  }
+}
+function send(action) {
+  try {
+    action();
+  } catch (error) {
+    if (!(error instanceof VoiceRequestError)) throw error;
+  }
+}
+function textOf(message) {
+  return message.content.filter((part) => part.type === "text").map((part) => part.text).join("\n");
+}
+function acknowledged(call, ack) {
+  return {
+    role: "toolResult",
+    toolCallId: call.id,
+    toolName: call.name,
+    content: [{ type: "text", text: ack }],
+    isError: false,
+    timestamp: Date.now()
+  };
+}
+function errorResult(call, error) {
+  return {
+    role: "toolResult",
+    toolCallId: call.id,
+    toolName: call.name,
+    content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
+    isError: true,
+    timestamp: Date.now()
+  };
+}
+function capResult(result, maxChars) {
+  let left = maxChars;
+  const content = result.content.map((part) => {
+    if (part.type !== "text") return part;
+    if (left <= 0) return { ...part, text: "" };
+    if (part.text.length <= left) {
+      left -= part.text.length;
+      return part;
+    }
+    const text = part.text.slice(0, left);
+    left = 0;
+    return { ...part, text };
+  });
+  return { ...result, content };
+}
+function bindThinker(session, thinker, direct, options) {
+  const captions = new PushStream();
+  const phases = new PushStream();
+  const done = run(session, thinker, direct, options, captions, phases).finally(() => {
+    captions.close();
+    phases.close();
+    captions.close();
+    phases.close();
+  });
+  return { captions, phases, done };
+}
+async function run(session, thinker, direct, options, captions, phases) {
+  const heard = [];
+  const running = /* @__PURE__ */ new Map();
+  for await (const event of session) {
+    if (event.type === "input_transcript" && event.final) heard.push(`user: ${event.text}`);
+    if (event.type === "output_transcript" && event.final) heard.push(`talker: ${event.text}`);
+    if (event.type === "tool_call_cancelled") {
+      const job2 = running.get(event.callId);
+      if (!job2) continue;
+      job2.abort.abort();
+      if (!job2.answered) {
+        job2.answered = true;
+        send(() => session.respond(event.callId, errorResult(job2.call, new Error("cancelled"))));
+      }
+      continue;
+    }
+    if (event.type !== "tool_call") continue;
+    const call = event.call;
+    if (call.name !== askAgent.name) {
+      phases.push({
+        phase: "tool",
+        toolName: call.name,
+        toolCallId: call.id,
+        args: call.arguments
+      });
+      void direct(call, AbortSignal.timeout(options.directTimeoutMs)).then((result) => {
+        send(() => session.respond(call.id, capResult(result, options.directMaxChars)));
+        phases.push({ phase: "idle" });
+      }).catch((error) => {
+        send(() => session.respond(call.id, errorResult(call, error)));
+        phases.push({ phase: "idle" });
+      });
+      continue;
+    }
+    const job = { abort: new AbortController(), answered: event.blocking, call };
+    running.set(call.id, job);
+    phases.push({ phase: "thinking" });
+    if (event.blocking) send(() => session.respond(call.id, acknowledged(call, options.ack)));
+    void followAsk(
+      session,
+      thinker,
+      options,
+      captions,
+      phases,
+      running,
+      heard,
+      call,
+      event.blocking,
+      job
+    );
+  }
+}
+async function followAsk(session, thinker, options, captions, phases, running, heard, call, blocking, job) {
+  try {
+    for await (const update of thinker.ask(
+      String(call.arguments.request),
+      heard.slice(-heardLimit),
+      job.abort.signal
+    )) {
+      if (job.abort.signal.aborted) break;
+      if (update.type === "progress") {
+        captions.push({ type: "thinker_progress", callId: call.id, text: update.text });
+        send(() => session.inject(update.text, options.progress));
+        continue;
+      }
+      job.answered = true;
+      captions.push({
+        type: "thinker_result",
+        callId: call.id,
+        summary: textOf(update.message),
+        isError: update.message.isError
+      });
+      if (blocking) send(() => session.inject(textOf(update.message), options.result));
+      else send(() => session.respond(call.id, update.message, options.result));
+    }
+  } catch (error) {
+    if (!job.abort.signal.aborted && !job.answered) {
+      job.answered = true;
+      send(() => session.respond(call.id, errorResult(call, error)));
+    }
+  } finally {
+    running.delete(call.id);
+    phases.push({ phase: "idle" });
+  }
+}
+export {
+  askAgent,
+  bindThinker,
+  talkerTools
+};
+//# sourceMappingURL=talker.js.map

package/dist/talker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/talker.ts"],
+  "sourcesContent": ["import {\n  type JsonObject,\n  type Tool,\n  type ToolCall,\n  type ToolResultMessage,\n  Type,\n} from '@earendil-works/pi-ai';\nimport { VoiceRequestError } from './errors.ts';\nimport type { Delivery } from './models.ts';\nimport type { RealtimeSession } from './realtime.ts';\n\nconst heardLimit = 20;\n\nexport const talkerTools = ['ask_agent', 'read'] as const;\n\nexport const askAgent: Tool = {\n  name: 'ask_agent',\n  description:\n    'Hand a request to the coding agent. Use for anything that needs files, the browser, tools or more than a few seconds of thought.',\n  parameters: Type.Object({\n    request: Type.String(),\n    context: Type.Optional(Type.String()),\n  }),\n};\n\nexport type ThinkerUpdate =\n  | { type: 'progress'; text: string }\n  | { type: 'result'; message: ToolResultMessage };\n\nexport interface Thinker {\n  ask(request: string, heard: string[], signal: AbortSignal): AsyncIterable<ThinkerUpdate>;\n}\n\nexport interface BindThinkerOptions {\n  progress: Delivery;\n  result: Delivery;\n  ack: string;\n  directTimeoutMs: number;\n  directMaxChars: number;\n}\n\nexport type ThinkerCaption =\n  | { type: 'thinker_progress'; callId: string; text: string }\n  | { type: 'thinker_result'; callId: string; summary: string; isError: boolean };\n\nexport type ThinkerPhase =\n  | { phase: 'thinking' }\n  | { phase: 'tool'; toolName: string; toolCallId: string; args: JsonObject }\n  | { phase: 'idle' };\n\nexport interface BoundThinker {\n  captions: AsyncIterable<ThinkerCaption>;\n  phases: AsyncIterable<ThinkerPhase>;\n  done: Promise<void>;\n}\n\nclass PushStream<T> implements AsyncIterable<T> {\n  readonly #queue: T[] = [];\n  readonly #waiters: Array<(result: { value?: T; done: boolean }) => void> = [];\n  #closed = false;\n\n  push(value: T): void {\n    if (this.#closed) return;\n    const waiter = this.#waiters.shift();\n    if (waiter) waiter({ value, done: false });\n    else this.#queue.push(value);\n  }\n\n  close(): void {\n    if (this.#closed) return;\n    this.#closed = true;\n    while (this.#waiters.length > 0) this.#waiters.shift()?.({ done: true });\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<T> {\n    while (true) {\n      const queued = this.#queue.shift();\n      if (queued !== undefined) {\n        yield queued;\n        continue;\n      }\n      if (this.#closed) return;\n      const next = await new Promise<{ value?: T; done: boolean }>((resolve) =>\n        this.#waiters.push(resolve)\n      );\n      if (next.done || next.value === undefined) return;\n      yield next.value;\n    }\n  }\n}\n\nfunction send(action: () => void): void {\n  try {\n    action();\n  } catch (error) {\n    if (!(error instanceof VoiceRequestError)) throw error;\n  }\n}\n\nfunction textOf(message: ToolResultMessage): string {\n  return message.content\n    .filter((part) => part.type === 'text')\n    .map((part) => part.text)\n    .join('\\n');\n}\n\nfunction acknowledged(call: ToolCall, ack: string): ToolResultMessage {\n  return {\n    role: 'toolResult',\n    toolCallId: call.id,\n    toolName: call.name,\n    content: [{ type: 'text', text: ack }],\n    isError: false,\n    timestamp: Date.now(),\n  };\n}\n\nfunction errorResult(call: ToolCall, error: unknown): ToolResultMessage {\n  return {\n    role: 'toolResult',\n    toolCallId: call.id,\n    toolName: call.name,\n    content: [{ type: 'text', text: error instanceof Error ? error.message : String(error) }],\n    isError: true,\n    timestamp: Date.now(),\n  };\n}\n\nfunction capResult(result: ToolResultMessage, maxChars: number): ToolResultMessage {\n  let left = maxChars;\n  const content = result.content.map((part) => {\n    if (part.type !== 'text') return part;\n    if (left <= 0) return { ...part, text: '' };\n    if (part.text.length <= left) {\n      left -= part.text.length;\n      return part;\n    }\n    const text = part.text.slice(0, left);\n    left = 0;\n    return { ...part, text };\n  });\n  return { ...result, content };\n}\n\ninterface AskJob {\n  abort: AbortController;\n  answered: boolean;\n  call: ToolCall;\n}\n\nexport function bindThinker(\n  session: RealtimeSession,\n  thinker: Thinker,\n  direct: (call: ToolCall, signal: AbortSignal) => Promise<ToolResultMessage>,\n  options: BindThinkerOptions\n): BoundThinker {\n  const captions = new PushStream<ThinkerCaption>();\n  const phases = new PushStream<ThinkerPhase>();\n  const done = run(session, thinker, direct, options, captions, phases).finally(() => {\n    captions.close();\n    phases.close();\n    captions.close();\n    phases.close();\n  });\n  return { captions, phases, done };\n}\n\nasync function run(\n  session: RealtimeSession,\n  thinker: Thinker,\n  direct: (call: ToolCall, signal: AbortSignal) => Promise<ToolResultMessage>,\n  options: BindThinkerOptions,\n  captions: PushStream<ThinkerCaption>,\n  phases: PushStream<ThinkerPhase>\n): Promise<void> {\n  const heard: string[] = [];\n  const running = new Map<string, AskJob>();\n  for await (const event of session) {\n    if (event.type === 'input_transcript' && event.final) heard.push(`user: ${event.text}`);\n    if (event.type === 'output_transcript' && event.final) heard.push(`talker: ${event.text}`);\n    if (event.type === 'tool_call_cancelled') {\n      const job = running.get(event.callId);\n      if (!job) continue;\n      job.abort.abort();\n      if (!job.answered) {\n        job.answered = true;\n        send(() => session.respond(event.callId, errorResult(job.call, new Error('cancelled'))));\n      }\n      continue;\n    }\n    if (event.type !== 'tool_call') continue;\n    const call = event.call;\n    if (call.name !== askAgent.name) {\n      phases.push({\n        phase: 'tool',\n        toolName: call.name,\n        toolCallId: call.id,\n        args: call.arguments,\n      });\n      void direct(call, AbortSignal.timeout(options.directTimeoutMs))\n        .then((result) => {\n          send(() => session.respond(call.id, capResult(result, options.directMaxChars)));\n          phases.push({ phase: 'idle' });\n        })\n        .catch((error: unknown) => {\n          send(() => session.respond(call.id, errorResult(call, error)));\n          phases.push({ phase: 'idle' });\n        });\n      continue;\n    }\n    const job: AskJob = { abort: new AbortController(), answered: event.blocking, call };\n    running.set(call.id, job);\n    phases.push({ phase: 'thinking' });\n    if (event.blocking) send(() => session.respond(call.id, acknowledged(call, options.ack)));\n    void followAsk(\n      session,\n      thinker,\n      options,\n      captions,\n      phases,\n      running,\n      heard,\n      call,\n      event.blocking,\n      job\n    );\n  }\n}\n\nasync function followAsk(\n  session: RealtimeSession,\n  thinker: Thinker,\n  options: BindThinkerOptions,\n  captions: PushStream<ThinkerCaption>,\n  phases: PushStream<ThinkerPhase>,\n  running: Map<string, AskJob>,\n  heard: string[],\n  call: ToolCall,\n  blocking: boolean,\n  job: AskJob\n): Promise<void> {\n  try {\n    for await (const update of thinker.ask(\n      String(call.arguments.request),\n      heard.slice(-heardLimit),\n      job.abort.signal\n    )) {\n      if (job.abort.signal.aborted) break;\n      if (update.type === 'progress') {\n        captions.push({ type: 'thinker_progress', callId: call.id, text: update.text });\n        send(() => session.inject(update.text, options.progress));\n        continue;\n      }\n      job.answered = true;\n      captions.push({\n        type: 'thinker_result',\n        callId: call.id,\n        summary: textOf(update.message),\n        isError: update.message.isError,\n      });\n      if (blocking) send(() => session.inject(textOf(update.message), options.result));\n      else send(() => session.respond(call.id, update.message, options.result));\n    }\n  } catch (error) {\n    if (!job.abort.signal.aborted && !job.answered) {\n      job.answered = true;\n      send(() => session.respond(call.id, errorResult(call, error)));\n    }\n  } finally {\n    running.delete(call.id);\n    phases.push({ phase: 'idle' });\n  }\n}\n"],
+  "mappings": "AAAA;AAAA,EAKE;AAAA,OACK;AACP,SAAS,yBAAyB;AAIlC,MAAM,aAAa;AAEZ,MAAM,cAAc,CAAC,aAAa,MAAM;AAExC,MAAM,WAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,aACE;AAAA,EACF,YAAY,KAAK,OAAO;AAAA,IACtB,SAAS,KAAK,OAAO;AAAA,IACrB,SAAS,KAAK,SAAS,KAAK,OAAO,CAAC;AAAA,EACtC,CAAC;AACH;AAiCA,MAAM,WAA0C;AAAA,EACrC,SAAc,CAAC;AAAA,EACf,WAAkE,CAAC;AAAA,EAC5E,UAAU;AAAA,EAEV,KAAK,OAAgB;AACnB,QAAI,KAAK,QAAS;AAClB,UAAM,SAAS,KAAK,SAAS,MAAM;AACnC,QAAI,OAAQ,QAAO,EAAE,OAAO,MAAM,MAAM,CAAC;AAAA,QACpC,MAAK,OAAO,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,QAAc;AACZ,QAAI,KAAK,QAAS;AAClB,SAAK,UAAU;AACf,WAAO,KAAK,SAAS,SAAS,EAAG,MAAK,SAAS,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,OAAO,MAAM;AACjC,UAAI,WAAW,QAAW;AACxB,cAAM;AACN;AAAA,MACF;AACA,UAAI,KAAK,QAAS;AAClB,YAAM,OAAO,MAAM,IAAI;AAAA,QAAsC,CAAC,YAC5D,KAAK,SAAS,KAAK,OAAO;AAAA,MAC5B;AACA,UAAI,KAAK,QAAQ,KAAK,UAAU,OAAW;AAC3C,YAAM,KAAK;AAAA,IACb;AAAA,EACF;AACF;AAEA,SAAS,KAAK,QAA0B;AACtC,MAAI;AACF,WAAO;AAAA,EACT,SAAS,OAAO;AACd,QAAI,EAAE,iBAAiB,mBAAoB,OAAM;AAAA,EACnD;AACF;AAEA,SAAS,OAAO,SAAoC;AAClD,SAAO,QAAQ,QACZ,OAAO,CAAC,SAAS,KAAK,SAAS,MAAM,EACrC,IAAI,CAAC,SAAS,KAAK,IAAI,EACvB,KAAK,IAAI;AACd;AAEA,SAAS,aAAa,MAAgB,KAAgC;AACpE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,YAAY,KAAK;AAAA,IACjB,UAAU,KAAK;AAAA,IACf,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC;AAAA,IACrC,SAAS;AAAA,IACT,WAAW,KAAK,IAAI;AAAA,EACtB;AACF;AAEA,SAAS,YAAY,MAAgB,OAAmC;AACtE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,YAAY,KAAK;AAAA,IACjB,UAAU,KAAK;AAAA,IACf,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE,CAAC;AAAA,IACxF,SAAS;AAAA,IACT,WAAW,KAAK,IAAI;AAAA,EACtB;AACF;AAEA,SAAS,UAAU,QAA2B,UAAqC;AACjF,MAAI,OAAO;AACX,QAAM,UAAU,OAAO,QAAQ,IAAI,CAAC,SAAS;AAC3C,QAAI,KAAK,SAAS,OAAQ,QAAO;AACjC,QAAI,QAAQ,EAAG,QAAO,EAAE,GAAG,MAAM,MAAM,GAAG;AAC1C,QAAI,KAAK,KAAK,UAAU,MAAM;AAC5B,cAAQ,KAAK,KAAK;AAClB,aAAO;AAAA,IACT;AACA,UAAM,OAAO,KAAK,KAAK,MAAM,GAAG,IAAI;AACpC,WAAO;AACP,WAAO,EAAE,GAAG,MAAM,KAAK;AAAA,EACzB,CAAC;AACD,SAAO,EAAE,GAAG,QAAQ,QAAQ;AAC9B;AAQO,SAAS,YACd,SACA,SACA,QACA,SACc;AACd,QAAM,WAAW,IAAI,WAA2B;AAChD,QAAM,SAAS,IAAI,WAAyB;AAC5C,QAAM,OAAO,IAAI,SAAS,SAAS,QAAQ,SAAS,UAAU,MAAM,EAAE,QAAQ,MAAM;AAClF,aAAS,MAAM;AACf,WAAO,MAAM;AACb,aAAS,MAAM;AACf,WAAO,MAAM;AAAA,EACf,CAAC;AACD,SAAO,EAAE,UAAU,QAAQ,KAAK;AAClC;AAEA,eAAe,IACb,SACA,SACA,QACA,SACA,UACA,QACe;AACf,QAAM,QAAkB,CAAC;AACzB,QAAM,UAAU,oBAAI,IAAoB;AACxC,mBAAiB,SAAS,SAAS;AACjC,QAAI,MAAM,SAAS,sBAAsB,MAAM,MAAO,OAAM,KAAK,SAAS,MAAM,IAAI,EAAE;AACtF,QAAI,MAAM,SAAS,uBAAuB,MAAM,MAAO,OAAM,KAAK,WAAW,MAAM,IAAI,EAAE;AACzF,QAAI,MAAM,SAAS,uBAAuB;AACxC,YAAMA,OAAM,QAAQ,IAAI,MAAM,MAAM;AACpC,UAAI,CAACA,KAAK;AACV,MAAAA,KAAI,MAAM,MAAM;AAChB,UAAI,CAACA,KAAI,UAAU;AACjB,QAAAA,KAAI,WAAW;AACf,aAAK,MAAM,QAAQ,QAAQ,MAAM,QAAQ,YAAYA,KAAI,MAAM,IAAI,MAAM,WAAW,CAAC,CAAC,CAAC;AAAA,MACzF;AACA;AAAA,IACF;AACA,QAAI,MAAM,SAAS,YAAa;AAChC,UAAM,OAAO,MAAM;AACnB,QAAI,KAAK,SAAS,SAAS,MAAM;AAC/B,aAAO,KAAK;AAAA,QACV,OAAO;AAAA,QACP,UAAU,KAAK;AAAA,QACf,YAAY,KAAK;AAAA,QACjB,MAAM,KAAK;AAAA,MACb,CAAC;AACD,WAAK,OAAO,MAAM,YAAY,QAAQ,QAAQ,eAAe,CAAC,EAC3D,KAAK,CAAC,WAAW;AAChB,aAAK,MAAM,QAAQ,QAAQ,KAAK,IAAI,UAAU,QAAQ,QAAQ,cAAc,CAAC,CAAC;AAC9E,eAAO,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC/B,CAAC,EACA,MAAM,CAAC,UAAmB;AACzB,aAAK,MAAM,QAAQ,QAAQ,KAAK,IAAI,YAAY,MAAM,KAAK,CAAC,CAAC;AAC7D,eAAO,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC/B,CAAC;AACH;AAAA,IACF;AACA,UAAM,MAAc,EAAE,OAAO,IAAI,gBAAgB,GAAG,UAAU,MAAM,UAAU,KAAK;AACnF,YAAQ,IAAI,KAAK,IAAI,GAAG;AACxB,WAAO,KAAK,EAAE,OAAO,WAAW,CAAC;AACjC,QAAI,MAAM,SAAU,MAAK,MAAM,QAAQ,QAAQ,KAAK,IAAI,aAAa,MAAM,QAAQ,GAAG,CAAC,CAAC;AACxF,SAAK;AAAA,MACH;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM;AAAA,MACN;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,UACb,SACA,SACA,SACA,UACA,QACA,SACA,OACA,MACA,UACA,KACe;AACf,MAAI;AACF,qBAAiB,UAAU,QAAQ;AAAA,MACjC,OAAO,KAAK,UAAU,OAAO;AAAA,MAC7B,MAAM,MAAM,CAAC,UAAU;AAAA,MACvB,IAAI,MAAM;AAAA,IACZ,GAAG;AACD,UAAI,IAAI,MAAM,OAAO,QAAS;AAC9B,UAAI,OAAO,SAAS,YAAY;AAC9B,iBAAS,KAAK,EAAE,MAAM,oBAAoB,QAAQ,KAAK,IAAI,MAAM,OAAO,KAAK,CAAC;AAC9E,aAAK,MAAM,QAAQ,OAAO,OAAO,MAAM,QAAQ,QAAQ,CAAC;AACxD;AAAA,MACF;AACA,UAAI,WAAW;AACf,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,QAAQ,KAAK;AAAA,QACb,SAAS,OAAO,OAAO,OAAO;AAAA,QAC9B,SAAS,OAAO,QAAQ;AAAA,MAC1B,CAAC;AACD,UAAI,SAAU,MAAK,MAAM,QAAQ,OAAO,OAAO,OAAO,OAAO,GAAG,QAAQ,MAAM,CAAC;AAAA,UAC1E,MAAK,MAAM,QAAQ,QAAQ,KAAK,IAAI,OAAO,SAAS,QAAQ,MAAM,CAAC;AAAA,IAC1E;AAAA,EACF,SAAS,OAAO;AACd,QAAI,CAAC,IAAI,MAAM,OAAO,WAAW,CAAC,IAAI,UAAU;AAC9C,UAAI,WAAW;AACf,WAAK,MAAM,QAAQ,QAAQ,KAAK,IAAI,YAAY,MAAM,KAAK,CAAC,CAAC;AAAA,IAC/D;AAAA,EACF,UAAE;AACA,YAAQ,OAAO,KAAK,EAAE;AACtB,WAAO,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,EAC/B;AACF;",
+  "names": ["job"]
+}

package/dist/types/audio.d.ts

@@ -34,3 +34,5 @@ export interface AudioSink {
     readonly playedMs: number;
     onPlayback(listener: (tag: AudioTag, offsetMs: number) => void): () => void;
 }
+export declare function chunkSeconds(chunk: AudioChunk): number;
+export declare function alignmentSeconds(alignment: CharAlignment | undefined): number;

package/dist/types/faux/realtime.d.ts

@@ -1,7 +1,7 @@
 import type { AudioChunk } from '../audio.ts';
 import { type Delivery, type DeliverySupport, type RealtimeModel } from '../models.ts';
-import type { RealtimeEvent, RealtimeSession } from '../realtime.ts';
-import { type RealtimeAudioCost } from '../usage.ts';
+import { type RealtimeEvent, type RealtimeSession, type RealtimeSocket, type TruncationPlan } from '../realtime.ts';
+import type { RealtimeAudioCost } from '../usage.ts';
 export type FauxRealtimeCall = {
     method: 'appendAudio';
     chunk: AudioChunk;
@@ -15,17 +15,20 @@ export type FauxRealtimeCall = {
     method: 'inject';
     text: string;
     delivery: Delivery;
+    support: DeliverySupport;
 } | {
     method: 'respond';
     callId: string;
     result: Parameters<RealtimeSession['respond']>[1];
     delivery?: Delivery;
+    support?: DeliverySupport;
 } | {
     method: 'interrupt';
     heard?: {
         itemId: string;
         heardMs: number;
     };
+    plan: TruncationPlan;
 } | {
     method: 'update';
     patch: Parameters<RealtimeSession['update']>[0];
@@ -48,6 +51,7 @@ export interface FauxRealtimeOptions extends FauxRealtimeFlags {
 export interface FauxRealtime {
     model: RealtimeModel<string>;
     calls: FauxRealtimeCall[];
+    readonly socket: RealtimeSocket | undefined;
     emit(event: RealtimeEvent): void;
     setFlags(flags: FauxRealtimeFlags): void;
     unregister(): void;

package/dist/types/index.d.ts

@@ -5,10 +5,13 @@ export { convertAudio, convertAudioSource, decodeAudioBase64, encodeAudioBase64,
 export { VoiceRequestError } from './errors.ts';
 export type { Delivery, DeliverySupport, KnownRealtimeApi, KnownSpeechApi, KnownTranscriptionApi, RealtimeApi, RealtimeModel, SpeechApi, SpeechModel, TranscriptionApi, TranscriptionModel, VoiceInfo, VoiceModel, VoiceModelType, } from './models.ts';
 export { getVoiceModel, listVoiceModels, registerVoiceModel, } from './models.ts';
-export type { RealtimeApiProvider, RealtimeContext, RealtimeEvent, RealtimeOptions, RealtimeSession, RealtimeToken, } from './realtime.ts';
+export type { DeliveryPlan, RealtimeApiProvider, RealtimeContext, RealtimeEvent, RealtimeOptions, RealtimeSession, RealtimeSocket, RealtimeToken, TruncationPlan, } from './realtime.ts';
+export { planDelivery, planTruncation, RealtimeLedger, realtimeSocket } from './realtime.ts';
 export { connectRealtime, listen, registerRealtimeApiProvider, registerSpeechApiProvider, registerTranscriptionApiProvider, speak, transcribe, } from './registry.ts';
 export type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';
 export { SpeechEventStream } from './speech.ts';
+export type { BindThinkerOptions, BoundThinker, Thinker, ThinkerCaption, ThinkerPhase, ThinkerUpdate, } from './talker.ts';
+export { askAgent, bindThinker, talkerTools } from './talker.ts';
 export type { AudioInput, ListenOptions, Transcript, TranscriptionApiProvider, TranscriptionEvent, TranscriptSegment, } from './transcription.ts';
 export { TranscriptionEventStream } from './transcription.ts';
 export type { AudioCost, AudioCounters, AudioUsage, RealtimeAudioCost, SpeechAudioCost, TranscriptionAudioCost, } from './usage.ts';

package/dist/types/realtime.d.ts

@@ -1,6 +1,7 @@
 import type { Message, ProviderRequestOptions, Tool, ToolCall, ToolResultMessage, Usage } from '@earendil-works/pi-ai';
-import type { AudioChunk, CharAlignment, TextSpan } from './audio.ts';
-import type { Delivery, RealtimeModel } from './models.ts';
+import { type AudioChunk, type CharAlignment, type TextSpan } from './audio.ts';
+import type { Delivery, DeliverySupport, RealtimeModel } from './models.ts';
+import { type RealtimeAudioCost } from './usage.ts';
 export interface RealtimeContext {
     instructions: string;
     tools: Tool[];
@@ -85,3 +86,38 @@ export interface RealtimeApiProvider<TApi extends string = string> {
     api: TApi;
     connect: (model: RealtimeModel<TApi>, context: RealtimeContext, options?: RealtimeOptions) => Promise<RealtimeSession>;
 }
+export interface DeliveryPlan {
+    delivery: Delivery;
+    support: DeliverySupport;
+    waitForIdle: boolean;
+}
+export interface TruncationPlan {
+    item?: {
+        itemId: string;
+        audioEndMs: number;
+    };
+}
+export interface RealtimeSocket {
+    url: string;
+    protocols: string[];
+}
+export declare function planDelivery(delivery: Record<Delivery, DeliverySupport>, requested: Delivery): DeliveryPlan;
+export declare function planTruncation(truncation: RealtimeModel<string>['truncation'], heard?: {
+    itemId: string;
+    heardMs: number;
+}): TruncationPlan;
+export declare function realtimeSocket(url: string, options?: Pick<RealtimeOptions, 'gateway'>, protocols?: readonly string[]): RealtimeSocket;
+export declare class RealtimeLedger {
+    #private;
+    get inputSeconds(): number;
+    get outputSeconds(): number;
+    get textMessages(): number;
+    get busy(): boolean;
+    addInput(chunk: AudioChunk): void;
+    addOutput(chunk: AudioChunk, alignment?: CharAlignment): void;
+    addText(): void;
+    observe(event: RealtimeEvent): void;
+    submit(delivery: Record<Delivery, DeliverySupport>, requested: Delivery, run: () => void): DeliveryPlan;
+    settle(): void;
+    usage(rates: RealtimeAudioCost): Usage;
+}

package/dist/types/talker.d.ts

@@ -0,0 +1,48 @@
+import { type JsonObject, type Tool, type ToolCall, type ToolResultMessage } from '@earendil-works/pi-ai';
+import type { Delivery } from './models.ts';
+import type { RealtimeSession } from './realtime.ts';
+export declare const talkerTools: readonly ['ask_agent', 'read'];
+export declare const askAgent: Tool;
+export type ThinkerUpdate = {
+    type: 'progress';
+    text: string;
+} | {
+    type: 'result';
+    message: ToolResultMessage;
+};
+export interface Thinker {
+    ask(request: string, heard: string[], signal: AbortSignal): AsyncIterable<ThinkerUpdate>;
+}
+export interface BindThinkerOptions {
+    progress: Delivery;
+    result: Delivery;
+    ack: string;
+    directTimeoutMs: number;
+    directMaxChars: number;
+}
+export type ThinkerCaption = {
+    type: 'thinker_progress';
+    callId: string;
+    text: string;
+} | {
+    type: 'thinker_result';
+    callId: string;
+    summary: string;
+    isError: boolean;
+};
+export type ThinkerPhase = {
+    phase: 'thinking';
+} | {
+    phase: 'tool';
+    toolName: string;
+    toolCallId: string;
+    args: JsonObject;
+} | {
+    phase: 'idle';
+};
+export interface BoundThinker {
+    captions: AsyncIterable<ThinkerCaption>;
+    phases: AsyncIterable<ThinkerPhase>;
+    done: Promise<void>;
+}
+export declare function bindThinker(session: RealtimeSession, thinker: Thinker, direct: (call: ToolCall, signal: AbortSignal) => Promise<ToolResultMessage>, options: BindThinkerOptions): BoundThinker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-ecoverse/pi-audio",
-  "version": "1.5.0",
+  "version": "1.7.0",
   "description": "Voice bindings for pi: speech-to-text, text-to-speech and realtime speech for pi-coding-agent plugins and slicc-agent",
   "type": "module",
   "license": "Apache-2.0",