@ai-ecoverse/pi-audio 0.0.0 → 1.1.0

package/dist/types/faux/shared.d.ts

@@ -0,0 +1,16 @@
+import type { AudioFormat } from '../audio.ts';
+export declare const zeroCost: {
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheWrite: number;
+};
+export declare function pcm(sampleRate?: number): AudioFormat;
+export declare function modelKey(model: {
+    provider: string;
+    id: string;
+}): string;
+export declare function decodeBytes(data: Uint8Array): string;
+export declare function encodeChars(text: string): Uint8Array;
+export declare function errorMessage(error: unknown): string;
+export declare function shareApi(registrations: Map<string, () => void>, api: string, register: () => () => void, stillUsed: () => boolean): () => void;

package/dist/types/faux/speech.d.ts

@@ -0,0 +1,18 @@
+import { type SpeechModel } from '../models.ts';
+import { type SpeechInput } from '../speech.ts';
+export interface FauxSpeechOptions {
+    api?: string;
+    provider?: string;
+    id?: string;
+    name?: string;
+    alignment?: boolean;
+    chunkChars?: number;
+    textStreaming?: boolean;
+}
+export interface FauxSpeech {
+    model: SpeechModel<string>;
+    calls: SpeechInput[];
+    setAlignment(alignment: boolean): void;
+    unregister(): void;
+}
+export declare function registerFauxSpeech(options?: FauxSpeechOptions): FauxSpeech;

package/dist/types/faux/transcription.d.ts

@@ -0,0 +1,27 @@
+import type { AudioChunk } from '../audio.ts';
+import { type TranscriptionModel } from '../models.ts';
+import { type AudioInput, type TranscriptionEvent } from '../transcription.ts';
+export interface FauxTranscriptionCue {
+    at: number | 'end';
+    events: TranscriptionEvent[];
+}
+export interface FauxTranscriptionOptions {
+    api?: string;
+    provider?: string;
+    id?: string;
+    name?: string;
+    turnDetection?: TranscriptionModel<string>['turnDetection'];
+    cues?: FauxTranscriptionCue[];
+}
+export interface FauxTranscriptionCall {
+    kind: 'transcribe' | 'listen';
+    input?: AudioInput;
+    chunks: AudioChunk[];
+}
+export interface FauxTranscription {
+    model: TranscriptionModel<string>;
+    calls: FauxTranscriptionCall[];
+    setCues(cues: FauxTranscriptionCue[] | undefined): void;
+    unregister(): void;
+}
+export declare function registerFauxTranscription(options?: FauxTranscriptionOptions): FauxTranscription;

package/dist/types/index.d.ts

@@ -0,0 +1,12 @@
+export type { AudioChunk, AudioEncoding, AudioFormat, AudioSink, AudioSource, AudioTag, CharAlignment, TextSpan, } from './audio.ts';
+export { resolveVoiceAuth } from './auth.ts';
+export { blankUsage } from './blank-usage.ts';
+export { VoiceRequestError } from './errors.ts';
+export type { Delivery, DeliverySupport, KnownRealtimeApi, KnownSpeechApi, KnownTranscriptionApi, RealtimeApi, RealtimeModel, SpeechApi, SpeechModel, TranscriptionApi, TranscriptionModel, VoiceInfo, VoiceModel, VoiceModelType, } from './models.ts';
+export { getVoiceModel, listVoiceModels, registerVoiceModel, } from './models.ts';
+export type { RealtimeApiProvider, RealtimeContext, RealtimeEvent, RealtimeOptions, RealtimeSession, RealtimeToken, } from './realtime.ts';
+export { connectRealtime, listen, registerRealtimeApiProvider, registerSpeechApiProvider, registerTranscriptionApiProvider, speak, transcribe, } from './registry.ts';
+export type { SpeechApiProvider, SpeechEvent, SpeechInput } from './speech.ts';
+export { SpeechEventStream } from './speech.ts';
+export type { AudioInput, ListenOptions, Transcript, TranscriptionApiProvider, TranscriptionEvent, TranscriptSegment, } from './transcription.ts';
+export { TranscriptionEventStream } from './transcription.ts';

package/dist/types/models.d.ts

@@ -0,0 +1,73 @@
+import type { BaseModel } from '@earendil-works/pi-ai';
+import type { AudioFormat } from './audio.ts';
+export type KnownTranscriptionApi = 'openai-transcriptions' | 'openai-realtime-transcription' | 'xai-stt' | 'elevenlabs-scribe' | 'mistral-transcriptions' | 'fish-asr' | 'cloudflare-ai-run';
+export type TranscriptionApi = KnownTranscriptionApi | (string & {});
+export type KnownSpeechApi = 'openai-speech' | 'xai-tts' | 'elevenlabs-tts' | 'mistral-speech' | 'fish-tts' | 'google-tts' | 'cloudflare-ai-run' | 'wavespeed';
+export type SpeechApi = KnownSpeechApi | (string & {});
+export type KnownRealtimeApi = 'openai-realtime' | 'xai-realtime' | 'gemini-live' | 'elevenlabs-agents' | 'fish-agents';
+export type RealtimeApi = KnownRealtimeApi | (string & {});
+export type Delivery = 'interrupt' | 'when_idle' | 'silent';
+export type DeliverySupport = 'native' | 'approximate' | 'unsupported';
+export interface VoiceInfo {
+    id: string;
+    name: string;
+    languages?: string[];
+}
+export interface TranscriptionModel<TApi extends TranscriptionApi> extends Omit<BaseModel<TApi>, 'input'> {
+    type: 'transcription';
+    streaming: boolean;
+    turnDetection: 'none' | 'vad' | 'semantic';
+    inputFormats: AudioFormat[];
+    languages?: string[];
+    diarization?: boolean;
+}
+export interface SpeechModel<TApi extends SpeechApi> extends Omit<BaseModel<TApi>, 'input'> {
+    type: 'speech';
+    textStreaming: boolean;
+    outputFormats: AudioFormat[];
+    voices?: VoiceInfo[];
+    cloning?: boolean;
+}
+export interface RealtimeModel<TApi extends RealtimeApi> extends Omit<BaseModel<TApi>, 'input'> {
+    type: 'realtime';
+    transports: ('websocket' | 'webrtc')[];
+    toolCalls: 'blocking' | 'async';
+    inputFormat: AudioFormat;
+    outputFormat: AudioFormat;
+    video?: {
+        maxFps: number;
+    };
+    maxSessionMs?: number;
+    resumable: boolean;
+    ephemeralAuth: boolean;
+    voices: VoiceInfo[];
+    cascade?: {
+        stt: string;
+        llm: string;
+        tts: string;
+    };
+    provisioning: 'none' | 'agent';
+    delivery: Record<Delivery, DeliverySupport>;
+    truncation: 'client' | 'server' | 'none';
+    liveToolUpdate: boolean;
+    toolTimeoutMs?: {
+        min: number;
+        max: number;
+    };
+}
+declare module '@earendil-works/pi-ai' {
+    interface ModelTypeMap {
+        transcription: TranscriptionModel<TranscriptionApi>;
+        speech: SpeechModel<SpeechApi>;
+        realtime: RealtimeModel<RealtimeApi>;
+    }
+}
+export type VoiceModel = TranscriptionModel<TranscriptionApi> | SpeechModel<SpeechApi> | RealtimeModel<RealtimeApi>;
+export type VoiceModelType = VoiceModel['type'];
+export declare function registerVoiceModel(model: VoiceModel): () => void;
+export declare function getVoiceModel<TType extends VoiceModelType>(type: TType, provider: string, id: string): Extract<VoiceModel, {
+    type: TType;
+}> | undefined;
+export declare function listVoiceModels<TType extends VoiceModelType>(type: TType, provider?: string): Extract<VoiceModel, {
+    type: TType;
+}>[];

package/dist/types/realtime.d.ts

@@ -0,0 +1,87 @@
+import type { Message, ProviderRequestOptions, Tool, ToolCall, ToolResultMessage, Usage } from '@earendil-works/pi-ai';
+import type { AudioChunk, CharAlignment, TextSpan } from './audio.ts';
+import type { Delivery, RealtimeModel } from './models.ts';
+export interface RealtimeContext {
+    instructions: string;
+    tools: Tool[];
+    voice: string;
+    history?: Message[];
+}
+export interface RealtimeToken {
+    value: string;
+    expiresAt: number;
+    model: string;
+}
+export interface RealtimeOptions extends ProviderRequestOptions {
+    token?: RealtimeToken;
+    transport?: 'websocket' | 'webrtc';
+    gateway?: {
+        url: string;
+        token: string;
+    };
+    resumeHandle?: string;
+}
+export type RealtimeEvent = {
+    type: 'session_start';
+    sessionId: string;
+} | {
+    type: 'input_speech_start';
+} | {
+    type: 'input_transcript';
+    itemId: string;
+    text: string;
+    final: boolean;
+} | {
+    type: 'output_audio';
+    itemId: string;
+    chunk: AudioChunk;
+    span?: TextSpan;
+    alignment?: CharAlignment;
+} | {
+    type: 'output_transcript';
+    itemId: string;
+    text: string;
+    final: boolean;
+} | {
+    type: 'tool_call';
+    call: ToolCall;
+    blocking: boolean;
+} | {
+    type: 'tool_call_cancelled';
+    callId: string;
+} | {
+    type: 'response_done';
+    itemId: string;
+    interrupted: boolean;
+    usage: Usage;
+} | {
+    type: 'go_away';
+    timeLeftMs: number;
+} | {
+    type: 'resumption';
+    handle: string;
+} | {
+    type: 'error';
+    error: string;
+    fatal: boolean;
+};
+export interface RealtimeSession extends AsyncIterable<RealtimeEvent> {
+    readonly model: RealtimeModel<string>;
+    appendAudio(chunk: AudioChunk): void;
+    appendVideoFrame?(jpeg: Uint8Array): void;
+    sendText(text: string): void;
+    inject(text: string, delivery: Delivery): void;
+    respond(callId: string, result: ToolResultMessage, delivery?: Delivery): void;
+    interrupt(heard?: {
+        itemId: string;
+        heardMs: number;
+    }): void;
+    update(patch: Partial<Pick<RealtimeContext, 'instructions' | 'tools' | 'voice'>>): void;
+    close(): Promise<{
+        usage: Usage;
+    }>;
+}
+export interface RealtimeApiProvider<TApi extends string = string> {
+    api: TApi;
+    connect: (model: RealtimeModel<TApi>, context: RealtimeContext, options?: RealtimeOptions) => Promise<RealtimeSession>;
+}

package/dist/types/registry.d.ts

@@ -0,0 +1,13 @@
+import type { ProviderRequestOptions } from '@earendil-works/pi-ai';
+import type { AudioSource } from './audio.ts';
+import type { RealtimeModel, SpeechModel, TranscriptionModel } from './models.ts';
+import type { RealtimeApiProvider, RealtimeContext, RealtimeOptions, RealtimeSession } from './realtime.ts';
+import type { SpeechApiProvider, SpeechEventStream, SpeechInput } from './speech.ts';
+import type { AudioInput, ListenOptions, Transcript, TranscriptionApiProvider, TranscriptionEventStream } from './transcription.ts';
+export declare function registerTranscriptionApiProvider(provider: TranscriptionApiProvider): () => void;
+export declare function registerSpeechApiProvider(provider: SpeechApiProvider): () => void;
+export declare function registerRealtimeApiProvider(provider: RealtimeApiProvider): () => void;
+export declare function transcribe(model: TranscriptionModel<string>, input: AudioInput, options?: ProviderRequestOptions): Promise<Transcript>;
+export declare function listen(model: TranscriptionModel<string>, audio: AudioSource, options?: ListenOptions): TranscriptionEventStream;
+export declare function speak(model: SpeechModel<string>, input: SpeechInput, options?: ProviderRequestOptions): SpeechEventStream;
+export declare function connectRealtime(model: RealtimeModel<string>, context: RealtimeContext, options?: RealtimeOptions): Promise<RealtimeSession>;

package/dist/types/speech.d.ts

@@ -0,0 +1,38 @@
+import type { ProviderRequestOptions, Usage } from '@earendil-works/pi-ai';
+import { EventStream } from '@earendil-works/pi-ai';
+import type { AudioChunk, AudioFormat, CharAlignment, TextSpan } from './audio.ts';
+import type { SpeechModel } from './models.ts';
+export interface SpeechInput {
+    text: string | AsyncIterable<string>;
+    voice: string;
+    language?: string;
+    instructions?: string;
+    format?: AudioFormat;
+}
+export type SpeechEvent = {
+    type: 'start';
+    format: AudioFormat;
+} | {
+    type: 'audio';
+    chunk: AudioChunk;
+    span: TextSpan;
+    alignment?: CharAlignment;
+} | {
+    type: 'done';
+    usage: Usage;
+} | {
+    type: 'error';
+    reason: 'aborted' | 'error';
+    error: string;
+};
+export declare class SpeechEventStream extends EventStream<SpeechEvent, Usage> {
+    #private;
+    constructor();
+    push(event: SpeechEvent): void;
+    end(result?: Usage): void;
+    result(): Promise<Usage>;
+}
+export interface SpeechApiProvider<TApi extends string = string> {
+    api: TApi;
+    speak: (model: SpeechModel<TApi>, input: SpeechInput, options?: ProviderRequestOptions) => SpeechEventStream;
+}

package/dist/types/stream-result.d.ts

@@ -0,0 +1,6 @@
+export interface StreamResult<T> {
+    readonly result: Promise<T>;
+    finish(value: T): void;
+    fail(error: Error): void;
+}
+export declare function streamResult<T>(): StreamResult<T>;

package/dist/types/transcription.d.ts

@@ -0,0 +1,62 @@
+import type { ProviderRequestOptions, Usage } from '@earendil-works/pi-ai';
+import { EventStream } from '@earendil-works/pi-ai';
+import type { AudioSource } from './audio.ts';
+import type { TranscriptionModel } from './models.ts';
+export type AudioInput = {
+    data: Uint8Array;
+    mimeType: string;
+} | {
+    url: string;
+};
+export interface TranscriptSegment {
+    text: string;
+    start: number;
+    end: number;
+    speaker?: string;
+}
+export interface Transcript {
+    text: string;
+    language?: string;
+    segments?: TranscriptSegment[];
+    usage: Usage;
+}
+export type TranscriptionEvent = {
+    type: 'speech_start';
+    at: number;
+} | {
+    type: 'partial';
+    segmentId: string;
+    text: string;
+} | {
+    type: 'final';
+    segmentId: string;
+    text: string;
+    confidence?: number;
+} | {
+    type: 'turn_end';
+    text: string;
+    at: number;
+} | {
+    type: 'done';
+    transcript: Transcript;
+} | {
+    type: 'error';
+    reason: 'aborted' | 'error';
+    error: string;
+};
+export interface ListenOptions extends ProviderRequestOptions {
+    language?: string;
+    vocabulary?: string[];
+}
+export declare class TranscriptionEventStream extends EventStream<TranscriptionEvent, Transcript> {
+    #private;
+    constructor();
+    push(event: TranscriptionEvent): void;
+    end(result?: Transcript): void;
+    result(): Promise<Transcript>;
+}
+export interface TranscriptionApiProvider<TApi extends string = string> {
+    api: TApi;
+    transcribe?: (model: TranscriptionModel<TApi>, input: AudioInput, options?: ProviderRequestOptions) => Promise<Transcript>;
+    listen?: (model: TranscriptionModel<TApi>, audio: AudioSource, options?: ListenOptions) => TranscriptionEventStream;
+}

package/package.json

@@ -1,10 +1,65 @@
 {
   "name": "@ai-ecoverse/pi-audio",
-  "version": "0.0.0",
+  "version": "1.1.0",
   "description": "Voice bindings for pi: speech-to-text, text-to-speech and realtime speech for pi-coding-agent plugins and slicc-agent",
+  "type": "module",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ai-ecoverse/pi-audio.git"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "fledgling": {
+    "provider": "github",
+    "workflow": "release.yml",
+    "repo": "ai-ecoverse/pi-audio",
+    "publish": true
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./faux": {
+      "types": "./dist/types/faux/index.d.ts",
+      "default": "./dist/faux/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "prepare": "lefthook install",
+    "build": "node build.mjs && tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "pretest": "npm run build",
+    "test": "node --test 'test/integration/*.test.mjs'",
+    "test:unit": "NODE_NO_WARNINGS=1 mcr -c node_modules/@ai-ecoverse/slicc-shared-web/mcr.config.mjs node --test 'test/unit/**/*.test.mjs'",
+    "typecheck": "tsc",
+    "lint": "slicc-lint",
+    "format": "biome check --write .",
+    "trust": "npx fledgling sync -y --skip-publish",
+    "trust:dry": "npx fledgling sync --dry-run -y --skip-publish",
+    "release": "semantic-release"
+  },
+  "devDependencies": {
+    "@ai-ecoverse/slicc-shared-web": "1.21.0",
+    "@biomejs/biome": "2.5.15",
+    "@earendil-works/pi-ai": "1.1.0",
+    "@types/node": "24.19.1",
+    "esbuild": "0.28.2",
+    "lefthook": "2.2.1",
+    "monocart-coverage-reports": "2.13.2",
+    "semantic-release": "25.0.9",
+    "typescript": "7.0.2"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "1.1.0"
   }
 }