@perkos/perkos-voice 0.1.0

package/dist/adapters/livekit.js

@@ -0,0 +1,236 @@
+import { AudioFrame, AudioSource, AudioStream, LocalAudioTrack, RemoteAudioTrack, Room, RoomEvent, TrackPublishOptions, TrackSource } from "@livekit/rtc-node";
+import { runVoiceSessionStage } from "../mediaStages.js";
+import { acceptRemoteMicrophone, EchoSuppressionGate } from "../echoSuppression.js";
+import { TtsError } from "../speechErrors.js";
+const LIVEKIT_CONNECT_TIMEOUT_MS = 15_000;
+const TRACK_PUBLISH_TIMEOUT_MS = 10_000;
+export class LiveKitMediaRoom {
+    observe;
+    #room = new Room();
+    #turns = [];
+    #waiters = [];
+    #publishes = new Map();
+    #source;
+    #track;
+    #activePublishTurn;
+    #bargeInHandler;
+    #agentIdentity;
+    #echoGate;
+    constructor(observe) {
+        this.observe = observe;
+        this.#echoGate = new EchoSuppressionGate(Date.now, 500, () => this.observe?.("turn_based_input_suppressed"));
+        this.#room.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
+            if (!(track instanceof RemoteAudioTrack) || !acceptRemoteMicrophone(publication.source, TrackSource.SOURCE_MICROPHONE, participant.identity, this.#room.localParticipant?.identity, this.#agentIdentity)) {
+                this.observe?.("remote_track_rejected");
+                return;
+            }
+            this.observe?.("track_subscribed");
+            void this.#segment(track);
+        });
+    }
+    async #segment(track) {
+        const stream = new AudioStream(track, { sampleRate: 48_000, numChannels: 1, frameSizeMs: 20 });
+        let chunks = [];
+        let speaking = false;
+        let silentFrames = 0;
+        let totalFrames = 0;
+        let voicedFrames = 0;
+        const emit = () => {
+            if (!chunks.length)
+                return;
+            const buffered = chunks;
+            const accepted = voicedFrames >= 10;
+            chunks = [];
+            speaking = false;
+            silentFrames = 0;
+            totalFrames = 0;
+            voicedFrames = 0;
+            if (!accepted)
+                return;
+            const turn = { turnId: crypto.randomUUID(), audio: (async function* () { for (const chunk of buffered)
+                    yield chunk; })() };
+            this.observe?.("turn_received");
+            const waiter = this.#waiters.shift();
+            if (waiter)
+                waiter.resolve(turn);
+            else
+                this.#turns.push(turn);
+        };
+        for await (const frame of stream) {
+            if (this.#echoGate.shouldSuppress()) {
+                chunks = [];
+                speaking = false;
+                silentFrames = 0;
+                totalFrames = 0;
+                voicedFrames = 0;
+                continue;
+            }
+            const samples = frame.data;
+            let energy = 0;
+            for (let index = 0; index < samples.length; index += 1)
+                energy += Math.abs(samples[index]);
+            const voiced = samples.length > 0 && energy / samples.length >= 500;
+            if (!speaking && voiced && this.#activePublishTurn && this.#bargeInHandler) {
+                this.observe?.("barge_in_candidate");
+                const interrupted = this.#activePublishTurn;
+                const barrier = this.#bargeInHandler(interrupted);
+                this.#echoGate.start(barrier);
+                this.#echoGate.shouldSuppress();
+                chunks = [];
+                speaking = false;
+                silentFrames = 0;
+                totalFrames = 0;
+                continue;
+            }
+            if (!speaking && !voiced)
+                continue;
+            speaking = true;
+            totalFrames += 1;
+            if (voiced)
+                voicedFrames += 1;
+            const copy = new Uint8Array(frame.data.byteLength);
+            copy.set(new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength));
+            chunks.push(copy);
+            silentFrames = voiced ? 0 : silentFrames + 1;
+            if (silentFrames >= 60 || totalFrames >= 1_000)
+                emit();
+        }
+        emit();
+    }
+    async join(grant) {
+        this.#agentIdentity = grant.agentIdentity;
+        const cleanup = () => this.#cleanupJoin();
+        await runVoiceSessionStage("livekit_connect", () => {
+            if (!grant.url)
+                throw new Error("unavailable");
+            return this.#room.connect(grant.url, grant.token, { autoSubscribe: true, dynacast: false });
+        }, { timeoutMs: LIVEKIT_CONNECT_TIMEOUT_MS, cleanup });
+        this.#source = await runVoiceSessionStage("audio_source_create", () => new AudioSource(48_000, 1), { cleanup });
+        this.#track = await runVoiceSessionStage("track_create", () => LocalAudioTrack.createAudioTrack("voice", this.#source), { cleanup });
+        const options = new TrackPublishOptions();
+        options.source = TrackSource.SOURCE_MICROPHONE;
+        const participant = this.#room.localParticipant;
+        if (!participant) {
+            await runVoiceSessionStage("track_publish", () => { throw new Error("unavailable"); }, { cleanup });
+            return;
+        }
+        await runVoiceSessionStage("track_publish", () => participant.publishTrack(this.#track, options), { timeoutMs: TRACK_PUBLISH_TIMEOUT_MS, cleanup });
+    }
+    async receiveTurn() {
+        const turn = this.#turns.shift();
+        if (turn)
+            return turn;
+        return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
+    }
+    setInputSuppressed(suppressed) { if (suppressed)
+        this.#echoGate.beginOutput();
+    else
+        this.#echoGate.endOutput(); }
+    async publish(turnId, audio) {
+        if (!this.#source)
+            throw new Error("LiveKit room is not joined");
+        const controller = new AbortController();
+        const iterator = audio[Symbol.asyncIterator]();
+        let resolveSettled;
+        const settled = new Promise((resolve) => { resolveSettled = resolve; });
+        const context = { controller, iterator, settled, resolveSettled };
+        this.#publishes.set(turnId, context);
+        this.#activePublishTurn = turnId;
+        this.#echoGate.beginOutput();
+        let started = false;
+        try {
+            while (true) {
+                const next = await this.#raceCancellation(iterator.next(), controller.signal);
+                if (next.status === "cancelled" || next.value.done)
+                    break;
+                const chunk = next.value.value;
+                if (!started) {
+                    started = true;
+                    this.observe?.("audio_publish_started");
+                }
+                const aligned = chunk.byteLength - (chunk.byteLength % 2);
+                const bytes = new Uint8Array(aligned);
+                bytes.set(chunk.subarray(0, aligned));
+                const samples = new Int16Array(bytes.buffer);
+                const captured = await this.#raceCancellation(this.#source.captureFrame(new AudioFrame(samples, 48_000, 1, samples.length)), controller.signal);
+                if (captured.status === "cancelled")
+                    break;
+            }
+            if (controller.signal.aborted) {
+                this.observe?.("publish_cancelled");
+                return "cancelled";
+            }
+            if (started)
+                this.observe?.("audio_publish_completed");
+            return "completed";
+        }
+        catch (error) {
+            if (error instanceof TtsError && error.category === "stream_stalled") {
+                controller.abort();
+                this.observe?.("publish_cancelled");
+                return "stream_stalled";
+            }
+            throw error;
+        }
+        finally {
+            this.#echoGate.endOutput();
+            context.resolveSettled();
+            this.#publishes.delete(turnId);
+            if (this.#activePublishTurn === turnId)
+                this.#activePublishTurn = undefined;
+        }
+    }
+    setBargeInHandler(handler) { this.#bargeInHandler = handler; }
+    async cancelPublish(turnId) {
+        const context = this.#publishes.get(turnId);
+        if (!context) {
+            await this.#source?.clearQueue();
+            return;
+        }
+        context.controller.abort();
+        try {
+            await this.#bounded(context.iterator.return?.());
+        }
+        catch { /* cancellation cleanup */ }
+        try {
+            await this.#bounded(this.#source?.clearQueue());
+        }
+        catch { /* cancellation cleanup */ }
+        await Promise.race([context.settled, new Promise((resolve) => setTimeout(resolve, 1_000))]);
+    }
+    async #bounded(operation) {
+        if (!operation)
+            return;
+        operation.catch(() => { });
+        await Promise.race([operation, new Promise((resolve) => setTimeout(resolve, 1_000))]);
+    }
+    async #raceCancellation(operation, signal) {
+        operation.catch(() => { });
+        if (signal.aborted)
+            return { status: "cancelled" };
+        return Promise.race([operation.then((value) => ({ status: "completed", value })), new Promise((resolve) => signal.addEventListener("abort", () => resolve({ status: "cancelled" }), { once: true }))]);
+    }
+    async #cleanupJoin() {
+        try {
+            await this.#track?.close();
+        }
+        catch { /* cleanup is best effort */ }
+        this.#track = undefined;
+        this.#source = undefined;
+        try {
+            await this.#room.disconnect();
+        }
+        catch { /* cleanup is best effort */ }
+    }
+    async close() {
+        if (this.#publishes.size > 0)
+            this.observe?.("publish_cancelled_session_close");
+        await Promise.allSettled([...this.#publishes.keys()].map((turnId) => this.cancelPublish(turnId)));
+        if (this.#turns.length > 0)
+            this.observe?.("queued_turns_discarded");
+        this.#turns.splice(0);
+        for (const waiter of this.#waiters.splice(0))
+            waiter.reject(new Error("LiveKit room closed"));
+        await this.#cleanupJoin();
+    }
+}

package/dist/adapters/openaiSpeech.d.ts

@@ -0,0 +1,24 @@
+import type { SpeechPipeline } from "../ports.js";
+import type { TranscriptPolicy, VoiceMode } from "../types.js";
+import type { MediaSuccessObserver } from "../mediaMetrics.js";
+export interface OpenAiTurnSpeechOptions {
+    apiKey: string;
+    baseUrl?: string;
+    transcriptionModel?: string;
+    speechModel?: string;
+    voice?: string;
+    fetcher?: typeof fetch;
+    observe?: MediaSuccessObserver;
+    ttsIdleTimeoutMs?: number;
+    ttsAbsoluteTimeoutMs?: number;
+    logFailure?: (line: string) => void;
+}
+export declare class OpenAiTurnSpeechAdapter implements SpeechPipeline {
+    #private;
+    private readonly options;
+    constructor(options: OpenAiTurnSpeechOptions);
+    probe(): Promise<void>;
+    transcribe(turnId: string, audio: AsyncIterable<Uint8Array>, policy: TranscriptPolicy): Promise<string>;
+    synthesize(turnId: string, text: string, _mode: VoiceMode, speechVoice?: string): AsyncIterable<Uint8Array>;
+    cancel(turnId: string): Promise<void>;
+}

package/dist/adapters/openaiSpeech.js

@@ -0,0 +1,194 @@
+import { NoSpeechError, TranscriptionError, TtsError } from "../speechErrors.js";
+function wavPcm48Mono(pcm) {
+    const output = new Uint8Array(44 + pcm.byteLength);
+    const view = new DataView(output.buffer);
+    const ascii = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0)));
+    ascii(0, "RIFF");
+    view.setUint32(4, 36 + pcm.byteLength, true);
+    ascii(8, "WAVE");
+    ascii(12, "fmt ");
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, 48_000, true);
+    view.setUint32(28, 96_000, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    ascii(36, "data");
+    view.setUint32(40, pcm.byteLength, true);
+    output.set(pcm, 44);
+    return output;
+}
+async function collect(audio, maxBytes = 3_840_000) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of audio) {
+        size += chunk.byteLength;
+        if (size > maxBytes)
+            throw new Error("speech turn exceeds maximum size");
+        chunks.push(chunk);
+    }
+    const result = new Uint8Array(size);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return result;
+}
+function pcm24To48(input) {
+    const aligned = input.byteLength - (input.byteLength % 2);
+    const source = new Int16Array(input.buffer, input.byteOffset, aligned / 2);
+    const result = new Int16Array(source.length * 2);
+    for (let index = 0; index < source.length; index += 1) {
+        result[index * 2] = source[index];
+        result[index * 2 + 1] = source[index];
+    }
+    return new Uint8Array(result.buffer);
+}
+export class OpenAiTurnSpeechAdapter {
+    options;
+    #controllers = new Map();
+    #baseUrl;
+    #fetcher;
+    constructor(options) {
+        this.options = options;
+        this.#baseUrl = (options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "");
+        this.#fetcher = options.fetcher ?? fetch;
+    }
+    async probe() {
+        for (const model of [this.options.transcriptionModel ?? "gpt-4o-mini-transcribe", this.options.speechModel ?? "gpt-4o-mini-tts"]) {
+            const response = await this.#fetcher(`${this.#baseUrl}/models/${encodeURIComponent(model)}`, { headers: this.#headers(), signal: AbortSignal.timeout(10_000) });
+            if (!response.ok)
+                throw new Error(`speech provider probe failed (${response.status})`);
+        }
+    }
+    async transcribe(turnId, audio, policy) {
+        if (policy === "saved")
+            throw new Error("saved transcripts are not supported");
+        const controller = new AbortController();
+        this.#controllers.set(turnId, controller);
+        try {
+            const pcm = await collect(audio);
+            const form = new FormData();
+            form.set("model", this.options.transcriptionModel ?? "gpt-4o-mini-transcribe");
+            form.set("response_format", "json");
+            const wav = wavPcm48Mono(pcm);
+            form.set("file", new Blob([wav.buffer], { type: "audio/wav" }), "turn.wav");
+            let response;
+            try {
+                response = await this.#fetcher(`${this.#baseUrl}/audio/transcriptions`, { method: "POST", headers: this.#headers(), body: form, signal: controller.signal });
+            }
+            catch (error) {
+                const name = error instanceof Error ? error.name : "";
+                if (name === "AbortError" || name === "TimeoutError")
+                    throw new TranscriptionError("timeout_abort");
+                if (error instanceof TypeError)
+                    throw new TranscriptionError("network");
+                throw new TranscriptionError("other");
+            }
+            if (!response.ok)
+                throw new TranscriptionError(response.status >= 500 ? "http_5xx" : response.status >= 400 ? "http_4xx" : "other");
+            let body;
+            try {
+                body = await response.json();
+            }
+            catch {
+                throw new TranscriptionError("other");
+            }
+            if (typeof body.text !== "string" || !body.text.trim())
+                throw new NoSpeechError();
+            return body.text;
+        }
+        finally {
+            this.#controllers.delete(turnId);
+        }
+    }
+    async *synthesize(turnId, text, _mode, speechVoice) {
+        const controller = new AbortController();
+        this.#controllers.set(turnId, controller);
+        try {
+            let response;
+            try {
+                response = await this.#fetcher(`${this.#baseUrl}/audio/speech`, {
+                    method: "POST", headers: { ...this.#headers(), "content-type": "application/json" },
+                    signal: AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]),
+                    body: JSON.stringify({ model: this.options.speechModel ?? "gpt-4o-mini-tts", voice: speechVoice ?? this.options.voice ?? "alloy", input: text, response_format: "pcm" }),
+                });
+            }
+            catch (error) {
+                throw this.#ttsTransportError(error);
+            }
+            if (!response.ok)
+                throw new TtsError(response.status >= 500 ? "http_5xx" : response.status >= 400 ? "http_4xx" : "other");
+            if (!response.body)
+                throw new TtsError("malformed");
+            let carry = new Uint8Array();
+            let first = true;
+            let reader;
+            const deadline = Date.now() + (this.options.ttsAbsoluteTimeoutMs ?? 30_000);
+            try {
+                reader = response.body.getReader();
+                while (true) {
+                    const remaining = deadline - Date.now();
+                    if (remaining <= 0)
+                        throw this.#streamStalled();
+                    const timeoutMs = first ? remaining : Math.min(this.options.ttsIdleTimeoutMs ?? 5_000, remaining);
+                    let timer;
+                    const timeout = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(this.#streamStalled()), timeoutMs); });
+                    const result = await Promise.race([reader.read(), timeout]).finally(() => { if (timer)
+                        clearTimeout(timer); });
+                    if (result.done)
+                        break;
+                    const chunk = result.value;
+                    const incoming = new Uint8Array(chunk);
+                    const combined = new Uint8Array(carry.byteLength + incoming.byteLength);
+                    combined.set(carry);
+                    combined.set(incoming, carry.byteLength);
+                    const aligned = combined.byteLength - (combined.byteLength % 2);
+                    carry = combined.slice(aligned);
+                    if (!aligned)
+                        continue;
+                    if (first) {
+                        first = false;
+                        this.options.observe?.("tts_first_byte");
+                    }
+                    yield pcm24To48(combined.subarray(0, aligned));
+                }
+            }
+            catch (error) {
+                if (error instanceof TtsError)
+                    throw error;
+                throw this.#ttsTransportError(error);
+            }
+            finally {
+                if (reader) {
+                    try {
+                        await reader.cancel();
+                    }
+                    catch { /* best-effort stream cleanup */ }
+                }
+            }
+            if (carry.byteLength || first)
+                throw new TtsError("malformed");
+        }
+        finally {
+            this.#controllers.delete(turnId);
+        }
+    }
+    async cancel(turnId) { this.#controllers.get(turnId)?.abort(); this.#controllers.delete(turnId); }
+    #headers() { return { authorization: `Bearer ${this.options.apiKey}` }; }
+    #ttsTransportError(error) {
+        const name = error instanceof Error ? error.name : "";
+        if (name === "AbortError" || name === "TimeoutError")
+            return new TtsError("timeout_abort");
+        if (error instanceof TypeError)
+            return new TtsError("network");
+        return new TtsError("other");
+    }
+    #streamStalled() {
+        const error = new TtsError("stream_stalled");
+        this.options.logFailure?.("voice tts failed: stream_stalled");
+        return error;
+    }
+}

package/dist/adapters/openclaw.d.ts

@@ -0,0 +1,58 @@
+import type { AgentRuntime } from "../ports.js";
+/** Uses OpenClaw's documented local OpenResponses API; this is not Talk integration. */
+export declare class OpenClawResponsesAdapter implements AgentRuntime {
+    #private;
+    private readonly endpoint;
+    private readonly token;
+    private readonly fetcher;
+    constructor(endpoint: string, token: string, fetcher?: typeof fetch);
+    respond(turnId: string, input: string, signal: AbortSignal): Promise<string>;
+    cancel(turnId: string): Promise<void>;
+}
+export interface OpenClawChatCompletionsOptions {
+    endpoint: string;
+    token: string;
+    model: string;
+    backendModel?: string;
+    /** Short spoken identity only (e.g. "Athena"). Must NOT contain tools/docs/history. */
+    spokenName?: string;
+    /**
+     * Optional Working Call brief from bound chat (API claim). Already sanitized/bounded
+     * by API; still re-checked here. Private Call must leave this empty.
+     */
+    workChatBrief?: string;
+    /** Max prior final pairs kept in-call (default 4). */
+    inCallHistoryPairs?: number;
+    /** Completion budget for spoken turns. Default 80. */
+    maxTokens?: number;
+    timeoutMs?: number;
+    observe?: (event: "work_chat_brief_loaded" | "work_chat_brief_empty" | "openclaw_model_default" | "openclaw_model_canary" | "openclaw_latency_fast" | "openclaw_latency_acceptable" | "openclaw_latency_slow" | "openclaw_response_filtered" | "openclaw_language_retry" | "openclaw_safe_fallback") => void;
+}
+export declare const WORK_CHAT_BRIEF_MAX_CHARS = 2400;
+/** Sanitize optional work-chat brief for thin system policy. */
+export declare function sanitizeWorkChatBrief(value: unknown): string | undefined;
+/** Build the thin spoken system policy. Identity is optional and must stay short. */
+export declare function buildSpokenPolicy(spokenName?: string, repair?: boolean, workChatBrief?: string): string;
+/**
+ * Thin chat-completions runtime for voice turns.
+ *
+ * IMPORTANT: Point OPENCLAW_ENDPOINT at a **thin** OpenAI-compatible LLM
+ * (direct model API or a voice-only route). Do NOT point it at a full agent
+ * stack (Hermes api_server / OpenClaw with tools+memory+skills) — those inject
+ * tens of thousands of system tokens per turn and destroy call latency.
+ *
+ * Payload: short spoken system policy (+ optional Working Call brief) +
+ * last few in-call final pairs + current user transcript.
+ */
+export declare class OpenClawChatCompletionsAdapter implements AgentRuntime {
+    #private;
+    private readonly options;
+    private readonly fetcher;
+    constructor(options: OpenClawChatCompletionsOptions, fetcher?: typeof fetch);
+    respond(turnId: string, input: string, signal: AbortSignal): Promise<string>;
+    cancel(turnId: string): Promise<void>;
+}
+export declare class UnavailableOpenClawTalkAdapter implements AgentRuntime {
+    respond(): Promise<string>;
+    cancel(): Promise<void>;
+}