@ouro.bot/cli 0.1.0-alpha.887 → 0.1.0-alpha.889

package/dist/senses/voice/local-audio.js

@@ -0,0 +1,281 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PacedFrameFeeder = exports.CaptureChunker = exports.PacedPlaybackQueue = exports.MULAW_SILENCE_BYTE = exports.MULAW_FRAME_BYTES = void 0;
+exports.mulawToLinear = mulawToLinear;
+exports.isDigitalSilence = isDigitalSilence;
+exports.mulawFrameRms = mulawFrameRms;
+const runtime_1 = require("../../nerves/runtime");
+/** 20 ms of 8 kHz mu-law audio. */
+exports.MULAW_FRAME_BYTES = 160;
+/** mu-law encoding of zero amplitude. */
+exports.MULAW_SILENCE_BYTE = 0xff;
+const FRAME_MS = 20;
+const DEFAULT_MAX_LEAD_MS = 60;
+function silenceFrame() {
+    return Buffer.alloc(exports.MULAW_FRAME_BYTES, exports.MULAW_SILENCE_BYTE);
+}
+/**
+ * Writes one 160-byte mu-law frame per 20 ms to a sink in real time, never more than
+ * `maxLeadMs` ahead of the clock. Idle periods write silence so the timeline (and mark
+ * delivery) keeps advancing.
+ */
+class PacedPlaybackQueue {
+    options;
+    items = [];
+    pendingMarks = new Set();
+    maxLeadMs;
+    outputLatencyMs;
+    running = false;
+    drainWaiters = [];
+    timer = undefined;
+    nextDue = 0;
+    constructor(options) {
+        this.options = options;
+        this.maxLeadMs = options.maxLeadMs ?? DEFAULT_MAX_LEAD_MS;
+        this.outputLatencyMs = options.outputLatencyMs ?? 0;
+    }
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.nextDue = this.options.now();
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_playback_started",
+            message: "local paced playback queue started",
+            meta: { maxLeadMs: this.maxLeadMs, outputLatencyMs: this.outputLatencyMs },
+        });
+        this.tick();
+    }
+    stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.options.clearTimer(this.timer);
+        this.timer = undefined;
+        for (const pending of this.pendingMarks)
+            this.options.clearTimer(pending.timer);
+        this.pendingMarks.clear();
+        this.items.length = 0;
+        this.releaseDrainWaiters();
+    }
+    /** Resolves once every queued frame has been written (or the queue stopped), so a file output gets the whole reply. */
+    drain() {
+        if (!this.running || !this.hasQueuedFrames())
+            return Promise.resolve();
+        return new Promise((resolve) => { this.drainWaiters.push(resolve); });
+    }
+    enqueue(payload) {
+        for (let offset = 0; offset < payload.length; offset += exports.MULAW_FRAME_BYTES) {
+            const slice = payload.subarray(offset, offset + exports.MULAW_FRAME_BYTES);
+            const frame = slice.length === exports.MULAW_FRAME_BYTES
+                ? Buffer.from(slice)
+                : Buffer.concat([slice, Buffer.alloc(exports.MULAW_FRAME_BYTES - slice.length, exports.MULAW_SILENCE_BYTE)]);
+            this.items.push({ kind: "frame", frame });
+        }
+    }
+    mark(name, onPlayed) {
+        this.items.push({ kind: "mark", name, onPlayed });
+    }
+    /** Drops queued audio and fires every pending mark now (Twilio returns marks on clear). */
+    clear() {
+        const dropped = this.items.splice(0);
+        const firing = [];
+        for (const pending of this.pendingMarks) {
+            this.options.clearTimer(pending.timer);
+            firing.push(pending);
+        }
+        this.pendingMarks.clear();
+        for (const item of dropped) {
+            if (item.kind === "mark")
+                firing.push(item);
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_playback_cleared",
+            message: "local paced playback queue cleared",
+            meta: { droppedFrames: dropped.filter((item) => item.kind === "frame").length, firedMarks: firing.length },
+        });
+        for (const mark of firing)
+            mark.onPlayed(mark.name);
+    }
+    queuedMs() {
+        return this.items.filter((item) => item.kind === "frame").length * FRAME_MS;
+    }
+    tick() {
+        const now = this.options.now();
+        // Underrun (event loop stall): restart the timeline instead of bursting to catch up.
+        if (this.nextDue < now)
+            this.nextDue = now;
+        for (;;) {
+            this.drainMarks();
+            if (this.nextDue + FRAME_MS - now > this.maxLeadMs)
+                break;
+            const head = this.items[0];
+            if (head && head.kind === "frame") {
+                this.items.shift();
+                this.options.write(head.frame);
+            }
+            else {
+                this.options.write(silenceFrame());
+            }
+            this.nextDue += FRAME_MS;
+        }
+        if (!this.hasQueuedFrames())
+            this.releaseDrainWaiters();
+        this.timer = this.options.setTimer(() => this.tick(), this.nextDue + FRAME_MS - this.maxLeadMs - now);
+    }
+    hasQueuedFrames() {
+        return this.items.some((item) => item.kind === "frame");
+    }
+    releaseDrainWaiters() {
+        const waiters = this.drainWaiters;
+        this.drainWaiters = [];
+        for (const resolve of waiters)
+            resolve();
+    }
+    drainMarks() {
+        for (;;) {
+            const head = this.items[0];
+            if (!head || head.kind !== "mark")
+                return;
+            this.items.shift();
+            // `nextDue` is the playhead: the scheduled end of the last frame written before this mark.
+            const delay = this.nextDue + this.outputLatencyMs - this.options.now();
+            if (delay <= 0) {
+                head.onPlayed(head.name);
+                continue;
+            }
+            const pending = { name: head.name, onPlayed: head.onPlayed, timer: undefined };
+            pending.timer = this.options.setTimer(() => {
+                this.pendingMarks.delete(pending);
+                pending.onPlayed(pending.name);
+            }, delay);
+            this.pendingMarks.add(pending);
+        }
+    }
+}
+exports.PacedPlaybackQueue = PacedPlaybackQueue;
+/** Re-chunks arbitrary byte chunks into 160-byte frames. */
+class CaptureChunker {
+    onFrame;
+    pending = Buffer.alloc(0);
+    constructor(onFrame) {
+        this.onFrame = onFrame;
+    }
+    push(chunk) {
+        this.pending = this.pending.length === 0 ? chunk : Buffer.concat([this.pending, chunk]);
+        while (this.pending.length >= exports.MULAW_FRAME_BYTES) {
+            this.onFrame(Buffer.from(this.pending.subarray(0, exports.MULAW_FRAME_BYTES)));
+            this.pending = this.pending.subarray(exports.MULAW_FRAME_BYTES);
+        }
+    }
+    /** Emits any remainder padded with silence. */
+    flush() {
+        if (this.pending.length === 0)
+            return;
+        const padded = Buffer.concat([this.pending, Buffer.alloc(exports.MULAW_FRAME_BYTES - this.pending.length, exports.MULAW_SILENCE_BYTE)]);
+        this.pending = Buffer.alloc(0);
+        this.onFrame(padded);
+    }
+}
+exports.CaptureChunker = CaptureChunker;
+/** Decodes one mu-law byte to 16-bit linear PCM (G.711). Both zero codes (0xff, 0x7f) decode to 0. */
+function mulawToLinear(byte) {
+    const inverted = ~byte & 0xff;
+    const exponent = (inverted >> 4) & 0x07;
+    const mantissa = inverted & 0x0f;
+    const magnitude = (((mantissa << 3) + 0x84) << exponent) - 0x84;
+    return inverted & 0x80 ? -magnitude : magnitude;
+}
+/** True when every byte decodes to linear PCM within `threshold` of zero. */
+function isDigitalSilence(frames, threshold = 2) {
+    for (const frame of frames) {
+        for (const byte of frame) {
+            if (Math.abs(mulawToLinear(byte)) > threshold)
+                return false;
+        }
+    }
+    return true;
+}
+/** Root-mean-square level of a mu-law frame in 16-bit linear units. */
+function mulawFrameRms(frame) {
+    if (frame.length === 0)
+        return 0;
+    let sum = 0;
+    for (const byte of frame) {
+        const sample = mulawToLinear(byte);
+        sum += sample * sample;
+    }
+    return Math.sqrt(sum / frame.length);
+}
+/**
+ * Turns a fast source (sox decoding a recorded file) into a real-time capture stream: one 160-byte
+ * frame per 20 ms, then silence for as long as it runs, like a quiet room. This is the file-driven
+ * "microphone" for tests without a human.
+ */
+class PacedFrameFeeder {
+    options;
+    buffered = Buffer.alloc(0);
+    drained = false;
+    running = false;
+    timer = undefined;
+    nextDue = 0;
+    onData = (chunk) => {
+        this.buffered = Buffer.concat([this.buffered, chunk]);
+    };
+    onEnd = () => {
+        this.drained = true;
+    };
+    constructor(options) {
+        this.options = options;
+    }
+    sourceDrained() {
+        return this.drained && this.buffered.length === 0;
+    }
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.nextDue = this.options.now();
+        this.options.source.on("data", this.onData);
+        this.options.source.on("end", this.onEnd);
+        this.options.source.on("error", this.onEnd);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_file_capture_started",
+            message: "paced file capture started",
+            meta: {},
+        });
+        this.tick();
+    }
+    stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.options.clearTimer(this.timer);
+        this.options.source.off("data", this.onData);
+        this.options.source.off("end", this.onEnd);
+        this.options.source.off("error", this.onEnd);
+        this.options.sink.end();
+    }
+    tick() {
+        const now = this.options.now();
+        // Stall: restart the timeline instead of bursting.
+        if (this.nextDue < now - FRAME_MS * 5)
+            this.nextDue = now;
+        while (this.nextDue <= now) {
+            this.options.sink.write(this.nextFrame());
+            this.nextDue += FRAME_MS;
+        }
+        this.timer = this.options.setTimer(() => this.tick(), this.nextDue - now);
+    }
+    nextFrame() {
+        const take = Math.min(this.buffered.length, exports.MULAW_FRAME_BYTES);
+        const frame = Buffer.alloc(exports.MULAW_FRAME_BYTES, exports.MULAW_SILENCE_BYTE);
+        this.buffered.copy(frame, 0, 0, take);
+        this.buffered = this.buffered.subarray(take);
+        return frame;
+    }
+}
+exports.PacedFrameFeeder = PacedFrameFeeder;

package/dist/senses/voice/local-media-stream.js

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalMediaStreamSocket = void 0;
+const events_1 = require("events");
+const runtime_1 = require("../../nerves/runtime");
+const local_audio_1 = require("./local-audio");
+const FRAME_MS = 20;
+/**
+ * Speaks the Twilio Media Stream protocol in-process: captured audio becomes `start`, `media` and
+ * `stop` messages for the session; the session's `media`, `clear` and `mark` messages drive the
+ * paced playback queue. It sends no token and no identity parameters; identity is injected into
+ * the session by whoever constructs it.
+ */
+class LocalMediaStreamSocket extends events_1.EventEmitter {
+    options;
+    opened = false;
+    closed = false;
+    sequence = 0;
+    chunk = 0;
+    chunker;
+    onCaptureData = (data) => this.chunker.push(data);
+    onCaptureEnd = () => this.endCapture();
+    onCaptureError = (error) => {
+        (0, runtime_1.emitNervesEvent)({
+            level: "error",
+            component: "senses",
+            event: "senses.voice_local_capture_error",
+            message: "local capture stream failed",
+            meta: { callSid: this.options.callSid, error: error.message },
+        });
+        this.endCapture();
+    };
+    constructor(options) {
+        super();
+        this.options = options;
+        this.chunker = new local_audio_1.CaptureChunker((frame) => this.emitMedia(frame));
+    }
+    isOpen() {
+        return this.opened && !this.closed;
+    }
+    open() {
+        if (this.opened || this.closed)
+            return;
+        this.opened = true;
+        const { streamSid, callSid } = this.options;
+        this.options.playback.start();
+        this.emitMessage({
+            event: "start",
+            start: {
+                streamSid,
+                callSid,
+                tracks: ["inbound"],
+                mediaFormat: { encoding: "audio/x-mulaw", sampleRate: 8000, channels: 1 },
+            },
+        });
+        this.options.capture.on("data", this.onCaptureData);
+        this.options.capture.on("end", this.onCaptureEnd);
+        this.options.capture.on("close", this.onCaptureEnd);
+        this.options.capture.on("error", this.onCaptureError);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_stream_opened",
+            message: "local media stream opened",
+            meta: { callSid, streamSid },
+        });
+    }
+    send(data) {
+        if (!this.isOpen())
+            return;
+        let message;
+        try {
+            message = JSON.parse(data.toString());
+        }
+        catch {
+            message = null;
+        }
+        if (!message || typeof message !== "object") {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "senses",
+                event: "senses.voice_local_stream_message_rejected",
+                message: "local media stream ignored a message that was not a JSON object",
+                meta: { callSid: this.options.callSid },
+            });
+            return;
+        }
+        const { event, media, mark } = message;
+        if (event === "media" && typeof media?.payload === "string") {
+            const audio = Buffer.from(media.payload, "base64");
+            this.options.onPlaybackAudio?.(audio.length);
+            this.options.playback.enqueue(audio);
+        }
+        else if (event === "clear") {
+            this.options.playback.clear();
+        }
+        else if (event === "mark" && typeof mark?.name === "string") {
+            this.options.playback.mark(mark.name, (name) => {
+                if (this.isOpen())
+                    this.emitMessage({ event: "mark", mark: { name } });
+            });
+        }
+    }
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.options.capture.off("data", this.onCaptureData);
+        this.options.capture.off("end", this.onCaptureEnd);
+        this.options.capture.off("close", this.onCaptureEnd);
+        this.options.capture.off("error", this.onCaptureError);
+        this.options.playback.stop();
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_stream_closed",
+            message: "local media stream closed",
+            meta: { callSid: this.options.callSid },
+        });
+        this.emit("close");
+        this.options.onEnded();
+    }
+    endCapture() {
+        this.chunker.flush();
+        this.emitMessage({ event: "stop", stop: { callSid: this.options.callSid } });
+        this.close();
+    }
+    emitMedia(frame) {
+        this.chunk += 1;
+        this.options.onCaptureFrame?.(frame);
+        this.emitMessage({
+            event: "media",
+            media: {
+                track: "inbound",
+                chunk: String(this.chunk),
+                timestamp: String(this.chunk * FRAME_MS),
+                payload: frame.toString("base64"),
+            },
+        });
+    }
+    emitMessage(body) {
+        this.sequence += 1;
+        const message = { ...body, sequenceNumber: String(this.sequence), streamSid: this.options.streamSid };
+        this.emit("message", Buffer.from(JSON.stringify(message)));
+    }
+}
+exports.LocalMediaStreamSocket = LocalMediaStreamSocket;

package/dist/senses/voice/twilio-phone-runtime.js

@@ -38,6 +38,7 @@ exports.resolveTwilioPhoneTransportRuntime = resolveTwilioPhoneTransportRuntime;
 exports.startConfiguredTwilioPhoneTransport = startConfiguredTwilioPhoneTransport;
 exports.placeConfiguredTwilioPhoneCall = placeConfiguredTwilioPhoneCall;
 exports.closeTwilioPhoneBridgeServer = closeTwilioPhoneBridgeServer;
+exports.resolveLocalAudioRealtimeOptions = resolveLocalAudioRealtimeOptions;
 const path = __importStar(require("path"));
 const crypto = __importStar(require("node:crypto"));
 const identity_1 = require("../../heart/identity");
@@ -818,3 +819,18 @@ function closeTwilioPhoneBridgeServer(server) {
         }),
     ]).then(() => undefined);
 }
+/**
+ * The OpenAI Realtime settings for a local audio join. Local audio needs no public URL or Twilio
+ * credentials, only the Realtime key and voice settings, so the phone resolver is asked for a
+ * placeholder-enabled media-stream setup and only its Realtime half is returned.
+ */
+async function resolveLocalAudioRealtimeOptions(agentName, deps = defaultTwilioPhoneOutboundCallRuntimeDeps) {
+    const settings = await readFreshRuntimeSettings(agentName, {
+        enabled: true,
+        publicBaseUrl: "https://local-audio.invalid",
+        transportMode: "media-stream",
+        conversationEngine: "openai-realtime",
+        outboundConversationEngine: "openai-realtime",
+    }, undefined, false, deps, { preferCached: true });
+    return settings.openaiRealtime;
+}