@ouro.bot/cli 0.1.0-alpha.92 → 0.1.0-alpha.921

package/dist/senses/voice/local-audio-devices.js

@@ -0,0 +1,516 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalAudioDevices = exports.KILL_GRACE_MS = exports.defaultProcessHost = exports.PS_ENV = exports.PLAYBACK_DEVICE_NAME = exports.CAPTURE_DEVICE_NAME = void 0;
+exports.soxCaptureArgs = soxCaptureArgs;
+exports.ffmpegPlaybackArgs = ffmpegPlaybackArgs;
+exports.audioToolboxListArgs = audioToolboxListArgs;
+exports.findAudioToolboxDeviceIndex = findAudioToolboxDeviceIndex;
+exports.defaultListAudioToolboxDevices = defaultListAudioToolboxDevices;
+exports.fileCaptureArgs = fileCaptureArgs;
+exports.filePlaybackArgs = filePlaybackArgs;
+exports.defaultKillGroup = defaultKillGroup;
+exports.defaultInspectProcess = defaultInspectProcess;
+exports.defaultInspectProcessAsync = defaultInspectProcessAsync;
+exports.checkLocalAudioRouting = checkLocalAudioRouting;
+exports.soxProbeToneArgs = soxProbeToneArgs;
+exports.probeCaptureLoopback = probeCaptureLoopback;
+exports.capturedNothingReason = capturedNothingReason;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const runtime_1 = require("../../nerves/runtime");
+const local_audio_1 = require("./local-audio");
+const local_audio_mute_1 = require("./local-audio-mute");
+const RAW_MULAW = ["-t", "raw", "-r", "8000", "-e", "mu-law", "-b", "8", "-c", "1"];
+const AUDIO_PROCESS_NAMES = new Set(["sox", "ffmpeg"]);
+exports.CAPTURE_DEVICE_NAME = "BlackHole 16ch";
+exports.PLAYBACK_DEVICE_NAME = "BlackHole 2ch";
+/** Capture a CoreAudio device as mono 8 kHz mu-law on stdout (channels 1 and 2 mixed). */
+function soxCaptureArgs(device) {
+    // BlackHole 16ch spreads a stereo call over many channels; `remix 1,2` mixes the first two to mono.
+    // `--buffer` sizes sox's writes to stdout (1024 bytes is 128 ms of 8 kHz mu-law), while the larger
+    // `--input-buffer` keeps the CoreAudio reads big enough to avoid dropped audio. A single
+    // `--buffer 8192` delayed everything the agent heard by about a second.
+    return ["-q", "--buffer", "1024", "--input-buffer", "8192", "-t", "coreaudio", device, ...RAW_MULAW, "-", "remix", "1,2"];
+}
+/**
+ * Play raw mono 8 kHz mu-law from stdin to an AudioToolbox output device, chosen by index.
+ *
+ * sox's CoreAudio output can deadlock when the process runs at lowered priority (a zsh background
+ * job is niced by default, and so is a launchd Background job): its main thread holds a lock inside
+ * `AudioDeviceStart` while the device IO thread waits for that lock, so nothing is ever played.
+ * Observed live: 5 of 5 niced joins were silent with sox and 2 of 2 spoke with ffmpeg. The probe
+ * flags keep ffmpeg from buffering input before it starts playing.
+ */
+function ffmpegPlaybackArgs(deviceIndex) {
+    return [
+        "-hide_banner", "-loglevel", "error", "-nostdin",
+        "-probesize", "32", "-analyzeduration", "0", "-fflags", "nobuffer",
+        "-f", "mulaw", "-ar", "8000", "-ac", "1", "-i", "pipe:0",
+        "-f", "audiotoolbox", "-audio_device_index", String(deviceIndex), "-",
+    ];
+}
+/** ffmpeg prints the AudioToolbox devices to its log; `-t 0` means nothing is played. */
+function audioToolboxListArgs() {
+    return ["-hide_banner", "-f", "lavfi", "-i", "anullsrc", "-t", "0", "-f", "audiotoolbox", "-list_devices", "true", "-"];
+}
+// "[AudioToolbox @ 0x7b514003c0] [1]                  BlackHole 2ch, BlackHole2ch_UID"
+const AUDIOTOOLBOX_DEVICE_LINE = /\[(\d+)\]\s+(.+),\s*[^,]*$/;
+/** The index ffmpeg uses for a device with exactly this name, or undefined when it is not listed. */
+function findAudioToolboxDeviceIndex(listing, deviceName) {
+    for (const line of listing.split("\n")) {
+        const match = AUDIOTOOLBOX_DEVICE_LINE.exec(line.trim());
+        if (match && match[2].trim() === deviceName)
+            return Number(match[1]);
+    }
+    return undefined;
+}
+/** Runs ffmpeg's device listing and returns its log. Rejects only when ffmpeg itself cannot run. */
+function defaultListAudioToolboxDevices(ffmpegPath = "ffmpeg", run = (file, args, callback) => {
+    (0, child_process_1.execFile)(file, args, { encoding: "utf8", timeout: 5000 }, (error, stdout, stderr) => callback(error, stdout, stderr));
+}) {
+    return new Promise((resolve, reject) => {
+        run(ffmpegPath, audioToolboxListArgs(), (error, _stdout, stderr) => {
+            if (error && typeof error.code !== "number")
+                reject(error);
+            else
+                resolve(stderr);
+        });
+    });
+}
+/** Decode any audio file to the capture stream format (stdout). The caller paces it. */
+function fileCaptureArgs(inputPath) {
+    return ["-q", inputPath, ...RAW_MULAW, "-"];
+}
+/** Write raw mono 8 kHz mu-law from stdin to a WAV file. */
+function filePlaybackArgs(outputPath) {
+    return ["-q", ...RAW_MULAW, "-", "-t", "wav", outputPath];
+}
+/** Signals a whole process group. Returns false when the group is already gone. */
+function defaultKillGroup(pid, signal) {
+    try {
+        process.kill(-pid, signal);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// `ps -o lstart=` prints "Fri Oct  9 11:59:46 2026": weekday, month, day, time, year. The words are not
+// matched by name, so a locale that spells them differently still parses (and LC_ALL=C pins the format).
+const PS_LINE = /^(\S+\s+\S+\s+\d+\s+[\d:]{8}\s+\d{4})\s+(.+)$/;
+exports.PS_ENV = { ...process.env, LC_ALL: "C" };
+const PS_ARGS = (pid) => ["-p", String(pid), "-o", "lstart=", "-o", "comm="];
+function parsePsLine(out) {
+    const match = PS_LINE.exec(out.trim());
+    return match ? { startTime: match[1], command: match[2] } : null;
+}
+/** Command and start time of a live process, or null when it is gone. Used to avoid killing a reused pid. */
+function defaultInspectProcess(pid, runPs = (args) => (0, child_process_1.execFileSync)("ps", args, { encoding: "utf8", timeout: 2000, env: exports.PS_ENV })) {
+    try {
+        return parsePsLine(runPs(PS_ARGS(pid)));
+    }
+    catch {
+        return null;
+    }
+}
+/** The same lookup without blocking the event loop (audio is flowing while this runs). */
+function defaultInspectProcessAsync(pid, runPs = (args) => new Promise((resolve, reject) => {
+    (0, child_process_1.execFile)("ps", args, { encoding: "utf8", timeout: 2000, env: exports.PS_ENV }, (error, stdout) => (error ? reject(error) : resolve(stdout)));
+})) {
+    return runPs(PS_ARGS(pid)).then(parsePsLine, () => null);
+}
+exports.defaultProcessHost = {
+    get pid() { return process.pid; },
+    once: (event, listener) => process.once(event, listener),
+    off: (event, listener) => process.off(event, listener),
+    kill: (pid, signal) => process.kill(pid, signal),
+    listenerCount: (event) => process.listenerCount(event),
+};
+/**
+ * sox does not act on SIGTERM while it is blocked in a CoreAudio call or in a stdin read (observed
+ * live: a tone process stayed alive for minutes). After this grace, anything left gets SIGKILL.
+ */
+exports.KILL_GRACE_MS = 500;
+/**
+ * Owns the capture and playback sox processes: each runs detached in its own process group,
+ * pids are recorded in a pid file so a later start can sweep leftovers from a crashed run.
+ */
+class LocalAudioDevices {
+    options;
+    running = [];
+    killGroup;
+    inspectProcess;
+    inspectProcessAsync;
+    host;
+    setTimer;
+    signalCleanup = null;
+    constructor(options) {
+        this.options = options;
+        this.killGroup = options.killGroup ?? defaultKillGroup;
+        this.setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms));
+        this.inspectProcess = options.inspectProcess ?? defaultInspectProcess;
+        this.inspectProcessAsync = options.inspectProcessAsync ?? defaultInspectProcessAsync;
+        this.host = options.processHost ?? exports.defaultProcessHost;
+    }
+    start() {
+        this.stop();
+        const command = this.options.soxPath ?? "sox";
+        const capture = this.spawnTracked(command, this.options.captureArgs);
+        const playback = this.spawnTracked(this.options.playbackCommand ?? command, this.options.playbackArgs);
+        this.writePidFile();
+        this.recordStartTimes([...this.running]);
+        this.watchParentSignals();
+        if (!capture.stdout || !playback.stdin) {
+            this.stop();
+            throw new Error("sox did not provide capture stdout / playback stdin stdio");
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_audio_devices_started",
+            message: "local audio sox processes started",
+            meta: { capturePid: capture.pid ?? null, playbackPid: playback.pid ?? null },
+        });
+        return { capture: capture.stdout, playback: playback.stdin };
+    }
+    /**
+     * `force` is for paths where no timer will get to run (the parent is exiting or being signalled):
+     * SIGKILL straight away. Otherwise SIGTERM now and SIGKILL after a short grace to whatever is
+     * still alive.
+     */
+    stop(force = false) {
+        this.signalCleanup?.();
+        const stopping = this.running;
+        this.running = [];
+        const send = (entry, signal) => {
+            if (entry.pid === undefined || !this.killGroup(entry.pid, signal))
+                entry.child.kill(signal);
+        };
+        for (const entry of stopping)
+            send(entry, force ? "SIGKILL" : "SIGTERM");
+        if (!force && stopping.length > 0) {
+            this.setTimer(() => {
+                for (const entry of stopping) {
+                    // A child that exited cleared its pid (the pid may be reused by now): leave it alone.
+                    if (entry.pid !== undefined)
+                        send(entry, "SIGKILL");
+                }
+            }, exports.KILL_GRACE_MS);
+        }
+        if (stopping.length > 0) {
+            fs.rmSync(this.options.pidFile, { force: true });
+            (0, runtime_1.emitNervesEvent)({
+                component: "senses",
+                event: "senses.voice_local_audio_devices_stopped",
+                message: "local audio sox processes stopped",
+                meta: { processes: stopping.length },
+            });
+        }
+    }
+    /**
+     * Graceful end for file output: close the playback stream and let sox finalize the WAV header,
+     * then stop everything. Gives up waiting after `timeoutMs`.
+     */
+    async finish(timeoutMs = 3000) {
+        const playback = this.running[1];
+        if (playback && playback.pid !== undefined) {
+            await new Promise((resolve) => {
+                const timer = setTimeout(resolve, timeoutMs);
+                playback.child.on("exit", () => {
+                    clearTimeout(timer);
+                    resolve();
+                });
+                playback.child.stdin?.end();
+            });
+        }
+        this.stop();
+    }
+    /** Kills sox groups left behind in the pid file by a previous run. Returns how many were killed. */
+    sweepStale() {
+        let text;
+        try {
+            text = fs.readFileSync(this.options.pidFile, "utf8");
+        }
+        catch {
+            return 0;
+        }
+        let killed = 0;
+        const terminated = [];
+        for (const line of text.split("\n")) {
+            const [pidText = "", recordedStart = ""] = line.split("\t");
+            const pid = Number.parseInt(pidText.trim(), 10);
+            if (!Number.isInteger(pid) || pid <= 1)
+                continue;
+            if (!this.isRecordedSox(pid, recordedStart))
+                continue;
+            if (this.killGroup(pid, "SIGTERM")) {
+                killed++;
+                terminated.push({ pid, recordedStart });
+            }
+        }
+        if (terminated.length > 0) {
+            // sox can ignore SIGTERM while blocked in CoreAudio or a stdin read: whatever is still the same sox gets SIGKILL.
+            this.setTimer(() => {
+                for (const { pid, recordedStart } of terminated) {
+                    if (this.isRecordedSox(pid, recordedStart))
+                        this.killGroup(pid, "SIGKILL");
+                }
+            }, exports.KILL_GRACE_MS);
+        }
+        fs.rmSync(this.options.pidFile, { force: true });
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_audio_stale_swept",
+            message: "stale local audio sox processes swept",
+            meta: { killed },
+        });
+        return killed;
+    }
+    /** Only a process that is still one of our audio tools AND started when we recorded it is ours to kill (pids get reused). */
+    isRecordedSox(pid, recordedStart) {
+        const info = this.inspectProcess(pid);
+        if (!info || !AUDIO_PROCESS_NAMES.has(path.basename(info.command)))
+            return false;
+        return recordedStart !== "" && info.startTime === recordedStart;
+    }
+    /**
+     * sox lives in its own process group, so a Ctrl-C or kill aimed at this process would not reach
+     * it. Kill the groups on SIGINT, SIGTERM and exit; for signals, re-raise afterwards so the
+     * process still terminates the way the sender intended.
+     */
+    watchParentSignals() {
+        const handlers = [
+            ["SIGINT", () => this.onSignal("SIGINT")],
+            ["SIGTERM", () => this.onSignal("SIGTERM")],
+            ["exit", () => this.stop(true)],
+        ];
+        for (const [event, listener] of handlers)
+            this.host.once(event, listener);
+        this.signalCleanup = () => {
+            this.signalCleanup = null;
+            for (const [event, listener] of handlers)
+                this.host.off(event, listener);
+        };
+    }
+    onSignal(signal) {
+        (0, runtime_1.emitNervesEvent)({
+            level: "warn",
+            component: "senses",
+            event: "senses.voice_local_audio_parent_signal",
+            message: "local audio parent received a stop signal; killing sox",
+            meta: { signal },
+        });
+        if (this.options.gracefulPlayback) {
+            this.winddownFileWriter();
+            this.setTimer(() => {
+                this.stop(true);
+                this.reraise(signal);
+            }, exports.KILL_GRACE_MS);
+            return;
+        }
+        this.stop(true);
+        this.reraise(signal);
+    }
+    /** The WAV writer finalizes its header on EOF or SIGTERM; everything else is not worth waiting for. */
+    winddownFileWriter() {
+        this.running.forEach((entry, index) => {
+            if (index === 0) {
+                this.sendSignal(entry, "SIGKILL");
+                return;
+            }
+            entry.child.stdin?.end();
+            this.sendSignal(entry, "SIGTERM");
+        });
+    }
+    sendSignal(entry, signal) {
+        if (entry.pid === undefined || !this.killGroup(entry.pid, signal))
+            entry.child.kill(signal);
+    }
+    /** Re-raise so the process still ends the way the sender intended, unless another listener owns that. */
+    reraise(signal) {
+        if (this.host.listenerCount(signal) === 0)
+            this.host.kill(this.host.pid, signal);
+    }
+    spawnTracked(command, args) {
+        const child = this.options.spawner.spawn(command, args, { detached: true });
+        const entry = { child, pid: child.pid };
+        this.running.push(entry);
+        child.on("exit", () => {
+            entry.pid = undefined;
+        });
+        return Object.assign(child, { pid: child.pid });
+    }
+    writePidFile() {
+        const lines = [];
+        for (const entry of this.running) {
+            if (entry.pid === undefined)
+                continue;
+            lines.push(`${entry.pid}\t`);
+        }
+        fs.mkdirSync(path.dirname(this.options.pidFile), { recursive: true });
+        fs.writeFileSync(this.options.pidFile, `${lines.join("\n")}\n`);
+    }
+    /**
+     * The pid file first lists bare pids (a later sweep will not kill a pid without a start time), then
+     * gains start times once `ps` answers, so the event loop never waits on `ps` while audio flows.
+     */
+    recordStartTimes(entries) {
+        const pids = entries.map((entry) => entry.pid);
+        void Promise.all(pids.map((pid) => (pid === undefined ? Promise.resolve(null) : this.inspectProcessAsync(pid)))).then((infos) => {
+            if (entries.some((entry) => !this.running.includes(entry)))
+                return;
+            const lines = [];
+            entries.forEach((entry, index) => {
+                if (entry.pid === pids[index] && entry.pid !== undefined)
+                    lines.push(`${entry.pid}\t${infos[index]?.startTime ?? ""}`);
+            });
+            try {
+                fs.writeFileSync(this.options.pidFile, `${lines.join("\n")}\n`);
+            }
+            catch {
+                // The state directory is gone (the run ended): nothing to record.
+            }
+        });
+    }
+}
+exports.LocalAudioDevices = LocalAudioDevices;
+async function checkLocalAudioRouting(inspect, queryMute) {
+    const inspection = await inspect({ captureDeviceName: exports.CAPTURE_DEVICE_NAME, outputDeviceName: exports.PLAYBACK_DEVICE_NAME });
+    const steps = [];
+    const muteSteps = inspection.missing.length === 0 && queryMute
+        ? (0, local_audio_mute_1.muteFixSteps)((await queryMute([exports.CAPTURE_DEVICE_NAME, exports.PLAYBACK_DEVICE_NAME])) ?? [])
+        : [];
+    steps.push(...muteSteps);
+    if (inspection.status === "unknown") {
+        // The inspection itself failed, so nothing is known about the devices: do not tell the owner to install them.
+        const error = inspection.error ?? "unknown error";
+        steps.push(/ENOENT|not found/i.test(error)
+            ? "The device check needs SwitchAudioSource: brew install switchaudio-osx"
+            : `Audio device inspection failed: ${error}`);
+    }
+    else {
+        const missing = inspection.missing.map((name) => name.toLowerCase().replace(/\s+/g, "-"));
+        if (missing.length > 0) {
+            steps.push(`Install the missing audio devices: brew install --cask ${missing.join(" ")} (then restart macOS audio or reboot).`);
+        }
+    }
+    steps.push(`In the call app, set the microphone to "${exports.PLAYBACK_DEVICE_NAME}" (the agent speaks into it).`, `In the call app, set the speaker output to "${exports.CAPTURE_DEVICE_NAME}" (the agent listens to it).`, `To hear the call yourself, create a Multi-Output Device in Audio MIDI Setup that includes "${exports.CAPTURE_DEVICE_NAME}" and your speakers, and use it as the call app's speaker output.`);
+    return { ok: inspection.status === "ready" && muteSteps.length === 0, steps };
+}
+/** Plays a quiet tone into a CoreAudio device with a separate sox process (stopped as soon as it is heard). */
+function soxProbeToneArgs(device) {
+    return ["-q", "-n", "-t", "coreaudio", device, "synth", "3", "sine", "1000", "vol", "0.2"];
+}
+/**
+ * A quiet room is not a permission failure: BlackHole returns zeros whenever the call app plays
+ * nothing. So before the join we play a short probe tone into the capture device and require a
+ * capture to hear it. The probe uses its own capture and tone processes, so it never touches the
+ * live capture stream: the first seconds of real audio are never consumed or discarded, and the
+ * tone never reaches the agent. The tone starts only after the probe capture has delivered its
+ * first chunk (proof that it is open and listening), so a slow CoreAudio open cannot miss the tone.
+ * Only a missing probe fails the join: either macOS microphone permission is missing for this
+ * process, or the device is muted.
+ */
+function probeCaptureLoopback(options) {
+    const timeoutMs = options.timeoutMs ?? 6000;
+    const openTimeoutMs = options.openTimeoutMs ?? 60_000;
+    const device = options.device ?? exports.CAPTURE_DEVICE_NAME;
+    const command = options.soxPath ?? "sox";
+    const setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms));
+    const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
+    const killGroup = options.killGroup ?? defaultKillGroup;
+    if (options.signal?.aborted)
+        return Promise.resolve({ ok: false, reason: "cancelled" });
+    const listener = options.spawner.spawn(command, soxCaptureArgs(device), { detached: true });
+    let tone;
+    return new Promise((resolve) => {
+        let timer;
+        let done = false;
+        const kill = (child) => {
+            const send = (signal) => {
+                if (child.pid === undefined || !killGroup(child.pid, signal))
+                    child.kill(signal);
+            };
+            let exited = false;
+            child.on("exit", () => { exited = true; });
+            send("SIGTERM");
+            setTimer(() => { if (!exited)
+                send("SIGKILL"); }, exports.KILL_GRACE_MS);
+        };
+        const finish = (result) => {
+            if (done)
+                return;
+            done = true;
+            listener.stdout?.off("data", onData);
+            clearTimer(timer);
+            options.signal?.removeEventListener("abort", onAbort);
+            kill(listener);
+            if (tone)
+                kill(tone);
+            (0, runtime_1.emitNervesEvent)({
+                level: result.ok ? "info" : "error",
+                component: "senses",
+                event: result.ok ? "senses.voice_local_capture_probe_heard" : "senses.voice_local_capture_probe_missing",
+                message: result.ok ? "local capture heard the probe tone" : "local capture did not hear the probe tone",
+                meta: { timeoutMs, reason: result.reason ?? null },
+            });
+            resolve(result);
+        };
+        const onAbort = () => finish({ ok: false, reason: "cancelled" });
+        const onData = (chunk) => {
+            if (!tone) {
+                // The capture is open and delivering: now it is safe to play the tone.
+                clearTimer(timer);
+                tone = options.spawner.spawn(command, soxProbeToneArgs(device), { detached: true });
+                timer = setTimer(() => {
+                    finish({ ok: false, reason: capturedNothingReason(device, `capture heard nothing from the probe tone within ${timeoutMs} ms`) });
+                }, timeoutMs);
+            }
+            if (!(0, local_audio_1.isDigitalSilence)([chunk]))
+                finish({ ok: true });
+        };
+        timer = setTimer(() => {
+            finish({ ok: false, reason: capturedNothingReason(device, `the probe capture delivered no audio at all within ${openTimeoutMs} ms`) });
+        }, openTimeoutMs);
+        options.signal?.addEventListener("abort", onAbort);
+        listener.on("exit", () => finish({ ok: false, reason: capturedNothingReason(device, "the probe capture process exited before hearing the tone") }));
+        listener.stdout?.on("data", onData);
+    });
+}
+/** Digital silence has two usual causes on macOS; always name both fixes. */
+function capturedNothingReason(device, what) {
+    return `${what} on ${device}. Either the device is muted (open Audio MIDI Setup, select ${device}, and uncheck Mute for Input and Output) or macOS microphone permission is missing for the app running this command (System Settings > Privacy & Security > Microphone).`;
+}

package/dist/senses/voice/local-audio-disclosure.js

@@ -0,0 +1,174 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DisclosureTracker = exports.disclosureTimings = void 0;
+const runtime_1 = require("../../nerves/runtime");
+/** Tunable so tests can run the real state machine quickly. */
+exports.disclosureTimings = {
+    /** The join leaves if the disclosure has not been spoken this long after it was first requested. */
+    deadlineMs: 20_000,
+    /** Pause before asking again after a cancelled or cut attempt. */
+    retryDelayMs: 500,
+    /** How long to wait for the response to start before checking whether the request was lost. */
+    createWaitMs: 4_000,
+};
+/** The spoken notice must actually say it is an AI; an answer to the room instead of the notice does not count. */
+const SAYS_AI = /\b(A\.?I\b\.?|artificial intelligence)/iu;
+/**
+ * Makes the spoken disclosure a guarantee instead of a hope. A local audio join asks the model to say
+ * the notice, but the floor gate can delay or overwrite that request and a barge-in can cut its audio.
+ * The tracker follows the one response that carries the notice and counts it as spoken only when the
+ * response completed, its audio transcript is done, no barge-in cut it, and its audio played out.
+ * Anything else asks again, until the deadline, when the join gives up and leaves.
+ */
+class DisclosureTracker {
+    options;
+    state = "idle";
+    responseId;
+    transcriptSeen = false;
+    cut = false;
+    sent = false;
+    attempts = 0;
+    markName = "";
+    deadlineTimer;
+    createTimer;
+    retryTimer;
+    now;
+    setTimer;
+    clearTimer;
+    constructor(options) {
+        this.options = options;
+        this.now = options.now ?? Date.now;
+        this.setTimer = options.setTimer ?? ((cb, ms) => {
+            const handle = setTimeout(cb, ms);
+            handle.unref();
+            return handle;
+        });
+        this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
+    }
+    start() {
+        this.deadlineTimer = this.setTimer(() => this.fail(), exports.disclosureTimings.deadlineMs);
+        this.attempt();
+    }
+    isSpoken() {
+        return this.state === "spoken";
+    }
+    stop() {
+        this.finish("stopped");
+    }
+    /** The session put a response.create on the wire. */
+    noteSent(body) {
+        if (body === this.options.body && this.state === "requested")
+            this.sent = true;
+    }
+    handleRealtimeEvent(event) {
+        if (this.isTerminal())
+            return;
+        const type = event.type;
+        const response = event.response;
+        if (type === "response.created") {
+            if (!this.sent || this.responseId || typeof response?.id !== "string")
+                return;
+            this.responseId = response.id;
+            this.sent = false;
+            this.state = "speaking";
+            this.clear(this.createTimer);
+            return;
+        }
+        if (type === "response.output_audio_transcript.done") {
+            if (this.responseId && event.response_id === this.responseId && typeof event.transcript === "string" && SAYS_AI.test(event.transcript)) {
+                this.transcriptSeen = true;
+            }
+            return;
+        }
+        if (type === "response.done" && this.responseId && response?.id === this.responseId) {
+            if (response.status === "completed" && this.transcriptSeen && !this.cut) {
+                this.state = "playing";
+                this.markName = `disclosure-${this.attempts}`;
+                this.options.host.sendMark(this.markName);
+            }
+            else {
+                this.retry(`response ${String(response.status)}${this.cut ? " after a barge-in" : ""}`);
+            }
+        }
+    }
+    /** The caller's speech cleared the playback queue. */
+    noteCleared() {
+        if (this.state === "speaking" || this.state === "playing")
+            this.cut = true;
+    }
+    /** A playback mark came back. Returns true when it was the disclosure's own. */
+    noteMark(name) {
+        if (this.state !== "playing" || name !== this.markName)
+            return false;
+        if (this.cut) {
+            this.retry("audio was cut before it played out");
+            return true;
+        }
+        this.finish("spoken");
+        const atMs = this.now();
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_local_disclosure_spoken",
+            message: "local audio disclosure notice was spoken and played out",
+            meta: { attempts: this.attempts },
+        });
+        this.options.host.onSpoken(atMs);
+        return true;
+    }
+    attempt() {
+        this.attempts++;
+        this.state = "requested";
+        this.responseId = undefined;
+        this.transcriptSeen = false;
+        this.cut = false;
+        this.sent = false;
+        this.options.host.request(this.options.body);
+        this.createTimer = this.setTimer(() => this.checkRequest(), exports.disclosureTimings.createWaitMs);
+    }
+    /** The response never started: if the session no longer holds the request (the gate overwrote it), ask again. */
+    checkRequest() {
+        if (this.options.host.isQueued(this.options.body)) {
+            this.createTimer = this.setTimer(() => this.checkRequest(), exports.disclosureTimings.createWaitMs);
+            return;
+        }
+        this.attempt();
+    }
+    retry(why) {
+        this.state = "retrying";
+        (0, runtime_1.emitNervesEvent)({
+            level: "warn",
+            component: "senses",
+            event: "senses.voice_local_disclosure_retry",
+            message: "local audio disclosure notice was not spoken; asking again",
+            meta: { why, attempts: this.attempts },
+        });
+        this.retryTimer = this.setTimer(() => this.attempt(), exports.disclosureTimings.retryDelayMs);
+    }
+    fail() {
+        this.finish("failed");
+        (0, runtime_1.emitNervesEvent)({
+            level: "error",
+            component: "senses",
+            event: "senses.voice_local_disclosure_failed",
+            message: "local audio disclosure notice could not be spoken in time; leaving",
+            meta: { attempts: this.attempts, deadlineMs: exports.disclosureTimings.deadlineMs },
+        });
+        this.options.host.onFailed();
+    }
+    isTerminal() {
+        return this.state === "spoken" || this.state === "failed" || this.state === "stopped";
+    }
+    finish(state) {
+        if (this.isTerminal())
+            return;
+        this.state = state;
+        this.clear(this.deadlineTimer);
+        this.clear(this.createTimer);
+        this.clear(this.retryTimer);
+    }
+    clear(handle) {
+        if (handle !== undefined)
+            this.clearTimer(handle);
+    }
+}
+exports.DisclosureTracker = DisclosureTracker;