agent-voice 0.2.3 → 0.3.0

package/dist/ask-5J4JCHM4.js

@@ -0,0 +1,307 @@
+import {
+  createRealtimeSession
+} from "./chunk-3YEHGYHI.js";
+import {
+  DEFAULT_VOICE,
+  SAMPLE_RATE
+} from "./chunk-YU5FF2L7.js";
+
+// src/ask.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function pcm16Rms(pcm16) {
+  const samples = Math.floor(pcm16.length / 2);
+  if (samples === 0) return 0;
+  let sumSquares = 0;
+  for (let i = 0; i < samples; i++) {
+    const value = pcm16.readInt16LE(i * 2);
+    sumSquares += value * value;
+  }
+  return Math.sqrt(sumSquares / samples);
+}
+function readEnvInt(name, fallback) {
+  const raw = process.env[name];
+  if (raw == null) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+async function ask(message, options = {}) {
+  const {
+    voice = DEFAULT_VOICE,
+    timeout = 30,
+    ack = false,
+    auth,
+    createSession,
+    createAudioEngine,
+    onTrace,
+    onAudioFrameSent,
+    onAssistantAudio,
+    onMicAudio
+  } = options;
+  const { AudioEngine } = require2("agent-voice-audio");
+  const streamDelayMs = readEnvInt("AGENT_VOICE_AEC_STREAM_DELAY_MS", 30);
+  const engine = (createAudioEngine ?? ((engineOptions) => new AudioEngine(engineOptions)))({
+    sampleRate: SAMPLE_RATE,
+    channels: 1,
+    enableAec: true,
+    streamDelayMs
+  });
+  engine.start();
+  const debug = process.env.AGENT_VOICE_DEBUG_ASK_EVENTS === "1";
+  const startMs = Date.now();
+  function logEvent(event, detail) {
+    if (!debug) return;
+    const elapsed = Date.now() - startMs;
+    const suffix = detail ? ` ${detail}` : "";
+    process.stderr.write(`[ask ${elapsed}ms] ${event}${suffix}
+`);
+  }
+  function trace(event, detail) {
+    onTrace?.({ atMs: Date.now() - startMs, event, detail });
+  }
+  logEvent("start");
+  trace("start");
+  return new Promise((resolve, reject) => {
+    let transcript = "";
+    let timeoutTimer = null;
+    let responseStartTimer = null;
+    let transcriptTimer = null;
+    let capturePollTimer = null;
+    let speechDetected = false;
+    let speechStartedAtMs = 0;
+    let initialResponseDone = false;
+    let heardAssistantAudio = false;
+    let lastAssistantAudioAt = 0;
+    let nearEndEvidenceSeen = false;
+    let nearEndEvidenceAtMs = 0;
+    let nearEndEvidenceConfirmed = false;
+    let cleaned = false;
+    let settled = false;
+    async function cleanup() {
+      if (cleaned) return;
+      cleaned = true;
+      logEvent("cleanup:start");
+      trace("cleanup:start");
+      if (timeoutTimer) clearTimeout(timeoutTimer);
+      if (responseStartTimer) clearTimeout(responseStartTimer);
+      if (transcriptTimer) clearTimeout(transcriptTimer);
+      if (capturePollTimer) clearInterval(capturePollTimer);
+      try {
+        engine.stop();
+        engine.close();
+      } catch {
+      }
+      session.close();
+      logEvent("cleanup:done");
+      trace("cleanup:done");
+    }
+    function resolveOnce(value) {
+      if (settled) return;
+      settled = true;
+      cleanup().then(() => resolve(value));
+    }
+    function rejectOnce(error) {
+      if (settled) return;
+      settled = true;
+      cleanup().then(() => reject(error));
+    }
+    capturePollTimer = setInterval(() => {
+      if (settled) return;
+      let rawFrames = [];
+      let processedFrames = [];
+      try {
+        rawFrames = engine.readRawCapture(64);
+        processedFrames = engine.readProcessedCapture(64);
+      } catch (err) {
+        rejectOnce(
+          new Error(
+            `audio engine capture read failed: ${err instanceof Error ? err.message : String(err)}`
+          )
+        );
+        trace("audio:capture_read_error", {
+          error: err instanceof Error ? err.message : String(err)
+        });
+        return;
+      }
+      for (const frame of rawFrames) onMicAudio?.(frame);
+      if (!heardAssistantAudio) return;
+      for (const frame of processedFrames) {
+        const rms = pcm16Rms(frame);
+        const configuredMinSpeechRms = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS",
+          220
+        );
+        const relaxAfterMs = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS_RELAX_AFTER_MS",
+          500
+        );
+        const relaxedMinSpeechRms = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS_RELAXED",
+          120
+        );
+        const minSpeechRms = speechDetected && speechStartedAtMs > 0 && Date.now() - speechStartedAtMs >= relaxAfterMs ? relaxedMinSpeechRms : configuredMinSpeechRms;
+        if (rms >= minSpeechRms) {
+          nearEndEvidenceSeen = true;
+          nearEndEvidenceAtMs = Date.now();
+          if (!nearEndEvidenceConfirmed && speechStartedAtMs > 0) {
+            const evidencePreRollMs = readEnvInt(
+              "AGENT_VOICE_SPEECH_EVIDENCE_PREROLL_MS",
+              200
+            );
+            const evidencePostRollMs = readEnvInt(
+              "AGENT_VOICE_SPEECH_EVIDENCE_POSTROLL_MS",
+              1500
+            );
+            if (nearEndEvidenceAtMs >= speechStartedAtMs - evidencePreRollMs && nearEndEvidenceAtMs <= speechStartedAtMs + evidencePostRollMs) {
+              nearEndEvidenceConfirmed = true;
+            }
+          }
+          trace("audio:near_end_evidence", { rms, minSpeechRms });
+        }
+        onAudioFrameSent?.(frame);
+        session.sendAudio(frame);
+      }
+      if (processedFrames.length > 0) {
+        trace("audio:sent_capture", { frames: processedFrames.length });
+      }
+    }, 10);
+    const session = (createSession ?? createRealtimeSession)({
+      voice,
+      mode: "default",
+      ack,
+      auth,
+      onAudioDelta(pcm16) {
+        logEvent("realtime:audio_delta", `bytes=${pcm16.length}`);
+        trace("realtime:audio_delta", { bytes: pcm16.length });
+        heardAssistantAudio = true;
+        lastAssistantAudioAt = Date.now();
+        onAssistantAudio?.(pcm16);
+        engine.play(pcm16);
+      },
+      onTranscript(text) {
+        const echoGuardMs = readEnvInt("AGENT_VOICE_ECHO_GUARD_MS", 1500);
+        const sinceAssistantMs = Date.now() - lastAssistantAudioAt;
+        if (heardAssistantAudio && sinceAssistantMs < echoGuardMs) {
+          logEvent(
+            "realtime:transcript_ignored_echo_guard",
+            `since_assistant_ms=${sinceAssistantMs} text="${text}"`
+          );
+          trace("realtime:transcript_ignored_echo_guard", {
+            sinceAssistantMs,
+            text
+          });
+          return;
+        }
+        logEvent("realtime:transcript", `text="${text}"`);
+        trace("realtime:transcript", { text });
+        if (speechDetected && !nearEndEvidenceConfirmed) {
+          trace("realtime:transcript_ignored_no_near_end_evidence", {
+            text,
+            speechStartedAtMs,
+            nearEndEvidenceSeen,
+            nearEndEvidenceAtMs
+          });
+          return;
+        }
+        if (transcriptTimer) {
+          clearTimeout(transcriptTimer);
+          transcriptTimer = null;
+        }
+        transcript = text;
+        if (!ack) resolveOnce(transcript);
+      },
+      onSpeechStarted() {
+        logEvent("realtime:speech_started");
+        trace("realtime:speech_started");
+        speechDetected = true;
+        speechStartedAtMs = Date.now();
+        if (nearEndEvidenceSeen && !nearEndEvidenceConfirmed) {
+          const evidencePreRollMs = readEnvInt(
+            "AGENT_VOICE_SPEECH_EVIDENCE_PREROLL_MS",
+            200
+          );
+          if (nearEndEvidenceAtMs >= speechStartedAtMs - evidencePreRollMs) {
+            nearEndEvidenceConfirmed = true;
+          }
+        }
+        if (timeoutTimer) {
+          clearTimeout(timeoutTimer);
+          timeoutTimer = null;
+        }
+        if (transcriptTimer) clearTimeout(transcriptTimer);
+        transcriptTimer = setTimeout(() => {
+          logEvent("timeout:no_transcript_after_speech");
+          trace("timeout:no_transcript_after_speech", {
+            timeoutSeconds: timeout
+          });
+          rejectOnce(
+            new Error(
+              `No transcript received within ${timeout}s after speech started`
+            )
+          );
+        }, timeout * 1e3);
+        if (!initialResponseDone && heardAssistantAudio) {
+          try {
+            engine.play(Buffer.alloc(0));
+          } catch {
+          }
+        }
+      },
+      onInitialResponseDone() {
+        logEvent("realtime:initial_response_done");
+        trace("realtime:initial_response_done");
+        initialResponseDone = true;
+        timeoutTimer = setTimeout(() => {
+          if (!speechDetected) {
+            logEvent("timeout:no_speech");
+            trace("timeout:no_speech", { timeoutSeconds: timeout });
+            rejectOnce(
+              new Error(`No speech detected within ${timeout}s timeout`)
+            );
+          }
+        }, timeout * 1e3);
+      },
+      onDone() {
+        logEvent("realtime:done");
+        trace("realtime:done");
+        if (ack) resolveOnce(transcript);
+      },
+      onError(error) {
+        logEvent("realtime:error", error);
+        trace("realtime:error", { error });
+        rejectOnce(new Error(error));
+      }
+    });
+    session.connect().then(
+      () => {
+        logEvent("realtime:connected");
+        trace("realtime:connected");
+        logEvent("realtime:send_message");
+        trace("realtime:send_message");
+        session.sendMessage(message);
+        responseStartTimer = setTimeout(() => {
+          if (!heardAssistantAudio) {
+            logEvent("timeout:no_assistant_audio");
+            trace("timeout:no_assistant_audio");
+            rejectOnce(
+              new Error("No assistant audio received after sending message")
+            );
+          }
+        }, 1e4);
+      },
+      (err) => {
+        logEvent(
+          "realtime:connect_error",
+          err instanceof Error ? err.message : String(err)
+        );
+        trace("realtime:connect_error", {
+          error: err instanceof Error ? err.message : String(err)
+        });
+        rejectOnce(err instanceof Error ? err : new Error(String(err)));
+      }
+    );
+  });
+}
+export {
+  ask
+};

package/dist/{ask-OIE6HL2H.js → ask-F6CPRZ22.js}

@@ -75,6 +75,7 @@ async function ask(message, options = {}) {
     let lastAssistantAudioAt = 0;
     let nearEndEvidenceSeen = false;
     let nearEndEvidenceAtMs = 0;
+    let nearEndEvidenceConfirmed = false;
     let cleaned = false;
     let settled = false;
     async function cleanup() {
@@ -127,10 +128,35 @@ async function ask(message, options = {}) {
       if (!heardAssistantAudio) return;
       for (const frame of processedFrames) {
         const rms = pcm16Rms(frame);
-        const minSpeechRms = readEnvInt("AGENT_VOICE_MIN_SPEECH_RMS", 550);
+        const configuredMinSpeechRms = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS",
+          220
+        );
+        const relaxAfterMs = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS_RELAX_AFTER_MS",
+          500
+        );
+        const relaxedMinSpeechRms = readEnvInt(
+          "AGENT_VOICE_MIN_SPEECH_RMS_RELAXED",
+          120
+        );
+        const minSpeechRms = speechDetected && speechStartedAtMs > 0 && Date.now() - speechStartedAtMs >= relaxAfterMs ? relaxedMinSpeechRms : configuredMinSpeechRms;
         if (rms >= minSpeechRms) {
           nearEndEvidenceSeen = true;
           nearEndEvidenceAtMs = Date.now();
+          if (!nearEndEvidenceConfirmed && speechStartedAtMs > 0) {
+            const evidencePreRollMs = readEnvInt(
+              "AGENT_VOICE_SPEECH_EVIDENCE_PREROLL_MS",
+              200
+            );
+            const evidencePostRollMs = readEnvInt(
+              "AGENT_VOICE_SPEECH_EVIDENCE_POSTROLL_MS",
+              1500
+            );
+            if (nearEndEvidenceAtMs >= speechStartedAtMs - evidencePreRollMs && nearEndEvidenceAtMs <= speechStartedAtMs + evidencePostRollMs) {
+              nearEndEvidenceConfirmed = true;
+            }
+          }
           trace("audio:near_end_evidence", { rms, minSpeechRms });
         }
         onAudioFrameSent?.(frame);
@@ -169,23 +195,14 @@ async function ask(message, options = {}) {
         }
         logEvent("realtime:transcript", `text="${text}"`);
         trace("realtime:transcript", { text });
-        if (speechDetected) {
-          const evidenceWindowMs = readEnvInt(
-            "AGENT_VOICE_SPEECH_EVIDENCE_WINDOW_MS",
-            1200
-          );
-          const evidenceAgeMs = nearEndEvidenceSeen ? Math.abs(nearEndEvidenceAtMs - speechStartedAtMs) : Number.POSITIVE_INFINITY;
-          if (!nearEndEvidenceSeen || evidenceAgeMs > evidenceWindowMs) {
-            trace("realtime:transcript_ignored_no_near_end_evidence", {
-              text,
-              speechStartedAtMs,
-              nearEndEvidenceSeen,
-              nearEndEvidenceAtMs,
-              evidenceAgeMs,
-              evidenceWindowMs
-            });
-            return;
-          }
+        if (speechDetected && !nearEndEvidenceConfirmed) {
+          trace("realtime:transcript_ignored_no_near_end_evidence", {
+            text,
+            speechStartedAtMs,
+            nearEndEvidenceSeen,
+            nearEndEvidenceAtMs
+          });
+          return;
         }
         if (transcriptTimer) {
           clearTimeout(transcriptTimer);
@@ -199,6 +216,15 @@ async function ask(message, options = {}) {
         trace("realtime:speech_started");
         speechDetected = true;
         speechStartedAtMs = Date.now();
+        if (nearEndEvidenceSeen && !nearEndEvidenceConfirmed) {
+          const evidencePreRollMs = readEnvInt(
+            "AGENT_VOICE_SPEECH_EVIDENCE_PREROLL_MS",
+            200
+          );
+          if (nearEndEvidenceAtMs >= speechStartedAtMs - evidencePreRollMs) {
+            nearEndEvidenceConfirmed = true;
+          }
+        }
         if (timeoutTimer) {
           clearTimeout(timeoutTimer);
           timeoutTimer = null;

package/dist/{auth-KET5DNSE.js → auth-4VUEFCFK.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   writeAuthConfig
-} from "./chunk-RGYWLATZ.js";
+} from "./chunk-ZNUQXGGO.js";
 import "./chunk-AHLLYIEW.js";
 
 // src/auth.ts

package/dist/chunk-3YEHGYHI.js

@@ -0,0 +1,115 @@
+// src/realtime.ts
+import { OpenAIRealtimeWS } from "openai/beta/realtime/ws";
+var SYSTEM_INSTRUCTIONS = `
+# Role
+Voice relay between an AI agent and a human.
+
+# Instructions
+- When given a text message, read it aloud EXACTLY as written. Do not add, remove, or rephrase anything.
+- After the human responds, acknowledge briefly \u2014 a few words only. Vary your phrasing.
+- NEVER repeat back what the user said verbatim.
+- NEVER ask follow-up questions.
+- Keep every response under one sentence.
+
+# Tone
+- Calm, neutral, concise.
+`.trim();
+function createRealtimeSession(options) {
+  let rt;
+  let responseCount = 0;
+  function configureSession() {
+    const turnDetection = options.mode === "say" ? void 0 : {
+      type: "semantic_vad",
+      eagerness: "medium",
+      create_response: options.ack,
+      interrupt_response: true
+    };
+    rt.send({
+      type: "session.update",
+      session: {
+        instructions: SYSTEM_INSTRUCTIONS,
+        voice: options.voice,
+        input_audio_format: "pcm16",
+        output_audio_format: "pcm16",
+        input_audio_transcription: { model: "gpt-4o-transcribe" },
+        turn_detection: turnDetection
+      }
+    });
+  }
+  function bindEvents() {
+    rt.on("response.audio.delta", (event) => {
+      const pcm16 = Buffer.from(event.delta, "base64");
+      options.onAudioDelta(pcm16);
+    });
+    rt.on("response.audio.done", () => {
+      options.onAudioDone?.();
+    });
+    rt.on("conversation.item.input_audio_transcription.completed", (event) => {
+      options.onTranscript(event.transcript);
+    });
+    rt.on("input_audio_buffer.speech_started", () => {
+      options.onSpeechStarted();
+    });
+    rt.on("response.done", () => {
+      responseCount++;
+      if (responseCount === 1) {
+        options.onInitialResponseDone();
+      } else if (responseCount === 2) {
+        options.onDone();
+      }
+    });
+    rt.on("error", (event) => {
+      options.onError(event.error?.message ?? "Unknown realtime error");
+    });
+  }
+  return {
+    connect() {
+      return new Promise((resolve, reject) => {
+        const client = options.auth ? {
+          apiKey: options.auth.apiKey,
+          baseURL: options.auth.baseUrl ?? "https://api.openai.com/v1"
+        } : void 0;
+        rt = new OpenAIRealtimeWS({ model: "gpt-4o-realtime-preview" }, client);
+        rt.socket.on("open", () => {
+          configureSession();
+          bindEvents();
+          resolve();
+        });
+        rt.socket.on("error", (err) => {
+          reject(new Error(`WebSocket connection failed: ${err.message}`));
+        });
+      });
+    },
+    sendMessage(text) {
+      rt.send({
+        type: "conversation.item.create",
+        item: {
+          type: "message",
+          role: "user",
+          content: [
+            {
+              type: "input_text",
+              text: `Read this aloud exactly as written, word for word. Do not add, remove, or change anything:
+
+${text}`
+            }
+          ]
+        }
+      });
+      rt.send({ type: "response.create" });
+    },
+    sendAudio(pcm16) {
+      rt.send({
+        type: "input_audio_buffer.append",
+        audio: pcm16.toString("base64")
+      });
+    },
+    close() {
+      rt?.close();
+    }
+  };
+}
+
+export {
+  createRealtimeSession
+};