@sphoro/bugcatch-web-sdk 0.13.1 → 0.14.0

package/src/voice.js

@@ -1,372 +0,0 @@
-// Talking a report through, out loud.
-//
-// When somebody on the team picks up a report, the reporter is offered a call
-// with a triage agent: it asks the three questions the form did not, and what
-// it learns lands in the bug's thread. This is the browser end of that — the
-// microphone, the socket, and the codec between them.
-//
-// Nobody from the team is ever on the line. The agent is software, the copy the
-// reporter sees says so, and the team reads the transcript afterwards.
-//
-// # Why this file is bigger than the rest of the SDK
-//
-// sphoro.voice speaks µ-law at 8 kHz in both directions — the format a
-// telephone carrier hands it, which is what makes a browser call and a phone
-// call indistinguishable to everything downstream. A browser has neither: it
-// captures 32-bit floats at whatever rate the sound card runs at. The gap
-// between those two is G.711 and a resampler, and neither is available from the
-// platform, so both are here.
-//
-// Nothing external is imported to do it. The SDK runs on someone else's page
-// and has no dependencies, and that rule does not bend for audio — the codec
-// and the resampler are a sibling file, shared with the live call, which needs
-// the same µ-law for the copy of the microphone it uplinks.
-
-import {
-  WIRE_RATE, CAPTURE_BLOCK, createContext, resumeContext,
-  downsample, encodeMuLaw, decodeMuLaw,
-} from './audio.js';
-
-// Re-exported because this is where they lived and where the codec test still
-// reaches for them. Moving a hand-rolled G.711 out from under its own test is
-// how a silent call ships.
-export { downsample, encodeMuLaw, decodeMuLaw };
-
-/**
- * What sphoro.voice's realtime protocol is called. Sent as the WebSocket
- * subprotocol so a future v2 can share the endpoint instead of breaking every
- * deployed client on one day.
- */
-const SUBPROTOCOL = 'voiceai.realtime.v1';
-
-/**
- * Start a call.
- *
- * Resolves once the microphone is live and the socket is open, so a caller can
- * show "connected" without guessing. Everything after that arrives through the
- * callbacks.
- *
- * @param {object} opts
- * @param {string} opts.realtimeUrl  From the API. Carries its own credential —
- *   never log it, and never build one by hand.
- * @param {(state: string, detail?: object) => void} [opts.onState]
- *   'connecting' | 'listening' | 'speaking' | 'ended' | 'failed'.
- * @param {(line: {role: string, text: string, final: boolean}) => void} [opts.onTranscript]
- * @returns {Promise<{hangUp: () => void}>}
- */
-export async function startVoiceCall({ realtimeUrl, onState, onTranscript }) {
-  const emit = (state, detail) => { try { onState?.(state, detail); } catch { /* theirs */ } };
-  const say = (line) => { try { onTranscript?.(line); } catch { /* theirs */ } };
-
-  emit('connecting');
-
-  // Asked for before the socket, deliberately. A permission prompt takes as
-  // long as the person takes to read it, and a socket opened first would have
-  // the agent greeting an empty room — sphoro.voice starts talking on connect.
-  let stream;
-  try {
-    stream = await navigator.mediaDevices.getUserMedia({
-      audio: {
-        // A speakerphone conversation with no echo cancellation is one where
-        // the agent hears itself and answers its own greeting.
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-        channelCount: 1,
-      },
-    });
-  } catch (err) {
-    emit('failed', { reason: 'microphone', error: String(err?.name || err) });
-    throw err;
-  }
-
-  const context = createContext();
-  // Started explicitly, and before anything is wired to it.
-  //
-  // A context constructed while the page has no user activation starts
-  // *suspended*, and a suspended context is silent in both directions: nothing
-  // plays out of it, and a ScriptProcessorNode attached to it never fires — so
-  // the reporter hears no greeting and the agent hears no reporter, with no
-  // error anywhere to say why. The microphone prompt above is what puts this
-  // code past the gesture that would have carried it: Safari does not treat
-  // activation as sticky across the await, so the context that follows it is
-  // suspended on exactly the browsers that need the resume most.
-  //
-  // Failure is not fatal here. `resume()` rejects when there is no activation to
-  // spend, and the call still has to be attempted — the state listener below
-  // picks it up if the browser starts the context later.
-  await resumeContext(context);
-  const source = context.createMediaStreamSource(stream);
-
-  let socket;
-  try {
-    socket = await openSocket(realtimeUrl);
-  } catch (err) {
-    stopEverything(stream, context, null, null);
-    emit('failed', { reason: 'connect' });
-    throw err;
-  }
-
-  // --- capture ---------------------------------------------------------------
-
-  // ScriptProcessorNode is deprecated in favour of AudioWorklet, and is used
-  // anyway. A worklet's processor has to be loaded from a URL, which on this
-  // SDK means a blob: URL — and a blob: worklet is refused outright by the
-  // Content-Security-Policy of a great many of the sites this code runs on.
-  // A deprecated node that works on a customer's page beats a modern one that
-  // is blocked by their headers.
-  const capture = context.createScriptProcessor(CAPTURE_BLOCK, 1, 1);
-  let sending = true;
-
-  capture.onaudioprocess = (event) => {
-    if (!sending || socket.readyState !== WebSocket.OPEN) return;
-    const input = event.inputBuffer.getChannelData(0);
-    const frames = context.sampleRate === WIRE_RATE
-      ? input
-      : downsample(input, context.sampleRate, WIRE_RATE);
-    socket.send(encodeMuLaw(frames));
-  };
-
-  source.connect(capture);
-  // A ScriptProcessorNode does not run unless it is connected to a destination,
-  // even when nothing is meant to come out of it. Zero gain is what keeps the
-  // reporter from hearing their own voice played back at them.
-  const mute = context.createGain();
-  mute.gain.value = 0;
-  capture.connect(mute);
-  mute.connect(context.destination);
-
-  // --- playback --------------------------------------------------------------
-
-  // Chunks arrive faster than they play, so each is scheduled to start where
-  // the last one ends. Playing them on arrival instead would overlap them into
-  // noise; waiting for each to finish would gap them into stutter.
-  //
-  // The scheduled nodes are held onto because scheduling is the problem: by the
-  // time somebody talks over the agent, most of what the agent is about to say
-  // has already been handed to the audio clock, and dropping the queue is not
-  // enough to stop it coming out of the speaker.
-  let playHead = 0;
-  const scheduled = new Set();
-
-  const play = (bytes) => {
-    // A context can be suspended long after it started — a backgrounded tab, an
-    // OS audio-session change — and a suspended one accepts scheduled buffers
-    // and plays none of them. Nudged on the way past rather than checked once at
-    // the top, because the failure is a call that goes quiet mid-sentence and
-    // never comes back.
-    if (context.state === 'suspended') void resumeContext(context);
-
-    const samples = decodeMuLaw(bytes);
-    // Declared at the wire's rate rather than the context's: the browser
-    // resamples an AudioBuffer on playback, and it does it better than the
-    // twenty lines it would take to do it here.
-    const buffer = context.createBuffer(1, samples.length, WIRE_RATE);
-    buffer.getChannelData(0).set(samples);
-
-    const node = context.createBufferSource();
-    node.buffer = buffer;
-    node.connect(context.destination);
-    node.onended = () => scheduled.delete(node);
-
-    const now = context.currentTime;
-    // A play head in the past means the queue drained — the agent stopped
-    // talking — so the next chunk starts now rather than at a stale offset.
-    if (playHead < now) playHead = now;
-    node.start(playHead);
-    playHead += buffer.duration;
-    scheduled.add(node);
-  };
-
-  /**
-   * Stop talking, now.
-   *
-   * Called when the caller interrupts and when the call ends. Without it, an
-   * agent that has been talked over keeps playing to the end of whatever was
-   * already scheduled — which is the behaviour barge-in exists to prevent, and
-   * which sounds like the agent ignoring the person.
-   */
-  const stopPlayback = () => {
-    for (const node of scheduled) {
-      try { node.stop(); } catch { /* already finished */ }
-    }
-    scheduled.clear();
-    playHead = 0;
-  };
-
-  // --- the conversation ------------------------------------------------------
-
-  socket.addEventListener('message', (event) => {
-    // Binary frames are not part of the downstream protocol; audio arrives
-    // base64-encoded inside a JSON envelope like everything else.
-    if (typeof event.data !== 'string') return;
-
-    let message;
-    try {
-      message = JSON.parse(event.data);
-    } catch {
-      return;
-    }
-
-    switch (message.type) {
-      case 'session.created':
-        emit('listening');
-        break;
-
-      case 'transcript.partial':
-        if (isSpeech(message.text)) say({ role: 'reporter', text: message.text, final: false });
-        break;
-
-      case 'transcript.final':
-        if (isSpeech(message.text)) say({ role: 'reporter', text: message.text, final: true });
-        break;
-
-      case 'response.started':
-        emit('speaking');
-        break;
-
-      case 'response.audio.delta':
-        if (message.audio) play(base64ToBytes(message.audio));
-        break;
-
-      case 'response.text.delta':
-        // Deltas, not whole lines: the caller concatenates until `response.done`
-        // if it is rendering captions.
-        //
-        // Filtered, because this field is not always text. sphoro.voice sends
-        // the caption and the audio from the same place, and against a real
-        // synthesiser it has been observed putting the µ-law bytes it is about
-        // to play into both — so an unguarded client renders a RIFF header into
-        // somebody's page.
-        if (isSpeech(message.text)) say({ role: 'agent', text: message.text, final: false });
-        break;
-
-      case 'response.done':
-        say({ role: 'agent', text: '', final: true });
-        emit('listening');
-        break;
-
-      case 'state.changed':
-        // The server's own view — it says 'listening' after a barge-in, which
-        // is the one transition this client cannot infer for itself. Going back
-        // to listening mid-answer *is* the barge-in, so the queued speech goes
-        // with it.
-        //
-        // 'interrupted' is the same event arriving by the other route: the
-        // server's own voice detection heard the reporter start talking and
-        // dropped what it was synthesizing. Only 'listening' — the answer to a
-        // barge-in this client asked for — used to be honoured here, so a
-        // reporter who simply talked over the agent kept hearing it: most of a
-        // reply is already handed to the audio clock by the time anybody
-        // interrupts, and dropping the queue is the only thing that stops it.
-        if (message.state === 'listening' || message.state === 'interrupted') stopPlayback();
-        if (message.state) emit(message.state);
-        break;
-
-      case 'session.ended':
-        finish('ended', { reason: message.data?.reason || null });
-        break;
-
-      case 'error':
-        finish('failed', { reason: message.error?.detail || 'the call failed' });
-        break;
-
-      default:
-        break;
-    }
-  });
-
-  socket.addEventListener('close', () => finish('ended'));
-  socket.addEventListener('error', () => finish('failed', { reason: 'transport' }));
-
-  let over = false;
-  function finish(state, detail) {
-    if (over) return;
-    over = true;
-    sending = false;
-    stopPlayback();
-    stopEverything(stream, context, capture, socket);
-    emit(state, detail);
-  }
-
-  return {
-    /** End the call. Safe to call twice, and after it has already ended. */
-    hangUp() {
-      if (!over && socket.readyState === WebSocket.OPEN) {
-        // Told, not just dropped: a socket that simply closes leaves
-        // sphoro.voice waiting out its own timeout before the call is marked
-        // finished, and the transcript does not reach the bug until it does.
-        try { socket.send(JSON.stringify({ type: 'session.end' })); } catch { /* closing */ }
-      }
-      finish('ended', { reason: 'hung_up' });
-    },
-  };
-}
-
-// --- plumbing ----------------------------------------------------------------
-
-/** Open the socket, resolving when it is actually open. */
-function openSocket(url) {
-  return new Promise((resolve, reject) => {
-    let socket;
-    try {
-      socket = new WebSocket(url, SUBPROTOCOL);
-    } catch (err) {
-      reject(err);
-      return;
-    }
-    socket.binaryType = 'arraybuffer';
-    const onOpen = () => { cleanup(); resolve(socket); };
-    const onFail = () => { cleanup(); reject(new Error('the call could not be connected')); };
-    const cleanup = () => {
-      socket.removeEventListener('open', onOpen);
-      socket.removeEventListener('error', onFail);
-      socket.removeEventListener('close', onFail);
-    };
-    socket.addEventListener('open', onOpen);
-    socket.addEventListener('error', onFail);
-    socket.addEventListener('close', onFail);
-  });
-}
-
-/**
- * Let go of everything.
- *
- * The microphone track especially: a page that closes its socket and leaves the
- * track live shows a recording indicator in the tab for as long as it stays
- * open, which reads — correctly — as being listened to after hanging up.
- */
-function stopEverything(stream, context, capture, socket) {
-  try { capture?.disconnect(); } catch { /* already gone */ }
-  if (capture) capture.onaudioprocess = null;
-  for (const track of stream?.getTracks() ?? []) {
-    try { track.stop(); } catch { /* already stopped */ }
-  }
-  try { context?.close(); } catch { /* already closed */ }
-  try { socket?.close(); } catch { /* already closing */ }
-}
-
-/**
- * Whether this is something somebody said, or bytes wearing a string's type.
- *
- * The caption fields on this protocol are typed as text and are not reliably
- * text: the same call site sends the audio and the caption, and against a real
- * synthesiser both have carried µ-law. A NUL settles it — no transcription
- * contains one — and a line that is mostly control characters is not speech
- * however it is typed.
- */
-function isSpeech(text) {
-  if (typeof text !== 'string' || !text) return false;
-  if (text.includes('\u0000')) return false;
-  // eslint-disable-next-line no-control-regex
-  const control = (text.match(/[\u0001-\u0008\u000e-\u001f]/g) || []).length;
-  return control <= text.length / 10;
-}
-
-/** Base64 to bytes, without pulling in anything to do it. */
-function base64ToBytes(value) {
-  const binary = window.atob(value);
-  const bytes = new Uint8Array(binary.length);
-  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
-  return bytes;
-}