@absolutejs/voice 0.0.22-beta.319 → 0.0.22-beta.320

package/dist/react/index.js

@@ -5023,6 +5023,412 @@ var serverMessageToAction = (message) => {
   }
 };
 
+// node_modules/@absolutejs/media/dist/index.js
+var formatLabel = (format) => `${format.container}/${format.encoding}/${String(format.sampleRateHz)}hz/${String(format.channels)}ch`;
+var formatMatches = (actual, expected) => actual.container === expected.container && actual.encoding === expected.encoding && actual.sampleRateHz === expected.sampleRateHz && actual.channels === expected.channels;
+var pushIssue = (issues, severity, code, message) => {
+  issues.push({ code, message, severity });
+};
+var numericMetadata = (frame, key) => {
+  const value = frame.metadata?.[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+};
+var average = (values) => values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0) / values.length;
+var max = (values) => values.length === 0 ? undefined : Math.max(...values);
+var min = (values) => values.length === 0 ? undefined : Math.min(...values);
+var numericStat = (stat, key) => {
+  const value = stat[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+};
+var booleanStat = (stat, key) => {
+  const value = stat[key];
+  return typeof value === "boolean" ? value : undefined;
+};
+var stringStat = (stat, key) => {
+  const value = stat[key];
+  return typeof value === "string" ? value : undefined;
+};
+var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
+var normalizeWebRTCStat = (stat) => {
+  const sample = {};
+  for (const [key, value] of Object.entries(stat)) {
+    if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") {
+      sample[key] = value;
+    }
+  }
+  return sample;
+};
+var buildMediaResamplingPlan = (input) => {
+  const required = !formatMatches(input.inputFormat, input.outputFormat);
+  return {
+    inputFormat: input.inputFormat,
+    outputFormat: input.outputFormat,
+    ratio: input.outputFormat.sampleRateHz / input.inputFormat.sampleRateHz,
+    required,
+    status: input.inputFormat.container === input.outputFormat.container && input.inputFormat.encoding === input.outputFormat.encoding && input.inputFormat.channels === input.outputFormat.channels ? "pass" : "warn"
+  };
+};
+var speechProbability = (frame) => {
+  if (frame.metadata?.isSpeech === true) {
+    return 1;
+  }
+  if (frame.metadata?.isSpeech === false) {
+    return 0;
+  }
+  for (const key of ["speechProbability", "voiceProbability", "rms", "energy"]) {
+    const value = numericMetadata(frame, key);
+    if (value !== undefined) {
+      return value;
+    }
+  }
+  return 0;
+};
+var buildMediaVadReport = (input = {}) => {
+  const frames = (input.frames ?? []).filter((frame) => frame.kind === "input-audio");
+  const speechStartThreshold = input.speechStartThreshold ?? 0.6;
+  const speechEndThreshold = input.speechEndThreshold ?? 0.35;
+  const minSpeechFrames = input.minSpeechFrames ?? 1;
+  const maxSilenceFrames = input.maxSilenceFrames ?? 1;
+  const segments = [];
+  let activeFrames = [];
+  let silenceFrames = 0;
+  const closeSegment = () => {
+    if (activeFrames.length < minSpeechFrames) {
+      activeFrames = [];
+      silenceFrames = 0;
+      return;
+    }
+    const first = activeFrames[0];
+    const last = activeFrames.at(-1);
+    if (!first) {
+      return;
+    }
+    segments.push({
+      durationMs: first.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined,
+      endAt: last?.at !== undefined ? last.at + (last.durationMs ?? 0) : undefined,
+      frameCount: activeFrames.length,
+      segmentId: `vad:${String(segments.length + 1)}`,
+      sessionId: first.sessionId,
+      startAt: first.at,
+      turnId: first.turnId
+    });
+    activeFrames = [];
+    silenceFrames = 0;
+  };
+  for (const frame of frames) {
+    const probability = speechProbability(frame);
+    if (activeFrames.length === 0) {
+      if (probability >= speechStartThreshold) {
+        activeFrames.push(frame);
+      }
+      continue;
+    }
+    activeFrames.push(frame);
+    if (probability <= speechEndThreshold) {
+      silenceFrames += 1;
+    } else {
+      silenceFrames = 0;
+    }
+    if (silenceFrames > maxSilenceFrames) {
+      closeSegment();
+    }
+  }
+  closeSegment();
+  return {
+    checkedAt: Date.now(),
+    inputAudioFrames: frames.length,
+    segments,
+    status: frames.length === 0 ? "warn" : "pass"
+  };
+};
+var buildMediaInterruptionReport = (input = {}) => {
+  const issues = [];
+  const interruptionFrames = (input.frames ?? []).filter((frame) => frame.kind === "interruption");
+  const latenciesMs = interruptionFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
+  const maxInterruptionLatencyMs = input.maxInterruptionLatencyMs;
+  if (interruptionFrames.length === 0) {
+    pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
+  }
+  if (maxInterruptionLatencyMs !== undefined && latenciesMs.some((latency) => latency > maxInterruptionLatencyMs)) {
+    pushIssue(issues, "error", "media.interruption_latency", `Interruption latency exceeded ${String(maxInterruptionLatencyMs)}ms.`);
+  }
+  return {
+    checkedAt: Date.now(),
+    interruptionFrames: interruptionFrames.length,
+    issues,
+    latenciesMs,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass"
+  };
+};
+var buildMediaQualityReport = (input = {}) => {
+  const frames = [...input.frames ?? []].sort((a, b) => (a.at ?? 0) - (b.at ?? 0));
+  const audioFrames = frames.filter((frame) => frame.kind === "input-audio" || frame.kind === "assistant-audio");
+  const inputAudioFrames = frames.filter((frame) => frame.kind === "input-audio");
+  const assistantAudioFrames = frames.filter((frame) => frame.kind === "assistant-audio");
+  const issues = [];
+  const gapsMs = [];
+  for (const [index, frame] of audioFrames.entries()) {
+    const previous = audioFrames[index - 1];
+    if (previous?.at === undefined || frame.at === undefined || previous.durationMs === undefined) {
+      continue;
+    }
+    const gap = frame.at - (previous.at + previous.durationMs);
+    if (gap > 0) {
+      gapsMs.push(gap);
+    }
+  }
+  const jitterMs = audioFrames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined).at(-1) ?? max(gapsMs);
+  const first = audioFrames.find((frame) => frame.at !== undefined);
+  const last = audioFrames.toReversed().find((frame) => frame.at !== undefined);
+  const durationMs = first?.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined;
+  const expectedDurationMs = audioFrames.length > 0 ? audioFrames.reduce((total, frame) => total + (frame.durationMs ?? 0), 0) : undefined;
+  const timestampDriftMs = durationMs !== undefined && expectedDurationMs !== undefined ? Math.max(0, durationMs - expectedDurationMs) : undefined;
+  const speechScores = inputAudioFrames.map(speechProbability);
+  const speechFrames = speechScores.filter((score) => score >= 0.6).length;
+  const silenceFrames = speechScores.filter((score) => score <= 0.35).length;
+  const unknownSpeechFrames = Math.max(0, inputAudioFrames.length - speechFrames - silenceFrames);
+  const speechRatio = inputAudioFrames.length === 0 ? 0 : speechFrames / inputAudioFrames.length;
+  const silenceRatio = inputAudioFrames.length === 0 ? 0 : silenceFrames / inputAudioFrames.length;
+  const levels = audioFrames.map((frame) => numericMetadata(frame, "level") ?? numericMetadata(frame, "rms") ?? numericMetadata(frame, "energy")).filter((value) => value !== undefined);
+  const backpressureEvents = input.transport?.backpressureEvents ?? 0;
+  const maxGapMs = input.maxGapMs;
+  if (maxGapMs !== undefined && gapsMs.some((gap) => gap > maxGapMs)) {
+    pushIssue(issues, "warning", "media.quality_gap", `Observed media gap above ${String(maxGapMs)}ms.`);
+  }
+  if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
+    pushIssue(issues, "warning", "media.quality_jitter", `Observed jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
+  }
+  if (input.maxTimestampDriftMs !== undefined && timestampDriftMs !== undefined && timestampDriftMs > input.maxTimestampDriftMs) {
+    pushIssue(issues, "warning", "media.quality_timestamp_drift", `Observed timestamp drift ${String(timestampDriftMs)}ms above ${String(input.maxTimestampDriftMs)}ms.`);
+  }
+  if (input.minSpeechRatio !== undefined && inputAudioFrames.length > 0 && speechRatio < input.minSpeechRatio) {
+    pushIssue(issues, "warning", "media.quality_speech_ratio", `Observed speech ratio ${String(speechRatio)} below ${String(input.minSpeechRatio)}.`);
+  }
+  if (input.maxBackpressureEvents !== undefined && backpressureEvents > input.maxBackpressureEvents) {
+    pushIssue(issues, "warning", "media.quality_backpressure", `Observed ${String(backpressureEvents)} backpressure event(s), above ${String(input.maxBackpressureEvents)}.`);
+  }
+  return {
+    assistantAudioFrames: assistantAudioFrames.length,
+    backpressureEvents,
+    checkedAt: Date.now(),
+    durationMs,
+    gapCount: gapsMs.length,
+    gapsMs,
+    inputAudioFrames: inputAudioFrames.length,
+    issues,
+    jitterMs,
+    levelAverage: average(levels),
+    levelMax: max(levels),
+    levelMin: min(levels),
+    silenceFrames,
+    silenceRatio,
+    speechFrames,
+    speechRatio,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
+    timestampDriftMs,
+    totalFrames: frames.length,
+    unknownSpeechFrames
+  };
+};
+var buildMediaWebRTCStatsReport = (input = {}) => {
+  const stats = input.stats ?? [];
+  const issues = [];
+  const inbound = stats.filter((stat) => stat.type === "inbound-rtp" && stringStat(stat, "kind") !== "video");
+  const outbound = stats.filter((stat) => stat.type === "outbound-rtp" && stringStat(stat, "kind") !== "video");
+  const candidatePairs = stats.filter((stat) => stat.type === "candidate-pair");
+  const audioTracks = stats.filter((stat) => (stat.type === "track" || stat.type === "media-source") && stringStat(stat, "kind") === "audio");
+  const activeCandidatePairs = candidatePairs.filter((stat) => booleanStat(stat, "selected") === true || booleanStat(stat, "nominated") === true || stringStat(stat, "state") === "succeeded").length;
+  const liveAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") !== "ended" && stringStat(stat, "trackState") !== "ended" && booleanStat(stat, "ended") !== true).length;
+  const endedAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") === "ended" || stringStat(stat, "trackState") === "ended" || booleanStat(stat, "ended") === true).length;
+  const inboundPackets = inbound.reduce((total, stat) => total + (numericStat(stat, "packetsReceived") ?? 0), 0);
+  const outboundPackets = outbound.reduce((total, stat) => total + (numericStat(stat, "packetsSent") ?? 0), 0);
+  const packetsLost = [...inbound, ...outbound].reduce((total, stat) => total + Math.max(0, numericStat(stat, "packetsLost") ?? 0), 0);
+  const packetLossDenominator = inboundPackets + packetsLost;
+  const packetLossRatio = packetLossDenominator === 0 ? 0 : packetsLost / packetLossDenominator;
+  const bytesReceived = inbound.reduce((total, stat) => total + (numericStat(stat, "bytesReceived") ?? 0), 0);
+  const bytesSent = outbound.reduce((total, stat) => total + (numericStat(stat, "bytesSent") ?? 0), 0);
+  const roundTripTimeMs = max(candidatePairs.map((stat) => secondsToMs(numericStat(stat, "currentRoundTripTime") ?? numericStat(stat, "roundTripTime"))).filter((value) => value !== undefined));
+  const jitterMs = max([...inbound, ...outbound].map((stat) => secondsToMs(numericStat(stat, "jitter"))).filter((value) => value !== undefined));
+  const jitterBufferDelayMs = max(inbound.map((stat) => {
+    const delay = numericStat(stat, "jitterBufferDelay");
+    const emitted = numericStat(stat, "jitterBufferEmittedCount");
+    return delay !== undefined && emitted !== undefined && emitted > 0 ? delay / emitted * 1000 : undefined;
+  }).filter((value) => value !== undefined));
+  const audioLevels = audioTracks.map((stat) => numericStat(stat, "audioLevel")).filter((value) => value !== undefined);
+  if (input.requireConnectedCandidatePair && candidatePairs.length > 0 && activeCandidatePairs === 0) {
+    pushIssue(issues, "error", "media.webrtc_candidate_pair_missing", "No active WebRTC candidate pair was observed.");
+  }
+  if (input.requireLiveAudioTrack && liveAudioTracks === 0) {
+    pushIssue(issues, "error", "media.webrtc_audio_track_missing", "No live WebRTC audio track was observed.");
+  }
+  if (input.maxPacketLossRatio !== undefined && packetLossRatio > input.maxPacketLossRatio) {
+    pushIssue(issues, "warning", "media.webrtc_packet_loss", `Observed WebRTC packet loss ratio ${String(packetLossRatio)} above ${String(input.maxPacketLossRatio)}.`);
+  }
+  if (input.maxRoundTripTimeMs !== undefined && roundTripTimeMs !== undefined && roundTripTimeMs > input.maxRoundTripTimeMs) {
+    pushIssue(issues, "warning", "media.webrtc_round_trip_time", `Observed WebRTC RTT ${String(roundTripTimeMs)}ms above ${String(input.maxRoundTripTimeMs)}ms.`);
+  }
+  if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
+    pushIssue(issues, "warning", "media.webrtc_jitter", `Observed WebRTC jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
+  }
+  return {
+    activeCandidatePairs,
+    audioLevelAverage: average(audioLevels),
+    bytesReceived,
+    bytesSent,
+    checkedAt: Date.now(),
+    endedAudioTracks,
+    inboundPackets,
+    issues,
+    jitterBufferDelayMs,
+    jitterMs,
+    liveAudioTracks,
+    outboundPackets,
+    packetLossRatio,
+    packetsLost,
+    roundTripTimeMs,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
+    totalStats: stats.length
+  };
+};
+var collectMediaWebRTCStats = async (input) => {
+  const report = await input.peerConnection.getStats(input.selector ?? null);
+  return [...report.values()].map(normalizeWebRTCStat);
+};
+var collectMediaWebRTCStatsReport = async (input) => {
+  const stats = await collectMediaWebRTCStats(input);
+  return buildMediaWebRTCStatsReport({
+    ...input,
+    stats
+  });
+};
+var buildMediaPipelineCalibrationReport = (input = {}) => {
+  const frames = input.frames ?? [];
+  const issues = [];
+  const inputFrames = frames.filter((frame) => frame.kind === "input-audio");
+  const assistantFrames = frames.filter((frame) => frame.kind === "assistant-audio");
+  const turnCommitFrames = frames.filter((frame) => frame.kind === "turn-commit");
+  const interruptionFrameRecords = frames.filter((frame) => frame.kind === "interruption");
+  const traceLinkedFrames = frames.filter((frame) => frame.traceEventId).length;
+  const backpressureFrames = frames.filter((frame) => Boolean(frame.metadata?.backpressure)).length;
+  const audioLatencies = assistantFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
+  const firstAudioLatencyMs = audioLatencies.length > 0 ? Math.min(...audioLatencies) : undefined;
+  const jitterValues = frames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined);
+  const jitterMs = jitterValues.length > 0 ? Math.max(...jitterValues) : undefined;
+  const inputFormat = input.inputFormat ?? inputFrames.find((frame) => frame.format)?.format;
+  const outputFormat = input.outputFormat ?? assistantFrames.find((frame) => frame.format)?.format;
+  const resamplingRequired = Boolean(input.expectedInputFormat && inputFormat && inputFormat.sampleRateHz !== input.expectedInputFormat.sampleRateHz) || Boolean(input.expectedOutputFormat && outputFormat && outputFormat.sampleRateHz !== input.expectedOutputFormat.sampleRateHz);
+  const resamplingTargetHz = resamplingRequired && input.expectedInputFormat ? input.expectedInputFormat.sampleRateHz : resamplingRequired ? input.expectedOutputFormat?.sampleRateHz : undefined;
+  if (inputFrames.length === 0) {
+    pushIssue(issues, "warning", "media.input_audio_missing", "No input audio frames were observed.");
+  }
+  if (assistantFrames.length === 0) {
+    pushIssue(issues, "warning", "media.assistant_audio_missing", "No assistant audio frames were observed.");
+  }
+  if (input.expectedInputFormat && inputFormat && !formatMatches(inputFormat, input.expectedInputFormat)) {
+    pushIssue(issues, inputFormat.sampleRateHz === input.expectedInputFormat.sampleRateHz ? "warning" : "error", "media.input_format_mismatch", `Input format ${formatLabel(inputFormat)} does not match expected ${formatLabel(input.expectedInputFormat)}.`);
+  }
+  if (input.expectedOutputFormat && outputFormat && !formatMatches(outputFormat, input.expectedOutputFormat)) {
+    pushIssue(issues, outputFormat.sampleRateHz === input.expectedOutputFormat.sampleRateHz ? "warning" : "error", "media.output_format_mismatch", `Output format ${formatLabel(outputFormat)} does not match expected ${formatLabel(input.expectedOutputFormat)}.`);
+  }
+  if (firstAudioLatencyMs !== undefined && input.maxFirstAudioLatencyMs !== undefined && firstAudioLatencyMs > input.maxFirstAudioLatencyMs) {
+    pushIssue(issues, "error", "media.first_audio_latency", `First audio latency ${String(firstAudioLatencyMs)}ms exceeds budget ${String(input.maxFirstAudioLatencyMs)}ms.`);
+  }
+  if (jitterMs !== undefined && input.maxJitterMs !== undefined && jitterMs > input.maxJitterMs) {
+    pushIssue(issues, "warning", "media.jitter", `Media jitter ${String(jitterMs)}ms exceeds budget ${String(input.maxJitterMs)}ms.`);
+  }
+  if (input.maxBackpressureFrames !== undefined && backpressureFrames > input.maxBackpressureFrames) {
+    pushIssue(issues, "warning", "media.backpressure", `Backpressure frame count ${String(backpressureFrames)} exceeds budget ${String(input.maxBackpressureFrames)}.`);
+  }
+  if (input.requireInterruptionFrame && interruptionFrameRecords.length === 0) {
+    pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
+  }
+  if (input.requireTraceEvidence && traceLinkedFrames === 0) {
+    pushIssue(issues, "warning", "media.trace_evidence_missing", "No media frames were linked to trace evidence.");
+  }
+  return {
+    assistantAudioFrames: assistantFrames.length,
+    backpressureFrames,
+    checkedAt: Date.now(),
+    firstAudioLatencyMs,
+    inputAudioFrames: inputFrames.length,
+    inputFormat,
+    interruptionFrames: interruptionFrameRecords.length,
+    issues,
+    jitterMs,
+    outputFormat,
+    resamplingRequired,
+    resamplingTargetHz,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
+    surface: input.surface ?? "media-pipeline",
+    traceLinkedFrames,
+    turnCommitFrames: turnCommitFrames.length
+  };
+};
+
+// src/client/browserMedia.ts
+var DEFAULT_BROWSER_MEDIA_PATH = "/api/voice/browser-media";
+var DEFAULT_BROWSER_MEDIA_INTERVAL_MS = 5000;
+var resolvePeerConnection = async (options) => options.peerConnection ?? await options.getPeerConnection?.() ?? null;
+var postBrowserMediaReport = async (payload, options) => {
+  const requestFetch = options.fetch ?? globalThis.fetch;
+  if (!requestFetch) {
+    return;
+  }
+  await requestFetch(options.path ?? DEFAULT_BROWSER_MEDIA_PATH, {
+    body: JSON.stringify(payload),
+    headers: {
+      "Content-Type": "application/json"
+    },
+    keepalive: true,
+    method: "POST"
+  });
+};
+var createVoiceBrowserMediaReporter = (options) => {
+  let interval = null;
+  const reportOnce = async () => {
+    const peerConnection = await resolvePeerConnection(options);
+    if (!peerConnection) {
+      return;
+    }
+    const report = await collectMediaWebRTCStatsReport({
+      ...options,
+      peerConnection
+    });
+    const payload = {
+      at: Date.now(),
+      report,
+      scenarioId: options.getScenarioId?.() ?? null,
+      sessionId: options.getSessionId?.() ?? null
+    };
+    options.onReport?.(payload);
+    await postBrowserMediaReport(payload, options);
+    return payload;
+  };
+  const run = () => {
+    reportOnce().catch((error) => {
+      options.onError?.(error);
+    });
+  };
+  const stop = () => {
+    if (interval) {
+      clearInterval(interval);
+      interval = null;
+    }
+  };
+  return {
+    close: stop,
+    reportOnce,
+    start: () => {
+      if (interval) {
+        return;
+      }
+      run();
+      interval = setInterval(run, options.intervalMs ?? DEFAULT_BROWSER_MEDIA_INTERVAL_MS);
+    },
+    stop
+  };
+};
+
 // src/client/connection.ts
 var WS_OPEN = 1;
 var WS_CLOSED = 3;
@@ -5455,12 +5861,18 @@ var createVoiceStreamStore = () => {
 var createVoiceStream = (path, options = {}) => {
   const connection = createVoiceConnection(path, options);
   const store = createVoiceStreamStore();
+  const browserMediaReporter = options.browserMedia && typeof window !== "undefined" ? createVoiceBrowserMediaReporter({
+    ...options.browserMedia,
+    getScenarioId: () => options.browserMedia ? options.browserMedia.getScenarioId?.() ?? connection.getScenarioId() : connection.getScenarioId(),
+    getSessionId: () => options.browserMedia ? options.browserMedia.getSessionId?.() ?? connection.getSessionId() : connection.getSessionId()
+  }) : null;
   const subscribers = new Set;
   const start = (input) => Promise.resolve().then(() => {
     if (!input?.sessionId && !input?.scenarioId) {
       return;
     }
     connection.start(input);
+    browserMediaReporter?.start();
   });
   const notify = () => {
     subscribers.forEach((subscriber) => subscriber());
@@ -5502,6 +5914,7 @@ var createVoiceStream = (path, options = {}) => {
     },
     close() {
       unsubscribeConnection();
+      browserMediaReporter?.close();
       connection.close();
       store.dispatch({ type: "disconnected" });
       notify();