@absolutejs/voice 0.0.22-beta.321 → 0.0.22-beta.323

package/README.md

@@ -4913,6 +4913,13 @@ Shared stream options:
 
 ```ts
 const browserMedia = {
+	continuity: {
+		maxGapMs: 7000,
+		maxInboundPacketStallMs: 7000,
+		maxOutboundPacketStallMs: 7000,
+		requireInboundAudio: true,
+		requireOutboundAudio: true
+	},
 	getPeerConnection: () => peerConnection,
 	maxJitterMs: 30,
 	maxPacketLossRatio: 0.02,

package/dist/angular/index.js

@@ -1412,7 +1412,65 @@ var stringStat = (stat, key) => {
   const value = stat[key];
   return typeof value === "string" ? value : undefined;
 };
+var statKey = (stat) => String(stat.id ?? stringStat(stat, "ssrc") ?? numericStat(stat, "ssrc") ?? stringStat(stat, "trackIdentifier") ?? stringStat(stat, "mid") ?? "unknown");
 var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
+var DEFAULT_TELEPHONY_FORMAT = {
+  channels: 1,
+  container: "raw",
+  encoding: "mulaw",
+  sampleRateHz: 8000
+};
+var bytesToBase64 = (audio) => {
+  const bytes = audio instanceof ArrayBuffer ? new Uint8Array(audio) : new Uint8Array(audio.buffer, audio.byteOffset, audio.byteLength);
+  return Buffer.from(bytes).toString("base64");
+};
+var base64ToBytes = (value) => new Uint8Array(Buffer.from(value, "base64"));
+var unknownRecord = (value) => value && typeof value === "object" ? value : {};
+var firstString = (records, keys) => {
+  for (const record of records) {
+    for (const key of keys) {
+      const value = record[key];
+      if (typeof value === "string" && value.length > 0) {
+        return value;
+      }
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return String(value);
+      }
+    }
+  }
+  return;
+};
+var firstNumber = (records, keys) => {
+  for (const record of records) {
+    for (const key of keys) {
+      const value = record[key];
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+      if (typeof value === "string") {
+        const parsed = Number(value);
+        if (Number.isFinite(parsed)) {
+          return parsed;
+        }
+      }
+    }
+  }
+  return;
+};
+var telephonyDirection = (track) => {
+  const normalized = track?.toLowerCase();
+  if (!normalized) {
+    return "unknown";
+  }
+  if (normalized.includes("inbound") || normalized.includes("caller") || normalized.includes("in")) {
+    return "inbound";
+  }
+  if (normalized.includes("outbound") || normalized.includes("assistant") || normalized.includes("out")) {
+    return "outbound";
+  }
+  return "unknown";
+};
+var telephonyFrameKind = (direction) => direction === "outbound" ? "assistant-audio" : "input-audio";
 var normalizeWebRTCStat = (stat) => {
   const sample = {};
   for (const [key, value] of Object.entries(stat)) {
@@ -1422,6 +1480,113 @@ var normalizeWebRTCStat = (stat) => {
   }
   return sample;
 };
+var parseTelephonyMediaFrame = (input) => {
+  const envelope = input.envelope;
+  const media = unknownRecord(envelope.media);
+  const payload = firstString([media, envelope], ["payload", "audio", "data"]) ?? firstString([unknownRecord(envelope.message)], ["payload"]);
+  if (!payload) {
+    return;
+  }
+  const carrier = input.carrier ?? firstString([envelope], ["provider"]) ?? "telephony";
+  const streamId = firstString([media, envelope], ["streamSid", "stream_id", "streamId", "streamId", "callSid", "call_id"]);
+  const sequenceNumber = firstString([media, envelope], ["sequenceNumber", "sequence_number", "chunk"]);
+  const track = firstString([media, envelope], ["track", "direction"]);
+  const direction = telephonyDirection(track);
+  const timestamp = firstNumber([media, envelope], ["timestamp", "time", "startedAt"]);
+  return {
+    at: timestamp,
+    audio: base64ToBytes(payload),
+    format: input.format ?? DEFAULT_TELEPHONY_FORMAT,
+    id: [
+      carrier,
+      streamId ?? input.sessionId ?? "stream",
+      sequenceNumber ?? timestamp ?? Date.now()
+    ].join(":"),
+    kind: telephonyFrameKind(direction),
+    metadata: {
+      carrier,
+      direction,
+      event: firstString([envelope], ["event", "type"]),
+      sequenceNumber,
+      streamId,
+      track
+    },
+    sessionId: input.sessionId ?? streamId,
+    source: "telephony"
+  };
+};
+var serializeTelephonyMediaFrame = (input) => {
+  const carrier = input.carrier ?? input.frame.metadata?.carrier ?? "telephony";
+  const streamId = input.streamId ?? (typeof input.frame.metadata?.streamId === "string" ? input.frame.metadata.streamId : input.frame.sessionId);
+  const sequenceNumber = input.sequenceNumber ?? (typeof input.frame.metadata?.sequenceNumber === "string" || typeof input.frame.metadata?.sequenceNumber === "number" ? input.frame.metadata.sequenceNumber : undefined);
+  const direction = input.frame.kind === "assistant-audio" ? "outbound" : "inbound";
+  const payload = input.frame.audio ? bytesToBase64(input.frame.audio) : "";
+  if (carrier === "twilio") {
+    return {
+      event: "media",
+      sequenceNumber,
+      streamSid: streamId,
+      media: {
+        payload,
+        timestamp: input.frame.at,
+        track: direction
+      }
+    };
+  }
+  if (carrier === "telnyx") {
+    return {
+      event: "media",
+      stream_id: streamId,
+      sequence_number: sequenceNumber,
+      media: {
+        payload,
+        timestamp: input.frame.at,
+        track: direction
+      }
+    };
+  }
+  if (carrier === "plivo") {
+    return {
+      event: "media",
+      streamId,
+      sequenceNumber,
+      media: {
+        payload,
+        timestamp: input.frame.at,
+        track: direction
+      }
+    };
+  }
+  return {
+    event: "media",
+    provider: carrier,
+    sequenceNumber,
+    streamId,
+    media: {
+      payload,
+      timestamp: input.frame.at,
+      track: direction
+    }
+  };
+};
+var createTelephonyMediaSerializer = (input) => {
+  const format = input.format ?? DEFAULT_TELEPHONY_FORMAT;
+  return {
+    carrier: input.carrier,
+    format,
+    parse: (envelope) => parseTelephonyMediaFrame({
+      carrier: input.carrier,
+      envelope,
+      format,
+      sessionId: input.sessionId ?? input.streamId
+    }),
+    serialize: (frame) => serializeTelephonyMediaFrame({
+      carrier: input.carrier,
+      frame,
+      streamId: input.streamId
+    })
+  };
+};
 var buildMediaResamplingPlan = (input) => {
   const required = !formatMatches(input.inputFormat, input.outputFormat);
   return {
@@ -1658,12 +1823,64 @@ var collectMediaWebRTCStats = async (input) => {
   const report = await input.peerConnection.getStats(input.selector ?? null);
   return [...report.values()].map(normalizeWebRTCStat);
 };
-var collectMediaWebRTCStatsReport = async (input) => {
-  const stats = await collectMediaWebRTCStats(input);
-  return buildMediaWebRTCStatsReport({
-    ...input,
-    stats
+var buildMediaWebRTCStreamContinuityReport = (input = {}) => {
+  const stats = input.stats ?? [];
+  const previousStats = input.previousStats ?? [];
+  const issues = [];
+  const previousByKey = new Map(previousStats.map((stat) => [statKey(stat), stat]));
+  const audioRtp = stats.filter((stat) => (stat.type === "inbound-rtp" || stat.type === "outbound-rtp") && stringStat(stat, "kind") !== "video" && stringStat(stat, "mediaType") !== "video");
+  const streams = audioRtp.map((stat) => {
+    const direction = stat.type === "outbound-rtp" ? "outbound" : "inbound";
+    const packetsKey = direction === "outbound" ? "packetsSent" : "packetsReceived";
+    const bytesKey = direction === "outbound" ? "bytesSent" : "bytesReceived";
+    const previous = previousByKey.get(statKey(stat));
+    const currentPackets = numericStat(stat, packetsKey);
+    const previousPackets = previous ? numericStat(previous, packetsKey) : undefined;
+    const currentBytes = numericStat(stat, bytesKey);
+    const previousBytes = previous ? numericStat(previous, bytesKey) : undefined;
+    const timeDeltaMs = stat.timestamp !== undefined && previous?.timestamp !== undefined ? stat.timestamp - previous.timestamp : undefined;
+    return {
+      bytesDelta: currentBytes !== undefined && previousBytes !== undefined ? currentBytes - previousBytes : undefined,
+      currentPackets,
+      direction,
+      id: statKey(stat),
+      packetDelta: currentPackets !== undefined && previousPackets !== undefined ? currentPackets - previousPackets : undefined,
+      previousPackets,
+      timeDeltaMs
+    };
   });
+  const inbound = streams.filter((stream) => stream.direction === "inbound");
+  const outbound = streams.filter((stream) => stream.direction === "outbound");
+  const maxObservedGapMs = max(streams.map((stream) => stream.timeDeltaMs).filter((value) => value !== undefined));
+  const stalledInboundStreams = inbound.filter((stream) => input.maxInboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxInboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
+  const stalledOutboundStreams = outbound.filter((stream) => input.maxOutboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxOutboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
+  if (input.requireInboundAudio && inbound.length === 0) {
+    pushIssue(issues, "error", "media.webrtc_inbound_audio_missing", "No inbound WebRTC audio RTP stream was observed.");
+  }
+  if (input.requireOutboundAudio && outbound.length === 0) {
+    pushIssue(issues, "error", "media.webrtc_outbound_audio_missing", "No outbound WebRTC audio RTP stream was observed.");
+  }
+  if (input.maxGapMs !== undefined && maxObservedGapMs !== undefined && maxObservedGapMs > input.maxGapMs) {
+    pushIssue(issues, "warning", "media.webrtc_stream_gap", `Observed WebRTC stream sample gap ${String(maxObservedGapMs)}ms above ${String(input.maxGapMs)}ms.`);
+  }
+  if (stalledInboundStreams > 0) {
+    pushIssue(issues, "error", "media.webrtc_inbound_stalled", `${String(stalledInboundStreams)} inbound WebRTC audio stream(s) stopped receiving packets.`);
+  }
+  if (stalledOutboundStreams > 0) {
+    pushIssue(issues, "error", "media.webrtc_outbound_stalled", `${String(stalledOutboundStreams)} outbound WebRTC audio stream(s) stopped sending packets.`);
+  }
+  return {
+    checkedAt: Date.now(),
+    inboundAudioStreams: inbound.length,
+    issues,
+    maxObservedGapMs,
+    outboundAudioStreams: outbound.length,
+    stalledInboundStreams,
+    stalledOutboundStreams,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
+    streams,
+    totalStats: stats.length
+  };
 };
 var buildMediaPipelineCalibrationReport = (input = {}) => {
   const frames = input.frames ?? [];
@@ -1749,21 +1966,30 @@ var postBrowserMediaReport = async (payload, options) => {
 };
 var createVoiceBrowserMediaReporter = (options) => {
   let interval = null;
+  let previousStats = [];
   const reportOnce = async () => {
     const peerConnection = await resolvePeerConnection(options);
     if (!peerConnection) {
       return;
     }
-    const report = await collectMediaWebRTCStatsReport({
+    const stats = await collectMediaWebRTCStats({ peerConnection });
+    const report = buildMediaWebRTCStatsReport({
       ...options,
-      peerConnection
+      stats
+    });
+    const continuity = options.continuity === false ? undefined : buildMediaWebRTCStreamContinuityReport({
+      ...options.continuity,
+      previousStats,
+      stats
     });
     const payload = {
       at: Date.now(),
+      continuity,
       report,
       scenarioId: options.getScenarioId?.() ?? null,
       sessionId: options.getSessionId?.() ?? null
     };
+    previousStats = stats;
     options.onReport?.(payload);
     await postBrowserMediaReport(payload, options);
     return payload;

package/dist/browserMediaRoutes.d.ts

@@ -1,9 +1,10 @@
 import { Elysia } from 'elysia';
-import type { MediaWebRTCStatsReport } from '@absolutejs/media';
+import type { MediaWebRTCStatsReport, MediaWebRTCStreamContinuityReport } from '@absolutejs/media';
 import type { VoiceTraceEventStore } from './trace';
 export type VoiceBrowserMediaStatus = 'empty' | 'fail' | 'pass' | 'warn';
 export type VoiceBrowserMediaSample = {
     at: number;
+    continuity?: MediaWebRTCStreamContinuityReport;
     report: MediaWebRTCStatsReport;
     scenarioId?: string;
     sessionId: string;

package/dist/client/htmxBootstrap.js

@@ -260,6 +260,7 @@ var stringStat = (stat, key) => {
   const value = stat[key];
   return typeof value === "string" ? value : undefined;
 };
+var statKey = (stat) => String(stat.id ?? stringStat(stat, "ssrc") ?? numericStat(stat, "ssrc") ?? stringStat(stat, "trackIdentifier") ?? stringStat(stat, "mid") ?? "unknown");
 var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
 var normalizeWebRTCStat = (stat) => {
   const sample = {};
@@ -334,12 +335,64 @@ var collectMediaWebRTCStats = async (input) => {
   const report = await input.peerConnection.getStats(input.selector ?? null);
   return [...report.values()].map(normalizeWebRTCStat);
 };
-var collectMediaWebRTCStatsReport = async (input) => {
-  const stats = await collectMediaWebRTCStats(input);
-  return buildMediaWebRTCStatsReport({
-    ...input,
-    stats
+var buildMediaWebRTCStreamContinuityReport = (input = {}) => {
+  const stats = input.stats ?? [];
+  const previousStats = input.previousStats ?? [];
+  const issues = [];
+  const previousByKey = new Map(previousStats.map((stat) => [statKey(stat), stat]));
+  const audioRtp = stats.filter((stat) => (stat.type === "inbound-rtp" || stat.type === "outbound-rtp") && stringStat(stat, "kind") !== "video" && stringStat(stat, "mediaType") !== "video");
+  const streams = audioRtp.map((stat) => {
+    const direction = stat.type === "outbound-rtp" ? "outbound" : "inbound";
+    const packetsKey = direction === "outbound" ? "packetsSent" : "packetsReceived";
+    const bytesKey = direction === "outbound" ? "bytesSent" : "bytesReceived";
+    const previous = previousByKey.get(statKey(stat));
+    const currentPackets = numericStat(stat, packetsKey);
+    const previousPackets = previous ? numericStat(previous, packetsKey) : undefined;
+    const currentBytes = numericStat(stat, bytesKey);
+    const previousBytes = previous ? numericStat(previous, bytesKey) : undefined;
+    const timeDeltaMs = stat.timestamp !== undefined && previous?.timestamp !== undefined ? stat.timestamp - previous.timestamp : undefined;
+    return {
+      bytesDelta: currentBytes !== undefined && previousBytes !== undefined ? currentBytes - previousBytes : undefined,
+      currentPackets,
+      direction,
+      id: statKey(stat),
+      packetDelta: currentPackets !== undefined && previousPackets !== undefined ? currentPackets - previousPackets : undefined,
+      previousPackets,
+      timeDeltaMs
+    };
   });
+  const inbound = streams.filter((stream) => stream.direction === "inbound");
+  const outbound = streams.filter((stream) => stream.direction === "outbound");
+  const maxObservedGapMs = max(streams.map((stream) => stream.timeDeltaMs).filter((value) => value !== undefined));
+  const stalledInboundStreams = inbound.filter((stream) => input.maxInboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxInboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
+  const stalledOutboundStreams = outbound.filter((stream) => input.maxOutboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxOutboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
+  if (input.requireInboundAudio && inbound.length === 0) {
+    pushIssue(issues, "error", "media.webrtc_inbound_audio_missing", "No inbound WebRTC audio RTP stream was observed.");
+  }
+  if (input.requireOutboundAudio && outbound.length === 0) {
+    pushIssue(issues, "error", "media.webrtc_outbound_audio_missing", "No outbound WebRTC audio RTP stream was observed.");
+  }
+  if (input.maxGapMs !== undefined && maxObservedGapMs !== undefined && maxObservedGapMs > input.maxGapMs) {
+    pushIssue(issues, "warning", "media.webrtc_stream_gap", `Observed WebRTC stream sample gap ${String(maxObservedGapMs)}ms above ${String(input.maxGapMs)}ms.`);
+  }
+  if (stalledInboundStreams > 0) {
+    pushIssue(issues, "error", "media.webrtc_inbound_stalled", `${String(stalledInboundStreams)} inbound WebRTC audio stream(s) stopped receiving packets.`);
+  }
+  if (stalledOutboundStreams > 0) {
+    pushIssue(issues, "error", "media.webrtc_outbound_stalled", `${String(stalledOutboundStreams)} outbound WebRTC audio stream(s) stopped sending packets.`);
+  }
+  return {
+    checkedAt: Date.now(),
+    inboundAudioStreams: inbound.length,
+    issues,
+    maxObservedGapMs,
+    outboundAudioStreams: outbound.length,
+    stalledInboundStreams,
+    stalledOutboundStreams,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
+    streams,
+    totalStats: stats.length
+  };
 };
 
 // src/client/browserMedia.ts
@@ -362,21 +415,30 @@ var postBrowserMediaReport = async (payload, options) => {
 };
 var createVoiceBrowserMediaReporter = (options) => {
   let interval = null;
+  let previousStats = [];
   const reportOnce = async () => {
     const peerConnection = await resolvePeerConnection(options);
     if (!peerConnection) {
       return;
     }
-    const report = await collectMediaWebRTCStatsReport({
+    const stats = await collectMediaWebRTCStats({ peerConnection });
+    const report = buildMediaWebRTCStatsReport({
       ...options,
-      peerConnection
+      stats
+    });
+    const continuity = options.continuity === false ? undefined : buildMediaWebRTCStreamContinuityReport({
+      ...options.continuity,
+      previousStats,
+      stats
     });
     const payload = {
       at: Date.now(),
+      continuity,
       report,
       scenarioId: options.getScenarioId?.() ?? null,
       sessionId: options.getSessionId?.() ?? null
     };
+    previousStats = stats;
     options.onReport?.(payload);
     await postBrowserMediaReport(payload, options);
     return payload;