@absolutejs/voice 0.0.22-beta.464 → 0.0.22-beta.465
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/angular/index.js +5 -927
- package/dist/client/htmxBootstrap.js +7 -330
- package/dist/client/index.js +5 -927
- package/dist/generated/htmxBootstrapBundle.d.ts +1 -1
- package/dist/index.js +98 -1005
- package/dist/react/index.js +5 -927
- package/dist/svelte/index.js +5 -927
- package/dist/testing/index.js +51 -973
- package/dist/vue/index.js +5 -927
- package/package.json +3 -3
package/dist/angular/index.js
CHANGED
|
@@ -2008,934 +2008,12 @@ var serverMessageToAction = (message) => {
|
|
|
2008
2008
|
}
|
|
2009
2009
|
};
|
|
2010
2010
|
|
|
2011
|
-
// node_modules/@absolutejs/media/dist/index.js
|
|
2012
|
-
import { mkdir, writeFile } from "fs/promises";
|
|
2013
|
-
import { join } from "path";
|
|
2014
|
-
var formatLabel = (format) => `${format.container}/${format.encoding}/${String(format.sampleRateHz)}hz/${String(format.channels)}ch`;
|
|
2015
|
-
var formatMatches = (actual, expected) => actual.container === expected.container && actual.encoding === expected.encoding && actual.sampleRateHz === expected.sampleRateHz && actual.channels === expected.channels;
|
|
2016
|
-
var pushIssue = (issues, severity, code, message) => {
|
|
2017
|
-
issues.push({ code, message, severity });
|
|
2018
|
-
};
|
|
2019
|
-
var numericMetadata = (frame, key) => {
|
|
2020
|
-
const value = frame.metadata?.[key];
|
|
2021
|
-
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
2022
|
-
};
|
|
2023
|
-
var average = (values) => values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0) / values.length;
|
|
2024
|
-
var max = (values) => values.length === 0 ? undefined : Math.max(...values);
|
|
2025
|
-
var min = (values) => values.length === 0 ? undefined : Math.min(...values);
|
|
2026
|
-
var numericStat = (stat, key) => {
|
|
2027
|
-
const value = stat[key];
|
|
2028
|
-
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
2029
|
-
};
|
|
2030
|
-
var booleanStat = (stat, key) => {
|
|
2031
|
-
const value = stat[key];
|
|
2032
|
-
return typeof value === "boolean" ? value : undefined;
|
|
2033
|
-
};
|
|
2034
|
-
var stringStat = (stat, key) => {
|
|
2035
|
-
const value = stat[key];
|
|
2036
|
-
return typeof value === "string" ? value : undefined;
|
|
2037
|
-
};
|
|
2038
|
-
var statKey = (stat) => String(stat.id ?? stringStat(stat, "ssrc") ?? numericStat(stat, "ssrc") ?? stringStat(stat, "trackIdentifier") ?? stringStat(stat, "mid") ?? "unknown");
|
|
2039
|
-
var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
|
|
2040
|
-
var DEFAULT_TELEPHONY_FORMAT = {
|
|
2041
|
-
channels: 1,
|
|
2042
|
-
container: "raw",
|
|
2043
|
-
encoding: "mulaw",
|
|
2044
|
-
sampleRateHz: 8000
|
|
2045
|
-
};
|
|
2046
|
-
var bytesToBase64 = (audio) => {
|
|
2047
|
-
const bytes = audio instanceof ArrayBuffer ? new Uint8Array(audio) : new Uint8Array(audio.buffer, audio.byteOffset, audio.byteLength);
|
|
2048
|
-
return Buffer.from(bytes).toString("base64");
|
|
2049
|
-
};
|
|
2050
|
-
var base64ToBytes = (value) => new Uint8Array(Buffer.from(value, "base64"));
|
|
2051
|
-
var unknownRecord = (value) => value && typeof value === "object" ? value : {};
|
|
2052
|
-
var firstString = (records, keys) => {
|
|
2053
|
-
for (const record of records) {
|
|
2054
|
-
for (const key of keys) {
|
|
2055
|
-
const value = record[key];
|
|
2056
|
-
if (typeof value === "string" && value.length > 0) {
|
|
2057
|
-
return value;
|
|
2058
|
-
}
|
|
2059
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
2060
|
-
return String(value);
|
|
2061
|
-
}
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
return;
|
|
2065
|
-
};
|
|
2066
|
-
var firstNumber = (records, keys) => {
|
|
2067
|
-
for (const record of records) {
|
|
2068
|
-
for (const key of keys) {
|
|
2069
|
-
const value = record[key];
|
|
2070
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
2071
|
-
return value;
|
|
2072
|
-
}
|
|
2073
|
-
if (typeof value === "string") {
|
|
2074
|
-
const parsed = Number(value);
|
|
2075
|
-
if (Number.isFinite(parsed)) {
|
|
2076
|
-
return parsed;
|
|
2077
|
-
}
|
|
2078
|
-
}
|
|
2079
|
-
}
|
|
2080
|
-
}
|
|
2081
|
-
return;
|
|
2082
|
-
};
|
|
2083
|
-
var telephonyDirection = (track) => {
|
|
2084
|
-
const normalized = track?.toLowerCase();
|
|
2085
|
-
if (!normalized) {
|
|
2086
|
-
return "unknown";
|
|
2087
|
-
}
|
|
2088
|
-
if (normalized.includes("inbound") || normalized.includes("caller") || normalized.includes("in")) {
|
|
2089
|
-
return "inbound";
|
|
2090
|
-
}
|
|
2091
|
-
if (normalized.includes("outbound") || normalized.includes("assistant") || normalized.includes("out")) {
|
|
2092
|
-
return "outbound";
|
|
2093
|
-
}
|
|
2094
|
-
return "unknown";
|
|
2095
|
-
};
|
|
2096
|
-
var telephonyFrameKind = (direction) => direction === "outbound" ? "assistant-audio" : "input-audio";
|
|
2097
|
-
var telephonyEventKind = (envelope) => {
|
|
2098
|
-
const raw = firstString([envelope], ["event", "type", "eventType"]) ?? firstString([unknownRecord(envelope.message)], ["event", "type"]);
|
|
2099
|
-
const normalized = raw?.toLowerCase().replace(/[_\s-]+/g, "-");
|
|
2100
|
-
if (!normalized) {
|
|
2101
|
-
return "unknown";
|
|
2102
|
-
}
|
|
2103
|
-
if (normalized.includes("connected")) {
|
|
2104
|
-
return "connected";
|
|
2105
|
-
}
|
|
2106
|
-
if (normalized.includes("start")) {
|
|
2107
|
-
return "start";
|
|
2108
|
-
}
|
|
2109
|
-
if (normalized.includes("media")) {
|
|
2110
|
-
return "media";
|
|
2111
|
-
}
|
|
2112
|
-
if (normalized.includes("stop") || normalized.includes("closed")) {
|
|
2113
|
-
return "stop";
|
|
2114
|
-
}
|
|
2115
|
-
if (normalized.includes("error") || normalized.includes("failed")) {
|
|
2116
|
-
return "error";
|
|
2117
|
-
}
|
|
2118
|
-
return "unknown";
|
|
2119
|
-
};
|
|
2120
|
-
var normalizeWebRTCStat = (stat) => {
|
|
2121
|
-
const sample = {};
|
|
2122
|
-
for (const [key, value] of Object.entries(stat)) {
|
|
2123
|
-
if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") {
|
|
2124
|
-
sample[key] = value;
|
|
2125
|
-
}
|
|
2126
|
-
}
|
|
2127
|
-
return sample;
|
|
2128
|
-
};
|
|
2129
|
-
var parseTelephonyMediaFrame = (input) => {
|
|
2130
|
-
const envelope = input.envelope;
|
|
2131
|
-
const media = unknownRecord(envelope.media);
|
|
2132
|
-
const payload = firstString([media, envelope], ["payload", "audio", "data"]) ?? firstString([unknownRecord(envelope.message)], ["payload"]);
|
|
2133
|
-
if (!payload) {
|
|
2134
|
-
return;
|
|
2135
|
-
}
|
|
2136
|
-
const carrier = input.carrier ?? firstString([envelope], ["provider"]) ?? "telephony";
|
|
2137
|
-
const streamId = firstString([media, envelope], ["streamSid", "stream_id", "streamId", "streamId", "callSid", "call_id"]);
|
|
2138
|
-
const sequenceNumber = firstString([media, envelope], ["sequenceNumber", "sequence_number", "chunk"]);
|
|
2139
|
-
const track = firstString([media, envelope], ["track", "direction"]);
|
|
2140
|
-
const direction = telephonyDirection(track);
|
|
2141
|
-
const timestamp = firstNumber([media, envelope], ["timestamp", "time", "startedAt"]);
|
|
2142
|
-
return {
|
|
2143
|
-
at: timestamp,
|
|
2144
|
-
audio: base64ToBytes(payload),
|
|
2145
|
-
format: input.format ?? DEFAULT_TELEPHONY_FORMAT,
|
|
2146
|
-
id: [
|
|
2147
|
-
carrier,
|
|
2148
|
-
streamId ?? input.sessionId ?? "stream",
|
|
2149
|
-
sequenceNumber ?? timestamp ?? Date.now()
|
|
2150
|
-
].join(":"),
|
|
2151
|
-
kind: telephonyFrameKind(direction),
|
|
2152
|
-
metadata: {
|
|
2153
|
-
carrier,
|
|
2154
|
-
direction,
|
|
2155
|
-
event: firstString([envelope], ["event", "type"]),
|
|
2156
|
-
sequenceNumber,
|
|
2157
|
-
streamId,
|
|
2158
|
-
track
|
|
2159
|
-
},
|
|
2160
|
-
sessionId: input.sessionId ?? streamId,
|
|
2161
|
-
source: "telephony"
|
|
2162
|
-
};
|
|
2163
|
-
};
|
|
2164
|
-
var serializeTelephonyMediaFrame = (input) => {
|
|
2165
|
-
const carrier = input.carrier ?? input.frame.metadata?.carrier ?? "telephony";
|
|
2166
|
-
const streamId = input.streamId ?? (typeof input.frame.metadata?.streamId === "string" ? input.frame.metadata.streamId : input.frame.sessionId);
|
|
2167
|
-
const sequenceNumber = input.sequenceNumber ?? (typeof input.frame.metadata?.sequenceNumber === "string" || typeof input.frame.metadata?.sequenceNumber === "number" ? input.frame.metadata.sequenceNumber : undefined);
|
|
2168
|
-
const direction = input.frame.kind === "assistant-audio" ? "outbound" : "inbound";
|
|
2169
|
-
const payload = input.frame.audio ? bytesToBase64(input.frame.audio) : "";
|
|
2170
|
-
if (carrier === "twilio") {
|
|
2171
|
-
return {
|
|
2172
|
-
event: "media",
|
|
2173
|
-
sequenceNumber,
|
|
2174
|
-
streamSid: streamId,
|
|
2175
|
-
media: {
|
|
2176
|
-
payload,
|
|
2177
|
-
timestamp: input.frame.at,
|
|
2178
|
-
track: direction
|
|
2179
|
-
}
|
|
2180
|
-
};
|
|
2181
|
-
}
|
|
2182
|
-
if (carrier === "telnyx") {
|
|
2183
|
-
return {
|
|
2184
|
-
event: "media",
|
|
2185
|
-
stream_id: streamId,
|
|
2186
|
-
sequence_number: sequenceNumber,
|
|
2187
|
-
media: {
|
|
2188
|
-
payload,
|
|
2189
|
-
timestamp: input.frame.at,
|
|
2190
|
-
track: direction
|
|
2191
|
-
}
|
|
2192
|
-
};
|
|
2193
|
-
}
|
|
2194
|
-
if (carrier === "plivo") {
|
|
2195
|
-
return {
|
|
2196
|
-
event: "media",
|
|
2197
|
-
streamId,
|
|
2198
|
-
sequenceNumber,
|
|
2199
|
-
media: {
|
|
2200
|
-
payload,
|
|
2201
|
-
timestamp: input.frame.at,
|
|
2202
|
-
track: direction
|
|
2203
|
-
}
|
|
2204
|
-
};
|
|
2205
|
-
}
|
|
2206
|
-
return {
|
|
2207
|
-
event: "media",
|
|
2208
|
-
provider: carrier,
|
|
2209
|
-
sequenceNumber,
|
|
2210
|
-
streamId,
|
|
2211
|
-
media: {
|
|
2212
|
-
payload,
|
|
2213
|
-
timestamp: input.frame.at,
|
|
2214
|
-
track: direction
|
|
2215
|
-
}
|
|
2216
|
-
};
|
|
2217
|
-
};
|
|
2218
|
-
var createTelephonyMediaSerializer = (input) => {
|
|
2219
|
-
const format = input.format ?? DEFAULT_TELEPHONY_FORMAT;
|
|
2220
|
-
return {
|
|
2221
|
-
carrier: input.carrier,
|
|
2222
|
-
format,
|
|
2223
|
-
parse: (envelope) => parseTelephonyMediaFrame({
|
|
2224
|
-
carrier: input.carrier,
|
|
2225
|
-
envelope,
|
|
2226
|
-
format,
|
|
2227
|
-
sessionId: input.sessionId ?? input.streamId
|
|
2228
|
-
}),
|
|
2229
|
-
serialize: (frame) => serializeTelephonyMediaFrame({
|
|
2230
|
-
carrier: input.carrier,
|
|
2231
|
-
frame,
|
|
2232
|
-
streamId: input.streamId
|
|
2233
|
-
})
|
|
2234
|
-
};
|
|
2235
|
-
};
|
|
2236
|
-
var parseTelephonyStreamEvent = (input) => {
|
|
2237
|
-
const envelope = input.envelope;
|
|
2238
|
-
const media = unknownRecord(envelope.media);
|
|
2239
|
-
const start = unknownRecord(envelope.start);
|
|
2240
|
-
const stop = unknownRecord(envelope.stop);
|
|
2241
|
-
const errorRecord = unknownRecord(envelope.error);
|
|
2242
|
-
const kind = telephonyEventKind(envelope);
|
|
2243
|
-
const carrier = input.carrier ?? firstString([envelope], ["provider", "carrier"]) ?? "telephony";
|
|
2244
|
-
const frame = kind === "media" ? parseTelephonyMediaFrame({
|
|
2245
|
-
carrier,
|
|
2246
|
-
envelope,
|
|
2247
|
-
format: input.format,
|
|
2248
|
-
sessionId: input.sessionId
|
|
2249
|
-
}) : undefined;
|
|
2250
|
-
const streamId = firstString([media, start, stop, envelope], ["streamSid", "stream_id", "streamId", "callSid", "call_id"]) ?? input.sessionId;
|
|
2251
|
-
const sequenceNumber = firstString([media, envelope], ["sequenceNumber", "sequence_number", "chunk"]);
|
|
2252
|
-
const track = firstString([media, envelope], ["track", "direction"]);
|
|
2253
|
-
return {
|
|
2254
|
-
audioBytes: frame?.audio ? frame.audio instanceof ArrayBuffer ? frame.audio.byteLength : frame.audio.byteLength : 0,
|
|
2255
|
-
at: frame?.at ?? firstNumber([media, start, stop, envelope], ["timestamp", "time", "startedAt"]),
|
|
2256
|
-
carrier,
|
|
2257
|
-
direction: telephonyDirection(track),
|
|
2258
|
-
error: firstString([errorRecord, envelope], ["message", "error", "reason"]),
|
|
2259
|
-
kind,
|
|
2260
|
-
sequenceNumber,
|
|
2261
|
-
streamId
|
|
2262
|
-
};
|
|
2263
|
-
};
|
|
2264
|
-
var buildMediaTelephonyStreamLifecycleReport = (input = {}) => {
|
|
2265
|
-
const envelopes = input.envelopes ?? [];
|
|
2266
|
-
const events = envelopes.map((envelope) => parseTelephonyStreamEvent({
|
|
2267
|
-
carrier: input.carrier,
|
|
2268
|
-
envelope
|
|
2269
|
-
}));
|
|
2270
|
-
const issues = [];
|
|
2271
|
-
const startedIndex = events.findIndex((event) => event.kind === "start");
|
|
2272
|
-
const firstMediaIndex = events.findIndex((event) => event.kind === "media");
|
|
2273
|
-
const stoppedIndex = events.findIndex((event) => event.kind === "stop");
|
|
2274
|
-
const started = startedIndex >= 0;
|
|
2275
|
-
const stopped = stoppedIndex >= 0;
|
|
2276
|
-
const mediaEvents = events.filter((event) => event.kind === "media");
|
|
2277
|
-
const audioBytes = events.reduce((total, event) => total + event.audioBytes, 0);
|
|
2278
|
-
const minAudioBytes = input.minAudioBytes ?? 1;
|
|
2279
|
-
const streamIds = Array.from(new Set(events.map((event) => event.streamId).filter(Boolean)));
|
|
2280
|
-
if ((input.requireStart ?? true) && !started) {
|
|
2281
|
-
pushIssue(issues, "error", "media.telephony_missing_start", "Telephony media stream did not include a start event.");
|
|
2282
|
-
}
|
|
2283
|
-
if ((input.requireMedia ?? true) && mediaEvents.length === 0) {
|
|
2284
|
-
pushIssue(issues, "error", "media.telephony_missing_media", "Telephony media stream did not include media payload events.");
|
|
2285
|
-
}
|
|
2286
|
-
if ((input.requireStop ?? true) && !stopped) {
|
|
2287
|
-
pushIssue(issues, input.maxMissingStop === false ? "warning" : "error", "media.telephony_missing_stop", "Telephony media stream did not include a stop event.");
|
|
2288
|
-
}
|
|
2289
|
-
if (started && firstMediaIndex >= 0 && firstMediaIndex < startedIndex) {
|
|
2290
|
-
pushIssue(issues, "error", "media.telephony_media_before_start", "Telephony media payload arrived before the stream start event.");
|
|
2291
|
-
}
|
|
2292
|
-
if (stopped && firstMediaIndex >= 0 && stoppedIndex < firstMediaIndex) {
|
|
2293
|
-
pushIssue(issues, "error", "media.telephony_stop_before_media", "Telephony media stream stopped before any media payload arrived.");
|
|
2294
|
-
}
|
|
2295
|
-
if (mediaEvents.length > 0 && audioBytes < minAudioBytes) {
|
|
2296
|
-
pushIssue(issues, "error", "media.telephony_no_audio_bytes", `Telephony media stream parsed ${String(audioBytes)} audio byte(s), below required ${String(minAudioBytes)}.`);
|
|
2297
|
-
}
|
|
2298
|
-
for (const event of events) {
|
|
2299
|
-
if (event.kind === "error") {
|
|
2300
|
-
pushIssue(issues, "error", "media.telephony_stream_error", event.error ?? "Telephony media stream emitted an error event.");
|
|
2301
|
-
}
|
|
2302
|
-
}
|
|
2303
|
-
return {
|
|
2304
|
-
audioBytes,
|
|
2305
|
-
carrier: input.carrier,
|
|
2306
|
-
checkedAt: Date.now(),
|
|
2307
|
-
events,
|
|
2308
|
-
issues,
|
|
2309
|
-
mediaEvents: mediaEvents.length,
|
|
2310
|
-
started,
|
|
2311
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
2312
|
-
stopped,
|
|
2313
|
-
streamIds
|
|
2314
|
-
};
|
|
2315
|
-
};
|
|
2316
|
-
var buildMediaResamplingPlan = (input) => {
|
|
2317
|
-
const required = !formatMatches(input.inputFormat, input.outputFormat);
|
|
2318
|
-
return {
|
|
2319
|
-
inputFormat: input.inputFormat,
|
|
2320
|
-
outputFormat: input.outputFormat,
|
|
2321
|
-
ratio: input.outputFormat.sampleRateHz / input.inputFormat.sampleRateHz,
|
|
2322
|
-
required,
|
|
2323
|
-
status: input.inputFormat.container === input.outputFormat.container && input.inputFormat.encoding === input.outputFormat.encoding && input.inputFormat.channels === input.outputFormat.channels ? "pass" : "warn"
|
|
2324
|
-
};
|
|
2325
|
-
};
|
|
2326
|
-
var speechProbability = (frame) => {
|
|
2327
|
-
if (frame.metadata?.isSpeech === true) {
|
|
2328
|
-
return 1;
|
|
2329
|
-
}
|
|
2330
|
-
if (frame.metadata?.isSpeech === false) {
|
|
2331
|
-
return 0;
|
|
2332
|
-
}
|
|
2333
|
-
for (const key of ["speechProbability", "voiceProbability", "rms", "energy"]) {
|
|
2334
|
-
const value = numericMetadata(frame, key);
|
|
2335
|
-
if (value !== undefined) {
|
|
2336
|
-
return value;
|
|
2337
|
-
}
|
|
2338
|
-
}
|
|
2339
|
-
return 0;
|
|
2340
|
-
};
|
|
2341
|
-
var buildMediaVadReport = (input = {}) => {
|
|
2342
|
-
const frames = (input.frames ?? []).filter((frame) => frame.kind === "input-audio");
|
|
2343
|
-
const speechStartThreshold = input.speechStartThreshold ?? 0.6;
|
|
2344
|
-
const speechEndThreshold = input.speechEndThreshold ?? 0.35;
|
|
2345
|
-
const minSpeechFrames = input.minSpeechFrames ?? 1;
|
|
2346
|
-
const maxSilenceFrames = input.maxSilenceFrames ?? 1;
|
|
2347
|
-
const segments = [];
|
|
2348
|
-
let activeFrames = [];
|
|
2349
|
-
let silenceFrames = 0;
|
|
2350
|
-
const closeSegment = () => {
|
|
2351
|
-
if (activeFrames.length < minSpeechFrames) {
|
|
2352
|
-
activeFrames = [];
|
|
2353
|
-
silenceFrames = 0;
|
|
2354
|
-
return;
|
|
2355
|
-
}
|
|
2356
|
-
const first = activeFrames[0];
|
|
2357
|
-
const last = activeFrames.at(-1);
|
|
2358
|
-
if (!first) {
|
|
2359
|
-
return;
|
|
2360
|
-
}
|
|
2361
|
-
segments.push({
|
|
2362
|
-
durationMs: first.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined,
|
|
2363
|
-
endAt: last?.at !== undefined ? last.at + (last.durationMs ?? 0) : undefined,
|
|
2364
|
-
frameCount: activeFrames.length,
|
|
2365
|
-
segmentId: `vad:${String(segments.length + 1)}`,
|
|
2366
|
-
sessionId: first.sessionId,
|
|
2367
|
-
startAt: first.at,
|
|
2368
|
-
turnId: first.turnId
|
|
2369
|
-
});
|
|
2370
|
-
activeFrames = [];
|
|
2371
|
-
silenceFrames = 0;
|
|
2372
|
-
};
|
|
2373
|
-
for (const frame of frames) {
|
|
2374
|
-
const probability = speechProbability(frame);
|
|
2375
|
-
if (activeFrames.length === 0) {
|
|
2376
|
-
if (probability >= speechStartThreshold) {
|
|
2377
|
-
activeFrames.push(frame);
|
|
2378
|
-
}
|
|
2379
|
-
continue;
|
|
2380
|
-
}
|
|
2381
|
-
activeFrames.push(frame);
|
|
2382
|
-
if (probability <= speechEndThreshold) {
|
|
2383
|
-
silenceFrames += 1;
|
|
2384
|
-
} else {
|
|
2385
|
-
silenceFrames = 0;
|
|
2386
|
-
}
|
|
2387
|
-
if (silenceFrames > maxSilenceFrames) {
|
|
2388
|
-
closeSegment();
|
|
2389
|
-
}
|
|
2390
|
-
}
|
|
2391
|
-
closeSegment();
|
|
2392
|
-
return {
|
|
2393
|
-
checkedAt: Date.now(),
|
|
2394
|
-
inputAudioFrames: frames.length,
|
|
2395
|
-
segments,
|
|
2396
|
-
status: frames.length === 0 ? "warn" : "pass"
|
|
2397
|
-
};
|
|
2398
|
-
};
|
|
2399
|
-
var buildMediaInterruptionReport = (input = {}) => {
|
|
2400
|
-
const issues = [];
|
|
2401
|
-
const interruptionFrames = (input.frames ?? []).filter((frame) => frame.kind === "interruption");
|
|
2402
|
-
const latenciesMs = interruptionFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
|
|
2403
|
-
const maxInterruptionLatencyMs = input.maxInterruptionLatencyMs;
|
|
2404
|
-
if (interruptionFrames.length === 0) {
|
|
2405
|
-
pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
|
|
2406
|
-
}
|
|
2407
|
-
if (maxInterruptionLatencyMs !== undefined && latenciesMs.some((latency) => latency > maxInterruptionLatencyMs)) {
|
|
2408
|
-
pushIssue(issues, "error", "media.interruption_latency", `Interruption latency exceeded ${String(maxInterruptionLatencyMs)}ms.`);
|
|
2409
|
-
}
|
|
2410
|
-
return {
|
|
2411
|
-
checkedAt: Date.now(),
|
|
2412
|
-
interruptionFrames: interruptionFrames.length,
|
|
2413
|
-
issues,
|
|
2414
|
-
latenciesMs,
|
|
2415
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass"
|
|
2416
|
-
};
|
|
2417
|
-
};
|
|
2418
|
-
var buildMediaQualityReport = (input = {}) => {
|
|
2419
|
-
const frames = [...input.frames ?? []].sort((a, b) => (a.at ?? 0) - (b.at ?? 0));
|
|
2420
|
-
const audioFrames = frames.filter((frame) => frame.kind === "input-audio" || frame.kind === "assistant-audio");
|
|
2421
|
-
const inputAudioFrames = frames.filter((frame) => frame.kind === "input-audio");
|
|
2422
|
-
const assistantAudioFrames = frames.filter((frame) => frame.kind === "assistant-audio");
|
|
2423
|
-
const issues = [];
|
|
2424
|
-
const gapsMs = [];
|
|
2425
|
-
for (const [index, frame] of audioFrames.entries()) {
|
|
2426
|
-
const previous = audioFrames[index - 1];
|
|
2427
|
-
if (previous?.at === undefined || frame.at === undefined || previous.durationMs === undefined) {
|
|
2428
|
-
continue;
|
|
2429
|
-
}
|
|
2430
|
-
const gap = frame.at - (previous.at + previous.durationMs);
|
|
2431
|
-
if (gap > 0) {
|
|
2432
|
-
gapsMs.push(gap);
|
|
2433
|
-
}
|
|
2434
|
-
}
|
|
2435
|
-
const jitterMs = audioFrames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined).at(-1) ?? max(gapsMs);
|
|
2436
|
-
const first = audioFrames.find((frame) => frame.at !== undefined);
|
|
2437
|
-
const last = audioFrames.toReversed().find((frame) => frame.at !== undefined);
|
|
2438
|
-
const durationMs = first?.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined;
|
|
2439
|
-
const expectedDurationMs = audioFrames.length > 0 ? audioFrames.reduce((total, frame) => total + (frame.durationMs ?? 0), 0) : undefined;
|
|
2440
|
-
const timestampDriftMs = durationMs !== undefined && expectedDurationMs !== undefined ? Math.max(0, durationMs - expectedDurationMs) : undefined;
|
|
2441
|
-
const speechScores = inputAudioFrames.map(speechProbability);
|
|
2442
|
-
const speechFrames = speechScores.filter((score) => score >= 0.6).length;
|
|
2443
|
-
const silenceFrames = speechScores.filter((score) => score <= 0.35).length;
|
|
2444
|
-
const unknownSpeechFrames = Math.max(0, inputAudioFrames.length - speechFrames - silenceFrames);
|
|
2445
|
-
const speechRatio = inputAudioFrames.length === 0 ? 0 : speechFrames / inputAudioFrames.length;
|
|
2446
|
-
const silenceRatio = inputAudioFrames.length === 0 ? 0 : silenceFrames / inputAudioFrames.length;
|
|
2447
|
-
const levels = audioFrames.map((frame) => numericMetadata(frame, "level") ?? numericMetadata(frame, "rms") ?? numericMetadata(frame, "energy")).filter((value) => value !== undefined);
|
|
2448
|
-
const backpressureEvents = input.transport?.backpressureEvents ?? 0;
|
|
2449
|
-
const maxGapMs = input.maxGapMs;
|
|
2450
|
-
if (maxGapMs !== undefined && gapsMs.some((gap) => gap > maxGapMs)) {
|
|
2451
|
-
pushIssue(issues, "warning", "media.quality_gap", `Observed media gap above ${String(maxGapMs)}ms.`);
|
|
2452
|
-
}
|
|
2453
|
-
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
2454
|
-
pushIssue(issues, "warning", "media.quality_jitter", `Observed jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
2455
|
-
}
|
|
2456
|
-
if (input.maxTimestampDriftMs !== undefined && timestampDriftMs !== undefined && timestampDriftMs > input.maxTimestampDriftMs) {
|
|
2457
|
-
pushIssue(issues, "warning", "media.quality_timestamp_drift", `Observed timestamp drift ${String(timestampDriftMs)}ms above ${String(input.maxTimestampDriftMs)}ms.`);
|
|
2458
|
-
}
|
|
2459
|
-
if (input.minSpeechRatio !== undefined && inputAudioFrames.length > 0 && speechRatio < input.minSpeechRatio) {
|
|
2460
|
-
pushIssue(issues, "warning", "media.quality_speech_ratio", `Observed speech ratio ${String(speechRatio)} below ${String(input.minSpeechRatio)}.`);
|
|
2461
|
-
}
|
|
2462
|
-
if (input.maxBackpressureEvents !== undefined && backpressureEvents > input.maxBackpressureEvents) {
|
|
2463
|
-
pushIssue(issues, "warning", "media.quality_backpressure", `Observed ${String(backpressureEvents)} backpressure event(s), above ${String(input.maxBackpressureEvents)}.`);
|
|
2464
|
-
}
|
|
2465
|
-
return {
|
|
2466
|
-
assistantAudioFrames: assistantAudioFrames.length,
|
|
2467
|
-
backpressureEvents,
|
|
2468
|
-
checkedAt: Date.now(),
|
|
2469
|
-
durationMs,
|
|
2470
|
-
gapCount: gapsMs.length,
|
|
2471
|
-
gapsMs,
|
|
2472
|
-
inputAudioFrames: inputAudioFrames.length,
|
|
2473
|
-
issues,
|
|
2474
|
-
jitterMs,
|
|
2475
|
-
levelAverage: average(levels),
|
|
2476
|
-
levelMax: max(levels),
|
|
2477
|
-
levelMin: min(levels),
|
|
2478
|
-
silenceFrames,
|
|
2479
|
-
silenceRatio,
|
|
2480
|
-
speechFrames,
|
|
2481
|
-
speechRatio,
|
|
2482
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
2483
|
-
timestampDriftMs,
|
|
2484
|
-
totalFrames: frames.length,
|
|
2485
|
-
unknownSpeechFrames
|
|
2486
|
-
};
|
|
2487
|
-
};
|
|
2488
|
-
var buildMediaWebRTCStatsReport = (input = {}) => {
|
|
2489
|
-
const stats = input.stats ?? [];
|
|
2490
|
-
const issues = [];
|
|
2491
|
-
const inbound = stats.filter((stat) => stat.type === "inbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
2492
|
-
const outbound = stats.filter((stat) => stat.type === "outbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
2493
|
-
const candidatePairs = stats.filter((stat) => stat.type === "candidate-pair");
|
|
2494
|
-
const audioTracks = stats.filter((stat) => (stat.type === "track" || stat.type === "media-source") && stringStat(stat, "kind") === "audio");
|
|
2495
|
-
const activeCandidatePairs = candidatePairs.filter((stat) => booleanStat(stat, "selected") === true || booleanStat(stat, "nominated") === true || stringStat(stat, "state") === "succeeded").length;
|
|
2496
|
-
const liveAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") !== "ended" && stringStat(stat, "trackState") !== "ended" && booleanStat(stat, "ended") !== true).length;
|
|
2497
|
-
const endedAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") === "ended" || stringStat(stat, "trackState") === "ended" || booleanStat(stat, "ended") === true).length;
|
|
2498
|
-
const inboundPackets = inbound.reduce((total, stat) => total + (numericStat(stat, "packetsReceived") ?? 0), 0);
|
|
2499
|
-
const outboundPackets = outbound.reduce((total, stat) => total + (numericStat(stat, "packetsSent") ?? 0), 0);
|
|
2500
|
-
const packetsLost = [...inbound, ...outbound].reduce((total, stat) => total + Math.max(0, numericStat(stat, "packetsLost") ?? 0), 0);
|
|
2501
|
-
const packetLossDenominator = inboundPackets + packetsLost;
|
|
2502
|
-
const packetLossRatio = packetLossDenominator === 0 ? 0 : packetsLost / packetLossDenominator;
|
|
2503
|
-
const bytesReceived = inbound.reduce((total, stat) => total + (numericStat(stat, "bytesReceived") ?? 0), 0);
|
|
2504
|
-
const bytesSent = outbound.reduce((total, stat) => total + (numericStat(stat, "bytesSent") ?? 0), 0);
|
|
2505
|
-
const roundTripTimeMs = max(candidatePairs.map((stat) => secondsToMs(numericStat(stat, "currentRoundTripTime") ?? numericStat(stat, "roundTripTime"))).filter((value) => value !== undefined));
|
|
2506
|
-
const jitterMs = max([...inbound, ...outbound].map((stat) => secondsToMs(numericStat(stat, "jitter"))).filter((value) => value !== undefined));
|
|
2507
|
-
const jitterBufferDelayMs = max(inbound.map((stat) => {
|
|
2508
|
-
const delay = numericStat(stat, "jitterBufferDelay");
|
|
2509
|
-
const emitted = numericStat(stat, "jitterBufferEmittedCount");
|
|
2510
|
-
return delay !== undefined && emitted !== undefined && emitted > 0 ? delay / emitted * 1000 : undefined;
|
|
2511
|
-
}).filter((value) => value !== undefined));
|
|
2512
|
-
const audioLevels = audioTracks.map((stat) => numericStat(stat, "audioLevel")).filter((value) => value !== undefined);
|
|
2513
|
-
if (input.requireConnectedCandidatePair && candidatePairs.length > 0 && activeCandidatePairs === 0) {
|
|
2514
|
-
pushIssue(issues, "error", "media.webrtc_candidate_pair_missing", "No active WebRTC candidate pair was observed.");
|
|
2515
|
-
}
|
|
2516
|
-
if (input.requireLiveAudioTrack && liveAudioTracks === 0) {
|
|
2517
|
-
pushIssue(issues, "error", "media.webrtc_audio_track_missing", "No live WebRTC audio track was observed.");
|
|
2518
|
-
}
|
|
2519
|
-
if (input.maxPacketLossRatio !== undefined && packetLossRatio > input.maxPacketLossRatio) {
|
|
2520
|
-
pushIssue(issues, "warning", "media.webrtc_packet_loss", `Observed WebRTC packet loss ratio ${String(packetLossRatio)} above ${String(input.maxPacketLossRatio)}.`);
|
|
2521
|
-
}
|
|
2522
|
-
if (input.maxRoundTripTimeMs !== undefined && roundTripTimeMs !== undefined && roundTripTimeMs > input.maxRoundTripTimeMs) {
|
|
2523
|
-
pushIssue(issues, "warning", "media.webrtc_round_trip_time", `Observed WebRTC RTT ${String(roundTripTimeMs)}ms above ${String(input.maxRoundTripTimeMs)}ms.`);
|
|
2524
|
-
}
|
|
2525
|
-
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
2526
|
-
pushIssue(issues, "warning", "media.webrtc_jitter", `Observed WebRTC jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
2527
|
-
}
|
|
2528
|
-
return {
|
|
2529
|
-
activeCandidatePairs,
|
|
2530
|
-
audioLevelAverage: average(audioLevels),
|
|
2531
|
-
bytesReceived,
|
|
2532
|
-
bytesSent,
|
|
2533
|
-
checkedAt: Date.now(),
|
|
2534
|
-
endedAudioTracks,
|
|
2535
|
-
inboundPackets,
|
|
2536
|
-
issues,
|
|
2537
|
-
jitterBufferDelayMs,
|
|
2538
|
-
jitterMs,
|
|
2539
|
-
liveAudioTracks,
|
|
2540
|
-
outboundPackets,
|
|
2541
|
-
packetLossRatio,
|
|
2542
|
-
packetsLost,
|
|
2543
|
-
roundTripTimeMs,
|
|
2544
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
2545
|
-
totalStats: stats.length
|
|
2546
|
-
};
|
|
2547
|
-
};
|
|
2548
|
-
var collectMediaWebRTCStats = async (input) => {
|
|
2549
|
-
const report = await input.peerConnection.getStats(input.selector ?? null);
|
|
2550
|
-
return [...report.values()].map(normalizeWebRTCStat);
|
|
2551
|
-
};
|
|
2552
|
-
var buildMediaWebRTCStreamContinuityReport = (input = {}) => {
|
|
2553
|
-
const stats = input.stats ?? [];
|
|
2554
|
-
const previousStats = input.previousStats ?? [];
|
|
2555
|
-
const issues = [];
|
|
2556
|
-
const previousByKey = new Map(previousStats.map((stat) => [statKey(stat), stat]));
|
|
2557
|
-
const audioRtp = stats.filter((stat) => (stat.type === "inbound-rtp" || stat.type === "outbound-rtp") && stringStat(stat, "kind") !== "video" && stringStat(stat, "mediaType") !== "video");
|
|
2558
|
-
const streams = audioRtp.map((stat) => {
|
|
2559
|
-
const direction = stat.type === "outbound-rtp" ? "outbound" : "inbound";
|
|
2560
|
-
const packetsKey = direction === "outbound" ? "packetsSent" : "packetsReceived";
|
|
2561
|
-
const bytesKey = direction === "outbound" ? "bytesSent" : "bytesReceived";
|
|
2562
|
-
const previous = previousByKey.get(statKey(stat));
|
|
2563
|
-
const currentPackets = numericStat(stat, packetsKey);
|
|
2564
|
-
const previousPackets = previous ? numericStat(previous, packetsKey) : undefined;
|
|
2565
|
-
const currentBytes = numericStat(stat, bytesKey);
|
|
2566
|
-
const previousBytes = previous ? numericStat(previous, bytesKey) : undefined;
|
|
2567
|
-
const timeDeltaMs = stat.timestamp !== undefined && previous?.timestamp !== undefined ? stat.timestamp - previous.timestamp : undefined;
|
|
2568
|
-
return {
|
|
2569
|
-
bytesDelta: currentBytes !== undefined && previousBytes !== undefined ? currentBytes - previousBytes : undefined,
|
|
2570
|
-
currentPackets,
|
|
2571
|
-
direction,
|
|
2572
|
-
id: statKey(stat),
|
|
2573
|
-
packetDelta: currentPackets !== undefined && previousPackets !== undefined ? currentPackets - previousPackets : undefined,
|
|
2574
|
-
previousPackets,
|
|
2575
|
-
timeDeltaMs
|
|
2576
|
-
};
|
|
2577
|
-
});
|
|
2578
|
-
const inbound = streams.filter((stream) => stream.direction === "inbound");
|
|
2579
|
-
const outbound = streams.filter((stream) => stream.direction === "outbound");
|
|
2580
|
-
const maxObservedGapMs = max(streams.map((stream) => stream.timeDeltaMs).filter((value) => value !== undefined));
|
|
2581
|
-
const stalledInboundStreams = inbound.filter((stream) => input.maxInboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxInboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
2582
|
-
const stalledOutboundStreams = outbound.filter((stream) => input.maxOutboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxOutboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
2583
|
-
if (input.requireInboundAudio && inbound.length === 0) {
|
|
2584
|
-
pushIssue(issues, "error", "media.webrtc_inbound_audio_missing", "No inbound WebRTC audio RTP stream was observed.");
|
|
2585
|
-
}
|
|
2586
|
-
if (input.requireOutboundAudio && outbound.length === 0) {
|
|
2587
|
-
pushIssue(issues, "error", "media.webrtc_outbound_audio_missing", "No outbound WebRTC audio RTP stream was observed.");
|
|
2588
|
-
}
|
|
2589
|
-
if (input.maxGapMs !== undefined && maxObservedGapMs !== undefined && maxObservedGapMs > input.maxGapMs) {
|
|
2590
|
-
pushIssue(issues, "warning", "media.webrtc_stream_gap", `Observed WebRTC stream sample gap ${String(maxObservedGapMs)}ms above ${String(input.maxGapMs)}ms.`);
|
|
2591
|
-
}
|
|
2592
|
-
if (stalledInboundStreams > 0) {
|
|
2593
|
-
pushIssue(issues, "error", "media.webrtc_inbound_stalled", `${String(stalledInboundStreams)} inbound WebRTC audio stream(s) stopped receiving packets.`);
|
|
2594
|
-
}
|
|
2595
|
-
if (stalledOutboundStreams > 0) {
|
|
2596
|
-
pushIssue(issues, "error", "media.webrtc_outbound_stalled", `${String(stalledOutboundStreams)} outbound WebRTC audio stream(s) stopped sending packets.`);
|
|
2597
|
-
}
|
|
2598
|
-
return {
|
|
2599
|
-
checkedAt: Date.now(),
|
|
2600
|
-
inboundAudioStreams: inbound.length,
|
|
2601
|
-
issues,
|
|
2602
|
-
maxObservedGapMs,
|
|
2603
|
-
outboundAudioStreams: outbound.length,
|
|
2604
|
-
stalledInboundStreams,
|
|
2605
|
-
stalledOutboundStreams,
|
|
2606
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
2607
|
-
streams,
|
|
2608
|
-
totalStats: stats.length
|
|
2609
|
-
};
|
|
2610
|
-
};
|
|
2611
|
-
var buildMediaPipelineCalibrationReport = (input = {}) => {
|
|
2612
|
-
const frames = input.frames ?? [];
|
|
2613
|
-
const issues = [];
|
|
2614
|
-
const inputFrames = frames.filter((frame) => frame.kind === "input-audio");
|
|
2615
|
-
const assistantFrames = frames.filter((frame) => frame.kind === "assistant-audio");
|
|
2616
|
-
const turnCommitFrames = frames.filter((frame) => frame.kind === "turn-commit");
|
|
2617
|
-
const interruptionFrameRecords = frames.filter((frame) => frame.kind === "interruption");
|
|
2618
|
-
const traceLinkedFrames = frames.filter((frame) => frame.traceEventId).length;
|
|
2619
|
-
const backpressureFrames = frames.filter((frame) => Boolean(frame.metadata?.backpressure)).length;
|
|
2620
|
-
const audioLatencies = assistantFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
|
|
2621
|
-
const firstAudioLatencyMs = audioLatencies.length > 0 ? Math.min(...audioLatencies) : undefined;
|
|
2622
|
-
const jitterValues = frames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined);
|
|
2623
|
-
const jitterMs = jitterValues.length > 0 ? Math.max(...jitterValues) : undefined;
|
|
2624
|
-
const inputFormat = input.inputFormat ?? inputFrames.find((frame) => frame.format)?.format;
|
|
2625
|
-
const outputFormat = input.outputFormat ?? assistantFrames.find((frame) => frame.format)?.format;
|
|
2626
|
-
const resamplingRequired = Boolean(input.expectedInputFormat && inputFormat && inputFormat.sampleRateHz !== input.expectedInputFormat.sampleRateHz) || Boolean(input.expectedOutputFormat && outputFormat && outputFormat.sampleRateHz !== input.expectedOutputFormat.sampleRateHz);
|
|
2627
|
-
const resamplingTargetHz = resamplingRequired && input.expectedInputFormat ? input.expectedInputFormat.sampleRateHz : resamplingRequired ? input.expectedOutputFormat?.sampleRateHz : undefined;
|
|
2628
|
-
if (inputFrames.length === 0) {
|
|
2629
|
-
pushIssue(issues, "warning", "media.input_audio_missing", "No input audio frames were observed.");
|
|
2630
|
-
}
|
|
2631
|
-
if (assistantFrames.length === 0) {
|
|
2632
|
-
pushIssue(issues, "warning", "media.assistant_audio_missing", "No assistant audio frames were observed.");
|
|
2633
|
-
}
|
|
2634
|
-
if (input.expectedInputFormat && inputFormat && !formatMatches(inputFormat, input.expectedInputFormat)) {
|
|
2635
|
-
pushIssue(issues, inputFormat.sampleRateHz === input.expectedInputFormat.sampleRateHz ? "warning" : "error", "media.input_format_mismatch", `Input format ${formatLabel(inputFormat)} does not match expected ${formatLabel(input.expectedInputFormat)}.`);
|
|
2636
|
-
}
|
|
2637
|
-
if (input.expectedOutputFormat && outputFormat && !formatMatches(outputFormat, input.expectedOutputFormat)) {
|
|
2638
|
-
pushIssue(issues, outputFormat.sampleRateHz === input.expectedOutputFormat.sampleRateHz ? "warning" : "error", "media.output_format_mismatch", `Output format ${formatLabel(outputFormat)} does not match expected ${formatLabel(input.expectedOutputFormat)}.`);
|
|
2639
|
-
}
|
|
2640
|
-
if (firstAudioLatencyMs !== undefined && input.maxFirstAudioLatencyMs !== undefined && firstAudioLatencyMs > input.maxFirstAudioLatencyMs) {
|
|
2641
|
-
pushIssue(issues, "error", "media.first_audio_latency", `First audio latency ${String(firstAudioLatencyMs)}ms exceeds budget ${String(input.maxFirstAudioLatencyMs)}ms.`);
|
|
2642
|
-
}
|
|
2643
|
-
if (jitterMs !== undefined && input.maxJitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
2644
|
-
pushIssue(issues, "warning", "media.jitter", `Media jitter ${String(jitterMs)}ms exceeds budget ${String(input.maxJitterMs)}ms.`);
|
|
2645
|
-
}
|
|
2646
|
-
if (input.maxBackpressureFrames !== undefined && backpressureFrames > input.maxBackpressureFrames) {
|
|
2647
|
-
pushIssue(issues, "warning", "media.backpressure", `Backpressure frame count ${String(backpressureFrames)} exceeds budget ${String(input.maxBackpressureFrames)}.`);
|
|
2648
|
-
}
|
|
2649
|
-
if (input.requireInterruptionFrame && interruptionFrameRecords.length === 0) {
|
|
2650
|
-
pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
|
|
2651
|
-
}
|
|
2652
|
-
if (input.requireTraceEvidence && traceLinkedFrames === 0) {
|
|
2653
|
-
pushIssue(issues, "warning", "media.trace_evidence_missing", "No media frames were linked to trace evidence.");
|
|
2654
|
-
}
|
|
2655
|
-
return {
|
|
2656
|
-
assistantAudioFrames: assistantFrames.length,
|
|
2657
|
-
backpressureFrames,
|
|
2658
|
-
checkedAt: Date.now(),
|
|
2659
|
-
firstAudioLatencyMs,
|
|
2660
|
-
inputAudioFrames: inputFrames.length,
|
|
2661
|
-
inputFormat,
|
|
2662
|
-
interruptionFrames: interruptionFrameRecords.length,
|
|
2663
|
-
issues,
|
|
2664
|
-
jitterMs,
|
|
2665
|
-
outputFormat,
|
|
2666
|
-
resamplingRequired,
|
|
2667
|
-
resamplingTargetHz,
|
|
2668
|
-
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
2669
|
-
surface: input.surface ?? "media-pipeline",
|
|
2670
|
-
traceLinkedFrames,
|
|
2671
|
-
turnCommitFrames: turnCommitFrames.length
|
|
2672
|
-
};
|
|
2673
|
-
};
|
|
2674
|
-
var DEFAULT_METADATA_DENY = [
|
|
2675
|
-
"audioPayload",
|
|
2676
|
-
"auth",
|
|
2677
|
-
"authorization",
|
|
2678
|
-
"cookie",
|
|
2679
|
-
"email",
|
|
2680
|
-
"phone",
|
|
2681
|
-
"phoneNumber",
|
|
2682
|
-
"rawPayload",
|
|
2683
|
-
"secret",
|
|
2684
|
-
"token",
|
|
2685
|
-
"transcript",
|
|
2686
|
-
"utterance"
|
|
2687
|
-
];
|
|
2688
|
-
var DEFAULT_TRUNCATE = 8;
|
|
2689
|
-
var issueCodes = (issues) => Array.from(new Set(issues.map((issue) => issue.code)));
|
|
2690
|
-
var lastEventKind = (events) => events[events.length - 1]?.kind;
|
|
2691
|
-
var formatOptionalMs = (value) => value === undefined ? "n/a" : `${String(Math.round(value))}ms`;
|
|
2692
|
-
var formatRatio = (value) => `${(value * 100).toFixed(1)}%`;
|
|
2693
|
-
var escapeMarkdownCell = (value) => value.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
2694
|
-
var renderIssuesTable = (issues) => {
|
|
2695
|
-
if (issues.length === 0) {
|
|
2696
|
-
return `- No issues.
|
|
2697
|
-
`;
|
|
2698
|
-
}
|
|
2699
|
-
const rows = issues.map((issue) => `| ${issue.severity} | ${escapeMarkdownCell(issue.code)} | ${escapeMarkdownCell(issue.message)} |`).join(`
|
|
2700
|
-
`);
|
|
2701
|
-
return `| Severity | Code | Message |
|
|
2702
|
-
| --- | --- | --- |
|
|
2703
|
-
${rows}
|
|
2704
|
-
`;
|
|
2705
|
-
};
|
|
2706
|
-
var summarizeMediaQualityReport = (report) => ({
|
|
2707
|
-
backpressureEvents: report.backpressureEvents,
|
|
2708
|
-
description: `${report.totalFrames} frame(s), ${report.gapCount} gap(s), speech ${formatRatio(report.speechRatio)}, status ${report.status}.`,
|
|
2709
|
-
driftMs: report.timestampDriftMs,
|
|
2710
|
-
frameCount: report.totalFrames,
|
|
2711
|
-
gapCount: report.gapCount,
|
|
2712
|
-
issueCodes: issueCodes(report.issues),
|
|
2713
|
-
issueCount: report.issues.length,
|
|
2714
|
-
jitterMs: report.jitterMs,
|
|
2715
|
-
silenceRatio: report.silenceRatio,
|
|
2716
|
-
speechRatio: report.speechRatio,
|
|
2717
|
-
status: report.status
|
|
2718
|
-
});
|
|
2719
|
-
var summarizeMediaTransportReport = (report) => ({
|
|
2720
|
-
backpressureEvents: report.backpressureEvents,
|
|
2721
|
-
description: `${report.name}: ${report.state}, in ${report.inputFrames}, out ${report.outputFrames}, backpressure ${report.backpressureEvents}.`,
|
|
2722
|
-
errors: report.events.filter((event) => event.kind === "error").length,
|
|
2723
|
-
inputFrames: report.inputFrames,
|
|
2724
|
-
lastEventKind: lastEventKind(report.events),
|
|
2725
|
-
name: report.name,
|
|
2726
|
-
outputFrames: report.outputFrames,
|
|
2727
|
-
state: report.state,
|
|
2728
|
-
status: report.status
|
|
2729
|
-
});
|
|
2730
|
-
var summarizeMediaProcessorGraphReport = (report) => {
|
|
2731
|
-
const errorIssueCodes = Array.from(new Set(report.errors.map((event) => event.kind)));
|
|
2732
|
-
return {
|
|
2733
|
-
backpressureEvents: report.backpressure.events.length,
|
|
2734
|
-
description: `${report.name}: ${report.state}, ${report.nodes.length} node(s), in ${report.inputFrames}, out ${report.emittedFrames}, dropped ${report.droppedFrames}.`,
|
|
2735
|
-
droppedFrames: report.droppedFrames,
|
|
2736
|
-
edgeCount: report.edges.length,
|
|
2737
|
-
edgeEventCount: report.edgeEvents.length,
|
|
2738
|
-
emittedFrames: report.emittedFrames,
|
|
2739
|
-
errorCount: report.errors.length,
|
|
2740
|
-
inputFrames: report.inputFrames,
|
|
2741
|
-
issueCodes: errorIssueCodes,
|
|
2742
|
-
lifecycleEventCount: report.lifecycleEvents.length,
|
|
2743
|
-
name: report.name,
|
|
2744
|
-
nodeCount: report.nodes.length,
|
|
2745
|
-
state: report.state,
|
|
2746
|
-
status: report.status,
|
|
2747
|
-
timingMaxMs: report.timing.maxNodeMs
|
|
2748
|
-
};
|
|
2749
|
-
};
|
|
2750
|
-
var renderMediaQualityMarkdown = (report, options = {}) => {
|
|
2751
|
-
const title = options.title ?? "Media Quality Report";
|
|
2752
|
-
const lines = [
|
|
2753
|
-
`# ${title}`,
|
|
2754
|
-
"",
|
|
2755
|
-
`Status: **${report.status}**`,
|
|
2756
|
-
"",
|
|
2757
|
-
"| Metric | Value |",
|
|
2758
|
-
"| --- | ---: |",
|
|
2759
|
-
`| Total frames | ${report.totalFrames} |`,
|
|
2760
|
-
`| Input audio | ${report.inputAudioFrames} |`,
|
|
2761
|
-
`| Assistant audio | ${report.assistantAudioFrames} |`,
|
|
2762
|
-
`| Gaps | ${report.gapCount} |`,
|
|
2763
|
-
`| Jitter | ${formatOptionalMs(report.jitterMs)} |`,
|
|
2764
|
-
`| Timestamp drift | ${formatOptionalMs(report.timestampDriftMs)} |`,
|
|
2765
|
-
`| Speech ratio | ${formatRatio(report.speechRatio)} |`,
|
|
2766
|
-
`| Silence ratio | ${formatRatio(report.silenceRatio)} |`,
|
|
2767
|
-
`| Backpressure events | ${report.backpressureEvents} |`,
|
|
2768
|
-
"",
|
|
2769
|
-
"## Issues",
|
|
2770
|
-
"",
|
|
2771
|
-
renderIssuesTable(report.issues).trimEnd()
|
|
2772
|
-
];
|
|
2773
|
-
return `${lines.join(`
|
|
2774
|
-
`)}
|
|
2775
|
-
`;
|
|
2776
|
-
};
|
|
2777
|
-
var renderMediaTransportMarkdown = (report, options = {}) => {
|
|
2778
|
-
const title = options.title ?? `Media Transport: ${report.name}`;
|
|
2779
|
-
const limit = options.redact?.truncateArraysAt ?? DEFAULT_TRUNCATE;
|
|
2780
|
-
const events = report.events.slice(-limit);
|
|
2781
|
-
const eventRows = events.length === 0 ? "- No transport events recorded." : ["| At | Kind | State | Buffered | Error |", "| --- | --- | --- | ---: | --- |"].concat(events.map((event) => `| ${event.at} | ${event.kind} | ${event.state} | ${event.bufferedFrames ?? ""} | ${escapeMarkdownCell(event.error ?? "")} |`)).join(`
|
|
2782
|
-
`);
|
|
2783
|
-
const lines = [
|
|
2784
|
-
`# ${title}`,
|
|
2785
|
-
"",
|
|
2786
|
-
`Status: **${report.status}** \xB7 State: **${report.state}**`,
|
|
2787
|
-
"",
|
|
2788
|
-
"| Metric | Value |",
|
|
2789
|
-
"| --- | ---: |",
|
|
2790
|
-
`| Input frames | ${report.inputFrames} |`,
|
|
2791
|
-
`| Output frames | ${report.outputFrames} |`,
|
|
2792
|
-
`| Backpressure events | ${report.backpressureEvents} |`,
|
|
2793
|
-
`| Connected | ${report.connected ? "yes" : "no"} |`,
|
|
2794
|
-
`| Closed | ${report.closed ? "yes" : "no"} |`,
|
|
2795
|
-
`| Failed | ${report.failed ? "yes" : "no"} |`,
|
|
2796
|
-
"",
|
|
2797
|
-
`## Last ${events.length} event(s)`,
|
|
2798
|
-
"",
|
|
2799
|
-
eventRows
|
|
2800
|
-
];
|
|
2801
|
-
return `${lines.join(`
|
|
2802
|
-
`)}
|
|
2803
|
-
`;
|
|
2804
|
-
};
|
|
2805
|
-
var renderMediaProcessorGraphMarkdown = (report, options = {}) => {
|
|
2806
|
-
const title = options.title ?? `Media Processor Graph: ${report.name}`;
|
|
2807
|
-
const limit = options.redact?.truncateArraysAt ?? DEFAULT_TRUNCATE;
|
|
2808
|
-
const nodeRows = report.nodes.map((node) => `| ${escapeMarkdownCell(node.name)} | ${node.kind} | ${node.status} | ${node.inputFrames} | ${node.emittedFrames} | ${node.droppedFrames} | ${node.errors.length} |`).join(`
|
|
2809
|
-
`);
|
|
2810
|
-
const edgeRows = report.edges.slice(0, limit).map((edge) => `| ${escapeMarkdownCell(edge.from)} | ${escapeMarkdownCell(edge.to)} | ${edge.status} | ${edge.emittedFrames} |`).join(`
|
|
2811
|
-
`);
|
|
2812
|
-
const issues = report.errors.map((event) => ({
|
|
2813
|
-
code: event.kind,
|
|
2814
|
-
message: event.error ?? `Processor graph ${event.kind} (state ${event.state}).`,
|
|
2815
|
-
severity: "error"
|
|
2816
|
-
}));
|
|
2817
|
-
const lines = [
|
|
2818
|
-
`# ${title}`,
|
|
2819
|
-
"",
|
|
2820
|
-
`Status: **${report.status}** \xB7 State: **${report.state}**`,
|
|
2821
|
-
"",
|
|
2822
|
-
"| Metric | Value |",
|
|
2823
|
-
"| --- | ---: |",
|
|
2824
|
-
`| Nodes | ${report.nodes.length} |`,
|
|
2825
|
-
`| Input frames | ${report.inputFrames} |`,
|
|
2826
|
-
`| Emitted frames | ${report.emittedFrames} |`,
|
|
2827
|
-
`| Dropped frames | ${report.droppedFrames} |`,
|
|
2828
|
-
`| Lifecycle events | ${report.lifecycleEvents.length} |`,
|
|
2829
|
-
`| Edge events | ${report.edgeEvents.length} |`,
|
|
2830
|
-
`| Backpressure events | ${report.backpressure.events.length} |`,
|
|
2831
|
-
`| Timing max | ${formatOptionalMs(report.timing.maxNodeMs)} |`,
|
|
2832
|
-
`| Timing average | ${formatOptionalMs(report.timing.averageNodeMs)} |`,
|
|
2833
|
-
"",
|
|
2834
|
-
"## Nodes",
|
|
2835
|
-
"",
|
|
2836
|
-
nodeRows ? `| Node | Kind | Status | In | Out | Dropped | Errors |
|
|
2837
|
-
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
|
2838
|
-
${nodeRows}` : "- No nodes.",
|
|
2839
|
-
"",
|
|
2840
|
-
`## Edges (showing up to ${limit})`,
|
|
2841
|
-
"",
|
|
2842
|
-
edgeRows ? `| From | To | Status | Frames |
|
|
2843
|
-
| --- | --- | --- | ---: |
|
|
2844
|
-
${edgeRows}` : "- No edges.",
|
|
2845
|
-
"",
|
|
2846
|
-
"## Errors",
|
|
2847
|
-
"",
|
|
2848
|
-
renderIssuesTable(issues).trimEnd()
|
|
2849
|
-
];
|
|
2850
|
-
return `${lines.join(`
|
|
2851
|
-
`)}
|
|
2852
|
-
`;
|
|
2853
|
-
};
|
|
2854
|
-
var truncateArrays = (value, limit, seen) => {
|
|
2855
|
-
if (Array.isArray(value)) {
|
|
2856
|
-
const head = value.slice(0, limit).map((entry) => truncateArrays(entry, limit, seen));
|
|
2857
|
-
if (value.length > limit) {
|
|
2858
|
-
return [...head, { truncated: value.length - limit }];
|
|
2859
|
-
}
|
|
2860
|
-
return head;
|
|
2861
|
-
}
|
|
2862
|
-
if (value && typeof value === "object") {
|
|
2863
|
-
if (seen.has(value))
|
|
2864
|
-
return value;
|
|
2865
|
-
seen.add(value);
|
|
2866
|
-
const next = {};
|
|
2867
|
-
for (const [key, entry] of Object.entries(value)) {
|
|
2868
|
-
next[key] = truncateArrays(entry, limit, seen);
|
|
2869
|
-
}
|
|
2870
|
-
return next;
|
|
2871
|
-
}
|
|
2872
|
-
return value;
|
|
2873
|
-
};
|
|
2874
|
-
var applyRedaction = (value, options, seen) => {
|
|
2875
|
-
const mode = options.mode ?? "omit";
|
|
2876
|
-
const maskValue = options.maskValue ?? "[redacted]";
|
|
2877
|
-
const allow = new Set(options.metadataAllow ?? []);
|
|
2878
|
-
const deny = new Set(options.metadataDeny ?? DEFAULT_METADATA_DENY);
|
|
2879
|
-
const walk = (input) => {
|
|
2880
|
-
if (Array.isArray(input)) {
|
|
2881
|
-
return input.map((entry) => walk(entry));
|
|
2882
|
-
}
|
|
2883
|
-
if (input && typeof input === "object") {
|
|
2884
|
-
if (seen.has(input))
|
|
2885
|
-
return input;
|
|
2886
|
-
seen.add(input);
|
|
2887
|
-
const next = {};
|
|
2888
|
-
for (const [key, entry] of Object.entries(input)) {
|
|
2889
|
-
if (allow.has(key)) {
|
|
2890
|
-
next[key] = entry;
|
|
2891
|
-
continue;
|
|
2892
|
-
}
|
|
2893
|
-
if (deny.has(key)) {
|
|
2894
|
-
if (mode === "mask")
|
|
2895
|
-
next[key] = maskValue;
|
|
2896
|
-
continue;
|
|
2897
|
-
}
|
|
2898
|
-
next[key] = walk(entry);
|
|
2899
|
-
}
|
|
2900
|
-
return next;
|
|
2901
|
-
}
|
|
2902
|
-
return input;
|
|
2903
|
-
};
|
|
2904
|
-
return walk(value);
|
|
2905
|
-
};
|
|
2906
|
-
var redactMediaReport = (report, options = {}) => {
|
|
2907
|
-
const limit = options.truncateArraysAt ?? DEFAULT_TRUNCATE;
|
|
2908
|
-
const truncated = truncateArrays(report, limit, new WeakSet);
|
|
2909
|
-
return applyRedaction(truncated, options, new WeakSet);
|
|
2910
|
-
};
|
|
2911
|
-
var buildArtifactPair = (report, summary, markdown, options) => {
|
|
2912
|
-
const jsonValue = options.redact ? redactMediaReport(report, options.redact) : report;
|
|
2913
|
-
return {
|
|
2914
|
-
json: JSON.stringify(jsonValue, null, 2),
|
|
2915
|
-
jsonValue,
|
|
2916
|
-
markdown,
|
|
2917
|
-
summary
|
|
2918
|
-
};
|
|
2919
|
-
};
|
|
2920
|
-
var buildMediaQualityArtifact = (report, options = {}) => buildArtifactPair(report, summarizeMediaQualityReport(report), renderMediaQualityMarkdown(report, options), options);
|
|
2921
|
-
var buildMediaTransportArtifact = (report, options = {}) => buildArtifactPair(report, summarizeMediaTransportReport(report), renderMediaTransportMarkdown(report, options), options);
|
|
2922
|
-
var buildMediaProcessorGraphArtifact = (report, options = {}) => buildArtifactPair(report, summarizeMediaProcessorGraphReport(report), renderMediaProcessorGraphMarkdown(report, options), options);
|
|
2923
|
-
var writeMediaArtifact = async (input) => {
|
|
2924
|
-
await mkdir(input.dir, { recursive: true });
|
|
2925
|
-
const jsonPath = join(input.dir, `${input.slug}.json`);
|
|
2926
|
-
const markdownPath = join(input.dir, `${input.slug}.md`);
|
|
2927
|
-
await Promise.all([
|
|
2928
|
-
writeFile(jsonPath, input.json, "utf8"),
|
|
2929
|
-
writeFile(markdownPath, input.markdown, "utf8")
|
|
2930
|
-
]);
|
|
2931
|
-
return {
|
|
2932
|
-
jsonPath,
|
|
2933
|
-
markdownPath,
|
|
2934
|
-
summary: input.summary
|
|
2935
|
-
};
|
|
2936
|
-
};
|
|
2937
|
-
|
|
2938
2011
|
// src/client/browserMedia.ts
|
|
2012
|
+
import {
|
|
2013
|
+
buildMediaWebRTCStatsReport,
|
|
2014
|
+
buildMediaWebRTCStreamContinuityReport,
|
|
2015
|
+
collectMediaWebRTCStats
|
|
2016
|
+
} from "@absolutejs/media";
|
|
2939
2017
|
var DEFAULT_BROWSER_MEDIA_PATH = "/api/voice/browser-media";
|
|
2940
2018
|
var DEFAULT_BROWSER_MEDIA_INTERVAL_MS = 5000;
|
|
2941
2019
|
var resolvePeerConnection = async (options) => options.peerConnection ?? await options.getPeerConnection?.() ?? null;
|