@absolutejs/voice 0.0.22-beta.319 → 0.0.22-beta.320
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/angular/index.js +413 -0
- package/dist/browserMediaRoutes.d.ts +61 -0
- package/dist/client/browserMedia.d.ts +8 -0
- package/dist/client/htmxBootstrap.js +172 -1
- package/dist/client/index.d.ts +2 -0
- package/dist/client/index.js +414 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +587 -404
- package/dist/react/index.js +413 -0
- package/dist/svelte/index.js +413 -0
- package/dist/testing/index.js +441 -28
- package/dist/trace.d.ts +1 -1
- package/dist/types.d.ts +19 -0
- package/dist/vue/index.js +413 -0
- package/package.json +1 -1
package/dist/angular/index.js
CHANGED
|
@@ -1387,6 +1387,412 @@ var serverMessageToAction = (message) => {
|
|
|
1387
1387
|
}
|
|
1388
1388
|
};
|
|
1389
1389
|
|
|
1390
|
+
// node_modules/@absolutejs/media/dist/index.js
|
|
1391
|
+
var formatLabel = (format) => `${format.container}/${format.encoding}/${String(format.sampleRateHz)}hz/${String(format.channels)}ch`;
|
|
1392
|
+
var formatMatches = (actual, expected) => actual.container === expected.container && actual.encoding === expected.encoding && actual.sampleRateHz === expected.sampleRateHz && actual.channels === expected.channels;
|
|
1393
|
+
var pushIssue = (issues, severity, code, message) => {
|
|
1394
|
+
issues.push({ code, message, severity });
|
|
1395
|
+
};
|
|
1396
|
+
var numericMetadata = (frame, key) => {
|
|
1397
|
+
const value = frame.metadata?.[key];
|
|
1398
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
1399
|
+
};
|
|
1400
|
+
var average = (values) => values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0) / values.length;
|
|
1401
|
+
var max = (values) => values.length === 0 ? undefined : Math.max(...values);
|
|
1402
|
+
var min = (values) => values.length === 0 ? undefined : Math.min(...values);
|
|
1403
|
+
var numericStat = (stat, key) => {
|
|
1404
|
+
const value = stat[key];
|
|
1405
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
1406
|
+
};
|
|
1407
|
+
var booleanStat = (stat, key) => {
|
|
1408
|
+
const value = stat[key];
|
|
1409
|
+
return typeof value === "boolean" ? value : undefined;
|
|
1410
|
+
};
|
|
1411
|
+
var stringStat = (stat, key) => {
|
|
1412
|
+
const value = stat[key];
|
|
1413
|
+
return typeof value === "string" ? value : undefined;
|
|
1414
|
+
};
|
|
1415
|
+
var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
|
|
1416
|
+
var normalizeWebRTCStat = (stat) => {
|
|
1417
|
+
const sample = {};
|
|
1418
|
+
for (const [key, value] of Object.entries(stat)) {
|
|
1419
|
+
if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") {
|
|
1420
|
+
sample[key] = value;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
return sample;
|
|
1424
|
+
};
|
|
1425
|
+
var buildMediaResamplingPlan = (input) => {
|
|
1426
|
+
const required = !formatMatches(input.inputFormat, input.outputFormat);
|
|
1427
|
+
return {
|
|
1428
|
+
inputFormat: input.inputFormat,
|
|
1429
|
+
outputFormat: input.outputFormat,
|
|
1430
|
+
ratio: input.outputFormat.sampleRateHz / input.inputFormat.sampleRateHz,
|
|
1431
|
+
required,
|
|
1432
|
+
status: input.inputFormat.container === input.outputFormat.container && input.inputFormat.encoding === input.outputFormat.encoding && input.inputFormat.channels === input.outputFormat.channels ? "pass" : "warn"
|
|
1433
|
+
};
|
|
1434
|
+
};
|
|
1435
|
+
var speechProbability = (frame) => {
|
|
1436
|
+
if (frame.metadata?.isSpeech === true) {
|
|
1437
|
+
return 1;
|
|
1438
|
+
}
|
|
1439
|
+
if (frame.metadata?.isSpeech === false) {
|
|
1440
|
+
return 0;
|
|
1441
|
+
}
|
|
1442
|
+
for (const key of ["speechProbability", "voiceProbability", "rms", "energy"]) {
|
|
1443
|
+
const value = numericMetadata(frame, key);
|
|
1444
|
+
if (value !== undefined) {
|
|
1445
|
+
return value;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
return 0;
|
|
1449
|
+
};
|
|
1450
|
+
var buildMediaVadReport = (input = {}) => {
|
|
1451
|
+
const frames = (input.frames ?? []).filter((frame) => frame.kind === "input-audio");
|
|
1452
|
+
const speechStartThreshold = input.speechStartThreshold ?? 0.6;
|
|
1453
|
+
const speechEndThreshold = input.speechEndThreshold ?? 0.35;
|
|
1454
|
+
const minSpeechFrames = input.minSpeechFrames ?? 1;
|
|
1455
|
+
const maxSilenceFrames = input.maxSilenceFrames ?? 1;
|
|
1456
|
+
const segments = [];
|
|
1457
|
+
let activeFrames = [];
|
|
1458
|
+
let silenceFrames = 0;
|
|
1459
|
+
const closeSegment = () => {
|
|
1460
|
+
if (activeFrames.length < minSpeechFrames) {
|
|
1461
|
+
activeFrames = [];
|
|
1462
|
+
silenceFrames = 0;
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
const first = activeFrames[0];
|
|
1466
|
+
const last = activeFrames.at(-1);
|
|
1467
|
+
if (!first) {
|
|
1468
|
+
return;
|
|
1469
|
+
}
|
|
1470
|
+
segments.push({
|
|
1471
|
+
durationMs: first.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined,
|
|
1472
|
+
endAt: last?.at !== undefined ? last.at + (last.durationMs ?? 0) : undefined,
|
|
1473
|
+
frameCount: activeFrames.length,
|
|
1474
|
+
segmentId: `vad:${String(segments.length + 1)}`,
|
|
1475
|
+
sessionId: first.sessionId,
|
|
1476
|
+
startAt: first.at,
|
|
1477
|
+
turnId: first.turnId
|
|
1478
|
+
});
|
|
1479
|
+
activeFrames = [];
|
|
1480
|
+
silenceFrames = 0;
|
|
1481
|
+
};
|
|
1482
|
+
for (const frame of frames) {
|
|
1483
|
+
const probability = speechProbability(frame);
|
|
1484
|
+
if (activeFrames.length === 0) {
|
|
1485
|
+
if (probability >= speechStartThreshold) {
|
|
1486
|
+
activeFrames.push(frame);
|
|
1487
|
+
}
|
|
1488
|
+
continue;
|
|
1489
|
+
}
|
|
1490
|
+
activeFrames.push(frame);
|
|
1491
|
+
if (probability <= speechEndThreshold) {
|
|
1492
|
+
silenceFrames += 1;
|
|
1493
|
+
} else {
|
|
1494
|
+
silenceFrames = 0;
|
|
1495
|
+
}
|
|
1496
|
+
if (silenceFrames > maxSilenceFrames) {
|
|
1497
|
+
closeSegment();
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
closeSegment();
|
|
1501
|
+
return {
|
|
1502
|
+
checkedAt: Date.now(),
|
|
1503
|
+
inputAudioFrames: frames.length,
|
|
1504
|
+
segments,
|
|
1505
|
+
status: frames.length === 0 ? "warn" : "pass"
|
|
1506
|
+
};
|
|
1507
|
+
};
|
|
1508
|
+
var buildMediaInterruptionReport = (input = {}) => {
|
|
1509
|
+
const issues = [];
|
|
1510
|
+
const interruptionFrames = (input.frames ?? []).filter((frame) => frame.kind === "interruption");
|
|
1511
|
+
const latenciesMs = interruptionFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
|
|
1512
|
+
const maxInterruptionLatencyMs = input.maxInterruptionLatencyMs;
|
|
1513
|
+
if (interruptionFrames.length === 0) {
|
|
1514
|
+
pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
|
|
1515
|
+
}
|
|
1516
|
+
if (maxInterruptionLatencyMs !== undefined && latenciesMs.some((latency) => latency > maxInterruptionLatencyMs)) {
|
|
1517
|
+
pushIssue(issues, "error", "media.interruption_latency", `Interruption latency exceeded ${String(maxInterruptionLatencyMs)}ms.`);
|
|
1518
|
+
}
|
|
1519
|
+
return {
|
|
1520
|
+
checkedAt: Date.now(),
|
|
1521
|
+
interruptionFrames: interruptionFrames.length,
|
|
1522
|
+
issues,
|
|
1523
|
+
latenciesMs,
|
|
1524
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass"
|
|
1525
|
+
};
|
|
1526
|
+
};
|
|
1527
|
+
var buildMediaQualityReport = (input = {}) => {
|
|
1528
|
+
const frames = [...input.frames ?? []].sort((a, b) => (a.at ?? 0) - (b.at ?? 0));
|
|
1529
|
+
const audioFrames = frames.filter((frame) => frame.kind === "input-audio" || frame.kind === "assistant-audio");
|
|
1530
|
+
const inputAudioFrames = frames.filter((frame) => frame.kind === "input-audio");
|
|
1531
|
+
const assistantAudioFrames = frames.filter((frame) => frame.kind === "assistant-audio");
|
|
1532
|
+
const issues = [];
|
|
1533
|
+
const gapsMs = [];
|
|
1534
|
+
for (const [index, frame] of audioFrames.entries()) {
|
|
1535
|
+
const previous = audioFrames[index - 1];
|
|
1536
|
+
if (previous?.at === undefined || frame.at === undefined || previous.durationMs === undefined) {
|
|
1537
|
+
continue;
|
|
1538
|
+
}
|
|
1539
|
+
const gap = frame.at - (previous.at + previous.durationMs);
|
|
1540
|
+
if (gap > 0) {
|
|
1541
|
+
gapsMs.push(gap);
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
const jitterMs = audioFrames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined).at(-1) ?? max(gapsMs);
|
|
1545
|
+
const first = audioFrames.find((frame) => frame.at !== undefined);
|
|
1546
|
+
const last = audioFrames.toReversed().find((frame) => frame.at !== undefined);
|
|
1547
|
+
const durationMs = first?.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined;
|
|
1548
|
+
const expectedDurationMs = audioFrames.length > 0 ? audioFrames.reduce((total, frame) => total + (frame.durationMs ?? 0), 0) : undefined;
|
|
1549
|
+
const timestampDriftMs = durationMs !== undefined && expectedDurationMs !== undefined ? Math.max(0, durationMs - expectedDurationMs) : undefined;
|
|
1550
|
+
const speechScores = inputAudioFrames.map(speechProbability);
|
|
1551
|
+
const speechFrames = speechScores.filter((score) => score >= 0.6).length;
|
|
1552
|
+
const silenceFrames = speechScores.filter((score) => score <= 0.35).length;
|
|
1553
|
+
const unknownSpeechFrames = Math.max(0, inputAudioFrames.length - speechFrames - silenceFrames);
|
|
1554
|
+
const speechRatio = inputAudioFrames.length === 0 ? 0 : speechFrames / inputAudioFrames.length;
|
|
1555
|
+
const silenceRatio = inputAudioFrames.length === 0 ? 0 : silenceFrames / inputAudioFrames.length;
|
|
1556
|
+
const levels = audioFrames.map((frame) => numericMetadata(frame, "level") ?? numericMetadata(frame, "rms") ?? numericMetadata(frame, "energy")).filter((value) => value !== undefined);
|
|
1557
|
+
const backpressureEvents = input.transport?.backpressureEvents ?? 0;
|
|
1558
|
+
const maxGapMs = input.maxGapMs;
|
|
1559
|
+
if (maxGapMs !== undefined && gapsMs.some((gap) => gap > maxGapMs)) {
|
|
1560
|
+
pushIssue(issues, "warning", "media.quality_gap", `Observed media gap above ${String(maxGapMs)}ms.`);
|
|
1561
|
+
}
|
|
1562
|
+
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
1563
|
+
pushIssue(issues, "warning", "media.quality_jitter", `Observed jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
1564
|
+
}
|
|
1565
|
+
if (input.maxTimestampDriftMs !== undefined && timestampDriftMs !== undefined && timestampDriftMs > input.maxTimestampDriftMs) {
|
|
1566
|
+
pushIssue(issues, "warning", "media.quality_timestamp_drift", `Observed timestamp drift ${String(timestampDriftMs)}ms above ${String(input.maxTimestampDriftMs)}ms.`);
|
|
1567
|
+
}
|
|
1568
|
+
if (input.minSpeechRatio !== undefined && inputAudioFrames.length > 0 && speechRatio < input.minSpeechRatio) {
|
|
1569
|
+
pushIssue(issues, "warning", "media.quality_speech_ratio", `Observed speech ratio ${String(speechRatio)} below ${String(input.minSpeechRatio)}.`);
|
|
1570
|
+
}
|
|
1571
|
+
if (input.maxBackpressureEvents !== undefined && backpressureEvents > input.maxBackpressureEvents) {
|
|
1572
|
+
pushIssue(issues, "warning", "media.quality_backpressure", `Observed ${String(backpressureEvents)} backpressure event(s), above ${String(input.maxBackpressureEvents)}.`);
|
|
1573
|
+
}
|
|
1574
|
+
return {
|
|
1575
|
+
assistantAudioFrames: assistantAudioFrames.length,
|
|
1576
|
+
backpressureEvents,
|
|
1577
|
+
checkedAt: Date.now(),
|
|
1578
|
+
durationMs,
|
|
1579
|
+
gapCount: gapsMs.length,
|
|
1580
|
+
gapsMs,
|
|
1581
|
+
inputAudioFrames: inputAudioFrames.length,
|
|
1582
|
+
issues,
|
|
1583
|
+
jitterMs,
|
|
1584
|
+
levelAverage: average(levels),
|
|
1585
|
+
levelMax: max(levels),
|
|
1586
|
+
levelMin: min(levels),
|
|
1587
|
+
silenceFrames,
|
|
1588
|
+
silenceRatio,
|
|
1589
|
+
speechFrames,
|
|
1590
|
+
speechRatio,
|
|
1591
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
1592
|
+
timestampDriftMs,
|
|
1593
|
+
totalFrames: frames.length,
|
|
1594
|
+
unknownSpeechFrames
|
|
1595
|
+
};
|
|
1596
|
+
};
|
|
1597
|
+
var buildMediaWebRTCStatsReport = (input = {}) => {
|
|
1598
|
+
const stats = input.stats ?? [];
|
|
1599
|
+
const issues = [];
|
|
1600
|
+
const inbound = stats.filter((stat) => stat.type === "inbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
1601
|
+
const outbound = stats.filter((stat) => stat.type === "outbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
1602
|
+
const candidatePairs = stats.filter((stat) => stat.type === "candidate-pair");
|
|
1603
|
+
const audioTracks = stats.filter((stat) => (stat.type === "track" || stat.type === "media-source") && stringStat(stat, "kind") === "audio");
|
|
1604
|
+
const activeCandidatePairs = candidatePairs.filter((stat) => booleanStat(stat, "selected") === true || booleanStat(stat, "nominated") === true || stringStat(stat, "state") === "succeeded").length;
|
|
1605
|
+
const liveAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") !== "ended" && stringStat(stat, "trackState") !== "ended" && booleanStat(stat, "ended") !== true).length;
|
|
1606
|
+
const endedAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") === "ended" || stringStat(stat, "trackState") === "ended" || booleanStat(stat, "ended") === true).length;
|
|
1607
|
+
const inboundPackets = inbound.reduce((total, stat) => total + (numericStat(stat, "packetsReceived") ?? 0), 0);
|
|
1608
|
+
const outboundPackets = outbound.reduce((total, stat) => total + (numericStat(stat, "packetsSent") ?? 0), 0);
|
|
1609
|
+
const packetsLost = [...inbound, ...outbound].reduce((total, stat) => total + Math.max(0, numericStat(stat, "packetsLost") ?? 0), 0);
|
|
1610
|
+
const packetLossDenominator = inboundPackets + packetsLost;
|
|
1611
|
+
const packetLossRatio = packetLossDenominator === 0 ? 0 : packetsLost / packetLossDenominator;
|
|
1612
|
+
const bytesReceived = inbound.reduce((total, stat) => total + (numericStat(stat, "bytesReceived") ?? 0), 0);
|
|
1613
|
+
const bytesSent = outbound.reduce((total, stat) => total + (numericStat(stat, "bytesSent") ?? 0), 0);
|
|
1614
|
+
const roundTripTimeMs = max(candidatePairs.map((stat) => secondsToMs(numericStat(stat, "currentRoundTripTime") ?? numericStat(stat, "roundTripTime"))).filter((value) => value !== undefined));
|
|
1615
|
+
const jitterMs = max([...inbound, ...outbound].map((stat) => secondsToMs(numericStat(stat, "jitter"))).filter((value) => value !== undefined));
|
|
1616
|
+
const jitterBufferDelayMs = max(inbound.map((stat) => {
|
|
1617
|
+
const delay = numericStat(stat, "jitterBufferDelay");
|
|
1618
|
+
const emitted = numericStat(stat, "jitterBufferEmittedCount");
|
|
1619
|
+
return delay !== undefined && emitted !== undefined && emitted > 0 ? delay / emitted * 1000 : undefined;
|
|
1620
|
+
}).filter((value) => value !== undefined));
|
|
1621
|
+
const audioLevels = audioTracks.map((stat) => numericStat(stat, "audioLevel")).filter((value) => value !== undefined);
|
|
1622
|
+
if (input.requireConnectedCandidatePair && candidatePairs.length > 0 && activeCandidatePairs === 0) {
|
|
1623
|
+
pushIssue(issues, "error", "media.webrtc_candidate_pair_missing", "No active WebRTC candidate pair was observed.");
|
|
1624
|
+
}
|
|
1625
|
+
if (input.requireLiveAudioTrack && liveAudioTracks === 0) {
|
|
1626
|
+
pushIssue(issues, "error", "media.webrtc_audio_track_missing", "No live WebRTC audio track was observed.");
|
|
1627
|
+
}
|
|
1628
|
+
if (input.maxPacketLossRatio !== undefined && packetLossRatio > input.maxPacketLossRatio) {
|
|
1629
|
+
pushIssue(issues, "warning", "media.webrtc_packet_loss", `Observed WebRTC packet loss ratio ${String(packetLossRatio)} above ${String(input.maxPacketLossRatio)}.`);
|
|
1630
|
+
}
|
|
1631
|
+
if (input.maxRoundTripTimeMs !== undefined && roundTripTimeMs !== undefined && roundTripTimeMs > input.maxRoundTripTimeMs) {
|
|
1632
|
+
pushIssue(issues, "warning", "media.webrtc_round_trip_time", `Observed WebRTC RTT ${String(roundTripTimeMs)}ms above ${String(input.maxRoundTripTimeMs)}ms.`);
|
|
1633
|
+
}
|
|
1634
|
+
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
1635
|
+
pushIssue(issues, "warning", "media.webrtc_jitter", `Observed WebRTC jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
1636
|
+
}
|
|
1637
|
+
return {
|
|
1638
|
+
activeCandidatePairs,
|
|
1639
|
+
audioLevelAverage: average(audioLevels),
|
|
1640
|
+
bytesReceived,
|
|
1641
|
+
bytesSent,
|
|
1642
|
+
checkedAt: Date.now(),
|
|
1643
|
+
endedAudioTracks,
|
|
1644
|
+
inboundPackets,
|
|
1645
|
+
issues,
|
|
1646
|
+
jitterBufferDelayMs,
|
|
1647
|
+
jitterMs,
|
|
1648
|
+
liveAudioTracks,
|
|
1649
|
+
outboundPackets,
|
|
1650
|
+
packetLossRatio,
|
|
1651
|
+
packetsLost,
|
|
1652
|
+
roundTripTimeMs,
|
|
1653
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
1654
|
+
totalStats: stats.length
|
|
1655
|
+
};
|
|
1656
|
+
};
|
|
1657
|
+
var collectMediaWebRTCStats = async (input) => {
|
|
1658
|
+
const report = await input.peerConnection.getStats(input.selector ?? null);
|
|
1659
|
+
return [...report.values()].map(normalizeWebRTCStat);
|
|
1660
|
+
};
|
|
1661
|
+
var collectMediaWebRTCStatsReport = async (input) => {
|
|
1662
|
+
const stats = await collectMediaWebRTCStats(input);
|
|
1663
|
+
return buildMediaWebRTCStatsReport({
|
|
1664
|
+
...input,
|
|
1665
|
+
stats
|
|
1666
|
+
});
|
|
1667
|
+
};
|
|
1668
|
+
var buildMediaPipelineCalibrationReport = (input = {}) => {
|
|
1669
|
+
const frames = input.frames ?? [];
|
|
1670
|
+
const issues = [];
|
|
1671
|
+
const inputFrames = frames.filter((frame) => frame.kind === "input-audio");
|
|
1672
|
+
const assistantFrames = frames.filter((frame) => frame.kind === "assistant-audio");
|
|
1673
|
+
const turnCommitFrames = frames.filter((frame) => frame.kind === "turn-commit");
|
|
1674
|
+
const interruptionFrameRecords = frames.filter((frame) => frame.kind === "interruption");
|
|
1675
|
+
const traceLinkedFrames = frames.filter((frame) => frame.traceEventId).length;
|
|
1676
|
+
const backpressureFrames = frames.filter((frame) => Boolean(frame.metadata?.backpressure)).length;
|
|
1677
|
+
const audioLatencies = assistantFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
|
|
1678
|
+
const firstAudioLatencyMs = audioLatencies.length > 0 ? Math.min(...audioLatencies) : undefined;
|
|
1679
|
+
const jitterValues = frames.map((frame) => numericMetadata(frame, "jitterMs")).filter((value) => value !== undefined);
|
|
1680
|
+
const jitterMs = jitterValues.length > 0 ? Math.max(...jitterValues) : undefined;
|
|
1681
|
+
const inputFormat = input.inputFormat ?? inputFrames.find((frame) => frame.format)?.format;
|
|
1682
|
+
const outputFormat = input.outputFormat ?? assistantFrames.find((frame) => frame.format)?.format;
|
|
1683
|
+
const resamplingRequired = Boolean(input.expectedInputFormat && inputFormat && inputFormat.sampleRateHz !== input.expectedInputFormat.sampleRateHz) || Boolean(input.expectedOutputFormat && outputFormat && outputFormat.sampleRateHz !== input.expectedOutputFormat.sampleRateHz);
|
|
1684
|
+
const resamplingTargetHz = resamplingRequired && input.expectedInputFormat ? input.expectedInputFormat.sampleRateHz : resamplingRequired ? input.expectedOutputFormat?.sampleRateHz : undefined;
|
|
1685
|
+
if (inputFrames.length === 0) {
|
|
1686
|
+
pushIssue(issues, "warning", "media.input_audio_missing", "No input audio frames were observed.");
|
|
1687
|
+
}
|
|
1688
|
+
if (assistantFrames.length === 0) {
|
|
1689
|
+
pushIssue(issues, "warning", "media.assistant_audio_missing", "No assistant audio frames were observed.");
|
|
1690
|
+
}
|
|
1691
|
+
if (input.expectedInputFormat && inputFormat && !formatMatches(inputFormat, input.expectedInputFormat)) {
|
|
1692
|
+
pushIssue(issues, inputFormat.sampleRateHz === input.expectedInputFormat.sampleRateHz ? "warning" : "error", "media.input_format_mismatch", `Input format ${formatLabel(inputFormat)} does not match expected ${formatLabel(input.expectedInputFormat)}.`);
|
|
1693
|
+
}
|
|
1694
|
+
if (input.expectedOutputFormat && outputFormat && !formatMatches(outputFormat, input.expectedOutputFormat)) {
|
|
1695
|
+
pushIssue(issues, outputFormat.sampleRateHz === input.expectedOutputFormat.sampleRateHz ? "warning" : "error", "media.output_format_mismatch", `Output format ${formatLabel(outputFormat)} does not match expected ${formatLabel(input.expectedOutputFormat)}.`);
|
|
1696
|
+
}
|
|
1697
|
+
if (firstAudioLatencyMs !== undefined && input.maxFirstAudioLatencyMs !== undefined && firstAudioLatencyMs > input.maxFirstAudioLatencyMs) {
|
|
1698
|
+
pushIssue(issues, "error", "media.first_audio_latency", `First audio latency ${String(firstAudioLatencyMs)}ms exceeds budget ${String(input.maxFirstAudioLatencyMs)}ms.`);
|
|
1699
|
+
}
|
|
1700
|
+
if (jitterMs !== undefined && input.maxJitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
1701
|
+
pushIssue(issues, "warning", "media.jitter", `Media jitter ${String(jitterMs)}ms exceeds budget ${String(input.maxJitterMs)}ms.`);
|
|
1702
|
+
}
|
|
1703
|
+
if (input.maxBackpressureFrames !== undefined && backpressureFrames > input.maxBackpressureFrames) {
|
|
1704
|
+
pushIssue(issues, "warning", "media.backpressure", `Backpressure frame count ${String(backpressureFrames)} exceeds budget ${String(input.maxBackpressureFrames)}.`);
|
|
1705
|
+
}
|
|
1706
|
+
if (input.requireInterruptionFrame && interruptionFrameRecords.length === 0) {
|
|
1707
|
+
pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
|
|
1708
|
+
}
|
|
1709
|
+
if (input.requireTraceEvidence && traceLinkedFrames === 0) {
|
|
1710
|
+
pushIssue(issues, "warning", "media.trace_evidence_missing", "No media frames were linked to trace evidence.");
|
|
1711
|
+
}
|
|
1712
|
+
return {
|
|
1713
|
+
assistantAudioFrames: assistantFrames.length,
|
|
1714
|
+
backpressureFrames,
|
|
1715
|
+
checkedAt: Date.now(),
|
|
1716
|
+
firstAudioLatencyMs,
|
|
1717
|
+
inputAudioFrames: inputFrames.length,
|
|
1718
|
+
inputFormat,
|
|
1719
|
+
interruptionFrames: interruptionFrameRecords.length,
|
|
1720
|
+
issues,
|
|
1721
|
+
jitterMs,
|
|
1722
|
+
outputFormat,
|
|
1723
|
+
resamplingRequired,
|
|
1724
|
+
resamplingTargetHz,
|
|
1725
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
1726
|
+
surface: input.surface ?? "media-pipeline",
|
|
1727
|
+
traceLinkedFrames,
|
|
1728
|
+
turnCommitFrames: turnCommitFrames.length
|
|
1729
|
+
};
|
|
1730
|
+
};
|
|
1731
|
+
|
|
1732
|
+
// src/client/browserMedia.ts
|
|
1733
|
+
var DEFAULT_BROWSER_MEDIA_PATH = "/api/voice/browser-media";
|
|
1734
|
+
var DEFAULT_BROWSER_MEDIA_INTERVAL_MS = 5000;
|
|
1735
|
+
var resolvePeerConnection = async (options) => options.peerConnection ?? await options.getPeerConnection?.() ?? null;
|
|
1736
|
+
var postBrowserMediaReport = async (payload, options) => {
|
|
1737
|
+
const requestFetch = options.fetch ?? globalThis.fetch;
|
|
1738
|
+
if (!requestFetch) {
|
|
1739
|
+
return;
|
|
1740
|
+
}
|
|
1741
|
+
await requestFetch(options.path ?? DEFAULT_BROWSER_MEDIA_PATH, {
|
|
1742
|
+
body: JSON.stringify(payload),
|
|
1743
|
+
headers: {
|
|
1744
|
+
"Content-Type": "application/json"
|
|
1745
|
+
},
|
|
1746
|
+
keepalive: true,
|
|
1747
|
+
method: "POST"
|
|
1748
|
+
});
|
|
1749
|
+
};
|
|
1750
|
+
var createVoiceBrowserMediaReporter = (options) => {
|
|
1751
|
+
let interval = null;
|
|
1752
|
+
const reportOnce = async () => {
|
|
1753
|
+
const peerConnection = await resolvePeerConnection(options);
|
|
1754
|
+
if (!peerConnection) {
|
|
1755
|
+
return;
|
|
1756
|
+
}
|
|
1757
|
+
const report = await collectMediaWebRTCStatsReport({
|
|
1758
|
+
...options,
|
|
1759
|
+
peerConnection
|
|
1760
|
+
});
|
|
1761
|
+
const payload = {
|
|
1762
|
+
at: Date.now(),
|
|
1763
|
+
report,
|
|
1764
|
+
scenarioId: options.getScenarioId?.() ?? null,
|
|
1765
|
+
sessionId: options.getSessionId?.() ?? null
|
|
1766
|
+
};
|
|
1767
|
+
options.onReport?.(payload);
|
|
1768
|
+
await postBrowserMediaReport(payload, options);
|
|
1769
|
+
return payload;
|
|
1770
|
+
};
|
|
1771
|
+
const run = () => {
|
|
1772
|
+
reportOnce().catch((error) => {
|
|
1773
|
+
options.onError?.(error);
|
|
1774
|
+
});
|
|
1775
|
+
};
|
|
1776
|
+
const stop = () => {
|
|
1777
|
+
if (interval) {
|
|
1778
|
+
clearInterval(interval);
|
|
1779
|
+
interval = null;
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
return {
|
|
1783
|
+
close: stop,
|
|
1784
|
+
reportOnce,
|
|
1785
|
+
start: () => {
|
|
1786
|
+
if (interval) {
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
run();
|
|
1790
|
+
interval = setInterval(run, options.intervalMs ?? DEFAULT_BROWSER_MEDIA_INTERVAL_MS);
|
|
1791
|
+
},
|
|
1792
|
+
stop
|
|
1793
|
+
};
|
|
1794
|
+
};
|
|
1795
|
+
|
|
1390
1796
|
// src/client/connection.ts
|
|
1391
1797
|
var WS_OPEN = 1;
|
|
1392
1798
|
var WS_CLOSED = 3;
|
|
@@ -1819,12 +2225,18 @@ var createVoiceStreamStore = () => {
|
|
|
1819
2225
|
var createVoiceStream = (path, options = {}) => {
|
|
1820
2226
|
const connection = createVoiceConnection(path, options);
|
|
1821
2227
|
const store = createVoiceStreamStore();
|
|
2228
|
+
const browserMediaReporter = options.browserMedia && typeof window !== "undefined" ? createVoiceBrowserMediaReporter({
|
|
2229
|
+
...options.browserMedia,
|
|
2230
|
+
getScenarioId: () => options.browserMedia ? options.browserMedia.getScenarioId?.() ?? connection.getScenarioId() : connection.getScenarioId(),
|
|
2231
|
+
getSessionId: () => options.browserMedia ? options.browserMedia.getSessionId?.() ?? connection.getSessionId() : connection.getSessionId()
|
|
2232
|
+
}) : null;
|
|
1822
2233
|
const subscribers = new Set;
|
|
1823
2234
|
const start = (input) => Promise.resolve().then(() => {
|
|
1824
2235
|
if (!input?.sessionId && !input?.scenarioId) {
|
|
1825
2236
|
return;
|
|
1826
2237
|
}
|
|
1827
2238
|
connection.start(input);
|
|
2239
|
+
browserMediaReporter?.start();
|
|
1828
2240
|
});
|
|
1829
2241
|
const notify = () => {
|
|
1830
2242
|
subscribers.forEach((subscriber) => subscriber());
|
|
@@ -1866,6 +2278,7 @@ var createVoiceStream = (path, options = {}) => {
|
|
|
1866
2278
|
},
|
|
1867
2279
|
close() {
|
|
1868
2280
|
unsubscribeConnection();
|
|
2281
|
+
browserMediaReporter?.close();
|
|
1869
2282
|
connection.close();
|
|
1870
2283
|
store.dispatch({ type: "disconnected" });
|
|
1871
2284
|
notify();
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { Elysia } from 'elysia';
|
|
2
|
+
import type { MediaWebRTCStatsReport } from '@absolutejs/media';
|
|
3
|
+
import type { VoiceTraceEventStore } from './trace';
|
|
4
|
+
export type VoiceBrowserMediaStatus = 'empty' | 'fail' | 'pass' | 'warn';
|
|
5
|
+
export type VoiceBrowserMediaSample = {
|
|
6
|
+
at: number;
|
|
7
|
+
report: MediaWebRTCStatsReport;
|
|
8
|
+
scenarioId?: string;
|
|
9
|
+
sessionId: string;
|
|
10
|
+
traceId?: string;
|
|
11
|
+
};
|
|
12
|
+
export type VoiceBrowserMediaReport = {
|
|
13
|
+
checkedAt: number;
|
|
14
|
+
latest?: VoiceBrowserMediaSample;
|
|
15
|
+
recent: VoiceBrowserMediaSample[];
|
|
16
|
+
status: VoiceBrowserMediaStatus;
|
|
17
|
+
stale: boolean;
|
|
18
|
+
total: number;
|
|
19
|
+
};
|
|
20
|
+
export type VoiceBrowserMediaRoutesOptions = {
|
|
21
|
+
headers?: HeadersInit;
|
|
22
|
+
htmlPath?: false | string;
|
|
23
|
+
maxAgeMs?: number;
|
|
24
|
+
name?: string;
|
|
25
|
+
path?: string;
|
|
26
|
+
store: VoiceTraceEventStore;
|
|
27
|
+
title?: string;
|
|
28
|
+
};
|
|
29
|
+
export declare const summarizeVoiceBrowserMedia: (options: Pick<VoiceBrowserMediaRoutesOptions, "maxAgeMs" | "store">) => Promise<VoiceBrowserMediaReport>;
|
|
30
|
+
export declare const getLatestVoiceBrowserMediaReport: (options: Pick<VoiceBrowserMediaRoutesOptions, "maxAgeMs" | "store">) => Promise<MediaWebRTCStatsReport | undefined>;
|
|
31
|
+
export declare const renderVoiceBrowserMediaHTML: (report: VoiceBrowserMediaReport, options?: {
|
|
32
|
+
title?: string;
|
|
33
|
+
}) => string;
|
|
34
|
+
export declare const createVoiceBrowserMediaRoutes: (options: VoiceBrowserMediaRoutesOptions) => Elysia<"", {
|
|
35
|
+
decorator: {};
|
|
36
|
+
store: {};
|
|
37
|
+
derive: {};
|
|
38
|
+
resolve: {};
|
|
39
|
+
}, {
|
|
40
|
+
typebox: {};
|
|
41
|
+
error: {};
|
|
42
|
+
}, {
|
|
43
|
+
schema: {};
|
|
44
|
+
standaloneSchema: {};
|
|
45
|
+
macro: {};
|
|
46
|
+
macroFn: {};
|
|
47
|
+
parser: {};
|
|
48
|
+
response: {};
|
|
49
|
+
}, {}, {
|
|
50
|
+
derive: {};
|
|
51
|
+
resolve: {};
|
|
52
|
+
schema: {};
|
|
53
|
+
standaloneSchema: {};
|
|
54
|
+
response: {};
|
|
55
|
+
}, {
|
|
56
|
+
derive: {};
|
|
57
|
+
resolve: {};
|
|
58
|
+
schema: {};
|
|
59
|
+
standaloneSchema: {};
|
|
60
|
+
response: {};
|
|
61
|
+
}>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { VoiceBrowserMediaReporterOptions, VoiceBrowserMediaReportPayload } from '../types';
|
|
2
|
+
export type VoiceBrowserMediaReporter = {
|
|
3
|
+
close: () => void;
|
|
4
|
+
reportOnce: () => Promise<VoiceBrowserMediaReportPayload | undefined>;
|
|
5
|
+
start: () => void;
|
|
6
|
+
stop: () => void;
|
|
7
|
+
};
|
|
8
|
+
export declare const createVoiceBrowserMediaReporter: (options: VoiceBrowserMediaReporterOptions) => VoiceBrowserMediaReporter;
|