@absolutejs/voice 0.0.22-beta.4 → 0.0.22-beta.400
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3537 -55
- package/dist/agent.d.ts +62 -0
- package/dist/agentSquadContract.d.ts +98 -0
- package/dist/angular/index.d.ts +19 -0
- package/dist/angular/index.js +4599 -1041
- package/dist/angular/voice-agent-squad-status.service.d.ts +12 -0
- package/dist/angular/voice-campaign-dialer-proof.service.d.ts +14 -0
- package/dist/angular/voice-controller.service.d.ts +1 -0
- package/dist/angular/voice-delivery-runtime.component.d.ts +17 -0
- package/dist/angular/voice-delivery-runtime.service.d.ts +16 -0
- package/dist/angular/voice-live-ops.service.d.ts +11 -0
- package/dist/angular/voice-ops-action-center.service.d.ts +13 -0
- package/dist/angular/voice-ops-status.component.d.ts +15 -0
- package/dist/angular/voice-ops-status.service.d.ts +12 -0
- package/dist/angular/voice-platform-coverage.service.d.ts +12 -0
- package/dist/angular/voice-profile-comparison.service.d.ts +12 -0
- package/dist/angular/voice-proof-trends.service.d.ts +12 -0
- package/dist/angular/voice-provider-capabilities.service.d.ts +12 -0
- package/dist/angular/voice-provider-contracts.service.d.ts +12 -0
- package/dist/angular/voice-provider-status.service.d.ts +12 -0
- package/dist/angular/voice-readiness-failures.service.d.ts +13 -0
- package/dist/angular/voice-routing-status.service.d.ts +11 -0
- package/dist/angular/voice-session-snapshot.service.d.ts +13 -0
- package/dist/angular/voice-stream.service.d.ts +4 -0
- package/dist/angular/voice-trace-timeline.service.d.ts +12 -0
- package/dist/angular/voice-turn-latency.service.d.ts +13 -0
- package/dist/angular/voice-turn-quality.service.d.ts +12 -0
- package/dist/angular/voice-workflow-status.service.d.ts +12 -0
- package/dist/assistantHealth.d.ts +81 -0
- package/dist/audit.d.ts +128 -0
- package/dist/auditDeliveryRoutes.d.ts +85 -0
- package/dist/auditExport.d.ts +34 -0
- package/dist/auditRoutes.d.ts +66 -0
- package/dist/auditSinks.d.ts +151 -0
- package/dist/bargeInRoutes.d.ts +56 -0
- package/dist/browserCallProfiles.d.ts +120 -0
- package/dist/browserMediaRoutes.d.ts +62 -0
- package/dist/campaign.d.ts +794 -0
- package/dist/campaignDialers.d.ts +111 -0
- package/dist/client/actions.d.ts +116 -0
- package/dist/client/agentSquadStatus.d.ts +37 -0
- package/dist/client/agentSquadStatusWidget.d.ts +24 -0
- package/dist/client/bargeInMonitor.d.ts +7 -0
- package/dist/client/browserMedia.d.ts +8 -0
- package/dist/client/campaignDialerProof.d.ts +23 -0
- package/dist/client/connection.d.ts +3 -0
- package/dist/client/deliveryRuntime.d.ts +34 -0
- package/dist/client/deliveryRuntimeWidget.d.ts +37 -0
- package/dist/client/duplex.d.ts +1 -1
- package/dist/client/htmxBootstrap.js +994 -16
- package/dist/client/index.d.ts +85 -0
- package/dist/client/index.js +9689 -10
- package/dist/client/liveOps.d.ts +22 -0
- package/dist/client/liveOpsWidget.d.ts +23 -0
- package/dist/client/liveTurnLatency.d.ts +41 -0
- package/dist/client/opsActionCenter.d.ts +54 -0
- package/dist/client/opsActionCenterWidget.d.ts +29 -0
- package/dist/client/opsActionHistory.d.ts +19 -0
- package/dist/client/opsActionHistoryWidget.d.ts +11 -0
- package/dist/client/opsStatus.d.ts +19 -0
- package/dist/client/opsStatusWidget.d.ts +40 -0
- package/dist/client/platformCoverage.d.ts +19 -0
- package/dist/client/platformCoverageWidget.d.ts +37 -0
- package/dist/client/profileComparison.d.ts +19 -0
- package/dist/client/profileComparisonWidget.d.ts +41 -0
- package/dist/client/profileSwitchRecommendation.d.ts +19 -0
- package/dist/client/profileSwitchRecommendationWidget.d.ts +12 -0
- package/dist/client/proofTrends.d.ts +19 -0
- package/dist/client/proofTrendsWidget.d.ts +37 -0
- package/dist/client/providerCapabilities.d.ts +19 -0
- package/dist/client/providerCapabilitiesWidget.d.ts +32 -0
- package/dist/client/providerContracts.d.ts +19 -0
- package/dist/client/providerContractsWidget.d.ts +37 -0
- package/dist/client/providerSimulationControls.d.ts +33 -0
- package/dist/client/providerSimulationControlsWidget.d.ts +20 -0
- package/dist/client/providerStatus.d.ts +19 -0
- package/dist/client/providerStatusWidget.d.ts +32 -0
- package/dist/client/readinessFailures.d.ts +19 -0
- package/dist/client/readinessFailuresWidget.d.ts +42 -0
- package/dist/client/routingStatus.d.ts +19 -0
- package/dist/client/routingStatusWidget.d.ts +32 -0
- package/dist/client/sessionSnapshot.d.ts +21 -0
- package/dist/client/sessionSnapshotWidget.d.ts +28 -0
- package/dist/client/traceTimeline.d.ts +19 -0
- package/dist/client/traceTimelineWidget.d.ts +36 -0
- package/dist/client/turnLatency.d.ts +22 -0
- package/dist/client/turnLatencyWidget.d.ts +33 -0
- package/dist/client/turnQuality.d.ts +19 -0
- package/dist/client/turnQualityWidget.d.ts +32 -0
- package/dist/client/workflowStatus.d.ts +19 -0
- package/dist/competitiveCoverage.d.ts +141 -0
- package/dist/dataControl.d.ts +180 -0
- package/dist/deliveryRuntime.d.ts +158 -0
- package/dist/deliverySinkRoutes.d.ts +117 -0
- package/dist/demoReadyRoutes.d.ts +98 -0
- package/dist/diagnosticsRoutes.d.ts +44 -0
- package/dist/evalRoutes.d.ts +219 -0
- package/dist/fileStore.d.ts +14 -2
- package/dist/guardrails.d.ts +128 -0
- package/dist/handoff.d.ts +54 -0
- package/dist/handoffHealth.d.ts +94 -0
- package/dist/incidentBundle.d.ts +116 -0
- package/dist/index.d.ts +172 -9
- package/dist/index.js +34332 -3644
- package/dist/latencySlo.d.ts +56 -0
- package/dist/liveLatency.d.ts +78 -0
- package/dist/liveOps.d.ts +190 -0
- package/dist/mediaPipelineRoutes.d.ts +117 -0
- package/dist/modelAdapters.d.ts +151 -0
- package/dist/observabilityExport.d.ts +481 -0
- package/dist/openaiTTS.d.ts +18 -0
- package/dist/operationsRecord.d.ts +351 -0
- package/dist/opsActionAuditRoutes.d.ts +99 -0
- package/dist/opsConsoleRoutes.d.ts +80 -0
- package/dist/opsRecovery.d.ts +137 -0
- package/dist/opsStatus.d.ts +76 -0
- package/dist/opsStatusRoutes.d.ts +33 -0
- package/dist/opsWebhook.d.ts +126 -0
- package/dist/outcomeContract.d.ts +146 -0
- package/dist/phoneAgent.d.ts +139 -0
- package/dist/phoneAgentProductionSmoke.d.ts +115 -0
- package/dist/platformCoverage.d.ts +91 -0
- package/dist/postCallAnalysis.d.ts +98 -0
- package/dist/postgresStore.d.ts +13 -2
- package/dist/productionReadiness.d.ts +691 -0
- package/dist/profileSwitchRecommendation.d.ts +350 -0
- package/dist/proofAssertions.d.ts +32 -0
- package/dist/proofRunner.d.ts +79 -0
- package/dist/proofTrends.d.ts +715 -0
- package/dist/providerAdapters.d.ts +48 -0
- package/dist/providerCapabilities.d.ts +92 -0
- package/dist/providerDecisionTraces.d.ts +130 -0
- package/dist/providerHealth.d.ts +79 -0
- package/dist/providerOrchestration.d.ts +109 -0
- package/dist/providerRouterTraces.d.ts +35 -0
- package/dist/providerRoutingContract.d.ts +71 -0
- package/dist/providerSlo.d.ts +142 -0
- package/dist/providerStackRecommendations.d.ts +187 -0
- package/dist/qualityRoutes.d.ts +76 -0
- package/dist/queue.d.ts +61 -0
- package/dist/react/VoiceAgentSquadStatus.d.ts +5 -0
- package/dist/react/VoiceDeliveryRuntime.d.ts +7 -0
- package/dist/react/VoiceOpsActionCenter.d.ts +5 -0
- package/dist/react/VoiceOpsStatus.d.ts +6 -0
- package/dist/react/VoicePlatformCoverage.d.ts +6 -0
- package/dist/react/VoiceProfileComparison.d.ts +6 -0
- package/dist/react/VoiceProfileSwitchRecommendation.d.ts +6 -0
- package/dist/react/VoiceProofTrends.d.ts +6 -0
- package/dist/react/VoiceProviderCapabilities.d.ts +6 -0
- package/dist/react/VoiceProviderContracts.d.ts +6 -0
- package/dist/react/VoiceProviderSimulationControls.d.ts +5 -0
- package/dist/react/VoiceProviderStatus.d.ts +6 -0
- package/dist/react/VoiceReadinessFailures.d.ts +6 -0
- package/dist/react/VoiceRoutingStatus.d.ts +6 -0
- package/dist/react/VoiceSessionSnapshot.d.ts +6 -0
- package/dist/react/VoiceTraceTimeline.d.ts +6 -0
- package/dist/react/VoiceTurnLatency.d.ts +6 -0
- package/dist/react/VoiceTurnQuality.d.ts +6 -0
- package/dist/react/index.d.ts +39 -0
- package/dist/react/index.js +10432 -14
- package/dist/react/useVoiceAgentSquadStatus.d.ts +8 -0
- package/dist/react/useVoiceCampaignDialerProof.d.ts +10 -0
- package/dist/react/useVoiceController.d.ts +4 -0
- package/dist/react/useVoiceDeliveryRuntime.d.ts +13 -0
- package/dist/react/useVoiceLiveOps.d.ts +9 -0
- package/dist/react/useVoiceOpsActionCenter.d.ts +11 -0
- package/dist/react/useVoiceOpsStatus.d.ts +8 -0
- package/dist/react/useVoicePlatformCoverage.d.ts +8 -0
- package/dist/react/useVoiceProfileComparison.d.ts +8 -0
- package/dist/react/useVoiceProfileSwitchRecommendation.d.ts +8 -0
- package/dist/react/useVoiceProofTrends.d.ts +8 -0
- package/dist/react/useVoiceProviderCapabilities.d.ts +8 -0
- package/dist/react/useVoiceProviderContracts.d.ts +8 -0
- package/dist/react/useVoiceProviderSimulationControls.d.ts +10 -0
- package/dist/react/useVoiceProviderStatus.d.ts +8 -0
- package/dist/react/useVoiceReadinessFailures.d.ts +8 -0
- package/dist/react/useVoiceRoutingStatus.d.ts +8 -0
- package/dist/react/useVoiceSessionSnapshot.d.ts +9 -0
- package/dist/react/useVoiceStream.d.ts +4 -0
- package/dist/react/useVoiceTraceTimeline.d.ts +8 -0
- package/dist/react/useVoiceTurnLatency.d.ts +9 -0
- package/dist/react/useVoiceTurnQuality.d.ts +8 -0
- package/dist/react/useVoiceWorkflowStatus.d.ts +8 -0
- package/dist/readinessProfiles.d.ts +38 -0
- package/dist/realtimeChannel.d.ts +136 -0
- package/dist/realtimeProviderContracts.d.ts +133 -0
- package/dist/reconnectContract.d.ts +88 -0
- package/dist/resilienceRoutes.d.ts +146 -0
- package/dist/sessionReplay.d.ts +187 -0
- package/dist/sessionSnapshot.d.ts +98 -0
- package/dist/simulationSuite.d.ts +143 -0
- package/dist/sloCalibration.d.ts +185 -0
- package/dist/sqliteStore.d.ts +13 -2
- package/dist/svelte/createVoiceAgentSquadStatus.d.ts +9 -0
- package/dist/svelte/createVoiceCampaignDialerProof.d.ts +9 -0
- package/dist/svelte/createVoiceDeliveryRuntime.d.ts +11 -0
- package/dist/svelte/createVoiceLiveOps.d.ts +13 -0
- package/dist/svelte/createVoiceOpsActionCenter.d.ts +10 -0
- package/dist/svelte/createVoiceOpsStatus.d.ts +9 -0
- package/dist/svelte/createVoicePlatformCoverage.d.ts +7 -0
- package/dist/svelte/createVoiceProfileComparison.d.ts +7 -0
- package/dist/svelte/createVoiceProofTrends.d.ts +7 -0
- package/dist/svelte/createVoiceProviderCapabilities.d.ts +10 -0
- package/dist/svelte/createVoiceProviderContracts.d.ts +10 -0
- package/dist/svelte/createVoiceProviderSimulationControls.d.ts +11 -0
- package/dist/svelte/createVoiceProviderStatus.d.ts +10 -0
- package/dist/svelte/createVoiceReadinessFailures.d.ts +7 -0
- package/dist/svelte/createVoiceRoutingStatus.d.ts +10 -0
- package/dist/svelte/createVoiceSessionSnapshot.d.ts +13 -0
- package/dist/svelte/createVoiceTraceTimeline.d.ts +10 -0
- package/dist/svelte/createVoiceTurnLatency.d.ts +11 -0
- package/dist/svelte/createVoiceTurnQuality.d.ts +10 -0
- package/dist/svelte/createVoiceWorkflowStatus.d.ts +8 -0
- package/dist/svelte/index.d.ts +20 -0
- package/dist/svelte/index.js +6188 -408
- package/dist/telephony/contract.d.ts +61 -0
- package/dist/telephony/matrix.d.ts +97 -0
- package/dist/telephony/plivo.d.ts +303 -0
- package/dist/telephony/security.d.ts +182 -0
- package/dist/telephony/telnyx.d.ts +291 -0
- package/dist/telephony/twilio.d.ts +136 -2
- package/dist/telephonyMediaRoutes.d.ts +72 -0
- package/dist/telephonyOutcome.d.ts +273 -0
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +9133 -2560
- package/dist/testing/ioProviderSimulator.d.ts +41 -0
- package/dist/testing/providerSimulator.d.ts +44 -0
- package/dist/testing/telephony.d.ts +25 -0
- package/dist/toolContract.d.ts +161 -0
- package/dist/toolRuntime.d.ts +50 -0
- package/dist/trace.d.ts +31 -1
- package/dist/traceDeliveryRoutes.d.ts +86 -0
- package/dist/traceTimeline.d.ts +97 -0
- package/dist/turnLatency.d.ts +95 -0
- package/dist/turnQuality.d.ts +94 -0
- package/dist/types.d.ts +240 -4
- package/dist/voiceMonitoring.d.ts +444 -0
- package/dist/vue/VoiceDeliveryRuntime.d.ts +30 -0
- package/dist/vue/VoiceOpsActionCenter.d.ts +13 -0
- package/dist/vue/VoiceOpsStatus.d.ts +30 -0
- package/dist/vue/VoicePlatformCoverage.d.ts +23 -0
- package/dist/vue/VoiceProofTrends.d.ts +21 -0
- package/dist/vue/VoiceProviderCapabilities.d.ts +51 -0
- package/dist/vue/VoiceProviderContracts.d.ts +21 -0
- package/dist/vue/VoiceProviderSimulationControls.d.ts +88 -0
- package/dist/vue/VoiceProviderStatus.d.ts +51 -0
- package/dist/vue/VoiceReadinessFailures.d.ts +21 -0
- package/dist/vue/VoiceRoutingStatus.d.ts +51 -0
- package/dist/vue/VoiceSessionSnapshot.d.ts +68 -0
- package/dist/vue/VoiceTurnLatency.d.ts +69 -0
- package/dist/vue/VoiceTurnQuality.d.ts +51 -0
- package/dist/vue/index.d.ts +34 -0
- package/dist/vue/index.js +9874 -31
- package/dist/vue/useVoiceAgentSquadStatus.d.ts +9 -0
- package/dist/vue/useVoiceCampaignDialerProof.d.ts +11 -0
- package/dist/vue/useVoiceController.d.ts +2 -1
- package/dist/vue/useVoiceDeliveryRuntime.d.ts +13 -0
- package/dist/vue/useVoiceLiveOps.d.ts +9 -0
- package/dist/vue/useVoiceOpsActionCenter.d.ts +11 -0
- package/dist/vue/useVoiceOpsStatus.d.ts +9 -0
- package/dist/vue/useVoicePlatformCoverage.d.ts +9 -0
- package/dist/vue/useVoiceProfileComparison.d.ts +9 -0
- package/dist/vue/useVoiceProofTrends.d.ts +9 -0
- package/dist/vue/useVoiceProviderCapabilities.d.ts +9 -0
- package/dist/vue/useVoiceProviderContracts.d.ts +9 -0
- package/dist/vue/useVoiceProviderSimulationControls.d.ts +24 -0
- package/dist/vue/useVoiceProviderStatus.d.ts +9 -0
- package/dist/vue/useVoiceReadinessFailures.d.ts +899 -0
- package/dist/vue/useVoiceRoutingStatus.d.ts +8 -0
- package/dist/vue/useVoiceSessionSnapshot.d.ts +10 -0
- package/dist/vue/useVoiceStream.d.ts +5 -1
- package/dist/vue/useVoiceTraceTimeline.d.ts +9 -0
- package/dist/vue/useVoiceTurnLatency.d.ts +10 -0
- package/dist/vue/useVoiceTurnQuality.d.ts +9 -0
- package/dist/vue/useVoiceWorkflowStatus.d.ts +9 -0
- package/dist/workflowContract.d.ts +91 -0
- package/package.json +4 -1
|
@@ -188,6 +188,17 @@ var serverMessageToAction = (message) => {
|
|
|
188
188
|
sessionId: message.sessionId,
|
|
189
189
|
type: "complete"
|
|
190
190
|
};
|
|
191
|
+
case "connection":
|
|
192
|
+
return {
|
|
193
|
+
reconnect: message.reconnect,
|
|
194
|
+
type: "connection"
|
|
195
|
+
};
|
|
196
|
+
case "call_lifecycle":
|
|
197
|
+
return {
|
|
198
|
+
event: message.event,
|
|
199
|
+
sessionId: message.sessionId,
|
|
200
|
+
type: "call_lifecycle"
|
|
201
|
+
};
|
|
191
202
|
case "error":
|
|
192
203
|
return {
|
|
193
204
|
message: normalizeErrorMessage(message.message),
|
|
@@ -203,9 +214,22 @@ var serverMessageToAction = (message) => {
|
|
|
203
214
|
transcript: message.transcript,
|
|
204
215
|
type: "partial"
|
|
205
216
|
};
|
|
217
|
+
case "replay":
|
|
218
|
+
return {
|
|
219
|
+
assistantTexts: message.assistantTexts,
|
|
220
|
+
call: message.call,
|
|
221
|
+
partial: message.partial,
|
|
222
|
+
scenarioId: message.scenarioId,
|
|
223
|
+
sessionId: message.sessionId,
|
|
224
|
+
sessionMetadata: message.sessionMetadata,
|
|
225
|
+
status: message.status,
|
|
226
|
+
turns: message.turns,
|
|
227
|
+
type: "replay"
|
|
228
|
+
};
|
|
206
229
|
case "session":
|
|
207
230
|
return {
|
|
208
231
|
sessionId: message.sessionId,
|
|
232
|
+
sessionMetadata: message.sessionMetadata,
|
|
209
233
|
scenarioId: message.scenarioId,
|
|
210
234
|
status: message.status,
|
|
211
235
|
type: "session"
|
|
@@ -220,6 +244,232 @@ var serverMessageToAction = (message) => {
|
|
|
220
244
|
}
|
|
221
245
|
};
|
|
222
246
|
|
|
247
|
+
// node_modules/@absolutejs/media/dist/index.js
|
|
248
|
+
var pushIssue = (issues, severity, code, message) => {
|
|
249
|
+
issues.push({ code, message, severity });
|
|
250
|
+
};
|
|
251
|
+
var average = (values) => values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0) / values.length;
|
|
252
|
+
var max = (values) => values.length === 0 ? undefined : Math.max(...values);
|
|
253
|
+
var numericStat = (stat, key) => {
|
|
254
|
+
const value = stat[key];
|
|
255
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
256
|
+
};
|
|
257
|
+
var booleanStat = (stat, key) => {
|
|
258
|
+
const value = stat[key];
|
|
259
|
+
return typeof value === "boolean" ? value : undefined;
|
|
260
|
+
};
|
|
261
|
+
var stringStat = (stat, key) => {
|
|
262
|
+
const value = stat[key];
|
|
263
|
+
return typeof value === "string" ? value : undefined;
|
|
264
|
+
};
|
|
265
|
+
var statKey = (stat) => String(stat.id ?? stringStat(stat, "ssrc") ?? numericStat(stat, "ssrc") ?? stringStat(stat, "trackIdentifier") ?? stringStat(stat, "mid") ?? "unknown");
|
|
266
|
+
var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
|
|
267
|
+
var normalizeWebRTCStat = (stat) => {
|
|
268
|
+
const sample = {};
|
|
269
|
+
for (const [key, value] of Object.entries(stat)) {
|
|
270
|
+
if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") {
|
|
271
|
+
sample[key] = value;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return sample;
|
|
275
|
+
};
|
|
276
|
+
var buildMediaWebRTCStatsReport = (input = {}) => {
|
|
277
|
+
const stats = input.stats ?? [];
|
|
278
|
+
const issues = [];
|
|
279
|
+
const inbound = stats.filter((stat) => stat.type === "inbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
280
|
+
const outbound = stats.filter((stat) => stat.type === "outbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
281
|
+
const candidatePairs = stats.filter((stat) => stat.type === "candidate-pair");
|
|
282
|
+
const audioTracks = stats.filter((stat) => (stat.type === "track" || stat.type === "media-source") && stringStat(stat, "kind") === "audio");
|
|
283
|
+
const activeCandidatePairs = candidatePairs.filter((stat) => booleanStat(stat, "selected") === true || booleanStat(stat, "nominated") === true || stringStat(stat, "state") === "succeeded").length;
|
|
284
|
+
const liveAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") !== "ended" && stringStat(stat, "trackState") !== "ended" && booleanStat(stat, "ended") !== true).length;
|
|
285
|
+
const endedAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") === "ended" || stringStat(stat, "trackState") === "ended" || booleanStat(stat, "ended") === true).length;
|
|
286
|
+
const inboundPackets = inbound.reduce((total, stat) => total + (numericStat(stat, "packetsReceived") ?? 0), 0);
|
|
287
|
+
const outboundPackets = outbound.reduce((total, stat) => total + (numericStat(stat, "packetsSent") ?? 0), 0);
|
|
288
|
+
const packetsLost = [...inbound, ...outbound].reduce((total, stat) => total + Math.max(0, numericStat(stat, "packetsLost") ?? 0), 0);
|
|
289
|
+
const packetLossDenominator = inboundPackets + packetsLost;
|
|
290
|
+
const packetLossRatio = packetLossDenominator === 0 ? 0 : packetsLost / packetLossDenominator;
|
|
291
|
+
const bytesReceived = inbound.reduce((total, stat) => total + (numericStat(stat, "bytesReceived") ?? 0), 0);
|
|
292
|
+
const bytesSent = outbound.reduce((total, stat) => total + (numericStat(stat, "bytesSent") ?? 0), 0);
|
|
293
|
+
const roundTripTimeMs = max(candidatePairs.map((stat) => secondsToMs(numericStat(stat, "currentRoundTripTime") ?? numericStat(stat, "roundTripTime"))).filter((value) => value !== undefined));
|
|
294
|
+
const jitterMs = max([...inbound, ...outbound].map((stat) => secondsToMs(numericStat(stat, "jitter"))).filter((value) => value !== undefined));
|
|
295
|
+
const jitterBufferDelayMs = max(inbound.map((stat) => {
|
|
296
|
+
const delay = numericStat(stat, "jitterBufferDelay");
|
|
297
|
+
const emitted = numericStat(stat, "jitterBufferEmittedCount");
|
|
298
|
+
return delay !== undefined && emitted !== undefined && emitted > 0 ? delay / emitted * 1000 : undefined;
|
|
299
|
+
}).filter((value) => value !== undefined));
|
|
300
|
+
const audioLevels = audioTracks.map((stat) => numericStat(stat, "audioLevel")).filter((value) => value !== undefined);
|
|
301
|
+
if (input.requireConnectedCandidatePair && candidatePairs.length > 0 && activeCandidatePairs === 0) {
|
|
302
|
+
pushIssue(issues, "error", "media.webrtc_candidate_pair_missing", "No active WebRTC candidate pair was observed.");
|
|
303
|
+
}
|
|
304
|
+
if (input.requireLiveAudioTrack && liveAudioTracks === 0) {
|
|
305
|
+
pushIssue(issues, "error", "media.webrtc_audio_track_missing", "No live WebRTC audio track was observed.");
|
|
306
|
+
}
|
|
307
|
+
if (input.maxPacketLossRatio !== undefined && packetLossRatio > input.maxPacketLossRatio) {
|
|
308
|
+
pushIssue(issues, "warning", "media.webrtc_packet_loss", `Observed WebRTC packet loss ratio ${String(packetLossRatio)} above ${String(input.maxPacketLossRatio)}.`);
|
|
309
|
+
}
|
|
310
|
+
if (input.maxRoundTripTimeMs !== undefined && roundTripTimeMs !== undefined && roundTripTimeMs > input.maxRoundTripTimeMs) {
|
|
311
|
+
pushIssue(issues, "warning", "media.webrtc_round_trip_time", `Observed WebRTC RTT ${String(roundTripTimeMs)}ms above ${String(input.maxRoundTripTimeMs)}ms.`);
|
|
312
|
+
}
|
|
313
|
+
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
314
|
+
pushIssue(issues, "warning", "media.webrtc_jitter", `Observed WebRTC jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
315
|
+
}
|
|
316
|
+
return {
|
|
317
|
+
activeCandidatePairs,
|
|
318
|
+
audioLevelAverage: average(audioLevels),
|
|
319
|
+
bytesReceived,
|
|
320
|
+
bytesSent,
|
|
321
|
+
checkedAt: Date.now(),
|
|
322
|
+
endedAudioTracks,
|
|
323
|
+
inboundPackets,
|
|
324
|
+
issues,
|
|
325
|
+
jitterBufferDelayMs,
|
|
326
|
+
jitterMs,
|
|
327
|
+
liveAudioTracks,
|
|
328
|
+
outboundPackets,
|
|
329
|
+
packetLossRatio,
|
|
330
|
+
packetsLost,
|
|
331
|
+
roundTripTimeMs,
|
|
332
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
333
|
+
totalStats: stats.length
|
|
334
|
+
};
|
|
335
|
+
};
|
|
336
|
+
var collectMediaWebRTCStats = async (input) => {
|
|
337
|
+
const report = await input.peerConnection.getStats(input.selector ?? null);
|
|
338
|
+
return [...report.values()].map(normalizeWebRTCStat);
|
|
339
|
+
};
|
|
340
|
+
var buildMediaWebRTCStreamContinuityReport = (input = {}) => {
|
|
341
|
+
const stats = input.stats ?? [];
|
|
342
|
+
const previousStats = input.previousStats ?? [];
|
|
343
|
+
const issues = [];
|
|
344
|
+
const previousByKey = new Map(previousStats.map((stat) => [statKey(stat), stat]));
|
|
345
|
+
const audioRtp = stats.filter((stat) => (stat.type === "inbound-rtp" || stat.type === "outbound-rtp") && stringStat(stat, "kind") !== "video" && stringStat(stat, "mediaType") !== "video");
|
|
346
|
+
const streams = audioRtp.map((stat) => {
|
|
347
|
+
const direction = stat.type === "outbound-rtp" ? "outbound" : "inbound";
|
|
348
|
+
const packetsKey = direction === "outbound" ? "packetsSent" : "packetsReceived";
|
|
349
|
+
const bytesKey = direction === "outbound" ? "bytesSent" : "bytesReceived";
|
|
350
|
+
const previous = previousByKey.get(statKey(stat));
|
|
351
|
+
const currentPackets = numericStat(stat, packetsKey);
|
|
352
|
+
const previousPackets = previous ? numericStat(previous, packetsKey) : undefined;
|
|
353
|
+
const currentBytes = numericStat(stat, bytesKey);
|
|
354
|
+
const previousBytes = previous ? numericStat(previous, bytesKey) : undefined;
|
|
355
|
+
const timeDeltaMs = stat.timestamp !== undefined && previous?.timestamp !== undefined ? stat.timestamp - previous.timestamp : undefined;
|
|
356
|
+
return {
|
|
357
|
+
bytesDelta: currentBytes !== undefined && previousBytes !== undefined ? currentBytes - previousBytes : undefined,
|
|
358
|
+
currentPackets,
|
|
359
|
+
direction,
|
|
360
|
+
id: statKey(stat),
|
|
361
|
+
packetDelta: currentPackets !== undefined && previousPackets !== undefined ? currentPackets - previousPackets : undefined,
|
|
362
|
+
previousPackets,
|
|
363
|
+
timeDeltaMs
|
|
364
|
+
};
|
|
365
|
+
});
|
|
366
|
+
const inbound = streams.filter((stream) => stream.direction === "inbound");
|
|
367
|
+
const outbound = streams.filter((stream) => stream.direction === "outbound");
|
|
368
|
+
const maxObservedGapMs = max(streams.map((stream) => stream.timeDeltaMs).filter((value) => value !== undefined));
|
|
369
|
+
const stalledInboundStreams = inbound.filter((stream) => input.maxInboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxInboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
370
|
+
const stalledOutboundStreams = outbound.filter((stream) => input.maxOutboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxOutboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
371
|
+
if (input.requireInboundAudio && inbound.length === 0) {
|
|
372
|
+
pushIssue(issues, "error", "media.webrtc_inbound_audio_missing", "No inbound WebRTC audio RTP stream was observed.");
|
|
373
|
+
}
|
|
374
|
+
if (input.requireOutboundAudio && outbound.length === 0) {
|
|
375
|
+
pushIssue(issues, "error", "media.webrtc_outbound_audio_missing", "No outbound WebRTC audio RTP stream was observed.");
|
|
376
|
+
}
|
|
377
|
+
if (input.maxGapMs !== undefined && maxObservedGapMs !== undefined && maxObservedGapMs > input.maxGapMs) {
|
|
378
|
+
pushIssue(issues, "warning", "media.webrtc_stream_gap", `Observed WebRTC stream sample gap ${String(maxObservedGapMs)}ms above ${String(input.maxGapMs)}ms.`);
|
|
379
|
+
}
|
|
380
|
+
if (stalledInboundStreams > 0) {
|
|
381
|
+
pushIssue(issues, "error", "media.webrtc_inbound_stalled", `${String(stalledInboundStreams)} inbound WebRTC audio stream(s) stopped receiving packets.`);
|
|
382
|
+
}
|
|
383
|
+
if (stalledOutboundStreams > 0) {
|
|
384
|
+
pushIssue(issues, "error", "media.webrtc_outbound_stalled", `${String(stalledOutboundStreams)} outbound WebRTC audio stream(s) stopped sending packets.`);
|
|
385
|
+
}
|
|
386
|
+
return {
|
|
387
|
+
checkedAt: Date.now(),
|
|
388
|
+
inboundAudioStreams: inbound.length,
|
|
389
|
+
issues,
|
|
390
|
+
maxObservedGapMs,
|
|
391
|
+
outboundAudioStreams: outbound.length,
|
|
392
|
+
stalledInboundStreams,
|
|
393
|
+
stalledOutboundStreams,
|
|
394
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
395
|
+
streams,
|
|
396
|
+
totalStats: stats.length
|
|
397
|
+
};
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
// src/client/browserMedia.ts
|
|
401
|
+
var DEFAULT_BROWSER_MEDIA_PATH = "/api/voice/browser-media";
|
|
402
|
+
var DEFAULT_BROWSER_MEDIA_INTERVAL_MS = 5000;
|
|
403
|
+
var resolvePeerConnection = async (options) => options.peerConnection ?? await options.getPeerConnection?.() ?? null;
|
|
404
|
+
var postBrowserMediaReport = async (payload, options) => {
|
|
405
|
+
const requestFetch = options.fetch ?? globalThis.fetch;
|
|
406
|
+
if (!requestFetch) {
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
await requestFetch(options.path ?? DEFAULT_BROWSER_MEDIA_PATH, {
|
|
410
|
+
body: JSON.stringify(payload),
|
|
411
|
+
headers: {
|
|
412
|
+
"Content-Type": "application/json"
|
|
413
|
+
},
|
|
414
|
+
keepalive: true,
|
|
415
|
+
method: "POST"
|
|
416
|
+
});
|
|
417
|
+
};
|
|
418
|
+
var createVoiceBrowserMediaReporter = (options) => {
|
|
419
|
+
let interval = null;
|
|
420
|
+
let previousStats = [];
|
|
421
|
+
const reportOnce = async () => {
|
|
422
|
+
const peerConnection = await resolvePeerConnection(options);
|
|
423
|
+
if (!peerConnection) {
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
const stats = await collectMediaWebRTCStats({ peerConnection });
|
|
427
|
+
const report = buildMediaWebRTCStatsReport({
|
|
428
|
+
...options,
|
|
429
|
+
stats
|
|
430
|
+
});
|
|
431
|
+
const continuity = options.continuity === false ? undefined : buildMediaWebRTCStreamContinuityReport({
|
|
432
|
+
...options.continuity,
|
|
433
|
+
previousStats,
|
|
434
|
+
stats
|
|
435
|
+
});
|
|
436
|
+
const payload = {
|
|
437
|
+
at: Date.now(),
|
|
438
|
+
continuity,
|
|
439
|
+
report,
|
|
440
|
+
scenarioId: options.getScenarioId?.() ?? null,
|
|
441
|
+
sessionId: options.getSessionId?.() ?? null
|
|
442
|
+
};
|
|
443
|
+
previousStats = stats;
|
|
444
|
+
options.onReport?.(payload);
|
|
445
|
+
await postBrowserMediaReport(payload, options);
|
|
446
|
+
return payload;
|
|
447
|
+
};
|
|
448
|
+
const run = () => {
|
|
449
|
+
reportOnce().catch((error) => {
|
|
450
|
+
options.onError?.(error);
|
|
451
|
+
});
|
|
452
|
+
};
|
|
453
|
+
const stop = () => {
|
|
454
|
+
if (interval) {
|
|
455
|
+
clearInterval(interval);
|
|
456
|
+
interval = null;
|
|
457
|
+
}
|
|
458
|
+
};
|
|
459
|
+
return {
|
|
460
|
+
close: stop,
|
|
461
|
+
reportOnce,
|
|
462
|
+
start: () => {
|
|
463
|
+
if (interval) {
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
run();
|
|
467
|
+
interval = setInterval(run, options.intervalMs ?? DEFAULT_BROWSER_MEDIA_INTERVAL_MS);
|
|
468
|
+
},
|
|
469
|
+
stop
|
|
470
|
+
};
|
|
471
|
+
};
|
|
472
|
+
|
|
223
473
|
// src/client/connection.ts
|
|
224
474
|
var WS_OPEN = 1;
|
|
225
475
|
var WS_CLOSED = 3;
|
|
@@ -231,7 +481,7 @@ var DEFAULT_SCENARIO_QUERY_PARAM = "scenarioId";
|
|
|
231
481
|
var noop = () => {};
|
|
232
482
|
var noopUnsubscribe = () => noop;
|
|
233
483
|
var NOOP_CONNECTION = {
|
|
234
|
-
|
|
484
|
+
callControl: noop,
|
|
235
485
|
close: noop,
|
|
236
486
|
endTurn: noop,
|
|
237
487
|
getReadyState: () => WS_CLOSED,
|
|
@@ -239,6 +489,7 @@ var NOOP_CONNECTION = {
|
|
|
239
489
|
getSessionId: () => "",
|
|
240
490
|
send: noop,
|
|
241
491
|
sendAudio: noop,
|
|
492
|
+
start: () => {},
|
|
242
493
|
subscribe: noopUnsubscribe
|
|
243
494
|
};
|
|
244
495
|
var createSessionId = () => crypto.randomUUID();
|
|
@@ -260,11 +511,14 @@ var isVoiceServerMessage = (value) => {
|
|
|
260
511
|
switch (value.type) {
|
|
261
512
|
case "audio":
|
|
262
513
|
case "assistant":
|
|
514
|
+
case "call_lifecycle":
|
|
263
515
|
case "complete":
|
|
516
|
+
case "connection":
|
|
264
517
|
case "error":
|
|
265
518
|
case "final":
|
|
266
519
|
case "partial":
|
|
267
520
|
case "pong":
|
|
521
|
+
case "replay":
|
|
268
522
|
case "session":
|
|
269
523
|
case "turn":
|
|
270
524
|
return true;
|
|
@@ -301,6 +555,9 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
301
555
|
sessionId: options.sessionId ?? createSessionId(),
|
|
302
556
|
ws: null
|
|
303
557
|
};
|
|
558
|
+
const emitConnection = (reconnect) => {
|
|
559
|
+
listeners.forEach((listener) => listener(reconnect));
|
|
560
|
+
};
|
|
304
561
|
const clearTimers = () => {
|
|
305
562
|
if (state.pingInterval) {
|
|
306
563
|
clearInterval(state.pingInterval);
|
|
@@ -323,9 +580,28 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
323
580
|
}
|
|
324
581
|
};
|
|
325
582
|
const scheduleReconnect = () => {
|
|
583
|
+
const nextAttemptAt = Date.now() + RECONNECT_DELAY_MS;
|
|
326
584
|
state.reconnectAttempts += 1;
|
|
585
|
+
emitConnection({
|
|
586
|
+
reconnect: {
|
|
587
|
+
attempts: state.reconnectAttempts,
|
|
588
|
+
lastDisconnectAt: Date.now(),
|
|
589
|
+
maxAttempts: maxReconnectAttempts,
|
|
590
|
+
nextAttemptAt,
|
|
591
|
+
status: "reconnecting"
|
|
592
|
+
},
|
|
593
|
+
type: "connection"
|
|
594
|
+
});
|
|
327
595
|
state.reconnectTimeout = setTimeout(() => {
|
|
328
596
|
if (state.reconnectAttempts > maxReconnectAttempts) {
|
|
597
|
+
emitConnection({
|
|
598
|
+
reconnect: {
|
|
599
|
+
attempts: state.reconnectAttempts,
|
|
600
|
+
maxAttempts: maxReconnectAttempts,
|
|
601
|
+
status: "exhausted"
|
|
602
|
+
},
|
|
603
|
+
type: "connection"
|
|
604
|
+
});
|
|
329
605
|
return;
|
|
330
606
|
}
|
|
331
607
|
connect();
|
|
@@ -335,9 +611,21 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
335
611
|
const ws = new WebSocket(buildWsUrl(path, state.sessionId, state.scenarioId));
|
|
336
612
|
ws.binaryType = "arraybuffer";
|
|
337
613
|
ws.onopen = () => {
|
|
614
|
+
const wasReconnecting = state.reconnectAttempts > 0;
|
|
338
615
|
state.isConnected = true;
|
|
339
|
-
state.reconnectAttempts = 0;
|
|
340
616
|
flushPendingMessages();
|
|
617
|
+
if (wasReconnecting) {
|
|
618
|
+
emitConnection({
|
|
619
|
+
reconnect: {
|
|
620
|
+
attempts: state.reconnectAttempts,
|
|
621
|
+
lastResumedAt: Date.now(),
|
|
622
|
+
maxAttempts: maxReconnectAttempts,
|
|
623
|
+
status: "resumed"
|
|
624
|
+
},
|
|
625
|
+
type: "connection"
|
|
626
|
+
});
|
|
627
|
+
state.reconnectAttempts = 0;
|
|
628
|
+
}
|
|
341
629
|
listeners.forEach((listener) => listener({
|
|
342
630
|
scenarioId: state.scenarioId ?? undefined,
|
|
343
631
|
sessionId: state.sessionId,
|
|
@@ -367,6 +655,16 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
367
655
|
const reconnectable = shouldReconnect && event.code !== WS_NORMAL_CLOSURE && state.reconnectAttempts < maxReconnectAttempts;
|
|
368
656
|
if (reconnectable) {
|
|
369
657
|
scheduleReconnect();
|
|
658
|
+
} else if (shouldReconnect && event.code !== WS_NORMAL_CLOSURE) {
|
|
659
|
+
emitConnection({
|
|
660
|
+
reconnect: {
|
|
661
|
+
attempts: state.reconnectAttempts,
|
|
662
|
+
lastDisconnectAt: Date.now(),
|
|
663
|
+
maxAttempts: maxReconnectAttempts,
|
|
664
|
+
status: "exhausted"
|
|
665
|
+
},
|
|
666
|
+
type: "connection"
|
|
667
|
+
});
|
|
370
668
|
}
|
|
371
669
|
};
|
|
372
670
|
state.ws = ws;
|
|
@@ -400,6 +698,12 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
400
698
|
const endTurn = () => {
|
|
401
699
|
send({ type: "end_turn" });
|
|
402
700
|
};
|
|
701
|
+
const callControl = (message) => {
|
|
702
|
+
send({
|
|
703
|
+
...message,
|
|
704
|
+
type: "call_control"
|
|
705
|
+
});
|
|
706
|
+
};
|
|
403
707
|
const close = () => {
|
|
404
708
|
clearTimers();
|
|
405
709
|
if (state.ws) {
|
|
@@ -417,7 +721,7 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
417
721
|
};
|
|
418
722
|
connect();
|
|
419
723
|
return {
|
|
420
|
-
|
|
724
|
+
callControl,
|
|
421
725
|
close,
|
|
422
726
|
endTurn,
|
|
423
727
|
getReadyState: () => state.ws?.readyState ?? WS_CLOSED,
|
|
@@ -425,18 +729,27 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
425
729
|
getSessionId: () => state.sessionId,
|
|
426
730
|
send,
|
|
427
731
|
sendAudio,
|
|
732
|
+
start,
|
|
428
733
|
subscribe
|
|
429
734
|
};
|
|
430
735
|
};
|
|
431
736
|
|
|
432
737
|
// src/client/store.ts
|
|
738
|
+
var createInitialReconnectState = () => ({
|
|
739
|
+
attempts: 0,
|
|
740
|
+
maxAttempts: 0,
|
|
741
|
+
status: "idle"
|
|
742
|
+
});
|
|
433
743
|
var createInitialState = () => ({
|
|
434
744
|
assistantAudio: [],
|
|
435
745
|
assistantTexts: [],
|
|
746
|
+
call: null,
|
|
436
747
|
error: null,
|
|
437
748
|
isConnected: false,
|
|
749
|
+
sessionMetadata: null,
|
|
438
750
|
scenarioId: null,
|
|
439
751
|
partial: "",
|
|
752
|
+
reconnect: createInitialReconnectState(),
|
|
440
753
|
sessionId: null,
|
|
441
754
|
status: "idle",
|
|
442
755
|
turns: []
|
|
@@ -476,10 +789,36 @@ var createVoiceStreamStore = () => {
|
|
|
476
789
|
status: "completed"
|
|
477
790
|
};
|
|
478
791
|
break;
|
|
792
|
+
case "call_lifecycle":
|
|
793
|
+
state = {
|
|
794
|
+
...state,
|
|
795
|
+
call: {
|
|
796
|
+
...state.call,
|
|
797
|
+
disposition: action.event.type === "end" ? action.event.disposition : state.call?.disposition,
|
|
798
|
+
endedAt: action.event.type === "end" ? action.event.at : state.call?.endedAt,
|
|
799
|
+
events: [...state.call?.events ?? [], action.event],
|
|
800
|
+
lastEventAt: action.event.at,
|
|
801
|
+
startedAt: state.call?.startedAt ?? action.event.at
|
|
802
|
+
},
|
|
803
|
+
sessionId: action.sessionId
|
|
804
|
+
};
|
|
805
|
+
break;
|
|
479
806
|
case "connected":
|
|
480
807
|
state = {
|
|
481
808
|
...state,
|
|
482
|
-
isConnected: true
|
|
809
|
+
isConnected: true,
|
|
810
|
+
reconnect: state.reconnect.status === "reconnecting" ? {
|
|
811
|
+
...state.reconnect,
|
|
812
|
+
lastResumedAt: Date.now(),
|
|
813
|
+
nextAttemptAt: undefined,
|
|
814
|
+
status: "resumed"
|
|
815
|
+
} : state.reconnect
|
|
816
|
+
};
|
|
817
|
+
break;
|
|
818
|
+
case "connection":
|
|
819
|
+
state = {
|
|
820
|
+
...state,
|
|
821
|
+
reconnect: action.reconnect
|
|
483
822
|
};
|
|
484
823
|
break;
|
|
485
824
|
case "disconnected":
|
|
@@ -507,6 +846,27 @@ var createVoiceStreamStore = () => {
|
|
|
507
846
|
partial: action.transcript.text
|
|
508
847
|
};
|
|
509
848
|
break;
|
|
849
|
+
case "replay":
|
|
850
|
+
state = {
|
|
851
|
+
...state,
|
|
852
|
+
assistantTexts: [...action.assistantTexts],
|
|
853
|
+
call: action.call ?? null,
|
|
854
|
+
error: null,
|
|
855
|
+
isConnected: action.status === "active",
|
|
856
|
+
partial: action.partial,
|
|
857
|
+
reconnect: state.reconnect.status === "reconnecting" ? {
|
|
858
|
+
...state.reconnect,
|
|
859
|
+
lastResumedAt: Date.now(),
|
|
860
|
+
nextAttemptAt: undefined,
|
|
861
|
+
status: "resumed"
|
|
862
|
+
} : state.reconnect,
|
|
863
|
+
scenarioId: action.scenarioId ?? state.scenarioId,
|
|
864
|
+
sessionId: action.sessionId,
|
|
865
|
+
sessionMetadata: action.sessionMetadata ?? state.sessionMetadata,
|
|
866
|
+
status: action.status,
|
|
867
|
+
turns: [...action.turns]
|
|
868
|
+
};
|
|
869
|
+
break;
|
|
510
870
|
case "session":
|
|
511
871
|
state = {
|
|
512
872
|
...state,
|
|
@@ -514,6 +874,7 @@ var createVoiceStreamStore = () => {
|
|
|
514
874
|
scenarioId: action.scenarioId ?? state.scenarioId,
|
|
515
875
|
isConnected: action.status === "active",
|
|
516
876
|
sessionId: action.sessionId,
|
|
877
|
+
sessionMetadata: action.sessionMetadata ?? state.sessionMetadata,
|
|
517
878
|
status: action.status
|
|
518
879
|
};
|
|
519
880
|
break;
|
|
@@ -544,26 +905,60 @@ var createVoiceStreamStore = () => {
|
|
|
544
905
|
var createVoiceStream = (path, options = {}) => {
|
|
545
906
|
const connection = createVoiceConnection(path, options);
|
|
546
907
|
const store = createVoiceStreamStore();
|
|
908
|
+
const browserMediaReporter = options.browserMedia && typeof window !== "undefined" ? createVoiceBrowserMediaReporter({
|
|
909
|
+
...options.browserMedia,
|
|
910
|
+
getScenarioId: () => options.browserMedia ? options.browserMedia.getScenarioId?.() ?? connection.getScenarioId() : connection.getScenarioId(),
|
|
911
|
+
getSessionId: () => options.browserMedia ? options.browserMedia.getSessionId?.() ?? connection.getSessionId() : connection.getSessionId()
|
|
912
|
+
}) : null;
|
|
547
913
|
const subscribers = new Set;
|
|
548
914
|
const start = (input) => Promise.resolve().then(() => {
|
|
549
915
|
if (!input?.sessionId && !input?.scenarioId) {
|
|
550
916
|
return;
|
|
551
917
|
}
|
|
552
918
|
connection.start(input);
|
|
919
|
+
browserMediaReporter?.start();
|
|
553
920
|
});
|
|
554
921
|
const notify = () => {
|
|
555
922
|
subscribers.forEach((subscriber) => subscriber());
|
|
556
923
|
};
|
|
924
|
+
const reportReconnect = () => {
|
|
925
|
+
if (!options.reconnectReportPath || typeof fetch === "undefined") {
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
const snapshot = store.getSnapshot();
|
|
929
|
+
const body = JSON.stringify({
|
|
930
|
+
at: Date.now(),
|
|
931
|
+
reconnect: snapshot.reconnect,
|
|
932
|
+
scenarioId: snapshot.scenarioId,
|
|
933
|
+
sessionId: connection.getSessionId(),
|
|
934
|
+
turnIds: snapshot.turns.map((turn) => turn.id)
|
|
935
|
+
});
|
|
936
|
+
fetch(options.reconnectReportPath, {
|
|
937
|
+
body,
|
|
938
|
+
headers: {
|
|
939
|
+
"Content-Type": "application/json"
|
|
940
|
+
},
|
|
941
|
+
keepalive: true,
|
|
942
|
+
method: "POST"
|
|
943
|
+
}).catch(() => {});
|
|
944
|
+
};
|
|
557
945
|
const unsubscribeConnection = connection.subscribe((message) => {
|
|
558
946
|
const action = serverMessageToAction(message);
|
|
559
947
|
if (action) {
|
|
560
948
|
store.dispatch(action);
|
|
949
|
+
if (message.type === "connection") {
|
|
950
|
+
reportReconnect();
|
|
951
|
+
}
|
|
561
952
|
notify();
|
|
562
953
|
}
|
|
563
954
|
});
|
|
564
955
|
return {
|
|
956
|
+
callControl(message) {
|
|
957
|
+
connection.callControl(message);
|
|
958
|
+
},
|
|
565
959
|
close() {
|
|
566
960
|
unsubscribeConnection();
|
|
961
|
+
browserMediaReporter?.close();
|
|
567
962
|
connection.close();
|
|
568
963
|
store.dispatch({ type: "disconnected" });
|
|
569
964
|
notify();
|
|
@@ -586,10 +981,16 @@ var createVoiceStream = (path, options = {}) => {
|
|
|
586
981
|
get scenarioId() {
|
|
587
982
|
return store.getSnapshot().scenarioId;
|
|
588
983
|
},
|
|
984
|
+
get sessionMetadata() {
|
|
985
|
+
return store.getSnapshot().sessionMetadata;
|
|
986
|
+
},
|
|
589
987
|
start,
|
|
590
988
|
get partial() {
|
|
591
989
|
return store.getSnapshot().partial;
|
|
592
990
|
},
|
|
991
|
+
get reconnect() {
|
|
992
|
+
return store.getSnapshot().reconnect;
|
|
993
|
+
},
|
|
593
994
|
get sessionId() {
|
|
594
995
|
return connection.getSessionId();
|
|
595
996
|
},
|
|
@@ -605,6 +1006,9 @@ var createVoiceStream = (path, options = {}) => {
|
|
|
605
1006
|
get assistantAudio() {
|
|
606
1007
|
return store.getSnapshot().assistantAudio;
|
|
607
1008
|
},
|
|
1009
|
+
get call() {
|
|
1010
|
+
return store.getSnapshot().call;
|
|
1011
|
+
},
|
|
608
1012
|
sendAudio(audio) {
|
|
609
1013
|
connection.sendAudio(audio);
|
|
610
1014
|
},
|
|
@@ -900,12 +1304,15 @@ var resolveVoiceRuntimePreset = (name = "default") => {
|
|
|
900
1304
|
var createInitialState2 = (stream) => ({
|
|
901
1305
|
assistantAudio: [...stream.assistantAudio],
|
|
902
1306
|
assistantTexts: [...stream.assistantTexts],
|
|
1307
|
+
call: stream.call,
|
|
903
1308
|
error: stream.error,
|
|
904
1309
|
isConnected: stream.isConnected,
|
|
905
1310
|
isRecording: false,
|
|
906
1311
|
partial: stream.partial,
|
|
1312
|
+
reconnect: stream.reconnect,
|
|
907
1313
|
recordingError: null,
|
|
908
1314
|
sessionId: stream.sessionId,
|
|
1315
|
+
sessionMetadata: stream.sessionMetadata,
|
|
909
1316
|
scenarioId: stream.scenarioId,
|
|
910
1317
|
status: stream.status,
|
|
911
1318
|
turns: [...stream.turns]
|
|
@@ -929,10 +1336,13 @@ var createVoiceController = (path, options = {}) => {
|
|
|
929
1336
|
...state,
|
|
930
1337
|
assistantAudio: [...stream.assistantAudio],
|
|
931
1338
|
assistantTexts: [...stream.assistantTexts],
|
|
1339
|
+
call: stream.call,
|
|
932
1340
|
error: stream.error,
|
|
933
1341
|
isConnected: stream.isConnected,
|
|
934
1342
|
partial: stream.partial,
|
|
1343
|
+
reconnect: stream.reconnect,
|
|
935
1344
|
sessionId: stream.sessionId,
|
|
1345
|
+
sessionMetadata: stream.sessionMetadata,
|
|
936
1346
|
scenarioId: stream.scenarioId,
|
|
937
1347
|
status: stream.status,
|
|
938
1348
|
turns: [...stream.turns]
|
|
@@ -956,7 +1366,13 @@ var createVoiceController = (path, options = {}) => {
|
|
|
956
1366
|
capture = createMicrophoneCapture({
|
|
957
1367
|
channelCount: options.capture?.channelCount ?? preset.capture.channelCount,
|
|
958
1368
|
onLevel: options.capture?.onLevel,
|
|
959
|
-
onAudio: (audio) =>
|
|
1369
|
+
onAudio: (audio) => {
|
|
1370
|
+
if (options.capture?.onAudio) {
|
|
1371
|
+
options.capture.onAudio(audio, stream.sendAudio);
|
|
1372
|
+
return;
|
|
1373
|
+
}
|
|
1374
|
+
stream.sendAudio(audio);
|
|
1375
|
+
},
|
|
960
1376
|
sampleRateHz: options.capture?.sampleRateHz ?? preset.capture.sampleRateHz
|
|
961
1377
|
});
|
|
962
1378
|
return capture;
|
|
@@ -1006,6 +1422,7 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1006
1422
|
bindHTMX(bindingOptions) {
|
|
1007
1423
|
return bindVoiceHTMX(stream, bindingOptions);
|
|
1008
1424
|
},
|
|
1425
|
+
callControl: (message) => stream.callControl(message),
|
|
1009
1426
|
close,
|
|
1010
1427
|
endTurn: () => stream.endTurn(),
|
|
1011
1428
|
get error() {
|
|
@@ -1025,10 +1442,16 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1025
1442
|
get recordingError() {
|
|
1026
1443
|
return state.recordingError;
|
|
1027
1444
|
},
|
|
1445
|
+
get reconnect() {
|
|
1446
|
+
return state.reconnect;
|
|
1447
|
+
},
|
|
1028
1448
|
sendAudio: (audio) => stream.sendAudio(audio),
|
|
1029
1449
|
get sessionId() {
|
|
1030
1450
|
return state.sessionId;
|
|
1031
1451
|
},
|
|
1452
|
+
get sessionMetadata() {
|
|
1453
|
+
return state.sessionMetadata;
|
|
1454
|
+
},
|
|
1032
1455
|
get scenarioId() {
|
|
1033
1456
|
return state.scenarioId;
|
|
1034
1457
|
},
|
|
@@ -1058,6 +1481,478 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1058
1481
|
},
|
|
1059
1482
|
get assistantAudio() {
|
|
1060
1483
|
return state.assistantAudio;
|
|
1484
|
+
},
|
|
1485
|
+
get call() {
|
|
1486
|
+
return state.call;
|
|
1487
|
+
}
|
|
1488
|
+
};
|
|
1489
|
+
};
|
|
1490
|
+
|
|
1491
|
+
// src/client/audioPlayer.ts
|
|
1492
|
+
var DEFAULT_LOOKAHEAD_MS = 15;
|
|
1493
|
+
var createInitialState3 = () => ({
|
|
1494
|
+
activeSourceCount: 0,
|
|
1495
|
+
error: null,
|
|
1496
|
+
isActive: false,
|
|
1497
|
+
isPlaying: false,
|
|
1498
|
+
lastInterruptLatencyMs: undefined,
|
|
1499
|
+
lastPlaybackStopLatencyMs: undefined,
|
|
1500
|
+
processedChunkCount: 0,
|
|
1501
|
+
queuedChunkCount: 0
|
|
1502
|
+
});
|
|
1503
|
+
var getAudioContextCtor = () => {
|
|
1504
|
+
if (typeof window === "undefined") {
|
|
1505
|
+
return typeof AudioContext === "undefined" ? undefined : AudioContext;
|
|
1506
|
+
}
|
|
1507
|
+
return window.AudioContext ?? window.webkitAudioContext;
|
|
1508
|
+
};
|
|
1509
|
+
var decodePCM16LEChunk = (audioContext, chunk) => {
|
|
1510
|
+
const format = chunk.format;
|
|
1511
|
+
if (format.container !== "raw" || format.encoding !== "pcm_s16le") {
|
|
1512
|
+
throw new Error(`Unsupported assistant audio format: ${format.container}/${format.encoding}`);
|
|
1513
|
+
}
|
|
1514
|
+
const bytes = chunk.chunk;
|
|
1515
|
+
const channels = Math.max(1, format.channels);
|
|
1516
|
+
const sampleCount = Math.floor(bytes.byteLength / 2);
|
|
1517
|
+
const frameCount = Math.max(1, Math.floor(sampleCount / channels));
|
|
1518
|
+
const audioBuffer = audioContext.createBuffer(channels, frameCount, format.sampleRateHz);
|
|
1519
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1520
|
+
for (let channelIndex = 0;channelIndex < channels; channelIndex += 1) {
|
|
1521
|
+
const channelData = audioBuffer.getChannelData(channelIndex);
|
|
1522
|
+
for (let frameIndex = 0;frameIndex < frameCount; frameIndex += 1) {
|
|
1523
|
+
const sampleIndex = frameIndex * channels + channelIndex;
|
|
1524
|
+
const sampleOffset = sampleIndex * 2;
|
|
1525
|
+
if (sampleOffset + 1 >= bytes.byteLength) {
|
|
1526
|
+
channelData[frameIndex] = 0;
|
|
1527
|
+
continue;
|
|
1528
|
+
}
|
|
1529
|
+
channelData[frameIndex] = view.getInt16(sampleOffset, true) / 32768;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
return audioBuffer;
|
|
1533
|
+
};
|
|
1534
|
+
var createVoiceAudioPlayer = (source, options = {}) => {
|
|
1535
|
+
const subscribers = new Set;
|
|
1536
|
+
const sourceNodes = new Set;
|
|
1537
|
+
const lookaheadSeconds = (options.lookaheadMs ?? DEFAULT_LOOKAHEAD_MS) / 1000;
|
|
1538
|
+
let state = createInitialState3();
|
|
1539
|
+
let audioContext = null;
|
|
1540
|
+
let outputNode = null;
|
|
1541
|
+
let queueEndTime = 0;
|
|
1542
|
+
let syncPromise = Promise.resolve();
|
|
1543
|
+
let interruptStartedAt = null;
|
|
1544
|
+
let interruptPromise = null;
|
|
1545
|
+
let resolveInterruptPromise = null;
|
|
1546
|
+
let interruptFallbackTimer = null;
|
|
1547
|
+
const notify = () => {
|
|
1548
|
+
for (const subscriber of subscribers) {
|
|
1549
|
+
subscriber();
|
|
1550
|
+
}
|
|
1551
|
+
};
|
|
1552
|
+
const setState = (next) => {
|
|
1553
|
+
state = {
|
|
1554
|
+
...state,
|
|
1555
|
+
...next
|
|
1556
|
+
};
|
|
1557
|
+
notify();
|
|
1558
|
+
};
|
|
1559
|
+
const clearError = () => {
|
|
1560
|
+
if (state.error !== null) {
|
|
1561
|
+
setState({ error: null });
|
|
1562
|
+
}
|
|
1563
|
+
};
|
|
1564
|
+
const clearInterruptTimer = () => {
|
|
1565
|
+
if (interruptFallbackTimer !== null) {
|
|
1566
|
+
clearTimeout(interruptFallbackTimer);
|
|
1567
|
+
interruptFallbackTimer = null;
|
|
1568
|
+
}
|
|
1569
|
+
};
|
|
1570
|
+
const resolveInterrupt = (latencyMs) => {
|
|
1571
|
+
clearInterruptTimer();
|
|
1572
|
+
interruptStartedAt = null;
|
|
1573
|
+
setState({
|
|
1574
|
+
activeSourceCount: sourceNodes.size,
|
|
1575
|
+
isPlaying: false,
|
|
1576
|
+
lastInterruptLatencyMs: latencyMs,
|
|
1577
|
+
lastPlaybackStopLatencyMs: state.lastPlaybackStopLatencyMs ?? latencyMs
|
|
1578
|
+
});
|
|
1579
|
+
resolveInterruptPromise?.();
|
|
1580
|
+
resolveInterruptPromise = null;
|
|
1581
|
+
interruptPromise = null;
|
|
1582
|
+
};
|
|
1583
|
+
const estimateOutputStopLatencyMs = (context) => {
|
|
1584
|
+
if (!context) {
|
|
1585
|
+
return 0;
|
|
1586
|
+
}
|
|
1587
|
+
return Math.max(0, ((context.baseLatency ?? 0) + (context.outputLatency ?? 0)) * 1000);
|
|
1588
|
+
};
|
|
1589
|
+
const restoreOutputGain = (context) => {
|
|
1590
|
+
if (!outputNode) {
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
const gainValue = 1;
|
|
1594
|
+
if (outputNode.gain.setValueAtTime) {
|
|
1595
|
+
outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
|
|
1596
|
+
return;
|
|
1597
|
+
}
|
|
1598
|
+
outputNode.gain.value = gainValue;
|
|
1599
|
+
};
|
|
1600
|
+
const muteOutputGain = (context) => {
|
|
1601
|
+
if (!outputNode) {
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
const gainValue = 0;
|
|
1605
|
+
if (outputNode.gain.setValueAtTime) {
|
|
1606
|
+
outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
1609
|
+
outputNode.gain.value = gainValue;
|
|
1610
|
+
};
|
|
1611
|
+
const maybeResolveInterrupt = () => {
|
|
1612
|
+
if (interruptStartedAt === null || sourceNodes.size > 0) {
|
|
1613
|
+
return;
|
|
1614
|
+
}
|
|
1615
|
+
resolveInterrupt(Date.now() - interruptStartedAt);
|
|
1616
|
+
};
|
|
1617
|
+
const ensureAudioContext = async () => {
|
|
1618
|
+
if (audioContext) {
|
|
1619
|
+
return audioContext;
|
|
1620
|
+
}
|
|
1621
|
+
if (options.createAudioContext) {
|
|
1622
|
+
audioContext = options.createAudioContext();
|
|
1623
|
+
} else {
|
|
1624
|
+
const AudioContextCtor = getAudioContextCtor();
|
|
1625
|
+
if (!AudioContextCtor) {
|
|
1626
|
+
throw new Error("Assistant audio playback requires AudioContext support.");
|
|
1627
|
+
}
|
|
1628
|
+
audioContext = new AudioContextCtor;
|
|
1629
|
+
}
|
|
1630
|
+
if (audioContext.createGain) {
|
|
1631
|
+
outputNode = audioContext.createGain();
|
|
1632
|
+
outputNode.connect?.(audioContext.destination);
|
|
1633
|
+
}
|
|
1634
|
+
queueEndTime = audioContext.currentTime;
|
|
1635
|
+
return audioContext;
|
|
1636
|
+
};
|
|
1637
|
+
const scheduleChunk = async (chunk) => {
|
|
1638
|
+
const context = await ensureAudioContext();
|
|
1639
|
+
const buffer = decodePCM16LEChunk(context, chunk);
|
|
1640
|
+
const node = context.createBufferSource();
|
|
1641
|
+
node.buffer = buffer;
|
|
1642
|
+
node.connect(outputNode ?? context.destination);
|
|
1643
|
+
node.onended = () => {
|
|
1644
|
+
sourceNodes.delete(node);
|
|
1645
|
+
node.disconnect?.();
|
|
1646
|
+
setState({
|
|
1647
|
+
activeSourceCount: sourceNodes.size,
|
|
1648
|
+
isPlaying: sourceNodes.size > 0 && state.isActive
|
|
1649
|
+
});
|
|
1650
|
+
maybeResolveInterrupt();
|
|
1651
|
+
};
|
|
1652
|
+
const startAt = Math.max(context.currentTime + lookaheadSeconds, queueEndTime);
|
|
1653
|
+
queueEndTime = startAt + buffer.duration;
|
|
1654
|
+
sourceNodes.add(node);
|
|
1655
|
+
setState({
|
|
1656
|
+
activeSourceCount: sourceNodes.size,
|
|
1657
|
+
isPlaying: true
|
|
1658
|
+
});
|
|
1659
|
+
node.start(startAt);
|
|
1660
|
+
};
|
|
1661
|
+
const stopQueuedPlayback = (options2) => {
|
|
1662
|
+
for (const node of [...sourceNodes]) {
|
|
1663
|
+
node.stop?.();
|
|
1664
|
+
}
|
|
1665
|
+
queueEndTime = audioContext ? audioContext.currentTime : 0;
|
|
1666
|
+
if (options2?.forceClear) {
|
|
1667
|
+
for (const node of sourceNodes) {
|
|
1668
|
+
node.disconnect?.();
|
|
1669
|
+
}
|
|
1670
|
+
sourceNodes.clear();
|
|
1671
|
+
maybeResolveInterrupt();
|
|
1672
|
+
}
|
|
1673
|
+
};
|
|
1674
|
+
const sync = async () => {
|
|
1675
|
+
if (!state.isActive) {
|
|
1676
|
+
return;
|
|
1677
|
+
}
|
|
1678
|
+
const nextChunks = source.assistantAudio.slice(state.processedChunkCount);
|
|
1679
|
+
if (nextChunks.length === 0) {
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1682
|
+
try {
|
|
1683
|
+
clearError();
|
|
1684
|
+
for (const chunk of nextChunks) {
|
|
1685
|
+
await scheduleChunk(chunk);
|
|
1686
|
+
}
|
|
1687
|
+
setState({
|
|
1688
|
+
processedChunkCount: source.assistantAudio.length,
|
|
1689
|
+
queuedChunkCount: state.queuedChunkCount + nextChunks.length
|
|
1690
|
+
});
|
|
1691
|
+
} catch (error) {
|
|
1692
|
+
setState({
|
|
1693
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
};
|
|
1697
|
+
const queueSync = () => {
|
|
1698
|
+
syncPromise = syncPromise.then(() => sync(), () => sync());
|
|
1699
|
+
return syncPromise;
|
|
1700
|
+
};
|
|
1701
|
+
const unsubscribeSource = source.subscribe(() => {
|
|
1702
|
+
if (options.autoStart && !state.isActive && source.assistantAudio.length > 0) {
|
|
1703
|
+
player.start();
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
if (state.isActive) {
|
|
1707
|
+
queueSync();
|
|
1708
|
+
}
|
|
1709
|
+
});
|
|
1710
|
+
const player = {
|
|
1711
|
+
close: async () => {
|
|
1712
|
+
unsubscribeSource();
|
|
1713
|
+
stopQueuedPlayback({ forceClear: true });
|
|
1714
|
+
clearInterruptTimer();
|
|
1715
|
+
resolveInterruptPromise?.();
|
|
1716
|
+
resolveInterruptPromise = null;
|
|
1717
|
+
interruptPromise = null;
|
|
1718
|
+
interruptStartedAt = null;
|
|
1719
|
+
if (audioContext && audioContext.state !== "closed") {
|
|
1720
|
+
await audioContext.close();
|
|
1721
|
+
}
|
|
1722
|
+
audioContext = null;
|
|
1723
|
+
outputNode?.disconnect?.();
|
|
1724
|
+
outputNode = null;
|
|
1725
|
+
queueEndTime = 0;
|
|
1726
|
+
setState({
|
|
1727
|
+
activeSourceCount: 0,
|
|
1728
|
+
isActive: false,
|
|
1729
|
+
isPlaying: false
|
|
1730
|
+
});
|
|
1731
|
+
},
|
|
1732
|
+
get activeSourceCount() {
|
|
1733
|
+
return state.activeSourceCount;
|
|
1734
|
+
},
|
|
1735
|
+
get error() {
|
|
1736
|
+
return state.error;
|
|
1737
|
+
},
|
|
1738
|
+
getSnapshot: () => state,
|
|
1739
|
+
get isActive() {
|
|
1740
|
+
return state.isActive;
|
|
1741
|
+
},
|
|
1742
|
+
get isPlaying() {
|
|
1743
|
+
return state.isPlaying;
|
|
1744
|
+
},
|
|
1745
|
+
interrupt: async () => {
|
|
1746
|
+
const startedAt = Date.now();
|
|
1747
|
+
const context = await ensureAudioContext();
|
|
1748
|
+
interruptStartedAt = startedAt;
|
|
1749
|
+
muteOutputGain(context);
|
|
1750
|
+
const playbackStopLatencyMs = Date.now() - startedAt + estimateOutputStopLatencyMs(context);
|
|
1751
|
+
setState({
|
|
1752
|
+
isActive: false,
|
|
1753
|
+
isPlaying: sourceNodes.size > 0,
|
|
1754
|
+
lastPlaybackStopLatencyMs: playbackStopLatencyMs
|
|
1755
|
+
});
|
|
1756
|
+
if (sourceNodes.size === 0) {
|
|
1757
|
+
resolveInterrupt(playbackStopLatencyMs);
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
if (!interruptPromise) {
|
|
1761
|
+
interruptPromise = new Promise((resolve) => {
|
|
1762
|
+
resolveInterruptPromise = resolve;
|
|
1763
|
+
});
|
|
1764
|
+
}
|
|
1765
|
+
clearInterruptTimer();
|
|
1766
|
+
interruptFallbackTimer = setTimeout(() => {
|
|
1767
|
+
for (const node of sourceNodes) {
|
|
1768
|
+
node.disconnect?.();
|
|
1769
|
+
}
|
|
1770
|
+
sourceNodes.clear();
|
|
1771
|
+
resolveInterrupt(Date.now() - startedAt);
|
|
1772
|
+
}, 250);
|
|
1773
|
+
stopQueuedPlayback();
|
|
1774
|
+
await interruptPromise;
|
|
1775
|
+
},
|
|
1776
|
+
get lastInterruptLatencyMs() {
|
|
1777
|
+
return state.lastInterruptLatencyMs;
|
|
1778
|
+
},
|
|
1779
|
+
get lastPlaybackStopLatencyMs() {
|
|
1780
|
+
return state.lastPlaybackStopLatencyMs;
|
|
1781
|
+
},
|
|
1782
|
+
pause: async () => {
|
|
1783
|
+
if (!audioContext) {
|
|
1784
|
+
setState({
|
|
1785
|
+
activeSourceCount: 0,
|
|
1786
|
+
isActive: false,
|
|
1787
|
+
isPlaying: false
|
|
1788
|
+
});
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
await audioContext.suspend();
|
|
1792
|
+
setState({
|
|
1793
|
+
activeSourceCount: sourceNodes.size,
|
|
1794
|
+
isActive: false,
|
|
1795
|
+
isPlaying: false
|
|
1796
|
+
});
|
|
1797
|
+
},
|
|
1798
|
+
get processedChunkCount() {
|
|
1799
|
+
return state.processedChunkCount;
|
|
1800
|
+
},
|
|
1801
|
+
get queuedChunkCount() {
|
|
1802
|
+
return state.queuedChunkCount;
|
|
1803
|
+
},
|
|
1804
|
+
start: async () => {
|
|
1805
|
+
try {
|
|
1806
|
+
clearError();
|
|
1807
|
+
const context = await ensureAudioContext();
|
|
1808
|
+
restoreOutputGain(context);
|
|
1809
|
+
if (context.state === "suspended") {
|
|
1810
|
+
await context.resume();
|
|
1811
|
+
}
|
|
1812
|
+
setState({
|
|
1813
|
+
activeSourceCount: sourceNodes.size,
|
|
1814
|
+
isActive: true,
|
|
1815
|
+
isPlaying: context.state === "running"
|
|
1816
|
+
});
|
|
1817
|
+
await queueSync();
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
setState({
|
|
1820
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1821
|
+
isActive: false,
|
|
1822
|
+
isPlaying: false
|
|
1823
|
+
});
|
|
1824
|
+
throw error;
|
|
1825
|
+
}
|
|
1826
|
+
},
|
|
1827
|
+
subscribe: (subscriber) => {
|
|
1828
|
+
subscribers.add(subscriber);
|
|
1829
|
+
return () => {
|
|
1830
|
+
subscribers.delete(subscriber);
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
};
|
|
1834
|
+
return player;
|
|
1835
|
+
};
|
|
1836
|
+
|
|
1837
|
+
// src/client/bargeInMonitor.ts
|
|
1838
|
+
var DEFAULT_THRESHOLD_MS = 250;
|
|
1839
|
+
var createEventId = () => `barge-in:${Date.now()}:${crypto.randomUUID?.() ?? Math.random().toString(36).slice(2)}`;
|
|
1840
|
+
var summarize = (events, thresholdMs) => {
|
|
1841
|
+
const stopped = events.filter((event) => event.status === "stopped");
|
|
1842
|
+
const latencies = stopped.map((event) => event.latencyMs).filter((value) => typeof value === "number");
|
|
1843
|
+
const failed = stopped.filter((event) => typeof event.latencyMs === "number" && event.latencyMs > thresholdMs).length;
|
|
1844
|
+
const passed = stopped.length - failed;
|
|
1845
|
+
return {
|
|
1846
|
+
averageLatencyMs: latencies.length > 0 ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : undefined,
|
|
1847
|
+
events: [...events],
|
|
1848
|
+
failed,
|
|
1849
|
+
lastEvent: events.at(-1),
|
|
1850
|
+
passed,
|
|
1851
|
+
status: events.length === 0 ? "empty" : failed > 0 ? "fail" : stopped.length === 0 ? "warn" : "pass",
|
|
1852
|
+
thresholdMs,
|
|
1853
|
+
total: stopped.length
|
|
1854
|
+
};
|
|
1855
|
+
};
|
|
1856
|
+
var createVoiceBargeInMonitor = (options = {}) => {
|
|
1857
|
+
const listeners = new Set;
|
|
1858
|
+
const thresholdMs = options.thresholdMs ?? DEFAULT_THRESHOLD_MS;
|
|
1859
|
+
const fetchImpl = options.fetch ?? globalThis.fetch;
|
|
1860
|
+
const events = [];
|
|
1861
|
+
const emit = () => {
|
|
1862
|
+
for (const listener of listeners) {
|
|
1863
|
+
listener();
|
|
1864
|
+
}
|
|
1865
|
+
};
|
|
1866
|
+
const postEvent = (event) => {
|
|
1867
|
+
if (!options.path || typeof fetchImpl !== "function") {
|
|
1868
|
+
return;
|
|
1869
|
+
}
|
|
1870
|
+
fetchImpl(options.path, {
|
|
1871
|
+
body: JSON.stringify(event),
|
|
1872
|
+
headers: {
|
|
1873
|
+
"Content-Type": "application/json"
|
|
1874
|
+
},
|
|
1875
|
+
method: "POST"
|
|
1876
|
+
}).catch(() => {});
|
|
1877
|
+
};
|
|
1878
|
+
const record = (status, input) => {
|
|
1879
|
+
const event = {
|
|
1880
|
+
at: Date.now(),
|
|
1881
|
+
id: createEventId(),
|
|
1882
|
+
latencyMs: input.latencyMs,
|
|
1883
|
+
playbackStopLatencyMs: input.playbackStopLatencyMs,
|
|
1884
|
+
reason: input.reason,
|
|
1885
|
+
sessionId: input.sessionId,
|
|
1886
|
+
status,
|
|
1887
|
+
thresholdMs
|
|
1888
|
+
};
|
|
1889
|
+
events.push(event);
|
|
1890
|
+
postEvent(event);
|
|
1891
|
+
emit();
|
|
1892
|
+
return event;
|
|
1893
|
+
};
|
|
1894
|
+
return {
|
|
1895
|
+
getSnapshot: () => summarize(events, thresholdMs),
|
|
1896
|
+
recordRequested: (input) => record("requested", input),
|
|
1897
|
+
recordSkipped: (input) => record("skipped", input),
|
|
1898
|
+
recordStopped: (input) => record("stopped", input),
|
|
1899
|
+
subscribe: (subscriber) => {
|
|
1900
|
+
listeners.add(subscriber);
|
|
1901
|
+
return () => {
|
|
1902
|
+
listeners.delete(subscriber);
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
};
|
|
1906
|
+
};
|
|
1907
|
+
|
|
1908
|
+
// src/client/duplex.ts
|
|
1909
|
+
var DEFAULT_INTERRUPT_THRESHOLD = 0.08;
|
|
1910
|
+
var shouldInterruptForLevel = (level, options = {}) => (options.enabled ?? true) && level >= (options.interruptThreshold ?? DEFAULT_INTERRUPT_THRESHOLD);
|
|
1911
|
+
var bindVoiceBargeIn = (controller, player, options = {}) => {
|
|
1912
|
+
let lastPartial = controller.partial;
|
|
1913
|
+
const interruptIfPlaying = (reason) => {
|
|
1914
|
+
if (!player.isPlaying || options.enabled === false) {
|
|
1915
|
+
options.monitor?.recordSkipped({
|
|
1916
|
+
reason,
|
|
1917
|
+
sessionId: controller.sessionId
|
|
1918
|
+
});
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
options.monitor?.recordRequested({
|
|
1922
|
+
reason,
|
|
1923
|
+
sessionId: controller.sessionId
|
|
1924
|
+
});
|
|
1925
|
+
player.interrupt().then(() => {
|
|
1926
|
+
options.monitor?.recordStopped({
|
|
1927
|
+
latencyMs: player.lastInterruptLatencyMs,
|
|
1928
|
+
playbackStopLatencyMs: player.lastPlaybackStopLatencyMs,
|
|
1929
|
+
reason,
|
|
1930
|
+
sessionId: controller.sessionId
|
|
1931
|
+
});
|
|
1932
|
+
});
|
|
1933
|
+
};
|
|
1934
|
+
const unsubscribe = controller.subscribe(() => {
|
|
1935
|
+
if (options.interruptOnPartial === false) {
|
|
1936
|
+
lastPartial = controller.partial;
|
|
1937
|
+
return;
|
|
1938
|
+
}
|
|
1939
|
+
if (!lastPartial && controller.partial) {
|
|
1940
|
+
interruptIfPlaying("partial-transcript");
|
|
1941
|
+
}
|
|
1942
|
+
lastPartial = controller.partial;
|
|
1943
|
+
});
|
|
1944
|
+
return {
|
|
1945
|
+
close: () => {
|
|
1946
|
+
unsubscribe();
|
|
1947
|
+
},
|
|
1948
|
+
handleLevel: (level) => {
|
|
1949
|
+
if (shouldInterruptForLevel(level, options)) {
|
|
1950
|
+
interruptIfPlaying("input-level");
|
|
1951
|
+
}
|
|
1952
|
+
},
|
|
1953
|
+
sendAudio: (audio) => {
|
|
1954
|
+
interruptIfPlaying("manual-audio");
|
|
1955
|
+
controller.sendAudio(audio);
|
|
1061
1956
|
}
|
|
1062
1957
|
};
|
|
1063
1958
|
};
|
|
@@ -1084,7 +1979,7 @@ var DEFAULT_GUIDED_PROMPTS = [
|
|
|
1084
1979
|
"Now describe what you are trying to do or test.",
|
|
1085
1980
|
"Finish with any detail that feels blocked, risky, or unclear."
|
|
1086
1981
|
];
|
|
1087
|
-
var clamp = (value, min,
|
|
1982
|
+
var clamp = (value, min, max2) => Math.min(max2, Math.max(min, value));
|
|
1088
1983
|
var escapeHtml = (value) => value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
1089
1984
|
var readErrorField = (value, key) => {
|
|
1090
1985
|
const candidate = value[key];
|
|
@@ -1118,6 +2013,17 @@ var formatErrorMessage = (error) => {
|
|
|
1118
2013
|
}
|
|
1119
2014
|
return "Unexpected error";
|
|
1120
2015
|
};
|
|
2016
|
+
var formatReconnectState = (reconnect) => {
|
|
2017
|
+
const pieces = [reconnect.status];
|
|
2018
|
+
if (reconnect.attempts > 0 || reconnect.maxAttempts > 0) {
|
|
2019
|
+
pieces.push(`${reconnect.attempts}/${reconnect.maxAttempts} attempts`);
|
|
2020
|
+
}
|
|
2021
|
+
if (reconnect.nextAttemptAt) {
|
|
2022
|
+
const waitMs = Math.max(0, reconnect.nextAttemptAt - Date.now());
|
|
2023
|
+
pieces.push(`retry in ${Math.ceil(waitMs / 100) / 10}s`);
|
|
2024
|
+
}
|
|
2025
|
+
return pieces.join(" · ");
|
|
2026
|
+
};
|
|
1121
2027
|
var createInitialVoiceWaveLevels = (count = VOICE_WAVE_POINTS) => Array.from({ length: count }, () => 0);
|
|
1122
2028
|
var pushVoiceWaveLevel = (levels, nextLevel, count = VOICE_WAVE_POINTS) => {
|
|
1123
2029
|
const next = levels.slice(-(count - 1));
|
|
@@ -1174,6 +2080,17 @@ var parsePromptList = (value) => {
|
|
|
1174
2080
|
} catch {}
|
|
1175
2081
|
return DEFAULT_GUIDED_PROMPTS;
|
|
1176
2082
|
};
|
|
2083
|
+
var parseOptionalNumber = (value) => {
|
|
2084
|
+
if (!value) {
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
const parsed = Number(value);
|
|
2088
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
2089
|
+
};
|
|
2090
|
+
var resolveElement2 = (root, selector, ctor) => {
|
|
2091
|
+
const value = selector ? document.querySelector(selector) : root.querySelector(selector ?? "");
|
|
2092
|
+
return value instanceof ctor ? value : null;
|
|
2093
|
+
};
|
|
1177
2094
|
var requireElement = (root, selector, ctor, name) => {
|
|
1178
2095
|
const value = selector ? document.querySelector(selector) : null;
|
|
1179
2096
|
if (value instanceof ctor) {
|
|
@@ -1224,11 +2141,20 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1224
2141
|
const guidedPrompts = parsePromptList(root.dataset.voiceGuidedPrompts);
|
|
1225
2142
|
const guidedLabel = root.dataset.voiceGuidedLabel ?? DEFAULT_GUIDED_LABEL;
|
|
1226
2143
|
const generalLabel = root.dataset.voiceGeneralLabel ?? DEFAULT_GENERAL_LABEL;
|
|
2144
|
+
const reconnectReportPath = root.dataset.voiceReconnectReportPath;
|
|
2145
|
+
const bargeInPath = root.dataset.voiceBargeInPath;
|
|
2146
|
+
const bargeInMonitor = bargeInPath ? createVoiceBargeInMonitor({
|
|
2147
|
+
path: bargeInPath,
|
|
2148
|
+
thresholdMs: parseOptionalNumber(root.dataset.voiceBargeInThresholdMs)
|
|
2149
|
+
}) : null;
|
|
2150
|
+
const bargeInRecentWindowMs = parseOptionalNumber(root.dataset.voiceBargeInRecentWindowMs) ?? 4000;
|
|
2151
|
+
const bargeInSpeechThreshold = parseOptionalNumber(root.dataset.voiceBargeInSpeechThreshold) ?? 0.04;
|
|
1227
2152
|
const syncElement = requireElement(document, root.dataset.voiceSync, HTMLElement, "voice-htmx-sync");
|
|
1228
2153
|
const connectionMetric = requireElement(root, root.dataset.voiceConnection, HTMLElement, "metric-connection");
|
|
1229
2154
|
const errorStatus = requireElement(root, root.dataset.voiceError, HTMLElement, "status-error");
|
|
1230
2155
|
const microphoneStatus = requireElement(root, root.dataset.voiceMicrophone, HTMLElement, "status-mic");
|
|
1231
2156
|
const promptStatus = requireElement(root, root.dataset.voicePrompt, HTMLElement, "status-prompt");
|
|
2157
|
+
const reconnectStatus = resolveElement2(root, root.dataset.voiceReconnect, HTMLElement);
|
|
1232
2158
|
const chatList = requireElement(root, root.dataset.voiceChat, HTMLElement, "chat-list");
|
|
1233
2159
|
const startGuidedButton = requireElement(root, root.dataset.voiceStartGuided, HTMLButtonElement, "start-guided");
|
|
1234
2160
|
const startGeneralButton = requireElement(root, root.dataset.voiceStartGeneral, HTMLButtonElement, "start-general");
|
|
@@ -1237,35 +2163,70 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1237
2163
|
const voiceMonitorCopy = requireElement(root, root.dataset.voiceMonitorCopy, HTMLElement, "voice-monitor-copy");
|
|
1238
2164
|
const voiceWaveGlow = requireElement(root, root.dataset.voiceWaveGlow, SVGPathElement, "voice-wave-glow");
|
|
1239
2165
|
const voiceWavePath = requireElement(root, root.dataset.voiceWavePath, SVGPathElement, "voice-wave-path");
|
|
2166
|
+
let activeMode = null;
|
|
2167
|
+
let hasStartedModes = {
|
|
2168
|
+
general: false,
|
|
2169
|
+
guided: false
|
|
2170
|
+
};
|
|
2171
|
+
let isCapturing = false;
|
|
2172
|
+
let micError = null;
|
|
2173
|
+
let waveLevels = createInitialVoiceWaveLevels();
|
|
2174
|
+
let guidedBargeInBinding = null;
|
|
2175
|
+
let generalBargeInBinding = null;
|
|
1240
2176
|
const guidedVoice = createVoiceController(guidedPath, {
|
|
1241
2177
|
capture: {
|
|
2178
|
+
onAudio: (audio, sendAudio) => {
|
|
2179
|
+
if (guidedBargeInBinding) {
|
|
2180
|
+
guidedBargeInBinding.sendAudio(audio);
|
|
2181
|
+
return;
|
|
2182
|
+
}
|
|
2183
|
+
sendAudio(audio);
|
|
2184
|
+
},
|
|
1242
2185
|
onLevel: (level) => {
|
|
2186
|
+
guidedBargeInBinding?.handleLevel(level);
|
|
1243
2187
|
waveLevels = pushVoiceWaveLevel(waveLevels, level);
|
|
1244
2188
|
renderWave();
|
|
1245
2189
|
}
|
|
1246
2190
|
},
|
|
2191
|
+
connection: {
|
|
2192
|
+
reconnectReportPath
|
|
2193
|
+
},
|
|
1247
2194
|
preset: "guided-intake"
|
|
1248
2195
|
});
|
|
1249
2196
|
const generalVoice = createVoiceController(generalPath, {
|
|
1250
2197
|
capture: {
|
|
2198
|
+
onAudio: (audio, sendAudio) => {
|
|
2199
|
+
if (generalBargeInBinding) {
|
|
2200
|
+
generalBargeInBinding.sendAudio(audio);
|
|
2201
|
+
return;
|
|
2202
|
+
}
|
|
2203
|
+
sendAudio(audio);
|
|
2204
|
+
},
|
|
1251
2205
|
onLevel: (level) => {
|
|
2206
|
+
generalBargeInBinding?.handleLevel(level);
|
|
1252
2207
|
waveLevels = pushVoiceWaveLevel(waveLevels, level);
|
|
1253
2208
|
renderWave();
|
|
1254
2209
|
}
|
|
1255
2210
|
},
|
|
2211
|
+
connection: {
|
|
2212
|
+
reconnectReportPath
|
|
2213
|
+
},
|
|
1256
2214
|
preset: "dictation"
|
|
1257
2215
|
});
|
|
1258
2216
|
const stopGuidedBinding = guidedVoice.bindHTMX({ element: syncElement });
|
|
1259
2217
|
const stopGeneralBinding = generalVoice.bindHTMX({ element: syncElement });
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
2218
|
+
const guidedAudioPlayer = createVoiceAudioPlayer(guidedVoice);
|
|
2219
|
+
const generalAudioPlayer = createVoiceAudioPlayer(generalVoice);
|
|
2220
|
+
guidedBargeInBinding = bindVoiceBargeIn(guidedVoice, guidedAudioPlayer, {
|
|
2221
|
+
interruptThreshold: bargeInSpeechThreshold,
|
|
2222
|
+
monitor: bargeInMonitor ?? undefined
|
|
2223
|
+
});
|
|
2224
|
+
generalBargeInBinding = bindVoiceBargeIn(generalVoice, generalAudioPlayer, {
|
|
2225
|
+
interruptThreshold: bargeInSpeechThreshold,
|
|
2226
|
+
monitor: bargeInMonitor ?? undefined
|
|
2227
|
+
});
|
|
1268
2228
|
const currentVoice = () => activeMode === "general" ? generalVoice : guidedVoice;
|
|
2229
|
+
const currentAudioPlayer = () => activeMode === "general" ? generalAudioPlayer : guidedAudioPlayer;
|
|
1269
2230
|
const renderWave = () => {
|
|
1270
2231
|
const path = createVoiceWavePath(waveLevels);
|
|
1271
2232
|
voiceWaveGlow.setAttribute("d", path);
|
|
@@ -1280,6 +2241,9 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1280
2241
|
const status = voice.status;
|
|
1281
2242
|
connectionMetric.textContent = voice.isConnected ? "Connected" : "Waiting";
|
|
1282
2243
|
errorStatus.textContent = micError || voice.error || "None";
|
|
2244
|
+
if (reconnectStatus) {
|
|
2245
|
+
reconnectStatus.textContent = formatReconnectState(voice.reconnect);
|
|
2246
|
+
}
|
|
1283
2247
|
microphoneStatus.textContent = isCapturing ? DEFAULT_MIC_LIVE : DEFAULT_MIC_IDLE;
|
|
1284
2248
|
promptStatus.textContent = resolvePromptMessage({
|
|
1285
2249
|
guidedPrompts,
|
|
@@ -1343,8 +2307,18 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1343
2307
|
render();
|
|
1344
2308
|
}
|
|
1345
2309
|
};
|
|
1346
|
-
guidedVoice.subscribe(
|
|
1347
|
-
|
|
2310
|
+
guidedVoice.subscribe(() => {
|
|
2311
|
+
if (guidedVoice.assistantAudio.length > 0) {
|
|
2312
|
+
guidedAudioPlayer.start().catch(() => {});
|
|
2313
|
+
}
|
|
2314
|
+
render();
|
|
2315
|
+
});
|
|
2316
|
+
generalVoice.subscribe(() => {
|
|
2317
|
+
if (generalVoice.assistantAudio.length > 0) {
|
|
2318
|
+
generalAudioPlayer.start().catch(() => {});
|
|
2319
|
+
}
|
|
2320
|
+
render();
|
|
2321
|
+
});
|
|
1348
2322
|
startGuidedButton.addEventListener("click", () => {
|
|
1349
2323
|
startMode("guided");
|
|
1350
2324
|
});
|
|
@@ -1357,6 +2331,10 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1357
2331
|
window.addEventListener("beforeunload", () => {
|
|
1358
2332
|
guidedVoice.stopRecording();
|
|
1359
2333
|
generalVoice.stopRecording();
|
|
2334
|
+
guidedBargeInBinding?.close();
|
|
2335
|
+
generalBargeInBinding?.close();
|
|
2336
|
+
guidedAudioPlayer.close();
|
|
2337
|
+
generalAudioPlayer.close();
|
|
1360
2338
|
stopGuidedBinding();
|
|
1361
2339
|
stopGeneralBinding();
|
|
1362
2340
|
guidedVoice.close();
|