@absolutejs/voice 0.0.22-beta.66 → 0.0.22-beta.661
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +88 -0
- package/README.md +4550 -1180
- package/dist/angular/index.d.ts +36 -7
- package/dist/angular/index.js +4427 -521
- package/dist/angular/voice-agent-squad-status.service.d.ts +12 -0
- package/dist/angular/voice-call-debugger.service.d.ts +12 -0
- package/dist/angular/voice-call-player.service.d.ts +19 -0
- package/dist/angular/voice-campaign-dialer-proof.service.d.ts +14 -0
- package/dist/angular/voice-controller.service.d.ts +4 -1
- package/dist/angular/voice-cost-dashboard.service.d.ts +15 -0
- package/dist/angular/voice-delivery-runtime.service.d.ts +16 -0
- package/dist/angular/voice-live-agent-console.service.d.ts +16 -0
- package/dist/angular/voice-live-call-viewer.service.d.ts +16 -0
- package/dist/angular/voice-live-ops.service.d.ts +11 -0
- package/dist/angular/voice-ops-action-center.service.d.ts +13 -0
- package/dist/angular/voice-ops-status.service.d.ts +12 -0
- package/dist/angular/voice-platform-coverage.service.d.ts +12 -0
- package/dist/angular/voice-profile-comparison.service.d.ts +12 -0
- package/dist/angular/voice-proof-trends.service.d.ts +12 -0
- package/dist/angular/voice-provider-capabilities.service.d.ts +2 -2
- package/dist/angular/voice-provider-contracts.service.d.ts +12 -0
- package/dist/angular/voice-provider-status.service.d.ts +2 -2
- package/dist/angular/voice-readiness-failures.service.d.ts +13 -0
- package/dist/angular/voice-reconnect-profile-evidence.service.d.ts +12 -0
- package/dist/angular/voice-replay-timeline.service.d.ts +13 -0
- package/dist/angular/voice-routing-status.service.d.ts +1 -1
- package/dist/angular/voice-session-observability.service.d.ts +12 -0
- package/dist/angular/voice-session-snapshot.service.d.ts +13 -0
- package/dist/angular/voice-stream.service.d.ts +6 -2
- package/dist/angular/voice-trace-timeline.service.d.ts +12 -0
- package/dist/angular/voice-turn-latency.service.d.ts +13 -0
- package/dist/angular/voice-turn-quality.service.d.ts +12 -0
- package/dist/angular/voice-widget.service.d.ts +18 -0
- package/dist/angular/voice-workflow-status.service.d.ts +2 -2
- package/dist/client/actions.d.ts +140 -2
- package/dist/client/agentSquadStatus.d.ts +37 -0
- package/dist/client/agentSquadStatusWidget.d.ts +24 -0
- package/dist/client/audioPlayer.d.ts +12 -2
- package/dist/client/bargeInMonitor.d.ts +7 -0
- package/dist/client/browserMedia.d.ts +8 -0
- package/dist/client/browserNoiseSuppression.d.ts +42 -0
- package/dist/client/browserVoiceSupport.d.ts +22 -0
- package/dist/client/callDebugger.d.ts +21 -0
- package/dist/client/callDebuggerWidget.d.ts +30 -0
- package/dist/client/callPlayer.d.ts +41 -0
- package/dist/client/campaignDialerProof.d.ts +25 -0
- package/dist/client/connection.d.ts +13 -4
- package/dist/client/controller.d.ts +1 -1
- package/dist/client/conversationAnalytics.d.ts +30 -0
- package/dist/client/costDashboard.d.ts +27 -0
- package/dist/client/createVoiceStream.d.ts +1 -1
- package/dist/client/deliveryRuntime.d.ts +36 -0
- package/dist/client/deliveryRuntimeWidget.d.ts +37 -0
- package/dist/client/duplex.d.ts +2 -2
- package/dist/client/htmx.d.ts +1 -1
- package/dist/client/htmxAttributes.d.ts +24 -0
- package/dist/client/htmxBootstrap.js +1759 -152
- package/dist/client/htmxDashboardRenderers.d.ts +71 -0
- package/dist/client/index.d.ts +109 -29
- package/dist/client/index.js +10885 -586
- package/dist/client/liveAgentConsole.d.ts +28 -0
- package/dist/client/liveCallViewer.d.ts +42 -0
- package/dist/client/liveOps.d.ts +22 -0
- package/dist/client/liveOpsWidget.d.ts +23 -0
- package/dist/client/liveTurnLatency.d.ts +41 -0
- package/dist/client/microphone.d.ts +5 -4
- package/dist/client/opsActionCenter.d.ts +56 -0
- package/dist/client/opsActionCenterWidget.d.ts +29 -0
- package/dist/client/opsActionHistory.d.ts +21 -0
- package/dist/client/opsActionHistoryWidget.d.ts +11 -0
- package/dist/client/opsStatus.d.ts +21 -0
- package/dist/client/opsStatusWidget.d.ts +10 -10
- package/dist/client/platformCoverage.d.ts +21 -0
- package/dist/client/platformCoverageWidget.d.ts +38 -0
- package/dist/client/profileComparison.d.ts +21 -0
- package/dist/client/profileComparisonWidget.d.ts +41 -0
- package/dist/client/profileSwitchRecommendation.d.ts +21 -0
- package/dist/client/profileSwitchRecommendationWidget.d.ts +12 -0
- package/dist/client/proofTrends.d.ts +21 -0
- package/dist/client/proofTrendsWidget.d.ts +38 -0
- package/dist/client/providerCapabilities.d.ts +5 -3
- package/dist/client/providerCapabilitiesWidget.d.ts +6 -6
- package/dist/client/providerContracts.d.ts +21 -0
- package/dist/client/providerContractsWidget.d.ts +37 -0
- package/dist/client/providerSimulationControls.d.ts +3 -3
- package/dist/client/providerSimulationControlsWidget.d.ts +5 -5
- package/dist/client/providerStatus.d.ts +5 -3
- package/dist/client/providerStatusWidget.d.ts +6 -6
- package/dist/client/reactiveSource.d.ts +8 -0
- package/dist/client/readinessFailures.d.ts +21 -0
- package/dist/client/readinessFailuresWidget.d.ts +42 -0
- package/dist/client/reconnectProfileEvidence.d.ts +21 -0
- package/dist/client/reconnectProfileEvidenceWidget.d.ts +39 -0
- package/dist/client/replayTimeline.d.ts +26 -0
- package/dist/client/routingStatus.d.ts +5 -3
- package/dist/client/routingStatusWidget.d.ts +10 -6
- package/dist/client/sessionObservability.d.ts +21 -0
- package/dist/client/sessionObservabilityWidget.d.ts +31 -0
- package/dist/client/sessionSnapshot.d.ts +23 -0
- package/dist/client/sessionSnapshotWidget.d.ts +33 -0
- package/dist/client/store.d.ts +1 -1
- package/dist/client/timeStretch.d.ts +5 -0
- package/dist/client/traceTimeline.d.ts +21 -0
- package/dist/client/traceTimelineWidget.d.ts +36 -0
- package/dist/client/turnLatency.d.ts +24 -0
- package/dist/client/turnLatencyWidget.d.ts +33 -0
- package/dist/client/turnQuality.d.ts +21 -0
- package/dist/client/turnQualityWidget.d.ts +32 -0
- package/dist/client/voiceWidgetView.d.ts +48 -0
- package/dist/client/workflowStatus.d.ts +5 -3
- package/dist/{agent.d.ts → core/agent.d.ts} +110 -7
- package/dist/core/agentPerformanceReport.d.ts +40 -0
- package/dist/core/agentSquadContract.d.ts +98 -0
- package/dist/core/agentState.d.ts +12 -0
- package/dist/core/agentTools.d.ts +132 -0
- package/dist/core/aiScorecard.d.ts +32 -0
- package/dist/core/aiVoiceModel.d.ts +15 -0
- package/dist/core/amdDetector.d.ts +25 -0
- package/dist/{assistant.d.ts → core/assistant.d.ts} +21 -14
- package/dist/core/assistantExperiment.d.ts +42 -0
- package/dist/{assistantHealth.d.ts → core/assistantHealth.d.ts} +9 -9
- package/dist/{assistantMemory.d.ts → core/assistantMemory.d.ts} +10 -10
- package/dist/core/assistantMode.d.ts +22 -0
- package/dist/{audioConditioning.d.ts → core/audioConditioning.d.ts} +2 -2
- package/dist/core/audit.d.ts +131 -0
- package/dist/core/auditDeliveryRoutes.d.ts +85 -0
- package/dist/core/auditExport.d.ts +34 -0
- package/dist/core/auditRoutes.d.ts +66 -0
- package/dist/core/auditSinks.d.ts +151 -0
- package/dist/core/backchannel.d.ts +25 -0
- package/dist/core/bargeInDetector.d.ts +51 -0
- package/dist/core/bargeInRoutes.d.ts +56 -0
- package/dist/core/bookingFlow.d.ts +43 -0
- package/dist/core/browserCallProfiles.d.ts +120 -0
- package/dist/core/browserMediaRoutes.d.ts +62 -0
- package/dist/core/cachedTTS.d.ts +54 -0
- package/dist/core/calendarAdapter.d.ts +47 -0
- package/dist/core/calendarSlots.d.ts +35 -0
- package/dist/core/callDebugger.d.ts +66 -0
- package/dist/core/callDisposition.d.ts +38 -0
- package/dist/core/callQuota.d.ts +54 -0
- package/dist/core/callScorecard.d.ts +53 -0
- package/dist/core/callerCRMLinker.d.ts +29 -0
- package/dist/core/callerMemory.d.ts +37 -0
- package/dist/core/callingWindow.d.ts +26 -0
- package/dist/core/campaign.d.ts +795 -0
- package/dist/core/campaignControls.d.ts +37 -0
- package/dist/core/campaignDialers.d.ts +111 -0
- package/dist/core/campaignTemplate.d.ts +16 -0
- package/dist/core/competitiveCoverage.d.ts +141 -0
- package/dist/core/conversationSimulator.d.ts +73 -0
- package/dist/{correction.d.ts → core/correction.d.ts} +5 -4
- package/dist/core/costAccounting.d.ts +90 -0
- package/dist/core/costPredictor.d.ts +74 -0
- package/dist/core/crmCallLogger.d.ts +37 -0
- package/dist/core/crmContract.d.ts +70 -0
- package/dist/core/dataControl.d.ts +180 -0
- package/dist/core/debugTiming.d.ts +11 -0
- package/dist/core/defineVoiceAssistant.d.ts +68 -0
- package/dist/core/deliveryRuntime.d.ts +159 -0
- package/dist/core/deliverySinkRoutes.d.ts +117 -0
- package/dist/core/demoReadyRoutes.d.ts +98 -0
- package/dist/{diagnosticsRoutes.d.ts → core/diagnosticsRoutes.d.ts} +2 -2
- package/dist/core/dncRegistry.d.ts +38 -0
- package/dist/core/dtmfCollector.d.ts +37 -0
- package/dist/{evalRoutes.d.ts → core/evalRoutes.d.ts} +26 -20
- package/dist/{fileStore.d.ts → core/fileStore.d.ts} +34 -20
- package/dist/core/guardrails.d.ts +128 -0
- package/dist/{handoff.d.ts → core/handoff.d.ts} +10 -10
- package/dist/{handoffHealth.d.ts → core/handoffHealth.d.ts} +9 -9
- package/dist/core/hardenedFetch.d.ts +1 -0
- package/dist/core/holdAudio.d.ts +23 -0
- package/dist/{htmx.d.ts → core/htmx.d.ts} +2 -2
- package/dist/core/htmxDashboardRoutes.d.ts +250 -0
- package/dist/core/iceServers.d.ts +34 -0
- package/dist/core/incidentBundle.d.ts +119 -0
- package/dist/core/incidentTimeline.d.ts +260 -0
- package/dist/core/ivrPlan.d.ts +40 -0
- package/dist/core/latencySlo.d.ts +56 -0
- package/dist/core/liveCoach.d.ts +43 -0
- package/dist/core/liveLatency.d.ts +78 -0
- package/dist/core/liveOps.d.ts +190 -0
- package/dist/core/llmJudge.d.ts +45 -0
- package/dist/{logger.d.ts → core/logger.d.ts} +1 -2
- package/dist/core/mcpToolset.d.ts +58 -0
- package/dist/core/mediaPipelineRoutes.d.ts +171 -0
- package/dist/core/mediaPipelineSurfaces.d.ts +48 -0
- package/dist/{memoryStore.d.ts → core/memoryStore.d.ts} +1 -1
- package/dist/core/midCallSummary.d.ts +27 -0
- package/dist/{modelAdapters.d.ts → core/modelAdapters.d.ts} +64 -7
- package/dist/core/monitor.d.ts +148 -0
- package/dist/core/multilingualProof.d.ts +77 -0
- package/dist/core/noShowPredictor.d.ts +46 -0
- package/dist/core/numberNormalizer.d.ts +1 -0
- package/dist/core/oauth2TokenSource.d.ts +21 -0
- package/dist/core/observabilityExport.d.ts +501 -0
- package/dist/core/openaiTTS.d.ts +18 -0
- package/dist/core/operationalStatus.d.ts +87 -0
- package/dist/core/operationsRecord.d.ts +371 -0
- package/dist/{ops.d.ts → core/ops.d.ts} +70 -70
- package/dist/core/opsActionAuditRoutes.d.ts +99 -0
- package/dist/{opsConsoleRoutes.d.ts → core/opsConsoleRoutes.d.ts} +11 -8
- package/dist/{opsPresets.d.ts → core/opsPresets.d.ts} +2 -2
- package/dist/core/opsRecovery.d.ts +137 -0
- package/dist/{opsRuntime.d.ts → core/opsRuntime.d.ts} +6 -6
- package/dist/{opsSinks.d.ts → core/opsSinks.d.ts} +19 -19
- package/dist/core/opsStatus.d.ts +76 -0
- package/dist/core/opsStatusRoutes.d.ts +33 -0
- package/dist/{opsWebhook.d.ts → core/opsWebhook.d.ts} +15 -15
- package/dist/core/otelExporter.d.ts +83 -0
- package/dist/core/outcomeContract.d.ts +146 -0
- package/dist/{outcomeRecipes.d.ts → core/outcomeRecipes.d.ts} +4 -4
- package/dist/core/pathway.d.ts +94 -0
- package/dist/core/pathwayCompiler.d.ts +31 -0
- package/dist/core/pathwayGenerator.d.ts +27 -0
- package/dist/core/pathwayRuntime.d.ts +57 -0
- package/dist/core/pathwaySlotCollector.d.ts +29 -0
- package/dist/core/pathwayVisualizer.d.ts +8 -0
- package/dist/core/phoneAgent.d.ts +139 -0
- package/dist/core/phoneAgentProductionSmoke.d.ts +115 -0
- package/dist/core/phoneProvisioning.d.ts +29 -0
- package/dist/core/platformCoverage.d.ts +91 -0
- package/dist/{plugin.d.ts → core/plugin.d.ts} +11 -8
- package/dist/core/postCallAnalysis.d.ts +98 -0
- package/dist/core/postCallSurvey.d.ts +41 -0
- package/dist/{postgresStore.d.ts → core/postgresStore.d.ts} +20 -9
- package/dist/{presets.d.ts → core/presets.d.ts} +3 -3
- package/dist/core/productionReadiness.d.ts +757 -0
- package/dist/core/profileSwitchRecommendation.d.ts +350 -0
- package/dist/core/promptInjectionGuard.d.ts +30 -0
- package/dist/core/proofAssertions.d.ts +32 -0
- package/dist/core/proofPack.d.ts +211 -0
- package/dist/core/proofRunner.d.ts +79 -0
- package/dist/core/proofTrends.d.ts +966 -0
- package/dist/{providerAdapters.d.ts → core/providerAdapters.d.ts} +5 -5
- package/dist/{providerCapabilities.d.ts → core/providerCapabilities.d.ts} +9 -9
- package/dist/core/providerDecisionTraces.d.ts +130 -0
- package/dist/{providerHealth.d.ts → core/providerHealth.d.ts} +15 -5
- package/dist/core/providerOrchestration.d.ts +109 -0
- package/dist/core/providerRouterTraces.d.ts +35 -0
- package/dist/core/providerRoutingContract.d.ts +71 -0
- package/dist/core/providerSlo.d.ts +142 -0
- package/dist/core/providerStackRecommendations.d.ts +188 -0
- package/dist/core/qualityDriftDetector.d.ts +44 -0
- package/dist/{qualityRoutes.d.ts → core/qualityRoutes.d.ts} +7 -7
- package/dist/{queue.d.ts → core/queue.d.ts} +48 -39
- package/dist/core/ragTool.d.ts +52 -0
- package/dist/core/readinessProfiles.d.ts +45 -0
- package/dist/core/realtimeChannel.d.ts +136 -0
- package/dist/core/realtimeProviderContracts.d.ts +133 -0
- package/dist/core/reconnectContract.d.ts +177 -0
- package/dist/core/recordingRedaction.d.ts +47 -0
- package/dist/core/recordingStore.d.ts +60 -0
- package/dist/core/redaction.d.ts +13 -0
- package/dist/core/reminderScheduler.d.ts +43 -0
- package/dist/{resilienceRoutes.d.ts → core/resilienceRoutes.d.ts} +34 -5
- package/dist/core/retention.d.ts +37 -0
- package/dist/core/retryPolicy.d.ts +38 -0
- package/dist/core/routeAuth.d.ts +58 -0
- package/dist/{routing.d.ts → core/routing.d.ts} +2 -2
- package/dist/{runtimeOps.d.ts → core/runtimeOps.d.ts} +3 -3
- package/dist/{s3Store.d.ts → core/s3Store.d.ts} +12 -3
- package/dist/core/scorecardCalibration.d.ts +31 -0
- package/dist/core/scribe.d.ts +50 -0
- package/dist/core/semanticTurn.d.ts +37 -0
- package/dist/{session.d.ts → core/session.d.ts} +1 -1
- package/dist/core/sessionObservability.d.ts +145 -0
- package/dist/{sessionReplay.d.ts → core/sessionReplay.d.ts} +31 -19
- package/dist/core/sessionSnapshot.d.ts +109 -0
- package/dist/core/simulationSuite.d.ts +144 -0
- package/dist/core/sloCalibration.d.ts +185 -0
- package/dist/{sqliteStore.d.ts → core/sqliteStore.d.ts} +20 -9
- package/dist/{store.d.ts → core/store.d.ts} +1 -1
- package/dist/core/supervisorPermissions.d.ts +33 -0
- package/dist/core/supervisorPresence.d.ts +49 -0
- package/dist/core/telephonyMediaRoutes.d.ts +72 -0
- package/dist/core/telephonyOutcome.d.ts +269 -0
- package/dist/{toolContract.d.ts → core/toolContract.d.ts} +46 -15
- package/dist/{toolRuntime.d.ts → core/toolRuntime.d.ts} +4 -4
- package/dist/{trace.d.ts → core/trace.d.ts} +61 -22
- package/dist/core/traceDeliveryRoutes.d.ts +86 -0
- package/dist/core/traceTimeline.d.ts +97 -0
- package/dist/core/transcriptAnnotator.d.ts +41 -0
- package/dist/{turnDetection.d.ts → core/turnDetection.d.ts} +2 -1
- package/dist/core/turnLatency.d.ts +95 -0
- package/dist/core/turnProfiles.d.ts +3 -0
- package/dist/core/turnQuality.d.ts +94 -0
- package/dist/core/types.d.ts +1759 -0
- package/dist/core/vapiAdapter.d.ts +160 -0
- package/dist/core/variableAnalytics.d.ts +47 -0
- package/dist/core/voiceConfiguration.d.ts +8 -0
- package/dist/core/voiceMonitoring.d.ts +444 -0
- package/dist/core/webhookFanout.d.ts +48 -0
- package/dist/core/webhookVerification.d.ts +27 -0
- package/dist/core/whisperChannel.d.ts +50 -0
- package/dist/{workflowContract.d.ts → core/workflowContract.d.ts} +21 -21
- package/dist/core/writeBehindStore.d.ts +41 -0
- package/dist/core/zeroDataRetention.d.ts +31 -0
- package/dist/drizzle/assistantMemory.d.ts +100 -0
- package/dist/drizzle/eval.d.ts +55 -0
- package/dist/drizzle/handoff.d.ts +56 -0
- package/dist/drizzle/incidentBundle.d.ts +55 -0
- package/dist/drizzle/index.d.ts +991 -0
- package/dist/drizzle/index.js +3059 -0
- package/dist/drizzle/observabilityExport.d.ts +55 -0
- package/dist/drizzle/proofTrends.d.ts +114 -0
- package/dist/drizzle/runtimeStorage.d.ts +1183 -0
- package/dist/drizzle/shared.d.ts +69 -0
- package/dist/embed/index.d.ts +38 -0
- package/dist/embed/index.js +2058 -0
- package/dist/embed/voice-widget.js +10 -0
- package/dist/index.d.ts +379 -80
- package/dist/index.js +48316 -8042
- package/dist/internal/evidence.d.ts +10 -0
- package/dist/internal/html.d.ts +6 -0
- package/dist/internal/status.d.ts +7 -0
- package/dist/manifest.d.ts +17 -0
- package/dist/manifest.js +424 -0
- package/dist/manifest.json +427 -0
- package/dist/react/VoiceAgentSquadStatus.d.ts +5 -0
- package/dist/react/VoiceCallDebuggerLaunch.d.ts +6 -0
- package/dist/react/VoiceCallPlayer.d.ts +11 -0
- package/dist/react/VoiceCostDashboard.d.ts +10 -0
- package/dist/react/VoiceDeliveryRuntime.d.ts +7 -0
- package/dist/react/VoiceLiveAgentConsole.d.ts +11 -0
- package/dist/react/VoiceLiveCallViewer.d.ts +9 -0
- package/dist/react/VoiceOpsActionCenter.d.ts +5 -0
- package/dist/react/VoiceOpsStatus.d.ts +1 -1
- package/dist/react/VoicePlatformCoverage.d.ts +6 -0
- package/dist/react/VoiceProfileComparison.d.ts +6 -0
- package/dist/react/VoiceProfileSwitchRecommendation.d.ts +6 -0
- package/dist/react/VoiceProofTrends.d.ts +6 -0
- package/dist/react/VoiceProviderCapabilities.d.ts +1 -1
- package/dist/react/VoiceProviderContracts.d.ts +6 -0
- package/dist/react/VoiceProviderSimulationControls.d.ts +1 -1
- package/dist/react/VoiceProviderStatus.d.ts +1 -1
- package/dist/react/VoiceReadinessFailures.d.ts +6 -0
- package/dist/react/VoiceReconnectProfileEvidence.d.ts +6 -0
- package/dist/react/VoiceReplayTimeline.d.ts +6 -0
- package/dist/react/VoiceRoutingStatus.d.ts +1 -1
- package/dist/react/VoiceSessionObservability.d.ts +6 -0
- package/dist/react/VoiceSessionSnapshot.d.ts +6 -0
- package/dist/react/VoiceTraceTimeline.d.ts +6 -0
- package/dist/react/VoiceTurnLatency.d.ts +6 -0
- package/dist/react/VoiceTurnQuality.d.ts +6 -0
- package/dist/react/VoiceWidget.d.ts +13 -0
- package/dist/react/index.d.ts +80 -13
- package/dist/react/index.js +13718 -1780
- package/dist/react/useVoiceAgentSquadStatus.d.ts +8 -0
- package/dist/react/useVoiceCallDebugger.d.ts +8 -0
- package/dist/react/useVoiceCampaignDialerProof.d.ts +10 -0
- package/dist/react/useVoiceController.d.ts +6 -2
- package/dist/react/useVoiceDeliveryRuntime.d.ts +13 -0
- package/dist/react/useVoiceLiveOps.d.ts +9 -0
- package/dist/react/useVoiceOpsActionCenter.d.ts +11 -0
- package/dist/react/useVoiceOpsStatus.d.ts +8 -0
- package/dist/react/useVoicePlatformCoverage.d.ts +8 -0
- package/dist/react/useVoiceProfileComparison.d.ts +8 -0
- package/dist/react/useVoiceProfileSwitchRecommendation.d.ts +8 -0
- package/dist/react/useVoiceProofTrends.d.ts +8 -0
- package/dist/react/useVoiceProviderCapabilities.d.ts +1 -1
- package/dist/react/useVoiceProviderContracts.d.ts +8 -0
- package/dist/react/useVoiceProviderSimulationControls.d.ts +1 -1
- package/dist/react/useVoiceProviderStatus.d.ts +1 -1
- package/dist/react/useVoiceReadinessFailures.d.ts +8 -0
- package/dist/react/useVoiceReconnectProfileEvidence.d.ts +8 -0
- package/dist/react/useVoiceRoutingStatus.d.ts +1 -1
- package/dist/react/useVoiceSessionObservability.d.ts +8 -0
- package/dist/react/useVoiceSessionSnapshot.d.ts +9 -0
- package/dist/react/useVoiceStream.d.ts +6 -2
- package/dist/react/useVoiceTraceTimeline.d.ts +8 -0
- package/dist/react/useVoiceTurnLatency.d.ts +9 -0
- package/dist/react/useVoiceTurnQuality.d.ts +8 -0
- package/dist/react/useVoiceWorkflowStatus.d.ts +1 -1
- package/dist/svelte/createVoiceAgentSquadStatus.d.ts +9 -0
- package/dist/svelte/createVoiceCallDebugger.d.ts +10 -0
- package/dist/svelte/createVoiceCallPlayer.d.ts +33 -0
- package/dist/svelte/createVoiceCampaignDialerProof.d.ts +9 -0
- package/dist/svelte/createVoiceCostDashboard.d.ts +13 -0
- package/dist/svelte/createVoiceDeliveryRuntime.d.ts +11 -0
- package/dist/svelte/createVoiceLiveAgentConsole.d.ts +23 -0
- package/dist/svelte/createVoiceLiveCallViewer.d.ts +26 -0
- package/dist/svelte/createVoiceLiveOps.d.ts +13 -0
- package/dist/svelte/createVoiceOpsActionCenter.d.ts +10 -0
- package/dist/svelte/createVoiceOpsStatus.d.ts +4 -4
- package/dist/svelte/createVoicePlatformCoverage.d.ts +7 -0
- package/dist/svelte/createVoiceProfileComparison.d.ts +7 -0
- package/dist/svelte/createVoiceProofTrends.d.ts +7 -0
- package/dist/svelte/createVoiceProviderCapabilities.d.ts +2 -2
- package/dist/svelte/createVoiceProviderContracts.d.ts +10 -0
- package/dist/svelte/createVoiceProviderSimulationControls.d.ts +2 -2
- package/dist/svelte/createVoiceProviderStatus.d.ts +2 -2
- package/dist/svelte/createVoiceReadinessFailures.d.ts +7 -0
- package/dist/svelte/createVoiceReconnectProfileEvidence.d.ts +7 -0
- package/dist/svelte/createVoiceReplayTimeline.d.ts +13 -0
- package/dist/svelte/createVoiceRoutingStatus.d.ts +2 -2
- package/dist/svelte/createVoiceSessionObservability.d.ts +10 -0
- package/dist/svelte/createVoiceSessionSnapshot.d.ts +11 -0
- package/dist/svelte/createVoiceStream.d.ts +1 -1
- package/dist/svelte/createVoiceTraceTimeline.d.ts +10 -0
- package/dist/svelte/createVoiceTurnLatency.d.ts +11 -0
- package/dist/svelte/createVoiceTurnQuality.d.ts +10 -0
- package/dist/svelte/createVoiceWidget.d.ts +19 -0
- package/dist/svelte/createVoiceWorkflowStatus.d.ts +2 -2
- package/dist/svelte/index.d.ts +37 -9
- package/dist/svelte/index.js +7493 -1545
- package/dist/telephony/contract.d.ts +61 -0
- package/dist/telephony/matrix.d.ts +97 -0
- package/dist/telephony/plivo.d.ts +303 -0
- package/dist/telephony/response.d.ts +1 -1
- package/dist/telephony/security.d.ts +182 -0
- package/dist/telephony/telnyx.d.ts +291 -0
- package/dist/telephony/twilio.d.ts +153 -15
- package/dist/testing/accuracy.d.ts +18 -1
- package/dist/testing/audioMatrix.d.ts +22 -0
- package/dist/testing/benchmark.d.ts +24 -15
- package/dist/testing/confidenceCalibration.d.ts +19 -0
- package/dist/testing/conformance.d.ts +11 -0
- package/dist/testing/corrected.d.ts +9 -9
- package/dist/testing/criticalFields.d.ts +22 -0
- package/dist/testing/duplex.d.ts +5 -5
- package/dist/testing/fixtures.d.ts +7 -4
- package/dist/testing/index.d.ts +21 -13
- package/dist/testing/index.js +9019 -1285
- package/dist/testing/ioProviderSimulator.d.ts +5 -5
- package/dist/testing/outcomes.d.ts +12 -0
- package/dist/testing/provenance.d.ts +44 -0
- package/dist/testing/providerSimulator.d.ts +5 -5
- package/dist/testing/review.d.ts +8 -8
- package/dist/testing/routingBenchmark.d.ts +16 -0
- package/dist/testing/sessionBenchmark.d.ts +13 -13
- package/dist/testing/statistics.d.ts +30 -0
- package/dist/testing/stt.d.ts +3 -3
- package/dist/testing/telephony.d.ts +27 -2
- package/dist/testing/tts.d.ts +2 -2
- package/dist/vue/VoiceCallDebuggerLaunch.d.ts +72 -0
- package/dist/vue/VoiceCallPlayer.d.ts +40 -0
- package/dist/vue/VoiceCostDashboard.d.ts +57 -0
- package/dist/vue/VoiceDeliveryRuntime.d.ts +34 -0
- package/dist/vue/VoiceLiveAgentConsole.d.ts +50 -0
- package/dist/vue/VoiceLiveCallViewer.d.ts +35 -0
- package/dist/vue/VoiceOpsActionCenter.d.ts +13 -0
- package/dist/vue/VoiceOpsStatus.d.ts +4 -0
- package/dist/vue/VoicePlatformCoverage.d.ts +27 -0
- package/dist/vue/VoiceProofTrends.d.ts +25 -0
- package/dist/vue/VoiceProviderCapabilities.d.ts +4 -0
- package/dist/vue/VoiceProviderContracts.d.ts +25 -0
- package/dist/vue/VoiceProviderSimulationControls.d.ts +3 -3
- package/dist/vue/VoiceProviderStatus.d.ts +4 -0
- package/dist/vue/VoiceReadinessFailures.d.ts +25 -0
- package/dist/vue/VoiceReconnectProfileEvidence.d.ts +25 -0
- package/dist/vue/VoiceReplayTimeline.d.ts +17 -0
- package/dist/vue/VoiceRoutingStatus.d.ts +4 -0
- package/dist/vue/VoiceSessionObservability.d.ts +27 -0
- package/dist/vue/VoiceSessionSnapshot.d.ts +72 -0
- package/dist/vue/VoiceTurnLatency.d.ts +73 -0
- package/dist/vue/VoiceTurnQuality.d.ts +55 -0
- package/dist/vue/VoiceWidget.d.ts +77 -0
- package/dist/vue/index.d.ts +49 -13
- package/dist/vue/index.js +13049 -1936
- package/dist/vue/useVoiceAgentSquadStatus.d.ts +9 -0
- package/dist/vue/useVoiceCallDebugger.d.ts +10 -0
- package/dist/vue/useVoiceCampaignDialerProof.d.ts +11 -0
- package/dist/vue/useVoiceController.d.ts +10 -7
- package/dist/vue/useVoiceDeliveryRuntime.d.ts +13 -0
- package/dist/vue/useVoiceLiveOps.d.ts +9 -0
- package/dist/vue/useVoiceOpsActionCenter.d.ts +11 -0
- package/dist/vue/useVoiceOpsStatus.d.ts +9 -0
- package/dist/vue/useVoicePlatformCoverage.d.ts +9 -0
- package/dist/vue/useVoiceProfileComparison.d.ts +9 -0
- package/dist/vue/useVoiceProofTrends.d.ts +9 -0
- package/dist/vue/useVoiceProviderCapabilities.d.ts +3 -3
- package/dist/vue/useVoiceProviderContracts.d.ts +9 -0
- package/dist/vue/useVoiceProviderSimulationControls.d.ts +2 -2
- package/dist/vue/useVoiceProviderStatus.d.ts +3 -3
- package/dist/vue/useVoiceReadinessFailures.d.ts +959 -0
- package/dist/vue/useVoiceReconnectProfileEvidence.d.ts +9 -0
- package/dist/vue/useVoiceRoutingStatus.d.ts +2 -2
- package/dist/vue/useVoiceSessionObservability.d.ts +9 -0
- package/dist/vue/useVoiceSessionSnapshot.d.ts +10 -0
- package/dist/vue/useVoiceStream.d.ts +10 -6
- package/dist/vue/useVoiceTraceTimeline.d.ts +9 -0
- package/dist/vue/useVoiceTurnLatency.d.ts +10 -0
- package/dist/vue/useVoiceTurnQuality.d.ts +9 -0
- package/dist/vue/useVoiceWorkflowStatus.d.ts +3 -3
- package/package.json +247 -256
- package/dist/angular/voice-app-kit-status.service.d.ts +0 -12
- package/dist/angular/voice-ops-status.component.d.ts +0 -15
- package/dist/appKit.d.ts +0 -94
- package/dist/client/appKitStatus.d.ts +0 -19
- package/dist/react/useVoiceAppKitStatus.d.ts +0 -8
- package/dist/svelte/createVoiceAppKitStatus.d.ts +0 -8
- package/dist/turnProfiles.d.ts +0 -6
- package/dist/types.d.ts +0 -968
- package/dist/vue/useVoiceAppKitStatus.d.ts +0 -9
- package/fixtures/README.md +0 -57
- package/fixtures/manifest.json +0 -199
- package/fixtures/pcm/dialogue-three-clean.pcm +0 -0
- package/fixtures/pcm/dialogue-three-mixed.pcm +0 -0
- package/fixtures/pcm/dialogue-two-clean.pcm +0 -0
- package/fixtures/pcm/dialogue-two-noisy.pcm +0 -0
- package/fixtures/pcm/multiturn-three-mixed.pcm +0 -0
- package/fixtures/pcm/multiturn-two-clean.pcm +0 -0
- package/fixtures/pcm/quietly-alone-clean.pcm +0 -0
- package/fixtures/pcm/rainstorms-noisy.pcm +0 -0
- package/fixtures/pcm/stella-bulgaria-bulgarian20.pcm +0 -0
- package/fixtures/pcm/stella-ghana-english507.pcm +0 -0
- package/fixtures/pcm/stella-india-english37.pcm +0 -0
- package/fixtures/pcm/stella-jamaica-jamaican-creole-english1.pcm +0 -0
- package/fixtures/pcm/stella-liberia-liberian-pidgin-english2.pcm +0 -0
- package/fixtures/pcm/stella-pakistan-english519.pcm +0 -0
- package/fixtures/pcm/stella-sierra-leone-krio5.pcm +0 -0
- package/fixtures/pcm/stella-singapore-english655.pcm +0 -0
- package/fixtures/pcm/traveled-back-route-clean.pcm +0 -0
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// src/internal/html.ts
|
|
2
|
+
var escapeHtml = (value) => String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
3
|
+
|
|
1
4
|
// src/client/htmx.ts
|
|
2
5
|
var DEFAULT_EVENT_NAME = "voice-refresh";
|
|
3
6
|
var DEFAULT_QUERY_PARAM = "sessionId";
|
|
@@ -100,37 +103,89 @@ var createMicrophoneCapture = (options) => {
|
|
|
100
103
|
let sourceNode = null;
|
|
101
104
|
let processorNode = null;
|
|
102
105
|
let mediaStream = null;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
106
|
+
let lifecycleGeneration = 0;
|
|
107
|
+
let startPromise = null;
|
|
108
|
+
const release = (resources) => {
|
|
109
|
+
resources.processorNode?.disconnect();
|
|
110
|
+
resources.sourceNode?.disconnect();
|
|
111
|
+
resources.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
112
|
+
resources.audioContext?.close();
|
|
113
|
+
};
|
|
114
|
+
const start = () => {
|
|
115
|
+
if (mediaStream)
|
|
116
|
+
return Promise.resolve();
|
|
117
|
+
if (startPromise)
|
|
118
|
+
return startPromise;
|
|
119
|
+
const generation = ++lifecycleGeneration;
|
|
120
|
+
const operation = async () => {
|
|
121
|
+
if (!options.stream && (typeof navigator === "undefined" || !navigator.mediaDevices?.getUserMedia)) {
|
|
122
|
+
throw new Error("Browser microphone capture requires navigator.mediaDevices.getUserMedia.");
|
|
123
|
+
}
|
|
124
|
+
const AudioContextCtor = (typeof window !== "undefined" ? window.AudioContext ?? window.webkitAudioContext : undefined) ?? AudioContext;
|
|
125
|
+
if (!AudioContextCtor) {
|
|
126
|
+
throw new Error("Browser microphone capture requires AudioContext support.");
|
|
127
|
+
}
|
|
128
|
+
let pendingAudioContext = null;
|
|
129
|
+
let pendingMediaStream = null;
|
|
130
|
+
let pendingProcessorNode = null;
|
|
131
|
+
let pendingSourceNode = null;
|
|
132
|
+
try {
|
|
133
|
+
pendingMediaStream = options.stream ?? await navigator.mediaDevices.getUserMedia({
|
|
134
|
+
audio: {
|
|
135
|
+
autoGainControl: true,
|
|
136
|
+
channelCount: options.channelCount ?? 1,
|
|
137
|
+
echoCancellation: true,
|
|
138
|
+
noiseSuppression: true
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
if (generation !== lifecycleGeneration) {
|
|
142
|
+
throw new Error("Microphone capture stopped during startup.");
|
|
143
|
+
}
|
|
144
|
+
pendingAudioContext = new AudioContextCtor;
|
|
145
|
+
if (pendingAudioContext.state === "suspended") {
|
|
146
|
+
await pendingAudioContext.resume();
|
|
147
|
+
}
|
|
148
|
+
if (generation !== lifecycleGeneration) {
|
|
149
|
+
throw new Error("Microphone capture stopped during startup.");
|
|
150
|
+
}
|
|
151
|
+
pendingSourceNode = pendingAudioContext.createMediaStreamSource(pendingMediaStream);
|
|
152
|
+
pendingProcessorNode = pendingAudioContext.createScriptProcessor(4096, 1, 1);
|
|
153
|
+
pendingProcessorNode.onaudioprocess = (event) => {
|
|
154
|
+
const channel = event.inputBuffer.getChannelData(0);
|
|
155
|
+
const downsampled = downsampleBuffer(channel, pendingAudioContext?.sampleRate ?? 48000, options.sampleRateHz ?? 16000);
|
|
156
|
+
const pcm = floatTo16BitPCM(downsampled);
|
|
157
|
+
options.onLevel?.(getPcmLevel(pcm));
|
|
158
|
+
options.onAudio(pcm);
|
|
159
|
+
};
|
|
160
|
+
pendingSourceNode.connect(pendingProcessorNode);
|
|
161
|
+
pendingProcessorNode.connect(pendingAudioContext.destination);
|
|
162
|
+
if (generation !== lifecycleGeneration) {
|
|
163
|
+
throw new Error("Microphone capture stopped during startup.");
|
|
164
|
+
}
|
|
165
|
+
audioContext = pendingAudioContext;
|
|
166
|
+
mediaStream = pendingMediaStream;
|
|
167
|
+
processorNode = pendingProcessorNode;
|
|
168
|
+
sourceNode = pendingSourceNode;
|
|
169
|
+
} catch (error) {
|
|
170
|
+
release({
|
|
171
|
+
audioContext: pendingAudioContext,
|
|
172
|
+
mediaStream: pendingMediaStream,
|
|
173
|
+
processorNode: pendingProcessorNode,
|
|
174
|
+
sourceNode: pendingSourceNode
|
|
175
|
+
});
|
|
176
|
+
throw error;
|
|
114
177
|
}
|
|
115
|
-
});
|
|
116
|
-
audioContext = new AudioContextCtor;
|
|
117
|
-
sourceNode = audioContext.createMediaStreamSource(mediaStream);
|
|
118
|
-
processorNode = audioContext.createScriptProcessor(4096, 1, 1);
|
|
119
|
-
processorNode.onaudioprocess = (event) => {
|
|
120
|
-
const channel = event.inputBuffer.getChannelData(0);
|
|
121
|
-
const downsampled = downsampleBuffer(channel, audioContext?.sampleRate ?? 48000, options.sampleRateHz ?? 16000);
|
|
122
|
-
const pcm = floatTo16BitPCM(downsampled);
|
|
123
|
-
options.onLevel?.(getPcmLevel(pcm));
|
|
124
|
-
options.onAudio(pcm);
|
|
125
178
|
};
|
|
126
|
-
|
|
127
|
-
|
|
179
|
+
const pending = operation().finally(() => {
|
|
180
|
+
if (startPromise === pending)
|
|
181
|
+
startPromise = null;
|
|
182
|
+
});
|
|
183
|
+
startPromise = pending;
|
|
184
|
+
return pending;
|
|
128
185
|
};
|
|
129
186
|
const stop = () => {
|
|
130
|
-
|
|
131
|
-
sourceNode
|
|
132
|
-
mediaStream?.getTracks().forEach((track) => track.stop());
|
|
133
|
-
audioContext?.close();
|
|
187
|
+
lifecycleGeneration += 1;
|
|
188
|
+
release({ audioContext, mediaStream, processorNode, sourceNode });
|
|
134
189
|
options.onLevel?.(0);
|
|
135
190
|
audioContext = null;
|
|
136
191
|
mediaStream = null;
|
|
@@ -181,13 +236,25 @@ var serverMessageToAction = (message) => {
|
|
|
181
236
|
case "assistant":
|
|
182
237
|
return {
|
|
183
238
|
text: message.text,
|
|
239
|
+
turnId: message.turnId,
|
|
184
240
|
type: "assistant"
|
|
185
241
|
};
|
|
242
|
+
case "assistant_delta":
|
|
243
|
+
return {
|
|
244
|
+
delta: message.delta,
|
|
245
|
+
turnId: message.turnId,
|
|
246
|
+
type: "assistant_delta"
|
|
247
|
+
};
|
|
186
248
|
case "complete":
|
|
187
249
|
return {
|
|
188
250
|
sessionId: message.sessionId,
|
|
189
251
|
type: "complete"
|
|
190
252
|
};
|
|
253
|
+
case "connection":
|
|
254
|
+
return {
|
|
255
|
+
reconnect: message.reconnect,
|
|
256
|
+
type: "connection"
|
|
257
|
+
};
|
|
191
258
|
case "call_lifecycle":
|
|
192
259
|
return {
|
|
193
260
|
event: message.event,
|
|
@@ -197,8 +264,11 @@ var serverMessageToAction = (message) => {
|
|
|
197
264
|
case "error":
|
|
198
265
|
return {
|
|
199
266
|
message: normalizeErrorMessage(message.message),
|
|
267
|
+
recoverable: message.recoverable,
|
|
200
268
|
type: "error"
|
|
201
269
|
};
|
|
270
|
+
case "call_control_ack":
|
|
271
|
+
return null;
|
|
202
272
|
case "final":
|
|
203
273
|
return {
|
|
204
274
|
transcript: message.transcript,
|
|
@@ -209,9 +279,22 @@ var serverMessageToAction = (message) => {
|
|
|
209
279
|
transcript: message.transcript,
|
|
210
280
|
type: "partial"
|
|
211
281
|
};
|
|
282
|
+
case "replay":
|
|
283
|
+
return {
|
|
284
|
+
assistantTexts: message.assistantTexts,
|
|
285
|
+
call: message.call,
|
|
286
|
+
partial: message.partial,
|
|
287
|
+
scenarioId: message.scenarioId,
|
|
288
|
+
sessionId: message.sessionId,
|
|
289
|
+
sessionMetadata: message.sessionMetadata,
|
|
290
|
+
status: message.status,
|
|
291
|
+
turns: message.turns,
|
|
292
|
+
type: "replay"
|
|
293
|
+
};
|
|
212
294
|
case "session":
|
|
213
295
|
return {
|
|
214
296
|
sessionId: message.sessionId,
|
|
297
|
+
sessionMetadata: message.sessionMetadata,
|
|
215
298
|
scenarioId: message.scenarioId,
|
|
216
299
|
status: message.status,
|
|
217
300
|
type: "session"
|
|
@@ -226,30 +309,276 @@ var serverMessageToAction = (message) => {
|
|
|
226
309
|
}
|
|
227
310
|
};
|
|
228
311
|
|
|
312
|
+
// node_modules/@absolutejs/media/dist/index.js
|
|
313
|
+
var TAU = Math.PI * 2;
|
|
314
|
+
var pushIssue = (issues, severity, code, message) => {
|
|
315
|
+
issues.push({ code, message, severity });
|
|
316
|
+
};
|
|
317
|
+
var average = (values) => values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0) / values.length;
|
|
318
|
+
var max = (values) => values.length === 0 ? undefined : Math.max(...values);
|
|
319
|
+
var numericStat = (stat, key) => {
|
|
320
|
+
const value = stat[key];
|
|
321
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
322
|
+
};
|
|
323
|
+
var booleanStat = (stat, key) => {
|
|
324
|
+
const value = stat[key];
|
|
325
|
+
return typeof value === "boolean" ? value : undefined;
|
|
326
|
+
};
|
|
327
|
+
var stringStat = (stat, key) => {
|
|
328
|
+
const value = stat[key];
|
|
329
|
+
return typeof value === "string" ? value : undefined;
|
|
330
|
+
};
|
|
331
|
+
var statKey = (stat) => String(stat.id ?? stringStat(stat, "ssrc") ?? numericStat(stat, "ssrc") ?? stringStat(stat, "trackIdentifier") ?? stringStat(stat, "mid") ?? "unknown");
|
|
332
|
+
var secondsToMs = (value) => value === undefined ? undefined : value * 1000;
|
|
333
|
+
var normalizeWebRTCStat = (stat) => {
|
|
334
|
+
const sample = {};
|
|
335
|
+
for (const [key, value] of Object.entries(stat)) {
|
|
336
|
+
if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") {
|
|
337
|
+
sample[key] = value;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return sample;
|
|
341
|
+
};
|
|
342
|
+
var buildMediaWebRTCStatsReport = (input = {}) => {
|
|
343
|
+
const stats = input.stats ?? [];
|
|
344
|
+
const issues = [];
|
|
345
|
+
const inbound = stats.filter((stat) => stat.type === "inbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
346
|
+
const outbound = stats.filter((stat) => stat.type === "outbound-rtp" && stringStat(stat, "kind") !== "video");
|
|
347
|
+
const candidatePairs = stats.filter((stat) => stat.type === "candidate-pair");
|
|
348
|
+
const audioTracks = stats.filter((stat) => (stat.type === "track" || stat.type === "media-source") && stringStat(stat, "kind") === "audio");
|
|
349
|
+
const activeCandidatePairs = candidatePairs.filter((stat) => booleanStat(stat, "selected") === true || booleanStat(stat, "nominated") === true || stringStat(stat, "state") === "succeeded").length;
|
|
350
|
+
const liveAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") !== "ended" && stringStat(stat, "trackState") !== "ended" && booleanStat(stat, "ended") !== true).length;
|
|
351
|
+
const endedAudioTracks = audioTracks.filter((stat) => stringStat(stat, "readyState") === "ended" || stringStat(stat, "trackState") === "ended" || booleanStat(stat, "ended") === true).length;
|
|
352
|
+
const inboundPackets = inbound.reduce((total, stat) => total + (numericStat(stat, "packetsReceived") ?? 0), 0);
|
|
353
|
+
const outboundPackets = outbound.reduce((total, stat) => total + (numericStat(stat, "packetsSent") ?? 0), 0);
|
|
354
|
+
const packetsLost = [...inbound, ...outbound].reduce((total, stat) => total + Math.max(0, numericStat(stat, "packetsLost") ?? 0), 0);
|
|
355
|
+
const packetLossDenominator = inboundPackets + packetsLost;
|
|
356
|
+
const packetLossRatio = packetLossDenominator === 0 ? 0 : packetsLost / packetLossDenominator;
|
|
357
|
+
const bytesReceived = inbound.reduce((total, stat) => total + (numericStat(stat, "bytesReceived") ?? 0), 0);
|
|
358
|
+
const bytesSent = outbound.reduce((total, stat) => total + (numericStat(stat, "bytesSent") ?? 0), 0);
|
|
359
|
+
const roundTripTimeMs = max(candidatePairs.map((stat) => secondsToMs(numericStat(stat, "currentRoundTripTime") ?? numericStat(stat, "roundTripTime"))).filter((value) => value !== undefined));
|
|
360
|
+
const jitterMs = max([...inbound, ...outbound].map((stat) => secondsToMs(numericStat(stat, "jitter"))).filter((value) => value !== undefined));
|
|
361
|
+
const jitterBufferDelayMs = max(inbound.map((stat) => {
|
|
362
|
+
const delay = numericStat(stat, "jitterBufferDelay");
|
|
363
|
+
const emitted = numericStat(stat, "jitterBufferEmittedCount");
|
|
364
|
+
return delay !== undefined && emitted !== undefined && emitted > 0 ? delay / emitted * 1000 : undefined;
|
|
365
|
+
}).filter((value) => value !== undefined));
|
|
366
|
+
const audioLevels = audioTracks.map((stat) => numericStat(stat, "audioLevel")).filter((value) => value !== undefined);
|
|
367
|
+
if (input.requireConnectedCandidatePair && candidatePairs.length > 0 && activeCandidatePairs === 0) {
|
|
368
|
+
pushIssue(issues, "error", "media.webrtc_candidate_pair_missing", "No active WebRTC candidate pair was observed.");
|
|
369
|
+
}
|
|
370
|
+
if (input.requireLiveAudioTrack && liveAudioTracks === 0) {
|
|
371
|
+
pushIssue(issues, "error", "media.webrtc_audio_track_missing", "No live WebRTC audio track was observed.");
|
|
372
|
+
}
|
|
373
|
+
if (input.maxPacketLossRatio !== undefined && packetLossRatio > input.maxPacketLossRatio) {
|
|
374
|
+
pushIssue(issues, "warning", "media.webrtc_packet_loss", `Observed WebRTC packet loss ratio ${String(packetLossRatio)} above ${String(input.maxPacketLossRatio)}.`);
|
|
375
|
+
}
|
|
376
|
+
if (input.maxRoundTripTimeMs !== undefined && roundTripTimeMs !== undefined && roundTripTimeMs > input.maxRoundTripTimeMs) {
|
|
377
|
+
pushIssue(issues, "warning", "media.webrtc_round_trip_time", `Observed WebRTC RTT ${String(roundTripTimeMs)}ms above ${String(input.maxRoundTripTimeMs)}ms.`);
|
|
378
|
+
}
|
|
379
|
+
if (input.maxJitterMs !== undefined && jitterMs !== undefined && jitterMs > input.maxJitterMs) {
|
|
380
|
+
pushIssue(issues, "warning", "media.webrtc_jitter", `Observed WebRTC jitter ${String(jitterMs)}ms above ${String(input.maxJitterMs)}ms.`);
|
|
381
|
+
}
|
|
382
|
+
return {
|
|
383
|
+
activeCandidatePairs,
|
|
384
|
+
audioLevelAverage: average(audioLevels),
|
|
385
|
+
bytesReceived,
|
|
386
|
+
bytesSent,
|
|
387
|
+
checkedAt: Date.now(),
|
|
388
|
+
endedAudioTracks,
|
|
389
|
+
inboundPackets,
|
|
390
|
+
issues,
|
|
391
|
+
jitterBufferDelayMs,
|
|
392
|
+
jitterMs,
|
|
393
|
+
liveAudioTracks,
|
|
394
|
+
outboundPackets,
|
|
395
|
+
packetLossRatio,
|
|
396
|
+
packetsLost,
|
|
397
|
+
roundTripTimeMs,
|
|
398
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
399
|
+
totalStats: stats.length
|
|
400
|
+
};
|
|
401
|
+
};
|
|
402
|
+
var collectMediaWebRTCStats = async (input) => {
|
|
403
|
+
const report = await input.peerConnection.getStats(input.selector ?? null);
|
|
404
|
+
return [...report.values()].map(normalizeWebRTCStat);
|
|
405
|
+
};
|
|
406
|
+
var buildMediaWebRTCStreamContinuityReport = (input = {}) => {
|
|
407
|
+
const stats = input.stats ?? [];
|
|
408
|
+
const previousStats = input.previousStats ?? [];
|
|
409
|
+
const issues = [];
|
|
410
|
+
const previousByKey = new Map(previousStats.map((stat) => [statKey(stat), stat]));
|
|
411
|
+
const audioRtp = stats.filter((stat) => (stat.type === "inbound-rtp" || stat.type === "outbound-rtp") && stringStat(stat, "kind") !== "video" && stringStat(stat, "mediaType") !== "video");
|
|
412
|
+
const streams = audioRtp.map((stat) => {
|
|
413
|
+
const direction = stat.type === "outbound-rtp" ? "outbound" : "inbound";
|
|
414
|
+
const packetsKey = direction === "outbound" ? "packetsSent" : "packetsReceived";
|
|
415
|
+
const bytesKey = direction === "outbound" ? "bytesSent" : "bytesReceived";
|
|
416
|
+
const previous = previousByKey.get(statKey(stat));
|
|
417
|
+
const currentPackets = numericStat(stat, packetsKey);
|
|
418
|
+
const previousPackets = previous ? numericStat(previous, packetsKey) : undefined;
|
|
419
|
+
const currentBytes = numericStat(stat, bytesKey);
|
|
420
|
+
const previousBytes = previous ? numericStat(previous, bytesKey) : undefined;
|
|
421
|
+
const timeDeltaMs = stat.timestamp !== undefined && previous?.timestamp !== undefined ? stat.timestamp - previous.timestamp : undefined;
|
|
422
|
+
return {
|
|
423
|
+
bytesDelta: currentBytes !== undefined && previousBytes !== undefined ? currentBytes - previousBytes : undefined,
|
|
424
|
+
currentPackets,
|
|
425
|
+
direction,
|
|
426
|
+
id: statKey(stat),
|
|
427
|
+
packetDelta: currentPackets !== undefined && previousPackets !== undefined ? currentPackets - previousPackets : undefined,
|
|
428
|
+
previousPackets,
|
|
429
|
+
timeDeltaMs
|
|
430
|
+
};
|
|
431
|
+
});
|
|
432
|
+
const inbound = streams.filter((stream) => stream.direction === "inbound");
|
|
433
|
+
const outbound = streams.filter((stream) => stream.direction === "outbound");
|
|
434
|
+
const maxObservedGapMs = max(streams.map((stream) => stream.timeDeltaMs).filter((value) => value !== undefined));
|
|
435
|
+
const stalledInboundStreams = inbound.filter((stream) => input.maxInboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxInboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
436
|
+
const stalledOutboundStreams = outbound.filter((stream) => input.maxOutboundPacketStallMs !== undefined && stream.timeDeltaMs !== undefined && stream.timeDeltaMs >= input.maxOutboundPacketStallMs && stream.packetDelta !== undefined && stream.packetDelta <= 0).length;
|
|
437
|
+
if (input.requireInboundAudio && inbound.length === 0) {
|
|
438
|
+
pushIssue(issues, "error", "media.webrtc_inbound_audio_missing", "No inbound WebRTC audio RTP stream was observed.");
|
|
439
|
+
}
|
|
440
|
+
if (input.requireOutboundAudio && outbound.length === 0) {
|
|
441
|
+
pushIssue(issues, "error", "media.webrtc_outbound_audio_missing", "No outbound WebRTC audio RTP stream was observed.");
|
|
442
|
+
}
|
|
443
|
+
if (input.maxGapMs !== undefined && maxObservedGapMs !== undefined && maxObservedGapMs > input.maxGapMs) {
|
|
444
|
+
pushIssue(issues, "warning", "media.webrtc_stream_gap", `Observed WebRTC stream sample gap ${String(maxObservedGapMs)}ms above ${String(input.maxGapMs)}ms.`);
|
|
445
|
+
}
|
|
446
|
+
if (stalledInboundStreams > 0) {
|
|
447
|
+
pushIssue(issues, "error", "media.webrtc_inbound_stalled", `${String(stalledInboundStreams)} inbound WebRTC audio stream(s) stopped receiving packets.`);
|
|
448
|
+
}
|
|
449
|
+
if (stalledOutboundStreams > 0) {
|
|
450
|
+
pushIssue(issues, "error", "media.webrtc_outbound_stalled", `${String(stalledOutboundStreams)} outbound WebRTC audio stream(s) stopped sending packets.`);
|
|
451
|
+
}
|
|
452
|
+
return {
|
|
453
|
+
checkedAt: Date.now(),
|
|
454
|
+
inboundAudioStreams: inbound.length,
|
|
455
|
+
issues,
|
|
456
|
+
maxObservedGapMs,
|
|
457
|
+
outboundAudioStreams: outbound.length,
|
|
458
|
+
stalledInboundStreams,
|
|
459
|
+
stalledOutboundStreams,
|
|
460
|
+
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
461
|
+
streams,
|
|
462
|
+
totalStats: stats.length
|
|
463
|
+
};
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
// src/client/browserMedia.ts
|
|
467
|
+
var DEFAULT_BROWSER_MEDIA_PATH = "/api/voice/browser-media";
|
|
468
|
+
var DEFAULT_BROWSER_MEDIA_INTERVAL_MS = 5000;
|
|
469
|
+
var resolvePeerConnection = async (options) => options.peerConnection ?? await options.getPeerConnection?.() ?? null;
|
|
470
|
+
var postBrowserMediaReport = async (payload, options) => {
|
|
471
|
+
const requestFetch = options.fetch ?? globalThis.fetch;
|
|
472
|
+
if (!requestFetch) {
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
await requestFetch(options.path ?? DEFAULT_BROWSER_MEDIA_PATH, {
|
|
476
|
+
body: JSON.stringify(payload),
|
|
477
|
+
headers: {
|
|
478
|
+
"Content-Type": "application/json"
|
|
479
|
+
},
|
|
480
|
+
keepalive: true,
|
|
481
|
+
method: "POST"
|
|
482
|
+
});
|
|
483
|
+
};
|
|
484
|
+
var createVoiceBrowserMediaReporter = (options) => {
|
|
485
|
+
let interval = null;
|
|
486
|
+
let previousStats = [];
|
|
487
|
+
const reportOnce = async () => {
|
|
488
|
+
const peerConnection = await resolvePeerConnection(options);
|
|
489
|
+
if (!peerConnection) {
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
const stats = await collectMediaWebRTCStats({ peerConnection });
|
|
493
|
+
const report = buildMediaWebRTCStatsReport({
|
|
494
|
+
...options,
|
|
495
|
+
stats
|
|
496
|
+
});
|
|
497
|
+
const continuity = options.continuity === false ? undefined : buildMediaWebRTCStreamContinuityReport({
|
|
498
|
+
...options.continuity,
|
|
499
|
+
previousStats,
|
|
500
|
+
stats
|
|
501
|
+
});
|
|
502
|
+
const payload = {
|
|
503
|
+
at: Date.now(),
|
|
504
|
+
continuity,
|
|
505
|
+
report,
|
|
506
|
+
scenarioId: options.getScenarioId?.() ?? null,
|
|
507
|
+
sessionId: options.getSessionId?.() ?? null
|
|
508
|
+
};
|
|
509
|
+
previousStats = stats;
|
|
510
|
+
options.onReport?.(payload);
|
|
511
|
+
await postBrowserMediaReport(payload, options);
|
|
512
|
+
return payload;
|
|
513
|
+
};
|
|
514
|
+
const run = () => {
|
|
515
|
+
reportOnce().catch((error) => {
|
|
516
|
+
options.onError?.(error);
|
|
517
|
+
});
|
|
518
|
+
};
|
|
519
|
+
const stop = () => {
|
|
520
|
+
if (interval) {
|
|
521
|
+
clearInterval(interval);
|
|
522
|
+
interval = null;
|
|
523
|
+
}
|
|
524
|
+
};
|
|
525
|
+
return {
|
|
526
|
+
close: stop,
|
|
527
|
+
reportOnce,
|
|
528
|
+
stop,
|
|
529
|
+
start: () => {
|
|
530
|
+
if (interval) {
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
run();
|
|
534
|
+
interval = setInterval(run, options.intervalMs ?? DEFAULT_BROWSER_MEDIA_INTERVAL_MS);
|
|
535
|
+
}
|
|
536
|
+
};
|
|
537
|
+
};
|
|
538
|
+
|
|
229
539
|
// src/client/connection.ts
|
|
230
540
|
var WS_OPEN = 1;
|
|
231
541
|
var WS_CLOSED = 3;
|
|
232
542
|
var WS_NORMAL_CLOSURE = 1000;
|
|
233
|
-
var
|
|
543
|
+
var WS_UNAUTHORIZED = 4401;
|
|
544
|
+
var WS_FORBIDDEN = 4403;
|
|
545
|
+
var WS_INITIALIZATION_FAILED = 4500;
|
|
546
|
+
var DEFAULT_MAX_RECONNECT_ATTEMPTS = 15;
|
|
234
547
|
var DEFAULT_PING_INTERVAL = 30000;
|
|
235
|
-
var
|
|
548
|
+
var DEFAULT_RECONNECT_RESET_AFTER_MS = 30000;
|
|
549
|
+
var DEFAULT_CALL_CONTROL_ACK_TIMEOUT_MS = 5000;
|
|
550
|
+
var DEFAULT_PREPARE_RECONNECT_TIMEOUT_MS = 1e4;
|
|
551
|
+
var MAX_PENDING_CONTROL_MESSAGES = 100;
|
|
552
|
+
var RECONNECT_BASE_DELAY_MS = 500;
|
|
553
|
+
var DEFAULT_RECONNECT_MAX_DELAY_MS = 8000;
|
|
554
|
+
var computeVoiceReconnectDelayMs = (attempt, baseMs, maxDelayMs) => Math.min(maxDelayMs, baseMs * 2 ** (Math.max(1, attempt) - 1));
|
|
236
555
|
var DEFAULT_SCENARIO_QUERY_PARAM = "scenarioId";
|
|
556
|
+
|
|
557
|
+
class VoiceReconnectRejectedError extends Error {
|
|
558
|
+
constructor(message = "Voice authorization could not be renewed.") {
|
|
559
|
+
super(message);
|
|
560
|
+
this.name = "VoiceReconnectRejectedError";
|
|
561
|
+
}
|
|
562
|
+
}
|
|
237
563
|
var noop = () => {};
|
|
564
|
+
var noopAsync = () => Promise.resolve();
|
|
238
565
|
var noopUnsubscribe = () => noop;
|
|
239
566
|
var NOOP_CONNECTION = {
|
|
240
|
-
callControl:
|
|
567
|
+
callControl: noopAsync,
|
|
241
568
|
close: noop,
|
|
569
|
+
disconnect: noop,
|
|
242
570
|
endTurn: noop,
|
|
571
|
+
send: noop,
|
|
572
|
+
sendAudio: noop,
|
|
573
|
+
simulateDisconnect: noop,
|
|
574
|
+
subscribe: noopUnsubscribe,
|
|
243
575
|
getReadyState: () => WS_CLOSED,
|
|
244
576
|
getScenarioId: () => "",
|
|
245
577
|
getSessionId: () => "",
|
|
246
|
-
|
|
247
|
-
sendAudio: noop,
|
|
248
|
-
start: () => {},
|
|
249
|
-
subscribe: noopUnsubscribe
|
|
578
|
+
start: () => {}
|
|
250
579
|
};
|
|
251
580
|
var createSessionId = () => crypto.randomUUID();
|
|
252
|
-
var buildWsUrl = (path, sessionId, scenarioId) => {
|
|
581
|
+
var buildWsUrl = (path, sessionId, scenarioId, query) => {
|
|
253
582
|
const { hostname, port, protocol } = window.location;
|
|
254
583
|
const wsProtocol = protocol === "https:" ? "wss:" : "ws:";
|
|
255
584
|
const portSuffix = port ? `:${port}` : "";
|
|
@@ -258,6 +587,11 @@ var buildWsUrl = (path, sessionId, scenarioId) => {
|
|
|
258
587
|
if (scenarioId) {
|
|
259
588
|
url.searchParams.set(DEFAULT_SCENARIO_QUERY_PARAM, scenarioId);
|
|
260
589
|
}
|
|
590
|
+
for (const [key, value] of Object.entries(query)) {
|
|
591
|
+
if (key !== "sessionId" && key !== DEFAULT_SCENARIO_QUERY_PARAM && value) {
|
|
592
|
+
url.searchParams.set(key, value);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
261
595
|
return url.toString();
|
|
262
596
|
};
|
|
263
597
|
var isVoiceServerMessage = (value) => {
|
|
@@ -268,11 +602,14 @@ var isVoiceServerMessage = (value) => {
|
|
|
268
602
|
case "audio":
|
|
269
603
|
case "assistant":
|
|
270
604
|
case "call_lifecycle":
|
|
605
|
+
case "call_control_ack":
|
|
271
606
|
case "complete":
|
|
607
|
+
case "connection":
|
|
272
608
|
case "error":
|
|
273
609
|
case "final":
|
|
274
610
|
case "partial":
|
|
275
611
|
case "pong":
|
|
612
|
+
case "replay":
|
|
276
613
|
case "session":
|
|
277
614
|
case "turn":
|
|
278
615
|
return true;
|
|
@@ -296,19 +633,45 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
296
633
|
return NOOP_CONNECTION;
|
|
297
634
|
}
|
|
298
635
|
const listeners = new Set;
|
|
636
|
+
const pendingCallControls = new Map;
|
|
299
637
|
const shouldReconnect = options.reconnect !== false;
|
|
300
638
|
const maxReconnectAttempts = options.maxReconnectAttempts ?? DEFAULT_MAX_RECONNECT_ATTEMPTS;
|
|
639
|
+
const reconnectMaxDelayMs = options.reconnectMaxDelayMs ?? DEFAULT_RECONNECT_MAX_DELAY_MS;
|
|
301
640
|
const pingInterval = options.pingInterval ?? DEFAULT_PING_INTERVAL;
|
|
641
|
+
const reconnectResetAfterMs = options.reconnectResetAfterMs ?? DEFAULT_RECONNECT_RESET_AFTER_MS;
|
|
642
|
+
let lifecycleGeneration = 0;
|
|
643
|
+
let disposed = false;
|
|
644
|
+
let reconnectPreparationAbort = null;
|
|
645
|
+
const computeReconnectDelayMs = (attempt) => computeVoiceReconnectDelayMs(attempt, RECONNECT_BASE_DELAY_MS, reconnectMaxDelayMs);
|
|
302
646
|
const state = {
|
|
303
647
|
isConnected: false,
|
|
304
648
|
pendingMessages: [],
|
|
305
649
|
scenarioId: options.scenarioId ?? null,
|
|
306
650
|
pingInterval: null,
|
|
307
651
|
reconnectAttempts: 0,
|
|
652
|
+
reconnectResetTimeout: null,
|
|
308
653
|
reconnectTimeout: null,
|
|
309
654
|
sessionId: options.sessionId ?? createSessionId(),
|
|
310
655
|
ws: null
|
|
311
656
|
};
|
|
657
|
+
const emitConnection = (reconnect) => {
|
|
658
|
+
listeners.forEach((listener) => listener(reconnect));
|
|
659
|
+
};
|
|
660
|
+
const emitTerminalFailure = (message) => {
|
|
661
|
+
listeners.forEach((listener) => listener({
|
|
662
|
+
message,
|
|
663
|
+
recoverable: false,
|
|
664
|
+
type: "error"
|
|
665
|
+
}));
|
|
666
|
+
emitConnection({
|
|
667
|
+
reconnect: {
|
|
668
|
+
attempts: state.reconnectAttempts,
|
|
669
|
+
maxAttempts: maxReconnectAttempts,
|
|
670
|
+
status: "exhausted"
|
|
671
|
+
},
|
|
672
|
+
type: "connection"
|
|
673
|
+
});
|
|
674
|
+
};
|
|
312
675
|
const clearTimers = () => {
|
|
313
676
|
if (state.pingInterval) {
|
|
314
677
|
clearInterval(state.pingInterval);
|
|
@@ -318,6 +681,10 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
318
681
|
clearTimeout(state.reconnectTimeout);
|
|
319
682
|
state.reconnectTimeout = null;
|
|
320
683
|
}
|
|
684
|
+
if (state.reconnectResetTimeout) {
|
|
685
|
+
clearTimeout(state.reconnectResetTimeout);
|
|
686
|
+
state.reconnectResetTimeout = null;
|
|
687
|
+
}
|
|
321
688
|
};
|
|
322
689
|
const flushPendingMessages = () => {
|
|
323
690
|
if (state.ws?.readyState !== WS_OPEN) {
|
|
@@ -330,28 +697,111 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
330
697
|
}
|
|
331
698
|
}
|
|
332
699
|
};
|
|
700
|
+
const rejectPendingCallControls = (message) => {
|
|
701
|
+
for (const pending of pendingCallControls.values()) {
|
|
702
|
+
clearTimeout(pending.timeout);
|
|
703
|
+
pending.reject(new Error(message));
|
|
704
|
+
}
|
|
705
|
+
pendingCallControls.clear();
|
|
706
|
+
};
|
|
333
707
|
const scheduleReconnect = () => {
|
|
708
|
+
if (disposed)
|
|
709
|
+
return;
|
|
710
|
+
const scheduledGeneration = lifecycleGeneration;
|
|
334
711
|
state.reconnectAttempts += 1;
|
|
335
|
-
|
|
712
|
+
const delayMs = computeReconnectDelayMs(state.reconnectAttempts);
|
|
713
|
+
const nextAttemptAt = Date.now() + delayMs;
|
|
714
|
+
emitConnection({
|
|
715
|
+
reconnect: {
|
|
716
|
+
attempts: state.reconnectAttempts,
|
|
717
|
+
lastDisconnectAt: Date.now(),
|
|
718
|
+
maxAttempts: maxReconnectAttempts,
|
|
719
|
+
nextAttemptAt,
|
|
720
|
+
status: "reconnecting"
|
|
721
|
+
},
|
|
722
|
+
type: "connection"
|
|
723
|
+
});
|
|
724
|
+
state.reconnectTimeout = setTimeout(async () => {
|
|
725
|
+
if (disposed || lifecycleGeneration !== scheduledGeneration)
|
|
726
|
+
return;
|
|
336
727
|
if (state.reconnectAttempts > maxReconnectAttempts) {
|
|
728
|
+
emitTerminalFailure("Voice connection could not be restored.");
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
try {
|
|
732
|
+
if (options.prepareReconnect) {
|
|
733
|
+
const preparationAbort = new AbortController;
|
|
734
|
+
reconnectPreparationAbort = preparationAbort;
|
|
735
|
+
const timeout = setTimeout(() => preparationAbort.abort("Voice reconnect preparation timed out"), options.prepareReconnectTimeoutMs ?? DEFAULT_PREPARE_RECONNECT_TIMEOUT_MS);
|
|
736
|
+
try {
|
|
737
|
+
await Promise.race([
|
|
738
|
+
options.prepareReconnect({
|
|
739
|
+
attempt: state.reconnectAttempts,
|
|
740
|
+
path,
|
|
741
|
+
scenarioId: state.scenarioId,
|
|
742
|
+
sessionId: state.sessionId,
|
|
743
|
+
signal: preparationAbort.signal
|
|
744
|
+
}),
|
|
745
|
+
new Promise((_, reject) => {
|
|
746
|
+
preparationAbort.signal.addEventListener("abort", () => reject(new Error("Voice reconnect preparation timed out.")), { once: true });
|
|
747
|
+
})
|
|
748
|
+
]);
|
|
749
|
+
} finally {
|
|
750
|
+
clearTimeout(timeout);
|
|
751
|
+
if (reconnectPreparationAbort === preparationAbort) {
|
|
752
|
+
reconnectPreparationAbort = null;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
} catch (error) {
|
|
757
|
+
if (disposed || lifecycleGeneration !== scheduledGeneration)
|
|
758
|
+
return;
|
|
759
|
+
if (error instanceof VoiceReconnectRejectedError) {
|
|
760
|
+
emitTerminalFailure(error.message);
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
if (state.reconnectAttempts < maxReconnectAttempts) {
|
|
764
|
+
scheduleReconnect();
|
|
765
|
+
} else {
|
|
766
|
+
emitTerminalFailure("Voice authorization could not be renewed for reconnect.");
|
|
767
|
+
}
|
|
337
768
|
return;
|
|
338
769
|
}
|
|
770
|
+
if (disposed || lifecycleGeneration !== scheduledGeneration)
|
|
771
|
+
return;
|
|
339
772
|
connect();
|
|
340
|
-
},
|
|
773
|
+
}, delayMs);
|
|
341
774
|
};
|
|
342
775
|
const connect = () => {
|
|
343
|
-
|
|
776
|
+
if (disposed)
|
|
777
|
+
return;
|
|
778
|
+
const ws = new WebSocket(buildWsUrl(path, state.sessionId, state.scenarioId, options.query ?? {}));
|
|
344
779
|
ws.binaryType = "arraybuffer";
|
|
345
780
|
ws.onopen = () => {
|
|
781
|
+
if (state.ws !== ws) {
|
|
782
|
+
ws.close(WS_NORMAL_CLOSURE);
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
const wasReconnecting = state.reconnectAttempts > 0;
|
|
346
786
|
state.isConnected = true;
|
|
347
|
-
state.reconnectAttempts = 0;
|
|
348
787
|
flushPendingMessages();
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
788
|
+
if (wasReconnecting) {
|
|
789
|
+
emitConnection({
|
|
790
|
+
reconnect: {
|
|
791
|
+
attempts: state.reconnectAttempts,
|
|
792
|
+
lastResumedAt: Date.now(),
|
|
793
|
+
maxAttempts: maxReconnectAttempts,
|
|
794
|
+
status: "resumed"
|
|
795
|
+
},
|
|
796
|
+
type: "connection"
|
|
797
|
+
});
|
|
798
|
+
state.reconnectResetTimeout = setTimeout(() => {
|
|
799
|
+
if (state.ws === ws && ws.readyState === WS_OPEN) {
|
|
800
|
+
state.reconnectAttempts = 0;
|
|
801
|
+
}
|
|
802
|
+
state.reconnectResetTimeout = null;
|
|
803
|
+
}, reconnectResetAfterMs);
|
|
804
|
+
}
|
|
355
805
|
state.pingInterval = setInterval(() => {
|
|
356
806
|
if (ws.readyState === WS_OPEN) {
|
|
357
807
|
ws.send(JSON.stringify({ type: "ping" }));
|
|
@@ -359,6 +809,8 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
359
809
|
}, pingInterval);
|
|
360
810
|
};
|
|
361
811
|
ws.onmessage = (event) => {
|
|
812
|
+
if (state.ws !== ws)
|
|
813
|
+
return;
|
|
362
814
|
const parsed = parseServerMessage(event);
|
|
363
815
|
if (!parsed) {
|
|
364
816
|
return;
|
|
@@ -367,29 +819,86 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
367
819
|
state.sessionId = parsed.sessionId;
|
|
368
820
|
state.scenarioId = parsed.scenarioId ?? state.scenarioId;
|
|
369
821
|
}
|
|
822
|
+
if (parsed.type === "call_control_ack") {
|
|
823
|
+
const pending = pendingCallControls.get(parsed.requestId);
|
|
824
|
+
if (pending) {
|
|
825
|
+
clearTimeout(pending.timeout);
|
|
826
|
+
pendingCallControls.delete(parsed.requestId);
|
|
827
|
+
if (parsed.ok) {
|
|
828
|
+
pending.resolve();
|
|
829
|
+
} else {
|
|
830
|
+
pending.reject(new Error(parsed.message ?? `${parsed.action} was not applied`));
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
370
834
|
listeners.forEach((listener) => listener(parsed));
|
|
371
835
|
};
|
|
372
836
|
ws.onclose = (event) => {
|
|
837
|
+
if (state.ws !== ws)
|
|
838
|
+
return;
|
|
839
|
+
state.ws = null;
|
|
373
840
|
state.isConnected = false;
|
|
374
841
|
clearTimers();
|
|
375
|
-
|
|
842
|
+
rejectPendingCallControls("Voice connection interrupted");
|
|
843
|
+
const reconnectable = shouldReconnect && event.code !== WS_NORMAL_CLOSURE && event.code !== WS_UNAUTHORIZED && event.code !== WS_FORBIDDEN && event.code !== WS_INITIALIZATION_FAILED && state.reconnectAttempts < maxReconnectAttempts;
|
|
844
|
+
const authorizationFailure = event.code === WS_UNAUTHORIZED || event.code === WS_FORBIDDEN;
|
|
845
|
+
const initializationFailure = event.code === WS_INITIALIZATION_FAILED;
|
|
846
|
+
if (authorizationFailure) {
|
|
847
|
+
listeners.forEach((listener) => listener({
|
|
848
|
+
message: event.code === WS_UNAUTHORIZED ? "Voice authorization expired or was rejected." : "Voice authorization was forbidden.",
|
|
849
|
+
recoverable: false,
|
|
850
|
+
type: "error"
|
|
851
|
+
}));
|
|
852
|
+
}
|
|
853
|
+
if (initializationFailure) {
|
|
854
|
+
listeners.forEach((listener) => listener({
|
|
855
|
+
message: "Voice provider initialization failed.",
|
|
856
|
+
recoverable: false,
|
|
857
|
+
type: "error"
|
|
858
|
+
}));
|
|
859
|
+
}
|
|
376
860
|
if (reconnectable) {
|
|
377
861
|
scheduleReconnect();
|
|
862
|
+
} else if (shouldReconnect && event.code !== WS_NORMAL_CLOSURE) {
|
|
863
|
+
if (authorizationFailure) {
|
|
864
|
+
emitConnection({
|
|
865
|
+
reconnect: {
|
|
866
|
+
attempts: state.reconnectAttempts,
|
|
867
|
+
lastDisconnectAt: Date.now(),
|
|
868
|
+
maxAttempts: maxReconnectAttempts,
|
|
869
|
+
status: "exhausted"
|
|
870
|
+
},
|
|
871
|
+
type: "connection"
|
|
872
|
+
});
|
|
873
|
+
} else if (!initializationFailure) {
|
|
874
|
+
emitTerminalFailure("Voice connection could not be restored.");
|
|
875
|
+
}
|
|
378
876
|
}
|
|
379
877
|
};
|
|
380
878
|
state.ws = ws;
|
|
381
879
|
};
|
|
382
880
|
const sendSerialized = (value) => {
|
|
881
|
+
if (disposed)
|
|
882
|
+
return;
|
|
383
883
|
if (state.ws?.readyState === WS_OPEN) {
|
|
384
884
|
state.ws.send(value);
|
|
385
885
|
return;
|
|
386
886
|
}
|
|
887
|
+
if (typeof value !== "string") {
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
if (state.pendingMessages.length >= MAX_PENDING_CONTROL_MESSAGES) {
|
|
891
|
+
state.pendingMessages.shift();
|
|
892
|
+
}
|
|
387
893
|
state.pendingMessages.push(value);
|
|
388
894
|
};
|
|
389
895
|
const send = (message) => {
|
|
390
896
|
sendSerialized(JSON.stringify(message));
|
|
391
897
|
};
|
|
392
898
|
const start = (input = {}) => {
|
|
899
|
+
if (input.sessionId && input.sessionId !== state.sessionId) {
|
|
900
|
+
throw new Error("Voice session switching requires a new connection instance.");
|
|
901
|
+
}
|
|
393
902
|
if (input.sessionId) {
|
|
394
903
|
state.sessionId = input.sessionId;
|
|
395
904
|
}
|
|
@@ -397,9 +906,9 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
397
906
|
state.scenarioId = input.scenarioId;
|
|
398
907
|
}
|
|
399
908
|
send({
|
|
400
|
-
|
|
909
|
+
scenarioId: state.scenarioId ?? undefined,
|
|
401
910
|
sessionId: state.sessionId,
|
|
402
|
-
|
|
911
|
+
type: "start"
|
|
403
912
|
});
|
|
404
913
|
};
|
|
405
914
|
const sendAudio = (audio) => {
|
|
@@ -409,20 +918,63 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
409
918
|
send({ type: "end_turn" });
|
|
410
919
|
};
|
|
411
920
|
const callControl = (message) => {
|
|
412
|
-
|
|
921
|
+
if (state.ws?.readyState !== WS_OPEN) {
|
|
922
|
+
return Promise.reject(new Error(`${message.action} is unavailable while disconnected`));
|
|
923
|
+
}
|
|
924
|
+
const requestId = crypto.randomUUID();
|
|
925
|
+
const { promise, reject, resolve } = Promise.withResolvers();
|
|
926
|
+
const timeout = setTimeout(() => {
|
|
927
|
+
pendingCallControls.delete(requestId);
|
|
928
|
+
reject(new Error(`${message.action} acknowledgement timed out`));
|
|
929
|
+
}, DEFAULT_CALL_CONTROL_ACK_TIMEOUT_MS);
|
|
930
|
+
pendingCallControls.set(requestId, { reject, resolve, timeout });
|
|
931
|
+
state.ws.send(JSON.stringify({
|
|
413
932
|
...message,
|
|
933
|
+
requestId,
|
|
414
934
|
type: "call_control"
|
|
415
|
-
});
|
|
935
|
+
}));
|
|
936
|
+
return promise;
|
|
937
|
+
};
|
|
938
|
+
const close = (reason = "client-close") => {
|
|
939
|
+
disposed = true;
|
|
940
|
+
reconnectPreparationAbort?.abort();
|
|
941
|
+
reconnectPreparationAbort = null;
|
|
942
|
+
lifecycleGeneration += 1;
|
|
943
|
+
clearTimers();
|
|
944
|
+
rejectPendingCallControls("Voice connection closed");
|
|
945
|
+
state.pendingMessages.length = 0;
|
|
946
|
+
if (state.ws) {
|
|
947
|
+
const ws = state.ws;
|
|
948
|
+
state.ws = null;
|
|
949
|
+
if (ws.readyState === WS_OPEN) {
|
|
950
|
+
ws.send(JSON.stringify({ reason, type: "close" }));
|
|
951
|
+
}
|
|
952
|
+
ws.close(WS_NORMAL_CLOSURE);
|
|
953
|
+
}
|
|
954
|
+
state.isConnected = false;
|
|
955
|
+
listeners.clear();
|
|
416
956
|
};
|
|
417
|
-
const
|
|
957
|
+
const disconnect = () => {
|
|
958
|
+
disposed = true;
|
|
959
|
+
reconnectPreparationAbort?.abort();
|
|
960
|
+
reconnectPreparationAbort = null;
|
|
961
|
+
lifecycleGeneration += 1;
|
|
418
962
|
clearTimers();
|
|
963
|
+
rejectPendingCallControls("Voice connection disconnected");
|
|
964
|
+
state.pendingMessages.length = 0;
|
|
419
965
|
if (state.ws) {
|
|
420
|
-
state.ws
|
|
966
|
+
const ws = state.ws;
|
|
421
967
|
state.ws = null;
|
|
968
|
+
ws.close(WS_NORMAL_CLOSURE);
|
|
422
969
|
}
|
|
423
970
|
state.isConnected = false;
|
|
424
971
|
listeners.clear();
|
|
425
972
|
};
|
|
973
|
+
const simulateDisconnect = () => {
|
|
974
|
+
if (state.ws?.readyState === WS_OPEN) {
|
|
975
|
+
state.ws.close(4000, "absolutejs-voice-reconnect-proof");
|
|
976
|
+
}
|
|
977
|
+
};
|
|
426
978
|
const subscribe = (callback) => {
|
|
427
979
|
listeners.add(callback);
|
|
428
980
|
return () => {
|
|
@@ -433,32 +985,64 @@ var createVoiceConnection = (path, options = {}) => {
|
|
|
433
985
|
return {
|
|
434
986
|
callControl,
|
|
435
987
|
close,
|
|
988
|
+
disconnect,
|
|
436
989
|
endTurn,
|
|
437
|
-
getReadyState: () => state.ws?.readyState ?? WS_CLOSED,
|
|
438
|
-
getScenarioId: () => state.scenarioId ?? "",
|
|
439
|
-
getSessionId: () => state.sessionId,
|
|
440
990
|
send,
|
|
441
991
|
sendAudio,
|
|
992
|
+
simulateDisconnect,
|
|
442
993
|
start,
|
|
443
|
-
subscribe
|
|
994
|
+
subscribe,
|
|
995
|
+
getReadyState: () => state.ws?.readyState ?? WS_CLOSED,
|
|
996
|
+
getScenarioId: () => state.scenarioId ?? "",
|
|
997
|
+
getSessionId: () => state.sessionId
|
|
444
998
|
};
|
|
445
999
|
};
|
|
446
1000
|
|
|
447
1001
|
// src/client/store.ts
|
|
1002
|
+
var createInitialReconnectState = () => ({
|
|
1003
|
+
attempts: 0,
|
|
1004
|
+
maxAttempts: 0,
|
|
1005
|
+
status: "idle"
|
|
1006
|
+
});
|
|
1007
|
+
var appendSegmentText = (accumulated, next) => {
|
|
1008
|
+
const nextText = next.trim().replace(/\s+/g, " ");
|
|
1009
|
+
if (!nextText)
|
|
1010
|
+
return accumulated;
|
|
1011
|
+
if (!accumulated)
|
|
1012
|
+
return nextText;
|
|
1013
|
+
if (accumulated === nextText || accumulated.endsWith(nextText)) {
|
|
1014
|
+
return accumulated;
|
|
1015
|
+
}
|
|
1016
|
+
if (nextText.includes(accumulated))
|
|
1017
|
+
return nextText;
|
|
1018
|
+
return `${accumulated} ${nextText}`;
|
|
1019
|
+
};
|
|
1020
|
+
var joinPartial = (finalized, interim) => {
|
|
1021
|
+
const interimText = interim.trim().replace(/\s+/g, " ");
|
|
1022
|
+
if (!finalized)
|
|
1023
|
+
return interimText;
|
|
1024
|
+
if (!interimText || finalized.endsWith(interimText))
|
|
1025
|
+
return finalized;
|
|
1026
|
+
return `${finalized} ${interimText}`;
|
|
1027
|
+
};
|
|
448
1028
|
var createInitialState = () => ({
|
|
449
1029
|
assistantAudio: [],
|
|
1030
|
+
assistantStreamingText: "",
|
|
450
1031
|
assistantTexts: [],
|
|
451
1032
|
call: null,
|
|
452
1033
|
error: null,
|
|
453
1034
|
isConnected: false,
|
|
454
|
-
scenarioId: null,
|
|
455
1035
|
partial: "",
|
|
1036
|
+
reconnect: createInitialReconnectState(),
|
|
1037
|
+
scenarioId: null,
|
|
456
1038
|
sessionId: null,
|
|
1039
|
+
sessionMetadata: null,
|
|
457
1040
|
status: "idle",
|
|
458
1041
|
turns: []
|
|
459
1042
|
});
|
|
460
1043
|
var createVoiceStreamStore = () => {
|
|
461
1044
|
let state = createInitialState();
|
|
1045
|
+
let turnFinalText = "";
|
|
462
1046
|
const subscribers = new Set;
|
|
463
1047
|
const notify = () => {
|
|
464
1048
|
subscribers.forEach((subscriber) => subscriber());
|
|
@@ -482,9 +1066,16 @@ var createVoiceStreamStore = () => {
|
|
|
482
1066
|
case "assistant":
|
|
483
1067
|
state = {
|
|
484
1068
|
...state,
|
|
1069
|
+
assistantStreamingText: "",
|
|
485
1070
|
assistantTexts: [...state.assistantTexts, action.text]
|
|
486
1071
|
};
|
|
487
1072
|
break;
|
|
1073
|
+
case "assistant_delta":
|
|
1074
|
+
state = {
|
|
1075
|
+
...state,
|
|
1076
|
+
assistantStreamingText: `${state.assistantStreamingText}${action.delta}`
|
|
1077
|
+
};
|
|
1078
|
+
break;
|
|
488
1079
|
case "complete":
|
|
489
1080
|
state = {
|
|
490
1081
|
...state,
|
|
@@ -509,7 +1100,19 @@ var createVoiceStreamStore = () => {
|
|
|
509
1100
|
case "connected":
|
|
510
1101
|
state = {
|
|
511
1102
|
...state,
|
|
512
|
-
isConnected: true
|
|
1103
|
+
isConnected: true,
|
|
1104
|
+
reconnect: state.reconnect.status === "reconnecting" ? {
|
|
1105
|
+
...state.reconnect,
|
|
1106
|
+
lastResumedAt: Date.now(),
|
|
1107
|
+
nextAttemptAt: undefined,
|
|
1108
|
+
status: "resumed"
|
|
1109
|
+
} : state.reconnect
|
|
1110
|
+
};
|
|
1111
|
+
break;
|
|
1112
|
+
case "connection":
|
|
1113
|
+
state = {
|
|
1114
|
+
...state,
|
|
1115
|
+
reconnect: action.reconnect
|
|
513
1116
|
};
|
|
514
1117
|
break;
|
|
515
1118
|
case "disconnected":
|
|
@@ -521,20 +1124,44 @@ var createVoiceStreamStore = () => {
|
|
|
521
1124
|
case "error":
|
|
522
1125
|
state = {
|
|
523
1126
|
...state,
|
|
524
|
-
error: action.message
|
|
1127
|
+
error: action.message,
|
|
1128
|
+
...action.recoverable === false ? { isConnected: false, status: "failed" } : {}
|
|
525
1129
|
};
|
|
526
1130
|
break;
|
|
527
1131
|
case "final":
|
|
1132
|
+
turnFinalText = appendSegmentText(turnFinalText, action.transcript.text);
|
|
528
1133
|
state = {
|
|
529
1134
|
...state,
|
|
530
|
-
partial:
|
|
531
|
-
turns: state.turns.map((turn) => turn)
|
|
1135
|
+
partial: turnFinalText
|
|
532
1136
|
};
|
|
533
1137
|
break;
|
|
534
1138
|
case "partial":
|
|
535
1139
|
state = {
|
|
536
1140
|
...state,
|
|
537
|
-
partial: action.transcript.text
|
|
1141
|
+
partial: joinPartial(turnFinalText, action.transcript.text)
|
|
1142
|
+
};
|
|
1143
|
+
break;
|
|
1144
|
+
case "replay":
|
|
1145
|
+
turnFinalText = action.partial;
|
|
1146
|
+
state = {
|
|
1147
|
+
...state,
|
|
1148
|
+
assistantStreamingText: "",
|
|
1149
|
+
assistantTexts: [...action.assistantTexts],
|
|
1150
|
+
call: action.call ?? null,
|
|
1151
|
+
error: null,
|
|
1152
|
+
isConnected: action.status === "active",
|
|
1153
|
+
partial: action.partial,
|
|
1154
|
+
reconnect: state.reconnect.status === "reconnecting" ? {
|
|
1155
|
+
...state.reconnect,
|
|
1156
|
+
lastResumedAt: Date.now(),
|
|
1157
|
+
nextAttemptAt: undefined,
|
|
1158
|
+
status: "resumed"
|
|
1159
|
+
} : state.reconnect,
|
|
1160
|
+
scenarioId: action.scenarioId ?? state.scenarioId,
|
|
1161
|
+
sessionId: action.sessionId,
|
|
1162
|
+
sessionMetadata: action.sessionMetadata ?? state.sessionMetadata,
|
|
1163
|
+
status: action.status,
|
|
1164
|
+
turns: [...action.turns]
|
|
538
1165
|
};
|
|
539
1166
|
break;
|
|
540
1167
|
case "session":
|
|
@@ -544,10 +1171,12 @@ var createVoiceStreamStore = () => {
|
|
|
544
1171
|
scenarioId: action.scenarioId ?? state.scenarioId,
|
|
545
1172
|
isConnected: action.status === "active",
|
|
546
1173
|
sessionId: action.sessionId,
|
|
1174
|
+
sessionMetadata: action.sessionMetadata ?? state.sessionMetadata,
|
|
547
1175
|
status: action.status
|
|
548
1176
|
};
|
|
549
1177
|
break;
|
|
550
1178
|
case "turn":
|
|
1179
|
+
turnFinalText = "";
|
|
551
1180
|
state = {
|
|
552
1181
|
...state,
|
|
553
1182
|
partial: "",
|
|
@@ -574,30 +1203,86 @@ var createVoiceStreamStore = () => {
|
|
|
574
1203
|
var createVoiceStream = (path, options = {}) => {
|
|
575
1204
|
const connection = createVoiceConnection(path, options);
|
|
576
1205
|
const store = createVoiceStreamStore();
|
|
1206
|
+
const browserMediaReporter = options.browserMedia && typeof window !== "undefined" ? createVoiceBrowserMediaReporter({
|
|
1207
|
+
...options.browserMedia,
|
|
1208
|
+
getScenarioId: () => options.browserMedia ? options.browserMedia.getScenarioId?.() ?? connection.getScenarioId() : connection.getScenarioId(),
|
|
1209
|
+
getSessionId: () => options.browserMedia ? options.browserMedia.getSessionId?.() ?? connection.getSessionId() : connection.getSessionId()
|
|
1210
|
+
}) : null;
|
|
577
1211
|
const subscribers = new Set;
|
|
1212
|
+
let disposed = false;
|
|
578
1213
|
const start = (input) => Promise.resolve().then(() => {
|
|
1214
|
+
if (disposed)
|
|
1215
|
+
return;
|
|
579
1216
|
if (!input?.sessionId && !input?.scenarioId) {
|
|
580
1217
|
return;
|
|
581
1218
|
}
|
|
582
1219
|
connection.start(input);
|
|
1220
|
+
browserMediaReporter?.start();
|
|
583
1221
|
});
|
|
584
1222
|
const notify = () => {
|
|
585
1223
|
subscribers.forEach((subscriber) => subscriber());
|
|
586
1224
|
};
|
|
1225
|
+
const reportReconnect = () => {
|
|
1226
|
+
if (!options.reconnectReportPath || typeof fetch === "undefined") {
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
const snapshot = store.getSnapshot();
|
|
1230
|
+
const body = JSON.stringify({
|
|
1231
|
+
at: Date.now(),
|
|
1232
|
+
reconnect: snapshot.reconnect,
|
|
1233
|
+
scenarioId: snapshot.scenarioId,
|
|
1234
|
+
sessionId: connection.getSessionId(),
|
|
1235
|
+
turnIds: snapshot.turns.map((turn) => turn.id)
|
|
1236
|
+
});
|
|
1237
|
+
fetch(options.reconnectReportPath, {
|
|
1238
|
+
body,
|
|
1239
|
+
headers: {
|
|
1240
|
+
"Content-Type": "application/json"
|
|
1241
|
+
},
|
|
1242
|
+
keepalive: true,
|
|
1243
|
+
method: "POST"
|
|
1244
|
+
}).catch(() => {});
|
|
1245
|
+
};
|
|
587
1246
|
const unsubscribeConnection = connection.subscribe((message) => {
|
|
588
1247
|
const action = serverMessageToAction(message);
|
|
589
1248
|
if (action) {
|
|
590
1249
|
store.dispatch(action);
|
|
1250
|
+
if (message.type === "connection") {
|
|
1251
|
+
reportReconnect();
|
|
1252
|
+
}
|
|
591
1253
|
notify();
|
|
592
1254
|
}
|
|
593
1255
|
});
|
|
594
1256
|
return {
|
|
1257
|
+
start,
|
|
1258
|
+
get assistantAudio() {
|
|
1259
|
+
return store.getSnapshot().assistantAudio;
|
|
1260
|
+
},
|
|
1261
|
+
get assistantTexts() {
|
|
1262
|
+
return store.getSnapshot().assistantTexts;
|
|
1263
|
+
},
|
|
1264
|
+
get assistantStreamingText() {
|
|
1265
|
+
return store.getSnapshot().assistantStreamingText;
|
|
1266
|
+
},
|
|
1267
|
+
get call() {
|
|
1268
|
+
return store.getSnapshot().call;
|
|
1269
|
+
},
|
|
595
1270
|
callControl(message) {
|
|
596
|
-
connection.callControl(message);
|
|
1271
|
+
return connection.callControl(message);
|
|
597
1272
|
},
|
|
598
|
-
close() {
|
|
1273
|
+
close(reason) {
|
|
1274
|
+
disposed = true;
|
|
599
1275
|
unsubscribeConnection();
|
|
600
|
-
|
|
1276
|
+
browserMediaReporter?.close();
|
|
1277
|
+
connection.close(reason);
|
|
1278
|
+
store.dispatch({ type: "disconnected" });
|
|
1279
|
+
notify();
|
|
1280
|
+
},
|
|
1281
|
+
disconnect() {
|
|
1282
|
+
disposed = true;
|
|
1283
|
+
unsubscribeConnection();
|
|
1284
|
+
browserMediaReporter?.close();
|
|
1285
|
+
connection.disconnect();
|
|
601
1286
|
store.dispatch({ type: "disconnected" });
|
|
602
1287
|
notify();
|
|
603
1288
|
},
|
|
@@ -616,44 +1301,43 @@ var createVoiceStream = (path, options = {}) => {
|
|
|
616
1301
|
get isConnected() {
|
|
617
1302
|
return store.getSnapshot().isConnected;
|
|
618
1303
|
},
|
|
619
|
-
get scenarioId() {
|
|
620
|
-
return store.getSnapshot().scenarioId;
|
|
621
|
-
},
|
|
622
|
-
start,
|
|
623
1304
|
get partial() {
|
|
624
1305
|
return store.getSnapshot().partial;
|
|
625
1306
|
},
|
|
626
|
-
get
|
|
627
|
-
return
|
|
1307
|
+
get reconnect() {
|
|
1308
|
+
return store.getSnapshot().reconnect;
|
|
628
1309
|
},
|
|
629
|
-
get
|
|
630
|
-
return store.getSnapshot().
|
|
1310
|
+
get scenarioId() {
|
|
1311
|
+
return store.getSnapshot().scenarioId;
|
|
631
1312
|
},
|
|
632
|
-
|
|
633
|
-
|
|
1313
|
+
sendAudio(audio) {
|
|
1314
|
+
connection.sendAudio(audio);
|
|
634
1315
|
},
|
|
635
|
-
get
|
|
636
|
-
return
|
|
1316
|
+
get sessionId() {
|
|
1317
|
+
return connection.getSessionId();
|
|
637
1318
|
},
|
|
638
|
-
get
|
|
639
|
-
return store.getSnapshot().
|
|
1319
|
+
get sessionMetadata() {
|
|
1320
|
+
return store.getSnapshot().sessionMetadata;
|
|
640
1321
|
},
|
|
641
|
-
|
|
642
|
-
|
|
1322
|
+
simulateDisconnect() {
|
|
1323
|
+
connection.simulateDisconnect();
|
|
643
1324
|
},
|
|
644
|
-
|
|
645
|
-
|
|
1325
|
+
get status() {
|
|
1326
|
+
return store.getSnapshot().status;
|
|
646
1327
|
},
|
|
647
1328
|
subscribe(subscriber) {
|
|
648
1329
|
subscribers.add(subscriber);
|
|
649
1330
|
return () => {
|
|
650
1331
|
subscribers.delete(subscriber);
|
|
651
1332
|
};
|
|
1333
|
+
},
|
|
1334
|
+
get turns() {
|
|
1335
|
+
return store.getSnapshot().turns;
|
|
652
1336
|
}
|
|
653
1337
|
};
|
|
654
1338
|
};
|
|
655
1339
|
|
|
656
|
-
// src/audioConditioning.ts
|
|
1340
|
+
// src/core/audioConditioning.ts
|
|
657
1341
|
var DEFAULT_TARGET_LEVEL = 0.08;
|
|
658
1342
|
var DEFAULT_MAX_GAIN = 3;
|
|
659
1343
|
var DEFAULT_NOISE_GATE_THRESHOLD = 0.006;
|
|
@@ -671,34 +1355,37 @@ var resolveAudioConditioningConfig = (config) => {
|
|
|
671
1355
|
};
|
|
672
1356
|
};
|
|
673
1357
|
|
|
674
|
-
// src/turnProfiles.ts
|
|
1358
|
+
// src/core/turnProfiles.ts
|
|
675
1359
|
var TURN_PROFILE_DEFAULTS = {
|
|
676
1360
|
balanced: {
|
|
677
1361
|
qualityProfile: "general",
|
|
1362
|
+
minSilenceMs: 400,
|
|
678
1363
|
silenceMs: 1400,
|
|
679
1364
|
speechThreshold: 0.012,
|
|
680
1365
|
transcriptStabilityMs: 1000
|
|
681
1366
|
},
|
|
682
1367
|
fast: {
|
|
683
1368
|
qualityProfile: "general",
|
|
1369
|
+
minSilenceMs: 300,
|
|
684
1370
|
silenceMs: 700,
|
|
685
1371
|
speechThreshold: 0.015,
|
|
686
1372
|
transcriptStabilityMs: 450
|
|
687
1373
|
},
|
|
688
1374
|
"long-form": {
|
|
689
1375
|
qualityProfile: "general",
|
|
1376
|
+
minSilenceMs: 600,
|
|
690
1377
|
silenceMs: 2200,
|
|
691
1378
|
speechThreshold: 0.01,
|
|
692
1379
|
transcriptStabilityMs: 1500
|
|
693
1380
|
}
|
|
694
1381
|
};
|
|
695
1382
|
var QUALITY_PROFILE_DEFAULTS = {
|
|
696
|
-
general: {},
|
|
697
1383
|
"accent-heavy": {
|
|
698
1384
|
silenceMs: 1200,
|
|
699
1385
|
speechThreshold: 0.01,
|
|
700
1386
|
transcriptStabilityMs: 1200
|
|
701
1387
|
},
|
|
1388
|
+
general: {},
|
|
702
1389
|
"noisy-room": {
|
|
703
1390
|
silenceMs: 2000,
|
|
704
1391
|
speechThreshold: 0.02,
|
|
@@ -717,16 +1404,18 @@ var resolveTurnDetectionConfig = (config) => {
|
|
|
717
1404
|
const qualityProfile = config?.qualityProfile ?? DEFAULT_QUALITY_PROFILE;
|
|
718
1405
|
const preset = TURN_PROFILE_DEFAULTS[profile];
|
|
719
1406
|
const quality = QUALITY_PROFILE_DEFAULTS[qualityProfile];
|
|
1407
|
+
const silenceMs = config?.silenceMs ?? quality.silenceMs ?? preset.silenceMs;
|
|
720
1408
|
return {
|
|
721
1409
|
profile,
|
|
722
1410
|
qualityProfile,
|
|
723
|
-
|
|
1411
|
+
minSilenceMs: Math.min(silenceMs, config?.minSilenceMs ?? quality.minSilenceMs ?? preset.minSilenceMs),
|
|
1412
|
+
silenceMs,
|
|
724
1413
|
speechThreshold: config?.speechThreshold ?? quality.speechThreshold ?? preset.speechThreshold,
|
|
725
1414
|
transcriptStabilityMs: config?.transcriptStabilityMs ?? quality.transcriptStabilityMs ?? preset.transcriptStabilityMs
|
|
726
1415
|
};
|
|
727
1416
|
};
|
|
728
1417
|
|
|
729
|
-
// src/presets.ts
|
|
1418
|
+
// src/core/presets.ts
|
|
730
1419
|
var PRESET_INPUTS = {
|
|
731
1420
|
chat: {
|
|
732
1421
|
audioConditioning: {
|
|
@@ -747,8 +1436,8 @@ var PRESET_INPUTS = {
|
|
|
747
1436
|
},
|
|
748
1437
|
sttLifecycle: "continuous",
|
|
749
1438
|
turnDetection: {
|
|
750
|
-
|
|
751
|
-
|
|
1439
|
+
profile: "balanced",
|
|
1440
|
+
qualityProfile: "short-command"
|
|
752
1441
|
}
|
|
753
1442
|
},
|
|
754
1443
|
default: {
|
|
@@ -763,8 +1452,8 @@ var PRESET_INPUTS = {
|
|
|
763
1452
|
},
|
|
764
1453
|
sttLifecycle: "continuous",
|
|
765
1454
|
turnDetection: {
|
|
766
|
-
|
|
767
|
-
|
|
1455
|
+
profile: "fast",
|
|
1456
|
+
qualityProfile: "general"
|
|
768
1457
|
}
|
|
769
1458
|
},
|
|
770
1459
|
dictation: {
|
|
@@ -786,8 +1475,8 @@ var PRESET_INPUTS = {
|
|
|
786
1475
|
},
|
|
787
1476
|
sttLifecycle: "continuous",
|
|
788
1477
|
turnDetection: {
|
|
789
|
-
|
|
790
|
-
|
|
1478
|
+
profile: "long-form",
|
|
1479
|
+
qualityProfile: "accent-heavy"
|
|
791
1480
|
}
|
|
792
1481
|
},
|
|
793
1482
|
"guided-intake": {
|
|
@@ -809,8 +1498,8 @@ var PRESET_INPUTS = {
|
|
|
809
1498
|
},
|
|
810
1499
|
sttLifecycle: "turn-scoped",
|
|
811
1500
|
turnDetection: {
|
|
812
|
-
|
|
813
|
-
|
|
1501
|
+
profile: "long-form",
|
|
1502
|
+
qualityProfile: "accent-heavy"
|
|
814
1503
|
}
|
|
815
1504
|
},
|
|
816
1505
|
"noisy-room": {
|
|
@@ -832,8 +1521,8 @@ var PRESET_INPUTS = {
|
|
|
832
1521
|
},
|
|
833
1522
|
sttLifecycle: "continuous",
|
|
834
1523
|
turnDetection: {
|
|
835
|
-
qualityProfile: "noisy-room",
|
|
836
1524
|
profile: "long-form",
|
|
1525
|
+
qualityProfile: "noisy-room",
|
|
837
1526
|
silenceMs: 2100,
|
|
838
1527
|
speechThreshold: 0.02,
|
|
839
1528
|
transcriptStabilityMs: 1650
|
|
@@ -858,8 +1547,8 @@ var PRESET_INPUTS = {
|
|
|
858
1547
|
},
|
|
859
1548
|
sttLifecycle: "continuous",
|
|
860
1549
|
turnDetection: {
|
|
861
|
-
qualityProfile: "noisy-room",
|
|
862
1550
|
profile: "long-form",
|
|
1551
|
+
qualityProfile: "noisy-room",
|
|
863
1552
|
silenceMs: 660,
|
|
864
1553
|
speechThreshold: 0.012,
|
|
865
1554
|
transcriptStabilityMs: 300
|
|
@@ -884,8 +1573,8 @@ var PRESET_INPUTS = {
|
|
|
884
1573
|
},
|
|
885
1574
|
sttLifecycle: "continuous",
|
|
886
1575
|
turnDetection: {
|
|
887
|
-
qualityProfile: "noisy-room",
|
|
888
1576
|
profile: "long-form",
|
|
1577
|
+
qualityProfile: "noisy-room",
|
|
889
1578
|
silenceMs: 620,
|
|
890
1579
|
speechThreshold: 0.012,
|
|
891
1580
|
transcriptStabilityMs: 280
|
|
@@ -910,8 +1599,8 @@ var PRESET_INPUTS = {
|
|
|
910
1599
|
},
|
|
911
1600
|
sttLifecycle: "continuous",
|
|
912
1601
|
turnDetection: {
|
|
913
|
-
|
|
914
|
-
|
|
1602
|
+
profile: "long-form",
|
|
1603
|
+
qualityProfile: "noisy-room"
|
|
915
1604
|
}
|
|
916
1605
|
}
|
|
917
1606
|
};
|
|
@@ -935,14 +1624,17 @@ var resolveVoiceRuntimePreset = (name = "default") => {
|
|
|
935
1624
|
// src/client/controller.ts
|
|
936
1625
|
var createInitialState2 = (stream) => ({
|
|
937
1626
|
assistantAudio: [...stream.assistantAudio],
|
|
1627
|
+
assistantStreamingText: stream.assistantStreamingText,
|
|
938
1628
|
assistantTexts: [...stream.assistantTexts],
|
|
939
1629
|
call: stream.call,
|
|
940
1630
|
error: stream.error,
|
|
941
1631
|
isConnected: stream.isConnected,
|
|
942
1632
|
isRecording: false,
|
|
943
1633
|
partial: stream.partial,
|
|
1634
|
+
reconnect: stream.reconnect,
|
|
944
1635
|
recordingError: null,
|
|
945
1636
|
sessionId: stream.sessionId,
|
|
1637
|
+
sessionMetadata: stream.sessionMetadata,
|
|
946
1638
|
scenarioId: stream.scenarioId,
|
|
947
1639
|
status: stream.status,
|
|
948
1640
|
turns: [...stream.turns]
|
|
@@ -954,6 +1646,9 @@ var createVoiceController = (path, options = {}) => {
|
|
|
954
1646
|
...options.connection
|
|
955
1647
|
});
|
|
956
1648
|
let capture = null;
|
|
1649
|
+
let captureGeneration = 0;
|
|
1650
|
+
let disposed = false;
|
|
1651
|
+
let startRecordingPromise = null;
|
|
957
1652
|
let state = createInitialState2(stream);
|
|
958
1653
|
const subscribers = new Set;
|
|
959
1654
|
const notify = () => {
|
|
@@ -965,12 +1660,15 @@ var createVoiceController = (path, options = {}) => {
|
|
|
965
1660
|
state = {
|
|
966
1661
|
...state,
|
|
967
1662
|
assistantAudio: [...stream.assistantAudio],
|
|
1663
|
+
assistantStreamingText: stream.assistantStreamingText,
|
|
968
1664
|
assistantTexts: [...stream.assistantTexts],
|
|
969
1665
|
call: stream.call,
|
|
970
1666
|
error: stream.error,
|
|
971
1667
|
isConnected: stream.isConnected,
|
|
972
1668
|
partial: stream.partial,
|
|
1669
|
+
reconnect: stream.reconnect,
|
|
973
1670
|
sessionId: stream.sessionId,
|
|
1671
|
+
sessionMetadata: stream.sessionMetadata,
|
|
974
1672
|
scenarioId: stream.scenarioId,
|
|
975
1673
|
status: stream.status,
|
|
976
1674
|
turns: [...stream.turns]
|
|
@@ -994,12 +1692,20 @@ var createVoiceController = (path, options = {}) => {
|
|
|
994
1692
|
capture = createMicrophoneCapture({
|
|
995
1693
|
channelCount: options.capture?.channelCount ?? preset.capture.channelCount,
|
|
996
1694
|
onLevel: options.capture?.onLevel,
|
|
997
|
-
onAudio: (audio) =>
|
|
998
|
-
|
|
1695
|
+
onAudio: (audio) => {
|
|
1696
|
+
if (options.capture?.onAudio) {
|
|
1697
|
+
options.capture.onAudio(audio, stream.sendAudio);
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
stream.sendAudio(audio);
|
|
1701
|
+
},
|
|
1702
|
+
sampleRateHz: options.capture?.sampleRateHz ?? preset.capture.sampleRateHz,
|
|
1703
|
+
...options.capture?.stream ? { stream: options.capture.stream } : {}
|
|
999
1704
|
});
|
|
1000
1705
|
return capture;
|
|
1001
1706
|
};
|
|
1002
1707
|
const stopRecording = () => {
|
|
1708
|
+
captureGeneration += 1;
|
|
1003
1709
|
capture?.stop();
|
|
1004
1710
|
capture = null;
|
|
1005
1711
|
state = {
|
|
@@ -1009,43 +1715,88 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1009
1715
|
notify();
|
|
1010
1716
|
};
|
|
1011
1717
|
const startRecording = async () => {
|
|
1718
|
+
if (disposed) {
|
|
1719
|
+
throw new Error("Voice controller is closed.");
|
|
1720
|
+
}
|
|
1012
1721
|
if (state.isRecording) {
|
|
1013
1722
|
return;
|
|
1014
1723
|
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1724
|
+
if (startRecordingPromise)
|
|
1725
|
+
return startRecordingPromise;
|
|
1726
|
+
const generation = ++captureGeneration;
|
|
1727
|
+
const activeCapture = ensureCapture();
|
|
1728
|
+
const operation = (async () => {
|
|
1729
|
+
try {
|
|
1730
|
+
state = {
|
|
1731
|
+
...state,
|
|
1732
|
+
recordingError: null
|
|
1733
|
+
};
|
|
1734
|
+
notify();
|
|
1735
|
+
await activeCapture.start();
|
|
1736
|
+
if (disposed || generation !== captureGeneration || capture !== activeCapture) {
|
|
1737
|
+
activeCapture.stop();
|
|
1738
|
+
return;
|
|
1739
|
+
}
|
|
1740
|
+
state = {
|
|
1741
|
+
...state,
|
|
1742
|
+
isRecording: true
|
|
1743
|
+
};
|
|
1744
|
+
notify();
|
|
1745
|
+
} catch (error) {
|
|
1746
|
+
activeCapture.stop();
|
|
1747
|
+
if (generation !== captureGeneration || capture !== activeCapture) {
|
|
1748
|
+
throw error;
|
|
1749
|
+
}
|
|
1750
|
+
capture = null;
|
|
1751
|
+
state = {
|
|
1752
|
+
...state,
|
|
1753
|
+
isRecording: false,
|
|
1754
|
+
recordingError: error instanceof Error ? error.message : String(error)
|
|
1755
|
+
};
|
|
1756
|
+
notify();
|
|
1757
|
+
throw error;
|
|
1758
|
+
}
|
|
1759
|
+
})();
|
|
1760
|
+
const pending = operation.finally(() => {
|
|
1761
|
+
if (startRecordingPromise === pending)
|
|
1762
|
+
startRecordingPromise = null;
|
|
1763
|
+
});
|
|
1764
|
+
startRecordingPromise = pending;
|
|
1765
|
+
return startRecordingPromise;
|
|
1037
1766
|
};
|
|
1038
|
-
const close = () => {
|
|
1767
|
+
const close = (reason) => {
|
|
1768
|
+
disposed = true;
|
|
1039
1769
|
unsubscribeStream();
|
|
1040
1770
|
stopRecording();
|
|
1041
|
-
stream.close();
|
|
1771
|
+
stream.close(reason);
|
|
1772
|
+
};
|
|
1773
|
+
const disconnect = () => {
|
|
1774
|
+
disposed = true;
|
|
1775
|
+
unsubscribeStream();
|
|
1776
|
+
stopRecording();
|
|
1777
|
+
stream.disconnect();
|
|
1042
1778
|
};
|
|
1043
1779
|
return {
|
|
1044
|
-
|
|
1780
|
+
close,
|
|
1781
|
+
disconnect,
|
|
1782
|
+
startRecording,
|
|
1783
|
+
stopRecording,
|
|
1784
|
+
get assistantAudio() {
|
|
1785
|
+
return state.assistantAudio;
|
|
1786
|
+
},
|
|
1787
|
+
get assistantTexts() {
|
|
1788
|
+
return state.assistantTexts;
|
|
1789
|
+
},
|
|
1790
|
+
get assistantStreamingText() {
|
|
1791
|
+
return state.assistantStreamingText;
|
|
1792
|
+
},
|
|
1793
|
+
bindHTMX(bindingOptions) {
|
|
1045
1794
|
return bindVoiceHTMX(stream, bindingOptions);
|
|
1046
1795
|
},
|
|
1796
|
+
get call() {
|
|
1797
|
+
return state.call;
|
|
1798
|
+
},
|
|
1047
1799
|
callControl: (message) => stream.callControl(message),
|
|
1048
|
-
close,
|
|
1049
1800
|
endTurn: () => stream.endTurn(),
|
|
1050
1801
|
get error() {
|
|
1051
1802
|
return state.error;
|
|
@@ -1061,21 +1812,26 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1061
1812
|
get partial() {
|
|
1062
1813
|
return state.partial;
|
|
1063
1814
|
},
|
|
1815
|
+
get reconnect() {
|
|
1816
|
+
return state.reconnect;
|
|
1817
|
+
},
|
|
1064
1818
|
get recordingError() {
|
|
1065
1819
|
return state.recordingError;
|
|
1066
1820
|
},
|
|
1821
|
+
get scenarioId() {
|
|
1822
|
+
return state.scenarioId;
|
|
1823
|
+
},
|
|
1067
1824
|
sendAudio: (audio) => stream.sendAudio(audio),
|
|
1068
1825
|
get sessionId() {
|
|
1069
1826
|
return state.sessionId;
|
|
1070
1827
|
},
|
|
1071
|
-
get
|
|
1072
|
-
return state.
|
|
1828
|
+
get sessionMetadata() {
|
|
1829
|
+
return state.sessionMetadata;
|
|
1073
1830
|
},
|
|
1074
|
-
|
|
1831
|
+
simulateDisconnect: () => stream.simulateDisconnect(),
|
|
1075
1832
|
get status() {
|
|
1076
1833
|
return state.status;
|
|
1077
1834
|
},
|
|
1078
|
-
stopRecording,
|
|
1079
1835
|
subscribe: (subscriber) => {
|
|
1080
1836
|
subscribers.add(subscriber);
|
|
1081
1837
|
return () => {
|
|
@@ -1091,15 +1847,778 @@ var createVoiceController = (path, options = {}) => {
|
|
|
1091
1847
|
},
|
|
1092
1848
|
get turns() {
|
|
1093
1849
|
return state.turns;
|
|
1850
|
+
}
|
|
1851
|
+
};
|
|
1852
|
+
};
|
|
1853
|
+
|
|
1854
|
+
// src/client/timeStretch.ts
|
|
1855
|
+
var HOP_MS = 10;
|
|
1856
|
+
var SEEK_MS = 5;
|
|
1857
|
+
var ENERGY_EPSILON = 0.000001;
|
|
1858
|
+
var HALF = 0.5;
|
|
1859
|
+
var MS_PER_SECOND = 1000;
|
|
1860
|
+
var makeHann = (length) => {
|
|
1861
|
+
const weights = new Float32Array(length);
|
|
1862
|
+
for (let index = 0;index < length; index += 1) {
|
|
1863
|
+
weights[index] = HALF - HALF * Math.cos(2 * Math.PI * index / length);
|
|
1864
|
+
}
|
|
1865
|
+
return weights;
|
|
1866
|
+
};
|
|
1867
|
+
var correlationScore = (base, start, ref, length) => {
|
|
1868
|
+
let dot = 0;
|
|
1869
|
+
let energy = 0;
|
|
1870
|
+
for (let index = 0;index < length; index += 1) {
|
|
1871
|
+
const sample = base[start + index] ?? 0;
|
|
1872
|
+
dot += sample * (ref[index] ?? 0);
|
|
1873
|
+
energy += sample * sample;
|
|
1874
|
+
}
|
|
1875
|
+
return dot / Math.sqrt(energy + ENERGY_EPSILON);
|
|
1876
|
+
};
|
|
1877
|
+
var overlapAddGrain = (src, off, tail, weights, hop) => {
|
|
1878
|
+
const out = new Float32Array(hop);
|
|
1879
|
+
const nextTail = new Float32Array(hop);
|
|
1880
|
+
for (let index = 0;index < hop; index += 1) {
|
|
1881
|
+
out[index] = (tail[index] ?? 0) + (src[off + index] ?? 0) * (weights[index] ?? 0);
|
|
1882
|
+
nextTail[index] = (src[off + hop + index] ?? 0) * (weights[hop + index] ?? 0);
|
|
1883
|
+
}
|
|
1884
|
+
return { nextTail, out };
|
|
1885
|
+
};
|
|
1886
|
+
var createTimeStretcher = () => {
|
|
1887
|
+
let sampleRate = 0;
|
|
1888
|
+
let channelCount = 0;
|
|
1889
|
+
let hop = 0;
|
|
1890
|
+
let frameLen = 0;
|
|
1891
|
+
let seek = 0;
|
|
1892
|
+
let weights = new Float32Array(0);
|
|
1893
|
+
let buffers = [];
|
|
1894
|
+
let inputStart = 0;
|
|
1895
|
+
let analysisPos = 0;
|
|
1896
|
+
let olaTail = [];
|
|
1897
|
+
let naturalRef = null;
|
|
1898
|
+
const init = (rate, channels) => {
|
|
1899
|
+
sampleRate = rate;
|
|
1900
|
+
channelCount = channels;
|
|
1901
|
+
hop = Math.max(1, Math.round(sampleRate * HOP_MS / MS_PER_SECOND));
|
|
1902
|
+
frameLen = hop * 2;
|
|
1903
|
+
seek = Math.max(1, Math.round(sampleRate * SEEK_MS / MS_PER_SECOND));
|
|
1904
|
+
weights = makeHann(frameLen);
|
|
1905
|
+
buffers = Array.from({ length: channels }, () => new Float32Array(0));
|
|
1906
|
+
olaTail = Array.from({ length: channels }, () => new Float32Array(hop));
|
|
1907
|
+
inputStart = 0;
|
|
1908
|
+
analysisPos = seek;
|
|
1909
|
+
naturalRef = null;
|
|
1910
|
+
};
|
|
1911
|
+
const reset = () => {
|
|
1912
|
+
buffers = buffers.map(() => new Float32Array(0));
|
|
1913
|
+
olaTail = olaTail.map(() => new Float32Array(hop));
|
|
1914
|
+
inputStart = 0;
|
|
1915
|
+
analysisPos = seek;
|
|
1916
|
+
naturalRef = null;
|
|
1917
|
+
};
|
|
1918
|
+
const append = (input) => {
|
|
1919
|
+
for (let channel = 0;channel < channelCount; channel += 1) {
|
|
1920
|
+
const incoming = input[channel] ?? input[0] ?? new Float32Array(0);
|
|
1921
|
+
const existing = buffers[channel] ?? new Float32Array(0);
|
|
1922
|
+
const merged = new Float32Array(existing.length + incoming.length);
|
|
1923
|
+
merged.set(existing, 0);
|
|
1924
|
+
merged.set(incoming, existing.length);
|
|
1925
|
+
buffers[channel] = merged;
|
|
1926
|
+
}
|
|
1927
|
+
};
|
|
1928
|
+
const inputEnd = () => inputStart + (buffers[0]?.length ?? 0);
|
|
1929
|
+
const compact = () => {
|
|
1930
|
+
const keepFrom = Math.max(inputStart, Math.floor(analysisPos) - seek - 1);
|
|
1931
|
+
if (keepFrom <= inputStart)
|
|
1932
|
+
return;
|
|
1933
|
+
const drop = keepFrom - inputStart;
|
|
1934
|
+
for (let channel = 0;channel < channelCount; channel += 1) {
|
|
1935
|
+
buffers[channel] = (buffers[channel] ?? new Float32Array(0)).slice(drop);
|
|
1936
|
+
}
|
|
1937
|
+
inputStart = keepFrom;
|
|
1938
|
+
};
|
|
1939
|
+
const bestOffset = (center) => {
|
|
1940
|
+
if (!naturalRef)
|
|
1941
|
+
return 0;
|
|
1942
|
+
const [base] = buffers;
|
|
1943
|
+
if (!base)
|
|
1944
|
+
return 0;
|
|
1945
|
+
let bestDelta = 0;
|
|
1946
|
+
let bestScore = -Infinity;
|
|
1947
|
+
for (let delta = -seek;delta <= seek; delta += 1) {
|
|
1948
|
+
const score = correlationScore(base, center + delta - inputStart, naturalRef, frameLen);
|
|
1949
|
+
if (score <= bestScore)
|
|
1950
|
+
continue;
|
|
1951
|
+
bestScore = score;
|
|
1952
|
+
bestDelta = delta;
|
|
1953
|
+
}
|
|
1954
|
+
return bestDelta;
|
|
1955
|
+
};
|
|
1956
|
+
const process = (input, speed, rate) => {
|
|
1957
|
+
const channels = Math.max(1, input.length);
|
|
1958
|
+
if (sampleRate !== rate || channelCount !== channels)
|
|
1959
|
+
init(rate, channels);
|
|
1960
|
+
append(input);
|
|
1961
|
+
const analysisHop = hop * speed;
|
|
1962
|
+
const segments = Array.from({ length: channelCount }, () => []);
|
|
1963
|
+
const emitGrain = (pos) => {
|
|
1964
|
+
const off = pos - inputStart;
|
|
1965
|
+
for (let channel = 0;channel < channelCount; channel += 1) {
|
|
1966
|
+
const src = buffers[channel];
|
|
1967
|
+
const tail = olaTail[channel];
|
|
1968
|
+
if (!src || !tail)
|
|
1969
|
+
continue;
|
|
1970
|
+
const grain = overlapAddGrain(src, off, tail, weights, hop);
|
|
1971
|
+
olaTail[channel] = grain.nextTail;
|
|
1972
|
+
segments[channel]?.push(grain.out);
|
|
1973
|
+
}
|
|
1974
|
+
};
|
|
1975
|
+
const captureRef = (pos) => {
|
|
1976
|
+
const ref = new Float32Array(frameLen);
|
|
1977
|
+
const refOff = pos + hop - inputStart;
|
|
1978
|
+
const [base] = buffers;
|
|
1979
|
+
if (base)
|
|
1980
|
+
ref.set(base.subarray(refOff, refOff + frameLen));
|
|
1981
|
+
naturalRef = ref;
|
|
1982
|
+
};
|
|
1983
|
+
const canEmit = () => Math.floor(analysisPos) - seek >= inputStart && Math.floor(analysisPos) + seek + frameLen + hop <= inputEnd();
|
|
1984
|
+
while (canEmit()) {
|
|
1985
|
+
const center = Math.round(analysisPos);
|
|
1986
|
+
const pos = center + bestOffset(center);
|
|
1987
|
+
emitGrain(pos);
|
|
1988
|
+
captureRef(pos);
|
|
1989
|
+
analysisPos += analysisHop;
|
|
1990
|
+
}
|
|
1991
|
+
compact();
|
|
1992
|
+
return segments.map((channelSegments) => {
|
|
1993
|
+
const total = channelSegments.reduce((sum, seg) => sum + seg.length, 0);
|
|
1994
|
+
const merged = new Float32Array(total);
|
|
1995
|
+
let offset = 0;
|
|
1996
|
+
for (const seg of channelSegments) {
|
|
1997
|
+
merged.set(seg, offset);
|
|
1998
|
+
offset += seg.length;
|
|
1999
|
+
}
|
|
2000
|
+
return merged;
|
|
2001
|
+
});
|
|
2002
|
+
};
|
|
2003
|
+
return { process, reset };
|
|
2004
|
+
};
|
|
2005
|
+
|
|
2006
|
+
// src/client/audioPlayer.ts
|
|
2007
|
+
var activeAudioPlayers = new Set;
|
|
2008
|
+
var DEFAULT_LOOKAHEAD_MS = 15;
|
|
2009
|
+
var DEFAULT_VOLUME = 1;
|
|
2010
|
+
var DEFAULT_PLAYBACK_RATE = 1;
|
|
2011
|
+
var MIN_PLAYBACK_RATE = 0.5;
|
|
2012
|
+
var MAX_PLAYBACK_RATE = 2;
|
|
2013
|
+
var STRETCH_BYPASS_EPSILON = 0.01;
|
|
2014
|
+
var ANALYSER_FFT_SIZE = 256;
|
|
2015
|
+
var PCM_BYTE_MIDPOINT = 128;
|
|
2016
|
+
var GAP_EPSILON_S = 0.03;
|
|
2017
|
+
var createInitialState3 = () => ({
|
|
2018
|
+
activeSourceCount: 0,
|
|
2019
|
+
error: null,
|
|
2020
|
+
isActive: false,
|
|
2021
|
+
isPlaying: false,
|
|
2022
|
+
lastInterruptLatencyMs: undefined,
|
|
2023
|
+
lastPlaybackStopLatencyMs: undefined,
|
|
2024
|
+
processedChunkCount: 0,
|
|
2025
|
+
queuedChunkCount: 0
|
|
2026
|
+
});
|
|
2027
|
+
var getAudioContextCtor = () => {
|
|
2028
|
+
if (typeof window === "undefined") {
|
|
2029
|
+
return typeof AudioContext === "undefined" ? undefined : AudioContext;
|
|
2030
|
+
}
|
|
2031
|
+
return window.AudioContext ?? window.webkitAudioContext;
|
|
2032
|
+
};
|
|
2033
|
+
var clampVolume = (volume) => {
|
|
2034
|
+
if (typeof volume !== "number" || !Number.isFinite(volume)) {
|
|
2035
|
+
return DEFAULT_VOLUME;
|
|
2036
|
+
}
|
|
2037
|
+
return Math.min(1, Math.max(0, volume));
|
|
2038
|
+
};
|
|
2039
|
+
var clampPlaybackRate = (rate) => {
|
|
2040
|
+
if (typeof rate !== "number" || !Number.isFinite(rate)) {
|
|
2041
|
+
return DEFAULT_PLAYBACK_RATE;
|
|
2042
|
+
}
|
|
2043
|
+
return Math.min(MAX_PLAYBACK_RATE, Math.max(MIN_PLAYBACK_RATE, rate));
|
|
2044
|
+
};
|
|
2045
|
+
var decodePCM16LEChunk = (audioContext, chunk) => {
|
|
2046
|
+
const { format } = chunk;
|
|
2047
|
+
if (format.container !== "raw" || format.encoding !== "pcm_s16le") {
|
|
2048
|
+
throw new Error(`Unsupported assistant audio format: ${format.container}/${format.encoding}`);
|
|
2049
|
+
}
|
|
2050
|
+
const bytes = chunk.chunk;
|
|
2051
|
+
const channels = Math.max(1, format.channels);
|
|
2052
|
+
const sampleCount = Math.floor(bytes.byteLength / 2);
|
|
2053
|
+
const frameCount = Math.max(1, Math.floor(sampleCount / channels));
|
|
2054
|
+
const audioBuffer = audioContext.createBuffer(channels, frameCount, format.sampleRateHz);
|
|
2055
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2056
|
+
for (let channelIndex = 0;channelIndex < channels; channelIndex += 1) {
|
|
2057
|
+
const channelData = audioBuffer.getChannelData(channelIndex);
|
|
2058
|
+
for (let frameIndex = 0;frameIndex < frameCount; frameIndex += 1) {
|
|
2059
|
+
const sampleIndex = frameIndex * channels + channelIndex;
|
|
2060
|
+
const sampleOffset = sampleIndex * 2;
|
|
2061
|
+
if (sampleOffset + 1 >= bytes.byteLength) {
|
|
2062
|
+
channelData[frameIndex] = 0;
|
|
2063
|
+
continue;
|
|
2064
|
+
}
|
|
2065
|
+
channelData[frameIndex] = view.getInt16(sampleOffset, true) / 32768;
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
return audioBuffer;
|
|
2069
|
+
};
|
|
2070
|
+
var createVoiceAudioPlayer = (source, options = {}) => {
|
|
2071
|
+
const subscribers = new Set;
|
|
2072
|
+
const sourceNodes = new Set;
|
|
2073
|
+
const lookaheadSeconds = (options.lookaheadMs ?? DEFAULT_LOOKAHEAD_MS) / 1000;
|
|
2074
|
+
let state = createInitialState3();
|
|
2075
|
+
let audioContext = null;
|
|
2076
|
+
let outputNode = null;
|
|
2077
|
+
let analyserNode = null;
|
|
2078
|
+
let analyserBuffer = null;
|
|
2079
|
+
let volume = clampVolume(options.volume);
|
|
2080
|
+
let playbackRate = clampPlaybackRate(options.playbackRate);
|
|
2081
|
+
let stretcher = null;
|
|
2082
|
+
let queueEndTime = 0;
|
|
2083
|
+
let syncPromise = Promise.resolve();
|
|
2084
|
+
let interruptStartedAt = null;
|
|
2085
|
+
let interruptPromise = null;
|
|
2086
|
+
let resolveInterruptPromise = null;
|
|
2087
|
+
let interruptFallbackTimer = null;
|
|
2088
|
+
let concurrencyCounted = false;
|
|
2089
|
+
let observedPeakConcurrency = 0;
|
|
2090
|
+
let scheduledDurationMs = 0;
|
|
2091
|
+
let gapCount = 0;
|
|
2092
|
+
let totalGapMs = 0;
|
|
2093
|
+
let maxGapMs = 0;
|
|
2094
|
+
let errorCount = 0;
|
|
2095
|
+
let hasScheduledAnyChunk = false;
|
|
2096
|
+
const sampleConcurrency = () => {
|
|
2097
|
+
if (activeAudioPlayers.size > observedPeakConcurrency) {
|
|
2098
|
+
observedPeakConcurrency = activeAudioPlayers.size;
|
|
2099
|
+
}
|
|
2100
|
+
};
|
|
2101
|
+
const markPlayerActive = () => {
|
|
2102
|
+
if (concurrencyCounted) {
|
|
2103
|
+
return;
|
|
2104
|
+
}
|
|
2105
|
+
concurrencyCounted = true;
|
|
2106
|
+
activeAudioPlayers.add(sampleConcurrency);
|
|
2107
|
+
for (const sample of activeAudioPlayers) {
|
|
2108
|
+
sample();
|
|
2109
|
+
}
|
|
2110
|
+
};
|
|
2111
|
+
const markPlayerInactive = () => {
|
|
2112
|
+
if (!concurrencyCounted) {
|
|
2113
|
+
return;
|
|
2114
|
+
}
|
|
2115
|
+
concurrencyCounted = false;
|
|
2116
|
+
activeAudioPlayers.delete(sampleConcurrency);
|
|
2117
|
+
};
|
|
2118
|
+
const notify = () => {
|
|
2119
|
+
for (const subscriber of subscribers) {
|
|
2120
|
+
subscriber();
|
|
2121
|
+
}
|
|
2122
|
+
};
|
|
2123
|
+
const setState = (next) => {
|
|
2124
|
+
state = {
|
|
2125
|
+
...state,
|
|
2126
|
+
...next
|
|
2127
|
+
};
|
|
2128
|
+
notify();
|
|
2129
|
+
};
|
|
2130
|
+
const clearError = () => {
|
|
2131
|
+
if (state.error !== null) {
|
|
2132
|
+
setState({ error: null });
|
|
2133
|
+
}
|
|
2134
|
+
};
|
|
2135
|
+
const clearInterruptTimer = () => {
|
|
2136
|
+
if (interruptFallbackTimer !== null) {
|
|
2137
|
+
clearTimeout(interruptFallbackTimer);
|
|
2138
|
+
interruptFallbackTimer = null;
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
const resolveInterrupt = (latencyMs) => {
|
|
2142
|
+
clearInterruptTimer();
|
|
2143
|
+
interruptStartedAt = null;
|
|
2144
|
+
stretcher?.reset();
|
|
2145
|
+
setState({
|
|
2146
|
+
activeSourceCount: sourceNodes.size,
|
|
2147
|
+
isPlaying: false,
|
|
2148
|
+
lastInterruptLatencyMs: latencyMs,
|
|
2149
|
+
lastPlaybackStopLatencyMs: state.lastPlaybackStopLatencyMs ?? latencyMs
|
|
2150
|
+
});
|
|
2151
|
+
resolveInterruptPromise?.();
|
|
2152
|
+
resolveInterruptPromise = null;
|
|
2153
|
+
interruptPromise = null;
|
|
2154
|
+
};
|
|
2155
|
+
const estimateOutputStopLatencyMs = (context) => {
|
|
2156
|
+
if (!context) {
|
|
2157
|
+
return 0;
|
|
2158
|
+
}
|
|
2159
|
+
return Math.max(0, ((context.baseLatency ?? 0) + (context.outputLatency ?? 0)) * 1000);
|
|
2160
|
+
};
|
|
2161
|
+
const applyOutputGain = (context) => {
|
|
2162
|
+
if (!outputNode) {
|
|
2163
|
+
return;
|
|
2164
|
+
}
|
|
2165
|
+
const gainValue = volume;
|
|
2166
|
+
if (outputNode.gain.setValueAtTime) {
|
|
2167
|
+
outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
|
|
2168
|
+
return;
|
|
2169
|
+
}
|
|
2170
|
+
outputNode.gain.value = gainValue;
|
|
2171
|
+
};
|
|
2172
|
+
const muteOutputGain = (context) => {
|
|
2173
|
+
if (!outputNode) {
|
|
2174
|
+
return;
|
|
2175
|
+
}
|
|
2176
|
+
const gainValue = 0;
|
|
2177
|
+
if (outputNode.gain.setValueAtTime) {
|
|
2178
|
+
outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
|
|
2179
|
+
return;
|
|
2180
|
+
}
|
|
2181
|
+
outputNode.gain.value = gainValue;
|
|
2182
|
+
};
|
|
2183
|
+
const maybeResolveInterrupt = () => {
|
|
2184
|
+
if (interruptStartedAt === null || sourceNodes.size > 0) {
|
|
2185
|
+
return;
|
|
2186
|
+
}
|
|
2187
|
+
resolveInterrupt(Date.now() - interruptStartedAt);
|
|
2188
|
+
};
|
|
2189
|
+
const ensureAudioContext = async () => {
|
|
2190
|
+
if (audioContext) {
|
|
2191
|
+
return audioContext;
|
|
2192
|
+
}
|
|
2193
|
+
if (options.createAudioContext) {
|
|
2194
|
+
audioContext = options.createAudioContext();
|
|
2195
|
+
} else {
|
|
2196
|
+
const AudioContextCtor = getAudioContextCtor();
|
|
2197
|
+
if (!AudioContextCtor) {
|
|
2198
|
+
throw new Error("Assistant audio playback requires AudioContext support.");
|
|
2199
|
+
}
|
|
2200
|
+
audioContext = new AudioContextCtor;
|
|
2201
|
+
}
|
|
2202
|
+
if (audioContext.createGain) {
|
|
2203
|
+
outputNode = audioContext.createGain();
|
|
2204
|
+
outputNode.connect?.(audioContext.destination);
|
|
2205
|
+
if (audioContext.createAnalyser) {
|
|
2206
|
+
analyserNode = audioContext.createAnalyser();
|
|
2207
|
+
analyserNode.fftSize = ANALYSER_FFT_SIZE;
|
|
2208
|
+
analyserBuffer = new Uint8Array(analyserNode.fftSize);
|
|
2209
|
+
outputNode.connect?.(analyserNode);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
queueEndTime = audioContext.currentTime;
|
|
2213
|
+
return audioContext;
|
|
2214
|
+
};
|
|
2215
|
+
const scheduleBuffer = (context, buffer, rate) => {
|
|
2216
|
+
const node = context.createBufferSource();
|
|
2217
|
+
node.buffer = buffer;
|
|
2218
|
+
if (node.playbackRate) {
|
|
2219
|
+
node.playbackRate.value = rate;
|
|
2220
|
+
}
|
|
2221
|
+
node.connect(outputNode ?? context.destination);
|
|
2222
|
+
node.onended = () => {
|
|
2223
|
+
sourceNodes.delete(node);
|
|
2224
|
+
node.disconnect?.();
|
|
2225
|
+
setState({
|
|
2226
|
+
activeSourceCount: sourceNodes.size,
|
|
2227
|
+
isPlaying: sourceNodes.size > 0 && state.isActive
|
|
2228
|
+
});
|
|
2229
|
+
maybeResolveInterrupt();
|
|
2230
|
+
};
|
|
2231
|
+
const earliestStart = context.currentTime + lookaheadSeconds;
|
|
2232
|
+
const startAt = Math.max(earliestStart, queueEndTime);
|
|
2233
|
+
if (hasScheduledAnyChunk && earliestStart > queueEndTime + GAP_EPSILON_S) {
|
|
2234
|
+
const gapMs = (earliestStart - queueEndTime) * 1000;
|
|
2235
|
+
gapCount += 1;
|
|
2236
|
+
totalGapMs += gapMs;
|
|
2237
|
+
maxGapMs = Math.max(maxGapMs, gapMs);
|
|
2238
|
+
}
|
|
2239
|
+
hasScheduledAnyChunk = true;
|
|
2240
|
+
scheduledDurationMs += buffer.duration / rate * 1000;
|
|
2241
|
+
sampleConcurrency();
|
|
2242
|
+
queueEndTime = startAt + buffer.duration / rate;
|
|
2243
|
+
sourceNodes.add(node);
|
|
2244
|
+
setState({
|
|
2245
|
+
activeSourceCount: sourceNodes.size,
|
|
2246
|
+
isPlaying: true
|
|
2247
|
+
});
|
|
2248
|
+
node.start(startAt);
|
|
2249
|
+
};
|
|
2250
|
+
const scheduleChunk = async (chunk) => {
|
|
2251
|
+
const context = await ensureAudioContext();
|
|
2252
|
+
const buffer = decodePCM16LEChunk(context, chunk);
|
|
2253
|
+
if (Math.abs(playbackRate - 1) <= STRETCH_BYPASS_EPSILON) {
|
|
2254
|
+
stretcher?.reset();
|
|
2255
|
+
scheduleBuffer(context, buffer, playbackRate);
|
|
2256
|
+
return;
|
|
2257
|
+
}
|
|
2258
|
+
const channels = Math.max(1, chunk.format.channels);
|
|
2259
|
+
const input = [];
|
|
2260
|
+
for (let channelIndex = 0;channelIndex < channels; channelIndex += 1) {
|
|
2261
|
+
input.push(buffer.getChannelData(channelIndex));
|
|
2262
|
+
}
|
|
2263
|
+
stretcher ??= createTimeStretcher();
|
|
2264
|
+
const stretched = stretcher.process(input, playbackRate, chunk.format.sampleRateHz);
|
|
2265
|
+
const outLength = stretched[0]?.length ?? 0;
|
|
2266
|
+
if (outLength === 0) {
|
|
2267
|
+
return;
|
|
2268
|
+
}
|
|
2269
|
+
const outBuffer = context.createBuffer(channels, outLength, chunk.format.sampleRateHz);
|
|
2270
|
+
for (let channelIndex = 0;channelIndex < channels; channelIndex += 1) {
|
|
2271
|
+
const channelOut = stretched[channelIndex];
|
|
2272
|
+
if (!channelOut)
|
|
2273
|
+
continue;
|
|
2274
|
+
outBuffer.getChannelData(channelIndex).set(channelOut);
|
|
2275
|
+
}
|
|
2276
|
+
scheduleBuffer(context, outBuffer, 1);
|
|
2277
|
+
};
|
|
2278
|
+
const stopQueuedPlayback = (options2) => {
|
|
2279
|
+
for (const node of [...sourceNodes]) {
|
|
2280
|
+
node.stop?.();
|
|
2281
|
+
}
|
|
2282
|
+
queueEndTime = audioContext ? audioContext.currentTime : 0;
|
|
2283
|
+
if (options2?.forceClear) {
|
|
2284
|
+
for (const node of sourceNodes) {
|
|
2285
|
+
node.disconnect?.();
|
|
2286
|
+
}
|
|
2287
|
+
sourceNodes.clear();
|
|
2288
|
+
maybeResolveInterrupt();
|
|
2289
|
+
}
|
|
2290
|
+
};
|
|
2291
|
+
const sync = async () => {
|
|
2292
|
+
if (!state.isActive) {
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
const nextChunks = source.assistantAudio.slice(state.processedChunkCount);
|
|
2296
|
+
if (nextChunks.length === 0) {
|
|
2297
|
+
return;
|
|
2298
|
+
}
|
|
2299
|
+
try {
|
|
2300
|
+
clearError();
|
|
2301
|
+
for (const chunk of nextChunks) {
|
|
2302
|
+
await scheduleChunk(chunk);
|
|
2303
|
+
}
|
|
2304
|
+
setState({
|
|
2305
|
+
processedChunkCount: source.assistantAudio.length,
|
|
2306
|
+
queuedChunkCount: state.queuedChunkCount + nextChunks.length
|
|
2307
|
+
});
|
|
2308
|
+
} catch (error) {
|
|
2309
|
+
errorCount += 1;
|
|
2310
|
+
setState({
|
|
2311
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2312
|
+
});
|
|
2313
|
+
}
|
|
2314
|
+
};
|
|
2315
|
+
const queueSync = () => {
|
|
2316
|
+
syncPromise = syncPromise.then(() => sync(), () => sync());
|
|
2317
|
+
return syncPromise;
|
|
2318
|
+
};
|
|
2319
|
+
const unsubscribeSource = source.subscribe(() => {
|
|
2320
|
+
if (options.autoStart && !state.isActive && source.assistantAudio.length > 0) {
|
|
2321
|
+
player.start();
|
|
2322
|
+
return;
|
|
2323
|
+
}
|
|
2324
|
+
if (state.isActive) {
|
|
2325
|
+
queueSync();
|
|
2326
|
+
}
|
|
2327
|
+
});
|
|
2328
|
+
const player = {
|
|
2329
|
+
get activeSourceCount() {
|
|
2330
|
+
return state.activeSourceCount;
|
|
1094
2331
|
},
|
|
1095
|
-
|
|
1096
|
-
|
|
2332
|
+
close: async () => {
|
|
2333
|
+
markPlayerInactive();
|
|
2334
|
+
unsubscribeSource();
|
|
2335
|
+
stopQueuedPlayback({ forceClear: true });
|
|
2336
|
+
clearInterruptTimer();
|
|
2337
|
+
resolveInterruptPromise?.();
|
|
2338
|
+
resolveInterruptPromise = null;
|
|
2339
|
+
interruptPromise = null;
|
|
2340
|
+
interruptStartedAt = null;
|
|
2341
|
+
if (audioContext && audioContext.state !== "closed") {
|
|
2342
|
+
await audioContext.close();
|
|
2343
|
+
}
|
|
2344
|
+
audioContext = null;
|
|
2345
|
+
outputNode?.disconnect?.();
|
|
2346
|
+
outputNode = null;
|
|
2347
|
+
analyserNode?.disconnect?.();
|
|
2348
|
+
analyserNode = null;
|
|
2349
|
+
analyserBuffer = null;
|
|
2350
|
+
queueEndTime = 0;
|
|
2351
|
+
setState({
|
|
2352
|
+
activeSourceCount: 0,
|
|
2353
|
+
isActive: false,
|
|
2354
|
+
isPlaying: false
|
|
2355
|
+
});
|
|
1097
2356
|
},
|
|
1098
|
-
get
|
|
1099
|
-
return state.
|
|
2357
|
+
get error() {
|
|
2358
|
+
return state.error;
|
|
1100
2359
|
},
|
|
1101
|
-
|
|
1102
|
-
|
|
2360
|
+
getOutputLevel: () => {
|
|
2361
|
+
if (!analyserNode || !analyserBuffer) {
|
|
2362
|
+
return 0;
|
|
2363
|
+
}
|
|
2364
|
+
analyserNode.getByteTimeDomainData(analyserBuffer);
|
|
2365
|
+
let sumSquares = 0;
|
|
2366
|
+
for (const sample of analyserBuffer) {
|
|
2367
|
+
const centered = (sample - PCM_BYTE_MIDPOINT) / PCM_BYTE_MIDPOINT;
|
|
2368
|
+
sumSquares += centered * centered;
|
|
2369
|
+
}
|
|
2370
|
+
return Math.sqrt(sumSquares / analyserBuffer.length);
|
|
2371
|
+
},
|
|
2372
|
+
getSnapshot: () => state,
|
|
2373
|
+
getIntegritySummary: () => {
|
|
2374
|
+
const chunksReceived = source.assistantAudio.length;
|
|
2375
|
+
const chunksScheduled = state.processedChunkCount;
|
|
2376
|
+
return {
|
|
2377
|
+
chunksReceived,
|
|
2378
|
+
chunksScheduled,
|
|
2379
|
+
errorCount,
|
|
2380
|
+
gapCount,
|
|
2381
|
+
maxConcurrentPlayers: observedPeakConcurrency,
|
|
2382
|
+
maxGapMs: Math.round(maxGapMs),
|
|
2383
|
+
ok: observedPeakConcurrency <= 1 && errorCount === 0 && chunksScheduled >= chunksReceived,
|
|
2384
|
+
scheduledDurationMs: Math.round(scheduledDurationMs),
|
|
2385
|
+
totalGapMs: Math.round(totalGapMs)
|
|
2386
|
+
};
|
|
2387
|
+
},
|
|
2388
|
+
interrupt: async () => {
|
|
2389
|
+
const startedAt = Date.now();
|
|
2390
|
+
const context = await ensureAudioContext();
|
|
2391
|
+
interruptStartedAt = startedAt;
|
|
2392
|
+
muteOutputGain(context);
|
|
2393
|
+
const playbackStopLatencyMs = Date.now() - startedAt + estimateOutputStopLatencyMs(context);
|
|
2394
|
+
setState({
|
|
2395
|
+
isActive: false,
|
|
2396
|
+
isPlaying: sourceNodes.size > 0,
|
|
2397
|
+
lastPlaybackStopLatencyMs: playbackStopLatencyMs
|
|
2398
|
+
});
|
|
2399
|
+
if (sourceNodes.size === 0) {
|
|
2400
|
+
resolveInterrupt(playbackStopLatencyMs);
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
if (!interruptPromise) {
|
|
2404
|
+
interruptPromise = new Promise((resolve) => {
|
|
2405
|
+
resolveInterruptPromise = resolve;
|
|
2406
|
+
});
|
|
2407
|
+
}
|
|
2408
|
+
clearInterruptTimer();
|
|
2409
|
+
interruptFallbackTimer = setTimeout(() => {
|
|
2410
|
+
for (const node of sourceNodes) {
|
|
2411
|
+
node.disconnect?.();
|
|
2412
|
+
}
|
|
2413
|
+
sourceNodes.clear();
|
|
2414
|
+
resolveInterrupt(Date.now() - startedAt);
|
|
2415
|
+
}, 250);
|
|
2416
|
+
stopQueuedPlayback();
|
|
2417
|
+
await interruptPromise;
|
|
2418
|
+
},
|
|
2419
|
+
get isActive() {
|
|
2420
|
+
return state.isActive;
|
|
2421
|
+
},
|
|
2422
|
+
get isPlaying() {
|
|
2423
|
+
return state.isPlaying;
|
|
2424
|
+
},
|
|
2425
|
+
get lastInterruptLatencyMs() {
|
|
2426
|
+
return state.lastInterruptLatencyMs;
|
|
2427
|
+
},
|
|
2428
|
+
get lastPlaybackStopLatencyMs() {
|
|
2429
|
+
return state.lastPlaybackStopLatencyMs;
|
|
2430
|
+
},
|
|
2431
|
+
pause: async () => {
|
|
2432
|
+
if (!audioContext) {
|
|
2433
|
+
setState({
|
|
2434
|
+
activeSourceCount: 0,
|
|
2435
|
+
isActive: false,
|
|
2436
|
+
isPlaying: false
|
|
2437
|
+
});
|
|
2438
|
+
return;
|
|
2439
|
+
}
|
|
2440
|
+
await audioContext.suspend();
|
|
2441
|
+
markPlayerInactive();
|
|
2442
|
+
setState({
|
|
2443
|
+
activeSourceCount: sourceNodes.size,
|
|
2444
|
+
isActive: false,
|
|
2445
|
+
isPlaying: false
|
|
2446
|
+
});
|
|
2447
|
+
},
|
|
2448
|
+
get playbackRate() {
|
|
2449
|
+
return playbackRate;
|
|
2450
|
+
},
|
|
2451
|
+
get processedChunkCount() {
|
|
2452
|
+
return state.processedChunkCount;
|
|
2453
|
+
},
|
|
2454
|
+
get queuedChunkCount() {
|
|
2455
|
+
return state.queuedChunkCount;
|
|
2456
|
+
},
|
|
2457
|
+
setPlaybackRate: (nextRate) => {
|
|
2458
|
+
playbackRate = clampPlaybackRate(nextRate);
|
|
2459
|
+
},
|
|
2460
|
+
setVolume: (nextVolume) => {
|
|
2461
|
+
volume = clampVolume(nextVolume);
|
|
2462
|
+
applyOutputGain(audioContext);
|
|
2463
|
+
},
|
|
2464
|
+
start: async () => {
|
|
2465
|
+
try {
|
|
2466
|
+
clearError();
|
|
2467
|
+
const context = await ensureAudioContext();
|
|
2468
|
+
applyOutputGain(context);
|
|
2469
|
+
if (context.state === "suspended") {
|
|
2470
|
+
await context.resume();
|
|
2471
|
+
}
|
|
2472
|
+
markPlayerActive();
|
|
2473
|
+
setState({
|
|
2474
|
+
activeSourceCount: sourceNodes.size,
|
|
2475
|
+
isActive: true,
|
|
2476
|
+
isPlaying: context.state === "running"
|
|
2477
|
+
});
|
|
2478
|
+
await queueSync();
|
|
2479
|
+
} catch (error) {
|
|
2480
|
+
errorCount += 1;
|
|
2481
|
+
markPlayerInactive();
|
|
2482
|
+
setState({
|
|
2483
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2484
|
+
isActive: false,
|
|
2485
|
+
isPlaying: false
|
|
2486
|
+
});
|
|
2487
|
+
throw error;
|
|
2488
|
+
}
|
|
2489
|
+
},
|
|
2490
|
+
subscribe: (subscriber) => {
|
|
2491
|
+
subscribers.add(subscriber);
|
|
2492
|
+
return () => {
|
|
2493
|
+
subscribers.delete(subscriber);
|
|
2494
|
+
};
|
|
2495
|
+
},
|
|
2496
|
+
get volume() {
|
|
2497
|
+
return volume;
|
|
2498
|
+
}
|
|
2499
|
+
};
|
|
2500
|
+
return player;
|
|
2501
|
+
};
|
|
2502
|
+
|
|
2503
|
+
// src/client/bargeInMonitor.ts
|
|
2504
|
+
var DEFAULT_THRESHOLD_MS = 250;
|
|
2505
|
+
var createEventId = () => `barge-in:${Date.now()}:${crypto.randomUUID?.() ?? Math.random().toString(36).slice(2)}`;
|
|
2506
|
+
var summarize = (events, thresholdMs) => {
|
|
2507
|
+
const stopped = events.filter((event) => event.status === "stopped");
|
|
2508
|
+
const latencies = stopped.map((event) => event.latencyMs).filter((value) => typeof value === "number");
|
|
2509
|
+
const failed = stopped.filter((event) => typeof event.latencyMs === "number" && event.latencyMs > thresholdMs).length;
|
|
2510
|
+
const passed = stopped.length - failed;
|
|
2511
|
+
return {
|
|
2512
|
+
averageLatencyMs: latencies.length > 0 ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : undefined,
|
|
2513
|
+
events: [...events],
|
|
2514
|
+
failed,
|
|
2515
|
+
lastEvent: events.at(-1),
|
|
2516
|
+
passed,
|
|
2517
|
+
status: events.length === 0 ? "empty" : failed > 0 ? "fail" : stopped.length === 0 ? "warn" : "pass",
|
|
2518
|
+
thresholdMs,
|
|
2519
|
+
total: stopped.length
|
|
2520
|
+
};
|
|
2521
|
+
};
|
|
2522
|
+
var createVoiceBargeInMonitor = (options = {}) => {
|
|
2523
|
+
const listeners = new Set;
|
|
2524
|
+
const thresholdMs = options.thresholdMs ?? DEFAULT_THRESHOLD_MS;
|
|
2525
|
+
const fetchImpl = options.fetch ?? globalThis.fetch;
|
|
2526
|
+
const events = [];
|
|
2527
|
+
const emit = () => {
|
|
2528
|
+
for (const listener of listeners) {
|
|
2529
|
+
listener();
|
|
2530
|
+
}
|
|
2531
|
+
};
|
|
2532
|
+
const postEvent = (event) => {
|
|
2533
|
+
if (!options.path || typeof fetchImpl !== "function") {
|
|
2534
|
+
return;
|
|
2535
|
+
}
|
|
2536
|
+
fetchImpl(options.path, {
|
|
2537
|
+
body: JSON.stringify(event),
|
|
2538
|
+
headers: {
|
|
2539
|
+
"Content-Type": "application/json"
|
|
2540
|
+
},
|
|
2541
|
+
method: "POST"
|
|
2542
|
+
}).catch(() => {});
|
|
2543
|
+
};
|
|
2544
|
+
const record = (status, input) => {
|
|
2545
|
+
const event = {
|
|
2546
|
+
at: Date.now(),
|
|
2547
|
+
id: createEventId(),
|
|
2548
|
+
latencyMs: input.latencyMs,
|
|
2549
|
+
playbackStopLatencyMs: input.playbackStopLatencyMs,
|
|
2550
|
+
reason: input.reason,
|
|
2551
|
+
sessionId: input.sessionId,
|
|
2552
|
+
status,
|
|
2553
|
+
thresholdMs
|
|
2554
|
+
};
|
|
2555
|
+
events.push(event);
|
|
2556
|
+
postEvent(event);
|
|
2557
|
+
emit();
|
|
2558
|
+
return event;
|
|
2559
|
+
};
|
|
2560
|
+
return {
|
|
2561
|
+
getSnapshot: () => summarize(events, thresholdMs),
|
|
2562
|
+
recordRequested: (input) => record("requested", input),
|
|
2563
|
+
recordSkipped: (input) => record("skipped", input),
|
|
2564
|
+
recordStopped: (input) => record("stopped", input),
|
|
2565
|
+
subscribe: (subscriber) => {
|
|
2566
|
+
listeners.add(subscriber);
|
|
2567
|
+
return () => {
|
|
2568
|
+
listeners.delete(subscriber);
|
|
2569
|
+
};
|
|
2570
|
+
}
|
|
2571
|
+
};
|
|
2572
|
+
};
|
|
2573
|
+
|
|
2574
|
+
// src/client/duplex.ts
|
|
2575
|
+
var DEFAULT_INTERRUPT_THRESHOLD = 0.08;
|
|
2576
|
+
var shouldInterruptForLevel = (level, options = {}) => (options.enabled ?? true) && level >= (options.interruptThreshold ?? DEFAULT_INTERRUPT_THRESHOLD);
|
|
2577
|
+
var bindVoiceBargeIn = (controller, player, options = {}) => {
|
|
2578
|
+
let lastPartial = controller.partial;
|
|
2579
|
+
const interruptIfPlaying = (reason) => {
|
|
2580
|
+
if (!player.isPlaying || options.enabled === false) {
|
|
2581
|
+
options.monitor?.recordSkipped({
|
|
2582
|
+
reason,
|
|
2583
|
+
sessionId: controller.sessionId
|
|
2584
|
+
});
|
|
2585
|
+
return;
|
|
2586
|
+
}
|
|
2587
|
+
options.monitor?.recordRequested({
|
|
2588
|
+
reason,
|
|
2589
|
+
sessionId: controller.sessionId
|
|
2590
|
+
});
|
|
2591
|
+
player.interrupt().then(() => {
|
|
2592
|
+
options.monitor?.recordStopped({
|
|
2593
|
+
latencyMs: player.lastInterruptLatencyMs,
|
|
2594
|
+
playbackStopLatencyMs: player.lastPlaybackStopLatencyMs,
|
|
2595
|
+
reason,
|
|
2596
|
+
sessionId: controller.sessionId
|
|
2597
|
+
});
|
|
2598
|
+
});
|
|
2599
|
+
};
|
|
2600
|
+
const unsubscribe = controller.subscribe(() => {
|
|
2601
|
+
if (options.interruptOnPartial === false) {
|
|
2602
|
+
lastPartial = controller.partial;
|
|
2603
|
+
return;
|
|
2604
|
+
}
|
|
2605
|
+
if (!lastPartial && controller.partial) {
|
|
2606
|
+
interruptIfPlaying("partial-transcript");
|
|
2607
|
+
}
|
|
2608
|
+
lastPartial = controller.partial;
|
|
2609
|
+
});
|
|
2610
|
+
return {
|
|
2611
|
+
close: () => {
|
|
2612
|
+
unsubscribe();
|
|
2613
|
+
},
|
|
2614
|
+
handleLevel: (level) => {
|
|
2615
|
+
if (shouldInterruptForLevel(level, options)) {
|
|
2616
|
+
interruptIfPlaying("input-level");
|
|
2617
|
+
}
|
|
2618
|
+
},
|
|
2619
|
+
sendAudio: (audio) => {
|
|
2620
|
+
interruptIfPlaying("manual-audio");
|
|
2621
|
+
controller.sendAudio(audio);
|
|
1103
2622
|
}
|
|
1104
2623
|
};
|
|
1105
2624
|
};
|
|
@@ -1126,8 +2645,7 @@ var DEFAULT_GUIDED_PROMPTS = [
|
|
|
1126
2645
|
"Now describe what you are trying to do or test.",
|
|
1127
2646
|
"Finish with any detail that feels blocked, risky, or unclear."
|
|
1128
2647
|
];
|
|
1129
|
-
var clamp = (value, min,
|
|
1130
|
-
var escapeHtml = (value) => value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
2648
|
+
var clamp = (value, min, max2) => Math.min(max2, Math.max(min, value));
|
|
1131
2649
|
var readErrorField = (value, key) => {
|
|
1132
2650
|
const candidate = value[key];
|
|
1133
2651
|
if (typeof candidate === "string" && candidate.trim()) {
|
|
@@ -1160,6 +2678,17 @@ var formatErrorMessage = (error) => {
|
|
|
1160
2678
|
}
|
|
1161
2679
|
return "Unexpected error";
|
|
1162
2680
|
};
|
|
2681
|
+
var formatReconnectState = (reconnect) => {
|
|
2682
|
+
const pieces = [reconnect.status];
|
|
2683
|
+
if (reconnect.attempts > 0 || reconnect.maxAttempts > 0) {
|
|
2684
|
+
pieces.push(`${reconnect.attempts}/${reconnect.maxAttempts} attempts`);
|
|
2685
|
+
}
|
|
2686
|
+
if (reconnect.nextAttemptAt) {
|
|
2687
|
+
const waitMs = Math.max(0, reconnect.nextAttemptAt - Date.now());
|
|
2688
|
+
pieces.push(`retry in ${Math.ceil(waitMs / 100) / 10}s`);
|
|
2689
|
+
}
|
|
2690
|
+
return pieces.join(" · ");
|
|
2691
|
+
};
|
|
1163
2692
|
var createInitialVoiceWaveLevels = (count = VOICE_WAVE_POINTS) => Array.from({ length: count }, () => 0);
|
|
1164
2693
|
var pushVoiceWaveLevel = (levels, nextLevel, count = VOICE_WAVE_POINTS) => {
|
|
1165
2694
|
const next = levels.slice(-(count - 1));
|
|
@@ -1216,6 +2745,20 @@ var parsePromptList = (value) => {
|
|
|
1216
2745
|
} catch {}
|
|
1217
2746
|
return DEFAULT_GUIDED_PROMPTS;
|
|
1218
2747
|
};
|
|
2748
|
+
var parseOptionalNumber = (value) => {
|
|
2749
|
+
if (!value) {
|
|
2750
|
+
return;
|
|
2751
|
+
}
|
|
2752
|
+
const parsed = Number(value);
|
|
2753
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
2754
|
+
};
|
|
2755
|
+
var resolveElement2 = (root, selector, ctor) => {
|
|
2756
|
+
if (!selector) {
|
|
2757
|
+
return null;
|
|
2758
|
+
}
|
|
2759
|
+
const value = document.querySelector(selector);
|
|
2760
|
+
return value instanceof ctor ? value : null;
|
|
2761
|
+
};
|
|
1219
2762
|
var requireElement = (root, selector, ctor, name) => {
|
|
1220
2763
|
const value = selector ? document.querySelector(selector) : null;
|
|
1221
2764
|
if (value instanceof ctor) {
|
|
@@ -1266,11 +2809,20 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1266
2809
|
const guidedPrompts = parsePromptList(root.dataset.voiceGuidedPrompts);
|
|
1267
2810
|
const guidedLabel = root.dataset.voiceGuidedLabel ?? DEFAULT_GUIDED_LABEL;
|
|
1268
2811
|
const generalLabel = root.dataset.voiceGeneralLabel ?? DEFAULT_GENERAL_LABEL;
|
|
2812
|
+
const reconnectReportPath = root.dataset.voiceReconnectReportPath;
|
|
2813
|
+
const bargeInPath = root.dataset.voiceBargeInPath;
|
|
2814
|
+
const bargeInMonitor = bargeInPath ? createVoiceBargeInMonitor({
|
|
2815
|
+
path: bargeInPath,
|
|
2816
|
+
thresholdMs: parseOptionalNumber(root.dataset.voiceBargeInThresholdMs)
|
|
2817
|
+
}) : null;
|
|
2818
|
+
const bargeInRecentWindowMs = parseOptionalNumber(root.dataset.voiceBargeInRecentWindowMs) ?? 4000;
|
|
2819
|
+
const bargeInSpeechThreshold = parseOptionalNumber(root.dataset.voiceBargeInSpeechThreshold) ?? 0.04;
|
|
1269
2820
|
const syncElement = requireElement(document, root.dataset.voiceSync, HTMLElement, "voice-htmx-sync");
|
|
1270
2821
|
const connectionMetric = requireElement(root, root.dataset.voiceConnection, HTMLElement, "metric-connection");
|
|
1271
2822
|
const errorStatus = requireElement(root, root.dataset.voiceError, HTMLElement, "status-error");
|
|
1272
2823
|
const microphoneStatus = requireElement(root, root.dataset.voiceMicrophone, HTMLElement, "status-mic");
|
|
1273
2824
|
const promptStatus = requireElement(root, root.dataset.voicePrompt, HTMLElement, "status-prompt");
|
|
2825
|
+
const reconnectStatus = resolveElement2(root, root.dataset.voiceReconnect, HTMLElement);
|
|
1274
2826
|
const chatList = requireElement(root, root.dataset.voiceChat, HTMLElement, "chat-list");
|
|
1275
2827
|
const startGuidedButton = requireElement(root, root.dataset.voiceStartGuided, HTMLButtonElement, "start-guided");
|
|
1276
2828
|
const startGeneralButton = requireElement(root, root.dataset.voiceStartGeneral, HTMLButtonElement, "start-general");
|
|
@@ -1279,35 +2831,70 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1279
2831
|
const voiceMonitorCopy = requireElement(root, root.dataset.voiceMonitorCopy, HTMLElement, "voice-monitor-copy");
|
|
1280
2832
|
const voiceWaveGlow = requireElement(root, root.dataset.voiceWaveGlow, SVGPathElement, "voice-wave-glow");
|
|
1281
2833
|
const voiceWavePath = requireElement(root, root.dataset.voiceWavePath, SVGPathElement, "voice-wave-path");
|
|
2834
|
+
let activeMode = null;
|
|
2835
|
+
let hasStartedModes = {
|
|
2836
|
+
general: false,
|
|
2837
|
+
guided: false
|
|
2838
|
+
};
|
|
2839
|
+
let isCapturing = false;
|
|
2840
|
+
let micError = null;
|
|
2841
|
+
let waveLevels = createInitialVoiceWaveLevels();
|
|
2842
|
+
let guidedBargeInBinding = null;
|
|
2843
|
+
let generalBargeInBinding = null;
|
|
1282
2844
|
const guidedVoice = createVoiceController(guidedPath, {
|
|
1283
2845
|
capture: {
|
|
2846
|
+
onAudio: (audio, sendAudio) => {
|
|
2847
|
+
if (guidedBargeInBinding) {
|
|
2848
|
+
guidedBargeInBinding.sendAudio(audio);
|
|
2849
|
+
return;
|
|
2850
|
+
}
|
|
2851
|
+
sendAudio(audio);
|
|
2852
|
+
},
|
|
1284
2853
|
onLevel: (level) => {
|
|
2854
|
+
guidedBargeInBinding?.handleLevel(level);
|
|
1285
2855
|
waveLevels = pushVoiceWaveLevel(waveLevels, level);
|
|
1286
2856
|
renderWave();
|
|
1287
2857
|
}
|
|
1288
2858
|
},
|
|
2859
|
+
connection: {
|
|
2860
|
+
reconnectReportPath
|
|
2861
|
+
},
|
|
1289
2862
|
preset: "guided-intake"
|
|
1290
2863
|
});
|
|
1291
2864
|
const generalVoice = createVoiceController(generalPath, {
|
|
1292
2865
|
capture: {
|
|
2866
|
+
onAudio: (audio, sendAudio) => {
|
|
2867
|
+
if (generalBargeInBinding) {
|
|
2868
|
+
generalBargeInBinding.sendAudio(audio);
|
|
2869
|
+
return;
|
|
2870
|
+
}
|
|
2871
|
+
sendAudio(audio);
|
|
2872
|
+
},
|
|
1293
2873
|
onLevel: (level) => {
|
|
2874
|
+
generalBargeInBinding?.handleLevel(level);
|
|
1294
2875
|
waveLevels = pushVoiceWaveLevel(waveLevels, level);
|
|
1295
2876
|
renderWave();
|
|
1296
2877
|
}
|
|
1297
2878
|
},
|
|
2879
|
+
connection: {
|
|
2880
|
+
reconnectReportPath
|
|
2881
|
+
},
|
|
1298
2882
|
preset: "dictation"
|
|
1299
2883
|
});
|
|
1300
2884
|
const stopGuidedBinding = guidedVoice.bindHTMX({ element: syncElement });
|
|
1301
2885
|
const stopGeneralBinding = generalVoice.bindHTMX({ element: syncElement });
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
2886
|
+
const guidedAudioPlayer = createVoiceAudioPlayer(guidedVoice);
|
|
2887
|
+
const generalAudioPlayer = createVoiceAudioPlayer(generalVoice);
|
|
2888
|
+
guidedBargeInBinding = bindVoiceBargeIn(guidedVoice, guidedAudioPlayer, {
|
|
2889
|
+
interruptThreshold: bargeInSpeechThreshold,
|
|
2890
|
+
monitor: bargeInMonitor ?? undefined
|
|
2891
|
+
});
|
|
2892
|
+
generalBargeInBinding = bindVoiceBargeIn(generalVoice, generalAudioPlayer, {
|
|
2893
|
+
interruptThreshold: bargeInSpeechThreshold,
|
|
2894
|
+
monitor: bargeInMonitor ?? undefined
|
|
2895
|
+
});
|
|
1310
2896
|
const currentVoice = () => activeMode === "general" ? generalVoice : guidedVoice;
|
|
2897
|
+
const currentAudioPlayer = () => activeMode === "general" ? generalAudioPlayer : guidedAudioPlayer;
|
|
1311
2898
|
const renderWave = () => {
|
|
1312
2899
|
const path = createVoiceWavePath(waveLevels);
|
|
1313
2900
|
voiceWaveGlow.setAttribute("d", path);
|
|
@@ -1319,9 +2906,12 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1319
2906
|
const render = () => {
|
|
1320
2907
|
const voice = currentVoice();
|
|
1321
2908
|
const hasStarted = (activeMode ? hasStartedModes[activeMode] : false) || voice.turns.length > 0;
|
|
1322
|
-
const status = voice
|
|
2909
|
+
const { status } = voice;
|
|
1323
2910
|
connectionMetric.textContent = voice.isConnected ? "Connected" : "Waiting";
|
|
1324
2911
|
errorStatus.textContent = micError || voice.error || "None";
|
|
2912
|
+
if (reconnectStatus) {
|
|
2913
|
+
reconnectStatus.textContent = formatReconnectState(voice.reconnect);
|
|
2914
|
+
}
|
|
1325
2915
|
microphoneStatus.textContent = isCapturing ? DEFAULT_MIC_LIVE : DEFAULT_MIC_IDLE;
|
|
1326
2916
|
promptStatus.textContent = resolvePromptMessage({
|
|
1327
2917
|
guidedPrompts,
|
|
@@ -1385,8 +2975,18 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1385
2975
|
render();
|
|
1386
2976
|
}
|
|
1387
2977
|
};
|
|
1388
|
-
guidedVoice.subscribe(
|
|
1389
|
-
|
|
2978
|
+
guidedVoice.subscribe(() => {
|
|
2979
|
+
if (guidedVoice.assistantAudio.length > 0) {
|
|
2980
|
+
guidedAudioPlayer.start().catch(() => {});
|
|
2981
|
+
}
|
|
2982
|
+
render();
|
|
2983
|
+
});
|
|
2984
|
+
generalVoice.subscribe(() => {
|
|
2985
|
+
if (generalVoice.assistantAudio.length > 0) {
|
|
2986
|
+
generalAudioPlayer.start().catch(() => {});
|
|
2987
|
+
}
|
|
2988
|
+
render();
|
|
2989
|
+
});
|
|
1390
2990
|
startGuidedButton.addEventListener("click", () => {
|
|
1391
2991
|
startMode("guided");
|
|
1392
2992
|
});
|
|
@@ -1396,9 +2996,16 @@ var initVoiceHTMXRoot = (root) => {
|
|
|
1396
2996
|
stopButton.addEventListener("click", () => {
|
|
1397
2997
|
stopMic();
|
|
1398
2998
|
});
|
|
2999
|
+
root.addEventListener("absolute-voice-simulate-disconnect", () => {
|
|
3000
|
+
currentVoice().simulateDisconnect();
|
|
3001
|
+
});
|
|
1399
3002
|
window.addEventListener("beforeunload", () => {
|
|
1400
3003
|
guidedVoice.stopRecording();
|
|
1401
3004
|
generalVoice.stopRecording();
|
|
3005
|
+
guidedBargeInBinding?.close();
|
|
3006
|
+
generalBargeInBinding?.close();
|
|
3007
|
+
guidedAudioPlayer.close();
|
|
3008
|
+
generalAudioPlayer.close();
|
|
1402
3009
|
stopGuidedBinding();
|
|
1403
3010
|
stopGeneralBinding();
|
|
1404
3011
|
guidedVoice.close();
|