@xopcai/xopc 0.0.242 → 0.0.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/manifest.json +1 -1
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{activity-store-MU5XrVv8.js → activity-store-CAah-_8h.js} +1 -1
- package/dist/gateway/static/root/assets/{agent-avatar-dicebear-B_Wc0wuL.js → agent-avatar-dicebear-BhXGsupm.js} +1 -1
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-Blzn2H6t.js +1 -0
- package/dist/gateway/static/root/assets/agents-Cmp-z8Y_.js +1 -0
- package/dist/gateway/static/root/assets/{api-DLyc8BQp.js → api-B2tKxbcz.js} +1 -1
- package/dist/gateway/static/root/assets/app-management-settings-panel-CezjAP1K.js +1 -0
- package/dist/gateway/static/root/assets/{appearance-settings-DB5gEQhN.js → appearance-settings-C-CfS05x.js} +1 -1
- package/dist/gateway/static/root/assets/apps-page-7Y13_bqZ.js +1 -0
- package/dist/gateway/static/root/assets/{archive-plugin-BOx0EJZs.js → archive-plugin-BmSx7wzY.js} +1 -1
- package/dist/gateway/static/root/assets/{attachment-load-CYnYIyLr.js → attachment-load-DI4CoIga.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-page-ChTflpmA.js → automations-page-D04Xceck.js} +1 -1
- package/dist/gateway/static/root/assets/automations-workspace-wEO1OWk0.js +6 -0
- package/dist/gateway/static/root/assets/{binary-plugins-BjbLfNwG.js → binary-plugins-BJQe8TRu.js} +1 -1
- package/dist/gateway/static/root/assets/block-editor-BSzGjJC6.js +185 -0
- package/dist/gateway/static/root/assets/browser-settings-page-DjAZ_LF2.js +8 -0
- package/dist/gateway/static/root/assets/{browser-workflow-inputs-DGQxD0vD.js → browser-workflow-inputs-CRnkgZHI.js} +1 -1
- package/dist/gateway/static/root/assets/{browser-workflows-page-B9gfdd7L.js → browser-workflows-page-CwbxA3B7.js} +2 -2
- package/dist/gateway/static/root/assets/{button-CSEbl4zT.js → button-CtTvuElV.js} +1 -1
- package/dist/gateway/static/root/assets/capabilities-settings-panel-jt9bqUMK.js +7 -0
- package/dist/gateway/static/root/assets/channels-settings-Cz_TWY3a.js +1 -0
- package/dist/gateway/static/root/assets/{channels-status-swr-TkU42f-W.js → channels-status-swr-DuY4MGxe.js} +1 -1
- package/dist/gateway/static/root/assets/{chat-terminal-dock-CWMPcnoN.js → chat-terminal-dock-D3gj2D3U.js} +2 -2
- package/dist/gateway/static/root/assets/connectors-page-Shs5vbzx.js +2 -0
- package/dist/gateway/static/root/assets/{cron-expression-3h3rR0cf.js → cron-expression-DgWHw0jX.js} +1 -1
- package/dist/gateway/static/root/assets/{date-picker-Bz1sLBEx.js → date-picker-CKg-RPbs.js} +1 -1
- package/dist/gateway/static/root/assets/{dependency-picker-p9DWR8uU.js → dependency-picker-CVYkw-NN.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-D1TDB_gR.js → desktop-pet-Bf46oXTY.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-settings-DWPZpoB7.js → desktop-pet-settings-BJO_klc7.js} +1 -1
- package/dist/gateway/static/root/assets/dist-CpzYSoz7.js +1 -0
- package/dist/gateway/static/root/assets/{dist-DaxPZj9e.js → dist-DBS3OB8K.js} +1 -1
- package/dist/gateway/static/root/assets/dist-DLmwHyAh.js +1 -0
- package/dist/gateway/static/root/assets/es2015-BvLh9GaE.js +41 -0
- package/dist/gateway/static/root/assets/{extension-debug-page-BrRbK2rJ.js → extension-debug-page-T2PmFFOW.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-CD65czfS.js → extension-page-2wAE2RCe.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-BRDDKhpi.js → extension-settings-page-Y9ZiCJ21.js} +1 -1
- package/dist/gateway/static/root/assets/{files-DQjhZjr2.js → files-D6vf3S30.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-api-CjbBsr5M.js → gateway-config-api-DaYY1Gkn.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-swr-HT7sLeO9.js → gateway-config-swr-BC3pgb_P.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-settings-BQjmfYD4.js → gateway-settings-eO3IzW_J.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-startup-retry-iBCtaAuT.js → gateway-startup-retry-uMcSfLru.js} +1 -1
- package/dist/gateway/static/root/assets/{heartbeat-settings-BzZxLgrw.js → heartbeat-settings-BAgrkyHc.js} +1 -1
- package/dist/gateway/static/root/assets/home-page-GTjKH7LK.js +1 -0
- package/dist/gateway/static/root/assets/{image-generation-api-dj9TNEn2.js → image-generation-api-D1qlQA-A.js} +1 -1
- package/dist/gateway/static/root/assets/index-DUz0O54y.js +71 -0
- package/dist/gateway/static/root/assets/index-Dbh5SXyV.css +2 -0
- package/dist/gateway/static/root/assets/{keyboard-shortcuts-settings-BEnuEPD3.js → keyboard-shortcuts-settings-CWGqL6o5.js} +1 -1
- package/dist/gateway/static/root/assets/local-app-workbench-page-DKJ-rks0.js +2 -0
- package/dist/gateway/static/root/assets/{local-apps-page-DXQtppWq.js → local-apps-page-Bfidv29d.js} +1 -1
- package/dist/gateway/static/root/assets/{locale-store-DVzTJSj2.js → locale-store-CMZljTJx.js} +1 -1
- package/dist/gateway/static/root/assets/logs-page-DwAHrPbF.js +2 -0
- package/dist/gateway/static/root/assets/{management-settings-DIpBMWrA.js → management-settings-BG7j1I9f.js} +1 -1
- package/dist/gateway/static/root/assets/{markdown-split-BFHyGwNF.js → markdown-split-AnmmpLs6.js} +1 -1
- package/dist/gateway/static/root/assets/{markdown-view-DR3yVwtV.js → markdown-view-PPCLt7fu.js} +3 -3
- package/dist/gateway/static/root/assets/{media-plugins-BjjMxC5_.js → media-plugins-CGS-05cE.js} +1 -1
- package/dist/gateway/static/root/assets/messages-CTSgFlE4.js +3 -0
- package/dist/gateway/static/root/assets/mobile-pairing-wizard-DoKK5GfV.js +1 -0
- package/dist/gateway/static/root/assets/models-hub-cache-ZrbGRiGY.js +1 -0
- package/dist/gateway/static/root/assets/{notes-page-BbvtelJa.js → notes-page-Lr8rsFAR.js} +1 -1
- package/dist/gateway/static/root/assets/notes-workbench-ISWnKLxO.js +4 -0
- package/dist/gateway/static/root/assets/{oauth-api-BB-VYJ5j.js → oauth-api-4VeMbkTJ.js} +1 -1
- package/dist/gateway/static/root/assets/popover-select-C-Hj-nXR.js +1 -0
- package/dist/gateway/static/root/assets/{product-delivery-dUJgmBQZ.js → product-delivery-CV4pDOtG.js} +1 -1
- package/dist/gateway/static/root/assets/product-open-page-BU0HiZtv.js +1 -0
- package/dist/gateway/static/root/assets/project-detail-page-BZqr4yam.js +1 -0
- package/dist/gateway/static/root/assets/projects-page-Dau8733A.js +1 -0
- package/dist/gateway/static/root/assets/{registry-api-DXsR8oGN.js → registry-api-CWOtPFrE.js} +1 -1
- package/dist/gateway/static/root/assets/remote-access-hub-DdChlvGP.js +1 -0
- package/dist/gateway/static/root/assets/{runtime-tools-settings-panel-DoEfLjPB.js → runtime-tools-settings-panel-CmZTNv1W.js} +2 -2
- package/dist/gateway/static/root/assets/{schema-form-BsqpbBDv.js → schema-form-XlVPyiVQ.js} +1 -1
- package/dist/gateway/static/root/assets/{schemas-zIBW3wiV.js → schemas-BHS7bo06.js} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-C3kADsd4.js +1 -0
- package/dist/gateway/static/root/assets/settings-advanced-gate-CSYy6SUa.js +1 -0
- package/dist/gateway/static/root/assets/{settings-page-CE7ZcCTK.js → settings-page-DblUeM3V.js} +1 -1
- package/dist/gateway/static/root/assets/{setup-status-panel-CGlne9i0.js → setup-status-panel-bDfRfSnq.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-Bn0qMX8V.js → share-preview-page-DBMWFMN-.js} +1 -1
- package/dist/gateway/static/root/assets/shares-settings-Bu6lIStj.js +1 -0
- package/dist/gateway/static/root/assets/skill-reload-api-BNMc4l04.js +1 -0
- package/dist/gateway/static/root/assets/{skills-page-dRkeYh5P.js → skills-page-CSOseeQe.js} +1 -1
- package/dist/gateway/static/root/assets/skills-page.utils-CniRGhkJ.js +1 -0
- package/dist/gateway/static/root/assets/{system-settings-panel-C0ZCuCM4.js → system-settings-panel-D0GqNpKy.js} +1 -1
- package/dist/gateway/static/root/assets/task-detail-page-BRCq-nR7.js +1 -0
- package/dist/gateway/static/root/assets/{theme-store-Cth-eChE.js → theme-store-uxUwr10D.js} +1 -1
- package/dist/gateway/static/root/assets/tunnel-api-BtjS2nrM.js +1 -0
- package/dist/gateway/static/root/assets/{use-autosave-PSV89QuQ.js → use-autosave-TwuAycG8.js} +1 -1
- package/dist/gateway/static/root/assets/user-context-page-CbqPu8xZ.js +1 -0
- package/dist/gateway/static/root/assets/{voice-api-key-field-DXWdJ8cR.js → voice-api-key-field-CzwG5plu.js} +1 -1
- package/dist/gateway/static/root/assets/voice-config-api-MqyI6cgq.js +1 -0
- package/dist/gateway/static/root/assets/work-discovery-overlay-DS15FzQA.js +1 -0
- package/dist/gateway/static/root/assets/workflow-run-setup-panel-C_boCemi.js +3 -0
- package/dist/gateway/static/root/assets/workflows-page-CaI2ruyy.js +5 -0
- package/dist/gateway/static/root/index.html +21 -22
- package/dist/package.js +1 -1
- package/dist/packages/realtime-protocol/src/voice-audio.js +20 -0
- package/dist/packages/realtime-protocol/src/voice.js +208 -0
- package/dist/src/agent/agent-manager.js +4 -4
- package/dist/src/agent/background-review/run-background-review.js +1 -1
- package/dist/src/agent/embedded/model-runtime.js +1 -1
- package/dist/src/agent/embedded/run-for-session.d.ts +1 -0
- package/dist/src/agent/embedded/run-for-session.js +3 -2
- package/dist/src/agent/image/tool-model-config.js +1 -1
- package/dist/src/agent/inbound/turn-dispatcher.d.ts +1 -0
- package/dist/src/agent/inbound/turn-dispatcher.js +2 -1
- package/dist/src/agent/ipc/inbox.js +1 -1
- package/dist/src/agent/mcp/oauth/mcp-oauth-store.js +1 -1
- package/dist/src/agent/prompt/voice-presentation.d.ts +2 -0
- package/dist/src/agent/prompt/voice-presentation.js +12 -0
- package/dist/src/agent/service/direct-turn-helpers.d.ts +1 -0
- package/dist/src/agent/service/direct-turn-helpers.js +1 -0
- package/dist/src/agent/service/process-direct-streaming.d.ts +1 -0
- package/dist/src/agent/service/process-direct-streaming.js +2 -1
- package/dist/src/agent/service.js +2 -2
- package/dist/src/agent/session/session-config-service.js +4 -2
- package/dist/src/agent/session/session-inspector.js +1 -1
- package/dist/src/agent/skills/hub-lock.js +1 -1
- package/dist/src/agent/tasks/task-judge-service.js +3 -3
- package/dist/src/ai-assist/text-assist.js +1 -1
- package/dist/src/auth/credentials.js +2 -2
- package/dist/src/auth/profiles/store.js +1 -1
- package/dist/src/auth/sync-provider-auth.js +2 -2
- package/dist/src/browser/providers/browser-ext-install.js +1 -1
- package/dist/src/cli/commands/agent.js +2 -2
- package/dist/src/cli/commands/doctor/checks/provider-auth.js +1 -1
- package/dist/src/cli/commands/gateway/lifecycle.js +1 -1
- package/dist/src/cli/commands/models.js +2 -2
- package/dist/src/cli/commands/onboard.js +1 -1
- package/dist/src/cli/commands/profile.js +1 -1
- package/dist/src/cli/commands/tunnel.js +1 -1
- package/dist/src/cli/commands/update.js +1 -1
- package/dist/src/cli/extension-cli-register.js +1 -1
- package/dist/src/config/index.js +5 -5
- package/dist/src/config/loader.js +1 -1
- package/dist/src/config/models-json.d.ts +8 -8
- package/dist/src/config/models-json.js +1 -1
- package/dist/src/config/schema.d.ts +60 -0
- package/dist/src/config/schema.js +31 -1
- package/dist/src/extensions/lockfile.js +1 -1
- package/dist/src/files/file-service.d.ts +1 -0
- package/dist/src/files/file-service.js +4 -0
- package/dist/src/gateway/agents-admin.js +1 -1
- package/dist/src/gateway/chat-stream/thinking-delta-coalescer.js +8 -3
- package/dist/src/gateway/file-path-classifier.js +1 -1
- package/dist/src/gateway/heartbeat/service.js +1 -1
- package/dist/src/gateway/hono/lib/config-payload.d.ts +1 -11
- package/dist/src/gateway/hono/lib/config-payload.js +3 -3
- package/dist/src/gateway/hono/lib/safe-voice-config.d.ts +14 -0
- package/dist/src/gateway/hono/lib/safe-voice-config.js +29 -1
- package/dist/src/gateway/hono/routes/config-patch/misc.js +2 -2
- package/dist/src/gateway/hono/routes/connectors.js +23 -23
- package/dist/src/gateway/hono/routes/files.js +7 -0
- package/dist/src/gateway/hono/routes/lazy-fallback.js +18 -4
- package/dist/src/gateway/hono/routes/models.js +1 -1
- package/dist/src/gateway/hono/routes/session-input-handler.js +14 -0
- package/dist/src/gateway/hono/routes/sessions.js +3 -0
- package/dist/src/gateway/hono/routes/voice.js +139 -2
- package/dist/src/gateway/run-loop.js +1 -1
- package/dist/src/gateway/server.js +1 -0
- package/dist/src/gateway/service/agent-runner.d.ts +1 -0
- package/dist/src/gateway/service/agent-runner.js +1 -6
- package/dist/src/gateway/service/config-coordinator.js +1 -1
- package/dist/src/gateway/service/run-gateway-agent.d.ts +1 -0
- package/dist/src/gateway/service/run-gateway-agent.js +19 -1
- package/dist/src/gateway/service.d.ts +2 -0
- package/dist/src/gateway/service.js +78 -5
- package/dist/src/heartbeat/index.js +1 -1
- package/dist/src/media-understanding/types.d.ts +61 -0
- package/dist/src/providers/auth-runtime/resolve-auth.js +1 -1
- package/dist/src/providers/index.js +2 -2
- package/dist/src/providers/model-registry.js +1 -1
- package/dist/src/providers/register-runtime-providers.js +1 -1
- package/dist/src/session/client-history.js +12 -0
- package/dist/src/session/manager.d.ts +1 -0
- package/dist/src/session/session-context-for-llm.js +8 -0
- package/dist/src/session/store.d.ts +1 -0
- package/dist/src/session/store.js +1 -0
- package/dist/src/session/stored-rows-to-file-entries.js +5 -1
- package/dist/src/session/voice-transcript.d.ts +9 -0
- package/dist/src/session/voice-transcript.js +45 -0
- package/dist/src/share/hosted-session-share.js +1 -1
- package/dist/src/storage/sqlite/config-repository.js +9 -0
- package/dist/src/storage/sqlite/remote-context-repository.d.ts +3 -0
- package/dist/src/storage/sqlite/remote-context-repository.js +40 -0
- package/dist/src/storage/sqlite/session-repository.js +13 -0
- package/dist/src/storage/sqlite/user-context-repository.d.ts +3 -1
- package/dist/src/storage/sqlite/user-context-repository.js +50 -4
- package/dist/src/tui/tui-startup-resources.js +1 -1
- package/dist/src/tui/tui.js +3 -2
- package/dist/src/user-context/automatic-context.d.ts +17 -0
- package/dist/src/user-context/automatic-context.js +85 -0
- package/dist/src/user-context/changes.d.ts +7 -0
- package/dist/src/user-context/changes.js +28 -0
- package/dist/src/user-context/planner.js +1 -24
- package/dist/src/user-context/retriever.js +10 -11
- package/dist/src/user-context/selection-policy.d.ts +20 -0
- package/dist/src/user-context/selection-policy.js +33 -0
- package/dist/src/user-context/sources/index.js +2 -2
- package/dist/src/user-context/sources/repository.d.ts +2 -1
- package/dist/src/user-context/sources/repository.js +26 -5
- package/dist/src/voice/dashscope/streaming-stt-session.d.ts +2 -0
- package/dist/src/voice/dashscope/streaming-stt-session.js +249 -0
- package/dist/src/voice/dashscope/streaming-tts-stream.d.ts +12 -0
- package/dist/src/voice/dashscope/streaming-tts-stream.js +153 -0
- package/dist/src/voice/realtime/agentEngine.d.ts +10 -0
- package/dist/src/voice/realtime/agentEngine.js +403 -0
- package/dist/src/voice/realtime/audio-playback-window.d.ts +9 -0
- package/dist/src/voice/realtime/audio-playback-window.js +49 -0
- package/dist/src/voice/realtime/conversation-context.d.ts +11 -0
- package/dist/src/voice/realtime/conversation-context.js +76 -0
- package/dist/src/voice/realtime/engine.d.ts +13 -0
- package/dist/src/voice/realtime/engine.js +1 -0
- package/dist/src/voice/realtime/memory-context.d.ts +17 -0
- package/dist/src/voice/realtime/memory-context.js +100 -0
- package/dist/src/voice/realtime/omniEngine.d.ts +18 -0
- package/dist/src/voice/realtime/omniEngine.js +443 -0
- package/dist/src/voice/realtime/omniRoute.d.ts +10 -0
- package/dist/src/voice/realtime/omniRoute.js +31 -0
- package/dist/src/voice/realtime/runtime.d.ts +99 -0
- package/dist/src/voice/realtime/runtime.js +622 -0
- package/dist/src/voice/realtime/speakable-segmenter.d.ts +10 -0
- package/dist/src/voice/realtime/speakable-segmenter.js +87 -0
- package/dist/src/voice/stt/list-providers.js +2 -1
- package/dist/src/voice/stt/providers/alibaba-transcription.js +13 -1
- package/dist/src/voice/stt/providers/xopc-cloud-transcription.js +23 -0
- package/dist/src/voice/tts/factory.js +1 -0
- package/dist/src/voice/tts/openai-compatible-speech.js +3 -1
- package/dist/src/voice/tts/providers/alibaba-speech.d.ts +1 -5
- package/dist/src/voice/tts/providers/alibaba-speech.js +25 -8
- package/dist/src/voice/tts/providers/edge-speech.js +21 -22
- package/dist/src/voice/tts/providers/minimax-speech.js +6 -3
- package/dist/src/voice/tts/providers/xopc-cloud-speech.js +26 -1
- package/dist/src/voice/tts/speak-core.d.ts +6 -2
- package/dist/src/voice/tts/speak-core.js +46 -34
- package/dist/src/voice/tts/speech-provider-types.d.ts +1 -0
- package/dist/src/workflows/service/workflow-run-service.js +1 -1
- package/package.json +7 -7
- package/dist/gateway/static/root/assets/Combination-14nmIgxH.js +0 -41
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-Cf7U2aUi.js +0 -1
- package/dist/gateway/static/root/assets/agents-Ce9oOnwE.js +0 -1
- package/dist/gateway/static/root/assets/app-chrome-BeB4KgOf.js +0 -1
- package/dist/gateway/static/root/assets/app-management-settings-panel-CACOOS-a.js +0 -1
- package/dist/gateway/static/root/assets/apps-page-BaTFOCmR.js +0 -1
- package/dist/gateway/static/root/assets/automations-workspace-DK6EreCr.js +0 -6
- package/dist/gateway/static/root/assets/block-editor-DgAs5aJc.js +0 -184
- package/dist/gateway/static/root/assets/browser-settings-page-CHStw5UW.js +0 -8
- package/dist/gateway/static/root/assets/capabilities-settings-panel-JGH6Kfkr.js +0 -7
- package/dist/gateway/static/root/assets/channels-settings-YkUaaWkY.js +0 -1
- package/dist/gateway/static/root/assets/connectors-page-BI0FG3gA.js +0 -2
- package/dist/gateway/static/root/assets/dist-BPoKqzeH.js +0 -1
- package/dist/gateway/static/root/assets/dist-DqPZM2MH.js +0 -1
- package/dist/gateway/static/root/assets/home-page--9fKq0mL.js +0 -1
- package/dist/gateway/static/root/assets/index-D499B1qS.css +0 -2
- package/dist/gateway/static/root/assets/index-J3FsIvTs.js +0 -70
- package/dist/gateway/static/root/assets/local-app-workbench-page-FqG00zS1.js +0 -2
- package/dist/gateway/static/root/assets/logs-page-c6CfZ8PA.js +0 -2
- package/dist/gateway/static/root/assets/messages-_tYhRAHD.js +0 -3
- package/dist/gateway/static/root/assets/mobile-pairing-wizard-CHnXZTwb.js +0 -1
- package/dist/gateway/static/root/assets/models-hub-cache-DiXjKuuY.js +0 -1
- package/dist/gateway/static/root/assets/notes-workbench-DIR37b-K.js +0 -4
- package/dist/gateway/static/root/assets/popover-select-DrILF6mR.js +0 -1
- package/dist/gateway/static/root/assets/product-open-page-CRm9Q66h.js +0 -1
- package/dist/gateway/static/root/assets/project-detail-page-DHiLTiRW.js +0 -1
- package/dist/gateway/static/root/assets/projects-page-God7NCcI.js +0 -1
- package/dist/gateway/static/root/assets/remote-access-hub-YpF0-mx2.js +0 -1
- package/dist/gateway/static/root/assets/sessions-page-CP8O1TxR.js +0 -1
- package/dist/gateway/static/root/assets/settings-advanced-gate-DzdbK3j9.js +0 -1
- package/dist/gateway/static/root/assets/shares-settings-B7NeVl4g.js +0 -1
- package/dist/gateway/static/root/assets/skill-reload-api-DDTmi4a7.js +0 -1
- package/dist/gateway/static/root/assets/skills-page.utils--q8p7MAn.js +0 -1
- package/dist/gateway/static/root/assets/task-detail-page-yiMzii2b.js +0 -1
- package/dist/gateway/static/root/assets/tunnel-api-CC3jOB4n.js +0 -1
- package/dist/gateway/static/root/assets/user-context-page-Vtwg7fQc.js +0 -1
- package/dist/gateway/static/root/assets/voice-config-api-BDoeXgww.js +0 -1
- package/dist/gateway/static/root/assets/work-discovery-overlay-BV_Jj5iD.js +0 -1
- package/dist/gateway/static/root/assets/workflow-run-setup-panel-DNHbWGPq.js +0 -3
- package/dist/gateway/static/root/assets/workflows-page-BkFIbMrz.js +0 -5
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
//#region src/voice/dashscope/streaming-tts-stream.ts
|
|
4
|
+
const { WebSocket } = createRequire(import.meta.url)("ws");
|
|
5
|
+
const DEFAULT_REALTIME_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime";
|
|
6
|
+
function realtimeUrl(baseUrl, model) {
|
|
7
|
+
const configured = baseUrl ? new URL(baseUrl) : new URL(DEFAULT_REALTIME_URL);
|
|
8
|
+
const wasWebSocket = configured.protocol === "ws:" || configured.protocol === "wss:";
|
|
9
|
+
configured.protocol = configured.protocol === "http:" ? "ws:" : configured.protocol === "https:" ? "wss:" : configured.protocol;
|
|
10
|
+
if (configured.protocol !== "ws:" && configured.protocol !== "wss:") throw new Error("DashScope realtime TTS base URL must use HTTP or WebSocket");
|
|
11
|
+
if (!wasWebSocket) configured.pathname = "/api-ws/v1/realtime";
|
|
12
|
+
configured.searchParams.set("model", model);
|
|
13
|
+
return configured.toString();
|
|
14
|
+
}
|
|
15
|
+
function rawText(data) {
|
|
16
|
+
if (typeof data === "string") return data;
|
|
17
|
+
if (data instanceof ArrayBuffer) return Buffer.from(data).toString("utf8");
|
|
18
|
+
if (Array.isArray(data)) return Buffer.concat(data).toString("utf8");
|
|
19
|
+
return data.toString("utf8");
|
|
20
|
+
}
|
|
21
|
+
function event(type, extra = {}) {
|
|
22
|
+
return JSON.stringify({
|
|
23
|
+
event_id: crypto.randomUUID(),
|
|
24
|
+
type,
|
|
25
|
+
...extra
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
async function openDashScopeStreamingTts(request) {
|
|
29
|
+
if (!request.apiKey) throw new Error("DashScope TTS API key is unavailable");
|
|
30
|
+
if (request.signal.aborted) throw request.signal.reason;
|
|
31
|
+
const socket = new WebSocket(realtimeUrl(request.baseUrl, request.model), {
|
|
32
|
+
headers: { Authorization: `Bearer ${request.apiKey}` },
|
|
33
|
+
perMessageDeflate: false
|
|
34
|
+
});
|
|
35
|
+
let controller;
|
|
36
|
+
let settled = false;
|
|
37
|
+
let streamClosed = false;
|
|
38
|
+
let resolveReady;
|
|
39
|
+
let rejectReady;
|
|
40
|
+
const readyPromise = new Promise((resolve, reject) => {
|
|
41
|
+
resolveReady = resolve;
|
|
42
|
+
rejectReady = reject;
|
|
43
|
+
});
|
|
44
|
+
const audioStream = new ReadableStream({
|
|
45
|
+
start(next) {
|
|
46
|
+
controller = next;
|
|
47
|
+
},
|
|
48
|
+
cancel() {
|
|
49
|
+
streamClosed = true;
|
|
50
|
+
socket.close(1e3, "Consumer cancelled");
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
const fail = (error, closeSocket = true) => {
|
|
54
|
+
if (!settled) rejectReady(error);
|
|
55
|
+
if (!streamClosed) {
|
|
56
|
+
streamClosed = true;
|
|
57
|
+
controller?.error(error);
|
|
58
|
+
}
|
|
59
|
+
if (closeSocket) {
|
|
60
|
+
if (socket.readyState === WebSocket.OPEN) socket.close(1011, "Provider stream failed");
|
|
61
|
+
else if (socket.readyState === WebSocket.CONNECTING) socket.terminate();
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const finishStream = () => {
|
|
65
|
+
if (streamClosed) return;
|
|
66
|
+
streamClosed = true;
|
|
67
|
+
controller?.close();
|
|
68
|
+
};
|
|
69
|
+
const onAbort = () => {
|
|
70
|
+
fail(request.signal.reason instanceof Error ? request.signal.reason : new DOMException("Realtime TTS aborted", "AbortError"), false);
|
|
71
|
+
if (socket.readyState === WebSocket.OPEN) socket.close(1e3, "Aborted");
|
|
72
|
+
else if (socket.readyState === WebSocket.CONNECTING) socket.terminate();
|
|
73
|
+
};
|
|
74
|
+
request.signal.addEventListener("abort", onAbort, { once: true });
|
|
75
|
+
socket.on("message", (data, isBinary) => {
|
|
76
|
+
if (streamClosed || request.signal.aborted) return;
|
|
77
|
+
if (isBinary) return;
|
|
78
|
+
let message;
|
|
79
|
+
try {
|
|
80
|
+
message = JSON.parse(rawText(data));
|
|
81
|
+
} catch {
|
|
82
|
+
fail(/* @__PURE__ */ new Error("DashScope realtime TTS returned invalid JSON"));
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
if (message.type === "session.created") {
|
|
86
|
+
socket.send(event("session.update", { session: {
|
|
87
|
+
voice: request.voice,
|
|
88
|
+
mode: "commit",
|
|
89
|
+
language_type: "Auto",
|
|
90
|
+
response_format: "pcm",
|
|
91
|
+
sample_rate: 24e3,
|
|
92
|
+
...request.instructions ? {
|
|
93
|
+
instructions: request.instructions,
|
|
94
|
+
optimize_instructions: false
|
|
95
|
+
} : {}
|
|
96
|
+
} }));
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (message.type === "session.updated") {
|
|
100
|
+
settled = true;
|
|
101
|
+
resolveReady();
|
|
102
|
+
socket.send(event("input_text_buffer.append", { text: request.text }));
|
|
103
|
+
socket.send(event("input_text_buffer.commit"));
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (message.type === "response.audio.delta") {
|
|
107
|
+
if (streamClosed) return;
|
|
108
|
+
if (typeof message.delta !== "string") {
|
|
109
|
+
fail(/* @__PURE__ */ new Error("DashScope realtime TTS returned invalid audio"));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
controller?.enqueue(Buffer.from(message.delta, "base64"));
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (message.type === "response.audio.done") {
|
|
116
|
+
finishStream();
|
|
117
|
+
socket.send(event("session.finish"));
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (message.type === "error") {
|
|
121
|
+
const code = typeof message.error?.code === "string" ? message.error.code : void 0;
|
|
122
|
+
const detail = typeof message.error?.message === "string" ? message.error.message : void 0;
|
|
123
|
+
fail(new Error([code, detail].filter(Boolean).join(": ") || "DashScope realtime TTS failed"));
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
socket.on("error", (error) => fail(error));
|
|
127
|
+
socket.on("close", () => {
|
|
128
|
+
request.signal.removeEventListener("abort", onAbort);
|
|
129
|
+
if (!streamClosed) fail(/* @__PURE__ */ new Error("DashScope realtime TTS connection closed unexpectedly"));
|
|
130
|
+
});
|
|
131
|
+
const timeout = setTimeout(() => {
|
|
132
|
+
fail(/* @__PURE__ */ new Error("DashScope realtime TTS setup timed out"));
|
|
133
|
+
socket.close(1011, "Setup timeout");
|
|
134
|
+
}, request.timeoutMs);
|
|
135
|
+
try {
|
|
136
|
+
await readyPromise;
|
|
137
|
+
} finally {
|
|
138
|
+
clearTimeout(timeout);
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
audioStream,
|
|
142
|
+
outputFormat: "pcm",
|
|
143
|
+
fileExtension: "pcm",
|
|
144
|
+
voiceCompatible: false,
|
|
145
|
+
release: async () => {
|
|
146
|
+
request.signal.removeEventListener("abort", onAbort);
|
|
147
|
+
finishStream();
|
|
148
|
+
if (socket.readyState === WebSocket.OPEN) socket.close(1e3, "Released");
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
//#endregion
|
|
153
|
+
export { openDashScopeStreamingTts };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { VoiceTicketClaim, VoiceRealtimeRuntimeOptions } from './runtime.js';
|
|
2
|
+
import type { VoiceEngine, VoiceEventSink } from './engine.js';
|
|
3
|
+
export declare function createAgentVoiceEngine(options: {
|
|
4
|
+
claim: VoiceTicketClaim;
|
|
5
|
+
runtime: VoiceRealtimeRuntimeOptions;
|
|
6
|
+
signal: AbortSignal;
|
|
7
|
+
send: VoiceEventSink;
|
|
8
|
+
sendAudio: (responseId: string, bytes: Uint8Array) => void;
|
|
9
|
+
onClose: (reason: string, notify: boolean) => Promise<void>;
|
|
10
|
+
}): VoiceEngine;
|
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
import { createLogger } from "../../utils/logger/index.js";
|
|
2
|
+
import { init_logger } from "../../utils/logger.js";
|
|
3
|
+
import { speakStream } from "../tts/speak-core.js";
|
|
4
|
+
import { AudioPlaybackWindow } from "./audio-playback-window.js";
|
|
5
|
+
import { SpeakableSegmenter } from "./speakable-segmenter.js";
|
|
6
|
+
import crypto from "node:crypto";
|
|
7
|
+
//#region src/voice/realtime/agentEngine.ts
|
|
8
|
+
init_logger();
|
|
9
|
+
const log = createLogger("Voice:Agent");
|
|
10
|
+
function createAgentVoiceEngine(options) {
|
|
11
|
+
const claim = options.claim;
|
|
12
|
+
const stt = claim.stt;
|
|
13
|
+
if (!stt) throw new Error("Agent voice requires a transcription route");
|
|
14
|
+
const send = options.send;
|
|
15
|
+
let closed = false;
|
|
16
|
+
let sttSession;
|
|
17
|
+
let activeResponse;
|
|
18
|
+
let conversationTail = Promise.resolve();
|
|
19
|
+
let interruptionWrites = Promise.resolve();
|
|
20
|
+
let closing;
|
|
21
|
+
let queuedTurns = 0;
|
|
22
|
+
let inputGeneration = 0;
|
|
23
|
+
let muted = false;
|
|
24
|
+
let sttGeneration = 0;
|
|
25
|
+
let sttAbort = new AbortController();
|
|
26
|
+
let inputReset;
|
|
27
|
+
let bufferedAudio = [];
|
|
28
|
+
let bufferedBytes = 0;
|
|
29
|
+
let finalCount = 0;
|
|
30
|
+
let committing = false;
|
|
31
|
+
const finalizedUtterances = /* @__PURE__ */ new Set();
|
|
32
|
+
function cancelActiveResponse(reason) {
|
|
33
|
+
const response = activeResponse;
|
|
34
|
+
if (!response) return false;
|
|
35
|
+
activeResponse = void 0;
|
|
36
|
+
response.abortController.abort(reason);
|
|
37
|
+
send("response.cancelled", {
|
|
38
|
+
responseId: response.id,
|
|
39
|
+
reason
|
|
40
|
+
});
|
|
41
|
+
const sessionKey = claim.request.sessionKey;
|
|
42
|
+
if (sessionKey && reason !== "session_closed") interruptionWrites = interruptionWrites.then(() => options.runtime.recordInterruption({
|
|
43
|
+
sessionKey,
|
|
44
|
+
responseId: response.id,
|
|
45
|
+
reason,
|
|
46
|
+
generatedCharacters: response.text.length,
|
|
47
|
+
interruptedDuring: response.audioStarted ? "speaking" : "thinking"
|
|
48
|
+
})).catch((error) => {
|
|
49
|
+
log.warn({
|
|
50
|
+
err: error,
|
|
51
|
+
sessionId: claim.sessionId,
|
|
52
|
+
responseId: response.id
|
|
53
|
+
}, "Voice interruption audit failed");
|
|
54
|
+
});
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
async function speakPhrase(response, phrase) {
|
|
58
|
+
if (!claim.tts || response.abortController.signal.aborted) return;
|
|
59
|
+
const result = await speakStream(phrase, claim.tts.config, {
|
|
60
|
+
appConfig: claim.config,
|
|
61
|
+
parseDirectives: false,
|
|
62
|
+
signal: response.abortController.signal,
|
|
63
|
+
allowFallback: false
|
|
64
|
+
});
|
|
65
|
+
try {
|
|
66
|
+
if (response.abortController.signal.aborted || activeResponse !== response || closed) return;
|
|
67
|
+
if (result.outputFormat !== "pcm") throw new Error(`Realtime TTS returned unsupported format: ${result.outputFormat}`);
|
|
68
|
+
const reader = result.audioStream.getReader();
|
|
69
|
+
while (true) {
|
|
70
|
+
const item = await reader.read();
|
|
71
|
+
if (item.done) break;
|
|
72
|
+
if (response.abortController.signal.aborted || activeResponse !== response) return;
|
|
73
|
+
if (!response.audioStarted) {
|
|
74
|
+
response.audioStarted = true;
|
|
75
|
+
log.info({
|
|
76
|
+
sessionId: claim.sessionId,
|
|
77
|
+
responseId: response.id,
|
|
78
|
+
latencyMs: Date.now() - response.startedAt
|
|
79
|
+
}, "Realtime voice first audio ready");
|
|
80
|
+
send("response.audio.started", {
|
|
81
|
+
responseId: response.id,
|
|
82
|
+
format: {
|
|
83
|
+
encoding: "pcm_s16le",
|
|
84
|
+
sampleRate: 24e3,
|
|
85
|
+
channels: 1
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
for (let offset = 0; offset < item.value.byteLength; offset += 24e3) {
|
|
90
|
+
const chunk = item.value.subarray(offset, offset + 24e3);
|
|
91
|
+
await response.playback.reserve(chunk.byteLength, response.abortController.signal);
|
|
92
|
+
if (activeResponse !== response || closed) return;
|
|
93
|
+
options.sendAudio(response.id, chunk);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
} finally {
|
|
97
|
+
await result.release();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function queuePhrases(response, phrases) {
|
|
101
|
+
for (const phrase of phrases) {
|
|
102
|
+
if (response.speechError || response.abortController.signal.aborted) return;
|
|
103
|
+
if (response.queuedSpeechCharacters + phrase.length > 8e3) {
|
|
104
|
+
response.speechError = /* @__PURE__ */ new Error("Voice synthesis queue limit reached");
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
response.queuedSpeechCharacters += phrase.length;
|
|
108
|
+
response.speechTail = response.speechTail.then(async () => {
|
|
109
|
+
if (response.speechError || response.abortController.signal.aborted) return;
|
|
110
|
+
try {
|
|
111
|
+
await speakPhrase(response, phrase);
|
|
112
|
+
} catch (error) {
|
|
113
|
+
if (!response.abortController.signal.aborted) response.speechError = error instanceof Error ? error : new Error(String(error));
|
|
114
|
+
}
|
|
115
|
+
}).finally(() => {
|
|
116
|
+
response.queuedSpeechCharacters -= phrase.length;
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async function runConversationTurn(text) {
|
|
121
|
+
if (!claim.tts || !claim.request.sessionKey || !text.trim() || closed) return;
|
|
122
|
+
cancelActiveResponse("barge_in");
|
|
123
|
+
const response = {
|
|
124
|
+
id: `resp_${crypto.randomUUID()}`,
|
|
125
|
+
playback: new AudioPlaybackWindow(),
|
|
126
|
+
abortController: new AbortController(),
|
|
127
|
+
segmenter: new SpeakableSegmenter(),
|
|
128
|
+
text: "",
|
|
129
|
+
audioStarted: false,
|
|
130
|
+
speechTail: Promise.resolve(),
|
|
131
|
+
queuedSpeechCharacters: 0,
|
|
132
|
+
startedAt: Date.now(),
|
|
133
|
+
firstTextSeen: false,
|
|
134
|
+
awaitingClarification: false
|
|
135
|
+
};
|
|
136
|
+
activeResponse = response;
|
|
137
|
+
send("response.created", { responseId: response.id });
|
|
138
|
+
try {
|
|
139
|
+
await interruptionWrites;
|
|
140
|
+
if (response.abortController.signal.aborted || closed) return;
|
|
141
|
+
for await (const event of options.runtime.runAgent(text, claim.request.sessionKey, response.abortController.signal)) {
|
|
142
|
+
if (activeResponse !== response || response.abortController.signal.aborted) return;
|
|
143
|
+
if (event.type === "assistant_delta" || event.type === "tool_end") response.awaitingClarification = false;
|
|
144
|
+
if (event.type === "assistant_delta" && typeof event.payload?.delta === "string") {
|
|
145
|
+
if (response.text.length + event.payload.delta.length > 32e3) throw new Error("Voice response text limit reached");
|
|
146
|
+
if (!response.firstTextSeen) {
|
|
147
|
+
response.firstTextSeen = true;
|
|
148
|
+
log.info({
|
|
149
|
+
sessionId: claim.sessionId,
|
|
150
|
+
responseId: response.id,
|
|
151
|
+
latencyMs: Date.now() - response.startedAt
|
|
152
|
+
}, "Realtime voice first response text ready");
|
|
153
|
+
}
|
|
154
|
+
response.text += event.payload.delta;
|
|
155
|
+
send("response.text.delta", {
|
|
156
|
+
responseId: response.id,
|
|
157
|
+
delta: event.payload.delta
|
|
158
|
+
});
|
|
159
|
+
queuePhrases(response, response.segmenter.push(event.payload.delta));
|
|
160
|
+
}
|
|
161
|
+
if ((event.type === "tool_start" || event.type === "tool_end") && typeof event.payload?.toolCallId === "string" && typeof event.payload?.toolName === "string") send("response.activity", {
|
|
162
|
+
responseId: response.id,
|
|
163
|
+
toolCallId: event.payload.toolCallId.slice(0, 160),
|
|
164
|
+
toolName: event.payload.toolName.slice(0, 256),
|
|
165
|
+
status: event.type === "tool_start" ? "running" : event.payload.status === "error" ? "failed" : "completed"
|
|
166
|
+
});
|
|
167
|
+
if (event.type === "clarify_request" && typeof event.payload?.requestId === "string" && typeof event.payload?.question === "string") {
|
|
168
|
+
response.awaitingClarification = true;
|
|
169
|
+
send("response.clarification", {
|
|
170
|
+
responseId: response.id,
|
|
171
|
+
requestId: event.payload.requestId.slice(0, 160),
|
|
172
|
+
question: event.payload.question.slice(0, 8e3),
|
|
173
|
+
...Array.isArray(event.payload.choices) ? { choices: event.payload.choices.filter((choice) => typeof choice === "string").slice(0, 20).map((choice) => choice.slice(0, 1e3)) } : {}
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
if (event.type === "error") throw new Error(typeof event.payload?.message === "string" ? event.payload.message : "Agent response failed");
|
|
177
|
+
}
|
|
178
|
+
if (activeResponse !== response || response.abortController.signal.aborted || closed) return;
|
|
179
|
+
queuePhrases(response, response.segmenter.flush());
|
|
180
|
+
send("response.text.done", { responseId: response.id });
|
|
181
|
+
await response.speechTail;
|
|
182
|
+
if (response.speechError) throw response.speechError;
|
|
183
|
+
await response.playback.drain(response.abortController.signal);
|
|
184
|
+
if (activeResponse !== response || response.abortController.signal.aborted) return;
|
|
185
|
+
if (response.audioStarted) send("response.audio.done", { responseId: response.id });
|
|
186
|
+
send("response.done", {
|
|
187
|
+
responseId: response.id,
|
|
188
|
+
finishReason: response.audioStarted ? "completed" : "text_only",
|
|
189
|
+
audio: response.audioStarted
|
|
190
|
+
});
|
|
191
|
+
activeResponse = void 0;
|
|
192
|
+
} catch (error) {
|
|
193
|
+
if (response.abortController.signal.aborted || closed) return;
|
|
194
|
+
log.warn({
|
|
195
|
+
err: error,
|
|
196
|
+
sessionId: claim.sessionId,
|
|
197
|
+
responseId: response.id
|
|
198
|
+
}, "Realtime voice response failed");
|
|
199
|
+
send("session.error", {
|
|
200
|
+
code: "RESPONSE_FAILED",
|
|
201
|
+
message: "Voice response failed",
|
|
202
|
+
recoverable: true
|
|
203
|
+
});
|
|
204
|
+
if (response.audioStarted) send("response.audio.done", { responseId: response.id });
|
|
205
|
+
send("response.done", {
|
|
206
|
+
responseId: response.id,
|
|
207
|
+
finishReason: response.audioStarted ? "audio_partial" : "text_only",
|
|
208
|
+
audio: response.audioStarted
|
|
209
|
+
});
|
|
210
|
+
response.abortController.abort("provider_error");
|
|
211
|
+
if (activeResponse === response) activeResponse = void 0;
|
|
212
|
+
} finally {
|
|
213
|
+
await response.speechTail;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async function openSttSession(consumed) {
|
|
217
|
+
const generation = sttGeneration;
|
|
218
|
+
return stt.plugin.openAudioStream({
|
|
219
|
+
model: stt.model,
|
|
220
|
+
inputFormat: {
|
|
221
|
+
encoding: "pcm_s16le",
|
|
222
|
+
sampleRate: 16e3,
|
|
223
|
+
channels: 1
|
|
224
|
+
},
|
|
225
|
+
...stt.apiKey ? { apiKey: stt.apiKey } : {},
|
|
226
|
+
...stt.baseUrl ? { baseUrl: stt.baseUrl } : {},
|
|
227
|
+
...stt.headers ? { headers: stt.headers } : {},
|
|
228
|
+
...stt.language ? { language: stt.language } : {},
|
|
229
|
+
...stt.prompt ? { prompt: stt.prompt } : {},
|
|
230
|
+
turnDetection: {
|
|
231
|
+
mode: consumed.inputMode,
|
|
232
|
+
silenceDurationMs: consumed.silenceDurationMs
|
|
233
|
+
},
|
|
234
|
+
timeoutMs: 15e3,
|
|
235
|
+
signal: AbortSignal.any([options.signal, sttAbort.signal]),
|
|
236
|
+
onEvent: (event) => {
|
|
237
|
+
if (generation === sttGeneration) onSttEvent(event);
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
function onSttEvent(event) {
|
|
242
|
+
if (closed) return;
|
|
243
|
+
if (event.type === "ready" || event.type === "usage") return;
|
|
244
|
+
if (event.type === "error") {
|
|
245
|
+
log.warn({
|
|
246
|
+
err: event.error,
|
|
247
|
+
sessionId: claim.sessionId,
|
|
248
|
+
provider: stt.route.provider
|
|
249
|
+
}, "Realtime STT failed");
|
|
250
|
+
send("session.error", {
|
|
251
|
+
code: "PROVIDER_ERROR",
|
|
252
|
+
message: "Streaming transcription failed",
|
|
253
|
+
recoverable: false
|
|
254
|
+
});
|
|
255
|
+
options.onClose("provider_error", true);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (muted || finalizedUtterances.has(event.utteranceId)) return;
|
|
259
|
+
if (event.type === "speech_started") {
|
|
260
|
+
if (claim.request.purpose === "conversation" && claim.config.voice?.realtime?.bargeIn && !activeResponse?.awaitingClarification) cancelActiveResponse("barge_in");
|
|
261
|
+
send("input.speech_started", { utteranceId: event.utteranceId });
|
|
262
|
+
}
|
|
263
|
+
if (event.type === "speech_stopped") send("input.speech_stopped", { utteranceId: event.utteranceId });
|
|
264
|
+
if (event.type === "transcript_delta") send("input.transcript.delta", {
|
|
265
|
+
utteranceId: event.utteranceId,
|
|
266
|
+
revision: event.revision,
|
|
267
|
+
text: event.text
|
|
268
|
+
});
|
|
269
|
+
if (event.type === "transcript_final") {
|
|
270
|
+
const text = event.text.trim();
|
|
271
|
+
if (!text) return;
|
|
272
|
+
if (finalizedUtterances.size >= 1e4) {
|
|
273
|
+
options.onClose("utterance_limit", true);
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
finalizedUtterances.add(event.utteranceId);
|
|
277
|
+
finalCount += 1;
|
|
278
|
+
send("input.transcript.final", {
|
|
279
|
+
utteranceId: event.utteranceId,
|
|
280
|
+
revision: event.revision,
|
|
281
|
+
text,
|
|
282
|
+
...event.language === "zh" || event.language === "en" ? { language: event.language } : {}
|
|
283
|
+
});
|
|
284
|
+
if (claim.request.purpose === "conversation") {
|
|
285
|
+
if (activeResponse?.awaitingClarification) return;
|
|
286
|
+
if (claim.config.voice?.realtime?.bargeIn) cancelActiveResponse("barge_in");
|
|
287
|
+
if (queuedTurns >= 8) {
|
|
288
|
+
send("session.error", {
|
|
289
|
+
code: "INPUT_BACKPRESSURE",
|
|
290
|
+
message: "Too many queued voice turns",
|
|
291
|
+
recoverable: false
|
|
292
|
+
});
|
|
293
|
+
options.onClose("input_backpressure", true);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
queuedTurns += 1;
|
|
297
|
+
const generation = inputGeneration;
|
|
298
|
+
conversationTail = conversationTail.then(() => {
|
|
299
|
+
if (generation === inputGeneration) return runConversationTurn(text);
|
|
300
|
+
}).finally(() => {
|
|
301
|
+
if (generation === inputGeneration) queuedTurns -= 1;
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
function discardInput() {
|
|
307
|
+
inputGeneration += 1;
|
|
308
|
+
queuedTurns = 0;
|
|
309
|
+
bufferedAudio = [];
|
|
310
|
+
bufferedBytes = 0;
|
|
311
|
+
const generation = ++sttGeneration;
|
|
312
|
+
sttAbort.abort("input_discarded");
|
|
313
|
+
sttAbort = new AbortController();
|
|
314
|
+
sttSession?.abort("input_discarded");
|
|
315
|
+
sttSession = void 0;
|
|
316
|
+
inputReset = openSttSession(claim).then((opened) => {
|
|
317
|
+
if (closed || generation !== sttGeneration) {
|
|
318
|
+
opened.abort("input_replaced");
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
sttSession = opened;
|
|
322
|
+
for (const bytes of bufferedAudio) opened.appendAudio(bytes);
|
|
323
|
+
bufferedAudio = [];
|
|
324
|
+
bufferedBytes = 0;
|
|
325
|
+
}).catch((error) => {
|
|
326
|
+
if (!closed && generation === sttGeneration) throw error;
|
|
327
|
+
});
|
|
328
|
+
return inputReset;
|
|
329
|
+
}
|
|
330
|
+
return {
|
|
331
|
+
async start() {
|
|
332
|
+
const opened = await openSttSession(claim);
|
|
333
|
+
if (closed || options.signal.aborted) {
|
|
334
|
+
opened.abort("session_closed");
|
|
335
|
+
throw new Error("Voice session closed during initialization");
|
|
336
|
+
}
|
|
337
|
+
sttSession = opened;
|
|
338
|
+
},
|
|
339
|
+
async setInputMuted(next) {
|
|
340
|
+
if (closed || muted === next) return;
|
|
341
|
+
muted = next;
|
|
342
|
+
if (!next) return inputReset;
|
|
343
|
+
return discardInput();
|
|
344
|
+
},
|
|
345
|
+
appendAudio(bytes) {
|
|
346
|
+
if (muted) return;
|
|
347
|
+
if (!closed && !sttSession && inputReset) {
|
|
348
|
+
if (bufferedBytes + bytes.byteLength > 64e3) throw new Error("Voice input reset exceeded audio buffer");
|
|
349
|
+
bufferedAudio.push(Uint8Array.from(bytes));
|
|
350
|
+
bufferedBytes += bytes.byteLength;
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
if (closed || !sttSession || committing) throw new Error("Voice input is not ready");
|
|
354
|
+
sttSession.appendAudio(bytes);
|
|
355
|
+
},
|
|
356
|
+
async commit() {
|
|
357
|
+
if (claim.request.purpose !== "dictation" || committing || !sttSession || closed) throw new Error("Input cannot be committed");
|
|
358
|
+
committing = true;
|
|
359
|
+
await sttSession.commit();
|
|
360
|
+
if (closed) return;
|
|
361
|
+
if (finalCount === 0) send("session.error", {
|
|
362
|
+
code: "EMPTY_UTTERANCE",
|
|
363
|
+
message: "No speech was recognized",
|
|
364
|
+
recoverable: true
|
|
365
|
+
});
|
|
366
|
+
await options.onClose("input_committed", true);
|
|
367
|
+
},
|
|
368
|
+
cancel(responseId, reason) {
|
|
369
|
+
if (activeResponse?.id !== responseId) return false;
|
|
370
|
+
if (reason === "client_cancelled") discardInput().catch((error) => {
|
|
371
|
+
log.warn({
|
|
372
|
+
err: error,
|
|
373
|
+
sessionId: claim.sessionId
|
|
374
|
+
}, "Voice input reset after interruption failed");
|
|
375
|
+
send("session.error", {
|
|
376
|
+
code: "PROVIDER_ERROR",
|
|
377
|
+
message: "Could not resume microphone input",
|
|
378
|
+
recoverable: false
|
|
379
|
+
});
|
|
380
|
+
options.onClose("provider_error", true);
|
|
381
|
+
});
|
|
382
|
+
return cancelActiveResponse(reason);
|
|
383
|
+
},
|
|
384
|
+
acknowledge(responseId, playedBytes) {
|
|
385
|
+
if (activeResponse?.id === responseId) activeResponse.playback.acknowledge(playedBytes);
|
|
386
|
+
},
|
|
387
|
+
close() {
|
|
388
|
+
if (closing) return closing;
|
|
389
|
+
closed = true;
|
|
390
|
+
cancelActiveResponse("session_closed");
|
|
391
|
+
sttAbort.abort("session_closed");
|
|
392
|
+
sttSession?.abort("session_closed");
|
|
393
|
+
closing = Promise.all([
|
|
394
|
+
conversationTail,
|
|
395
|
+
interruptionWrites,
|
|
396
|
+
inputReset
|
|
397
|
+
]).then(() => {});
|
|
398
|
+
return closing;
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
//#endregion
|
|
403
|
+
export { createAgentVoiceEngine };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare class AudioPlaybackWindow {
|
|
2
|
+
private sentBytes;
|
|
3
|
+
private playedBytes;
|
|
4
|
+
private wake?;
|
|
5
|
+
acknowledge(playedBytes: number): void;
|
|
6
|
+
reserve(bytes: number, signal: AbortSignal): Promise<void>;
|
|
7
|
+
drain(signal: AbortSignal): Promise<void>;
|
|
8
|
+
private wait;
|
|
9
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
//#region src/voice/realtime/audio-playback-window.ts
|
|
2
|
+
const MAX_PENDING_BYTES = 96e3;
|
|
3
|
+
const PLAYBACK_TIMEOUT_MS = 15e3;
|
|
4
|
+
var AudioPlaybackWindow = class {
|
|
5
|
+
sentBytes = 0;
|
|
6
|
+
playedBytes = 0;
|
|
7
|
+
wake;
|
|
8
|
+
acknowledge(playedBytes) {
|
|
9
|
+
if (playedBytes > this.sentBytes) throw new Error("Playback acknowledgement exceeds sent audio");
|
|
10
|
+
if (playedBytes <= this.playedBytes) return;
|
|
11
|
+
this.playedBytes = playedBytes;
|
|
12
|
+
this.wake?.();
|
|
13
|
+
}
|
|
14
|
+
async reserve(bytes, signal) {
|
|
15
|
+
if (bytes > MAX_PENDING_BYTES) throw new Error("Audio frame exceeds playback window");
|
|
16
|
+
while (this.sentBytes - this.playedBytes + bytes > MAX_PENDING_BYTES) await this.wait(signal);
|
|
17
|
+
signal.throwIfAborted();
|
|
18
|
+
this.sentBytes += bytes;
|
|
19
|
+
}
|
|
20
|
+
async drain(signal) {
|
|
21
|
+
while (this.sentBytes > this.playedBytes) await this.wait(signal);
|
|
22
|
+
signal.throwIfAborted();
|
|
23
|
+
}
|
|
24
|
+
wait(signal) {
|
|
25
|
+
signal.throwIfAborted();
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const cleanup = () => {
|
|
28
|
+
clearTimeout(timer);
|
|
29
|
+
signal.removeEventListener("abort", onAbort);
|
|
30
|
+
this.wake = void 0;
|
|
31
|
+
};
|
|
32
|
+
const onAbort = () => {
|
|
33
|
+
cleanup();
|
|
34
|
+
reject(signal.reason);
|
|
35
|
+
};
|
|
36
|
+
const timer = setTimeout(() => {
|
|
37
|
+
cleanup();
|
|
38
|
+
reject(/* @__PURE__ */ new Error("Voice playback acknowledgement timed out"));
|
|
39
|
+
}, PLAYBACK_TIMEOUT_MS);
|
|
40
|
+
this.wake = () => {
|
|
41
|
+
cleanup();
|
|
42
|
+
resolve();
|
|
43
|
+
};
|
|
44
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
//#endregion
|
|
49
|
+
export { AudioPlaybackWindow };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { AgentMessage } from '@earendil-works/pi-agent-core';
|
|
2
|
+
import type { VoiceMemorySnapshot } from './memory-context.js';
|
|
3
|
+
export interface VoiceConversationContext {
|
|
4
|
+
identity: string;
|
|
5
|
+
history: AgentMessage[];
|
|
6
|
+
memory?: VoiceMemorySnapshot;
|
|
7
|
+
}
|
|
8
|
+
/** Use the canonical, already-compacted Session context. No tools or hidden reasoning. */
|
|
9
|
+
export declare function buildVoiceHistory(messages: AgentMessage[], budget?: number): string;
|
|
10
|
+
export declare function voiceMemoryBudget(base: string, context: VoiceConversationContext): number;
|
|
11
|
+
export declare function voiceConversationInstructions(base: string, context: VoiceConversationContext): string;
|