@xopcai/xopc 0.0.241 → 0.0.243
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/manifest.json +1 -1
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{activity-store-CLnbVm9G.js → activity-store-CAah-_8h.js} +1 -1
- package/dist/gateway/static/root/assets/{agent-avatar-dicebear-DuBewmbW.js → agent-avatar-dicebear-DDH4HgP7.js} +1 -1
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-BDCfJZhh.js +1 -0
- package/dist/gateway/static/root/assets/agents-B0ZWgf-U.js +1 -0
- package/dist/gateway/static/root/assets/{api-DjBZe303.js → api-B2tKxbcz.js} +1 -1
- package/dist/gateway/static/root/assets/app-management-settings-panel-Blfwszzn.js +1 -0
- package/dist/gateway/static/root/assets/{appearance-settings-B-iul0RG.js → appearance-settings-C3V7Na75.js} +1 -1
- package/dist/gateway/static/root/assets/apps-page-BiEks0-F.js +1 -0
- package/dist/gateway/static/root/assets/{archive-plugin-BOx0EJZs.js → archive-plugin-BmSx7wzY.js} +1 -1
- package/dist/gateway/static/root/assets/{attachment-load-Cfk1lwBo.js → attachment-load-BS-S4LJ9.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-page-BTSGvaeS.js → automations-page-C7Y3L30D.js} +1 -1
- package/dist/gateway/static/root/assets/automations-workspace-A3zPkPuo.js +6 -0
- package/dist/gateway/static/root/assets/{binary-plugins-ChZyZK45.js → binary-plugins-BM9B5kcm.js} +1 -1
- package/dist/gateway/static/root/assets/block-editor-BbnpkYP9.js +185 -0
- package/dist/gateway/static/root/assets/browser-settings-page-Dk4FQ7fY.js +8 -0
- package/dist/gateway/static/root/assets/{browser-workflow-inputs-jvfcgFG-.js → browser-workflow-inputs-CRnkgZHI.js} +1 -1
- package/dist/gateway/static/root/assets/{browser-workflows-page-DrHW7XNi.js → browser-workflows-page-B9qGL-it.js} +2 -2
- package/dist/gateway/static/root/assets/{button-CSEbl4zT.js → button-CtTvuElV.js} +1 -1
- package/dist/gateway/static/root/assets/capabilities-settings-panel--tR_El9e.js +7 -0
- package/dist/gateway/static/root/assets/channels-settings-D347vl_J.js +1 -0
- package/dist/gateway/static/root/assets/{channels-status-swr-CC9MFway.js → channels-status-swr-DuY4MGxe.js} +1 -1
- package/dist/gateway/static/root/assets/{chat-terminal-dock-DeeZrVbV.js → chat-terminal-dock-DABPvwBT.js} +2 -2
- package/dist/gateway/static/root/assets/connectors-page-B-tvJkRJ.js +2 -0
- package/dist/gateway/static/root/assets/{cron-expression-CiqQOeQ3.js → cron-expression-D3bsKbNd.js} +1 -1
- package/dist/gateway/static/root/assets/{date-picker-CGAMGPfU.js → date-picker-BsGM6Yty.js} +1 -1
- package/dist/gateway/static/root/assets/{dependency-picker-CxWeUTsc.js → dependency-picker-Dj3g6YXZ.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-C3QmYO75.js → desktop-pet-C1QT0Sur.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-settings-CAADkWYy.js → desktop-pet-settings-r6kAOnfY.js} +1 -1
- package/dist/gateway/static/root/assets/dist-CpzYSoz7.js +1 -0
- package/dist/gateway/static/root/assets/{dist-DaxPZj9e.js → dist-DBS3OB8K.js} +1 -1
- package/dist/gateway/static/root/assets/dist-DLmwHyAh.js +1 -0
- package/dist/gateway/static/root/assets/es2015-BvLh9GaE.js +41 -0
- package/dist/gateway/static/root/assets/{extension-debug-page-FQXjDJ8P.js → extension-debug-page-DOkf7nh9.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-BJE1D5Ji.js → extension-page-Cq2IA4Ho.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-5Ik2tNry.js → extension-settings-page-DQn4qjDZ.js} +1 -1
- package/dist/gateway/static/root/assets/{files-DQjhZjr2.js → files-D6vf3S30.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-api-cTj71t2a.js → gateway-config-api-DM3poL22.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-swr-Bxe_R-JH.js → gateway-config-swr-BC3pgb_P.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-settings-KXBUjNcx.js → gateway-settings-DPgH1hnh.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-startup-retry-DJG-I0wW.js → gateway-startup-retry-uMcSfLru.js} +1 -1
- package/dist/gateway/static/root/assets/{heartbeat-settings-C8cVzIcs.js → heartbeat-settings-Bk80-E3b.js} +1 -1
- package/dist/gateway/static/root/assets/home-page-DKo5JiMR.js +1 -0
- package/dist/gateway/static/root/assets/{image-generation-api-Dc_ojQuj.js → image-generation-api-D1qlQA-A.js} +1 -1
- package/dist/gateway/static/root/assets/index-B0WJL9N1.js +71 -0
- package/dist/gateway/static/root/assets/index-Dbh5SXyV.css +2 -0
- package/dist/gateway/static/root/assets/{keyboard-shortcuts-settings-pOKl7blR.js → keyboard-shortcuts-settings-BEwvd8ZU.js} +1 -1
- package/dist/gateway/static/root/assets/local-app-workbench-page--eJ5rmNj.js +2 -0
- package/dist/gateway/static/root/assets/{local-apps-page-Bi6wwKgm.js → local-apps-page-CgkY9E-r.js} +1 -1
- package/dist/gateway/static/root/assets/{locale-store-C5wr3_pc.js → locale-store-CMZljTJx.js} +1 -1
- package/dist/gateway/static/root/assets/logs-page-Z49Td0J9.js +2 -0
- package/dist/gateway/static/root/assets/{management-settings-DGlVizWi.js → management-settings-Dv8NSJKt.js} +1 -1
- package/dist/gateway/static/root/assets/{markdown-split-CXP5zBx_.js → markdown-split-Cm0RBB-t.js} +1 -1
- package/dist/gateway/static/root/assets/{markdown-view-D6LxlDxr.js → markdown-view-PPCLt7fu.js} +3 -3
- package/dist/gateway/static/root/assets/{media-plugins-BsENGLr2.js → media-plugins-Bcr5sU1f.js} +1 -1
- package/dist/gateway/static/root/assets/messages-CTSgFlE4.js +3 -0
- package/dist/gateway/static/root/assets/mobile-pairing-wizard-kKDiXofY.js +1 -0
- package/dist/gateway/static/root/assets/{models-hub-cache-Y7GUjxHx.js → models-hub-cache-ZrbGRiGY.js} +1 -1
- package/dist/gateway/static/root/assets/{notes-page-D2EqpDyo.js → notes-page-CgsaRaGi.js} +1 -1
- package/dist/gateway/static/root/assets/notes-workbench-C1DEE09k.js +4 -0
- package/dist/gateway/static/root/assets/{oauth-api-C5NVcsKE.js → oauth-api-4VeMbkTJ.js} +1 -1
- package/dist/gateway/static/root/assets/popover-select-C-Hj-nXR.js +1 -0
- package/dist/gateway/static/root/assets/{product-delivery-dUJgmBQZ.js → product-delivery-CV4pDOtG.js} +1 -1
- package/dist/gateway/static/root/assets/product-open-page-BU0HiZtv.js +1 -0
- package/dist/gateway/static/root/assets/project-detail-page-isRapnp5.js +1 -0
- package/dist/gateway/static/root/assets/projects-page-zvsjg0mi.js +1 -0
- package/dist/gateway/static/root/assets/{registry-api-FXdFaoWM.js → registry-api-CWOtPFrE.js} +1 -1
- package/dist/gateway/static/root/assets/remote-access-hub-C9TGZ9Hc.js +1 -0
- package/dist/gateway/static/root/assets/runtime-tools-settings-panel-2zm9edMA.js +5 -0
- package/dist/gateway/static/root/assets/{schema-form-BUzdHWx6.js → schema-form-54A5gzkN.js} +1 -1
- package/dist/gateway/static/root/assets/{schemas-zIBW3wiV.js → schemas-BHS7bo06.js} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-BP8RyiUn.js +1 -0
- package/dist/gateway/static/root/assets/settings-advanced-gate-D4EMxWAZ.js +1 -0
- package/dist/gateway/static/root/assets/{settings-page-C0OBLqpu.js → settings-page-DzDhoyxv.js} +1 -1
- package/dist/gateway/static/root/assets/{setup-status-panel-NpLQkFCh.js → setup-status-panel-uDicHU1w.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-BtbPYaYC.js → share-preview-page-lOmLCWFS.js} +1 -1
- package/dist/gateway/static/root/assets/shares-settings-T4Yvouvv.js +1 -0
- package/dist/gateway/static/root/assets/skill-reload-api-DVb7nrXz.js +1 -0
- package/dist/gateway/static/root/assets/{skills-page-CYk8pN97.js → skills-page-CnyNH3To.js} +1 -1
- package/dist/gateway/static/root/assets/skills-page.utils-CgDtf2qs.js +1 -0
- package/dist/gateway/static/root/assets/{system-settings-panel-DmCRSvtd.js → system-settings-panel-C8yWfwtR.js} +1 -1
- package/dist/gateway/static/root/assets/task-detail-page-Tzfjlbf3.js +1 -0
- package/dist/gateway/static/root/assets/{theme-store-DUko1ZnE.js → theme-store-uxUwr10D.js} +1 -1
- package/dist/gateway/static/root/assets/tunnel-api-Bvg679no.js +1 -0
- package/dist/gateway/static/root/assets/{use-autosave-W_P8Kiv4.js → use-autosave-D88VGoyG.js} +1 -1
- package/dist/gateway/static/root/assets/user-context-page-RijPtyfO.js +1 -0
- package/dist/gateway/static/root/assets/{voice-api-key-field-BZD0j1jc.js → voice-api-key-field-CBHLsMcc.js} +1 -1
- package/dist/gateway/static/root/assets/voice-config-api-MqyI6cgq.js +1 -0
- package/dist/gateway/static/root/assets/work-discovery-overlay-GGe2TQ98.js +1 -0
- package/dist/gateway/static/root/assets/workflow-run-setup-panel-C0layUE9.js +3 -0
- package/dist/gateway/static/root/assets/workflows-page-DQXIzBAc.js +5 -0
- package/dist/gateway/static/root/index.html +21 -22
- package/dist/package.js +1 -1
- package/dist/packages/realtime-protocol/src/voice-audio.js +20 -0
- package/dist/packages/realtime-protocol/src/voice.js +208 -0
- package/dist/src/agent/agent-manager.js +1 -0
- package/dist/src/agent/embedded/run-for-session.d.ts +1 -0
- package/dist/src/agent/embedded/run-for-session.js +2 -1
- package/dist/src/agent/inbound/turn-dispatcher.d.ts +1 -0
- package/dist/src/agent/inbound/turn-dispatcher.js +2 -1
- package/dist/src/agent/prompt/voice-presentation.d.ts +2 -0
- package/dist/src/agent/prompt/voice-presentation.js +12 -0
- package/dist/src/agent/service/direct-turn-helpers.d.ts +1 -0
- package/dist/src/agent/service/direct-turn-helpers.js +1 -0
- package/dist/src/agent/service/process-direct-streaming.d.ts +1 -0
- package/dist/src/agent/service/process-direct-streaming.js +2 -1
- package/dist/src/agent/session/session-config-service.js +4 -2
- package/dist/src/agent/skills/installer.d.ts +1 -0
- package/dist/src/agent/skills/installer.js +9 -6
- package/dist/src/agent/tools/factory.d.ts +2 -0
- package/dist/src/agent/tools/factory.js +5 -1
- package/dist/src/automations/draft/automation-draft-service.js +1 -1
- package/dist/src/config/models-json.d.ts +8 -8
- package/dist/src/config/schema.d.ts +60 -0
- package/dist/src/config/schema.js +36 -4
- package/dist/src/extensions/install.js +1 -1
- package/dist/src/files/file-service.d.ts +1 -0
- package/dist/src/files/file-service.js +4 -0
- package/dist/src/gateway/chat-stream/thinking-delta-coalescer.js +8 -3
- package/dist/src/gateway/hono/lib/config-payload.d.ts +1 -11
- package/dist/src/gateway/hono/lib/config-payload.js +2 -2
- package/dist/src/gateway/hono/lib/safe-voice-config.d.ts +14 -0
- package/dist/src/gateway/hono/lib/safe-voice-config.js +29 -1
- package/dist/src/gateway/hono/routes/config-patch/misc.js +2 -2
- package/dist/src/gateway/hono/routes/files.js +7 -0
- package/dist/src/gateway/hono/routes/lazy-fallback.js +18 -4
- package/dist/src/gateway/hono/routes/runtime-tools.js +7 -1
- package/dist/src/gateway/hono/routes/session-input-handler.js +14 -0
- package/dist/src/gateway/hono/routes/sessions.js +4 -1
- package/dist/src/gateway/hono/routes/voice.js +140 -3
- package/dist/src/gateway/server.js +1 -0
- package/dist/src/gateway/service/agent-runner.d.ts +1 -0
- package/dist/src/gateway/service/agent-runner.js +1 -6
- package/dist/src/gateway/service/run-gateway-agent.d.ts +1 -0
- package/dist/src/gateway/service/run-gateway-agent.js +19 -1
- package/dist/src/gateway/service.d.ts +2 -0
- package/dist/src/gateway/service.js +73 -0
- package/dist/src/media-understanding/types.d.ts +61 -0
- package/dist/src/runtime-tools/archive.js +17 -3
- package/dist/src/runtime-tools/command.d.ts +1 -0
- package/dist/src/runtime-tools/command.js +29 -8
- package/dist/src/runtime-tools/environment.d.ts +1 -0
- package/dist/src/runtime-tools/environment.js +11 -16
- package/dist/src/runtime-tools/install-lock.d.ts +1 -1
- package/dist/src/runtime-tools/install-lock.js +34 -21
- package/dist/src/runtime-tools/manager.d.ts +1 -1
- package/dist/src/runtime-tools/manager.js +99 -64
- package/dist/src/runtime-tools/manifest-store.d.ts +2 -0
- package/dist/src/runtime-tools/manifest-store.js +29 -8
- package/dist/src/runtime-tools/paths.d.ts +2 -1
- package/dist/src/runtime-tools/paths.js +19 -8
- package/dist/src/runtime-tools/probe.d.ts +2 -0
- package/dist/src/runtime-tools/probe.js +13 -1
- package/dist/src/runtime-tools/prune.js +21 -7
- package/dist/src/session/client-history.js +12 -0
- package/dist/src/session/manager.d.ts +1 -0
- package/dist/src/session/session-context-for-llm.js +8 -0
- package/dist/src/session/store.d.ts +1 -0
- package/dist/src/session/store.js +1 -0
- package/dist/src/session/stored-rows-to-file-entries.js +5 -1
- package/dist/src/session/voice-transcript.d.ts +9 -0
- package/dist/src/session/voice-transcript.js +45 -0
- package/dist/src/storage/sqlite/config-repository.js +9 -0
- package/dist/src/storage/sqlite/remote-context-repository.d.ts +3 -0
- package/dist/src/storage/sqlite/remote-context-repository.js +40 -0
- package/dist/src/storage/sqlite/session-repository.js +13 -0
- package/dist/src/storage/sqlite/user-context-repository.d.ts +3 -1
- package/dist/src/storage/sqlite/user-context-repository.js +50 -4
- package/dist/src/tui/tui.js +2 -1
- package/dist/src/user-context/automatic-context.d.ts +17 -0
- package/dist/src/user-context/automatic-context.js +85 -0
- package/dist/src/user-context/changes.d.ts +7 -0
- package/dist/src/user-context/changes.js +28 -0
- package/dist/src/user-context/planner.js +1 -24
- package/dist/src/user-context/retriever.js +10 -11
- package/dist/src/user-context/selection-policy.d.ts +20 -0
- package/dist/src/user-context/selection-policy.js +33 -0
- package/dist/src/user-context/sources/index.js +2 -2
- package/dist/src/user-context/sources/repository.d.ts +2 -1
- package/dist/src/user-context/sources/repository.js +26 -5
- package/dist/src/voice/dashscope/streaming-stt-session.d.ts +2 -0
- package/dist/src/voice/dashscope/streaming-stt-session.js +249 -0
- package/dist/src/voice/dashscope/streaming-tts-stream.d.ts +12 -0
- package/dist/src/voice/dashscope/streaming-tts-stream.js +153 -0
- package/dist/src/voice/realtime/agentEngine.d.ts +10 -0
- package/dist/src/voice/realtime/agentEngine.js +403 -0
- package/dist/src/voice/realtime/audio-playback-window.d.ts +9 -0
- package/dist/src/voice/realtime/audio-playback-window.js +49 -0
- package/dist/src/voice/realtime/conversation-context.d.ts +11 -0
- package/dist/src/voice/realtime/conversation-context.js +76 -0
- package/dist/src/voice/realtime/engine.d.ts +13 -0
- package/dist/src/voice/realtime/engine.js +1 -0
- package/dist/src/voice/realtime/memory-context.d.ts +17 -0
- package/dist/src/voice/realtime/memory-context.js +100 -0
- package/dist/src/voice/realtime/omniEngine.d.ts +18 -0
- package/dist/src/voice/realtime/omniEngine.js +443 -0
- package/dist/src/voice/realtime/omniRoute.d.ts +10 -0
- package/dist/src/voice/realtime/omniRoute.js +31 -0
- package/dist/src/voice/realtime/runtime.d.ts +99 -0
- package/dist/src/voice/realtime/runtime.js +622 -0
- package/dist/src/voice/realtime/speakable-segmenter.d.ts +10 -0
- package/dist/src/voice/realtime/speakable-segmenter.js +87 -0
- package/dist/src/voice/stt/list-providers.js +2 -1
- package/dist/src/voice/stt/providers/alibaba-transcription.js +13 -1
- package/dist/src/voice/stt/providers/xopc-cloud-transcription.js +23 -0
- package/dist/src/voice/tts/factory.js +1 -0
- package/dist/src/voice/tts/openai-compatible-speech.js +3 -1
- package/dist/src/voice/tts/providers/alibaba-speech.d.ts +1 -5
- package/dist/src/voice/tts/providers/alibaba-speech.js +25 -8
- package/dist/src/voice/tts/providers/edge-speech.js +21 -22
- package/dist/src/voice/tts/providers/minimax-speech.js +6 -3
- package/dist/src/voice/tts/providers/xopc-cloud-speech.js +26 -1
- package/dist/src/voice/tts/speak-core.d.ts +6 -2
- package/dist/src/voice/tts/speak-core.js +46 -34
- package/dist/src/voice/tts/speech-provider-types.d.ts +1 -0
- package/package.json +6 -6
- package/dist/gateway/static/root/assets/Combination-14nmIgxH.js +0 -41
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-DbdgX6AA.js +0 -1
- package/dist/gateway/static/root/assets/agents-BX3uRVm4.js +0 -1
- package/dist/gateway/static/root/assets/app-chrome-BeB4KgOf.js +0 -1
- package/dist/gateway/static/root/assets/app-management-settings-panel-D8evZyFQ.js +0 -1
- package/dist/gateway/static/root/assets/apps-page-CoZd_cZu.js +0 -1
- package/dist/gateway/static/root/assets/automations-workspace-C8RRKT-a.js +0 -6
- package/dist/gateway/static/root/assets/block-editor-DCH9b2Ht.js +0 -184
- package/dist/gateway/static/root/assets/browser-settings-page-D1GPwIfx.js +0 -8
- package/dist/gateway/static/root/assets/capabilities-settings-panel-CrXB1Q-l.js +0 -7
- package/dist/gateway/static/root/assets/channels-settings-DWMSYaNZ.js +0 -1
- package/dist/gateway/static/root/assets/connectors-page-DXTl1fNB.js +0 -2
- package/dist/gateway/static/root/assets/dist-BPoKqzeH.js +0 -1
- package/dist/gateway/static/root/assets/dist-DqPZM2MH.js +0 -1
- package/dist/gateway/static/root/assets/home-page-NdWtBda5.js +0 -1
- package/dist/gateway/static/root/assets/index-7i2P3312.js +0 -70
- package/dist/gateway/static/root/assets/index-DTo3IcbW.css +0 -2
- package/dist/gateway/static/root/assets/local-app-workbench-page-C_j_od5f.js +0 -2
- package/dist/gateway/static/root/assets/logs-page-D6PDZEVP.js +0 -2
- package/dist/gateway/static/root/assets/messages-_tYhRAHD.js +0 -3
- package/dist/gateway/static/root/assets/mobile-pairing-wizard-DcEz7qcX.js +0 -1
- package/dist/gateway/static/root/assets/notes-workbench-Bu0a83Oj.js +0 -4
- package/dist/gateway/static/root/assets/popover-select-DrILF6mR.js +0 -1
- package/dist/gateway/static/root/assets/product-open-page-DMhf1rdz.js +0 -1
- package/dist/gateway/static/root/assets/project-detail-page-DDHi0Y4Q.js +0 -1
- package/dist/gateway/static/root/assets/projects-page-CVMFKpmu.js +0 -1
- package/dist/gateway/static/root/assets/remote-access-hub-B74aErBY.js +0 -1
- package/dist/gateway/static/root/assets/runtime-tools-settings-panel-Bv-akndY.js +0 -5
- package/dist/gateway/static/root/assets/sessions-page-D2wmGNZE.js +0 -1
- package/dist/gateway/static/root/assets/settings-advanced-gate-CYCCUMEt.js +0 -1
- package/dist/gateway/static/root/assets/shares-settings-rf8f1OGq.js +0 -1
- package/dist/gateway/static/root/assets/skill-reload-api-BkvclZ_o.js +0 -1
- package/dist/gateway/static/root/assets/skills-page.utils-DGjnz-NG.js +0 -1
- package/dist/gateway/static/root/assets/task-detail-page-6a2RdDl2.js +0 -1
- package/dist/gateway/static/root/assets/tunnel-api-Chfiba_x.js +0 -1
- package/dist/gateway/static/root/assets/user-context-page-SeOHP6XO.js +0 -1
- package/dist/gateway/static/root/assets/voice-config-api-BO6A8V3Z.js +0 -1
- package/dist/gateway/static/root/assets/work-discovery-overlay-BfwEoOGL.js +0 -1
- package/dist/gateway/static/root/assets/workflow-run-setup-panel-X7Z5_lRX.js +0 -3
- package/dist/gateway/static/root/assets/workflows-page-BklMzkg8.js +0 -5
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { createLogger } from "../../utils/logger/index.js";
|
|
2
|
+
import { init_logger } from "../../utils/logger.js";
|
|
3
|
+
import { onUserContextChange } from "../../user-context/changes.js";
|
|
4
|
+
import { getUnderstanding, getUserProfile } from "../../storage/sqlite/user-context-repository.js";
|
|
5
|
+
import { parseAgentSessionKey } from "../../routing/session-key-utils.js";
|
|
6
|
+
import { getSessionMetadata } from "../../storage/sqlite/session-repository.js";
|
|
7
|
+
import { getSessionConfig } from "../../storage/sqlite/config-repository.js";
|
|
8
|
+
import { init_session_key, parseSessionKey } from "../../routing/session-key.js";
|
|
9
|
+
import { getUserFocus } from "../../user-context/sources/repository.js";
|
|
10
|
+
import { allowsAutomaticDisclosure, focusRejectionReason, rejectionReason } from "../../user-context/selection-policy.js";
|
|
11
|
+
import { stripRuntimeContextFromUserMessage } from "../../session/user-message-display.js";
|
|
12
|
+
import { remoteContextEligibleIds } from "../../storage/sqlite/remote-context-repository.js";
|
|
13
|
+
import { selectAutomaticContext } from "../../user-context/automatic-context.js";
|
|
14
|
+
//#region src/voice/realtime/memory-context.ts
|
|
15
|
+
init_session_key();
|
|
16
|
+
init_logger();
|
|
17
|
+
const log = createLogger("Voice:Memory");
|
|
18
|
+
const policyVersion = (config) => JSON.stringify({
|
|
19
|
+
userContext: config.userContext,
|
|
20
|
+
agents: config.agents
|
|
21
|
+
});
|
|
22
|
+
function voiceMemoryEnabled(config, sessionKey) {
|
|
23
|
+
const parsed = parseSessionKey(sessionKey);
|
|
24
|
+
const rest = parseAgentSessionKey(sessionKey)?.rest;
|
|
25
|
+
const parts = rest?.split(":") ?? [];
|
|
26
|
+
const explicitlyPrivate = rest === "main" || parts.every(Boolean) && !["cron", "subagent"].includes(parts[0] ?? "") && (parts[0] === "direct" && parts.length >= 2 || parts[1] === "direct" && parts.length >= 3 || parts[2] === "direct" && parts.length >= 4);
|
|
27
|
+
const user = config.userContext;
|
|
28
|
+
if (!parsed || !explicitlyPrivate || parsed.peerKind !== "direct" || !user.enabled || !user.understanding.enabled || user.memory.mode === "off" || !user.memory.sources.includes("understanding")) return false;
|
|
29
|
+
const mode = getSessionConfig(sessionKey)?.userContextMode;
|
|
30
|
+
return mode === void 0 || mode === "enabled";
|
|
31
|
+
}
|
|
32
|
+
function voiceMemoryQuery(history) {
|
|
33
|
+
return history.filter((message) => message.role === "user").slice(-2).map((message) => {
|
|
34
|
+
return stripRuntimeContextFromUserMessage(typeof message.content === "string" ? message.content : message.content.filter((part) => part.type === "text").map((part) => part.type === "text" ? part.text : "").join("\n")).slice(-300);
|
|
35
|
+
}).join("\n").slice(-600);
|
|
36
|
+
}
|
|
37
|
+
function buildVoiceMemoryContext(input) {
|
|
38
|
+
const started = performance.now();
|
|
39
|
+
try {
|
|
40
|
+
if (!voiceMemoryEnabled(input.getConfig(), input.sessionKey) || input.maxChars < 128) return;
|
|
41
|
+
const policy = policyVersion(input.getConfig());
|
|
42
|
+
const workspace = getSessionConfig(input.sessionKey)?.workingDirectoryOverride;
|
|
43
|
+
const sessionId = getSessionMetadata(input.sessionKey)?.sessionId;
|
|
44
|
+
const selection = selectAutomaticContext({
|
|
45
|
+
...input,
|
|
46
|
+
query: voiceMemoryQuery(input.history),
|
|
47
|
+
deadline: started + 150
|
|
48
|
+
});
|
|
49
|
+
if (!selection.block) return;
|
|
50
|
+
let invalid = false;
|
|
51
|
+
const isCurrent = () => {
|
|
52
|
+
try {
|
|
53
|
+
if (invalid || !voiceMemoryEnabled(input.getConfig(), input.sessionKey) || policy !== policyVersion(input.getConfig()) || workspace !== getSessionConfig(input.sessionKey)?.workingDirectoryOverride) return false;
|
|
54
|
+
const session = getSessionMetadata(input.sessionKey);
|
|
55
|
+
if (session?.sessionId !== sessionId || session?.projectId !== input.projectId) return false;
|
|
56
|
+
const understandingIds = selection.references.filter((r) => r.kind === "understanding").map((r) => r.id);
|
|
57
|
+
const focusIds = selection.references.filter((r) => r.kind === "focus").map((r) => r.id);
|
|
58
|
+
const allowed = new Set([...remoteContextEligibleIds("understanding", understandingIds), ...remoteContextEligibleIds("focus", focusIds)]);
|
|
59
|
+
return selection.references.every((ref) => {
|
|
60
|
+
if (ref.kind === "profile") return ref.version === JSON.stringify(getUserProfile());
|
|
61
|
+
if (!allowed.has(ref.id)) return false;
|
|
62
|
+
if (ref.kind === "focus") {
|
|
63
|
+
const item = getUserFocus(ref.id);
|
|
64
|
+
return !!item && item.versionId === ref.version && allowsAutomaticDisclosure(item) && !focusRejectionReason(item, input, Date.now());
|
|
65
|
+
}
|
|
66
|
+
const item = getUnderstanding(ref.id);
|
|
67
|
+
return !!item && item.status === "active" && item.versionId === ref.version && allowsAutomaticDisclosure(item) && !rejectionReason(item, input, Date.now(), []);
|
|
68
|
+
});
|
|
69
|
+
} catch {
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
log.debug({
|
|
74
|
+
sessionKey: input.sessionKey,
|
|
75
|
+
selectedCount: selection.references.length,
|
|
76
|
+
contextChars: selection.block.length,
|
|
77
|
+
durationMs: performance.now() - started
|
|
78
|
+
}, "Native voice memory selected");
|
|
79
|
+
return {
|
|
80
|
+
...selection,
|
|
81
|
+
isCurrent,
|
|
82
|
+
subscribe: (invalidate) => onUserContextChange((change) => {
|
|
83
|
+
if (change.kind === "policy" || change.kind === "session" && change.id === input.sessionKey && !isCurrent() || selection.references.some((ref) => ref.kind === change.kind && ref.id === change.id)) {
|
|
84
|
+
if (!isCurrent()) {
|
|
85
|
+
invalid = true;
|
|
86
|
+
invalidate();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
})
|
|
90
|
+
};
|
|
91
|
+
} catch {
|
|
92
|
+
log.warn({
|
|
93
|
+
sessionKey: input.sessionKey,
|
|
94
|
+
durationMs: performance.now() - started
|
|
95
|
+
}, "Native voice memory unavailable; continuing with chat history");
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
//#endregion
|
|
100
|
+
export { buildVoiceMemoryContext, voiceMemoryEnabled, voiceMemoryQuery };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { VoiceEngine, VoiceEventSink } from './engine.js';
|
|
2
|
+
import type { OmniRoute } from './omniRoute.js';
|
|
3
|
+
export interface OmniTranscript {
|
|
4
|
+
itemId: string;
|
|
5
|
+
role: 'user' | 'assistant';
|
|
6
|
+
text: string;
|
|
7
|
+
interrupted: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare function createOmniVoiceEngine(options: {
|
|
10
|
+
callId: string;
|
|
11
|
+
route: OmniRoute;
|
|
12
|
+
silenceDurationMs: number;
|
|
13
|
+
bargeIn: boolean;
|
|
14
|
+
send: VoiceEventSink;
|
|
15
|
+
sendAudio: (responseId: string, bytes: Uint8Array) => void;
|
|
16
|
+
record: (entry: OmniTranscript) => Promise<void>;
|
|
17
|
+
onClose: (reason: string, notify: boolean) => Promise<void>;
|
|
18
|
+
}): VoiceEngine;
|
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
import { createLogger } from "../../utils/logger/index.js";
|
|
2
|
+
import { init_logger } from "../../utils/logger.js";
|
|
3
|
+
import { AudioPlaybackWindow } from "./audio-playback-window.js";
|
|
4
|
+
import crypto from "node:crypto";
|
|
5
|
+
import WebSocket from "ws";
|
|
6
|
+
//#region src/voice/realtime/omniEngine.ts
|
|
7
|
+
init_logger();
|
|
8
|
+
const log = createLogger("Voice:Omni");
|
|
9
|
+
const MAX_QUEUED_AUDIO_BYTES = 24e3 * 2 * 60;
|
|
10
|
+
const MAX_PENDING_INPUT_BYTES = 64 * 1024;
|
|
11
|
+
const UPLOAD_TIMEOUT_MS = 1e4;
|
|
12
|
+
const FAILURE_MESSAGES = {
|
|
13
|
+
OMNI_UPLOAD_TIMEOUT: "The connection to the voice service stopped uploading audio. Check the network to your configured endpoint and reconnect.",
|
|
14
|
+
OMNI_INPUT_RESET_TIMEOUT: "The voice service did not acknowledge clearing microphone input. Reconnect to continue.",
|
|
15
|
+
OMNI_START_TIMEOUT: "The voice service did not become ready in time.",
|
|
16
|
+
OMNI_CONNECTION_FAILED: "Could not connect to the voice service. Check network and service availability.",
|
|
17
|
+
OMNI_CONNECTION_REJECTED: "The voice service rejected the connection. Check account access, quota and route configuration.",
|
|
18
|
+
OMNI_CONNECTION_CLOSED: "The voice service closed the connection.",
|
|
19
|
+
OMNI_PROVIDER_ERROR: "The voice service reported an error.",
|
|
20
|
+
OMNI_RESPONSE_FAILED: "The voice service could not complete its reply.",
|
|
21
|
+
OMNI_INVALID_TRANSCRIPT: "The voice service returned an invalid transcript.",
|
|
22
|
+
OMNI_INVALID_AUDIO: "The voice service returned invalid audio.",
|
|
23
|
+
OMNI_TRANSCRIPT_LIMIT: "The voice reply exceeded the transcript limit.",
|
|
24
|
+
OMNI_PROTOCOL_ERROR: "The voice service returned an unexpected message.",
|
|
25
|
+
OMNI_ITEM_LIMIT: "This call reached its conversation limit.",
|
|
26
|
+
TRANSCRIPT_WRITE_FAILED: "The conversation could not be saved. Check gateway storage."
|
|
27
|
+
};
|
|
28
|
+
function createOmniVoiceEngine(options) {
|
|
29
|
+
let socket;
|
|
30
|
+
let closed = false;
|
|
31
|
+
let ready = false;
|
|
32
|
+
let muted = false;
|
|
33
|
+
let inputBlocked = false;
|
|
34
|
+
let clearingInput = false;
|
|
35
|
+
let clearTimer;
|
|
36
|
+
const discardedInputs = /* @__PURE__ */ new Set();
|
|
37
|
+
const pendingInputs = /* @__PURE__ */ new Set();
|
|
38
|
+
let failed = false;
|
|
39
|
+
let platformRequestId;
|
|
40
|
+
let active;
|
|
41
|
+
let rejectStart;
|
|
42
|
+
let timer;
|
|
43
|
+
const recorded = /* @__PURE__ */ new Set();
|
|
44
|
+
let writes = Promise.resolve();
|
|
45
|
+
let cancellationPending = false;
|
|
46
|
+
let inputQueue = [];
|
|
47
|
+
let queuedInputBytes = 0;
|
|
48
|
+
let uploadingBytes = 0;
|
|
49
|
+
let uploadTimer;
|
|
50
|
+
const send = (type, fields = {}) => {
|
|
51
|
+
if (closed || socket?.readyState !== WebSocket.OPEN) throw new Error("Omni connection is closed");
|
|
52
|
+
socket.send(JSON.stringify({
|
|
53
|
+
type,
|
|
54
|
+
event_id: crypto.randomUUID(),
|
|
55
|
+
...fields
|
|
56
|
+
}));
|
|
57
|
+
};
|
|
58
|
+
function pumpInput() {
|
|
59
|
+
if (closed || failed || uploadingBytes || muted || clearingInput) return;
|
|
60
|
+
const bytes = inputQueue.shift();
|
|
61
|
+
if (!bytes) return;
|
|
62
|
+
queuedInputBytes -= bytes.length;
|
|
63
|
+
uploadingBytes = bytes.length;
|
|
64
|
+
uploadTimer = setTimeout(() => fail("OMNI_UPLOAD_TIMEOUT"), UPLOAD_TIMEOUT_MS);
|
|
65
|
+
try {
|
|
66
|
+
if (socket?.readyState !== WebSocket.OPEN) throw new Error("Omni connection is closed");
|
|
67
|
+
socket.send(JSON.stringify({
|
|
68
|
+
type: "input_audio_buffer.append",
|
|
69
|
+
event_id: crypto.randomUUID(),
|
|
70
|
+
audio: bytes.toString("base64")
|
|
71
|
+
}), (error) => {
|
|
72
|
+
clearTimeout(uploadTimer);
|
|
73
|
+
uploadingBytes = 0;
|
|
74
|
+
if (closed || failed) return;
|
|
75
|
+
if (error) {
|
|
76
|
+
fail("OMNI_CONNECTION_FAILED");
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
pumpInput();
|
|
80
|
+
});
|
|
81
|
+
} catch {
|
|
82
|
+
clearTimeout(uploadTimer);
|
|
83
|
+
fail("OMNI_CONNECTION_FAILED");
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const save = (entry) => {
|
|
87
|
+
if (!entry.text.trim() || recorded.has(entry.itemId)) return;
|
|
88
|
+
if (recorded.size >= 1e4) {
|
|
89
|
+
fail("OMNI_ITEM_LIMIT");
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
recorded.add(entry.itemId);
|
|
93
|
+
writes = writes.then(() => options.record(entry)).catch((err) => {
|
|
94
|
+
log.warn({
|
|
95
|
+
err,
|
|
96
|
+
sessionId: options.callId
|
|
97
|
+
}, "Native voice transcript write failed");
|
|
98
|
+
fail("TRANSCRIPT_WRITE_FAILED");
|
|
99
|
+
});
|
|
100
|
+
};
|
|
101
|
+
const cancel = (reason) => {
|
|
102
|
+
const response = active;
|
|
103
|
+
if (!response) return false;
|
|
104
|
+
active = void 0;
|
|
105
|
+
response.abort.abort(reason);
|
|
106
|
+
save({
|
|
107
|
+
itemId: response.id,
|
|
108
|
+
role: "assistant",
|
|
109
|
+
text: response.text,
|
|
110
|
+
interrupted: true
|
|
111
|
+
});
|
|
112
|
+
options.send("response.cancelled", {
|
|
113
|
+
responseId: response.id,
|
|
114
|
+
reason
|
|
115
|
+
});
|
|
116
|
+
if (reason === "client_cancelled" && response.generating && socket?.readyState === WebSocket.OPEN) {
|
|
117
|
+
cancellationPending = true;
|
|
118
|
+
send("response.cancel");
|
|
119
|
+
}
|
|
120
|
+
return true;
|
|
121
|
+
};
|
|
122
|
+
const finish = async (response) => {
|
|
123
|
+
await response.tail;
|
|
124
|
+
if (active !== response || closed) return;
|
|
125
|
+
await response.playback.drain(response.abort.signal);
|
|
126
|
+
if (active !== response || closed) return;
|
|
127
|
+
save({
|
|
128
|
+
itemId: response.id,
|
|
129
|
+
role: "assistant",
|
|
130
|
+
text: response.text,
|
|
131
|
+
interrupted: false
|
|
132
|
+
});
|
|
133
|
+
options.send("response.audio.done", { responseId: response.id });
|
|
134
|
+
options.send("response.done", {
|
|
135
|
+
responseId: response.id,
|
|
136
|
+
audio: response.audio,
|
|
137
|
+
finishReason: response.audio ? "completed" : "text_only"
|
|
138
|
+
});
|
|
139
|
+
active = void 0;
|
|
140
|
+
};
|
|
141
|
+
const stopReply = (response, code) => {
|
|
142
|
+
if (closed || active !== response) return;
|
|
143
|
+
log.warn({
|
|
144
|
+
sessionId: options.callId,
|
|
145
|
+
platformRequestId,
|
|
146
|
+
responseId: response.id,
|
|
147
|
+
code,
|
|
148
|
+
queuedBytes: response.queuedBytes
|
|
149
|
+
}, "Native voice reply stopped; call remains connected");
|
|
150
|
+
cancel("client_cancelled");
|
|
151
|
+
options.send("session.error", {
|
|
152
|
+
code: "RESPONSE_FAILED",
|
|
153
|
+
message: "Voice playback could not keep up. This reply was stopped; you can continue speaking.",
|
|
154
|
+
recoverable: true
|
|
155
|
+
});
|
|
156
|
+
};
|
|
157
|
+
function fail(code, details = {}) {
|
|
158
|
+
if (closed || failed) return;
|
|
159
|
+
failed = true;
|
|
160
|
+
log.warn({
|
|
161
|
+
sessionId: options.callId,
|
|
162
|
+
platformRequestId,
|
|
163
|
+
responseId: active?.id,
|
|
164
|
+
code,
|
|
165
|
+
provider: options.route.route.provider,
|
|
166
|
+
upstreamHost: new URL(options.route.url).hostname,
|
|
167
|
+
queuedInputBytes,
|
|
168
|
+
uploadingBytes,
|
|
169
|
+
bufferedAmount: socket?.bufferedAmount,
|
|
170
|
+
...details
|
|
171
|
+
}, `Native voice conversation failed: ${code}`);
|
|
172
|
+
rejectStart?.(new Error(code));
|
|
173
|
+
const reference = platformRequestId ?? options.callId;
|
|
174
|
+
options.send("session.error", {
|
|
175
|
+
code,
|
|
176
|
+
message: `${FAILURE_MESSAGES[code] ?? "Natural conversation failed."} (${code}; reference: ${reference})`,
|
|
177
|
+
recoverable: false
|
|
178
|
+
});
|
|
179
|
+
options.onClose("omni_error", true);
|
|
180
|
+
}
|
|
181
|
+
function discardInput() {
|
|
182
|
+
inputQueue = [];
|
|
183
|
+
queuedInputBytes = 0;
|
|
184
|
+
inputBlocked = true;
|
|
185
|
+
if (clearingInput) return;
|
|
186
|
+
clearingInput = true;
|
|
187
|
+
clearTimer = setTimeout(() => fail("OMNI_INPUT_RESET_TIMEOUT"), uploadingBytes ? UPLOAD_TIMEOUT_MS + 5e3 : 5e3);
|
|
188
|
+
for (const id of pendingInputs) discardedInputs.add(id);
|
|
189
|
+
pendingInputs.clear();
|
|
190
|
+
if (discardedInputs.size > 1e4) {
|
|
191
|
+
fail("OMNI_ITEM_LIMIT");
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
send("input_audio_buffer.clear");
|
|
195
|
+
}
|
|
196
|
+
const engine = {
|
|
197
|
+
start() {
|
|
198
|
+
return new Promise((resolve, reject) => {
|
|
199
|
+
rejectStart = reject;
|
|
200
|
+
socket = new WebSocket(options.route.url, {
|
|
201
|
+
headers: { Authorization: `Bearer ${options.route.apiKey}` },
|
|
202
|
+
maxPayload: 1024 * 1024,
|
|
203
|
+
perMessageDeflate: false,
|
|
204
|
+
handshakeTimeout: 15e3
|
|
205
|
+
});
|
|
206
|
+
socket.on("upgrade", (response) => {
|
|
207
|
+
const id = response.headers["x-xopc-request-id"];
|
|
208
|
+
if (typeof id === "string" && /^[a-zA-Z0-9-]{1,160}$/.test(id)) platformRequestId = id;
|
|
209
|
+
log.info({
|
|
210
|
+
sessionId: options.callId,
|
|
211
|
+
platformRequestId,
|
|
212
|
+
model: options.route.route.model,
|
|
213
|
+
provider: options.route.route.provider
|
|
214
|
+
}, "Native voice upstream connected");
|
|
215
|
+
});
|
|
216
|
+
timer = setTimeout(() => {
|
|
217
|
+
fail("OMNI_START_TIMEOUT");
|
|
218
|
+
engine.close();
|
|
219
|
+
}, 15e3);
|
|
220
|
+
socket.on("error", () => fail("OMNI_CONNECTION_FAILED"));
|
|
221
|
+
socket.on("close", (closeCode) => {
|
|
222
|
+
if (!closed) fail("OMNI_CONNECTION_CLOSED", { closeCode });
|
|
223
|
+
});
|
|
224
|
+
socket.on("unexpected-response", (_request, response) => {
|
|
225
|
+
response.resume();
|
|
226
|
+
fail("OMNI_CONNECTION_REJECTED", { upstreamStatus: response.statusCode });
|
|
227
|
+
});
|
|
228
|
+
socket.on("message", (raw, binary) => {
|
|
229
|
+
if (closed) return;
|
|
230
|
+
try {
|
|
231
|
+
if (binary) throw new Error("Unexpected Omni binary frame");
|
|
232
|
+
const event = JSON.parse(raw.toString());
|
|
233
|
+
if (event.type === "error") {
|
|
234
|
+
if (cancellationPending && event.error?.type === "invalid_request_error" && event.error?.message === "Conversation has none active response") {
|
|
235
|
+
cancellationPending = false;
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
fail("OMNI_PROVIDER_ERROR");
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (event.type === "session.created") send("session.update", { session: {
|
|
242
|
+
modalities: ["text", "audio"],
|
|
243
|
+
voice: options.route.voice,
|
|
244
|
+
instructions: options.route.instructions,
|
|
245
|
+
input_audio_format: "pcm",
|
|
246
|
+
output_audio_format: "pcm",
|
|
247
|
+
input_audio_transcription: { model: "gummy-realtime-v1" },
|
|
248
|
+
turn_detection: {
|
|
249
|
+
type: "server_vad",
|
|
250
|
+
threshold: .5,
|
|
251
|
+
silence_duration_ms: options.silenceDurationMs,
|
|
252
|
+
create_response: true,
|
|
253
|
+
interrupt_response: options.bargeIn
|
|
254
|
+
}
|
|
255
|
+
} });
|
|
256
|
+
else if (event.type === "session.updated" && !ready) {
|
|
257
|
+
ready = true;
|
|
258
|
+
clearTimeout(timer);
|
|
259
|
+
rejectStart = void 0;
|
|
260
|
+
resolve();
|
|
261
|
+
} else if (event.type === "input_audio_buffer.cleared") {
|
|
262
|
+
clearingInput = false;
|
|
263
|
+
clearTimeout(clearTimer);
|
|
264
|
+
} else if (event.type === "input_audio_buffer.speech_started") {
|
|
265
|
+
if (muted || clearingInput) {
|
|
266
|
+
discardedInputs.add(String(event.item_id));
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
inputBlocked = false;
|
|
270
|
+
pendingInputs.add(String(event.item_id));
|
|
271
|
+
if (options.bargeIn) cancel("barge_in");
|
|
272
|
+
options.send("input.speech_started", { utteranceId: String(event.item_id) });
|
|
273
|
+
} else if (event.type === "input_audio_buffer.speech_stopped") {
|
|
274
|
+
if (muted || inputBlocked || discardedInputs.has(String(event.item_id))) return;
|
|
275
|
+
options.send("input.speech_stopped", { utteranceId: String(event.item_id) });
|
|
276
|
+
} else if (event.type === "conversation.item.input_audio_transcription.completed") {
|
|
277
|
+
if (typeof event.transcript !== "string" || event.transcript.length > 32e3 || typeof event.item_id !== "string") {
|
|
278
|
+
fail("OMNI_INVALID_TRANSCRIPT");
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (muted || inputBlocked || discardedInputs.has(event.item_id) || recorded.has(event.item_id)) return;
|
|
282
|
+
pendingInputs.delete(event.item_id);
|
|
283
|
+
save({
|
|
284
|
+
itemId: event.item_id,
|
|
285
|
+
role: "user",
|
|
286
|
+
text: event.transcript,
|
|
287
|
+
interrupted: false
|
|
288
|
+
});
|
|
289
|
+
options.send("input.transcript.final", {
|
|
290
|
+
utteranceId: event.item_id,
|
|
291
|
+
revision: 1,
|
|
292
|
+
text: event.transcript
|
|
293
|
+
});
|
|
294
|
+
} else if (event.type === "response.created") {
|
|
295
|
+
if (muted || inputBlocked) {
|
|
296
|
+
cancellationPending = true;
|
|
297
|
+
send("response.cancel");
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
cancellationPending = false;
|
|
301
|
+
if (active) cancel("barge_in");
|
|
302
|
+
const id = event.response?.id;
|
|
303
|
+
if (typeof id !== "string" || !id.length || id.length > 160) throw new Error("Invalid response ID");
|
|
304
|
+
active = {
|
|
305
|
+
id,
|
|
306
|
+
text: "",
|
|
307
|
+
audio: false,
|
|
308
|
+
generating: true,
|
|
309
|
+
queuedBytes: 0,
|
|
310
|
+
abort: new AbortController(),
|
|
311
|
+
playback: new AudioPlaybackWindow(),
|
|
312
|
+
tail: Promise.resolve()
|
|
313
|
+
};
|
|
314
|
+
options.send("response.created", { responseId: id });
|
|
315
|
+
} else if (active && event.response_id === active.id && event.type === "response.audio_transcript.delta") {
|
|
316
|
+
if (typeof event.delta !== "string" || active.text.length + event.delta.length > 32e3) {
|
|
317
|
+
fail("OMNI_TRANSCRIPT_LIMIT");
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
active.text += event.delta;
|
|
321
|
+
options.send("response.text.delta", {
|
|
322
|
+
responseId: active.id,
|
|
323
|
+
delta: event.delta
|
|
324
|
+
});
|
|
325
|
+
} else if (active && event.response_id === active.id && event.type === "response.audio.delta") {
|
|
326
|
+
if (typeof event.delta !== "string" || !/^[A-Za-z0-9+/]*={0,2}$/.test(event.delta)) {
|
|
327
|
+
fail("OMNI_INVALID_AUDIO");
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
const audio = Buffer.from(event.delta, "base64");
|
|
331
|
+
if (!audio.length || audio.length % 2) {
|
|
332
|
+
fail("OMNI_INVALID_AUDIO");
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
const response = active;
|
|
336
|
+
if (response.queuedBytes + audio.length > MAX_QUEUED_AUDIO_BYTES) {
|
|
337
|
+
stopReply(response, "OMNI_OUTPUT_BACKPRESSURE");
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
response.queuedBytes += audio.length;
|
|
341
|
+
response.tail = response.tail.then(async () => {
|
|
342
|
+
for (let offset = 0; offset < audio.length; offset += 24e3) {
|
|
343
|
+
if (closed || active !== response) return;
|
|
344
|
+
const chunk = audio.subarray(offset, offset + 24e3);
|
|
345
|
+
await response.playback.reserve(chunk.length, response.abort.signal);
|
|
346
|
+
if (closed || active !== response) return;
|
|
347
|
+
if (!response.audio) {
|
|
348
|
+
response.audio = true;
|
|
349
|
+
options.send("response.audio.started", {
|
|
350
|
+
responseId: response.id,
|
|
351
|
+
format: {
|
|
352
|
+
encoding: "pcm_s16le",
|
|
353
|
+
channels: 1,
|
|
354
|
+
sampleRate: 24e3
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
options.sendAudio(response.id, chunk);
|
|
359
|
+
}
|
|
360
|
+
}).catch(() => {
|
|
361
|
+
if (!response.abort.signal.aborted) stopReply(response, "OMNI_PLAYBACK_FAILED");
|
|
362
|
+
}).finally(() => {
|
|
363
|
+
response.queuedBytes -= audio.length;
|
|
364
|
+
});
|
|
365
|
+
} else if (event.type === "response.done" && active?.id === event.response?.id) {
|
|
366
|
+
active.generating = false;
|
|
367
|
+
if (event.response.status === "cancelled") cancel("barge_in");
|
|
368
|
+
else if (event.response.status !== "completed") fail("OMNI_RESPONSE_FAILED");
|
|
369
|
+
else {
|
|
370
|
+
const response = active;
|
|
371
|
+
options.send("response.text.done", { responseId: response.id });
|
|
372
|
+
finish(response).catch(() => {
|
|
373
|
+
if (!response.abort.signal.aborted) stopReply(response, "OMNI_PLAYBACK_FAILED");
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
} catch {
|
|
378
|
+
fail("OMNI_PROTOCOL_ERROR");
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
});
|
|
382
|
+
},
|
|
383
|
+
setInputMuted(next) {
|
|
384
|
+
muted = next;
|
|
385
|
+
if (next) discardInput();
|
|
386
|
+
},
|
|
387
|
+
appendAudio(bytes) {
|
|
388
|
+
if (closed || failed || muted || clearingInput) return;
|
|
389
|
+
if (!ready || !bytes.length || bytes.length % 2 || bytes.length > 64 * 1024) throw new Error("Invalid Omni input");
|
|
390
|
+
if (queuedInputBytes + uploadingBytes + bytes.length > MAX_PENDING_INPUT_BYTES) {
|
|
391
|
+
log.warn({
|
|
392
|
+
sessionId: options.callId,
|
|
393
|
+
provider: options.route.route.provider,
|
|
394
|
+
upstreamHost: new URL(options.route.url).hostname,
|
|
395
|
+
queuedInputBytes,
|
|
396
|
+
uploadingBytes,
|
|
397
|
+
bufferedAmount: socket?.bufferedAmount
|
|
398
|
+
}, "Native voice input discarded after upload congestion; call remains connected");
|
|
399
|
+
discardInput();
|
|
400
|
+
options.send("session.error", {
|
|
401
|
+
code: "INPUT_DROPPED",
|
|
402
|
+
message: "The voice connection is slow. This microphone input was discarded; please repeat it once the connection recovers.",
|
|
403
|
+
recoverable: true
|
|
404
|
+
});
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
for (let offset = 0; offset < bytes.length; offset += 24e3) {
|
|
408
|
+
const chunk = Buffer.from(bytes.subarray(offset, offset + 24e3));
|
|
409
|
+
inputQueue.push(chunk);
|
|
410
|
+
queuedInputBytes += chunk.length;
|
|
411
|
+
}
|
|
412
|
+
pumpInput();
|
|
413
|
+
},
|
|
414
|
+
async commit() {
|
|
415
|
+
throw new Error("Natural conversation uses automatic turn detection");
|
|
416
|
+
},
|
|
417
|
+
cancel(responseId, reason) {
|
|
418
|
+
if (active?.id !== responseId) return false;
|
|
419
|
+
if (reason === "client_cancelled") discardInput();
|
|
420
|
+
return cancel(reason);
|
|
421
|
+
},
|
|
422
|
+
acknowledge(responseId, playedBytes) {
|
|
423
|
+
if (active?.id === responseId) active.playback.acknowledge(playedBytes);
|
|
424
|
+
},
|
|
425
|
+
close() {
|
|
426
|
+
if (closed) return writes;
|
|
427
|
+
closed = true;
|
|
428
|
+
cancel("session_closed");
|
|
429
|
+
clearTimeout(timer);
|
|
430
|
+
clearTimeout(clearTimer);
|
|
431
|
+
clearTimeout(uploadTimer);
|
|
432
|
+
inputQueue = [];
|
|
433
|
+
queuedInputBytes = 0;
|
|
434
|
+
rejectStart?.(/* @__PURE__ */ new Error("Omni connection closed"));
|
|
435
|
+
rejectStart = void 0;
|
|
436
|
+
socket?.terminate();
|
|
437
|
+
return writes;
|
|
438
|
+
}
|
|
439
|
+
};
|
|
440
|
+
return engine;
|
|
441
|
+
}
|
|
442
|
+
//#endregion
|
|
443
|
+
export { createOmniVoiceEngine };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { VoiceProviderRoute } from '@xopcai/realtime-protocol/voice';
|
|
2
|
+
import type { Config } from '../../config/schema.js';
|
|
3
|
+
export interface OmniRoute {
|
|
4
|
+
route: VoiceProviderRoute;
|
|
5
|
+
url: string;
|
|
6
|
+
apiKey: string;
|
|
7
|
+
voice: string;
|
|
8
|
+
instructions: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function resolveOmniRoute(config: Config): Promise<OmniRoute>;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { init_xopc_cloud_config, resolveXopcModelRouterUrl } from "../../providers/xopc-cloud-config.js";
|
|
2
|
+
import { getProviderAuthService, init_provider_auth_service } from "../../providers/provider-auth-service.js";
|
|
3
|
+
//#region src/voice/realtime/omniRoute.ts
|
|
4
|
+
init_provider_auth_service();
|
|
5
|
+
init_xopc_cloud_config();
|
|
6
|
+
async function resolveOmniRoute(config) {
|
|
7
|
+
const slice = config.voice?.realtime?.omni;
|
|
8
|
+
if (!slice) throw new Error("Natural conversation is not configured");
|
|
9
|
+
if (slice.model !== "qwen3-omni-flash-realtime") throw new Error("This Omni model has not been certified");
|
|
10
|
+
const managed = slice.provider === "xopc-cloud";
|
|
11
|
+
const url = new URL(managed ? `${resolveXopcModelRouterUrl(slice.baseUrl)}/audio/conversations/realtime` : slice.baseUrl ?? "wss://dashscope.aliyuncs.com/api-ws/v1/realtime");
|
|
12
|
+
if (managed && url.protocol === "https:") url.protocol = "wss:";
|
|
13
|
+
if (url.protocol !== "wss:" || url.username || url.password || url.hash || url.port) throw new Error("Omni requires a secure WebSocket endpoint");
|
|
14
|
+
if (!managed && (!(new Set(["dashscope.aliyuncs.com", "dashscope-intl.aliyuncs.com"]).has(url.hostname) || /^[a-zA-Z0-9-]+\.(cn-beijing|ap-southeast-1)\.maas\.aliyuncs\.com$/.test(url.hostname)) || url.pathname !== "/api-ws/v1/realtime")) throw new Error("Unsupported DashScope realtime endpoint");
|
|
15
|
+
const apiKey = managed ? await getProviderAuthService().resolveApiKey("xopc-cloud") : slice.apiKey ?? config.tools?.media?.audio?.providers?.alibaba?.apiKey ?? await getProviderAuthService().resolveApiKey("dashscope");
|
|
16
|
+
if (!apiKey) throw new Error("Omni credentials are unavailable");
|
|
17
|
+
url.search = new URLSearchParams({ model: slice.model }).toString();
|
|
18
|
+
return {
|
|
19
|
+
route: {
|
|
20
|
+
provider: slice.provider,
|
|
21
|
+
model: slice.model,
|
|
22
|
+
managed
|
|
23
|
+
},
|
|
24
|
+
url: url.toString(),
|
|
25
|
+
apiKey,
|
|
26
|
+
voice: slice.voice,
|
|
27
|
+
instructions: slice.instructions
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
31
|
+
export { resolveOmniRoute };
|