@xopcai/xopc 0.0.368 → 0.0.369
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/_locales/en/messages.json +30 -1
- package/dist/browser-ext/_locales/zh_CN/messages.json +30 -1
- package/dist/browser-ext/dist/assets/{auth-beGtbLPS.js → page-context-BeworyNB.js} +1331 -1355
- package/dist/browser-ext/dist/assets/{sidepanel-BT0ojEzG.css → sidepanel-Ccl0eSa1.css} +43 -0
- package/dist/browser-ext/dist/background.js +680 -182
- package/dist/browser-ext/dist/sidepanel.html +2 -2
- package/dist/browser-ext/dist/sidepanel.js +372 -193
- package/dist/browser-ext/manifest.json +2 -8
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{activity-store-D9ZMh5ZR.js → activity-store-BQ3PMp2q.js} +1 -1
- package/dist/gateway/static/root/assets/{agent-avatar-dicebear-Bj3rvcTm.js → agent-avatar-dicebear-Stj17S9u.js} +1 -1
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-CCRfvuov.js +1 -0
- package/dist/gateway/static/root/assets/{agent-plugin-dialog-BjdgKCnk.js → agent-plugin-dialog-CRlzd8El.js} +1 -1
- package/dist/gateway/static/root/assets/agents-_ug2LCQb.js +66 -0
- package/dist/gateway/static/root/assets/{agents-admin-api-B5QDepbW.js → agents-admin-api-HBZvVyzF.js} +1 -1
- package/dist/gateway/static/root/assets/{app-management-settings-panel-2rTEtB_X.js → app-management-settings-panel-pJFe6M6c.js} +1 -1
- package/dist/gateway/static/root/assets/{appearance-settings-BPVBS80k.js → appearance-settings-BrxSpBWa.js} +1 -1
- package/dist/gateway/static/root/assets/{apps-page-1FuKlZYP.js → apps-page-DegSl5Lh.js} +1 -1
- package/dist/gateway/static/root/assets/{archive-plugin-BbT1O7oK.js → archive-plugin-BO3I9zRP.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-page-DjwP_h8h.js → automations-page-BZQ_VHjp.js} +1 -1
- package/dist/gateway/static/root/assets/automations-workspace-BQywN6nG.js +6 -0
- package/dist/gateway/static/root/assets/{binary-plugins-gS7Wg68v.js → binary-plugins-gFD7vH9R.js} +1 -1
- package/dist/gateway/static/root/assets/{block-editor-DbwMaivP.js → block-editor-3XjFv3kK.js} +3 -3
- package/dist/gateway/static/root/assets/browser-automation-inputs-DxRoKFun.js +1 -0
- package/dist/gateway/static/root/assets/browser-automations-page-yiEuG1v7.js +3 -0
- package/dist/gateway/static/root/assets/browser-settings-page-ch2_bQF0.js +3 -0
- package/dist/gateway/static/root/assets/{capabilities-Dd1PZRQw.js → capabilities-B48r0DDa.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-Lswa0hv8.js → capabilities-DO-vfNKI.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-page-djBsC_tG.js → capabilities-page-BEnFt_dj.js} +2 -2
- package/dist/gateway/static/root/assets/{capabilities-settings-panel-DTHR1DrJ.js → capabilities-settings-panel-NckMsJ5N.js} +4 -4
- package/dist/gateway/static/root/assets/{capability-header-actions-494dnJsj.js → capability-header-actions-CY8KCO9j.js} +1 -1
- package/dist/gateway/static/root/assets/channels-CHOS2CGa.js +1 -0
- package/dist/gateway/static/root/assets/{chat-terminal-dock-Bl7-Zk-x.js → chat-terminal-dock-Bwknnb4c.js} +2 -2
- package/dist/gateway/static/root/assets/{computer-settings-page-ByExKr_6.js → computer-settings-page-B2xeFCSt.js} +1 -1
- package/dist/gateway/static/root/assets/{connector-service-page-BK_5aFxO.js → connector-service-page-C83DnAqK.js} +1 -1
- package/dist/gateway/static/root/assets/connectors-page-D6wTmWA6.js +2 -0
- package/dist/gateway/static/root/assets/{date-picker-BXJv6472.js → date-picker-Bq9GR-DU.js} +1 -1
- package/dist/gateway/static/root/assets/{dependency-picker-CWbbhKZa.js → dependency-picker-BTERkA9X.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-DWNEC3mk.js → desktop-pet-BwsbQw8V.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-settings-BYwlxGiN.js → desktop-pet-settings-sL9i44pP.js} +1 -1
- package/dist/gateway/static/root/assets/device-pairing-wizard-DI2bqaKO.js +1 -0
- package/dist/gateway/static/root/assets/{directory-picker-path-field-D26tSY8Z.js → directory-picker-path-field-BFlDmsS5.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-debug-page-xYIGNegB.js → extension-debug-page-7Z1Tp7-w.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-BoQ1-3AW.js → extension-page-uGqvb8QX.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-provider-Cw_JU6RR.js → extension-provider-xlSyz4Z6.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-BH5TFGPe.js → extension-settings-page-CKHgrjOs.js} +1 -1
- package/dist/gateway/static/root/assets/{fallback-models-04Wqmw14.js → fallback-models-BwQg0G1D.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-swr-CNf0ZeRw.js → gateway-config-swr-7RKqgU_U.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-realtime-C4KtwMY3.js → gateway-realtime-DDWymi-o.js} +1 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-BheKaO6d.js +1 -0
- package/dist/gateway/static/root/assets/{gateway-settings-BqvCJVUC.js → gateway-settings-BAQnSEMJ.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-startup-retry-DIxHacUw.js → gateway-startup-retry-jiV5sbEX.js} +1 -1
- package/dist/gateway/static/root/assets/home-page-BZDjvwet.js +2 -0
- package/dist/gateway/static/root/assets/{image-generation-api-CS0q0Kto.js → image-generation-api-DO4mXb6I.js} +1 -1
- package/dist/gateway/static/root/assets/{imports-page-n_rSLo6b.js → imports-page-re1_Nh64.js} +1 -1
- package/dist/gateway/static/root/assets/index-BB1z8QxA.js +106 -0
- package/dist/gateway/static/root/assets/index-D9R6tU7r.css +1 -0
- package/dist/gateway/static/root/assets/{keyboard-shortcuts-settings-tVkrHbYF.js → keyboard-shortcuts-settings-B_xja9LN.js} +1 -1
- package/dist/gateway/static/root/assets/{local-app-workbench-page-L8tYuas0.js → local-app-workbench-page-CErdCrPX.js} +1 -1
- package/dist/gateway/static/root/assets/{local-apps-page-SSJ9B1bH.js → local-apps-page-BPL2RXdK.js} +1 -1
- package/dist/gateway/static/root/assets/{local-session-drafts-B_McurIj.js → local-session-drafts-DRPsDCj1.js} +1 -1
- package/dist/gateway/static/root/assets/{locale-store-D6_rF-qT.js → locale-store-COVscAqC.js} +1 -1
- package/dist/gateway/static/root/assets/{logs-page-BWD63eg1.js → logs-page-DL-BM0xq.js} +2 -2
- package/dist/gateway/static/root/assets/management-settings-BOVRHhXk.js +1 -0
- package/dist/gateway/static/root/assets/{markdown-view-_xwUuKce.js → markdown-view-B_qnAcW2.js} +1 -1
- package/dist/gateway/static/root/assets/{media-plugins-C631CDu4.js → media-plugins-C0Pc-wSY.js} +1 -1
- package/dist/gateway/static/root/assets/messages-Bb2GK-Nj.js +3 -0
- package/dist/gateway/static/root/assets/models-hub-cache-DyPPksMP.js +1 -0
- package/dist/gateway/static/root/assets/note-detail-panel-DJJK2yL4.js +5 -0
- package/dist/gateway/static/root/assets/{notes-api-vIZWQ9Yt.js → notes-api-sBgBcyLW.js} +1 -1
- package/dist/gateway/static/root/assets/notes-page-oCxO7ZTv.js +8 -0
- package/dist/gateway/static/root/assets/{page-context-capture-button-BlPDM7GM.js → page-context-capture-button-tf77xIz3.js} +1 -1
- package/dist/gateway/static/root/assets/personal-page-DOk9gUK5.js +1 -0
- package/dist/gateway/static/root/assets/{product-open-page-BJjWvoER.js → product-open-page-Dc3Y-Y7J.js} +1 -1
- package/dist/gateway/static/root/assets/{product-preview-content-CEd4E7Gf.js → product-preview-content-MG_hmVA3.js} +1 -1
- package/dist/gateway/static/root/assets/{project-detail-page-DM4_VLNX.js → project-detail-page-3uAzbW4T.js} +1 -1
- package/dist/gateway/static/root/assets/{projects-page-lszV8HuS.js → projects-page-MllfNLGA.js} +1 -1
- package/dist/gateway/static/root/assets/{registry-api-uJnjqTcn.js → registry-api-Bh9UBb-f.js} +1 -1
- package/dist/gateway/static/root/assets/{remote-access-hub-O6j9CWX0.js → remote-access-hub-Bs-C70el.js} +1 -1
- package/dist/gateway/static/root/assets/{runtime-tools-settings-panel-CjnNQYW8.js → runtime-tools-settings-panel-DpJ5jNzR.js} +1 -1
- package/dist/gateway/static/root/assets/{scenes-page-B3T3dDOR.js → scenes-page-Dpxamkw6.js} +2 -2
- package/dist/gateway/static/root/assets/{schema-form-DT3c4Vd_.js → schema-form-DUeyQ1zh.js} +1 -1
- package/dist/gateway/static/root/assets/{send-session-input-CICkGwn-.js → send-session-input-Dgn4BRk_.js} +1 -1
- package/dist/gateway/static/root/assets/{session-api-BcQodoAu.js → session-api-yjpEz4pP.js} +1 -1
- package/dist/gateway/static/root/assets/{session-manager-B0tOE4qz.js → session-manager-LInoZ_Yi.js} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-BCVlNHtq.js +1 -0
- package/dist/gateway/static/root/assets/settings-advanced-gate-Bz-ue9fZ.js +1 -0
- package/dist/gateway/static/root/assets/{settings-page-D-PZjcss.js → settings-page-DouHuBXt.js} +1 -1
- package/dist/gateway/static/root/assets/{setup-status-panel-Ce3gTbT3.js → setup-status-panel-B5BoZ-1y.js} +1 -1
- package/dist/gateway/static/root/assets/{share-link-dialog-CWS8EkJS.js → share-link-dialog-DJTabLAM.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-23NJAKiq.js → share-preview-page-jw7h4t_B.js} +1 -1
- package/dist/gateway/static/root/assets/{shares-api-iKhHbvBD.js → shares-api-c0dfWz6h.js} +1 -1
- package/dist/gateway/static/root/assets/shares-settings-BlnlzGjH.js +1 -0
- package/dist/gateway/static/root/assets/{skill-api-C-MWdQwl.js → skill-api-CfyX_0zN.js} +1 -1
- package/dist/gateway/static/root/assets/{skill-reload-api-iy02aB13.js → skill-reload-api-Ge5XymHa.js} +1 -1
- package/dist/gateway/static/root/assets/skills-page-DjdTeYzm.js +2 -0
- package/dist/gateway/static/root/assets/system-settings-panel-DWf256TE.js +1 -0
- package/dist/gateway/static/root/assets/{task-detail-page-B2SIEyhZ.js → task-detail-page-DE6N1ZQe.js} +2 -2
- package/dist/gateway/static/root/assets/{task-expandable-content-D2LRR4BC.js → task-expandable-content-hYcgCEE3.js} +1 -1
- package/dist/gateway/static/root/assets/{theme-store-BJfhz-Zq.js → theme-store-BOfCKgl6.js} +1 -1
- package/dist/gateway/static/root/assets/{tracing-settings-page-aHD56iaS.js → tracing-settings-page-DDWFXcST.js} +1 -1
- package/dist/gateway/static/root/assets/{tunnel-api-BCf8CfTC.js → tunnel-api-BBm9esrY.js} +1 -1
- package/dist/gateway/static/root/assets/{usage-page-B4XUkuJd.js → usage-page-C93x89uW.js} +1 -1
- package/dist/gateway/static/root/assets/{use-autosave-BnnTw9RE.js → use-autosave-CSckcjFx.js} +1 -1
- package/dist/gateway/static/root/assets/{use-composer-attachments-Itczg0y1.js → use-composer-attachments-C7Gaaai1.js} +1 -1
- package/dist/gateway/static/root/assets/{use-file-preview-expanded-DJ5vyrvM.js → use-file-preview-expanded-Db2VYg1b.js} +2 -2
- package/dist/gateway/static/root/assets/user-model-page-B74LkVbp.js +2 -0
- package/dist/gateway/static/root/assets/{voice-api-key-field-BphJku8x.js → voice-api-key-field-GEw5x2fU.js} +1 -1
- package/dist/gateway/static/root/assets/{voice-config-api-B4u4coFj.js → voice-config-api-3yyVwXln.js} +1 -1
- package/dist/gateway/static/root/assets/{work-discovery-overlay-jgOEP1xw.js → work-discovery-overlay-Bs2MKiTv.js} +1 -1
- package/dist/gateway/static/root/assets/{workflow-run-setup-panel-N69crzRu.js → workflow-run-setup-panel-BR_U_snV.js} +2 -2
- package/dist/gateway/static/root/assets/workflows-page-Crk4SxtC.js +6 -0
- package/dist/gateway/static/root/assets/{workspace-api-Lzv6GDO8.js → workspace-api-Si2o3inw.js} +2 -2
- package/dist/gateway/static/root/index.html +27 -27
- package/dist/package.js +1 -1
- package/dist/packages/browser-control-contract/src/automation.js +213 -0
- package/dist/packages/browser-control-contract/src/index.js +43 -12
- package/dist/packages/gateway-contract/src/gateway-scopes.js +3 -1
- package/dist/packages/gateway-contract/src/notifications.js +1 -0
- package/dist/packages/realtime-protocol/src/voice.js +9 -3
- package/dist/src/agent/coding/repository-instructions.js +8 -1
- package/dist/src/agent/tool-manuals/browser.d.ts +1 -1
- package/dist/src/agent/tool-manuals/browser.js +3 -3
- package/dist/src/agent/tools/browser/tool/browser-use-tool.js +1 -1
- package/dist/src/agent/tools/browser/tool/schemas.d.ts +0 -13
- package/dist/src/agent/tools/browser/tool/schemas.js +0 -4
- package/dist/src/agent/tools/browser-automation-tool.d.ts +74 -1
- package/dist/src/agent/tools/browser-automation-tool.js +52 -6
- package/dist/src/automations/capabilities/write.d.ts +2 -0
- package/dist/src/automations/domain/types.d.ts +11 -0
- package/dist/src/automations/domain/validation.d.ts +4 -0
- package/dist/src/automations/domain/validation.js +1 -0
- package/dist/src/automations/draft/automation-draft-validator.d.ts +1 -0
- package/dist/src/automations/service/action-executor.js +11 -3
- package/dist/src/automations/service/automation-service.d.ts +1 -0
- package/dist/src/automations/service/automation-service.js +13 -1
- package/dist/src/bootstrap/application-state.js +1 -1
- package/dist/src/browser/automations/repository.d.ts +4 -0
- package/dist/src/browser/automations/repository.js +27 -6
- package/dist/src/browser/automations/runner.js +101 -9
- package/dist/src/browser/automations/runtime-service.js +4 -3
- package/dist/src/browser/automations/schema.d.ts +84 -1
- package/dist/src/browser/automations/schema.js +68 -2
- package/dist/src/browser/automations/service.d.ts +78 -1
- package/dist/src/browser/automations/service.js +36 -4
- package/dist/src/browser/automations/types.d.ts +19 -4
- package/dist/src/browser/drivers/extension-driver.d.ts +5 -5
- package/dist/src/browser/drivers/extension-driver.js +10 -10
- package/dist/src/browser/drivers/playwright-driver.js +46 -0
- package/dist/src/browser/index.d.ts +1 -2
- package/dist/src/browser/index.js +2 -3
- package/dist/src/browser/policy/browser-policy.d.ts +0 -1
- package/dist/src/browser/policy/browser-policy.js +1 -4
- package/dist/src/browser/providers/index.d.ts +1 -1
- package/dist/src/browser/providers/index.js +2 -2
- package/dist/src/browser/providers/realtime-extension.d.ts +1 -1
- package/dist/src/browser/providers/realtime-extension.js +5 -1
- package/dist/src/browser/recordings/service.d.ts +22 -0
- package/dist/src/browser/recordings/service.js +160 -0
- package/dist/src/browser/runtime/browser-runtime.d.ts +1 -1
- package/dist/src/browser/runtime/browser-runtime.js +19 -56
- package/dist/src/cli/commands/doctor/checks/database-schema.js +1 -1
- package/dist/src/computer/runtime.js +2 -2
- package/dist/src/config/schema.d.ts +12 -12
- package/dist/src/config/schema.js +12 -12
- package/dist/src/endpoint-tools/policy.js +4 -3
- package/dist/src/gateway/hono/routes/browser-automations.js +23 -7
- package/dist/src/gateway/hono/routes/browser-recordings.d.ts +3 -0
- package/dist/src/gateway/hono/routes/browser-recordings.js +78 -0
- package/dist/src/gateway/hono/routes/browser.js +1 -34
- package/dist/src/gateway/hono/routes/device-push.js +1 -0
- package/dist/src/gateway/hono/routes/home.js +1 -1
- package/dist/src/gateway/hono/routes/lazy-bundles.js +8 -0
- package/dist/src/gateway/hono/routes/personal-agent.js +35 -0
- package/dist/src/gateway/hono/routes/voice.js +5 -2
- package/dist/src/gateway/security/gateway-scopes.js +3 -1
- package/dist/src/gateway/service/agent-runner.js +1 -0
- package/dist/src/home-intelligence/generator.d.ts +2 -0
- package/dist/src/home-intelligence/generator.js +53 -4
- package/dist/src/home-intelligence/host.d.ts +2 -1
- package/dist/src/home-intelligence/host.js +23 -3
- package/dist/src/migrations/cloud-public-models.js +1 -0
- package/dist/src/notifications/clarification.d.ts +5 -0
- package/dist/src/notifications/clarification.js +66 -0
- package/dist/src/notifications/device-store.js +1 -0
- package/dist/src/notifications/harmony-push.d.ts +1 -0
- package/dist/src/notifications/harmony-push.js +3 -1
- package/dist/src/notifications/planner.d.ts +1 -0
- package/dist/src/notifications/service.d.ts +2 -2
- package/dist/src/notifications/service.js +20 -4
- package/dist/src/notifications/store.d.ts +2 -0
- package/dist/src/notifications/store.js +7 -1
- package/dist/src/notifications/types.d.ts +1 -0
- package/dist/src/notifications/types.js +1 -0
- package/dist/src/storage/sqlite/clarification-wait-repository.js +1 -0
- package/dist/src/storage/sqlite/migrations/238_browser_record_replay.sql +45 -0
- package/dist/src/storage/sqlite/migrations/239_clarification_notifications.sql +26 -0
- package/dist/src/storage/sqlite/migrations/runner.d.ts +1 -1
- package/dist/src/storage/sqlite/migrations/runner.js +3 -3
- package/dist/src/storage/sqlite/schema.js +1 -1
- package/dist/src/tasks/home-query-service.d.ts +1 -1
- package/dist/src/tasks/home-query-service.js +6 -2
- package/dist/src/voice/dashscope/streaming-tts-stream.d.ts +2 -1
- package/dist/src/voice/dashscope/streaming-tts-stream.js +135 -79
- package/dist/src/voice/realtime/agentEngine.js +78 -1
- package/dist/src/voice/realtime/speakable-segmenter.d.ts +2 -0
- package/dist/src/voice/realtime/speakable-segmenter.js +30 -0
- package/dist/src/voice/tts/providers/alibaba-speech.js +17 -1
- package/dist/src/voice/tts/speak-core.d.ts +3 -0
- package/dist/src/voice/tts/speak-core.js +21 -1
- package/dist/src/voice/tts/speech-provider-types.d.ts +8 -0
- package/package.json +3 -3
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-D0tD3kCe.js +0 -1
- package/dist/gateway/static/root/assets/agents-CWXKzX5L.js +0 -66
- package/dist/gateway/static/root/assets/automations-workspace-C9ZT6TAU.js +0 -6
- package/dist/gateway/static/root/assets/browser-automation-inputs-CXG_JveC.js +0 -1
- package/dist/gateway/static/root/assets/browser-automations-page-CJc7OKR3.js +0 -3
- package/dist/gateway/static/root/assets/browser-settings-page-BjaTEUR8.js +0 -3
- package/dist/gateway/static/root/assets/channels-BaYRJtz3.js +0 -1
- package/dist/gateway/static/root/assets/connectors-page-DyqRwSH2.js +0 -2
- package/dist/gateway/static/root/assets/device-pairing-wizard-BAc4F35b.js +0 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-ChZUzVbe.js +0 -1
- package/dist/gateway/static/root/assets/home-page-DhPPLbdD.js +0 -2
- package/dist/gateway/static/root/assets/index-CK4O2xMa.css +0 -1
- package/dist/gateway/static/root/assets/index-DpLD-7Hc.js +0 -106
- package/dist/gateway/static/root/assets/management-settings-aKSibklM.js +0 -1
- package/dist/gateway/static/root/assets/messages-Bnx3GMDB.js +0 -3
- package/dist/gateway/static/root/assets/models-hub-cache-Di83uIPw.js +0 -1
- package/dist/gateway/static/root/assets/note-detail-panel-B_fve3_2.js +0 -5
- package/dist/gateway/static/root/assets/notes-page-DnChyp8S.js +0 -8
- package/dist/gateway/static/root/assets/personal-page-BM2qAgg_.js +0 -1
- package/dist/gateway/static/root/assets/sessions-page-BIixHXlQ.js +0 -1
- package/dist/gateway/static/root/assets/settings-advanced-gate-49SUEWPh.js +0 -1
- package/dist/gateway/static/root/assets/shares-settings-CNEZjF66.js +0 -1
- package/dist/gateway/static/root/assets/skills-page-BndNdYsJ.js +0 -2
- package/dist/gateway/static/root/assets/system-settings-panel-EpZeMlvp.js +0 -1
- package/dist/gateway/static/root/assets/user-model-page-Bz5lkFG9.js +0 -2
- package/dist/gateway/static/root/assets/workflows-page-CPCgOfHh.js +0 -6
- package/dist/src/browser/policy/approval-store.d.ts +0 -17
- package/dist/src/browser/policy/approval-store.js +0 -62
- package/dist/src/browser/providers/extension-ws-acquire.d.ts +0 -19
- package/dist/src/browser/providers/extension-ws-acquire.js +0 -78
- package/dist/src/browser/providers/extension.d.ts +0 -61
- package/dist/src/browser/providers/extension.js +0 -378
|
@@ -47,6 +47,18 @@ function event(type, extra = {}) {
|
|
|
47
47
|
});
|
|
48
48
|
}
|
|
49
49
|
async function openDashScopeStreamingTts(request) {
|
|
50
|
+
const session = await openDashScopeTtsSession(request);
|
|
51
|
+
try {
|
|
52
|
+
return {
|
|
53
|
+
...await session.synthesize(request.text),
|
|
54
|
+
release: () => session.close()
|
|
55
|
+
};
|
|
56
|
+
} catch (error) {
|
|
57
|
+
await session.close();
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async function openDashScopeTtsSession(request) {
|
|
50
62
|
const volume = request.volume ?? DEFAULT_REALTIME_VOLUME;
|
|
51
63
|
if (!Number.isInteger(volume) || volume < 0 || volume > 100) throw new Error("DashScope realtime TTS volume must be an integer from 0 to 100");
|
|
52
64
|
request = {
|
|
@@ -79,58 +91,48 @@ async function openDashScopeStreamingTts(request) {
|
|
|
79
91
|
}
|
|
80
92
|
async function openStreamingTtsAttempt(request) {
|
|
81
93
|
if (!request.apiKey) throw new Error("DashScope TTS API key is unavailable");
|
|
82
|
-
|
|
94
|
+
request.signal.throwIfAborted();
|
|
95
|
+
const startedAt = Date.now();
|
|
83
96
|
const socket = new WebSocket(realtimeUrl(request.baseUrl, request.model), {
|
|
84
97
|
headers: { Authorization: `Bearer ${request.apiKey}` },
|
|
85
98
|
perMessageDeflate: false
|
|
86
99
|
});
|
|
87
|
-
let controller;
|
|
88
100
|
let settled = false;
|
|
89
|
-
let
|
|
101
|
+
let closed = false;
|
|
102
|
+
let failure;
|
|
103
|
+
let active;
|
|
90
104
|
let resolveReady;
|
|
91
105
|
let rejectReady;
|
|
92
|
-
const
|
|
106
|
+
const ready = new Promise((resolve, reject) => {
|
|
93
107
|
resolveReady = resolve;
|
|
94
108
|
rejectReady = reject;
|
|
95
109
|
});
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
const fail = (error, closeSocket = true) => {
|
|
110
|
+
const closeSocket = (code, reason) => {
|
|
111
|
+
request.signal.removeEventListener("abort", onAbort);
|
|
112
|
+
if (socket.readyState === WebSocket.OPEN) socket.close(code, reason);
|
|
113
|
+
else if (socket.readyState === WebSocket.CONNECTING) socket.terminate();
|
|
114
|
+
};
|
|
115
|
+
const fail = (error, code = 1011) => {
|
|
116
|
+
if (closed) return;
|
|
117
|
+
failure = error;
|
|
118
|
+
closed = true;
|
|
106
119
|
if (!settled) rejectReady(error);
|
|
107
|
-
if (
|
|
108
|
-
|
|
109
|
-
controller
|
|
110
|
-
|
|
111
|
-
if (closeSocket) {
|
|
112
|
-
if (socket.readyState === WebSocket.OPEN) socket.close(1011, "Provider stream failed");
|
|
113
|
-
else if (socket.readyState === WebSocket.CONNECTING) socket.terminate();
|
|
120
|
+
if (active) {
|
|
121
|
+
clearTimeout(active.timer);
|
|
122
|
+
if (!active.audioDone) active.controller.error(error);
|
|
123
|
+
active.reject(error);
|
|
114
124
|
}
|
|
125
|
+
closeSocket(code, code === 1e3 ? "Aborted" : "Provider stream failed");
|
|
115
126
|
};
|
|
116
|
-
const
|
|
117
|
-
if (streamClosed) return;
|
|
118
|
-
streamClosed = true;
|
|
119
|
-
controller?.close();
|
|
120
|
-
};
|
|
121
|
-
const onAbort = () => {
|
|
122
|
-
fail(request.signal.reason instanceof Error ? request.signal.reason : new DOMException("Realtime TTS aborted", "AbortError"), false);
|
|
123
|
-
if (socket.readyState === WebSocket.OPEN) socket.close(1e3, "Aborted");
|
|
124
|
-
else if (socket.readyState === WebSocket.CONNECTING) socket.terminate();
|
|
125
|
-
};
|
|
127
|
+
const onAbort = () => fail(request.signal.reason instanceof Error ? request.signal.reason : new DOMException("Realtime TTS aborted", "AbortError"), 1e3);
|
|
126
128
|
request.signal.addEventListener("abort", onAbort, { once: true });
|
|
129
|
+
if (request.signal.aborted) onAbort();
|
|
127
130
|
socket.on("unexpected-response", (_request, response) => {
|
|
128
131
|
fail(new TtsHandshakeError(response.statusCode ?? 0, parseRetryAfter(response.headers["retry-after"])));
|
|
129
132
|
response.destroy();
|
|
130
133
|
});
|
|
131
134
|
socket.on("message", (data, isBinary) => {
|
|
132
|
-
if (
|
|
133
|
-
if (isBinary) return;
|
|
135
|
+
if (closed || isBinary) return;
|
|
134
136
|
let message;
|
|
135
137
|
try {
|
|
136
138
|
message = JSON.parse(rawText(data));
|
|
@@ -138,43 +140,54 @@ async function openStreamingTtsAttempt(request) {
|
|
|
138
140
|
fail(/* @__PURE__ */ new Error("DashScope realtime TTS returned invalid JSON"));
|
|
139
141
|
return;
|
|
140
142
|
}
|
|
141
|
-
if (message.type === "session.created") {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
if (message.type === "session.updated") {
|
|
143
|
+
if (message.type === "session.created") socket.send(event("session.update", { session: {
|
|
144
|
+
voice: request.voice,
|
|
145
|
+
mode: "commit",
|
|
146
|
+
language_type: "Auto",
|
|
147
|
+
response_format: "pcm",
|
|
148
|
+
sample_rate: 24e3,
|
|
149
|
+
volume: request.volume,
|
|
150
|
+
...request.instructions ? {
|
|
151
|
+
instructions: request.instructions,
|
|
152
|
+
optimize_instructions: false
|
|
153
|
+
} : {}
|
|
154
|
+
} }));
|
|
155
|
+
else if (message.type === "session.updated" && !settled) {
|
|
157
156
|
settled = true;
|
|
157
|
+
log.info({
|
|
158
|
+
model: request.model,
|
|
159
|
+
phase: "tts_connection_ready",
|
|
160
|
+
setupMs: Date.now() - startedAt
|
|
161
|
+
}, "Realtime TTS connection ready");
|
|
158
162
|
resolveReady();
|
|
159
|
-
|
|
160
|
-
socket.send(event("input_text_buffer.commit"));
|
|
161
|
-
return;
|
|
162
|
-
}
|
|
163
|
-
if (message.type === "response.audio.delta") {
|
|
164
|
-
if (streamClosed) return;
|
|
163
|
+
} else if (message.type === "response.audio.delta" && active && !active.audioDone) {
|
|
165
164
|
if (typeof message.delta !== "string") {
|
|
166
165
|
fail(/* @__PURE__ */ new Error("DashScope realtime TTS returned invalid audio"));
|
|
167
166
|
return;
|
|
168
167
|
}
|
|
169
|
-
|
|
170
|
-
return;
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
168
|
+
const bytes = Buffer.from(message.delta, "base64");
|
|
169
|
+
if (!bytes.byteLength) return;
|
|
170
|
+
if (!active.firstAudio) {
|
|
171
|
+
active.firstAudio = true;
|
|
172
|
+
log.info({
|
|
173
|
+
model: request.model,
|
|
174
|
+
phase: "tts_first_audio",
|
|
175
|
+
synthesisMs: Date.now() - active.submittedAt
|
|
176
|
+
}, "Realtime TTS first audio received");
|
|
177
|
+
}
|
|
178
|
+
active.controller.enqueue(bytes);
|
|
179
|
+
} else if (message.type === "response.audio.done" && active && !active.audioDone) {
|
|
180
|
+
active.audioDone = true;
|
|
181
|
+
active.controller.close();
|
|
182
|
+
} else if (message.type === "response.done" && active) {
|
|
183
|
+
if (!active.audioDone) {
|
|
184
|
+
active.audioDone = true;
|
|
185
|
+
active.controller.close();
|
|
186
|
+
}
|
|
187
|
+
clearTimeout(active.timer);
|
|
188
|
+
active.resolve();
|
|
189
|
+
active = void 0;
|
|
190
|
+
} else if (message.type === "error") {
|
|
178
191
|
const code = typeof message.error?.code === "string" ? message.error.code : void 0;
|
|
179
192
|
const detail = typeof message.error?.message === "string" ? message.error.message : void 0;
|
|
180
193
|
fail(new Error([code, detail].filter(Boolean).join(": ") || "DashScope realtime TTS failed"));
|
|
@@ -183,28 +196,71 @@ async function openStreamingTtsAttempt(request) {
|
|
|
183
196
|
socket.on("error", (error) => fail(error));
|
|
184
197
|
socket.on("close", () => {
|
|
185
198
|
request.signal.removeEventListener("abort", onAbort);
|
|
186
|
-
if (!
|
|
199
|
+
if (!closed) fail(/* @__PURE__ */ new Error("DashScope realtime TTS connection closed unexpectedly"));
|
|
187
200
|
});
|
|
188
|
-
const timeout = setTimeout(() =>
|
|
189
|
-
fail(/* @__PURE__ */ new Error("DashScope realtime TTS setup timed out"));
|
|
190
|
-
socket.close(1011, "Setup timeout");
|
|
191
|
-
}, request.timeoutMs);
|
|
201
|
+
const timeout = setTimeout(() => fail(/* @__PURE__ */ new Error("DashScope realtime TTS setup timed out")), request.timeoutMs);
|
|
192
202
|
try {
|
|
193
|
-
await
|
|
203
|
+
await ready;
|
|
194
204
|
} finally {
|
|
195
205
|
clearTimeout(timeout);
|
|
196
206
|
}
|
|
197
207
|
return {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
async synthesize(text) {
|
|
209
|
+
request.signal.throwIfAborted();
|
|
210
|
+
if (closed) throw failure ?? /* @__PURE__ */ new Error("Realtime TTS session closed");
|
|
211
|
+
if (active) throw new Error("Realtime TTS synthesis already active");
|
|
212
|
+
let resolve;
|
|
213
|
+
let reject;
|
|
214
|
+
const done = new Promise((yes, no) => {
|
|
215
|
+
resolve = yes;
|
|
216
|
+
reject = no;
|
|
217
|
+
});
|
|
218
|
+
done.catch(() => {});
|
|
219
|
+
const audioStream = new ReadableStream({
|
|
220
|
+
start(controller) {
|
|
221
|
+
active = {
|
|
222
|
+
controller,
|
|
223
|
+
audioDone: false,
|
|
224
|
+
done,
|
|
225
|
+
resolve,
|
|
226
|
+
reject,
|
|
227
|
+
submittedAt: Date.now(),
|
|
228
|
+
firstAudio: false,
|
|
229
|
+
timer: setTimeout(() => fail(/* @__PURE__ */ new Error("Realtime TTS synthesis timed out")), request.timeoutMs)
|
|
230
|
+
};
|
|
231
|
+
},
|
|
232
|
+
cancel() {
|
|
233
|
+
fail(new DOMException("Realtime TTS consumer cancelled", "AbortError"), 1e3);
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
socket.send(event("input_text_buffer.append", { text }));
|
|
237
|
+
socket.send(event("input_text_buffer.commit"));
|
|
238
|
+
return {
|
|
239
|
+
audioStream,
|
|
240
|
+
outputFormat: "pcm",
|
|
241
|
+
fileExtension: "pcm",
|
|
242
|
+
voiceCompatible: false,
|
|
243
|
+
release: async () => {
|
|
244
|
+
await done;
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
},
|
|
248
|
+
async close() {
|
|
249
|
+
if (closed) return;
|
|
250
|
+
if (active && !active.audioDone) {
|
|
251
|
+
fail(new DOMException("Realtime TTS session released", "AbortError"), 1e3);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (active) {
|
|
255
|
+
clearTimeout(active.timer);
|
|
256
|
+
active.resolve();
|
|
257
|
+
active = void 0;
|
|
258
|
+
}
|
|
259
|
+
closed = true;
|
|
260
|
+
if (socket.readyState === WebSocket.OPEN) socket.send(event("session.finish"));
|
|
261
|
+
closeSocket(1e3, "Released");
|
|
206
262
|
}
|
|
207
263
|
};
|
|
208
264
|
}
|
|
209
265
|
//#endregion
|
|
210
|
-
export { openDashScopeStreamingTts };
|
|
266
|
+
export { openDashScopeStreamingTts, openDashScopeTtsSession };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createLogger } from "../../utils/logger/index.js";
|
|
2
2
|
import { init_logger } from "../../utils/logger.js";
|
|
3
|
-
import { speakStream } from "../tts/speak-core.js";
|
|
3
|
+
import { prepareSpeechStreamSession, speakStream } from "../tts/speak-core.js";
|
|
4
4
|
import { AudioPlaybackWindow } from "./audio-playback-window.js";
|
|
5
5
|
import { TurnCoordinator } from "./turnPolicy.js";
|
|
6
6
|
import { PlaybackEchoCandidates } from "./playback-echo.js";
|
|
@@ -12,6 +12,7 @@ init_logger();
|
|
|
12
12
|
const log = createLogger("Voice:Agent");
|
|
13
13
|
const AGENT_TTS_MAX_SEGMENT_CHARACTERS = 180;
|
|
14
14
|
const AGENT_TTS_MIN_SEGMENT_CHARACTERS = 24;
|
|
15
|
+
const FIRST_PHRASE_WAIT_MS = 300;
|
|
15
16
|
const PLAYBACK_ECHO_TAIL_MS = 3e3;
|
|
16
17
|
const TASK_UPDATE_QUIET_MS = 1500;
|
|
17
18
|
function createAgentVoiceEngine(options) {
|
|
@@ -101,6 +102,7 @@ function createAgentVoiceEngine(options) {
|
|
|
101
102
|
expiresAt: Date.now() + PLAYBACK_ECHO_TAIL_MS
|
|
102
103
|
};
|
|
103
104
|
activeResponse = void 0;
|
|
105
|
+
clearTimeout(response.firstPhraseTimer);
|
|
104
106
|
response.abortController.abort(reason);
|
|
105
107
|
send("response.cancelled", {
|
|
106
108
|
responseId: response.id,
|
|
@@ -124,6 +126,42 @@ function createAgentVoiceEngine(options) {
|
|
|
124
126
|
}
|
|
125
127
|
function openSpeech(response, job) {
|
|
126
128
|
if (job.opening) return job.opening;
|
|
129
|
+
if (response.firstSynthesisAtMs === void 0) {
|
|
130
|
+
response.firstSynthesisAtMs = Date.now();
|
|
131
|
+
log.info({
|
|
132
|
+
sessionId: claim.sessionId,
|
|
133
|
+
responseId: response.id,
|
|
134
|
+
conversationId: claim.request.conversationId,
|
|
135
|
+
phase: "first_synthesis_requested",
|
|
136
|
+
textToSynthesisMs: response.firstTextAtMs === void 0 ? void 0 : response.firstSynthesisAtMs - response.firstTextAtMs
|
|
137
|
+
}, "Realtime voice first synthesis requested");
|
|
138
|
+
}
|
|
139
|
+
if (response.preparedSpeech) {
|
|
140
|
+
job.opening = response.preparedSpeech.then(async (prepared) => {
|
|
141
|
+
if ("error" in prepared) return prepared;
|
|
142
|
+
response.abortController.signal.throwIfAborted();
|
|
143
|
+
if (response.firstSubmitAtMs === void 0) {
|
|
144
|
+
response.firstSubmitAtMs = Date.now();
|
|
145
|
+
log.info({
|
|
146
|
+
sessionId: claim.sessionId,
|
|
147
|
+
responseId: response.id,
|
|
148
|
+
conversationId: claim.request.conversationId,
|
|
149
|
+
phase: "first_tts_text_submitted",
|
|
150
|
+
phraseToSubmitMs: response.firstPhraseAtMs === void 0 ? void 0 : response.firstSubmitAtMs - response.firstPhraseAtMs
|
|
151
|
+
}, "Realtime voice first TTS text submitted");
|
|
152
|
+
}
|
|
153
|
+
const result = await prepared.session.synthesize(job.phrase);
|
|
154
|
+
return { result: {
|
|
155
|
+
...result,
|
|
156
|
+
provider: claim.tts.config.provider,
|
|
157
|
+
release: result.release ?? (async () => {}),
|
|
158
|
+
ttsText: job.phrase,
|
|
159
|
+
wasPreprocessed: false,
|
|
160
|
+
wasSummarized: false
|
|
161
|
+
} };
|
|
162
|
+
}).catch((error) => ({ error }));
|
|
163
|
+
return job.opening;
|
|
164
|
+
}
|
|
127
165
|
job.opening = speakStream(job.phrase, claim.tts.config, {
|
|
128
166
|
appConfig: claim.config,
|
|
129
167
|
parseDirectives: false,
|
|
@@ -133,6 +171,7 @@ function createAgentVoiceEngine(options) {
|
|
|
133
171
|
return job.opening;
|
|
134
172
|
}
|
|
135
173
|
function prefetchNextSpeech(response) {
|
|
174
|
+
if (response.preparedSpeech) return;
|
|
136
175
|
const next = response.speechQueue[0];
|
|
137
176
|
if (!next || next.opening || response.abortController.signal.aborted) return;
|
|
138
177
|
openSpeech(response, next);
|
|
@@ -165,6 +204,9 @@ function createAgentVoiceEngine(options) {
|
|
|
165
204
|
phase: "first_response_audio",
|
|
166
205
|
latencyMs: Date.now() - response.startedAt,
|
|
167
206
|
inputToAudioMs: Date.now() - response.inputReceivedAtMs,
|
|
207
|
+
submitToAudioMs: response.firstSubmitAtMs === void 0 ? void 0 : Date.now() - response.firstSubmitAtMs,
|
|
208
|
+
synthesisToAudioMs: response.firstSynthesisAtMs === void 0 ? void 0 : Date.now() - response.firstSynthesisAtMs,
|
|
209
|
+
phraseToAudioMs: response.firstPhraseAtMs === void 0 ? void 0 : Date.now() - response.firstPhraseAtMs,
|
|
168
210
|
textToAudioMs: response.firstTextAtMs === void 0 ? void 0 : Date.now() - response.firstTextAtMs
|
|
169
211
|
}, "Realtime voice first audio ready");
|
|
170
212
|
send("response.audio.started", {
|
|
@@ -233,6 +275,18 @@ function createAgentVoiceEngine(options) {
|
|
|
233
275
|
function queuePhrases(response, phrases) {
|
|
234
276
|
for (const phrase of phrases) {
|
|
235
277
|
if (response.speechError || response.abortController.signal.aborted) return;
|
|
278
|
+
if (response.firstPhraseAtMs === void 0) {
|
|
279
|
+
response.firstPhraseAtMs = Date.now();
|
|
280
|
+
clearTimeout(response.firstPhraseTimer);
|
|
281
|
+
log.info({
|
|
282
|
+
sessionId: claim.sessionId,
|
|
283
|
+
responseId: response.id,
|
|
284
|
+
conversationId: claim.request.conversationId,
|
|
285
|
+
phase: "first_speech_phrase",
|
|
286
|
+
characters: phrase.length,
|
|
287
|
+
textToPhraseMs: response.firstTextAtMs === void 0 ? void 0 : response.firstPhraseAtMs - response.firstTextAtMs
|
|
288
|
+
}, "Realtime voice first speech phrase ready");
|
|
289
|
+
}
|
|
236
290
|
if (response.queuedSpeechCharacters + phrase.length > 8e3) {
|
|
237
291
|
response.speechError = /* @__PURE__ */ new Error("Voice synthesis queue limit reached");
|
|
238
292
|
return;
|
|
@@ -275,6 +329,21 @@ function createAgentVoiceEngine(options) {
|
|
|
275
329
|
activeResponse = response;
|
|
276
330
|
send("response.created", { responseId: response.id });
|
|
277
331
|
try {
|
|
332
|
+
const opening = prepareSpeechStreamSession(claim.tts.config, {
|
|
333
|
+
appConfig: claim.config,
|
|
334
|
+
signal: response.abortController.signal,
|
|
335
|
+
allowFallback: false
|
|
336
|
+
});
|
|
337
|
+
if (opening) response.preparedSpeech = opening.then((session) => {
|
|
338
|
+
log.info({
|
|
339
|
+
sessionId: claim.sessionId,
|
|
340
|
+
responseId: response.id,
|
|
341
|
+
conversationId: claim.request.conversationId,
|
|
342
|
+
phase: "response_tts_ready",
|
|
343
|
+
setupMs: Date.now() - response.startedAt
|
|
344
|
+
}, "Realtime voice TTS prepared");
|
|
345
|
+
return { session };
|
|
346
|
+
}, (error) => ({ error }));
|
|
278
347
|
await interruptionWrites;
|
|
279
348
|
if (response.abortController.signal.aborted || closed) return taskUpdateExposed();
|
|
280
349
|
if (!claim.conversationSessionId) throw new Error("Conversation identity is unavailable");
|
|
@@ -320,6 +389,10 @@ function createAgentVoiceEngine(options) {
|
|
|
320
389
|
delta: event.payload.delta
|
|
321
390
|
});
|
|
322
391
|
queuePhrases(response, response.segmenter.push(event.payload.delta));
|
|
392
|
+
if (response.firstPhraseAtMs === void 0 && response.firstPhraseTimer === void 0) response.firstPhraseTimer = setTimeout(() => {
|
|
393
|
+
response.firstPhraseTimer = void 0;
|
|
394
|
+
if (activeResponse === response && !response.abortController.signal.aborted) queuePhrases(response, response.segmenter.flushFirstPhrase());
|
|
395
|
+
}, FIRST_PHRASE_WAIT_MS);
|
|
323
396
|
}
|
|
324
397
|
if (event.type === "tool_start") queuePhrases(response, response.segmenter.flush());
|
|
325
398
|
if ((event.type === "tool_start" || event.type === "tool_end") && typeof event.payload?.toolCallId === "string" && typeof event.payload?.toolName === "string") send("task.activity", {
|
|
@@ -410,7 +483,11 @@ function createAgentVoiceEngine(options) {
|
|
|
410
483
|
if (activeResponse === response) activeResponse = void 0;
|
|
411
484
|
return taskUpdateExposed();
|
|
412
485
|
} finally {
|
|
486
|
+
clearTimeout(response.firstPhraseTimer);
|
|
413
487
|
await waitForSpeech(response);
|
|
488
|
+
response.abortController.abort("response_finished");
|
|
489
|
+
const prepared = await response.preparedSpeech;
|
|
490
|
+
if (prepared && "session" in prepared) await prepared.session.close();
|
|
414
491
|
}
|
|
415
492
|
}
|
|
416
493
|
async function openSttSession(consumed) {
|
|
@@ -8,6 +8,8 @@ export declare class SpeakableSegmenter {
|
|
|
8
8
|
private emitted;
|
|
9
9
|
constructor(maxCharacters?: number, minCharacters?: number, firstMinCharacters?: number);
|
|
10
10
|
push(delta: string): string[];
|
|
11
|
+
/** Releases only a complete short phrase; never cuts a word or Markdown construct. */
|
|
12
|
+
flushFirstPhrase(): string[];
|
|
11
13
|
flush(): string[];
|
|
12
14
|
private toSpeech;
|
|
13
15
|
}
|
|
@@ -80,6 +80,36 @@ var SpeakableSegmenter = class {
|
|
|
80
80
|
}
|
|
81
81
|
return phrases;
|
|
82
82
|
}
|
|
83
|
+
/** Releases only a complete short phrase; never cuts a word or Markdown construct. */
|
|
84
|
+
flushFirstPhrase() {
|
|
85
|
+
if (this.emitted || this.fence || /^\s*(```|~~~|\|)/.test(this.pending)) return [];
|
|
86
|
+
let brackets = 0;
|
|
87
|
+
let parentheses = 0;
|
|
88
|
+
let inlineCode = false;
|
|
89
|
+
for (let index = 0; index < this.pending.length; index += 1) {
|
|
90
|
+
if (/^https?:\/\//.test(this.pending.slice(index))) {
|
|
91
|
+
const end = this.pending.slice(index).search(/\s/);
|
|
92
|
+
if (end < 0) return [];
|
|
93
|
+
index += end - 1;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const char = this.pending[index];
|
|
97
|
+
if (char === "`") inlineCode = !inlineCode;
|
|
98
|
+
if (char === "[") brackets += 1;
|
|
99
|
+
if (char === "]") brackets = Math.max(0, brackets - 1);
|
|
100
|
+
if (char === "(") parentheses += 1;
|
|
101
|
+
if (char === ")") parentheses = Math.max(0, parentheses - 1);
|
|
102
|
+
if (inlineCode || brackets || parentheses) continue;
|
|
103
|
+
if (index >= 11 && /[,、,\s]/u.test(char)) {
|
|
104
|
+
const phrase = this.toSpeech(this.pending.slice(0, index + 1));
|
|
105
|
+
if (!phrase) continue;
|
|
106
|
+
this.pending = this.pending.slice(index + 1);
|
|
107
|
+
this.emitted = true;
|
|
108
|
+
return [phrase];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
83
113
|
flush() {
|
|
84
114
|
const phrase = this.fence || /^\s*(```|~~~)/.test(this.pending) ? "" : this.toSpeech(this.pending);
|
|
85
115
|
this.pending = "";
|
|
@@ -4,7 +4,7 @@ import { ProviderHttpError } from "../../../media-shared/http/provider-http-erro
|
|
|
4
4
|
import { fetchWithTimeoutGuarded, postJsonRequest } from "../../../media-shared/http/provider-http.js";
|
|
5
5
|
import "../../../media-shared/http/index.js";
|
|
6
6
|
import { registerSpeechProvider } from "../speech-registry.js";
|
|
7
|
-
import { openDashScopeStreamingTts } from "../../dashscope/streaming-tts-stream.js";
|
|
7
|
+
import { openDashScopeStreamingTts, openDashScopeTtsSession } from "../../dashscope/streaming-tts-stream.js";
|
|
8
8
|
import { alibabaVoicesForModel } from "./alibaba-voices.js";
|
|
9
9
|
//#region src/voice/tts/providers/alibaba-speech.ts
|
|
10
10
|
/**
|
|
@@ -154,6 +154,22 @@ const alibabaSpeechProvider = {
|
|
|
154
154
|
voiceCompatible: false
|
|
155
155
|
};
|
|
156
156
|
},
|
|
157
|
+
openStreamSession: async (req) => {
|
|
158
|
+
const config = readProviderConfig(req.providerConfig);
|
|
159
|
+
const apiKey = resolveApiKey(config);
|
|
160
|
+
if (!apiKey) throw new Error("Alibaba TTS API key missing");
|
|
161
|
+
const overrides = req.providerOverrides ?? {};
|
|
162
|
+
const instructions = trimToUndefined(overrides.instructions);
|
|
163
|
+
return openDashScopeTtsSession({
|
|
164
|
+
apiKey,
|
|
165
|
+
baseUrl: config.baseUrl,
|
|
166
|
+
model: ALIBABA_REALTIME_TTS_MODEL,
|
|
167
|
+
voice: trimToUndefined(overrides.voice ?? overrides.voiceId) ?? config.voice,
|
|
168
|
+
...instructions ? { instructions } : {},
|
|
169
|
+
signal: req.signal,
|
|
170
|
+
timeoutMs: req.timeoutMs
|
|
171
|
+
});
|
|
172
|
+
},
|
|
157
173
|
synthesizeStream: async (req) => {
|
|
158
174
|
const config = readProviderConfig(req.providerConfig);
|
|
159
175
|
const apiKey = resolveApiKey(config);
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import type { Config } from '../../config/schema.js';
|
|
18
18
|
import { type PreprocessOptions } from './preprocess.js';
|
|
19
|
+
import type { SpeechStreamSession } from './speech-provider-types.js';
|
|
19
20
|
import type { TTSConfig, TTSModelOverrideConfig, TTSOptions, TTSProvider, TTSResultWithTracking } from './types.js';
|
|
20
21
|
export interface SpeakOptions {
|
|
21
22
|
tts?: TTSOptions;
|
|
@@ -56,3 +57,5 @@ export interface SpeakStreamResult {
|
|
|
56
57
|
* Providers without `synthesizeStream` are wrapped via wrapBufferAsStream.
|
|
57
58
|
*/
|
|
58
59
|
export declare function speakStream(text: string, config: TTSConfig, options?: SpeakStreamOptions): Promise<SpeakStreamResult>;
|
|
60
|
+
/** Prepares a single frozen route while the Agent is generating its first text. */
|
|
61
|
+
export declare function prepareSpeechStreamSession(config: TTSConfig, options: SpeakStreamOptions): Promise<SpeechStreamSession> | undefined;
|
|
@@ -292,5 +292,25 @@ function wrapBufferAsStream(result) {
|
|
|
292
292
|
voiceCompatible: result.voiceCompatible
|
|
293
293
|
};
|
|
294
294
|
}
|
|
295
|
+
/** Prepares a single frozen route while the Agent is generating its first text. */
|
|
296
|
+
function prepareSpeechStreamSession(config, options) {
|
|
297
|
+
options.signal?.throwIfAborted();
|
|
298
|
+
if (!config.enabled) return void 0;
|
|
299
|
+
const resolved = resolveSpeechProvider(config.provider, config);
|
|
300
|
+
if (!resolved?.plugin.openStreamSession) return void 0;
|
|
301
|
+
const sessionRequest = {
|
|
302
|
+
...buildSynthesisRequest(resolved, "", options.tts, void 0, options.appConfig, "voice-note", options.signal),
|
|
303
|
+
signal: options.signal ?? new AbortController().signal
|
|
304
|
+
};
|
|
305
|
+
return resolved.plugin.openStreamSession(sessionRequest).then((session) => ({
|
|
306
|
+
close: () => session.close(),
|
|
307
|
+
synthesize: (text) => session.synthesize(preprocessText(text, {
|
|
308
|
+
maxLength: config.maxTextLength || 4096,
|
|
309
|
+
stripMarkdown: true,
|
|
310
|
+
normalizeWhitespace: true,
|
|
311
|
+
...options.preprocess
|
|
312
|
+
}).text)
|
|
313
|
+
}));
|
|
314
|
+
}
|
|
295
315
|
//#endregion
|
|
296
|
-
export { speak, speakStream, speakWithProvider };
|
|
316
|
+
export { prepareSpeechStreamSession, speak, speakStream, speakWithProvider };
|
|
@@ -89,6 +89,12 @@ export interface SpeechSynthesisRequest {
|
|
|
89
89
|
signal: AbortSignal;
|
|
90
90
|
}
|
|
91
91
|
export type SpeechSynthesisStreamRequest = SpeechSynthesisRequest;
|
|
92
|
+
/** A prepared connection owned by one reply; syntheses must be consumed in order. */
|
|
93
|
+
export interface SpeechStreamSession {
|
|
94
|
+
synthesize(text: string): Promise<SpeechSynthesisStreamResult>;
|
|
95
|
+
close(): Promise<void>;
|
|
96
|
+
}
|
|
97
|
+
export type SpeechStreamSessionRequest = Omit<SpeechSynthesisStreamRequest, 'text'>;
|
|
92
98
|
/** Voice catalog entry returned by `listVoices`. */
|
|
93
99
|
export interface SpeechVoiceOption {
|
|
94
100
|
id: string;
|
|
@@ -154,6 +160,8 @@ export interface SpeechProviderPlugin {
|
|
|
154
160
|
* + `wrapBufferAsStream` (single-chunk stream). See speak-core §8.1.
|
|
155
161
|
*/
|
|
156
162
|
synthesizeStream?(req: SpeechSynthesisStreamRequest): Promise<SpeechSynthesisStreamResult>;
|
|
163
|
+
/** Optional connection preparation and reuse for live conversation replies. */
|
|
164
|
+
openStreamSession?(req: SpeechStreamSessionRequest): Promise<SpeechStreamSession>;
|
|
157
165
|
/** Voice discovery for Web UI / CLI. Optional; orchestrator returns [] when missing. */
|
|
158
166
|
listVoices?(req: SpeechListVoicesRequest): Promise<SpeechVoiceOption[]>;
|
|
159
167
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xopcai/xopc",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.369",
|
|
4
4
|
"description": "Local-first personal AI for the one-person company, with durable context, tasks, workflows, automations, and local or cloud models.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/src/index.js",
|
|
@@ -167,8 +167,8 @@
|
|
|
167
167
|
"@xopcai/endpoint-tools-protocol": "0.0.0",
|
|
168
168
|
"@xopcai/gateway-contract": "0.0.0",
|
|
169
169
|
"@xopcai/realtime-client": "0.0.0",
|
|
170
|
-
"@xopcai/
|
|
171
|
-
"@xopcai/
|
|
170
|
+
"@xopcai/realtime-protocol": "0.0.0",
|
|
171
|
+
"@xopcai/image-providers": "0.1.0"
|
|
172
172
|
},
|
|
173
173
|
"scripts": {
|
|
174
174
|
"assets:brand": "node scripts/generate-brand-assets.mjs",
|