@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/desktop-browser-cli.md +45 -4
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +665 -14
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +10 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +546 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +6 -6
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +25 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/llm-catalog-parity.test.ts +2 -2
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secure-keys.test.ts +43 -1
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +99 -41
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +31 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +6 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -8
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/browser.help.ts +4 -5
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +6 -1
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/bundled-marketplace.json +84 -34
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/config/__tests__/default-provider.test.ts +3 -1
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/profile-text-generation.test.ts +2 -2
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +3 -1
- package/src/config/schemas/mcp.ts +2 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +1 -11
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
- package/src/daemon/conversation-agent-loop.ts +219 -25
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +8 -3
- package/src/daemon/conversation.ts +143 -41
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +76 -68
- package/src/desktop/desktop-automation-lease.ts +41 -50
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +11 -122
- package/src/desktop/desktop-stream-bridge.ts +6 -8
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +52 -2
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
- package/src/mcp/__tests__/tool-caps.test.ts +28 -1
- package/src/mcp/manager.ts +3 -0
- package/src/mcp/tool-caps.ts +12 -4
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +1 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +0 -23
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/oauth-store.ts +1 -3
- package/src/oauth/provider-serializer.ts +7 -0
- package/src/oauth/seed-providers.ts +30 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +55 -6
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/steps.ts +17 -0
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +24 -5
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
- package/src/plugins/defaults/memory/context-search/format.ts +51 -7
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
- package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +1 -1
- package/src/providers/jev/client.test.ts +10 -0
- package/src/providers/jev/client.ts +20 -4
- package/src/providers/model-catalog.ts +4 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +4 -2
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +1 -1
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-reply-router.ts +28 -32
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +113 -27
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +4 -6
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +35 -38
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-providers.ts +7 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/secret-routes.ts +1 -1
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/worker-control.ts +15 -5
- package/src/security/secure-keys.ts +53 -103
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.ts +6 -4
- package/src/tools/browser/browser-execution.ts +22 -59
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +4 -1
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +11 -2
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
- package/src/tools/terminal/safe-env.ts +4 -0
- package/src/tools/terminal/shell.ts +2 -0
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -148,6 +148,23 @@ export async function streamLiveVoiceTtsAudio(
|
|
|
148
148
|
// a trailing odd byte is carried into the next chunk to keep frames aligned.
|
|
149
149
|
const pcmAligner = createPcmChunkAligner(2);
|
|
150
150
|
|
|
151
|
+
// Send the first available samples immediately, then frame PCM in 100ms
|
|
152
|
+
// blocks. Provider read sizes must not determine websocket queue pressure.
|
|
153
|
+
const frameBytes = Math.max(2, Math.floor(sampleRate / 10) * 2);
|
|
154
|
+
let pendingPcm: Buffer = Buffer.alloc(0);
|
|
155
|
+
const emitPcm = (audio: Buffer, flush = false): void => {
|
|
156
|
+
pendingPcm =
|
|
157
|
+
pendingPcm.length > 0 ? Buffer.concat([pendingPcm, audio]) : audio;
|
|
158
|
+
while (
|
|
159
|
+
pendingPcm.length >= frameBytes ||
|
|
160
|
+
(pendingPcm.length > 0 && (chunks === 0 || flush))
|
|
161
|
+
) {
|
|
162
|
+
const length = Math.min(frameBytes, pendingPcm.length);
|
|
163
|
+
emitAudioFrame(chunkContentType, pendingPcm.subarray(0, length));
|
|
164
|
+
pendingPcm = pendingPcm.subarray(length);
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
|
|
151
168
|
try {
|
|
152
169
|
const result = await synthesizeAndEmit({
|
|
153
170
|
provider,
|
|
@@ -160,16 +177,17 @@ export async function streamLiveVoiceTtsAudio(
|
|
|
160
177
|
signal: options.signal,
|
|
161
178
|
onChunk: (chunk) => {
|
|
162
179
|
if (canStreamChunks) {
|
|
163
|
-
|
|
164
|
-
chunk.contentType || chunkContentType,
|
|
165
|
-
pcmAligner.align(chunk.audio),
|
|
166
|
-
);
|
|
180
|
+
emitPcm(pcmAligner.align(chunk.audio));
|
|
167
181
|
} else {
|
|
168
182
|
bufferedAudio.push(chunk.audio);
|
|
169
183
|
}
|
|
170
184
|
},
|
|
171
185
|
});
|
|
172
186
|
|
|
187
|
+
if (canStreamChunks && !result.stopped && !options.signal?.aborted) {
|
|
188
|
+
emitPcm(Buffer.alloc(0), true);
|
|
189
|
+
}
|
|
190
|
+
|
|
173
191
|
// A dangling final byte is malformed provider output — drop it rather
|
|
174
192
|
// than emit a torn sample.
|
|
175
193
|
if (pcmAligner.carryLength() > 0) {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
|
|
3
|
+
const IDLE_TIMEOUT_MS = 1_000;
|
|
4
|
+
const PRE_ROLL_MS = 200;
|
|
5
|
+
|
|
6
|
+
/** Gates PCM16 mono room audio while preserving the provider's audio clock. */
|
|
7
|
+
export class MicrophoneIdleGate {
|
|
8
|
+
private readonly tailBytes: number;
|
|
9
|
+
private readonly preRollBytes: number;
|
|
10
|
+
private quietBytes = 0;
|
|
11
|
+
private pending: Buffer = Buffer.alloc(0);
|
|
12
|
+
|
|
13
|
+
constructor(sampleRate: number) {
|
|
14
|
+
this.tailBytes = Math.round((sampleRate * IDLE_TIMEOUT_MS) / 1_000) * 2;
|
|
15
|
+
this.preRollBytes = Math.round((sampleRate * PRE_ROLL_MS) / 1_000) * 2;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
get closed(): boolean {
|
|
19
|
+
return this.quietBytes >= this.tailBytes;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
process(chunk: Buffer, hasSpeech: boolean): Buffer {
|
|
23
|
+
if (hasSpeech) {
|
|
24
|
+
this.quietBytes = 0;
|
|
25
|
+
const pending = this.pending;
|
|
26
|
+
this.pending = Buffer.alloc(0);
|
|
27
|
+
return pending.byteLength > 0 ? Buffer.concat([pending, chunk]) : chunk;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const passBytes = Math.min(
|
|
31
|
+
chunk.byteLength,
|
|
32
|
+
this.tailBytes - this.quietBytes,
|
|
33
|
+
);
|
|
34
|
+
this.quietBytes = Math.min(
|
|
35
|
+
this.tailBytes,
|
|
36
|
+
this.quietBytes + chunk.byteLength,
|
|
37
|
+
);
|
|
38
|
+
if (passBytes === chunk.byteLength) {
|
|
39
|
+
return chunk;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Hold only the newest onset audio. Older samples become silence before
|
|
43
|
+
// submission, so reopening never replays an already-sent time interval.
|
|
44
|
+
const buffered = Buffer.concat([this.pending, chunk.subarray(passBytes)]);
|
|
45
|
+
const silenceBytes = Math.max(0, buffered.byteLength - this.preRollBytes);
|
|
46
|
+
this.pending = Buffer.from(buffered.subarray(silenceBytes));
|
|
47
|
+
const output = Buffer.alloc(passBytes + silenceBytes);
|
|
48
|
+
chunk.copy(output, 0, 0, passBytes);
|
|
49
|
+
return output;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
reset(): void {
|
|
53
|
+
this.quietBytes = 0;
|
|
54
|
+
this.pending = Buffer.alloc(0);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { VoiceEscalationProfileSource } from "../calls/voice-escalation-target.js";
|
|
1
2
|
import { type ClientOs, parseClientOs } from "../channels/types.js";
|
|
2
3
|
|
|
3
4
|
const LIVE_VOICE_CLIENT_FRAME_TYPES = [
|
|
@@ -9,6 +10,8 @@ const LIVE_VOICE_CLIENT_FRAME_TYPES = [
|
|
|
9
10
|
"update_config",
|
|
10
11
|
"attach_image",
|
|
11
12
|
"attach_frame",
|
|
13
|
+
"sight_start",
|
|
14
|
+
"sight_end",
|
|
12
15
|
"sight_frame",
|
|
13
16
|
"text",
|
|
14
17
|
] as const;
|
|
@@ -67,6 +70,20 @@ type LiveVoiceParseResult<T> =
|
|
|
67
70
|
| { ok: true; frame: T }
|
|
68
71
|
| { ok: false; error: LiveVoiceProtocolError };
|
|
69
72
|
|
|
73
|
+
type LiveVoiceParseFailure = Extract<
|
|
74
|
+
LiveVoiceParseResult<never>,
|
|
75
|
+
{ ok: false }
|
|
76
|
+
>;
|
|
77
|
+
|
|
78
|
+
function liveVoiceParseFailure(
|
|
79
|
+
code: LiveVoiceProtocolErrorCode,
|
|
80
|
+
message: string,
|
|
81
|
+
field: string,
|
|
82
|
+
frameType: string,
|
|
83
|
+
): LiveVoiceParseFailure {
|
|
84
|
+
return { ok: false, error: { code, message, field, frameType } };
|
|
85
|
+
}
|
|
86
|
+
|
|
70
87
|
export interface LiveVoiceAudioConfig {
|
|
71
88
|
readonly mimeType: "audio/pcm";
|
|
72
89
|
readonly sampleRate: number;
|
|
@@ -363,6 +380,8 @@ export interface LiveVoiceClientAttachFrameFrame {
|
|
|
363
380
|
export interface LiveVoiceClientSightFrameFrame {
|
|
364
381
|
readonly type: "sight_frame";
|
|
365
382
|
readonly attachmentId: string;
|
|
383
|
+
readonly cameraEpoch?: number;
|
|
384
|
+
readonly source?: LiveVoiceSightSource;
|
|
366
385
|
/**
|
|
367
386
|
* How long the client's half of the frame took, for the daemon's log. The
|
|
368
387
|
* daemon adds its own half and the distance from the speech onset it
|
|
@@ -372,6 +391,19 @@ export interface LiveVoiceClientSightFrameFrame {
|
|
|
372
391
|
readonly timing?: LiveVoiceSightFrameTiming;
|
|
373
392
|
}
|
|
374
393
|
|
|
394
|
+
export type LiveVoiceSightSource = "live" | "ambient";
|
|
395
|
+
|
|
396
|
+
export interface LiveVoiceClientSightStartFrame {
|
|
397
|
+
readonly type: "sight_start";
|
|
398
|
+
readonly cameraEpoch: number;
|
|
399
|
+
readonly source?: LiveVoiceSightSource;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export interface LiveVoiceClientSightEndFrame {
|
|
403
|
+
readonly type: "sight_end";
|
|
404
|
+
readonly cameraEpoch: number;
|
|
405
|
+
}
|
|
406
|
+
|
|
375
407
|
/**
|
|
376
408
|
* The client leg of one kept frame, as durations between its own marks.
|
|
377
409
|
*
|
|
@@ -451,6 +483,8 @@ export type LiveVoiceClientFrame =
|
|
|
451
483
|
| LiveVoiceClientUpdateConfigFrame
|
|
452
484
|
| LiveVoiceClientAttachImageFrame
|
|
453
485
|
| LiveVoiceClientAttachFrameFrame
|
|
486
|
+
| LiveVoiceClientSightStartFrame
|
|
487
|
+
| LiveVoiceClientSightEndFrame
|
|
454
488
|
| LiveVoiceClientSightFrameFrame
|
|
455
489
|
| LiveVoiceClientTextTurnFrame;
|
|
456
490
|
|
|
@@ -481,6 +515,8 @@ export interface LiveVoiceReadyServerFrame extends LiveVoiceServerFrameBase {
|
|
|
481
515
|
* every typed turn with an `unknown_type` error.
|
|
482
516
|
*/
|
|
483
517
|
readonly textInput?: boolean;
|
|
518
|
+
/** Whether this session accepts camera lifecycle epochs. */
|
|
519
|
+
readonly sightSessions?: boolean;
|
|
484
520
|
/**
|
|
485
521
|
* Whether the session's speech-to-text leg is live. Absent means yes, which
|
|
486
522
|
* is the only thing an older daemon can have meant: it rejects a session it
|
|
@@ -575,8 +611,9 @@ export interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
|
|
|
575
611
|
* the room) can otherwise only say "Thinking...". The label is composed here
|
|
576
612
|
* rather than by each surface for the same reason phase wording is composed
|
|
577
613
|
* once: the Live Activity has two independent drivers (this socket and an APNs
|
|
578
|
-
* push the daemon dispatches), they must carry
|
|
579
|
-
*
|
|
614
|
+
* push the daemon dispatches), they must carry compatible content. Structured
|
|
615
|
+
* kinds let in-conversation surfaces use localized copy while system-level
|
|
616
|
+
* surfaces can suppress internal detail.
|
|
580
617
|
*
|
|
581
618
|
* An empty `label` means "no current activity", which is what a turn's end
|
|
582
619
|
* sends. Emitted only on change, never per tool result.
|
|
@@ -585,6 +622,12 @@ export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
|
|
|
585
622
|
readonly type: "activity";
|
|
586
623
|
readonly turnId: string;
|
|
587
624
|
readonly label: string;
|
|
625
|
+
/** Structured reason for the activity, when a client needs custom display. */
|
|
626
|
+
readonly kind?: "escalation";
|
|
627
|
+
/** Selected inference profile for diagnostics, never default UI copy. */
|
|
628
|
+
readonly profile?: string;
|
|
629
|
+
/** Why the selected profile won for this leg. */
|
|
630
|
+
readonly profileSource?: VoiceEscalationProfileSource;
|
|
588
631
|
/**
|
|
589
632
|
* The confirmation this turn is blocked on, when the label describes a wait
|
|
590
633
|
* rather than work in flight. Absent otherwise.
|
|
@@ -896,6 +939,10 @@ export function validateLiveVoiceClientFrame(
|
|
|
896
939
|
return validateAttachImageFrame(value);
|
|
897
940
|
case "attach_frame":
|
|
898
941
|
return validateAttachFrameFrame(value);
|
|
942
|
+
case "sight_start":
|
|
943
|
+
return validateSightStartFrame(value);
|
|
944
|
+
case "sight_end":
|
|
945
|
+
return validateSightEndFrame(value);
|
|
899
946
|
case "sight_frame":
|
|
900
947
|
return validateSightFrameFrame(value);
|
|
901
948
|
case "text":
|
|
@@ -1044,10 +1091,19 @@ function validateSightFrameFrame(
|
|
|
1044
1091
|
);
|
|
1045
1092
|
}
|
|
1046
1093
|
|
|
1094
|
+
const lifecycle = validateOptionalSightLifecycle(value, "sight_frame");
|
|
1095
|
+
if (!lifecycle.ok) {
|
|
1096
|
+
return lifecycle;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1047
1099
|
if (!("timing" in value) || value.timing === undefined) {
|
|
1048
1100
|
return {
|
|
1049
1101
|
ok: true,
|
|
1050
|
-
frame: {
|
|
1102
|
+
frame: {
|
|
1103
|
+
type: "sight_frame",
|
|
1104
|
+
attachmentId: value.attachmentId,
|
|
1105
|
+
...lifecycle.fields,
|
|
1106
|
+
},
|
|
1051
1107
|
};
|
|
1052
1108
|
}
|
|
1053
1109
|
|
|
@@ -1063,10 +1119,121 @@ function validateSightFrameFrame(
|
|
|
1063
1119
|
|
|
1064
1120
|
return {
|
|
1065
1121
|
ok: true,
|
|
1066
|
-
frame: {
|
|
1122
|
+
frame: {
|
|
1123
|
+
type: "sight_frame",
|
|
1124
|
+
attachmentId: value.attachmentId,
|
|
1125
|
+
...lifecycle.fields,
|
|
1126
|
+
timing,
|
|
1127
|
+
},
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
function validateSightStartFrame(
|
|
1132
|
+
value: Record<string, unknown>,
|
|
1133
|
+
): LiveVoiceParseResult<LiveVoiceClientSightStartFrame> {
|
|
1134
|
+
const lifecycle = validateRequiredSightLifecycle(value, "sight_start");
|
|
1135
|
+
if (!lifecycle.ok) {
|
|
1136
|
+
return lifecycle;
|
|
1137
|
+
}
|
|
1138
|
+
return {
|
|
1139
|
+
ok: true,
|
|
1140
|
+
frame: { type: "sight_start", ...lifecycle.fields },
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
function validateSightEndFrame(
|
|
1145
|
+
value: Record<string, unknown>,
|
|
1146
|
+
): LiveVoiceParseResult<LiveVoiceClientSightEndFrame> {
|
|
1147
|
+
if (!isPositiveSafeInteger(value.cameraEpoch)) {
|
|
1148
|
+
return protocolError(
|
|
1149
|
+
"invalid_field",
|
|
1150
|
+
"sight_end frame field cameraEpoch must be a positive safe integer",
|
|
1151
|
+
"cameraEpoch",
|
|
1152
|
+
"sight_end",
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
return {
|
|
1156
|
+
ok: true,
|
|
1157
|
+
frame: { type: "sight_end", cameraEpoch: value.cameraEpoch },
|
|
1067
1158
|
};
|
|
1068
1159
|
}
|
|
1069
1160
|
|
|
1161
|
+
function validateRequiredSightLifecycle(
|
|
1162
|
+
value: Record<string, unknown>,
|
|
1163
|
+
frameType: "sight_start",
|
|
1164
|
+
):
|
|
1165
|
+
| { ok: true; fields: { cameraEpoch: number; source?: LiveVoiceSightSource } }
|
|
1166
|
+
| LiveVoiceParseFailure {
|
|
1167
|
+
if (!isPositiveSafeInteger(value.cameraEpoch)) {
|
|
1168
|
+
return liveVoiceParseFailure(
|
|
1169
|
+
"invalid_field",
|
|
1170
|
+
`${frameType} frame field cameraEpoch must be a positive safe integer`,
|
|
1171
|
+
"cameraEpoch",
|
|
1172
|
+
frameType,
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
if (value.source !== undefined && !isLiveVoiceSightSource(value.source)) {
|
|
1176
|
+
return liveVoiceParseFailure(
|
|
1177
|
+
"invalid_field",
|
|
1178
|
+
`${frameType} frame field source must be live or ambient`,
|
|
1179
|
+
"source",
|
|
1180
|
+
frameType,
|
|
1181
|
+
);
|
|
1182
|
+
}
|
|
1183
|
+
return {
|
|
1184
|
+
ok: true,
|
|
1185
|
+
fields: {
|
|
1186
|
+
cameraEpoch: value.cameraEpoch,
|
|
1187
|
+
...(value.source ? { source: value.source } : {}),
|
|
1188
|
+
},
|
|
1189
|
+
};
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
function validateOptionalSightLifecycle(
|
|
1193
|
+
value: Record<string, unknown>,
|
|
1194
|
+
frameType: "sight_frame",
|
|
1195
|
+
):
|
|
1196
|
+
| {
|
|
1197
|
+
ok: true;
|
|
1198
|
+
fields: { cameraEpoch?: number; source?: LiveVoiceSightSource };
|
|
1199
|
+
}
|
|
1200
|
+
| LiveVoiceParseFailure {
|
|
1201
|
+
if (value.cameraEpoch === undefined && value.source === undefined) {
|
|
1202
|
+
return { ok: true, fields: {} };
|
|
1203
|
+
}
|
|
1204
|
+
if (!isPositiveSafeInteger(value.cameraEpoch)) {
|
|
1205
|
+
return liveVoiceParseFailure(
|
|
1206
|
+
"invalid_field",
|
|
1207
|
+
`${frameType} frame field cameraEpoch must be a positive safe integer when lifecycle fields are present`,
|
|
1208
|
+
"cameraEpoch",
|
|
1209
|
+
frameType,
|
|
1210
|
+
);
|
|
1211
|
+
}
|
|
1212
|
+
if (value.source !== undefined && !isLiveVoiceSightSource(value.source)) {
|
|
1213
|
+
return liveVoiceParseFailure(
|
|
1214
|
+
"invalid_field",
|
|
1215
|
+
`${frameType} frame field source must be live or ambient`,
|
|
1216
|
+
"source",
|
|
1217
|
+
frameType,
|
|
1218
|
+
);
|
|
1219
|
+
}
|
|
1220
|
+
return {
|
|
1221
|
+
ok: true,
|
|
1222
|
+
fields: {
|
|
1223
|
+
cameraEpoch: value.cameraEpoch,
|
|
1224
|
+
...(value.source ? { source: value.source } : {}),
|
|
1225
|
+
},
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
function isPositiveSafeInteger(value: unknown): value is number {
|
|
1230
|
+
return Number.isSafeInteger(value) && typeof value === "number" && value > 0;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
function isLiveVoiceSightSource(value: unknown): value is LiveVoiceSightSource {
|
|
1234
|
+
return value === "live" || value === "ambient";
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1070
1237
|
/**
|
|
1071
1238
|
* Parse a `sight_frame`'s `timing`. Null for anything off-shape: a timing is
|
|
1072
1239
|
* for the log, so nothing about it is coerced, but a malformed one still
|
|
@@ -44,7 +44,7 @@ const CLIENT_CONTROL_LINES: Record<LiveVoiceSessionControl, string> = {
|
|
|
44
44
|
const LOOK_FRAME_CONTROL_LINES: Partial<
|
|
45
45
|
Record<LiveVoiceSessionControl, string>
|
|
46
46
|
> = {
|
|
47
|
-
look_screen: `- To look at their screen (for example "take a look at my screen", or "can you see it now?" after something on it changed), say in a few words that you are taking a look, then end your reply with ${LOOK_SCREEN_MARKER}. Use it even when their screen is already shared with you, to see it as it is now. You get a fresh view right after you finish speaking
|
|
47
|
+
look_screen: `- To look at their screen (for example "take a look at my screen", or "can you see it now?" after something on it changed), say in a few words that you are taking a look, then end your reply with ${LOOK_SCREEN_MARKER}. Use it even when their screen is already shared with you, to see it as it is now. You get a fresh view right after you finish speaking, so do not describe the screen yet or ask them to say anything more. This only obtains a view; showing them where to click still needs screen annotation after the view arrives.`,
|
|
48
48
|
look_camera: `- To look through their camera (for example "look at this" or "can you see this?" while they hold something up), say in a few words that you are taking a look, then end your reply with ${LOOK_CAMERA_MARKER}. Use it even when the camera is already on, to see what it shows now. You get a fresh view right after you finish speaking and answer from that, so do not describe it yet and do not ask them to say anything more.`,
|
|
49
49
|
};
|
|
50
50
|
|
|
@@ -61,6 +61,11 @@ export type LookSessionControl = Extract<
|
|
|
61
61
|
"look_screen" | "look_camera"
|
|
62
62
|
>;
|
|
63
63
|
|
|
64
|
+
export interface LookFollowUp {
|
|
65
|
+
action: LookSessionControl;
|
|
66
|
+
callerUtterance: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
64
69
|
export function isLookSessionControl(
|
|
65
70
|
action: string,
|
|
66
71
|
): action is LookSessionControl {
|
|
@@ -78,9 +83,12 @@ export const LOOK_FOLLOW_UP_CONTENT = "(fresh view taken; answer from it now)";
|
|
|
78
83
|
* Appended to the control prompt of the turn that answers a look: the reply
|
|
79
84
|
* that asked for it only acknowledged, and the frame it asked for has landed.
|
|
80
85
|
*/
|
|
81
|
-
export function lookFollowUpNote(
|
|
86
|
+
export function lookFollowUpNote({
|
|
87
|
+
action,
|
|
88
|
+
callerUtterance,
|
|
89
|
+
}: LookFollowUp): string {
|
|
82
90
|
const what = action === "look_screen" ? "their screen" : "their camera";
|
|
83
|
-
return `You just took a fresh look at ${what}, and the newest image in the conversation is what it shows right now.
|
|
91
|
+
return `You just took a fresh look at ${what}, and the newest image in the conversation is what it shows right now. Resume the caller's request ${JSON.stringify(callerUtterance)} using that view and the latest conversation context. Complete any requested action, not just a spoken description.${action === "look_screen" ? " Showing or pointing out where to click needs the screen-annotation tools." : ""} If the request needs tools and this leg has none, escalate. Otherwise answer in a few spoken sentences. If the image does not show what they meant, say briefly what you do see and ask. You have already said you were taking a look, so do not say it again and do not end with a look marker.`;
|
|
84
92
|
}
|
|
85
93
|
|
|
86
94
|
// Always taught: narration is the session's own, so no client has to be able
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { SubagentParentNotification } from "../subagent/parent-notification.js";
|
|
2
|
+
|
|
3
|
+
export interface VoiceTaskOutcome extends SubagentParentNotification {
|
|
4
|
+
source: "subagent" | "continuation";
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
interface Delivery {
|
|
8
|
+
outcome: VoiceTaskOutcome | null;
|
|
9
|
+
interrupted: boolean;
|
|
10
|
+
lastPlayedReply: string | null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Session-local delivery receipts. Generated text and context exposure are not receipts. */
|
|
14
|
+
export class VoiceTaskOutcomes {
|
|
15
|
+
private readonly deliveries = new Map<string, Delivery>();
|
|
16
|
+
private playback: {
|
|
17
|
+
outcomes: readonly VoiceTaskOutcome[];
|
|
18
|
+
announcement: boolean;
|
|
19
|
+
reply: string;
|
|
20
|
+
untilMs: number;
|
|
21
|
+
} | null = null;
|
|
22
|
+
|
|
23
|
+
get hasPending(): boolean {
|
|
24
|
+
return this.next() !== undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
enqueue(outcome: VoiceTaskOutcome): void {
|
|
28
|
+
const previous = this.deliveries.get(outcome.taskId);
|
|
29
|
+
this.deliveries.set(outcome.taskId, {
|
|
30
|
+
outcome,
|
|
31
|
+
interrupted: previous?.interrupted ?? false,
|
|
32
|
+
lastPlayedReply: previous?.lastPlayedReply ?? null,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
next(): VoiceTaskOutcome | undefined {
|
|
37
|
+
for (const delivery of this.deliveries.values()) {
|
|
38
|
+
if (delivery.outcome !== null) {
|
|
39
|
+
return delivery.outcome;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
pendingOutcomes(): VoiceTaskOutcome[] {
|
|
46
|
+
return [...this.deliveries.values()].flatMap(({ outcome }) =>
|
|
47
|
+
outcome === null ? [] : [outcome],
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
deliveryContext(outcome: VoiceTaskOutcome): string {
|
|
52
|
+
const delivery = this.deliveries.get(outcome.taskId);
|
|
53
|
+
const status = delivery?.interrupted
|
|
54
|
+
? "An earlier attempt to deliver this task's findings was interrupted before playback completed. Resume the useful unheard findings briefly."
|
|
55
|
+
: "This update is still pending delivery.";
|
|
56
|
+
const heard = delivery?.lastPlayedReply;
|
|
57
|
+
return `${status} Generated assistant text in conversation history is not evidence that the user heard it. ${heard ? `The latest reply with completed audio playback for this task was: ${JSON.stringify(heard)}. Use that receipt to avoid repeating findings actually heard.` : "There is no completed audio playback receipt for this task."}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
finish(outcome: VoiceTaskOutcome, playbackUntilMs: number, reply = ""): void {
|
|
61
|
+
this.playback = {
|
|
62
|
+
outcomes: [outcome],
|
|
63
|
+
announcement: true,
|
|
64
|
+
reply,
|
|
65
|
+
untilMs: playbackUntilMs,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
finishContextReply(
|
|
70
|
+
outcomes: readonly VoiceTaskOutcome[],
|
|
71
|
+
playbackUntilMs: number,
|
|
72
|
+
reply: string,
|
|
73
|
+
): void {
|
|
74
|
+
if (outcomes.length > 0) {
|
|
75
|
+
this.playback = {
|
|
76
|
+
outcomes,
|
|
77
|
+
announcement: false,
|
|
78
|
+
reply,
|
|
79
|
+
untilMs: playbackUntilMs,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
acknowledgePlayback(now: number): void {
|
|
85
|
+
const playback = this.playback;
|
|
86
|
+
if (playback === null || now < playback.untilMs) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
for (const outcome of playback.outcomes) {
|
|
90
|
+
const delivery = this.deliveries.get(outcome.taskId);
|
|
91
|
+
if (delivery === undefined) {
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (playback.reply.length > 0) {
|
|
95
|
+
delivery.lastPlayedReply = playback.reply;
|
|
96
|
+
}
|
|
97
|
+
if (playback.announcement) {
|
|
98
|
+
delivery.interrupted = false;
|
|
99
|
+
}
|
|
100
|
+
if (playback.announcement && delivery.outcome === outcome) {
|
|
101
|
+
delivery.outcome = null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
this.playback = null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
interrupt(outcome: VoiceTaskOutcome): void {
|
|
108
|
+
const delivery = this.deliveries.get(outcome.taskId);
|
|
109
|
+
if (delivery?.outcome != null) {
|
|
110
|
+
delivery.interrupted = true;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
interruptPlayback(now: number): void {
|
|
115
|
+
this.acknowledgePlayback(now);
|
|
116
|
+
if (this.playback?.announcement) {
|
|
117
|
+
for (const outcome of this.playback.outcomes) {
|
|
118
|
+
this.interrupt(outcome);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
this.playback = null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
drain(now: number): VoiceTaskOutcome[] {
|
|
125
|
+
this.acknowledgePlayback(now);
|
|
126
|
+
const outcomes = this.pendingOutcomes();
|
|
127
|
+
this.deliveries.clear();
|
|
128
|
+
this.playback = null;
|
|
129
|
+
return outcomes;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
export interface VoiceInputSample {
|
|
2
|
+
receivedAtMs: number;
|
|
3
|
+
chunkMs: number;
|
|
4
|
+
meanAmplitude: number;
|
|
5
|
+
baseThreshold: number;
|
|
6
|
+
roomNoiseFloor: number | null;
|
|
7
|
+
echoEnergy: number;
|
|
8
|
+
echoOnsetLapsed: boolean;
|
|
9
|
+
echoWindowMs: number;
|
|
10
|
+
echoGuardCarryover: boolean;
|
|
11
|
+
echoReferenceMs: number;
|
|
12
|
+
echoProbeMs: number;
|
|
13
|
+
echoCorrelation: number | null;
|
|
14
|
+
playbackRemainingMs: number;
|
|
15
|
+
playbackEchoPossible: boolean;
|
|
16
|
+
speechMs: number;
|
|
17
|
+
silenceMs: number;
|
|
18
|
+
echoMs: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const SUMMARY_INTERVAL_MS = 5_000;
|
|
22
|
+
const TRACE_WINDOW_MS = 2_000;
|
|
23
|
+
const MAX_TRACE_SAMPLES = 40;
|
|
24
|
+
|
|
25
|
+
/** Bounded signal measurements only. Never retains PCM or transcript text. */
|
|
26
|
+
export class VoiceInputDiagnostics {
|
|
27
|
+
private recent: VoiceInputSample[] = [];
|
|
28
|
+
private lastArrivalAtMs: number | null = null;
|
|
29
|
+
private lastSttSubmissionAtMs: number | null = null;
|
|
30
|
+
private window = this.emptyWindow();
|
|
31
|
+
|
|
32
|
+
recordSttSubmission(atMs: number, audioMs: number): void {
|
|
33
|
+
this.window.sttSubmittedChunks += 1;
|
|
34
|
+
this.window.sttSubmittedAudioMs += audioMs;
|
|
35
|
+
if (this.lastSttSubmissionAtMs !== null) {
|
|
36
|
+
this.window.sttMaxSubmissionGapMs = Math.max(
|
|
37
|
+
this.window.sttMaxSubmissionGapMs,
|
|
38
|
+
atMs - this.lastSttSubmissionAtMs,
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
this.lastSttSubmissionAtMs = atMs;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
observe(sample: VoiceInputSample): Record<string, number> | null {
|
|
45
|
+
this.recent.push(sample);
|
|
46
|
+
while (
|
|
47
|
+
this.recent.length > MAX_TRACE_SAMPLES ||
|
|
48
|
+
(this.recent[0] &&
|
|
49
|
+
sample.receivedAtMs - this.recent[0].receivedAtMs > TRACE_WINDOW_MS)
|
|
50
|
+
) {
|
|
51
|
+
this.recent.shift();
|
|
52
|
+
}
|
|
53
|
+
const window = this.window;
|
|
54
|
+
window.firstAtMs ??= sample.receivedAtMs;
|
|
55
|
+
window.lastAtMs = sample.receivedAtMs;
|
|
56
|
+
window.chunks += 1;
|
|
57
|
+
window.audioMs += sample.chunkMs;
|
|
58
|
+
window.weightedAmplitude += sample.meanAmplitude * sample.chunkMs;
|
|
59
|
+
window.minAmplitude = Math.min(window.minAmplitude, sample.meanAmplitude);
|
|
60
|
+
window.maxAmplitude = Math.max(window.maxAmplitude, sample.meanAmplitude);
|
|
61
|
+
window.zeroAudioMs += sample.meanAmplitude === 0 ? sample.chunkMs : 0;
|
|
62
|
+
window.speechMs += sample.speechMs;
|
|
63
|
+
window.silenceMs += sample.silenceMs;
|
|
64
|
+
window.echoMs += sample.echoMs;
|
|
65
|
+
if (this.lastArrivalAtMs !== null) {
|
|
66
|
+
window.maxArrivalGapMs = Math.max(
|
|
67
|
+
window.maxArrivalGapMs,
|
|
68
|
+
sample.receivedAtMs - this.lastArrivalAtMs,
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
this.lastArrivalAtMs = sample.receivedAtMs;
|
|
72
|
+
return sample.receivedAtMs - window.firstAtMs >= SUMMARY_INTERVAL_MS
|
|
73
|
+
? this.flush()
|
|
74
|
+
: null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
snapshot(nowMs: number): VoiceInputSample[] {
|
|
78
|
+
return this.recent
|
|
79
|
+
.filter((entry) => nowMs - entry.receivedAtMs <= TRACE_WINDOW_MS)
|
|
80
|
+
.map((entry) => ({ ...entry }));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
flush(): Record<string, number> | null {
|
|
84
|
+
const window = this.window;
|
|
85
|
+
if (window.chunks === 0) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
this.window = this.emptyWindow();
|
|
89
|
+
const { weightedAmplitude, firstAtMs, ...summary } = window;
|
|
90
|
+
return {
|
|
91
|
+
...summary,
|
|
92
|
+
firstAtMs: firstAtMs!,
|
|
93
|
+
meanAmplitude:
|
|
94
|
+
window.audioMs > 0 ? weightedAmplitude / window.audioMs : 0,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private emptyWindow() {
|
|
99
|
+
return {
|
|
100
|
+
firstAtMs: null as number | null,
|
|
101
|
+
lastAtMs: 0,
|
|
102
|
+
chunks: 0,
|
|
103
|
+
audioMs: 0,
|
|
104
|
+
weightedAmplitude: 0,
|
|
105
|
+
minAmplitude: Infinity,
|
|
106
|
+
maxAmplitude: 0,
|
|
107
|
+
zeroAudioMs: 0,
|
|
108
|
+
speechMs: 0,
|
|
109
|
+
silenceMs: 0,
|
|
110
|
+
echoMs: 0,
|
|
111
|
+
maxArrivalGapMs: 0,
|
|
112
|
+
sttSubmittedChunks: 0,
|
|
113
|
+
sttSubmittedAudioMs: 0,
|
|
114
|
+
sttMaxSubmissionGapMs: 0,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
}
|