@vellumai/assistant 0.11.7 → 0.11.8-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +10 -10
- package/Dockerfile +1 -0
- package/docs/architecture/security.md +1 -1
- package/docs/error-handling.md +1 -2
- package/docs/guardian-request-flow.md +51 -5
- package/docs/trusted-contact-access.md +13 -12
- package/knip.json +1 -0
- package/node_modules/@vellumai/avatar-catalog/package.json +19 -0
- package/{src/avatar/character-components.ts → node_modules/@vellumai/avatar-catalog/src/catalog.ts} +12 -44
- package/node_modules/@vellumai/avatar-catalog/src/colors.ts +19 -0
- package/node_modules/@vellumai/avatar-catalog/src/index.ts +25 -0
- package/node_modules/@vellumai/avatar-catalog/src/types.ts +47 -0
- package/node_modules/@vellumai/avatar-catalog/tsconfig.json +20 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/node_modules/@vellumai/credential-storage/src/static-credentials.ts +43 -12
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/test-path-guard.test.ts +89 -0
- package/node_modules/@vellumai/environments/src/test-path-guard.ts +113 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/guardian-request-contract.test.ts +8 -18
- package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +46 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/invite-contract.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +46 -0
- package/node_modules/@vellumai/gateway-client/src/guardian-request-contract.ts +62 -21
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +14 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +53 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +11 -5
- package/node_modules/@vellumai/gateway-client/src/invite-contract.ts +1 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/openapi.yaml +481 -9
- package/package.json +3 -1
- package/scripts/sync-llm-catalog.ts +3 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +16 -0
- package/src/__tests__/agent-loop-compacted-history-transform.test.ts +304 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +3 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +3 -0
- package/src/__tests__/anthropic-provider.test.ts +56 -0
- package/src/__tests__/approval-cascade.test.ts +0 -13
- package/src/__tests__/attachments-store.test.ts +117 -6
- package/src/__tests__/attachments.test.ts +122 -0
- package/src/__tests__/bundled-asset.test.ts +22 -1
- package/src/__tests__/channel-approval.test.ts +14 -7
- package/src/__tests__/channel-approvals.test.ts +3 -3
- package/src/__tests__/channel-delivery-store.test.ts +58 -0
- package/src/__tests__/channel-inbound-disk-pressure.test.ts +1 -0
- package/src/__tests__/channel-invite-transport.test.ts +1 -0
- package/src/__tests__/channel-readiness-slack-remote.test.ts +3 -3
- package/src/__tests__/compaction-events.test.ts +0 -12
- package/src/__tests__/compactor-call-site-logging.test.ts +7 -1
- package/src/__tests__/compactor-retained-image-validation.test.ts +15 -0
- package/src/__tests__/config-schema.test.ts +35 -0
- package/src/__tests__/contacts-write.test.ts +141 -3
- package/src/__tests__/conversation-abort-tool-results.test.ts +0 -13
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +0 -12
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -13
- package/src/__tests__/conversation-agent-loop.test.ts +95 -13
- package/src/__tests__/conversation-confirmation-signals.test.ts +0 -13
- package/src/__tests__/conversation-load-history-repair.test.ts +418 -0
- package/src/__tests__/conversation-media-retry.test.ts +33 -2
- package/src/__tests__/conversation-pairing.test.ts +54 -0
- package/src/__tests__/conversation-process-callsite.test.ts +0 -12
- package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -13
- package/src/__tests__/conversation-queue.test.ts +4 -13
- package/src/__tests__/conversation-retry-route.test.ts +5 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +1 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +1 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -1
- package/src/__tests__/conversation-sight-frames.test.ts +615 -0
- package/src/__tests__/conversation-slash-queue.test.ts +0 -13
- package/src/__tests__/conversation-slash-unknown.test.ts +0 -13
- package/src/__tests__/conversation-speed-override.test.ts +0 -12
- package/src/__tests__/conversation-store-ephemeral.test.ts +0 -12
- package/src/__tests__/conversation-summarize-route.test.ts +5 -0
- package/src/__tests__/conversation-summarize-up-to.test.ts +102 -0
- package/src/__tests__/conversation-wait-for-idle.test.ts +0 -12
- package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -5
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -17
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -17
- package/src/__tests__/credential-health-service.test.ts +45 -0
- package/src/__tests__/credential-record-retire.test.ts +256 -0
- package/src/__tests__/credential-record-write-through.test.ts +8 -4
- package/src/__tests__/db-conversation-fork-lineage-migration.test.ts +0 -4
- package/src/__tests__/db-conversation-inference-profile-migration.test.ts +0 -4
- package/src/__tests__/db-llm-request-log-provider-migration.test.ts +0 -8
- package/src/__tests__/delete-propagation.test.ts +121 -3
- package/src/__tests__/evict-conversations-for-reload.test.ts +46 -6
- package/src/__tests__/file-ops-service.test.ts +8 -3
- package/src/__tests__/gemini-provider.test.ts +64 -0
- package/src/__tests__/guardian-action-service-scope.test.ts +118 -0
- package/src/__tests__/guardian-card-withdrawal.test.ts +299 -5
- package/src/__tests__/guardian-expiry-notifier.test.ts +117 -13
- package/src/__tests__/guardian-gateway-sim.test.ts +119 -0
- package/src/__tests__/guardian-gateway-sim.ts +56 -20
- package/src/__tests__/guardian-reply-router-answer-mode.test.ts +6 -0
- package/src/__tests__/guardian-request-status-sync.test.ts +162 -0
- package/src/__tests__/guardian-wait-controller.test.ts +6 -0
- package/src/__tests__/headless-browser-mode.test.ts +27 -0
- package/src/__tests__/helpers/mock-conversation.ts +2 -0
- package/src/__tests__/list-messages-system-card.test.ts +70 -0
- package/src/__tests__/llm-catalog-parity.test.ts +2 -0
- package/src/__tests__/managed-profile-guard.test.ts +2 -2
- package/src/__tests__/media-resolve-image-validation.test.ts +106 -0
- package/src/__tests__/media-stream-server-integration.test.ts +6 -0
- package/src/__tests__/non-member-access-request.test.ts +12 -0
- package/src/__tests__/notification-discord-adapter.test.ts +76 -4
- package/src/__tests__/oauth-commands-routes.test.ts +21 -0
- package/src/__tests__/openai-provider.test.ts +63 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +114 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +63 -1
- package/src/__tests__/persist-unsendable-image.test.ts +19 -0
- package/src/__tests__/plugin-app-serve-routes.test.ts +2 -2
- package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
- package/src/__tests__/process-message-background-slack.test.ts +1 -1
- package/src/__tests__/process-message-display-content.test.ts +1 -1
- package/src/__tests__/processing-flag-persist-failure.test.ts +0 -5
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +0 -17
- package/src/__tests__/qdrant-manager.test.ts +6 -0
- package/src/__tests__/reaction-persistence.test.ts +58 -62
- package/src/__tests__/retry-thinking-adaptive-only.test.ts +67 -0
- package/src/__tests__/server-history-render.test.ts +22 -0
- package/src/__tests__/sight-frame-retention.test.ts +362 -0
- package/src/__tests__/slack-reaction-approvals.test.ts +44 -24
- package/src/__tests__/slack-reaction-guardian-approval.test.ts +29 -1
- package/src/__tests__/subagent-fork-notifications.test.ts +2 -0
- package/src/__tests__/subagent-manager-notify.test.ts +2 -0
- package/src/__tests__/subagent-notify-parent.test.ts +2 -0
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +85 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +4 -4
- package/src/__tests__/terminal-tools.test.ts +9 -9
- package/src/__tests__/tool-result-spool.test.ts +5 -8
- package/src/__tests__/usage-attribution.test.ts +26 -0
- package/src/acp/__tests__/acp-auth-marker-store.test.ts +191 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +262 -0
- package/src/acp/__tests__/acp-connect-card-state.test.ts +17 -0
- package/src/acp/__tests__/helpers/acp-history-db.ts +10 -3
- package/src/acp/acp-auth-marker-store.ts +196 -0
- package/src/acp/acp-claude-oauth.ts +154 -4
- package/src/acp/acp-connect-card-state.ts +49 -8
- package/src/acp/agent-process.ts +1 -0
- package/src/acp/auto-install.ts +1 -0
- package/src/acp/client-handler.ts +1 -0
- package/src/acp/prepare-agent-env.ts +79 -1
- package/src/acp/resolve-agent.ts +12 -0
- package/src/acp/session-manager.test.ts +183 -0
- package/src/acp/session-manager.ts +110 -6
- package/src/acp/types.ts +17 -0
- package/src/agent/attachments.ts +7 -2
- package/src/agent/loop.ts +59 -1
- package/src/api/events/contact-form-closed.ts +32 -0
- package/src/api/events/contact-record-request.ts +63 -0
- package/src/api/events/contact-request.ts +10 -3
- package/src/api/index.ts +20 -0
- package/src/api/responses/conversation-message.ts +20 -0
- package/src/api/responses/home.ts +101 -0
- package/src/approvals/AGENTS.md +1 -1
- package/src/approvals/guardian-card-withdrawal.ts +127 -30
- package/src/approvals/guardian-decision-primitive.ts +9 -0
- package/src/approvals/guardian-expiry-notifier.ts +22 -16
- package/src/approvals/guardian-request-status-sync.ts +98 -0
- package/src/avatar/svg-compositor.ts +1 -1
- package/src/avatar/traits-png-sync.ts +1 -1
- package/src/bundler/app-compiler.ts +1 -0
- package/src/bundler/compiler-tools.ts +1 -0
- package/src/bundler/package-resolver.ts +1 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +183 -3
- package/src/calls/guardian-action-sweep.ts +6 -1
- package/src/calls/voice-session-bridge.ts +46 -1
- package/src/channels/__tests__/gateway-guardian-requests.test.ts +13 -18
- package/src/channels/gateway-guardian-requests.ts +37 -18
- package/src/cli/__tests__/contacts-record-prompt-cli.test.ts +329 -0
- package/src/cli/commands/__tests__/browser.test.ts +65 -1
- package/src/cli/commands/browser.ts +30 -10
- package/src/cli/commands/contacts.help.ts +173 -11
- package/src/cli/commands/contacts.ts +343 -4
- package/src/cli/commands/db/refresh.ts +1 -0
- package/src/cli/commands/oauth/index.help.ts +6 -3
- package/src/cli/commands/oauth/request.test.ts +37 -0
- package/src/cli/commands/oauth/request.ts +7 -6
- package/src/cli/lib/bundled-marketplace.json +1 -1
- package/src/cli/lib/install-from-github.ts +20 -19
- package/src/cli/lib/install-plugin-dependencies.ts +3 -5
- package/src/cli/lib/merge-plugin-tree.ts +1 -0
- package/src/cli/lib/publish-plugin.ts +4 -1
- package/src/config/__tests__/balanced-model-experiment.test.ts +63 -31
- package/src/config/__tests__/default-profile-catalog.test.ts +2 -2
- package/src/config/balanced-model-experiment.ts +15 -9
- package/src/config/bundled-skills/contacts/SKILL.md +61 -18
- package/src/config/default-profile-catalog.ts +12 -11
- package/src/config/feature-flag-registry.json +22 -13
- package/src/config/schema.ts +2 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +13 -0
- package/src/config/schemas/live-voice.ts +7 -0
- package/src/config/schemas/sight.ts +45 -0
- package/src/config/skills.ts +2 -1
- package/src/contacts/__tests__/mirror-reconciler.test.ts +352 -0
- package/src/contacts/contact-store.ts +47 -4
- package/src/contacts/contacts-write.ts +5 -2
- package/src/contacts/mirror-reconciler.ts +210 -0
- package/src/contacts/mirror-tombstones.ts +42 -0
- package/src/contacts/notify-contacts-changed.ts +11 -2
- package/src/context/compactor.ts +47 -19
- package/src/context/tool-result-spool.ts +21 -14
- package/src/conversations/message-consolidation.ts +2 -2
- package/src/credential-health/credential-health-service.ts +37 -6
- package/src/daemon/conversation-agent-loop-handlers.ts +19 -2
- package/src/daemon/conversation-agent-loop.ts +100 -1
- package/src/daemon/conversation-media-retry.ts +11 -6
- package/src/daemon/conversation-messaging.ts +293 -30
- package/src/daemon/conversation-runtime-assembly.ts +67 -0
- package/src/daemon/conversation-sight-frames.ts +187 -0
- package/src/daemon/conversation-store.ts +18 -28
- package/src/daemon/conversation-surfaces.ts +12 -2
- package/src/daemon/conversation-tool-setup.ts +39 -10
- package/src/daemon/conversation.ts +159 -16
- package/src/daemon/daemon-control.ts +6 -17
- package/src/daemon/handlers/config-model.test.ts +1 -0
- package/src/daemon/handlers/conversations.ts +10 -11
- package/src/daemon/install-assistant-command.ts +1 -0
- package/src/daemon/lifecycle.ts +10 -13
- package/src/daemon/message-types/sync.ts +4 -0
- package/src/daemon/process-message.ts +2 -17
- package/src/daemon/reaction-history-render.test.ts +162 -0
- package/src/daemon/reaction-history-render.ts +115 -0
- package/src/daemon/reaction-record.test.ts +162 -0
- package/src/daemon/reaction-record.ts +83 -0
- package/src/daemon/shutdown-handlers.ts +0 -2
- package/src/daemon/summarize-boundary.ts +12 -0
- package/src/daemon/video-thumbnail.ts +1 -1
- package/src/home/feed-types.ts +4 -0
- package/src/home/feed-writer.ts +25 -0
- package/src/live-voice/__tests__/live-voice-attach-frame.test.ts +679 -0
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +8 -5
- package/src/live-voice/__tests__/live-voice-photo.test.ts +637 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-inline.test.ts +787 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +347 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +109 -0
- package/src/live-voice/live-voice-photo.ts +420 -65
- package/src/live-voice/live-voice-session.ts +311 -27
- package/src/live-voice/protocol.ts +147 -11
- package/src/messaging/provider-message-metadata.ts +7 -0
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +113 -1
- package/src/messaging/providers/channel-transport.ts +37 -0
- package/src/messaging/providers/discord/api.ts +4 -4
- package/src/messaging/providers/discord/send.test.ts +177 -2
- package/src/messaging/providers/discord/send.ts +197 -10
- package/src/messaging/providers/discord/transport.ts +10 -0
- package/src/messaging/providers/index.ts +42 -0
- package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +129 -3
- package/src/messaging/providers/slack/__tests__/auth.test.ts +35 -0
- package/src/messaging/providers/slack/__tests__/message-content.test.ts +152 -0
- package/src/messaging/providers/slack/adapter.ts +112 -27
- package/src/messaging/providers/slack/api.test.ts +10 -0
- package/src/messaging/providers/slack/auth.ts +32 -6
- package/src/messaging/providers/slack/client.ts +0 -15
- package/src/messaging/providers/slack/message-content.ts +137 -0
- package/src/messaging/providers/slack/send.test.ts +43 -2
- package/src/messaging/providers/slack/send.ts +18 -5
- package/src/messaging/providers/slack/transport.ts +10 -0
- package/src/messaging/providers/slack/types.ts +21 -2
- package/src/messaging/providers/telegram-bot/adapter.ts +5 -1
- package/src/messaging/providers/telegram-bot/send.test.ts +49 -2
- package/src/messaging/providers/telegram-bot/send.ts +41 -1
- package/src/messaging/providers/telegram-bot/transport.ts +10 -0
- package/src/messaging/providers/telegram-bot/withdraw.ts +13 -5
- package/src/messaging/reaction-envelopes.ts +79 -0
- package/src/monitoring/control.ts +2 -2
- package/src/monitoring/page-cache.ts +5 -1
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +360 -0
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +8 -0
- package/src/notifications/adapters/discord.ts +62 -9
- package/src/notifications/conversation-pairing.ts +31 -4
- package/src/notifications/emit-signal.ts +8 -5
- package/src/notifications/guardian-delivery-recorder.ts +47 -6
- package/src/notifications/guardian-feed-projection.ts +492 -0
- package/src/notifications/home-feed-side-effect.ts +59 -11
- package/src/oauth/byo-connection.test.ts +22 -0
- package/src/oauth/byo-connection.ts +11 -5
- package/src/oauth/connection.ts +3 -2
- package/src/oauth/platform-connection.test.ts +35 -0
- package/src/oauth/platform-connection.ts +18 -13
- package/src/oauth/scope-utils.ts +48 -0
- package/src/oauth/seed-providers.ts +12 -0
- package/src/outbound-proxy/certs.ts +5 -4
- package/src/permissions/question-guardian-request.test.ts +7 -1
- package/src/persistence/attachments-store.ts +134 -31
- package/src/persistence/conversation-crud.ts +137 -4
- package/src/persistence/conversation-types.ts +109 -0
- package/src/persistence/db-connection.ts +3 -21
- package/src/persistence/delivery-crud.ts +42 -12
- package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +15 -2
- package/src/persistence/embeddings/embedding-local.ts +7 -112
- package/src/persistence/embeddings/qdrant-manager.ts +4 -1
- package/src/persistence/llm-request-log-store.ts +1 -1
- package/src/persistence/migrations/370-acp-session-history-auth-error-code.ts +36 -0
- package/src/persistence/migrations/371-acp-session-history-auth-error-credential.ts +40 -0
- package/src/persistence/migrations/372-create-acp-refused-credentials.ts +34 -0
- package/src/persistence/migrations/373-acp-auth-marker-index.ts +27 -0
- package/src/persistence/migrations/__tests__/370-acp-session-history-auth-error-code.test.ts +65 -0
- package/src/persistence/migrations/__tests__/371-acp-session-history-auth-error-credential.test.ts +68 -0
- package/src/persistence/migrations/__tests__/372-create-acp-refused-credentials.test.ts +64 -0
- package/src/persistence/migrations/__tests__/373-acp-auth-marker-index.test.ts +82 -0
- package/src/persistence/schema/acp.ts +25 -0
- package/src/persistence/steps.ts +8 -0
- package/src/platform/platform-patch-queue.test.ts +43 -41
- package/src/plugin-api/index.ts +12 -0
- package/src/plugins/defaults/compaction/window-manager.ts +2 -0
- package/src/plugins/defaults/image-recovery/recover.ts +12 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +166 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +3 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +3 -0
- package/src/plugins/defaults/memory/jobs-worker.ts +1 -2
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +53 -9
- package/src/plugins/defaults/memory/src/__tests__/memory-worker-routes.test.ts +4 -7
- package/src/plugins/defaults/memory/src/memory-worker-routes.ts +1 -1
- package/src/plugins/defaults/memory/v1/job-handlers/index-maintenance.ts +15 -1
- package/src/plugins/defaults/memory/worker-control.ts +1 -2
- package/src/providers/content-block-schema.ts +1 -0
- package/src/providers/fireworks/client.ts +9 -1
- package/src/providers/gemini/client.ts +22 -15
- package/src/providers/media-resolve.ts +24 -10
- package/src/providers/model-catalog.ts +91 -0
- package/src/providers/openai/chat-completions-provider.ts +44 -5
- package/src/providers/retry.ts +16 -11
- package/src/providers/types.ts +56 -0
- package/src/routes/__tests__/control.test.ts +4 -3
- package/src/routes/control.ts +1 -1
- package/src/routes/route-host-client.ts +1 -1
- package/src/runtime/AGENTS.md +30 -12
- package/src/runtime/__tests__/agent-wake.test.ts +71 -6
- package/src/runtime/__tests__/guardian-form-registry.test.ts +242 -0
- package/src/runtime/access-request-helper.ts +15 -7
- package/src/runtime/agent-wake.ts +16 -4
- package/src/runtime/channel-approval-types.ts +8 -5
- package/src/runtime/channel-invite-transport.ts +2 -0
- package/src/runtime/channel-invite-transports/discord.ts +27 -0
- package/src/runtime/channel-readiness-service.ts +4 -4
- package/src/runtime/channel-reply-delivery.ts +72 -28
- package/src/runtime/guardian-action-service.ts +9 -5
- package/src/runtime/guardian-form-registry.ts +256 -0
- package/src/runtime/guardian-reply-router.ts +33 -14
- package/src/runtime/invite-instruction-generator.ts +14 -2
- package/src/runtime/migrations/vbundle-metadata-merge.ts +4 -7
- package/src/runtime/no-response.ts +13 -74
- package/src/runtime/routes/__tests__/acp-routes.test.ts +527 -0
- package/src/runtime/routes/__tests__/app-management-shared-apps.test.ts +74 -0
- package/src/runtime/routes/__tests__/contact-form-kinds.test.ts +22 -0
- package/src/runtime/routes/__tests__/contact-record-prompt-route.test.ts +303 -0
- package/src/runtime/routes/__tests__/monitoring-routes.test.ts +3 -4
- package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +6 -0
- package/src/runtime/routes/__tests__/route-host-worker-routes.test.ts +3 -4
- package/src/runtime/routes/__tests__/schedule-worker-routes.test.ts +5 -6
- package/src/runtime/routes/acp-routes-list.test.ts +58 -0
- package/src/runtime/routes/acp-routes.ts +253 -47
- package/src/runtime/routes/app-management-routes.ts +80 -47
- package/src/runtime/routes/approval-routes.ts +9 -0
- package/src/runtime/routes/archive-utils.ts +6 -1
- package/src/runtime/routes/avatar-routes.ts +1 -1
- package/src/runtime/routes/channel-route-shared.ts +18 -20
- package/src/runtime/routes/contact-prompt-routes.ts +271 -105
- package/src/runtime/routes/conversation-routes.ts +41 -32
- package/src/runtime/routes/credential-prompt-routes.ts +10 -1
- package/src/runtime/routes/debug-bash-routes.ts +1 -0
- package/src/runtime/routes/guardian-expiry-sweep.ts +89 -31
- package/src/runtime/routes/guardian-form-routes.ts +115 -0
- package/src/runtime/routes/inbound-message-handler.ts +82 -29
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +6 -0
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +6 -0
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +6 -1
- package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +50 -17
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +97 -94
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/integrations/slack/channel.ts +39 -18
- package/src/runtime/routes/migration-routes.ts +1 -3
- package/src/runtime/routes/monitoring-routes.ts +0 -3
- package/src/runtime/routes/oauth-commands-routes.ts +18 -5
- package/src/runtime/routes/route-host-worker-routes.ts +1 -1
- package/src/runtime/routes/schedule-worker-routes.ts +1 -1
- package/src/runtime/tool-grant-request-helper.ts +4 -0
- package/src/schedule/run-script.ts +1 -0
- package/src/schedule/scheduler.ts +1 -1
- package/src/schedule/worker-control.ts +2 -2
- package/src/security/oauth2.ts +2 -11
- package/src/security/secure-keys.ts +55 -8
- package/src/skills/catalog-install.ts +7 -4
- package/src/skills/clawhub.ts +1 -0
- package/src/skills/inline-command-runner.ts +1 -0
- package/src/stt/__tests__/room-noise-floor.test.ts +108 -0
- package/src/stt/room-noise-floor.ts +140 -0
- package/src/subagent/notify.ts +49 -3
- package/src/tools/__tests__/tool-input-schemas.test.ts +2 -0
- package/src/tools/acp/spawn.ts +2 -2
- package/src/tools/browser/__tests__/browser-status.test.ts +9 -0
- package/src/tools/browser/browser-execution.ts +10 -43
- package/src/tools/browser/browser-manager.ts +1 -0
- package/src/tools/browser/browser-status-constants.ts +6 -5
- package/src/tools/browser/runtime-check.ts +2 -0
- package/src/tools/channel/react-to-message.test.ts +193 -0
- package/src/tools/channel/react-to-message.ts +159 -0
- package/src/tools/credentials/metadata-store.ts +115 -19
- package/src/tools/credentials/store.ts +0 -2
- package/src/tools/filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +24 -0
- package/src/tools/host-terminal/host-shell.ts +11 -38
- package/src/tools/shared/filesystem/file-ops-service.ts +10 -7
- package/src/tools/shared/shell-output.test.ts +104 -0
- package/src/tools/shared/shell-output.ts +112 -39
- package/src/tools/skills/sandbox-runner.ts +1 -0
- package/src/tools/system/request-permission.test.ts +34 -0
- package/src/tools/system/request-permission.ts +29 -5
- package/src/tools/terminal/shell.test.ts +63 -0
- package/src/tools/terminal/shell.ts +12 -44
- package/src/tools/tool-input-schemas.ts +2 -0
- package/src/tools/tool-manifest.ts +2 -0
- package/src/usage/attribution.ts +12 -4
- package/src/util/__tests__/worker-ownership.test.ts +77 -0
- package/src/util/__tests__/worker-process-command.test.ts +246 -1
- package/src/util/bun-runtime.ts +44 -13
- package/src/util/bundled-asset.ts +9 -1
- package/src/util/disk-usage.ts +1 -0
- package/src/util/file-use.ts +4 -1
- package/src/util/guardian-form-timeouts.ts +50 -0
- package/src/util/image-conversion.ts +1 -0
- package/src/util/oauth-request-body.test.ts +59 -0
- package/src/util/oauth-request-body.ts +56 -5
- package/src/util/platform.ts +69 -73
- package/src/util/process-table.ts +16 -5
- package/src/util/read-stdin.ts +5 -0
- package/src/util/spawn.ts +1 -0
- package/src/util/sqlite3-runtime.ts +25 -5
- package/src/util/worker-ownership.ts +193 -0
- package/src/util/worker-process.ts +375 -37
- package/src/workspace/git-service.ts +4 -12
- package/src/workspace/migrations/010-app-dir-rename.ts +1 -0
|
@@ -45,6 +45,10 @@ import {
|
|
|
45
45
|
recordLiveVoiceSessionStarted,
|
|
46
46
|
} from "../onboarding/onboarding-events-store.js";
|
|
47
47
|
import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
|
|
48
|
+
import {
|
|
49
|
+
attachmentExists,
|
|
50
|
+
deleteOrphanAttachments,
|
|
51
|
+
} from "../persistence/attachments-store.js";
|
|
48
52
|
import { ensureConversationExists } from "../persistence/conversation-crud.js";
|
|
49
53
|
import {
|
|
50
54
|
listProviderIds,
|
|
@@ -60,6 +64,7 @@ import {
|
|
|
60
64
|
voteDominantLanguage,
|
|
61
65
|
} from "../stt/language-metadata.js";
|
|
62
66
|
import { sttCatalogKeyForRole } from "../stt/roles.js";
|
|
67
|
+
import { RoomNoiseFloor } from "../stt/room-noise-floor.js";
|
|
63
68
|
import {
|
|
64
69
|
DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
65
70
|
pcm16MaxNormalizedCorrelation,
|
|
@@ -106,7 +111,10 @@ import {
|
|
|
106
111
|
type VoiceEndpointAction,
|
|
107
112
|
type VoiceEndpointSource,
|
|
108
113
|
} from "./live-voice-metrics.js";
|
|
109
|
-
import {
|
|
114
|
+
import {
|
|
115
|
+
persistLiveVoicePhoto,
|
|
116
|
+
persistLiveVoiceSightFrame,
|
|
117
|
+
} from "./live-voice-photo.js";
|
|
110
118
|
import {
|
|
111
119
|
type LiveVoiceSession as LiveVoiceSessionContract,
|
|
112
120
|
type LiveVoiceSessionCloseReason,
|
|
@@ -130,8 +138,10 @@ import {
|
|
|
130
138
|
PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
|
|
131
139
|
} from "./progress-phrases.js";
|
|
132
140
|
import {
|
|
141
|
+
type LiveVoiceClientAttachFrameFrame,
|
|
133
142
|
type LiveVoiceClientAttachImageFrame,
|
|
134
143
|
type LiveVoiceClientFrame,
|
|
144
|
+
type LiveVoiceClientSightFrameFrame,
|
|
135
145
|
type LiveVoiceClientTextTurnFrame,
|
|
136
146
|
type LiveVoiceClientUpdateConfigFrame,
|
|
137
147
|
LiveVoiceProtocolErrorCode,
|
|
@@ -200,6 +210,18 @@ const ECHO_ONSET_ELIGIBILITY_MS = 300;
|
|
|
200
210
|
// Client buffering makes audible playback trail the server's send-time
|
|
201
211
|
// estimate. Keep the echo window open briefly past that estimate.
|
|
202
212
|
const DEFAULT_ECHO_DRAIN_SLACK_MS = 300;
|
|
213
|
+
// The speech gate's base threshold is raised to this multiple of the room's
|
|
214
|
+
// measured noise floor. Speech has to stand clear of the room it is spoken in,
|
|
215
|
+
// and roughly 3x (about 10dB) is the separation an ordinary talking voice keeps
|
|
216
|
+
// over its own background. Mirrors the liveVoice.vad.noiseFloorMargin default.
|
|
217
|
+
const DEFAULT_NOISE_FLOOR_MARGIN = 3;
|
|
218
|
+
// Ceiling on the adaptive base, as a multiple of the configured threshold. The
|
|
219
|
+
// floor estimate is derived from the microphone, so a pathological room (or a
|
|
220
|
+
// stuck input) could in principle drive it high enough that nothing counts as
|
|
221
|
+
// speech and the user cannot interrupt at all. Barge-in getting harder is a
|
|
222
|
+
// nuisance; barge-in becoming impossible is a broken product, so the adaptation
|
|
223
|
+
// is bounded even though the bound should never be reached.
|
|
224
|
+
const NOISE_FLOOR_MAX_BASE_MULTIPLE = 4;
|
|
203
225
|
// Mirrors MediaTurnDetector's DEFAULT_SILENCE_THRESHOLD_MS: the session
|
|
204
226
|
// tracks the effective trailing-silence threshold (the detector keeps its own
|
|
205
227
|
// copy private) so the endpoint decider can report the pause length.
|
|
@@ -329,6 +351,14 @@ export interface LiveVoiceSessionOptions {
|
|
|
329
351
|
* `DEFAULT_SPEECH_ENERGY_THRESHOLD`.
|
|
330
352
|
*/
|
|
331
353
|
speechEnergyThreshold?: number;
|
|
354
|
+
/**
|
|
355
|
+
* Multiple of the room's measured noise floor that the speech gate's base
|
|
356
|
+
* threshold is raised to. The production factory seeds this from
|
|
357
|
+
* `liveVoice.vad.noiseFloorMargin` config when unset; defaults to
|
|
358
|
+
* `DEFAULT_NOISE_FLOOR_MARGIN`. Values at or below 0 disable the adaptation,
|
|
359
|
+
* pinning the gate to `speechEnergyThreshold` (used by fixed-gate tests).
|
|
360
|
+
*/
|
|
361
|
+
noiseFloorMargin?: number;
|
|
332
362
|
/**
|
|
333
363
|
* Sustained speech (ms) required before speech during assistant playback
|
|
334
364
|
* interrupts it (barge-in); 0 disables the guard. The production factory
|
|
@@ -716,6 +746,14 @@ interface ActiveAssistantTurn {
|
|
|
716
746
|
// that request's transcript, so the model can tell the user the work is
|
|
717
747
|
// still running instead of appearing to have dropped it.
|
|
718
748
|
handedOffRequest: string | null;
|
|
749
|
+
// The parked camera frame this turn claimed, hung on the turn's own
|
|
750
|
+
// persisted user message. Held here so a rolled-back dispatch can hand it
|
|
751
|
+
// back to the session (see restorePendingTurnContext).
|
|
752
|
+
turnAttachmentId: string | null;
|
|
753
|
+
// The frame the rollback could NOT hand back, because a newer one took the
|
|
754
|
+
// slot while this turn was in flight. Nothing will ever send it, so it is
|
|
755
|
+
// collected once the discard has removed the row still linking it.
|
|
756
|
+
staleFrameId: string | null;
|
|
719
757
|
// The queued announcement this turn consumed alongside `continuationResult`
|
|
720
758
|
// — the two are routes for one answer, so launching consumes both. Held here
|
|
721
759
|
// so a rolled-back speculative dispatch can re-arm it (see
|
|
@@ -1176,6 +1214,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1176
1214
|
// Base energy gate for server-VAD speech classification. During estimated
|
|
1177
1215
|
// playback, classifyVadEnergy raises this above the learned echo level.
|
|
1178
1216
|
private readonly speechEnergyThreshold: number | undefined;
|
|
1217
|
+
private readonly noiseFloorMargin: number;
|
|
1218
|
+
// Quietest recent second of microphone audio, learned only while the
|
|
1219
|
+
// assistant is silent (during playback the microphone hears echo, which is
|
|
1220
|
+
// what echoEnergyEma is for). Raises the base gate in a noisy room so the
|
|
1221
|
+
// room itself stops reading as speech.
|
|
1222
|
+
private readonly roomNoiseFloor = new RoomNoiseFloor();
|
|
1179
1223
|
private readonly echoBargeInMargin: number;
|
|
1180
1224
|
private readonly echoEmaHalfLifeMs: number;
|
|
1181
1225
|
private readonly echoDrainSlackMs: number;
|
|
@@ -1255,6 +1299,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1255
1299
|
// turn; cleared when the continuation finishes (by then the result note
|
|
1256
1300
|
// takes over and "still running" would be stale).
|
|
1257
1301
|
private pendingHandoffRequest: string | null = null;
|
|
1302
|
+
// The camera frame a client parked with `attach_frame`, waiting for the next
|
|
1303
|
+
// turn to ride. One slot, latest wins: the frame is ambient context, so an
|
|
1304
|
+
// older one is simply out of date. Consumed (and cleared) when a turn
|
|
1305
|
+
// launches, handed back if that turn is rolled back, and cleared on close.
|
|
1306
|
+
private pendingTurnAttachmentId: string | null = null;
|
|
1258
1307
|
private readonly maxPendingAudioBytes: number;
|
|
1259
1308
|
// Set on VAD speech onset; consumed when the first speech chunk is routed
|
|
1260
1309
|
// to an utterance so the metric lands on the right turn.
|
|
@@ -1388,6 +1437,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1388
1437
|
...(options.metricsClock ? { clock: options.metricsClock } : {}),
|
|
1389
1438
|
});
|
|
1390
1439
|
this.speechEnergyThreshold = options.speechEnergyThreshold;
|
|
1440
|
+
this.noiseFloorMargin =
|
|
1441
|
+
options.noiseFloorMargin ?? DEFAULT_NOISE_FLOOR_MARGIN;
|
|
1391
1442
|
// Precedence for the two sensitivity knobs: per-session start-frame
|
|
1392
1443
|
// override (the client's user setting) > daemon `liveVoice.vad` config
|
|
1393
1444
|
// (seeded into `options` by the factory) > in-code default.
|
|
@@ -1526,6 +1577,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1526
1577
|
case "attach_image":
|
|
1527
1578
|
this.persistPhoto(frame);
|
|
1528
1579
|
return;
|
|
1580
|
+
case "attach_frame":
|
|
1581
|
+
this.parkTurnFrame(frame);
|
|
1582
|
+
return;
|
|
1583
|
+
case "sight_frame":
|
|
1584
|
+
this.persistSightFrame(frame);
|
|
1585
|
+
return;
|
|
1529
1586
|
case "text":
|
|
1530
1587
|
await this.handleTextTurn(frame);
|
|
1531
1588
|
return;
|
|
@@ -1618,6 +1675,138 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1618
1675
|
);
|
|
1619
1676
|
}
|
|
1620
1677
|
|
|
1678
|
+
/**
|
|
1679
|
+
* Persist a kept camera frame into the conversation, running no turn.
|
|
1680
|
+
*
|
|
1681
|
+
* Fire-and-forget for the same reason the photo is: the persist waits out any
|
|
1682
|
+
* in-flight turn, and the socket must keep pumping audio meanwhile. Waiting
|
|
1683
|
+
* is also what keeps a frame that lands mid-reply from splitting that reply's
|
|
1684
|
+
* own rows.
|
|
1685
|
+
*
|
|
1686
|
+
* The frame becomes its own tagged user message rather than riding a turn, so
|
|
1687
|
+
* the transcript is the record of what the assistant saw and the model
|
|
1688
|
+
* correlates a frame with speech by adjacency. See `live-voice-photo.ts`.
|
|
1689
|
+
*
|
|
1690
|
+
* An id the attachment store cannot resolve is refused there, before the
|
|
1691
|
+
* persist waits on anything, so a bad frame is answered while the client can
|
|
1692
|
+
* still act on it rather than a turn's length later. That resolve is the only
|
|
1693
|
+
* existence check: a second one here would read the same row to learn the
|
|
1694
|
+
* same fact.
|
|
1695
|
+
*/
|
|
1696
|
+
private persistSightFrame(frame: LiveVoiceClientSightFrameFrame): void {
|
|
1697
|
+
void persistLiveVoiceSightFrame(
|
|
1698
|
+
this.conversationId,
|
|
1699
|
+
frame.attachmentId,
|
|
1700
|
+
).then((result) => {
|
|
1701
|
+
if (!result.ok && !this.isClosed) {
|
|
1702
|
+
void this.sendFrame({
|
|
1703
|
+
type: "error",
|
|
1704
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1705
|
+
message: "Could not attach that camera frame to the conversation.",
|
|
1706
|
+
// Names the frame as the casualty so the client can retract the
|
|
1707
|
+
// preview it already showed, rather than filing this with the
|
|
1708
|
+
// transient transcriber and TTS blips that share `recoverable`.
|
|
1709
|
+
frameType: "sight_frame",
|
|
1710
|
+
// The session is fine; only this frame failed.
|
|
1711
|
+
recoverable: true,
|
|
1712
|
+
});
|
|
1713
|
+
}
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
/**
|
|
1718
|
+
* Park a camera frame for the next turn to carry, or give up the parked one.
|
|
1719
|
+
*
|
|
1720
|
+
* Nothing is persisted and no turn runs: the frame is ambient context, and
|
|
1721
|
+
* on its own it says nothing the user asked to say. It rides the next turn's
|
|
1722
|
+
* own user message instead, so the picture and the words about it are one
|
|
1723
|
+
* message. A frame that arrives while a turn is already running belongs to
|
|
1724
|
+
* the turn after it, which the drain at launch takes care of.
|
|
1725
|
+
*
|
|
1726
|
+
* A null id unparks: the client's camera is off, so the slot must stop
|
|
1727
|
+
* answering for a view nobody can see any more. The frame goes with it, and
|
|
1728
|
+
* unparking an empty slot is a no-op.
|
|
1729
|
+
*
|
|
1730
|
+
* The id is checked against the attachment store here rather than at the
|
|
1731
|
+
* turn, for the same reason a failed photo is reported: an id that resolves
|
|
1732
|
+
* to nothing would otherwise leave the user believing the assistant can see
|
|
1733
|
+
* something it never received, one turn later and with nothing said.
|
|
1734
|
+
*/
|
|
1735
|
+
private parkTurnFrame(frame: LiveVoiceClientAttachFrameFrame): void {
|
|
1736
|
+
if (frame.attachmentId === null) {
|
|
1737
|
+
const parked = this.pendingTurnAttachmentId;
|
|
1738
|
+
this.pendingTurnAttachmentId = null;
|
|
1739
|
+
this.reclaimParkedFrame(parked);
|
|
1740
|
+
return;
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
let exists = false;
|
|
1744
|
+
try {
|
|
1745
|
+
exists = attachmentExists(frame.attachmentId);
|
|
1746
|
+
} catch (err) {
|
|
1747
|
+
log.warn(
|
|
1748
|
+
{ err, attachmentId: frame.attachmentId },
|
|
1749
|
+
"Live-voice camera frame lookup failed",
|
|
1750
|
+
);
|
|
1751
|
+
}
|
|
1752
|
+
if (!exists) {
|
|
1753
|
+
void this.sendFrame({
|
|
1754
|
+
type: "error",
|
|
1755
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1756
|
+
message: "Could not attach that camera frame to the conversation.",
|
|
1757
|
+
frameType: "attach_frame",
|
|
1758
|
+
// The session is fine; only this frame failed.
|
|
1759
|
+
recoverable: true,
|
|
1760
|
+
});
|
|
1761
|
+
return;
|
|
1762
|
+
}
|
|
1763
|
+
const displaced = this.pendingTurnAttachmentId;
|
|
1764
|
+
this.pendingTurnAttachmentId = frame.attachmentId;
|
|
1765
|
+
if (displaced !== null && displaced !== frame.attachmentId) {
|
|
1766
|
+
this.reclaimParkedFrame(displaced);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
/**
|
|
1771
|
+
* Give up a parked camera frame no turn will ever carry: one a newer frame
|
|
1772
|
+
* displaced, one the session end leaves behind, one a rollback could not
|
|
1773
|
+
* hand back. A client shooting the camera on a timer parks one every few
|
|
1774
|
+
* seconds, and attachment collection is scoped to message deletion with no
|
|
1775
|
+
* global sweep, so an id dropped without this leaves its row and its bytes
|
|
1776
|
+
* behind for good.
|
|
1777
|
+
*
|
|
1778
|
+
* Safe because `deleteOrphanAttachments` is link-aware: it removes only ids
|
|
1779
|
+
* with no message link, so a frame that already rode a turn is kept and one
|
|
1780
|
+
* that never left this slot is collected. Nothing waits on the result and a
|
|
1781
|
+
* store that cannot be reached must not take the call down, so a failure is
|
|
1782
|
+
* logged and dropped.
|
|
1783
|
+
*/
|
|
1784
|
+
private reclaimParkedFrame(attachmentId: string | null): void {
|
|
1785
|
+
if (attachmentId === null) {
|
|
1786
|
+
return;
|
|
1787
|
+
}
|
|
1788
|
+
try {
|
|
1789
|
+
deleteOrphanAttachments([attachmentId]);
|
|
1790
|
+
} catch (err) {
|
|
1791
|
+
log.warn(
|
|
1792
|
+
{ err, attachmentId },
|
|
1793
|
+
"Could not reclaim a parked live-voice camera frame",
|
|
1794
|
+
);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
/**
|
|
1799
|
+
* Collect the frame a turn's rollback could not hand back, once the discard
|
|
1800
|
+
* that owns the turn's row has finished. Ordering is the whole point: the
|
|
1801
|
+
* reclaim is link-aware, so running it while the doomed row still links the
|
|
1802
|
+
* frame finds the link, keeps the frame, and leaks it.
|
|
1803
|
+
*/
|
|
1804
|
+
private reclaimStaleTurnFrame(turn: ActiveAssistantTurn): void {
|
|
1805
|
+
const staleFrameId = turn.staleFrameId;
|
|
1806
|
+
turn.staleFrameId = null;
|
|
1807
|
+
this.reclaimParkedFrame(staleFrameId);
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1621
1810
|
/**
|
|
1622
1811
|
* Apply a mid-session `update_config` frame: retune the live turn detector's
|
|
1623
1812
|
* pause ("pause before reply") and/or the barge-in guard ("interrupt
|
|
@@ -1681,6 +1870,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1681
1870
|
this.turnDetector?.dispose();
|
|
1682
1871
|
this.clearEndpointExtensionTimer();
|
|
1683
1872
|
this.clearProviderTurnEndTimer();
|
|
1873
|
+
// No turn will ever carry it now, and the frame is the client's to shoot
|
|
1874
|
+
// again on its next session, so the staged row and bytes go with the call.
|
|
1875
|
+
this.reclaimParkedFrame(this.pendingTurnAttachmentId);
|
|
1876
|
+
this.pendingTurnAttachmentId = null;
|
|
1684
1877
|
// There is no longer anyone on the call to speak to, so a queued
|
|
1685
1878
|
// announcement cannot be spoken — but the answer behind it is finished
|
|
1686
1879
|
// work the user asked for, so it takes the same conversation route a
|
|
@@ -2166,18 +2359,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2166
2359
|
* confirmed echo while speech above the learned margin remains frozen out.
|
|
2167
2360
|
*/
|
|
2168
2361
|
private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
|
|
2169
|
-
const baseThreshold =
|
|
2170
|
-
this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
|
|
2171
2362
|
const meanAmplitude = pcm16MeanAmplitude(chunk);
|
|
2363
|
+
const chunkMs = pcm16DurationMs(
|
|
2364
|
+
chunk.byteLength,
|
|
2365
|
+
this.context.startFrame.audio.sampleRate,
|
|
2366
|
+
);
|
|
2367
|
+
const baseThreshold = this.effectiveBaseThreshold();
|
|
2172
2368
|
if (
|
|
2173
2369
|
this.echoBargeInMargin <= 1 ||
|
|
2174
2370
|
!this.isAssistantPlaybackEchoPossible()
|
|
2175
2371
|
) {
|
|
2176
2372
|
this.resetEchoReference();
|
|
2373
|
+
// Only learn the room while the assistant is silent. The floor is a
|
|
2374
|
+
// minimum over completed blocks, so this chunk cannot lower the
|
|
2375
|
+
// threshold that is about to judge it.
|
|
2376
|
+
this.roomNoiseFloor.observe(meanAmplitude, chunkMs);
|
|
2177
2377
|
return [
|
|
2178
2378
|
this.classifyAtFixedThreshold(chunk, baseThreshold, meanAmplitude),
|
|
2179
2379
|
];
|
|
2180
2380
|
}
|
|
2381
|
+
// Playback is starting or ongoing: stop measuring the room, and drop the
|
|
2382
|
+
// partial block rather than let it straddle the silence and the echo.
|
|
2383
|
+
this.roomNoiseFloor.interrupt();
|
|
2181
2384
|
|
|
2182
2385
|
if (this.echoWindowTotalAudioMs === 0) {
|
|
2183
2386
|
this.echoWindowGuardCarryover =
|
|
@@ -2186,10 +2389,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2186
2389
|
this.echoWindowGuardCarryover = false;
|
|
2187
2390
|
}
|
|
2188
2391
|
|
|
2189
|
-
const chunkMs = pcm16DurationMs(
|
|
2190
|
-
chunk.byteLength,
|
|
2191
|
-
this.context.startFrame.audio.sampleRate,
|
|
2192
|
-
);
|
|
2193
2392
|
const onsetWasEligible =
|
|
2194
2393
|
!this.echoOnsetLapsed &&
|
|
2195
2394
|
this.echoWindowTotalAudioMs < ECHO_ONSET_ELIGIBILITY_MS;
|
|
@@ -2296,6 +2495,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2296
2495
|
alpha * meanAmplitude + (1 - alpha) * this.echoEnergyEma;
|
|
2297
2496
|
}
|
|
2298
2497
|
|
|
2498
|
+
/**
|
|
2499
|
+
* The gate a chunk is judged against before any playback-echo adjustment.
|
|
2500
|
+
*
|
|
2501
|
+
* The configured `speechEnergyThreshold` is a floor, never a ceiling: the
|
|
2502
|
+
* adaptation can only make speech classification *stricter*. That asymmetry
|
|
2503
|
+
* is deliberate. A gate that could drop below the configured value would be
|
|
2504
|
+
* able to invent new self-interruption in a room quieter than the one the
|
|
2505
|
+
* constant was chosen for, which is the failure this whole change exists to
|
|
2506
|
+
* remove; a gate that only rises can, at worst, make a soft barge-in need
|
|
2507
|
+
* repeating. The result is bounded so it can never rise far enough to make
|
|
2508
|
+
* interrupting impossible.
|
|
2509
|
+
*/
|
|
2510
|
+
private effectiveBaseThreshold(): number {
|
|
2511
|
+
const configured =
|
|
2512
|
+
this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
|
|
2513
|
+
if (this.noiseFloorMargin <= 0) {
|
|
2514
|
+
return configured;
|
|
2515
|
+
}
|
|
2516
|
+
const floor = this.roomNoiseFloor.floor;
|
|
2517
|
+
if (floor === null) {
|
|
2518
|
+
return configured;
|
|
2519
|
+
}
|
|
2520
|
+
return Math.min(
|
|
2521
|
+
Math.max(configured, floor * this.noiseFloorMargin),
|
|
2522
|
+
configured * NOISE_FLOOR_MAX_BASE_MULTIPLE,
|
|
2523
|
+
);
|
|
2524
|
+
}
|
|
2525
|
+
|
|
2299
2526
|
private classifyAtFixedThreshold(
|
|
2300
2527
|
chunk: Buffer,
|
|
2301
2528
|
baseThreshold: number,
|
|
@@ -3852,9 +4079,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3852
4079
|
): void {
|
|
3853
4080
|
turn.speculativePending = false;
|
|
3854
4081
|
// The dispatch is being unwound, so the barge-in merge note, the finished
|
|
3855
|
-
// continuation's answer, its queued announcement, and any
|
|
3856
|
-
// all go back to the session. The turn that would have delivered
|
|
3857
|
-
// gone, and the utterance they belong to is about to be sent by
|
|
4082
|
+
// continuation's answer, its queued announcement, and any camera frame it
|
|
4083
|
+
// claimed all go back to the session. The turn that would have delivered
|
|
4084
|
+
// them is gone, and the utterance they belong to is about to be sent by
|
|
4085
|
+
// another.
|
|
3858
4086
|
this.restorePendingTurnContext(turn);
|
|
3859
4087
|
// Latched before the handle check: when the discard beats the bridge
|
|
3860
4088
|
// handle's resolution (startVoiceTurn still persisting), the handle's
|
|
@@ -3869,12 +4097,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3869
4097
|
);
|
|
3870
4098
|
const handle = turn.handle;
|
|
3871
4099
|
turn.handle = null;
|
|
3872
|
-
void handle
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
4100
|
+
void handle
|
|
4101
|
+
?.discard?.()
|
|
4102
|
+
.catch((err: unknown) => {
|
|
4103
|
+
log.warn(
|
|
4104
|
+
{ err, turnId: turn.turnId, reason },
|
|
4105
|
+
"Speculative voice turn discard failed",
|
|
4106
|
+
);
|
|
4107
|
+
})
|
|
4108
|
+
// Only once the discard has deleted the row: until then the row still
|
|
4109
|
+
// links a displaced frame, and the reclaim would keep it forever.
|
|
4110
|
+
.finally(() => {
|
|
4111
|
+
this.reclaimStaleTurnFrame(turn);
|
|
4112
|
+
});
|
|
3878
4113
|
turn.utterance.assistantTurnStarted = false;
|
|
3879
4114
|
log.info(
|
|
3880
4115
|
{ turnId: turn.turnId, reason },
|
|
@@ -4622,10 +4857,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4622
4857
|
}
|
|
4623
4858
|
|
|
4624
4859
|
/**
|
|
4625
|
-
* Take the barge-in merge context, the completed-continuation context,
|
|
4626
|
-
*
|
|
4627
|
-
* launched turn
|
|
4628
|
-
* context rides delivers the same answer
|
|
4860
|
+
* Take the barge-in merge context, the completed-continuation context, the
|
|
4861
|
+
* handoff note, and the parked camera frame for the turn being launched
|
|
4862
|
+
* (each feeds exactly the next launched turn) and cancel the queued
|
|
4863
|
+
* announcement, because the turn this context rides delivers the same answer
|
|
4864
|
+
* and would otherwise say it twice.
|
|
4629
4865
|
*
|
|
4630
4866
|
* Every real user turn consumes here, speculative or not: a speculative
|
|
4631
4867
|
* dispatch latches `assistantTurnStarted`, so a turn that skipped this would
|
|
@@ -4637,16 +4873,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4637
4873
|
continuationResult: string | null;
|
|
4638
4874
|
handedOffRequest: string | null;
|
|
4639
4875
|
announcement: ContinuationDelivery | null;
|
|
4876
|
+
turnAttachmentId: string | null;
|
|
4640
4877
|
} {
|
|
4641
4878
|
const consumed = {
|
|
4642
4879
|
interruptedRequest: this.pendingInterruptedRequest,
|
|
4643
4880
|
continuationResult: this.pendingContinuationResult,
|
|
4644
4881
|
handedOffRequest: this.pendingHandoffRequest,
|
|
4645
4882
|
announcement: this.pendingAnnouncement,
|
|
4883
|
+
turnAttachmentId: this.pendingTurnAttachmentId,
|
|
4646
4884
|
};
|
|
4647
4885
|
this.pendingInterruptedRequest = null;
|
|
4648
4886
|
this.pendingContinuationResult = null;
|
|
4649
4887
|
this.pendingHandoffRequest = null;
|
|
4888
|
+
this.pendingTurnAttachmentId = null;
|
|
4650
4889
|
this.clearContinuationAnnouncement();
|
|
4651
4890
|
return consumed;
|
|
4652
4891
|
}
|
|
@@ -4668,21 +4907,43 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4668
4907
|
const continuationResult = turn.continuationResult;
|
|
4669
4908
|
const handedOffRequest = turn.handedOffRequest;
|
|
4670
4909
|
const announcement = turn.consumedAnnouncement;
|
|
4910
|
+
const turnAttachmentId = turn.turnAttachmentId;
|
|
4671
4911
|
// One restore per turn: the rollback paths overlap (a discarded turn whose
|
|
4672
4912
|
// leg start also fails), and the second pass must be a no-op.
|
|
4673
4913
|
turn.interruptedRequest = null;
|
|
4674
4914
|
turn.continuationResult = null;
|
|
4675
4915
|
turn.handedOffRequest = null;
|
|
4676
4916
|
turn.consumedAnnouncement = null;
|
|
4917
|
+
turn.turnAttachmentId = null;
|
|
4918
|
+
// Nobody's to send unless the slot already holds it. A client that
|
|
4919
|
+
// retransmits the id this turn claimed parked the SAME frame rather than a
|
|
4920
|
+
// newer one, and collecting it would leave the slot pointing at a row that
|
|
4921
|
+
// is gone: the discard keeps the bytes alive precisely so the replay can
|
|
4922
|
+
// still send them.
|
|
4923
|
+
const orphanedFrameId =
|
|
4924
|
+
turnAttachmentId !== null &&
|
|
4925
|
+
this.pendingTurnAttachmentId !== turnAttachmentId
|
|
4926
|
+
? turnAttachmentId
|
|
4927
|
+
: null;
|
|
4677
4928
|
if (
|
|
4678
4929
|
this.isClosed ||
|
|
4679
4930
|
this.detachStopGeneration !== turn.pendingContextStopGeneration
|
|
4680
4931
|
) {
|
|
4932
|
+
// The drop is deliberate, so the frame it drops is nobody's to send.
|
|
4933
|
+
turn.staleFrameId = orphanedFrameId;
|
|
4681
4934
|
return;
|
|
4682
4935
|
}
|
|
4683
4936
|
if (this.pendingInterruptedRequest === null) {
|
|
4684
4937
|
this.pendingInterruptedRequest = interruptedRequest;
|
|
4685
4938
|
}
|
|
4939
|
+
// A frame parked since this turn launched is newer, so it wins outright
|
|
4940
|
+
// and the one being handed back is the stale one: collect it rather than
|
|
4941
|
+
// leave it staged for a turn that will never ask for it.
|
|
4942
|
+
if (this.pendingTurnAttachmentId === null) {
|
|
4943
|
+
this.pendingTurnAttachmentId = turnAttachmentId;
|
|
4944
|
+
} else {
|
|
4945
|
+
turn.staleFrameId = orphanedFrameId;
|
|
4946
|
+
}
|
|
4686
4947
|
if (this.pendingContinuationResult === null) {
|
|
4687
4948
|
this.pendingContinuationResult = continuationResult;
|
|
4688
4949
|
}
|
|
@@ -4779,6 +5040,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4779
5040
|
interruptedRequest: pending?.interruptedRequest ?? null,
|
|
4780
5041
|
continuationResult: pending?.continuationResult ?? null,
|
|
4781
5042
|
handedOffRequest: pending?.handedOffRequest ?? null,
|
|
5043
|
+
turnAttachmentId: pending?.turnAttachmentId ?? null,
|
|
5044
|
+
staleFrameId: null,
|
|
4782
5045
|
consumedAnnouncement: pending?.announcement ?? null,
|
|
4783
5046
|
pendingContextStopGeneration: this.detachStopGeneration,
|
|
4784
5047
|
continuationDelivery: opts?.continuationDelivery ?? null,
|
|
@@ -4803,6 +5066,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4803
5066
|
await this.sendFrame({ type: "thinking", turnId });
|
|
4804
5067
|
if (!this.isActiveAssistantTurn(token)) {
|
|
4805
5068
|
this.restorePendingTurnContext(activeTurn);
|
|
5069
|
+
// Nothing was persisted, so a frame the restore could not hand back
|
|
5070
|
+
// links to no row and is collectible right here.
|
|
5071
|
+
this.reclaimStaleTurnFrame(activeTurn);
|
|
4806
5072
|
return false;
|
|
4807
5073
|
}
|
|
4808
5074
|
} else {
|
|
@@ -4853,11 +5119,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4853
5119
|
content,
|
|
4854
5120
|
routingLeg: "front-door",
|
|
4855
5121
|
frontDoor: true,
|
|
5122
|
+
// Only this leg: an escalated leg persists its own continuation row, and
|
|
5123
|
+
// the frame belongs to the row carrying the user's words.
|
|
5124
|
+
...(activeTurn.turnAttachmentId
|
|
5125
|
+
? { attachments: [activeTurn.turnAttachmentId] }
|
|
5126
|
+
: {}),
|
|
4856
5127
|
});
|
|
4857
5128
|
if (!started) {
|
|
4858
5129
|
// Nothing was persisted or spoken, so the context this turn consumed goes
|
|
4859
|
-
// back to the session rather than dying with the failed dispatch.
|
|
5130
|
+
// back to the session rather than dying with the failed dispatch. A frame
|
|
5131
|
+
// a newer one displaced meanwhile links to no row, so it is collected
|
|
5132
|
+
// here rather than left staged.
|
|
4860
5133
|
this.restorePendingTurnContext(activeTurn);
|
|
5134
|
+
this.reclaimStaleTurnFrame(activeTurn);
|
|
4861
5135
|
}
|
|
4862
5136
|
return started;
|
|
4863
5137
|
}
|
|
@@ -4890,6 +5164,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4890
5164
|
routingLeg?: VoiceRoutingLeg;
|
|
4891
5165
|
frontDoor?: boolean;
|
|
4892
5166
|
spokenEscalationBridge?: string;
|
|
5167
|
+
attachments?: readonly string[];
|
|
4893
5168
|
},
|
|
4894
5169
|
): Promise<boolean> {
|
|
4895
5170
|
if (!this.startVoiceTurn) {
|
|
@@ -4994,6 +5269,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4994
5269
|
this.clearAwaitingApproval(activeTurn);
|
|
4995
5270
|
},
|
|
4996
5271
|
content: leg.content,
|
|
5272
|
+
...(leg.attachments ? { attachments: leg.attachments } : {}),
|
|
4997
5273
|
isInbound: true,
|
|
4998
5274
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
4999
5275
|
signal: activeTurn.abortController.signal,
|
|
@@ -5310,12 +5586,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5310
5586
|
// rollback: a plain abort would leave the discarded pause's user
|
|
5311
5587
|
// row in history.
|
|
5312
5588
|
if (activeTurn.discardRequested && handle.discard) {
|
|
5313
|
-
void handle
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5589
|
+
void handle
|
|
5590
|
+
.discard()
|
|
5591
|
+
.catch((err: unknown) => {
|
|
5592
|
+
log.warn(
|
|
5593
|
+
{ err, turnId: activeTurn.turnId },
|
|
5594
|
+
"Late speculative voice turn discard failed",
|
|
5595
|
+
);
|
|
5596
|
+
})
|
|
5597
|
+
// This discard owns the row, so it owns the reclaim of any frame
|
|
5598
|
+
// the rollback could not hand back (see reclaimStaleTurnFrame).
|
|
5599
|
+
.finally(() => {
|
|
5600
|
+
this.reclaimStaleTurnFrame(activeTurn);
|
|
5601
|
+
});
|
|
5319
5602
|
} else {
|
|
5320
5603
|
handle.abort();
|
|
5321
5604
|
}
|
|
@@ -6657,6 +6940,7 @@ export function createLiveVoiceSession(
|
|
|
6657
6940
|
: {}),
|
|
6658
6941
|
speechEnergyThreshold:
|
|
6659
6942
|
options.speechEnergyThreshold ?? vadConfig?.speechEnergyThreshold,
|
|
6943
|
+
noiseFloorMargin: options.noiseFloorMargin ?? vadConfig?.noiseFloorMargin,
|
|
6660
6944
|
bargeInMinSpeechMs:
|
|
6661
6945
|
options.bargeInMinSpeechMs ?? vadConfig?.bargeInMinSpeechMs,
|
|
6662
6946
|
echoBargeInMargin:
|