@vellumai/assistant 0.11.7 → 0.11.8-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +10 -10
- package/Dockerfile +1 -0
- package/docs/architecture/security.md +1 -1
- package/docs/error-handling.md +1 -2
- package/docs/guardian-request-flow.md +51 -5
- package/docs/trusted-contact-access.md +13 -12
- package/knip.json +1 -0
- package/node_modules/@vellumai/avatar-catalog/package.json +19 -0
- package/{src/avatar/character-components.ts → node_modules/@vellumai/avatar-catalog/src/catalog.ts} +12 -44
- package/node_modules/@vellumai/avatar-catalog/src/colors.ts +19 -0
- package/node_modules/@vellumai/avatar-catalog/src/index.ts +25 -0
- package/node_modules/@vellumai/avatar-catalog/src/types.ts +47 -0
- package/node_modules/@vellumai/avatar-catalog/tsconfig.json +20 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/node_modules/@vellumai/credential-storage/src/static-credentials.ts +43 -12
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/test-path-guard.test.ts +89 -0
- package/node_modules/@vellumai/environments/src/test-path-guard.ts +113 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/guardian-request-contract.test.ts +8 -18
- package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +46 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/invite-contract.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +46 -0
- package/node_modules/@vellumai/gateway-client/src/guardian-request-contract.ts +62 -21
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +14 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +53 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +11 -5
- package/node_modules/@vellumai/gateway-client/src/invite-contract.ts +1 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
- package/openapi.yaml +481 -9
- package/package.json +3 -1
- package/scripts/sync-llm-catalog.ts +3 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +16 -0
- package/src/__tests__/agent-loop-compacted-history-transform.test.ts +304 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +3 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +3 -0
- package/src/__tests__/anthropic-provider.test.ts +56 -0
- package/src/__tests__/approval-cascade.test.ts +0 -13
- package/src/__tests__/attachments-store.test.ts +117 -6
- package/src/__tests__/attachments.test.ts +122 -0
- package/src/__tests__/bundled-asset.test.ts +22 -1
- package/src/__tests__/channel-approval.test.ts +14 -7
- package/src/__tests__/channel-approvals.test.ts +3 -3
- package/src/__tests__/channel-delivery-store.test.ts +58 -0
- package/src/__tests__/channel-inbound-disk-pressure.test.ts +1 -0
- package/src/__tests__/channel-invite-transport.test.ts +1 -0
- package/src/__tests__/channel-readiness-slack-remote.test.ts +3 -3
- package/src/__tests__/compaction-events.test.ts +0 -12
- package/src/__tests__/compactor-call-site-logging.test.ts +7 -1
- package/src/__tests__/compactor-retained-image-validation.test.ts +15 -0
- package/src/__tests__/config-schema.test.ts +35 -0
- package/src/__tests__/contacts-write.test.ts +141 -3
- package/src/__tests__/conversation-abort-tool-results.test.ts +0 -13
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +0 -12
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -13
- package/src/__tests__/conversation-agent-loop.test.ts +95 -13
- package/src/__tests__/conversation-confirmation-signals.test.ts +0 -13
- package/src/__tests__/conversation-load-history-repair.test.ts +418 -0
- package/src/__tests__/conversation-media-retry.test.ts +33 -2
- package/src/__tests__/conversation-pairing.test.ts +54 -0
- package/src/__tests__/conversation-process-callsite.test.ts +0 -12
- package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -13
- package/src/__tests__/conversation-queue.test.ts +4 -13
- package/src/__tests__/conversation-retry-route.test.ts +5 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +1 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +1 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -1
- package/src/__tests__/conversation-sight-frames.test.ts +615 -0
- package/src/__tests__/conversation-slash-queue.test.ts +0 -13
- package/src/__tests__/conversation-slash-unknown.test.ts +0 -13
- package/src/__tests__/conversation-speed-override.test.ts +0 -12
- package/src/__tests__/conversation-store-ephemeral.test.ts +0 -12
- package/src/__tests__/conversation-summarize-route.test.ts +5 -0
- package/src/__tests__/conversation-summarize-up-to.test.ts +102 -0
- package/src/__tests__/conversation-wait-for-idle.test.ts +0 -12
- package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -5
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -17
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -17
- package/src/__tests__/credential-health-service.test.ts +45 -0
- package/src/__tests__/credential-record-retire.test.ts +256 -0
- package/src/__tests__/credential-record-write-through.test.ts +8 -4
- package/src/__tests__/db-conversation-fork-lineage-migration.test.ts +0 -4
- package/src/__tests__/db-conversation-inference-profile-migration.test.ts +0 -4
- package/src/__tests__/db-llm-request-log-provider-migration.test.ts +0 -8
- package/src/__tests__/delete-propagation.test.ts +121 -3
- package/src/__tests__/evict-conversations-for-reload.test.ts +46 -6
- package/src/__tests__/file-ops-service.test.ts +8 -3
- package/src/__tests__/gemini-provider.test.ts +64 -0
- package/src/__tests__/guardian-action-service-scope.test.ts +118 -0
- package/src/__tests__/guardian-card-withdrawal.test.ts +299 -5
- package/src/__tests__/guardian-expiry-notifier.test.ts +117 -13
- package/src/__tests__/guardian-gateway-sim.test.ts +119 -0
- package/src/__tests__/guardian-gateway-sim.ts +56 -20
- package/src/__tests__/guardian-reply-router-answer-mode.test.ts +6 -0
- package/src/__tests__/guardian-request-status-sync.test.ts +162 -0
- package/src/__tests__/guardian-wait-controller.test.ts +6 -0
- package/src/__tests__/headless-browser-mode.test.ts +27 -0
- package/src/__tests__/helpers/mock-conversation.ts +2 -0
- package/src/__tests__/list-messages-system-card.test.ts +70 -0
- package/src/__tests__/llm-catalog-parity.test.ts +2 -0
- package/src/__tests__/managed-profile-guard.test.ts +2 -2
- package/src/__tests__/media-resolve-image-validation.test.ts +106 -0
- package/src/__tests__/media-stream-server-integration.test.ts +6 -0
- package/src/__tests__/non-member-access-request.test.ts +12 -0
- package/src/__tests__/notification-discord-adapter.test.ts +76 -4
- package/src/__tests__/oauth-commands-routes.test.ts +21 -0
- package/src/__tests__/openai-provider.test.ts +63 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +114 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +63 -1
- package/src/__tests__/persist-unsendable-image.test.ts +19 -0
- package/src/__tests__/plugin-app-serve-routes.test.ts +2 -2
- package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
- package/src/__tests__/process-message-background-slack.test.ts +1 -1
- package/src/__tests__/process-message-display-content.test.ts +1 -1
- package/src/__tests__/processing-flag-persist-failure.test.ts +0 -5
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +0 -17
- package/src/__tests__/qdrant-manager.test.ts +6 -0
- package/src/__tests__/reaction-persistence.test.ts +58 -62
- package/src/__tests__/retry-thinking-adaptive-only.test.ts +67 -0
- package/src/__tests__/server-history-render.test.ts +22 -0
- package/src/__tests__/sight-frame-retention.test.ts +362 -0
- package/src/__tests__/slack-reaction-approvals.test.ts +44 -24
- package/src/__tests__/slack-reaction-guardian-approval.test.ts +29 -1
- package/src/__tests__/subagent-fork-notifications.test.ts +2 -0
- package/src/__tests__/subagent-manager-notify.test.ts +2 -0
- package/src/__tests__/subagent-notify-parent.test.ts +2 -0
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +85 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +4 -4
- package/src/__tests__/terminal-tools.test.ts +9 -9
- package/src/__tests__/tool-result-spool.test.ts +5 -8
- package/src/__tests__/usage-attribution.test.ts +26 -0
- package/src/acp/__tests__/acp-auth-marker-store.test.ts +191 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +262 -0
- package/src/acp/__tests__/acp-connect-card-state.test.ts +17 -0
- package/src/acp/__tests__/helpers/acp-history-db.ts +10 -3
- package/src/acp/acp-auth-marker-store.ts +196 -0
- package/src/acp/acp-claude-oauth.ts +154 -4
- package/src/acp/acp-connect-card-state.ts +49 -8
- package/src/acp/agent-process.ts +1 -0
- package/src/acp/auto-install.ts +1 -0
- package/src/acp/client-handler.ts +1 -0
- package/src/acp/prepare-agent-env.ts +79 -1
- package/src/acp/resolve-agent.ts +12 -0
- package/src/acp/session-manager.test.ts +183 -0
- package/src/acp/session-manager.ts +110 -6
- package/src/acp/types.ts +17 -0
- package/src/agent/attachments.ts +7 -2
- package/src/agent/loop.ts +59 -1
- package/src/api/events/contact-form-closed.ts +32 -0
- package/src/api/events/contact-record-request.ts +63 -0
- package/src/api/events/contact-request.ts +10 -3
- package/src/api/index.ts +20 -0
- package/src/api/responses/conversation-message.ts +20 -0
- package/src/api/responses/home.ts +101 -0
- package/src/approvals/AGENTS.md +1 -1
- package/src/approvals/guardian-card-withdrawal.ts +127 -30
- package/src/approvals/guardian-decision-primitive.ts +9 -0
- package/src/approvals/guardian-expiry-notifier.ts +22 -16
- package/src/approvals/guardian-request-status-sync.ts +98 -0
- package/src/avatar/svg-compositor.ts +1 -1
- package/src/avatar/traits-png-sync.ts +1 -1
- package/src/bundler/app-compiler.ts +1 -0
- package/src/bundler/compiler-tools.ts +1 -0
- package/src/bundler/package-resolver.ts +1 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +183 -3
- package/src/calls/guardian-action-sweep.ts +6 -1
- package/src/calls/voice-session-bridge.ts +46 -1
- package/src/channels/__tests__/gateway-guardian-requests.test.ts +13 -18
- package/src/channels/gateway-guardian-requests.ts +37 -18
- package/src/cli/__tests__/contacts-record-prompt-cli.test.ts +329 -0
- package/src/cli/commands/__tests__/browser.test.ts +65 -1
- package/src/cli/commands/browser.ts +30 -10
- package/src/cli/commands/contacts.help.ts +173 -11
- package/src/cli/commands/contacts.ts +343 -4
- package/src/cli/commands/db/refresh.ts +1 -0
- package/src/cli/commands/oauth/index.help.ts +6 -3
- package/src/cli/commands/oauth/request.test.ts +37 -0
- package/src/cli/commands/oauth/request.ts +7 -6
- package/src/cli/lib/bundled-marketplace.json +1 -1
- package/src/cli/lib/install-from-github.ts +20 -19
- package/src/cli/lib/install-plugin-dependencies.ts +3 -5
- package/src/cli/lib/merge-plugin-tree.ts +1 -0
- package/src/cli/lib/publish-plugin.ts +4 -1
- package/src/config/__tests__/balanced-model-experiment.test.ts +63 -31
- package/src/config/__tests__/default-profile-catalog.test.ts +2 -2
- package/src/config/balanced-model-experiment.ts +15 -9
- package/src/config/bundled-skills/contacts/SKILL.md +61 -18
- package/src/config/default-profile-catalog.ts +12 -11
- package/src/config/feature-flag-registry.json +22 -13
- package/src/config/schema.ts +2 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +13 -0
- package/src/config/schemas/live-voice.ts +7 -0
- package/src/config/schemas/sight.ts +45 -0
- package/src/config/skills.ts +2 -1
- package/src/contacts/__tests__/mirror-reconciler.test.ts +352 -0
- package/src/contacts/contact-store.ts +47 -4
- package/src/contacts/contacts-write.ts +5 -2
- package/src/contacts/mirror-reconciler.ts +210 -0
- package/src/contacts/mirror-tombstones.ts +42 -0
- package/src/contacts/notify-contacts-changed.ts +11 -2
- package/src/context/compactor.ts +47 -19
- package/src/context/tool-result-spool.ts +21 -14
- package/src/conversations/message-consolidation.ts +2 -2
- package/src/credential-health/credential-health-service.ts +37 -6
- package/src/daemon/conversation-agent-loop-handlers.ts +19 -2
- package/src/daemon/conversation-agent-loop.ts +100 -1
- package/src/daemon/conversation-media-retry.ts +11 -6
- package/src/daemon/conversation-messaging.ts +293 -30
- package/src/daemon/conversation-runtime-assembly.ts +67 -0
- package/src/daemon/conversation-sight-frames.ts +187 -0
- package/src/daemon/conversation-store.ts +18 -28
- package/src/daemon/conversation-surfaces.ts +12 -2
- package/src/daemon/conversation-tool-setup.ts +39 -10
- package/src/daemon/conversation.ts +159 -16
- package/src/daemon/daemon-control.ts +6 -17
- package/src/daemon/handlers/config-model.test.ts +1 -0
- package/src/daemon/handlers/conversations.ts +10 -11
- package/src/daemon/install-assistant-command.ts +1 -0
- package/src/daemon/lifecycle.ts +10 -13
- package/src/daemon/message-types/sync.ts +4 -0
- package/src/daemon/process-message.ts +2 -17
- package/src/daemon/reaction-history-render.test.ts +162 -0
- package/src/daemon/reaction-history-render.ts +115 -0
- package/src/daemon/reaction-record.test.ts +162 -0
- package/src/daemon/reaction-record.ts +83 -0
- package/src/daemon/shutdown-handlers.ts +0 -2
- package/src/daemon/summarize-boundary.ts +12 -0
- package/src/daemon/video-thumbnail.ts +1 -1
- package/src/home/feed-types.ts +4 -0
- package/src/home/feed-writer.ts +25 -0
- package/src/live-voice/__tests__/live-voice-attach-frame.test.ts +679 -0
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +8 -5
- package/src/live-voice/__tests__/live-voice-photo.test.ts +637 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-inline.test.ts +787 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +347 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +109 -0
- package/src/live-voice/live-voice-photo.ts +420 -65
- package/src/live-voice/live-voice-session.ts +311 -27
- package/src/live-voice/protocol.ts +147 -11
- package/src/messaging/provider-message-metadata.ts +7 -0
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +113 -1
- package/src/messaging/providers/channel-transport.ts +37 -0
- package/src/messaging/providers/discord/api.ts +4 -4
- package/src/messaging/providers/discord/send.test.ts +177 -2
- package/src/messaging/providers/discord/send.ts +197 -10
- package/src/messaging/providers/discord/transport.ts +10 -0
- package/src/messaging/providers/index.ts +42 -0
- package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +129 -3
- package/src/messaging/providers/slack/__tests__/auth.test.ts +35 -0
- package/src/messaging/providers/slack/__tests__/message-content.test.ts +152 -0
- package/src/messaging/providers/slack/adapter.ts +112 -27
- package/src/messaging/providers/slack/api.test.ts +10 -0
- package/src/messaging/providers/slack/auth.ts +32 -6
- package/src/messaging/providers/slack/client.ts +0 -15
- package/src/messaging/providers/slack/message-content.ts +137 -0
- package/src/messaging/providers/slack/send.test.ts +43 -2
- package/src/messaging/providers/slack/send.ts +18 -5
- package/src/messaging/providers/slack/transport.ts +10 -0
- package/src/messaging/providers/slack/types.ts +21 -2
- package/src/messaging/providers/telegram-bot/adapter.ts +5 -1
- package/src/messaging/providers/telegram-bot/send.test.ts +49 -2
- package/src/messaging/providers/telegram-bot/send.ts +41 -1
- package/src/messaging/providers/telegram-bot/transport.ts +10 -0
- package/src/messaging/providers/telegram-bot/withdraw.ts +13 -5
- package/src/messaging/reaction-envelopes.ts +79 -0
- package/src/monitoring/control.ts +2 -2
- package/src/monitoring/page-cache.ts +5 -1
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +360 -0
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +8 -0
- package/src/notifications/adapters/discord.ts +62 -9
- package/src/notifications/conversation-pairing.ts +31 -4
- package/src/notifications/emit-signal.ts +8 -5
- package/src/notifications/guardian-delivery-recorder.ts +47 -6
- package/src/notifications/guardian-feed-projection.ts +492 -0
- package/src/notifications/home-feed-side-effect.ts +59 -11
- package/src/oauth/byo-connection.test.ts +22 -0
- package/src/oauth/byo-connection.ts +11 -5
- package/src/oauth/connection.ts +3 -2
- package/src/oauth/platform-connection.test.ts +35 -0
- package/src/oauth/platform-connection.ts +18 -13
- package/src/oauth/scope-utils.ts +48 -0
- package/src/oauth/seed-providers.ts +12 -0
- package/src/outbound-proxy/certs.ts +5 -4
- package/src/permissions/question-guardian-request.test.ts +7 -1
- package/src/persistence/attachments-store.ts +134 -31
- package/src/persistence/conversation-crud.ts +137 -4
- package/src/persistence/conversation-types.ts +109 -0
- package/src/persistence/db-connection.ts +3 -21
- package/src/persistence/delivery-crud.ts +42 -12
- package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +15 -2
- package/src/persistence/embeddings/embedding-local.ts +7 -112
- package/src/persistence/embeddings/qdrant-manager.ts +4 -1
- package/src/persistence/llm-request-log-store.ts +1 -1
- package/src/persistence/migrations/370-acp-session-history-auth-error-code.ts +36 -0
- package/src/persistence/migrations/371-acp-session-history-auth-error-credential.ts +40 -0
- package/src/persistence/migrations/372-create-acp-refused-credentials.ts +34 -0
- package/src/persistence/migrations/373-acp-auth-marker-index.ts +27 -0
- package/src/persistence/migrations/__tests__/370-acp-session-history-auth-error-code.test.ts +65 -0
- package/src/persistence/migrations/__tests__/371-acp-session-history-auth-error-credential.test.ts +68 -0
- package/src/persistence/migrations/__tests__/372-create-acp-refused-credentials.test.ts +64 -0
- package/src/persistence/migrations/__tests__/373-acp-auth-marker-index.test.ts +82 -0
- package/src/persistence/schema/acp.ts +25 -0
- package/src/persistence/steps.ts +8 -0
- package/src/platform/platform-patch-queue.test.ts +43 -41
- package/src/plugin-api/index.ts +12 -0
- package/src/plugins/defaults/compaction/window-manager.ts +2 -0
- package/src/plugins/defaults/image-recovery/recover.ts +12 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +166 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +3 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +3 -0
- package/src/plugins/defaults/memory/jobs-worker.ts +1 -2
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +53 -9
- package/src/plugins/defaults/memory/src/__tests__/memory-worker-routes.test.ts +4 -7
- package/src/plugins/defaults/memory/src/memory-worker-routes.ts +1 -1
- package/src/plugins/defaults/memory/v1/job-handlers/index-maintenance.ts +15 -1
- package/src/plugins/defaults/memory/worker-control.ts +1 -2
- package/src/providers/content-block-schema.ts +1 -0
- package/src/providers/fireworks/client.ts +9 -1
- package/src/providers/gemini/client.ts +22 -15
- package/src/providers/media-resolve.ts +24 -10
- package/src/providers/model-catalog.ts +91 -0
- package/src/providers/openai/chat-completions-provider.ts +44 -5
- package/src/providers/retry.ts +16 -11
- package/src/providers/types.ts +56 -0
- package/src/routes/__tests__/control.test.ts +4 -3
- package/src/routes/control.ts +1 -1
- package/src/routes/route-host-client.ts +1 -1
- package/src/runtime/AGENTS.md +30 -12
- package/src/runtime/__tests__/agent-wake.test.ts +71 -6
- package/src/runtime/__tests__/guardian-form-registry.test.ts +242 -0
- package/src/runtime/access-request-helper.ts +15 -7
- package/src/runtime/agent-wake.ts +16 -4
- package/src/runtime/channel-approval-types.ts +8 -5
- package/src/runtime/channel-invite-transport.ts +2 -0
- package/src/runtime/channel-invite-transports/discord.ts +27 -0
- package/src/runtime/channel-readiness-service.ts +4 -4
- package/src/runtime/channel-reply-delivery.ts +72 -28
- package/src/runtime/guardian-action-service.ts +9 -5
- package/src/runtime/guardian-form-registry.ts +256 -0
- package/src/runtime/guardian-reply-router.ts +33 -14
- package/src/runtime/invite-instruction-generator.ts +14 -2
- package/src/runtime/migrations/vbundle-metadata-merge.ts +4 -7
- package/src/runtime/no-response.ts +13 -74
- package/src/runtime/routes/__tests__/acp-routes.test.ts +527 -0
- package/src/runtime/routes/__tests__/app-management-shared-apps.test.ts +74 -0
- package/src/runtime/routes/__tests__/contact-form-kinds.test.ts +22 -0
- package/src/runtime/routes/__tests__/contact-record-prompt-route.test.ts +303 -0
- package/src/runtime/routes/__tests__/monitoring-routes.test.ts +3 -4
- package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +6 -0
- package/src/runtime/routes/__tests__/route-host-worker-routes.test.ts +3 -4
- package/src/runtime/routes/__tests__/schedule-worker-routes.test.ts +5 -6
- package/src/runtime/routes/acp-routes-list.test.ts +58 -0
- package/src/runtime/routes/acp-routes.ts +253 -47
- package/src/runtime/routes/app-management-routes.ts +80 -47
- package/src/runtime/routes/approval-routes.ts +9 -0
- package/src/runtime/routes/archive-utils.ts +6 -1
- package/src/runtime/routes/avatar-routes.ts +1 -1
- package/src/runtime/routes/channel-route-shared.ts +18 -20
- package/src/runtime/routes/contact-prompt-routes.ts +271 -105
- package/src/runtime/routes/conversation-routes.ts +41 -32
- package/src/runtime/routes/credential-prompt-routes.ts +10 -1
- package/src/runtime/routes/debug-bash-routes.ts +1 -0
- package/src/runtime/routes/guardian-expiry-sweep.ts +89 -31
- package/src/runtime/routes/guardian-form-routes.ts +115 -0
- package/src/runtime/routes/inbound-message-handler.ts +82 -29
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +6 -0
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +6 -0
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +6 -1
- package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +50 -17
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +97 -94
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/integrations/slack/channel.ts +39 -18
- package/src/runtime/routes/migration-routes.ts +1 -3
- package/src/runtime/routes/monitoring-routes.ts +0 -3
- package/src/runtime/routes/oauth-commands-routes.ts +18 -5
- package/src/runtime/routes/route-host-worker-routes.ts +1 -1
- package/src/runtime/routes/schedule-worker-routes.ts +1 -1
- package/src/runtime/tool-grant-request-helper.ts +4 -0
- package/src/schedule/run-script.ts +1 -0
- package/src/schedule/scheduler.ts +1 -1
- package/src/schedule/worker-control.ts +2 -2
- package/src/security/oauth2.ts +2 -11
- package/src/security/secure-keys.ts +55 -8
- package/src/skills/catalog-install.ts +7 -4
- package/src/skills/clawhub.ts +1 -0
- package/src/skills/inline-command-runner.ts +1 -0
- package/src/stt/__tests__/room-noise-floor.test.ts +108 -0
- package/src/stt/room-noise-floor.ts +140 -0
- package/src/subagent/notify.ts +49 -3
- package/src/tools/__tests__/tool-input-schemas.test.ts +2 -0
- package/src/tools/acp/spawn.ts +2 -2
- package/src/tools/browser/__tests__/browser-status.test.ts +9 -0
- package/src/tools/browser/browser-execution.ts +10 -43
- package/src/tools/browser/browser-manager.ts +1 -0
- package/src/tools/browser/browser-status-constants.ts +6 -5
- package/src/tools/browser/runtime-check.ts +2 -0
- package/src/tools/channel/react-to-message.test.ts +193 -0
- package/src/tools/channel/react-to-message.ts +159 -0
- package/src/tools/credentials/metadata-store.ts +115 -19
- package/src/tools/credentials/store.ts +0 -2
- package/src/tools/filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +24 -0
- package/src/tools/host-terminal/host-shell.ts +11 -38
- package/src/tools/shared/filesystem/file-ops-service.ts +10 -7
- package/src/tools/shared/shell-output.test.ts +104 -0
- package/src/tools/shared/shell-output.ts +112 -39
- package/src/tools/skills/sandbox-runner.ts +1 -0
- package/src/tools/system/request-permission.test.ts +34 -0
- package/src/tools/system/request-permission.ts +29 -5
- package/src/tools/terminal/shell.test.ts +63 -0
- package/src/tools/terminal/shell.ts +12 -44
- package/src/tools/tool-input-schemas.ts +2 -0
- package/src/tools/tool-manifest.ts +2 -0
- package/src/usage/attribution.ts +12 -4
- package/src/util/__tests__/worker-ownership.test.ts +77 -0
- package/src/util/__tests__/worker-process-command.test.ts +246 -1
- package/src/util/bun-runtime.ts +44 -13
- package/src/util/bundled-asset.ts +9 -1
- package/src/util/disk-usage.ts +1 -0
- package/src/util/file-use.ts +4 -1
- package/src/util/guardian-form-timeouts.ts +50 -0
- package/src/util/image-conversion.ts +1 -0
- package/src/util/oauth-request-body.test.ts +59 -0
- package/src/util/oauth-request-body.ts +56 -5
- package/src/util/platform.ts +69 -73
- package/src/util/process-table.ts +16 -5
- package/src/util/read-stdin.ts +5 -0
- package/src/util/spawn.ts +1 -0
- package/src/util/sqlite3-runtime.ts +25 -5
- package/src/util/worker-ownership.ts +193 -0
- package/src/util/worker-process.ts +375 -37
- package/src/workspace/git-service.ts +4 -12
- package/src/workspace/migrations/010-app-dir-rename.ts +1 -0
|
@@ -0,0 +1,679 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ambient camera frames sent mid-call (`attach_frame`).
|
|
3
|
+
*
|
|
4
|
+
* The behaviour under test is the parking rule: a frame dispatches nothing of
|
|
5
|
+
* its own. It waits on the session and rides the next spoken turn's own user
|
|
6
|
+
* message, so the picture and the words about it are one message, and a frame
|
|
7
|
+
* nobody ever speaks over leaves no trace in the conversation.
|
|
8
|
+
*
|
|
9
|
+
* The contrast with `attach_image` is deliberate and is pinned here too: a
|
|
10
|
+
* deliberate snap persists immediately as its own message and parks nothing
|
|
11
|
+
* (`live-voice-attach-image.test.ts`).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { Buffer } from "node:buffer";
|
|
15
|
+
import { describe, expect, mock, test } from "bun:test";
|
|
16
|
+
|
|
17
|
+
import type {
|
|
18
|
+
VoiceTurnCallbacks,
|
|
19
|
+
VoiceTurnOptions,
|
|
20
|
+
} from "../../calls/voice-session-bridge.js";
|
|
21
|
+
import {
|
|
22
|
+
getAttachmentById,
|
|
23
|
+
linkAttachmentToMessage,
|
|
24
|
+
uploadAttachment,
|
|
25
|
+
} from "../../persistence/attachments-store.js";
|
|
26
|
+
import {
|
|
27
|
+
addMessage,
|
|
28
|
+
createConversation,
|
|
29
|
+
} from "../../persistence/conversation-crud.js";
|
|
30
|
+
import { initializeDb } from "../../persistence/db-init.js";
|
|
31
|
+
import type {
|
|
32
|
+
StreamingTranscriber,
|
|
33
|
+
SttStreamServerEvent,
|
|
34
|
+
} from "../../stt/types.js";
|
|
35
|
+
import {
|
|
36
|
+
createLiveVoiceSession,
|
|
37
|
+
type LiveVoiceSession,
|
|
38
|
+
type LiveVoiceTurnStarter,
|
|
39
|
+
} from "../live-voice-session.js";
|
|
40
|
+
import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
|
|
41
|
+
import type {
|
|
42
|
+
LiveVoiceTtsOptions,
|
|
43
|
+
LiveVoiceTtsResult,
|
|
44
|
+
} from "../live-voice-tts.js";
|
|
45
|
+
import {
|
|
46
|
+
createLiveVoiceServerFrameSequencer,
|
|
47
|
+
type LiveVoiceClientStartFrame,
|
|
48
|
+
type LiveVoiceServerFrame,
|
|
49
|
+
validateLiveVoiceClientFrame,
|
|
50
|
+
} from "../protocol.js";
|
|
51
|
+
|
|
52
|
+
await initializeDb();
|
|
53
|
+
|
|
54
|
+
const IMAGE_BASE64 =
|
|
55
|
+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk";
|
|
56
|
+
|
|
57
|
+
const START_FRAME = {
|
|
58
|
+
type: "start",
|
|
59
|
+
conversationId: "conversation-123",
|
|
60
|
+
audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
|
|
61
|
+
turnDetection: "server_vad",
|
|
62
|
+
} as const satisfies LiveVoiceClientStartFrame;
|
|
63
|
+
|
|
64
|
+
/** A real attachment row, so the park-time existence check has one to find. */
|
|
65
|
+
async function uploadFrame(): Promise<string> {
|
|
66
|
+
const attachment = await uploadAttachment(
|
|
67
|
+
"frame.png",
|
|
68
|
+
"image/png",
|
|
69
|
+
IMAGE_BASE64,
|
|
70
|
+
);
|
|
71
|
+
return attachment.id;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** True while the attachment's row is still in the store. */
|
|
75
|
+
function frameStored(attachmentId: string): boolean {
|
|
76
|
+
return getAttachmentById(attachmentId) !== null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function loudPcmChunk(amplitude: number, sampleCount = 240): Uint8Array {
|
|
80
|
+
const buffer = Buffer.alloc(sampleCount * 2);
|
|
81
|
+
for (let index = 0; index < sampleCount; index += 1) {
|
|
82
|
+
buffer.writeInt16LE(amplitude, index * 2);
|
|
83
|
+
}
|
|
84
|
+
return new Uint8Array(buffer);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** One fresh transcriber per utterance; `stop()` flushes its transcript. */
|
|
88
|
+
class FakeStreamingTranscriber implements StreamingTranscriber {
|
|
89
|
+
readonly providerId = "deepgram" as const;
|
|
90
|
+
readonly boundaryId = "daemon-streaming" as const;
|
|
91
|
+
private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
|
|
92
|
+
|
|
93
|
+
constructor(private readonly transcript: string) {}
|
|
94
|
+
|
|
95
|
+
async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
|
|
96
|
+
this.onEvent = onEvent;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
sendAudio(): void {
|
|
100
|
+
this.onEvent?.({ type: "partial", text: "wha" });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
stop(): void {
|
|
104
|
+
this.onEvent?.({ type: "final", text: this.transcript });
|
|
105
|
+
this.onEvent?.({ type: "closed" });
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function makeMessageComplete(): Parameters<
|
|
110
|
+
NonNullable<VoiceTurnCallbacks["message_complete"]>
|
|
111
|
+
>[0] {
|
|
112
|
+
return {
|
|
113
|
+
type: "message_complete",
|
|
114
|
+
conversationId: "conversation-123",
|
|
115
|
+
messageId: "assistant-message-123",
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function makeTtsResult(text: string): LiveVoiceTtsResult {
|
|
120
|
+
return {
|
|
121
|
+
provider: "fish-audio",
|
|
122
|
+
contentType: "audio/pcm",
|
|
123
|
+
sampleRate: 24_000,
|
|
124
|
+
chunks: 1,
|
|
125
|
+
bytes: Buffer.byteLength(text),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function createSessionHarness(startVoiceTurn: LiveVoiceTurnStarter) {
|
|
130
|
+
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
131
|
+
const frames: LiveVoiceServerFrame[] = [];
|
|
132
|
+
const context: LiveVoiceSessionFactoryContext = {
|
|
133
|
+
sessionId: "session-123",
|
|
134
|
+
startFrame: START_FRAME,
|
|
135
|
+
sendFrame: mock(async (payload) => {
|
|
136
|
+
const frame = sequencer.next(payload);
|
|
137
|
+
frames.push(frame);
|
|
138
|
+
return frame;
|
|
139
|
+
}),
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
let utterances = 0;
|
|
143
|
+
const session = createLiveVoiceSession(context, {
|
|
144
|
+
// Credential-free harness: every leg is injected, so skip the preflight.
|
|
145
|
+
resolveCredentialReadiness: null,
|
|
146
|
+
// One discrete mic chunk per utterance; keep the adaptive playback
|
|
147
|
+
// classifier out of the cycle timing.
|
|
148
|
+
echoBargeInMargin: 1,
|
|
149
|
+
resolveTranscriber: mock(async () => {
|
|
150
|
+
utterances += 1;
|
|
151
|
+
return new FakeStreamingTranscriber(`utterance ${utterances}`);
|
|
152
|
+
}),
|
|
153
|
+
startVoiceTurn,
|
|
154
|
+
streamTtsAudio: mock(async (options: LiveVoiceTtsOptions) =>
|
|
155
|
+
makeTtsResult(options.text),
|
|
156
|
+
),
|
|
157
|
+
createTurnId: () => `live-turn-${utterances}`,
|
|
158
|
+
emitMetrics: false,
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
return { frames, session };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async function waitFor(
|
|
165
|
+
predicate: () => boolean,
|
|
166
|
+
message: string,
|
|
167
|
+
): Promise<void> {
|
|
168
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
169
|
+
if (predicate()) {
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
173
|
+
}
|
|
174
|
+
throw new Error(message);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Drive one utterance through to its dispatched assistant turn. */
|
|
178
|
+
async function speakAndRelease(
|
|
179
|
+
session: LiveVoiceSession,
|
|
180
|
+
seen: () => number,
|
|
181
|
+
amplitude: number,
|
|
182
|
+
): Promise<void> {
|
|
183
|
+
const before = seen();
|
|
184
|
+
await session.handleBinaryAudio(loudPcmChunk(amplitude));
|
|
185
|
+
await session.handleClientFrame({ type: "ptt_release" });
|
|
186
|
+
await waitFor(
|
|
187
|
+
() => seen() > before,
|
|
188
|
+
"Timed out waiting for the live-voice turn to dispatch",
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function parkedFrameId(session: LiveVoiceSession): string | null {
|
|
193
|
+
return (session as unknown as { pendingTurnAttachmentId: string | null })
|
|
194
|
+
.pendingTurnAttachmentId;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
describe("live-voice attach_frame frame", () => {
|
|
198
|
+
test("accepts a well-formed frame", () => {
|
|
199
|
+
const result = validateLiveVoiceClientFrame({
|
|
200
|
+
type: "attach_frame",
|
|
201
|
+
attachmentId: "att-1",
|
|
202
|
+
});
|
|
203
|
+
expect(result).toEqual({
|
|
204
|
+
ok: true,
|
|
205
|
+
frame: { type: "attach_frame", attachmentId: "att-1" },
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
test("accepts a null attachmentId, which unparks", () => {
|
|
210
|
+
const result = validateLiveVoiceClientFrame({
|
|
211
|
+
type: "attach_frame",
|
|
212
|
+
attachmentId: null,
|
|
213
|
+
});
|
|
214
|
+
expect(result).toEqual({
|
|
215
|
+
ok: true,
|
|
216
|
+
frame: { type: "attach_frame", attachmentId: null },
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
test("rejects a missing attachmentId, naming the frame", () => {
|
|
221
|
+
// A missing field is not an unpark. A client that means to clear the slot
|
|
222
|
+
// says so with an explicit null; an absent field is a malformed frame.
|
|
223
|
+
const result = validateLiveVoiceClientFrame({ type: "attach_frame" });
|
|
224
|
+
expect(result.ok).toBe(false);
|
|
225
|
+
if (!result.ok) {
|
|
226
|
+
expect(result.error.code).toBe("missing_required_field");
|
|
227
|
+
expect(result.error.field).toBe("attachmentId");
|
|
228
|
+
expect(result.error.frameType).toBe("attach_frame");
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
test("rejects an empty attachmentId", () => {
|
|
233
|
+
const result = validateLiveVoiceClientFrame({
|
|
234
|
+
type: "attach_frame",
|
|
235
|
+
attachmentId: "",
|
|
236
|
+
});
|
|
237
|
+
expect(result.ok).toBe(false);
|
|
238
|
+
if (!result.ok) {
|
|
239
|
+
expect(result.error.code).toBe("invalid_field");
|
|
240
|
+
expect(result.error.frameType).toBe("attach_frame");
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
describe("live-voice camera frames parked for the next turn", () => {
|
|
246
|
+
test("a parked frame rides the next turn's user message", async () => {
|
|
247
|
+
const attachmentId = await uploadFrame();
|
|
248
|
+
const turns: VoiceTurnOptions[] = [];
|
|
249
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
250
|
+
async (options: VoiceTurnOptions) => {
|
|
251
|
+
turns.push(options);
|
|
252
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
253
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
254
|
+
},
|
|
255
|
+
);
|
|
256
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
257
|
+
await session.start();
|
|
258
|
+
|
|
259
|
+
await session.handleClientFrame({ type: "attach_frame", attachmentId });
|
|
260
|
+
// Nothing is dispatched for the frame itself; it waits for words.
|
|
261
|
+
expect(turns).toHaveLength(0);
|
|
262
|
+
expect(parkedFrameId(session)).toBe(attachmentId);
|
|
263
|
+
|
|
264
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
265
|
+
|
|
266
|
+
expect(turns[0]?.content).toBe("utterance 1");
|
|
267
|
+
expect(turns[0]?.attachments).toEqual([attachmentId]);
|
|
268
|
+
// Drained by the turn that took it, so it cannot ride a second one.
|
|
269
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
270
|
+
|
|
271
|
+
await session.close("websocket_close");
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
test("the newest frame replaces the one before it", async () => {
|
|
275
|
+
const stale = await uploadFrame();
|
|
276
|
+
const current = await uploadFrame();
|
|
277
|
+
const turns: VoiceTurnOptions[] = [];
|
|
278
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
279
|
+
async (options: VoiceTurnOptions) => {
|
|
280
|
+
turns.push(options);
|
|
281
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
282
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
283
|
+
},
|
|
284
|
+
);
|
|
285
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
286
|
+
await session.start();
|
|
287
|
+
|
|
288
|
+
await session.handleClientFrame({
|
|
289
|
+
type: "attach_frame",
|
|
290
|
+
attachmentId: stale,
|
|
291
|
+
});
|
|
292
|
+
await session.handleClientFrame({
|
|
293
|
+
type: "attach_frame",
|
|
294
|
+
attachmentId: current,
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
// A client shooting the camera on a timer parks one of these every few
|
|
298
|
+
// seconds. Nothing else ever collects them, so the displaced one is given
|
|
299
|
+
// up here or its row and bytes stay for good.
|
|
300
|
+
expect(frameStored(stale)).toBe(false);
|
|
301
|
+
expect(frameStored(current)).toBe(true);
|
|
302
|
+
|
|
303
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
304
|
+
|
|
305
|
+
expect(turns[0]?.attachments).toEqual([current]);
|
|
306
|
+
|
|
307
|
+
await session.close("websocket_close");
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
test("displacing a frame that already rode a message leaves it alone", async () => {
|
|
311
|
+
// The reclaim is link-aware, which is what makes it safe to run on an id
|
|
312
|
+
// the client may have sent before: a frame that reached a message is that
|
|
313
|
+
// message's, and re-sending its id must not delete it out from under the
|
|
314
|
+
// transcript.
|
|
315
|
+
const conversation = createConversation("Live voice frame reuse");
|
|
316
|
+
const message = await addMessage(conversation.id, "user", "what is this");
|
|
317
|
+
const sent = await uploadFrame();
|
|
318
|
+
linkAttachmentToMessage(message.id, sent, 0);
|
|
319
|
+
const replacement = await uploadFrame();
|
|
320
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
321
|
+
async (options: VoiceTurnOptions) => {
|
|
322
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
323
|
+
return { turnId: "bridge-turn-1", abort: mock() };
|
|
324
|
+
},
|
|
325
|
+
);
|
|
326
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
327
|
+
await session.start();
|
|
328
|
+
|
|
329
|
+
await session.handleClientFrame({
|
|
330
|
+
type: "attach_frame",
|
|
331
|
+
attachmentId: sent,
|
|
332
|
+
});
|
|
333
|
+
await session.handleClientFrame({
|
|
334
|
+
type: "attach_frame",
|
|
335
|
+
attachmentId: replacement,
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
expect(frameStored(sent)).toBe(true);
|
|
339
|
+
expect(parkedFrameId(session)).toBe(replacement);
|
|
340
|
+
|
|
341
|
+
await session.close("websocket_close");
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
test("a frame that arrives mid-turn rides the turn after it", async () => {
|
|
345
|
+
const first = await uploadFrame();
|
|
346
|
+
const second = await uploadFrame();
|
|
347
|
+
const turns: VoiceTurnOptions[] = [];
|
|
348
|
+
const sessionRef: { current: LiveVoiceSession | null } = { current: null };
|
|
349
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
350
|
+
async (options: VoiceTurnOptions) => {
|
|
351
|
+
turns.push(options);
|
|
352
|
+
if (turns.length === 1) {
|
|
353
|
+
// The camera keeps shooting while the assistant answers. This frame
|
|
354
|
+
// belongs to whatever the user says next, not to the turn in flight.
|
|
355
|
+
await sessionRef.current?.handleClientFrame({
|
|
356
|
+
type: "attach_frame",
|
|
357
|
+
attachmentId: second,
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
361
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
362
|
+
},
|
|
363
|
+
);
|
|
364
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
365
|
+
sessionRef.current = session;
|
|
366
|
+
await session.start();
|
|
367
|
+
|
|
368
|
+
await session.handleClientFrame({
|
|
369
|
+
type: "attach_frame",
|
|
370
|
+
attachmentId: first,
|
|
371
|
+
});
|
|
372
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
373
|
+
await speakAndRelease(session, () => turns.length, 9_000);
|
|
374
|
+
|
|
375
|
+
expect(turns[0]?.attachments).toEqual([first]);
|
|
376
|
+
expect(turns[1]?.attachments).toEqual([second]);
|
|
377
|
+
|
|
378
|
+
await session.close("websocket_close");
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
test("unparking clears the slot and gives the frame up", async () => {
|
|
382
|
+
// The client's viewfinder closed. Nothing should still be staged for a
|
|
383
|
+
// turn that would show a view the user can no longer see.
|
|
384
|
+
const attachmentId = await uploadFrame();
|
|
385
|
+
const turns: VoiceTurnOptions[] = [];
|
|
386
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
387
|
+
async (options: VoiceTurnOptions) => {
|
|
388
|
+
turns.push(options);
|
|
389
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
390
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
391
|
+
},
|
|
392
|
+
);
|
|
393
|
+
const { frames, session } = createSessionHarness(startVoiceTurn);
|
|
394
|
+
await session.start();
|
|
395
|
+
|
|
396
|
+
await session.handleClientFrame({ type: "attach_frame", attachmentId });
|
|
397
|
+
expect(parkedFrameId(session)).toBe(attachmentId);
|
|
398
|
+
const before = frames.length;
|
|
399
|
+
|
|
400
|
+
await session.handleClientFrame({
|
|
401
|
+
type: "attach_frame",
|
|
402
|
+
attachmentId: null,
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
406
|
+
expect(frameStored(attachmentId)).toBe(false);
|
|
407
|
+
// Nothing is answered for an unpark: it is the client tidying up after
|
|
408
|
+
// itself, not a request that can fail.
|
|
409
|
+
expect(frames.slice(before)).toEqual([]);
|
|
410
|
+
|
|
411
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
412
|
+
expect(turns[0]?.attachments).toBeUndefined();
|
|
413
|
+
|
|
414
|
+
await session.close("websocket_close");
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test("unparking an empty slot does nothing", async () => {
|
|
418
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
419
|
+
async (options: VoiceTurnOptions) => {
|
|
420
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
421
|
+
return { turnId: "bridge-turn-1", abort: mock() };
|
|
422
|
+
},
|
|
423
|
+
);
|
|
424
|
+
const { frames, session } = createSessionHarness(startVoiceTurn);
|
|
425
|
+
await session.start();
|
|
426
|
+
const before = frames.length;
|
|
427
|
+
|
|
428
|
+
await session.handleClientFrame({
|
|
429
|
+
type: "attach_frame",
|
|
430
|
+
attachmentId: null,
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
434
|
+
expect(frames.slice(before)).toEqual([]);
|
|
435
|
+
|
|
436
|
+
await session.close("websocket_close");
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
test("unparking leaves a frame that already rode a message alone", async () => {
|
|
440
|
+
// Same link-awareness the displacement path relies on: an unpark arriving
|
|
441
|
+
// after the turn drained the slot finds it empty, and the frame the
|
|
442
|
+
// transcript is showing is not this call's to collect.
|
|
443
|
+
const conversation = createConversation("Live voice frame unpark");
|
|
444
|
+
const message = await addMessage(conversation.id, "user", "what is this");
|
|
445
|
+
const sent = await uploadFrame();
|
|
446
|
+
linkAttachmentToMessage(message.id, sent, 0);
|
|
447
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
448
|
+
async (options: VoiceTurnOptions) => {
|
|
449
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
450
|
+
return { turnId: "bridge-turn-1", abort: mock() };
|
|
451
|
+
},
|
|
452
|
+
);
|
|
453
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
454
|
+
await session.start();
|
|
455
|
+
|
|
456
|
+
await session.handleClientFrame({
|
|
457
|
+
type: "attach_frame",
|
|
458
|
+
attachmentId: sent,
|
|
459
|
+
});
|
|
460
|
+
await session.handleClientFrame({
|
|
461
|
+
type: "attach_frame",
|
|
462
|
+
attachmentId: null,
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
466
|
+
expect(frameStored(sent)).toBe(true);
|
|
467
|
+
|
|
468
|
+
await session.close("websocket_close");
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
test("ending the session drops the parked frame", async () => {
|
|
472
|
+
const attachmentId = await uploadFrame();
|
|
473
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
474
|
+
async (options: VoiceTurnOptions) => {
|
|
475
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
476
|
+
return { turnId: "bridge-turn-1", abort: mock() };
|
|
477
|
+
},
|
|
478
|
+
);
|
|
479
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
480
|
+
await session.start();
|
|
481
|
+
|
|
482
|
+
await session.handleClientFrame({ type: "attach_frame", attachmentId });
|
|
483
|
+
expect(parkedFrameId(session)).toBe(attachmentId);
|
|
484
|
+
|
|
485
|
+
await session.close("websocket_close");
|
|
486
|
+
|
|
487
|
+
// A frame is only good for the call it was shot on, so the staged row
|
|
488
|
+
// goes with the call rather than outliving it.
|
|
489
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
490
|
+
expect(frameStored(attachmentId)).toBe(false);
|
|
491
|
+
});
|
|
492
|
+
|
|
493
|
+
test("a rolled-back turn hands the frame back for the replay", async () => {
|
|
494
|
+
// A turn that never reached the bridge (and a speculative one the hold
|
|
495
|
+
// verdict unwinds) must not take the frame down with it: the utterance is
|
|
496
|
+
// about to be sent again, and the user is still holding the thing up.
|
|
497
|
+
// The other half of this, that the rollback leaves the attachment itself
|
|
498
|
+
// alive, is `voice-session-bridge.test.ts`'s discard suite.
|
|
499
|
+
const attachmentId = await uploadFrame();
|
|
500
|
+
const turns: VoiceTurnOptions[] = [];
|
|
501
|
+
let dispatches = 0;
|
|
502
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
503
|
+
async (options: VoiceTurnOptions) => {
|
|
504
|
+
dispatches += 1;
|
|
505
|
+
if (dispatches === 1) {
|
|
506
|
+
throw new Error("bridge unavailable");
|
|
507
|
+
}
|
|
508
|
+
turns.push(options);
|
|
509
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
510
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
511
|
+
},
|
|
512
|
+
);
|
|
513
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
514
|
+
await session.start();
|
|
515
|
+
|
|
516
|
+
await session.handleClientFrame({ type: "attach_frame", attachmentId });
|
|
517
|
+
await speakAndRelease(session, () => dispatches, 8_000);
|
|
518
|
+
|
|
519
|
+
// Claimed by the failed turn, then handed straight back.
|
|
520
|
+
expect(parkedFrameId(session)).toBe(attachmentId);
|
|
521
|
+
|
|
522
|
+
await speakAndRelease(session, () => dispatches, 9_000);
|
|
523
|
+
|
|
524
|
+
expect(turns[0]?.attachments).toEqual([attachmentId]);
|
|
525
|
+
|
|
526
|
+
await session.close("websocket_close");
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
test("a rollback keeps the newer frame and gives up the one it replaced", async () => {
|
|
530
|
+
// The camera kept shooting while the doomed turn was in flight, so the
|
|
531
|
+
// frame coming back off it is already out of date. Latest still wins, and
|
|
532
|
+
// the one that loses is nobody's to send: it goes.
|
|
533
|
+
const claimed = await uploadFrame();
|
|
534
|
+
const newer = await uploadFrame();
|
|
535
|
+
const turns: VoiceTurnOptions[] = [];
|
|
536
|
+
let dispatches = 0;
|
|
537
|
+
const sessionRef: { current: LiveVoiceSession | null } = { current: null };
|
|
538
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
539
|
+
async (options: VoiceTurnOptions) => {
|
|
540
|
+
dispatches += 1;
|
|
541
|
+
if (dispatches === 1) {
|
|
542
|
+
await sessionRef.current?.handleClientFrame({
|
|
543
|
+
type: "attach_frame",
|
|
544
|
+
attachmentId: newer,
|
|
545
|
+
});
|
|
546
|
+
throw new Error("bridge unavailable");
|
|
547
|
+
}
|
|
548
|
+
turns.push(options);
|
|
549
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
550
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
551
|
+
},
|
|
552
|
+
);
|
|
553
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
554
|
+
sessionRef.current = session;
|
|
555
|
+
await session.start();
|
|
556
|
+
|
|
557
|
+
await session.handleClientFrame({
|
|
558
|
+
type: "attach_frame",
|
|
559
|
+
attachmentId: claimed,
|
|
560
|
+
});
|
|
561
|
+
await speakAndRelease(session, () => dispatches, 8_000);
|
|
562
|
+
|
|
563
|
+
expect(parkedFrameId(session)).toBe(newer);
|
|
564
|
+
expect(frameStored(claimed)).toBe(false);
|
|
565
|
+
|
|
566
|
+
await speakAndRelease(session, () => dispatches, 9_000);
|
|
567
|
+
|
|
568
|
+
expect(turns[0]?.attachments).toEqual([newer]);
|
|
569
|
+
|
|
570
|
+
await session.close("websocket_close");
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
test("a retransmitted id is the same frame, not a newer one", async () => {
|
|
574
|
+
// A client that re-sends the id it already sent, while the turn holding it
|
|
575
|
+
// is in flight, has parked the SAME frame. Reading that as a displacement
|
|
576
|
+
// would collect the very frame the slot is pointing at, and the replay
|
|
577
|
+
// would speak without the picture with nothing said.
|
|
578
|
+
const attachmentId = await uploadFrame();
|
|
579
|
+
const turns: VoiceTurnOptions[] = [];
|
|
580
|
+
let dispatches = 0;
|
|
581
|
+
const sessionRef: { current: LiveVoiceSession | null } = { current: null };
|
|
582
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
583
|
+
async (options: VoiceTurnOptions) => {
|
|
584
|
+
dispatches += 1;
|
|
585
|
+
if (dispatches === 1) {
|
|
586
|
+
await sessionRef.current?.handleClientFrame({
|
|
587
|
+
type: "attach_frame",
|
|
588
|
+
attachmentId,
|
|
589
|
+
});
|
|
590
|
+
throw new Error("bridge unavailable");
|
|
591
|
+
}
|
|
592
|
+
turns.push(options);
|
|
593
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
594
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
595
|
+
},
|
|
596
|
+
);
|
|
597
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
598
|
+
sessionRef.current = session;
|
|
599
|
+
await session.start();
|
|
600
|
+
|
|
601
|
+
await session.handleClientFrame({ type: "attach_frame", attachmentId });
|
|
602
|
+
await speakAndRelease(session, () => dispatches, 8_000);
|
|
603
|
+
|
|
604
|
+
expect(parkedFrameId(session)).toBe(attachmentId);
|
|
605
|
+
expect(frameStored(attachmentId)).toBe(true);
|
|
606
|
+
|
|
607
|
+
await speakAndRelease(session, () => dispatches, 9_000);
|
|
608
|
+
|
|
609
|
+
expect(turns[0]?.attachments).toEqual([attachmentId]);
|
|
610
|
+
|
|
611
|
+
await session.close("websocket_close");
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
test("an unknown attachment id is refused and nothing is parked", async () => {
|
|
615
|
+
const turns: VoiceTurnOptions[] = [];
|
|
616
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
617
|
+
async (options: VoiceTurnOptions) => {
|
|
618
|
+
turns.push(options);
|
|
619
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
620
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
621
|
+
},
|
|
622
|
+
);
|
|
623
|
+
const { frames, session } = createSessionHarness(startVoiceTurn);
|
|
624
|
+
await session.start();
|
|
625
|
+
const before = frames.length;
|
|
626
|
+
|
|
627
|
+
await session.handleClientFrame({
|
|
628
|
+
type: "attach_frame",
|
|
629
|
+
attachmentId: "att-missing",
|
|
630
|
+
});
|
|
631
|
+
await waitFor(
|
|
632
|
+
() => frames.length > before,
|
|
633
|
+
"Timed out waiting for the rejected camera frame's error",
|
|
634
|
+
);
|
|
635
|
+
|
|
636
|
+
// Attributed to the frame it is about, so the client can retract the
|
|
637
|
+
// preview it already showed instead of filing this with the transient
|
|
638
|
+
// transcriber and TTS blips that share `recoverable`.
|
|
639
|
+
expect(
|
|
640
|
+
frames.slice(before).find((frame) => frame.type === "error"),
|
|
641
|
+
).toMatchObject({
|
|
642
|
+
type: "error",
|
|
643
|
+
frameType: "attach_frame",
|
|
644
|
+
recoverable: true,
|
|
645
|
+
});
|
|
646
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
647
|
+
|
|
648
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
649
|
+
expect(turns[0]?.attachments).toBeUndefined();
|
|
650
|
+
|
|
651
|
+
await session.close("websocket_close");
|
|
652
|
+
});
|
|
653
|
+
|
|
654
|
+
test("attach_image parks nothing", async () => {
|
|
655
|
+
// A deliberate snap persists as its own message; the spoken turn that
|
|
656
|
+
// follows carries no attachments and reaches it through history.
|
|
657
|
+
const turns: VoiceTurnOptions[] = [];
|
|
658
|
+
const startVoiceTurn: LiveVoiceTurnStarter = mock(
|
|
659
|
+
async (options: VoiceTurnOptions) => {
|
|
660
|
+
turns.push(options);
|
|
661
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
662
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
663
|
+
},
|
|
664
|
+
);
|
|
665
|
+
const { session } = createSessionHarness(startVoiceTurn);
|
|
666
|
+
await session.start();
|
|
667
|
+
|
|
668
|
+
await session.handleClientFrame({
|
|
669
|
+
type: "attach_image",
|
|
670
|
+
attachmentId: "att-missing",
|
|
671
|
+
});
|
|
672
|
+
expect(parkedFrameId(session)).toBeNull();
|
|
673
|
+
|
|
674
|
+
await speakAndRelease(session, () => turns.length, 8_000);
|
|
675
|
+
expect(turns[0]?.attachments).toBeUndefined();
|
|
676
|
+
|
|
677
|
+
await session.close("websocket_close");
|
|
678
|
+
});
|
|
679
|
+
});
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Photos taken mid-call (the voice room's camera).
|
|
3
3
|
*
|
|
4
|
-
* The behaviour under test is
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
4
|
+
* The behaviour under test is that an `attach_image` frame stands alone: it
|
|
5
|
+
* persists the photo as its own user message the moment it arrives and
|
|
6
|
+
* dispatches no turn, so the spoken turn that follows carries no attachments
|
|
7
|
+
* of its own and reaches the picture through conversation history. That is
|
|
8
|
+
* what makes shutter-then-speak and speak-then-shutter answer the same way.
|
|
9
|
+
*
|
|
10
|
+
* The parking rule belongs to `attach_frame` instead
|
|
11
|
+
* (`live-voice-attach-frame.test.ts`), which carries ambient camera frames.
|
|
9
12
|
*/
|
|
10
13
|
|
|
11
14
|
import { describe, expect, mock, test } from "bun:test";
|