@vellumai/assistant 0.11.1 → 0.11.2-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -3
- package/README.md +1 -1
- package/eslint.config.mjs +28 -8
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/slack-text/src/index.test.ts +48 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +25 -6
- package/openapi.yaml +292 -5
- package/package.json +1 -1
- package/src/__tests__/agent-loop-resume-interrupted.test.ts +223 -0
- package/src/__tests__/byok-default-profile-ensure.test.ts +43 -0
- package/src/__tests__/cli-logger-boundary-guard.test.ts +87 -0
- package/src/__tests__/compaction-events.test.ts +139 -2
- package/src/__tests__/config-loader-backfill.test.ts +35 -3
- package/src/__tests__/config-schema.test.ts +1 -0
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +30 -0
- package/src/__tests__/conversation-agent-loop.test.ts +180 -7
- package/src/__tests__/conversation-error.test.ts +211 -4
- package/src/__tests__/conversation-options-turn-scoped-transport.test.ts +64 -0
- package/src/__tests__/conversation-queue.test.ts +126 -0
- package/src/__tests__/conversation-retry-route.test.ts +46 -7
- package/src/__tests__/conversation-slash.test.ts +4 -2
- package/src/__tests__/conversation-summarize-route.test.ts +7 -1
- package/src/__tests__/conversation-summarize-up-to.test.ts +71 -5
- package/src/__tests__/document-create-dedupe.test.ts +2 -1
- package/src/__tests__/document-find-replace.test.ts +2 -1
- package/src/__tests__/document-tool-security.test.ts +2 -1
- package/src/__tests__/document-update-default-surface.test.ts +2 -1
- package/src/__tests__/document-workspace-file.test.ts +467 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +290 -49
- package/src/__tests__/guardian-card-withdrawal.test.ts +91 -3
- package/src/__tests__/http-user-message-parity.test.ts +66 -0
- package/src/__tests__/list-messages-attachments.test.ts +87 -0
- package/src/__tests__/list-messages-provider-error.test.ts +143 -0
- package/src/__tests__/list-messages-system-card.test.ts +100 -0
- package/src/__tests__/llm-resolver.test.ts +19 -9
- package/src/__tests__/managed-profile-guard.test.ts +23 -0
- package/src/__tests__/notification-platform-adapter.test.ts +130 -2
- package/src/__tests__/notification-telegram-adapter.test.ts +6 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +45 -0
- package/src/__tests__/plugin-api-model-profiles.test.ts +10 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +3 -0
- package/src/__tests__/provider-error-scenarios.test.ts +140 -0
- package/src/__tests__/provider-send-message-override-profile.test.ts +95 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +66 -3
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +209 -0
- package/src/__tests__/skills.test.ts +27 -0
- package/src/__tests__/slack-channels-routes.test.ts +0 -2
- package/src/__tests__/slack-share-routes.test.ts +0 -3
- package/src/__tests__/slack-users-routes.test.ts +0 -2
- package/src/__tests__/subagent-tools.test.ts +11 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +58 -0
- package/src/__tests__/tool-result-spool.test.ts +5 -4
- package/src/__tests__/turn-boundary-resolution.test.ts +53 -0
- package/src/__tests__/turn-events-store.test.ts +26 -0
- package/src/__tests__/ui-visual-surface.test.ts +695 -0
- package/src/__tests__/unified-turn-context-visible-app.test.ts +99 -0
- package/src/__tests__/visible-app-context.test.ts +189 -0
- package/src/__tests__/workspace-git-service.test.ts +173 -33
- package/src/__tests__/workspace-migration-137-repair-retired-fireworks-minimax-model-id.test.ts +157 -0
- package/src/__tests__/workspace-migration-138-backfill-home-feed-titles.test.ts +373 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +137 -0
- package/src/agent/loop.ts +36 -7
- package/src/api/events/context-window-usage.ts +31 -0
- package/src/api/events/notification-intent.ts +8 -0
- package/src/api/events/ui-surface-pending.ts +35 -0
- package/src/api/index.ts +14 -0
- package/src/api/responses/conversation-message.ts +25 -4
- package/src/api/surfaces.ts +90 -1
- package/src/approvals/guardian-card-withdrawal.ts +66 -31
- package/src/approvals/guardian-decision-primitive.ts +4 -0
- package/src/calls/__tests__/call-setup-router.test.ts +156 -26
- package/src/calls/__tests__/voice-session-bridge.test.ts +201 -12
- package/src/calls/call-setup-router.ts +80 -36
- package/src/calls/voice-session-bridge.ts +173 -26
- package/src/cli/commands/inference.help.ts +3 -3
- package/src/cli/commands/notifications.help.ts +3 -2
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +42 -128
- package/src/cli/commands/platform/__tests__/credits.test.ts +9 -78
- package/src/cli/commands/platform/__tests__/helpers.ts +90 -0
- package/src/cli/commands/platform/__tests__/invoices.test.ts +238 -0
- package/src/cli/commands/platform/__tests__/plans.test.ts +9 -88
- package/src/cli/commands/platform/__tests__/status.test.ts +12 -87
- package/src/cli/commands/platform/__tests__/subscription.test.ts +9 -86
- package/src/cli/commands/platform/index.help.ts +92 -0
- package/src/cli/commands/platform/index.ts +7 -0
- package/src/cli/commands/platform/invoices.ts +132 -0
- package/src/cli/commands/usage.help.ts +1 -1
- package/src/cli/lib/list-installed-plugins.ts +2 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +7 -10
- package/src/config/__tests__/deployment-context-defaults.test.ts +27 -5
- package/src/config/assistant-feature-flags.ts +7 -2
- package/src/config/bundled-skills/app-builder/SKILL.md +3 -2
- package/src/config/bundled-skills/subagent/SKILL.md +3 -1
- package/src/config/bundled-skills/subagent/TOOLS.json +3 -3
- package/src/config/bundled-skills/visualize/SKILL.md +163 -0
- package/src/config/call-site-defaults.ts +3 -2
- package/src/config/default-profile-catalog.ts +75 -70
- package/src/config/default-profile-names.ts +13 -28
- package/src/config/env-registry.ts +1 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/llm-resolver.ts +21 -5
- package/src/config/loader.ts +18 -11
- package/src/config/schemas/llm.ts +1 -9
- package/src/config/schemas/memory-retrospective.ts +9 -0
- package/src/config/schemas/monitoring.ts +28 -2
- package/src/config/schemas/workspace-git.ts +15 -0
- package/src/config/seed-inference-profiles.ts +19 -5
- package/src/context/post-turn-tool-result-truncation.ts +2 -2
- package/src/conversations/__tests__/message-consolidation.test.ts +54 -0
- package/src/conversations/message-consolidation.ts +17 -15
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +182 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +181 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +90 -7
- package/src/daemon/conversation-agent-loop.ts +107 -22
- package/src/daemon/conversation-error.ts +169 -48
- package/src/daemon/conversation-messaging.ts +87 -4
- package/src/daemon/conversation-process.ts +57 -45
- package/src/daemon/conversation-runtime-assembly.ts +57 -0
- package/src/daemon/conversation-store.ts +36 -1
- package/src/daemon/conversation-surfaces.ts +68 -6
- package/src/daemon/conversation-turn-finalize.ts +76 -18
- package/src/daemon/conversation.ts +122 -27
- package/src/daemon/lifecycle.ts +9 -0
- package/src/daemon/message-types/conversations.ts +8 -0
- package/src/daemon/message-types/surfaces.ts +3 -0
- package/src/documents/document-store.ts +247 -10
- package/src/home/__tests__/feed-types.test.ts +43 -8
- package/src/home/__tests__/feed-writer.test.ts +59 -0
- package/src/home/feed-types.ts +34 -7
- package/src/home/feed-writer.ts +15 -6
- package/src/live-voice/__tests__/activity-label.test.ts +68 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +86 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +185 -25
- package/src/live-voice/__tests__/live-voice-events.test.ts +4 -4
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +8 -4
- package/src/live-voice/__tests__/live-voice-vad.test.ts +9 -1
- package/src/live-voice/activity-label.ts +141 -0
- package/src/live-voice/live-activity-reporter.ts +35 -6
- package/src/live-voice/live-voice-session.ts +171 -41
- package/src/live-voice/protocol.ts +23 -0
- package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +84 -2
- package/src/messaging/providers/slack/adapter.ts +86 -19
- package/src/messaging/providers/slack/api.test.ts +85 -1
- package/src/messaging/providers/slack/api.ts +121 -288
- package/src/messaging/providers/slack/client.ts +20 -251
- package/src/messaging/providers/slack/send.test.ts +4 -9
- package/src/messaging/providers/slack/send.ts +1 -1
- package/src/messaging/providers/slack/types.ts +5 -0
- package/src/messaging/providers/slack/web-api-transport.test.ts +181 -0
- package/src/messaging/providers/slack/web-api-transport.ts +367 -0
- package/src/messaging/providers/slack/withdraw.ts +6 -14
- package/src/messaging/providers/telegram-bot/send.test.ts +31 -1
- package/src/messaging/providers/telegram-bot/send.ts +23 -2
- package/src/messaging/providers/telegram-bot/withdraw.test.ts +152 -0
- package/src/messaging/providers/telegram-bot/withdraw.ts +166 -0
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +18 -4
- package/src/monitoring/__tests__/file-descriptors.test.ts +144 -0
- package/src/monitoring/file-descriptors.ts +262 -0
- package/src/monitoring/process-memory.ts +4 -11
- package/src/monitoring/resource-sampler.ts +5 -0
- package/src/monitoring/worker.ts +12 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +740 -0
- package/src/notifications/__tests__/broadcaster.test.ts +347 -7
- package/src/notifications/__tests__/copy-composer.test.ts +53 -3
- package/src/notifications/__tests__/decision-engine.test.ts +349 -29
- package/src/notifications/__tests__/deterministic-checks.test.ts +21 -0
- package/src/notifications/__tests__/edit-notification.test.ts +330 -0
- package/src/notifications/__tests__/guardian-delivery-recorder.test.ts +71 -0
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +170 -12
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/adapters/platform.ts +90 -14
- package/src/notifications/adapters/telegram.ts +6 -4
- package/src/notifications/assistant-reply-producer.ts +219 -0
- package/src/notifications/broadcaster.ts +523 -317
- package/src/notifications/copy-composer.ts +27 -6
- package/src/notifications/decision-engine.ts +154 -77
- package/src/notifications/deterministic-checks.ts +6 -7
- package/src/notifications/edit-notification.ts +8 -4
- package/src/notifications/emit-signal.ts +49 -16
- package/src/notifications/guardian-delivery-recorder.ts +8 -2
- package/src/notifications/home-feed-side-effect.ts +43 -6
- package/src/notifications/notification-utils.ts +39 -4
- package/src/notifications/signal.ts +10 -0
- package/src/notifications/types.ts +17 -0
- package/src/persistence/bookmark-crud.ts +18 -9
- package/src/persistence/conversation-attention-store.ts +26 -0
- package/src/persistence/conversation-crud.ts +79 -32
- package/src/persistence/conversation-queries.ts +30 -15
- package/src/persistence/conversation-title-service.ts +5 -170
- package/src/persistence/conversation-types.ts +192 -1
- package/src/persistence/db-init.ts +14 -1
- package/src/persistence/db-maintenance.ts +5 -4
- package/src/persistence/embeddings/__tests__/plugin-index-qdrant-init.test.ts +219 -0
- package/src/persistence/embeddings/__tests__/plugin-index.test.ts +22 -0
- package/src/persistence/embeddings/__tests__/worker-script-version.test.ts +76 -0
- package/src/persistence/embeddings/embedding-local.ts +2 -1
- package/src/persistence/embeddings/embedding-runtime-manager.ts +29 -5
- package/src/persistence/embeddings/plugin-index.ts +84 -12
- package/src/persistence/migrations/360-add-document-workspace-path.test.ts +110 -0
- package/src/persistence/migrations/360-add-document-workspace-path.ts +39 -0
- package/src/persistence/planner-statistics.ts +14 -0
- package/src/persistence/schema/documents.ts +26 -11
- package/src/persistence/steps.ts +2 -0
- package/src/platform/client.test.ts +172 -2
- package/src/platform/client.ts +178 -55
- package/src/plugin-api/conversation-turn.ts +13 -0
- package/src/plugin-api/model-profiles.test.ts +6 -2
- package/src/plugins/defaults/memory/__tests__/bookmark-crud.test.ts +34 -0
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +22 -0
- package/src/plugins/defaults/memory/__tests__/jobs-store-enqueue-gate.test.ts +11 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +199 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +98 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +105 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +24 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +524 -0
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +9 -4
- package/src/plugins/defaults/memory/host-utils.ts +5 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +105 -1
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +64 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +36 -1
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +12 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +1 -0
- package/src/plugins/defaults/memory/substrate/__tests__/skill-store.test.ts +195 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +1 -1
- package/src/plugins/defaults/memory/substrate/skill-store.ts +167 -29
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +131 -0
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +1 -0
- package/src/plugins/defaults/memory/v2/activation-log-store.ts +4 -0
- package/src/plugins/defaults/memory/v2/injection.ts +48 -3
- package/src/plugins/defaults/memory/v2/rerank-local.ts +6 -2
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +1 -1
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +1 -1
- package/src/plugins/defaults/turn-context/injectors.ts +1 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +26 -0
- package/src/plugins/types.ts +17 -0
- package/src/prompts/templates/system-sections.ts +2 -2
- package/src/providers/__tests__/dispatch-connection-routing.test.ts +43 -2
- package/src/providers/__tests__/vellum-mismatch-routing.test.ts +8 -0
- package/src/providers/call-site-routing.ts +77 -14
- package/src/providers/connection-resolution.ts +42 -2
- package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +127 -1
- package/src/providers/inference/adapter-factory.ts +70 -12
- package/src/providers/model-catalog.ts +0 -11
- package/src/providers/model-intents.ts +11 -0
- package/src/providers/registry.ts +8 -0
- package/src/providers/retry.ts +85 -17
- package/src/providers/types.ts +8 -1
- package/src/runtime/__tests__/agent-wake.test.ts +181 -0
- package/src/runtime/agent-wake.ts +132 -9
- package/src/runtime/channel-approval-types.ts +25 -0
- package/src/runtime/routes/__tests__/connection-routes-vs-cli-parity.test.ts +2 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +11 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +121 -9
- package/src/runtime/routes/__tests__/platform-invoice-routes.test.ts +392 -0
- package/src/runtime/routes/__tests__/slack-channel-routes.test.ts +7 -0
- package/src/runtime/routes/__tests__/workspace-commit-routes.test.ts +63 -0
- package/src/runtime/routes/consolidation-routes.ts +1 -1
- package/src/runtime/routes/conversation-management-routes.ts +18 -6
- package/src/runtime/routes/conversation-query-routes.ts +14 -23
- package/src/runtime/routes/conversation-routes.ts +113 -8
- package/src/runtime/routes/credential-routes.ts +1 -5
- package/src/runtime/routes/documents-routes.ts +207 -12
- package/src/runtime/routes/identity-routes.ts +3 -93
- package/src/runtime/routes/inbound-stages/admission-policy.test.ts +31 -1
- package/src/runtime/routes/inbound-stages/admission-policy.ts +18 -0
- package/src/runtime/routes/inference-provider-connection-routes.ts +52 -3
- package/src/runtime/routes/notification-routes.ts +4 -1
- package/src/runtime/routes/platform-routes.ts +238 -3
- package/src/runtime/routes/playground/guard.ts +1 -2
- package/src/runtime/routes/slack-channel-routes.ts +2 -4
- package/src/runtime/routes/workspace-commit-routes.ts +49 -2
- package/src/runtime/routes/workspace-routes.ts +11 -12
- package/src/runtime/routes/workspace-utils.ts +49 -2
- package/src/runtime/services/conversation-serializer.ts +2 -4
- package/src/subagent/__tests__/consult-context-gating.test.ts +98 -0
- package/src/subagent/__tests__/consult-context-skills.test.ts +82 -0
- package/src/subagent/__tests__/consult-context.test.ts +84 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +36 -0
- package/src/subagent/consult-context.ts +410 -0
- package/src/subagent/consult-prompt.ts +38 -5
- package/src/subagent/manager.ts +9 -4
- package/src/subagent/types.ts +8 -0
- package/src/telemetry/telemetry-event-sources.ts +7 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +1 -0
- package/src/telemetry/turn-events-store.ts +30 -1
- package/src/telemetry/types.ts +24 -0
- package/src/telemetry/usage-telemetry-reporter.test.ts +48 -0
- package/src/tools/browser/pinned-tabs.ts +3 -1
- package/src/tools/subagent/spawn.ts +23 -1
- package/src/tools/terminal/safe-env.ts +1 -0
- package/src/tools/ui-surface/definitions.ts +39 -1
- package/src/tools/ui-surface/surface-shape-docs.ts +38 -4
- package/src/tools/ui-surface/visual-validation.ts +787 -0
- package/src/tools/workflows/run-workflow.test.ts +1 -0
- package/src/util/__tests__/short-title.test.ts +229 -0
- package/src/util/__tests__/worker-compute.test.ts +65 -0
- package/src/util/cgroup-cpu.ts +93 -0
- package/src/util/errors.ts +27 -0
- package/src/util/process-tree.ts +19 -0
- package/src/util/short-title.ts +189 -0
- package/src/util/worker-compute.ts +83 -0
- package/src/workspace/byok-default-profile-ensure.ts +25 -15
- package/src/workspace/git-service.ts +79 -30
- package/src/workspace/migrations/137-repair-retired-fireworks-minimax-model-id.ts +131 -0
- package/src/workspace/migrations/138-backfill-home-feed-titles.ts +179 -0
- package/src/workspace/migrations/139-clear-renamed-cost-profile-label.ts +92 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/plugins/defaults/memory/substrate/constants.ts +0 -8
|
@@ -109,21 +109,45 @@ export function phaseForFrame(
|
|
|
109
109
|
*/
|
|
110
110
|
export class LiveActivityReporter {
|
|
111
111
|
private lastPhase: LiveActivityPhase | null = null;
|
|
112
|
+
private lastDetail = "";
|
|
112
113
|
private ended = false;
|
|
113
114
|
|
|
114
115
|
constructor(private readonly conversationId: string) {}
|
|
115
116
|
|
|
116
117
|
/**
|
|
117
|
-
* Report
|
|
118
|
-
* are on the session's send path
|
|
118
|
+
* Report what a frame implies about the phase or the activity line, if
|
|
119
|
+
* either changed. Fire-and-forget: callers are on the session's send path
|
|
120
|
+
* and must not await this.
|
|
121
|
+
*
|
|
122
|
+
* Both travel in one dispatch because they are two fields of one content
|
|
123
|
+
* state: ActivityKit replaces the whole thing per push, so a dispatch that
|
|
124
|
+
* carried only the field that moved would blank the other.
|
|
119
125
|
*/
|
|
120
126
|
report(frame: LiveVoiceServerFramePayload): void {
|
|
127
|
+
if (this.ended) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
121
130
|
const phase = phaseForFrame(frame, this.lastPhase);
|
|
122
|
-
|
|
131
|
+
// The session sends this frame only on a change it wants surfaced, and
|
|
132
|
+
// sends an empty label when the turn stops working, so it is taken
|
|
133
|
+
// verbatim rather than derived.
|
|
134
|
+
const detail = frame.type === "activity" ? frame.label : this.lastDetail;
|
|
135
|
+
const phaseMoved = phase !== null && phase !== this.lastPhase;
|
|
136
|
+
if (!phaseMoved && detail === this.lastDetail) {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
if (phase !== null) {
|
|
140
|
+
this.lastPhase = phase;
|
|
141
|
+
}
|
|
142
|
+
this.lastDetail = detail;
|
|
143
|
+
// An activity line can arrive before any phase has been established (a
|
|
144
|
+
// turn whose first tool starts before the phase-bearing frame lands).
|
|
145
|
+
// There is no content state to attach it to yet, and the phase the next
|
|
146
|
+
// frame sets will carry it.
|
|
147
|
+
if (this.lastPhase === null) {
|
|
123
148
|
return;
|
|
124
149
|
}
|
|
125
|
-
this.lastPhase
|
|
126
|
-
void this.dispatch(phase, "update");
|
|
150
|
+
void this.dispatch(this.lastPhase, "update", this.lastDetail);
|
|
127
151
|
}
|
|
128
152
|
|
|
129
153
|
/**
|
|
@@ -139,13 +163,17 @@ export class LiveActivityReporter {
|
|
|
139
163
|
return;
|
|
140
164
|
}
|
|
141
165
|
this.ended = true;
|
|
142
|
-
|
|
166
|
+
// No detail on the way out: whatever the turn was doing, it is not doing
|
|
167
|
+
// it any more, and this state is the one that lingers on the Lock Screen
|
|
168
|
+
// through the dismissal window.
|
|
169
|
+
void this.dispatch("ending", "end", "");
|
|
143
170
|
}
|
|
144
171
|
|
|
145
172
|
/** `protected` so a test can observe what would be sent without sending it. */
|
|
146
173
|
protected async dispatch(
|
|
147
174
|
phase: LiveActivityPhase,
|
|
148
175
|
event: "update" | "end",
|
|
176
|
+
detail: string,
|
|
149
177
|
): Promise<void> {
|
|
150
178
|
try {
|
|
151
179
|
const client = await VellumPlatformClient.create();
|
|
@@ -162,6 +190,7 @@ export class LiveActivityReporter {
|
|
|
162
190
|
conversation_id: this.conversationId,
|
|
163
191
|
phase,
|
|
164
192
|
event,
|
|
193
|
+
detail,
|
|
165
194
|
}),
|
|
166
195
|
});
|
|
167
196
|
if (!response.ok) {
|
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
9
9
|
import {
|
|
10
10
|
isIncompleteControlMarkerTail,
|
|
11
|
-
MINIMIZE_ROOM_MARKER,
|
|
12
11
|
stripInternalSpeechMarkers,
|
|
13
12
|
} from "../calls/voice-control-protocol.js";
|
|
14
13
|
import type {
|
|
@@ -59,6 +58,11 @@ import { getToolOwner } from "../tools/registry.js";
|
|
|
59
58
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
60
59
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
61
60
|
import { getLogger } from "../util/logger.js";
|
|
61
|
+
import {
|
|
62
|
+
activityLabelForTool,
|
|
63
|
+
dismissesUiSurface,
|
|
64
|
+
revealsUiSurface,
|
|
65
|
+
} from "./activity-label.js";
|
|
62
66
|
import {
|
|
63
67
|
createVoiceFrontDecider,
|
|
64
68
|
type VoiceFrontDecider,
|
|
@@ -478,9 +482,19 @@ interface ActiveAssistantTurn {
|
|
|
478
482
|
progress: TurnProgressState;
|
|
479
483
|
assistantCompleted: boolean;
|
|
480
484
|
ttsDone: boolean;
|
|
481
|
-
// Latched when the
|
|
485
|
+
// Latched when the turn puts something on screen (a ui tool ran); consumed
|
|
482
486
|
// once at TTS drain, where the minimize_room frame goes out after tts_done.
|
|
487
|
+
// Never set from anything the model says: the reveal is a consequence of
|
|
488
|
+
// showing a surface, not a token the model has to remember.
|
|
483
489
|
minimizeRequested: boolean;
|
|
490
|
+
// The activity label the client was last told about, so a run of tools that
|
|
491
|
+
// map to the same line sends one frame rather than one per call. Empty means
|
|
492
|
+
// the client believes nothing is running, which is also where a turn ends.
|
|
493
|
+
activityLabel: string;
|
|
494
|
+
// Set while the turn is blocked on a decision the user has to make. Suppresses
|
|
495
|
+
// progress narration, whose entire vocabulary ("still on it", "almost there")
|
|
496
|
+
// describes work in flight and would be false here.
|
|
497
|
+
awaitingApproval: boolean;
|
|
484
498
|
// A tts_audio frame actually went out to the client — latches on the first
|
|
485
499
|
// forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
|
|
486
500
|
ttsAudioStarted: boolean;
|
|
@@ -599,13 +613,16 @@ interface ActiveAssistantTurn {
|
|
|
599
613
|
* runs forward from the emitted boundary — not from the last "[" — so
|
|
600
614
|
* brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
|
|
601
615
|
* in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
|
|
602
|
-
* its terminator.
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
616
|
+
* its terminator.
|
|
617
|
+
*
|
|
618
|
+
* **Markers are stripped, never acted on.** Nothing the model can say
|
|
619
|
+
* minimizes the room any more: that is decided by whether a ui tool ran (see
|
|
620
|
+
* the `tool_use_start` handler), so the reveal cannot depend on the model
|
|
621
|
+
* remembering a token, and a reply whose content happens to contain "[-1]" (an
|
|
622
|
+
* array literal, a temperature) cannot move the room either. No prompt teaches
|
|
623
|
+
* a marker, so this stripping is defense against a model that emits one
|
|
624
|
+
* regardless: an unspoken, unpersisted "[-1]" is the correct handling of a
|
|
625
|
+
* token that now means nothing.
|
|
609
626
|
*/
|
|
610
627
|
function createControlMarkerHoldback(
|
|
611
628
|
turn: ActiveAssistantTurn,
|
|
@@ -613,13 +630,6 @@ function createControlMarkerHoldback(
|
|
|
613
630
|
): (raw: string, opts?: { force?: boolean }) => void {
|
|
614
631
|
let emitted = 0;
|
|
615
632
|
return (raw, opts) => {
|
|
616
|
-
if (
|
|
617
|
-
opts?.force === true &&
|
|
618
|
-
!turn.minimizeRequested &&
|
|
619
|
-
raw.trimEnd().endsWith(MINIMIZE_ROOM_MARKER)
|
|
620
|
-
) {
|
|
621
|
-
turn.minimizeRequested = true;
|
|
622
|
-
}
|
|
623
633
|
let safeEnd = raw.length;
|
|
624
634
|
if (opts?.force !== true) {
|
|
625
635
|
for (
|
|
@@ -644,30 +654,30 @@ function createControlMarkerHoldback(
|
|
|
644
654
|
// barge-in, the interruption merge note is appended to it (see
|
|
645
655
|
// buildInterruptionMergeNote) so the model reconciles the interrupted request
|
|
646
656
|
// with the new utterance.
|
|
657
|
+
// Spoken once when a turn starts waiting on the user's decision. Fixed rather
|
|
658
|
+
// than generated: this is a statement about the system's state, not about the
|
|
659
|
+
// work, and it has to be true every time. Kept in the shape of the progress
|
|
660
|
+
// phrases it displaces (short, neutral, no claim about tools).
|
|
661
|
+
const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
|
|
662
|
+
|
|
647
663
|
const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
648
|
-
"You are speaking in a local live voice session. Keep replies brief and conversational.
|
|
649
|
-
|
|
650
|
-
//
|
|
651
|
-
//
|
|
652
|
-
//
|
|
653
|
-
//
|
|
654
|
-
// token every character is spoken verbatim — teaching it the marker would
|
|
655
|
-
// contradict that rule and could only produce spurious minimizes.
|
|
656
|
-
//
|
|
657
|
-
// The teaching deliberately keeps the no-interactive-UI rule intact
|
|
658
|
-
// (live-voice turns are non-interactive, JARVIS-1291): the marker does not
|
|
659
|
-
// render UI — it asks the client to reveal the screen the call overlay
|
|
660
|
-
// covers. The gate in createControlMarkerHoldback strips it from speech, and
|
|
661
|
-
// completeTtsForTurn sends the minimize_room frame only after the reply's
|
|
662
|
-
// TTS drains, so "end your reply with it" is the whole contract the model
|
|
663
|
-
// needs.
|
|
664
|
+
"You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
|
|
665
|
+
|
|
666
|
+
// Appended for the legs that can actually put something on screen: the main
|
|
667
|
+
// leg and the escalated leg. The front-door (fast) leg never receives it, for
|
|
668
|
+
// the same reason it never received the marker this replaces: that leg is
|
|
669
|
+
// toolless, so it has nothing to show.
|
|
664
670
|
//
|
|
665
|
-
// The
|
|
666
|
-
//
|
|
667
|
-
//
|
|
668
|
-
//
|
|
669
|
-
|
|
670
|
-
|
|
671
|
+
// **The model no longer asks for the minimize; it is told one is coming.**
|
|
672
|
+
// Revealing the screen used to be a marker the model emitted at the end of a
|
|
673
|
+
// reply, which made "did the user see it" depend on the model remembering a
|
|
674
|
+
// token. It is now a consequence of showing a surface at all: the session
|
|
675
|
+
// latches the minimize when a ui tool runs, and the room opens after the
|
|
676
|
+
// reply's speech drains. What is left for the prompt is the part only the
|
|
677
|
+
// model can get right, which is speaking as though the thing is already in
|
|
678
|
+
// front of the user, because by the time it stops talking it is.
|
|
679
|
+
const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
|
|
680
|
+
"The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
|
|
671
681
|
|
|
672
682
|
// System-level guidance appended to a barge-in turn's control prompt so the
|
|
673
683
|
// model treats the new utterance as a continuation of the request it was cut
|
|
@@ -726,8 +736,8 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
726
736
|
}
|
|
727
737
|
|
|
728
738
|
// Assemble a leg's model-facing control prompt: the base live-voice rules,
|
|
729
|
-
// the
|
|
730
|
-
//
|
|
739
|
+
// the screen-reveal teaching (withheld from the front-door leg, see
|
|
740
|
+
// LIVE_VOICE_SCREEN_REVEAL_TEACHING), the shared no-setup-flows rule, plus
|
|
731
741
|
// any pending barge-in merge context, completed-continuation context, and/or
|
|
732
742
|
// the announcement instruction. A turn can carry several (a barge-in follow-up
|
|
733
743
|
// that also has a continuation result waiting); the notes are model-only and
|
|
@@ -738,7 +748,7 @@ function buildVoiceControlPrompt(
|
|
|
738
748
|
): string {
|
|
739
749
|
let prompt =
|
|
740
750
|
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
741
|
-
(leg.frontDoor === true ? "" :
|
|
751
|
+
(leg.frontDoor === true ? "" : LIVE_VOICE_SCREEN_REVEAL_TEACHING) +
|
|
742
752
|
VOICE_NO_SETUP_FLOWS_RULE;
|
|
743
753
|
if (turn.interruptedRequest) {
|
|
744
754
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
@@ -1920,6 +1930,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1920
1930
|
"barge_in",
|
|
1921
1931
|
turn.turnId,
|
|
1922
1932
|
);
|
|
1933
|
+
// A cancelled turn's tools are abandoned, not finished, so nothing will
|
|
1934
|
+
// arrive to clear the line it left behind.
|
|
1935
|
+
this.publishActivity(turn, "");
|
|
1923
1936
|
await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
|
|
1924
1937
|
await this.cancelAssistantTurn("barge_in");
|
|
1925
1938
|
// Keep the interrupted turn's work alive on a background subagent; the
|
|
@@ -2317,6 +2330,82 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2317
2330
|
}, drainMs + this.continuationAnnounceSilenceMs);
|
|
2318
2331
|
}
|
|
2319
2332
|
|
|
2333
|
+
/**
|
|
2334
|
+
* The line describing what this turn is doing right now: the newest tool
|
|
2335
|
+
* still running, or nothing when none is.
|
|
2336
|
+
*
|
|
2337
|
+
* Newest-first because parallel ops are the interesting case. A turn that
|
|
2338
|
+
* starts a search and a file read, then finishes the search, is still
|
|
2339
|
+
* reading a file, and a surface that went blank there would claim the turn
|
|
2340
|
+
* had gone idle while it plainly has not.
|
|
2341
|
+
*/
|
|
2342
|
+
private currentActivityLabel(turn: ActiveAssistantTurn): string {
|
|
2343
|
+
for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
|
|
2344
|
+
const op = turn.progress.ops[i];
|
|
2345
|
+
if (op !== undefined && op.completedAtMs === undefined) {
|
|
2346
|
+
return activityLabelForTool(op.toolName);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
return "";
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
/**
|
|
2353
|
+
* Open the room because a decision is waiting behind it.
|
|
2354
|
+
*
|
|
2355
|
+
* The approval card renders in the app, and the room covers the app, so
|
|
2356
|
+
* without this the turn simply goes quiet: nothing is spoken, nothing is
|
|
2357
|
+
* visible, and the only cue is a call that stopped talking. Sent
|
|
2358
|
+
* immediately rather than latched for the drain the way a shown surface is,
|
|
2359
|
+
* because there is no drain coming. The turn is blocked on this decision,
|
|
2360
|
+
* and the whole point is that the user can make it now.
|
|
2361
|
+
*
|
|
2362
|
+
* The latch is cleared as well, so a turn that also showed a surface does
|
|
2363
|
+
* not send a second minimize once its speech ends; the room is already open.
|
|
2364
|
+
*/
|
|
2365
|
+
private revealRoomForPendingApproval(turn: ActiveAssistantTurn): void {
|
|
2366
|
+
if (turn.awaitingApproval) {
|
|
2367
|
+
return;
|
|
2368
|
+
}
|
|
2369
|
+
turn.awaitingApproval = true;
|
|
2370
|
+
turn.minimizeRequested = false;
|
|
2371
|
+
void this.sendFrame(
|
|
2372
|
+
{ type: "minimize_room", turnId: turn.turnId },
|
|
2373
|
+
() => !this.isClosed,
|
|
2374
|
+
);
|
|
2375
|
+
// Spoken, because opening the room is only a cue for someone looking at
|
|
2376
|
+
// the screen, and the case this exists for is a phone the user has put
|
|
2377
|
+
// down. One line, not narration: the turn is not working, it is waiting,
|
|
2378
|
+
// and it says which.
|
|
2379
|
+
this.enqueueFillerPhrase(turn, APPROVAL_PENDING_PHRASE);
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** Clear the wait once a decision lands, so the turn narrates normally again. */
|
|
2383
|
+
private clearAwaitingApproval(turn: ActiveAssistantTurn): void {
|
|
2384
|
+
turn.awaitingApproval = false;
|
|
2385
|
+
}
|
|
2386
|
+
|
|
2387
|
+
/**
|
|
2388
|
+
* Tell the client what the turn is doing, if it changed.
|
|
2389
|
+
*
|
|
2390
|
+
* De-duplicated for the same reason the Live Activity reporter de-duplicates
|
|
2391
|
+
* phases: this frame reaches ActivityKit, whose update budget is finite and
|
|
2392
|
+
* whose overflow is dropped silently, and a turn that runs four file reads in
|
|
2393
|
+
* a row would otherwise spend that budget restating one line.
|
|
2394
|
+
*
|
|
2395
|
+
* Fire-and-forget. An activity label is a flourish, and nothing about the
|
|
2396
|
+
* conversation may wait on one.
|
|
2397
|
+
*/
|
|
2398
|
+
private publishActivity(turn: ActiveAssistantTurn, label: string): void {
|
|
2399
|
+
if (turn.activityLabel === label) {
|
|
2400
|
+
return;
|
|
2401
|
+
}
|
|
2402
|
+
turn.activityLabel = label;
|
|
2403
|
+
void this.sendFrame(
|
|
2404
|
+
{ type: "activity", turnId: turn.turnId, label },
|
|
2405
|
+
() => !this.isClosed,
|
|
2406
|
+
);
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2320
2409
|
private clearActiveAssistantTurn(token: symbol): void {
|
|
2321
2410
|
if (this.activeAssistantTurn?.token !== token) {
|
|
2322
2411
|
return;
|
|
@@ -3409,6 +3498,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3409
3498
|
assistantCompleted: false,
|
|
3410
3499
|
ttsDone: false,
|
|
3411
3500
|
minimizeRequested: false,
|
|
3501
|
+
activityLabel: "",
|
|
3502
|
+
awaitingApproval: false,
|
|
3412
3503
|
ttsAudioStarted: false,
|
|
3413
3504
|
finalized: false,
|
|
3414
3505
|
speculativePending: opts?.speculative === true,
|
|
@@ -3610,6 +3701,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3610
3701
|
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
3611
3702
|
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
3612
3703
|
}),
|
|
3704
|
+
onApprovalPending: () => {
|
|
3705
|
+
this.revealRoomForPendingApproval(activeTurn);
|
|
3706
|
+
},
|
|
3707
|
+
onApprovalsResolved: () => {
|
|
3708
|
+
this.clearAwaitingApproval(activeTurn);
|
|
3709
|
+
},
|
|
3613
3710
|
content: leg.content,
|
|
3614
3711
|
isInbound: true,
|
|
3615
3712
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
@@ -3835,6 +3932,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3835
3932
|
});
|
|
3836
3933
|
current.progress.opsSinceNarration += 1;
|
|
3837
3934
|
current.progress.stateEpoch += 1;
|
|
3935
|
+
this.publishActivity(current, this.currentActivityLabel(current));
|
|
3838
3936
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
3839
3937
|
// Definitive tool use means the turn is guaranteed slow: speak
|
|
3840
3938
|
// the floor-holding ack now instead of waiting out the
|
|
@@ -3888,6 +3986,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3888
3986
|
}
|
|
3889
3987
|
}
|
|
3890
3988
|
current.progress.stateEpoch += 1;
|
|
3989
|
+
// Showing a surface implies revealing it: the room is a
|
|
3990
|
+
// full-screen overlay, so a surface rendered behind it is a
|
|
3991
|
+
// surface nobody sees.
|
|
3992
|
+
//
|
|
3993
|
+
// **Latched on the result, not the tool start.** A ui call can be
|
|
3994
|
+
// rejected (no `surface_type`, an empty card, a client that never
|
|
3995
|
+
// acks), and a reveal driven by the attempt would minimize the room
|
|
3996
|
+
// to show nothing at all. Only a call that came back without an
|
|
3997
|
+
// error actually put something on screen.
|
|
3998
|
+
//
|
|
3999
|
+
// A dismissal clears it again, so a turn that shows a surface and
|
|
4000
|
+
// then takes it away does not reveal an empty screen. Last write
|
|
4001
|
+
// wins, which is the right reading of "what did this turn leave up"
|
|
4002
|
+
// without tracking surfaces individually.
|
|
4003
|
+
if (event.isError !== true) {
|
|
4004
|
+
if (revealsUiSurface(event.toolName)) {
|
|
4005
|
+
current.minimizeRequested = true;
|
|
4006
|
+
} else if (dismissesUiSurface(event.toolName)) {
|
|
4007
|
+
current.minimizeRequested = false;
|
|
4008
|
+
}
|
|
4009
|
+
}
|
|
4010
|
+
this.publishActivity(current, this.currentActivityLabel(current));
|
|
3891
4011
|
this.maybeNarrateProgress(current, trigger);
|
|
3892
4012
|
},
|
|
3893
4013
|
},
|
|
@@ -4259,6 +4379,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4259
4379
|
return (
|
|
4260
4380
|
this.isActiveAssistantTurn(turn.token) &&
|
|
4261
4381
|
!turn.assistantCompleted &&
|
|
4382
|
+
// Nothing is in flight while a decision is pending, so every phrase
|
|
4383
|
+
// narration has would be a lie about who the call is waiting on. The
|
|
4384
|
+
// turn says so once, when it starts waiting, and is quiet after that.
|
|
4385
|
+
!turn.awaitingApproval &&
|
|
4262
4386
|
this.turnAudioIdle(turn)
|
|
4263
4387
|
);
|
|
4264
4388
|
}
|
|
@@ -4569,6 +4693,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4569
4693
|
!this.isClosed,
|
|
4570
4694
|
);
|
|
4571
4695
|
|
|
4696
|
+
// The turn is over, so nothing is running. Ordinarily the line is
|
|
4697
|
+
// already clear (the last tool_result cleared it); this is the net
|
|
4698
|
+
// for a turn that ends with an op still open, which would otherwise
|
|
4699
|
+
// leave the last tool it touched on screen through the next silence.
|
|
4700
|
+
this.publishActivity(currentTurn, "");
|
|
4701
|
+
|
|
4572
4702
|
// Drain-scoped minimize: the latched marker is consumed here, after
|
|
4573
4703
|
// the turn's speech has fully drained — never mid-speech, never for
|
|
4574
4704
|
// a barged-in turn, at most once per turn.
|
|
@@ -18,6 +18,7 @@ const _LIVE_VOICE_SERVER_FRAME_TYPES = [
|
|
|
18
18
|
"stt_partial",
|
|
19
19
|
"stt_final",
|
|
20
20
|
"thinking",
|
|
21
|
+
"activity",
|
|
21
22
|
"assistant_text_delta",
|
|
22
23
|
"tts_audio",
|
|
23
24
|
"tts_done",
|
|
@@ -217,6 +218,26 @@ export interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
|
|
|
217
218
|
readonly turnId: string;
|
|
218
219
|
}
|
|
219
220
|
|
|
221
|
+
/**
|
|
222
|
+
* What the assistant is doing inside a turn, as one short user-facing line.
|
|
223
|
+
*
|
|
224
|
+
* Exists because a live-voice turn can go silent for a long time while it
|
|
225
|
+
* works, and the surfaces that show the session (the iOS Live Activity, and
|
|
226
|
+
* the room) can otherwise only say "Thinking...". The label is composed here
|
|
227
|
+
* rather than by each surface for the same reason phase wording is composed
|
|
228
|
+
* once: the Live Activity has two independent drivers (this socket and an APNs
|
|
229
|
+
* push the daemon dispatches), they must carry identical content, and the only
|
|
230
|
+
* way to guarantee that is for both to be handed the same string.
|
|
231
|
+
*
|
|
232
|
+
* An empty `label` means "no current activity", which is what a turn's end
|
|
233
|
+
* sends. Emitted only on change, never per tool result.
|
|
234
|
+
*/
|
|
235
|
+
export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
|
|
236
|
+
readonly type: "activity";
|
|
237
|
+
readonly turnId: string;
|
|
238
|
+
readonly label: string;
|
|
239
|
+
}
|
|
240
|
+
|
|
220
241
|
export interface LiveVoiceAssistantTextDeltaServerFrame extends LiveVoiceServerFrameBase {
|
|
221
242
|
readonly type: "assistant_text_delta";
|
|
222
243
|
readonly text: string;
|
|
@@ -321,6 +342,7 @@ export type LiveVoiceServerFrame =
|
|
|
321
342
|
| LiveVoiceSttPartialServerFrame
|
|
322
343
|
| LiveVoiceSttFinalServerFrame
|
|
323
344
|
| LiveVoiceThinkingServerFrame
|
|
345
|
+
| LiveVoiceActivityServerFrame
|
|
324
346
|
| LiveVoiceAssistantTextDeltaServerFrame
|
|
325
347
|
| LiveVoiceTtsAudioServerFrame
|
|
326
348
|
| LiveVoiceTtsDoneServerFrame
|
|
@@ -341,6 +363,7 @@ export type LiveVoiceServerFramePayload =
|
|
|
341
363
|
| WithoutSeq<LiveVoiceSttPartialServerFrame>
|
|
342
364
|
| WithoutSeq<LiveVoiceSttFinalServerFrame>
|
|
343
365
|
| WithoutSeq<LiveVoiceThinkingServerFrame>
|
|
366
|
+
| WithoutSeq<LiveVoiceActivityServerFrame>
|
|
344
367
|
| WithoutSeq<LiveVoiceAssistantTextDeltaServerFrame>
|
|
345
368
|
| WithoutSeq<LiveVoiceTtsAudioServerFrame>
|
|
346
369
|
| WithoutSeq<LiveVoiceTtsDoneServerFrame>
|
|
@@ -31,12 +31,13 @@ mock.module("../../../../contacts/contacts-write.js", () => ({
|
|
|
31
31
|
}));
|
|
32
32
|
|
|
33
33
|
import {
|
|
34
|
-
|
|
34
|
+
__resetSlackMentionCachesForTests,
|
|
35
35
|
slackProvider,
|
|
36
36
|
} from "../adapter.js";
|
|
37
37
|
|
|
38
38
|
const originalFetch = globalThis.fetch;
|
|
39
39
|
let userInfoCalls: string[] = [];
|
|
40
|
+
let channelInfoCalls: string[] = [];
|
|
40
41
|
|
|
41
42
|
function installFetchStub() {
|
|
42
43
|
globalThis.fetch = (async (
|
|
@@ -63,7 +64,49 @@ function fakeSlackResponse(url: string): Record<string, unknown> {
|
|
|
63
64
|
const parsed = new URL(url);
|
|
64
65
|
const method = parsed.pathname.split("/").at(-1);
|
|
65
66
|
|
|
67
|
+
if (method === "conversations.info") {
|
|
68
|
+
const channelId = parsed.searchParams.get("channel") ?? "";
|
|
69
|
+
channelInfoCalls.push(channelId);
|
|
70
|
+
if (channelId === "CPRODMODELS1") {
|
|
71
|
+
return {
|
|
72
|
+
ok: true,
|
|
73
|
+
channel: { id: "CPRODMODELS1", name: "prod-models" },
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return { ok: false, error: "channel_not_found" };
|
|
77
|
+
}
|
|
78
|
+
|
|
66
79
|
if (method === "conversations.history") {
|
|
80
|
+
if (parsed.searchParams.get("channel") === "C_CHANNEL_MENTIONS") {
|
|
81
|
+
return {
|
|
82
|
+
ok: true,
|
|
83
|
+
has_more: false,
|
|
84
|
+
messages: [
|
|
85
|
+
{
|
|
86
|
+
type: "message",
|
|
87
|
+
ts: "1700000009.001000",
|
|
88
|
+
user: "USENDER",
|
|
89
|
+
text: "post in <#CPRODMODELS1> and <#COTHER1|other-name> please",
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (parsed.searchParams.get("channel") === "C_PRIVATE_MENTION") {
|
|
96
|
+
return {
|
|
97
|
+
ok: true,
|
|
98
|
+
has_more: false,
|
|
99
|
+
messages: [
|
|
100
|
+
{
|
|
101
|
+
type: "message",
|
|
102
|
+
ts: "1700000010.001100",
|
|
103
|
+
user: "USENDER",
|
|
104
|
+
text: "see <#CINVISIBLE1> for details",
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
67
110
|
if (parsed.searchParams.get("channel") === "C_USERINFO_FAIL") {
|
|
68
111
|
return {
|
|
69
112
|
ok: true,
|
|
@@ -318,8 +361,9 @@ function makeOAuthConnection(
|
|
|
318
361
|
|
|
319
362
|
describe("Slack adapter mention rendering", () => {
|
|
320
363
|
beforeEach(async () => {
|
|
321
|
-
|
|
364
|
+
__resetSlackMentionCachesForTests();
|
|
322
365
|
userInfoCalls = [];
|
|
366
|
+
channelInfoCalls = [];
|
|
323
367
|
getSecureKeyAsyncMock.mockReset();
|
|
324
368
|
getSecureKeyAsyncMock.mockImplementation(async (key: string) => {
|
|
325
369
|
if (key === credentialKey("slack_channel", "bot_token")) {
|
|
@@ -516,6 +560,44 @@ describe("Slack adapter mention rendering", () => {
|
|
|
516
560
|
});
|
|
517
561
|
});
|
|
518
562
|
|
|
563
|
+
test("getHistory renders inline channel mentions without rebinding the message's own channel", async () => {
|
|
564
|
+
// A message read from channel A that mentions channels B and C: the
|
|
565
|
+
// message stays attributed to A while the inline mentions render B/C.
|
|
566
|
+
const messages = await slackProvider.getHistory(
|
|
567
|
+
undefined,
|
|
568
|
+
"C_CHANNEL_MENTIONS",
|
|
569
|
+
);
|
|
570
|
+
|
|
571
|
+
expect(messages).toHaveLength(1);
|
|
572
|
+
// Bare token resolves via conversations.info; pipe token uses its own
|
|
573
|
+
// embedded label with no lookup.
|
|
574
|
+
expect(messages[0].text).toBe(
|
|
575
|
+
"post in #prod-models and #other-name please",
|
|
576
|
+
);
|
|
577
|
+
expect(messages[0].conversationId).toBe("C_CHANNEL_MENTIONS");
|
|
578
|
+
expect(channelInfoCalls).toEqual(["CPRODMODELS1"]);
|
|
579
|
+
});
|
|
580
|
+
|
|
581
|
+
test("getHistory falls back to #unknown-channel for channels this auth cannot see, and caches the failure", async () => {
|
|
582
|
+
const first = await slackProvider.getHistory(
|
|
583
|
+
undefined,
|
|
584
|
+
"C_PRIVATE_MENTION",
|
|
585
|
+
);
|
|
586
|
+
const second = await slackProvider.getHistory(
|
|
587
|
+
undefined,
|
|
588
|
+
"C_PRIVATE_MENTION",
|
|
589
|
+
);
|
|
590
|
+
|
|
591
|
+
expect(first[0].text).toBe("see #unknown-channel for details");
|
|
592
|
+
expect(second[0].text).toBe("see #unknown-channel for details");
|
|
593
|
+
expect(first[0].text).not.toContain("CINVISIBLE1");
|
|
594
|
+
// channel_not_found is a definitive answer for this auth: one lookup,
|
|
595
|
+
// not one per history read.
|
|
596
|
+
expect(channelInfoCalls.filter((id) => id === "CINVISIBLE1")).toHaveLength(
|
|
597
|
+
1,
|
|
598
|
+
);
|
|
599
|
+
});
|
|
600
|
+
|
|
519
601
|
test("getThreadReplies renders Slack user mentions for model-facing text without changing sender identity", async () => {
|
|
520
602
|
const messages = await slackProvider.getThreadReplies!(
|
|
521
603
|
undefined,
|