@vellumai/assistant 0.11.1 → 0.11.2-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -3
- package/README.md +1 -1
- package/eslint.config.mjs +28 -8
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/slack-text/src/index.test.ts +48 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +25 -6
- package/openapi.yaml +292 -5
- package/package.json +1 -1
- package/src/__tests__/agent-loop-resume-interrupted.test.ts +223 -0
- package/src/__tests__/byok-default-profile-ensure.test.ts +43 -0
- package/src/__tests__/cli-logger-boundary-guard.test.ts +87 -0
- package/src/__tests__/compaction-events.test.ts +139 -2
- package/src/__tests__/config-loader-backfill.test.ts +35 -3
- package/src/__tests__/config-schema.test.ts +1 -0
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +30 -0
- package/src/__tests__/conversation-agent-loop.test.ts +180 -7
- package/src/__tests__/conversation-error.test.ts +211 -4
- package/src/__tests__/conversation-options-turn-scoped-transport.test.ts +64 -0
- package/src/__tests__/conversation-queue.test.ts +126 -0
- package/src/__tests__/conversation-retry-route.test.ts +46 -7
- package/src/__tests__/conversation-slash.test.ts +4 -2
- package/src/__tests__/conversation-summarize-route.test.ts +7 -1
- package/src/__tests__/conversation-summarize-up-to.test.ts +71 -5
- package/src/__tests__/document-create-dedupe.test.ts +2 -1
- package/src/__tests__/document-find-replace.test.ts +2 -1
- package/src/__tests__/document-tool-security.test.ts +2 -1
- package/src/__tests__/document-update-default-surface.test.ts +2 -1
- package/src/__tests__/document-workspace-file.test.ts +467 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +290 -49
- package/src/__tests__/guardian-card-withdrawal.test.ts +91 -3
- package/src/__tests__/http-user-message-parity.test.ts +66 -0
- package/src/__tests__/list-messages-attachments.test.ts +87 -0
- package/src/__tests__/list-messages-provider-error.test.ts +143 -0
- package/src/__tests__/list-messages-system-card.test.ts +100 -0
- package/src/__tests__/llm-resolver.test.ts +19 -9
- package/src/__tests__/managed-profile-guard.test.ts +23 -0
- package/src/__tests__/notification-platform-adapter.test.ts +130 -2
- package/src/__tests__/notification-telegram-adapter.test.ts +6 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +45 -0
- package/src/__tests__/plugin-api-model-profiles.test.ts +10 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +3 -0
- package/src/__tests__/provider-error-scenarios.test.ts +140 -0
- package/src/__tests__/provider-send-message-override-profile.test.ts +95 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +66 -3
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +209 -0
- package/src/__tests__/skills.test.ts +27 -0
- package/src/__tests__/slack-channels-routes.test.ts +0 -2
- package/src/__tests__/slack-share-routes.test.ts +0 -3
- package/src/__tests__/slack-users-routes.test.ts +0 -2
- package/src/__tests__/subagent-tools.test.ts +11 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +58 -0
- package/src/__tests__/tool-result-spool.test.ts +5 -4
- package/src/__tests__/turn-boundary-resolution.test.ts +53 -0
- package/src/__tests__/turn-events-store.test.ts +26 -0
- package/src/__tests__/ui-visual-surface.test.ts +695 -0
- package/src/__tests__/unified-turn-context-visible-app.test.ts +99 -0
- package/src/__tests__/visible-app-context.test.ts +189 -0
- package/src/__tests__/workspace-git-service.test.ts +173 -33
- package/src/__tests__/workspace-migration-137-repair-retired-fireworks-minimax-model-id.test.ts +157 -0
- package/src/__tests__/workspace-migration-138-backfill-home-feed-titles.test.ts +373 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +137 -0
- package/src/agent/loop.ts +36 -7
- package/src/api/events/context-window-usage.ts +31 -0
- package/src/api/events/notification-intent.ts +8 -0
- package/src/api/events/ui-surface-pending.ts +35 -0
- package/src/api/index.ts +14 -0
- package/src/api/responses/conversation-message.ts +25 -4
- package/src/api/surfaces.ts +90 -1
- package/src/approvals/guardian-card-withdrawal.ts +66 -31
- package/src/approvals/guardian-decision-primitive.ts +4 -0
- package/src/calls/__tests__/call-setup-router.test.ts +156 -26
- package/src/calls/__tests__/voice-session-bridge.test.ts +201 -12
- package/src/calls/call-setup-router.ts +80 -36
- package/src/calls/voice-session-bridge.ts +173 -26
- package/src/cli/commands/inference.help.ts +3 -3
- package/src/cli/commands/notifications.help.ts +3 -2
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +42 -128
- package/src/cli/commands/platform/__tests__/credits.test.ts +9 -78
- package/src/cli/commands/platform/__tests__/helpers.ts +90 -0
- package/src/cli/commands/platform/__tests__/invoices.test.ts +238 -0
- package/src/cli/commands/platform/__tests__/plans.test.ts +9 -88
- package/src/cli/commands/platform/__tests__/status.test.ts +12 -87
- package/src/cli/commands/platform/__tests__/subscription.test.ts +9 -86
- package/src/cli/commands/platform/index.help.ts +92 -0
- package/src/cli/commands/platform/index.ts +7 -0
- package/src/cli/commands/platform/invoices.ts +132 -0
- package/src/cli/commands/usage.help.ts +1 -1
- package/src/cli/lib/list-installed-plugins.ts +2 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +7 -10
- package/src/config/__tests__/deployment-context-defaults.test.ts +27 -5
- package/src/config/assistant-feature-flags.ts +7 -2
- package/src/config/bundled-skills/app-builder/SKILL.md +3 -2
- package/src/config/bundled-skills/subagent/SKILL.md +3 -1
- package/src/config/bundled-skills/subagent/TOOLS.json +3 -3
- package/src/config/bundled-skills/visualize/SKILL.md +163 -0
- package/src/config/call-site-defaults.ts +3 -2
- package/src/config/default-profile-catalog.ts +75 -70
- package/src/config/default-profile-names.ts +13 -28
- package/src/config/env-registry.ts +1 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/llm-resolver.ts +21 -5
- package/src/config/loader.ts +18 -11
- package/src/config/schemas/llm.ts +1 -9
- package/src/config/schemas/memory-retrospective.ts +9 -0
- package/src/config/schemas/monitoring.ts +28 -2
- package/src/config/schemas/workspace-git.ts +15 -0
- package/src/config/seed-inference-profiles.ts +19 -5
- package/src/context/post-turn-tool-result-truncation.ts +2 -2
- package/src/conversations/__tests__/message-consolidation.test.ts +54 -0
- package/src/conversations/message-consolidation.ts +17 -15
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +182 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +181 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +90 -7
- package/src/daemon/conversation-agent-loop.ts +107 -22
- package/src/daemon/conversation-error.ts +169 -48
- package/src/daemon/conversation-messaging.ts +87 -4
- package/src/daemon/conversation-process.ts +57 -45
- package/src/daemon/conversation-runtime-assembly.ts +57 -0
- package/src/daemon/conversation-store.ts +36 -1
- package/src/daemon/conversation-surfaces.ts +68 -6
- package/src/daemon/conversation-turn-finalize.ts +76 -18
- package/src/daemon/conversation.ts +122 -27
- package/src/daemon/lifecycle.ts +9 -0
- package/src/daemon/message-types/conversations.ts +8 -0
- package/src/daemon/message-types/surfaces.ts +3 -0
- package/src/documents/document-store.ts +247 -10
- package/src/home/__tests__/feed-types.test.ts +43 -8
- package/src/home/__tests__/feed-writer.test.ts +59 -0
- package/src/home/feed-types.ts +34 -7
- package/src/home/feed-writer.ts +15 -6
- package/src/live-voice/__tests__/activity-label.test.ts +95 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +86 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +310 -25
- package/src/live-voice/__tests__/live-voice-events.test.ts +4 -4
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +8 -4
- package/src/live-voice/__tests__/live-voice-vad.test.ts +9 -1
- package/src/live-voice/activity-label.ts +169 -0
- package/src/live-voice/live-activity-reporter.ts +35 -6
- package/src/live-voice/live-voice-session.ts +282 -41
- package/src/live-voice/protocol.ts +35 -0
- package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +84 -2
- package/src/messaging/providers/slack/adapter.ts +86 -19
- package/src/messaging/providers/slack/api.test.ts +85 -1
- package/src/messaging/providers/slack/api.ts +121 -288
- package/src/messaging/providers/slack/client.ts +20 -251
- package/src/messaging/providers/slack/send.test.ts +4 -9
- package/src/messaging/providers/slack/send.ts +1 -1
- package/src/messaging/providers/slack/types.ts +5 -0
- package/src/messaging/providers/slack/web-api-transport.test.ts +181 -0
- package/src/messaging/providers/slack/web-api-transport.ts +367 -0
- package/src/messaging/providers/slack/withdraw.ts +6 -14
- package/src/messaging/providers/telegram-bot/send.test.ts +31 -1
- package/src/messaging/providers/telegram-bot/send.ts +23 -2
- package/src/messaging/providers/telegram-bot/withdraw.test.ts +152 -0
- package/src/messaging/providers/telegram-bot/withdraw.ts +166 -0
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +18 -4
- package/src/monitoring/__tests__/file-descriptors.test.ts +144 -0
- package/src/monitoring/file-descriptors.ts +262 -0
- package/src/monitoring/process-memory.ts +4 -11
- package/src/monitoring/resource-sampler.ts +5 -0
- package/src/monitoring/worker.ts +12 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +740 -0
- package/src/notifications/__tests__/broadcaster.test.ts +347 -7
- package/src/notifications/__tests__/copy-composer.test.ts +53 -3
- package/src/notifications/__tests__/decision-engine.test.ts +349 -29
- package/src/notifications/__tests__/deterministic-checks.test.ts +21 -0
- package/src/notifications/__tests__/edit-notification.test.ts +330 -0
- package/src/notifications/__tests__/guardian-delivery-recorder.test.ts +71 -0
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +170 -12
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/adapters/platform.ts +90 -14
- package/src/notifications/adapters/telegram.ts +6 -4
- package/src/notifications/assistant-reply-producer.ts +219 -0
- package/src/notifications/broadcaster.ts +523 -317
- package/src/notifications/copy-composer.ts +27 -6
- package/src/notifications/decision-engine.ts +154 -77
- package/src/notifications/deterministic-checks.ts +6 -7
- package/src/notifications/edit-notification.ts +8 -4
- package/src/notifications/emit-signal.ts +49 -16
- package/src/notifications/guardian-delivery-recorder.ts +8 -2
- package/src/notifications/home-feed-side-effect.ts +43 -6
- package/src/notifications/notification-utils.ts +39 -4
- package/src/notifications/signal.ts +10 -0
- package/src/notifications/types.ts +17 -0
- package/src/persistence/bookmark-crud.ts +18 -9
- package/src/persistence/conversation-attention-store.ts +26 -0
- package/src/persistence/conversation-crud.ts +79 -32
- package/src/persistence/conversation-queries.ts +30 -15
- package/src/persistence/conversation-title-service.ts +5 -170
- package/src/persistence/conversation-types.ts +192 -1
- package/src/persistence/db-init.ts +14 -1
- package/src/persistence/db-maintenance.ts +5 -4
- package/src/persistence/embeddings/__tests__/plugin-index-qdrant-init.test.ts +219 -0
- package/src/persistence/embeddings/__tests__/plugin-index.test.ts +22 -0
- package/src/persistence/embeddings/__tests__/worker-script-version.test.ts +76 -0
- package/src/persistence/embeddings/embedding-local.ts +2 -1
- package/src/persistence/embeddings/embedding-runtime-manager.ts +29 -5
- package/src/persistence/embeddings/plugin-index.ts +84 -12
- package/src/persistence/migrations/360-add-document-workspace-path.test.ts +110 -0
- package/src/persistence/migrations/360-add-document-workspace-path.ts +39 -0
- package/src/persistence/planner-statistics.ts +14 -0
- package/src/persistence/schema/documents.ts +26 -11
- package/src/persistence/steps.ts +2 -0
- package/src/platform/client.test.ts +172 -2
- package/src/platform/client.ts +178 -55
- package/src/plugin-api/conversation-turn.ts +13 -0
- package/src/plugin-api/model-profiles.test.ts +6 -2
- package/src/plugins/defaults/memory/__tests__/bookmark-crud.test.ts +34 -0
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +22 -0
- package/src/plugins/defaults/memory/__tests__/jobs-store-enqueue-gate.test.ts +11 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +199 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +98 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +105 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +24 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +524 -0
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +9 -4
- package/src/plugins/defaults/memory/host-utils.ts +5 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +105 -1
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +64 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +36 -1
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +12 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +1 -0
- package/src/plugins/defaults/memory/substrate/__tests__/skill-store.test.ts +195 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +1 -1
- package/src/plugins/defaults/memory/substrate/skill-store.ts +167 -29
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +131 -0
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +1 -0
- package/src/plugins/defaults/memory/v2/activation-log-store.ts +4 -0
- package/src/plugins/defaults/memory/v2/injection.ts +48 -3
- package/src/plugins/defaults/memory/v2/rerank-local.ts +6 -2
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +1 -1
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +1 -1
- package/src/plugins/defaults/turn-context/injectors.ts +1 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +26 -0
- package/src/plugins/types.ts +17 -0
- package/src/prompts/templates/system-sections.ts +2 -2
- package/src/providers/__tests__/dispatch-connection-routing.test.ts +43 -2
- package/src/providers/__tests__/vellum-mismatch-routing.test.ts +8 -0
- package/src/providers/call-site-routing.ts +77 -14
- package/src/providers/connection-resolution.ts +42 -2
- package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +127 -1
- package/src/providers/inference/adapter-factory.ts +70 -12
- package/src/providers/model-catalog.ts +0 -11
- package/src/providers/model-intents.ts +11 -0
- package/src/providers/registry.ts +8 -0
- package/src/providers/retry.ts +85 -17
- package/src/providers/types.ts +8 -1
- package/src/runtime/__tests__/agent-wake.test.ts +181 -0
- package/src/runtime/agent-wake.ts +132 -9
- package/src/runtime/channel-approval-types.ts +25 -0
- package/src/runtime/routes/__tests__/connection-routes-vs-cli-parity.test.ts +2 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +11 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +121 -9
- package/src/runtime/routes/__tests__/platform-invoice-routes.test.ts +392 -0
- package/src/runtime/routes/__tests__/slack-channel-routes.test.ts +7 -0
- package/src/runtime/routes/__tests__/workspace-commit-routes.test.ts +63 -0
- package/src/runtime/routes/consolidation-routes.ts +1 -1
- package/src/runtime/routes/conversation-management-routes.ts +18 -6
- package/src/runtime/routes/conversation-query-routes.ts +14 -23
- package/src/runtime/routes/conversation-routes.ts +113 -8
- package/src/runtime/routes/credential-routes.ts +1 -5
- package/src/runtime/routes/documents-routes.ts +207 -12
- package/src/runtime/routes/identity-routes.ts +3 -93
- package/src/runtime/routes/inbound-stages/admission-policy.test.ts +31 -1
- package/src/runtime/routes/inbound-stages/admission-policy.ts +18 -0
- package/src/runtime/routes/inference-provider-connection-routes.ts +52 -3
- package/src/runtime/routes/notification-routes.ts +4 -1
- package/src/runtime/routes/platform-routes.ts +238 -3
- package/src/runtime/routes/playground/guard.ts +1 -2
- package/src/runtime/routes/slack-channel-routes.ts +2 -4
- package/src/runtime/routes/workspace-commit-routes.ts +49 -2
- package/src/runtime/routes/workspace-routes.ts +11 -12
- package/src/runtime/routes/workspace-utils.ts +49 -2
- package/src/runtime/services/conversation-serializer.ts +2 -4
- package/src/subagent/__tests__/consult-context-gating.test.ts +98 -0
- package/src/subagent/__tests__/consult-context-skills.test.ts +82 -0
- package/src/subagent/__tests__/consult-context.test.ts +84 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +36 -0
- package/src/subagent/consult-context.ts +410 -0
- package/src/subagent/consult-prompt.ts +38 -5
- package/src/subagent/manager.ts +9 -4
- package/src/subagent/types.ts +8 -0
- package/src/telemetry/telemetry-event-sources.ts +7 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +1 -0
- package/src/telemetry/turn-events-store.ts +30 -1
- package/src/telemetry/types.ts +24 -0
- package/src/telemetry/usage-telemetry-reporter.test.ts +48 -0
- package/src/tools/browser/pinned-tabs.ts +3 -1
- package/src/tools/subagent/spawn.ts +23 -1
- package/src/tools/terminal/safe-env.ts +1 -0
- package/src/tools/ui-surface/definitions.ts +39 -1
- package/src/tools/ui-surface/surface-shape-docs.ts +38 -4
- package/src/tools/ui-surface/visual-validation.ts +787 -0
- package/src/tools/workflows/run-workflow.test.ts +1 -0
- package/src/util/__tests__/short-title.test.ts +229 -0
- package/src/util/__tests__/worker-compute.test.ts +65 -0
- package/src/util/cgroup-cpu.ts +93 -0
- package/src/util/errors.ts +27 -0
- package/src/util/process-tree.ts +19 -0
- package/src/util/short-title.ts +189 -0
- package/src/util/worker-compute.ts +83 -0
- package/src/workspace/byok-default-profile-ensure.ts +25 -15
- package/src/workspace/git-service.ts +79 -30
- package/src/workspace/migrations/137-repair-retired-fireworks-minimax-model-id.ts +131 -0
- package/src/workspace/migrations/138-backfill-home-feed-titles.ts +179 -0
- package/src/workspace/migrations/139-clear-renamed-cost-profile-label.ts +92 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/plugins/defaults/memory/substrate/constants.ts +0 -8
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
9
9
|
import {
|
|
10
10
|
isIncompleteControlMarkerTail,
|
|
11
|
-
MINIMIZE_ROOM_MARKER,
|
|
12
11
|
stripInternalSpeechMarkers,
|
|
13
12
|
} from "../calls/voice-control-protocol.js";
|
|
14
13
|
import type {
|
|
@@ -59,6 +58,12 @@ import { getToolOwner } from "../tools/registry.js";
|
|
|
59
58
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
60
59
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
61
60
|
import { getLogger } from "../util/logger.js";
|
|
61
|
+
import {
|
|
62
|
+
activityLabelForTool,
|
|
63
|
+
approvalActivityLabel,
|
|
64
|
+
dismissesUiSurface,
|
|
65
|
+
revealsUiSurface,
|
|
66
|
+
} from "./activity-label.js";
|
|
62
67
|
import {
|
|
63
68
|
createVoiceFrontDecider,
|
|
64
69
|
type VoiceFrontDecider,
|
|
@@ -478,9 +483,38 @@ interface ActiveAssistantTurn {
|
|
|
478
483
|
progress: TurnProgressState;
|
|
479
484
|
assistantCompleted: boolean;
|
|
480
485
|
ttsDone: boolean;
|
|
481
|
-
// Latched when the
|
|
486
|
+
// Latched when the turn puts something on screen (a ui tool ran); consumed
|
|
482
487
|
// once at TTS drain, where the minimize_room frame goes out after tts_done.
|
|
488
|
+
// Never set from anything the model says: the reveal is a consequence of
|
|
489
|
+
// showing a surface, not a token the model has to remember.
|
|
483
490
|
minimizeRequested: boolean;
|
|
491
|
+
// The activity label the client was last told about, so a run of tools that
|
|
492
|
+
// map to the same line sends one frame rather than one per call. Empty means
|
|
493
|
+
// the client believes nothing is running, which is also where a turn ends.
|
|
494
|
+
activityLabel: string;
|
|
495
|
+
// The approval id that went out with it, so the de-duplication covers the
|
|
496
|
+
// whole frame rather than its wording. What the CLIENT believes, as against
|
|
497
|
+
// `pendingApproval`, which is what is true.
|
|
498
|
+
publishedApprovalRequestId: string | null;
|
|
499
|
+
// Set while the turn is blocked on a decision the user has to make, and null
|
|
500
|
+
// when it is not. Suppresses progress narration, whose entire vocabulary
|
|
501
|
+
// ("still on it", "almost there") describes work in flight and would be
|
|
502
|
+
// false here.
|
|
503
|
+
//
|
|
504
|
+
// Carries the request id so a surface that is not the app — the Live
|
|
505
|
+
// Activity's buttons — can answer *that* request rather than whatever is
|
|
506
|
+
// pending by the time the tap arrives, and the wording that named it, which
|
|
507
|
+
// is captured once at the reveal rather than recomputed: a parallel op
|
|
508
|
+
// starting mid-wait moves `currentActivityLabel` on, and the line beside an
|
|
509
|
+
// Approve button must keep naming the thing being approved.
|
|
510
|
+
//
|
|
511
|
+
// The FIRST one, on a turn that leaves two decisions pending at once: the
|
|
512
|
+
// wait is announced once (see `revealRoomForPendingApproval`) and the pair
|
|
513
|
+
// resolves as one. A tap answering the first after it has already been
|
|
514
|
+
// decided is dropped client-side by the id check, which is the safe end of
|
|
515
|
+
// a rare case — the island never silently answers a request other than the
|
|
516
|
+
// one it named.
|
|
517
|
+
pendingApproval: { requestId: string; label: string } | null;
|
|
484
518
|
// A tts_audio frame actually went out to the client — latches on the first
|
|
485
519
|
// forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
|
|
486
520
|
ttsAudioStarted: boolean;
|
|
@@ -599,13 +633,16 @@ interface ActiveAssistantTurn {
|
|
|
599
633
|
* runs forward from the emitted boundary — not from the last "[" — so
|
|
600
634
|
* brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
|
|
601
635
|
* in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
|
|
602
|
-
* its terminator.
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
636
|
+
* its terminator.
|
|
637
|
+
*
|
|
638
|
+
* **Markers are stripped, never acted on.** Nothing the model can say
|
|
639
|
+
* minimizes the room any more: that is decided by whether a ui tool ran (see
|
|
640
|
+
* the `tool_use_start` handler), so the reveal cannot depend on the model
|
|
641
|
+
* remembering a token, and a reply whose content happens to contain "[-1]" (an
|
|
642
|
+
* array literal, a temperature) cannot move the room either. No prompt teaches
|
|
643
|
+
* a marker, so this stripping is defense against a model that emits one
|
|
644
|
+
* regardless: an unspoken, unpersisted "[-1]" is the correct handling of a
|
|
645
|
+
* token that now means nothing.
|
|
609
646
|
*/
|
|
610
647
|
function createControlMarkerHoldback(
|
|
611
648
|
turn: ActiveAssistantTurn,
|
|
@@ -613,13 +650,6 @@ function createControlMarkerHoldback(
|
|
|
613
650
|
): (raw: string, opts?: { force?: boolean }) => void {
|
|
614
651
|
let emitted = 0;
|
|
615
652
|
return (raw, opts) => {
|
|
616
|
-
if (
|
|
617
|
-
opts?.force === true &&
|
|
618
|
-
!turn.minimizeRequested &&
|
|
619
|
-
raw.trimEnd().endsWith(MINIMIZE_ROOM_MARKER)
|
|
620
|
-
) {
|
|
621
|
-
turn.minimizeRequested = true;
|
|
622
|
-
}
|
|
623
653
|
let safeEnd = raw.length;
|
|
624
654
|
if (opts?.force !== true) {
|
|
625
655
|
for (
|
|
@@ -644,30 +674,30 @@ function createControlMarkerHoldback(
|
|
|
644
674
|
// barge-in, the interruption merge note is appended to it (see
|
|
645
675
|
// buildInterruptionMergeNote) so the model reconciles the interrupted request
|
|
646
676
|
// with the new utterance.
|
|
677
|
+
// Spoken once when a turn starts waiting on the user's decision. Fixed rather
|
|
678
|
+
// than generated: this is a statement about the system's state, not about the
|
|
679
|
+
// work, and it has to be true every time. Kept in the shape of the progress
|
|
680
|
+
// phrases it displaces (short, neutral, no claim about tools).
|
|
681
|
+
const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
|
|
682
|
+
|
|
647
683
|
const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
648
|
-
"You are speaking in a local live voice session. Keep replies brief and conversational.
|
|
649
|
-
|
|
650
|
-
//
|
|
651
|
-
//
|
|
652
|
-
//
|
|
653
|
-
//
|
|
654
|
-
// token every character is spoken verbatim — teaching it the marker would
|
|
655
|
-
// contradict that rule and could only produce spurious minimizes.
|
|
656
|
-
//
|
|
657
|
-
// The teaching deliberately keeps the no-interactive-UI rule intact
|
|
658
|
-
// (live-voice turns are non-interactive, JARVIS-1291): the marker does not
|
|
659
|
-
// render UI — it asks the client to reveal the screen the call overlay
|
|
660
|
-
// covers. The gate in createControlMarkerHoldback strips it from speech, and
|
|
661
|
-
// completeTtsForTurn sends the minimize_room frame only after the reply's
|
|
662
|
-
// TTS drains, so "end your reply with it" is the whole contract the model
|
|
663
|
-
// needs.
|
|
684
|
+
"You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
|
|
685
|
+
|
|
686
|
+
// Appended for the legs that can actually put something on screen: the main
|
|
687
|
+
// leg and the escalated leg. The front-door (fast) leg never receives it, for
|
|
688
|
+
// the same reason it never received the marker this replaces: that leg is
|
|
689
|
+
// toolless, so it has nothing to show.
|
|
664
690
|
//
|
|
665
|
-
// The
|
|
666
|
-
//
|
|
667
|
-
//
|
|
668
|
-
//
|
|
669
|
-
|
|
670
|
-
|
|
691
|
+
// **The model no longer asks for the minimize; it is told one is coming.**
|
|
692
|
+
// Revealing the screen used to be a marker the model emitted at the end of a
|
|
693
|
+
// reply, which made "did the user see it" depend on the model remembering a
|
|
694
|
+
// token. It is now a consequence of showing a surface at all: the session
|
|
695
|
+
// latches the minimize when a ui tool runs, and the room opens after the
|
|
696
|
+
// reply's speech drains. What is left for the prompt is the part only the
|
|
697
|
+
// model can get right, which is speaking as though the thing is already in
|
|
698
|
+
// front of the user, because by the time it stops talking it is.
|
|
699
|
+
const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
|
|
700
|
+
"The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
|
|
671
701
|
|
|
672
702
|
// System-level guidance appended to a barge-in turn's control prompt so the
|
|
673
703
|
// model treats the new utterance as a continuation of the request it was cut
|
|
@@ -726,8 +756,8 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
726
756
|
}
|
|
727
757
|
|
|
728
758
|
// Assemble a leg's model-facing control prompt: the base live-voice rules,
|
|
729
|
-
// the
|
|
730
|
-
//
|
|
759
|
+
// the screen-reveal teaching (withheld from the front-door leg, see
|
|
760
|
+
// LIVE_VOICE_SCREEN_REVEAL_TEACHING), the shared no-setup-flows rule, plus
|
|
731
761
|
// any pending barge-in merge context, completed-continuation context, and/or
|
|
732
762
|
// the announcement instruction. A turn can carry several (a barge-in follow-up
|
|
733
763
|
// that also has a continuation result waiting); the notes are model-only and
|
|
@@ -738,7 +768,7 @@ function buildVoiceControlPrompt(
|
|
|
738
768
|
): string {
|
|
739
769
|
let prompt =
|
|
740
770
|
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
741
|
-
(leg.frontDoor === true ? "" :
|
|
771
|
+
(leg.frontDoor === true ? "" : LIVE_VOICE_SCREEN_REVEAL_TEACHING) +
|
|
742
772
|
VOICE_NO_SETUP_FLOWS_RULE;
|
|
743
773
|
if (turn.interruptedRequest) {
|
|
744
774
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
@@ -1920,6 +1950,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1920
1950
|
"barge_in",
|
|
1921
1951
|
turn.turnId,
|
|
1922
1952
|
);
|
|
1953
|
+
// A cancelled turn's tools are abandoned, not finished, so nothing will
|
|
1954
|
+
// arrive to clear the line it left behind.
|
|
1955
|
+
this.publishActivity(turn, "");
|
|
1923
1956
|
await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
|
|
1924
1957
|
await this.cancelAssistantTurn("barge_in");
|
|
1925
1958
|
// Keep the interrupted turn's work alive on a background subagent; the
|
|
@@ -2317,6 +2350,172 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2317
2350
|
}, drainMs + this.continuationAnnounceSilenceMs);
|
|
2318
2351
|
}
|
|
2319
2352
|
|
|
2353
|
+
/**
|
|
2354
|
+
* The line describing what this turn is doing right now: the newest tool
|
|
2355
|
+
* still running, or nothing when none is.
|
|
2356
|
+
*
|
|
2357
|
+
* Newest-first because parallel ops are the interesting case. A turn that
|
|
2358
|
+
* starts a search and a file read, then finishes the search, is still
|
|
2359
|
+
* reading a file, and a surface that went blank there would claim the turn
|
|
2360
|
+
* had gone idle while it plainly has not.
|
|
2361
|
+
*/
|
|
2362
|
+
private currentActivityLabel(turn: ActiveAssistantTurn): string {
|
|
2363
|
+
for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
|
|
2364
|
+
const op = turn.progress.ops[i];
|
|
2365
|
+
if (op !== undefined && op.completedAtMs === undefined) {
|
|
2366
|
+
return activityLabelForTool(op.toolName);
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
return "";
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
/**
|
|
2373
|
+
* The line for a turn that is waiting on a decision about its newest running
|
|
2374
|
+
* op.
|
|
2375
|
+
*
|
|
2376
|
+
* The approval gate sits behind `tool_use_start`, so the tool being waited on
|
|
2377
|
+
* is the one the turn last said it was running — which is precisely why the
|
|
2378
|
+
* wait has to be published at all: without it the surfaces keep showing that
|
|
2379
|
+
* tool as *running* for the whole time it is doing nothing of the kind.
|
|
2380
|
+
* Falls back to the tool-less phrase when no op is open, which is the case
|
|
2381
|
+
* for a confirmation raised by a prompter outside the tool pipeline.
|
|
2382
|
+
*
|
|
2383
|
+
* Read once, at the reveal — see `pendingApproval` for why it is then held
|
|
2384
|
+
* rather than recomputed.
|
|
2385
|
+
*/
|
|
2386
|
+
private pendingApprovalLabel(turn: ActiveAssistantTurn): string {
|
|
2387
|
+
for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
|
|
2388
|
+
const op = turn.progress.ops[i];
|
|
2389
|
+
if (op !== undefined && op.completedAtMs === undefined) {
|
|
2390
|
+
return approvalActivityLabel(op.toolName);
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
return approvalActivityLabel("");
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
/**
|
|
2397
|
+
* Publish whatever the turn's activity line should be right now: the
|
|
2398
|
+
* decision it is waiting on if it is waiting, and its newest running tool if
|
|
2399
|
+
* it is not.
|
|
2400
|
+
*
|
|
2401
|
+
* The single entry point for every caller that would otherwise reach for
|
|
2402
|
+
* `currentActivityLabel` directly. A turn can start and finish other ops
|
|
2403
|
+
* while it is blocked on an approval — a parallel `tool_use_start` or
|
|
2404
|
+
* `tool_result` lands mid-wait — and each of those would otherwise publish a
|
|
2405
|
+
* line composed as if nothing were pending, taking the request id down with
|
|
2406
|
+
* it and retiring the island's buttons while the turn was still waiting.
|
|
2407
|
+
*/
|
|
2408
|
+
private refreshActivity(turn: ActiveAssistantTurn): void {
|
|
2409
|
+
if (turn.pendingApproval !== null) {
|
|
2410
|
+
this.publishActivity(
|
|
2411
|
+
turn,
|
|
2412
|
+
turn.pendingApproval.label,
|
|
2413
|
+
turn.pendingApproval.requestId,
|
|
2414
|
+
);
|
|
2415
|
+
return;
|
|
2416
|
+
}
|
|
2417
|
+
this.publishActivity(turn, this.currentActivityLabel(turn));
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
/**
|
|
2421
|
+
* Open the room because a decision is waiting behind it.
|
|
2422
|
+
*
|
|
2423
|
+
* The approval card renders in the app, and the room covers the app, so
|
|
2424
|
+
* without this the turn simply goes quiet: nothing is spoken, nothing is
|
|
2425
|
+
* visible, and the only cue is a call that stopped talking. Sent
|
|
2426
|
+
* immediately rather than latched for the drain the way a shown surface is,
|
|
2427
|
+
* because there is no drain coming. The turn is blocked on this decision,
|
|
2428
|
+
* and the whole point is that the user can make it now.
|
|
2429
|
+
*
|
|
2430
|
+
* The latch is cleared as well, so a turn that also showed a surface does
|
|
2431
|
+
* not send a second minimize once its speech ends; the room is already open.
|
|
2432
|
+
*/
|
|
2433
|
+
private revealRoomForPendingApproval(
|
|
2434
|
+
turn: ActiveAssistantTurn,
|
|
2435
|
+
requestId: string,
|
|
2436
|
+
): void {
|
|
2437
|
+
if (turn.pendingApproval !== null) {
|
|
2438
|
+
return;
|
|
2439
|
+
}
|
|
2440
|
+
turn.pendingApproval = {
|
|
2441
|
+
requestId,
|
|
2442
|
+
label: this.pendingApprovalLabel(turn),
|
|
2443
|
+
};
|
|
2444
|
+
turn.minimizeRequested = false;
|
|
2445
|
+
// Say what the turn is actually doing on the surfaces that are not the
|
|
2446
|
+
// app. Without this the island keeps showing the tool as running for the
|
|
2447
|
+
// whole wait — and it is the surface most likely to be the only one the
|
|
2448
|
+
// user can see, since the case this exists for is a phone put down.
|
|
2449
|
+
// Carrying the request id is what makes the line answerable there rather
|
|
2450
|
+
// than merely accurate.
|
|
2451
|
+
this.refreshActivity(turn);
|
|
2452
|
+
void this.sendFrame(
|
|
2453
|
+
{ type: "minimize_room", turnId: turn.turnId },
|
|
2454
|
+
() => !this.isClosed,
|
|
2455
|
+
);
|
|
2456
|
+
// Spoken, because opening the room is only a cue for someone looking at
|
|
2457
|
+
// the screen, and the case this exists for is a phone the user has put
|
|
2458
|
+
// down. One line, not narration: the turn is not working, it is waiting,
|
|
2459
|
+
// and it says which.
|
|
2460
|
+
this.enqueueFillerPhrase(turn, APPROVAL_PENDING_PHRASE);
|
|
2461
|
+
}
|
|
2462
|
+
|
|
2463
|
+
/** Clear the wait once a decision lands, so the turn narrates normally again. */
|
|
2464
|
+
private clearAwaitingApproval(turn: ActiveAssistantTurn): void {
|
|
2465
|
+
turn.pendingApproval = null;
|
|
2466
|
+
// Put the activity line back to whatever the turn resumed doing, and
|
|
2467
|
+
// retire the request id with it, so the island's Approve/Deny buttons go
|
|
2468
|
+
// away the moment the decision is no longer the user's to make — including
|
|
2469
|
+
// when it was made somewhere else entirely (the card in the app, the
|
|
2470
|
+
// 45-second fallback, a superseding message).
|
|
2471
|
+
this.refreshActivity(turn);
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
/**
|
|
2475
|
+
* Tell the client what the turn is doing, if it changed.
|
|
2476
|
+
*
|
|
2477
|
+
* De-duplicated for the same reason the Live Activity reporter de-duplicates
|
|
2478
|
+
* phases: this frame reaches ActivityKit, whose update budget is finite and
|
|
2479
|
+
* whose overflow is dropped silently, and a turn that runs four file reads in
|
|
2480
|
+
* a row would otherwise spend that budget restating one line.
|
|
2481
|
+
*
|
|
2482
|
+
* Fire-and-forget. An activity label is a flourish, and nothing about the
|
|
2483
|
+
* conversation may wait on one.
|
|
2484
|
+
*
|
|
2485
|
+
* Callers whose line depends on turn state go through
|
|
2486
|
+
* {@link refreshActivity}; this is called directly only to clear the line
|
|
2487
|
+
* outright, which a cancelled or finished turn does regardless of what it
|
|
2488
|
+
* was waiting on.
|
|
2489
|
+
*/
|
|
2490
|
+
private publishActivity(
|
|
2491
|
+
turn: ActiveAssistantTurn,
|
|
2492
|
+
label: string,
|
|
2493
|
+
approvalRequestId?: string,
|
|
2494
|
+
): void {
|
|
2495
|
+
// De-duplicated on the request id as well as the wording. The two move
|
|
2496
|
+
// independently: a wait can be entered and left without the tool line
|
|
2497
|
+
// changing at all, and a label-only check would swallow the frame that
|
|
2498
|
+
// retires the approval — leaving the island's buttons up with nothing
|
|
2499
|
+
// behind them.
|
|
2500
|
+
if (
|
|
2501
|
+
turn.activityLabel === label &&
|
|
2502
|
+
turn.publishedApprovalRequestId === (approvalRequestId ?? null)
|
|
2503
|
+
) {
|
|
2504
|
+
return;
|
|
2505
|
+
}
|
|
2506
|
+
turn.activityLabel = label;
|
|
2507
|
+
turn.publishedApprovalRequestId = approvalRequestId ?? null;
|
|
2508
|
+
void this.sendFrame(
|
|
2509
|
+
{
|
|
2510
|
+
type: "activity",
|
|
2511
|
+
turnId: turn.turnId,
|
|
2512
|
+
label,
|
|
2513
|
+
...(approvalRequestId !== undefined ? { approvalRequestId } : {}),
|
|
2514
|
+
},
|
|
2515
|
+
() => !this.isClosed,
|
|
2516
|
+
);
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2320
2519
|
private clearActiveAssistantTurn(token: symbol): void {
|
|
2321
2520
|
if (this.activeAssistantTurn?.token !== token) {
|
|
2322
2521
|
return;
|
|
@@ -3409,6 +3608,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3409
3608
|
assistantCompleted: false,
|
|
3410
3609
|
ttsDone: false,
|
|
3411
3610
|
minimizeRequested: false,
|
|
3611
|
+
activityLabel: "",
|
|
3612
|
+
publishedApprovalRequestId: null,
|
|
3613
|
+
pendingApproval: null,
|
|
3412
3614
|
ttsAudioStarted: false,
|
|
3413
3615
|
finalized: false,
|
|
3414
3616
|
speculativePending: opts?.speculative === true,
|
|
@@ -3610,6 +3812,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3610
3812
|
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
3611
3813
|
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
3612
3814
|
}),
|
|
3815
|
+
onApprovalPending: (requestId) => {
|
|
3816
|
+
this.revealRoomForPendingApproval(activeTurn, requestId);
|
|
3817
|
+
},
|
|
3818
|
+
onApprovalsResolved: () => {
|
|
3819
|
+
this.clearAwaitingApproval(activeTurn);
|
|
3820
|
+
},
|
|
3613
3821
|
content: leg.content,
|
|
3614
3822
|
isInbound: true,
|
|
3615
3823
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
@@ -3835,6 +4043,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3835
4043
|
});
|
|
3836
4044
|
current.progress.opsSinceNarration += 1;
|
|
3837
4045
|
current.progress.stateEpoch += 1;
|
|
4046
|
+
this.refreshActivity(current);
|
|
3838
4047
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
3839
4048
|
// Definitive tool use means the turn is guaranteed slow: speak
|
|
3840
4049
|
// the floor-holding ack now instead of waiting out the
|
|
@@ -3888,6 +4097,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3888
4097
|
}
|
|
3889
4098
|
}
|
|
3890
4099
|
current.progress.stateEpoch += 1;
|
|
4100
|
+
// Showing a surface implies revealing it: the room is a
|
|
4101
|
+
// full-screen overlay, so a surface rendered behind it is a
|
|
4102
|
+
// surface nobody sees.
|
|
4103
|
+
//
|
|
4104
|
+
// **Latched on the result, not the tool start.** A ui call can be
|
|
4105
|
+
// rejected (no `surface_type`, an empty card, a client that never
|
|
4106
|
+
// acks), and a reveal driven by the attempt would minimize the room
|
|
4107
|
+
// to show nothing at all. Only a call that came back without an
|
|
4108
|
+
// error actually put something on screen.
|
|
4109
|
+
//
|
|
4110
|
+
// A dismissal clears it again, so a turn that shows a surface and
|
|
4111
|
+
// then takes it away does not reveal an empty screen. Last write
|
|
4112
|
+
// wins, which is the right reading of "what did this turn leave up"
|
|
4113
|
+
// without tracking surfaces individually.
|
|
4114
|
+
if (event.isError !== true) {
|
|
4115
|
+
if (revealsUiSurface(event.toolName)) {
|
|
4116
|
+
current.minimizeRequested = true;
|
|
4117
|
+
} else if (dismissesUiSurface(event.toolName)) {
|
|
4118
|
+
current.minimizeRequested = false;
|
|
4119
|
+
}
|
|
4120
|
+
}
|
|
4121
|
+
this.refreshActivity(current);
|
|
3891
4122
|
this.maybeNarrateProgress(current, trigger);
|
|
3892
4123
|
},
|
|
3893
4124
|
},
|
|
@@ -4259,6 +4490,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4259
4490
|
return (
|
|
4260
4491
|
this.isActiveAssistantTurn(turn.token) &&
|
|
4261
4492
|
!turn.assistantCompleted &&
|
|
4493
|
+
// Nothing is in flight while a decision is pending, so every phrase
|
|
4494
|
+
// narration has would be a lie about who the call is waiting on. The
|
|
4495
|
+
// turn says so once, when it starts waiting, and is quiet after that.
|
|
4496
|
+
turn.pendingApproval === null &&
|
|
4262
4497
|
this.turnAudioIdle(turn)
|
|
4263
4498
|
);
|
|
4264
4499
|
}
|
|
@@ -4569,6 +4804,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4569
4804
|
!this.isClosed,
|
|
4570
4805
|
);
|
|
4571
4806
|
|
|
4807
|
+
// The turn is over, so nothing is running. Ordinarily the line is
|
|
4808
|
+
// already clear (the last tool_result cleared it); this is the net
|
|
4809
|
+
// for a turn that ends with an op still open, which would otherwise
|
|
4810
|
+
// leave the last tool it touched on screen through the next silence.
|
|
4811
|
+
this.publishActivity(currentTurn, "");
|
|
4812
|
+
|
|
4572
4813
|
// Drain-scoped minimize: the latched marker is consumed here, after
|
|
4573
4814
|
// the turn's speech has fully drained — never mid-speech, never for
|
|
4574
4815
|
// a barged-in turn, at most once per turn.
|
|
@@ -18,6 +18,7 @@ const _LIVE_VOICE_SERVER_FRAME_TYPES = [
|
|
|
18
18
|
"stt_partial",
|
|
19
19
|
"stt_final",
|
|
20
20
|
"thinking",
|
|
21
|
+
"activity",
|
|
21
22
|
"assistant_text_delta",
|
|
22
23
|
"tts_audio",
|
|
23
24
|
"tts_done",
|
|
@@ -217,6 +218,38 @@ export interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
|
|
|
217
218
|
readonly turnId: string;
|
|
218
219
|
}
|
|
219
220
|
|
|
221
|
+
/**
|
|
222
|
+
* What the assistant is doing inside a turn, as one short user-facing line.
|
|
223
|
+
*
|
|
224
|
+
* Exists because a live-voice turn can go silent for a long time while it
|
|
225
|
+
* works, and the surfaces that show the session (the iOS Live Activity, and
|
|
226
|
+
* the room) can otherwise only say "Thinking...". The label is composed here
|
|
227
|
+
* rather than by each surface for the same reason phase wording is composed
|
|
228
|
+
* once: the Live Activity has two independent drivers (this socket and an APNs
|
|
229
|
+
* push the daemon dispatches), they must carry identical content, and the only
|
|
230
|
+
* way to guarantee that is for both to be handed the same string.
|
|
231
|
+
*
|
|
232
|
+
* An empty `label` means "no current activity", which is what a turn's end
|
|
233
|
+
* sends. Emitted only on change, never per tool result.
|
|
234
|
+
*/
|
|
235
|
+
export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
|
|
236
|
+
readonly type: "activity";
|
|
237
|
+
readonly turnId: string;
|
|
238
|
+
readonly label: string;
|
|
239
|
+
/**
|
|
240
|
+
* The confirmation this turn is blocked on, when the label describes a wait
|
|
241
|
+
* rather than work in flight. Absent otherwise.
|
|
242
|
+
*
|
|
243
|
+
* It travels so that a surface outside the app — the Live Activity's
|
|
244
|
+
* Approve/Deny buttons — can answer the request it was drawn against rather
|
|
245
|
+
* than whatever is pending by the time the press lands. Content on a Lock
|
|
246
|
+
* Screen can be seconds old, and a decision is the one thing that must not
|
|
247
|
+
* be re-pointed when it is: the id lets a client drop a press aimed at a
|
|
248
|
+
* request already answered, timed out, or superseded.
|
|
249
|
+
*/
|
|
250
|
+
readonly approvalRequestId?: string;
|
|
251
|
+
}
|
|
252
|
+
|
|
220
253
|
export interface LiveVoiceAssistantTextDeltaServerFrame extends LiveVoiceServerFrameBase {
|
|
221
254
|
readonly type: "assistant_text_delta";
|
|
222
255
|
readonly text: string;
|
|
@@ -321,6 +354,7 @@ export type LiveVoiceServerFrame =
|
|
|
321
354
|
| LiveVoiceSttPartialServerFrame
|
|
322
355
|
| LiveVoiceSttFinalServerFrame
|
|
323
356
|
| LiveVoiceThinkingServerFrame
|
|
357
|
+
| LiveVoiceActivityServerFrame
|
|
324
358
|
| LiveVoiceAssistantTextDeltaServerFrame
|
|
325
359
|
| LiveVoiceTtsAudioServerFrame
|
|
326
360
|
| LiveVoiceTtsDoneServerFrame
|
|
@@ -341,6 +375,7 @@ export type LiveVoiceServerFramePayload =
|
|
|
341
375
|
| WithoutSeq<LiveVoiceSttPartialServerFrame>
|
|
342
376
|
| WithoutSeq<LiveVoiceSttFinalServerFrame>
|
|
343
377
|
| WithoutSeq<LiveVoiceThinkingServerFrame>
|
|
378
|
+
| WithoutSeq<LiveVoiceActivityServerFrame>
|
|
344
379
|
| WithoutSeq<LiveVoiceAssistantTextDeltaServerFrame>
|
|
345
380
|
| WithoutSeq<LiveVoiceTtsAudioServerFrame>
|
|
346
381
|
| WithoutSeq<LiveVoiceTtsDoneServerFrame>
|
|
@@ -31,12 +31,13 @@ mock.module("../../../../contacts/contacts-write.js", () => ({
|
|
|
31
31
|
}));
|
|
32
32
|
|
|
33
33
|
import {
|
|
34
|
-
|
|
34
|
+
__resetSlackMentionCachesForTests,
|
|
35
35
|
slackProvider,
|
|
36
36
|
} from "../adapter.js";
|
|
37
37
|
|
|
38
38
|
const originalFetch = globalThis.fetch;
|
|
39
39
|
let userInfoCalls: string[] = [];
|
|
40
|
+
let channelInfoCalls: string[] = [];
|
|
40
41
|
|
|
41
42
|
function installFetchStub() {
|
|
42
43
|
globalThis.fetch = (async (
|
|
@@ -63,7 +64,49 @@ function fakeSlackResponse(url: string): Record<string, unknown> {
|
|
|
63
64
|
const parsed = new URL(url);
|
|
64
65
|
const method = parsed.pathname.split("/").at(-1);
|
|
65
66
|
|
|
67
|
+
if (method === "conversations.info") {
|
|
68
|
+
const channelId = parsed.searchParams.get("channel") ?? "";
|
|
69
|
+
channelInfoCalls.push(channelId);
|
|
70
|
+
if (channelId === "CPRODMODELS1") {
|
|
71
|
+
return {
|
|
72
|
+
ok: true,
|
|
73
|
+
channel: { id: "CPRODMODELS1", name: "prod-models" },
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return { ok: false, error: "channel_not_found" };
|
|
77
|
+
}
|
|
78
|
+
|
|
66
79
|
if (method === "conversations.history") {
|
|
80
|
+
if (parsed.searchParams.get("channel") === "C_CHANNEL_MENTIONS") {
|
|
81
|
+
return {
|
|
82
|
+
ok: true,
|
|
83
|
+
has_more: false,
|
|
84
|
+
messages: [
|
|
85
|
+
{
|
|
86
|
+
type: "message",
|
|
87
|
+
ts: "1700000009.001000",
|
|
88
|
+
user: "USENDER",
|
|
89
|
+
text: "post in <#CPRODMODELS1> and <#COTHER1|other-name> please",
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (parsed.searchParams.get("channel") === "C_PRIVATE_MENTION") {
|
|
96
|
+
return {
|
|
97
|
+
ok: true,
|
|
98
|
+
has_more: false,
|
|
99
|
+
messages: [
|
|
100
|
+
{
|
|
101
|
+
type: "message",
|
|
102
|
+
ts: "1700000010.001100",
|
|
103
|
+
user: "USENDER",
|
|
104
|
+
text: "see <#CINVISIBLE1> for details",
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
67
110
|
if (parsed.searchParams.get("channel") === "C_USERINFO_FAIL") {
|
|
68
111
|
return {
|
|
69
112
|
ok: true,
|
|
@@ -318,8 +361,9 @@ function makeOAuthConnection(
|
|
|
318
361
|
|
|
319
362
|
describe("Slack adapter mention rendering", () => {
|
|
320
363
|
beforeEach(async () => {
|
|
321
|
-
|
|
364
|
+
__resetSlackMentionCachesForTests();
|
|
322
365
|
userInfoCalls = [];
|
|
366
|
+
channelInfoCalls = [];
|
|
323
367
|
getSecureKeyAsyncMock.mockReset();
|
|
324
368
|
getSecureKeyAsyncMock.mockImplementation(async (key: string) => {
|
|
325
369
|
if (key === credentialKey("slack_channel", "bot_token")) {
|
|
@@ -516,6 +560,44 @@ describe("Slack adapter mention rendering", () => {
|
|
|
516
560
|
});
|
|
517
561
|
});
|
|
518
562
|
|
|
563
|
+
test("getHistory renders inline channel mentions without rebinding the message's own channel", async () => {
|
|
564
|
+
// A message read from channel A that mentions channels B and C: the
|
|
565
|
+
// message stays attributed to A while the inline mentions render B/C.
|
|
566
|
+
const messages = await slackProvider.getHistory(
|
|
567
|
+
undefined,
|
|
568
|
+
"C_CHANNEL_MENTIONS",
|
|
569
|
+
);
|
|
570
|
+
|
|
571
|
+
expect(messages).toHaveLength(1);
|
|
572
|
+
// Bare token resolves via conversations.info; pipe token uses its own
|
|
573
|
+
// embedded label with no lookup.
|
|
574
|
+
expect(messages[0].text).toBe(
|
|
575
|
+
"post in #prod-models and #other-name please",
|
|
576
|
+
);
|
|
577
|
+
expect(messages[0].conversationId).toBe("C_CHANNEL_MENTIONS");
|
|
578
|
+
expect(channelInfoCalls).toEqual(["CPRODMODELS1"]);
|
|
579
|
+
});
|
|
580
|
+
|
|
581
|
+
test("getHistory falls back to #unknown-channel for channels this auth cannot see, and caches the failure", async () => {
|
|
582
|
+
const first = await slackProvider.getHistory(
|
|
583
|
+
undefined,
|
|
584
|
+
"C_PRIVATE_MENTION",
|
|
585
|
+
);
|
|
586
|
+
const second = await slackProvider.getHistory(
|
|
587
|
+
undefined,
|
|
588
|
+
"C_PRIVATE_MENTION",
|
|
589
|
+
);
|
|
590
|
+
|
|
591
|
+
expect(first[0].text).toBe("see #unknown-channel for details");
|
|
592
|
+
expect(second[0].text).toBe("see #unknown-channel for details");
|
|
593
|
+
expect(first[0].text).not.toContain("CINVISIBLE1");
|
|
594
|
+
// channel_not_found is a definitive answer for this auth: one lookup,
|
|
595
|
+
// not one per history read.
|
|
596
|
+
expect(channelInfoCalls.filter((id) => id === "CINVISIBLE1")).toHaveLength(
|
|
597
|
+
1,
|
|
598
|
+
);
|
|
599
|
+
});
|
|
600
|
+
|
|
519
601
|
test("getThreadReplies renders Slack user mentions for model-facing text without changing sender identity", async () => {
|
|
520
602
|
const messages = await slackProvider.getThreadReplies!(
|
|
521
603
|
undefined,
|