@vellumai/assistant 0.11.5 → 0.11.6-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +70 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +16 -1
- package/node_modules/@vellumai/gateway-client/src/index.ts +6 -2
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +121 -61
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/openapi.yaml +421 -15
- package/package.json +1 -1
- package/scripts/sync-web-search-catalog.ts +6 -0
- package/src/__tests__/app-pin-store.test.ts +149 -0
- package/src/__tests__/channel-availability-routes.test.ts +23 -1
- package/src/__tests__/channel-readiness-discord.test.ts +231 -0
- package/src/__tests__/channel-readiness-service.test.ts +126 -0
- package/src/__tests__/channel-readiness-slack-remote.test.ts +141 -0
- package/src/__tests__/channel-reply-delivery.test.ts +4 -4
- package/src/__tests__/client-os-metadata-persistence.test.ts +23 -10
- package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
- package/src/__tests__/conversation-error.test.ts +17 -0
- package/src/__tests__/conversation-seed-composer.test.ts +8 -0
- package/src/__tests__/conversation-slash-commands.test.ts +8 -0
- package/src/__tests__/disk-pressure-policy.test.ts +6 -0
- package/src/__tests__/gemini-provider.test.ts +138 -0
- package/src/__tests__/history-repair.test.ts +105 -3
- package/src/__tests__/identity-routes.test.ts +1 -0
- package/src/__tests__/llm-catalog-parity.test.ts +45 -0
- package/src/__tests__/migration-import-from-path.test.ts +349 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +102 -0
- package/src/__tests__/oauth-commands-routes.test.ts +89 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +7 -6
- package/src/__tests__/openai-provider.test.ts +18 -0
- package/src/__tests__/openai-responses-provider.test.ts +18 -0
- package/src/__tests__/platform-callback-registration.test.ts +184 -0
- package/src/__tests__/plugin-api-store-credential.test.ts +71 -3
- package/src/__tests__/pricing.test.ts +2 -2
- package/src/__tests__/public-ingress-urls.test.ts +36 -0
- package/src/__tests__/resolve-trust-class.test.ts +0 -48
- package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +49 -0
- package/src/__tests__/settings-routes.test.ts +85 -3
- package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
- package/src/agent/history-repair/history-repair.ts +45 -14
- package/src/agent/loop.ts +4 -1
- package/src/api/constants/profile-config-validation.ts +60 -0
- package/src/api/events/tool-result.ts +6 -1
- package/src/api/events/watch-retro-completed.ts +52 -0
- package/src/api/index.ts +11 -0
- package/src/apps/app-pin-reconciler.ts +92 -0
- package/src/apps/app-pin-store.ts +125 -0
- package/src/channels/gateway-channel-socket-health.ts +32 -0
- package/src/channels/gateway-discord-admission.ts +32 -0
- package/src/channels/types.ts +20 -0
- package/src/cli/commands/__tests__/conversations-slack.test.ts +1 -1
- package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
- package/src/cli/commands/__tests__/inference-providers.test.ts +67 -2
- package/src/cli/commands/channels/__tests__/channels.test.ts +85 -0
- package/src/cli/commands/channels/index.ts +45 -31
- package/src/cli/commands/inference-profiles.ts +56 -3
- package/src/cli/commands/inference-providers.ts +28 -2
- package/src/cli/commands/oauth/index.help.ts +7 -1
- package/src/cli/commands/oauth/request.test.ts +290 -0
- package/src/cli/commands/oauth/request.ts +57 -41
- package/src/cli/lib/bundled-marketplace.json +14 -1
- package/src/cli/lib/open-browser.test.ts +67 -0
- package/src/cli/lib/open-browser.ts +24 -5
- package/src/config/__tests__/profile-materialization.test.ts +26 -0
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +6 -0
- package/src/config/bundled-skills/schedule/SKILL.md +1 -1
- package/src/config/feature-flag-registry.json +17 -1
- package/src/config/profile-materialization.ts +29 -0
- package/src/config/sanitize-for-transfer.ts +16 -0
- package/src/config/schemas/llm.ts +7 -0
- package/src/config/schemas/services.ts +6 -0
- package/src/context/outbound-sanitize.ts +6 -0
- package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
- package/src/daemon/conversation-error.ts +24 -2
- package/src/daemon/conversation-slash.ts +6 -15
- package/src/daemon/daemon-control.ts +1 -0
- package/src/daemon/disk-pressure-policy.ts +7 -1
- package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
- package/src/daemon/handlers/config-ingress.ts +115 -5
- package/src/daemon/lifecycle.ts +26 -0
- package/src/daemon/message-types/web-activity.ts +3 -2
- package/src/daemon/trust-context.ts +0 -37
- package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
- package/src/inbound/platform-callback-registration.ts +28 -2
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/inbound/tunnel-probe.ts +261 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +25 -0
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +8 -2
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +212 -10
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +5 -2
- package/src/live-voice/live-voice-connection.ts +46 -8
- package/src/live-voice/live-voice-manager.ts +25 -0
- package/src/live-voice/live-voice-session-manager.ts +318 -2
- package/src/live-voice/live-voice-session.ts +52 -2
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +126 -68
- package/src/messaging/providers/channel-transport.ts +64 -47
- package/src/messaging/providers/discord/send.test.ts +46 -1
- package/src/messaging/providers/discord/send.ts +51 -0
- package/src/messaging/providers/discord/transport.ts +26 -3
- package/src/messaging/providers/index.ts +22 -47
- package/src/messaging/providers/slack/send.test.ts +83 -26
- package/src/messaging/providers/slack/send.ts +120 -51
- package/src/messaging/providers/slack/stream-tasks.test.ts +26 -0
- package/src/messaging/providers/slack/stream-tasks.ts +39 -0
- package/src/messaging/providers/slack/transport.ts +24 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +109 -12
- package/src/messaging/providers/telegram-bot/send.ts +43 -0
- package/src/messaging/providers/telegram-bot/transport.ts +25 -8
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +30 -7
- package/src/notifications/adapters/telegram.ts +48 -1
- package/src/notifications/assistant-reply-producer.ts +7 -7
- package/src/notifications/conversation-seed-composer.ts +7 -2
- package/src/oauth/byo-connection.test.ts +63 -0
- package/src/oauth/byo-connection.ts +16 -15
- package/src/oauth/connection.test.ts +111 -0
- package/src/oauth/connection.ts +142 -1
- package/src/oauth/platform-connection.test.ts +34 -0
- package/src/oauth/platform-connection.ts +28 -5
- package/src/oauth/seed-providers.ts +15 -1
- package/src/permissions/types.ts +3 -1
- package/src/persistence/conversation-crud.ts +46 -0
- package/src/persistence/conversation-types.ts +11 -9
- package/src/persistence/db-async-query.ts +2 -1
- package/src/persistence/db-maintenance.ts +15 -0
- package/src/persistence/embeddings/qdrant-manager.ts +1 -0
- package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
- package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
- package/src/persistence/migrations/369-create-app-pins.ts +37 -0
- package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
- package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +17 -0
- package/src/persistence/schema/watch.ts +29 -0
- package/src/persistence/steps.ts +6 -0
- package/src/plugins/mtime-cache.ts +11 -0
- package/src/providers/__tests__/retry-network-error.test.ts +84 -0
- package/src/providers/connection-resolution.ts +23 -1
- package/src/providers/content-blocks.ts +9 -0
- package/src/providers/fetch-provider-catalog.ts +19 -0
- package/src/providers/gemini/client.ts +13 -5
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
- package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
- package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
- package/src/providers/inference/adapter-factory.ts +0 -9
- package/src/providers/inference/credential-rotation.ts +61 -0
- package/src/providers/inference/endpoint-probe.ts +115 -0
- package/src/providers/inference/profile-probe.ts +256 -0
- package/src/providers/model-catalog.ts +170 -125
- package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +42 -60
- package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
- package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts +34 -2
- package/src/providers/openai/api-error-normalization.ts +16 -2
- package/src/providers/openai/chat-completions-provider.ts +75 -29
- package/src/providers/openai/responses-provider.ts +5 -2
- package/src/providers/openrouter/client.ts +0 -1
- package/src/providers/provider-send-message.ts +11 -0
- package/src/providers/retry.ts +6 -0
- package/src/providers/search-provider-catalog.ts +20 -0
- package/src/providers/vercel-ai-gateway/client.ts +0 -1
- package/src/runtime/AGENTS.md +1 -0
- package/src/runtime/__tests__/desktop-presence.test.ts +27 -4
- package/src/runtime/__tests__/host-observe.test.ts +302 -0
- package/src/runtime/channel-readiness-service.ts +214 -14
- package/src/runtime/channel-readiness-types.ts +49 -2
- package/src/runtime/channel-reply-delivery.ts +2 -2
- package/src/runtime/desktop-presence.ts +24 -21
- package/src/runtime/host-observe.ts +246 -0
- package/src/runtime/http-server.ts +181 -1
- package/src/runtime/migrations/__tests__/staged-import-path.test.ts +104 -0
- package/src/runtime/migrations/staged-import-path.ts +116 -0
- package/src/runtime/routes/__tests__/app-pin-routes.test.ts +383 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +35 -56
- package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
- package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
- package/src/runtime/routes/app-management-routes.ts +140 -29
- package/src/runtime/routes/channel-availability-routes.ts +1 -0
- package/src/runtime/routes/channel-readiness-routes.ts +14 -2
- package/src/runtime/routes/conversation-query-routes.ts +10 -0
- package/src/runtime/routes/guardian-approval-interception.ts +24 -33
- package/src/runtime/routes/host-cu-routes.ts +18 -0
- package/src/runtime/routes/identity-routes.ts +2 -0
- package/src/runtime/routes/inbound-message-handler.ts +10 -7
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +166 -308
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +158 -335
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/inference-profiles-routes.ts +232 -31
- package/src/runtime/routes/inference-provider-connection-routes.ts +24 -4
- package/src/runtime/routes/ingress-status-routes.ts +180 -0
- package/src/runtime/routes/live-voice-routes.test.ts +40 -1
- package/src/runtime/routes/live-voice-routes.ts +34 -0
- package/src/runtime/routes/migration-routes.ts +218 -10
- package/src/runtime/routes/oauth-commands-routes.ts +23 -16
- package/src/runtime/routes/plugins-routes.ts +12 -28
- package/src/runtime/routes/question-routes.ts +6 -0
- package/src/runtime/routes/secret-routes.ts +7 -27
- package/src/runtime/routes/settings-routes.ts +9 -6
- package/src/runtime/routes/watch-routes.ts +807 -0
- package/src/runtime/slack-reply-session.test.ts +230 -121
- package/src/runtime/slack-reply-session.ts +113 -81
- package/src/runtime/{slack-task-progress.test.ts → task-progress.test.ts} +1 -28
- package/src/runtime/{slack-task-progress.ts → task-progress.ts} +30 -51
- package/src/security/__tests__/untrusted-content.test.ts +42 -0
- package/src/security/untrusted-content.ts +28 -9
- package/src/telemetry/__tests__/live-voice-funnel.test.ts +108 -0
- package/src/telemetry/live-voice-funnel.ts +75 -8
- package/src/tools/credentials/store.ts +18 -6
- package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
- package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
- package/src/tools/network/__tests__/web-search.test.ts +97 -2
- package/src/tools/network/firecrawl-compat.ts +90 -0
- package/src/tools/network/web-fetch.ts +142 -62
- package/src/tools/network/web-search.ts +141 -55
- package/src/tools/types.ts +2 -1
- package/src/util/oauth-request-body.test.ts +74 -0
- package/src/util/oauth-request-body.ts +60 -0
- package/src/util/worker-process.ts +1 -0
- package/src/watch/__tests__/watch-retro.test.ts +665 -0
- package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
- package/src/watch/__tests__/watch-timeline.test.ts +670 -0
- package/src/watch/watch-retro.ts +480 -0
- package/src/watch/watch-session-manager.ts +575 -0
- package/src/watch/watch-timeline.ts +848 -0
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
*
|
|
20
20
|
* const connection = createLiveVoiceConnection({
|
|
21
21
|
* send: (frame) => socket.send(JSON.stringify(frame)),
|
|
22
|
+
* close: () => socket.close(),
|
|
22
23
|
* });
|
|
23
24
|
* socket.on("message", (msg) => void connection.handleMessage(msg));
|
|
24
25
|
* socket.on("close", () => connection.release());
|
|
@@ -26,8 +27,11 @@
|
|
|
26
27
|
|
|
27
28
|
import { getLogger } from "../util/logger.js";
|
|
28
29
|
import { getLiveVoiceSessionManager } from "./live-voice-manager.js";
|
|
29
|
-
import type {
|
|
30
|
-
|
|
30
|
+
import type {
|
|
31
|
+
LiveVoiceSessionCloseReason,
|
|
32
|
+
LiveVoiceSessionManager,
|
|
33
|
+
LiveVoiceStartSessionResult,
|
|
34
|
+
} from "./live-voice-session-manager.js";
|
|
31
35
|
import {
|
|
32
36
|
type LiveVoiceClientFrame,
|
|
33
37
|
type LiveVoiceProtocolError,
|
|
@@ -66,23 +70,39 @@ export interface LiveVoiceConnection {
|
|
|
66
70
|
release(reason?: LiveVoiceSessionCloseReason): void;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Create a live voice connection bound to the caller's `send` transport.
|
|
75
|
+
*
|
|
76
|
+
* `close` is how the daemon hangs up. It is optional only because a transport
|
|
77
|
+
* may not be able to (an in-process pipe), but a real one should always pass
|
|
78
|
+
* it: without it, a session the daemon reclaims from an absent client leaves
|
|
79
|
+
* the transport open behind it.
|
|
80
|
+
*/
|
|
70
81
|
export function createLiveVoiceConnection(options: {
|
|
71
82
|
send: LiveVoiceFrameSender;
|
|
83
|
+
close?: () => void;
|
|
72
84
|
}): LiveVoiceConnection {
|
|
73
85
|
return new LiveVoiceConnectionImpl(
|
|
74
86
|
options.send,
|
|
75
87
|
getLiveVoiceSessionManager(),
|
|
88
|
+
options.close,
|
|
76
89
|
);
|
|
77
90
|
}
|
|
78
91
|
|
|
79
92
|
class LiveVoiceConnectionImpl implements LiveVoiceConnection {
|
|
80
93
|
private activeSessionId: string | undefined;
|
|
81
94
|
private lastSeq = 0;
|
|
95
|
+
/**
|
|
96
|
+
* A `start` still waiting on the manager. It holds no session id yet, so
|
|
97
|
+
* without this handle a transport that closes mid-wait has nothing to
|
|
98
|
+
* release and the start would go on to build a session for a dead socket.
|
|
99
|
+
*/
|
|
100
|
+
private pendingStart: AbortController | null = null;
|
|
82
101
|
|
|
83
102
|
constructor(
|
|
84
103
|
private readonly send: LiveVoiceFrameSender,
|
|
85
104
|
private readonly manager: LiveVoiceSessionManager,
|
|
105
|
+
private readonly closeTransport?: () => void,
|
|
86
106
|
) {}
|
|
87
107
|
|
|
88
108
|
get sessionId(): string | undefined {
|
|
@@ -136,6 +156,8 @@ class LiveVoiceConnectionImpl implements LiveVoiceConnection {
|
|
|
136
156
|
}
|
|
137
157
|
|
|
138
158
|
release(reason: LiveVoiceSessionCloseReason = "transport_closed"): void {
|
|
159
|
+
this.pendingStart?.abort();
|
|
160
|
+
this.pendingStart = null;
|
|
139
161
|
const sessionId = this.activeSessionId;
|
|
140
162
|
this.activeSessionId = undefined;
|
|
141
163
|
if (!sessionId) {
|
|
@@ -168,11 +190,27 @@ class LiveVoiceConnectionImpl implements LiveVoiceConnection {
|
|
|
168
190
|
this.activeSessionId = undefined;
|
|
169
191
|
}
|
|
170
192
|
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
193
|
+
const pending = new AbortController();
|
|
194
|
+
this.pendingStart = pending;
|
|
195
|
+
let result: LiveVoiceStartSessionResult;
|
|
196
|
+
try {
|
|
197
|
+
result = await this.manager.startSession(
|
|
198
|
+
frame,
|
|
199
|
+
{
|
|
200
|
+
sendFrame: (serverFrame) => {
|
|
201
|
+
this.sendFrame(serverFrame);
|
|
202
|
+
},
|
|
203
|
+
...(this.closeTransport
|
|
204
|
+
? { closeTransport: this.closeTransport }
|
|
205
|
+
: {}),
|
|
206
|
+
},
|
|
207
|
+
{ signal: pending.signal },
|
|
208
|
+
);
|
|
209
|
+
} finally {
|
|
210
|
+
if (this.pendingStart === pending) {
|
|
211
|
+
this.pendingStart = null;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
176
214
|
if (result.status === "accepted") {
|
|
177
215
|
this.activeSessionId = result.sessionId;
|
|
178
216
|
}
|
|
@@ -11,10 +11,13 @@
|
|
|
11
11
|
|
|
12
12
|
import { createRequire } from "node:module";
|
|
13
13
|
|
|
14
|
+
import { getLogger } from "../util/logger.js";
|
|
14
15
|
import { LiveVoiceSessionManager } from "./live-voice-session-manager.js";
|
|
15
16
|
|
|
16
17
|
const require = createRequire(import.meta.url);
|
|
17
18
|
|
|
19
|
+
const log = getLogger("live-voice-manager");
|
|
20
|
+
|
|
18
21
|
type LiveVoiceSessionFactory =
|
|
19
22
|
typeof import("./live-voice-session.js").createLiveVoiceSession;
|
|
20
23
|
|
|
@@ -49,6 +52,28 @@ export function getLiveVoiceSessionManager(): LiveVoiceSessionManager {
|
|
|
49
52
|
).createLiveVoiceSession;
|
|
50
53
|
return createSession(context);
|
|
51
54
|
},
|
|
55
|
+
// The manager reclaims a slot on its own only when a session stopped
|
|
56
|
+
// behaving: either its teardown overran the close budget, or its client
|
|
57
|
+
// went silent on a mode that streams continuously. Both end somebody's
|
|
58
|
+
// call without them asking, and neither leaves a trace anywhere else, so
|
|
59
|
+
// they are logged at warn.
|
|
60
|
+
onSlotEvent: (event) => {
|
|
61
|
+
if (event.kind === "close_timed_out") {
|
|
62
|
+
log.warn(
|
|
63
|
+
{
|
|
64
|
+
sessionId: event.sessionId,
|
|
65
|
+
reason: event.reason,
|
|
66
|
+
timeoutMs: event.timeoutMs,
|
|
67
|
+
},
|
|
68
|
+
"Live voice session close overran its budget, freeing the slot anyway",
|
|
69
|
+
);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
log.warn(
|
|
73
|
+
{ sessionId: event.sessionId, timeoutMs: event.timeoutMs },
|
|
74
|
+
"Live voice client went silent, releasing the session",
|
|
75
|
+
);
|
|
76
|
+
},
|
|
52
77
|
});
|
|
53
78
|
}
|
|
54
79
|
return manager;
|
|
@@ -15,8 +15,37 @@ export type LiveVoiceSessionCloseReason =
|
|
|
15
15
|
| "error"
|
|
16
16
|
| "websocket_close"
|
|
17
17
|
| "transport_closed"
|
|
18
|
+
| "client_timeout"
|
|
19
|
+
| "forced_end"
|
|
18
20
|
| "manager_shutdown";
|
|
19
21
|
|
|
22
|
+
/**
|
|
23
|
+
* How long a session's own teardown may run before the manager stops waiting
|
|
24
|
+
* on it and frees the slot anyway.
|
|
25
|
+
*
|
|
26
|
+
* `close()` is not a formality: it delivers a pending continuation into the
|
|
27
|
+
* conversation, aborts an in-flight assistant turn, and flushes metrics, all
|
|
28
|
+
* of which reach the network. Awaiting it unbounded is what turned a single
|
|
29
|
+
* wedged teardown into a daemon that could never run live voice again, because
|
|
30
|
+
* a closing session still reads as the active one to the busy check while
|
|
31
|
+
* being un-releasable by any retry (LUM-3440).
|
|
32
|
+
*/
|
|
33
|
+
export const DEFAULT_LIVE_VOICE_CLOSE_TIMEOUT_MS = 10_000;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* How long a `server_vad` session may go without a single inbound frame
|
|
37
|
+
* before the manager assumes its client is gone and releases the slot.
|
|
38
|
+
*
|
|
39
|
+
* The daemon's WebSocket peer is the gateway, not the client. A half-open
|
|
40
|
+
* transport anywhere further out (a phone that left the network mid-call, a
|
|
41
|
+
* dropped velay tunnel) delivers no close event here, so transport liveness
|
|
42
|
+
* proves nothing about whether anyone is still on the call. Inbound frames do:
|
|
43
|
+
* a `server_vad` client streams PCM continuously by construction (muted turns
|
|
44
|
+
* send silence rather than nothing, so the server VAD is never starved), so a
|
|
45
|
+
* full minute of wire silence means the client is gone, not quiet.
|
|
46
|
+
*/
|
|
47
|
+
export const DEFAULT_LIVE_VOICE_CLIENT_SILENCE_TIMEOUT_MS = 60_000;
|
|
48
|
+
|
|
20
49
|
export interface LiveVoiceSession {
|
|
21
50
|
start(): MaybePromise<void>;
|
|
22
51
|
handleClientFrame(frame: LiveVoiceClientFrame): MaybePromise<void>;
|
|
@@ -26,6 +55,14 @@ export interface LiveVoiceSession {
|
|
|
26
55
|
|
|
27
56
|
export interface LiveVoiceServerFrameSink {
|
|
28
57
|
sendFrame(frame: LiveVoiceServerFrame): MaybePromise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Hang up the transport this session runs on. Optional, and called when the
|
|
60
|
+
* slot is taken back rather than given up (the silence watchdog, or a forced
|
|
61
|
+
* end). Reclaiming without hanging up leaves a client streaming into a
|
|
62
|
+
* session that no longer exists, still showing an active call, with every
|
|
63
|
+
* socket in the chain out to it open and nothing behind them.
|
|
64
|
+
*/
|
|
65
|
+
closeTransport?(): void;
|
|
29
66
|
}
|
|
30
67
|
|
|
31
68
|
export interface LiveVoiceSessionFactoryContext {
|
|
@@ -45,9 +82,79 @@ export type LiveVoiceSessionFactory = (
|
|
|
45
82
|
context: LiveVoiceSessionFactoryContext,
|
|
46
83
|
) => LiveVoiceSession;
|
|
47
84
|
|
|
85
|
+
/**
|
|
86
|
+
* A slot the manager reclaimed on the session's behalf rather than at its
|
|
87
|
+
* owner's request. Both mean a live voice session ended without anyone asking
|
|
88
|
+
* it to, so both are worth a log line and a metric: they are the only evidence
|
|
89
|
+
* that would explain a user who suddenly could not talk, and (for
|
|
90
|
+
* `close_timed_out`) the only warning that a teardown path has started
|
|
91
|
+
* hanging.
|
|
92
|
+
*/
|
|
93
|
+
export type LiveVoiceSlotEvent =
|
|
94
|
+
| {
|
|
95
|
+
kind: "close_timed_out";
|
|
96
|
+
sessionId: string;
|
|
97
|
+
reason: LiveVoiceSessionCloseReason;
|
|
98
|
+
timeoutMs: number;
|
|
99
|
+
}
|
|
100
|
+
| {
|
|
101
|
+
kind: "client_silence_timeout";
|
|
102
|
+
sessionId: string;
|
|
103
|
+
timeoutMs: number;
|
|
104
|
+
};
|
|
105
|
+
|
|
48
106
|
export interface LiveVoiceSessionManagerOptions {
|
|
49
107
|
createSession: LiveVoiceSessionFactory;
|
|
50
108
|
createSessionId?: () => string;
|
|
109
|
+
/** Overrides {@link DEFAULT_LIVE_VOICE_CLOSE_TIMEOUT_MS}; `0` disables. */
|
|
110
|
+
closeTimeoutMs?: number;
|
|
111
|
+
/**
|
|
112
|
+
* Overrides {@link DEFAULT_LIVE_VOICE_CLIENT_SILENCE_TIMEOUT_MS}; `0`
|
|
113
|
+
* disables the watchdog.
|
|
114
|
+
*/
|
|
115
|
+
clientSilenceTimeoutMs?: number;
|
|
116
|
+
/** Reports a slot the manager reclaimed itself. */
|
|
117
|
+
onSlotEvent?: (event: LiveVoiceSlotEvent) => void;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Close reasons where the manager took the slot back rather than its owner
|
|
122
|
+
* handing it over. Only these hang up the transport: a session that ended
|
|
123
|
+
* because its own socket closed has no transport left to close, and one that
|
|
124
|
+
* failed post-`ready` deliberately keeps the socket so the client can start
|
|
125
|
+
* again on it.
|
|
126
|
+
*/
|
|
127
|
+
const RECLAIMED_CLOSE_REASONS: ReadonlySet<LiveVoiceSessionCloseReason> =
|
|
128
|
+
new Set(["client_timeout", "forced_end"]);
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Read through a call so the check is not narrowed away: an abort that lands
|
|
132
|
+
* while the manager awaits the outgoing slot is exactly the one that matters,
|
|
133
|
+
* and it happens after the first read.
|
|
134
|
+
*/
|
|
135
|
+
function isAborted(signal?: AbortSignal): boolean {
|
|
136
|
+
return signal?.aborted === true;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Resolve when the closing session lets go of the slot, or as soon as the
|
|
141
|
+
* waiting transport aborts, whichever happens first.
|
|
142
|
+
*/
|
|
143
|
+
function waitForSlotRelease(
|
|
144
|
+
released: Promise<void>,
|
|
145
|
+
signal?: AbortSignal,
|
|
146
|
+
): Promise<void> {
|
|
147
|
+
if (signal === undefined) {
|
|
148
|
+
return released;
|
|
149
|
+
}
|
|
150
|
+
return new Promise<void>((resolve) => {
|
|
151
|
+
const settle = () => {
|
|
152
|
+
signal.removeEventListener("abort", settle);
|
|
153
|
+
resolve();
|
|
154
|
+
};
|
|
155
|
+
signal.addEventListener("abort", settle, { once: true });
|
|
156
|
+
void released.then(settle, settle);
|
|
157
|
+
});
|
|
51
158
|
}
|
|
52
159
|
|
|
53
160
|
export class LiveVoiceSessionStartupError extends Error {
|
|
@@ -62,6 +169,13 @@ export type LiveVoiceStartSessionResult =
|
|
|
62
169
|
status: "accepted";
|
|
63
170
|
sessionId: string;
|
|
64
171
|
}
|
|
172
|
+
| {
|
|
173
|
+
/**
|
|
174
|
+
* The transport that asked went away before the session could be
|
|
175
|
+
* created. Nothing was started and nothing needs releasing.
|
|
176
|
+
*/
|
|
177
|
+
status: "aborted";
|
|
178
|
+
}
|
|
65
179
|
| {
|
|
66
180
|
status: "failed";
|
|
67
181
|
sessionId: string;
|
|
@@ -93,17 +207,31 @@ export type LiveVoiceSessionReleaseResult =
|
|
|
93
207
|
interface ActiveLiveVoiceSession {
|
|
94
208
|
sessionId: string;
|
|
95
209
|
session: LiveVoiceSession;
|
|
210
|
+
sink: LiveVoiceServerFrameSink;
|
|
96
211
|
closing: boolean;
|
|
212
|
+
silenceTimer: ReturnType<typeof setTimeout> | null;
|
|
213
|
+
/** Resolves once this session has let go of the slot, however it went. */
|
|
214
|
+
readonly released: Promise<void>;
|
|
215
|
+
resolveReleased: () => void;
|
|
97
216
|
}
|
|
98
217
|
|
|
99
218
|
export class LiveVoiceSessionManager {
|
|
100
219
|
private readonly createSession: LiveVoiceSessionFactory;
|
|
101
220
|
private readonly createSessionId: () => string;
|
|
221
|
+
private readonly closeTimeoutMs: number;
|
|
222
|
+
private readonly clientSilenceTimeoutMs: number;
|
|
223
|
+
private readonly onSlotEvent: (event: LiveVoiceSlotEvent) => void;
|
|
102
224
|
private activeSession: ActiveLiveVoiceSession | null = null;
|
|
103
225
|
|
|
104
226
|
constructor(options: LiveVoiceSessionManagerOptions) {
|
|
105
227
|
this.createSession = options.createSession;
|
|
106
228
|
this.createSessionId = options.createSessionId ?? randomUUID;
|
|
229
|
+
this.closeTimeoutMs =
|
|
230
|
+
options.closeTimeoutMs ?? DEFAULT_LIVE_VOICE_CLOSE_TIMEOUT_MS;
|
|
231
|
+
this.clientSilenceTimeoutMs =
|
|
232
|
+
options.clientSilenceTimeoutMs ??
|
|
233
|
+
DEFAULT_LIVE_VOICE_CLIENT_SILENCE_TIMEOUT_MS;
|
|
234
|
+
this.onSlotEvent = options.onSlotEvent ?? (() => {});
|
|
107
235
|
}
|
|
108
236
|
|
|
109
237
|
get activeSessionId(): string | null {
|
|
@@ -113,7 +241,37 @@ export class LiveVoiceSessionManager {
|
|
|
113
241
|
async startSession(
|
|
114
242
|
startFrame: LiveVoiceClientStartFrame,
|
|
115
243
|
sink: LiveVoiceServerFrameSink,
|
|
244
|
+
options: { signal?: AbortSignal } = {},
|
|
116
245
|
): Promise<LiveVoiceStartSessionResult> {
|
|
246
|
+
const signal = options.signal;
|
|
247
|
+
if (isAborted(signal)) {
|
|
248
|
+
return { status: "aborted" };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// A slot that is already tearing down is not a reason to turn a client
|
|
252
|
+
// away, it is a reason to wait: the client asking is usually the same one
|
|
253
|
+
// whose transport just died, reconnecting a second or two later, and its
|
|
254
|
+
// old session's close can easily outlast that gap (it delivers a pending
|
|
255
|
+
// continuation and unwinds an in-flight turn). Answering `busy` there is
|
|
256
|
+
// what turned a recoverable blip into "Another live-voice session is
|
|
257
|
+
// active." with nothing behind it (LUM-3440). Waiting keeps the invariant
|
|
258
|
+
// that matters (one session's audio devices are never live alongside the
|
|
259
|
+
// next one's), and the wait is bounded by the close budget.
|
|
260
|
+
const closingSession = this.activeSession?.closing
|
|
261
|
+
? this.activeSession
|
|
262
|
+
: null;
|
|
263
|
+
if (closingSession !== null) {
|
|
264
|
+
await waitForSlotRelease(closingSession.released, signal);
|
|
265
|
+
// The wait can outlive the transport that asked (its own connect
|
|
266
|
+
// timeout is the same order as the close budget, and the user can give
|
|
267
|
+
// up sooner). Building a session for a socket that is gone would hand
|
|
268
|
+
// the slot to nobody, which is the failure this change exists to
|
|
269
|
+
// remove, so the caller aborts and the start is abandoned here.
|
|
270
|
+
if (isAborted(signal)) {
|
|
271
|
+
return { status: "aborted" };
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
117
275
|
const existingSessionId = this.activeSessionId;
|
|
118
276
|
if (existingSessionId !== null) {
|
|
119
277
|
const busySequencer = createLiveVoiceServerFrameSequencer();
|
|
@@ -144,7 +302,27 @@ export class LiveVoiceSessionManager {
|
|
|
144
302
|
},
|
|
145
303
|
};
|
|
146
304
|
const session = this.createSession(context);
|
|
147
|
-
|
|
305
|
+
let resolveReleased = (): void => {};
|
|
306
|
+
const released = new Promise<void>((resolve) => {
|
|
307
|
+
resolveReleased = resolve;
|
|
308
|
+
});
|
|
309
|
+
const active: ActiveLiveVoiceSession = {
|
|
310
|
+
sessionId,
|
|
311
|
+
session,
|
|
312
|
+
sink,
|
|
313
|
+
closing: false,
|
|
314
|
+
silenceTimer: null,
|
|
315
|
+
released,
|
|
316
|
+
resolveReleased,
|
|
317
|
+
};
|
|
318
|
+
this.activeSession = active;
|
|
319
|
+
// Armed before `start()` rather than after it, so the watchdog also covers
|
|
320
|
+
// a startup that hangs: a provider dial with no timeout of its own would
|
|
321
|
+
// otherwise hold the slot with no session to release and no frame to
|
|
322
|
+
// explain it. Nothing refreshes the deadline during startup, since the
|
|
323
|
+
// transport rejects client audio until `start` resolves, so the first
|
|
324
|
+
// window is startup plus the client's first frame.
|
|
325
|
+
this.armClientSilenceWatchdog(active, startFrame);
|
|
148
326
|
|
|
149
327
|
try {
|
|
150
328
|
await session.start();
|
|
@@ -167,6 +345,7 @@ export class LiveVoiceSessionManager {
|
|
|
167
345
|
if (active === null) {
|
|
168
346
|
return { status: "not_found" };
|
|
169
347
|
}
|
|
348
|
+
this.noteClientActivity(active);
|
|
170
349
|
|
|
171
350
|
try {
|
|
172
351
|
await active.session.handleClientFrame(frame);
|
|
@@ -190,6 +369,7 @@ export class LiveVoiceSessionManager {
|
|
|
190
369
|
if (active === null) {
|
|
191
370
|
return { status: "not_found" };
|
|
192
371
|
}
|
|
372
|
+
this.noteClientActivity(active);
|
|
193
373
|
|
|
194
374
|
try {
|
|
195
375
|
await active.session.handleBinaryAudio(chunk);
|
|
@@ -211,16 +391,40 @@ export class LiveVoiceSessionManager {
|
|
|
211
391
|
}
|
|
212
392
|
|
|
213
393
|
active.closing = true;
|
|
394
|
+
this.clearClientSilenceWatchdog(active);
|
|
395
|
+
if (RECLAIMED_CLOSE_REASONS.has(reason)) {
|
|
396
|
+
this.hangUpTransport(active);
|
|
397
|
+
}
|
|
214
398
|
try {
|
|
215
|
-
await
|
|
399
|
+
await this.closeWithinBudget(active, reason);
|
|
216
400
|
} finally {
|
|
217
401
|
if (this.activeSession === active) {
|
|
218
402
|
this.activeSession = null;
|
|
219
403
|
}
|
|
404
|
+
active.resolveReleased();
|
|
220
405
|
}
|
|
221
406
|
return { released: true, sessionId };
|
|
222
407
|
}
|
|
223
408
|
|
|
409
|
+
/**
|
|
410
|
+
* Release whatever session currently holds the slot, whoever owns it.
|
|
411
|
+
*
|
|
412
|
+
* The recovery path for a client that has no other way back: its own
|
|
413
|
+
* transport is gone (so `end` cannot be sent) but the slot is still claimed,
|
|
414
|
+
* which is the "Another live-voice session is active." dead end. Reports
|
|
415
|
+
* `released: false` when nothing holds the slot, including when the holder
|
|
416
|
+
* is already closing, since that teardown frees the slot on its own budget.
|
|
417
|
+
*/
|
|
418
|
+
async endActiveSession(
|
|
419
|
+
reason: LiveVoiceSessionCloseReason = "forced_end",
|
|
420
|
+
): Promise<LiveVoiceSessionReleaseResult> {
|
|
421
|
+
const sessionId = this.activeSessionId;
|
|
422
|
+
if (sessionId === null) {
|
|
423
|
+
return { released: false };
|
|
424
|
+
}
|
|
425
|
+
return await this.releaseSession(sessionId, reason);
|
|
426
|
+
}
|
|
427
|
+
|
|
224
428
|
/**
|
|
225
429
|
* Whether the given session still holds the manager slot. Transports use
|
|
226
430
|
* this to heal a stale per-connection binding: a session that failed
|
|
@@ -246,4 +450,116 @@ export class LiveVoiceSessionManager {
|
|
|
246
450
|
// The original session error is more useful to callers than a cleanup error.
|
|
247
451
|
}
|
|
248
452
|
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Await the session's teardown, but only for as long as the close budget.
|
|
456
|
+
*
|
|
457
|
+
* A teardown that overruns is abandoned rather than cancelled: it keeps
|
|
458
|
+
* unwinding in the background (the session has already marked itself closed,
|
|
459
|
+
* so it cannot resurrect), while the caller (and the next client) stop
|
|
460
|
+
* waiting on it. Freeing the slot early can leave the outgoing session's
|
|
461
|
+
* providers briefly overlapping the next one's, which is a degraded call;
|
|
462
|
+
* holding it forever is no call at all, ever again, until the daemon
|
|
463
|
+
* restarts.
|
|
464
|
+
*/
|
|
465
|
+
private async closeWithinBudget(
|
|
466
|
+
active: ActiveLiveVoiceSession,
|
|
467
|
+
reason: LiveVoiceSessionCloseReason,
|
|
468
|
+
): Promise<void> {
|
|
469
|
+
const closed = Promise.resolve(active.session.close(reason));
|
|
470
|
+
// The race below stops awaiting `closed` on a timeout, so anchor a handler
|
|
471
|
+
// now: an abandoned teardown that rejects later must not surface as an
|
|
472
|
+
// unhandled rejection.
|
|
473
|
+
closed.catch(() => {});
|
|
474
|
+
|
|
475
|
+
if (this.closeTimeoutMs <= 0) {
|
|
476
|
+
await closed;
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
481
|
+
const budget = new Promise<"timed_out">((resolve) => {
|
|
482
|
+
timer = setTimeout(() => resolve("timed_out"), this.closeTimeoutMs);
|
|
483
|
+
timer.unref?.();
|
|
484
|
+
});
|
|
485
|
+
try {
|
|
486
|
+
const outcome = await Promise.race([
|
|
487
|
+
closed.then(() => "closed" as const),
|
|
488
|
+
budget,
|
|
489
|
+
]);
|
|
490
|
+
if (outcome === "timed_out") {
|
|
491
|
+
this.onSlotEvent({
|
|
492
|
+
kind: "close_timed_out",
|
|
493
|
+
sessionId: active.sessionId,
|
|
494
|
+
reason,
|
|
495
|
+
timeoutMs: this.closeTimeoutMs,
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
} finally {
|
|
499
|
+
clearTimeout(timer);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Start the client-silence watchdog for a session whose mode guarantees a
|
|
505
|
+
* continuous inbound stream. A `manual` session legitimately sits idle
|
|
506
|
+
* between push-to-talk segments, so wire silence says nothing about its
|
|
507
|
+
* client and it gets no watchdog.
|
|
508
|
+
*/
|
|
509
|
+
private armClientSilenceWatchdog(
|
|
510
|
+
active: ActiveLiveVoiceSession,
|
|
511
|
+
startFrame: LiveVoiceClientStartFrame,
|
|
512
|
+
): void {
|
|
513
|
+
if (
|
|
514
|
+
this.clientSilenceTimeoutMs <= 0 ||
|
|
515
|
+
startFrame.turnDetection !== "server_vad"
|
|
516
|
+
) {
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
this.scheduleClientSilenceWatchdog(active);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
private scheduleClientSilenceWatchdog(active: ActiveLiveVoiceSession): void {
|
|
523
|
+
const timer = setTimeout(() => {
|
|
524
|
+
active.silenceTimer = null;
|
|
525
|
+
if (this.activeSession !== active || active.closing) {
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
this.onSlotEvent({
|
|
529
|
+
kind: "client_silence_timeout",
|
|
530
|
+
sessionId: active.sessionId,
|
|
531
|
+
timeoutMs: this.clientSilenceTimeoutMs,
|
|
532
|
+
});
|
|
533
|
+
void this.releaseSession(active.sessionId, "client_timeout").catch(() => {
|
|
534
|
+
// Best-effort reclamation; the slot is dropped either way.
|
|
535
|
+
});
|
|
536
|
+
}, this.clientSilenceTimeoutMs);
|
|
537
|
+
timer.unref?.();
|
|
538
|
+
active.silenceTimer = timer;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** Push the watchdog deadline out: the client is demonstrably still there. */
|
|
542
|
+
private noteClientActivity(active: ActiveLiveVoiceSession): void {
|
|
543
|
+
if (active.silenceTimer === null) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
clearTimeout(active.silenceTimer);
|
|
547
|
+
this.scheduleClientSilenceWatchdog(active);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
private hangUpTransport(active: ActiveLiveVoiceSession): void {
|
|
551
|
+
try {
|
|
552
|
+
active.sink.closeTransport?.();
|
|
553
|
+
} catch {
|
|
554
|
+
// A transport that cannot be hung up is exactly the case the watchdog
|
|
555
|
+
// exists for. Reclaiming the slot still has to happen.
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private clearClientSilenceWatchdog(active: ActiveLiveVoiceSession): void {
|
|
560
|
+
if (active.silenceTimer !== null) {
|
|
561
|
+
clearTimeout(active.silenceTimer);
|
|
562
|
+
active.silenceTimer = null;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
249
565
|
}
|
|
@@ -71,7 +71,10 @@ import type {
|
|
|
71
71
|
SttStreamServerEvent,
|
|
72
72
|
} from "../stt/types.js";
|
|
73
73
|
import { getSubagentManager } from "../subagent/index.js";
|
|
74
|
-
import {
|
|
74
|
+
import {
|
|
75
|
+
liveVoiceEndScreen,
|
|
76
|
+
liveVoiceSilenceReason,
|
|
77
|
+
} from "../telemetry/live-voice-funnel.js";
|
|
75
78
|
import { getToolOwner } from "../tools/registry.js";
|
|
76
79
|
import {
|
|
77
80
|
createReasoningTagFilter,
|
|
@@ -1126,6 +1129,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1126
1129
|
* session that never failed.
|
|
1127
1130
|
*/
|
|
1128
1131
|
private failureCode: LiveVoiceProtocolErrorCode | null = null;
|
|
1132
|
+
/**
|
|
1133
|
+
* How far a session that never produced a turn actually got. Latched rather
|
|
1134
|
+
* than derived at close, because every one of these is a transient the
|
|
1135
|
+
* teardown has already destroyed by then: `state` has moved on, the current
|
|
1136
|
+
* utterance is gone, and the turn detector is disposed.
|
|
1137
|
+
*
|
|
1138
|
+
* A quarter of sessions end with no turn at all, and these three booleans are
|
|
1139
|
+
* what separate a microphone that never opened from one that was muted from a
|
|
1140
|
+
* user who simply left. See `telemetry/live-voice-funnel.ts`.
|
|
1141
|
+
*/
|
|
1142
|
+
private reachedActive = false;
|
|
1143
|
+
private receivedAudio = false;
|
|
1144
|
+
private detectedSpeech = false;
|
|
1145
|
+
private dispatchedTurn = false;
|
|
1129
1146
|
// Non-null iff the start frame requested turnDetection "server_vad".
|
|
1130
1147
|
private readonly turnDetector: MediaTurnDetector | null;
|
|
1131
1148
|
// Base energy gate for server-VAD speech classification. During estimated
|
|
@@ -1419,6 +1436,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1419
1436
|
// through the pending/pre-roll paths and flushes on arm. An arm failure
|
|
1420
1437
|
// surfaces as a non-recoverable error frame instead of a start rejection.
|
|
1421
1438
|
this.state = "active";
|
|
1439
|
+
this.reachedActive = true;
|
|
1422
1440
|
void this.armUtterance().catch(() => {});
|
|
1423
1441
|
this.metrics.markReady();
|
|
1424
1442
|
await this.sendFrame({
|
|
@@ -1522,9 +1540,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1522
1540
|
// `recorded_at` minus the started row's, so it must be written before the
|
|
1523
1541
|
// teardown below, which awaits a pending continuation and can run long.
|
|
1524
1542
|
const failed = this.state === "failed";
|
|
1543
|
+
// Only a session that never dispatched a turn gets a silence reason; for
|
|
1544
|
+
// every other session the question is meaningless.
|
|
1545
|
+
const silenceReason = this.dispatchedTurn
|
|
1546
|
+
? null
|
|
1547
|
+
: liveVoiceSilenceReason({
|
|
1548
|
+
reachedActive: this.reachedActive,
|
|
1549
|
+
receivedAudio: this.receivedAudio,
|
|
1550
|
+
detectedSpeech: this.detectedSpeech,
|
|
1551
|
+
});
|
|
1525
1552
|
recordLiveVoiceSessionEnded({
|
|
1526
1553
|
sessionId: this.context.sessionId,
|
|
1527
|
-
screen: liveVoiceEndScreen(
|
|
1554
|
+
screen: liveVoiceEndScreen(
|
|
1555
|
+
reason,
|
|
1556
|
+
failed ? this.failureCode : null,
|
|
1557
|
+
silenceReason,
|
|
1558
|
+
),
|
|
1528
1559
|
outcome: failed ? "failed" : "completed",
|
|
1529
1560
|
});
|
|
1530
1561
|
|
|
@@ -1831,6 +1862,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1831
1862
|
}
|
|
1832
1863
|
|
|
1833
1864
|
private async handleAudio(chunk: Buffer): Promise<void> {
|
|
1865
|
+
// Both transports funnel through here, and this runs before any of the
|
|
1866
|
+
// early returns below. The question it answers is "did the microphone ever
|
|
1867
|
+
// open", which a chunk arriving at all settles regardless of what the
|
|
1868
|
+
// session then does with it.
|
|
1869
|
+
this.receivedAudio = true;
|
|
1834
1870
|
if (this.turnDetector) {
|
|
1835
1871
|
await this.handleServerVadAudio(this.turnDetector, chunk);
|
|
1836
1872
|
return;
|
|
@@ -1849,6 +1885,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1849
1885
|
// The chunk belongs to an utterance the user is still holding the button
|
|
1850
1886
|
// for, whether or not the transcriber has produced any text for it yet.
|
|
1851
1887
|
utterance.manualAudioCaptured = true;
|
|
1888
|
+
// Manual sessions have no VAD, so the user holding the talk button is
|
|
1889
|
+
// the only speech signal available. Without this every abandoned manual
|
|
1890
|
+
// session would be misfiled as `no_turn` instead of `no_speech`.
|
|
1891
|
+
this.detectedSpeech = true;
|
|
1852
1892
|
this.collectUserAudio(utterance, chunk);
|
|
1853
1893
|
if (utterance.phase === "pending") {
|
|
1854
1894
|
// The transcriber is still arming (session start overlaps the STT
|
|
@@ -1968,6 +2008,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1968
2008
|
// resets it.
|
|
1969
2009
|
if (hasSpeech || this.vadPreRollHasSpeech) {
|
|
1970
2010
|
utterance.speechRouted = true;
|
|
2011
|
+
this.detectedSpeech = true;
|
|
1971
2012
|
}
|
|
1972
2013
|
for (const preRollChunk of this.takeVadPreRoll()) {
|
|
1973
2014
|
await this.routeVadAudio(utterance, preRollChunk);
|
|
@@ -2221,6 +2262,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2221
2262
|
}
|
|
2222
2263
|
if (preRollHadSpeech) {
|
|
2223
2264
|
utterance.speechRouted = true;
|
|
2265
|
+
this.detectedSpeech = true;
|
|
2224
2266
|
}
|
|
2225
2267
|
}
|
|
2226
2268
|
|
|
@@ -4725,6 +4767,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4725
4767
|
};
|
|
4726
4768
|
|
|
4727
4769
|
try {
|
|
4770
|
+
// Latched before the await, not after: this flag only decides whether the
|
|
4771
|
+
// end event carries a silence classification, and the dashboard decides
|
|
4772
|
+
// silence from persisted turn rows instead. Setting it after would let a
|
|
4773
|
+
// dispatch that persisted a turn and then threw stamp "this session was
|
|
4774
|
+
// silent" onto a session that has turns, a contradictory row. Setting it
|
|
4775
|
+
// before can at worst leave a silent session unexplained, which is a gap
|
|
4776
|
+
// rather than a false statement.
|
|
4777
|
+
this.dispatchedTurn = true;
|
|
4728
4778
|
const handle = await this.startVoiceTurn({
|
|
4729
4779
|
conversationId: this.conversationId,
|
|
4730
4780
|
voiceSessionId: this.context.sessionId,
|