@openclaw/discord 2026.9.1-beta.1 → 2026.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-inspect-BY-ve3yH.js +45 -0
- package/dist/account-inspect-api.js +1 -1
- package/dist/{accounts-CfQ-vSyf.js → accounts-CwJQoLjM.js} +89 -45
- package/dist/action-runtime-api.js +1 -1
- package/dist/{activities-api-q-9PaaHH.js → activities-api-CTcGq3Hf.js} +30 -17
- package/dist/activities-api.js +1 -1
- package/dist/{allow-list-BDMimASD.js → allow-list-DBTtR2dz.js} +13 -35
- package/dist/{api-DNVbJyEy.js → api-BfmEP6wp.js} +6 -5
- package/dist/api.js +18 -18
- package/dist/{approval-handler.runtime-B7qhK3fR.js → approval-handler.runtime-DUSe5fyH.js} +19 -12
- package/dist/{audit-CBEv_ERu.js → audit-DpPfwoJM.js} +8 -7
- package/dist/{channel-Dx1tMTfx.js → channel-BHajPgwR.js} +29 -23
- package/dist/{channel-actions.runtime-CLWWu9nC.js → channel-actions.runtime-DS8pXo0S.js} +38 -38
- package/dist/channel-config-api.js +1 -1
- package/dist/channel-plugin-api.js +1 -1
- package/dist/{channel.setup-DOzXS2Pk.js → channel.setup-BWmMe1tT.js} +18 -18
- package/dist/command-owners-D78sAoOz.js +10 -0
- package/dist/{components-B9V5o361.js → components-BimX47Mq.js} +111 -149
- package/dist/{config-schema-BR13IAx6.js → config-schema-B797-Spt.js} +19 -12
- package/dist/{conversation-binding-route-DJYrY2TQ.js → conversation-binding-route-DuR5NGIx.js} +1 -1
- package/dist/{directory-config-BK6eqekJ.js → directory-config-CD2v2Szs.js} +2 -2
- package/dist/directory-contract-api.js +1 -1
- package/dist/{directory-live-DSAJSrJk.js → directory-live-DaJzWJ2N.js} +4 -4
- package/dist/{doctor-Cm7Y1pJ2.js → doctor-D4HH1inA.js} +9 -6
- package/dist/{doctor-contract-DsWvKIXF.js → doctor-contract-Co5vRCZ6.js} +9 -6
- package/dist/doctor-contract-api.js +31 -28
- package/dist/{handle-action.guild-admin-BdwCRHmm.js → handle-action.guild-admin-4We6wWQU.js} +6 -3
- package/dist/index.js +1 -1
- package/dist/ingress-identity-XNmW-mZS.js +60 -0
- package/dist/{message-handler-BXpWZyni.js → message-handler-DhgBabyU.js} +61 -70
- package/dist/{message-handler.preflight-Ba5LAY_X.js → message-handler.preflight-oZ7dQCd0.js} +41 -101
- package/dist/{message-handler.process-3HEzmemS.js → message-handler.process-BzCYf1pa.js} +82 -108
- package/dist/{message-text-aiwO5R2F.js → message-text-DK4NnBg-.js} +11 -6
- package/dist/{outbound-adapter-BE4jSq2d.js → outbound-adapter-CtggmGh9.js} +52 -67
- package/dist/{outbound-components-oLpg8Qzb.js → outbound-components-Dn8SJWMS.js} +5 -4
- package/dist/{outbound-session-route-Cg15ZFJd.js → outbound-session-route-Q_HEusBQ.js} +57 -15
- package/dist/{owner-access-UlYi9t0M.js → owner-access-Cp8r6Ipf.js} +9 -7
- package/dist/{pluralkit-Tiyl70dx.js → pluralkit-BY75YcK0.js} +2 -2
- package/dist/{probe-n6ZXxK8V.js → probe-3ovFAp3b.js} +3 -3
- package/dist/{probe.runtime-CN4bdPfZ.js → probe.runtime-BckHn3cJ.js} +1 -1
- package/dist/{provider-session.runtime-B9xe-Qaf.js → provider-session.runtime-BL2lHBH7.js} +3 -3
- package/dist/{provider-B11xwilq.js → provider-uKmmrDM4.js} +299 -136
- package/dist/provider.runtime-C-IkWF2w.js +2 -0
- package/dist/{realtime-session.runtime-CU8IQUS-.js → realtime-session.runtime-CGYm1t_0.js} +798 -448
- package/dist/{resolve-allowlist-common-BLoYaxEG.js → resolve-allowlist-common-sKIKy-cF.js} +3 -3
- package/dist/{resolve-channels-C2sT_n67.js → resolve-channels-CoiMlWy0.js} +4 -4
- package/dist/{resolve-users-CzpjLwjy.js → resolve-users-DJpiokfW.js} +5 -4
- package/dist/{retry-Cv4jCU5Z.js → retry-DQt1zDgH.js} +356 -214
- package/dist/{retry-after-DdLPmPoa.js → retry-after-BrLjLUJi.js} +2 -1
- package/dist/{runtime-CY4PRMF-.js → runtime-BHsdEdr5.js} +2 -2
- package/dist/{runtime-BHrTlFkf.js → runtime-Ghtfh5ZA.js} +151 -109
- package/dist/runtime-api.js +20 -20
- package/dist/{secret-config-contract-B3i6WxV5.js → secret-config-contract-DFu9hfU8.js} +14 -6
- package/dist/secret-contract-api.js +1 -1
- package/dist/{security-audit-BfJhAXZp.js → security-audit-DKfK-iPY.js} +10 -11
- package/dist/security-audit-contract-api.js +1 -1
- package/dist/{security-audit.runtime-BVKF9VtY.js → security-audit.runtime-sKZyKgpx.js} +1 -1
- package/dist/{send-C49MFF9j.js → send-CZeWKUiA.js} +132 -100
- package/dist/{send.components-DKKAw5VX.js → send.components-BG5IDABj.js} +21 -15
- package/dist/{send.outbound-zam0TrZx.js → send.outbound-DHuFUL6W.js} +42 -63
- package/dist/{send.shared-JjljcV4O.js → send.shared-B0n_fWzu.js} +77 -149
- package/dist/{sender-identity-BJtfyise.js → sender-identity-B8Ncfi63.js} +1 -1
- package/dist/setup-plugin-api.js +1 -1
- package/dist/{shared-_5EcP4jW.js → shared-C3dKun_X.js} +24 -15
- package/dist/{subagent-hooks-DQzvOptn.js → subagent-hooks-ZoQXEI1E.js} +2 -2
- package/dist/subagent-hooks-api.js +1 -1
- package/dist/{system-events-C7yxaHWP.js → system-events-DFF_awl7.js} +2 -2
- package/dist/{target-resolver-vxIh6O3-.js → target-resolver-DLtHDwMK.js} +6 -5
- package/dist/targets-BObRyaKX.js +3 -0
- package/dist/{thread-bindings-DW5F9vat.js → thread-bindings-DKvsNjQR.js} +7 -6
- package/dist/{thread-bindings.manager-ByOtwjo_.js → thread-bindings.manager-pbbRT5gb.js} +38 -29
- package/dist/{thread-bindings.session-updates-CJb-BpPL.js → thread-bindings.session-updates-t6jeFB1-.js} +3 -2
- package/dist/{thread-bindings.state-D8OMdk4O.js → thread-bindings.state-DtgOqsbX.js} +34 -1
- package/dist/{threading-agH1T_6s.js → threading-BCx49X-6.js} +8 -6
- package/dist/{timeouts-B9cy6RvQ.js → timeouts-D86uWLt_.js} +1 -1
- package/dist/{transcripts-source-BR11iHA4.js → transcripts-source-Dc6Nnli5.js} +55 -7
- package/dist/transcripts-source-api.js +1 -1
- package/dist/{typing-Dc_YMY2e.js → typing-CuopV-Hi.js} +2 -2
- package/dist/{voice-runtime-DIeaa8fJ.js → voice-runtime-BQnILUbG.js} +492 -105
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/README.md +1 -1
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/conforms.d.ts +2 -0
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/package.json +3 -3
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/stubArray.d.ts +2 -0
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/stubObject.d.ts +2 -0
- package/node_modules/@discord/embedded-app-sdk/node_modules/@types/lodash/stubString.d.ts +2 -0
- package/node_modules/@discordjs/voice/node_modules/@types/node/README.md +1 -1
- package/node_modules/@discordjs/voice/node_modules/@types/node/buffer.buffer.d.ts +5 -5
- package/node_modules/@discordjs/voice/node_modules/@types/node/crypto.d.ts +2 -2
- package/node_modules/@discordjs/voice/node_modules/@types/node/http.d.ts +16 -0
- package/node_modules/@discordjs/voice/node_modules/@types/node/http2.d.ts +10 -2
- package/node_modules/@discordjs/voice/node_modules/@types/node/package.json +2 -2
- package/node_modules/@discordjs/voice/node_modules/@types/node/process.d.ts +52 -0
- package/node_modules/@discordjs/voice/node_modules/@types/node/quic.d.ts +210 -28
- package/node_modules/@discordjs/voice/node_modules/@types/node/sqlite.d.ts +6 -2
- package/node_modules/@discordjs/voice/node_modules/@types/node/test.d.ts +36 -0
- package/node_modules/typebox/build/format/email.mjs +1 -1
- package/node_modules/typebox/build/format/idna/idn-hostname.mjs +19 -1
- package/node_modules/typebox/build/format/json_pointer.d.mts +2 -2
- package/node_modules/typebox/build/format/json_pointer.mjs +2 -2
- package/node_modules/typebox/build/format/json_pointer_uri_fragment.d.mts +2 -2
- package/node_modules/typebox/build/format/json_pointer_uri_fragment.mjs +2 -2
- package/node_modules/typebox/build/format/relative_json_pointer.d.mts +2 -2
- package/node_modules/typebox/build/format/relative_json_pointer.mjs +2 -2
- package/node_modules/typebox/build/system/memory/assign.mjs +2 -1
- package/node_modules/typebox/build/system/memory/create.mjs +3 -4
- package/node_modules/typebox/build/system/memory/discard.mjs +2 -2
- package/node_modules/typebox/build/system/memory/freeze.d.mts +4 -0
- package/node_modules/typebox/build/system/memory/freeze.mjs +6 -0
- package/node_modules/typebox/build/system/memory/update.mjs +3 -2
- package/node_modules/typebox/package.json +1 -1
- package/node_modules/undici/docs/docs/api/Client.md +4 -0
- package/node_modules/undici/docs/docs/api/EnvHttpProxyAgent.md +3 -2
- package/node_modules/undici/docs/docs/api/EventSource.md +4 -0
- package/node_modules/undici/docs/docs/api/Interceptors.md +7 -3
- package/node_modules/undici/docs/docs/api/Socks5ProxyAgent.md +7 -1
- package/node_modules/undici/lib/cache/memory-cache-store.js +2 -2
- package/node_modules/undici/lib/core/symbols.js +2 -0
- package/node_modules/undici/lib/dispatcher/agent.js +7 -2
- package/node_modules/undici/lib/dispatcher/balanced-pool.js +7 -4
- package/node_modules/undici/lib/dispatcher/client-h1.js +12 -4
- package/node_modules/undici/lib/dispatcher/client-h2.js +43 -3
- package/node_modules/undici/lib/dispatcher/client.js +3 -2
- package/node_modules/undici/lib/dispatcher/dispatcher-base.js +13 -1
- package/node_modules/undici/lib/dispatcher/dispatcher.js +15 -1
- package/node_modules/undici/lib/dispatcher/dispatcher1-wrapper.js +3 -0
- package/node_modules/undici/lib/dispatcher/env-http-proxy-agent.js +10 -3
- package/node_modules/undici/lib/dispatcher/proxy-agent.js +18 -1
- package/node_modules/undici/lib/dispatcher/retry-agent.js +3 -0
- package/node_modules/undici/lib/dispatcher/round-robin-pool.js +1 -1
- package/node_modules/undici/lib/dispatcher/socks5-proxy-agent.js +91 -21
- package/node_modules/undici/lib/handler/cache-handler.js +29 -5
- package/node_modules/undici/lib/handler/decorator-handler.js +7 -1
- package/node_modules/undici/lib/handler/deduplication-handler.js +12 -6
- package/node_modules/undici/lib/handler/redirect-handler.js +21 -4
- package/node_modules/undici/lib/handler/retry-handler.js +148 -30
- package/node_modules/undici/lib/interceptor/cache.js +65 -12
- package/node_modules/undici/lib/interceptor/decompress.js +115 -13
- package/node_modules/undici/lib/interceptor/deduplicate.js +5 -4
- package/node_modules/undici/lib/interceptor/dns.js +7 -0
- package/node_modules/undici/lib/interceptor/dump.js +10 -23
- package/node_modules/undici/lib/mock/mock-agent.js +13 -1
- package/node_modules/undici/lib/mock/mock-utils.js +11 -10
- package/node_modules/undici/lib/util/cache.js +45 -3
- package/node_modules/undici/lib/web/cookies/parse.js +150 -147
- package/node_modules/undici/lib/web/eventsource/eventsource-stream.js +28 -1
- package/node_modules/undici/lib/web/eventsource/eventsource.js +13 -3
- package/node_modules/undici/lib/web/fetch/index.js +6 -0
- package/node_modules/undici/lib/web/fetch/request.js +2 -3
- package/node_modules/undici/lib/web/fetch/util.js +6 -1
- package/node_modules/undici/lib/web/webidl/index.js +3 -3
- package/node_modules/undici/lib/web/websocket/connection.js +1 -1
- package/node_modules/undici/lib/web/websocket/permessage-deflate.js +5 -0
- package/node_modules/undici/lib/web/websocket/stream/websocketstream.js +8 -10
- package/node_modules/undici/package.json +3 -3
- package/node_modules/undici/types/client.d.ts +10 -0
- package/node_modules/undici/types/interceptors.d.ts +2 -0
- package/node_modules/undici/types/mock-agent.d.ts +11 -10
- package/node_modules/undici/types/socks5-proxy-agent.d.ts +2 -0
- package/openclaw.plugin.json +15 -8
- package/package.json +7 -6
- package/skills/discord/SKILL.md +1 -1
- package/dist/account-inspect-BDfKyuvL.js +0 -89
- package/dist/provider.runtime-DiYaHvOz.js +0 -2
- package/dist/sdk-runtime-4geQHtXc.js +0 -338
- package/dist/targets-CtTKvntJ.js +0 -3
|
@@ -1,23 +1,107 @@
|
|
|
1
|
-
import { n as discordRealtimeVoiceSecretOwnerId } from "./secret-config-contract-
|
|
2
|
-
import {
|
|
1
|
+
import { n as discordRealtimeVoiceSecretOwnerId } from "./secret-config-contract-DFu9hfU8.js";
|
|
2
|
+
import { a as createDiscordOpusEncodeStream, c as formatVoiceLogPreview, i as convertRealtimePcm24kMonoToDiscordPcm48kStereo, l as loadDiscordVoiceSdk, n as logVoiceVerbose, o as formatVoiceIngressPrompt, r as convertDiscordPcm48kStereoToRealtimePcm24kMono, s as maybeControlDiscordVoiceAgentRun } from "./voice-runtime-BQnILUbG.js";
|
|
3
3
|
import { asDateTimestampMs, resolveExpiresAtMsFromDurationMs } from "openclaw/plugin-sdk/number-runtime";
|
|
4
4
|
import { formatErrorMessage } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
5
5
|
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
|
|
6
6
|
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
7
|
+
import { createDeferred } from "openclaw/plugin-sdk/extension-shared";
|
|
7
8
|
import { assertSecretOwnerAvailable, isSecretOwnerAvailable } from "openclaw/plugin-sdk/channel-secret-owner-runtime";
|
|
8
|
-
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AGENT_CONTROL_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, buildRealtimeVoiceSessionInstructions, buildRealtimeVoiceSpeakExactMessage, canonicalizeRealtimeVoiceProviderId, classifyRealtimeVoiceConsultToolCall, classifySkippableRealtimeVoiceConsultTranscript, controlRealtimeVoiceAgentRun, createRealtimeVoiceAgentTalkbackQueue,
|
|
9
|
+
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AGENT_CONTROL_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, buildRealtimeVoiceAgentErrorProviderResult, buildRealtimeVoiceSessionInstructions, buildRealtimeVoiceSpeakExactMessage, canonicalizeRealtimeVoiceProviderId, classifyRealtimeVoiceConsultToolCall, classifySkippableRealtimeVoiceConsultTranscript, controlRealtimeVoiceAgentRun, createRealtimeVoiceAgentTalkbackQueue, createRealtimeVoiceOutputActivityTracker, createRealtimeVoiceSessionHarness, isRealtimeVoiceWakeNameRequired, matchRealtimeVoiceActivationName, matchRealtimeVoiceConsultQuestions, parseRealtimeVoiceAgentControlToolArgs, realtimeVoiceAudioDurationMs, resolveConfiguredRealtimeVoiceProvider, resolveRealtimeVoiceAgentConsultTools, resolveRealtimeVoiceBargeIn, resolveRealtimeVoiceInterruptResponseOnInputAudio, resolveRealtimeVoiceMinBargeInAudioEndMs, resolveRealtimeVoiceSessionPolicy } from "openclaw/plugin-sdk/realtime-voice";
|
|
9
10
|
import { PassThrough, pipeline } from "node:stream";
|
|
11
|
+
//#region extensions/discord/src/voice/realtime-player.ts
|
|
12
|
+
/** One physical player serves every speaker lane in the room. */
|
|
13
|
+
var DiscordRealtimePlayer = class {
|
|
14
|
+
constructor(player) {
|
|
15
|
+
this.player = player;
|
|
16
|
+
this.queue = [];
|
|
17
|
+
this.lanes = /* @__PURE__ */ new Set();
|
|
18
|
+
this.changing = false;
|
|
19
|
+
this.closed = false;
|
|
20
|
+
this.onIdle = () => {
|
|
21
|
+
const request = this.current;
|
|
22
|
+
this.current = void 0;
|
|
23
|
+
if (request) this.transition(() => request.onIdle());
|
|
24
|
+
};
|
|
25
|
+
player.on(loadDiscordVoiceSdk().AudioPlayerStatus.Idle, this.onIdle);
|
|
26
|
+
}
|
|
27
|
+
registerLane(lane) {
|
|
28
|
+
this.lanes.add(lane);
|
|
29
|
+
return () => this.lanes.delete(lane);
|
|
30
|
+
}
|
|
31
|
+
enqueue(request) {
|
|
32
|
+
if (this.closed || this.current === request || this.queue.includes(request)) return;
|
|
33
|
+
this.queue.push(request);
|
|
34
|
+
this.drain();
|
|
35
|
+
}
|
|
36
|
+
cancel(request) {
|
|
37
|
+
this.queue = this.queue.filter((queued) => queued !== request);
|
|
38
|
+
if (this.current !== request) return;
|
|
39
|
+
this.current = void 0;
|
|
40
|
+
this.transition(() => this.player.stop(true));
|
|
41
|
+
}
|
|
42
|
+
handleBargeIn(reason = "barge-in") {
|
|
43
|
+
if (this.current) {
|
|
44
|
+
this.current.onBargeIn(reason);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
for (const lane of this.lanes) if (lane.hasOutput()) lane.onBargeIn(reason);
|
|
48
|
+
}
|
|
49
|
+
isActive() {
|
|
50
|
+
return this.current !== void 0 || Array.from(this.lanes).some((lane) => lane.hasOutput());
|
|
51
|
+
}
|
|
52
|
+
cancelForControl() {
|
|
53
|
+
this.transition(() => {
|
|
54
|
+
this.queue = [];
|
|
55
|
+
for (const lane of this.lanes) lane.cancelForControl();
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
close() {
|
|
59
|
+
this.closed = true;
|
|
60
|
+
this.queue = [];
|
|
61
|
+
this.lanes.clear();
|
|
62
|
+
this.current = void 0;
|
|
63
|
+
this.player.off(loadDiscordVoiceSdk().AudioPlayerStatus.Idle, this.onIdle);
|
|
64
|
+
this.player.stop(true);
|
|
65
|
+
}
|
|
66
|
+
transition(action) {
|
|
67
|
+
const wasChanging = this.changing;
|
|
68
|
+
this.changing = true;
|
|
69
|
+
try {
|
|
70
|
+
action();
|
|
71
|
+
} finally {
|
|
72
|
+
this.changing = wasChanging;
|
|
73
|
+
this.drain();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
drain() {
|
|
77
|
+
if (this.closed || this.changing || this.current) return;
|
|
78
|
+
const next = this.queue.shift();
|
|
79
|
+
if (!next) return;
|
|
80
|
+
this.current = next;
|
|
81
|
+
this.transition(() => {
|
|
82
|
+
try {
|
|
83
|
+
this.player.play(next.createResource());
|
|
84
|
+
next.onStart();
|
|
85
|
+
} catch (error) {
|
|
86
|
+
this.current = void 0;
|
|
87
|
+
this.player.stop(true);
|
|
88
|
+
next.onError(error);
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
//#endregion
|
|
10
94
|
//#region extensions/discord/src/voice/realtime-transcript.ts
|
|
95
|
+
const PARTIAL_TRANSCRIPT_MAX_CHARS = 240;
|
|
11
96
|
function mergeRealtimePartialTranscript(previous, next) {
|
|
12
97
|
const trimmed = next.trim();
|
|
13
98
|
if (!trimmed) return previous;
|
|
14
|
-
|
|
99
|
+
const merged = trimmed.startsWith(previous) ? trimmed : `${previous}${next}`;
|
|
100
|
+
return sliceUtf16Safe(merged, 0, PARTIAL_TRANSCRIPT_MAX_CHARS);
|
|
15
101
|
}
|
|
16
102
|
//#endregion
|
|
17
103
|
//#region extensions/discord/src/voice/realtime-turns.ts
|
|
18
|
-
const logger$
|
|
19
|
-
const DISCORD_REALTIME_PENDING_SPEAKER_CONTEXT_LIMIT = 32;
|
|
20
|
-
const DISCORD_REALTIME_IGNORED_WAKE_NAME_CONTEXT_TTL_MS = 1e4;
|
|
104
|
+
const logger$5 = createSubsystemLogger("discord/voice");
|
|
21
105
|
const DISCORD_REALTIME_WAKE_NAME_FOLLOWUP_TTL_MS = 1e4;
|
|
22
106
|
const REALTIME_PCM16_BYTES_PER_SAMPLE = 2;
|
|
23
107
|
const DISCORD_REALTIME_TRAILING_SILENCE_MIN_MS = 700;
|
|
@@ -25,48 +109,59 @@ const DISCORD_REALTIME_TRAILING_SILENCE_MAX_MS = 3e3;
|
|
|
25
109
|
var DiscordRealtimeTurns = class {
|
|
26
110
|
constructor(params) {
|
|
27
111
|
this.params = params;
|
|
28
|
-
this.
|
|
29
|
-
limit: DISCORD_REALTIME_PENDING_SPEAKER_CONTEXT_LIMIT,
|
|
30
|
-
ignoredContextTtlMs: DISCORD_REALTIME_IGNORED_WAKE_NAME_CONTEXT_TTL_MS,
|
|
31
|
-
deferUntilAudio: true
|
|
32
|
-
});
|
|
112
|
+
this.pendingAudio = false;
|
|
33
113
|
this.partialUserTranscript = "";
|
|
34
114
|
this.wakeNameAckedForTurn = false;
|
|
35
115
|
}
|
|
36
116
|
beginSpeakerTurn(context, userId) {
|
|
117
|
+
if (this.source && (this.source.userId !== userId || this.source.senderIsOwner !== context.senderIsOwner)) throw new Error("A Discord realtime connection cannot change speaker authority");
|
|
118
|
+
this.source ??= Object.freeze({
|
|
119
|
+
userId,
|
|
120
|
+
senderIsOwner: context.senderIsOwner
|
|
121
|
+
});
|
|
37
122
|
this.resetPartialWakeNameTracking();
|
|
38
|
-
const turn =
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
123
|
+
const turn = {
|
|
124
|
+
context: {
|
|
125
|
+
...context,
|
|
126
|
+
...this.source
|
|
127
|
+
},
|
|
128
|
+
startedAt: Date.now(),
|
|
129
|
+
hasAudio: false,
|
|
130
|
+
closed: false,
|
|
131
|
+
transcripts: this.params.entry.transcripts,
|
|
42
132
|
inputDiscordBytes: 0,
|
|
43
133
|
inputRealtimeBytes: 0,
|
|
44
134
|
inputChunks: 0,
|
|
45
135
|
interruptedPlayback: false
|
|
46
|
-
}
|
|
136
|
+
};
|
|
47
137
|
return {
|
|
48
138
|
sendInputAudio: (discordPcm48kStereo) => this.sendInputAudioForTurn(turn, discordPcm48kStereo),
|
|
49
139
|
close: () => {
|
|
50
140
|
this.sendRealtimeTrailingSilenceForTurn(turn);
|
|
51
141
|
this.logSpeakerTurnClosed(turn);
|
|
52
|
-
|
|
142
|
+
turn.closed = true;
|
|
53
143
|
}
|
|
54
144
|
};
|
|
55
145
|
}
|
|
56
146
|
handlePartialUserTranscript(text) {
|
|
57
|
-
if (!this.isWakeNameRequired() || this.wakeNameAckedForTurn) return;
|
|
147
|
+
if (!this.lastSpeakerTurn || !this.isWakeNameRequired() || this.wakeNameAckedForTurn) return;
|
|
58
148
|
this.partialUserTranscript = mergeRealtimePartialTranscript(this.partialUserTranscript, text);
|
|
59
149
|
const wakeNameResult = matchRealtimeVoiceActivationName(this.partialUserTranscript, this.params.wakeNames());
|
|
60
|
-
if (
|
|
150
|
+
if (wakeNameResult?.edge !== "leading" || wakeNameResult.match !== "exact") return;
|
|
61
151
|
this.wakeNameAckedForTurn = true;
|
|
62
152
|
this.params.playback.sendWakeNameAck(wakeNameResult);
|
|
63
153
|
}
|
|
64
154
|
async handleFinalUserTranscript(text) {
|
|
155
|
+
if (!this.lastSpeakerTurn || this.params.stopped()) return;
|
|
65
156
|
const providerEpoch = this.params.providerEpoch();
|
|
66
157
|
const trimmed = text.trim();
|
|
67
|
-
if (!trimmed)
|
|
158
|
+
if (!trimmed) {
|
|
159
|
+
this.pendingAudio = false;
|
|
160
|
+
this.resetPartialWakeNameTracking();
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
68
163
|
this.partialUserTranscript = "";
|
|
69
|
-
const transcriptsTurn = this.
|
|
164
|
+
const transcriptsTurn = this.lastSpeakerTurn;
|
|
70
165
|
let transcriptAttribution = this.transcriptAttributionFromTurn(transcriptsTurn);
|
|
71
166
|
const humanParticipantCount = this.params.getHumanParticipantCount();
|
|
72
167
|
const requireWakeName = this.isWakeNameRequired(humanParticipantCount);
|
|
@@ -77,12 +172,12 @@ var DiscordRealtimeTurns = class {
|
|
|
77
172
|
transcriptAttribution ??= pendingWakeNameFollowup;
|
|
78
173
|
if (!pendingWakeNameFollowup) {
|
|
79
174
|
this.recordTranscriptUtterance(trimmed, transcriptAttribution, providerEpoch);
|
|
80
|
-
this.
|
|
81
|
-
logger$
|
|
175
|
+
this.consumePendingSpeakerContext();
|
|
176
|
+
logger$5.info(`discord voice: realtime wake-name gate ignored transcript chars=${trimmed.length} humanParticipants=${humanParticipantCount} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId} wakeNames=${this.params.wakeNames().join(",") || "none"}`);
|
|
82
177
|
return;
|
|
83
178
|
}
|
|
84
179
|
forcedSpeakerContext = pendingWakeNameFollowup.context;
|
|
85
|
-
logger$
|
|
180
|
+
logger$5.info(`discord voice: realtime wake-name follow-up accepted chars=${trimmed.length} speaker=${forcedSpeakerContext.speakerLabel} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
86
181
|
}
|
|
87
182
|
this.recordTranscriptUtterance(trimmed, transcriptAttribution, providerEpoch);
|
|
88
183
|
const acceptedText = wakeNameResult.allowed ? wakeNameResult.text || trimmed : trimmed;
|
|
@@ -100,23 +195,24 @@ var DiscordRealtimeTurns = class {
|
|
|
100
195
|
resetProviderContinuity() {
|
|
101
196
|
this.partialUserTranscript = "";
|
|
102
197
|
this.pendingWakeNameFollowup = void 0;
|
|
198
|
+
this.pendingAudio = false;
|
|
199
|
+
this.lastSpeakerTurn = void 0;
|
|
103
200
|
}
|
|
104
201
|
clear() {
|
|
105
|
-
this.
|
|
202
|
+
this.pendingAudio = false;
|
|
203
|
+
this.lastSpeakerTurn = void 0;
|
|
106
204
|
this.resetPartialWakeNameTracking();
|
|
107
205
|
this.pendingWakeNameFollowup = void 0;
|
|
108
206
|
}
|
|
109
207
|
consumePendingSpeakerContext() {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
consumeRecentIgnoredWakeNameSpeakerContext() {
|
|
113
|
-
return this.speakerTurns.consumeIgnoredContext();
|
|
208
|
+
this.pendingAudio = false;
|
|
209
|
+
return this.speakerContext();
|
|
114
210
|
}
|
|
115
|
-
|
|
116
|
-
return this.
|
|
211
|
+
speakerContext() {
|
|
212
|
+
return this.lastSpeakerTurn?.context;
|
|
117
213
|
}
|
|
118
214
|
hasPendingSpeakerAudioContext() {
|
|
119
|
-
return this.
|
|
215
|
+
return this.pendingAudio;
|
|
120
216
|
}
|
|
121
217
|
sendInputAudioForTurn(turn, discordPcm48kStereo) {
|
|
122
218
|
const bridge = this.params.bridge();
|
|
@@ -127,27 +223,28 @@ var DiscordRealtimeTurns = class {
|
|
|
127
223
|
turn.inputDiscordBytes += discordPcm48kStereo.length;
|
|
128
224
|
turn.inputRealtimeBytes += realtimePcm.length;
|
|
129
225
|
turn.inputChunks += 1;
|
|
130
|
-
if (turn.inputChunks === 1) logger$
|
|
131
|
-
|
|
132
|
-
if (!turn.interruptedPlayback && this.params.playback.isBargeInEnabled() && outputActive) {
|
|
226
|
+
if (turn.inputChunks === 1) logger$5.info(`discord voice: realtime input audio started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length} outputAudioMs=${this.params.playback.outputAudioMs()} outputActive=${this.params.playback.isOutputAudioActive()}`);
|
|
227
|
+
if (!turn.interruptedPlayback && this.params.interruptRoomPlayback()) {
|
|
133
228
|
turn.interruptedPlayback = true;
|
|
134
229
|
logVoiceVerbose(`realtime barge-in from active speaker audio: guild ${this.params.entry.guildId} channel ${this.params.entry.channelId} user ${turn.context.userId}`);
|
|
135
|
-
logger$
|
|
136
|
-
this.params.playback.handleBargeIn("active-speaker-audio");
|
|
230
|
+
logger$5.info(`discord voice: realtime barge-in detected source=active-speaker-audio guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length}`);
|
|
137
231
|
}
|
|
138
232
|
if (this.params.recordInputAudio(realtimePcm)) bridge.sendAudio(realtimePcm);
|
|
139
233
|
}
|
|
140
234
|
}
|
|
141
235
|
registerSpeakerTurnAudioStarted(turn) {
|
|
236
|
+
this.pendingAudio = true;
|
|
237
|
+
this.lastSpeakerTurn = turn;
|
|
238
|
+
turn.lastAudioAt = Date.now();
|
|
142
239
|
if (turn.hasAudio) return;
|
|
143
|
-
|
|
144
|
-
logger$
|
|
240
|
+
turn.hasAudio = true;
|
|
241
|
+
logger$5.info(`discord voice: realtime speaker turn opened guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} owner=${turn.context.senderIsOwner}`);
|
|
145
242
|
}
|
|
146
243
|
logSpeakerTurnClosed(turn) {
|
|
147
244
|
if (turn.closed || !turn.hasAudio) return;
|
|
148
245
|
const elapsedMs = Date.now() - turn.startedAt;
|
|
149
246
|
const sinceLastAudioMs = turn.lastAudioAt ? Date.now() - turn.lastAudioAt : void 0;
|
|
150
|
-
logger$
|
|
247
|
+
logger$5.info(`discord voice: realtime speaker turn closed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} owner=${turn.context.senderIsOwner} hasAudio=${turn.hasAudio} chunks=${turn.inputChunks} discordBytes=${turn.inputDiscordBytes} realtimeBytes=${turn.inputRealtimeBytes} elapsedMs=${elapsedMs}${sinceLastAudioMs === void 0 ? "" : ` sinceLastAudioMs=${sinceLastAudioMs}`} interruptedPlayback=${turn.interruptedPlayback}`);
|
|
151
248
|
}
|
|
152
249
|
sendRealtimeTrailingSilenceForTurn(turn) {
|
|
153
250
|
const bridge = this.params.bridge();
|
|
@@ -158,7 +255,7 @@ var DiscordRealtimeTurns = class {
|
|
|
158
255
|
const silenceBytes = Math.ceil(24e3 * silenceMs / 1e3) * REALTIME_PCM16_BYTES_PER_SAMPLE;
|
|
159
256
|
const silence = Buffer.alloc(silenceBytes);
|
|
160
257
|
bridge.sendAudio(silence);
|
|
161
|
-
logger$
|
|
258
|
+
logger$5.info(`discord voice: realtime trailing silence sent guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} silenceMs=${silenceMs} realtimeBytes=${silence.length}`);
|
|
162
259
|
}
|
|
163
260
|
resolveWakeNameTranscript(text, requireWakeName) {
|
|
164
261
|
if (!requireWakeName) return {
|
|
@@ -171,7 +268,7 @@ var DiscordRealtimeTurns = class {
|
|
|
171
268
|
};
|
|
172
269
|
const wakeNameResult = matchRealtimeVoiceActivationName(text, this.params.wakeNames());
|
|
173
270
|
if (wakeNameResult) {
|
|
174
|
-
logger$
|
|
271
|
+
logger$5.info(`discord voice: realtime wake-name gate matched canonical=${wakeNameResult.activationName} heard=${wakeNameResult.heardName} match=${wakeNameResult.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
175
272
|
return wakeNameResult;
|
|
176
273
|
}
|
|
177
274
|
return {
|
|
@@ -185,12 +282,13 @@ var DiscordRealtimeTurns = class {
|
|
|
185
282
|
transcriptAttributionFromTurn(turn) {
|
|
186
283
|
return turn ? {
|
|
187
284
|
context: turn.context,
|
|
188
|
-
startedAt: turn.startedAt
|
|
285
|
+
startedAt: turn.startedAt,
|
|
286
|
+
transcripts: turn.transcripts
|
|
189
287
|
} : void 0;
|
|
190
288
|
}
|
|
191
289
|
recordTranscriptUtterance(text, attribution, providerEpoch) {
|
|
192
|
-
const transcripts =
|
|
193
|
-
if (!transcripts || !attribution) return;
|
|
290
|
+
const transcripts = attribution?.transcripts;
|
|
291
|
+
if (!transcripts || !attribution || this.params.entry.transcripts !== transcripts) return;
|
|
194
292
|
const context = attribution.context;
|
|
195
293
|
const utterance = {
|
|
196
294
|
sessionId: transcripts.sessionId,
|
|
@@ -209,27 +307,28 @@ var DiscordRealtimeTurns = class {
|
|
|
209
307
|
}
|
|
210
308
|
};
|
|
211
309
|
Promise.resolve().then(() => {
|
|
212
|
-
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
310
|
+
if (providerEpoch !== this.params.providerEpoch() || this.params.entry.transcripts !== transcripts) return;
|
|
213
311
|
return transcripts.onUtterance(utterance);
|
|
214
312
|
}).catch((error) => {
|
|
215
|
-
logger$
|
|
313
|
+
logger$5.warn(`discord voice: realtime transcripts utterance failed: ${formatErrorMessage(error)}`);
|
|
216
314
|
});
|
|
217
315
|
}
|
|
218
316
|
armWakeNameFollowup() {
|
|
219
|
-
const turn = this.
|
|
317
|
+
const turn = this.lastSpeakerTurn;
|
|
220
318
|
const context = this.consumePendingSpeakerContext();
|
|
221
319
|
if (!context) {
|
|
222
|
-
logger$
|
|
320
|
+
logger$5.warn(`discord voice: realtime wake-name follow-up has no speaker context voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
223
321
|
return;
|
|
224
322
|
}
|
|
225
323
|
const expiresAt = resolveExpiresAtMsFromDurationMs(DISCORD_REALTIME_WAKE_NAME_FOLLOWUP_TTL_MS);
|
|
226
324
|
if (expiresAt === void 0) return;
|
|
227
325
|
this.pendingWakeNameFollowup = {
|
|
228
326
|
context,
|
|
327
|
+
transcripts: turn?.transcripts,
|
|
229
328
|
startedAt: turn?.startedAt ?? Date.now(),
|
|
230
329
|
expiresAt
|
|
231
330
|
};
|
|
232
|
-
logger$
|
|
331
|
+
logger$5.info(`discord voice: realtime wake-name follow-up armed speaker=${context.speakerLabel} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
233
332
|
}
|
|
234
333
|
consumePendingWakeNameFollowup() {
|
|
235
334
|
const pending = this.pendingWakeNameFollowup;
|
|
@@ -237,37 +336,41 @@ var DiscordRealtimeTurns = class {
|
|
|
237
336
|
const now = asDateTimestampMs(Date.now());
|
|
238
337
|
const expiresAt = pending ? asDateTimestampMs(pending.expiresAt) : void 0;
|
|
239
338
|
if (!pending || now === void 0 || expiresAt === void 0 || now > expiresAt) return;
|
|
240
|
-
|
|
241
|
-
if (currentTurn && currentTurn.context.userId !== pending.context.userId) return;
|
|
242
|
-
if (currentTurn) this.consumePendingSpeakerContext();
|
|
339
|
+
this.consumePendingSpeakerContext();
|
|
243
340
|
return {
|
|
244
341
|
context: pending.context,
|
|
245
|
-
startedAt: pending.startedAt
|
|
342
|
+
startedAt: pending.startedAt,
|
|
343
|
+
transcripts: pending.transcripts
|
|
246
344
|
};
|
|
247
345
|
}
|
|
248
|
-
rememberIgnoredWakeNameSpeakerContext(context) {
|
|
249
|
-
this.speakerTurns.rememberIgnoredContext(context);
|
|
250
|
-
}
|
|
251
346
|
};
|
|
252
347
|
function isDiscordRealtimeSpeakerContext(value) {
|
|
253
348
|
return Boolean(value) && typeof value === "object" && typeof value.userId === "string" && typeof value.senderIsOwner === "boolean" && typeof value.speakerLabel === "string";
|
|
254
349
|
}
|
|
255
350
|
//#endregion
|
|
256
351
|
//#region extensions/discord/src/voice/realtime-consults.ts
|
|
257
|
-
const logger$
|
|
352
|
+
const logger$4 = createSubsystemLogger("discord/voice");
|
|
258
353
|
const DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS = 350;
|
|
259
354
|
const DISCORD_REALTIME_FALLBACK_TEXT = "I hit an error while checking that. Please try again.";
|
|
260
355
|
const DISCORD_REALTIME_FORCED_CONSULT_FALLBACK_DELAY_MS = 200;
|
|
261
356
|
const DISCORD_REALTIME_FORCED_CONSULT_REASON = "provider_final_transcript_without_openclaw_agent_consult";
|
|
357
|
+
const CANCELLED_CONSULT_RESULT = {
|
|
358
|
+
status: "cancelled",
|
|
359
|
+
message: "OpenClaw cancelled this consult before completion. Do not restart it."
|
|
360
|
+
};
|
|
262
361
|
var DiscordRealtimeConsults = class {
|
|
263
362
|
constructor(params) {
|
|
264
363
|
this.params = params;
|
|
364
|
+
this.providerDeliveries = /* @__PURE__ */ new Map();
|
|
265
365
|
this.talkback = this.createTalkbackQueue();
|
|
266
366
|
}
|
|
267
367
|
close() {
|
|
268
368
|
this.talkback.close();
|
|
269
369
|
this.clearProviderConsultState();
|
|
270
370
|
}
|
|
371
|
+
isIdle() {
|
|
372
|
+
return this.talkback.isIdle() && this.providerDeliveries.size === 0 && this.params.harness.forcedConsults.handles().every((handle) => handle.context?.result);
|
|
373
|
+
}
|
|
271
374
|
resetProviderContinuity() {
|
|
272
375
|
this.talkback.close();
|
|
273
376
|
this.talkback = this.createTalkbackQueue();
|
|
@@ -276,6 +379,10 @@ var DiscordRealtimeConsults = class {
|
|
|
276
379
|
async handleToolCall(event, session) {
|
|
277
380
|
const providerEpoch = this.params.providerEpoch();
|
|
278
381
|
const callId = event.callId || event.itemId || "unknown";
|
|
382
|
+
if (this.params.stopped() || !this.params.turns.speakerContext()) {
|
|
383
|
+
await session.submitToolResult(callId, { error: "No Discord speaker context available" });
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
279
386
|
if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) {
|
|
280
387
|
await this.handleAgentControlToolCall(event, session, callId, providerEpoch);
|
|
281
388
|
return;
|
|
@@ -291,23 +398,19 @@ var DiscordRealtimeConsults = class {
|
|
|
291
398
|
const outcome = classifyRealtimeVoiceConsultToolCall(event.args, { retainedExactSpeechTexts: this.params.playback.retainedExactSpeechTexts() });
|
|
292
399
|
switch (outcome.kind) {
|
|
293
400
|
case "exact-speech-echo":
|
|
294
|
-
logger$
|
|
401
|
+
logger$4.info(`discord voice: realtime exact speech consult bypassed call=${callId || "unknown"} answerChars=${outcome.text.length}`);
|
|
295
402
|
await session.submitToolResult(callId, { text: outcome.text });
|
|
296
403
|
return;
|
|
297
404
|
case "malformed":
|
|
298
|
-
logger$
|
|
405
|
+
logger$4.warn(`discord voice: realtime consult rejected malformed args call=${callId || "unknown"}: ${outcome.error}`);
|
|
299
406
|
await session.submitToolResult(callId, { error: outcome.error });
|
|
300
407
|
return;
|
|
301
|
-
case "consult": break;
|
|
302
408
|
}
|
|
303
409
|
const consultMessage = outcome.message;
|
|
304
|
-
logger$
|
|
410
|
+
logger$4.info(`discord voice: realtime consult requested call=${callId || "unknown"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} question=${formatVoiceLogPreview(consultMessage)}`);
|
|
305
411
|
const nativeConsult = this.params.harness.forcedConsults.recordNativeConsult(event.args, callId);
|
|
306
412
|
if (nativeConsult.kind === "already_delivered" && this.params.harness.forcedConsults.isCancelled(nativeConsult.handle)) {
|
|
307
|
-
await this.submitTerminalRealtimeToolResult(callId, session,
|
|
308
|
-
status: "cancelled",
|
|
309
|
-
message: "OpenClaw cancelled this consult before completion. Do not restart it."
|
|
310
|
-
});
|
|
413
|
+
await this.submitTerminalRealtimeToolResult(callId, session, CANCELLED_CONSULT_RESULT);
|
|
311
414
|
return;
|
|
312
415
|
}
|
|
313
416
|
const pendingConsult = nativeConsult.kind === "pending" ? nativeConsult.handle : void 0;
|
|
@@ -319,7 +422,7 @@ var DiscordRealtimeConsults = class {
|
|
|
319
422
|
if (recentConsult) {
|
|
320
423
|
const recentSpeaker = recentConsult.context?.speaker;
|
|
321
424
|
if (this.params.turns.hasPendingSpeakerAudioContext()) {
|
|
322
|
-
logger$
|
|
425
|
+
logger$4.info(`discord voice: realtime consult matched recent agent result but newer speaker audio is pending call=${callId} speaker=${recentSpeaker?.speakerLabel ?? "unknown"} owner=${recentSpeaker?.senderIsOwner ?? false}`);
|
|
323
426
|
await session.submitToolResult(callId, { error: "Discord speaker context changed before this realtime consult completed" });
|
|
324
427
|
return;
|
|
325
428
|
}
|
|
@@ -334,28 +437,18 @@ var DiscordRealtimeConsults = class {
|
|
|
334
437
|
});
|
|
335
438
|
}
|
|
336
439
|
if (!context) {
|
|
337
|
-
logger$
|
|
440
|
+
logger$4.warn(`discord voice: realtime consult has no speaker context call=${callId || "unknown"}`);
|
|
338
441
|
await session.submitToolResult(callId, { error: "No Discord speaker context available" });
|
|
339
442
|
return;
|
|
340
443
|
}
|
|
341
|
-
const
|
|
444
|
+
const result = await this.trackAgentProxyConsult(recent, this.runAgentTurn({
|
|
342
445
|
context,
|
|
343
446
|
message: consultMessage
|
|
344
|
-
});
|
|
345
|
-
if (recent) this.setRecentAgentProxyConsultPromise(recent, promise);
|
|
346
|
-
let text;
|
|
347
|
-
try {
|
|
348
|
-
text = await promise;
|
|
349
|
-
} catch (error) {
|
|
350
|
-
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
351
|
-
const message = formatErrorMessage(error);
|
|
352
|
-
logger$2.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`);
|
|
353
|
-
await session.submitToolResult(callId, { error: message });
|
|
354
|
-
return;
|
|
355
|
-
}
|
|
447
|
+
}));
|
|
356
448
|
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
357
|
-
logger$
|
|
358
|
-
|
|
449
|
+
if ("text" in result) logger$4.info(`discord voice: realtime consult answer (${result.text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(result.text)}`);
|
|
450
|
+
else if ("error" in result) logger$4.warn(`discord voice: realtime consult failed call=${callId}: ${result.error}`);
|
|
451
|
+
await this.submitAgentProxyConsultResult(callId, session, result);
|
|
359
452
|
}
|
|
360
453
|
async handleAcceptedTranscript(acceptedText, forcedSpeakerContext, providerEpoch) {
|
|
361
454
|
const usesRealtimeAgentHandoff = this.params.usesRealtimeAgentHandoff();
|
|
@@ -369,7 +462,7 @@ var DiscordRealtimeConsults = class {
|
|
|
369
462
|
});
|
|
370
463
|
} catch (error) {
|
|
371
464
|
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
372
|
-
logger$
|
|
465
|
+
logger$4.warn(`discord voice: realtime active-run control failed; falling back to normal transcript handling: ${formatErrorMessage(error)}`);
|
|
373
466
|
control = void 0;
|
|
374
467
|
}
|
|
375
468
|
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
@@ -391,7 +484,7 @@ var DiscordRealtimeConsults = class {
|
|
|
391
484
|
return createRealtimeVoiceAgentTalkbackQueue({
|
|
392
485
|
debounceMs: this.params.debounceMs() ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
|
|
393
486
|
isStopped: () => this.params.stopped() || providerEpoch !== this.params.providerEpoch(),
|
|
394
|
-
logger: logger$
|
|
487
|
+
logger: logger$4,
|
|
395
488
|
logPrefix: "[discord] realtime agent",
|
|
396
489
|
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
|
|
397
490
|
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
|
|
@@ -434,7 +527,7 @@ var DiscordRealtimeConsults = class {
|
|
|
434
527
|
});
|
|
435
528
|
}
|
|
436
529
|
logAgentControlResult(result) {
|
|
437
|
-
logger$
|
|
530
|
+
logger$4.info(`discord voice: realtime active-run control handled mode=${result.mode} ok=${result.ok} active=${result.active} reason=${result.reason ?? "none"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}`);
|
|
438
531
|
}
|
|
439
532
|
prepareForcedAgentProxyConsult(transcript, speakerContext) {
|
|
440
533
|
if (this.params.consultPolicy() !== "always" && this.params.wakeNamePolicy() === "never") return;
|
|
@@ -443,18 +536,17 @@ var DiscordRealtimeConsults = class {
|
|
|
443
536
|
const skipReason = classifySkippableRealtimeVoiceConsultTranscript(question);
|
|
444
537
|
if (skipReason) {
|
|
445
538
|
const context = this.params.turns.consumePendingSpeakerContext();
|
|
446
|
-
logger$
|
|
539
|
+
logger$4.info(`discord voice: realtime forced agent consult skipped reason=${skipReason} chars=${question.length} speaker=${context?.speakerLabel ?? "unknown"} transcript=${formatVoiceLogPreview(question)}`);
|
|
447
540
|
return;
|
|
448
541
|
}
|
|
449
|
-
|
|
450
|
-
if (!context) context = this.params.turns.consumeRecentIgnoredWakeNameSpeakerContext();
|
|
542
|
+
const context = speakerContext ?? this.params.turns.consumePendingSpeakerContext();
|
|
451
543
|
if (!context) {
|
|
452
544
|
const recent = this.findRecentAgentProxyConsultContext(question);
|
|
453
545
|
if (recent) {
|
|
454
546
|
logVoiceVerbose(`realtime forced agent consult skipped (already delegated): guild ${this.params.entry.guildId} channel ${this.params.entry.channelId} speaker ${recent.context?.speaker.userId ?? "unknown"}`);
|
|
455
547
|
return;
|
|
456
548
|
}
|
|
457
|
-
logger$
|
|
549
|
+
logger$4.warn("discord voice: realtime forced agent consult has no speaker context");
|
|
458
550
|
return;
|
|
459
551
|
}
|
|
460
552
|
return this.params.harness.forcedConsults.prepare(question, { context: {
|
|
@@ -479,27 +571,21 @@ var DiscordRealtimeConsults = class {
|
|
|
479
571
|
return;
|
|
480
572
|
}
|
|
481
573
|
const startedAt = Date.now();
|
|
482
|
-
logger$
|
|
483
|
-
logger$
|
|
484
|
-
if (this.params.playback.hasInterruptibleOutputAudio()) logger$
|
|
574
|
+
logger$4.info(`discord voice: realtime forced agent consult starting chars=${question.length} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}`);
|
|
575
|
+
logger$4.debug(`discord voice: realtime forced agent consult reason=${DISCORD_REALTIME_FORCED_CONSULT_REASON} consultPolicy=${this.params.consultPolicy()} wakeNamePolicy=${this.params.wakeNamePolicy()} requireWakeName=${this.params.isWakeNameRequired()} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel}`);
|
|
576
|
+
if (this.params.playback.hasInterruptibleOutputAudio()) logger$4.info(`discord voice: realtime forced agent consult preserving active playback guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.params.playback.outputAudioMs()} outputActive=${this.params.playback.isOutputAudioActive()} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
485
577
|
state.handledByForcedPlayback = true;
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
} catch (error) {
|
|
498
|
-
await state.providerDelivery;
|
|
499
|
-
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
500
|
-
logger$2.warn(`discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`);
|
|
501
|
-
if (state.handledByForcedPlayback) this.params.playback.enqueueExactSpeechMessage(DISCORD_REALTIME_FALLBACK_TEXT);
|
|
502
|
-
}
|
|
578
|
+
const result = await this.trackAgentProxyConsult(pending, this.runAgentTurn({
|
|
579
|
+
context,
|
|
580
|
+
message: question
|
|
581
|
+
}));
|
|
582
|
+
const deliveries = this.providerDeliveries.get(state);
|
|
583
|
+
await Promise.all(Array.from(deliveries ?? [], (delivery) => delivery.promise));
|
|
584
|
+
if (state.providerEpoch !== this.params.providerEpoch() || "status" in result) return;
|
|
585
|
+
if ("text" in result) logger$4.info(`discord voice: realtime forced agent consult answer (${result.text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(result.text)}`);
|
|
586
|
+
else logger$4.warn(`discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${result.error}`);
|
|
587
|
+
const text = "text" in result ? result.text : DISCORD_REALTIME_FALLBACK_TEXT;
|
|
588
|
+
if (text.trim() && state.handledByForcedPlayback) this.params.playback.enqueueExactSpeechMessage(text);
|
|
503
589
|
}
|
|
504
590
|
rememberRecentAgentProxyConsultContext(question, context, options = {}) {
|
|
505
591
|
const handle = this.params.harness.forcedConsults.prepare(question, {
|
|
@@ -513,26 +599,19 @@ var DiscordRealtimeConsults = class {
|
|
|
513
599
|
if (options.started) this.params.harness.forcedConsults.markStarted(handle);
|
|
514
600
|
return handle;
|
|
515
601
|
}
|
|
516
|
-
|
|
517
|
-
const state = recent
|
|
518
|
-
if (
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
};
|
|
527
|
-
this.params.harness.forcedConsults.markDelivered(recent);
|
|
528
|
-
}).catch((error) => {
|
|
529
|
-
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
530
|
-
state.result = {
|
|
531
|
-
status: "rejected",
|
|
532
|
-
error: formatErrorMessage(error)
|
|
533
|
-
};
|
|
534
|
-
this.params.harness.forcedConsults.markDelivered(recent);
|
|
602
|
+
trackAgentProxyConsult(recent, promise) {
|
|
603
|
+
const state = recent?.context;
|
|
604
|
+
if (recent) this.params.harness.forcedConsults.markStarted(recent);
|
|
605
|
+
const tracked = promise.then((text) => ({ text }), buildRealtimeVoiceAgentErrorProviderResult).then((result) => {
|
|
606
|
+
if (recent && state && state.providerEpoch === this.params.providerEpoch()) {
|
|
607
|
+
state.result = result;
|
|
608
|
+
if ("status" in result) this.params.harness.forcedConsults.markCancelled(recent);
|
|
609
|
+
else this.params.harness.forcedConsults.markDelivered(recent);
|
|
610
|
+
}
|
|
611
|
+
return result;
|
|
535
612
|
});
|
|
613
|
+
if (state) state.promise = tracked;
|
|
614
|
+
return tracked;
|
|
536
615
|
}
|
|
537
616
|
findRecentAgentProxyConsultContext(consultMessage) {
|
|
538
617
|
return this.params.harness.forcedConsults.findRecent(consultMessage);
|
|
@@ -548,139 +627,311 @@ var DiscordRealtimeConsults = class {
|
|
|
548
627
|
const state = recent.context;
|
|
549
628
|
if (!state) return false;
|
|
550
629
|
if (state.providerEpoch !== this.params.providerEpoch()) return true;
|
|
551
|
-
const
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
630
|
+
const pendingResult = state.result ?? state.promise;
|
|
631
|
+
if (!pendingResult) return false;
|
|
632
|
+
const providerDelivery = state.handledByForcedPlayback && !state.result && session.bridge.supportsToolResultSuppression === false ? createDeferred() : void 0;
|
|
633
|
+
if (providerDelivery) {
|
|
634
|
+
const deliveries = this.providerDeliveries.get(state) ?? /* @__PURE__ */ new Set();
|
|
635
|
+
deliveries.add(providerDelivery);
|
|
636
|
+
this.providerDeliveries.set(state, deliveries);
|
|
637
|
+
}
|
|
638
|
+
logger$4.info(`discord voice: realtime consult ${state.result ? "reused recent" : "joined in-flight"} agent result call=${callId} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`);
|
|
639
|
+
try {
|
|
640
|
+
const result = await pendingResult;
|
|
641
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return true;
|
|
642
|
+
await this.submitAgentProxyConsultResult(callId, session, result, Boolean(state.handledByForcedPlayback && !providerDelivery));
|
|
643
|
+
if (providerDelivery && state.providerEpoch === this.params.providerEpoch()) state.handledByForcedPlayback = false;
|
|
644
|
+
} finally {
|
|
645
|
+
if (providerDelivery) {
|
|
646
|
+
providerDelivery.resolve();
|
|
647
|
+
const deliveries = this.providerDeliveries.get(state);
|
|
648
|
+
deliveries?.delete(providerDelivery);
|
|
649
|
+
if (deliveries?.size === 0) this.providerDeliveries.delete(state);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
return true;
|
|
653
|
+
}
|
|
654
|
+
async submitAgentProxyConsultResult(callId, session, result, alreadyDelivered = false) {
|
|
655
|
+
if ("status" in result) await this.submitTerminalRealtimeToolResult(callId, session, CANCELLED_CONSULT_RESULT);
|
|
656
|
+
else if (alreadyDelivered) await this.submitTerminalRealtimeToolResult(callId, session, {
|
|
657
|
+
status: "already_delivered",
|
|
658
|
+
message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it."
|
|
659
|
+
});
|
|
660
|
+
else await session.submitToolResult(callId, result);
|
|
661
|
+
}
|
|
662
|
+
clearProviderConsultState() {
|
|
663
|
+
for (const [state, deliveries] of this.providerDeliveries) {
|
|
664
|
+
state.handledByForcedPlayback = false;
|
|
665
|
+
for (const delivery of deliveries) delivery.resolve();
|
|
666
|
+
}
|
|
667
|
+
this.providerDeliveries.clear();
|
|
668
|
+
this.params.harness.forcedConsults.clear();
|
|
669
|
+
}
|
|
670
|
+
};
|
|
671
|
+
//#endregion
|
|
672
|
+
//#region extensions/discord/src/voice/realtime-output.ts
|
|
673
|
+
const logger$3 = createSubsystemLogger("discord/voice");
|
|
674
|
+
const DISCORD_RAW_PCM_FRAME_BYTES = 3840;
|
|
675
|
+
const DISCORD_RAW_PCM_BYTES_PER_MS = 192;
|
|
676
|
+
const DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES = 25;
|
|
677
|
+
const DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS = 3e3;
|
|
678
|
+
/** One output stream retains ownership through queued and audible playback. */
|
|
679
|
+
var DiscordRealtimeOutput = class {
|
|
680
|
+
constructor(params) {
|
|
681
|
+
this.params = params;
|
|
682
|
+
this.activity = createRealtimeVoiceOutputActivityTracker();
|
|
683
|
+
this.stream = new PassThrough({ highWaterMark: DISCORD_RAW_PCM_FRAME_BYTES * 128 });
|
|
684
|
+
this.playbackMarks = [];
|
|
685
|
+
this.playedPcmBytes = 0;
|
|
686
|
+
this.reportedPlaybackMs = 0;
|
|
687
|
+
this.audioSpans = [];
|
|
688
|
+
this.buffers = [];
|
|
689
|
+
this.bufferedBytes = 0;
|
|
690
|
+
this.closed = false;
|
|
691
|
+
this.failed = false;
|
|
692
|
+
this.stream.once("close", () => {
|
|
693
|
+
if (!this.activity.snapshot().playbackStarted) this.close("stream-close");
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
pendingBytes() {
|
|
697
|
+
return this.closed ? 0 : this.bufferedBytes + this.stream.writableLength + this.stream.readableLength;
|
|
698
|
+
}
|
|
699
|
+
playbackItems() {
|
|
700
|
+
const playedMs = Math.min(this.playedPcmBytes / DISCORD_RAW_PCM_BYTES_PER_MS, this.activity.snapshot().audioMs);
|
|
701
|
+
for (const span of this.audioSpans) span.item.audioEndMs += Math.max(0, Math.min(playedMs, span.endMs) - Math.max(this.reportedPlaybackMs, span.startMs));
|
|
702
|
+
this.reportedPlaybackMs = playedMs;
|
|
703
|
+
return this.audioSpans.filter(({ endMs }) => endMs > playedMs).map(({ item }) => item);
|
|
704
|
+
}
|
|
705
|
+
markPlayback(acknowledge) {
|
|
706
|
+
if (!this.closed) this.playbackMarks.push({
|
|
707
|
+
endBytes: this.activity.snapshot().sinkAudioBytes,
|
|
708
|
+
acknowledge
|
|
556
709
|
});
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
710
|
+
}
|
|
711
|
+
append(pcm, activity, item) {
|
|
712
|
+
if (this.closed) return;
|
|
713
|
+
const startMs = this.activity.snapshot().audioMs;
|
|
714
|
+
if (item) {
|
|
715
|
+
const previous = this.audioSpans.at(-1);
|
|
716
|
+
const endMs = startMs + activity.audioMs;
|
|
717
|
+
if (previous?.item === item && previous.endMs === startMs) previous.endMs = endMs;
|
|
718
|
+
else this.audioSpans.push({
|
|
719
|
+
item,
|
|
720
|
+
startMs,
|
|
721
|
+
endMs
|
|
562
722
|
});
|
|
563
|
-
};
|
|
564
|
-
const submitResult = async (result) => {
|
|
565
|
-
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
566
|
-
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
|
|
567
|
-
await submitAlreadyDelivered();
|
|
568
|
-
return;
|
|
569
|
-
}
|
|
570
|
-
if (result.status === "fulfilled") {
|
|
571
|
-
await session.submitToolResult(callId, { text: result.text });
|
|
572
|
-
return;
|
|
573
|
-
}
|
|
574
|
-
await session.submitToolResult(callId, { error: result.error });
|
|
575
|
-
};
|
|
576
|
-
if (state.result) {
|
|
577
|
-
logger$2.info(`discord voice: realtime consult reused recent agent result call=${callId || "unknown"} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`);
|
|
578
|
-
await submitResult(state.result);
|
|
579
|
-
return true;
|
|
580
723
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
724
|
+
this.activity.markAudio(activity);
|
|
725
|
+
if (this.activity.snapshot().playbackStarted && !this.drainHandler) {
|
|
726
|
+
if (!this.stream.write(pcm)) this.waitForDrain();
|
|
727
|
+
return;
|
|
728
|
+
}
|
|
729
|
+
this.buffers.push(pcm);
|
|
730
|
+
this.bufferedBytes += pcm.length;
|
|
731
|
+
if (!this.drainHandler && this.bufferedBytes >= DISCORD_RAW_PCM_FRAME_BYTES * DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES) this.startPlayback();
|
|
732
|
+
}
|
|
733
|
+
finish(reason, playBuffered) {
|
|
734
|
+
if (this.closed) return;
|
|
735
|
+
this.activity.markStreamEnding();
|
|
736
|
+
const activity = this.activity.snapshot();
|
|
737
|
+
logger$3.info(`discord voice: realtime audio playback finishing reason=${reason} ${this.params.logContext} audioMs=${Math.floor(activity.audioMs)} chunks=${activity.chunks}`);
|
|
738
|
+
if (!playBuffered) {
|
|
739
|
+
this.close(reason);
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
this.startPlayback();
|
|
743
|
+
if (this.activity.snapshot().playbackStarted) {
|
|
744
|
+
this.scheduleWatchdog(reason);
|
|
745
|
+
if (!this.drainHandler) this.stream.end();
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
close(reason) {
|
|
749
|
+
if (this.closed) return;
|
|
750
|
+
const playbackRetirement = reason === "player-idle" || reason === "output-pipeline-error" || reason === "playback-watchdog";
|
|
751
|
+
const heardMarks = playbackRetirement ? this.playbackMarks.filter((mark) => mark.endBytes <= this.playedPcmBytes) : [];
|
|
752
|
+
const lostPlaybackMarks = playbackRetirement && this.playbackMarks.some((mark) => mark.endBytes > this.playedPcmBytes);
|
|
753
|
+
this.closed = true;
|
|
754
|
+
this.playbackMarks = [];
|
|
755
|
+
this.playbackItems();
|
|
756
|
+
this.clearWatchdog();
|
|
757
|
+
const activity = this.activity.snapshot();
|
|
758
|
+
logger$3.info(`discord voice: realtime audio playback stopped reason=${reason} ${this.params.logContext} audioMs=${Math.floor(activity.audioMs)} elapsedMs=${this.activity.elapsedPlaybackMs()} chunks=${activity.chunks} discordBytes=${activity.sinkAudioBytes} realtimeBytes=${activity.sourceAudioBytes}`);
|
|
759
|
+
this.buffers = [];
|
|
760
|
+
this.bufferedBytes = 0;
|
|
761
|
+
if (this.drainHandler) {
|
|
762
|
+
this.stream.off("drain", this.drainHandler);
|
|
763
|
+
this.drainHandler = void 0;
|
|
764
|
+
}
|
|
765
|
+
this.stream.end();
|
|
766
|
+
this.stream.destroy();
|
|
767
|
+
if (lostPlaybackMarks) this.params.onError(/* @__PURE__ */ new Error(`Discord realtime audio stopped before playback completed: ${reason}`));
|
|
768
|
+
this.params.onClose(this, reason);
|
|
769
|
+
if (this.request) {
|
|
770
|
+
this.params.player.cancel(this.request);
|
|
771
|
+
this.request = void 0;
|
|
588
772
|
}
|
|
589
|
-
let result;
|
|
590
773
|
try {
|
|
591
|
-
|
|
592
|
-
status: "fulfilled",
|
|
593
|
-
text: await state.promise
|
|
594
|
-
};
|
|
774
|
+
for (const mark of heardMarks) mark.acknowledge();
|
|
595
775
|
} catch (error) {
|
|
596
|
-
|
|
597
|
-
status: "rejected",
|
|
598
|
-
error: formatErrorMessage(error)
|
|
599
|
-
};
|
|
776
|
+
this.params.onError(error);
|
|
600
777
|
}
|
|
601
|
-
|
|
778
|
+
}
|
|
779
|
+
startPlayback() {
|
|
780
|
+
if (this.closed || this.request) return;
|
|
781
|
+
this.request = {
|
|
782
|
+
createResource: () => this.createResource(),
|
|
783
|
+
onStart: () => {
|
|
784
|
+
this.activity.markPlaybackStarted();
|
|
785
|
+
this.params.onStart();
|
|
786
|
+
if (this.activity.snapshot().streamEnding) {
|
|
787
|
+
this.scheduleWatchdog("player-start");
|
|
788
|
+
if (!this.drainHandler) this.stream.end();
|
|
789
|
+
}
|
|
790
|
+
},
|
|
791
|
+
onIdle: () => this.close(this.failed ? "output-pipeline-error" : "player-idle"),
|
|
792
|
+
onBargeIn: this.params.onBargeIn,
|
|
793
|
+
onError: this.params.onError
|
|
794
|
+
};
|
|
795
|
+
this.params.player.enqueue(this.request);
|
|
796
|
+
}
|
|
797
|
+
createResource() {
|
|
798
|
+
const voiceSdk = loadDiscordVoiceSdk();
|
|
799
|
+
const opusStream = createDiscordOpusEncodeStream();
|
|
800
|
+
opusStream.once("error", () => {
|
|
801
|
+
this.failed = true;
|
|
802
|
+
});
|
|
803
|
+
pipeline(this.stream, opusStream, (error) => {
|
|
804
|
+
if (!error || this.closed) return;
|
|
805
|
+
logger$3.warn(`discord voice: realtime output pipeline failed ${this.params.logContext}: ${formatErrorMessage(error)}`);
|
|
806
|
+
this.close("output-pipeline-error");
|
|
807
|
+
});
|
|
808
|
+
const buffered = Buffer.concat(this.buffers, this.bufferedBytes);
|
|
809
|
+
this.buffers = [];
|
|
810
|
+
this.bufferedBytes = 0;
|
|
811
|
+
if (buffered.length > 0 && !this.stream.write(buffered)) this.waitForDrain();
|
|
812
|
+
const resource = voiceSdk.createAudioResource(opusStream, { inputType: voiceSdk.StreamType.Opus });
|
|
813
|
+
const read = resource.read.bind(resource);
|
|
814
|
+
resource.read = () => {
|
|
815
|
+
let packet;
|
|
816
|
+
try {
|
|
817
|
+
packet = read();
|
|
818
|
+
if (!packet && opusStream.flushPartialFrame()) packet = read();
|
|
819
|
+
if (packet) this.playedPcmBytes += opusStream.takePcmBytes(packet);
|
|
820
|
+
} catch (error) {
|
|
821
|
+
opusStream.destroy(error instanceof Error ? error : new Error(formatErrorMessage(error)));
|
|
822
|
+
return null;
|
|
823
|
+
}
|
|
824
|
+
if (this.playbackMarks.length > 0) queueMicrotask(() => this.acknowledgePlayedMarks());
|
|
825
|
+
return packet;
|
|
826
|
+
};
|
|
827
|
+
return resource;
|
|
828
|
+
}
|
|
829
|
+
acknowledgePlayedMarks() {
|
|
602
830
|
try {
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
831
|
+
while (!this.closed) {
|
|
832
|
+
const mark = this.playbackMarks[0];
|
|
833
|
+
if (!mark || mark.endBytes > this.playedPcmBytes) return;
|
|
834
|
+
this.playbackMarks.shift();
|
|
835
|
+
mark.acknowledge();
|
|
608
836
|
}
|
|
609
837
|
} catch (error) {
|
|
610
|
-
|
|
611
|
-
resolveProviderDelivery?.(false);
|
|
612
|
-
throw error;
|
|
838
|
+
this.params.onError(error);
|
|
613
839
|
}
|
|
614
|
-
return true;
|
|
615
840
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
841
|
+
waitForDrain() {
|
|
842
|
+
if (this.drainHandler || this.closed) return;
|
|
843
|
+
logger$3.info(`discord voice: realtime audio playback buffering ${this.params.logContext} bufferedBytes=${this.stream.writableLength + this.stream.readableLength}`);
|
|
844
|
+
this.drainHandler = () => {
|
|
845
|
+
this.drainHandler = void 0;
|
|
846
|
+
if (this.closed) return;
|
|
847
|
+
let refreshWatchdog = this.activity.snapshot().streamEnding;
|
|
848
|
+
while (this.buffers.length > 0) {
|
|
849
|
+
const buffered = this.buffers.shift();
|
|
850
|
+
if (!buffered) break;
|
|
851
|
+
this.bufferedBytes -= buffered.length;
|
|
852
|
+
const writable = this.stream.write(buffered);
|
|
853
|
+
if (refreshWatchdog) {
|
|
854
|
+
this.scheduleWatchdog("output-drain");
|
|
855
|
+
refreshWatchdog = false;
|
|
856
|
+
}
|
|
857
|
+
if (!writable) {
|
|
858
|
+
this.waitForDrain();
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
if (this.activity.snapshot().streamEnding) this.stream.end();
|
|
863
|
+
};
|
|
864
|
+
this.stream.once("drain", this.drainHandler);
|
|
865
|
+
}
|
|
866
|
+
scheduleWatchdog(reason) {
|
|
867
|
+
this.clearWatchdog();
|
|
868
|
+
const timeoutMs = this.activity.playbackWatchdogDelayMs({
|
|
869
|
+
marginMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS,
|
|
870
|
+
minMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS
|
|
871
|
+
});
|
|
872
|
+
if (timeoutMs === void 0) return;
|
|
873
|
+
this.watchdog = setTimeout(() => {
|
|
874
|
+
this.watchdog = void 0;
|
|
875
|
+
logger$3.warn(`discord voice: realtime audio playback watchdog fired reason=${reason} ${this.params.logContext} audioMs=${Math.floor(this.activity.snapshot().audioMs)} elapsedMs=${this.activity.elapsedPlaybackMs()}`);
|
|
876
|
+
this.close("playback-watchdog");
|
|
877
|
+
}, timeoutMs);
|
|
878
|
+
}
|
|
879
|
+
clearWatchdog() {
|
|
880
|
+
clearTimeout(this.watchdog);
|
|
881
|
+
this.watchdog = void 0;
|
|
626
882
|
}
|
|
627
883
|
};
|
|
628
884
|
//#endregion
|
|
629
885
|
//#region extensions/discord/src/voice/realtime-playback.ts
|
|
630
|
-
const logger$
|
|
886
|
+
const logger$2 = createSubsystemLogger("discord/voice");
|
|
631
887
|
const DISCORD_REALTIME_CONTROL_SPEECH_DEDUPE_MS = 5e3;
|
|
632
|
-
const
|
|
633
|
-
const
|
|
634
|
-
const DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_BYTES = 32 * 1024;
|
|
888
|
+
const DISCORD_REALTIME_MAX_RETAINED_RESPONSES = 32;
|
|
889
|
+
const DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_BYTES = 32768;
|
|
635
890
|
const DISCORD_REALTIME_WAKE_ACKS = [
|
|
636
891
|
"Yeah.",
|
|
637
892
|
"Mm-hmm.",
|
|
638
893
|
"Got it.",
|
|
639
894
|
"One sec."
|
|
640
895
|
];
|
|
641
|
-
const
|
|
642
|
-
const DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES = 25;
|
|
643
|
-
const DISCORD_REALTIME_MAX_PENDING_OUTPUT_BYTES = DISCORD_RAW_PCM_FRAME_BYTES * 6e3;
|
|
896
|
+
const DISCORD_REALTIME_MAX_PENDING_OUTPUT_BYTES = 2304e4;
|
|
644
897
|
function normalizeControlSpeechText(text) {
|
|
645
898
|
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
646
899
|
}
|
|
647
900
|
var DiscordRealtimePlayback = class {
|
|
648
901
|
constructor(params) {
|
|
649
902
|
this.params = params;
|
|
650
|
-
this.
|
|
651
|
-
this.
|
|
652
|
-
this.
|
|
903
|
+
this.outputs = /* @__PURE__ */ new Set();
|
|
904
|
+
this.generatingItems = /* @__PURE__ */ new Map();
|
|
905
|
+
this.responseAudio = "completed";
|
|
653
906
|
this.queuedExactSpeechMessages = [];
|
|
654
907
|
this.exactSpeechState = { status: "idle" };
|
|
655
908
|
this.wakeNameAckIndex = 0;
|
|
656
|
-
this.
|
|
657
|
-
|
|
658
|
-
this.
|
|
659
|
-
|
|
660
|
-
};
|
|
661
|
-
}
|
|
662
|
-
attachPlayer() {
|
|
663
|
-
const voiceSdk = loadDiscordVoiceSdk();
|
|
664
|
-
this.params.entry.player.on(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
|
|
909
|
+
this.unregisterPlayerLane = this.params.player.registerLane({
|
|
910
|
+
hasOutput: () => this.isOutputAudioActive(),
|
|
911
|
+
onBargeIn: (reason) => this.handleBargeIn(reason),
|
|
912
|
+
cancelForControl: () => this.cancelForControl()
|
|
913
|
+
});
|
|
665
914
|
}
|
|
666
915
|
close() {
|
|
916
|
+
this.unregisterPlayerLane();
|
|
667
917
|
this.queuedExactSpeechMessages = [];
|
|
668
918
|
this.exactSpeechState = { status: "idle" };
|
|
669
919
|
this.clearOutputAudio("session-close");
|
|
670
|
-
const voiceSdk = loadDiscordVoiceSdk();
|
|
671
|
-
this.params.entry.player.off(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
|
|
672
920
|
}
|
|
673
921
|
handleBargeIn(reason = "barge-in") {
|
|
674
922
|
if (!this.isBargeInEnabled()) {
|
|
675
|
-
logger$
|
|
923
|
+
logger$2.info(`discord voice: realtime barge-in ignored reason=${reason} bargeIn=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}`);
|
|
676
924
|
return;
|
|
677
925
|
}
|
|
678
926
|
if (!this.hasInterruptibleOutputAudio()) {
|
|
679
|
-
logger$
|
|
927
|
+
logger$2.info(`discord voice: realtime barge-in ignored reason=${reason} outputActive=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
680
928
|
return;
|
|
681
929
|
}
|
|
682
|
-
logger$
|
|
683
|
-
this.params.
|
|
930
|
+
logger$2.info(`discord voice: realtime barge-in requested reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
931
|
+
const bridge = this.params.bridge();
|
|
932
|
+
this.params.harness.handleBargeIn({ audioPlaybackActive: true }, () => {
|
|
933
|
+
if (!bridge?.bridge.handleBargeIn) this.clearOutputAudio(reason);
|
|
934
|
+
});
|
|
684
935
|
}
|
|
685
936
|
isBargeInEnabled() {
|
|
686
937
|
if (this.params.wakeNameRequired()) return false;
|
|
@@ -693,67 +944,89 @@ var DiscordRealtimePlayback = class {
|
|
|
693
944
|
}
|
|
694
945
|
hasInterruptibleOutputAudio() {
|
|
695
946
|
this.params.bridge()?.setMediaTimestamp(this.outputAudioMs());
|
|
696
|
-
|
|
697
|
-
|
|
947
|
+
return this.isOutputAudioActive();
|
|
948
|
+
}
|
|
949
|
+
getPlaybackState() {
|
|
950
|
+
const items = /* @__PURE__ */ new Set();
|
|
951
|
+
for (const output of this.outputs) {
|
|
952
|
+
const outputItems = output.playbackItems();
|
|
953
|
+
if (outputItems.some((item) => this.generatingItems.get(item.itemId) === item)) for (const item of this.generatingItems.values()) items.add(item);
|
|
954
|
+
for (const item of outputItems) items.add(item);
|
|
955
|
+
}
|
|
956
|
+
for (const item of this.generatingItems.values()) items.add(item);
|
|
957
|
+
return Array.from(items, (item) => ({
|
|
958
|
+
...item,
|
|
959
|
+
audioEndMs: Math.floor(item.audioEndMs)
|
|
960
|
+
}));
|
|
961
|
+
}
|
|
962
|
+
beginResponse() {
|
|
963
|
+
if (this.responseAudio === "completed") this.responseAudio = "accepting";
|
|
964
|
+
}
|
|
965
|
+
sendOutputMark(acknowledge) {
|
|
966
|
+
if (!this.params.stopped() && this.responseAudio === "accepting") this.generatingOutput?.markPlayback(acknowledge);
|
|
698
967
|
}
|
|
699
|
-
sendOutputAudio(realtimePcm24kMono) {
|
|
968
|
+
sendOutputAudio(realtimePcm24kMono, metadata) {
|
|
700
969
|
this.params.markProviderGenerationObserved();
|
|
701
|
-
if (this.params.stopped()) return;
|
|
970
|
+
if (this.params.stopped() || this.responseAudio === "discarding") return;
|
|
702
971
|
const discordPcm = convertRealtimePcm24kMonoToDiscordPcm48kStereo(realtimePcm24kMono);
|
|
703
972
|
if (discordPcm.length === 0) return;
|
|
704
973
|
this.params.bridge()?.setMediaTimestamp(this.outputAudioMs());
|
|
705
|
-
|
|
706
|
-
|
|
974
|
+
const pendingBytes = Array.from(this.outputs).reduce((total, output) => total + output.pendingBytes(), 0);
|
|
975
|
+
if (discordPcm.length > DISCORD_REALTIME_MAX_PENDING_OUTPUT_BYTES - pendingBytes || !this.generatingOutput && this.outputs.size >= DISCORD_REALTIME_MAX_RETAINED_RESPONSES) {
|
|
976
|
+
this.stopAfterPlaybackFailure("output-audio-overflow", /* @__PURE__ */ new Error(`Discord realtime audio playback overflow: responses=${this.outputs.size} pendingBytes=${pendingBytes} incomingBytes=${discordPcm.length}`));
|
|
707
977
|
return;
|
|
708
978
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
audioStarted: true
|
|
713
|
-
};
|
|
714
|
-
this.params.harness.recordOutputAudio(realtimePcm24kMono, {
|
|
979
|
+
this.beginResponse();
|
|
980
|
+
const output = this.generatingOutput ?? this.createOutput();
|
|
981
|
+
const activity = {
|
|
715
982
|
audioMs: realtimeVoiceAudioDurationMs(REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, realtimePcm24kMono.byteLength),
|
|
716
983
|
sourceAudioBytes: realtimePcm24kMono.length,
|
|
717
984
|
sinkAudioBytes: discordPcm.length
|
|
718
|
-
}
|
|
719
|
-
|
|
985
|
+
};
|
|
986
|
+
let item;
|
|
987
|
+
if (metadata) {
|
|
988
|
+
item = this.generatingItems.get(metadata.itemId);
|
|
989
|
+
if (!item) {
|
|
990
|
+
item = {
|
|
991
|
+
itemId: metadata.itemId,
|
|
992
|
+
audioEndMs: 0
|
|
993
|
+
};
|
|
994
|
+
this.generatingItems.set(metadata.itemId, item);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
this.params.harness.recordOutputAudio(realtimePcm24kMono, activity);
|
|
998
|
+
output.append(discordPcm, activity, item);
|
|
720
999
|
}
|
|
721
1000
|
clearOutputAudio(reason = "clear") {
|
|
722
|
-
this.
|
|
723
|
-
this.
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
this.params.harness.outputActivity.
|
|
729
|
-
|
|
730
|
-
if (playBuffered) {
|
|
731
|
-
this.startOutputPlayback(stream);
|
|
732
|
-
this.scheduleOutputPlaybackWatchdog(reason, stream);
|
|
733
|
-
} else {
|
|
734
|
-
this.resetOutputStream(reason);
|
|
735
|
-
this.params.entry.player.stop(true);
|
|
736
|
-
this.completeExactSpeechResponse(reason);
|
|
737
|
-
return;
|
|
738
|
-
}
|
|
739
|
-
if (!this.outputDrainHandler) stream.end();
|
|
1001
|
+
if (this.responseAudio === "accepting") this.responseAudio = "discarding";
|
|
1002
|
+
this.generatingOutput = void 0;
|
|
1003
|
+
this.generatingItems.clear();
|
|
1004
|
+
const outputs = Array.from(this.outputs);
|
|
1005
|
+
this.outputs.clear();
|
|
1006
|
+
for (const output of outputs.toReversed()) output.close(reason);
|
|
1007
|
+
this.params.harness.outputActivity.reset();
|
|
1008
|
+
this.completeExactSpeechResponse(reason);
|
|
740
1009
|
}
|
|
741
1010
|
handleResponseDone(outcome) {
|
|
742
|
-
|
|
743
|
-
this.
|
|
1011
|
+
const output = this.generatingOutput;
|
|
1012
|
+
this.generatingOutput = void 0;
|
|
1013
|
+
this.generatingItems.clear();
|
|
1014
|
+
this.responseAudio = "completed";
|
|
1015
|
+
output?.finish(outcome.status, outcome.status === "completed");
|
|
1016
|
+
this.completeExactSpeechResponse(outcome.status);
|
|
744
1017
|
}
|
|
745
1018
|
enqueueExactSpeechMessage(text) {
|
|
746
1019
|
if (this.params.stopped() || !text.trim()) return;
|
|
747
1020
|
const retainedMessages = this.queuedExactSpeechMessages.length + (this.exactSpeechState.status === "active" ? 1 : 0);
|
|
748
1021
|
const retainedBytes = this.queuedExactSpeechMessages.reduce((total, message) => total + Buffer.byteLength(message, "utf8"), 0) + Buffer.byteLength(this.exactSpeechState.status === "active" ? this.exactSpeechState.message : "", "utf8");
|
|
749
1022
|
const incomingBytes = Buffer.byteLength(text, "utf8");
|
|
750
|
-
if (retainedMessages >=
|
|
751
|
-
this.
|
|
1023
|
+
if (retainedMessages >= DISCORD_REALTIME_MAX_RETAINED_RESPONSES || retainedBytes + incomingBytes > DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_BYTES) {
|
|
1024
|
+
this.stopAfterPlaybackFailure("exact-speech-overflow", /* @__PURE__ */ new Error(`Discord realtime exact speech overflow: retained=${retainedMessages} retainedBytes=${retainedBytes} incomingBytes=${incomingBytes}`));
|
|
752
1025
|
return;
|
|
753
1026
|
}
|
|
754
|
-
if (!this.params.bridgeReady() || this.exactSpeechState.status === "active" || this.hasInterruptibleOutputAudio()) {
|
|
1027
|
+
if (!this.params.bridgeReady() || this.responseAudio !== "completed" || this.exactSpeechState.status === "active" || this.hasInterruptibleOutputAudio()) {
|
|
755
1028
|
this.queuedExactSpeechMessages.push(text);
|
|
756
|
-
logger$
|
|
1029
|
+
logger$2.info(`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`);
|
|
757
1030
|
return;
|
|
758
1031
|
}
|
|
759
1032
|
this.sendExactSpeechMessage(text);
|
|
@@ -762,32 +1035,27 @@ var DiscordRealtimePlayback = class {
|
|
|
762
1035
|
return [...this.exactSpeechState.status === "active" ? [this.exactSpeechState.message] : [], ...this.queuedExactSpeechMessages];
|
|
763
1036
|
}
|
|
764
1037
|
drainQueuedExactSpeechMessages(reason) {
|
|
765
|
-
if (this.params.stopped() || !this.params.bridgeReady() || this.exactSpeechState.status === "active" || this.queuedExactSpeechMessages.length === 0 || this.hasInterruptibleOutputAudio()) return;
|
|
1038
|
+
if (this.params.stopped() || !this.params.bridgeReady() || this.responseAudio !== "completed" || this.exactSpeechState.status === "active" || this.queuedExactSpeechMessages.length === 0 || this.hasInterruptibleOutputAudio()) return;
|
|
766
1039
|
const next = this.queuedExactSpeechMessages.shift();
|
|
767
1040
|
if (!next) return;
|
|
768
|
-
logger$
|
|
1041
|
+
logger$2.info(`discord voice: realtime exact speech dequeued reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length}`);
|
|
769
1042
|
this.sendExactSpeechMessage(next);
|
|
770
1043
|
}
|
|
771
1044
|
sendWakeNameAck(result) {
|
|
772
1045
|
if (!result.allowed || this.params.stopped() || this.exactSpeechState.status === "active") return;
|
|
773
|
-
if (this.hasInterruptibleOutputAudio()) {
|
|
774
|
-
logger$
|
|
1046
|
+
if (this.params.player.isActive() || this.hasInterruptibleOutputAudio()) {
|
|
1047
|
+
logger$2.info(`discord voice: realtime wake-name ack skipped outputActive=true voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
775
1048
|
return;
|
|
776
1049
|
}
|
|
777
1050
|
const ack = DISCORD_REALTIME_WAKE_ACKS[this.wakeNameAckIndex % DISCORD_REALTIME_WAKE_ACKS.length];
|
|
778
1051
|
this.wakeNameAckIndex += 1;
|
|
779
|
-
logger$
|
|
1052
|
+
logger$2.info(`discord voice: realtime wake-name ack canonical=${result.activationName} heard=${result.heardName} match=${result.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
780
1053
|
this.enqueueExactSpeechMessage(ack ?? "Yeah.");
|
|
781
1054
|
}
|
|
782
1055
|
speakControlResult(text) {
|
|
783
1056
|
const trimmed = text.trim();
|
|
784
1057
|
if (this.params.stopped() || !trimmed) return;
|
|
785
|
-
this.
|
|
786
|
-
this.completeExactSpeechResponse("active-run-control", { drain: false });
|
|
787
|
-
this.params.harness.handleBargeIn({
|
|
788
|
-
audioPlaybackActive: true,
|
|
789
|
-
force: true
|
|
790
|
-
}, () => this.clearOutputAudio("active-run-control"));
|
|
1058
|
+
this.params.player.cancelForControl();
|
|
791
1059
|
this.lastControlSpeech = {
|
|
792
1060
|
normalizedText: normalizeControlSpeechText(trimmed),
|
|
793
1061
|
sentAt: Date.now(),
|
|
@@ -795,6 +1063,15 @@ var DiscordRealtimePlayback = class {
|
|
|
795
1063
|
};
|
|
796
1064
|
this.enqueueExactSpeechMessage(trimmed);
|
|
797
1065
|
}
|
|
1066
|
+
cancelForControl() {
|
|
1067
|
+
this.queuedExactSpeechMessages = [];
|
|
1068
|
+
this.exactSpeechState = { status: "idle" };
|
|
1069
|
+
if (this.responseAudio === "completed" && this.outputs.size === 0) return;
|
|
1070
|
+
this.params.harness.handleBargeIn({
|
|
1071
|
+
audioPlaybackActive: true,
|
|
1072
|
+
force: true
|
|
1073
|
+
}, () => this.clearOutputAudio("active-run-control"));
|
|
1074
|
+
}
|
|
798
1075
|
suppressDuplicateControlSpeech(text) {
|
|
799
1076
|
const recent = this.lastControlSpeech;
|
|
800
1077
|
if (!recent) return;
|
|
@@ -805,7 +1082,7 @@ var DiscordRealtimePlayback = class {
|
|
|
805
1082
|
if (normalizeControlSpeechText(text) !== recent.normalizedText) return;
|
|
806
1083
|
recent.assistantTranscriptCount += 1;
|
|
807
1084
|
if (recent.assistantTranscriptCount <= 1) return;
|
|
808
|
-
logger$
|
|
1085
|
+
logger$2.info(`discord voice: realtime duplicate active-run control speech suppressed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}`);
|
|
809
1086
|
this.params.harness.handleBargeIn({
|
|
810
1087
|
audioPlaybackActive: true,
|
|
811
1088
|
force: true
|
|
@@ -813,168 +1090,73 @@ var DiscordRealtimePlayback = class {
|
|
|
813
1090
|
}
|
|
814
1091
|
resetProviderContinuity(reason) {
|
|
815
1092
|
this.lastControlSpeech = void 0;
|
|
816
|
-
const replayExactSpeech = this.exactSpeechState.status === "active" && !this.
|
|
1093
|
+
const replayExactSpeech = this.exactSpeechState.status === "active" && !this.exactSpeechState.output?.activity.snapshot().playbackStarted ? this.exactSpeechState.message : void 0;
|
|
817
1094
|
this.exactSpeechState = { status: "idle" };
|
|
818
1095
|
if (replayExactSpeech) this.queuedExactSpeechMessages.unshift(replayExactSpeech);
|
|
1096
|
+
this.responseAudio = "discarding";
|
|
819
1097
|
this.params.harness.flushOutput(() => this.clearOutputAudio(reason));
|
|
1098
|
+
this.responseAudio = "completed";
|
|
820
1099
|
this.params.harness.finishOutputAudio(reason);
|
|
821
1100
|
}
|
|
822
1101
|
outputAudioMs() {
|
|
823
1102
|
return Math.floor(this.params.harness.outputActivity.snapshot().audioMs);
|
|
824
1103
|
}
|
|
825
1104
|
isOutputAudioActive() {
|
|
826
|
-
return this.
|
|
1105
|
+
return this.outputs.size > 0 || this.generatingItems.size > 0;
|
|
827
1106
|
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
}
|
|
831
|
-
stopAfterOverflow(reason, error) {
|
|
832
|
-
this.params.stopTerminally(reason);
|
|
1107
|
+
stopAfterPlaybackFailure(reason, error) {
|
|
1108
|
+
this.params.stopTerminally();
|
|
833
1109
|
this.queuedExactSpeechMessages = [];
|
|
834
1110
|
this.exactSpeechState = { status: "idle" };
|
|
835
1111
|
this.clearOutputAudio(reason);
|
|
836
1112
|
this.params.onTerminalError(error);
|
|
837
1113
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
const
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
const pendingBytes = this.outputPacedBytes + stream.writableLength + stream.readableLength;
|
|
854
|
-
if (discordPcm.length > DISCORD_REALTIME_MAX_PENDING_OUTPUT_BYTES - pendingBytes) {
|
|
855
|
-
this.stopAfterOverflow("output-audio-overflow", /* @__PURE__ */ new Error(`Discord realtime audio playback overflow: pendingBytes=${pendingBytes} incomingBytes=${discordPcm.length}`));
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
if (this.params.harness.outputActivity.snapshot().playbackStarted && !this.outputDrainHandler) {
|
|
859
|
-
if (!stream.write(discordPcm)) this.waitForOutputDrain(stream);
|
|
860
|
-
return;
|
|
861
|
-
}
|
|
862
|
-
this.outputPacedBuffers.push(discordPcm);
|
|
863
|
-
this.outputPacedBytes += discordPcm.length;
|
|
864
|
-
if (!this.outputDrainHandler && this.outputPacedBytes >= DISCORD_RAW_PCM_FRAME_BYTES * DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES) this.startOutputPlayback(stream);
|
|
865
|
-
}
|
|
866
|
-
waitForOutputDrain(stream) {
|
|
867
|
-
if (this.outputDrainHandler || this.outputStream !== stream) return;
|
|
868
|
-
logger$1.info(`discord voice: realtime audio playback buffering guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} bufferedBytes=${stream.writableLength + stream.readableLength}`);
|
|
869
|
-
this.outputDrainHandler = () => {
|
|
870
|
-
if (this.params.stopped() || this.outputStream !== stream || stream.destroyed) return;
|
|
871
|
-
this.outputDrainHandler = void 0;
|
|
872
|
-
let refreshWatchdog = this.params.harness.outputActivity.snapshot().streamEnding;
|
|
873
|
-
while (this.outputPacedBuffers.length > 0) {
|
|
874
|
-
const buffered = this.outputPacedBuffers.shift();
|
|
875
|
-
if (!buffered) break;
|
|
876
|
-
this.outputPacedBytes -= buffered.length;
|
|
877
|
-
const writable = stream.write(buffered);
|
|
878
|
-
if (refreshWatchdog) {
|
|
879
|
-
this.scheduleOutputPlaybackWatchdog("output-drain", stream);
|
|
880
|
-
refreshWatchdog = false;
|
|
881
|
-
}
|
|
882
|
-
if (!writable) {
|
|
883
|
-
this.waitForOutputDrain(stream);
|
|
884
|
-
return;
|
|
1114
|
+
createOutput() {
|
|
1115
|
+
const logContext = `guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}`;
|
|
1116
|
+
const output = new DiscordRealtimeOutput({
|
|
1117
|
+
player: this.params.player,
|
|
1118
|
+
logContext,
|
|
1119
|
+
onStart: () => {
|
|
1120
|
+
this.params.harness.outputActivity.markPlaybackStarted();
|
|
1121
|
+
const config = this.params.realtimeConfig();
|
|
1122
|
+
logger$2.info(`discord voice: realtime audio playback started ${logContext} mode=${this.params.mode} model=${config?.model ?? "provider-default"} voice=${config?.speakerVoice ?? config?.speakerVoiceId ?? "provider-default"}`);
|
|
1123
|
+
},
|
|
1124
|
+
onClose: (closed, reason) => {
|
|
1125
|
+
if (!this.outputs.delete(closed)) return;
|
|
1126
|
+
if (this.generatingOutput === closed) {
|
|
1127
|
+
this.generatingOutput = void 0;
|
|
1128
|
+
if (reason !== "player-idle") this.responseAudio = "discarding";
|
|
885
1129
|
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
startOutputPlayback(stream) {
|
|
892
|
-
if (this.params.harness.outputActivity.snapshot().playbackStarted || stream.destroyed) return;
|
|
893
|
-
const voiceSdk = loadDiscordVoiceSdk();
|
|
894
|
-
const opusStream = createDiscordOpusEncodeStream();
|
|
895
|
-
pipeline(stream, opusStream, (err) => {
|
|
896
|
-
if (!err || this.outputStream !== stream) return;
|
|
897
|
-
logger$1.warn(`discord voice: realtime output pipeline failed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}: ${formatErrorMessage(err)}`);
|
|
898
|
-
this.clearOutputAudio("output-pipeline-error");
|
|
899
|
-
this.completeExactSpeechResponse("output-pipeline-error");
|
|
900
|
-
});
|
|
901
|
-
const buffered = Buffer.concat(this.outputPacedBuffers, this.outputPacedBytes);
|
|
902
|
-
this.outputPacedBuffers = [];
|
|
903
|
-
this.outputPacedBytes = 0;
|
|
904
|
-
if (buffered.length > 0 && !stream.write(buffered)) this.waitForOutputDrain(stream);
|
|
905
|
-
const resource = voiceSdk.createAudioResource(opusStream, { inputType: voiceSdk.StreamType.Opus });
|
|
906
|
-
this.params.entry.player.play(resource);
|
|
907
|
-
this.params.harness.outputActivity.markPlaybackStarted();
|
|
908
|
-
const realtimeConfig = this.params.realtimeConfig();
|
|
909
|
-
logger$1.info(`discord voice: realtime audio playback started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} mode=${this.params.mode} model=${realtimeConfig?.model ?? "provider-default"} voice=${realtimeConfig?.speakerVoice ?? realtimeConfig?.speakerVoiceId ?? "provider-default"}`);
|
|
910
|
-
}
|
|
911
|
-
resetOutputStream(reason = "reset") {
|
|
912
|
-
const stream = this.outputStream;
|
|
913
|
-
this.clearOutputPlaybackWatchdog();
|
|
914
|
-
this.logOutputAudioStopped(reason);
|
|
915
|
-
this.outputStream = null;
|
|
916
|
-
this.outputPacedBuffers = [];
|
|
917
|
-
this.outputPacedBytes = 0;
|
|
918
|
-
if (this.outputDrainHandler) {
|
|
919
|
-
stream?.off("drain", this.outputDrainHandler);
|
|
920
|
-
this.outputDrainHandler = void 0;
|
|
921
|
-
}
|
|
922
|
-
this.params.harness.outputActivity.reset();
|
|
923
|
-
stream?.end();
|
|
924
|
-
stream?.destroy();
|
|
925
|
-
}
|
|
926
|
-
scheduleOutputPlaybackWatchdog(reason, stream) {
|
|
927
|
-
this.clearOutputPlaybackWatchdog();
|
|
928
|
-
const timeoutMs = this.params.harness.outputActivity.playbackWatchdogDelayMs({
|
|
929
|
-
marginMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS,
|
|
930
|
-
minMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS
|
|
1130
|
+
if (this.outputs.size === 0) this.params.harness.outputActivity.reset();
|
|
1131
|
+
this.completeExactSpeechResponse(reason);
|
|
1132
|
+
},
|
|
1133
|
+
onBargeIn: (reason) => this.handleBargeIn(reason),
|
|
1134
|
+
onError: (error) => this.stopAfterPlaybackFailure("output-playback-error", error instanceof Error ? error : new Error(formatErrorMessage(error)))
|
|
931
1135
|
});
|
|
932
|
-
if (
|
|
933
|
-
this.
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
this.completeExactSpeechResponse("playback-watchdog");
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
logger$1.warn(`discord voice: realtime audio playback watchdog fired reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${this.outputAudioMs()} elapsedMs=${this.params.harness.outputActivity.elapsedPlaybackMs()}`);
|
|
941
|
-
this.clearOutputAudio("playback-watchdog");
|
|
942
|
-
this.completeExactSpeechResponse("playback-watchdog");
|
|
943
|
-
}, timeoutMs);
|
|
944
|
-
}
|
|
945
|
-
clearOutputPlaybackWatchdog() {
|
|
946
|
-
if (!this.outputPlaybackWatchdog) return;
|
|
947
|
-
clearTimeout(this.outputPlaybackWatchdog);
|
|
948
|
-
this.outputPlaybackWatchdog = void 0;
|
|
1136
|
+
if (this.outputs.size === 0) this.params.harness.outputActivity.markStreamOpened();
|
|
1137
|
+
this.outputs.add(output);
|
|
1138
|
+
this.generatingOutput = output;
|
|
1139
|
+
if (this.exactSpeechState.status === "active") this.exactSpeechState.output ??= output;
|
|
1140
|
+
return output;
|
|
949
1141
|
}
|
|
950
1142
|
sendExactSpeechMessage(text) {
|
|
951
1143
|
if (this.params.stopped() || !text.trim()) return;
|
|
952
1144
|
this.exactSpeechState = {
|
|
953
1145
|
status: "active",
|
|
954
|
-
message: text
|
|
955
|
-
audioStarted: false
|
|
1146
|
+
message: text
|
|
956
1147
|
};
|
|
1148
|
+
this.beginResponse();
|
|
957
1149
|
this.params.bridge()?.sendUserMessage(this.params.buildSpeakExactMessage(text));
|
|
958
1150
|
}
|
|
959
|
-
completeExactSpeechResponse(reason
|
|
960
|
-
if (this.
|
|
1151
|
+
completeExactSpeechResponse(reason) {
|
|
1152
|
+
if (this.responseAudio !== "completed" || this.outputs.size > 0) return;
|
|
961
1153
|
this.exactSpeechState = { status: "idle" };
|
|
962
|
-
if (options?.drain === false) return;
|
|
963
1154
|
this.drainQueuedExactSpeechMessages(reason);
|
|
964
1155
|
}
|
|
965
|
-
logOutputAudioStopped(reason) {
|
|
966
|
-
const activity = this.params.harness.outputActivity.snapshot();
|
|
967
|
-
const audioMs = Math.floor(activity.audioMs);
|
|
968
|
-
const chunks = activity.chunks;
|
|
969
|
-
const discordBytes = activity.sinkAudioBytes;
|
|
970
|
-
const realtimeBytes = activity.sourceAudioBytes;
|
|
971
|
-
const elapsedMs = this.params.harness.outputActivity.elapsedPlaybackMs();
|
|
972
|
-
if (this.outputStream || chunks > 0 || audioMs > 0) logger$1.info(`discord voice: realtime audio playback stopped reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${audioMs} elapsedMs=${elapsedMs} chunks=${chunks} discordBytes=${discordBytes} realtimeBytes=${realtimeBytes}`);
|
|
973
|
-
}
|
|
974
1156
|
};
|
|
975
1157
|
//#endregion
|
|
976
|
-
//#region extensions/discord/src/voice/realtime-session.
|
|
977
|
-
const logger = createSubsystemLogger("discord/voice");
|
|
1158
|
+
//#region extensions/discord/src/voice/realtime-speaker-session.ts
|
|
1159
|
+
const logger$1 = createSubsystemLogger("discord/voice");
|
|
978
1160
|
const DISCORD_REALTIME_DUPLICATE_ERROR_SUPPRESS_MS = 6e4;
|
|
979
1161
|
const discordRealtimeTalkPayload = () => ({});
|
|
980
1162
|
const DISCORD_REALTIME_VERBOSE_OMITTED_EVENTS = /* @__PURE__ */ new Set([
|
|
@@ -1010,7 +1192,7 @@ function readProviderConfigString(config, key) {
|
|
|
1010
1192
|
function isDiscordAgentProxyVoiceMode(mode) {
|
|
1011
1193
|
return mode === "agent-proxy";
|
|
1012
1194
|
}
|
|
1013
|
-
var
|
|
1195
|
+
var DiscordRealtimeSpeakerSession = class {
|
|
1014
1196
|
constructor(params) {
|
|
1015
1197
|
this.params = params;
|
|
1016
1198
|
this.bridge = null;
|
|
@@ -1018,16 +1200,19 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1018
1200
|
status: "inactive",
|
|
1019
1201
|
generation: 0
|
|
1020
1202
|
};
|
|
1021
|
-
this.nextLifecycleGeneration = 0;
|
|
1022
1203
|
this.consultToolPolicy = "safe-read-only";
|
|
1023
1204
|
this.consultPolicy = "auto";
|
|
1024
1205
|
this.wakeNamePolicy = "never";
|
|
1025
1206
|
this.wakeNames = [];
|
|
1026
1207
|
this.providerGenerationObserved = false;
|
|
1027
1208
|
this.providerContinuityEpoch = 0;
|
|
1209
|
+
this.captures = /* @__PURE__ */ new Set();
|
|
1210
|
+
this.inputOpen = true;
|
|
1211
|
+
this.activeOperations = 0;
|
|
1212
|
+
this.lastActivityAt = Date.now();
|
|
1028
1213
|
this.harness = createRealtimeVoiceSessionHarness({
|
|
1029
1214
|
talk: {
|
|
1030
|
-
sessionId:
|
|
1215
|
+
sessionId: this.params.sessionId,
|
|
1031
1216
|
mode: "realtime",
|
|
1032
1217
|
transport: "gateway-relay",
|
|
1033
1218
|
brain: "agent-consult"
|
|
@@ -1054,14 +1239,15 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1054
1239
|
surfaceLabel: "the Discord voice channel"
|
|
1055
1240
|
}),
|
|
1056
1241
|
entry: this.params.entry,
|
|
1242
|
+
player: this.params.player,
|
|
1057
1243
|
harness: this.harness,
|
|
1058
1244
|
markProviderGenerationObserved: () => this.markProviderGenerationObserved(),
|
|
1059
1245
|
mode: this.params.mode,
|
|
1060
1246
|
onTerminalError: this.params.onTerminalError,
|
|
1061
1247
|
providerId: () => this.realtimeProviderId,
|
|
1062
1248
|
realtimeConfig: () => this.realtimeConfig,
|
|
1063
|
-
stopTerminally: (
|
|
1064
|
-
this.
|
|
1249
|
+
stopTerminally: () => {
|
|
1250
|
+
this.lifecycle.status = "stopped";
|
|
1065
1251
|
this.consults.close();
|
|
1066
1252
|
},
|
|
1067
1253
|
stopped: () => this.isStopped(),
|
|
@@ -1071,6 +1257,11 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1071
1257
|
bridge: () => this.bridge,
|
|
1072
1258
|
entry: this.params.entry,
|
|
1073
1259
|
getHumanParticipantCount: () => this.humanParticipantCount(),
|
|
1260
|
+
interruptRoomPlayback: () => {
|
|
1261
|
+
if (!this.playback.isBargeInEnabled() || !this.params.player.isActive()) return false;
|
|
1262
|
+
this.params.player.handleBargeIn("active-speaker-audio");
|
|
1263
|
+
return true;
|
|
1264
|
+
},
|
|
1074
1265
|
onAcceptedTranscript: (text, context, providerEpoch) => this.consults.handleAcceptedTranscript(text, context, providerEpoch),
|
|
1075
1266
|
playback: this.playback,
|
|
1076
1267
|
providerEpoch: () => this.providerContinuityEpoch,
|
|
@@ -1092,7 +1283,7 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1092
1283
|
isWakeNameRequired: () => this.isWakeNameRequired(),
|
|
1093
1284
|
playback: this.playback,
|
|
1094
1285
|
providerEpoch: () => this.providerContinuityEpoch,
|
|
1095
|
-
runAgentTurn: this.params.runAgentTurn,
|
|
1286
|
+
runAgentTurn: (turn) => this.trackOperation(() => this.params.runAgentTurn(turn)),
|
|
1096
1287
|
stopped: () => this.isStopped(),
|
|
1097
1288
|
turns: this.turns,
|
|
1098
1289
|
usesRealtimeAgentHandoff: () => this.params.mode === "bidi" || this.consultToolPolicy !== "none",
|
|
@@ -1100,11 +1291,10 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1100
1291
|
});
|
|
1101
1292
|
}
|
|
1102
1293
|
async connect() {
|
|
1103
|
-
const lifecycleGeneration =
|
|
1294
|
+
const lifecycleGeneration = this.lifecycle.generation + 1;
|
|
1104
1295
|
this.lifecycle = {
|
|
1105
1296
|
status: "starting",
|
|
1106
|
-
generation: lifecycleGeneration
|
|
1107
|
-
instance: this
|
|
1297
|
+
generation: lifecycleGeneration
|
|
1108
1298
|
};
|
|
1109
1299
|
const configuredProviderId = this.realtimeConfig?.provider?.trim();
|
|
1110
1300
|
if (configuredProviderId) {
|
|
@@ -1157,6 +1347,10 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1157
1347
|
toolPolicy,
|
|
1158
1348
|
consultPolicy
|
|
1159
1349
|
});
|
|
1350
|
+
const onReady = () => {
|
|
1351
|
+
this.markProviderGenerationObserved();
|
|
1352
|
+
if (this.markLifecycleReady(lifecycleGeneration)) this.playback.drainQueuedExactSpeechMessages("provider-ready");
|
|
1353
|
+
};
|
|
1160
1354
|
this.bridge = this.harness.createBridge({
|
|
1161
1355
|
provider: resolved.provider,
|
|
1162
1356
|
cfg: this.params.cfg,
|
|
@@ -1166,53 +1360,67 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1166
1360
|
instructions,
|
|
1167
1361
|
autoRespondToAudio,
|
|
1168
1362
|
interruptResponseOnInputAudio,
|
|
1169
|
-
markStrategy: "
|
|
1363
|
+
markStrategy: "transport",
|
|
1170
1364
|
tools: usesRealtimeAgentHandoff ? resolveRealtimeVoiceAgentConsultTools(toolPolicy, toolPolicy !== "none" ? [REALTIME_VOICE_AGENT_CONTROL_TOOL] : []) : [],
|
|
1171
1365
|
audioSink: {
|
|
1172
1366
|
isOpen: () => !this.isStopped(),
|
|
1173
|
-
sendAudio: (audio) => this.playback.sendOutputAudio(audio),
|
|
1367
|
+
sendAudio: (audio, metadata) => this.playback.sendOutputAudio(audio, metadata),
|
|
1368
|
+
sendMark: (markName, acknowledge) => {
|
|
1369
|
+
if (acknowledge) this.playback.sendOutputMark(acknowledge);
|
|
1370
|
+
else this.bridge?.acknowledgeMark(markName);
|
|
1371
|
+
},
|
|
1372
|
+
getPlaybackState: () => this.playback.getPlaybackState(),
|
|
1174
1373
|
clearAudio: () => {
|
|
1175
1374
|
this.markProviderGenerationObserved();
|
|
1176
1375
|
this.harness.flushOutput(() => this.playback.clearOutputAudio("provider-clear-audio"));
|
|
1177
1376
|
}
|
|
1178
1377
|
},
|
|
1179
1378
|
onTranscript: (role, text, isFinal) => {
|
|
1379
|
+
if (this.isStopped()) return;
|
|
1180
1380
|
this.markProviderGenerationObserved();
|
|
1181
|
-
if (isFinal && text.trim()) logger.info(`discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`);
|
|
1381
|
+
if (isFinal && text.trim()) logger$1.info(`discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`);
|
|
1182
1382
|
if (isFinal && role === "assistant") this.playback.suppressDuplicateControlSpeech(text);
|
|
1183
1383
|
if (role !== "user") return;
|
|
1184
1384
|
if (!isFinal) {
|
|
1185
1385
|
this.turns.handlePartialUserTranscript(text);
|
|
1186
1386
|
return;
|
|
1187
1387
|
}
|
|
1188
|
-
this.turns.handleFinalUserTranscript(text);
|
|
1388
|
+
this.trackOperation(() => this.turns.handleFinalUserTranscript(text)).catch((error) => this.logRealtimeError(formatErrorMessage(error)));
|
|
1189
1389
|
},
|
|
1190
1390
|
onToolCall: (event, session) => {
|
|
1391
|
+
if (this.isStopped()) return;
|
|
1191
1392
|
this.markProviderGenerationObserved();
|
|
1192
|
-
return this.consults.handleToolCall(event, session);
|
|
1393
|
+
return this.trackOperation(() => this.consults.handleToolCall(event, session));
|
|
1193
1394
|
},
|
|
1194
|
-
onReady
|
|
1195
|
-
|
|
1196
|
-
if (this.
|
|
1395
|
+
onReady,
|
|
1396
|
+
onEvent: (event) => {
|
|
1397
|
+
if (this.isStopped()) return;
|
|
1398
|
+
this.handleBridgeEvent(event);
|
|
1399
|
+
if (event.direction === "client" && event.type === "session.reconnect.ready") onReady();
|
|
1197
1400
|
},
|
|
1198
|
-
onEvent: (event) => this.handleBridgeEvent(event),
|
|
1199
1401
|
onResponseDone: (outcome) => {
|
|
1402
|
+
if (this.isStopped()) return;
|
|
1200
1403
|
this.markProviderGenerationObserved();
|
|
1201
1404
|
this.playback.handleResponseDone(outcome);
|
|
1202
|
-
if (outcome.status === "cancelled") logger.info(`discord voice: realtime model interrupt confirmed server:response.done status=cancelled${outcome.reason ? ` reason=${outcome.reason}` : ""}`);
|
|
1405
|
+
if (outcome.status === "cancelled") logger$1.info(`discord voice: realtime model interrupt confirmed server:response.done status=cancelled${outcome.reason ? ` reason=${outcome.reason}` : ""}`);
|
|
1203
1406
|
else if (outcome.status === "failed" || outcome.status === "incomplete") this.logRealtimeError(outcome.message);
|
|
1204
1407
|
},
|
|
1205
1408
|
onError: (error) => this.logRealtimeError(formatErrorMessage(error)),
|
|
1206
1409
|
onClose: (reason) => {
|
|
1207
|
-
this.
|
|
1208
|
-
|
|
1410
|
+
if (!this.isStopped()) {
|
|
1411
|
+
this.lifecycle.status = "stopped";
|
|
1412
|
+
this.params.onTerminalError(/* @__PURE__ */ new Error(`Realtime provider closed unexpectedly: ${reason}`));
|
|
1413
|
+
}
|
|
1209
1414
|
}
|
|
1210
1415
|
});
|
|
1416
|
+
if (this.isStopped()) {
|
|
1417
|
+
this.close();
|
|
1418
|
+
return;
|
|
1419
|
+
}
|
|
1211
1420
|
const resolvedModel = readProviderConfigString(resolved.providerConfig, "model") ?? resolved.provider.defaultModel;
|
|
1212
1421
|
const resolvedVoice = readProviderConfigString(resolved.providerConfig, "voice");
|
|
1213
1422
|
const humanParticipantCount = this.humanParticipantCount();
|
|
1214
|
-
logger.info(`discord voice: realtime bridge starting mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"} consultPolicy=${consultPolicy} toolPolicy=${toolPolicy} autoRespond=${autoRespondToAudio} wakeNamePolicy=${this.wakeNamePolicy} requireWakeName=${this.isWakeNameRequired(humanParticipantCount)} humanParticipants=${humanParticipantCount} wakeNames=${this.wakeNames.join(",") || "none"} interruptResponse=${interruptResponseOnInputAudio} bargeIn=${bargeIn} minBargeInAudioEndMs=${minBargeInAudioEndMs}`);
|
|
1215
|
-
this.playback.attachPlayer();
|
|
1423
|
+
logger$1.info(`discord voice: realtime bridge starting mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"} consultPolicy=${consultPolicy} toolPolicy=${toolPolicy} autoRespond=${autoRespondToAudio} wakeNamePolicy=${this.wakeNamePolicy} requireWakeName=${this.isWakeNameRequired(humanParticipantCount)} humanParticipants=${humanParticipantCount} wakeNames=${this.wakeNames.join(",") || "none"} interruptResponse=${interruptResponseOnInputAudio} bargeIn=${bargeIn} minBargeInAudioEndMs=${minBargeInAudioEndMs}`);
|
|
1216
1424
|
await this.bridge.connect();
|
|
1217
1425
|
if (!this.markLifecycleReady(lifecycleGeneration)) {
|
|
1218
1426
|
this.bridge?.close();
|
|
@@ -1220,10 +1428,11 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1220
1428
|
}
|
|
1221
1429
|
this.markProviderGenerationObserved();
|
|
1222
1430
|
this.playback.drainQueuedExactSpeechMessages("provider-connected");
|
|
1223
|
-
logger.info(`discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`);
|
|
1431
|
+
logger$1.info(`discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`);
|
|
1224
1432
|
}
|
|
1225
1433
|
close() {
|
|
1226
|
-
this.
|
|
1434
|
+
this.lifecycle.status = "stopped";
|
|
1435
|
+
this.drain();
|
|
1227
1436
|
this.providerContinuityEpoch += 1;
|
|
1228
1437
|
this.flushSuppressedRealtimeErrors();
|
|
1229
1438
|
this.consults.close();
|
|
@@ -1235,7 +1444,46 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1235
1444
|
this.realtimeProviderId = void 0;
|
|
1236
1445
|
}
|
|
1237
1446
|
beginSpeakerTurn(context, userId) {
|
|
1238
|
-
|
|
1447
|
+
if (!this.inputOpen || this.isStopped()) throw new Error("Discord realtime speaker input is closed");
|
|
1448
|
+
const turn = this.turns.beginSpeakerTurn(context, userId);
|
|
1449
|
+
let closed = false;
|
|
1450
|
+
const capture = {
|
|
1451
|
+
sendInputAudio: (audio) => {
|
|
1452
|
+
if (!closed) {
|
|
1453
|
+
this.lastActivityAt = Date.now();
|
|
1454
|
+
turn.sendInputAudio(audio);
|
|
1455
|
+
}
|
|
1456
|
+
},
|
|
1457
|
+
close: () => {
|
|
1458
|
+
if (closed) return;
|
|
1459
|
+
closed = true;
|
|
1460
|
+
this.captures.delete(capture);
|
|
1461
|
+
this.lastActivityAt = Date.now();
|
|
1462
|
+
turn.close();
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
this.captures.add(capture);
|
|
1466
|
+
return capture;
|
|
1467
|
+
}
|
|
1468
|
+
drain() {
|
|
1469
|
+
this.inputOpen = false;
|
|
1470
|
+
for (const capture of this.captures) capture.close();
|
|
1471
|
+
}
|
|
1472
|
+
releaseReasonBefore(cutoff) {
|
|
1473
|
+
if (!(this.lastActivityAt < cutoff && this.captures.size === 0 && this.activeOperations === 0 && this.consults.isIdle() && !this.playback.isOutputAudioActive() && this.playback.retainedExactSpeechTexts().length === 0)) return;
|
|
1474
|
+
return this.turns.hasPendingSpeakerAudioContext() ? "input-timeout" : "idle";
|
|
1475
|
+
}
|
|
1476
|
+
notify(text) {
|
|
1477
|
+
this.playback.enqueueExactSpeechMessage(text);
|
|
1478
|
+
}
|
|
1479
|
+
async trackOperation(operation) {
|
|
1480
|
+
this.activeOperations += 1;
|
|
1481
|
+
try {
|
|
1482
|
+
return await operation();
|
|
1483
|
+
} finally {
|
|
1484
|
+
this.activeOperations -= 1;
|
|
1485
|
+
this.lastActivityAt = Date.now();
|
|
1486
|
+
}
|
|
1239
1487
|
}
|
|
1240
1488
|
handleBargeIn(reason = "barge-in") {
|
|
1241
1489
|
this.playback.handleBargeIn(reason);
|
|
@@ -1255,21 +1503,9 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1255
1503
|
}
|
|
1256
1504
|
markLifecycleReady(generation) {
|
|
1257
1505
|
if (this.lifecycle.status !== "starting" && this.lifecycle.status !== "active" || this.lifecycle.generation !== generation) return false;
|
|
1258
|
-
this.lifecycle =
|
|
1259
|
-
status: "active",
|
|
1260
|
-
generation,
|
|
1261
|
-
instance: this
|
|
1262
|
-
};
|
|
1506
|
+
this.lifecycle.status = "active";
|
|
1263
1507
|
return true;
|
|
1264
1508
|
}
|
|
1265
|
-
stopLifecycle(reason) {
|
|
1266
|
-
const generation = this.lifecycle.generation;
|
|
1267
|
-
this.lifecycle = {
|
|
1268
|
-
status: "stopped",
|
|
1269
|
-
generation,
|
|
1270
|
-
reason
|
|
1271
|
-
};
|
|
1272
|
-
}
|
|
1273
1509
|
humanParticipantCount() {
|
|
1274
1510
|
return this.params.getHumanParticipantCount?.() ?? 0;
|
|
1275
1511
|
}
|
|
@@ -1280,24 +1516,22 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1280
1516
|
if (!(event.direction === "client" && event.type === "session.continuity.reset")) this.markProviderGenerationObserved();
|
|
1281
1517
|
const detail = event.detail ? ` ${event.detail}` : "";
|
|
1282
1518
|
if (event.direction === "client" && event.type === "session.continuity.reset") this.resetProviderContinuity(event.type);
|
|
1519
|
+
if (event.direction === "server" && event.type === "response.created") this.playback.beginResponse();
|
|
1283
1520
|
if (event.direction === "server" && event.type === "input_audio_buffer.speech_started") this.turns.resetPartialWakeNameTracking();
|
|
1284
1521
|
if (shouldLogRealtimeVerboseEvent(event)) logVoiceVerbose(`realtime ${event.direction}:${event.type}${detail}`);
|
|
1285
1522
|
const interruptionLog = formatRealtimeInterruptionLog(event);
|
|
1286
|
-
if (interruptionLog) logger.info(interruptionLog);
|
|
1523
|
+
if (interruptionLog) logger$1.info(interruptionLog);
|
|
1287
1524
|
const lifecycleLog = formatRealtimeLifecycleLog(event);
|
|
1288
|
-
if (lifecycleLog) logger.info(lifecycleLog);
|
|
1525
|
+
if (lifecycleLog) logger$1.info(lifecycleLog);
|
|
1289
1526
|
}
|
|
1290
1527
|
markProviderGenerationObserved() {
|
|
1528
|
+
this.lastActivityAt = Date.now();
|
|
1291
1529
|
this.providerGenerationObserved = true;
|
|
1292
1530
|
}
|
|
1293
1531
|
resetProviderContinuity(reason) {
|
|
1294
1532
|
if (!this.providerGenerationObserved) return;
|
|
1295
1533
|
this.providerGenerationObserved = false;
|
|
1296
|
-
if (this.lifecycle.status === "active") this.lifecycle =
|
|
1297
|
-
status: "starting",
|
|
1298
|
-
generation: this.lifecycle.generation,
|
|
1299
|
-
instance: this
|
|
1300
|
-
};
|
|
1534
|
+
if (this.lifecycle.status === "active") this.lifecycle.status = "starting";
|
|
1301
1535
|
this.providerContinuityEpoch += 1;
|
|
1302
1536
|
this.consults.resetProviderContinuity();
|
|
1303
1537
|
this.turns.resetProviderContinuity();
|
|
@@ -1315,11 +1549,11 @@ var DiscordRealtimeVoiceSession = class {
|
|
|
1315
1549
|
suppressed: 0,
|
|
1316
1550
|
lastLoggedAt: now
|
|
1317
1551
|
};
|
|
1318
|
-
logger.warn(`discord voice: realtime error: ${message}`);
|
|
1552
|
+
logger$1.warn(`discord voice: realtime error: ${message}`);
|
|
1319
1553
|
}
|
|
1320
1554
|
flushSuppressedRealtimeErrors() {
|
|
1321
1555
|
if (!this.lastRealtimeError || this.lastRealtimeError.suppressed === 0) return;
|
|
1322
|
-
logger.warn(`discord voice: suppressed ${this.lastRealtimeError.suppressed} duplicate realtime errors: ${this.lastRealtimeError.message}`);
|
|
1556
|
+
logger$1.warn(`discord voice: suppressed ${this.lastRealtimeError.suppressed} duplicate realtime errors: ${this.lastRealtimeError.message}`);
|
|
1323
1557
|
this.lastRealtimeError.suppressed = 0;
|
|
1324
1558
|
}
|
|
1325
1559
|
};
|
|
@@ -1336,4 +1570,120 @@ function buildProviderConfigOverrides(realtimeConfig) {
|
|
|
1336
1570
|
return Object.keys(overrides).length > 0 ? overrides : void 0;
|
|
1337
1571
|
}
|
|
1338
1572
|
//#endregion
|
|
1573
|
+
//#region extensions/discord/src/voice/realtime-session.runtime.ts
|
|
1574
|
+
const logger = createSubsystemLogger("discord/voice");
|
|
1575
|
+
const MAX_REALTIME_SPEAKERS = 8;
|
|
1576
|
+
const REALTIME_SPEAKER_IDLE_MS = 6e4;
|
|
1577
|
+
/** The room shares its agent and player; each provider connection has one immutable speaker. */
|
|
1578
|
+
var DiscordRealtimeVoiceSession = class {
|
|
1579
|
+
constructor(params) {
|
|
1580
|
+
this.params = params;
|
|
1581
|
+
this.speakers = /* @__PURE__ */ new Map();
|
|
1582
|
+
this.sessions = /* @__PURE__ */ new Set();
|
|
1583
|
+
this.nextSessionId = 0;
|
|
1584
|
+
this.closed = false;
|
|
1585
|
+
this.player = new DiscordRealtimePlayer(params.entry.player);
|
|
1586
|
+
}
|
|
1587
|
+
async connect() {
|
|
1588
|
+
if (this.closed) throw new Error("Discord realtime voice session is closed");
|
|
1589
|
+
const session = this.createSession();
|
|
1590
|
+
this.warmSession = session;
|
|
1591
|
+
await session.connect();
|
|
1592
|
+
if (this.closed) {
|
|
1593
|
+
session.close();
|
|
1594
|
+
return;
|
|
1595
|
+
}
|
|
1596
|
+
this.idleTimer = setInterval(() => this.releaseIdleSpeakers(), REALTIME_SPEAKER_IDLE_MS);
|
|
1597
|
+
this.idleTimer.unref?.();
|
|
1598
|
+
}
|
|
1599
|
+
close() {
|
|
1600
|
+
if (this.closed) return;
|
|
1601
|
+
this.closed = true;
|
|
1602
|
+
clearInterval(this.idleTimer);
|
|
1603
|
+
this.idleTimer = void 0;
|
|
1604
|
+
this.player.close();
|
|
1605
|
+
this.warmSession?.close();
|
|
1606
|
+
this.warmSession = void 0;
|
|
1607
|
+
for (const { session } of this.sessions) session.close();
|
|
1608
|
+
this.speakers.clear();
|
|
1609
|
+
this.sessions.clear();
|
|
1610
|
+
}
|
|
1611
|
+
beginSpeakerTurn(context, userId) {
|
|
1612
|
+
if (this.closed) throw new Error("Discord realtime voice session is closed");
|
|
1613
|
+
for (const previous of this.sessions) if (previous.userId === userId && previous.senderIsOwner !== context.senderIsOwner) this.retireSpeaker(previous, "admission-changed");
|
|
1614
|
+
let speaker = this.speakers.get(userId);
|
|
1615
|
+
if (speaker && speaker.transcripts !== this.params.entry.transcripts) {
|
|
1616
|
+
this.speakers.delete(userId);
|
|
1617
|
+
speaker.session.drain();
|
|
1618
|
+
speaker = void 0;
|
|
1619
|
+
}
|
|
1620
|
+
if (!speaker) {
|
|
1621
|
+
this.releaseIdleSpeakers();
|
|
1622
|
+
if (this.sessions.size >= MAX_REALTIME_SPEAKERS) {
|
|
1623
|
+
const message = "Voice is busy with other speakers. Please try again after their replies.";
|
|
1624
|
+
this.notify(message);
|
|
1625
|
+
throw new Error(message);
|
|
1626
|
+
}
|
|
1627
|
+
const warm = this.warmSession;
|
|
1628
|
+
this.warmSession = void 0;
|
|
1629
|
+
const session = warm ?? this.createSession();
|
|
1630
|
+
speaker = {
|
|
1631
|
+
userId,
|
|
1632
|
+
senderIsOwner: context.senderIsOwner,
|
|
1633
|
+
transcripts: this.params.entry.transcripts,
|
|
1634
|
+
session
|
|
1635
|
+
};
|
|
1636
|
+
this.speakers.set(userId, speaker);
|
|
1637
|
+
this.sessions.add(speaker);
|
|
1638
|
+
if (!warm) session.connect().catch((error) => this.handleSpeakerFailure(session, error));
|
|
1639
|
+
}
|
|
1640
|
+
return speaker.session.beginSpeakerTurn(context, userId);
|
|
1641
|
+
}
|
|
1642
|
+
handleBargeIn(reason = "barge-in") {
|
|
1643
|
+
this.player.handleBargeIn(reason);
|
|
1644
|
+
}
|
|
1645
|
+
isBargeInEnabled() {
|
|
1646
|
+
return (this.warmSession ?? this.sessions.values().next().value?.session)?.isBargeInEnabled() ?? false;
|
|
1647
|
+
}
|
|
1648
|
+
createSession() {
|
|
1649
|
+
const session = new DiscordRealtimeSpeakerSession({
|
|
1650
|
+
...this.params,
|
|
1651
|
+
player: this.player,
|
|
1652
|
+
sessionId: `discord:${this.params.entry.voiceSessionKey}:realtime:${++this.nextSessionId}`,
|
|
1653
|
+
onTerminalError: (error) => this.handleSpeakerFailure(session, error)
|
|
1654
|
+
});
|
|
1655
|
+
return session;
|
|
1656
|
+
}
|
|
1657
|
+
handleSpeakerFailure(session, error) {
|
|
1658
|
+
if (this.closed) return;
|
|
1659
|
+
if (this.warmSession === session) {
|
|
1660
|
+
this.params.onTerminalError(error instanceof Error ? error : new Error(String(error)));
|
|
1661
|
+
return;
|
|
1662
|
+
}
|
|
1663
|
+
for (const speaker of this.sessions) {
|
|
1664
|
+
if (speaker.session !== session) continue;
|
|
1665
|
+
logger.warn(`discord voice: realtime speaker failed user=${speaker.userId}: ${formatErrorMessage(error)}`);
|
|
1666
|
+
this.retireSpeaker(speaker, "provider-failed");
|
|
1667
|
+
this.notify("I lost a speaker's voice connection. Please try speaking again.");
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
notify(text) {
|
|
1672
|
+
(this.warmSession ?? this.sessions.values().next().value?.session)?.notify(text);
|
|
1673
|
+
}
|
|
1674
|
+
releaseIdleSpeakers() {
|
|
1675
|
+
const cutoff = Date.now() - REALTIME_SPEAKER_IDLE_MS;
|
|
1676
|
+
for (const speaker of this.sessions) {
|
|
1677
|
+
const reason = speaker.session.releaseReasonBefore(cutoff);
|
|
1678
|
+
if (reason) this.retireSpeaker(speaker, reason);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
retireSpeaker(speaker, reason) {
|
|
1682
|
+
if (this.speakers.get(speaker.userId) === speaker) this.speakers.delete(speaker.userId);
|
|
1683
|
+
this.sessions.delete(speaker);
|
|
1684
|
+
speaker.session.close();
|
|
1685
|
+
logger.info(`discord voice: realtime speaker retired user=${speaker.userId} reason=${reason}${reason === "input-timeout" ? "; idle speaker input expired; speak again to reconnect" : ""}`);
|
|
1686
|
+
}
|
|
1687
|
+
};
|
|
1688
|
+
//#endregion
|
|
1339
1689
|
export { DiscordRealtimeVoiceSession };
|