@openclaw/discord 2026.7.2-beta.7 → 2026.8.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{account-inspect-SXiCBCgP.js → account-inspect-CqzrH14k.js} +48 -42
- package/dist/account-inspect-api.js +1 -1
- package/dist/action-runtime-api.js +1 -1
- package/dist/{active-turn-thread-route-MlKWp8RD.js → active-turn-thread-route-fmHl1HNd.js} +9 -11
- package/dist/{activities-api-DTKsuDxf.js → activities-api-DS7vpdtp.js} +4 -4
- package/dist/activities-api.js +1 -1
- package/dist/{allow-list-DzKrtfah.js → allow-list-DnxIlPP2.js} +2 -3
- package/dist/{api-DnrK9gML.js → api-DNVbJyEy.js} +7 -2
- package/dist/api.js +15 -15
- package/dist/{approval-handler.runtime-HymvFTYn.js → approval-handler.runtime-B6yEV79q.js} +5 -5
- package/dist/{approval-message-safety-DEyKVsAw.js → approval-message-safety-AcIxvrf3.js} +1 -1
- package/dist/{audit-CjsrabP8.js → audit-Db-rlntl.js} +4 -4
- package/dist/{channel-vKa-76xY.js → channel-CnnbJSt5.js} +46 -72
- package/dist/{channel-actions-uXFTJX3o.js → channel-actions-5s_hjYKd.js} +4 -4
- package/dist/{channel-actions.runtime-CIYU90sl.js → channel-actions.runtime-CLKrx6tr.js} +53 -36
- package/dist/channel-config-api.js +1 -1
- package/dist/channel-plugin-api.js +1 -1
- package/dist/{channel.setup-CmcwZ_qd.js → channel.setup-BFAO-V1P.js} +15 -53
- package/dist/{config-schema-WHRQcY19.js → config-schema-BgEskFAs.js} +9 -4
- package/dist/{directory-config-D_LojtCe.js → directory-config-BVX7Cflc.js} +1 -1
- package/dist/directory-contract-api.js +1 -1
- package/dist/{directory-live-Bw5Ephzv.js → directory-live-DGIGxpIo.js} +6 -3
- package/dist/{doctor-Bc4S1qyd.js → doctor-BzC-TLnB.js} +4 -4
- package/dist/{doctor-contract-CCFMIDk9.js → doctor-contract-DLa-BTiU.js} +2 -3
- package/dist/doctor-contract-api.js +147 -2
- package/dist/{handle-action.guild-admin-D6VRRW-C.js → handle-action.guild-admin-BNVTUISF.js} +4 -7
- package/dist/{inbound-event-delivery-CEPlt2uz.js → inbound-event-delivery-De2qxTrP.js} +4 -25
- package/dist/index.js +4 -2
- package/dist/{message-handler-CeiSulIw.js → message-handler-Unk_f10F.js} +41 -32
- package/dist/{message-handler.preflight-DibgQFf2.js → message-handler.preflight-C9zecxR9.js} +50 -26
- package/dist/{message-handler.process-Dnc_oz9y.js → message-handler.process-DWQ6UHHu.js} +130 -166
- package/dist/{message-utils-BgTwT_q7.js → message-utils-BSCPVgy0.js} +4 -4
- package/dist/{model-picker-preference-primitives-D25pnCOu.js → model-picker-preference-primitives-BHT1LkR_.js} +3 -3
- package/dist/{outbound-adapter-DtE-i2OJ.js → outbound-adapter-D9nRivT1.js} +91 -32
- package/dist/{outbound-components-C7GVyfPh.js → outbound-components-cTzwBJ60.js} +3 -3
- package/dist/{outbound-session-route-H-HoyH5U.js → outbound-session-route-DxLkrgKQ.js} +68 -95
- package/dist/{pluralkit-CD4B11JB.js → pluralkit-DdiJ9KMK.js} +1 -1
- package/dist/{preflight-audio-BjnQZ5yS.js → preflight-audio-DrnqFA8U.js} +22 -28
- package/dist/{probe-BeWKhjqX.js → probe-Bl-k5_Ku.js} +36 -12
- package/dist/probe.runtime-M9fREKvp.js +2 -0
- package/dist/{provider-DfQJX3z5.js → provider-DvSNb-2v.js} +969 -1248
- package/dist/{provider-session.runtime-Bo88k0pV.js → provider-session.runtime-BUEqD0rA.js} +3 -3
- package/dist/provider.runtime-C9PnJZbZ.js +2 -0
- package/dist/realtime-session.runtime-C7wen-aF.js +1340 -0
- package/dist/{resolve-allowlist-common-DHERApxi.js → resolve-allowlist-common-D04lpEzb.js} +2 -2
- package/dist/{resolve-channels-BjepAj8b.js → resolve-channels-Dm7jEeg6.js} +4 -4
- package/dist/{resolve-users-Bc28htgt.js → resolve-users-bktrCUUU.js} +3 -3
- package/dist/{chunk-DX4jpFQP.js → retry-CLpJcXXf.js} +291 -68
- package/dist/{runtime-_aRC6V2s.js → runtime-DGsSyDTS.js} +30 -25
- package/dist/{runtime-BmW_j_hl.js → runtime-DoHjckYm.js} +4 -6
- package/dist/runtime-api.actions.js +2 -2
- package/dist/runtime-api.js +22 -23
- package/dist/runtime-api.lookup.js +5 -5
- package/dist/runtime-api.monitor-CHxrFPnW.js +5 -0
- package/dist/runtime-api.monitor.js +5 -5
- package/dist/runtime-api.send.js +6 -6
- package/dist/runtime-api.threads.js +5 -6
- package/dist/sdk-runtime-QpY8PvCh.js +370 -0
- package/dist/{send-CdkZ6zeW.js → send-B7Q6Oc1I.js} +175 -86
- package/dist/{send.components-5uqwkjgE.js → send.components-iJyFXw9L.js} +24 -28
- package/dist/{send.outbound-D-80Ej-t.js → send.outbound-DS5tHwfR.js} +37 -24
- package/dist/{send.reactions-CtlmGd_o.js → send.reactions-BAn0iiVQ.js} +14 -21
- package/dist/{send.receipt-CuwRrpFz.js → send.receipt-IPI1p-SP.js} +19 -1
- package/dist/{send.shared-jpvAG_u6.js → send.shared-BmwFjW6T.js} +16 -85
- package/dist/{sender-identity-mG_zsdox.js → sender-identity-QtalnCYv.js} +1 -1
- package/dist/setup-entry.js +0 -5
- package/dist/setup-plugin-api.js +1 -1
- package/dist/{shared-CXF1tup1.js → shared-BIXIX6uJ.js} +16 -17
- package/dist/{subagent-hooks-Dxy7Fmnq.js → subagent-hooks-CTt5KCJr.js} +2 -2
- package/dist/subagent-hooks-api.js +2 -6
- package/dist/subagent-progress-DhaE_h8a.js +109 -0
- package/dist/{system-events-DdUqf91B.js → system-events-CTqDf6_s.js} +2 -2
- package/dist/{target-resolver-C3s_2m0t.js → target-resolver-maOKSqO0.js} +2 -2
- package/dist/targets-yUFTHoAx.js +3 -0
- package/dist/{thread-bindings-C-M-ly7P.js → thread-bindings-3L8p82GV.js} +5 -7
- package/dist/{thread-bindings.manager-DYvSlrva.js → thread-bindings.manager-DCIH6WDC.js} +257 -29
- package/dist/{thread-bindings.session-updates-DAIlp7Px.js → thread-bindings.session-updates-bJ2KQFY6.js} +1 -1
- package/dist/{thread-bindings.state-Cgi5uRn-.js → thread-bindings.state-aeV8HNgV.js} +2 -15
- package/dist/{transcripts-source-DIKGsTGT.js → transcripts-source-CfFMexGt.js} +1 -1
- package/dist/transcripts-source-api.js +1 -1
- package/dist/{typing--Mzo2kM7.js → typing-CS-nNVoe.js} +1 -1
- package/dist/voice-runtime-BEdqNcST.js +2459 -0
- package/openclaw.plugin.json +203 -19
- package/package.json +15 -9
- package/skills/discord/SKILL.md +22 -115
- package/dist/legacy-state-migrations-api.js +0 -138
- package/dist/manager.runtime-Bjav3pfC.js +0 -3655
- package/dist/preflight-audio.runtime-CoCXMM8r.js +0 -7
- package/dist/probe.runtime-DqFGFMbZ.js +0 -2
- package/dist/provider.runtime-Cy7XZmyh.js +0 -2
- package/dist/runtime-api.monitor-CCRubd3-.js +0 -5
- package/dist/session-binding-contract-api.js +0 -2
- package/dist/subagent-progress-DI3sYLLo.js +0 -664
- package/dist/targets-Dv_RN7vT.js +0 -3
- package/dist/thread-bindings.discord-api-BoDNqamb.js +0 -246
|
@@ -0,0 +1,1340 @@
|
|
|
1
|
+
import { S as formatVoiceLogPreview, b as formatVoiceIngressPrompt, h as createDiscordOpusEncodeStream, m as convertRealtimePcm24kMonoToDiscordPcm48kStereo, p as convertDiscordPcm48kStereoToRealtimePcm24kMono, t as loadDiscordVoiceSdk, u as logVoiceVerbose, x as maybeControlDiscordVoiceAgentRun } from "./sdk-runtime-QpY8PvCh.js";
|
|
2
|
+
import { asDateTimestampMs, resolveExpiresAtMsFromDurationMs } from "openclaw/plugin-sdk/number-runtime";
|
|
3
|
+
import { formatErrorMessage } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
4
|
+
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
|
|
5
|
+
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
6
|
+
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AGENT_CONTROL_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, buildRealtimeVoiceSessionInstructions, buildRealtimeVoiceSpeakExactMessage, classifyRealtimeVoiceConsultToolCall, classifySkippableRealtimeVoiceConsultTranscript, controlRealtimeVoiceAgentRun, createRealtimeVoiceAgentTalkbackQueue, createRealtimeVoiceSessionHarness, createRealtimeVoiceTurnContextTracker, isRealtimeVoiceWakeNameRequired, matchRealtimeVoiceActivationName, matchRealtimeVoiceConsultQuestions, parseRealtimeVoiceAgentControlToolArgs, resolveConfiguredRealtimeVoiceProvider, resolveRealtimeVoiceAgentConsultTools, resolveRealtimeVoiceBargeIn, resolveRealtimeVoiceInterruptResponseOnInputAudio, resolveRealtimeVoiceMinBargeInAudioEndMs, resolveRealtimeVoiceSessionPolicy } from "openclaw/plugin-sdk/realtime-voice";
|
|
7
|
+
import { PassThrough, pipeline } from "node:stream";
|
|
8
|
+
//#region extensions/discord/src/voice/realtime-transcript.ts
|
|
9
|
+
function mergeRealtimePartialTranscript(previous, next) {
|
|
10
|
+
const trimmed = next.trim();
|
|
11
|
+
if (!trimmed) return previous;
|
|
12
|
+
return sliceUtf16Safe(trimmed.startsWith(previous) ? trimmed : `${previous}${next}`, -240);
|
|
13
|
+
}
|
|
14
|
+
//#endregion
|
|
15
|
+
//#region extensions/discord/src/voice/realtime-turns.ts
|
|
16
|
+
const logger$3 = createSubsystemLogger("discord/voice");
|
|
17
|
+
const DISCORD_REALTIME_PENDING_SPEAKER_CONTEXT_LIMIT = 32;
|
|
18
|
+
const DISCORD_REALTIME_IGNORED_WAKE_NAME_CONTEXT_TTL_MS = 1e4;
|
|
19
|
+
const DISCORD_REALTIME_WAKE_NAME_FOLLOWUP_TTL_MS = 1e4;
|
|
20
|
+
const REALTIME_PCM16_BYTES_PER_SAMPLE$1 = 2;
|
|
21
|
+
const DISCORD_REALTIME_TRAILING_SILENCE_MIN_MS = 700;
|
|
22
|
+
const DISCORD_REALTIME_TRAILING_SILENCE_MAX_MS = 3e3;
|
|
23
|
+
var DiscordRealtimeTurns = class {
|
|
24
|
+
constructor(params) {
|
|
25
|
+
this.params = params;
|
|
26
|
+
this.speakerTurns = createRealtimeVoiceTurnContextTracker({
|
|
27
|
+
limit: DISCORD_REALTIME_PENDING_SPEAKER_CONTEXT_LIMIT,
|
|
28
|
+
ignoredContextTtlMs: DISCORD_REALTIME_IGNORED_WAKE_NAME_CONTEXT_TTL_MS,
|
|
29
|
+
deferUntilAudio: true
|
|
30
|
+
});
|
|
31
|
+
this.partialUserTranscript = "";
|
|
32
|
+
this.wakeNameAckedForTurn = false;
|
|
33
|
+
}
|
|
34
|
+
beginSpeakerTurn(context, userId) {
|
|
35
|
+
this.resetPartialWakeNameTracking();
|
|
36
|
+
const turn = this.speakerTurns.open({
|
|
37
|
+
...context,
|
|
38
|
+
userId
|
|
39
|
+
}, {
|
|
40
|
+
inputDiscordBytes: 0,
|
|
41
|
+
inputRealtimeBytes: 0,
|
|
42
|
+
inputChunks: 0,
|
|
43
|
+
interruptedPlayback: false
|
|
44
|
+
});
|
|
45
|
+
return {
|
|
46
|
+
sendInputAudio: (discordPcm48kStereo) => this.sendInputAudioForTurn(turn, discordPcm48kStereo),
|
|
47
|
+
close: () => {
|
|
48
|
+
this.sendRealtimeTrailingSilenceForTurn(turn);
|
|
49
|
+
this.logSpeakerTurnClosed(turn);
|
|
50
|
+
this.speakerTurns.close(turn);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
handlePartialUserTranscript(text) {
|
|
55
|
+
if (!this.isWakeNameRequired() || this.wakeNameAckedForTurn) return;
|
|
56
|
+
this.partialUserTranscript = mergeRealtimePartialTranscript(this.partialUserTranscript, text);
|
|
57
|
+
const wakeNameResult = matchRealtimeVoiceActivationName(this.partialUserTranscript, this.params.wakeNames());
|
|
58
|
+
if (!wakeNameResult || wakeNameResult.edge !== "leading") return;
|
|
59
|
+
this.wakeNameAckedForTurn = true;
|
|
60
|
+
this.params.playback.sendWakeNameAck(wakeNameResult);
|
|
61
|
+
}
|
|
62
|
+
async handleFinalUserTranscript(text) {
|
|
63
|
+
const providerEpoch = this.params.providerEpoch();
|
|
64
|
+
const trimmed = text.trim();
|
|
65
|
+
if (!trimmed) return;
|
|
66
|
+
this.partialUserTranscript = "";
|
|
67
|
+
const transcriptsTurn = this.peekPendingSpeakerTurn();
|
|
68
|
+
let transcriptAttribution = this.transcriptAttributionFromTurn(transcriptsTurn);
|
|
69
|
+
const humanParticipantCount = this.params.getHumanParticipantCount();
|
|
70
|
+
const requireWakeName = this.isWakeNameRequired(humanParticipantCount);
|
|
71
|
+
const wakeNameResult = this.resolveWakeNameTranscript(trimmed, requireWakeName);
|
|
72
|
+
let forcedSpeakerContext;
|
|
73
|
+
if (!wakeNameResult.allowed) {
|
|
74
|
+
const pendingWakeNameFollowup = this.consumePendingWakeNameFollowup();
|
|
75
|
+
transcriptAttribution ??= pendingWakeNameFollowup;
|
|
76
|
+
if (!pendingWakeNameFollowup) {
|
|
77
|
+
this.recordTranscriptUtterance(trimmed, transcriptAttribution, providerEpoch);
|
|
78
|
+
this.rememberIgnoredWakeNameSpeakerContext(this.consumePendingSpeakerContext());
|
|
79
|
+
logger$3.info(`discord voice: realtime wake-name gate ignored transcript chars=${trimmed.length} humanParticipants=${humanParticipantCount} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId} wakeNames=${this.params.wakeNames().join(",") || "none"}`);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
forcedSpeakerContext = pendingWakeNameFollowup.context;
|
|
83
|
+
logger$3.info(`discord voice: realtime wake-name follow-up accepted chars=${trimmed.length} speaker=${forcedSpeakerContext.speakerLabel} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
84
|
+
}
|
|
85
|
+
this.recordTranscriptUtterance(trimmed, transcriptAttribution, providerEpoch);
|
|
86
|
+
const acceptedText = wakeNameResult.allowed ? wakeNameResult.text || trimmed : trimmed;
|
|
87
|
+
if (wakeNameResult.allowed && !wakeNameResult.text.trim()) {
|
|
88
|
+
this.armWakeNameFollowup();
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (wakeNameResult.allowed) this.pendingWakeNameFollowup = void 0;
|
|
92
|
+
await this.params.onAcceptedTranscript(acceptedText, forcedSpeakerContext, providerEpoch);
|
|
93
|
+
}
|
|
94
|
+
resetPartialWakeNameTracking() {
|
|
95
|
+
this.partialUserTranscript = "";
|
|
96
|
+
this.wakeNameAckedForTurn = false;
|
|
97
|
+
}
|
|
98
|
+
resetProviderContinuity() {
|
|
99
|
+
this.partialUserTranscript = "";
|
|
100
|
+
this.pendingWakeNameFollowup = void 0;
|
|
101
|
+
}
|
|
102
|
+
clear() {
|
|
103
|
+
this.speakerTurns.clear();
|
|
104
|
+
this.resetPartialWakeNameTracking();
|
|
105
|
+
this.pendingWakeNameFollowup = void 0;
|
|
106
|
+
}
|
|
107
|
+
consumePendingSpeakerContext() {
|
|
108
|
+
return this.speakerTurns.consumeAudioContext();
|
|
109
|
+
}
|
|
110
|
+
consumeRecentIgnoredWakeNameSpeakerContext() {
|
|
111
|
+
return this.speakerTurns.consumeIgnoredContext();
|
|
112
|
+
}
|
|
113
|
+
peekPendingSpeakerTurn() {
|
|
114
|
+
return this.speakerTurns.peekAudioTurn();
|
|
115
|
+
}
|
|
116
|
+
hasPendingSpeakerAudioContext() {
|
|
117
|
+
return this.speakerTurns.hasAudioContext();
|
|
118
|
+
}
|
|
119
|
+
sendInputAudioForTurn(turn, discordPcm48kStereo) {
|
|
120
|
+
const bridge = this.params.bridge();
|
|
121
|
+
if (!bridge || this.params.stopped()) return;
|
|
122
|
+
const realtimePcm = convertDiscordPcm48kStereoToRealtimePcm24kMono(discordPcm48kStereo);
|
|
123
|
+
if (realtimePcm.length > 0) {
|
|
124
|
+
this.registerSpeakerTurnAudioStarted(turn);
|
|
125
|
+
turn.inputDiscordBytes += discordPcm48kStereo.length;
|
|
126
|
+
turn.inputRealtimeBytes += realtimePcm.length;
|
|
127
|
+
turn.inputChunks += 1;
|
|
128
|
+
if (turn.inputChunks === 1) logger$3.info(`discord voice: realtime input audio started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length} outputAudioMs=${this.params.playback.outputAudioMs()} outputActive=${this.params.playback.isOutputAudioActive()}`);
|
|
129
|
+
const outputActive = this.params.playback.hasInterruptibleOutputAudio();
|
|
130
|
+
if (!turn.interruptedPlayback && this.params.playback.isBargeInEnabled() && outputActive) {
|
|
131
|
+
turn.interruptedPlayback = true;
|
|
132
|
+
logVoiceVerbose(`realtime barge-in from active speaker audio: guild ${this.params.entry.guildId} channel ${this.params.entry.channelId} user ${turn.context.userId}`);
|
|
133
|
+
logger$3.info(`discord voice: realtime barge-in detected source=active-speaker-audio guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} outputAudioMs=${this.params.playback.outputAudioMs()} outputActive=${this.params.playback.isOutputAudioActive()} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length}`);
|
|
134
|
+
this.params.playback.handleBargeIn("active-speaker-audio");
|
|
135
|
+
}
|
|
136
|
+
if (this.params.recordInputAudio(realtimePcm)) bridge.sendAudio(realtimePcm);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
registerSpeakerTurnAudioStarted(turn) {
|
|
140
|
+
if (turn.hasAudio) return;
|
|
141
|
+
this.speakerTurns.markAudio(turn);
|
|
142
|
+
logger$3.info(`discord voice: realtime speaker turn opened guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} owner=${turn.context.senderIsOwner} pendingTurns=${this.speakerTurns.size()}`);
|
|
143
|
+
}
|
|
144
|
+
logSpeakerTurnClosed(turn) {
|
|
145
|
+
if (turn.closed || !turn.hasAudio) return;
|
|
146
|
+
const elapsedMs = Date.now() - turn.startedAt;
|
|
147
|
+
const sinceLastAudioMs = turn.lastAudioAt ? Date.now() - turn.lastAudioAt : void 0;
|
|
148
|
+
logger$3.info(`discord voice: realtime speaker turn closed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} owner=${turn.context.senderIsOwner} hasAudio=${turn.hasAudio} chunks=${turn.inputChunks} discordBytes=${turn.inputDiscordBytes} realtimeBytes=${turn.inputRealtimeBytes} elapsedMs=${elapsedMs}${sinceLastAudioMs === void 0 ? "" : ` sinceLastAudioMs=${sinceLastAudioMs}`} interruptedPlayback=${turn.interruptedPlayback}`);
|
|
149
|
+
}
|
|
150
|
+
sendRealtimeTrailingSilenceForTurn(turn) {
|
|
151
|
+
const bridge = this.params.bridge();
|
|
152
|
+
if (!bridge || this.params.stopped() || turn.closed || !turn.hasAudio) return;
|
|
153
|
+
const providerId = this.params.providerId() ?? this.params.realtimeConfig()?.provider ?? "openai";
|
|
154
|
+
const rawSilenceDurationMs = (this.params.realtimeConfig()?.providers?.[providerId])?.silenceDurationMs;
|
|
155
|
+
const silenceMs = Math.min(DISCORD_REALTIME_TRAILING_SILENCE_MAX_MS, Math.max(DISCORD_REALTIME_TRAILING_SILENCE_MIN_MS, typeof rawSilenceDurationMs === "number" && Number.isFinite(rawSilenceDurationMs) ? rawSilenceDurationMs : 0));
|
|
156
|
+
const silenceBytes = Math.ceil(24e3 * silenceMs / 1e3) * REALTIME_PCM16_BYTES_PER_SAMPLE$1;
|
|
157
|
+
const silence = Buffer.alloc(silenceBytes);
|
|
158
|
+
bridge.sendAudio(silence);
|
|
159
|
+
logger$3.info(`discord voice: realtime trailing silence sent guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} silenceMs=${silenceMs} realtimeBytes=${silence.length}`);
|
|
160
|
+
}
|
|
161
|
+
resolveWakeNameTranscript(text, requireWakeName) {
|
|
162
|
+
if (!requireWakeName) return {
|
|
163
|
+
allowed: true,
|
|
164
|
+
text,
|
|
165
|
+
activationName: "",
|
|
166
|
+
heardName: "",
|
|
167
|
+
match: "exact",
|
|
168
|
+
edge: "leading"
|
|
169
|
+
};
|
|
170
|
+
const wakeNameResult = matchRealtimeVoiceActivationName(text, this.params.wakeNames());
|
|
171
|
+
if (wakeNameResult) {
|
|
172
|
+
logger$3.info(`discord voice: realtime wake-name gate matched canonical=${wakeNameResult.activationName} heard=${wakeNameResult.heardName} match=${wakeNameResult.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
173
|
+
return wakeNameResult;
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
allowed: false,
|
|
177
|
+
text
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
isWakeNameRequired(humanParticipantCount = this.params.getHumanParticipantCount()) {
|
|
181
|
+
return isRealtimeVoiceWakeNameRequired(this.params.wakeNamePolicy(), humanParticipantCount);
|
|
182
|
+
}
|
|
183
|
+
transcriptAttributionFromTurn(turn) {
|
|
184
|
+
return turn ? {
|
|
185
|
+
context: turn.context,
|
|
186
|
+
startedAt: turn.startedAt
|
|
187
|
+
} : void 0;
|
|
188
|
+
}
|
|
189
|
+
recordTranscriptUtterance(text, attribution, providerEpoch) {
|
|
190
|
+
const transcripts = this.params.entry.transcripts;
|
|
191
|
+
if (!transcripts || !attribution) return;
|
|
192
|
+
const context = attribution.context;
|
|
193
|
+
const utterance = {
|
|
194
|
+
sessionId: transcripts.sessionId,
|
|
195
|
+
startedAt: new Date(attribution.startedAt).toISOString(),
|
|
196
|
+
final: true,
|
|
197
|
+
speaker: {
|
|
198
|
+
id: context.userId,
|
|
199
|
+
label: context.speakerLabel
|
|
200
|
+
},
|
|
201
|
+
text,
|
|
202
|
+
metadata: {
|
|
203
|
+
channel: "discord",
|
|
204
|
+
guildId: this.params.entry.guildId,
|
|
205
|
+
channelId: this.params.entry.channelId,
|
|
206
|
+
voiceSessionKey: this.params.entry.voiceSessionKey
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
Promise.resolve().then(() => {
|
|
210
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
211
|
+
return transcripts.onUtterance(utterance);
|
|
212
|
+
}).catch((error) => {
|
|
213
|
+
logger$3.warn(`discord voice: realtime transcripts utterance failed: ${formatErrorMessage(error)}`);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
armWakeNameFollowup() {
|
|
217
|
+
const turn = this.peekPendingSpeakerTurn();
|
|
218
|
+
const context = this.consumePendingSpeakerContext();
|
|
219
|
+
if (!context) {
|
|
220
|
+
logger$3.warn(`discord voice: realtime wake-name follow-up has no speaker context voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const expiresAt = resolveExpiresAtMsFromDurationMs(DISCORD_REALTIME_WAKE_NAME_FOLLOWUP_TTL_MS);
|
|
224
|
+
if (expiresAt === void 0) return;
|
|
225
|
+
this.pendingWakeNameFollowup = {
|
|
226
|
+
context,
|
|
227
|
+
startedAt: turn?.startedAt ?? Date.now(),
|
|
228
|
+
expiresAt
|
|
229
|
+
};
|
|
230
|
+
logger$3.info(`discord voice: realtime wake-name follow-up armed speaker=${context.speakerLabel} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
231
|
+
}
|
|
232
|
+
consumePendingWakeNameFollowup() {
|
|
233
|
+
const pending = this.pendingWakeNameFollowup;
|
|
234
|
+
this.pendingWakeNameFollowup = void 0;
|
|
235
|
+
const now = asDateTimestampMs(Date.now());
|
|
236
|
+
const expiresAt = pending ? asDateTimestampMs(pending.expiresAt) : void 0;
|
|
237
|
+
if (!pending || now === void 0 || expiresAt === void 0 || now > expiresAt) return;
|
|
238
|
+
const currentTurn = this.peekPendingSpeakerTurn();
|
|
239
|
+
if (currentTurn && currentTurn.context.userId !== pending.context.userId) return;
|
|
240
|
+
if (currentTurn) this.consumePendingSpeakerContext();
|
|
241
|
+
return {
|
|
242
|
+
context: pending.context,
|
|
243
|
+
startedAt: pending.startedAt
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
rememberIgnoredWakeNameSpeakerContext(context) {
|
|
247
|
+
this.speakerTurns.rememberIgnoredContext(context);
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
function isDiscordRealtimeSpeakerContext(value) {
|
|
251
|
+
return Boolean(value) && typeof value === "object" && typeof value.userId === "string" && typeof value.senderIsOwner === "boolean" && typeof value.speakerLabel === "string";
|
|
252
|
+
}
|
|
253
|
+
//#endregion
|
|
254
|
+
//#region extensions/discord/src/voice/realtime-consults.ts
|
|
255
|
+
const logger$2 = createSubsystemLogger("discord/voice");
|
|
256
|
+
const DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS = 350;
|
|
257
|
+
const DISCORD_REALTIME_FALLBACK_TEXT = "I hit an error while checking that. Please try again.";
|
|
258
|
+
const DISCORD_REALTIME_FORCED_CONSULT_FALLBACK_DELAY_MS = 200;
|
|
259
|
+
const DISCORD_REALTIME_FORCED_CONSULT_REASON = "provider_final_transcript_without_openclaw_agent_consult";
|
|
260
|
+
var DiscordRealtimeConsults = class {
|
|
261
|
+
constructor(params) {
|
|
262
|
+
this.params = params;
|
|
263
|
+
this.talkback = this.createTalkbackQueue();
|
|
264
|
+
}
|
|
265
|
+
close() {
|
|
266
|
+
this.talkback.close();
|
|
267
|
+
this.clearProviderConsultState();
|
|
268
|
+
}
|
|
269
|
+
resetProviderContinuity() {
|
|
270
|
+
this.talkback.close();
|
|
271
|
+
this.talkback = this.createTalkbackQueue();
|
|
272
|
+
this.clearProviderConsultState();
|
|
273
|
+
}
|
|
274
|
+
async handleToolCall(event, session) {
|
|
275
|
+
const providerEpoch = this.params.providerEpoch();
|
|
276
|
+
const callId = event.callId || event.itemId || "unknown";
|
|
277
|
+
if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) {
|
|
278
|
+
await this.handleAgentControlToolCall(event, session, callId, providerEpoch);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
|
282
|
+
await session.submitToolResult(callId, { error: `Tool "${event.name}" not available` });
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
if (this.params.consultToolPolicy() === "none") {
|
|
286
|
+
await session.submitToolResult(callId, { error: `Tool "${event.name}" not available` });
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
const outcome = classifyRealtimeVoiceConsultToolCall(event.args, { retainedExactSpeechTexts: this.params.playback.retainedExactSpeechTexts() });
|
|
290
|
+
switch (outcome.kind) {
|
|
291
|
+
case "exact-speech-echo":
|
|
292
|
+
logger$2.info(`discord voice: realtime exact speech consult bypassed call=${callId || "unknown"} answerChars=${outcome.text.length}`);
|
|
293
|
+
await session.submitToolResult(callId, { text: outcome.text });
|
|
294
|
+
return;
|
|
295
|
+
case "malformed":
|
|
296
|
+
logger$2.warn(`discord voice: realtime consult rejected malformed args call=${callId || "unknown"}: ${outcome.error}`);
|
|
297
|
+
await session.submitToolResult(callId, { error: outcome.error });
|
|
298
|
+
return;
|
|
299
|
+
case "consult": break;
|
|
300
|
+
}
|
|
301
|
+
const consultMessage = outcome.message;
|
|
302
|
+
logger$2.info(`discord voice: realtime consult requested call=${callId || "unknown"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} question=${formatVoiceLogPreview(consultMessage)}`);
|
|
303
|
+
const nativeConsult = this.params.harness.forcedConsults.recordNativeConsult(event.args, callId);
|
|
304
|
+
if (nativeConsult.kind === "already_delivered" && this.params.harness.forcedConsults.isCancelled(nativeConsult.handle)) {
|
|
305
|
+
await this.submitTerminalRealtimeToolResult(callId, session, {
|
|
306
|
+
status: "cancelled",
|
|
307
|
+
message: "OpenClaw cancelled this consult before completion. Do not restart it."
|
|
308
|
+
});
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
const pendingConsult = nativeConsult.kind === "pending" ? nativeConsult.handle : void 0;
|
|
312
|
+
if (pendingConsult) this.params.harness.forcedConsults.rememberQuestion(pendingConsult, consultMessage);
|
|
313
|
+
let context = pendingConsult?.context?.speaker;
|
|
314
|
+
let recent = pendingConsult;
|
|
315
|
+
if (!context) {
|
|
316
|
+
const recentConsult = nativeConsult.kind === "in_flight" || nativeConsult.kind === "already_delivered" ? nativeConsult.handle : this.findRecentAgentProxyConsultContext(consultMessage);
|
|
317
|
+
if (recentConsult) {
|
|
318
|
+
const recentSpeaker = recentConsult.context?.speaker;
|
|
319
|
+
if (this.params.turns.hasPendingSpeakerAudioContext()) {
|
|
320
|
+
logger$2.info(`discord voice: realtime consult matched recent agent result but newer speaker audio is pending call=${callId} speaker=${recentSpeaker?.speakerLabel ?? "unknown"} owner=${recentSpeaker?.senderIsOwner ?? false}`);
|
|
321
|
+
await session.submitToolResult(callId, { error: "Discord speaker context changed before this realtime consult completed" });
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
if (await this.submitRecentAgentProxyConsultResult(callId, recentConsult, session)) return;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
if (!context) {
|
|
328
|
+
context = this.params.turns.consumePendingSpeakerContext();
|
|
329
|
+
if (context) recent = this.rememberRecentAgentProxyConsultContext(consultMessage, context, {
|
|
330
|
+
...callId === "unknown" ? {} : { id: `native-consult:${callId}` },
|
|
331
|
+
started: true
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
if (!context) {
|
|
335
|
+
logger$2.warn(`discord voice: realtime consult has no speaker context call=${callId || "unknown"}`);
|
|
336
|
+
await session.submitToolResult(callId, { error: "No Discord speaker context available" });
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
const promise = this.runAgentTurn({
|
|
340
|
+
context,
|
|
341
|
+
message: consultMessage
|
|
342
|
+
});
|
|
343
|
+
if (recent) this.setRecentAgentProxyConsultPromise(recent, promise);
|
|
344
|
+
let text;
|
|
345
|
+
try {
|
|
346
|
+
text = await promise;
|
|
347
|
+
} catch (error) {
|
|
348
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
349
|
+
const message = formatErrorMessage(error);
|
|
350
|
+
logger$2.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`);
|
|
351
|
+
await session.submitToolResult(callId, { error: message });
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
355
|
+
logger$2.info(`discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`);
|
|
356
|
+
await session.submitToolResult(callId, { text });
|
|
357
|
+
}
|
|
358
|
+
async handleAcceptedTranscript(acceptedText, forcedSpeakerContext, providerEpoch) {
|
|
359
|
+
const pendingForcedConsult = this.params.isAgentProxy && this.params.usesRealtimeAgentHandoff() ? this.prepareForcedAgentProxyConsult(acceptedText, forcedSpeakerContext) : void 0;
|
|
360
|
+
let control;
|
|
361
|
+
try {
|
|
362
|
+
control = await maybeControlDiscordVoiceAgentRun({
|
|
363
|
+
entry: this.params.entry,
|
|
364
|
+
text: acceptedText
|
|
365
|
+
});
|
|
366
|
+
} catch (error) {
|
|
367
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
368
|
+
logger$2.warn(`discord voice: realtime active-run control failed; falling back to normal transcript handling: ${formatErrorMessage(error)}`);
|
|
369
|
+
control = void 0;
|
|
370
|
+
}
|
|
371
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
372
|
+
if (control?.handled) {
|
|
373
|
+
if (pendingForcedConsult) this.params.harness.forcedConsults.remove(pendingForcedConsult);
|
|
374
|
+
this.logAgentControlResult(control.result);
|
|
375
|
+
if (control.speakText) this.params.playback.speakControlResult(control.speakText);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (!this.params.isAgentProxy) return;
|
|
379
|
+
if (this.params.usesRealtimeAgentHandoff()) {
|
|
380
|
+
if (pendingForcedConsult) this.schedulePreparedForcedAgentProxyConsult(pendingForcedConsult);
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
this.talkback.enqueue(acceptedText, forcedSpeakerContext ?? this.params.turns.consumePendingSpeakerContext());
|
|
384
|
+
}
|
|
385
|
+
createTalkbackQueue() {
|
|
386
|
+
const providerEpoch = this.params.providerEpoch();
|
|
387
|
+
return createRealtimeVoiceAgentTalkbackQueue({
|
|
388
|
+
debounceMs: this.params.debounceMs() ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
|
|
389
|
+
isStopped: () => this.params.stopped() || providerEpoch !== this.params.providerEpoch(),
|
|
390
|
+
logger: logger$2,
|
|
391
|
+
logPrefix: "[discord] realtime agent",
|
|
392
|
+
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
|
|
393
|
+
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
|
|
394
|
+
consult: async ({ question, responseStyle, metadata }) => {
|
|
395
|
+
const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : void 0;
|
|
396
|
+
return { text: await this.runAgentTurn({
|
|
397
|
+
context,
|
|
398
|
+
message: formatVoiceIngressPrompt([question, responseStyle ? `Spoken style: ${responseStyle}` : void 0].filter(Boolean).join("\n\n"), context?.speakerLabel ?? "Discord voice speaker")
|
|
399
|
+
}) };
|
|
400
|
+
},
|
|
401
|
+
deliver: (text) => this.params.playback.enqueueExactSpeechMessage(text)
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
async handleAgentControlToolCall(event, session, callId, providerEpoch) {
|
|
405
|
+
let result;
|
|
406
|
+
try {
|
|
407
|
+
const parsed = parseRealtimeVoiceAgentControlToolArgs(event.args);
|
|
408
|
+
result = await controlRealtimeVoiceAgentRun({
|
|
409
|
+
sessionKey: this.params.entry.route.sessionKey,
|
|
410
|
+
text: parsed.text,
|
|
411
|
+
mode: parsed.mode
|
|
412
|
+
});
|
|
413
|
+
} catch (error) {
|
|
414
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
415
|
+
await session.submitToolResult(callId, { error: formatErrorMessage(error) });
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
418
|
+
if (providerEpoch !== this.params.providerEpoch()) return;
|
|
419
|
+
this.logAgentControlResult(result);
|
|
420
|
+
await session.submitToolResult(callId, result);
|
|
421
|
+
}
|
|
422
|
+
async runAgentTurn(params) {
|
|
423
|
+
const context = params.context;
|
|
424
|
+
if (!context) return "";
|
|
425
|
+
return this.params.runAgentTurn({
|
|
426
|
+
context,
|
|
427
|
+
message: params.message,
|
|
428
|
+
toolsAllow: this.params.consultToolsAllow(),
|
|
429
|
+
userId: context.userId
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
logAgentControlResult(result) {
|
|
433
|
+
logger$2.info(`discord voice: realtime active-run control handled mode=${result.mode} ok=${result.ok} active=${result.active} reason=${result.reason ?? "none"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}`);
|
|
434
|
+
}
|
|
435
|
+
prepareForcedAgentProxyConsult(transcript, speakerContext) {
|
|
436
|
+
if (this.params.consultPolicy() !== "always" && this.params.wakeNamePolicy() === "never") return;
|
|
437
|
+
const question = transcript.trim();
|
|
438
|
+
if (!question) return;
|
|
439
|
+
const skipReason = classifySkippableRealtimeVoiceConsultTranscript(question);
|
|
440
|
+
if (skipReason) {
|
|
441
|
+
const context = this.params.turns.consumePendingSpeakerContext();
|
|
442
|
+
logger$2.info(`discord voice: realtime forced agent consult skipped reason=${skipReason} chars=${question.length} speaker=${context?.speakerLabel ?? "unknown"} transcript=${formatVoiceLogPreview(question)}`);
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
let context = speakerContext ?? this.params.turns.consumePendingSpeakerContext();
|
|
446
|
+
if (!context) context = this.params.turns.consumeRecentIgnoredWakeNameSpeakerContext();
|
|
447
|
+
if (!context) {
|
|
448
|
+
const recent = this.findRecentAgentProxyConsultContext(question);
|
|
449
|
+
if (recent) {
|
|
450
|
+
logVoiceVerbose(`realtime forced agent consult skipped (already delegated): guild ${this.params.entry.guildId} channel ${this.params.entry.channelId} speaker ${recent.context?.speaker.userId ?? "unknown"}`);
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
logger$2.warn("discord voice: realtime forced agent consult has no speaker context");
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
return this.params.harness.forcedConsults.prepare(question, { context: {
|
|
457
|
+
speaker: context,
|
|
458
|
+
providerEpoch: this.params.providerEpoch()
|
|
459
|
+
} });
|
|
460
|
+
}
|
|
461
|
+
schedulePreparedForcedAgentProxyConsult(pending) {
|
|
462
|
+
this.params.harness.forcedConsults.schedule(pending, DISCORD_REALTIME_FORCED_CONSULT_FALLBACK_DELAY_MS, (handle) => void this.runForcedAgentProxyConsult(handle));
|
|
463
|
+
}
|
|
464
|
+
async runForcedAgentProxyConsult(pending) {
|
|
465
|
+
this.params.harness.forcedConsults.markStarted(pending);
|
|
466
|
+
const state = pending.context;
|
|
467
|
+
if (!state) {
|
|
468
|
+
this.params.harness.forcedConsults.markCancelled(pending);
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
const context = state.speaker;
|
|
472
|
+
const { question } = pending;
|
|
473
|
+
if (this.params.stopped() || state.providerEpoch !== this.params.providerEpoch()) {
|
|
474
|
+
this.params.harness.forcedConsults.markCancelled(pending);
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
const startedAt = Date.now();
|
|
478
|
+
logger$2.info(`discord voice: realtime forced agent consult starting chars=${question.length} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}`);
|
|
479
|
+
logger$2.debug(`discord voice: realtime forced agent consult reason=${DISCORD_REALTIME_FORCED_CONSULT_REASON} consultPolicy=${this.params.consultPolicy()} wakeNamePolicy=${this.params.wakeNamePolicy()} requireWakeName=${this.params.isWakeNameRequired()} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel}`);
|
|
480
|
+
if (this.params.playback.hasInterruptibleOutputAudio()) logger$2.info(`discord voice: realtime forced agent consult preserving active playback guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.params.playback.outputAudioMs()} outputActive=${this.params.playback.isOutputAudioActive()} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
481
|
+
state.handledByForcedPlayback = true;
|
|
482
|
+
try {
|
|
483
|
+
const promise = this.runAgentTurn({
|
|
484
|
+
context,
|
|
485
|
+
message: question
|
|
486
|
+
});
|
|
487
|
+
this.setRecentAgentProxyConsultPromise(pending, promise);
|
|
488
|
+
const text = await promise;
|
|
489
|
+
await state.providerDelivery;
|
|
490
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
491
|
+
logger$2.info(`discord voice: realtime forced agent consult answer (${text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(text)}`);
|
|
492
|
+
if (text.trim() && state.handledByForcedPlayback) this.params.playback.enqueueExactSpeechMessage(text);
|
|
493
|
+
} catch (error) {
|
|
494
|
+
await state.providerDelivery;
|
|
495
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
496
|
+
logger$2.warn(`discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`);
|
|
497
|
+
if (state.handledByForcedPlayback) this.params.playback.enqueueExactSpeechMessage(DISCORD_REALTIME_FALLBACK_TEXT);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
rememberRecentAgentProxyConsultContext(question, context, options = {}) {
|
|
501
|
+
const handle = this.params.harness.forcedConsults.prepare(question, {
|
|
502
|
+
context: {
|
|
503
|
+
speaker: context,
|
|
504
|
+
providerEpoch: this.params.providerEpoch()
|
|
505
|
+
},
|
|
506
|
+
...options.id ? { id: options.id } : {}
|
|
507
|
+
});
|
|
508
|
+
if (!handle) throw new Error("Discord realtime consult context requires a non-empty question");
|
|
509
|
+
if (options.started) this.params.harness.forcedConsults.markStarted(handle);
|
|
510
|
+
return handle;
|
|
511
|
+
}
|
|
512
|
+
setRecentAgentProxyConsultPromise(recent, promise) {
|
|
513
|
+
const state = recent.context;
|
|
514
|
+
if (!state) return;
|
|
515
|
+
this.params.harness.forcedConsults.markStarted(recent);
|
|
516
|
+
state.promise = promise;
|
|
517
|
+
promise.then((text) => {
|
|
518
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
519
|
+
state.result = {
|
|
520
|
+
status: "fulfilled",
|
|
521
|
+
text
|
|
522
|
+
};
|
|
523
|
+
this.params.harness.forcedConsults.markDelivered(recent);
|
|
524
|
+
}).catch((error) => {
|
|
525
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
526
|
+
state.result = {
|
|
527
|
+
status: "rejected",
|
|
528
|
+
error: formatErrorMessage(error)
|
|
529
|
+
};
|
|
530
|
+
this.params.harness.forcedConsults.markDelivered(recent);
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
findRecentAgentProxyConsultContext(consultMessage) {
|
|
534
|
+
return this.params.harness.forcedConsults.findRecent(consultMessage);
|
|
535
|
+
}
|
|
536
|
+
async submitTerminalRealtimeToolResult(callId, session, result) {
|
|
537
|
+
if (session.bridge.supportsToolResultSuppression === false) {
|
|
538
|
+
await session.submitToolResult(callId, result);
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
await session.submitToolResult(callId, result, { suppressResponse: true });
|
|
542
|
+
}
|
|
543
|
+
async submitRecentAgentProxyConsultResult(callId, recent, session) {
|
|
544
|
+
const state = recent.context;
|
|
545
|
+
if (!state) return false;
|
|
546
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return true;
|
|
547
|
+
const providerOwnsDelivery = Boolean(state.handledByForcedPlayback && state.promise && !state.result && session.bridge.supportsToolResultSuppression === false);
|
|
548
|
+
let resolveProviderDelivery;
|
|
549
|
+
if (providerOwnsDelivery) state.providerDelivery = new Promise((resolve) => {
|
|
550
|
+
resolveProviderDelivery = resolve;
|
|
551
|
+
state.settleProviderDelivery = resolve;
|
|
552
|
+
});
|
|
553
|
+
const submitAlreadyDelivered = async () => {
|
|
554
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
555
|
+
await this.submitTerminalRealtimeToolResult(callId, session, {
|
|
556
|
+
status: "already_delivered",
|
|
557
|
+
message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it."
|
|
558
|
+
});
|
|
559
|
+
};
|
|
560
|
+
const submitResult = async (result) => {
|
|
561
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return;
|
|
562
|
+
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
|
|
563
|
+
await submitAlreadyDelivered();
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
if (result.status === "fulfilled") {
|
|
567
|
+
await session.submitToolResult(callId, { text: result.text });
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
await session.submitToolResult(callId, { error: result.error });
|
|
571
|
+
};
|
|
572
|
+
if (state.result) {
|
|
573
|
+
logger$2.info(`discord voice: realtime consult reused recent agent result call=${callId || "unknown"} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`);
|
|
574
|
+
await submitResult(state.result);
|
|
575
|
+
return true;
|
|
576
|
+
}
|
|
577
|
+
if (!state.promise) return false;
|
|
578
|
+
logger$2.info(`discord voice: realtime consult joined in-flight agent result call=${callId || "unknown"} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`);
|
|
579
|
+
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
|
|
580
|
+
await state.promise.catch(() => void 0);
|
|
581
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return true;
|
|
582
|
+
await submitAlreadyDelivered();
|
|
583
|
+
return true;
|
|
584
|
+
}
|
|
585
|
+
let result;
|
|
586
|
+
try {
|
|
587
|
+
result = {
|
|
588
|
+
status: "fulfilled",
|
|
589
|
+
text: await state.promise
|
|
590
|
+
};
|
|
591
|
+
} catch (error) {
|
|
592
|
+
result = {
|
|
593
|
+
status: "rejected",
|
|
594
|
+
error: formatErrorMessage(error)
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
if (state.providerEpoch !== this.params.providerEpoch()) return true;
|
|
598
|
+
try {
|
|
599
|
+
await submitResult(result);
|
|
600
|
+
if (providerOwnsDelivery) {
|
|
601
|
+
state.handledByForcedPlayback = false;
|
|
602
|
+
state.settleProviderDelivery = void 0;
|
|
603
|
+
resolveProviderDelivery?.(true);
|
|
604
|
+
}
|
|
605
|
+
} catch (error) {
|
|
606
|
+
state.settleProviderDelivery = void 0;
|
|
607
|
+
resolveProviderDelivery?.(false);
|
|
608
|
+
throw error;
|
|
609
|
+
}
|
|
610
|
+
return true;
|
|
611
|
+
}
|
|
612
|
+
clearProviderConsultState() {
|
|
613
|
+
for (const handle of this.params.harness.forcedConsults.handles()) {
|
|
614
|
+
const state = handle.context;
|
|
615
|
+
if (!state) continue;
|
|
616
|
+
state.handledByForcedPlayback = false;
|
|
617
|
+
state.settleProviderDelivery?.(false);
|
|
618
|
+
state.settleProviderDelivery = void 0;
|
|
619
|
+
state.providerDelivery = void 0;
|
|
620
|
+
}
|
|
621
|
+
this.params.harness.forcedConsults.clear();
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
//#endregion
|
|
625
|
+
//#region extensions/discord/src/voice/realtime-playback.ts
|
|
626
|
+
const logger$1 = createSubsystemLogger("discord/voice");
|
|
627
|
+
const DISCORD_REALTIME_CONTROL_SPEECH_DEDUPE_MS = 5e3;
|
|
628
|
+
const DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS = 1500;
|
|
629
|
+
const DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_MESSAGES = 32;
|
|
630
|
+
const DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_BYTES = 32 * 1024;
|
|
631
|
+
const DISCORD_REALTIME_CANCELLATION_RACE_DETAIL = "Cancellation failed: no active response found";
|
|
632
|
+
const DISCORD_REALTIME_WAKE_ACKS = [
|
|
633
|
+
"Yeah.",
|
|
634
|
+
"Mm-hmm.",
|
|
635
|
+
"Got it.",
|
|
636
|
+
"One sec."
|
|
637
|
+
];
|
|
638
|
+
const REALTIME_PCM16_BYTES_PER_SAMPLE = 2;
|
|
639
|
+
const DISCORD_RAW_PCM_FRAME_BYTES = 3840;
|
|
640
|
+
const DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES = 25;
|
|
641
|
+
function isRealtimeResponseCancellationRace(event) {
|
|
642
|
+
return event.direction === "server" && event.type === "error" && event.detail === DISCORD_REALTIME_CANCELLATION_RACE_DETAIL;
|
|
643
|
+
}
|
|
644
|
+
function normalizeControlSpeechText(text) {
|
|
645
|
+
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
646
|
+
}
|
|
647
|
+
function pcm16MonoDurationMs(audio, sampleRate) {
|
|
648
|
+
if (audio.length === 0 || sampleRate <= 0) return 0;
|
|
649
|
+
return audio.length / REALTIME_PCM16_BYTES_PER_SAMPLE * 1e3 / sampleRate;
|
|
650
|
+
}
|
|
651
|
+
var DiscordRealtimePlayback = class {
|
|
652
|
+
constructor(params) {
|
|
653
|
+
this.params = params;
|
|
654
|
+
this.outputStream = null;
|
|
655
|
+
this.outputPacedBuffer = Buffer.alloc(0);
|
|
656
|
+
this.playbackState = { status: "idle" };
|
|
657
|
+
this.queuedExactSpeechMessages = [];
|
|
658
|
+
this.exactSpeechState = { status: "idle" };
|
|
659
|
+
this.wakeNameAckIndex = 0;
|
|
660
|
+
this.playerIdleHandler = () => {
|
|
661
|
+
const hadOutputAudio = this.isOutputAudioActive();
|
|
662
|
+
this.resetOutputStream("player-idle");
|
|
663
|
+
if (hadOutputAudio) this.completeExactSpeechResponse("player-idle");
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
attachPlayer() {
|
|
667
|
+
const voiceSdk = loadDiscordVoiceSdk();
|
|
668
|
+
this.params.entry.player.on(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
|
|
669
|
+
}
|
|
670
|
+
close() {
|
|
671
|
+
this.playbackState = { status: "idle" };
|
|
672
|
+
this.queuedExactSpeechMessages = [];
|
|
673
|
+
this.exactSpeechState = { status: "idle" };
|
|
674
|
+
this.clearOutputAudio("session-close");
|
|
675
|
+
const voiceSdk = loadDiscordVoiceSdk();
|
|
676
|
+
this.params.entry.player.off(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
|
|
677
|
+
}
|
|
678
|
+
handleBargeIn(reason = "barge-in") {
|
|
679
|
+
if (!this.isBargeInEnabled()) {
|
|
680
|
+
logger$1.info(`discord voice: realtime barge-in ignored reason=${reason} bargeIn=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}`);
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
if (!this.hasInterruptibleOutputAudio()) {
|
|
684
|
+
logger$1.info(`discord voice: realtime barge-in ignored reason=${reason} outputActive=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
logger$1.info(`discord voice: realtime barge-in requested reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
688
|
+
this.params.harness.handleBargeIn({ audioPlaybackActive: true }, () => {});
|
|
689
|
+
}
|
|
690
|
+
isBargeInEnabled() {
|
|
691
|
+
if (this.params.wakeNameRequired()) return false;
|
|
692
|
+
const providerId = this.params.providerId() ?? this.params.realtimeConfig()?.provider ?? "openai";
|
|
693
|
+
const realtimeConfig = this.params.realtimeConfig();
|
|
694
|
+
return resolveRealtimeVoiceBargeIn({
|
|
695
|
+
configuredBargeIn: realtimeConfig?.bargeIn,
|
|
696
|
+
interruptResponseOnInputAudio: realtimeConfig?.providers?.[providerId]?.interruptResponseOnInputAudio
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
hasInterruptibleOutputAudio() {
|
|
700
|
+
this.params.bridge()?.setMediaTimestamp(this.outputAudioMs());
|
|
701
|
+
const streamActive = Boolean(this.outputStream && !this.outputStream.destroyed);
|
|
702
|
+
return this.params.harness.outputActivity.isInterruptible(streamActive);
|
|
703
|
+
}
|
|
704
|
+
sendOutputAudio(realtimePcm24kMono) {
|
|
705
|
+
this.params.markProviderGenerationObserved();
|
|
706
|
+
if (this.params.stopped() || this.playbackState.status === "backpressured") return;
|
|
707
|
+
const discordPcm = convertRealtimePcm24kMonoToDiscordPcm48kStereo(realtimePcm24kMono);
|
|
708
|
+
if (discordPcm.length === 0) return;
|
|
709
|
+
this.params.bridge()?.setMediaTimestamp(this.outputAudioMs());
|
|
710
|
+
if (this.params.harness.outputActivity.snapshot().streamEnding) {
|
|
711
|
+
logVoiceVerbose(`realtime output audio ignored after stream ending: guild ${this.params.entry.guildId} channel ${this.params.entry.channelId}`);
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
const stream = this.ensureOutputStream();
|
|
715
|
+
if (this.exactSpeechState.status === "active") this.exactSpeechState = {
|
|
716
|
+
...this.exactSpeechState,
|
|
717
|
+
audioStarted: true
|
|
718
|
+
};
|
|
719
|
+
this.params.harness.recordOutputAudio(realtimePcm24kMono, {
|
|
720
|
+
audioMs: pcm16MonoDurationMs(realtimePcm24kMono, 24e3),
|
|
721
|
+
sourceAudioBytes: realtimePcm24kMono.length,
|
|
722
|
+
sinkAudioBytes: discordPcm.length
|
|
723
|
+
});
|
|
724
|
+
this.queueOutputAudio(stream, discordPcm);
|
|
725
|
+
}
|
|
726
|
+
clearOutputAudio(reason = "clear") {
|
|
727
|
+
this.resetOutputStream(reason);
|
|
728
|
+
this.params.entry.player.stop(true);
|
|
729
|
+
}
|
|
730
|
+
finishOutputAudioStream(reason, { playBuffered = true } = {}) {
|
|
731
|
+
const stream = this.outputStream;
|
|
732
|
+
if (!stream || stream.destroyed || this.params.harness.outputActivity.snapshot().streamEnding) return;
|
|
733
|
+
this.params.harness.outputActivity.markStreamEnding();
|
|
734
|
+
logger$1.info(`discord voice: realtime audio playback finishing reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${this.outputAudioMs()} chunks=${this.params.harness.outputActivity.snapshot().chunks}`);
|
|
735
|
+
if (playBuffered) {
|
|
736
|
+
this.startOutputPlayback(stream);
|
|
737
|
+
this.scheduleOutputPlaybackWatchdog(reason, stream);
|
|
738
|
+
} else {
|
|
739
|
+
this.resetOutputStream(reason);
|
|
740
|
+
this.params.entry.player.stop(true);
|
|
741
|
+
this.completeExactSpeechResponse(reason);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
stream.end();
|
|
745
|
+
}
|
|
746
|
+
handleProviderEvent(event) {
|
|
747
|
+
if (!(this.playbackState.status === "backpressured" && isRealtimeResponseCancellationRace(event))) return;
|
|
748
|
+
const outputBackpressured = this.playbackState.status === "backpressured";
|
|
749
|
+
if (outputBackpressured) this.playbackState = { status: "idle" };
|
|
750
|
+
if (this.exactSpeechState.status === "active" && (outputBackpressured || !this.exactSpeechState.audioStarted)) this.completeExactSpeechResponse(event.type);
|
|
751
|
+
this.finishOutputAudioStream(event.type, { playBuffered: false });
|
|
752
|
+
}
|
|
753
|
+
handleResponseDone(outcome) {
|
|
754
|
+
const outputBackpressured = this.playbackState.status === "backpressured";
|
|
755
|
+
if (outputBackpressured) this.playbackState = { status: "idle" };
|
|
756
|
+
if (this.exactSpeechState.status === "active" && (outputBackpressured || !this.exactSpeechState.audioStarted)) this.completeExactSpeechResponse(outcome.status);
|
|
757
|
+
this.finishOutputAudioStream(outcome.status, { playBuffered: outcome.status === "completed" });
|
|
758
|
+
}
|
|
759
|
+
enqueueExactSpeechMessage(text) {
|
|
760
|
+
if (this.params.stopped() || !text.trim()) return;
|
|
761
|
+
const retainedMessages = this.queuedExactSpeechMessages.length + (this.exactSpeechState.status === "active" ? 1 : 0);
|
|
762
|
+
const retainedBytes = this.queuedExactSpeechMessages.reduce((total, message) => total + Buffer.byteLength(message, "utf8"), 0) + Buffer.byteLength(this.exactSpeechState.status === "active" ? this.exactSpeechState.message : "", "utf8");
|
|
763
|
+
const incomingBytes = Buffer.byteLength(text, "utf8");
|
|
764
|
+
if (retainedMessages >= DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_MESSAGES || retainedBytes + incomingBytes > DISCORD_REALTIME_MAX_RETAINED_EXACT_SPEECH_BYTES) {
|
|
765
|
+
this.params.stopTerminally();
|
|
766
|
+
this.queuedExactSpeechMessages = [];
|
|
767
|
+
this.exactSpeechState = { status: "idle" };
|
|
768
|
+
this.clearOutputAudio("exact-speech-overflow");
|
|
769
|
+
this.params.onTerminalError(/* @__PURE__ */ new Error(`Discord realtime exact speech overflow: retained=${retainedMessages} retainedBytes=${retainedBytes} incomingBytes=${incomingBytes}`));
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
if (!this.params.bridgeReady() || this.exactSpeechState.status === "active" || this.hasInterruptibleOutputAudio()) {
|
|
773
|
+
this.queuedExactSpeechMessages.push(text);
|
|
774
|
+
logger$1.info(`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
this.sendExactSpeechMessage(text);
|
|
778
|
+
}
|
|
779
|
+
retainedExactSpeechTexts() {
|
|
780
|
+
return [...this.exactSpeechState.status === "active" ? [this.exactSpeechState.message] : [], ...this.queuedExactSpeechMessages];
|
|
781
|
+
}
|
|
782
|
+
drainQueuedExactSpeechMessages(reason) {
|
|
783
|
+
if (this.params.stopped() || !this.params.bridgeReady() || this.exactSpeechState.status === "active" || this.queuedExactSpeechMessages.length === 0 || this.hasInterruptibleOutputAudio()) return;
|
|
784
|
+
const next = this.queuedExactSpeechMessages.shift();
|
|
785
|
+
if (!next) return;
|
|
786
|
+
logger$1.info(`discord voice: realtime exact speech dequeued reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length}`);
|
|
787
|
+
this.sendExactSpeechMessage(next);
|
|
788
|
+
}
|
|
789
|
+
sendWakeNameAck(result) {
|
|
790
|
+
if (!result.allowed || this.params.stopped() || this.exactSpeechState.status === "active") return;
|
|
791
|
+
if (this.hasInterruptibleOutputAudio()) {
|
|
792
|
+
logger$1.info(`discord voice: realtime wake-name ack skipped outputActive=true voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const ack = DISCORD_REALTIME_WAKE_ACKS[this.wakeNameAckIndex % DISCORD_REALTIME_WAKE_ACKS.length];
|
|
796
|
+
this.wakeNameAckIndex += 1;
|
|
797
|
+
logger$1.info(`discord voice: realtime wake-name ack canonical=${result.activationName} heard=${result.heardName} match=${result.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`);
|
|
798
|
+
this.enqueueExactSpeechMessage(ack ?? "Yeah.");
|
|
799
|
+
}
|
|
800
|
+
speakControlResult(text) {
|
|
801
|
+
const trimmed = text.trim();
|
|
802
|
+
if (this.params.stopped() || !trimmed) return;
|
|
803
|
+
this.queuedExactSpeechMessages = [];
|
|
804
|
+
this.completeExactSpeechResponse("active-run-control", { drain: false });
|
|
805
|
+
this.params.harness.handleBargeIn({
|
|
806
|
+
audioPlaybackActive: true,
|
|
807
|
+
force: true
|
|
808
|
+
}, () => this.clearOutputAudio("active-run-control"));
|
|
809
|
+
this.lastControlSpeech = {
|
|
810
|
+
normalizedText: normalizeControlSpeechText(trimmed),
|
|
811
|
+
sentAt: Date.now(),
|
|
812
|
+
assistantTranscriptCount: 0
|
|
813
|
+
};
|
|
814
|
+
this.enqueueExactSpeechMessage(trimmed);
|
|
815
|
+
}
|
|
816
|
+
suppressDuplicateControlSpeech(text) {
|
|
817
|
+
const recent = this.lastControlSpeech;
|
|
818
|
+
if (!recent) return;
|
|
819
|
+
if (Date.now() - recent.sentAt > DISCORD_REALTIME_CONTROL_SPEECH_DEDUPE_MS) {
|
|
820
|
+
this.lastControlSpeech = void 0;
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
823
|
+
if (normalizeControlSpeechText(text) !== recent.normalizedText) return;
|
|
824
|
+
recent.assistantTranscriptCount += 1;
|
|
825
|
+
if (recent.assistantTranscriptCount <= 1) return;
|
|
826
|
+
logger$1.info(`discord voice: realtime duplicate active-run control speech suppressed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}`);
|
|
827
|
+
this.params.harness.handleBargeIn({
|
|
828
|
+
audioPlaybackActive: true,
|
|
829
|
+
force: true
|
|
830
|
+
}, () => this.clearOutputAudio("duplicate-active-run-control"));
|
|
831
|
+
}
|
|
832
|
+
resetProviderContinuity(reason) {
|
|
833
|
+
this.lastControlSpeech = void 0;
|
|
834
|
+
const replayExactSpeech = this.exactSpeechState.status === "active" && !this.params.harness.outputActivity.snapshot().playbackStarted ? this.exactSpeechState.message : void 0;
|
|
835
|
+
this.exactSpeechState = { status: "idle" };
|
|
836
|
+
if (replayExactSpeech) this.queuedExactSpeechMessages.unshift(replayExactSpeech);
|
|
837
|
+
this.params.harness.flushOutput(() => this.clearOutputAudio(reason));
|
|
838
|
+
this.params.harness.finishOutputAudio(reason);
|
|
839
|
+
}
|
|
840
|
+
outputAudioMs() {
|
|
841
|
+
return Math.floor(this.params.harness.outputActivity.snapshot().audioMs);
|
|
842
|
+
}
|
|
843
|
+
isOutputAudioActive() {
|
|
844
|
+
return this.params.harness.outputActivity.isActive(Boolean(this.outputStream && !this.outputStream.destroyed));
|
|
845
|
+
}
|
|
846
|
+
currentOutputStream() {
|
|
847
|
+
return this.outputStream;
|
|
848
|
+
}
|
|
849
|
+
ensureOutputStream() {
|
|
850
|
+
if (this.outputStream && !this.outputStream.destroyed && !this.outputStream.writableEnded) return this.outputStream;
|
|
851
|
+
const stream = new PassThrough({ highWaterMark: DISCORD_RAW_PCM_FRAME_BYTES * 128 });
|
|
852
|
+
this.outputStream = stream;
|
|
853
|
+
this.playbackState = {
|
|
854
|
+
status: "buffering",
|
|
855
|
+
stream
|
|
856
|
+
};
|
|
857
|
+
this.outputPacedBuffer = Buffer.alloc(0);
|
|
858
|
+
this.params.harness.outputActivity.markStreamOpened();
|
|
859
|
+
stream.once("close", () => {
|
|
860
|
+
if (this.params.harness.outputActivity.snapshot().playbackStarted) return;
|
|
861
|
+
this.handleOutputStreamClosed(stream, "stream-close");
|
|
862
|
+
});
|
|
863
|
+
return stream;
|
|
864
|
+
}
|
|
865
|
+
handleOutputStreamClosed(stream, reason) {
|
|
866
|
+
if (this.outputStream !== stream) return;
|
|
867
|
+
this.logOutputAudioStopped(reason);
|
|
868
|
+
this.clearOutputPlaybackWatchdog();
|
|
869
|
+
this.outputStream = null;
|
|
870
|
+
if (this.playbackState.status !== "backpressured") this.playbackState = { status: "idle" };
|
|
871
|
+
this.outputPacedBuffer = Buffer.alloc(0);
|
|
872
|
+
this.params.harness.outputActivity.reset();
|
|
873
|
+
this.completeExactSpeechResponse(reason);
|
|
874
|
+
}
|
|
875
|
+
queueOutputAudio(stream, discordPcm) {
|
|
876
|
+
if (this.playbackState.status === "playing") {
|
|
877
|
+
if (!stream.write(discordPcm)) this.handleOutputBackpressure(stream);
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
this.outputPacedBuffer = this.outputPacedBuffer.length > 0 ? Buffer.concat([this.outputPacedBuffer, discordPcm]) : discordPcm;
|
|
881
|
+
if (this.outputPacedBuffer.length >= DISCORD_RAW_PCM_FRAME_BYTES * DISCORD_REALTIME_OUTPUT_PREROLL_FRAMES) this.startOutputPlayback(stream);
|
|
882
|
+
}
|
|
883
|
+
handleOutputBackpressure(stream) {
|
|
884
|
+
if (this.playbackState.status === "backpressured" || this.outputStream !== stream) return;
|
|
885
|
+
const token = Symbol("output-backpressure");
|
|
886
|
+
this.playbackState = {
|
|
887
|
+
status: "backpressured",
|
|
888
|
+
stream,
|
|
889
|
+
token
|
|
890
|
+
};
|
|
891
|
+
const bufferedBytes = stream.writableLength + stream.readableLength;
|
|
892
|
+
logger$1.warn(`discord voice: realtime audio playback backpressured guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} bufferedBytes=${bufferedBytes}`);
|
|
893
|
+
this.clearOutputAudio("output-backpressure");
|
|
894
|
+
queueMicrotask(() => {
|
|
895
|
+
if (this.params.stopped() || this.playbackState.status !== "backpressured" || this.playbackState.token !== token) return;
|
|
896
|
+
this.params.harness.handleBargeIn({
|
|
897
|
+
audioPlaybackActive: true,
|
|
898
|
+
force: true
|
|
899
|
+
}, () => {});
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
startOutputPlayback(stream) {
|
|
903
|
+
if (this.params.harness.outputActivity.snapshot().playbackStarted || stream.destroyed) return;
|
|
904
|
+
const voiceSdk = loadDiscordVoiceSdk();
|
|
905
|
+
const opusStream = createDiscordOpusEncodeStream();
|
|
906
|
+
opusStream.on("error", (err) => {
|
|
907
|
+
logger$1.warn(`discord voice: realtime opus encode failed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}: ${formatErrorMessage(err)}`);
|
|
908
|
+
this.resetOutputStream("opus-encode-error");
|
|
909
|
+
});
|
|
910
|
+
opusStream.once("close", () => this.handleOutputStreamClosed(stream, "stream-close"));
|
|
911
|
+
pipeline(stream, opusStream, (err) => {
|
|
912
|
+
if (!err) return;
|
|
913
|
+
logger$1.warn(`discord voice: realtime output pipeline failed guild=${this.params.entry.guildId} channel=${this.params.entry.channelId}: ${formatErrorMessage(err)}`);
|
|
914
|
+
this.resetOutputStream("output-pipeline-error");
|
|
915
|
+
});
|
|
916
|
+
if (this.outputPacedBuffer.length > 0) {
|
|
917
|
+
stream.write(this.outputPacedBuffer);
|
|
918
|
+
this.outputPacedBuffer = Buffer.alloc(0);
|
|
919
|
+
}
|
|
920
|
+
const resource = voiceSdk.createAudioResource(opusStream, { inputType: voiceSdk.StreamType.Opus });
|
|
921
|
+
this.params.entry.player.play(resource);
|
|
922
|
+
this.params.harness.outputActivity.markPlaybackStarted();
|
|
923
|
+
this.playbackState = {
|
|
924
|
+
status: "playing",
|
|
925
|
+
stream
|
|
926
|
+
};
|
|
927
|
+
const realtimeConfig = this.params.realtimeConfig();
|
|
928
|
+
logger$1.info(`discord voice: realtime audio playback started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} mode=${this.params.mode} model=${realtimeConfig?.model ?? "provider-default"} voice=${realtimeConfig?.speakerVoice ?? realtimeConfig?.speakerVoiceId ?? "provider-default"}`);
|
|
929
|
+
}
|
|
930
|
+
resetOutputStream(reason = "reset") {
|
|
931
|
+
const stream = this.outputStream;
|
|
932
|
+
this.clearOutputPlaybackWatchdog();
|
|
933
|
+
this.logOutputAudioStopped(reason);
|
|
934
|
+
this.outputStream = null;
|
|
935
|
+
if (this.playbackState.status !== "backpressured") this.playbackState = { status: "idle" };
|
|
936
|
+
this.outputPacedBuffer = Buffer.alloc(0);
|
|
937
|
+
this.params.harness.outputActivity.reset();
|
|
938
|
+
stream?.end();
|
|
939
|
+
stream?.destroy();
|
|
940
|
+
}
|
|
941
|
+
scheduleOutputPlaybackWatchdog(reason, stream) {
|
|
942
|
+
this.clearOutputPlaybackWatchdog();
|
|
943
|
+
const timeoutMs = this.params.harness.outputActivity.playbackWatchdogDelayMs({ marginMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS });
|
|
944
|
+
if (timeoutMs === void 0) return;
|
|
945
|
+
this.outputPlaybackWatchdog = setTimeout(() => {
|
|
946
|
+
this.outputPlaybackWatchdog = void 0;
|
|
947
|
+
if (this.outputStream && this.outputStream !== stream) return;
|
|
948
|
+
if (!this.outputStream && !this.isOutputAudioActive()) {
|
|
949
|
+
this.completeExactSpeechResponse("playback-watchdog");
|
|
950
|
+
return;
|
|
951
|
+
}
|
|
952
|
+
logger$1.warn(`discord voice: realtime audio playback watchdog fired reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${this.outputAudioMs()} elapsedMs=${this.params.harness.outputActivity.elapsedPlaybackMs()}`);
|
|
953
|
+
this.clearOutputAudio("playback-watchdog");
|
|
954
|
+
this.completeExactSpeechResponse("playback-watchdog");
|
|
955
|
+
}, timeoutMs);
|
|
956
|
+
}
|
|
957
|
+
clearOutputPlaybackWatchdog() {
|
|
958
|
+
if (!this.outputPlaybackWatchdog) return;
|
|
959
|
+
clearTimeout(this.outputPlaybackWatchdog);
|
|
960
|
+
this.outputPlaybackWatchdog = void 0;
|
|
961
|
+
}
|
|
962
|
+
sendExactSpeechMessage(text) {
|
|
963
|
+
if (this.params.stopped() || !text.trim()) return;
|
|
964
|
+
this.exactSpeechState = {
|
|
965
|
+
status: "active",
|
|
966
|
+
message: text,
|
|
967
|
+
audioStarted: false
|
|
968
|
+
};
|
|
969
|
+
this.params.bridge()?.sendUserMessage(this.params.buildSpeakExactMessage(text));
|
|
970
|
+
}
|
|
971
|
+
completeExactSpeechResponse(reason, options) {
|
|
972
|
+
if (this.exactSpeechState.status === "idle" && this.queuedExactSpeechMessages.length === 0) return;
|
|
973
|
+
this.exactSpeechState = { status: "idle" };
|
|
974
|
+
if (options?.drain === false) return;
|
|
975
|
+
this.drainQueuedExactSpeechMessages(reason);
|
|
976
|
+
}
|
|
977
|
+
logOutputAudioStopped(reason) {
|
|
978
|
+
const activity = this.params.harness.outputActivity.snapshot();
|
|
979
|
+
const audioMs = Math.floor(activity.audioMs);
|
|
980
|
+
const chunks = activity.chunks;
|
|
981
|
+
const discordBytes = activity.sinkAudioBytes;
|
|
982
|
+
const realtimeBytes = activity.sourceAudioBytes;
|
|
983
|
+
const elapsedMs = this.params.harness.outputActivity.elapsedPlaybackMs();
|
|
984
|
+
if (this.outputStream || chunks > 0 || audioMs > 0) logger$1.info(`discord voice: realtime audio playback stopped reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${audioMs} elapsedMs=${elapsedMs} chunks=${chunks} discordBytes=${discordBytes} realtimeBytes=${realtimeBytes}`);
|
|
985
|
+
}
|
|
986
|
+
};
|
|
987
|
+
//#endregion
|
|
988
|
+
//#region extensions/discord/src/voice/realtime-session.runtime.ts
|
|
989
|
+
const logger = createSubsystemLogger("discord/voice");
|
|
990
|
+
const DISCORD_REALTIME_DUPLICATE_ERROR_SUPPRESS_MS = 6e4;
|
|
991
|
+
const discordRealtimeTalkPayload = () => ({});
|
|
992
|
+
const DISCORD_REALTIME_VERBOSE_OMITTED_EVENTS = /* @__PURE__ */ new Set([
|
|
993
|
+
"conversation.output_audio.delta",
|
|
994
|
+
"input_audio_buffer.append",
|
|
995
|
+
"response.audio.delta",
|
|
996
|
+
"response.output_audio.delta"
|
|
997
|
+
]);
|
|
998
|
+
function formatRealtimeInterruptionLog(event) {
|
|
999
|
+
const detail = event.detail ? ` ${event.detail}` : "";
|
|
1000
|
+
if (event.direction === "client") {
|
|
1001
|
+
if (event.type === "response.cancel") return `discord voice: realtime model interrupt requested ${event.direction}:${event.type}${detail}`;
|
|
1002
|
+
if (event.type === "conversation.item.truncate.skipped") return `discord voice: realtime model interrupt ignored ${event.direction}:${event.type}${detail}`;
|
|
1003
|
+
if (event.type === "conversation.item.truncate") return `discord voice: realtime model audio truncated ${event.direction}:${event.type}${detail}`;
|
|
1004
|
+
}
|
|
1005
|
+
if (event.direction === "server") {
|
|
1006
|
+
if (event.type === "response.cancelled") return `discord voice: realtime model interrupt confirmed ${event.direction}:${event.type}${detail}`;
|
|
1007
|
+
if (event.type === "error" && event.detail === "Cancellation failed: no active response found") return `discord voice: realtime model interrupt raced ${event.direction}:${event.type}${detail}`;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
function formatRealtimeLifecycleLog(event) {
|
|
1011
|
+
if (!event.type.startsWith("session.")) return;
|
|
1012
|
+
const detail = event.detail ? ` ${event.detail}` : "";
|
|
1013
|
+
return `discord voice: realtime lifecycle ${event.direction}:${event.type}${detail}`;
|
|
1014
|
+
}
|
|
1015
|
+
function shouldLogRealtimeVerboseEvent(event) {
|
|
1016
|
+
return !DISCORD_REALTIME_VERBOSE_OMITTED_EVENTS.has(event.type);
|
|
1017
|
+
}
|
|
1018
|
+
function readProviderConfigString(config, key) {
|
|
1019
|
+
const value = config[key];
|
|
1020
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
1021
|
+
}
|
|
1022
|
+
function isDiscordAgentProxyVoiceMode(mode) {
|
|
1023
|
+
return mode === "agent-proxy";
|
|
1024
|
+
}
|
|
1025
|
+
var DiscordRealtimeVoiceSession = class {
|
|
1026
|
+
constructor(params) {
|
|
1027
|
+
this.params = params;
|
|
1028
|
+
this.bridge = null;
|
|
1029
|
+
this.lifecycle = {
|
|
1030
|
+
status: "inactive",
|
|
1031
|
+
generation: 0
|
|
1032
|
+
};
|
|
1033
|
+
this.nextLifecycleGeneration = 0;
|
|
1034
|
+
this.consultToolPolicy = "safe-read-only";
|
|
1035
|
+
this.consultPolicy = "auto";
|
|
1036
|
+
this.wakeNamePolicy = "never";
|
|
1037
|
+
this.wakeNames = [];
|
|
1038
|
+
this.providerGenerationObserved = false;
|
|
1039
|
+
this.providerContinuityEpoch = 0;
|
|
1040
|
+
this.harness = createRealtimeVoiceSessionHarness({
|
|
1041
|
+
talk: {
|
|
1042
|
+
sessionId: `discord:${this.params.entry.voiceSessionKey}:realtime`,
|
|
1043
|
+
mode: "realtime",
|
|
1044
|
+
transport: "gateway-relay",
|
|
1045
|
+
brain: "agent-consult"
|
|
1046
|
+
},
|
|
1047
|
+
talkPayloads: {
|
|
1048
|
+
turnStarted: discordRealtimeTalkPayload,
|
|
1049
|
+
turnEnded: discordRealtimeTalkPayload,
|
|
1050
|
+
inputAudioDelta: discordRealtimeTalkPayload,
|
|
1051
|
+
outputAudioStarted: discordRealtimeTalkPayload,
|
|
1052
|
+
outputAudioDelta: discordRealtimeTalkPayload,
|
|
1053
|
+
outputAudioDone: discordRealtimeTalkPayload
|
|
1054
|
+
},
|
|
1055
|
+
forcedConsults: {
|
|
1056
|
+
limit: 16,
|
|
1057
|
+
nativeDedupeMs: 15e3,
|
|
1058
|
+
questionsMatch: matchRealtimeVoiceConsultQuestions
|
|
1059
|
+
}
|
|
1060
|
+
});
|
|
1061
|
+
this.playback = new DiscordRealtimePlayback({
|
|
1062
|
+
bridge: () => this.bridge,
|
|
1063
|
+
bridgeReady: () => this.isReady(),
|
|
1064
|
+
buildSpeakExactMessage: (text) => buildRealtimeVoiceSpeakExactMessage({
|
|
1065
|
+
text,
|
|
1066
|
+
surfaceLabel: "the Discord voice channel"
|
|
1067
|
+
}),
|
|
1068
|
+
entry: this.params.entry,
|
|
1069
|
+
harness: this.harness,
|
|
1070
|
+
markProviderGenerationObserved: () => this.markProviderGenerationObserved(),
|
|
1071
|
+
mode: this.params.mode,
|
|
1072
|
+
onTerminalError: this.params.onTerminalError,
|
|
1073
|
+
providerId: () => this.realtimeProviderId,
|
|
1074
|
+
realtimeConfig: () => this.realtimeConfig,
|
|
1075
|
+
stopTerminally: () => {
|
|
1076
|
+
this.stopLifecycle("exact-speech overflow");
|
|
1077
|
+
this.consults.close();
|
|
1078
|
+
},
|
|
1079
|
+
stopped: () => this.isStopped(),
|
|
1080
|
+
wakeNameRequired: () => this.isWakeNameRequired()
|
|
1081
|
+
});
|
|
1082
|
+
this.turns = new DiscordRealtimeTurns({
|
|
1083
|
+
bridge: () => this.bridge,
|
|
1084
|
+
entry: this.params.entry,
|
|
1085
|
+
getHumanParticipantCount: () => this.humanParticipantCount(),
|
|
1086
|
+
onAcceptedTranscript: (text, context, providerEpoch) => this.consults.handleAcceptedTranscript(text, context, providerEpoch),
|
|
1087
|
+
playback: this.playback,
|
|
1088
|
+
providerEpoch: () => this.providerContinuityEpoch,
|
|
1089
|
+
providerId: () => this.realtimeProviderId,
|
|
1090
|
+
realtimeConfig: () => this.realtimeConfig,
|
|
1091
|
+
recordInputAudio: (audio) => this.harness.recordInputAudio(audio),
|
|
1092
|
+
stopped: () => this.isStopped(),
|
|
1093
|
+
wakeNamePolicy: () => this.wakeNamePolicy,
|
|
1094
|
+
wakeNames: () => this.wakeNames
|
|
1095
|
+
});
|
|
1096
|
+
this.consults = new DiscordRealtimeConsults({
|
|
1097
|
+
consultPolicy: () => this.consultPolicy,
|
|
1098
|
+
consultToolPolicy: () => this.consultToolPolicy,
|
|
1099
|
+
consultToolsAllow: () => this.consultToolsAllow,
|
|
1100
|
+
debounceMs: () => this.realtimeConfig?.debounceMs,
|
|
1101
|
+
entry: this.params.entry,
|
|
1102
|
+
harness: this.harness,
|
|
1103
|
+
isAgentProxy: isDiscordAgentProxyVoiceMode(this.params.mode),
|
|
1104
|
+
isWakeNameRequired: () => this.isWakeNameRequired(),
|
|
1105
|
+
playback: this.playback,
|
|
1106
|
+
providerEpoch: () => this.providerContinuityEpoch,
|
|
1107
|
+
runAgentTurn: this.params.runAgentTurn,
|
|
1108
|
+
stopped: () => this.isStopped(),
|
|
1109
|
+
turns: this.turns,
|
|
1110
|
+
usesRealtimeAgentHandoff: () => this.params.mode === "bidi" || this.consultToolPolicy !== "none",
|
|
1111
|
+
wakeNamePolicy: () => this.wakeNamePolicy
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
async connect() {
|
|
1115
|
+
const lifecycleGeneration = ++this.nextLifecycleGeneration;
|
|
1116
|
+
this.lifecycle = {
|
|
1117
|
+
status: "starting",
|
|
1118
|
+
generation: lifecycleGeneration,
|
|
1119
|
+
instance: this
|
|
1120
|
+
};
|
|
1121
|
+
const resolved = resolveConfiguredRealtimeVoiceProvider({
|
|
1122
|
+
configuredProviderId: this.realtimeConfig?.provider,
|
|
1123
|
+
providerConfigs: buildProviderConfigs(this.realtimeConfig),
|
|
1124
|
+
providerConfigOverrides: buildProviderConfigOverrides(this.realtimeConfig),
|
|
1125
|
+
cfg: this.params.cfg,
|
|
1126
|
+
defaultModel: this.realtimeConfig?.model,
|
|
1127
|
+
noRegisteredProviderMessage: "No configured realtime voice provider registered"
|
|
1128
|
+
});
|
|
1129
|
+
this.realtimeProviderId = resolved.provider.id;
|
|
1130
|
+
const isAgentProxy = isDiscordAgentProxyVoiceMode(this.params.mode);
|
|
1131
|
+
const { toolPolicy, consultToolsAllow, consultPolicy, wakeNamePolicy, wakeNames, autoRespondToAudio } = resolveRealtimeVoiceSessionPolicy({
|
|
1132
|
+
isAgentProxy,
|
|
1133
|
+
supportsActivationNameGating: resolved.provider.capabilities?.supportsActivationNameGating === true,
|
|
1134
|
+
configuredToolPolicy: this.realtimeConfig?.toolPolicy,
|
|
1135
|
+
configuredConsultPolicy: this.realtimeConfig?.consultPolicy,
|
|
1136
|
+
requireWakeName: this.realtimeConfig?.requireWakeName,
|
|
1137
|
+
configuredWakeNames: this.realtimeConfig?.wakeNames,
|
|
1138
|
+
cfg: this.params.cfg,
|
|
1139
|
+
agentId: this.params.entry.route.agentId
|
|
1140
|
+
});
|
|
1141
|
+
this.consultToolPolicy = toolPolicy;
|
|
1142
|
+
this.consultToolsAllow = consultToolsAllow;
|
|
1143
|
+
this.consultPolicy = consultPolicy;
|
|
1144
|
+
this.wakeNamePolicy = wakeNamePolicy;
|
|
1145
|
+
this.wakeNames = wakeNames;
|
|
1146
|
+
const usesRealtimeAgentHandoff = this.params.mode === "bidi" || toolPolicy !== "none";
|
|
1147
|
+
const providerInterruptResponseOnInputAudio = this.realtimeConfig?.providers?.[resolved.provider.id]?.interruptResponseOnInputAudio;
|
|
1148
|
+
const interruptResponseOnInputAudio = this.wakeNamePolicy === "never" && resolveRealtimeVoiceInterruptResponseOnInputAudio(providerInterruptResponseOnInputAudio);
|
|
1149
|
+
const bargeIn = resolveRealtimeVoiceBargeIn({
|
|
1150
|
+
configuredBargeIn: this.realtimeConfig?.bargeIn,
|
|
1151
|
+
interruptResponseOnInputAudio: providerInterruptResponseOnInputAudio
|
|
1152
|
+
});
|
|
1153
|
+
const minBargeInAudioEndMs = resolveRealtimeVoiceMinBargeInAudioEndMs(this.realtimeConfig?.minBargeInAudioEndMs);
|
|
1154
|
+
const instructions = buildRealtimeVoiceSessionInstructions({
|
|
1155
|
+
base: this.realtimeConfig?.instructions ?? ["You are OpenClaw's Discord voice interface.", "Keep spoken replies concise, natural, and suitable for a live Discord voice channel."].join("\n"),
|
|
1156
|
+
isAgentProxy,
|
|
1157
|
+
bootstrapContextInstructions: this.params.bootstrapContextInstructions,
|
|
1158
|
+
toolPolicy,
|
|
1159
|
+
consultPolicy
|
|
1160
|
+
});
|
|
1161
|
+
this.bridge = this.harness.createBridge({
|
|
1162
|
+
provider: resolved.provider,
|
|
1163
|
+
cfg: this.params.cfg,
|
|
1164
|
+
providerConfig: resolved.providerConfig,
|
|
1165
|
+
audioFormat: REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
|
|
1166
|
+
instructions,
|
|
1167
|
+
autoRespondToAudio,
|
|
1168
|
+
interruptResponseOnInputAudio,
|
|
1169
|
+
markStrategy: "ack-immediately",
|
|
1170
|
+
tools: usesRealtimeAgentHandoff ? resolveRealtimeVoiceAgentConsultTools(toolPolicy, toolPolicy !== "none" ? [REALTIME_VOICE_AGENT_CONTROL_TOOL] : []) : [],
|
|
1171
|
+
audioSink: {
|
|
1172
|
+
isOpen: () => !this.isStopped(),
|
|
1173
|
+
sendAudio: (audio) => this.playback.sendOutputAudio(audio),
|
|
1174
|
+
clearAudio: () => {
|
|
1175
|
+
this.markProviderGenerationObserved();
|
|
1176
|
+
this.harness.flushOutput(() => this.playback.clearOutputAudio("provider-clear-audio"));
|
|
1177
|
+
}
|
|
1178
|
+
},
|
|
1179
|
+
onTranscript: (role, text, isFinal) => {
|
|
1180
|
+
this.markProviderGenerationObserved();
|
|
1181
|
+
if (isFinal && text.trim()) logger.info(`discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`);
|
|
1182
|
+
if (isFinal && role === "assistant") this.playback.suppressDuplicateControlSpeech(text);
|
|
1183
|
+
if (role !== "user") return;
|
|
1184
|
+
if (!isFinal) {
|
|
1185
|
+
this.turns.handlePartialUserTranscript(text);
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
this.turns.handleFinalUserTranscript(text);
|
|
1189
|
+
},
|
|
1190
|
+
onToolCall: (event, session) => {
|
|
1191
|
+
this.markProviderGenerationObserved();
|
|
1192
|
+
return this.consults.handleToolCall(event, session);
|
|
1193
|
+
},
|
|
1194
|
+
onReady: () => {
|
|
1195
|
+
this.markProviderGenerationObserved();
|
|
1196
|
+
if (this.markLifecycleReady(lifecycleGeneration)) this.playback.drainQueuedExactSpeechMessages("provider-ready");
|
|
1197
|
+
},
|
|
1198
|
+
onEvent: (event) => this.handleBridgeEvent(event),
|
|
1199
|
+
onResponseDone: (outcome) => {
|
|
1200
|
+
this.markProviderGenerationObserved();
|
|
1201
|
+
this.playback.handleResponseDone(outcome);
|
|
1202
|
+
if (outcome.status === "cancelled") logger.info(`discord voice: realtime model interrupt confirmed server:response.done status=cancelled${outcome.reason ? ` reason=${outcome.reason}` : ""}`);
|
|
1203
|
+
else if (outcome.status === "failed" || outcome.status === "incomplete") this.logRealtimeError(outcome.message);
|
|
1204
|
+
},
|
|
1205
|
+
onError: (error) => this.logRealtimeError(formatErrorMessage(error)),
|
|
1206
|
+
onClose: (reason) => {
|
|
1207
|
+
this.flushSuppressedRealtimeErrors();
|
|
1208
|
+
logVoiceVerbose(`realtime closed: ${reason}`);
|
|
1209
|
+
}
|
|
1210
|
+
});
|
|
1211
|
+
const resolvedModel = readProviderConfigString(resolved.providerConfig, "model") ?? resolved.provider.defaultModel;
|
|
1212
|
+
const resolvedVoice = readProviderConfigString(resolved.providerConfig, "voice");
|
|
1213
|
+
const humanParticipantCount = this.humanParticipantCount();
|
|
1214
|
+
logger.info(`discord voice: realtime bridge starting mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"} consultPolicy=${consultPolicy} toolPolicy=${toolPolicy} autoRespond=${autoRespondToAudio} wakeNamePolicy=${this.wakeNamePolicy} requireWakeName=${this.isWakeNameRequired(humanParticipantCount)} humanParticipants=${humanParticipantCount} wakeNames=${this.wakeNames.join(",") || "none"} interruptResponse=${interruptResponseOnInputAudio} bargeIn=${bargeIn} minBargeInAudioEndMs=${minBargeInAudioEndMs}`);
|
|
1215
|
+
this.playback.attachPlayer();
|
|
1216
|
+
await this.bridge.connect();
|
|
1217
|
+
if (!this.markLifecycleReady(lifecycleGeneration)) {
|
|
1218
|
+
this.bridge?.close();
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
this.markProviderGenerationObserved();
|
|
1222
|
+
this.playback.drainQueuedExactSpeechMessages("provider-connected");
|
|
1223
|
+
logger.info(`discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`);
|
|
1224
|
+
}
|
|
1225
|
+
close() {
|
|
1226
|
+
this.stopLifecycle("session close");
|
|
1227
|
+
this.providerContinuityEpoch += 1;
|
|
1228
|
+
this.flushSuppressedRealtimeErrors();
|
|
1229
|
+
this.consults.close();
|
|
1230
|
+
this.harness.close();
|
|
1231
|
+
this.turns.clear();
|
|
1232
|
+
this.playback.close();
|
|
1233
|
+
this.bridge?.close();
|
|
1234
|
+
this.bridge = null;
|
|
1235
|
+
this.realtimeProviderId = void 0;
|
|
1236
|
+
}
|
|
1237
|
+
beginSpeakerTurn(context, userId) {
|
|
1238
|
+
return this.turns.beginSpeakerTurn(context, userId);
|
|
1239
|
+
}
|
|
1240
|
+
handleBargeIn(reason = "barge-in") {
|
|
1241
|
+
this.playback.handleBargeIn(reason);
|
|
1242
|
+
}
|
|
1243
|
+
isBargeInEnabled() {
|
|
1244
|
+
if (this.isWakeNameRequired()) return false;
|
|
1245
|
+
return this.playback.isBargeInEnabled();
|
|
1246
|
+
}
|
|
1247
|
+
get realtimeConfig() {
|
|
1248
|
+
return this.params.discordConfig.voice?.realtime;
|
|
1249
|
+
}
|
|
1250
|
+
isStopped() {
|
|
1251
|
+
return this.lifecycle.status === "stopped";
|
|
1252
|
+
}
|
|
1253
|
+
isReady() {
|
|
1254
|
+
return this.lifecycle.status === "active";
|
|
1255
|
+
}
|
|
1256
|
+
markLifecycleReady(generation) {
|
|
1257
|
+
if (this.lifecycle.status !== "starting" && this.lifecycle.status !== "active" || this.lifecycle.generation !== generation) return false;
|
|
1258
|
+
this.lifecycle = {
|
|
1259
|
+
status: "active",
|
|
1260
|
+
generation,
|
|
1261
|
+
instance: this
|
|
1262
|
+
};
|
|
1263
|
+
return true;
|
|
1264
|
+
}
|
|
1265
|
+
stopLifecycle(reason) {
|
|
1266
|
+
const generation = this.lifecycle.generation;
|
|
1267
|
+
this.lifecycle = {
|
|
1268
|
+
status: "stopped",
|
|
1269
|
+
generation,
|
|
1270
|
+
reason
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
humanParticipantCount() {
|
|
1274
|
+
return this.params.getHumanParticipantCount?.() ?? 0;
|
|
1275
|
+
}
|
|
1276
|
+
isWakeNameRequired(humanParticipantCount = this.humanParticipantCount()) {
|
|
1277
|
+
return isRealtimeVoiceWakeNameRequired(this.wakeNamePolicy, humanParticipantCount);
|
|
1278
|
+
}
|
|
1279
|
+
handleBridgeEvent(event) {
|
|
1280
|
+
if (!(event.direction === "client" && event.type === "session.continuity.reset")) this.markProviderGenerationObserved();
|
|
1281
|
+
const detail = event.detail ? ` ${event.detail}` : "";
|
|
1282
|
+
if (event.direction === "client" && event.type === "session.continuity.reset") this.resetProviderContinuity(event.type);
|
|
1283
|
+
if (event.direction === "server" && event.type === "input_audio_buffer.speech_started") this.turns.resetPartialWakeNameTracking();
|
|
1284
|
+
if (shouldLogRealtimeVerboseEvent(event)) logVoiceVerbose(`realtime ${event.direction}:${event.type}${detail}`);
|
|
1285
|
+
this.playback.handleProviderEvent(event);
|
|
1286
|
+
const interruptionLog = formatRealtimeInterruptionLog(event);
|
|
1287
|
+
if (interruptionLog) logger.info(interruptionLog);
|
|
1288
|
+
const lifecycleLog = formatRealtimeLifecycleLog(event);
|
|
1289
|
+
if (lifecycleLog) logger.info(lifecycleLog);
|
|
1290
|
+
}
|
|
1291
|
+
markProviderGenerationObserved() {
|
|
1292
|
+
this.providerGenerationObserved = true;
|
|
1293
|
+
}
|
|
1294
|
+
resetProviderContinuity(reason) {
|
|
1295
|
+
if (!this.providerGenerationObserved) return;
|
|
1296
|
+
this.providerGenerationObserved = false;
|
|
1297
|
+
if (this.lifecycle.status === "active") this.lifecycle = {
|
|
1298
|
+
status: "starting",
|
|
1299
|
+
generation: this.lifecycle.generation,
|
|
1300
|
+
instance: this
|
|
1301
|
+
};
|
|
1302
|
+
this.providerContinuityEpoch += 1;
|
|
1303
|
+
this.consults.resetProviderContinuity();
|
|
1304
|
+
this.turns.resetProviderContinuity();
|
|
1305
|
+
this.playback.resetProviderContinuity(reason);
|
|
1306
|
+
}
|
|
1307
|
+
logRealtimeError(message) {
|
|
1308
|
+
const now = Date.now();
|
|
1309
|
+
if (this.lastRealtimeError?.message === message && now - this.lastRealtimeError.lastLoggedAt < DISCORD_REALTIME_DUPLICATE_ERROR_SUPPRESS_MS) {
|
|
1310
|
+
this.lastRealtimeError.suppressed += 1;
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
this.flushSuppressedRealtimeErrors();
|
|
1314
|
+
this.lastRealtimeError = {
|
|
1315
|
+
message,
|
|
1316
|
+
suppressed: 0,
|
|
1317
|
+
lastLoggedAt: now
|
|
1318
|
+
};
|
|
1319
|
+
logger.warn(`discord voice: realtime error: ${message}`);
|
|
1320
|
+
}
|
|
1321
|
+
flushSuppressedRealtimeErrors() {
|
|
1322
|
+
if (!this.lastRealtimeError || this.lastRealtimeError.suppressed === 0) return;
|
|
1323
|
+
logger.warn(`discord voice: suppressed ${this.lastRealtimeError.suppressed} duplicate realtime errors: ${this.lastRealtimeError.message}`);
|
|
1324
|
+
this.lastRealtimeError.suppressed = 0;
|
|
1325
|
+
}
|
|
1326
|
+
};
|
|
1327
|
+
function buildProviderConfigs(realtimeConfig) {
|
|
1328
|
+
const configs = realtimeConfig?.providers;
|
|
1329
|
+
return configs && Object.keys(configs).length > 0 ? { ...configs } : void 0;
|
|
1330
|
+
}
|
|
1331
|
+
function buildProviderConfigOverrides(realtimeConfig) {
|
|
1332
|
+
const overrides = {
|
|
1333
|
+
...realtimeConfig?.model ? { model: realtimeConfig.model } : {},
|
|
1334
|
+
...realtimeConfig?.speakerVoice ? { voice: realtimeConfig.speakerVoice } : realtimeConfig?.speakerVoiceId ? { voice: realtimeConfig.speakerVoiceId } : {},
|
|
1335
|
+
...typeof realtimeConfig?.minBargeInAudioEndMs === "number" ? { minBargeInAudioEndMs: realtimeConfig.minBargeInAudioEndMs } : {}
|
|
1336
|
+
};
|
|
1337
|
+
return Object.keys(overrides).length > 0 ? overrides : void 0;
|
|
1338
|
+
}
|
|
1339
|
+
//#endregion
|
|
1340
|
+
export { DiscordRealtimeVoiceSession };
|