@openclaw/google-meet 2026.5.3 → 2026.5.4-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -6,16 +6,29 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
|
6
6
|
import { normalizeOptionalLowercaseString, normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
|
7
7
|
import { Type } from "typebox";
|
|
8
8
|
import { isBlockedHostnameOrIp } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
9
|
-
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, buildRealtimeVoiceAgentConsultWorkingResponse, consultRealtimeVoiceAgent, createRealtimeVoiceBridgeSession, resolveConfiguredRealtimeVoiceProvider, resolveRealtimeVoiceAgentConsultToolPolicy, resolveRealtimeVoiceAgentConsultTools, resolveRealtimeVoiceAgentConsultToolsAllow } from "openclaw/plugin-sdk/realtime-voice";
|
|
9
|
+
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, buildRealtimeVoiceAgentConsultWorkingResponse, consultRealtimeVoiceAgent, convertPcmToMulaw8k, createRealtimeVoiceBridgeSession, mulawToPcm, resamplePcm, resolveConfiguredRealtimeVoiceProvider, resolveRealtimeVoiceAgentConsultToolPolicy, resolveRealtimeVoiceAgentConsultTools, resolveRealtimeVoiceAgentConsultToolsAllow } from "openclaw/plugin-sdk/realtime-voice";
|
|
10
10
|
import { spawn, spawnSync } from "node:child_process";
|
|
11
11
|
import { randomUUID } from "node:crypto";
|
|
12
12
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
13
13
|
import { normalizeAgentId } from "openclaw/plugin-sdk/routing";
|
|
14
|
+
import { getRealtimeTranscriptionProvider, listRealtimeTranscriptionProviders } from "openclaw/plugin-sdk/realtime-transcription";
|
|
14
15
|
import fs from "node:fs";
|
|
15
16
|
import os from "node:os";
|
|
16
17
|
import path from "node:path";
|
|
17
18
|
//#region extensions/google-meet/src/config.ts
|
|
18
|
-
const
|
|
19
|
+
const SOX_DEFAULT_BUFFER_BYTES = 8192;
|
|
20
|
+
const SOX_MIN_BUFFER_BYTES = 17;
|
|
21
|
+
const DEFAULT_GOOGLE_MEET_AUDIO_BUFFER_BYTES = SOX_DEFAULT_BUFFER_BYTES / 2;
|
|
22
|
+
function withSoxBuffer(command, bufferBytes) {
|
|
23
|
+
return [
|
|
24
|
+
command[0] ?? "sox",
|
|
25
|
+
"-q",
|
|
26
|
+
"--buffer",
|
|
27
|
+
String(bufferBytes),
|
|
28
|
+
...command.slice(2)
|
|
29
|
+
];
|
|
30
|
+
}
|
|
31
|
+
const DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND_BASE = [
|
|
19
32
|
"sox",
|
|
20
33
|
"-q",
|
|
21
34
|
"-t",
|
|
@@ -34,7 +47,7 @@ const DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND = [
|
|
|
34
47
|
"-L",
|
|
35
48
|
"-"
|
|
36
49
|
];
|
|
37
|
-
const
|
|
50
|
+
const DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND_BASE = [
|
|
38
51
|
"sox",
|
|
39
52
|
"-q",
|
|
40
53
|
"-t",
|
|
@@ -53,7 +66,7 @@ const DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND = [
|
|
|
53
66
|
"coreaudio",
|
|
54
67
|
"BlackHole 2ch"
|
|
55
68
|
];
|
|
56
|
-
const
|
|
69
|
+
const LEGACY_GOOGLE_MEET_AUDIO_INPUT_COMMAND_BASE = [
|
|
57
70
|
"rec",
|
|
58
71
|
"-q",
|
|
59
72
|
"-t",
|
|
@@ -68,7 +81,7 @@ const LEGACY_GOOGLE_MEET_AUDIO_INPUT_COMMAND = [
|
|
|
68
81
|
"8",
|
|
69
82
|
"-"
|
|
70
83
|
];
|
|
71
|
-
const
|
|
84
|
+
const LEGACY_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND_BASE = [
|
|
72
85
|
"play",
|
|
73
86
|
"-q",
|
|
74
87
|
"-t",
|
|
@@ -83,21 +96,24 @@ const LEGACY_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND = [
|
|
|
83
96
|
"8",
|
|
84
97
|
"-"
|
|
85
98
|
];
|
|
99
|
+
const DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND = withSoxBuffer(DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND_BASE, DEFAULT_GOOGLE_MEET_AUDIO_BUFFER_BYTES);
|
|
100
|
+
const DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND = withSoxBuffer(DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND_BASE, DEFAULT_GOOGLE_MEET_AUDIO_BUFFER_BYTES);
|
|
86
101
|
const DEFAULT_GOOGLE_MEET_CHROME_AUDIO_FORMAT = "pcm16-24khz";
|
|
87
102
|
const DEFAULT_GOOGLE_MEET_BARGE_IN_RMS_THRESHOLD = 650;
|
|
88
103
|
const DEFAULT_GOOGLE_MEET_BARGE_IN_PEAK_THRESHOLD = 2500;
|
|
89
104
|
const DEFAULT_GOOGLE_MEET_BARGE_IN_COOLDOWN_MS = 900;
|
|
90
|
-
const DEFAULT_GOOGLE_MEET_REALTIME_INSTRUCTIONS = `You are joining a private Google Meet as an OpenClaw
|
|
105
|
+
const DEFAULT_GOOGLE_MEET_REALTIME_INSTRUCTIONS = `You are joining a private Google Meet as an OpenClaw voice transport. Keep spoken replies brief and natural. In agent mode, wait for OpenClaw consult results and speak them exactly. In bidi mode, answer directly and call ${REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME} for deeper reasoning, current information, or tools.`;
|
|
91
106
|
const DEFAULT_GOOGLE_MEET_REALTIME_INTRO_MESSAGE = "Say exactly: I'm here and listening.";
|
|
92
107
|
const DEFAULT_GOOGLE_MEET_CONFIG = {
|
|
93
108
|
enabled: true,
|
|
94
109
|
defaults: {},
|
|
95
110
|
preview: { enrollmentAcknowledged: false },
|
|
96
111
|
defaultTransport: "chrome",
|
|
97
|
-
defaultMode: "
|
|
112
|
+
defaultMode: "agent",
|
|
98
113
|
chrome: {
|
|
99
114
|
audioBackend: "blackhole-2ch",
|
|
100
115
|
audioFormat: DEFAULT_GOOGLE_MEET_CHROME_AUDIO_FORMAT,
|
|
116
|
+
audioBufferBytes: DEFAULT_GOOGLE_MEET_AUDIO_BUFFER_BYTES,
|
|
101
117
|
launch: true,
|
|
102
118
|
guestName: "OpenClaw Agent",
|
|
103
119
|
reuseExistingTab: true,
|
|
@@ -119,7 +135,9 @@ const DEFAULT_GOOGLE_MEET_CONFIG = {
|
|
|
119
135
|
postDtmfSpeechDelayMs: 5e3
|
|
120
136
|
},
|
|
121
137
|
realtime: {
|
|
138
|
+
strategy: "agent",
|
|
122
139
|
provider: "openai",
|
|
140
|
+
transcriptionProvider: "openai",
|
|
123
141
|
instructions: DEFAULT_GOOGLE_MEET_REALTIME_INSTRUCTIONS,
|
|
124
142
|
introMessage: DEFAULT_GOOGLE_MEET_REALTIME_INTRO_MESSAGE,
|
|
125
143
|
toolPolicy: "safe-read-only",
|
|
@@ -157,6 +175,9 @@ function readEnvString(env, keys) {
|
|
|
157
175
|
if (value) return value;
|
|
158
176
|
}
|
|
159
177
|
}
|
|
178
|
+
function normalizeStringAllowEmpty(value) {
|
|
179
|
+
return typeof value === "string" ? value.trim() : void 0;
|
|
180
|
+
}
|
|
160
181
|
function readEnvBoolean(env, keys) {
|
|
161
182
|
const normalized = normalizeOptionalLowercaseString(readEnvString(env, keys));
|
|
162
183
|
if (!normalized) return;
|
|
@@ -197,7 +218,12 @@ function resolveTransport$1(value, fallback) {
|
|
|
197
218
|
}
|
|
198
219
|
function resolveMode$1(value, fallback) {
|
|
199
220
|
const normalized = normalizeOptionalLowercaseString(value);
|
|
200
|
-
|
|
221
|
+
if (normalized === "realtime") return "agent";
|
|
222
|
+
return normalized === "agent" || normalized === "bidi" || normalized === "transcribe" ? normalized : fallback;
|
|
223
|
+
}
|
|
224
|
+
function resolveRealtimeStrategy(value, fallback) {
|
|
225
|
+
const normalized = normalizeOptionalLowercaseString(value);
|
|
226
|
+
return normalized === "agent" || normalized === "bidi" ? normalized : fallback;
|
|
201
227
|
}
|
|
202
228
|
function resolveChromeAudioFormat(value) {
|
|
203
229
|
switch (normalizeOptionalString(value)?.toLowerCase().replaceAll("_", "-")) {
|
|
@@ -213,11 +239,16 @@ function resolveChromeAudioFormat(value) {
|
|
|
213
239
|
default: return;
|
|
214
240
|
}
|
|
215
241
|
}
|
|
216
|
-
function
|
|
217
|
-
|
|
242
|
+
function resolveAudioBufferBytes(value, fallback) {
|
|
243
|
+
const number = resolveNumber(value, fallback);
|
|
244
|
+
if (!Number.isFinite(number) || number <= 0) return fallback;
|
|
245
|
+
return Math.max(SOX_MIN_BUFFER_BYTES, Math.trunc(number));
|
|
246
|
+
}
|
|
247
|
+
function defaultAudioInputCommand(format, bufferBytes) {
|
|
248
|
+
return withSoxBuffer(format === "g711-ulaw-8khz" ? LEGACY_GOOGLE_MEET_AUDIO_INPUT_COMMAND_BASE : DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND_BASE, bufferBytes);
|
|
218
249
|
}
|
|
219
|
-
function defaultAudioOutputCommand(format) {
|
|
220
|
-
return format === "g711-ulaw-8khz" ?
|
|
250
|
+
function defaultAudioOutputCommand(format, bufferBytes) {
|
|
251
|
+
return withSoxBuffer(format === "g711-ulaw-8khz" ? LEGACY_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND_BASE : DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND_BASE, bufferBytes);
|
|
221
252
|
}
|
|
222
253
|
function resolveGoogleMeetConfig(input) {
|
|
223
254
|
return resolveGoogleMeetConfigWithEnv(input);
|
|
@@ -231,10 +262,13 @@ function resolveGoogleMeetConfigWithEnv(input, env = process.env) {
|
|
|
231
262
|
const configuredAudioOutputCommand = resolveStringArray(chrome.audioOutputCommand);
|
|
232
263
|
const hasCustomAudioCommand = configuredAudioInputCommand !== void 0 || configuredAudioOutputCommand !== void 0;
|
|
233
264
|
const audioFormat = resolveChromeAudioFormat(chrome.audioFormat) ?? (hasCustomAudioCommand ? "g711-ulaw-8khz" : DEFAULT_GOOGLE_MEET_CONFIG.chrome.audioFormat);
|
|
265
|
+
const audioBufferBytes = resolveAudioBufferBytes(chrome.audioBufferBytes, DEFAULT_GOOGLE_MEET_CONFIG.chrome.audioBufferBytes);
|
|
234
266
|
const chromeNode = asRecord$3(raw.chromeNode);
|
|
235
267
|
const twilio = asRecord$3(raw.twilio);
|
|
236
268
|
const voiceCall = asRecord$3(raw.voiceCall);
|
|
237
269
|
const realtime = asRecord$3(raw.realtime);
|
|
270
|
+
const realtimeProvider = normalizeOptionalString(realtime.provider);
|
|
271
|
+
const resolvedRealtimeProvider = realtimeProvider ?? DEFAULT_GOOGLE_MEET_CONFIG.realtime.provider;
|
|
238
272
|
const oauth = asRecord$3(raw.oauth);
|
|
239
273
|
const auth = asRecord$3(raw.auth);
|
|
240
274
|
return {
|
|
@@ -246,6 +280,7 @@ function resolveGoogleMeetConfigWithEnv(input, env = process.env) {
|
|
|
246
280
|
chrome: {
|
|
247
281
|
audioBackend: "blackhole-2ch",
|
|
248
282
|
audioFormat,
|
|
283
|
+
audioBufferBytes,
|
|
249
284
|
launch: resolveBoolean(chrome.launch, DEFAULT_GOOGLE_MEET_CONFIG.chrome.launch),
|
|
250
285
|
browserProfile: normalizeOptionalString(chrome.browserProfile),
|
|
251
286
|
guestName: normalizeOptionalString(chrome.guestName) ?? DEFAULT_GOOGLE_MEET_CONFIG.chrome.guestName,
|
|
@@ -253,8 +288,8 @@ function resolveGoogleMeetConfigWithEnv(input, env = process.env) {
|
|
|
253
288
|
autoJoin: resolveBoolean(chrome.autoJoin, DEFAULT_GOOGLE_MEET_CONFIG.chrome.autoJoin),
|
|
254
289
|
joinTimeoutMs: resolveNumber(chrome.joinTimeoutMs, DEFAULT_GOOGLE_MEET_CONFIG.chrome.joinTimeoutMs),
|
|
255
290
|
waitForInCallMs: resolveNumber(chrome.waitForInCallMs, DEFAULT_GOOGLE_MEET_CONFIG.chrome.waitForInCallMs),
|
|
256
|
-
audioInputCommand: configuredAudioInputCommand ??
|
|
257
|
-
audioOutputCommand: configuredAudioOutputCommand ??
|
|
291
|
+
audioInputCommand: configuredAudioInputCommand ?? defaultAudioInputCommand(audioFormat, audioBufferBytes),
|
|
292
|
+
audioOutputCommand: configuredAudioOutputCommand ?? defaultAudioOutputCommand(audioFormat, audioBufferBytes),
|
|
258
293
|
bargeInInputCommand: resolveStringArray(chrome.bargeInInputCommand),
|
|
259
294
|
bargeInRmsThreshold: resolveNumber(chrome.bargeInRmsThreshold, DEFAULT_GOOGLE_MEET_CONFIG.chrome.bargeInRmsThreshold),
|
|
260
295
|
bargeInPeakThreshold: resolveNumber(chrome.bargeInPeakThreshold, DEFAULT_GOOGLE_MEET_CONFIG.chrome.bargeInPeakThreshold),
|
|
@@ -278,10 +313,13 @@ function resolveGoogleMeetConfigWithEnv(input, env = process.env) {
|
|
|
278
313
|
introMessage: normalizeOptionalString(voiceCall.introMessage)
|
|
279
314
|
},
|
|
280
315
|
realtime: {
|
|
281
|
-
|
|
316
|
+
strategy: resolveRealtimeStrategy(realtime.strategy, DEFAULT_GOOGLE_MEET_CONFIG.realtime.strategy),
|
|
317
|
+
provider: resolvedRealtimeProvider,
|
|
318
|
+
transcriptionProvider: normalizeOptionalString(realtime.transcriptionProvider) ?? (realtimeProvider && realtimeProvider !== "google" ? resolvedRealtimeProvider : DEFAULT_GOOGLE_MEET_CONFIG.realtime.transcriptionProvider),
|
|
319
|
+
voiceProvider: normalizeOptionalString(realtime.voiceProvider),
|
|
282
320
|
model: normalizeOptionalString(realtime.model) ?? DEFAULT_GOOGLE_MEET_CONFIG.realtime.model,
|
|
283
321
|
instructions: normalizeOptionalString(realtime.instructions) ?? DEFAULT_GOOGLE_MEET_CONFIG.realtime.instructions,
|
|
284
|
-
introMessage:
|
|
322
|
+
introMessage: normalizeStringAllowEmpty(realtime.introMessage) ?? DEFAULT_GOOGLE_MEET_CONFIG.realtime.introMessage,
|
|
285
323
|
agentId: normalizeOptionalString(realtime.agentId),
|
|
286
324
|
toolPolicy: resolveRealtimeVoiceAgentConsultToolPolicy(realtime.toolPolicy, DEFAULT_GOOGLE_MEET_CONFIG.realtime.toolPolicy),
|
|
287
325
|
providers: resolveProvidersConfig(realtime.providers)
|
|
@@ -320,7 +358,8 @@ function submitGoogleMeetConsultWorkingResponse(session, callId) {
|
|
|
320
358
|
}
|
|
321
359
|
async function consultOpenClawAgentForGoogleMeet(params) {
|
|
322
360
|
const agentId = normalizeAgentId(params.config.realtime.agentId);
|
|
323
|
-
const
|
|
361
|
+
const requesterSessionKey = normalizeOptionalString(params.requesterSessionKey) ?? `agent:${agentId}:main`;
|
|
362
|
+
const sessionKey = `agent:${agentId}:subagent:google-meet:${params.meetingSessionId}`;
|
|
324
363
|
return await consultRealtimeVoiceAgent({
|
|
325
364
|
cfg: params.fullConfig,
|
|
326
365
|
agentRuntime: params.runtime.agent,
|
|
@@ -330,6 +369,8 @@ async function consultOpenClawAgentForGoogleMeet(params) {
|
|
|
330
369
|
messageProvider: "google-meet",
|
|
331
370
|
lane: "google-meet",
|
|
332
371
|
runIdPrefix: `google-meet:${params.meetingSessionId}`,
|
|
372
|
+
spawnedBy: requesterSessionKey,
|
|
373
|
+
contextMode: "fork",
|
|
333
374
|
args: params.args,
|
|
334
375
|
transcript: params.transcript,
|
|
335
376
|
surface: "a private Google Meet",
|
|
@@ -362,7 +403,9 @@ function getGoogleMeetRealtimeTranscriptHealth(transcript) {
|
|
|
362
403
|
recentRealtimeTranscript: transcript.slice(-5)
|
|
363
404
|
};
|
|
364
405
|
}
|
|
406
|
+
const GOOGLE_MEET_OUTPUT_ECHO_SUPPRESSION_TAIL_MS = 3e3;
|
|
365
407
|
function recordGoogleMeetRealtimeEvent(events, event) {
|
|
408
|
+
if (event.direction === "client" && event.type === "input_audio_buffer.append") return;
|
|
366
409
|
events.push({
|
|
367
410
|
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
368
411
|
...event
|
|
@@ -401,12 +444,88 @@ function readPcm16Stats(audio) {
|
|
|
401
444
|
peak
|
|
402
445
|
};
|
|
403
446
|
}
|
|
447
|
+
function normalizeTranscriptForEchoMatch(text) {
|
|
448
|
+
return text.toLowerCase().replace(/['’]/g, "").replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter((token) => token.length > 1);
|
|
449
|
+
}
|
|
450
|
+
function hasMeaningfulEchoOverlap(userTokens, assistantTokens) {
|
|
451
|
+
if (userTokens.length < 4 || assistantTokens.length < 4) return false;
|
|
452
|
+
const uniqueUserTokens = [...new Set(userTokens)];
|
|
453
|
+
if (uniqueUserTokens.length < 4) return false;
|
|
454
|
+
const assistantTokenSet = new Set(assistantTokens);
|
|
455
|
+
return uniqueUserTokens.filter((token) => assistantTokenSet.has(token)).length / uniqueUserTokens.length >= .58;
|
|
456
|
+
}
|
|
457
|
+
function isGoogleMeetLikelyAssistantEchoTranscript(params) {
|
|
458
|
+
const userTokens = normalizeTranscriptForEchoMatch(params.text);
|
|
459
|
+
if (userTokens.length < 4) return false;
|
|
460
|
+
const nowMs = params.nowMs ?? Date.now();
|
|
461
|
+
const recentAssistantText = params.transcript.filter((entry) => {
|
|
462
|
+
if (entry.role !== "assistant") return false;
|
|
463
|
+
const at = Date.parse(entry.at);
|
|
464
|
+
return Number.isFinite(at) && nowMs - at <= 45e3;
|
|
465
|
+
}).slice(-6).map((entry) => entry.text).join(" ");
|
|
466
|
+
if (!recentAssistantText.trim()) return false;
|
|
467
|
+
const userNormalized = userTokens.join(" ");
|
|
468
|
+
const assistantTokens = normalizeTranscriptForEchoMatch(recentAssistantText);
|
|
469
|
+
const assistantNormalized = assistantTokens.join(" ");
|
|
470
|
+
return userNormalized.length >= 18 && assistantNormalized.includes(userNormalized) || assistantNormalized.length >= 18 && userNormalized.includes(assistantNormalized) || hasMeaningfulEchoOverlap(userTokens, assistantTokens);
|
|
471
|
+
}
|
|
472
|
+
function extendGoogleMeetOutputEchoSuppression(params) {
|
|
473
|
+
const bytesPerMs = params.audioFormat === "g711-ulaw-8khz" ? 8 : 48;
|
|
474
|
+
const durationMs = Math.ceil(params.audio.byteLength / bytesPerMs);
|
|
475
|
+
const playbackEndMs = Math.max(params.nowMs, params.lastOutputPlayableUntilMs) + durationMs;
|
|
476
|
+
return {
|
|
477
|
+
durationMs,
|
|
478
|
+
lastOutputPlayableUntilMs: playbackEndMs,
|
|
479
|
+
suppressInputUntilMs: Math.max(params.suppressInputUntilMs, playbackEndMs + GOOGLE_MEET_OUTPUT_ECHO_SUPPRESSION_TAIL_MS)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
404
482
|
function resolveGoogleMeetRealtimeAudioFormat(config) {
|
|
405
483
|
return config.chrome.audioFormat === "g711-ulaw-8khz" ? REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ : REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ;
|
|
406
484
|
}
|
|
485
|
+
function convertGoogleMeetBridgeAudioForStt(audio, config) {
|
|
486
|
+
if (config.chrome.audioFormat === "g711-ulaw-8khz") return audio;
|
|
487
|
+
return convertPcmToMulaw8k(audio, 24e3);
|
|
488
|
+
}
|
|
489
|
+
function convertGoogleMeetTtsAudioForBridge(audio, sampleRate, config, outputFormat) {
|
|
490
|
+
const sourceFormat = sourceTelephonyTtsFormat(outputFormat);
|
|
491
|
+
if (config.chrome.audioFormat === "g711-ulaw-8khz" && sourceFormat === "mulaw" && sampleRate === 8e3) return audio;
|
|
492
|
+
const pcm = decodeGoogleMeetTelephonyTtsAudio(audio, sourceFormat);
|
|
493
|
+
return config.chrome.audioFormat === "g711-ulaw-8khz" ? convertPcmToMulaw8k(pcm, sampleRate) : resamplePcm(pcm, sampleRate, 24e3);
|
|
494
|
+
}
|
|
495
|
+
function sourceTelephonyTtsFormat(outputFormat) {
|
|
496
|
+
const normalized = outputFormat?.trim().toLowerCase().replaceAll("_", "-") ?? "";
|
|
497
|
+
if (!normalized || normalized === "pcm" || normalized.startsWith("pcm-") || normalized.includes("pcm16") || normalized.includes("16bit-mono-pcm")) return "pcm";
|
|
498
|
+
if (normalized === "mulaw" || normalized === "ulaw" || normalized.includes("mu-law") || normalized.includes("mulaw") || normalized.includes("ulaw")) return "mulaw";
|
|
499
|
+
if (normalized === "alaw" || normalized.includes("a-law") || normalized.includes("alaw")) return "alaw";
|
|
500
|
+
throw new Error(`Unsupported telephony TTS output format for Google Meet: ${outputFormat}`);
|
|
501
|
+
}
|
|
502
|
+
function decodeGoogleMeetTelephonyTtsAudio(audio, sourceFormat) {
|
|
503
|
+
switch (sourceFormat) {
|
|
504
|
+
case "pcm": return audio;
|
|
505
|
+
case "mulaw": return mulawToPcm(audio);
|
|
506
|
+
case "alaw": return alawToPcm(audio);
|
|
507
|
+
}
|
|
508
|
+
return unsupportedGoogleMeetTelephonyTtsFormat(sourceFormat);
|
|
509
|
+
}
|
|
510
|
+
function unsupportedGoogleMeetTelephonyTtsFormat(_format) {
|
|
511
|
+
throw new Error("Unsupported telephony TTS output format for Google Meet");
|
|
512
|
+
}
|
|
513
|
+
function alawToPcm(alaw) {
|
|
514
|
+
const pcm = Buffer.alloc(alaw.length * 2);
|
|
515
|
+
for (let index = 0; index < alaw.length; index += 1) pcm.writeInt16LE(alawByteToLinear(alaw[index] ?? 0), index * 2);
|
|
516
|
+
return pcm;
|
|
517
|
+
}
|
|
518
|
+
function alawByteToLinear(value) {
|
|
519
|
+
const aLaw = value ^ 85;
|
|
520
|
+
const sign = aLaw & 128;
|
|
521
|
+
const exponent = (aLaw & 112) >> 4;
|
|
522
|
+
const mantissa = aLaw & 15;
|
|
523
|
+
let sample = exponent === 0 ? (mantissa << 4) + 8 : (mantissa << 4) + 264 << exponent - 1;
|
|
524
|
+
return sign ? sample : -sample;
|
|
525
|
+
}
|
|
407
526
|
function resolveGoogleMeetRealtimeProvider(params) {
|
|
408
527
|
return resolveConfiguredRealtimeVoiceProvider({
|
|
409
|
-
configuredProviderId: params.config.realtime.provider,
|
|
528
|
+
configuredProviderId: params.config.realtime.voiceProvider ?? params.config.realtime.provider,
|
|
410
529
|
providerConfigs: params.config.realtime.providers,
|
|
411
530
|
cfg: params.fullConfig,
|
|
412
531
|
providers: params.providers,
|
|
@@ -414,6 +533,322 @@ function resolveGoogleMeetRealtimeProvider(params) {
|
|
|
414
533
|
noRegisteredProviderMessage: "No configured realtime voice provider registered"
|
|
415
534
|
});
|
|
416
535
|
}
|
|
536
|
+
function resolveGoogleMeetRealtimeTranscriptionProvider(params) {
|
|
537
|
+
const providers = params.providers ?? listRealtimeTranscriptionProviders(params.fullConfig);
|
|
538
|
+
if (providers.length === 0) throw new Error("No configured realtime transcription provider registered");
|
|
539
|
+
const providerId = params.config.realtime.transcriptionProvider ?? params.config.realtime.provider;
|
|
540
|
+
const provider = (providerId ? params.providers?.find((entry) => entry.id === providerId || entry.aliases?.includes(providerId)) ?? getRealtimeTranscriptionProvider(providerId, params.fullConfig) : void 0) ?? providers[0];
|
|
541
|
+
if (!provider) throw new Error("No configured realtime transcription provider registered");
|
|
542
|
+
const rawConfig = providerId ? params.config.realtime.providers[providerId] ?? params.config.realtime.providers[provider.id] ?? {} : params.config.realtime.providers[provider.id] ?? {};
|
|
543
|
+
const providerConfig = provider.resolveConfig ? provider.resolveConfig({
|
|
544
|
+
cfg: params.fullConfig,
|
|
545
|
+
rawConfig
|
|
546
|
+
}) : rawConfig;
|
|
547
|
+
if (!provider.isConfigured({
|
|
548
|
+
cfg: params.fullConfig,
|
|
549
|
+
providerConfig
|
|
550
|
+
})) throw new Error(`Realtime transcription provider "${provider.id}" is not configured`);
|
|
551
|
+
return {
|
|
552
|
+
provider,
|
|
553
|
+
providerConfig
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
function buildGoogleMeetSpeakExactUserMessage(text) {
|
|
557
|
+
return ["Speak this exact OpenClaw answer to the meeting, without adding, removing, or rephrasing words.", `Answer: ${JSON.stringify(text)}`].join("\n");
|
|
558
|
+
}
|
|
559
|
+
function readLogString(value) {
|
|
560
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
561
|
+
}
|
|
562
|
+
function formatLogValue(value) {
|
|
563
|
+
return value?.replace(/\s+/g, "_").slice(0, 180) || "unknown";
|
|
564
|
+
}
|
|
565
|
+
function resolveProviderModelForLog(params) {
|
|
566
|
+
return readLogString(params.providerConfig.model) ?? readLogString(params.providerConfig.modelId) ?? readLogString(params.fallbackModel) ?? readLogString(params.provider.defaultModel) ?? "provider-default";
|
|
567
|
+
}
|
|
568
|
+
function formatGoogleMeetRealtimeVoiceModelLog(params) {
|
|
569
|
+
return [
|
|
570
|
+
`[google-meet] realtime voice bridge starting: strategy=${formatLogValue(params.strategy)}`,
|
|
571
|
+
`provider=${formatLogValue(params.provider.id)}`,
|
|
572
|
+
`model=${formatLogValue(resolveProviderModelForLog({
|
|
573
|
+
provider: params.provider,
|
|
574
|
+
providerConfig: params.providerConfig,
|
|
575
|
+
fallbackModel: params.fallbackModel
|
|
576
|
+
}))}`,
|
|
577
|
+
`audioFormat=${formatLogValue(params.audioFormat)}`
|
|
578
|
+
].join(" ");
|
|
579
|
+
}
|
|
580
|
+
function formatGoogleMeetAgentAudioModelLog(params) {
|
|
581
|
+
return [
|
|
582
|
+
`[google-meet] agent audio bridge starting: transcriptionProvider=${formatLogValue(params.provider.id)}`,
|
|
583
|
+
`transcriptionModel=${formatLogValue(resolveProviderModelForLog({
|
|
584
|
+
provider: params.provider,
|
|
585
|
+
providerConfig: params.providerConfig
|
|
586
|
+
}))}`,
|
|
587
|
+
"tts=telephony",
|
|
588
|
+
`audioFormat=${formatLogValue(params.audioFormat)}`
|
|
589
|
+
].join(" ");
|
|
590
|
+
}
|
|
591
|
+
function formatGoogleMeetAgentTtsResultLog(prefix, result) {
|
|
592
|
+
return [
|
|
593
|
+
`[google-meet] ${prefix} TTS: provider=${formatLogValue(result.provider)}`,
|
|
594
|
+
`model=${formatLogValue(result.providerModel)}`,
|
|
595
|
+
`voice=${formatLogValue(result.providerVoice)}`,
|
|
596
|
+
`outputFormat=${formatLogValue(result.outputFormat)}`,
|
|
597
|
+
`sampleRate=${result.sampleRate ?? "unknown"}`,
|
|
598
|
+
...result.fallbackFrom ? [`fallbackFrom=${formatLogValue(result.fallbackFrom)}`] : []
|
|
599
|
+
].join(" ");
|
|
600
|
+
}
|
|
601
|
+
function normalizeGoogleMeetTtsPromptText$1(text) {
|
|
602
|
+
const trimmed = text?.trim();
|
|
603
|
+
if (!trimmed) return;
|
|
604
|
+
const sayExactly = trimmed.match(/^say exactly:\s*(?<text>.+)$/is)?.groups?.text?.trim();
|
|
605
|
+
if (sayExactly) return sayExactly.replace(/^["']|["']$/g, "").trim() || trimmed;
|
|
606
|
+
return trimmed;
|
|
607
|
+
}
|
|
608
|
+
async function startCommandAgentAudioBridge(params) {
|
|
609
|
+
const input = splitCommand$1(params.inputCommand);
|
|
610
|
+
const output = splitCommand$1(params.outputCommand);
|
|
611
|
+
const spawnFn = params.spawn ?? ((command, args, options) => spawn(command, args, options));
|
|
612
|
+
const outputProcess = spawnFn(output.command, output.args, { stdio: [
|
|
613
|
+
"pipe",
|
|
614
|
+
"ignore",
|
|
615
|
+
"pipe"
|
|
616
|
+
] });
|
|
617
|
+
const inputProcess = spawnFn(input.command, input.args, { stdio: [
|
|
618
|
+
"ignore",
|
|
619
|
+
"pipe",
|
|
620
|
+
"pipe"
|
|
621
|
+
] });
|
|
622
|
+
let stopped = false;
|
|
623
|
+
let sttSession = null;
|
|
624
|
+
let realtimeReady = false;
|
|
625
|
+
let lastInputAt;
|
|
626
|
+
let lastOutputAt;
|
|
627
|
+
let lastInputBytes = 0;
|
|
628
|
+
let lastOutputBytes = 0;
|
|
629
|
+
let suppressedInputBytes = 0;
|
|
630
|
+
let lastSuppressedInputAt;
|
|
631
|
+
let suppressInputUntil = 0;
|
|
632
|
+
let lastOutputPlayableUntilMs = 0;
|
|
633
|
+
let agentConsultActive = false;
|
|
634
|
+
let pendingAgentQuestion;
|
|
635
|
+
let agentConsultDebounceTimer;
|
|
636
|
+
let ttsQueue = Promise.resolve();
|
|
637
|
+
const transcript = [];
|
|
638
|
+
const resolved = resolveGoogleMeetRealtimeTranscriptionProvider({
|
|
639
|
+
config: params.config,
|
|
640
|
+
fullConfig: params.fullConfig,
|
|
641
|
+
providers: params.providers
|
|
642
|
+
});
|
|
643
|
+
params.logger.info(formatGoogleMeetAgentAudioModelLog({
|
|
644
|
+
provider: resolved.provider,
|
|
645
|
+
providerConfig: resolved.providerConfig,
|
|
646
|
+
audioFormat: params.config.chrome.audioFormat
|
|
647
|
+
}));
|
|
648
|
+
const terminateProcess = (proc, signal = "SIGTERM") => {
|
|
649
|
+
if (proc.killed && signal !== "SIGKILL") return;
|
|
650
|
+
let exited = false;
|
|
651
|
+
proc.on("exit", () => {
|
|
652
|
+
exited = true;
|
|
653
|
+
});
|
|
654
|
+
try {
|
|
655
|
+
proc.kill(signal);
|
|
656
|
+
} catch {
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
if (signal === "SIGKILL") return;
|
|
660
|
+
setTimeout(() => {
|
|
661
|
+
if (!exited) try {
|
|
662
|
+
proc.kill("SIGKILL");
|
|
663
|
+
} catch {}
|
|
664
|
+
}, 1e3).unref?.();
|
|
665
|
+
};
|
|
666
|
+
const stop = async () => {
|
|
667
|
+
if (stopped) return;
|
|
668
|
+
stopped = true;
|
|
669
|
+
if (agentConsultDebounceTimer) {
|
|
670
|
+
clearTimeout(agentConsultDebounceTimer);
|
|
671
|
+
agentConsultDebounceTimer = void 0;
|
|
672
|
+
}
|
|
673
|
+
try {
|
|
674
|
+
sttSession?.close();
|
|
675
|
+
} catch (error) {
|
|
676
|
+
params.logger.debug?.(`[google-meet] agent transcription bridge close ignored: ${formatErrorMessage(error)}`);
|
|
677
|
+
}
|
|
678
|
+
terminateProcess(inputProcess);
|
|
679
|
+
terminateProcess(outputProcess);
|
|
680
|
+
};
|
|
681
|
+
const fail = (label) => (error) => {
|
|
682
|
+
params.logger.warn(`[google-meet] ${label} failed: ${formatErrorMessage(error)}`);
|
|
683
|
+
stop();
|
|
684
|
+
};
|
|
685
|
+
inputProcess.on("error", fail("audio input command"));
|
|
686
|
+
inputProcess.on("exit", (code, signal) => {
|
|
687
|
+
if (!stopped) {
|
|
688
|
+
params.logger.warn(`[google-meet] audio input command exited (${code ?? signal ?? "done"})`);
|
|
689
|
+
stop();
|
|
690
|
+
}
|
|
691
|
+
});
|
|
692
|
+
inputProcess.stderr?.on("data", (chunk) => {
|
|
693
|
+
params.logger.debug?.(`[google-meet] audio input: ${String(chunk).trim()}`);
|
|
694
|
+
});
|
|
695
|
+
outputProcess.on("error", fail("audio output command"));
|
|
696
|
+
outputProcess.stdin?.on?.("error", fail("audio output command"));
|
|
697
|
+
outputProcess.on("exit", (code, signal) => {
|
|
698
|
+
if (!stopped) {
|
|
699
|
+
params.logger.warn(`[google-meet] audio output command exited (${code ?? signal ?? "done"})`);
|
|
700
|
+
stop();
|
|
701
|
+
}
|
|
702
|
+
});
|
|
703
|
+
outputProcess.stderr?.on("data", (chunk) => {
|
|
704
|
+
params.logger.debug?.(`[google-meet] audio output: ${String(chunk).trim()}`);
|
|
705
|
+
});
|
|
706
|
+
const writeOutputAudio = (audio) => {
|
|
707
|
+
const suppression = extendGoogleMeetOutputEchoSuppression({
|
|
708
|
+
audio,
|
|
709
|
+
audioFormat: params.config.chrome.audioFormat,
|
|
710
|
+
nowMs: Date.now(),
|
|
711
|
+
lastOutputPlayableUntilMs,
|
|
712
|
+
suppressInputUntilMs: suppressInputUntil
|
|
713
|
+
});
|
|
714
|
+
suppressInputUntil = suppression.suppressInputUntilMs;
|
|
715
|
+
lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
|
|
716
|
+
lastOutputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
717
|
+
lastOutputBytes += audio.byteLength;
|
|
718
|
+
try {
|
|
719
|
+
outputProcess.stdin?.write(audio);
|
|
720
|
+
} catch (error) {
|
|
721
|
+
fail("audio output command")(error);
|
|
722
|
+
}
|
|
723
|
+
};
|
|
724
|
+
const enqueueSpeakText = (text) => {
|
|
725
|
+
const normalized = normalizeGoogleMeetTtsPromptText$1(text);
|
|
726
|
+
if (!normalized || stopped) return;
|
|
727
|
+
ttsQueue = ttsQueue.then(async () => {
|
|
728
|
+
if (stopped) return;
|
|
729
|
+
recordGoogleMeetRealtimeTranscript(transcript, "assistant", normalized);
|
|
730
|
+
params.logger.info(`[google-meet] agent assistant: ${normalized}`);
|
|
731
|
+
const result = await params.runtime.tts.textToSpeechTelephony({
|
|
732
|
+
text: normalized,
|
|
733
|
+
cfg: params.fullConfig
|
|
734
|
+
});
|
|
735
|
+
if (!result.success || !result.audioBuffer || !result.sampleRate) throw new Error(result.error ?? "TTS conversion failed");
|
|
736
|
+
params.logger.info(formatGoogleMeetAgentTtsResultLog("agent", result));
|
|
737
|
+
writeOutputAudio(convertGoogleMeetTtsAudioForBridge(result.audioBuffer, result.sampleRate, params.config, result.outputFormat));
|
|
738
|
+
}).catch((error) => {
|
|
739
|
+
params.logger.warn(`[google-meet] agent TTS failed: ${formatErrorMessage(error)}`);
|
|
740
|
+
});
|
|
741
|
+
};
|
|
742
|
+
const runAgentConsultForUserTranscript = async (question) => {
|
|
743
|
+
const trimmed = question.trim();
|
|
744
|
+
if (!trimmed || stopped) return;
|
|
745
|
+
if (agentConsultActive) {
|
|
746
|
+
pendingAgentQuestion = trimmed;
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
agentConsultActive = true;
|
|
750
|
+
let nextQuestion = trimmed;
|
|
751
|
+
try {
|
|
752
|
+
while (nextQuestion) {
|
|
753
|
+
if (stopped) return;
|
|
754
|
+
const currentQuestion = nextQuestion;
|
|
755
|
+
pendingAgentQuestion = void 0;
|
|
756
|
+
params.logger.info(`[google-meet] agent consult: ${currentQuestion}`);
|
|
757
|
+
enqueueSpeakText((await consultOpenClawAgentForGoogleMeet({
|
|
758
|
+
config: params.config,
|
|
759
|
+
fullConfig: params.fullConfig,
|
|
760
|
+
runtime: params.runtime,
|
|
761
|
+
logger: params.logger,
|
|
762
|
+
meetingSessionId: params.meetingSessionId,
|
|
763
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
764
|
+
args: {
|
|
765
|
+
question: currentQuestion,
|
|
766
|
+
responseStyle: "Brief, natural spoken answer for a live meeting."
|
|
767
|
+
},
|
|
768
|
+
transcript
|
|
769
|
+
})).text);
|
|
770
|
+
nextQuestion = pendingAgentQuestion;
|
|
771
|
+
}
|
|
772
|
+
} catch (error) {
|
|
773
|
+
params.logger.warn(`[google-meet] agent consult failed: ${formatErrorMessage(error)}`);
|
|
774
|
+
enqueueSpeakText("I hit an error while checking that. Please try again.");
|
|
775
|
+
} finally {
|
|
776
|
+
agentConsultActive = false;
|
|
777
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
778
|
+
pendingAgentQuestion = void 0;
|
|
779
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
780
|
+
}
|
|
781
|
+
};
|
|
782
|
+
const enqueueAgentConsultForUserTranscript = (question) => {
|
|
783
|
+
const trimmed = question.trim();
|
|
784
|
+
if (!trimmed || stopped) return;
|
|
785
|
+
pendingAgentQuestion = pendingAgentQuestion ? `${pendingAgentQuestion}\n${trimmed}` : trimmed;
|
|
786
|
+
if (agentConsultDebounceTimer) clearTimeout(agentConsultDebounceTimer);
|
|
787
|
+
agentConsultDebounceTimer = setTimeout(() => {
|
|
788
|
+
agentConsultDebounceTimer = void 0;
|
|
789
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
790
|
+
pendingAgentQuestion = void 0;
|
|
791
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
792
|
+
}, 900);
|
|
793
|
+
agentConsultDebounceTimer.unref?.();
|
|
794
|
+
};
|
|
795
|
+
sttSession = resolved.provider.createSession({
|
|
796
|
+
providerConfig: resolved.providerConfig,
|
|
797
|
+
onTranscript: (text) => {
|
|
798
|
+
const trimmed = text.trim();
|
|
799
|
+
if (!trimmed || stopped) return;
|
|
800
|
+
recordGoogleMeetRealtimeTranscript(transcript, "user", trimmed);
|
|
801
|
+
params.logger.info(`[google-meet] agent user: ${trimmed}`);
|
|
802
|
+
if (isGoogleMeetLikelyAssistantEchoTranscript({
|
|
803
|
+
transcript,
|
|
804
|
+
text: trimmed
|
|
805
|
+
})) {
|
|
806
|
+
params.logger.info(`[google-meet] agent ignored assistant echo transcript: ${trimmed}`);
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
enqueueAgentConsultForUserTranscript(trimmed);
|
|
810
|
+
},
|
|
811
|
+
onError: (error) => {
|
|
812
|
+
params.logger.warn(`[google-meet] agent transcription bridge failed: ${formatErrorMessage(error)}`);
|
|
813
|
+
stop();
|
|
814
|
+
}
|
|
815
|
+
});
|
|
816
|
+
await sttSession.connect();
|
|
817
|
+
realtimeReady = true;
|
|
818
|
+
inputProcess.stdout?.on("data", (chunk) => {
|
|
819
|
+
if (stopped) return;
|
|
820
|
+
const audio = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
821
|
+
if (Date.now() < suppressInputUntil) {
|
|
822
|
+
lastSuppressedInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
823
|
+
suppressedInputBytes += audio.byteLength;
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
lastInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
827
|
+
lastInputBytes += audio.byteLength;
|
|
828
|
+
sttSession?.sendAudio(convertGoogleMeetBridgeAudioForStt(audio, params.config));
|
|
829
|
+
});
|
|
830
|
+
return {
|
|
831
|
+
providerId: resolved.provider.id,
|
|
832
|
+
inputCommand: params.inputCommand,
|
|
833
|
+
outputCommand: params.outputCommand,
|
|
834
|
+
speak: enqueueSpeakText,
|
|
835
|
+
getHealth: () => ({
|
|
836
|
+
providerConnected: sttSession?.isConnected() ?? false,
|
|
837
|
+
realtimeReady,
|
|
838
|
+
audioInputActive: lastInputBytes > 0,
|
|
839
|
+
audioOutputActive: lastOutputBytes > 0,
|
|
840
|
+
lastInputAt,
|
|
841
|
+
lastOutputAt,
|
|
842
|
+
lastSuppressedInputAt,
|
|
843
|
+
lastInputBytes,
|
|
844
|
+
lastOutputBytes,
|
|
845
|
+
suppressedInputBytes,
|
|
846
|
+
...getGoogleMeetRealtimeTranscriptHealth(transcript),
|
|
847
|
+
bridgeClosed: stopped
|
|
848
|
+
}),
|
|
849
|
+
stop
|
|
850
|
+
};
|
|
851
|
+
}
|
|
417
852
|
async function startCommandRealtimeAudioBridge(params) {
|
|
418
853
|
const input = splitCommand$1(params.inputCommand);
|
|
419
854
|
const output = splitCommand$1(params.outputCommand);
|
|
@@ -444,12 +879,17 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
444
879
|
let lastOutputAtMs = 0;
|
|
445
880
|
let lastOutputPlayableUntilMs = 0;
|
|
446
881
|
let bargeInInputProcess;
|
|
882
|
+
let agentConsultDebounceTimer;
|
|
447
883
|
const suppressInputForOutput = (audio) => {
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
884
|
+
const suppression = extendGoogleMeetOutputEchoSuppression({
|
|
885
|
+
audio,
|
|
886
|
+
audioFormat: params.config.chrome.audioFormat,
|
|
887
|
+
nowMs: Date.now(),
|
|
888
|
+
lastOutputPlayableUntilMs,
|
|
889
|
+
suppressInputUntilMs: suppressInputUntil
|
|
890
|
+
});
|
|
891
|
+
suppressInputUntil = suppression.suppressInputUntilMs;
|
|
892
|
+
lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
|
|
453
893
|
};
|
|
454
894
|
const terminateProcess = (proc, signal = "SIGTERM") => {
|
|
455
895
|
if (proc.killed && signal !== "SIGKILL") return;
|
|
@@ -472,6 +912,10 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
472
912
|
const stop = async () => {
|
|
473
913
|
if (stopped) return;
|
|
474
914
|
stopped = true;
|
|
915
|
+
if (agentConsultDebounceTimer) {
|
|
916
|
+
clearTimeout(agentConsultDebounceTimer);
|
|
917
|
+
agentConsultDebounceTimer = void 0;
|
|
918
|
+
}
|
|
475
919
|
try {
|
|
476
920
|
bridge?.close();
|
|
477
921
|
} catch (error) {
|
|
@@ -490,6 +934,10 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
490
934
|
if (proc !== outputProcess) return;
|
|
491
935
|
fail("audio output command")(error);
|
|
492
936
|
});
|
|
937
|
+
proc.stdin?.on?.("error", (error) => {
|
|
938
|
+
if (proc !== outputProcess) return;
|
|
939
|
+
fail("audio output command")(error);
|
|
940
|
+
});
|
|
493
941
|
proc.on("exit", (code, signal) => {
|
|
494
942
|
if (proc !== outputProcess) return;
|
|
495
943
|
if (!stopped) {
|
|
@@ -513,6 +961,13 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
513
961
|
params.logger.debug?.(`[google-meet] cleared realtime audio output buffer by restarting playback command`);
|
|
514
962
|
terminateProcess(previousOutput, "SIGKILL");
|
|
515
963
|
};
|
|
964
|
+
const writeOutputAudio = (audio) => {
|
|
965
|
+
try {
|
|
966
|
+
outputProcess.stdin?.write(audio);
|
|
967
|
+
} catch (error) {
|
|
968
|
+
fail("audio output command")(error);
|
|
969
|
+
}
|
|
970
|
+
};
|
|
516
971
|
const startHumanBargeInMonitor = () => {
|
|
517
972
|
const commandArgv = params.config.chrome.bargeInInputCommand;
|
|
518
973
|
if (!commandArgv) return;
|
|
@@ -563,17 +1018,82 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
563
1018
|
fullConfig: params.fullConfig,
|
|
564
1019
|
providers: params.providers
|
|
565
1020
|
});
|
|
1021
|
+
const strategy = params.config.realtime.strategy;
|
|
1022
|
+
params.logger.info(formatGoogleMeetRealtimeVoiceModelLog({
|
|
1023
|
+
strategy,
|
|
1024
|
+
provider: resolved.provider,
|
|
1025
|
+
providerConfig: resolved.providerConfig,
|
|
1026
|
+
fallbackModel: params.config.realtime.model,
|
|
1027
|
+
audioFormat: params.config.chrome.audioFormat
|
|
1028
|
+
}));
|
|
566
1029
|
const transcript = [];
|
|
567
1030
|
const realtimeEvents = [];
|
|
1031
|
+
let agentConsultActive = false;
|
|
1032
|
+
let pendingAgentQuestion;
|
|
1033
|
+
const enqueueAgentConsultForUserTranscript = (question) => {
|
|
1034
|
+
const trimmed = question.trim();
|
|
1035
|
+
if (!trimmed || stopped) return;
|
|
1036
|
+
pendingAgentQuestion = pendingAgentQuestion ? `${pendingAgentQuestion}\n${trimmed}` : trimmed;
|
|
1037
|
+
if (agentConsultDebounceTimer) clearTimeout(agentConsultDebounceTimer);
|
|
1038
|
+
agentConsultDebounceTimer = setTimeout(() => {
|
|
1039
|
+
agentConsultDebounceTimer = void 0;
|
|
1040
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1041
|
+
pendingAgentQuestion = void 0;
|
|
1042
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1043
|
+
}, 900);
|
|
1044
|
+
agentConsultDebounceTimer.unref?.();
|
|
1045
|
+
};
|
|
1046
|
+
const runAgentConsultForUserTranscript = async (question) => {
|
|
1047
|
+
const trimmed = question.trim();
|
|
1048
|
+
if (!trimmed || stopped) return;
|
|
1049
|
+
if (agentConsultActive) {
|
|
1050
|
+
pendingAgentQuestion = trimmed;
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
agentConsultActive = true;
|
|
1054
|
+
let nextQuestion = trimmed;
|
|
1055
|
+
try {
|
|
1056
|
+
while (nextQuestion) {
|
|
1057
|
+
if (stopped) return;
|
|
1058
|
+
const currentQuestion = nextQuestion;
|
|
1059
|
+
pendingAgentQuestion = void 0;
|
|
1060
|
+
params.logger.info(`[google-meet] realtime agent consult: ${currentQuestion}`);
|
|
1061
|
+
const result = await consultOpenClawAgentForGoogleMeet({
|
|
1062
|
+
config: params.config,
|
|
1063
|
+
fullConfig: params.fullConfig,
|
|
1064
|
+
runtime: params.runtime,
|
|
1065
|
+
logger: params.logger,
|
|
1066
|
+
meetingSessionId: params.meetingSessionId,
|
|
1067
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
1068
|
+
args: {
|
|
1069
|
+
question: currentQuestion,
|
|
1070
|
+
responseStyle: "Brief, natural spoken answer for a live meeting."
|
|
1071
|
+
},
|
|
1072
|
+
transcript
|
|
1073
|
+
});
|
|
1074
|
+
if (!stopped && result.text.trim()) bridge?.sendUserMessage(buildGoogleMeetSpeakExactUserMessage(result.text.trim()));
|
|
1075
|
+
nextQuestion = pendingAgentQuestion;
|
|
1076
|
+
}
|
|
1077
|
+
} catch (error) {
|
|
1078
|
+
params.logger.warn(`[google-meet] realtime agent consult failed: ${formatErrorMessage(error)}`);
|
|
1079
|
+
if (!stopped) bridge?.sendUserMessage(buildGoogleMeetSpeakExactUserMessage("I hit an error while checking that. Please try again."));
|
|
1080
|
+
} finally {
|
|
1081
|
+
agentConsultActive = false;
|
|
1082
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1083
|
+
pendingAgentQuestion = void 0;
|
|
1084
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1085
|
+
}
|
|
1086
|
+
};
|
|
568
1087
|
bridge = createRealtimeVoiceBridgeSession({
|
|
569
1088
|
provider: resolved.provider,
|
|
570
1089
|
providerConfig: resolved.providerConfig,
|
|
571
1090
|
audioFormat: resolveGoogleMeetRealtimeAudioFormat(params.config),
|
|
572
1091
|
instructions: params.config.realtime.instructions,
|
|
573
1092
|
initialGreetingInstructions: params.config.realtime.introMessage,
|
|
1093
|
+
autoRespondToAudio: strategy === "bidi",
|
|
574
1094
|
triggerGreetingOnReady: false,
|
|
575
1095
|
markStrategy: "ack-immediately",
|
|
576
|
-
tools: resolveGoogleMeetRealtimeTools(params.config.realtime.toolPolicy),
|
|
1096
|
+
tools: strategy === "bidi" ? resolveGoogleMeetRealtimeTools(params.config.realtime.toolPolicy) : [],
|
|
577
1097
|
audioSink: {
|
|
578
1098
|
isOpen: () => !stopped,
|
|
579
1099
|
sendAudio: (audio) => {
|
|
@@ -581,7 +1101,7 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
581
1101
|
lastOutputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
582
1102
|
lastOutputBytes += audio.byteLength;
|
|
583
1103
|
suppressInputForOutput(audio);
|
|
584
|
-
|
|
1104
|
+
writeOutputAudio(audio);
|
|
585
1105
|
},
|
|
586
1106
|
clearAudio: clearOutputPlayback
|
|
587
1107
|
},
|
|
@@ -589,16 +1109,30 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
589
1109
|
if (isFinal) {
|
|
590
1110
|
recordGoogleMeetRealtimeTranscript(transcript, role, text);
|
|
591
1111
|
params.logger.info(`[google-meet] realtime ${role}: ${text}`);
|
|
1112
|
+
if (role === "user" && strategy === "agent") {
|
|
1113
|
+
if (isGoogleMeetLikelyAssistantEchoTranscript({
|
|
1114
|
+
transcript,
|
|
1115
|
+
text
|
|
1116
|
+
})) {
|
|
1117
|
+
params.logger.info(`[google-meet] realtime ignored assistant echo transcript: ${text}`);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
enqueueAgentConsultForUserTranscript(text);
|
|
1121
|
+
}
|
|
592
1122
|
}
|
|
593
1123
|
},
|
|
594
1124
|
onEvent: (event) => {
|
|
595
1125
|
recordGoogleMeetRealtimeEvent(realtimeEvents, event);
|
|
596
|
-
if (event.type === "error" || event.type === "response.done") {
|
|
1126
|
+
if (event.type === "error" || event.type === "response.done" || event.type === "input_audio_buffer.speech_started" || event.type === "input_audio_buffer.speech_stopped" || event.type === "conversation.item.input_audio_transcription.completed" || event.type === "conversation.item.input_audio_transcription.failed") {
|
|
597
1127
|
const detail = event.detail ? ` ${event.detail}` : "";
|
|
598
1128
|
params.logger.info(`[google-meet] realtime ${event.direction}:${event.type}${detail}`);
|
|
599
1129
|
}
|
|
600
1130
|
},
|
|
601
1131
|
onToolCall: (event, session) => {
|
|
1132
|
+
if (strategy !== "bidi") {
|
|
1133
|
+
session.submitToolResult(event.callId || event.itemId, { error: `Tool "${event.name}" is only available in bidi realtime strategy` });
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
602
1136
|
if (event.name !== GOOGLE_MEET_AGENT_CONSULT_TOOL_NAME) {
|
|
603
1137
|
session.submitToolResult(event.callId || event.itemId, { error: `Tool "${event.name}" not available` });
|
|
604
1138
|
return;
|
|
@@ -610,6 +1144,7 @@ async function startCommandRealtimeAudioBridge(params) {
|
|
|
610
1144
|
runtime: params.runtime,
|
|
611
1145
|
logger: params.logger,
|
|
612
1146
|
meetingSessionId: params.meetingSessionId,
|
|
1147
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
613
1148
|
args: event.args,
|
|
614
1149
|
transcript
|
|
615
1150
|
}).then((result) => {
|
|
@@ -677,6 +1212,252 @@ function asRecord$2(value) {
|
|
|
677
1212
|
function readString$1(value) {
|
|
678
1213
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
679
1214
|
}
|
|
1215
|
+
function normalizeGoogleMeetTtsPromptText(text) {
|
|
1216
|
+
const trimmed = text?.trim();
|
|
1217
|
+
if (!trimmed) return;
|
|
1218
|
+
const sayExactly = trimmed.match(/^say exactly:\s*(?<text>.+)$/is)?.groups?.text?.trim();
|
|
1219
|
+
if (sayExactly) return sayExactly.replace(/^["']|["']$/g, "").trim() || trimmed;
|
|
1220
|
+
return trimmed;
|
|
1221
|
+
}
|
|
1222
|
+
async function startNodeAgentAudioBridge(params) {
|
|
1223
|
+
let stopped = false;
|
|
1224
|
+
let sttSession = null;
|
|
1225
|
+
let realtimeReady = false;
|
|
1226
|
+
let lastInputAt;
|
|
1227
|
+
let lastOutputAt;
|
|
1228
|
+
let lastInputBytes = 0;
|
|
1229
|
+
let lastOutputBytes = 0;
|
|
1230
|
+
let suppressedInputBytes = 0;
|
|
1231
|
+
let lastSuppressedInputAt;
|
|
1232
|
+
let suppressInputUntil = 0;
|
|
1233
|
+
let lastOutputPlayableUntilMs = 0;
|
|
1234
|
+
let consecutiveInputErrors = 0;
|
|
1235
|
+
let lastInputError;
|
|
1236
|
+
const resolved = resolveGoogleMeetRealtimeTranscriptionProvider({
|
|
1237
|
+
config: params.config,
|
|
1238
|
+
fullConfig: params.fullConfig,
|
|
1239
|
+
providers: params.providers
|
|
1240
|
+
});
|
|
1241
|
+
params.logger.info(formatGoogleMeetAgentAudioModelLog({
|
|
1242
|
+
provider: resolved.provider,
|
|
1243
|
+
providerConfig: resolved.providerConfig,
|
|
1244
|
+
audioFormat: params.config.chrome.audioFormat
|
|
1245
|
+
}));
|
|
1246
|
+
const transcript = [];
|
|
1247
|
+
let agentConsultActive = false;
|
|
1248
|
+
let pendingAgentQuestion;
|
|
1249
|
+
let agentConsultDebounceTimer;
|
|
1250
|
+
let ttsQueue = Promise.resolve();
|
|
1251
|
+
const stop = async () => {
|
|
1252
|
+
if (stopped) return;
|
|
1253
|
+
stopped = true;
|
|
1254
|
+
if (agentConsultDebounceTimer) {
|
|
1255
|
+
clearTimeout(agentConsultDebounceTimer);
|
|
1256
|
+
agentConsultDebounceTimer = void 0;
|
|
1257
|
+
}
|
|
1258
|
+
try {
|
|
1259
|
+
sttSession?.close();
|
|
1260
|
+
} catch (error) {
|
|
1261
|
+
params.logger.debug?.(`[google-meet] node agent transcription bridge close ignored: ${formatErrorMessage(error)}`);
|
|
1262
|
+
}
|
|
1263
|
+
try {
|
|
1264
|
+
await params.runtime.nodes.invoke({
|
|
1265
|
+
nodeId: params.nodeId,
|
|
1266
|
+
command: "googlemeet.chrome",
|
|
1267
|
+
params: {
|
|
1268
|
+
action: "stop",
|
|
1269
|
+
bridgeId: params.bridgeId
|
|
1270
|
+
},
|
|
1271
|
+
timeoutMs: 5e3
|
|
1272
|
+
});
|
|
1273
|
+
} catch (error) {
|
|
1274
|
+
params.logger.debug?.(`[google-meet] node audio bridge stop ignored: ${formatErrorMessage(error)}`);
|
|
1275
|
+
}
|
|
1276
|
+
};
|
|
1277
|
+
const pushOutputAudio = async (audio) => {
|
|
1278
|
+
const suppression = extendGoogleMeetOutputEchoSuppression({
|
|
1279
|
+
audio,
|
|
1280
|
+
audioFormat: params.config.chrome.audioFormat,
|
|
1281
|
+
nowMs: Date.now(),
|
|
1282
|
+
lastOutputPlayableUntilMs,
|
|
1283
|
+
suppressInputUntilMs: suppressInputUntil
|
|
1284
|
+
});
|
|
1285
|
+
suppressInputUntil = suppression.suppressInputUntilMs;
|
|
1286
|
+
lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
|
|
1287
|
+
lastOutputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1288
|
+
lastOutputBytes += audio.byteLength;
|
|
1289
|
+
await params.runtime.nodes.invoke({
|
|
1290
|
+
nodeId: params.nodeId,
|
|
1291
|
+
command: "googlemeet.chrome",
|
|
1292
|
+
params: {
|
|
1293
|
+
action: "pushAudio",
|
|
1294
|
+
bridgeId: params.bridgeId,
|
|
1295
|
+
base64: Buffer.from(audio).toString("base64")
|
|
1296
|
+
},
|
|
1297
|
+
timeoutMs: 5e3
|
|
1298
|
+
});
|
|
1299
|
+
};
|
|
1300
|
+
const enqueueSpeakText = (text) => {
|
|
1301
|
+
const normalized = normalizeGoogleMeetTtsPromptText(text);
|
|
1302
|
+
if (!normalized || stopped) return;
|
|
1303
|
+
ttsQueue = ttsQueue.then(async () => {
|
|
1304
|
+
if (stopped) return;
|
|
1305
|
+
recordGoogleMeetRealtimeTranscript(transcript, "assistant", normalized);
|
|
1306
|
+
params.logger.info(`[google-meet] node agent assistant: ${normalized}`);
|
|
1307
|
+
const result = await params.runtime.tts.textToSpeechTelephony({
|
|
1308
|
+
text: normalized,
|
|
1309
|
+
cfg: params.fullConfig
|
|
1310
|
+
});
|
|
1311
|
+
if (!result.success || !result.audioBuffer || !result.sampleRate) throw new Error(result.error ?? "TTS conversion failed");
|
|
1312
|
+
params.logger.info(formatGoogleMeetAgentTtsResultLog("node agent", result));
|
|
1313
|
+
await pushOutputAudio(convertGoogleMeetTtsAudioForBridge(result.audioBuffer, result.sampleRate, params.config, result.outputFormat));
|
|
1314
|
+
}).catch((error) => {
|
|
1315
|
+
params.logger.warn(`[google-meet] node agent TTS failed: ${formatErrorMessage(error)}`);
|
|
1316
|
+
});
|
|
1317
|
+
};
|
|
1318
|
+
const runAgentConsultForUserTranscript = async (question) => {
|
|
1319
|
+
const trimmed = question.trim();
|
|
1320
|
+
if (!trimmed || stopped) return;
|
|
1321
|
+
if (agentConsultActive) {
|
|
1322
|
+
pendingAgentQuestion = trimmed;
|
|
1323
|
+
return;
|
|
1324
|
+
}
|
|
1325
|
+
agentConsultActive = true;
|
|
1326
|
+
let nextQuestion = trimmed;
|
|
1327
|
+
try {
|
|
1328
|
+
while (nextQuestion) {
|
|
1329
|
+
if (stopped) return;
|
|
1330
|
+
const currentQuestion = nextQuestion;
|
|
1331
|
+
pendingAgentQuestion = void 0;
|
|
1332
|
+
params.logger.info(`[google-meet] node agent consult: ${currentQuestion}`);
|
|
1333
|
+
enqueueSpeakText((await consultOpenClawAgentForGoogleMeet({
|
|
1334
|
+
config: params.config,
|
|
1335
|
+
fullConfig: params.fullConfig,
|
|
1336
|
+
runtime: params.runtime,
|
|
1337
|
+
logger: params.logger,
|
|
1338
|
+
meetingSessionId: params.meetingSessionId,
|
|
1339
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
1340
|
+
args: {
|
|
1341
|
+
question: currentQuestion,
|
|
1342
|
+
responseStyle: "Brief, natural spoken answer for a live meeting."
|
|
1343
|
+
},
|
|
1344
|
+
transcript
|
|
1345
|
+
})).text);
|
|
1346
|
+
nextQuestion = pendingAgentQuestion;
|
|
1347
|
+
}
|
|
1348
|
+
} catch (error) {
|
|
1349
|
+
params.logger.warn(`[google-meet] node agent consult failed: ${formatErrorMessage(error)}`);
|
|
1350
|
+
enqueueSpeakText("I hit an error while checking that. Please try again.");
|
|
1351
|
+
} finally {
|
|
1352
|
+
agentConsultActive = false;
|
|
1353
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1354
|
+
pendingAgentQuestion = void 0;
|
|
1355
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1356
|
+
}
|
|
1357
|
+
};
|
|
1358
|
+
const enqueueAgentConsultForUserTranscript = (question) => {
|
|
1359
|
+
const trimmed = question.trim();
|
|
1360
|
+
if (!trimmed || stopped) return;
|
|
1361
|
+
pendingAgentQuestion = pendingAgentQuestion ? `${pendingAgentQuestion}\n${trimmed}` : trimmed;
|
|
1362
|
+
if (agentConsultDebounceTimer) clearTimeout(agentConsultDebounceTimer);
|
|
1363
|
+
agentConsultDebounceTimer = setTimeout(() => {
|
|
1364
|
+
agentConsultDebounceTimer = void 0;
|
|
1365
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1366
|
+
pendingAgentQuestion = void 0;
|
|
1367
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1368
|
+
}, 900);
|
|
1369
|
+
agentConsultDebounceTimer.unref?.();
|
|
1370
|
+
};
|
|
1371
|
+
sttSession = resolved.provider.createSession({
|
|
1372
|
+
providerConfig: resolved.providerConfig,
|
|
1373
|
+
onTranscript: (text) => {
|
|
1374
|
+
const trimmed = text.trim();
|
|
1375
|
+
if (!trimmed || stopped) return;
|
|
1376
|
+
recordGoogleMeetRealtimeTranscript(transcript, "user", trimmed);
|
|
1377
|
+
params.logger.info(`[google-meet] node agent user: ${trimmed}`);
|
|
1378
|
+
if (isGoogleMeetLikelyAssistantEchoTranscript({
|
|
1379
|
+
transcript,
|
|
1380
|
+
text: trimmed
|
|
1381
|
+
})) {
|
|
1382
|
+
params.logger.info(`[google-meet] node agent ignored assistant echo transcript: ${trimmed}`);
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
enqueueAgentConsultForUserTranscript(trimmed);
|
|
1386
|
+
},
|
|
1387
|
+
onError: (error) => {
|
|
1388
|
+
params.logger.warn(`[google-meet] node agent transcription bridge failed: ${formatErrorMessage(error)}`);
|
|
1389
|
+
stop();
|
|
1390
|
+
}
|
|
1391
|
+
});
|
|
1392
|
+
await sttSession.connect();
|
|
1393
|
+
realtimeReady = true;
|
|
1394
|
+
(async () => {
|
|
1395
|
+
for (;;) {
|
|
1396
|
+
if (stopped) break;
|
|
1397
|
+
try {
|
|
1398
|
+
const raw = await params.runtime.nodes.invoke({
|
|
1399
|
+
nodeId: params.nodeId,
|
|
1400
|
+
command: "googlemeet.chrome",
|
|
1401
|
+
params: {
|
|
1402
|
+
action: "pullAudio",
|
|
1403
|
+
bridgeId: params.bridgeId,
|
|
1404
|
+
timeoutMs: 250
|
|
1405
|
+
},
|
|
1406
|
+
timeoutMs: 2e3
|
|
1407
|
+
});
|
|
1408
|
+
const result = asRecord$2(asRecord$2(raw).payload ?? raw);
|
|
1409
|
+
consecutiveInputErrors = 0;
|
|
1410
|
+
lastInputError = void 0;
|
|
1411
|
+
const base64 = readString$1(result.base64);
|
|
1412
|
+
if (base64) {
|
|
1413
|
+
const audio = Buffer.from(base64, "base64");
|
|
1414
|
+
if (Date.now() < suppressInputUntil) {
|
|
1415
|
+
lastSuppressedInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1416
|
+
suppressedInputBytes += audio.byteLength;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
lastInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1420
|
+
lastInputBytes += audio.byteLength;
|
|
1421
|
+
sttSession?.sendAudio(convertGoogleMeetBridgeAudioForStt(audio, params.config));
|
|
1422
|
+
}
|
|
1423
|
+
if (result.closed === true) await stop();
|
|
1424
|
+
} catch (error) {
|
|
1425
|
+
if (!stopped) {
|
|
1426
|
+
const message = formatErrorMessage(error);
|
|
1427
|
+
consecutiveInputErrors += 1;
|
|
1428
|
+
lastInputError = message;
|
|
1429
|
+
params.logger.warn(`[google-meet] node agent audio input failed (${consecutiveInputErrors}/5): ${message}`);
|
|
1430
|
+
if (consecutiveInputErrors >= 5 || /unknown bridgeId|bridge is not open/i.test(message)) await stop();
|
|
1431
|
+
else await new Promise((resolve) => setTimeout(resolve, 250));
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
})();
|
|
1436
|
+
return {
|
|
1437
|
+
type: "node-command-pair",
|
|
1438
|
+
providerId: resolved.provider.id,
|
|
1439
|
+
nodeId: params.nodeId,
|
|
1440
|
+
bridgeId: params.bridgeId,
|
|
1441
|
+
speak: enqueueSpeakText,
|
|
1442
|
+
getHealth: () => ({
|
|
1443
|
+
providerConnected: sttSession?.isConnected() ?? false,
|
|
1444
|
+
realtimeReady,
|
|
1445
|
+
audioInputActive: lastInputBytes > 0,
|
|
1446
|
+
audioOutputActive: lastOutputBytes > 0,
|
|
1447
|
+
lastInputAt,
|
|
1448
|
+
lastOutputAt,
|
|
1449
|
+
lastSuppressedInputAt,
|
|
1450
|
+
lastInputBytes,
|
|
1451
|
+
lastOutputBytes,
|
|
1452
|
+
suppressedInputBytes,
|
|
1453
|
+
...getGoogleMeetRealtimeTranscriptHealth(transcript),
|
|
1454
|
+
consecutiveInputErrors,
|
|
1455
|
+
lastInputError,
|
|
1456
|
+
bridgeClosed: stopped
|
|
1457
|
+
}),
|
|
1458
|
+
stop
|
|
1459
|
+
};
|
|
1460
|
+
}
|
|
680
1461
|
async function startNodeRealtimeAudioBridge(params) {
|
|
681
1462
|
let stopped = false;
|
|
682
1463
|
let bridge = null;
|
|
@@ -686,6 +1467,10 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
686
1467
|
let lastClearAt;
|
|
687
1468
|
let lastInputBytes = 0;
|
|
688
1469
|
let lastOutputBytes = 0;
|
|
1470
|
+
let suppressedInputBytes = 0;
|
|
1471
|
+
let lastSuppressedInputAt;
|
|
1472
|
+
let suppressInputUntil = 0;
|
|
1473
|
+
let lastOutputPlayableUntilMs = 0;
|
|
689
1474
|
let consecutiveInputErrors = 0;
|
|
690
1475
|
let lastInputError;
|
|
691
1476
|
let clearCount = 0;
|
|
@@ -696,9 +1481,78 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
696
1481
|
});
|
|
697
1482
|
const transcript = [];
|
|
698
1483
|
const realtimeEvents = [];
|
|
1484
|
+
const strategy = params.config.realtime.strategy;
|
|
1485
|
+
params.logger.info(formatGoogleMeetRealtimeVoiceModelLog({
|
|
1486
|
+
strategy,
|
|
1487
|
+
provider: resolved.provider,
|
|
1488
|
+
providerConfig: resolved.providerConfig,
|
|
1489
|
+
fallbackModel: params.config.realtime.model,
|
|
1490
|
+
audioFormat: params.config.chrome.audioFormat
|
|
1491
|
+
}));
|
|
1492
|
+
let agentConsultActive = false;
|
|
1493
|
+
let pendingAgentQuestion;
|
|
1494
|
+
let agentConsultDebounceTimer;
|
|
1495
|
+
const enqueueAgentConsultForUserTranscript = (question) => {
|
|
1496
|
+
const trimmed = question.trim();
|
|
1497
|
+
if (!trimmed || stopped) return;
|
|
1498
|
+
pendingAgentQuestion = pendingAgentQuestion ? `${pendingAgentQuestion}\n${trimmed}` : trimmed;
|
|
1499
|
+
if (agentConsultDebounceTimer) clearTimeout(agentConsultDebounceTimer);
|
|
1500
|
+
agentConsultDebounceTimer = setTimeout(() => {
|
|
1501
|
+
agentConsultDebounceTimer = void 0;
|
|
1502
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1503
|
+
pendingAgentQuestion = void 0;
|
|
1504
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1505
|
+
}, 900);
|
|
1506
|
+
agentConsultDebounceTimer.unref?.();
|
|
1507
|
+
};
|
|
1508
|
+
const runAgentConsultForUserTranscript = async (question) => {
|
|
1509
|
+
const trimmed = question.trim();
|
|
1510
|
+
if (!trimmed || stopped) return;
|
|
1511
|
+
if (agentConsultActive) {
|
|
1512
|
+
pendingAgentQuestion = trimmed;
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
agentConsultActive = true;
|
|
1516
|
+
let nextQuestion = trimmed;
|
|
1517
|
+
try {
|
|
1518
|
+
while (nextQuestion) {
|
|
1519
|
+
if (stopped) return;
|
|
1520
|
+
const currentQuestion = nextQuestion;
|
|
1521
|
+
pendingAgentQuestion = void 0;
|
|
1522
|
+
params.logger.info(`[google-meet] node realtime agent consult: ${currentQuestion}`);
|
|
1523
|
+
const result = await consultOpenClawAgentForGoogleMeet({
|
|
1524
|
+
config: params.config,
|
|
1525
|
+
fullConfig: params.fullConfig,
|
|
1526
|
+
runtime: params.runtime,
|
|
1527
|
+
logger: params.logger,
|
|
1528
|
+
meetingSessionId: params.meetingSessionId,
|
|
1529
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
1530
|
+
args: {
|
|
1531
|
+
question: currentQuestion,
|
|
1532
|
+
responseStyle: "Brief, natural spoken answer for a live meeting."
|
|
1533
|
+
},
|
|
1534
|
+
transcript
|
|
1535
|
+
});
|
|
1536
|
+
if (!stopped && result.text.trim()) bridge?.sendUserMessage(buildGoogleMeetSpeakExactUserMessage(result.text.trim()));
|
|
1537
|
+
nextQuestion = pendingAgentQuestion;
|
|
1538
|
+
}
|
|
1539
|
+
} catch (error) {
|
|
1540
|
+
params.logger.warn(`[google-meet] node realtime agent consult failed: ${formatErrorMessage(error)}`);
|
|
1541
|
+
if (!stopped) bridge?.sendUserMessage(buildGoogleMeetSpeakExactUserMessage("I hit an error while checking that. Please try again."));
|
|
1542
|
+
} finally {
|
|
1543
|
+
agentConsultActive = false;
|
|
1544
|
+
const queuedQuestion = pendingAgentQuestion;
|
|
1545
|
+
pendingAgentQuestion = void 0;
|
|
1546
|
+
if (queuedQuestion && !stopped) runAgentConsultForUserTranscript(queuedQuestion);
|
|
1547
|
+
}
|
|
1548
|
+
};
|
|
699
1549
|
const stop = async () => {
|
|
700
1550
|
if (stopped) return;
|
|
701
1551
|
stopped = true;
|
|
1552
|
+
if (agentConsultDebounceTimer) {
|
|
1553
|
+
clearTimeout(agentConsultDebounceTimer);
|
|
1554
|
+
agentConsultDebounceTimer = void 0;
|
|
1555
|
+
}
|
|
702
1556
|
try {
|
|
703
1557
|
bridge?.close();
|
|
704
1558
|
} catch (error) {
|
|
@@ -724,12 +1578,22 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
724
1578
|
audioFormat: resolveGoogleMeetRealtimeAudioFormat(params.config),
|
|
725
1579
|
instructions: params.config.realtime.instructions,
|
|
726
1580
|
initialGreetingInstructions: params.config.realtime.introMessage,
|
|
1581
|
+
autoRespondToAudio: strategy === "bidi",
|
|
727
1582
|
triggerGreetingOnReady: false,
|
|
728
1583
|
markStrategy: "ack-immediately",
|
|
729
|
-
tools: resolveGoogleMeetRealtimeTools(params.config.realtime.toolPolicy),
|
|
1584
|
+
tools: strategy === "bidi" ? resolveGoogleMeetRealtimeTools(params.config.realtime.toolPolicy) : [],
|
|
730
1585
|
audioSink: {
|
|
731
1586
|
isOpen: () => !stopped,
|
|
732
1587
|
sendAudio: (audio) => {
|
|
1588
|
+
const suppression = extendGoogleMeetOutputEchoSuppression({
|
|
1589
|
+
audio,
|
|
1590
|
+
audioFormat: params.config.chrome.audioFormat,
|
|
1591
|
+
nowMs: Date.now(),
|
|
1592
|
+
lastOutputPlayableUntilMs,
|
|
1593
|
+
suppressInputUntilMs: suppressInputUntil
|
|
1594
|
+
});
|
|
1595
|
+
suppressInputUntil = suppression.suppressInputUntilMs;
|
|
1596
|
+
lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
|
|
733
1597
|
lastOutputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
734
1598
|
lastOutputBytes += audio.byteLength;
|
|
735
1599
|
params.runtime.nodes.invoke({
|
|
@@ -749,6 +1613,8 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
749
1613
|
clearAudio: () => {
|
|
750
1614
|
lastClearAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
751
1615
|
clearCount += 1;
|
|
1616
|
+
suppressInputUntil = 0;
|
|
1617
|
+
lastOutputPlayableUntilMs = 0;
|
|
752
1618
|
params.runtime.nodes.invoke({
|
|
753
1619
|
nodeId: params.nodeId,
|
|
754
1620
|
command: "googlemeet.chrome",
|
|
@@ -767,16 +1633,30 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
767
1633
|
if (isFinal) {
|
|
768
1634
|
recordGoogleMeetRealtimeTranscript(transcript, role, text);
|
|
769
1635
|
params.logger.info(`[google-meet] node realtime ${role}: ${text}`);
|
|
1636
|
+
if (role === "user" && strategy === "agent") {
|
|
1637
|
+
if (isGoogleMeetLikelyAssistantEchoTranscript({
|
|
1638
|
+
transcript,
|
|
1639
|
+
text
|
|
1640
|
+
})) {
|
|
1641
|
+
params.logger.info(`[google-meet] node realtime ignored assistant echo transcript: ${text}`);
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
enqueueAgentConsultForUserTranscript(text);
|
|
1645
|
+
}
|
|
770
1646
|
}
|
|
771
1647
|
},
|
|
772
1648
|
onEvent: (event) => {
|
|
773
1649
|
recordGoogleMeetRealtimeEvent(realtimeEvents, event);
|
|
774
|
-
if (event.type === "error" || event.type === "response.done") {
|
|
1650
|
+
if (event.type === "error" || event.type === "response.done" || event.type === "input_audio_buffer.speech_started" || event.type === "input_audio_buffer.speech_stopped" || event.type === "conversation.item.input_audio_transcription.completed" || event.type === "conversation.item.input_audio_transcription.failed") {
|
|
775
1651
|
const detail = event.detail ? ` ${event.detail}` : "";
|
|
776
1652
|
params.logger.info(`[google-meet] node realtime ${event.direction}:${event.type}${detail}`);
|
|
777
1653
|
}
|
|
778
1654
|
},
|
|
779
1655
|
onToolCall: (event, session) => {
|
|
1656
|
+
if (strategy !== "bidi") {
|
|
1657
|
+
session.submitToolResult(event.callId || event.itemId, { error: `Tool "${event.name}" is only available in bidi realtime strategy` });
|
|
1658
|
+
return;
|
|
1659
|
+
}
|
|
780
1660
|
if (event.name !== GOOGLE_MEET_AGENT_CONSULT_TOOL_NAME) {
|
|
781
1661
|
session.submitToolResult(event.callId || event.itemId, { error: `Tool "${event.name}" not available` });
|
|
782
1662
|
return;
|
|
@@ -788,6 +1668,7 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
788
1668
|
runtime: params.runtime,
|
|
789
1669
|
logger: params.logger,
|
|
790
1670
|
meetingSessionId: params.meetingSessionId,
|
|
1671
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
791
1672
|
args: event.args,
|
|
792
1673
|
transcript
|
|
793
1674
|
}).then((result) => {
|
|
@@ -829,6 +1710,11 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
829
1710
|
const base64 = readString$1(result.base64);
|
|
830
1711
|
if (base64) {
|
|
831
1712
|
const audio = Buffer.from(base64, "base64");
|
|
1713
|
+
if (Date.now() < suppressInputUntil) {
|
|
1714
|
+
lastSuppressedInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1715
|
+
suppressedInputBytes += audio.byteLength;
|
|
1716
|
+
continue;
|
|
1717
|
+
}
|
|
832
1718
|
lastInputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
833
1719
|
lastInputBytes += audio.byteLength;
|
|
834
1720
|
bridge?.sendAudio(audio);
|
|
@@ -861,9 +1747,11 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
861
1747
|
audioOutputActive: lastOutputBytes > 0,
|
|
862
1748
|
lastInputAt,
|
|
863
1749
|
lastOutputAt,
|
|
1750
|
+
lastSuppressedInputAt,
|
|
864
1751
|
lastClearAt,
|
|
865
1752
|
lastInputBytes,
|
|
866
1753
|
lastOutputBytes,
|
|
1754
|
+
suppressedInputBytes,
|
|
867
1755
|
...getGoogleMeetRealtimeTranscriptHealth(transcript),
|
|
868
1756
|
...getGoogleMeetRealtimeEventHealth(realtimeEvents),
|
|
869
1757
|
consecutiveInputErrors,
|
|
@@ -878,6 +1766,9 @@ async function startNodeRealtimeAudioBridge(params) {
|
|
|
878
1766
|
//#region extensions/google-meet/src/transports/chrome.ts
|
|
879
1767
|
const GOOGLE_MEET_SYSTEM_PROFILER_COMMAND = "/usr/sbin/system_profiler";
|
|
880
1768
|
const chromeTransportDeps = { callGatewayFromCli };
|
|
1769
|
+
function isGoogleMeetTalkBackMode$2(mode) {
|
|
1770
|
+
return mode === "agent" || mode === "bidi";
|
|
1771
|
+
}
|
|
881
1772
|
function outputMentionsBlackHole2ch(output) {
|
|
882
1773
|
return /\bBlackHole\s+2ch\b/i.test(output);
|
|
883
1774
|
}
|
|
@@ -899,7 +1790,7 @@ async function assertBlackHole2chAvailable(params) {
|
|
|
899
1790
|
}
|
|
900
1791
|
async function launchChromeMeet(params) {
|
|
901
1792
|
const checkRealtimeAudioPrerequisites = async () => {
|
|
902
|
-
if (params.mode
|
|
1793
|
+
if (!isGoogleMeetTalkBackMode$2(params.mode)) return;
|
|
903
1794
|
await assertBlackHole2chAvailable({
|
|
904
1795
|
runtime: params.runtime,
|
|
905
1796
|
timeoutMs: Math.min(params.config.chrome.joinTimeoutMs, 1e4)
|
|
@@ -910,20 +1801,37 @@ async function launchChromeMeet(params) {
|
|
|
910
1801
|
}
|
|
911
1802
|
};
|
|
912
1803
|
const startRealtimeAudioBridge = async () => {
|
|
913
|
-
if (params.mode
|
|
1804
|
+
if (!isGoogleMeetTalkBackMode$2(params.mode)) return;
|
|
914
1805
|
if (params.config.chrome.audioBridgeCommand) {
|
|
1806
|
+
if (params.mode === "agent") throw new Error("Chrome agent mode requires chrome.audioInputCommand and chrome.audioOutputCommand so OpenClaw can run STT and regular TTS directly.");
|
|
915
1807
|
const bridge = await params.runtime.system.runCommandWithTimeout(params.config.chrome.audioBridgeCommand, { timeoutMs: params.config.chrome.joinTimeoutMs });
|
|
916
1808
|
if (bridge.code !== 0) throw new Error(`failed to start Chrome audio bridge: ${bridge.stderr || bridge.stdout || bridge.code}`);
|
|
917
1809
|
return { type: "external-command" };
|
|
918
1810
|
}
|
|
919
|
-
if (!params.config.chrome.audioInputCommand || !params.config.chrome.audioOutputCommand) throw new Error("Chrome
|
|
1811
|
+
if (!params.config.chrome.audioInputCommand || !params.config.chrome.audioOutputCommand) throw new Error("Chrome talk-back mode requires chrome.audioInputCommand and chrome.audioOutputCommand, or chrome.audioBridgeCommand for an external bridge.");
|
|
920
1812
|
return {
|
|
921
1813
|
type: "command-pair",
|
|
922
|
-
...await
|
|
1814
|
+
...params.mode === "agent" ? await startCommandAgentAudioBridge({
|
|
923
1815
|
config: params.config,
|
|
924
1816
|
fullConfig: params.fullConfig,
|
|
925
1817
|
runtime: params.runtime,
|
|
926
1818
|
meetingSessionId: params.meetingSessionId,
|
|
1819
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
1820
|
+
inputCommand: params.config.chrome.audioInputCommand,
|
|
1821
|
+
outputCommand: params.config.chrome.audioOutputCommand,
|
|
1822
|
+
logger: params.logger
|
|
1823
|
+
}) : await startCommandRealtimeAudioBridge({
|
|
1824
|
+
config: {
|
|
1825
|
+
...params.config,
|
|
1826
|
+
realtime: {
|
|
1827
|
+
...params.config.realtime,
|
|
1828
|
+
strategy: "bidi"
|
|
1829
|
+
}
|
|
1830
|
+
},
|
|
1831
|
+
fullConfig: params.fullConfig,
|
|
1832
|
+
runtime: params.runtime,
|
|
1833
|
+
meetingSessionId: params.meetingSessionId,
|
|
1834
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
927
1835
|
inputCommand: params.config.chrome.audioInputCommand,
|
|
928
1836
|
outputCommand: params.config.chrome.audioOutputCommand,
|
|
929
1837
|
logger: params.logger
|
|
@@ -941,7 +1849,7 @@ async function launchChromeMeet(params) {
|
|
|
941
1849
|
mode: params.mode,
|
|
942
1850
|
url: params.url
|
|
943
1851
|
});
|
|
944
|
-
const audioBridge = params.mode
|
|
1852
|
+
const audioBridge = isGoogleMeetTalkBackMode$2(params.mode) && result.browser?.inCall === true && result.browser.micMuted !== true && result.browser.manualActionRequired !== true ? await startRealtimeAudioBridge() : void 0;
|
|
945
1853
|
return {
|
|
946
1854
|
...result,
|
|
947
1855
|
audioBridge
|
|
@@ -968,6 +1876,9 @@ function parseMeetBrowserStatus(result) {
|
|
|
968
1876
|
lastCaptionSpeaker: parsed.lastCaptionSpeaker,
|
|
969
1877
|
lastCaptionText: parsed.lastCaptionText,
|
|
970
1878
|
recentTranscript: parsed.recentTranscript,
|
|
1879
|
+
audioOutputRouted: parsed.audioOutputRouted,
|
|
1880
|
+
audioOutputDeviceLabel: parsed.audioOutputDeviceLabel,
|
|
1881
|
+
audioOutputRouteError: parsed.audioOutputRouteError,
|
|
971
1882
|
manualActionRequired: parsed.manualActionRequired,
|
|
972
1883
|
manualActionReason: parsed.manualActionReason,
|
|
973
1884
|
manualActionMessage: parsed.manualActionMessage,
|
|
@@ -1022,7 +1933,7 @@ async function grantMeetMediaPermissions(params) {
|
|
|
1022
1933
|
}
|
|
1023
1934
|
}
|
|
1024
1935
|
function meetStatusScript(params) {
|
|
1025
|
-
return `() => {
|
|
1936
|
+
return `async () => {
|
|
1026
1937
|
const text = (node) => (node?.innerText || node?.textContent || "").trim();
|
|
1027
1938
|
const allowMicrophone = ${JSON.stringify(params.allowMicrophone)};
|
|
1028
1939
|
const captureCaptions = ${JSON.stringify(params.captureCaptions)};
|
|
@@ -1038,6 +1949,9 @@ function meetStatusScript(params) {
|
|
|
1038
1949
|
.join(" ");
|
|
1039
1950
|
const buttonLabels = buttons.map(buttonLabel).filter(Boolean);
|
|
1040
1951
|
const notes = [];
|
|
1952
|
+
let audioOutputRouted;
|
|
1953
|
+
let audioOutputDeviceLabel;
|
|
1954
|
+
let audioOutputRouteError;
|
|
1041
1955
|
const findButton = (pattern) =>
|
|
1042
1956
|
buttons.find((button) => {
|
|
1043
1957
|
const label = buttonLabel(button);
|
|
@@ -1071,7 +1985,7 @@ function meetStatusScript(params) {
|
|
|
1071
1985
|
}
|
|
1072
1986
|
if (!readOnly && allowMicrophone && mic && /turn on microphone/i.test(buttonLabel(mic))) {
|
|
1073
1987
|
mic.click();
|
|
1074
|
-
notes.push("Attempted to turn on the Meet microphone for
|
|
1988
|
+
notes.push("Attempted to turn on the Meet microphone for talk-back mode.");
|
|
1075
1989
|
}
|
|
1076
1990
|
if (!readOnly && !allowMicrophone && mic && /turn off microphone/i.test(mic.getAttribute('aria-label') || text(mic))) {
|
|
1077
1991
|
mic.click();
|
|
@@ -1091,6 +2005,55 @@ function meetStatusScript(params) {
|
|
|
1091
2005
|
notes.push("Skipped Meet microphone prompt for observe-only mode.");
|
|
1092
2006
|
}
|
|
1093
2007
|
const inCall = buttons.some((button) => /leave call/i.test(button.getAttribute('aria-label') || text(button)));
|
|
2008
|
+
const routeMeetAudioOutput = async () => {
|
|
2009
|
+
if (
|
|
2010
|
+
!allowMicrophone ||
|
|
2011
|
+
typeof navigator === 'undefined' ||
|
|
2012
|
+
!navigator.mediaDevices?.enumerateDevices
|
|
2013
|
+
) return;
|
|
2014
|
+
const mediaElements = [...document.querySelectorAll('audio, video')]
|
|
2015
|
+
.filter((el) => typeof el.setSinkId === 'function');
|
|
2016
|
+
if (mediaElements.length === 0) return;
|
|
2017
|
+
try {
|
|
2018
|
+
const devices = await navigator.mediaDevices.enumerateDevices();
|
|
2019
|
+
const output = devices.find((device) =>
|
|
2020
|
+
device.kind === 'audiooutput' && /\\bBlackHole\\s+2ch\\b/i.test(device.label || '')
|
|
2021
|
+
) || devices.find((device) =>
|
|
2022
|
+
device.kind === 'audiooutput' && /\\bBlackHole\\b/i.test(device.label || '')
|
|
2023
|
+
);
|
|
2024
|
+
if (!output?.deviceId) {
|
|
2025
|
+
if (devices.some((device) => device.kind === 'audiooutput')) {
|
|
2026
|
+
notes.push("BlackHole 2ch speaker output was not visible to Meet.");
|
|
2027
|
+
}
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
let routed = 0;
|
|
2031
|
+
for (const element of mediaElements) {
|
|
2032
|
+
if (element.sinkId !== output.deviceId) {
|
|
2033
|
+
if (readOnly) {
|
|
2034
|
+
continue;
|
|
2035
|
+
}
|
|
2036
|
+
await element.setSinkId(output.deviceId);
|
|
2037
|
+
routed += 1;
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
audioOutputRouted = mediaElements.some((element) => element.sinkId === output.deviceId);
|
|
2041
|
+
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
|
|
2042
|
+
if (!readOnly && audioOutputRouted) {
|
|
2043
|
+
notes.push(
|
|
2044
|
+
routed > 0
|
|
2045
|
+
? \`Routed Meet media output to \${audioOutputDeviceLabel}.\`
|
|
2046
|
+
: \`Meet media output already routed to \${audioOutputDeviceLabel}.\`
|
|
2047
|
+
);
|
|
2048
|
+
}
|
|
2049
|
+
} catch (error) {
|
|
2050
|
+
audioOutputRouteError = error?.message || String(error);
|
|
2051
|
+
notes.push(\`Could not route Meet speaker output to BlackHole 2ch: \${audioOutputRouteError}\`);
|
|
2052
|
+
}
|
|
2053
|
+
};
|
|
2054
|
+
if (inCall) {
|
|
2055
|
+
await routeMeetAudioOutput();
|
|
2056
|
+
}
|
|
1094
2057
|
let captioning = false;
|
|
1095
2058
|
let captionsEnabledAttempted = false;
|
|
1096
2059
|
let transcriptLines = 0;
|
|
@@ -1213,6 +2176,9 @@ function meetStatusScript(params) {
|
|
|
1213
2176
|
lastCaptionSpeaker,
|
|
1214
2177
|
lastCaptionText,
|
|
1215
2178
|
recentTranscript,
|
|
2179
|
+
audioOutputRouted,
|
|
2180
|
+
audioOutputDeviceLabel,
|
|
2181
|
+
audioOutputRouteError,
|
|
1216
2182
|
manualActionRequired: Boolean(manualActionReason),
|
|
1217
2183
|
manualActionReason,
|
|
1218
2184
|
manualActionMessage,
|
|
@@ -1275,7 +2241,7 @@ async function openMeetWithBrowserRequest(params) {
|
|
|
1275
2241
|
}
|
|
1276
2242
|
};
|
|
1277
2243
|
const permissionNotes = await grantMeetMediaPermissions({
|
|
1278
|
-
allowMicrophone: params.mode
|
|
2244
|
+
allowMicrophone: isGoogleMeetTalkBackMode$2(params.mode),
|
|
1279
2245
|
callBrowser: params.callBrowser,
|
|
1280
2246
|
targetId,
|
|
1281
2247
|
timeoutMs
|
|
@@ -1296,7 +2262,7 @@ async function openMeetWithBrowserRequest(params) {
|
|
|
1296
2262
|
kind: "evaluate",
|
|
1297
2263
|
targetId,
|
|
1298
2264
|
fn: meetStatusScript({
|
|
1299
|
-
allowMicrophone: params.mode
|
|
2265
|
+
allowMicrophone: isGoogleMeetTalkBackMode$2(params.mode),
|
|
1300
2266
|
captureCaptions: params.mode === "transcribe",
|
|
1301
2267
|
guestName: params.config.chrome.guestName,
|
|
1302
2268
|
autoJoin: params.config.chrome.autoJoin
|
|
@@ -1304,7 +2270,7 @@ async function openMeetWithBrowserRequest(params) {
|
|
|
1304
2270
|
},
|
|
1305
2271
|
timeoutMs: Math.min(timeoutMs, 1e4)
|
|
1306
2272
|
})) ?? browser, permissionNotes);
|
|
1307
|
-
if (browser?.inCall === true && (params.mode
|
|
2273
|
+
if (browser?.inCall === true && (!isGoogleMeetTalkBackMode$2(params.mode) || browser.micMuted !== true)) return {
|
|
1308
2274
|
launched: true,
|
|
1309
2275
|
browser
|
|
1310
2276
|
};
|
|
@@ -1492,11 +2458,18 @@ async function launchChromeMeetOnNode(params) {
|
|
|
1492
2458
|
}));
|
|
1493
2459
|
if (result.audioBridge?.type === "node-command-pair") {
|
|
1494
2460
|
if (!result.bridgeId) throw new Error("Google Meet node did not return an audio bridge id.");
|
|
1495
|
-
const bridge = await startNodeRealtimeAudioBridge({
|
|
1496
|
-
config: params.config
|
|
2461
|
+
const bridge = await (params.mode === "agent" ? startNodeAgentAudioBridge : startNodeRealtimeAudioBridge)({
|
|
2462
|
+
config: params.mode === "agent" ? params.config : {
|
|
2463
|
+
...params.config,
|
|
2464
|
+
realtime: {
|
|
2465
|
+
...params.config.realtime,
|
|
2466
|
+
strategy: "bidi"
|
|
2467
|
+
}
|
|
2468
|
+
},
|
|
1497
2469
|
fullConfig: params.fullConfig,
|
|
1498
2470
|
runtime: params.runtime,
|
|
1499
2471
|
meetingSessionId: params.meetingSessionId,
|
|
2472
|
+
requesterSessionKey: params.requesterSessionKey,
|
|
1500
2473
|
nodeId,
|
|
1501
2474
|
bridgeId: result.bridgeId,
|
|
1502
2475
|
logger: params.logger
|
|
@@ -1583,6 +2556,9 @@ function attachOutputProcessHandlers(session, outputProcess) {
|
|
|
1583
2556
|
outputProcess.on("error", () => {
|
|
1584
2557
|
if (session.output === outputProcess) stopSession(session);
|
|
1585
2558
|
});
|
|
2559
|
+
outputProcess.stdin?.on?.("error", () => {
|
|
2560
|
+
if (session.output === outputProcess) stopSession(session);
|
|
2561
|
+
});
|
|
1586
2562
|
}
|
|
1587
2563
|
function startOutputProcess(command) {
|
|
1588
2564
|
return spawn(command.command, command.args, { stdio: [
|
|
@@ -1670,7 +2646,12 @@ function pushAudio(params) {
|
|
|
1670
2646
|
const audio = Buffer.from(base64, "base64");
|
|
1671
2647
|
session.lastOutputAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1672
2648
|
session.lastOutputBytes += audio.byteLength;
|
|
1673
|
-
|
|
2649
|
+
try {
|
|
2650
|
+
session.output?.stdin?.write(audio);
|
|
2651
|
+
} catch {
|
|
2652
|
+
stopSession(session);
|
|
2653
|
+
throw new Error(`bridge is not open: ${bridgeId}`);
|
|
2654
|
+
}
|
|
1674
2655
|
return {
|
|
1675
2656
|
bridgeId,
|
|
1676
2657
|
ok: true
|
|
@@ -1701,7 +2682,7 @@ function startChrome(params) {
|
|
|
1701
2682
|
const mode = readString(params.mode);
|
|
1702
2683
|
let bridgeId;
|
|
1703
2684
|
let audioBridge;
|
|
1704
|
-
if (mode === "realtime") {
|
|
2685
|
+
if (mode === "agent" || mode === "bidi" || mode === "realtime") {
|
|
1705
2686
|
assertBlackHoleAvailable(Math.min(timeoutMs, 1e4));
|
|
1706
2687
|
const healthCommand = readStringArray(params.audioBridgeHealthCommand);
|
|
1707
2688
|
if (healthCommand) {
|
|
@@ -1710,6 +2691,7 @@ function startChrome(params) {
|
|
|
1710
2691
|
}
|
|
1711
2692
|
const bridgeCommand = readStringArray(params.audioBridgeCommand);
|
|
1712
2693
|
if (bridgeCommand) {
|
|
2694
|
+
if (mode === "agent") throw new Error("Chrome agent mode requires audioInputCommand and audioOutputCommand so OpenClaw can run STT and regular TTS directly.");
|
|
1713
2695
|
const bridge = runCommandWithTimeout(bridgeCommand, timeoutMs);
|
|
1714
2696
|
if (bridge.code !== 0) throw new Error(`failed to start Chrome audio bridge: ${bridge.stderr || bridge.stdout || bridge.code}`);
|
|
1715
2697
|
audioBridge = { type: "external-command" };
|
|
@@ -1925,7 +2907,7 @@ function getGoogleMeetSetupStatus(config, options) {
|
|
|
1925
2907
|
const fullConfig = asRecord(options?.fullConfig);
|
|
1926
2908
|
const mode = options?.mode ?? config.defaultMode;
|
|
1927
2909
|
const transport = options?.transport ?? config.defaultTransport;
|
|
1928
|
-
const needsChromeRealtimeAudio = mode === "
|
|
2910
|
+
const needsChromeRealtimeAudio = (mode === "agent" || mode === "bidi") && (transport === "chrome" || transport === "chrome-node");
|
|
1929
2911
|
const pluginEntries = asRecord(asRecord(fullConfig.plugins).entries);
|
|
1930
2912
|
const pluginAllow = asRecord(fullConfig.plugins).allow;
|
|
1931
2913
|
const voiceCallEntry = asRecord(pluginEntries["voice-call"]);
|
|
@@ -1948,12 +2930,16 @@ function getGoogleMeetSetupStatus(config, options) {
|
|
|
1948
2930
|
ok: true,
|
|
1949
2931
|
message: config.chrome.browserProfile ? "Local Chrome uses the OpenClaw browser profile; chrome.browserProfile is passed to chrome-node hosts" : "Local Chrome uses the OpenClaw browser profile; configure browser.defaultProfile to choose another profile"
|
|
1950
2932
|
});
|
|
1951
|
-
if (needsChromeRealtimeAudio)
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
2933
|
+
if (needsChromeRealtimeAudio) {
|
|
2934
|
+
const hasCommandPair = Boolean(config.chrome.audioInputCommand && config.chrome.audioOutputCommand);
|
|
2935
|
+
const hasExternalBridge = Boolean(config.chrome.audioBridgeCommand);
|
|
2936
|
+
const agentModeExternalBridgeInvalid = mode === "agent" && hasExternalBridge;
|
|
2937
|
+
checks.push({
|
|
2938
|
+
id: "audio-bridge",
|
|
2939
|
+
ok: mode === "agent" ? hasCommandPair && !agentModeExternalBridgeInvalid : hasExternalBridge || hasCommandPair,
|
|
2940
|
+
message: agentModeExternalBridgeInvalid ? "Chrome agent mode requires chrome.audioInputCommand and chrome.audioOutputCommand; chrome.audioBridgeCommand is bidi-only" : hasExternalBridge ? "Chrome audio bridge command configured" : hasCommandPair ? `Chrome command-pair talk-back audio bridge configured (${config.chrome.audioFormat})` : "Chrome talk-back audio bridge not configured"
|
|
2941
|
+
});
|
|
2942
|
+
} else if (transport === "chrome" || transport === "chrome-node") checks.push({
|
|
1957
2943
|
id: "audio-bridge",
|
|
1958
2944
|
ok: true,
|
|
1959
2945
|
message: "Chrome observe-only mode does not require a realtime audio bridge"
|
|
@@ -1985,7 +2971,7 @@ function getGoogleMeetSetupStatus(config, options) {
|
|
|
1985
2971
|
}
|
|
1986
2972
|
if (config.voiceCall.enabled && (transport === "twilio" || Boolean(config.twilio.defaultDialInNumber) || Object.hasOwn(pluginEntries, "voice-call"))) {
|
|
1987
2973
|
const voiceCallAllowed = !Array.isArray(pluginAllow) || pluginAllow.includes("voice-call");
|
|
1988
|
-
const voiceCallEnabled = voiceCallEntry.enabled !== false;
|
|
2974
|
+
const voiceCallEnabled = Object.hasOwn(pluginEntries, "voice-call") && voiceCallEntry.enabled !== false;
|
|
1989
2975
|
checks.push({
|
|
1990
2976
|
id: "twilio-voice-call-plugin",
|
|
1991
2977
|
ok: voiceCallAllowed && voiceCallEnabled,
|
|
@@ -2127,11 +3113,14 @@ async function joinMeetViaVoiceCallGateway(params) {
|
|
|
2127
3113
|
}
|
|
2128
3114
|
const spoken = await client.request("voicecall.speak", {
|
|
2129
3115
|
callId: start.callId,
|
|
3116
|
+
allowTwimlFallback: false,
|
|
2130
3117
|
message: params.message
|
|
2131
3118
|
}, { timeoutMs: params.config.voiceCall.requestTimeoutMs });
|
|
2132
|
-
if (spoken.success === false)
|
|
2133
|
-
|
|
2134
|
-
|
|
3119
|
+
if (spoken.success === false) params.logger?.warn?.(`[google-meet] Skipped intro speech because realtime bridge was not ready: ${spoken.error || "voicecall.speak failed"}`);
|
|
3120
|
+
else {
|
|
3121
|
+
introSent = true;
|
|
3122
|
+
params.logger?.info(`[google-meet] Intro speech requested after Meet dial sequence: callId=${start.callId}`);
|
|
3123
|
+
}
|
|
2135
3124
|
}
|
|
2136
3125
|
return {
|
|
2137
3126
|
callId: start.callId,
|
|
@@ -2186,7 +3175,10 @@ function resolveTransport(input, config) {
|
|
|
2186
3175
|
return input ?? config.defaultTransport;
|
|
2187
3176
|
}
|
|
2188
3177
|
function resolveMode(input, config) {
|
|
2189
|
-
return input ?? config.defaultMode;
|
|
3178
|
+
return input === "realtime" ? "agent" : input ?? config.defaultMode;
|
|
3179
|
+
}
|
|
3180
|
+
function isGoogleMeetTalkBackMode$1(mode) {
|
|
3181
|
+
return mode === "agent" || mode === "bidi";
|
|
2190
3182
|
}
|
|
2191
3183
|
function hasRealtimeAudioOutputAdvanced(health, startOutputBytes) {
|
|
2192
3184
|
return (health?.lastOutputBytes ?? 0) > startOutputBytes;
|
|
@@ -2215,7 +3207,7 @@ function isManagedChromeBrowserSession(session) {
|
|
|
2215
3207
|
return Boolean((session.transport === "chrome" || session.transport === "chrome-node") && session.chrome && session.chrome.launched);
|
|
2216
3208
|
}
|
|
2217
3209
|
function evaluateSpeechReadiness(session) {
|
|
2218
|
-
if (session.mode
|
|
3210
|
+
if (!isGoogleMeetTalkBackMode$1(session.mode) || !session.chrome) return { ready: true };
|
|
2219
3211
|
if (!isManagedChromeBrowserSession(session)) {
|
|
2220
3212
|
if (session.chrome.audioBridge) return { ready: true };
|
|
2221
3213
|
return {
|
|
@@ -2329,7 +3321,7 @@ var GoogleMeetRuntime = class {
|
|
|
2329
3321
|
message: formatErrorMessage(error)
|
|
2330
3322
|
});
|
|
2331
3323
|
}
|
|
2332
|
-
if (transport === "chrome" && mode
|
|
3324
|
+
if (transport === "chrome" && isGoogleMeetTalkBackMode$1(mode)) {
|
|
2333
3325
|
try {
|
|
2334
3326
|
await assertBlackHole2chAvailable({
|
|
2335
3327
|
runtime: this.params.runtime,
|
|
@@ -2357,7 +3349,7 @@ var GoogleMeetRuntime = class {
|
|
|
2357
3349
|
status = addGoogleMeetSetupCheck(status, {
|
|
2358
3350
|
id: "chrome-local-audio-commands",
|
|
2359
3351
|
ok: commands.length > 0 && missingCommands.length === 0,
|
|
2360
|
-
message: commands.length === 0 ? "Chrome
|
|
3352
|
+
message: commands.length === 0 ? "Chrome talk-back audio commands are not configured" : missingCommands.length === 0 ? `Chrome audio command${commands.length === 1 ? "" : "s"} available: ${commands.join(", ")}` : `Chrome audio command${missingCommands.length === 1 ? "" : "s"} missing: ${missingCommands.join(", ")}`
|
|
2361
3353
|
});
|
|
2362
3354
|
}
|
|
2363
3355
|
return status;
|
|
@@ -2394,7 +3386,7 @@ var GoogleMeetRuntime = class {
|
|
|
2394
3386
|
reusable.updatedAt = nowIso();
|
|
2395
3387
|
return {
|
|
2396
3388
|
session: reusable,
|
|
2397
|
-
spoken: mode
|
|
3389
|
+
spoken: isGoogleMeetTalkBackMode$1(mode) && speechInstructions ? await this.#speakWhenReady(reusable, speechInstructions) : false
|
|
2398
3390
|
};
|
|
2399
3391
|
}
|
|
2400
3392
|
const createdAt = nowIso();
|
|
@@ -2409,9 +3401,11 @@ var GoogleMeetRuntime = class {
|
|
|
2409
3401
|
updatedAt: createdAt,
|
|
2410
3402
|
participantIdentity: transport === "twilio" ? "Twilio phone participant" : transport === "chrome-node" ? "signed-in Google Chrome profile on a paired node" : "signed-in Google Chrome profile",
|
|
2411
3403
|
realtime: {
|
|
2412
|
-
enabled: mode
|
|
2413
|
-
|
|
2414
|
-
|
|
3404
|
+
enabled: isGoogleMeetTalkBackMode$1(mode),
|
|
3405
|
+
strategy: mode === "bidi" ? "bidi" : "agent",
|
|
3406
|
+
provider: mode === "bidi" ? this.params.config.realtime.voiceProvider ?? this.params.config.realtime.provider : void 0,
|
|
3407
|
+
model: mode === "bidi" ? this.params.config.realtime.model : void 0,
|
|
3408
|
+
transcriptionProvider: mode === "agent" ? this.params.config.realtime.transcriptionProvider ?? this.params.config.realtime.provider : void 0,
|
|
2415
3409
|
toolPolicy: this.params.config.realtime.toolPolicy
|
|
2416
3410
|
},
|
|
2417
3411
|
notes: []
|
|
@@ -2423,6 +3417,7 @@ var GoogleMeetRuntime = class {
|
|
|
2423
3417
|
config: this.params.config,
|
|
2424
3418
|
fullConfig: this.params.fullConfig,
|
|
2425
3419
|
meetingSessionId: session.id,
|
|
3420
|
+
requesterSessionKey: request.requesterSessionKey,
|
|
2426
3421
|
mode,
|
|
2427
3422
|
url,
|
|
2428
3423
|
logger: this.params.logger
|
|
@@ -2431,6 +3426,7 @@ var GoogleMeetRuntime = class {
|
|
|
2431
3426
|
config: this.params.config,
|
|
2432
3427
|
fullConfig: this.params.fullConfig,
|
|
2433
3428
|
meetingSessionId: session.id,
|
|
3429
|
+
requesterSessionKey: request.requesterSessionKey,
|
|
2434
3430
|
mode,
|
|
2435
3431
|
url,
|
|
2436
3432
|
logger: this.params.logger
|
|
@@ -2443,7 +3439,7 @@ var GoogleMeetRuntime = class {
|
|
|
2443
3439
|
health: "browser" in result ? result.browser : void 0
|
|
2444
3440
|
};
|
|
2445
3441
|
this.#attachChromeAudioBridge(session, result.audioBridge);
|
|
2446
|
-
session.notes.push(result.audioBridge ? transport === "chrome-node" ? "Chrome node transport joins as the signed-in Google profile on the selected node and routes realtime audio through the node bridge." : "Chrome transport joins as the signed-in Google profile and routes realtime audio through the configured bridge." : mode
|
|
3442
|
+
session.notes.push(result.audioBridge ? transport === "chrome-node" ? "Chrome node transport joins as the signed-in Google profile on the selected node and routes realtime audio through the node bridge." : "Chrome transport joins as the signed-in Google profile and routes realtime audio through the configured bridge." : isGoogleMeetTalkBackMode$1(mode) ? "Chrome transport joins as the signed-in Google profile and expects BlackHole 2ch audio routing." : "Chrome transport joins as the signed-in Google profile without starting the realtime audio bridge.");
|
|
2447
3443
|
this.#refreshSpeechReadiness(session);
|
|
2448
3444
|
} else {
|
|
2449
3445
|
const dialInNumber = normalizeDialInNumber(request.dialInNumber ?? this.params.config.twilio.defaultDialInNumber);
|
|
@@ -2457,7 +3453,7 @@ var GoogleMeetRuntime = class {
|
|
|
2457
3453
|
dialInNumber,
|
|
2458
3454
|
dtmfSequence,
|
|
2459
3455
|
logger: this.params.logger,
|
|
2460
|
-
message: mode
|
|
3456
|
+
message: isGoogleMeetTalkBackMode$1(mode) ? request.message ?? this.params.config.voiceCall.introMessage ?? this.params.config.realtime.introMessage : void 0
|
|
2461
3457
|
}) : void 0;
|
|
2462
3458
|
delegatedTwilioSpoken = Boolean(voiceCallResult?.introSent);
|
|
2463
3459
|
session.twilio = {
|
|
@@ -2483,7 +3479,7 @@ var GoogleMeetRuntime = class {
|
|
|
2483
3479
|
this.#sessions.set(session.id, session);
|
|
2484
3480
|
return {
|
|
2485
3481
|
session,
|
|
2486
|
-
spoken: transport === "twilio" ? delegatedTwilioSpoken : mode
|
|
3482
|
+
spoken: transport === "twilio" ? delegatedTwilioSpoken : isGoogleMeetTalkBackMode$1(mode) && speechInstructions ? await this.#speakWhenReady(session, speechInstructions) : false
|
|
2487
3483
|
};
|
|
2488
3484
|
}
|
|
2489
3485
|
async leave(sessionId) {
|
|
@@ -2563,22 +3559,22 @@ var GoogleMeetRuntime = class {
|
|
|
2563
3559
|
const health = result.session?.chrome?.health;
|
|
2564
3560
|
if (health?.manualActionRequired || result.session?.state !== "active") return false;
|
|
2565
3561
|
const blocked = health?.speechBlockedReason;
|
|
2566
|
-
if (blocked && blocked !== "not-in-call" && blocked !== "browser-unverified") return false;
|
|
3562
|
+
if (blocked && blocked !== "not-in-call" && blocked !== "browser-unverified" && blocked !== "meet-microphone-muted") return false;
|
|
2567
3563
|
}
|
|
2568
3564
|
return false;
|
|
2569
3565
|
}
|
|
2570
3566
|
async testSpeech(request) {
|
|
2571
|
-
if (request.mode === "transcribe") throw new Error("test_speech requires mode:
|
|
3567
|
+
if (request.mode === "transcribe") throw new Error("test_speech requires mode: agent or bidi; use join mode: transcribe for observe-only sessions.");
|
|
2572
3568
|
const url = normalizeMeetUrl(request.url);
|
|
2573
3569
|
const transport = resolveTransport(request.transport, this.params.config);
|
|
2574
3570
|
const beforeSessions = this.list();
|
|
2575
3571
|
const before = new Set(beforeSessions.map((session) => session.id));
|
|
2576
|
-
const startOutputBytes = beforeSessions.find((session) => session.state === "active" && isSameMeetUrlForReuse(session.url, url) && session.transport === transport && session.mode
|
|
3572
|
+
const startOutputBytes = beforeSessions.find((session) => session.state === "active" && isSameMeetUrlForReuse(session.url, url) && session.transport === transport && isGoogleMeetTalkBackMode$1(session.mode))?.chrome?.health?.lastOutputBytes ?? 0;
|
|
2577
3573
|
const result = await this.join({
|
|
2578
3574
|
...request,
|
|
2579
3575
|
transport,
|
|
2580
3576
|
url,
|
|
2581
|
-
mode: "
|
|
3577
|
+
mode: "agent",
|
|
2582
3578
|
message: request.message ?? "Say exactly: Google Meet speech test complete."
|
|
2583
3579
|
});
|
|
2584
3580
|
let health = result.session.chrome?.health;
|
|
@@ -2611,7 +3607,8 @@ var GoogleMeetRuntime = class {
|
|
|
2611
3607
|
};
|
|
2612
3608
|
}
|
|
2613
3609
|
async testListen(request) {
|
|
2614
|
-
|
|
3610
|
+
const requestedMode = request.mode ? resolveMode(request.mode, this.params.config) : void 0;
|
|
3611
|
+
if (requestedMode && isGoogleMeetTalkBackMode$1(requestedMode)) throw new Error("test_listen requires mode: transcribe; use test_speech for talk-back sessions.");
|
|
2615
3612
|
const url = normalizeMeetUrl(request.url);
|
|
2616
3613
|
const transport = resolveTransport(request.transport, this.params.config);
|
|
2617
3614
|
if (transport === "twilio") throw new Error("test_listen supports chrome or chrome-node transports");
|
|
@@ -2682,7 +3679,7 @@ var GoogleMeetRuntime = class {
|
|
|
2682
3679
|
this.#refreshSpeechReadiness(session);
|
|
2683
3680
|
return;
|
|
2684
3681
|
}
|
|
2685
|
-
if (!options.force && session.mode
|
|
3682
|
+
if (!options.force && isGoogleMeetTalkBackMode$1(session.mode) && evaluateSpeechReadiness(session).ready) {
|
|
2686
3683
|
this.#refreshSpeechReadiness(session);
|
|
2687
3684
|
return;
|
|
2688
3685
|
}
|
|
@@ -2724,7 +3721,7 @@ var GoogleMeetRuntime = class {
|
|
|
2724
3721
|
}
|
|
2725
3722
|
}
|
|
2726
3723
|
async #ensureChromeRealtimeBridge(session) {
|
|
2727
|
-
if (session.mode
|
|
3724
|
+
if (!isGoogleMeetTalkBackMode$1(session.mode) || session.transport !== "chrome" || session.state !== "active" || !session.chrome || session.chrome.audioBridge) return;
|
|
2728
3725
|
const health = session.chrome.health;
|
|
2729
3726
|
if (health?.inCall !== true || health.micMuted === true || health.manualActionRequired === true) return;
|
|
2730
3727
|
const result = await launchChromeMeet({
|
|
@@ -2792,7 +3789,7 @@ const googleMeetConfigSchema = {
|
|
|
2792
3789
|
},
|
|
2793
3790
|
defaultMode: {
|
|
2794
3791
|
label: "Default Mode",
|
|
2795
|
-
help: "
|
|
3792
|
+
help: "Agent uses realtime transcription plus regular OpenClaw TTS. Bidi uses the realtime voice model directly. Transcribe observes only."
|
|
2796
3793
|
},
|
|
2797
3794
|
"chrome.audioBackend": {
|
|
2798
3795
|
label: "Chrome Audio Backend",
|
|
@@ -2825,6 +3822,11 @@ const googleMeetConfigSchema = {
|
|
|
2825
3822
|
help: "Command-pair audio format. PCM16 24 kHz is the default Chrome/Meet path; G.711 mu-law 8 kHz remains available for legacy command pairs.",
|
|
2826
3823
|
advanced: true
|
|
2827
3824
|
},
|
|
3825
|
+
"chrome.audioBufferBytes": {
|
|
3826
|
+
label: "Audio Buffer Bytes",
|
|
3827
|
+
help: "SoX processing buffer for generated Chrome command-pair audio commands. Lower values reduce latency but may underrun on busy hosts.",
|
|
3828
|
+
advanced: true
|
|
3829
|
+
},
|
|
2828
3830
|
"chrome.audioInputCommand": {
|
|
2829
3831
|
label: "Audio Input Command",
|
|
2830
3832
|
help: "Command that writes meeting audio to stdout in chrome.audioFormat.",
|
|
@@ -2908,12 +3910,25 @@ const googleMeetConfigSchema = {
|
|
|
2908
3910
|
label: "Voice Call Intro Message",
|
|
2909
3911
|
advanced: true
|
|
2910
3912
|
},
|
|
3913
|
+
"realtime.strategy": {
|
|
3914
|
+
label: "Realtime Strategy",
|
|
3915
|
+
help: "Legacy realtime alias setting. Use mode=agent or mode=bidi for new Meet joins."
|
|
3916
|
+
},
|
|
2911
3917
|
"realtime.provider": {
|
|
2912
|
-
label: "
|
|
2913
|
-
help: "
|
|
3918
|
+
label: "Speech Provider",
|
|
3919
|
+
help: "Compatibility fallback for both realtime transcription and bidi voice. Prefer realtime.transcriptionProvider and realtime.voiceProvider for new configs."
|
|
3920
|
+
},
|
|
3921
|
+
"realtime.transcriptionProvider": {
|
|
3922
|
+
label: "Realtime Transcription Provider",
|
|
3923
|
+
help: "Agent mode uses this provider to transcribe meeting audio before regular OpenClaw TTS answers."
|
|
3924
|
+
},
|
|
3925
|
+
"realtime.voiceProvider": {
|
|
3926
|
+
label: "Bidi Voice Provider",
|
|
3927
|
+
help: "Bidi mode uses this realtime voice provider. Falls back to realtime.provider when unset."
|
|
2914
3928
|
},
|
|
2915
3929
|
"realtime.model": {
|
|
2916
|
-
label: "Realtime Model",
|
|
3930
|
+
label: "Bidi Realtime Model",
|
|
3931
|
+
help: "Only used by mode=bidi. Agent mode answers with the configured OpenClaw agent and regular TTS.",
|
|
2917
3932
|
advanced: true
|
|
2918
3933
|
},
|
|
2919
3934
|
"realtime.instructions": {
|
|
@@ -3001,8 +4016,12 @@ const GoogleMeetToolSchema = Type.Object({
|
|
|
3001
4016
|
description: "Join transport"
|
|
3002
4017
|
})),
|
|
3003
4018
|
mode: Type.Optional(Type.String({
|
|
3004
|
-
enum: [
|
|
3005
|
-
|
|
4019
|
+
enum: [
|
|
4020
|
+
"agent",
|
|
4021
|
+
"bidi",
|
|
4022
|
+
"transcribe"
|
|
4023
|
+
],
|
|
4024
|
+
description: "Join mode. agent uses realtime transcription, the configured OpenClaw agent, and regular TTS. bidi uses the realtime voice model directly. transcribe joins observe-only."
|
|
3006
4025
|
})),
|
|
3007
4026
|
dialInNumber: Type.Optional(Type.String({ description: "Meet dial-in phone number for Twilio. Required for Twilio unless twilio.defaultDialInNumber is configured; Meet URLs cannot be dialed directly." })),
|
|
3008
4027
|
pin: Type.Optional(Type.String({ description: "Meet phone PIN for Twilio; # is appended if omitted" })),
|
|
@@ -3047,7 +4066,11 @@ function normalizeTransport(value) {
|
|
|
3047
4066
|
return value === "chrome" || value === "chrome-node" || value === "twilio" ? value : void 0;
|
|
3048
4067
|
}
|
|
3049
4068
|
function normalizeMode(value) {
|
|
3050
|
-
|
|
4069
|
+
if (value === "realtime") return "agent";
|
|
4070
|
+
return value === "agent" || value === "bidi" || value === "transcribe" ? value : void 0;
|
|
4071
|
+
}
|
|
4072
|
+
function isGoogleMeetTalkBackMode(mode) {
|
|
4073
|
+
return mode === "agent" || mode === "bidi";
|
|
3051
4074
|
}
|
|
3052
4075
|
function resolveMeetingInput(config, value) {
|
|
3053
4076
|
const meeting = normalizeOptionalString(value) ?? config.defaults.meeting;
|
|
@@ -3091,12 +4114,12 @@ function isGoogleMeetAgentToolActionUnsupportedOnHost(params) {
|
|
|
3091
4114
|
const action = params.raw.action;
|
|
3092
4115
|
if (action !== "join" && action !== "test_speech" && !(action === "create" && shouldJoinCreatedMeet(params.raw))) return false;
|
|
3093
4116
|
const transport = normalizeTransport(params.raw.transport) ?? params.config.defaultTransport;
|
|
3094
|
-
const mode = action === "test_speech" ? "
|
|
3095
|
-
return transport === "chrome" && mode
|
|
4117
|
+
const mode = action === "test_speech" ? "agent" : normalizeMode(params.raw.mode) ?? params.config.defaultMode;
|
|
4118
|
+
return transport === "chrome" && isGoogleMeetTalkBackMode(mode);
|
|
3096
4119
|
}
|
|
3097
4120
|
function assertGoogleMeetAgentToolActionSupported(params) {
|
|
3098
4121
|
if (!isGoogleMeetAgentToolActionUnsupportedOnHost(params)) return;
|
|
3099
|
-
throw new Error("Google Meet local Chrome
|
|
4122
|
+
throw new Error("Google Meet local Chrome talk-back audio is macOS-only. On this host, use mode: transcribe, transport: twilio, or transport: chrome-node backed by a macOS node.");
|
|
3100
4123
|
}
|
|
3101
4124
|
function resolveGoogleMeetToolGatewayTimeoutMs(config) {
|
|
3102
4125
|
return Math.max(6e4, config.chrome.joinTimeoutMs + 3e4, config.voiceCall.requestTimeoutMs + 1e4);
|
|
@@ -3118,10 +4141,10 @@ async function callGoogleMeetGatewayFromTool(params) {
|
|
|
3118
4141
|
}
|
|
3119
4142
|
}
|
|
3120
4143
|
async function createMeetFromParams(params) {
|
|
3121
|
-
return (await import("./create-
|
|
4144
|
+
return (await import("./create-mHH3keLH.js")).createMeetFromParams(params);
|
|
3122
4145
|
}
|
|
3123
4146
|
async function createAndJoinMeetFromParams(params) {
|
|
3124
|
-
return (await import("./create-
|
|
4147
|
+
return (await import("./create-mHH3keLH.js")).createAndJoinMeetFromParams(params);
|
|
3125
4148
|
}
|
|
3126
4149
|
async function resolveGoogleMeetTokenFromParams(config, raw) {
|
|
3127
4150
|
const { resolveGoogleMeetAccessToken } = await import("./oauth-BJwzuzT-.js");
|
|
@@ -3213,7 +4236,7 @@ async function exportGoogleMeetBundleFromParams(config, raw) {
|
|
|
3213
4236
|
lateAfterMinutes: resolved.lateAfterMinutes,
|
|
3214
4237
|
earlyBeforeMinutes: resolved.earlyBeforeMinutes
|
|
3215
4238
|
})]);
|
|
3216
|
-
const { buildGoogleMeetExportManifest, googleMeetExportFileNames, writeMeetExportBundle } = await import("./cli-
|
|
4239
|
+
const { buildGoogleMeetExportManifest, googleMeetExportFileNames, writeMeetExportBundle } = await import("./cli-CP1gp7Wl.js");
|
|
3217
4240
|
const calendarId = normalizeOptionalString(raw.calendarId);
|
|
3218
4241
|
const request = {
|
|
3219
4242
|
...resolved.meeting ? { meeting: resolved.meeting } : {},
|
|
@@ -3290,7 +4313,8 @@ var google_meet_default = definePluginEntry({
|
|
|
3290
4313
|
dialInNumber: normalizeOptionalString(params?.dialInNumber),
|
|
3291
4314
|
pin: normalizeOptionalString(params?.pin),
|
|
3292
4315
|
dtmfSequence: normalizeOptionalString(params?.dtmfSequence),
|
|
3293
|
-
message: normalizeOptionalString(params?.message)
|
|
4316
|
+
message: normalizeOptionalString(params?.message),
|
|
4317
|
+
requesterSessionKey: normalizeOptionalString(params?.requesterSessionKey)
|
|
3294
4318
|
}));
|
|
3295
4319
|
} catch (err) {
|
|
3296
4320
|
sendError(respond, err);
|
|
@@ -3460,7 +4484,8 @@ var google_meet_default = definePluginEntry({
|
|
|
3460
4484
|
dialInNumber: normalizeOptionalString(params?.dialInNumber),
|
|
3461
4485
|
pin: normalizeOptionalString(params?.pin),
|
|
3462
4486
|
dtmfSequence: normalizeOptionalString(params?.dtmfSequence),
|
|
3463
|
-
message: normalizeOptionalString(params?.message)
|
|
4487
|
+
message: normalizeOptionalString(params?.message),
|
|
4488
|
+
requesterSessionKey: normalizeOptionalString(params?.requesterSessionKey)
|
|
3464
4489
|
}));
|
|
3465
4490
|
} catch (err) {
|
|
3466
4491
|
sendError(respond, err);
|
|
@@ -3478,13 +4503,18 @@ var google_meet_default = definePluginEntry({
|
|
|
3478
4503
|
sendError(respond, err);
|
|
3479
4504
|
}
|
|
3480
4505
|
});
|
|
3481
|
-
api.registerTool({
|
|
4506
|
+
api.registerTool((toolContext) => ({
|
|
3482
4507
|
name: "google_meet",
|
|
3483
4508
|
label: "Google Meet",
|
|
3484
|
-
description: "Join and track Google Meet sessions through Chrome or Twilio. Call setup_status before join/create/test_listen/test_speech; if it reports a Chrome node offline, local audio missing, or missing Twilio dial plan, surface that blocker instead of retrying or switching transports. Twilio cannot dial a Meet URL directly: provide dialInNumber plus optional pin/dtmfSequence, or configure twilio.defaultDialInNumber. Offline nodes are diagnostics only, not usable candidates. If local Chrome
|
|
4509
|
+
description: "Join and track Google Meet sessions through Chrome or Twilio. Call setup_status before join/create/test_listen/test_speech; if it reports a Chrome node offline, local audio missing, or missing Twilio dial plan, surface that blocker instead of retrying or switching transports. Twilio cannot dial a Meet URL directly: provide dialInNumber plus optional pin/dtmfSequence, or configure twilio.defaultDialInNumber. Offline nodes are diagnostics only, not usable candidates. If local Chrome talk-back audio is unsupported on this OS, use mode=transcribe, transport=twilio, or a macOS chrome-node for agent/bidi Chrome. If a Meet tab is already open after a timeout, call recover_current_tab before retrying join to report login, permission, or admission blockers without opening another tab.",
|
|
3485
4510
|
parameters: GoogleMeetToolSchema,
|
|
3486
4511
|
async execute(_toolCallId, params) {
|
|
3487
4512
|
const raw = asParamRecord(params);
|
|
4513
|
+
const requesterSessionKey = normalizeOptionalString(toolContext.sessionKey);
|
|
4514
|
+
const rawWithRequester = requesterSessionKey ? {
|
|
4515
|
+
...raw,
|
|
4516
|
+
requesterSessionKey
|
|
4517
|
+
} : raw;
|
|
3488
4518
|
try {
|
|
3489
4519
|
assertGoogleMeetAgentToolActionSupported({
|
|
3490
4520
|
config,
|
|
@@ -3494,17 +4524,17 @@ var google_meet_default = definePluginEntry({
|
|
|
3494
4524
|
case "join": return json(await callGoogleMeetGatewayFromTool({
|
|
3495
4525
|
config,
|
|
3496
4526
|
action: "join",
|
|
3497
|
-
raw
|
|
4527
|
+
raw: rawWithRequester
|
|
3498
4528
|
}));
|
|
3499
4529
|
case "create": return json(await callGoogleMeetGatewayFromTool({
|
|
3500
4530
|
config,
|
|
3501
4531
|
action: "create",
|
|
3502
|
-
raw
|
|
4532
|
+
raw: rawWithRequester
|
|
3503
4533
|
}));
|
|
3504
4534
|
case "test_speech": return json(await callGoogleMeetGatewayFromTool({
|
|
3505
4535
|
config,
|
|
3506
4536
|
action: "test_speech",
|
|
3507
|
-
raw
|
|
4537
|
+
raw: rawWithRequester
|
|
3508
4538
|
}));
|
|
3509
4539
|
case "test_listen": return json(await callGoogleMeetGatewayFromTool({
|
|
3510
4540
|
config,
|
|
@@ -3615,14 +4645,14 @@ var google_meet_default = definePluginEntry({
|
|
|
3615
4645
|
return json(formatGatewayError(err));
|
|
3616
4646
|
}
|
|
3617
4647
|
}
|
|
3618
|
-
});
|
|
4648
|
+
}), { name: "google_meet" });
|
|
3619
4649
|
api.registerNodeHostCommand({
|
|
3620
4650
|
command: "googlemeet.chrome",
|
|
3621
4651
|
cap: "google-meet",
|
|
3622
4652
|
handle: handleGoogleMeetNodeHostCommand
|
|
3623
4653
|
});
|
|
3624
4654
|
api.registerCli(async ({ program }) => {
|
|
3625
|
-
const { registerGoogleMeetCli } = await import("./cli-
|
|
4655
|
+
const { registerGoogleMeetCli } = await import("./cli-CP1gp7Wl.js");
|
|
3626
4656
|
registerGoogleMeetCli({
|
|
3627
4657
|
program,
|
|
3628
4658
|
config,
|