@vellumai/assistant 0.11.3-staging.1 → 0.11.3-staging.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +139 -5
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +168 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/events-dev-bypass-actor.test.ts +112 -1
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/approvals/guardian-request-resolvers.ts +6 -3
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +81 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +12 -0
- package/src/calls/media-stream-stt-session.ts +34 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +31 -3
- package/src/calls/voice-triage-escalate.ts +52 -5
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/loader.ts +5 -0
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/daemon/conversation-agent-loop.ts +22 -3
- package/src/daemon/conversation-messaging.ts +21 -1
- package/src/daemon/conversation-process.ts +55 -5
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation.ts +6 -0
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/live-voice/__tests__/front-decision.test.ts +120 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +1 -1
- package/src/live-voice/__tests__/live-voice-progress.test.ts +178 -11
- package/src/live-voice/__tests__/live-voice-stt.test.ts +304 -1
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +102 -2
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/front-decision.ts +50 -3
- package/src/live-voice/live-voice-session.ts +202 -25
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/seed-providers.ts +6 -3
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/provider-catalog.ts +38 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/assistant-event-hub.ts +23 -0
- package/src/runtime/local-actor-identity.ts +18 -5
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/conversation-routes.ts +10 -0
- package/src/runtime/routes/events-routes.ts +17 -16
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/types.ts +16 -0
- package/src/subagent/manager.ts +9 -2
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/language-subtag.ts +43 -0
- package/src/util/unicode.ts +1 -1
|
@@ -164,7 +164,11 @@ function makeConversationWithPendingConfirmation(
|
|
|
164
164
|
promoteToHead: (requestId: string) => ({ requestId }),
|
|
165
165
|
},
|
|
166
166
|
pendingSteerRepair: false,
|
|
167
|
-
|
|
167
|
+
// Stores rather than discarding, so a test can observe which trust the
|
|
168
|
+
// route resolved and move the slot afterwards.
|
|
169
|
+
setTrustContext(this: { trustContext: unknown }, ctx: unknown) {
|
|
170
|
+
this.trustContext = ctx;
|
|
171
|
+
},
|
|
168
172
|
updateClient: () => {},
|
|
169
173
|
emitConfirmationStateChanged: () => {},
|
|
170
174
|
emitActivityState: () => {},
|
|
@@ -354,3 +358,62 @@ describe("hidden sends to an idle conversation with a pending confirmation", ()
|
|
|
354
358
|
expect(spies.agentLoopOptions()?.isHiddenPrompt).toBeUndefined();
|
|
355
359
|
});
|
|
356
360
|
});
|
|
361
|
+
|
|
362
|
+
describe("POST /messages turn trust", () => {
|
|
363
|
+
test("the idle send path runs its turn under the trust resolved for the request", async () => {
|
|
364
|
+
// The idle web path does not go through `processMessage`; it persists and
|
|
365
|
+
// calls `runAgentLoop` directly. Without the captured trust travelling
|
|
366
|
+
// with that call, the loop re-reads the conversation slot when it opens,
|
|
367
|
+
// and the slot is writable in between by paths that do not own this turn
|
|
368
|
+
// (channel ingress for another actor, live-voice hydration, pointer
|
|
369
|
+
// elevation, the voice bridge).
|
|
370
|
+
//
|
|
371
|
+
// The defect here is at the call site, not inside the loop, so observing
|
|
372
|
+
// the options the route passes is sufficient. The re-read *inside* the
|
|
373
|
+
// loop is covered separately against a real Conversation, since a stub
|
|
374
|
+
// like this one cannot see it.
|
|
375
|
+
const spies = makeConversationWithPendingConfirmation(false);
|
|
376
|
+
setConversation(CONV_ID, spies.conversation);
|
|
377
|
+
|
|
378
|
+
const conversation = spies.conversation as unknown as {
|
|
379
|
+
trustContext: unknown;
|
|
380
|
+
persistUserMessage: () => Promise<{ id: string; deduplicated: boolean }>;
|
|
381
|
+
};
|
|
382
|
+
|
|
383
|
+
// Another actor writes the slot inside the window, after the route
|
|
384
|
+
// resolves this request's trust and before the loop call.
|
|
385
|
+
let slotMoved = false;
|
|
386
|
+
conversation.persistUserMessage = async () => {
|
|
387
|
+
conversation.trustContext = {
|
|
388
|
+
trustClass: "trusted_contact",
|
|
389
|
+
sourceChannel: "slack",
|
|
390
|
+
requesterExternalUserId: "U-other-actor",
|
|
391
|
+
};
|
|
392
|
+
slotMoved = true;
|
|
393
|
+
return { id: "persisted-user-id", deduplicated: false };
|
|
394
|
+
};
|
|
395
|
+
|
|
396
|
+
const res = await callHandler(
|
|
397
|
+
(args) => handleSendMessage(args, makeDeps(spies.conversation)),
|
|
398
|
+
makeRequest({ content: "run a command" }),
|
|
399
|
+
undefined,
|
|
400
|
+
202,
|
|
401
|
+
);
|
|
402
|
+
|
|
403
|
+
expect(res.status).toBe(202);
|
|
404
|
+
// Guard the test itself: if the injection stopped running the assertion
|
|
405
|
+
// below would pass for the wrong reason.
|
|
406
|
+
expect(slotMoved).toBe(true);
|
|
407
|
+
expect(
|
|
408
|
+
(conversation.trustContext as { trustClass?: string } | undefined)
|
|
409
|
+
?.trustClass,
|
|
410
|
+
).toBe("trusted_contact");
|
|
411
|
+
|
|
412
|
+
// The turn carries the trust this request resolved, not the moved slot.
|
|
413
|
+
const turnTrust = spies.agentLoopOptions()?.turnTrustContext as
|
|
414
|
+
| { trustClass?: string }
|
|
415
|
+
| undefined;
|
|
416
|
+
expect(turnTrust).toBeDefined();
|
|
417
|
+
expect(turnTrust?.trustClass).toBe("guardian");
|
|
418
|
+
});
|
|
419
|
+
});
|
|
@@ -27,6 +27,10 @@ let fakeGuardianPrincipalId: string | undefined = undefined;
|
|
|
27
27
|
// exactly when the SSE self-heal lookup completes.
|
|
28
28
|
const pendingAsyncResolutions: Array<(value: string | undefined) => void> = [];
|
|
29
29
|
|
|
30
|
+
/** Options each async resolution was called with, in the same order. */
|
|
31
|
+
const asyncResolutionOptions: Array<{ forceRefresh?: boolean } | undefined> =
|
|
32
|
+
[];
|
|
33
|
+
|
|
30
34
|
mock.module("../config/env.js", () => ({
|
|
31
35
|
isHttpAuthDisabled: () => fakeHttpAuthDisabled,
|
|
32
36
|
hasUngatedHttpAuthDisabled: () => false,
|
|
@@ -42,10 +46,14 @@ mock.module("../runtime/local-actor-identity.js", () => ({
|
|
|
42
46
|
}
|
|
43
47
|
return fakeGuardianPrincipalId;
|
|
44
48
|
},
|
|
45
|
-
resolveActorPrincipalIdForLocalGuardian: (
|
|
49
|
+
resolveActorPrincipalIdForLocalGuardian: (
|
|
50
|
+
rawHeader: string | undefined,
|
|
51
|
+
options?: { forceRefresh?: boolean },
|
|
52
|
+
) => {
|
|
46
53
|
if (rawHeader !== "dev-bypass" || !fakeHttpAuthDisabled) {
|
|
47
54
|
return Promise.resolve(rawHeader);
|
|
48
55
|
}
|
|
56
|
+
asyncResolutionOptions.push(options);
|
|
49
57
|
return new Promise<string | undefined>((resolve) => {
|
|
50
58
|
pendingAsyncResolutions.push(resolve);
|
|
51
59
|
});
|
|
@@ -57,6 +65,23 @@ async function flushMicrotasks(): Promise<void> {
|
|
|
57
65
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
58
66
|
}
|
|
59
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Poll until `predicate` holds, for assertions that depend on the heal's
|
|
70
|
+
* retry backoff timer rather than on promise settling alone.
|
|
71
|
+
*/
|
|
72
|
+
async function waitFor(
|
|
73
|
+
predicate: () => boolean,
|
|
74
|
+
timeoutMs = 3_000,
|
|
75
|
+
): Promise<void> {
|
|
76
|
+
const deadline = Date.now() + timeoutMs;
|
|
77
|
+
while (!predicate()) {
|
|
78
|
+
if (Date.now() >= deadline) {
|
|
79
|
+
throw new Error("waitFor timed out");
|
|
80
|
+
}
|
|
81
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
60
85
|
// ── Real imports (after mocks) ────────────────────────────────────────────
|
|
61
86
|
|
|
62
87
|
import { AssistantEventHub } from "../runtime/assistant-event-hub.js";
|
|
@@ -69,6 +94,7 @@ afterAll(() => {
|
|
|
69
94
|
describe("events SSE registration — dev-bypass actor translation", () => {
|
|
70
95
|
beforeEach(() => {
|
|
71
96
|
pendingAsyncResolutions.length = 0;
|
|
97
|
+
asyncResolutionOptions.length = 0;
|
|
72
98
|
});
|
|
73
99
|
|
|
74
100
|
test("translates 'dev-bypass' to the real guardian principalId when auth is disabled", () => {
|
|
@@ -261,6 +287,91 @@ describe("events SSE registration — dev-bypass actor translation", () => {
|
|
|
261
287
|
ac2.abort();
|
|
262
288
|
});
|
|
263
289
|
|
|
290
|
+
test("retries the heal after a failed lookup instead of giving up for the connection's lifetime", async () => {
|
|
291
|
+
// The gateway IPC returns null on any transport failure, so the first
|
|
292
|
+
// lookup can miss even when a guardian is bound. A subscription left
|
|
293
|
+
// principal-less rejects every host-proxy result it submits, so the heal
|
|
294
|
+
// has to outlive one miss rather than stopping there.
|
|
295
|
+
fakeHttpAuthDisabled = true;
|
|
296
|
+
fakeGuardianPrincipalId = undefined; // cold cache: sync resolution misses
|
|
297
|
+
|
|
298
|
+
const ac = new AbortController();
|
|
299
|
+
const hub = new AssistantEventHub();
|
|
300
|
+
|
|
301
|
+
handleSubscribeAssistantEvents(
|
|
302
|
+
{
|
|
303
|
+
headers: {
|
|
304
|
+
"x-vellum-client-id": "retry-client-001",
|
|
305
|
+
"x-vellum-interface-id": "chrome-extension",
|
|
306
|
+
"x-vellum-actor-principal-id": "dev-bypass",
|
|
307
|
+
},
|
|
308
|
+
abortSignal: ac.signal,
|
|
309
|
+
},
|
|
310
|
+
{ hub },
|
|
311
|
+
);
|
|
312
|
+
|
|
313
|
+
// First attempt fails (gateway not reachable yet).
|
|
314
|
+
expect(pendingAsyncResolutions).toHaveLength(1);
|
|
315
|
+
pendingAsyncResolutions.shift()!(undefined);
|
|
316
|
+
await flushMicrotasks();
|
|
317
|
+
expect(
|
|
318
|
+
hub.getActorPrincipalIdForClient("retry-client-001"),
|
|
319
|
+
).toBeUndefined();
|
|
320
|
+
|
|
321
|
+
// The retry fires on the backoff and succeeds this time.
|
|
322
|
+
await waitFor(() => pendingAsyncResolutions.length === 1);
|
|
323
|
+
pendingAsyncResolutions.shift()!("guardian-real-id");
|
|
324
|
+
await flushMicrotasks();
|
|
325
|
+
|
|
326
|
+
expect(hub.getActorPrincipalIdForClient("retry-client-001")).toBe(
|
|
327
|
+
"guardian-real-id",
|
|
328
|
+
);
|
|
329
|
+
|
|
330
|
+
// Every attempt must bypass the guardian-delivery cache. A successful read
|
|
331
|
+
// that finds no binding is cached for minutes, which outlives the retry
|
|
332
|
+
// schedule, so a cached read would spend all attempts on the same answer
|
|
333
|
+
// and never observe a binding created in between.
|
|
334
|
+
expect(asyncResolutionOptions).toEqual([
|
|
335
|
+
{ forceRefresh: true },
|
|
336
|
+
{ forceRefresh: true },
|
|
337
|
+
]);
|
|
338
|
+
|
|
339
|
+
ac.abort();
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
test("needsActorPrincipalHeal reports only live, principal-less client subscriptions", () => {
|
|
343
|
+
const hub = new AssistantEventHub();
|
|
344
|
+
const bare = hub.subscribe({
|
|
345
|
+
type: "client",
|
|
346
|
+
clientId: "needs-heal-001",
|
|
347
|
+
interfaceId: "chrome-extension",
|
|
348
|
+
capabilities: [],
|
|
349
|
+
callback: () => {},
|
|
350
|
+
});
|
|
351
|
+
const withPrincipal = hub.subscribe({
|
|
352
|
+
type: "client",
|
|
353
|
+
clientId: "needs-heal-002",
|
|
354
|
+
interfaceId: "macos",
|
|
355
|
+
capabilities: [],
|
|
356
|
+
actorPrincipalId: "guardian-real-id",
|
|
357
|
+
callback: () => {},
|
|
358
|
+
});
|
|
359
|
+
const process = hub.subscribe({ type: "process", callback: () => {} });
|
|
360
|
+
|
|
361
|
+
expect(hub.needsActorPrincipalHeal(bare.connectionId)).toBe(true);
|
|
362
|
+
expect(hub.needsActorPrincipalHeal(withPrincipal.connectionId)).toBe(false);
|
|
363
|
+
expect(hub.needsActorPrincipalHeal(process.connectionId)).toBe(false);
|
|
364
|
+
expect(hub.needsActorPrincipalHeal("conn-does-not-exist")).toBe(false);
|
|
365
|
+
|
|
366
|
+
// A disposed connection stops needing a heal, which is what ends the
|
|
367
|
+
// retry loop when a client disconnects mid-backoff.
|
|
368
|
+
bare.dispose();
|
|
369
|
+
expect(hub.needsActorPrincipalHeal(bare.connectionId)).toBe(false);
|
|
370
|
+
|
|
371
|
+
withPrincipal.dispose();
|
|
372
|
+
process.dispose();
|
|
373
|
+
});
|
|
374
|
+
|
|
264
375
|
test("fillClientActorPrincipalId never overwrites a present principal", () => {
|
|
265
376
|
const hub = new AssistantEventHub();
|
|
266
377
|
const sub = hub.subscribe({
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
} from "../calls/media-stream-audio-transcode.js";
|
|
27
27
|
import { MediaStreamOutput } from "../calls/media-stream-output.js";
|
|
28
28
|
import { resolveCallTtsProvider } from "../calls/resolve-call-tts-provider.js";
|
|
29
|
+
import { setConfig } from "./helpers/set-config.js";
|
|
29
30
|
|
|
30
31
|
const mockResolveCallTtsProvider = resolveCallTtsProvider as ReturnType<
|
|
31
32
|
typeof jest.fn
|
|
@@ -1213,6 +1214,180 @@ describe("MediaStreamOutput", () => {
|
|
|
1213
1214
|
});
|
|
1214
1215
|
});
|
|
1215
1216
|
|
|
1217
|
+
// ---------------------------------------------------------------------------
|
|
1218
|
+
// Synthesis language hint
|
|
1219
|
+
// ---------------------------------------------------------------------------
|
|
1220
|
+
|
|
1221
|
+
describe("synthesis language hint", () => {
|
|
1222
|
+
/** Synthesize one turn and return the provider request it produced. */
|
|
1223
|
+
async function synthesizeOneTurn(
|
|
1224
|
+
output: MediaStreamOutput,
|
|
1225
|
+
): Promise<{ language?: string }> {
|
|
1226
|
+
mockSynthesize.mockResolvedValue({
|
|
1227
|
+
audio: makeWavBuffer([1000, 2000, 3000, 4000]),
|
|
1228
|
+
contentType: "audio/wav",
|
|
1229
|
+
});
|
|
1230
|
+
output.sendTextToken("Hello caller.", true);
|
|
1231
|
+
await drain(() => mockSynthesize.mock.calls.length > 0);
|
|
1232
|
+
return mockSynthesize.mock.calls[0][0] as { language?: string };
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
test("the resolver's language rides the provider request", async () => {
|
|
1236
|
+
const { ws } = createMockWs();
|
|
1237
|
+
const output = makeOutput(ws, "stream-lang");
|
|
1238
|
+
output.setSynthesisLanguageResolver(() => "es");
|
|
1239
|
+
|
|
1240
|
+
const request = await synthesizeOneTurn(output);
|
|
1241
|
+
|
|
1242
|
+
expect(request.language).toBe("es");
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
test("fixed system copy synthesizes without the caller-language hint while model text keeps it", async () => {
|
|
1246
|
+
const { ws } = createMockWs();
|
|
1247
|
+
const output = makeOutput(ws, "stream-lang");
|
|
1248
|
+
output.setSynthesisLanguageResolver(() => "ja");
|
|
1249
|
+
mockSynthesize.mockResolvedValue({
|
|
1250
|
+
audio: makeWavBuffer([1000, 2000, 3000, 4000]),
|
|
1251
|
+
contentType: "audio/wav",
|
|
1252
|
+
});
|
|
1253
|
+
|
|
1254
|
+
output.sendTextToken("Model reply.", true);
|
|
1255
|
+
output.sendTextToken("Are you still there?", true, { systemCopy: true });
|
|
1256
|
+
await drain(() => mockSynthesize.mock.calls.length >= 2);
|
|
1257
|
+
|
|
1258
|
+
const requests = mockSynthesize.mock.calls.map(
|
|
1259
|
+
(call) => call[0] as { text: string; language?: string },
|
|
1260
|
+
);
|
|
1261
|
+
expect(requests[0]?.text).toBe("Model reply.");
|
|
1262
|
+
expect(requests[0]?.language).toBe("ja");
|
|
1263
|
+
expect(requests[1]?.text).toBe("Are you still there?");
|
|
1264
|
+
expect(requests[1]?.language).toBeUndefined();
|
|
1265
|
+
});
|
|
1266
|
+
|
|
1267
|
+
test("no language when nothing resolves (default multilingual pin)", async () => {
|
|
1268
|
+
const { ws } = createMockWs();
|
|
1269
|
+
const output = makeOutput(ws, "stream-lang");
|
|
1270
|
+
|
|
1271
|
+
const request = await synthesizeOneTurn(output);
|
|
1272
|
+
|
|
1273
|
+
expect(request.language).toBeUndefined();
|
|
1274
|
+
});
|
|
1275
|
+
|
|
1276
|
+
test("a monolingual pin on a manual-selection provider resolves as the hint", async () => {
|
|
1277
|
+
setConfig("services", {
|
|
1278
|
+
stt: { provider: "deepgram", language: "es-ES" },
|
|
1279
|
+
});
|
|
1280
|
+
try {
|
|
1281
|
+
const { ws } = createMockWs();
|
|
1282
|
+
const output = makeOutput(ws, "stream-lang");
|
|
1283
|
+
|
|
1284
|
+
const request = await synthesizeOneTurn(output);
|
|
1285
|
+
|
|
1286
|
+
expect(request.language).toBe("es");
|
|
1287
|
+
} finally {
|
|
1288
|
+
setConfig("services", {});
|
|
1289
|
+
}
|
|
1290
|
+
});
|
|
1291
|
+
|
|
1292
|
+
test("the pin is ignored for auto-detecting providers", async () => {
|
|
1293
|
+
setConfig("services", {
|
|
1294
|
+
stt: { provider: "openai-whisper", language: "es" },
|
|
1295
|
+
});
|
|
1296
|
+
try {
|
|
1297
|
+
const { ws } = createMockWs();
|
|
1298
|
+
const output = makeOutput(ws, "stream-lang");
|
|
1299
|
+
|
|
1300
|
+
const request = await synthesizeOneTurn(output);
|
|
1301
|
+
|
|
1302
|
+
expect(request.language).toBeUndefined();
|
|
1303
|
+
} finally {
|
|
1304
|
+
setConfig("services", {});
|
|
1305
|
+
}
|
|
1306
|
+
});
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
// ---------------------------------------------------------------------------
|
|
1310
|
+
// Per-language voice override (languageVoices)
|
|
1311
|
+
// ---------------------------------------------------------------------------
|
|
1312
|
+
|
|
1313
|
+
describe("per-language voice override", () => {
|
|
1314
|
+
afterEach(() => {
|
|
1315
|
+
setConfig("services", {});
|
|
1316
|
+
});
|
|
1317
|
+
|
|
1318
|
+
/** Seed a deepgram languageVoices entry and resolve deepgram as the provider. */
|
|
1319
|
+
function useDeepgramWithHindiVoice(): void {
|
|
1320
|
+
setConfig("services", {
|
|
1321
|
+
tts: {
|
|
1322
|
+
provider: "deepgram",
|
|
1323
|
+
providers: {
|
|
1324
|
+
deepgram: { languageVoices: { hi: "aura-2-hindi-voice" } },
|
|
1325
|
+
},
|
|
1326
|
+
},
|
|
1327
|
+
});
|
|
1328
|
+
useProvider({
|
|
1329
|
+
id: "deepgram",
|
|
1330
|
+
capabilities: { supportsStreaming: false, supportedFormats: ["wav"] },
|
|
1331
|
+
synthesize: mockSynthesize,
|
|
1332
|
+
});
|
|
1333
|
+
mockSynthesize.mockResolvedValue({
|
|
1334
|
+
audio: makeWavBuffer([1000, 2000, 3000, 4000]),
|
|
1335
|
+
contentType: "audio/wav",
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
test("a configured languageVoices entry for the resolved language rides the request as voiceId", async () => {
|
|
1340
|
+
useDeepgramWithHindiVoice();
|
|
1341
|
+
const { ws } = createMockWs();
|
|
1342
|
+
const output = makeOutput(ws, "stream-voice");
|
|
1343
|
+
output.setSynthesisLanguageResolver(() => "hi");
|
|
1344
|
+
|
|
1345
|
+
output.sendTextToken("Namaste.", true);
|
|
1346
|
+
await drain(() => mockSynthesize.mock.calls.length > 0);
|
|
1347
|
+
|
|
1348
|
+
const request = mockSynthesize.mock.calls[0][0] as {
|
|
1349
|
+
voiceId?: string;
|
|
1350
|
+
language?: string;
|
|
1351
|
+
};
|
|
1352
|
+
expect(request.language).toBe("hi");
|
|
1353
|
+
expect(request.voiceId).toBe("aura-2-hindi-voice");
|
|
1354
|
+
});
|
|
1355
|
+
|
|
1356
|
+
test("no voiceId when the map has no entry for the resolved language", async () => {
|
|
1357
|
+
useDeepgramWithHindiVoice();
|
|
1358
|
+
const { ws } = createMockWs();
|
|
1359
|
+
const output = makeOutput(ws, "stream-voice");
|
|
1360
|
+
output.setSynthesisLanguageResolver(() => "es");
|
|
1361
|
+
|
|
1362
|
+
output.sendTextToken("Hola.", true);
|
|
1363
|
+
await drain(() => mockSynthesize.mock.calls.length > 0);
|
|
1364
|
+
|
|
1365
|
+
const request = mockSynthesize.mock.calls[0][0] as {
|
|
1366
|
+
voiceId?: string;
|
|
1367
|
+
language?: string;
|
|
1368
|
+
};
|
|
1369
|
+
expect(request.language).toBe("es");
|
|
1370
|
+
expect(request.voiceId).toBeUndefined();
|
|
1371
|
+
});
|
|
1372
|
+
|
|
1373
|
+
test("system copy gets neither the language hint nor the language voice", async () => {
|
|
1374
|
+
useDeepgramWithHindiVoice();
|
|
1375
|
+
const { ws } = createMockWs();
|
|
1376
|
+
const output = makeOutput(ws, "stream-voice");
|
|
1377
|
+
output.setSynthesisLanguageResolver(() => "hi");
|
|
1378
|
+
|
|
1379
|
+
output.sendTextToken("Are you still there?", true, { systemCopy: true });
|
|
1380
|
+
await drain(() => mockSynthesize.mock.calls.length > 0);
|
|
1381
|
+
|
|
1382
|
+
const request = mockSynthesize.mock.calls[0][0] as {
|
|
1383
|
+
voiceId?: string;
|
|
1384
|
+
language?: string;
|
|
1385
|
+
};
|
|
1386
|
+
expect(request.language).toBeUndefined();
|
|
1387
|
+
expect(request.voiceId).toBeUndefined();
|
|
1388
|
+
});
|
|
1389
|
+
});
|
|
1390
|
+
|
|
1216
1391
|
// ---------------------------------------------------------------------------
|
|
1217
1392
|
// Streaming PCM synthesis (incremental transcode)
|
|
1218
1393
|
// ---------------------------------------------------------------------------
|
|
@@ -897,6 +897,73 @@ describe("MediaStreamSttSession", () => {
|
|
|
897
897
|
session.dispose();
|
|
898
898
|
});
|
|
899
899
|
|
|
900
|
+
// ── Detected language (latest tagged utterance) ─────────────────
|
|
901
|
+
|
|
902
|
+
test("a language switch retargets on the first final in the new language", async () => {
|
|
903
|
+
const { session, fake } = await startStreamingSession();
|
|
904
|
+
|
|
905
|
+
expect(session.currentLanguage()).toBeUndefined();
|
|
906
|
+
|
|
907
|
+
fake.emit({ type: "final", text: "hola", languages: ["es", "en"] });
|
|
908
|
+
fake.emit({ type: "final", text: "buenos dias", languages: ["es"] });
|
|
909
|
+
expect(session.currentLanguage()).toBe("es");
|
|
910
|
+
|
|
911
|
+
// The first ja-tagged utterance wins immediately: the caller does
|
|
912
|
+
// not have to outvote the session's es history. The secondary "es"
|
|
913
|
+
// tag does not dilute the utterance's dominant tag.
|
|
914
|
+
fake.emit({ type: "final", text: "konnichiwa", languages: ["ja", "es"] });
|
|
915
|
+
expect(session.currentLanguage()).toBe("ja");
|
|
916
|
+
|
|
917
|
+
session.dispose();
|
|
918
|
+
});
|
|
919
|
+
|
|
920
|
+
test("regional variants normalize to their base subtag", async () => {
|
|
921
|
+
const { session, fake } = await startStreamingSession();
|
|
922
|
+
|
|
923
|
+
fake.emit({ type: "final", text: "tudo bem", languages: ["pt-BR"] });
|
|
924
|
+
|
|
925
|
+
expect(session.currentLanguage()).toBe("pt");
|
|
926
|
+
|
|
927
|
+
session.dispose();
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
test("empty and untagged finals keep the previous value", async () => {
|
|
931
|
+
const { session, fake } = await startStreamingSession();
|
|
932
|
+
|
|
933
|
+
// A silence final can carry container-level tags describing no
|
|
934
|
+
// emitted words; it must not update the language.
|
|
935
|
+
fake.emit({ type: "final", text: " ", languages: ["fr"] });
|
|
936
|
+
expect(session.currentLanguage()).toBeUndefined();
|
|
937
|
+
|
|
938
|
+
fake.emit({ type: "final", text: "hola", languages: ["es"] });
|
|
939
|
+
expect(session.currentLanguage()).toBe("es");
|
|
940
|
+
|
|
941
|
+
// A committed final without tags keeps the previous value.
|
|
942
|
+
fake.emit({ type: "final", text: "hello" });
|
|
943
|
+
expect(session.currentLanguage()).toBe("es");
|
|
944
|
+
|
|
945
|
+
// So does a tagged silence final after a language is established.
|
|
946
|
+
fake.emit({ type: "final", text: " ", languages: ["fr"] });
|
|
947
|
+
expect(session.currentLanguage()).toBe("es");
|
|
948
|
+
|
|
949
|
+
session.dispose();
|
|
950
|
+
});
|
|
951
|
+
|
|
952
|
+
test("falling back to batch mid-call clears the detected language", async () => {
|
|
953
|
+
const { session, fake } = await startStreamingSession();
|
|
954
|
+
|
|
955
|
+
fake.emit({ type: "final", text: "konnichiwa", languages: ["ja"] });
|
|
956
|
+
expect(session.currentLanguage()).toBe("ja");
|
|
957
|
+
|
|
958
|
+
// Provider closes the stream unexpectedly: the session settles on
|
|
959
|
+
// batch, whose transcripts carry no language metadata. The stale ja
|
|
960
|
+
// detection must not keep hinting synthesis for English speech.
|
|
961
|
+
fake.emit({ type: "closed" });
|
|
962
|
+
expect(session.currentLanguage()).toBeUndefined();
|
|
963
|
+
|
|
964
|
+
session.dispose();
|
|
965
|
+
});
|
|
966
|
+
|
|
900
967
|
test("local VAD turn end never triggers batch transcription in streaming mode", async () => {
|
|
901
968
|
const onTranscriptFinal = jest.fn();
|
|
902
969
|
const { session } = await startStreamingSession(
|
|
@@ -1,7 +1,44 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
1
3
|
import { describe, expect, test } from "bun:test";
|
|
2
4
|
|
|
3
5
|
import { PROVIDER_SEED_DATA } from "../oauth/seed-providers.js";
|
|
4
6
|
|
|
7
|
+
const REPO_ROOT = dirname(dirname(dirname(import.meta.dir)));
|
|
8
|
+
const WEB_PUBLIC_DIR = join(REPO_ROOT, "clients/web/public");
|
|
9
|
+
const INTEGRATION_ICON_SOURCE = join(
|
|
10
|
+
REPO_ROOT,
|
|
11
|
+
"clients/web/src/components/integrations/integration-icon.tsx",
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Provider keys the web client draws without a `BUNDLED_LOGO_URLS` entry.
|
|
16
|
+
* Google's mark is multi-colour, so it ships as the `GoogleLogo` component
|
|
17
|
+
* and is returned before the bundled-asset lookup runs.
|
|
18
|
+
*/
|
|
19
|
+
const COMPONENT_RENDERED_PROVIDERS = new Set(["google"]);
|
|
20
|
+
|
|
21
|
+
/** `publicAsset("/images/...")` paths from `BUNDLED_LOGO_URLS`, by key. */
|
|
22
|
+
function readBundledLogoMap(): Map<string, string> {
|
|
23
|
+
const source = readFileSync(INTEGRATION_ICON_SOURCE, "utf8");
|
|
24
|
+
const block = source.match(
|
|
25
|
+
/const BUNDLED_LOGO_URLS: Record<string, string> = \{([\s\S]*?)\n\};/,
|
|
26
|
+
);
|
|
27
|
+
if (!block) {
|
|
28
|
+
throw new Error(
|
|
29
|
+
`Could not find BUNDLED_LOGO_URLS in ${INTEGRATION_ICON_SOURCE}. If it ` +
|
|
30
|
+
`was renamed or restructured, update this test rather than deleting it.`,
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
const entries = new Map<string, string>();
|
|
34
|
+
for (const [, key, path] of block[1]!.matchAll(
|
|
35
|
+
/(\w+):\s*publicAsset\("([^"]+)"\)/g,
|
|
36
|
+
)) {
|
|
37
|
+
entries.set(key!, path!);
|
|
38
|
+
}
|
|
39
|
+
return entries;
|
|
40
|
+
}
|
|
41
|
+
|
|
5
42
|
/**
|
|
6
43
|
* Allowed CDN prefixes for the ``logoUrl`` field on ``PROVIDER_SEED_DATA``
|
|
7
44
|
* (``assistant/src/oauth/seed-providers.ts``):
|
|
@@ -41,3 +78,31 @@ describe("PROVIDER_SEED_DATA logo URLs", () => {
|
|
|
41
78
|
expect(invalid).toEqual([]);
|
|
42
79
|
});
|
|
43
80
|
});
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The `logoUrl` above is a fallback for providers registered at runtime. Every
|
|
84
|
+
* provider we seed ourselves must draw from an asset we ship, because an icon
|
|
85
|
+
* CDN can drop a brand at any time and the client then degrades to an initials
|
|
86
|
+
* avatar with nothing in CI to notice: the prefix check above passes just as
|
|
87
|
+
* happily for a URL that 404s. Simple Icons hosts no Microsoft mark and no
|
|
88
|
+
* Slack mark, so the seeded URLs for those brands do not resolve there.
|
|
89
|
+
*/
|
|
90
|
+
describe("PROVIDER_SEED_DATA bundled logo coverage", () => {
|
|
91
|
+
test("every seeded provider renders from a bundled asset", () => {
|
|
92
|
+
const bundled = readBundledLogoMap();
|
|
93
|
+
const uncovered = Object.keys(PROVIDER_SEED_DATA).filter(
|
|
94
|
+
(provider) =>
|
|
95
|
+
!bundled.has(provider) && !COMPONENT_RENDERED_PROVIDERS.has(provider),
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
expect(uncovered).toEqual([]);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("every bundled logo path resolves to a file on disk", () => {
|
|
102
|
+
const missingFiles = [...readBundledLogoMap()]
|
|
103
|
+
.filter(([, path]) => !existsSync(join(WEB_PUBLIC_DIR, path)))
|
|
104
|
+
.map(([provider, path]) => ({ provider, path }));
|
|
105
|
+
|
|
106
|
+
expect(missingFiles).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
@@ -101,17 +101,20 @@ import {
|
|
|
101
101
|
getRegisteredKinds,
|
|
102
102
|
getResolver,
|
|
103
103
|
} from "../approvals/guardian-request-resolvers.js";
|
|
104
|
+
import { getConfig } from "../config/loader.js";
|
|
104
105
|
import { getDb } from "../persistence/db-connection.js";
|
|
105
106
|
import { getSqlite } from "../persistence/db-connection.js";
|
|
106
107
|
import { initializeDb } from "../persistence/db-init.js";
|
|
107
108
|
import { scopedApprovalGrants } from "../persistence/schema/index.js";
|
|
108
109
|
import {
|
|
110
|
+
TC_GRANT_WAIT_MAX_MS,
|
|
109
111
|
ToolApprovalHandler,
|
|
110
112
|
waitForInlineGrant,
|
|
111
113
|
} from "../tools/tool-approval-handler.js";
|
|
112
114
|
import type { ToolContext } from "../tools/types.js";
|
|
115
|
+
import { setConfig } from "./helpers/set-config.js";
|
|
113
116
|
|
|
114
|
-
/** Short wait config for tests
|
|
117
|
+
/** Short wait config for tests: keeps escalation cases off the real budget. */
|
|
115
118
|
const TEST_INLINE_WAIT_CONFIG = { maxWaitMs: 100, intervalMs: 20 };
|
|
116
119
|
|
|
117
120
|
await initializeDb();
|
|
@@ -490,6 +493,42 @@ describe("inline wait-and-resume", () => {
|
|
|
490
493
|
expect(elapsed).toBeLessThan(500);
|
|
491
494
|
});
|
|
492
495
|
|
|
496
|
+
test("waitForInlineGrant spends timeouts.permissionTimeoutSec when no explicit budget is given", async () => {
|
|
497
|
+
// The guardian's escalation window must track the configured approval
|
|
498
|
+
// budget rather than a compiled-in constant, so an operator raising
|
|
499
|
+
// permissionTimeoutSec actually widens the window a guardian has to
|
|
500
|
+
// answer in. Omitting maxWaitMs is what production does.
|
|
501
|
+
const priorTimeouts = getConfig().timeouts;
|
|
502
|
+
setConfig("timeouts", {
|
|
503
|
+
...priorTimeouts,
|
|
504
|
+
permissionTimeoutSec: 0.15,
|
|
505
|
+
});
|
|
506
|
+
try {
|
|
507
|
+
const req = seedGrantRequest("sha256:configbudget");
|
|
508
|
+
|
|
509
|
+
const start = Date.now();
|
|
510
|
+
const result = await waitForInlineGrant(
|
|
511
|
+
req.id,
|
|
512
|
+
{
|
|
513
|
+
toolName: "bash",
|
|
514
|
+
inputDigest: "sha256:configbudget",
|
|
515
|
+
consumingRequestId: "consume-config-budget",
|
|
516
|
+
},
|
|
517
|
+
{ intervalMs: 20 },
|
|
518
|
+
);
|
|
519
|
+
const elapsed = Date.now() - start;
|
|
520
|
+
|
|
521
|
+
expect(result.outcome).toBe("timeout");
|
|
522
|
+
// Bounded well under TC_GRANT_WAIT_MAX_MS: a regression that ignores
|
|
523
|
+
// config and falls back to the constant blows this ceiling by ~400x.
|
|
524
|
+
expect(elapsed).toBeGreaterThanOrEqual(120);
|
|
525
|
+
expect(elapsed).toBeLessThan(3_000);
|
|
526
|
+
expect(elapsed).toBeLessThan(TC_GRANT_WAIT_MAX_MS);
|
|
527
|
+
} finally {
|
|
528
|
+
setConfig("timeouts", priorTimeouts);
|
|
529
|
+
}
|
|
530
|
+
}, 10_000);
|
|
531
|
+
|
|
493
532
|
test("waitForInlineGrant returns aborted when signal fires during wait", async () => {
|
|
494
533
|
const req = seedGrantRequest("sha256:abortwait");
|
|
495
534
|
|