@vellumai/assistant 0.12.6-staging.3 → 0.12.7-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/ARCHITECTURE.md +14 -14
- package/docs/architecture/integrations.md +5 -5
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/node_modules/@vellumai/gateway-client/src/__tests__/route-trust-class.test.ts +111 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +9 -0
- package/node_modules/@vellumai/gateway-client/src/route-trust-class.ts +94 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/openapi.yaml +153 -5
- package/package.json +1 -1
- package/src/__tests__/channel-policy.test.ts +7 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +141 -2
- package/src/__tests__/identity-routes.test.ts +1 -0
- package/src/__tests__/list-messages-client-message-id.test.ts +43 -0
- package/src/__tests__/openai-provider.test.ts +27 -0
- package/src/__tests__/skill-execute-input.test.ts +11 -0
- package/src/__tests__/tool-approval-handler.test.ts +4 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +26 -0
- package/src/agent/loop-unparseable-tool-retry.test.ts +333 -0
- package/src/agent/loop.ts +133 -12
- package/src/calls/__tests__/call-controller.test.ts +273 -1
- package/src/calls/__tests__/media-stream-server-integration.test.ts +40 -0
- package/src/calls/__tests__/media-stream-stt-session.test.ts +33 -0
- package/src/calls/call-constants.ts +10 -0
- package/src/calls/call-controller.ts +359 -25
- package/src/calls/media-stream-server.ts +18 -0
- package/src/calls/media-stream-stt-session.ts +30 -9
- package/src/channels/config.ts +8 -0
- package/src/cli/lib/bundled-marketplace.json +39 -0
- package/src/config/composer-preferences.test.ts +97 -0
- package/src/config/composer-preferences.ts +95 -0
- package/src/config/feature-flag-registry.json +16 -8
- package/src/config/schemas/llm.ts +1 -0
- package/src/contacts/types.ts +1 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +18 -2
- package/src/daemon/conversation-agent-loop.ts +3 -0
- package/src/daemon/conversation-messaging.ts +4 -0
- package/src/daemon/conversation-process.ts +15 -0
- package/src/daemon/conversation-queue-manager.ts +2 -0
- package/src/daemon/conversation.ts +2 -0
- package/src/daemon/handlers/shared.ts +13 -2
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/process-message.ts +5 -0
- package/src/i18n/__tests__/i18n.test.ts +25 -0
- package/src/i18n/assistant-text.ts +18 -0
- package/src/i18n/index.ts +1 -0
- package/src/i18n/messages.ts +11 -0
- package/src/ipc/routes/__tests__/route-adapter.test.ts +40 -0
- package/src/ipc/routes/route-adapter.ts +9 -0
- package/src/notifications/README.md +1 -1
- package/src/notifications/signal.ts +4 -0
- package/src/persistence/conversation-key-store.ts +12 -11
- package/src/providers/__tests__/unparseable-tool-args.test.ts +25 -0
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/ionet/client.test.ts +143 -0
- package/src/providers/ionet/client.ts +37 -0
- package/src/providers/model-catalog.ts +40 -0
- package/src/providers/openai/chat-completions-provider.ts +33 -2
- package/src/providers/retry.ts +10 -4
- package/src/providers/unparseable-tool-args.ts +29 -5
- package/src/runtime/__tests__/http-router-trust.test.ts +360 -0
- package/src/runtime/__tests__/shared-principal-lookup.test.ts +346 -0
- package/src/runtime/auth/__tests__/guard-tests.test.ts +51 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +85 -1
- package/src/runtime/auth/__tests__/scopes.test.ts +36 -0
- package/src/runtime/auth/route-policy.ts +44 -0
- package/src/runtime/auth/scopes.ts +15 -0
- package/src/runtime/auth/types.ts +1 -0
- package/src/runtime/channel-reply-delivery.ts +2 -0
- package/src/runtime/http-router.ts +61 -1
- package/src/runtime/http-types.ts +3 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +35 -10
- package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +48 -0
- package/src/runtime/migrations/vbundle-builder.ts +19 -3
- package/src/runtime/migrations/vbundle-streaming-importer.ts +14 -19
- package/src/runtime/migrations/vbundle-streaming-validator.ts +14 -7
- package/src/runtime/migrations/vbundle-validator.ts +54 -1
- package/src/runtime/routes/composer-settings-routes.test.ts +71 -0
- package/src/runtime/routes/composer-settings-routes.ts +76 -0
- package/src/runtime/routes/conversation-routes.ts +54 -14
- package/src/runtime/routes/identity-routes.ts +2 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -0
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/migration-routes.ts +3 -1
- package/src/runtime/routes/types.ts +4 -1
- package/src/runtime/shared-principal-lookup.ts +235 -0
- package/src/tools/host-terminal/host-shell.ts +4 -2
- package/src/tools/shared/shell-output.test.ts +42 -0
- package/src/tools/shared/shell-output.ts +27 -4
- package/src/tools/skills/execute.ts +10 -7
- package/src/tools/terminal/shell.ts +2 -0
- package/src/tools/tool-approval-handler.ts +9 -1
|
@@ -18,7 +18,10 @@ import {
|
|
|
18
18
|
listGuardianRequestDeliveriesOrEmpty,
|
|
19
19
|
} from "../channels/gateway-guardian-requests.js";
|
|
20
20
|
import { loadConfig } from "../config/loader.js";
|
|
21
|
-
import type {
|
|
21
|
+
import type {
|
|
22
|
+
VoiceFrontModelConfig,
|
|
23
|
+
VoiceProgressConfig,
|
|
24
|
+
} from "../config/schemas/voice.js";
|
|
22
25
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
23
26
|
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
|
|
24
27
|
import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
|
|
@@ -42,6 +45,7 @@ import {
|
|
|
42
45
|
getSilenceTimeoutMs,
|
|
43
46
|
getUserConsultationTimeoutMs,
|
|
44
47
|
POST_GOODBYE_HANGUP_DELAY_MS,
|
|
48
|
+
PROVIDER_FINAL_FALLBACK_MARGIN_MS,
|
|
45
49
|
} from "./call-constants.js";
|
|
46
50
|
import {
|
|
47
51
|
formatDuration,
|
|
@@ -162,6 +166,31 @@ interface PhoneVoiceLeg {
|
|
|
162
166
|
content: string;
|
|
163
167
|
routingLeg: VoiceRoutingLeg;
|
|
164
168
|
spokenEscalationBridge?: string;
|
|
169
|
+
/**
|
|
170
|
+
* Teach the front-door leg the hold token, so it may answer the caller's
|
|
171
|
+
* words, hand off, or judge them unfinished. Front-door legs only.
|
|
172
|
+
*/
|
|
173
|
+
holdEnabled?: boolean;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* A front-door leg dispatched under the hold verdict, awaiting its decision.
|
|
178
|
+
* Nothing it produced has reached the caller yet, so its words can still be
|
|
179
|
+
* reclaimed into the utterance that eventually answers them.
|
|
180
|
+
*/
|
|
181
|
+
interface SpeculativePhoneTurn {
|
|
182
|
+
/** The caller's words this leg is judging, held transcript included. */
|
|
183
|
+
content: string;
|
|
184
|
+
/** `Date.now()` at dispatch, for the endpoint-decision latency mark. */
|
|
185
|
+
dispatchedAtMs: number;
|
|
186
|
+
/**
|
|
187
|
+
* The transcript anchor this leg consumed, restored on a hold so the turn
|
|
188
|
+
* that finally answers is still measured from the moment the caller
|
|
189
|
+
* stopped talking rather than from the replay.
|
|
190
|
+
*/
|
|
191
|
+
callerStoppedAtMs: number | null;
|
|
192
|
+
/** The first-partial anchor this leg consumed, restored alongside it. */
|
|
193
|
+
firstPartialAtMs: number | null;
|
|
165
194
|
}
|
|
166
195
|
|
|
167
196
|
export class CallController {
|
|
@@ -270,6 +299,45 @@ export class CallController {
|
|
|
270
299
|
* talking rather than the moment the controller got around to dispatching.
|
|
271
300
|
*/
|
|
272
301
|
private pendingFinalTranscriptAtMs: number | null = null;
|
|
302
|
+
/**
|
|
303
|
+
* When the provider first had words for the utterance being accumulated,
|
|
304
|
+
* held until the turn it dispatches opens. Against that turn's transcript
|
|
305
|
+
* anchor it measures how long the provider sat on speech it had already
|
|
306
|
+
* heard, which is the cost the endpointing work is trading against.
|
|
307
|
+
*/
|
|
308
|
+
private pendingFirstPartialAtMs: number | null = null;
|
|
309
|
+
/** Front-door endpointing tuning (voice.frontModel), shared with live voice. */
|
|
310
|
+
private readonly frontModelConfig: VoiceFrontModelConfig;
|
|
311
|
+
/**
|
|
312
|
+
* The caller's words the front door judged unfinished, carried into their
|
|
313
|
+
* next words so a sentence split across provider finals is answered whole.
|
|
314
|
+
* Empty when no hold is outstanding.
|
|
315
|
+
*/
|
|
316
|
+
private heldTranscript = "";
|
|
317
|
+
/**
|
|
318
|
+
* The newest interim transcript since the last final: what the caller has
|
|
319
|
+
* said that the provider has not committed yet. A held boundary replays on
|
|
320
|
+
* held transcript plus this, so the question the model answers is the whole
|
|
321
|
+
* one. Cleared by every final, which supersedes it.
|
|
322
|
+
*/
|
|
323
|
+
private latestPartialTranscript = "";
|
|
324
|
+
/**
|
|
325
|
+
* Replays a held boundary once the caller stays quiet: the pause the front
|
|
326
|
+
* door read as mid-thought was the end of the thought after all.
|
|
327
|
+
*/
|
|
328
|
+
private endpointExtensionTimer: ReturnType<typeof setTimeout> | null = null;
|
|
329
|
+
/**
|
|
330
|
+
* Consecutive hold verdicts on the utterance being accumulated, capped by
|
|
331
|
+
* `endpointMaxExtensions`. Reset when a turn commits.
|
|
332
|
+
*/
|
|
333
|
+
private endpointHoldCount = 0;
|
|
334
|
+
/** The front-door leg awaiting its hold verdict, or null. */
|
|
335
|
+
private speculativeTurn: SpeculativePhoneTurn | null = null;
|
|
336
|
+
/**
|
|
337
|
+
* Set when a speculative leg had to be rolled back before the bridge
|
|
338
|
+
* returned its handle. The handle's arrival runs the rollback instead.
|
|
339
|
+
*/
|
|
340
|
+
private speculativeDiscardPending = false;
|
|
273
341
|
|
|
274
342
|
constructor(
|
|
275
343
|
callSessionId: string,
|
|
@@ -283,6 +351,7 @@ export class CallController {
|
|
|
283
351
|
/** Test seam: an explicit null disables narration for the call. */
|
|
284
352
|
progressNarrator?: VoiceProgressNarrator | null;
|
|
285
353
|
progressConfig?: VoiceProgressConfig;
|
|
354
|
+
frontModelConfig?: VoiceFrontModelConfig;
|
|
286
355
|
},
|
|
287
356
|
) {
|
|
288
357
|
this.callSessionId = callSessionId;
|
|
@@ -295,8 +364,10 @@ export class CallController {
|
|
|
295
364
|
this.resolveSynthesisLanguage =
|
|
296
365
|
opts?.resolveSynthesisLanguage ??
|
|
297
366
|
(() => resolveTelephonySynthesisLanguage());
|
|
367
|
+
this.frontModelConfig =
|
|
368
|
+
opts?.frontModelConfig ?? loadConfig().voice.frontModel;
|
|
298
369
|
this.progressConfig =
|
|
299
|
-
opts?.progressConfig ??
|
|
370
|
+
opts?.progressConfig ?? this.frontModelConfig.progress;
|
|
300
371
|
this.progressNarrator =
|
|
301
372
|
opts?.progressNarrator !== undefined
|
|
302
373
|
? opts.progressNarrator
|
|
@@ -399,16 +470,46 @@ export class CallController {
|
|
|
399
470
|
await this.runTurn(CALL_VERIFICATION_COMPLETE_MARKER);
|
|
400
471
|
}
|
|
401
472
|
|
|
473
|
+
/**
|
|
474
|
+
* Handle an interim transcript from the call transport: the caller's words
|
|
475
|
+
* since the last final, as the provider currently hears them.
|
|
476
|
+
*
|
|
477
|
+
* Partials commit nothing. They keep the newest words to hand for a
|
|
478
|
+
* boundary the front door held mid-thought, so a replay of that boundary
|
|
479
|
+
* answers everything the caller has said rather than only what the
|
|
480
|
+
* provider has finalized. Each partial supersedes the last, and any
|
|
481
|
+
* partial re-arms the replay: words are still arriving, so the caller's
|
|
482
|
+
* pause has not started yet, and the re-armed wait now has to clear the
|
|
483
|
+
* provider's own commit window for those words too.
|
|
484
|
+
*/
|
|
485
|
+
handleCallerPartial(text: string): void {
|
|
486
|
+
if (this.destroyed) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
// The mark belongs to the turn this utterance will dispatch, not to the
|
|
490
|
+
// turn still open behind it, so it is held for that turn's seed.
|
|
491
|
+
this.pendingFirstPartialAtMs ??= Date.now();
|
|
492
|
+
this.latestPartialTranscript = text;
|
|
493
|
+
if (this.endpointExtensionTimer !== null) {
|
|
494
|
+
this.armEndpointExtensionTimer();
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
402
498
|
/**
|
|
403
499
|
* Handle a final caller utterance from the call transport.
|
|
404
500
|
* Caller utterances always trigger normal turns, even when a guardian
|
|
405
|
-
* consultation is pending
|
|
501
|
+
* consultation is pending: the consultation is tracked separately.
|
|
406
502
|
*/
|
|
407
503
|
async handleCallerUtterance(transcript: string): Promise<void> {
|
|
408
504
|
// Stamped before the teardown waits below, which can run for hundreds of
|
|
409
505
|
// milliseconds: the caller stopped talking now, not once the prior turn
|
|
410
506
|
// finished dying.
|
|
411
507
|
this.pendingFinalTranscriptAtMs = Date.now();
|
|
508
|
+
// This final supersedes the interim text and owns the boundary, so the
|
|
509
|
+
// replay timer for any outstanding hold stands down: the words it would
|
|
510
|
+
// have replayed are in this dispatch.
|
|
511
|
+
this.clearEndpointExtensionTimer();
|
|
512
|
+
this.latestPartialTranscript = "";
|
|
412
513
|
// If the caller speaks while an END_CALL teardown is pending (during the
|
|
413
514
|
// drain wait or the listen window), this is a deferral — the caller is
|
|
414
515
|
// re-engaging after we tried to hang up. Track it so we can cap repeats.
|
|
@@ -443,7 +544,10 @@ export class CallController {
|
|
|
443
544
|
|
|
444
545
|
this.state = "processing";
|
|
445
546
|
this.resetSilenceTimer();
|
|
446
|
-
|
|
547
|
+
// Words a hold verdict judged unfinished, and words a leg the caller
|
|
548
|
+
// talked over was still judging, lead the sentence this final completes.
|
|
549
|
+
// Read after the abort above, which is what returns the latter.
|
|
550
|
+
const callerContent = this.withHeldTranscript(transcript);
|
|
447
551
|
const shouldMarkOpeningAck = this.awaitingOpeningAck;
|
|
448
552
|
if (shouldMarkOpeningAck) {
|
|
449
553
|
this.awaitingOpeningAck = false;
|
|
@@ -455,7 +559,31 @@ export class CallController {
|
|
|
455
559
|
: callerContent;
|
|
456
560
|
|
|
457
561
|
this.lastSentWasOpener = false;
|
|
458
|
-
await this.runTurn(callerTurnContent);
|
|
562
|
+
await this.runTurn(callerTurnContent, this.holdAllowed());
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* The caller's full sentence so far: anything a hold left outstanding,
|
|
567
|
+
* followed by the words that just arrived. Consumes the held text, which
|
|
568
|
+
* belongs to the dispatch that carries it.
|
|
569
|
+
*/
|
|
570
|
+
private withHeldTranscript(transcript: string): string {
|
|
571
|
+
const held = this.heldTranscript;
|
|
572
|
+
this.heldTranscript = "";
|
|
573
|
+
if (held.length === 0) {
|
|
574
|
+
return transcript;
|
|
575
|
+
}
|
|
576
|
+
return transcript.length === 0 ? held : `${held} ${transcript}`;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Whether the next front-door leg may judge the caller mid-thought. A
|
|
581
|
+
* sentence that has already been held `endpointMaxExtensions` times is
|
|
582
|
+
* answered as it stands: one more hold would read as the assistant
|
|
583
|
+
* refusing to speak.
|
|
584
|
+
*/
|
|
585
|
+
private holdAllowed(): boolean {
|
|
586
|
+
return this.endpointHoldCount < this.frontModelConfig.endpointMaxExtensions;
|
|
459
587
|
}
|
|
460
588
|
|
|
461
589
|
/**
|
|
@@ -587,6 +715,13 @@ export class CallController {
|
|
|
587
715
|
// Restart silence detection so a barge-in that never yields a
|
|
588
716
|
// follow-up utterance doesn't leave the call without a watchdog.
|
|
589
717
|
this.resetSilenceTimer();
|
|
718
|
+
// The abort may have reclaimed the words of a leg still awaiting its
|
|
719
|
+
// verdict. Arm the replay so they are answered even if the caller stops
|
|
720
|
+
// there; their next final cancels it, and an interim transcript re-arms
|
|
721
|
+
// it for as long as they keep talking.
|
|
722
|
+
if (this.heldTranscript.length > 0) {
|
|
723
|
+
this.armEndpointExtensionTimer();
|
|
724
|
+
}
|
|
590
725
|
}
|
|
591
726
|
|
|
592
727
|
/**
|
|
@@ -613,6 +748,7 @@ export class CallController {
|
|
|
613
748
|
clearTimeout(this.durationEndTimer);
|
|
614
749
|
this.durationEndTimer = null;
|
|
615
750
|
}
|
|
751
|
+
this.clearEndpointExtensionTimer();
|
|
616
752
|
this.pendingInstructions = [];
|
|
617
753
|
this.llmRunVersion++;
|
|
618
754
|
this.abortCurrentTurn();
|
|
@@ -658,9 +794,25 @@ export class CallController {
|
|
|
658
794
|
* plus the local AbortController for signal propagation.
|
|
659
795
|
*/
|
|
660
796
|
private abortCurrentTurn(): void {
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
797
|
+
// A leg still awaiting its hold verdict said nothing to the caller, and
|
|
798
|
+
// its persisted user row holds half a sentence. Roll the row back and
|
|
799
|
+
// keep the words, so they are persisted once, on the turn that answers
|
|
800
|
+
// them. Without a discard the row survives and the sentence is stored
|
|
801
|
+
// twice, so the words are not reclaimed either.
|
|
802
|
+
const speculative = this.speculativeTurn;
|
|
803
|
+
this.speculativeTurn = null;
|
|
804
|
+
const handle = this.currentTurnHandle;
|
|
805
|
+
this.currentTurnHandle = null;
|
|
806
|
+
if (speculative !== null) {
|
|
807
|
+
this.heldTranscript = speculative.content;
|
|
808
|
+
if (handle === null) {
|
|
809
|
+
// The verdict beat the bridge's handle: the leg is still persisting
|
|
810
|
+
// its user row. Latch the rollback so the handle's arrival runs it.
|
|
811
|
+
this.speculativeDiscardPending = true;
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
if (handle) {
|
|
815
|
+
this.discardOrAbortLeg(handle, speculative !== null);
|
|
664
816
|
}
|
|
665
817
|
this.abortController.abort();
|
|
666
818
|
this.abortController = new AbortController();
|
|
@@ -683,17 +835,168 @@ export class CallController {
|
|
|
683
835
|
}
|
|
684
836
|
}
|
|
685
837
|
|
|
838
|
+
/**
|
|
839
|
+
* End a leg: roll its persisted user row back when it was speculative and
|
|
840
|
+
* the bridge offers a discard, else abort it. A bridge without discard
|
|
841
|
+
* degrades to abort, exactly as the handle documents.
|
|
842
|
+
*/
|
|
843
|
+
private discardOrAbortLeg(
|
|
844
|
+
handle: VoiceTurnHandle,
|
|
845
|
+
speculative: boolean,
|
|
846
|
+
): void {
|
|
847
|
+
if (speculative && handle.discard) {
|
|
848
|
+
void handle.discard().catch((err: unknown) => {
|
|
849
|
+
log.warn(
|
|
850
|
+
{ err, callSessionId: this.callSessionId },
|
|
851
|
+
"Speculative phone turn discard failed",
|
|
852
|
+
);
|
|
853
|
+
});
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
handle.abort();
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Front-door hold verdict: the caller's words are visibly unfinished, so
|
|
861
|
+
* the leg is thrown away and the call goes back to listening. Nothing was
|
|
862
|
+
* spoken and nothing persists, so the caller hears only the pause they
|
|
863
|
+
* were already in, and their next words are judged together with these.
|
|
864
|
+
*
|
|
865
|
+
* The replay timer covers the case where there are no next words: the
|
|
866
|
+
* caller really did stop, and the sentence is answered as it stands.
|
|
867
|
+
*/
|
|
868
|
+
private holdCallerTurn(runVersion: number): void {
|
|
869
|
+
const speculative = this.speculativeTurn;
|
|
870
|
+
if (!this.isCurrentRun(runVersion) || speculative === null) {
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
const latencyMs = Date.now() - speculative.dispatchedAtMs;
|
|
874
|
+
this.markTurnMetric((turnId) =>
|
|
875
|
+
this.metrics.markEndpointDecision(turnId, {
|
|
876
|
+
action: "hold",
|
|
877
|
+
latencyMs,
|
|
878
|
+
source: "front-door",
|
|
879
|
+
}),
|
|
880
|
+
);
|
|
881
|
+
this.cancelMetricsTurn("endpoint_hold");
|
|
882
|
+
this.endpointHoldCount += 1;
|
|
883
|
+
this.pendingFinalTranscriptAtMs = speculative.callerStoppedAtMs;
|
|
884
|
+
this.pendingFirstPartialAtMs = speculative.firstPartialAtMs;
|
|
885
|
+
// Supersede the leg before tearing it down, so its own completion and
|
|
886
|
+
// delta callbacks are inert by the time the abort unwinds them.
|
|
887
|
+
this.llmRunVersion++;
|
|
888
|
+
// Parks the leg's words in heldTranscript and rolls back its user row.
|
|
889
|
+
this.abortCurrentTurn();
|
|
890
|
+
this.state = "idle";
|
|
891
|
+
this.resetSilenceTimer();
|
|
892
|
+
this.armEndpointExtensionTimer();
|
|
893
|
+
log.info(
|
|
894
|
+
{
|
|
895
|
+
callSessionId: this.callSessionId,
|
|
896
|
+
latencyMs,
|
|
897
|
+
holdCount: this.endpointHoldCount,
|
|
898
|
+
},
|
|
899
|
+
"Front door held a phone turn: the caller is mid-thought",
|
|
900
|
+
);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* Arm the replay of a held boundary. Any interim transcript re-arms it
|
|
905
|
+
* (see {@link handleCallerPartial}) and the next final cancels it, so it
|
|
906
|
+
* fires only once the caller has genuinely stopped.
|
|
907
|
+
*/
|
|
908
|
+
private armEndpointExtensionTimer(): void {
|
|
909
|
+
this.clearEndpointExtensionTimer();
|
|
910
|
+
this.endpointExtensionTimer = setTimeout(() => {
|
|
911
|
+
this.endpointExtensionTimer = null;
|
|
912
|
+
void this.replayHeldBoundary().catch((err: unknown) => {
|
|
913
|
+
log.error(
|
|
914
|
+
{ err, callSessionId: this.callSessionId },
|
|
915
|
+
"Held phone turn replay failed",
|
|
916
|
+
);
|
|
917
|
+
});
|
|
918
|
+
}, this.endpointReplayDelayMs());
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* How long the replay waits before answering a held sentence.
|
|
923
|
+
*
|
|
924
|
+
* With no interim transcript outstanding the caller has said nothing the
|
|
925
|
+
* provider still owes a final for, so the hold extension is the whole
|
|
926
|
+
* wait. With one outstanding the provider does owe a final, and that
|
|
927
|
+
* final is the better boundary: it is the committed, corrected text, and
|
|
928
|
+
* it cancels this timer. Racing it would ask the same words twice, so the
|
|
929
|
+
* wait stretches past the provider's own utterance-end window, and firing
|
|
930
|
+
* at all then means no final is coming.
|
|
931
|
+
*/
|
|
932
|
+
private endpointReplayDelayMs(): number {
|
|
933
|
+
const extensionMs = this.frontModelConfig.endpointExtensionMs;
|
|
934
|
+
if (this.latestPartialTranscript.length === 0) {
|
|
935
|
+
return extensionMs;
|
|
936
|
+
}
|
|
937
|
+
return Math.max(
|
|
938
|
+
extensionMs,
|
|
939
|
+
loadConfig().calls.voice.utteranceEndMs +
|
|
940
|
+
PROVIDER_FINAL_FALLBACK_MARGIN_MS,
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
private clearEndpointExtensionTimer(): void {
|
|
945
|
+
if (this.endpointExtensionTimer !== null) {
|
|
946
|
+
clearTimeout(this.endpointExtensionTimer);
|
|
947
|
+
this.endpointExtensionTimer = null;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
/**
|
|
952
|
+
* The caller stayed quiet after a hold, so answer what they said. Any
|
|
953
|
+
* interim transcript rides along: reaching here with one outstanding
|
|
954
|
+
* means the provider is past its own commit window (see
|
|
955
|
+
* {@link endpointReplayDelayMs}), so those words are the only record of
|
|
956
|
+
* what the caller said and dropping them would lose the question.
|
|
957
|
+
*/
|
|
958
|
+
private async replayHeldBoundary(): Promise<void> {
|
|
959
|
+
if (this.destroyed) {
|
|
960
|
+
return;
|
|
961
|
+
}
|
|
962
|
+
if (this.state !== "idle") {
|
|
963
|
+
// Something else took the floor (an instruction turn, a nudge). Wait
|
|
964
|
+
// it out rather than dropping the sentence the caller is owed.
|
|
965
|
+
this.armEndpointExtensionTimer();
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
const partial = this.latestPartialTranscript;
|
|
969
|
+
this.latestPartialTranscript = "";
|
|
970
|
+
const content = this.withHeldTranscript(partial);
|
|
971
|
+
if (content.length === 0) {
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
this.state = "processing";
|
|
975
|
+
this.resetSilenceTimer();
|
|
976
|
+
this.lastSentWasOpener = false;
|
|
977
|
+
await this.runTurn(content, this.holdAllowed());
|
|
978
|
+
}
|
|
979
|
+
|
|
686
980
|
/**
|
|
687
981
|
* Execute a single voice turn through the conversation pipeline and stream
|
|
688
982
|
* the response back through the relay.
|
|
983
|
+
*
|
|
984
|
+
* `holdEnabled` marks the turn as the endpoint decision for the caller's
|
|
985
|
+
* words: the front-door leg may judge them unfinished and hand the floor
|
|
986
|
+
* back instead of answering. Only caller speech is dispatched that way;
|
|
987
|
+
* openers, instructions, and nudges are the assistant's own turns and
|
|
988
|
+
* always speak.
|
|
689
989
|
*/
|
|
690
|
-
private runTurn(content: string): Promise<void> {
|
|
691
|
-
const promise = this.runTurnInner(content);
|
|
990
|
+
private runTurn(content: string, holdEnabled = false): Promise<void> {
|
|
991
|
+
const promise = this.runTurnInner(content, holdEnabled);
|
|
692
992
|
this.currentTurnPromise = promise;
|
|
693
993
|
return promise;
|
|
694
994
|
}
|
|
695
995
|
|
|
696
|
-
private async runTurnInner(
|
|
996
|
+
private async runTurnInner(
|
|
997
|
+
content: string,
|
|
998
|
+
holdEnabled: boolean,
|
|
999
|
+
): Promise<void> {
|
|
697
1000
|
if (this.destroyed) {
|
|
698
1001
|
return;
|
|
699
1002
|
}
|
|
@@ -702,6 +1005,15 @@ export class CallController {
|
|
|
702
1005
|
// Stamped before any pre-bridge work so the bridge's dispatch-timing log
|
|
703
1006
|
// attributes the whole turn, TTS provider resolution included.
|
|
704
1007
|
const launchedAtMs = Date.now();
|
|
1008
|
+
this.speculativeDiscardPending = false;
|
|
1009
|
+
this.speculativeTurn = holdEnabled
|
|
1010
|
+
? {
|
|
1011
|
+
content,
|
|
1012
|
+
dispatchedAtMs: launchedAtMs,
|
|
1013
|
+
callerStoppedAtMs: this.pendingFinalTranscriptAtMs,
|
|
1014
|
+
firstPartialAtMs: this.pendingFirstPartialAtMs,
|
|
1015
|
+
}
|
|
1016
|
+
: null;
|
|
705
1017
|
this.openMetricsTurn(runVersion);
|
|
706
1018
|
|
|
707
1019
|
// Clear silence timer while actively processing. The caller said
|
|
@@ -724,6 +1036,7 @@ export class CallController {
|
|
|
724
1036
|
runVersion,
|
|
725
1037
|
runSignal,
|
|
726
1038
|
launchedAtMs,
|
|
1039
|
+
holdEnabled,
|
|
727
1040
|
);
|
|
728
1041
|
if (!this.isCurrentRun(runVersion)) {
|
|
729
1042
|
return;
|
|
@@ -801,15 +1114,16 @@ export class CallController {
|
|
|
801
1114
|
* escalated leg on the conversation's own model. Both legs stream through
|
|
802
1115
|
* the same TTS machinery and share this turn's single end-of-turn signal,
|
|
803
1116
|
* so the caller never gets a listening window between the bridge and the
|
|
804
|
-
* answer.
|
|
805
|
-
*
|
|
806
|
-
*
|
|
1117
|
+
* answer. A turn dispatched with `holdEnabled` adds the third verdict: the
|
|
1118
|
+
* caller's words are unfinished, so the leg is discarded and the floor
|
|
1119
|
+
* goes back to them (see {@link holdCallerTurn}).
|
|
807
1120
|
*/
|
|
808
1121
|
private async streamTtsTokens(
|
|
809
1122
|
content: string,
|
|
810
1123
|
runVersion: number,
|
|
811
1124
|
runSignal: AbortSignal,
|
|
812
1125
|
launchedAtMs: number,
|
|
1126
|
+
holdEnabled: boolean,
|
|
813
1127
|
): Promise<VoiceTurnOutcome> {
|
|
814
1128
|
// Resolve the active TTS provider through the global abstraction.
|
|
815
1129
|
// The catalog's callMode determines the call path: synthesized-play
|
|
@@ -1194,17 +1508,29 @@ export class CallController {
|
|
|
1194
1508
|
// Verdict-first: a front-door leg's leading tokens decide the turn's
|
|
1195
1509
|
// fate, so its raw stream is a control plane until they classify. The
|
|
1196
1510
|
// shared coordinator reads it and sequences the hand-off; only text it
|
|
1197
|
-
// releases is spoken or recorded.
|
|
1198
|
-
//
|
|
1511
|
+
// releases is spoken or recorded. A leg carrying the caller's words is
|
|
1512
|
+
// also the endpoint decision, so it is taught the hold verdict.
|
|
1199
1513
|
const coordinator: FrontDoorLegCoordinator | null =
|
|
1200
1514
|
leg.routingLeg === "front-door"
|
|
1201
1515
|
? createFrontDoorLegCoordinator({
|
|
1202
|
-
holdEnabled:
|
|
1516
|
+
holdEnabled: leg.holdEnabled === true,
|
|
1203
1517
|
host: {
|
|
1204
1518
|
isLive: () =>
|
|
1205
1519
|
this.isCurrentRun(runVersion) && !runSignal.aborted,
|
|
1206
1520
|
language: () => this.resolveSynthesisLanguage(),
|
|
1207
1521
|
progress: cadence,
|
|
1522
|
+
onHold: () => this.holdCallerTurn(runVersion),
|
|
1523
|
+
commit: () => {
|
|
1524
|
+
if (!this.isCurrentRun(runVersion)) {
|
|
1525
|
+
return false;
|
|
1526
|
+
}
|
|
1527
|
+
// The leg is answering or handing off: the sentence is
|
|
1528
|
+
// settled, so it stops being reclaimable and the hold cap
|
|
1529
|
+
// resets for the caller's next one.
|
|
1530
|
+
this.speculativeTurn = null;
|
|
1531
|
+
this.endpointHoldCount = 0;
|
|
1532
|
+
return true;
|
|
1533
|
+
},
|
|
1208
1534
|
onAnswerText: (text) => {
|
|
1209
1535
|
fullResponseText += text;
|
|
1210
1536
|
flushSafeText(fullResponseText);
|
|
@@ -1308,6 +1634,7 @@ export class CallController {
|
|
|
1308
1634
|
skipDisclosure: this.skipDisclosure,
|
|
1309
1635
|
launchedAtMs,
|
|
1310
1636
|
routingLeg: leg.routingLeg,
|
|
1637
|
+
...(leg.holdEnabled === true ? { unifiedVerdict: true } : {}),
|
|
1311
1638
|
...(leg.spokenEscalationBridge !== undefined
|
|
1312
1639
|
? { spokenEscalationBridge: leg.spokenEscalationBridge }
|
|
1313
1640
|
: {}),
|
|
@@ -1329,8 +1656,11 @@ export class CallController {
|
|
|
1329
1656
|
}
|
|
1330
1657
|
} else {
|
|
1331
1658
|
// Superseded, or the front-door leg handed off before its
|
|
1332
|
-
// handle arrived:
|
|
1333
|
-
|
|
1659
|
+
// handle arrived: end it immediately. A leg whose hold verdict
|
|
1660
|
+
// beat this resolution still owes its user row a rollback.
|
|
1661
|
+
const rollback = this.speculativeDiscardPending;
|
|
1662
|
+
this.speculativeDiscardPending = false;
|
|
1663
|
+
this.discardOrAbortLeg(handle, rollback);
|
|
1334
1664
|
}
|
|
1335
1665
|
})
|
|
1336
1666
|
.catch((err) => {
|
|
@@ -1385,7 +1715,7 @@ export class CallController {
|
|
|
1385
1715
|
// in the turn it occurs; the front-door leg is toolless and quick, so in
|
|
1386
1716
|
// practice the escalated leg's tool loops are what it covers.
|
|
1387
1717
|
cadence.arm();
|
|
1388
|
-
await runVoiceLeg({ content, routingLeg: "front-door" });
|
|
1718
|
+
await runVoiceLeg({ content, routingLeg: "front-door", holdEnabled });
|
|
1389
1719
|
if (!this.isCurrentRun(runVersion)) {
|
|
1390
1720
|
return settleSuperseded();
|
|
1391
1721
|
}
|
|
@@ -2054,16 +2384,20 @@ export class CallController {
|
|
|
2054
2384
|
const turnId = this.metricsTurnIdForRun(runVersion);
|
|
2055
2385
|
const finalTranscriptAtMs = this.pendingFinalTranscriptAtMs;
|
|
2056
2386
|
this.pendingFinalTranscriptAtMs = null;
|
|
2387
|
+
const firstPartialAtMs = this.pendingFirstPartialAtMs;
|
|
2388
|
+
this.pendingFirstPartialAtMs = null;
|
|
2057
2389
|
this.metricsTurnId = turnId;
|
|
2058
2390
|
// The provider's utterance-boundary final is both the transcript and the
|
|
2059
2391
|
// moment the caller stopped speaking, so it anchors the round trip too;
|
|
2060
2392
|
// a phone turn has no separate VAD end or push-to-talk release to use.
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2393
|
+
// Against it, the first interim transcript of the same utterance is how
|
|
2394
|
+
// long the provider sat on words it had already heard.
|
|
2395
|
+
this.metrics.startTurn(turnId, {
|
|
2396
|
+
...(finalTranscriptAtMs !== null
|
|
2064
2397
|
? { finalTranscriptAtMs, utteranceEndAtMs: finalTranscriptAtMs }
|
|
2065
|
-
: {},
|
|
2066
|
-
|
|
2398
|
+
: {}),
|
|
2399
|
+
...(firstPartialAtMs !== null ? { firstPartialAtMs } : {}),
|
|
2400
|
+
});
|
|
2067
2401
|
}
|
|
2068
2402
|
|
|
2069
2403
|
/**
|
|
@@ -31,6 +31,9 @@
|
|
|
31
31
|
* - STT `onTranscriptFinal` -> routed to the active setup flow's
|
|
32
32
|
* `pushTranscriptFinal()`, else the controller's
|
|
33
33
|
* `handleCallerUtterance()`.
|
|
34
|
+
* - STT `onTranscriptPartial` -> the controller's
|
|
35
|
+
* `handleCallerPartial()`, which keeps the newest interim words for a
|
|
36
|
+
* boundary the front door held mid-thought. Never persisted.
|
|
34
37
|
* - STT `onDtmf` -> routed to the active setup flow's
|
|
35
38
|
* `pushDtmfDigit()` for code-collection sub-flows.
|
|
36
39
|
* - STT `onSpeechStart` -> barge-in: clears outbound audio queue
|
|
@@ -227,6 +230,7 @@ export class MediaStreamCallSession {
|
|
|
227
230
|
onSpeechEnd: () => this.handleSpeechEnd(),
|
|
228
231
|
onTranscriptFinal: (text, durationMs) =>
|
|
229
232
|
this.handleTranscriptFinal(text, durationMs),
|
|
233
|
+
onTranscriptPartial: (text) => this.handleTranscriptPartial(text),
|
|
230
234
|
onDtmf: (digit) => this.handleDtmf(digit),
|
|
231
235
|
onMark: (name) => this.handleMarkEcho(name),
|
|
232
236
|
onStop: () => this.handleStreamStop(),
|
|
@@ -922,6 +926,20 @@ export class MediaStreamCallSession {
|
|
|
922
926
|
});
|
|
923
927
|
}
|
|
924
928
|
|
|
929
|
+
/**
|
|
930
|
+
* Interim transcript from the streaming provider. It is not the caller's
|
|
931
|
+
* committed words, so it is never persisted, never fires the transcript
|
|
932
|
+
* notifier, and never reaches a setup flow: it only tells the controller
|
|
933
|
+
* what has been said since the last final, so a boundary the front door
|
|
934
|
+
* held mid-thought is re-judged on the caller's newest words.
|
|
935
|
+
*/
|
|
936
|
+
private handleTranscriptPartial(text: string): void {
|
|
937
|
+
if (this.setupRouting || this.setupFlow || !this.controller) {
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
this.controller.handleCallerPartial(text);
|
|
941
|
+
}
|
|
942
|
+
|
|
925
943
|
private handleDtmf(digit: string): void {
|
|
926
944
|
// Drop DTMF arriving while setup routing is still pending so a
|
|
927
945
|
// not-yet-authorized / floor-denied caller's digit is never persisted
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
* inbound mu-law audio is decoded to 16 kHz PCM16 and fed to a
|
|
10
10
|
* {@link StreamingTranscriber} resolved for the configured
|
|
11
11
|
* `services.stt` provider. Replies trigger on the provider's
|
|
12
|
-
* utterance-boundary `final` events;
|
|
13
|
-
*
|
|
12
|
+
* utterance-boundary `final` events; interim `partial` events are
|
|
13
|
+
* forwarded as the freshest words spoken since that final but start no
|
|
14
|
+
* turn of their own, and barge-in (`onSpeechStart`) always fires from
|
|
15
|
+
* the local energy VAD, never from transcriber partials.
|
|
14
16
|
* Frames arriving while the provider session is still starting are
|
|
15
17
|
* held in a bounded buffer and flushed on start (overflow drops the
|
|
16
18
|
* oldest frames and is counted + logged at teardown).
|
|
@@ -20,10 +22,11 @@
|
|
|
20
22
|
* transcriber is available for the configured provider, or the
|
|
21
23
|
* provider closed the streaming session unexpectedly mid-call.
|
|
22
24
|
*
|
|
23
|
-
* This module is **transport-neutral
|
|
24
|
-
* (`onSpeechStart`, `onTranscriptFinal`, `
|
|
25
|
-
* driving any call flow itself;
|
|
26
|
-
* connects it to the
|
|
25
|
+
* This module is **transport-neutral**: it exposes callback hooks
|
|
26
|
+
* (`onSpeechStart`, `onTranscriptFinal`, `onTranscriptPartial`, `onDtmf`,
|
|
27
|
+
* `onStop`) rather than driving any call flow itself;
|
|
28
|
+
* `media-stream-server.ts` instantiates and connects it to the
|
|
29
|
+
* media-stream WebSocket ingress.
|
|
27
30
|
*
|
|
28
31
|
* Error handling:
|
|
29
32
|
* - When the telephony resolver returns a non-supported status, the
|
|
@@ -142,6 +145,19 @@ export interface MediaStreamSttSessionCallbacks {
|
|
|
142
145
|
*/
|
|
143
146
|
onTranscriptFinal?: (text: string, durationMs: number) => void;
|
|
144
147
|
|
|
148
|
+
/**
|
|
149
|
+
* Called for each interim transcript the streaming provider revises on
|
|
150
|
+
* its way to the next final. The text is the provider's running guess at
|
|
151
|
+
* the words spoken since that final, so each call supersedes the last and
|
|
152
|
+
* the run resets when {@link onTranscriptFinal} commits it. Empty
|
|
153
|
+
* partials are suppressed, and batch mode never emits one.
|
|
154
|
+
*
|
|
155
|
+
* Turn taking still runs off the finals: this is the freshest text a
|
|
156
|
+
* boundary the front door held mid-thought can be re-judged on, not a
|
|
157
|
+
* boundary of its own.
|
|
158
|
+
*/
|
|
159
|
+
onTranscriptPartial?: (text: string) => void;
|
|
160
|
+
|
|
145
161
|
/**
|
|
146
162
|
* Called when a DTMF digit is received from Twilio.
|
|
147
163
|
*/
|
|
@@ -542,8 +558,8 @@ export class MediaStreamSttSession {
|
|
|
542
558
|
/**
|
|
543
559
|
* Map streaming transcriber events onto the session callbacks.
|
|
544
560
|
*
|
|
545
|
-
* Partials are
|
|
546
|
-
* provider's utterance-boundary finals, and barge-in comes from the
|
|
561
|
+
* Partials are forwarded but own no turn taking: replies trigger only on
|
|
562
|
+
* the provider's utterance-boundary finals, and barge-in comes from the
|
|
547
563
|
* local VAD.
|
|
548
564
|
*/
|
|
549
565
|
private handleStreamingEvent(event: SttStreamServerEvent): void {
|
|
@@ -552,8 +568,13 @@ export class MediaStreamSttSession {
|
|
|
552
568
|
}
|
|
553
569
|
|
|
554
570
|
switch (event.type) {
|
|
555
|
-
case "partial":
|
|
571
|
+
case "partial": {
|
|
572
|
+
const text = event.text.trim();
|
|
573
|
+
if (text.length > 0) {
|
|
574
|
+
this.callbacks.onTranscriptPartial?.(text);
|
|
575
|
+
}
|
|
556
576
|
return;
|
|
577
|
+
}
|
|
557
578
|
case "finalized":
|
|
558
579
|
return;
|
|
559
580
|
case "turn-start":
|