@vellumai/assistant 0.12.6 → 0.12.7-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.env.example +2 -0
  2. package/ARCHITECTURE.md +14 -14
  3. package/docs/architecture/integrations.md +5 -5
  4. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
  5. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
  6. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
  7. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
  8. package/node_modules/@vellumai/gateway-client/src/__tests__/route-trust-class.test.ts +111 -0
  9. package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -2
  10. package/node_modules/@vellumai/gateway-client/src/index.ts +9 -0
  11. package/node_modules/@vellumai/gateway-client/src/route-trust-class.ts +94 -0
  12. package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
  13. package/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
  14. package/openapi.yaml +153 -5
  15. package/package.json +1 -1
  16. package/src/__tests__/channel-policy.test.ts +7 -0
  17. package/src/__tests__/conversation-routes-disk-view.test.ts +141 -2
  18. package/src/__tests__/identity-routes.test.ts +1 -0
  19. package/src/__tests__/list-messages-client-message-id.test.ts +43 -0
  20. package/src/__tests__/openai-provider.test.ts +27 -0
  21. package/src/__tests__/skill-execute-input.test.ts +11 -0
  22. package/src/__tests__/tool-approval-handler.test.ts +4 -0
  23. package/src/__tests__/tool-preview-lifecycle.test.ts +26 -0
  24. package/src/agent/loop-unparseable-tool-retry.test.ts +333 -0
  25. package/src/agent/loop.ts +133 -12
  26. package/src/calls/__tests__/call-controller.test.ts +273 -1
  27. package/src/calls/__tests__/media-stream-server-integration.test.ts +40 -0
  28. package/src/calls/__tests__/media-stream-stt-session.test.ts +33 -0
  29. package/src/calls/call-constants.ts +10 -0
  30. package/src/calls/call-controller.ts +359 -25
  31. package/src/calls/media-stream-server.ts +18 -0
  32. package/src/calls/media-stream-stt-session.ts +30 -9
  33. package/src/channels/config.ts +8 -0
  34. package/src/cli/lib/bundled-marketplace.json +39 -0
  35. package/src/config/composer-preferences.test.ts +97 -0
  36. package/src/config/composer-preferences.ts +95 -0
  37. package/src/config/feature-flag-registry.json +16 -8
  38. package/src/config/schemas/llm.ts +1 -0
  39. package/src/contacts/types.ts +1 -0
  40. package/src/daemon/conversation-agent-loop-handlers.ts +18 -2
  41. package/src/daemon/conversation-agent-loop.ts +3 -0
  42. package/src/daemon/conversation-messaging.ts +4 -0
  43. package/src/daemon/conversation-process.ts +15 -0
  44. package/src/daemon/conversation-queue-manager.ts +2 -0
  45. package/src/daemon/conversation.ts +2 -0
  46. package/src/daemon/handlers/shared.ts +13 -2
  47. package/src/daemon/message-types/sync.ts +1 -0
  48. package/src/daemon/process-message.ts +5 -0
  49. package/src/i18n/__tests__/i18n.test.ts +25 -0
  50. package/src/i18n/assistant-text.ts +18 -0
  51. package/src/i18n/index.ts +1 -0
  52. package/src/i18n/messages.ts +11 -0
  53. package/src/ipc/routes/__tests__/route-adapter.test.ts +40 -0
  54. package/src/ipc/routes/route-adapter.ts +9 -0
  55. package/src/notifications/README.md +1 -1
  56. package/src/notifications/signal.ts +4 -0
  57. package/src/persistence/conversation-key-store.ts +12 -11
  58. package/src/providers/__tests__/unparseable-tool-args.test.ts +25 -0
  59. package/src/providers/inference/adapter-factory.ts +6 -0
  60. package/src/providers/ionet/client.test.ts +143 -0
  61. package/src/providers/ionet/client.ts +37 -0
  62. package/src/providers/model-catalog.ts +40 -0
  63. package/src/providers/openai/chat-completions-provider.ts +33 -2
  64. package/src/providers/retry.ts +10 -4
  65. package/src/providers/unparseable-tool-args.ts +29 -5
  66. package/src/runtime/__tests__/http-router-trust.test.ts +360 -0
  67. package/src/runtime/__tests__/shared-principal-lookup.test.ts +346 -0
  68. package/src/runtime/auth/__tests__/guard-tests.test.ts +51 -1
  69. package/src/runtime/auth/__tests__/route-policy.test.ts +85 -1
  70. package/src/runtime/auth/__tests__/scopes.test.ts +36 -0
  71. package/src/runtime/auth/route-policy.ts +44 -0
  72. package/src/runtime/auth/scopes.ts +15 -0
  73. package/src/runtime/auth/types.ts +1 -0
  74. package/src/runtime/channel-reply-delivery.ts +2 -0
  75. package/src/runtime/http-router.ts +61 -1
  76. package/src/runtime/http-types.ts +3 -0
  77. package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +35 -10
  78. package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +48 -0
  79. package/src/runtime/migrations/vbundle-builder.ts +19 -3
  80. package/src/runtime/migrations/vbundle-streaming-importer.ts +14 -19
  81. package/src/runtime/migrations/vbundle-streaming-validator.ts +14 -7
  82. package/src/runtime/migrations/vbundle-validator.ts +54 -1
  83. package/src/runtime/routes/composer-settings-routes.test.ts +71 -0
  84. package/src/runtime/routes/composer-settings-routes.ts +76 -0
  85. package/src/runtime/routes/conversation-routes.ts +54 -14
  86. package/src/runtime/routes/identity-routes.ts +2 -0
  87. package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -0
  88. package/src/runtime/routes/index.ts +2 -0
  89. package/src/runtime/routes/migration-routes.ts +3 -1
  90. package/src/runtime/routes/types.ts +4 -1
  91. package/src/runtime/shared-principal-lookup.ts +235 -0
  92. package/src/tools/host-terminal/host-shell.ts +4 -2
  93. package/src/tools/shared/shell-output.test.ts +42 -0
  94. package/src/tools/shared/shell-output.ts +27 -4
  95. package/src/tools/skills/execute.ts +10 -7
  96. package/src/tools/terminal/shell.ts +2 -0
  97. package/src/tools/tool-approval-handler.ts +9 -1
@@ -18,7 +18,10 @@ import {
18
18
  listGuardianRequestDeliveriesOrEmpty,
19
19
  } from "../channels/gateway-guardian-requests.js";
20
20
  import { loadConfig } from "../config/loader.js";
21
- import type { VoiceProgressConfig } from "../config/schemas/voice.js";
21
+ import type {
22
+ VoiceFrontModelConfig,
23
+ VoiceProgressConfig,
24
+ } from "../config/schemas/voice.js";
22
25
  import type { TrustContext } from "../daemon/trust-context-types.js";
23
26
  import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
24
27
  import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
@@ -42,6 +45,7 @@ import {
42
45
  getSilenceTimeoutMs,
43
46
  getUserConsultationTimeoutMs,
44
47
  POST_GOODBYE_HANGUP_DELAY_MS,
48
+ PROVIDER_FINAL_FALLBACK_MARGIN_MS,
45
49
  } from "./call-constants.js";
46
50
  import {
47
51
  formatDuration,
@@ -162,6 +166,31 @@ interface PhoneVoiceLeg {
162
166
  content: string;
163
167
  routingLeg: VoiceRoutingLeg;
164
168
  spokenEscalationBridge?: string;
169
+ /**
170
+ * Teach the front-door leg the hold token, so it may answer the caller's
171
+ * words, hand off, or judge them unfinished. Front-door legs only.
172
+ */
173
+ holdEnabled?: boolean;
174
+ }
175
+
176
+ /**
177
+ * A front-door leg dispatched under the hold verdict, awaiting its decision.
178
+ * Nothing it produced has reached the caller yet, so its words can still be
179
+ * reclaimed into the utterance that eventually answers them.
180
+ */
181
+ interface SpeculativePhoneTurn {
182
+ /** The caller's words this leg is judging, held transcript included. */
183
+ content: string;
184
+ /** `Date.now()` at dispatch, for the endpoint-decision latency mark. */
185
+ dispatchedAtMs: number;
186
+ /**
187
+ * The transcript anchor this leg consumed, restored on a hold so the turn
188
+ * that finally answers is still measured from the moment the caller
189
+ * stopped talking rather than from the replay.
190
+ */
191
+ callerStoppedAtMs: number | null;
192
+ /** The first-partial anchor this leg consumed, restored alongside it. */
193
+ firstPartialAtMs: number | null;
165
194
  }
166
195
 
167
196
  export class CallController {
@@ -270,6 +299,45 @@ export class CallController {
270
299
  * talking rather than the moment the controller got around to dispatching.
271
300
  */
272
301
  private pendingFinalTranscriptAtMs: number | null = null;
302
+ /**
303
+ * When the provider first had words for the utterance being accumulated,
304
+ * held until the turn it dispatches opens. Against that turn's transcript
305
+ * anchor it measures how long the provider sat on speech it had already
306
+ * heard, which is the cost the endpointing work is trading against.
307
+ */
308
+ private pendingFirstPartialAtMs: number | null = null;
309
+ /** Front-door endpointing tuning (voice.frontModel), shared with live voice. */
310
+ private readonly frontModelConfig: VoiceFrontModelConfig;
311
+ /**
312
+ * The caller's words the front door judged unfinished, carried into their
313
+ * next words so a sentence split across provider finals is answered whole.
314
+ * Empty when no hold is outstanding.
315
+ */
316
+ private heldTranscript = "";
317
+ /**
318
+ * The newest interim transcript since the last final: what the caller has
319
+ * said that the provider has not committed yet. A held boundary replays on
320
+ * held transcript plus this, so the question the model answers is the whole
321
+ * one. Cleared by every final, which supersedes it.
322
+ */
323
+ private latestPartialTranscript = "";
324
+ /**
325
+ * Replays a held boundary once the caller stays quiet: the pause the front
326
+ * door read as mid-thought was the end of the thought after all.
327
+ */
328
+ private endpointExtensionTimer: ReturnType<typeof setTimeout> | null = null;
329
+ /**
330
+ * Consecutive hold verdicts on the utterance being accumulated, capped by
331
+ * `endpointMaxExtensions`. Reset when a turn commits.
332
+ */
333
+ private endpointHoldCount = 0;
334
+ /** The front-door leg awaiting its hold verdict, or null. */
335
+ private speculativeTurn: SpeculativePhoneTurn | null = null;
336
+ /**
337
+ * Set when a speculative leg had to be rolled back before the bridge
338
+ * returned its handle. The handle's arrival runs the rollback instead.
339
+ */
340
+ private speculativeDiscardPending = false;
273
341
 
274
342
  constructor(
275
343
  callSessionId: string,
@@ -283,6 +351,7 @@ export class CallController {
283
351
  /** Test seam: an explicit null disables narration for the call. */
284
352
  progressNarrator?: VoiceProgressNarrator | null;
285
353
  progressConfig?: VoiceProgressConfig;
354
+ frontModelConfig?: VoiceFrontModelConfig;
286
355
  },
287
356
  ) {
288
357
  this.callSessionId = callSessionId;
@@ -295,8 +364,10 @@ export class CallController {
295
364
  this.resolveSynthesisLanguage =
296
365
  opts?.resolveSynthesisLanguage ??
297
366
  (() => resolveTelephonySynthesisLanguage());
367
+ this.frontModelConfig =
368
+ opts?.frontModelConfig ?? loadConfig().voice.frontModel;
298
369
  this.progressConfig =
299
- opts?.progressConfig ?? loadConfig().voice.frontModel.progress;
370
+ opts?.progressConfig ?? this.frontModelConfig.progress;
300
371
  this.progressNarrator =
301
372
  opts?.progressNarrator !== undefined
302
373
  ? opts.progressNarrator
@@ -399,16 +470,46 @@ export class CallController {
399
470
  await this.runTurn(CALL_VERIFICATION_COMPLETE_MARKER);
400
471
  }
401
472
 
473
+ /**
474
+ * Handle an interim transcript from the call transport: the caller's words
475
+ * since the last final, as the provider currently hears them.
476
+ *
477
+ * Partials commit nothing. They keep the newest words to hand for a
478
+ * boundary the front door held mid-thought, so a replay of that boundary
479
+ * answers everything the caller has said rather than only what the
480
+ * provider has finalized. Each partial supersedes the last, and any
481
+ * partial re-arms the replay: words are still arriving, so the caller's
482
+ * pause has not started yet, and the re-armed wait now has to clear the
483
+ * provider's own commit window for those words too.
484
+ */
485
+ handleCallerPartial(text: string): void {
486
+ if (this.destroyed) {
487
+ return;
488
+ }
489
+ // The mark belongs to the turn this utterance will dispatch, not to the
490
+ // turn still open behind it, so it is held for that turn's seed.
491
+ this.pendingFirstPartialAtMs ??= Date.now();
492
+ this.latestPartialTranscript = text;
493
+ if (this.endpointExtensionTimer !== null) {
494
+ this.armEndpointExtensionTimer();
495
+ }
496
+ }
497
+
402
498
  /**
403
499
  * Handle a final caller utterance from the call transport.
404
500
  * Caller utterances always trigger normal turns, even when a guardian
405
- * consultation is pending — the consultation is tracked separately.
501
+ * consultation is pending: the consultation is tracked separately.
406
502
  */
407
503
  async handleCallerUtterance(transcript: string): Promise<void> {
408
504
  // Stamped before the teardown waits below, which can run for hundreds of
409
505
  // milliseconds: the caller stopped talking now, not once the prior turn
410
506
  // finished dying.
411
507
  this.pendingFinalTranscriptAtMs = Date.now();
508
+ // This final supersedes the interim text and owns the boundary, so the
509
+ // replay timer for any outstanding hold stands down: the words it would
510
+ // have replayed are in this dispatch.
511
+ this.clearEndpointExtensionTimer();
512
+ this.latestPartialTranscript = "";
412
513
  // If the caller speaks while an END_CALL teardown is pending (during the
413
514
  // drain wait or the listen window), this is a deferral — the caller is
414
515
  // re-engaging after we tried to hang up. Track it so we can cap repeats.
@@ -443,7 +544,10 @@ export class CallController {
443
544
 
444
545
  this.state = "processing";
445
546
  this.resetSilenceTimer();
446
- const callerContent = transcript;
547
+ // Words a hold verdict judged unfinished, and words a leg the caller
548
+ // talked over was still judging, lead the sentence this final completes.
549
+ // Read after the abort above, which is what returns the latter.
550
+ const callerContent = this.withHeldTranscript(transcript);
447
551
  const shouldMarkOpeningAck = this.awaitingOpeningAck;
448
552
  if (shouldMarkOpeningAck) {
449
553
  this.awaitingOpeningAck = false;
@@ -455,7 +559,31 @@ export class CallController {
455
559
  : callerContent;
456
560
 
457
561
  this.lastSentWasOpener = false;
458
- await this.runTurn(callerTurnContent);
562
+ await this.runTurn(callerTurnContent, this.holdAllowed());
563
+ }
564
+
565
+ /**
566
+ * The caller's full sentence so far: anything a hold left outstanding,
567
+ * followed by the words that just arrived. Consumes the held text, which
568
+ * belongs to the dispatch that carries it.
569
+ */
570
+ private withHeldTranscript(transcript: string): string {
571
+ const held = this.heldTranscript;
572
+ this.heldTranscript = "";
573
+ if (held.length === 0) {
574
+ return transcript;
575
+ }
576
+ return transcript.length === 0 ? held : `${held} ${transcript}`;
577
+ }
578
+
579
+ /**
580
+ * Whether the next front-door leg may judge the caller mid-thought. A
581
+ * sentence that has already been held `endpointMaxExtensions` times is
582
+ * answered as it stands: one more hold would read as the assistant
583
+ * refusing to speak.
584
+ */
585
+ private holdAllowed(): boolean {
586
+ return this.endpointHoldCount < this.frontModelConfig.endpointMaxExtensions;
459
587
  }
460
588
 
461
589
  /**
@@ -587,6 +715,13 @@ export class CallController {
587
715
  // Restart silence detection so a barge-in that never yields a
588
716
  // follow-up utterance doesn't leave the call without a watchdog.
589
717
  this.resetSilenceTimer();
718
+ // The abort may have reclaimed the words of a leg still awaiting its
719
+ // verdict. Arm the replay so they are answered even if the caller stops
720
+ // there; their next final cancels it, and an interim transcript re-arms
721
+ // it for as long as they keep talking.
722
+ if (this.heldTranscript.length > 0) {
723
+ this.armEndpointExtensionTimer();
724
+ }
590
725
  }
591
726
 
592
727
  /**
@@ -613,6 +748,7 @@ export class CallController {
613
748
  clearTimeout(this.durationEndTimer);
614
749
  this.durationEndTimer = null;
615
750
  }
751
+ this.clearEndpointExtensionTimer();
616
752
  this.pendingInstructions = [];
617
753
  this.llmRunVersion++;
618
754
  this.abortCurrentTurn();
@@ -658,9 +794,25 @@ export class CallController {
658
794
  * plus the local AbortController for signal propagation.
659
795
  */
660
796
  private abortCurrentTurn(): void {
661
- if (this.currentTurnHandle) {
662
- this.currentTurnHandle.abort();
663
- this.currentTurnHandle = null;
797
+ // A leg still awaiting its hold verdict said nothing to the caller, and
798
+ // its persisted user row holds half a sentence. Roll the row back and
799
+ // keep the words, so they are persisted once, on the turn that answers
800
+ // them. Without a discard the row survives and the sentence is stored
801
+ // twice, so the words are not reclaimed either.
802
+ const speculative = this.speculativeTurn;
803
+ this.speculativeTurn = null;
804
+ const handle = this.currentTurnHandle;
805
+ this.currentTurnHandle = null;
806
+ if (speculative !== null) {
807
+ this.heldTranscript = speculative.content;
808
+ if (handle === null) {
809
+ // The verdict beat the bridge's handle: the leg is still persisting
810
+ // its user row. Latch the rollback so the handle's arrival runs it.
811
+ this.speculativeDiscardPending = true;
812
+ }
813
+ }
814
+ if (handle) {
815
+ this.discardOrAbortLeg(handle, speculative !== null);
664
816
  }
665
817
  this.abortController.abort();
666
818
  this.abortController = new AbortController();
@@ -683,17 +835,168 @@ export class CallController {
683
835
  }
684
836
  }
685
837
 
838
+ /**
839
+ * End a leg: roll its persisted user row back when it was speculative and
840
+ * the bridge offers a discard, else abort it. A bridge without discard
841
+ * degrades to abort, exactly as the handle documents.
842
+ */
843
+ private discardOrAbortLeg(
844
+ handle: VoiceTurnHandle,
845
+ speculative: boolean,
846
+ ): void {
847
+ if (speculative && handle.discard) {
848
+ void handle.discard().catch((err: unknown) => {
849
+ log.warn(
850
+ { err, callSessionId: this.callSessionId },
851
+ "Speculative phone turn discard failed",
852
+ );
853
+ });
854
+ return;
855
+ }
856
+ handle.abort();
857
+ }
858
+
859
+ /**
860
+ * Front-door hold verdict: the caller's words are visibly unfinished, so
861
+ * the leg is thrown away and the call goes back to listening. Nothing was
862
+ * spoken and nothing persists, so the caller hears only the pause they
863
+ * were already in, and their next words are judged together with these.
864
+ *
865
+ * The replay timer covers the case where there are no next words: the
866
+ * caller really did stop, and the sentence is answered as it stands.
867
+ */
868
+ private holdCallerTurn(runVersion: number): void {
869
+ const speculative = this.speculativeTurn;
870
+ if (!this.isCurrentRun(runVersion) || speculative === null) {
871
+ return;
872
+ }
873
+ const latencyMs = Date.now() - speculative.dispatchedAtMs;
874
+ this.markTurnMetric((turnId) =>
875
+ this.metrics.markEndpointDecision(turnId, {
876
+ action: "hold",
877
+ latencyMs,
878
+ source: "front-door",
879
+ }),
880
+ );
881
+ this.cancelMetricsTurn("endpoint_hold");
882
+ this.endpointHoldCount += 1;
883
+ this.pendingFinalTranscriptAtMs = speculative.callerStoppedAtMs;
884
+ this.pendingFirstPartialAtMs = speculative.firstPartialAtMs;
885
+ // Supersede the leg before tearing it down, so its own completion and
886
+ // delta callbacks are inert by the time the abort unwinds them.
887
+ this.llmRunVersion++;
888
+ // Parks the leg's words in heldTranscript and rolls back its user row.
889
+ this.abortCurrentTurn();
890
+ this.state = "idle";
891
+ this.resetSilenceTimer();
892
+ this.armEndpointExtensionTimer();
893
+ log.info(
894
+ {
895
+ callSessionId: this.callSessionId,
896
+ latencyMs,
897
+ holdCount: this.endpointHoldCount,
898
+ },
899
+ "Front door held a phone turn: the caller is mid-thought",
900
+ );
901
+ }
902
+
903
+ /**
904
+ * Arm the replay of a held boundary. Any interim transcript re-arms it
905
+ * (see {@link handleCallerPartial}) and the next final cancels it, so it
906
+ * fires only once the caller has genuinely stopped.
907
+ */
908
+ private armEndpointExtensionTimer(): void {
909
+ this.clearEndpointExtensionTimer();
910
+ this.endpointExtensionTimer = setTimeout(() => {
911
+ this.endpointExtensionTimer = null;
912
+ void this.replayHeldBoundary().catch((err: unknown) => {
913
+ log.error(
914
+ { err, callSessionId: this.callSessionId },
915
+ "Held phone turn replay failed",
916
+ );
917
+ });
918
+ }, this.endpointReplayDelayMs());
919
+ }
920
+
921
+ /**
922
+ * How long the replay waits before answering a held sentence.
923
+ *
924
+ * With no interim transcript outstanding the caller has said nothing the
925
+ * provider still owes a final for, so the hold extension is the whole
926
+ * wait. With one outstanding the provider does owe a final, and that
927
+ * final is the better boundary: it is the committed, corrected text, and
928
+ * it cancels this timer. Racing it would ask the same words twice, so the
929
+ * wait stretches past the provider's own utterance-end window, and firing
930
+ * at all then means no final is coming.
931
+ */
932
+ private endpointReplayDelayMs(): number {
933
+ const extensionMs = this.frontModelConfig.endpointExtensionMs;
934
+ if (this.latestPartialTranscript.length === 0) {
935
+ return extensionMs;
936
+ }
937
+ return Math.max(
938
+ extensionMs,
939
+ loadConfig().calls.voice.utteranceEndMs +
940
+ PROVIDER_FINAL_FALLBACK_MARGIN_MS,
941
+ );
942
+ }
943
+
944
+ private clearEndpointExtensionTimer(): void {
945
+ if (this.endpointExtensionTimer !== null) {
946
+ clearTimeout(this.endpointExtensionTimer);
947
+ this.endpointExtensionTimer = null;
948
+ }
949
+ }
950
+
951
+ /**
952
+ * The caller stayed quiet after a hold, so answer what they said. Any
953
+ * interim transcript rides along: reaching here with one outstanding
954
+ * means the provider is past its own commit window (see
955
+ * {@link endpointReplayDelayMs}), so those words are the only record of
956
+ * what the caller said and dropping them would lose the question.
957
+ */
958
+ private async replayHeldBoundary(): Promise<void> {
959
+ if (this.destroyed) {
960
+ return;
961
+ }
962
+ if (this.state !== "idle") {
963
+ // Something else took the floor (an instruction turn, a nudge). Wait
964
+ // it out rather than dropping the sentence the caller is owed.
965
+ this.armEndpointExtensionTimer();
966
+ return;
967
+ }
968
+ const partial = this.latestPartialTranscript;
969
+ this.latestPartialTranscript = "";
970
+ const content = this.withHeldTranscript(partial);
971
+ if (content.length === 0) {
972
+ return;
973
+ }
974
+ this.state = "processing";
975
+ this.resetSilenceTimer();
976
+ this.lastSentWasOpener = false;
977
+ await this.runTurn(content, this.holdAllowed());
978
+ }
979
+
686
980
  /**
687
981
  * Execute a single voice turn through the conversation pipeline and stream
688
982
  * the response back through the relay.
983
+ *
984
+ * `holdEnabled` marks the turn as the endpoint decision for the caller's
985
+ * words: the front-door leg may judge them unfinished and hand the floor
986
+ * back instead of answering. Only caller speech is dispatched that way;
987
+ * openers, instructions, and nudges are the assistant's own turns and
988
+ * always speak.
689
989
  */
690
- private runTurn(content: string): Promise<void> {
691
- const promise = this.runTurnInner(content);
990
+ private runTurn(content: string, holdEnabled = false): Promise<void> {
991
+ const promise = this.runTurnInner(content, holdEnabled);
692
992
  this.currentTurnPromise = promise;
693
993
  return promise;
694
994
  }
695
995
 
696
- private async runTurnInner(content: string): Promise<void> {
996
+ private async runTurnInner(
997
+ content: string,
998
+ holdEnabled: boolean,
999
+ ): Promise<void> {
697
1000
  if (this.destroyed) {
698
1001
  return;
699
1002
  }
@@ -702,6 +1005,15 @@ export class CallController {
702
1005
  // Stamped before any pre-bridge work so the bridge's dispatch-timing log
703
1006
  // attributes the whole turn, TTS provider resolution included.
704
1007
  const launchedAtMs = Date.now();
1008
+ this.speculativeDiscardPending = false;
1009
+ this.speculativeTurn = holdEnabled
1010
+ ? {
1011
+ content,
1012
+ dispatchedAtMs: launchedAtMs,
1013
+ callerStoppedAtMs: this.pendingFinalTranscriptAtMs,
1014
+ firstPartialAtMs: this.pendingFirstPartialAtMs,
1015
+ }
1016
+ : null;
705
1017
  this.openMetricsTurn(runVersion);
706
1018
 
707
1019
  // Clear silence timer while actively processing. The caller said
@@ -724,6 +1036,7 @@ export class CallController {
724
1036
  runVersion,
725
1037
  runSignal,
726
1038
  launchedAtMs,
1039
+ holdEnabled,
727
1040
  );
728
1041
  if (!this.isCurrentRun(runVersion)) {
729
1042
  return;
@@ -801,15 +1114,16 @@ export class CallController {
801
1114
  * escalated leg on the conversation's own model. Both legs stream through
802
1115
  * the same TTS machinery and share this turn's single end-of-turn signal,
803
1116
  * so the caller never gets a listening window between the bridge and the
804
- * answer. Phone turns commit on the STT provider's utterance-boundary
805
- * final and carry no partial transcript, so the front-door leg never
806
- * holds: routing is escalate-only.
1117
+ * answer. A turn dispatched with `holdEnabled` adds the third verdict: the
1118
+ * caller's words are unfinished, so the leg is discarded and the floor
1119
+ * goes back to them (see {@link holdCallerTurn}).
807
1120
  */
808
1121
  private async streamTtsTokens(
809
1122
  content: string,
810
1123
  runVersion: number,
811
1124
  runSignal: AbortSignal,
812
1125
  launchedAtMs: number,
1126
+ holdEnabled: boolean,
813
1127
  ): Promise<VoiceTurnOutcome> {
814
1128
  // Resolve the active TTS provider through the global abstraction.
815
1129
  // The catalog's callMode determines the call path: synthesized-play
@@ -1194,17 +1508,29 @@ export class CallController {
1194
1508
  // Verdict-first: a front-door leg's leading tokens decide the turn's
1195
1509
  // fate, so its raw stream is a control plane until they classify. The
1196
1510
  // shared coordinator reads it and sequences the hand-off; only text it
1197
- // releases is spoken or recorded. Phone has no partial transcripts,
1198
- // so the hold verdict is never taught.
1511
+ // releases is spoken or recorded. A leg carrying the caller's words is
1512
+ // also the endpoint decision, so it is taught the hold verdict.
1199
1513
  const coordinator: FrontDoorLegCoordinator | null =
1200
1514
  leg.routingLeg === "front-door"
1201
1515
  ? createFrontDoorLegCoordinator({
1202
- holdEnabled: false,
1516
+ holdEnabled: leg.holdEnabled === true,
1203
1517
  host: {
1204
1518
  isLive: () =>
1205
1519
  this.isCurrentRun(runVersion) && !runSignal.aborted,
1206
1520
  language: () => this.resolveSynthesisLanguage(),
1207
1521
  progress: cadence,
1522
+ onHold: () => this.holdCallerTurn(runVersion),
1523
+ commit: () => {
1524
+ if (!this.isCurrentRun(runVersion)) {
1525
+ return false;
1526
+ }
1527
+ // The leg is answering or handing off: the sentence is
1528
+ // settled, so it stops being reclaimable and the hold cap
1529
+ // resets for the caller's next one.
1530
+ this.speculativeTurn = null;
1531
+ this.endpointHoldCount = 0;
1532
+ return true;
1533
+ },
1208
1534
  onAnswerText: (text) => {
1209
1535
  fullResponseText += text;
1210
1536
  flushSafeText(fullResponseText);
@@ -1308,6 +1634,7 @@ export class CallController {
1308
1634
  skipDisclosure: this.skipDisclosure,
1309
1635
  launchedAtMs,
1310
1636
  routingLeg: leg.routingLeg,
1637
+ ...(leg.holdEnabled === true ? { unifiedVerdict: true } : {}),
1311
1638
  ...(leg.spokenEscalationBridge !== undefined
1312
1639
  ? { spokenEscalationBridge: leg.spokenEscalationBridge }
1313
1640
  : {}),
@@ -1329,8 +1656,11 @@ export class CallController {
1329
1656
  }
1330
1657
  } else {
1331
1658
  // Superseded, or the front-door leg handed off before its
1332
- // handle arrived: abort immediately.
1333
- handle.abort();
1659
+ // handle arrived: end it immediately. A leg whose hold verdict
1660
+ // beat this resolution still owes its user row a rollback.
1661
+ const rollback = this.speculativeDiscardPending;
1662
+ this.speculativeDiscardPending = false;
1663
+ this.discardOrAbortLeg(handle, rollback);
1334
1664
  }
1335
1665
  })
1336
1666
  .catch((err) => {
@@ -1385,7 +1715,7 @@ export class CallController {
1385
1715
  // in the turn it occurs; the front-door leg is toolless and quick, so in
1386
1716
  // practice the escalated leg's tool loops are what it covers.
1387
1717
  cadence.arm();
1388
- await runVoiceLeg({ content, routingLeg: "front-door" });
1718
+ await runVoiceLeg({ content, routingLeg: "front-door", holdEnabled });
1389
1719
  if (!this.isCurrentRun(runVersion)) {
1390
1720
  return settleSuperseded();
1391
1721
  }
@@ -2054,16 +2384,20 @@ export class CallController {
2054
2384
  const turnId = this.metricsTurnIdForRun(runVersion);
2055
2385
  const finalTranscriptAtMs = this.pendingFinalTranscriptAtMs;
2056
2386
  this.pendingFinalTranscriptAtMs = null;
2387
+ const firstPartialAtMs = this.pendingFirstPartialAtMs;
2388
+ this.pendingFirstPartialAtMs = null;
2057
2389
  this.metricsTurnId = turnId;
2058
2390
  // The provider's utterance-boundary final is both the transcript and the
2059
2391
  // moment the caller stopped speaking, so it anchors the round trip too;
2060
2392
  // a phone turn has no separate VAD end or push-to-talk release to use.
2061
- this.metrics.startTurn(
2062
- turnId,
2063
- finalTranscriptAtMs !== null
2393
+ // Against it, the first interim transcript of the same utterance is how
2394
+ // long the provider sat on words it had already heard.
2395
+ this.metrics.startTurn(turnId, {
2396
+ ...(finalTranscriptAtMs !== null
2064
2397
  ? { finalTranscriptAtMs, utteranceEndAtMs: finalTranscriptAtMs }
2065
- : {},
2066
- );
2398
+ : {}),
2399
+ ...(firstPartialAtMs !== null ? { firstPartialAtMs } : {}),
2400
+ });
2067
2401
  }
2068
2402
 
2069
2403
  /**
@@ -31,6 +31,9 @@
31
31
  * - STT `onTranscriptFinal` -> routed to the active setup flow's
32
32
  * `pushTranscriptFinal()`, else the controller's
33
33
  * `handleCallerUtterance()`.
34
+ * - STT `onTranscriptPartial` -> the controller's
35
+ * `handleCallerPartial()`, which keeps the newest interim words for a
36
+ * boundary the front door held mid-thought. Never persisted.
34
37
  * - STT `onDtmf` -> routed to the active setup flow's
35
38
  * `pushDtmfDigit()` for code-collection sub-flows.
36
39
  * - STT `onSpeechStart` -> barge-in: clears outbound audio queue
@@ -227,6 +230,7 @@ export class MediaStreamCallSession {
227
230
  onSpeechEnd: () => this.handleSpeechEnd(),
228
231
  onTranscriptFinal: (text, durationMs) =>
229
232
  this.handleTranscriptFinal(text, durationMs),
233
+ onTranscriptPartial: (text) => this.handleTranscriptPartial(text),
230
234
  onDtmf: (digit) => this.handleDtmf(digit),
231
235
  onMark: (name) => this.handleMarkEcho(name),
232
236
  onStop: () => this.handleStreamStop(),
@@ -922,6 +926,20 @@ export class MediaStreamCallSession {
922
926
  });
923
927
  }
924
928
 
929
+ /**
930
+ * Interim transcript from the streaming provider. It is not the caller's
931
+ * committed words, so it is never persisted, never fires the transcript
932
+ * notifier, and never reaches a setup flow: it only tells the controller
933
+ * what has been said since the last final, so a boundary the front door
934
+ * held mid-thought is re-judged on the caller's newest words.
935
+ */
936
+ private handleTranscriptPartial(text: string): void {
937
+ if (this.setupRouting || this.setupFlow || !this.controller) {
938
+ return;
939
+ }
940
+ this.controller.handleCallerPartial(text);
941
+ }
942
+
925
943
  private handleDtmf(digit: string): void {
926
944
  // Drop DTMF arriving while setup routing is still pending so a
927
945
  // not-yet-authorized / floor-denied caller's digit is never persisted
@@ -9,8 +9,10 @@
9
9
  * inbound mu-law audio is decoded to 16 kHz PCM16 and fed to a
10
10
  * {@link StreamingTranscriber} resolved for the configured
11
11
  * `services.stt` provider. Replies trigger on the provider's
12
- * utterance-boundary `final` events; barge-in (`onSpeechStart`) always
13
- * fires from the local energy VAD, never from transcriber partials.
12
+ * utterance-boundary `final` events; interim `partial` events are
13
+ * forwarded as the freshest words spoken since that final but start no
14
+ * turn of their own, and barge-in (`onSpeechStart`) always fires from
15
+ * the local energy VAD, never from transcriber partials.
14
16
  * Frames arriving while the provider session is still starting are
15
17
  * held in a bounded buffer and flushed on start (overflow drops the
16
18
  * oldest frames and is counted + logged at teardown).
@@ -20,10 +22,11 @@
20
22
  * transcriber is available for the configured provider, or the
21
23
  * provider closed the streaming session unexpectedly mid-call.
22
24
  *
23
- * This module is **transport-neutral** — it exposes callback hooks
24
- * (`onSpeechStart`, `onTranscriptFinal`, `onDtmf`, `onStop`) rather than
25
- * driving any call flow itself; `media-stream-server.ts` instantiates and
26
- * connects it to the media-stream WebSocket ingress.
25
+ * This module is **transport-neutral**: it exposes callback hooks
26
+ * (`onSpeechStart`, `onTranscriptFinal`, `onTranscriptPartial`, `onDtmf`,
27
+ * `onStop`) rather than driving any call flow itself;
28
+ * `media-stream-server.ts` instantiates and connects it to the
29
+ * media-stream WebSocket ingress.
27
30
  *
28
31
  * Error handling:
29
32
  * - When the telephony resolver returns a non-supported status, the
@@ -142,6 +145,19 @@ export interface MediaStreamSttSessionCallbacks {
142
145
  */
143
146
  onTranscriptFinal?: (text: string, durationMs: number) => void;
144
147
 
148
+ /**
149
+ * Called for each interim transcript the streaming provider revises on
150
+ * its way to the next final. The text is the provider's running guess at
151
+ * the words spoken since that final, so each call supersedes the last and
152
+ * the run resets when {@link onTranscriptFinal} commits it. Empty
153
+ * partials are suppressed, and batch mode never emits one.
154
+ *
155
+ * Turn taking still runs off the finals: this is the freshest text a
156
+ * boundary the front door held mid-thought can be re-judged on, not a
157
+ * boundary of its own.
158
+ */
159
+ onTranscriptPartial?: (text: string) => void;
160
+
145
161
  /**
146
162
  * Called when a DTMF digit is received from Twilio.
147
163
  */
@@ -542,8 +558,8 @@ export class MediaStreamSttSession {
542
558
  /**
543
559
  * Map streaming transcriber events onto the session callbacks.
544
560
  *
545
- * Partials are ignored for turn-taking: replies trigger only on the
546
- * provider's utterance-boundary finals, and barge-in comes from the
561
+ * Partials are forwarded but own no turn taking: replies trigger only on
562
+ * the provider's utterance-boundary finals, and barge-in comes from the
547
563
  * local VAD.
548
564
  */
549
565
  private handleStreamingEvent(event: SttStreamServerEvent): void {
@@ -552,8 +568,13 @@ export class MediaStreamSttSession {
552
568
  }
553
569
 
554
570
  switch (event.type) {
555
- case "partial":
571
+ case "partial": {
572
+ const text = event.text.trim();
573
+ if (text.length > 0) {
574
+ this.callbacks.onTranscriptPartial?.(text);
575
+ }
556
576
  return;
577
+ }
557
578
  case "finalized":
558
579
  return;
559
580
  case "turn-start":