@vellumai/assistant 0.10.4-dev.202607011307.b79a0c5 → 0.10.4-dev.202607011452.4dbd565
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/call-controller.test.ts +292 -34
- package/src/__tests__/voice-session-bridge-processing-wait.test.ts +27 -0
- package/src/calls/call-controller.ts +75 -8
- package/src/calls/voice-session-bridge.ts +25 -1
- package/src/config/feature-flag-registry.json +8 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +52 -0
- package/src/config/schemas/memory-v3.ts +76 -0
- package/src/daemon/abort-watchdog.ts +7 -0
- package/src/daemon/conversation-agent-loop.ts +19 -21
- package/src/plugins/defaults/memory/v3/__tests__/carry-integration.test.ts +6 -0
- package/src/plugins/defaults/memory/v3/__tests__/dense.test.ts +76 -1
- package/src/plugins/defaults/memory/v3/__tests__/gate-flag.test.ts +56 -0
- package/src/plugins/defaults/memory/v3/__tests__/gate.test.ts +181 -0
- package/src/plugins/defaults/memory/v3/__tests__/live-integration.test.ts +6 -0
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +230 -3
- package/src/plugins/defaults/memory/v3/__tests__/section-needle.test.ts +64 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-integration.test.ts +9 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +43 -1
- package/src/plugins/defaults/memory/v3/dense.ts +44 -9
- package/src/plugins/defaults/memory/v3/gate-flag.ts +15 -0
- package/src/plugins/defaults/memory/v3/gate.ts +158 -0
- package/src/plugins/defaults/memory/v3/orchestrate.ts +154 -25
- package/src/plugins/defaults/memory/v3/section-needle.ts +27 -7
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +8 -0
package/package.json
CHANGED
|
@@ -1025,6 +1025,52 @@ describe("call-controller", () => {
|
|
|
1025
1025
|
controller.destroy();
|
|
1026
1026
|
});
|
|
1027
1027
|
|
|
1028
|
+
test("lock-contention rejection: speaks a brief re-prompt, no technical-issue speech, returns to idle", async () => {
|
|
1029
|
+
mockStartVoiceTurn.mockImplementation(async () => {
|
|
1030
|
+
throw new Error("Conversation is already processing a message");
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
const { relay, controller } = setupController();
|
|
1034
|
+
|
|
1035
|
+
await controller.handleCallerUtterance("Hello");
|
|
1036
|
+
|
|
1037
|
+
// A wedged prior turn must never surface a technical-error message.
|
|
1038
|
+
const errorTokens = relay.sentTokens.filter((t) =>
|
|
1039
|
+
t.token.includes("technical issue"),
|
|
1040
|
+
);
|
|
1041
|
+
expect(errorTokens.length).toBe(0);
|
|
1042
|
+
|
|
1043
|
+
// A brief natural re-prompt (last=true) is spoken so the caller knows to
|
|
1044
|
+
// repeat themselves and the relay transitions back to listening.
|
|
1045
|
+
const reprompts = relay.sentTokens.filter(
|
|
1046
|
+
(t) => t.token === "Sorry, could you say that again?" && t.last === true,
|
|
1047
|
+
);
|
|
1048
|
+
expect(reprompts.length).toBeGreaterThan(0);
|
|
1049
|
+
|
|
1050
|
+
// Controller goes idle and re-arms the silence watchdog.
|
|
1051
|
+
expect(controller.getState()).toBe("idle");
|
|
1052
|
+
|
|
1053
|
+
controller.destroy();
|
|
1054
|
+
});
|
|
1055
|
+
|
|
1056
|
+
test("genuine non-lock-contention error still speaks the technical-issue fallback", async () => {
|
|
1057
|
+
mockStartVoiceTurn.mockImplementation(async () => {
|
|
1058
|
+
throw new Error("boom");
|
|
1059
|
+
});
|
|
1060
|
+
|
|
1061
|
+
const { relay, controller } = setupController();
|
|
1062
|
+
|
|
1063
|
+
await controller.handleCallerUtterance("Hello");
|
|
1064
|
+
|
|
1065
|
+
const errorTokens = relay.sentTokens.filter((t) =>
|
|
1066
|
+
t.token.includes("technical issue"),
|
|
1067
|
+
);
|
|
1068
|
+
expect(errorTokens.length).toBeGreaterThan(0);
|
|
1069
|
+
expect(controller.getState()).toBe("idle");
|
|
1070
|
+
|
|
1071
|
+
controller.destroy();
|
|
1072
|
+
});
|
|
1073
|
+
|
|
1028
1074
|
test("handleUserAnswer: returns false when no pending consultation exists", async () => {
|
|
1029
1075
|
const { controller } = setupController();
|
|
1030
1076
|
|
|
@@ -1054,17 +1100,13 @@ describe("call-controller", () => {
|
|
|
1054
1100
|
onComplete: () => void;
|
|
1055
1101
|
}) => {
|
|
1056
1102
|
return new Promise((resolve) => {
|
|
1057
|
-
//
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
opts.onComplete();
|
|
1061
|
-
resolve({ turnId: "run-1", abort: () => {} });
|
|
1062
|
-
}, 1000);
|
|
1103
|
+
// Emit a token right away so the assistant is actively speaking
|
|
1104
|
+
// (state flips to `speaking`) before the interrupt arrives.
|
|
1105
|
+
opts.onTextDelta("This should be interrupted");
|
|
1063
1106
|
|
|
1064
1107
|
opts.signal?.addEventListener(
|
|
1065
1108
|
"abort",
|
|
1066
1109
|
() => {
|
|
1067
|
-
clearTimeout(timeout);
|
|
1068
1110
|
// In the real system, generation_cancelled triggers
|
|
1069
1111
|
// onComplete via the event sink. The AbortSignal listener
|
|
1070
1112
|
// in call-controller also resolves turnComplete defensively.
|
|
@@ -2123,7 +2165,10 @@ describe("call-controller", () => {
|
|
|
2123
2165
|
"Are you still there?",
|
|
2124
2166
|
);
|
|
2125
2167
|
await new Promise((r) => setTimeout(r, 10));
|
|
2126
|
-
|
|
2168
|
+
// The turn is paused before emitting any token, so the controller is still
|
|
2169
|
+
// in the pre-speech `processing` phase (it only flips to `speaking` once
|
|
2170
|
+
// real outbound audio/tokens begin).
|
|
2171
|
+
expect(controller.getState()).toBe("processing");
|
|
2127
2172
|
|
|
2128
2173
|
// Answer arrives while the controller is processing/speaking
|
|
2129
2174
|
const accepted = await controller.handleUserAnswer("3pm works");
|
|
@@ -2682,6 +2727,65 @@ describe("call-controller", () => {
|
|
|
2682
2727
|
controller.destroy();
|
|
2683
2728
|
});
|
|
2684
2729
|
|
|
2730
|
+
test("synthesized provider: a superseded run does not emit native fallback text", async () => {
|
|
2731
|
+
const cfg = loadConfig();
|
|
2732
|
+
cfg.services.tts.provider = "fish-audio";
|
|
2733
|
+
cfg.services.tts.providers["fish-audio"].referenceId = "fish-ref-123";
|
|
2734
|
+
|
|
2735
|
+
_resetTtsProviderRegistry();
|
|
2736
|
+
// elevenlabs present so native fallback is available for fish-audio.
|
|
2737
|
+
const elevenlabs: TtsProvider = {
|
|
2738
|
+
id: "elevenlabs",
|
|
2739
|
+
capabilities: { supportsStreaming: false, supportedFormats: ["mp3"] },
|
|
2740
|
+
async synthesize() {
|
|
2741
|
+
return { audio: Buffer.from(""), contentType: "audio/mpeg" };
|
|
2742
|
+
},
|
|
2743
|
+
};
|
|
2744
|
+
registerTtsProvider(elevenlabs);
|
|
2745
|
+
|
|
2746
|
+
let releaseSynth: (() => void) | undefined;
|
|
2747
|
+
const synthGate = new Promise<void>((resolve) => {
|
|
2748
|
+
releaseSynth = resolve;
|
|
2749
|
+
});
|
|
2750
|
+
const fishAudioGatedFailing: TtsProvider = {
|
|
2751
|
+
id: "fish-audio",
|
|
2752
|
+
capabilities: {
|
|
2753
|
+
supportsStreaming: true,
|
|
2754
|
+
supportedFormats: ["mp3", "wav", "opus"],
|
|
2755
|
+
},
|
|
2756
|
+
async synthesize() {
|
|
2757
|
+
throw new Error("fish-audio synth failure");
|
|
2758
|
+
},
|
|
2759
|
+
async synthesizeStream() {
|
|
2760
|
+
// Ignore the abort signal, block, then fail with no audio — this would
|
|
2761
|
+
// normally trigger the native fallback text emission.
|
|
2762
|
+
await synthGate;
|
|
2763
|
+
throw new Error("fish-audio stream failure");
|
|
2764
|
+
},
|
|
2765
|
+
};
|
|
2766
|
+
registerTtsProvider(fishAudioGatedFailing);
|
|
2767
|
+
|
|
2768
|
+
mockStartVoiceTurn.mockImplementation(
|
|
2769
|
+
createMockVoiceTurn(["Stale synthesized response"]),
|
|
2770
|
+
);
|
|
2771
|
+
const { relay, controller } = setupController();
|
|
2772
|
+
|
|
2773
|
+
const turn = controller.handleCallerUtterance("Hi");
|
|
2774
|
+
await new Promise((r) => setTimeout(r, 20)); // reach synthesis, blocked
|
|
2775
|
+
|
|
2776
|
+
// Supersede the turn mid-synthesis, then let synthesis fail.
|
|
2777
|
+
controller.handleInterrupt();
|
|
2778
|
+
releaseSynth?.();
|
|
2779
|
+
await turn;
|
|
2780
|
+
|
|
2781
|
+
// The stale run must NOT leak its native fallback text into the relay.
|
|
2782
|
+
const emitted = relay.sentTokens.map((t) => t.token).join("");
|
|
2783
|
+
expect(emitted).not.toContain("Stale synthesized response");
|
|
2784
|
+
expect(relay.sentPlayUrls.length).toBe(0);
|
|
2785
|
+
|
|
2786
|
+
controller.destroy();
|
|
2787
|
+
});
|
|
2788
|
+
|
|
2685
2789
|
test("synthesized provider: play URL uses public base URL", async () => {
|
|
2686
2790
|
const cfg = loadConfig();
|
|
2687
2791
|
cfg.ingress.publicBaseUrl = "https://twilio.example.com/";
|
|
@@ -2726,6 +2830,113 @@ describe("call-controller", () => {
|
|
|
2726
2830
|
controller.destroy();
|
|
2727
2831
|
});
|
|
2728
2832
|
|
|
2833
|
+
test("synthesized provider: stays 'processing' during synthesis latency, so barge-in cannot abort an inaudible turn", async () => {
|
|
2834
|
+
const cfg = loadConfig();
|
|
2835
|
+
cfg.services.tts.provider = "fish-audio";
|
|
2836
|
+
cfg.services.tts.providers["fish-audio"].referenceId = "fish-ref-123";
|
|
2837
|
+
|
|
2838
|
+
_resetTtsProviderRegistry();
|
|
2839
|
+
let releaseChunk: (() => void) | undefined;
|
|
2840
|
+
const chunkGate = new Promise<void>((resolve) => {
|
|
2841
|
+
releaseChunk = resolve;
|
|
2842
|
+
});
|
|
2843
|
+
const fishAudioStreaming: TtsProvider = {
|
|
2844
|
+
id: "fish-audio",
|
|
2845
|
+
capabilities: {
|
|
2846
|
+
supportsStreaming: true,
|
|
2847
|
+
supportedFormats: ["mp3", "wav", "opus"],
|
|
2848
|
+
},
|
|
2849
|
+
async synthesize() {
|
|
2850
|
+
return { audio: Buffer.from("x"), contentType: "audio/mpeg" };
|
|
2851
|
+
},
|
|
2852
|
+
async synthesizeStream(_request, onChunk) {
|
|
2853
|
+
// Simulate provider latency: emit no audio until the gate releases.
|
|
2854
|
+
await chunkGate;
|
|
2855
|
+
onChunk(Buffer.from("fish-audio-stream"));
|
|
2856
|
+
return {
|
|
2857
|
+
audio: Buffer.from("fish-audio-stream"),
|
|
2858
|
+
contentType: "audio/mpeg",
|
|
2859
|
+
};
|
|
2860
|
+
},
|
|
2861
|
+
};
|
|
2862
|
+
registerTtsProvider(fishAudioStreaming);
|
|
2863
|
+
|
|
2864
|
+
mockStartVoiceTurn.mockImplementation(
|
|
2865
|
+
createMockVoiceTurn(["Hello from synthesized path."]),
|
|
2866
|
+
);
|
|
2867
|
+
const { relay, controller } = setupController();
|
|
2868
|
+
|
|
2869
|
+
const turn = controller.handleCallerUtterance("Hi");
|
|
2870
|
+
// Let the turn generate its text and reach synthesis, blocked on the gate.
|
|
2871
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
2872
|
+
|
|
2873
|
+
// No audio has reached the caller yet (play URL not sent), so the controller
|
|
2874
|
+
// must stay in `processing` — a barge-in here would abort a turn the caller
|
|
2875
|
+
// cannot hear. See JARVIS-1232.
|
|
2876
|
+
expect(relay.sentPlayUrls.length).toBe(0);
|
|
2877
|
+
expect(controller.getState()).toBe("processing");
|
|
2878
|
+
expect(controller.handleBargeIn()).toBe(false);
|
|
2879
|
+
|
|
2880
|
+
// Release audio → play URL sent → turn finishes and returns to idle.
|
|
2881
|
+
releaseChunk?.();
|
|
2882
|
+
await turn;
|
|
2883
|
+
expect(relay.sentPlayUrls.length).toBeGreaterThan(0);
|
|
2884
|
+
expect(controller.getState()).toBe("idle");
|
|
2885
|
+
|
|
2886
|
+
controller.destroy();
|
|
2887
|
+
});
|
|
2888
|
+
|
|
2889
|
+
test("synthesized provider: a superseded turn does not send a stale play URL", async () => {
|
|
2890
|
+
const cfg = loadConfig();
|
|
2891
|
+
cfg.services.tts.provider = "fish-audio";
|
|
2892
|
+
cfg.services.tts.providers["fish-audio"].referenceId = "fish-ref-123";
|
|
2893
|
+
|
|
2894
|
+
_resetTtsProviderRegistry();
|
|
2895
|
+
let releaseChunk: (() => void) | undefined;
|
|
2896
|
+
const chunkGate = new Promise<void>((resolve) => {
|
|
2897
|
+
releaseChunk = resolve;
|
|
2898
|
+
});
|
|
2899
|
+
const fishAudioStreaming: TtsProvider = {
|
|
2900
|
+
id: "fish-audio",
|
|
2901
|
+
capabilities: {
|
|
2902
|
+
supportsStreaming: true,
|
|
2903
|
+
supportedFormats: ["mp3", "wav", "opus"],
|
|
2904
|
+
},
|
|
2905
|
+
async synthesize() {
|
|
2906
|
+
return { audio: Buffer.from("x"), contentType: "audio/mpeg" };
|
|
2907
|
+
},
|
|
2908
|
+
async synthesizeStream(_request, onChunk) {
|
|
2909
|
+
// Block until released so we can supersede the turn mid-synthesis.
|
|
2910
|
+
await chunkGate;
|
|
2911
|
+
onChunk(Buffer.from("stale-audio"));
|
|
2912
|
+
return { audio: Buffer.from("stale-audio"), contentType: "audio/mpeg" };
|
|
2913
|
+
},
|
|
2914
|
+
};
|
|
2915
|
+
registerTtsProvider(fishAudioStreaming);
|
|
2916
|
+
|
|
2917
|
+
mockStartVoiceTurn.mockImplementation(
|
|
2918
|
+
createMockVoiceTurn(["Hello from synthesized path."]),
|
|
2919
|
+
);
|
|
2920
|
+
const { relay, controller } = setupController();
|
|
2921
|
+
|
|
2922
|
+
const turn = controller.handleCallerUtterance("first");
|
|
2923
|
+
// Let the turn generate text and block inside provider synthesis.
|
|
2924
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
2925
|
+
expect(relay.sentPlayUrls.length).toBe(0);
|
|
2926
|
+
|
|
2927
|
+
// Supersede the in-flight synthesized turn (bumps run version + aborts synthesis).
|
|
2928
|
+
controller.handleInterrupt();
|
|
2929
|
+
|
|
2930
|
+
// Releasing the stale synthesis must NOT send a play URL for the old run,
|
|
2931
|
+
// nor inject a stale end-of-turn marker into the relay stream.
|
|
2932
|
+
releaseChunk?.();
|
|
2933
|
+
await turn;
|
|
2934
|
+
expect(relay.sentPlayUrls.length).toBe(0);
|
|
2935
|
+
expect(relay.sentTokens.filter((t) => t.last).length).toBe(0);
|
|
2936
|
+
|
|
2937
|
+
controller.destroy();
|
|
2938
|
+
});
|
|
2939
|
+
|
|
2729
2940
|
test("Deepgram selected path resolves useSynthesizedPath to true", () => {
|
|
2730
2941
|
const cfg = loadConfig();
|
|
2731
2942
|
cfg.services.tts.provider = "deepgram";
|
|
@@ -2841,16 +3052,13 @@ describe("call-controller", () => {
|
|
|
2841
3052
|
onComplete: () => void;
|
|
2842
3053
|
}) => {
|
|
2843
3054
|
return new Promise((resolve) => {
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
resolve({ turnId: "run-1", abort: () => {} });
|
|
2848
|
-
}, 1000);
|
|
3055
|
+
// Emit a token right away so the assistant is actively speaking
|
|
3056
|
+
// before the interrupt arrives.
|
|
3057
|
+
opts.onTextDelta("This should be interrupted");
|
|
2849
3058
|
|
|
2850
3059
|
opts.signal?.addEventListener(
|
|
2851
3060
|
"abort",
|
|
2852
3061
|
() => {
|
|
2853
|
-
clearTimeout(timeout);
|
|
2854
3062
|
opts.onComplete();
|
|
2855
3063
|
resolve({ turnId: "run-1", abort: () => {} });
|
|
2856
3064
|
},
|
|
@@ -2972,49 +3180,99 @@ describe("call-controller", () => {
|
|
|
2972
3180
|
controller.destroy();
|
|
2973
3181
|
});
|
|
2974
3182
|
|
|
2975
|
-
test("handleBargeIn returns false
|
|
2976
|
-
//
|
|
2977
|
-
//
|
|
3183
|
+
test("handleBargeIn returns false and does not abort while still processing (no output yet)", async () => {
|
|
3184
|
+
// Simulate a turn stuck waiting for the processing lock: no tokens
|
|
3185
|
+
// emitted, no completion. The controller must stay in `processing` and
|
|
3186
|
+
// NOT flip to `speaking`, so barge-in can't abort a silent turn.
|
|
2978
3187
|
mockStartVoiceTurn.mockImplementation(
|
|
2979
3188
|
async (opts: {
|
|
2980
3189
|
onTextDelta: (t: string) => void;
|
|
2981
3190
|
onComplete: () => void;
|
|
2982
3191
|
signal?: AbortSignal;
|
|
2983
3192
|
}) => {
|
|
2984
|
-
//
|
|
3193
|
+
// Never emit tokens or complete — remain in the pre-speech phase.
|
|
2985
3194
|
return { turnId: "run-slow", abort: () => opts.onComplete() };
|
|
2986
3195
|
},
|
|
2987
3196
|
);
|
|
2988
3197
|
|
|
2989
3198
|
const { relay, controller } = setupController();
|
|
2990
|
-
// Kick off a turn (moves to speaking state)
|
|
2991
3199
|
const turnPromise = controller.handleCallerUtterance("Hello");
|
|
2992
3200
|
|
|
2993
3201
|
// Wait for microtasks to settle
|
|
2994
3202
|
for (let i = 0; i < 5; i++) await Promise.resolve();
|
|
2995
3203
|
|
|
2996
|
-
//
|
|
2997
|
-
|
|
2998
|
-
// But if no text has been emitted yet, the state is "speaking" per
|
|
2999
|
-
// the implementation (state is set to speaking at the start of
|
|
3000
|
-
// runTurnInner). So handleBargeIn should accept. Let's verify the
|
|
3001
|
-
// state and behavior.
|
|
3002
|
-
const bargeResult = controller.handleBargeIn();
|
|
3204
|
+
// No outbound audio/tokens yet → still processing.
|
|
3205
|
+
expect(controller.getState()).toBe("processing");
|
|
3003
3206
|
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
)
|
|
3010
|
-
|
|
3011
|
-
|
|
3207
|
+
const bargeResult = controller.handleBargeIn();
|
|
3208
|
+
expect(bargeResult).toBe(false);
|
|
3209
|
+
// Still processing (not aborted), and no interrupt/end-of-turn token sent.
|
|
3210
|
+
expect(controller.getState()).toBe("processing");
|
|
3211
|
+
const endTokens = relay.sentTokens.filter(
|
|
3212
|
+
(t) => t.last === true && t.token === "",
|
|
3213
|
+
);
|
|
3214
|
+
expect(endTokens.length).toBe(0);
|
|
3012
3215
|
|
|
3013
3216
|
// Cleanup: abort the pending turn
|
|
3014
3217
|
controller.destroy();
|
|
3015
3218
|
await turnPromise.catch(() => {});
|
|
3016
3219
|
});
|
|
3017
3220
|
|
|
3221
|
+
test("stays in processing until first token, then flips to speaking and barge-in is accepted", async () => {
|
|
3222
|
+
// Gate the first token so we can observe the pre-speech processing phase
|
|
3223
|
+
// (e.g. the lock-wait / generation window) before any audio is emitted.
|
|
3224
|
+
let emitFirstToken!: () => void;
|
|
3225
|
+
const firstTokenGate = new Promise<void>((r) => {
|
|
3226
|
+
emitFirstToken = r;
|
|
3227
|
+
});
|
|
3228
|
+
let releaseTurn!: () => void;
|
|
3229
|
+
const completeGate = new Promise<void>((r) => {
|
|
3230
|
+
releaseTurn = r;
|
|
3231
|
+
});
|
|
3232
|
+
|
|
3233
|
+
mockStartVoiceTurn.mockImplementation(
|
|
3234
|
+
async (opts: {
|
|
3235
|
+
onTextDelta: (t: string) => void;
|
|
3236
|
+
onComplete: () => void;
|
|
3237
|
+
signal?: AbortSignal;
|
|
3238
|
+
}) => {
|
|
3239
|
+
await firstTokenGate;
|
|
3240
|
+
if (opts.signal?.aborted) {
|
|
3241
|
+
const err = new Error("aborted");
|
|
3242
|
+
err.name = "AbortError";
|
|
3243
|
+
throw err;
|
|
3244
|
+
}
|
|
3245
|
+
opts.onTextDelta("Hello");
|
|
3246
|
+
// Hold the turn open so state remains `speaking` until we barge in.
|
|
3247
|
+
opts.signal?.addEventListener("abort", () => releaseTurn());
|
|
3248
|
+
await completeGate;
|
|
3249
|
+
opts.onComplete();
|
|
3250
|
+
return { turnId: "run-gate", abort: () => releaseTurn() };
|
|
3251
|
+
},
|
|
3252
|
+
);
|
|
3253
|
+
|
|
3254
|
+
const { controller } = setupController();
|
|
3255
|
+
const turnPromise = controller.handleCallerUtterance("Hi");
|
|
3256
|
+
for (let i = 0; i < 5; i++) await Promise.resolve();
|
|
3257
|
+
|
|
3258
|
+
// Before any token: processing, barge-in ignored (turn not aborted).
|
|
3259
|
+
expect(controller.getState()).toBe("processing");
|
|
3260
|
+
expect(controller.handleBargeIn()).toBe(false);
|
|
3261
|
+
expect(controller.getState()).toBe("processing");
|
|
3262
|
+
|
|
3263
|
+
// Release the first token → controller flips to speaking.
|
|
3264
|
+
emitFirstToken();
|
|
3265
|
+
for (let i = 0; i < 5; i++) await Promise.resolve();
|
|
3266
|
+
expect(controller.getState()).toBe("speaking");
|
|
3267
|
+
|
|
3268
|
+
// Now barge-in is accepted and interrupts the turn.
|
|
3269
|
+
expect(controller.handleBargeIn()).toBe(true);
|
|
3270
|
+
expect(controller.getState()).toBe("idle");
|
|
3271
|
+
|
|
3272
|
+
controller.destroy();
|
|
3273
|
+
await turnPromise.catch(() => {});
|
|
3274
|
+
});
|
|
3275
|
+
|
|
3018
3276
|
test("handleBargeIn returns true and interrupts when controller is speaking", async () => {
|
|
3019
3277
|
// Create a turn that holds the speaking state long enough to test barge-in
|
|
3020
3278
|
let resolveComplete: () => void;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Mocks — declared before importing voice-session-bridge so its module-level
|
|
5
|
+
// imports (logger, config loader) stay side-effect free for this pure helper.
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
|
|
8
|
+
mock.module("../util/logger.js", () => ({
|
|
9
|
+
getLogger: () =>
|
|
10
|
+
new Proxy({} as Record<string, unknown>, {
|
|
11
|
+
get: () => () => {},
|
|
12
|
+
}),
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
mock.module("../config/loader.js", () => ({
|
|
16
|
+
getConfig: () => ({}),
|
|
17
|
+
}));
|
|
18
|
+
|
|
19
|
+
import { resolveProcessingWaitMs } from "../calls/voice-session-bridge.js";
|
|
20
|
+
|
|
21
|
+
describe("resolveProcessingWaitMs", () => {
|
|
22
|
+
test("sums commit window, abort unwind budget, and fixed margin", () => {
|
|
23
|
+
expect(resolveProcessingWaitMs(4000, 5000)).toBe(10000);
|
|
24
|
+
expect(resolveProcessingWaitMs(1000, 5000)).toBe(7000);
|
|
25
|
+
expect(resolveProcessingWaitMs(4000, 0)).toBe(5000);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
@@ -395,11 +395,6 @@ export class CallController {
|
|
|
395
395
|
const wasSpeaking = this.state === "speaking";
|
|
396
396
|
this.abortCurrentTurn();
|
|
397
397
|
this.llmRunVersion++;
|
|
398
|
-
// Cancel in-flight synthesized TTS on barge-in
|
|
399
|
-
if (this.activeSynthesisAbort) {
|
|
400
|
-
this.activeSynthesisAbort.abort();
|
|
401
|
-
this.activeSynthesisAbort = null;
|
|
402
|
-
}
|
|
403
398
|
// Explicitly terminate the in-progress TTS turn so the relay can
|
|
404
399
|
// immediately hand control back to the caller after barge-in.
|
|
405
400
|
if (wasSpeaking) {
|
|
@@ -483,6 +478,12 @@ export class CallController {
|
|
|
483
478
|
}
|
|
484
479
|
this.abortController.abort();
|
|
485
480
|
this.abortController = new AbortController();
|
|
481
|
+
// Abort any in-flight synthesized-TTS playback too, so a superseded or
|
|
482
|
+
// torn-down turn's audio isn't streamed to the caller after they move on.
|
|
483
|
+
if (this.activeSynthesisAbort) {
|
|
484
|
+
this.activeSynthesisAbort.abort();
|
|
485
|
+
this.activeSynthesisAbort = null;
|
|
486
|
+
}
|
|
486
487
|
}
|
|
487
488
|
|
|
488
489
|
private formatCallerUtterance(
|
|
@@ -523,7 +524,10 @@ export class CallController {
|
|
|
523
524
|
}
|
|
524
525
|
|
|
525
526
|
try {
|
|
526
|
-
|
|
527
|
+
// Stay in `processing` through the lock-wait and LLM generation; flip to
|
|
528
|
+
// `speaking` only when real outbound audio/tokens start (see
|
|
529
|
+
// beginSpeaking). This keeps barge-in from aborting a silent turn.
|
|
530
|
+
this.state = "processing";
|
|
527
531
|
|
|
528
532
|
const fullResponseText = await this.streamTtsTokens(
|
|
529
533
|
content,
|
|
@@ -561,6 +565,21 @@ export class CallController {
|
|
|
561
565
|
);
|
|
562
566
|
return;
|
|
563
567
|
}
|
|
568
|
+
if (this.isLockContentionError(err) && this.isCurrentRun(runVersion)) {
|
|
569
|
+
log.debug(
|
|
570
|
+
{ callSessionId: this.callSessionId },
|
|
571
|
+
"Prior voice turn wedged past lock-hold budget; re-prompting caller",
|
|
572
|
+
);
|
|
573
|
+
// Reaching here means the prior turn is genuinely wedged past the full
|
|
574
|
+
// lock-hold wait budget, so surface a brief natural re-prompt (never a
|
|
575
|
+
// technical-error message) and re-arm listening. last=true doubles as
|
|
576
|
+
// the end-of-turn marker.
|
|
577
|
+
this.transport.sendTextToken("Sorry, could you say that again?", true);
|
|
578
|
+
this.state = "idle";
|
|
579
|
+
this.resetSilenceTimer();
|
|
580
|
+
this.flushPendingInstructions();
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
564
583
|
log.error({ err, callSessionId: this.callSessionId }, "Voice turn error");
|
|
565
584
|
this.transport.sendTextToken(
|
|
566
585
|
"I'm sorry, I encountered a technical issue. Could you repeat that?",
|
|
@@ -613,6 +632,7 @@ export class CallController {
|
|
|
613
632
|
if (useSynthesizedPath) {
|
|
614
633
|
synthesizedTextBuffer += cleaned;
|
|
615
634
|
} else {
|
|
635
|
+
this.beginSpeaking(runVersion);
|
|
616
636
|
this.transport.sendTextToken(cleaned, false);
|
|
617
637
|
}
|
|
618
638
|
};
|
|
@@ -732,6 +752,10 @@ export class CallController {
|
|
|
732
752
|
const sanitizedSynthText = sanitizeForTts(synthesizedTextBuffer.trim());
|
|
733
753
|
if (useSynthesizedPath && provider && sanitizedSynthText.length > 0) {
|
|
734
754
|
if (!this.isCurrentRun(runVersion)) return fullResponseText;
|
|
755
|
+
// Do NOT flip to `speaking` here — provider synthesis latency (or the
|
|
756
|
+
// no-audio fallback window) would still be silent. The transition happens
|
|
757
|
+
// inside synthesizeAndStreamAudio when the play URL / first audio chunk
|
|
758
|
+
// (or native fallback token) is actually emitted.
|
|
735
759
|
await this.synthesizeAndStreamAudio(
|
|
736
760
|
provider,
|
|
737
761
|
sanitizedSynthText,
|
|
@@ -740,6 +764,12 @@ export class CallController {
|
|
|
740
764
|
);
|
|
741
765
|
}
|
|
742
766
|
|
|
767
|
+
// Synthesized playback (and its native fallback) can await provider
|
|
768
|
+
// latency; re-check the run wasn't superseded meanwhile so a stale turn
|
|
769
|
+
// doesn't inject its end-of-turn marker (or fallback text) into the next
|
|
770
|
+
// turn's relay stream.
|
|
771
|
+
if (!this.isCurrentRun(runVersion)) return fullResponseText;
|
|
772
|
+
|
|
743
773
|
// Signal end of this turn's speech. An empty token with `last: true`
|
|
744
774
|
// tells ConversationRelay to start listening — it does NOT trigger TTS
|
|
745
775
|
// synthesis. This is required even when a synthesized provider handled
|
|
@@ -764,7 +794,7 @@ export class CallController {
|
|
|
764
794
|
private async synthesizeAndStreamAudio(
|
|
765
795
|
provider: TtsProvider,
|
|
766
796
|
text: string,
|
|
767
|
-
|
|
797
|
+
runVersion: number,
|
|
768
798
|
format: "mp3" | "wav" | "opus" = "mp3",
|
|
769
799
|
): Promise<void> {
|
|
770
800
|
let handle: ReturnType<typeof createStreamingEntry> | null = null;
|
|
@@ -788,6 +818,12 @@ export class CallController {
|
|
|
788
818
|
const url = `${baseUrl}/v1/audio/${handle.audioId}`;
|
|
789
819
|
const sendPlayUrlOnce = (): void => {
|
|
790
820
|
if (playUrlSent) return;
|
|
821
|
+
// Superseded/aborted while synthesis was pending — don't start playing
|
|
822
|
+
// a stale response after the caller has already moved on.
|
|
823
|
+
if (!this.isCurrentRun(runVersion)) return;
|
|
824
|
+
// Audio is now actually reaching the caller — flip to `speaking` so
|
|
825
|
+
// barge-in can interrupt (it stays `processing` until this point).
|
|
826
|
+
this.beginSpeaking(runVersion);
|
|
791
827
|
this.transport.sendPlayUrl(url);
|
|
792
828
|
playUrlSent = true;
|
|
793
829
|
};
|
|
@@ -874,7 +910,14 @@ export class CallController {
|
|
|
874
910
|
// token-based speech on ConversationRelay so the caller still
|
|
875
911
|
// hears a response instead of silence. This fallback is only
|
|
876
912
|
// used for providers whose catalog entry allows native fallback.
|
|
877
|
-
|
|
913
|
+
// Skip it entirely for a superseded run so a stale response can't
|
|
914
|
+
// leak into the next caller turn.
|
|
915
|
+
if (
|
|
916
|
+
!playUrlSent &&
|
|
917
|
+
!this.transport.requiresWavAudio &&
|
|
918
|
+
this.isCurrentRun(runVersion)
|
|
919
|
+
) {
|
|
920
|
+
this.beginSpeaking(runVersion);
|
|
878
921
|
this.transport.sendTextToken(text, false);
|
|
879
922
|
}
|
|
880
923
|
}
|
|
@@ -1221,6 +1264,30 @@ export class CallController {
|
|
|
1221
1264
|
return err.name === "AbortError" || err.name === "APIUserAbortError";
|
|
1222
1265
|
}
|
|
1223
1266
|
|
|
1267
|
+
/**
|
|
1268
|
+
* Transient teardown race: a new voice turn reached the session bridge
|
|
1269
|
+
* before the previous turn released the conversation processing lock.
|
|
1270
|
+
* This is not a real error and must never be spoken to the caller.
|
|
1271
|
+
*/
|
|
1272
|
+
private isLockContentionError(err: unknown): boolean {
|
|
1273
|
+
return (
|
|
1274
|
+
err instanceof Error &&
|
|
1275
|
+
err.message.includes("already processing a message")
|
|
1276
|
+
);
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/**
|
|
1280
|
+
* Flip from the pre-speech `processing` phase to `speaking` at the moment the
|
|
1281
|
+
* first real outbound audio/token is emitted. Guarded so a superseded or
|
|
1282
|
+
* aborted (idle) turn never (re)enters `speaking`, and so barge-in
|
|
1283
|
+
* (handleBargeIn, gated on `speaking`) can't abort a turn that is still
|
|
1284
|
+
* waiting for the processing lock or generating with no audio yet.
|
|
1285
|
+
*/
|
|
1286
|
+
private beginSpeaking(runVersion: number): void {
|
|
1287
|
+
if (!this.isCurrentRun(runVersion)) return;
|
|
1288
|
+
if (this.state === "processing") this.state = "speaking";
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1224
1291
|
private isCurrentRun(runVersion: number): boolean {
|
|
1225
1292
|
return runVersion === this.llmRunVersion;
|
|
1226
1293
|
}
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
TurnInterfaceContext,
|
|
19
19
|
} from "../channels/types.js";
|
|
20
20
|
import { getConfig } from "../config/loader.js";
|
|
21
|
+
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
21
22
|
import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
|
|
22
23
|
import { getOrCreateConversation } from "../daemon/conversation-store.js";
|
|
23
24
|
import type { ServerMessage } from "../daemon/message-protocol.js";
|
|
@@ -35,6 +36,21 @@ import {
|
|
|
35
36
|
|
|
36
37
|
const log = getLogger("voice-session-bridge");
|
|
37
38
|
|
|
39
|
+
const PROCESSING_WAIT_MARGIN_MS = 1000;
|
|
40
|
+
/**
|
|
41
|
+
* How long startVoiceTurn waits for a prior turn to release the processing
|
|
42
|
+
* lock before giving up. The prior turn can hold the lock for the abort
|
|
43
|
+
* unwind budget PLUS the awaited turn-boundary commit window, so the wait
|
|
44
|
+
* must cover both (+ margin) or a barge-in can still surface
|
|
45
|
+
* "Conversation is already processing a message".
|
|
46
|
+
*/
|
|
47
|
+
export function resolveProcessingWaitMs(
|
|
48
|
+
turnCommitMaxWaitMs: number,
|
|
49
|
+
abortUnwindMs: number,
|
|
50
|
+
): number {
|
|
51
|
+
return turnCommitMaxWaitMs + abortUnwindMs + PROCESSING_WAIT_MARGIN_MS;
|
|
52
|
+
}
|
|
53
|
+
|
|
38
54
|
// ---------------------------------------------------------------------------
|
|
39
55
|
// Module-level dependency injection
|
|
40
56
|
// ---------------------------------------------------------------------------
|
|
@@ -360,7 +376,11 @@ export async function startVoiceTurn(
|
|
|
360
376
|
if (conversation.isProcessing()) {
|
|
361
377
|
// Voice barge-in can race with turn teardown. Wait briefly for the
|
|
362
378
|
// previous turn to finish aborting before giving up.
|
|
363
|
-
const
|
|
379
|
+
const config = getConfig();
|
|
380
|
+
const maxWaitMs = resolveProcessingWaitMs(
|
|
381
|
+
config.workspaceGit?.turnCommitMaxWaitMs ?? 4000,
|
|
382
|
+
ABORT_WATCHDOG_MS,
|
|
383
|
+
);
|
|
364
384
|
const pollIntervalMs = 50;
|
|
365
385
|
let waited = 0;
|
|
366
386
|
while (conversation.isProcessing() && waited < maxWaitMs) {
|
|
@@ -374,6 +394,10 @@ export async function startVoiceTurn(
|
|
|
374
394
|
throw new Error("Turn aborted while waiting for conversation");
|
|
375
395
|
}
|
|
376
396
|
if (conversation.isProcessing()) {
|
|
397
|
+
// Waited the full budget (see resolveProcessingWaitMs) without the lock
|
|
398
|
+
// releasing, so the prior turn is genuinely wedged. The controller
|
|
399
|
+
// catches this terminal error and speaks a brief non-technical
|
|
400
|
+
// re-prompt rather than staying silent.
|
|
377
401
|
throw new Error("Conversation is already processing a message");
|
|
378
402
|
}
|
|
379
403
|
}
|
|
@@ -354,6 +354,14 @@
|
|
|
354
354
|
"description": "Expose the developer-only Memory Router Playground tab in macOS Settings and the /assistant/memory-router-playground web page for dry-running v4 router config overrides against the live page index. Dev-only; default off.",
|
|
355
355
|
"defaultEnabled": false
|
|
356
356
|
},
|
|
357
|
+
{
|
|
358
|
+
"id": "memory-v3-injection-gate",
|
|
359
|
+
"scope": "assistant",
|
|
360
|
+
"key": "memory-v3-injection-gate",
|
|
361
|
+
"label": "Memory v3 injection gate",
|
|
362
|
+
"description": "Per-turn injection gate for memory v3: after the BM25F needle + dense cosine finder lanes run, skip the selectPool LLM call (inject nothing) when finder-lane scores fall below the configured thresholds (memory.v3.gate). Default off.",
|
|
363
|
+
"defaultEnabled": false
|
|
364
|
+
},
|
|
357
365
|
{
|
|
358
366
|
"id": "self-intro-greeting",
|
|
359
367
|
"scope": "both",
|