@kuralle-syrinx/core 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/package.json +21 -2
- package/src/confidence-to-wait.test.ts +21 -0
- package/src/confidence-to-wait.ts +30 -0
- package/src/hedge-throwing-backend.test.ts +46 -0
- package/src/incremental-unit.ts +19 -0
- package/src/index.ts +40 -0
- package/src/interaction-coordinator.test.ts +425 -0
- package/src/interaction-coordinator.ts +240 -0
- package/src/interaction-policy.ts +104 -0
- package/src/iu-ledger.test.ts +276 -0
- package/src/iu-ledger.ts +110 -0
- package/src/packet-factories.test.ts +20 -1
- package/src/packet-factories.ts +27 -0
- package/src/packets.ts +24 -3
- package/src/policies/defer.test.ts +86 -0
- package/src/policies/defer.ts +15 -0
- package/src/policies/rule-based.test.ts +269 -0
- package/src/policies/rule-based.ts +140 -0
- package/src/reasoner-hedge.test.ts +361 -0
- package/src/reasoner-hedge.ts +216 -0
- package/src/reasoner-route.test.ts +248 -0
- package/src/reasoner-route.ts +113 -0
- package/src/route-throwing-spec.test.ts +44 -0
- package/src/turn-arbiter.ts +11 -2
- package/src/voice-agent-session.test.ts +439 -2
- package/src/voice-agent-session.ts +232 -22
package/README.md
CHANGED
|
@@ -4,6 +4,10 @@ The Syrinx Kernel v2 — the framework-agnostic core of the voice engine: the `P
|
|
|
4
4
|
|
|
5
5
|
This README documents the **`Reasoner` seam** added for the Reasoner-bridge generalization; the rest of the kernel surface is in `src/index.ts`.
|
|
6
6
|
|
|
7
|
+
## Incremental-Unit ledger (dormant)
|
|
8
|
+
|
|
9
|
+
The IU substrate (`IncrementalUnit`, `InMemoryIuLedger`) models speculative voice work as **hypothesize → ground → discard**: `add` registers a hypothesis, `commit` grounds it (optionally with a heard prefix in chars/ms), and `revoke` discards it. State is monotonic (`hypothesized` → `committed` or `revoked`; terminal ops are idempotent). The ledger is **dormant** in Sprint 0 — no pipeline consumer is wired yet; C2+ will re-express speculative generation, barge-in truncation, and poison-set bookkeeping on this primitive.
|
|
10
|
+
|
|
7
11
|
## The Reasoner seam
|
|
8
12
|
|
|
9
13
|
A normalized pull-stream that lets one bridge ([`ReasoningBridge`](../aisdk/README.md)) drive **any** reasoning backend (Vercel AI SDK `ToolLoopAgent`/`streamText`, Mastra `Agent`, …) without changing the pipeline primitive. Defined in `src/reasoner.ts`:
|
package/package.json
CHANGED
|
@@ -1,9 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/core",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"
|
|
5
|
+
"description": "Voice pipeline kernel for Syrinx — pipeline bus, turn arbitration, playout clock, packets, and the framework-agnostic Reasoner seam",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"voice",
|
|
8
|
+
"voice-agent",
|
|
9
|
+
"speech",
|
|
10
|
+
"syrinx",
|
|
11
|
+
"voice-pipeline",
|
|
12
|
+
"barge-in",
|
|
13
|
+
"turn-taking"
|
|
14
|
+
],
|
|
6
15
|
"license": "MIT",
|
|
16
|
+
"homepage": "https://github.com/kuralle/syrinx#readme",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/kuralle/syrinx.git",
|
|
20
|
+
"directory": "packages/core"
|
|
21
|
+
},
|
|
22
|
+
"bugs": {
|
|
23
|
+
"url": "https://github.com/kuralle/syrinx/issues"
|
|
24
|
+
},
|
|
25
|
+
"type": "module",
|
|
7
26
|
"main": "./src/index.ts",
|
|
8
27
|
"types": "./src/index.ts",
|
|
9
28
|
"exports": {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import { confidenceToWaitMs } from "./confidence-to-wait.js";
|
|
6
|
+
|
|
7
|
+
describe("confidenceToWaitMs", () => {
|
|
8
|
+
it("maps high confidence to 150ms and low confidence to 2000ms", () => {
|
|
9
|
+
expect(confidenceToWaitMs(1)).toBe(150);
|
|
10
|
+
expect(confidenceToWaitMs(0)).toBe(2000);
|
|
11
|
+
expect(confidenceToWaitMs(2)).toBe(150);
|
|
12
|
+
expect(confidenceToWaitMs(-1)).toBe(2000);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it("is monotonic and supports a bounded custom range", () => {
|
|
16
|
+
const waits = [0, 0.25, 0.5, 0.75, 1].map((confidence) => confidenceToWaitMs(confidence));
|
|
17
|
+
expect(waits).toEqual([...waits].sort((a, b) => b - a));
|
|
18
|
+
expect(confidenceToWaitMs(0.5, { minWaitMs: 100, maxWaitMs: 1100 })).toBe(600);
|
|
19
|
+
expect(() => confidenceToWaitMs(0.5, { minWaitMs: 500, maxWaitMs: 100 })).toThrow(/must be >=/);
|
|
20
|
+
});
|
|
21
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
export interface ConfidenceToWaitConfig {
|
|
4
|
+
/** Wait at confidence = 1.0 (high EOT certainty). Default: 150 ms. */
|
|
5
|
+
readonly minWaitMs?: number;
|
|
6
|
+
/** Wait at confidence = 0.0 (low EOT certainty). Default: 2000 ms. */
|
|
7
|
+
readonly maxWaitMs?: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
const DEFAULT_MIN_WAIT_MS = 150;
|
|
11
|
+
const DEFAULT_MAX_WAIT_MS = 2000;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Maps an end-of-turn confidence score to a bounded finalize wait (LiveKit-style).
|
|
15
|
+
* Higher confidence → shorter wait. Monotonic, clamped, deterministic.
|
|
16
|
+
*/
|
|
17
|
+
export function confidenceToWaitMs(
|
|
18
|
+
confidence: number,
|
|
19
|
+
config: ConfidenceToWaitConfig = {},
|
|
20
|
+
): number {
|
|
21
|
+
const minWaitMs = config.minWaitMs ?? DEFAULT_MIN_WAIT_MS;
|
|
22
|
+
const maxWaitMs = config.maxWaitMs ?? DEFAULT_MAX_WAIT_MS;
|
|
23
|
+
if (maxWaitMs < minWaitMs) {
|
|
24
|
+
throw new Error(`maxWaitMs (${String(maxWaitMs)}) must be >= minWaitMs (${String(minWaitMs)})`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const clamped = Math.min(1, Math.max(0, confidence));
|
|
28
|
+
const span = maxWaitMs - minWaitMs;
|
|
29
|
+
return Math.round(maxWaitMs - clamped * span);
|
|
30
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
// Regression guard (RL-WBS-1): a backend whose next() REJECTS (throws) — e.g. an adapter that
|
|
3
|
+
// throws AbortError on barge-in — must NOT hang the hedge race; it fails over / surfaces an error.
|
|
4
|
+
import { describe, expect, it } from "vitest";
|
|
5
|
+
import { HedgedReasoner } from "./reasoner-hedge.js";
|
|
6
|
+
import type { Reasoner, ReasonerTurn, ReasoningPart } from "./reasoner.js";
|
|
7
|
+
import type { Scheduler } from "./scheduler.js";
|
|
8
|
+
|
|
9
|
+
class ThrowingReasoner implements Reasoner {
|
|
10
|
+
async *stream(_turn: ReasonerTurn): AsyncGenerator<ReasoningPart> {
|
|
11
|
+
throw new Error("backend boom"); // rejects on first next()
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
class GoodReasoner implements Reasoner {
|
|
15
|
+
async *stream(_turn: ReasonerTurn): AsyncGenerator<ReasoningPart> {
|
|
16
|
+
yield { type: "text-delta", text: "backup ok" };
|
|
17
|
+
yield { type: "finish", reason: "stop", text: "backup ok" };
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
// scheduler that fires immediately so backup is available
|
|
21
|
+
const eagerScheduler: Scheduler = { schedule: (_k, _ms, cb) => cb(), cancel: () => {} };
|
|
22
|
+
|
|
23
|
+
function turn(): ReasonerTurn {
|
|
24
|
+
return { userText: "hi", messages: [], signal: new AbortController().signal };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
describe("hedge reject repro", () => {
|
|
28
|
+
it("does not hang when the primary backend THROWS (rejects next())", async () => {
|
|
29
|
+
const hedged = new HedgedReasoner({
|
|
30
|
+
primary: new ThrowingReasoner(),
|
|
31
|
+
backup: new GoodReasoner(),
|
|
32
|
+
hedgeAfterMs: 0,
|
|
33
|
+
scheduler: eagerScheduler,
|
|
34
|
+
});
|
|
35
|
+
const parts: string[] = [];
|
|
36
|
+
const collect = (async () => {
|
|
37
|
+
for await (const p of hedged.stream(turn())) {
|
|
38
|
+
if (p.type === "text-delta") parts.push(p.text);
|
|
39
|
+
}
|
|
40
|
+
})();
|
|
41
|
+
const timeout = new Promise((_r, rej) => setTimeout(() => rej(new Error("HUNG: HedgedReasoner never completed")), 3000));
|
|
42
|
+
await Promise.race([collect, timeout]);
|
|
43
|
+
// If it doesn't hang, it should have failed over to the backup.
|
|
44
|
+
expect(parts).toContain("backup ok");
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Incremental-Unit identity + state (IU substrate, RFC incremental-unit-substrate)
|
|
4
|
+
|
|
5
|
+
export type IuState = "hypothesized" | "committed" | "revoked";
|
|
6
|
+
|
|
7
|
+
export interface IncrementalUnitId {
|
|
8
|
+
readonly contextId: string;
|
|
9
|
+
readonly iuId: string;
|
|
10
|
+
readonly epoch: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface IncrementalUnit {
|
|
14
|
+
readonly id: IncrementalUnitId;
|
|
15
|
+
readonly kind: "user_turn" | "assistant_response" | "tts_segment";
|
|
16
|
+
state: IuState;
|
|
17
|
+
/** For assistant/tts IUs: the committed character/ms prefix (heard). */
|
|
18
|
+
committedPrefix?: { chars?: number; ms?: number };
|
|
19
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -30,6 +30,7 @@ export {
|
|
|
30
30
|
type VadSpeechActivityPacket,
|
|
31
31
|
type SpeechToTextAudioPacket,
|
|
32
32
|
type SttInterimPacket,
|
|
33
|
+
type SttPartialPacket,
|
|
33
34
|
type SttResultPacket,
|
|
34
35
|
type FinalizeSttPacket,
|
|
35
36
|
type SttErrorPacket,
|
|
@@ -78,6 +79,18 @@ export {
|
|
|
78
79
|
type ReasonerSessionStore,
|
|
79
80
|
} from "./reasoner-session-store.js";
|
|
80
81
|
|
|
82
|
+
// Incremental-Unit ledger (IU substrate — dormant until C2)
|
|
83
|
+
export {
|
|
84
|
+
type IuState,
|
|
85
|
+
type IncrementalUnitId,
|
|
86
|
+
type IncrementalUnit,
|
|
87
|
+
} from "./incremental-unit.js";
|
|
88
|
+
export {
|
|
89
|
+
type IuLedger,
|
|
90
|
+
type IuLedgerAnomaly,
|
|
91
|
+
InMemoryIuLedger,
|
|
92
|
+
} from "./iu-ledger.js";
|
|
93
|
+
|
|
81
94
|
// Pipeline packets — TTS
|
|
82
95
|
export {
|
|
83
96
|
type TextToSpeechTextPacket,
|
|
@@ -187,6 +200,23 @@ export {
|
|
|
187
200
|
type SyrinxAudioEnvelopeHeader,
|
|
188
201
|
} from "./audio-envelope.js";
|
|
189
202
|
|
|
203
|
+
export { StreamingPcm16Resampler } from "./audio/index.js";
|
|
204
|
+
|
|
205
|
+
// Interaction policy seam (IP-C1)
|
|
206
|
+
export {
|
|
207
|
+
type InteractionPolicy,
|
|
208
|
+
type LifecycleInteractionPolicy,
|
|
209
|
+
isLifecycleInteractionPolicy,
|
|
210
|
+
type InteractionObservation,
|
|
211
|
+
type InteractionDecision,
|
|
212
|
+
type WordTiming,
|
|
213
|
+
} from "./interaction-policy.js";
|
|
214
|
+
export { confidenceToWaitMs, type ConfidenceToWaitConfig } from "./confidence-to-wait.js";
|
|
215
|
+
export { RuleBasedInteractionPolicy } from "./policies/rule-based.js";
|
|
216
|
+
export { DeferInteractionPolicy } from "./policies/defer.js";
|
|
217
|
+
export { InteractionCoordinator, type InteractionCaps } from "./interaction-coordinator.js";
|
|
218
|
+
export { type InteractionBackchannelPacket } from "./packets.js";
|
|
219
|
+
|
|
190
220
|
// VoiceAgentSession
|
|
191
221
|
export { VoiceAgentSession, type VoiceAgentSessionConfig, type VoiceAgentSessionEvents } from "./voice-agent-session.js";
|
|
192
222
|
|
|
@@ -228,3 +258,13 @@ export {
|
|
|
228
258
|
type ReasonerMessage,
|
|
229
259
|
type ReasoningPart,
|
|
230
260
|
} from "./reasoner.js";
|
|
261
|
+
|
|
262
|
+
// Hedged reasoner (Lever C — reasoner-latency RFC)
|
|
263
|
+
export { HedgedReasoner, type HedgedReasonerOptions } from "./reasoner-hedge.js";
|
|
264
|
+
|
|
265
|
+
// Routing reasoner (Lever B — reasoner-latency RFC)
|
|
266
|
+
export {
|
|
267
|
+
RoutingReasoner,
|
|
268
|
+
type RoutingReasonerOptions,
|
|
269
|
+
type ReasonerRoute,
|
|
270
|
+
} from "./reasoner-route.js";
|
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { describe, it, expect, vi } from "vitest";
|
|
4
|
+
import { PipelineBusImpl, Route } from "./pipeline-bus.js";
|
|
5
|
+
import { InteractionCoordinator } from "./interaction-coordinator.js";
|
|
6
|
+
import type { InteractionDecision, InteractionObservation, InteractionPolicy } from "./interaction-policy.js";
|
|
7
|
+
import { TurnArbiter } from "./turn-arbiter.js";
|
|
8
|
+
import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
|
|
9
|
+
import { TtsPlayoutClock } from "./tts-playout-clock.js";
|
|
10
|
+
import { TimerScheduler } from "./scheduler.js";
|
|
11
|
+
import type { EndOfSpeechPacket, InteractionBackchannelPacket, InterruptionDetectedPacket } from "./packets.js";
|
|
12
|
+
|
|
13
|
+
class StubPolicy implements InteractionPolicy {
|
|
14
|
+
constructor(private readonly decisions: InteractionDecision[]) {}
|
|
15
|
+
|
|
16
|
+
observe(_obs: InteractionObservation): readonly InteractionDecision[] {
|
|
17
|
+
return this.decisions;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
reset(): void {}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
class StatefulPolicy implements InteractionPolicy {
|
|
24
|
+
private step = 0;
|
|
25
|
+
|
|
26
|
+
constructor(private readonly steps: Array<readonly InteractionDecision[]>) {}
|
|
27
|
+
|
|
28
|
+
observe(_obs: InteractionObservation): readonly InteractionDecision[] {
|
|
29
|
+
const out = this.steps[this.step] ?? [];
|
|
30
|
+
this.step += 1;
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
reset(): void {
|
|
35
|
+
this.step = 0;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function createCoordinator(
|
|
40
|
+
decisions: InteractionDecision[],
|
|
41
|
+
options: {
|
|
42
|
+
emitsBackchannel?: boolean;
|
|
43
|
+
isUserSpeaking?: () => boolean;
|
|
44
|
+
isTtsActive?: () => boolean;
|
|
45
|
+
hasCueAsset?: (cueId: string) => boolean;
|
|
46
|
+
} = {},
|
|
47
|
+
) {
|
|
48
|
+
const bus = new PipelineBusImpl();
|
|
49
|
+
void bus.start();
|
|
50
|
+
const ttsPlayout = new TtsPlayoutClock();
|
|
51
|
+
const executor = new TurnArbiter({
|
|
52
|
+
bus,
|
|
53
|
+
primarySpeakerGate: new PrimarySpeakerGate(),
|
|
54
|
+
ttsPlayout,
|
|
55
|
+
minInterruptionMs: 280,
|
|
56
|
+
});
|
|
57
|
+
const coordinator = new InteractionCoordinator({
|
|
58
|
+
bus,
|
|
59
|
+
policy: new StubPolicy(decisions),
|
|
60
|
+
executor,
|
|
61
|
+
scheduler: new TimerScheduler(),
|
|
62
|
+
caps: { emitsBackchannel: options.emitsBackchannel },
|
|
63
|
+
isUserSpeaking: options.isUserSpeaking,
|
|
64
|
+
isTtsActive: options.isTtsActive,
|
|
65
|
+
hasCueAsset: options.hasCueAsset,
|
|
66
|
+
});
|
|
67
|
+
coordinator.initialize();
|
|
68
|
+
return { bus, coordinator, executor };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function drainBus(): Promise<void> {
|
|
72
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function metricNames(bus: PipelineBusImpl): string[] {
|
|
76
|
+
const names: string[] = [];
|
|
77
|
+
bus.on("metric.conversation", (pkt) => {
|
|
78
|
+
names.push((pkt as unknown as { name: string }).name);
|
|
79
|
+
});
|
|
80
|
+
return names;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe("InteractionCoordinator", () => {
|
|
84
|
+
it("maps interrupt decision to executor.emitInterruptDetected", async () => {
|
|
85
|
+
const { bus, coordinator, executor } = await createCoordinator([
|
|
86
|
+
{ kind: "interrupt", interruptedContextId: "assistant-turn" },
|
|
87
|
+
]);
|
|
88
|
+
const interrupts: InterruptionDetectedPacket[] = [];
|
|
89
|
+
bus.on("interrupt.detected", (pkt) => {
|
|
90
|
+
interrupts.push(pkt as InterruptionDetectedPacket);
|
|
91
|
+
});
|
|
92
|
+
const emitSpy = vi.spyOn(executor, "emitInterruptDetected");
|
|
93
|
+
|
|
94
|
+
coordinator.observe({
|
|
95
|
+
kind: "vad_speech_activity",
|
|
96
|
+
contextId: "user",
|
|
97
|
+
timestampMs: 1000,
|
|
98
|
+
});
|
|
99
|
+
await drainBus();
|
|
100
|
+
|
|
101
|
+
expect(emitSpy).toHaveBeenCalledOnce();
|
|
102
|
+
expect(emitSpy).toHaveBeenCalledWith("assistant-turn");
|
|
103
|
+
expect(interrupts).toHaveLength(1);
|
|
104
|
+
expect(interrupts[0]!.contextId).toBe("assistant-turn");
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("emits interaction.backchannel for a backchannel decision", async () => {
|
|
108
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "backchannel", cue: "mm_hmm" }]);
|
|
109
|
+
const packets: InteractionBackchannelPacket[] = [];
|
|
110
|
+
const metrics = metricNames(bus);
|
|
111
|
+
bus.on("interaction.backchannel", (pkt) => {
|
|
112
|
+
packets.push(pkt as InteractionBackchannelPacket);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
coordinator.observe({
|
|
116
|
+
kind: "delegate_state",
|
|
117
|
+
contextId: "turn-1",
|
|
118
|
+
timestampMs: 2000,
|
|
119
|
+
toolCallPhase: "delayed",
|
|
120
|
+
});
|
|
121
|
+
await drainBus();
|
|
122
|
+
|
|
123
|
+
expect(packets).toHaveLength(1);
|
|
124
|
+
expect(packets[0]).toMatchObject({ contextId: "turn-1", cue: "mm_hmm" });
|
|
125
|
+
expect(metrics).toContain("backchannel.candidate");
|
|
126
|
+
expect(metrics).toContain("backchannel.emitted");
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it("suppresses backchannel when caps.emitsBackchannel is true", async () => {
|
|
130
|
+
const { bus, coordinator } = await createCoordinator(
|
|
131
|
+
[{ kind: "backchannel", cue: "mm_hmm" }],
|
|
132
|
+
{ emitsBackchannel: true },
|
|
133
|
+
);
|
|
134
|
+
const packets: InteractionBackchannelPacket[] = [];
|
|
135
|
+
const metrics = metricNames(bus);
|
|
136
|
+
bus.on("interaction.backchannel", (pkt) => {
|
|
137
|
+
packets.push(pkt as InteractionBackchannelPacket);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
coordinator.observe({
|
|
141
|
+
kind: "delegate_state",
|
|
142
|
+
contextId: "turn-1",
|
|
143
|
+
timestampMs: 2000,
|
|
144
|
+
toolCallPhase: "delayed",
|
|
145
|
+
});
|
|
146
|
+
await drainBus();
|
|
147
|
+
|
|
148
|
+
expect(packets).toEqual([]);
|
|
149
|
+
expect(metrics).toContain("backchannel.suppressed_caps");
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it("suppresses backchannel when TTS is active", async () => {
|
|
153
|
+
const { bus, coordinator } = await createCoordinator(
|
|
154
|
+
[{ kind: "backchannel", cue: "mm_hmm" }],
|
|
155
|
+
{ isTtsActive: () => true },
|
|
156
|
+
);
|
|
157
|
+
const metrics = metricNames(bus);
|
|
158
|
+
|
|
159
|
+
coordinator.observe({
|
|
160
|
+
kind: "delegate_state",
|
|
161
|
+
contextId: "turn-1",
|
|
162
|
+
timestampMs: 2000,
|
|
163
|
+
toolCallPhase: "delayed",
|
|
164
|
+
});
|
|
165
|
+
await drainBus();
|
|
166
|
+
|
|
167
|
+
expect(metrics).toContain("backchannel.suppressed_tts_active");
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
it("suppresses backchannel when the user is speaking", async () => {
|
|
171
|
+
const { bus, coordinator } = await createCoordinator(
|
|
172
|
+
[{ kind: "backchannel", cue: "mm_hmm" }],
|
|
173
|
+
{ isUserSpeaking: () => true },
|
|
174
|
+
);
|
|
175
|
+
const metrics = metricNames(bus);
|
|
176
|
+
|
|
177
|
+
coordinator.observe({
|
|
178
|
+
kind: "delegate_state",
|
|
179
|
+
contextId: "turn-1",
|
|
180
|
+
timestampMs: 2000,
|
|
181
|
+
toolCallPhase: "delayed",
|
|
182
|
+
});
|
|
183
|
+
await drainBus();
|
|
184
|
+
|
|
185
|
+
expect(metrics).toContain("backchannel.suppressed_user_speaking");
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
it("suppresses backchannel when the cue asset is missing", async () => {
|
|
189
|
+
const { bus, coordinator } = await createCoordinator(
|
|
190
|
+
[{ kind: "backchannel", cue: "mm_hmm" }],
|
|
191
|
+
{ hasCueAsset: () => false },
|
|
192
|
+
);
|
|
193
|
+
const metrics = metricNames(bus);
|
|
194
|
+
|
|
195
|
+
coordinator.observe({
|
|
196
|
+
kind: "delegate_state",
|
|
197
|
+
contextId: "turn-1",
|
|
198
|
+
timestampMs: 2000,
|
|
199
|
+
toolCallPhase: "delayed",
|
|
200
|
+
});
|
|
201
|
+
await drainBus();
|
|
202
|
+
|
|
203
|
+
expect(metrics).toContain("backchannel.suppressed_missing_asset");
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it("maps take_turn to stt.finalize and a delayed eos.turn_complete", async () => {
|
|
207
|
+
vi.useFakeTimers();
|
|
208
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "take_turn", confidence: 1 }]);
|
|
209
|
+
const finalizeRequests: string[] = [];
|
|
210
|
+
const completions: Array<{ contextId: string; text: string }> = [];
|
|
211
|
+
bus.on("stt.finalize", (pkt) => {
|
|
212
|
+
finalizeRequests.push(pkt.contextId);
|
|
213
|
+
});
|
|
214
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
215
|
+
const eos = pkt as EndOfSpeechPacket;
|
|
216
|
+
completions.push({ contextId: eos.contextId, text: eos.text });
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
coordinator.observe({
|
|
220
|
+
kind: "vad_speech_ended",
|
|
221
|
+
contextId: "turn-1",
|
|
222
|
+
timestampMs: 1000,
|
|
223
|
+
hasActiveTts: false,
|
|
224
|
+
});
|
|
225
|
+
bus.push(Route.Main, {
|
|
226
|
+
kind: "stt.result",
|
|
227
|
+
contextId: "turn-1",
|
|
228
|
+
timestampMs: 1100,
|
|
229
|
+
text: "hello there",
|
|
230
|
+
confidence: 0.95,
|
|
231
|
+
language: "en-US",
|
|
232
|
+
});
|
|
233
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
234
|
+
expect(finalizeRequests).toEqual(["turn-1"]);
|
|
235
|
+
expect(completions).toEqual([]);
|
|
236
|
+
|
|
237
|
+
await vi.advanceTimersByTimeAsync(149);
|
|
238
|
+
expect(completions).toEqual([]);
|
|
239
|
+
|
|
240
|
+
await vi.advanceTimersByTimeAsync(1);
|
|
241
|
+
expect(completions).toEqual([{ contextId: "turn-1", text: "hello there" }]);
|
|
242
|
+
vi.useRealTimers();
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
it("uses a transcript that arrived before take_turn and requests STT finalization only once", async () => {
|
|
246
|
+
vi.useFakeTimers();
|
|
247
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "take_turn", confidence: 1 }]);
|
|
248
|
+
const finalizeRequests: string[] = [];
|
|
249
|
+
const completions: EndOfSpeechPacket[] = [];
|
|
250
|
+
bus.on("stt.finalize", (pkt) => {
|
|
251
|
+
finalizeRequests.push(pkt.contextId);
|
|
252
|
+
});
|
|
253
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
254
|
+
completions.push(pkt as EndOfSpeechPacket);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
bus.push(Route.Main, {
|
|
258
|
+
kind: "stt.result",
|
|
259
|
+
contextId: "turn-cached",
|
|
260
|
+
timestampMs: 900,
|
|
261
|
+
text: "already final",
|
|
262
|
+
confidence: 0.95,
|
|
263
|
+
language: "en-US",
|
|
264
|
+
});
|
|
265
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
266
|
+
const observation = {
|
|
267
|
+
kind: "audio_frame" as const,
|
|
268
|
+
contextId: "turn-cached",
|
|
269
|
+
timestampMs: 1000,
|
|
270
|
+
audio: new Int16Array(320),
|
|
271
|
+
};
|
|
272
|
+
coordinator.observe(observation);
|
|
273
|
+
coordinator.observe(observation);
|
|
274
|
+
|
|
275
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
276
|
+
expect(finalizeRequests).toEqual(["turn-cached"]);
|
|
277
|
+
await vi.advanceTimersByTimeAsync(150);
|
|
278
|
+
expect(completions).toEqual([
|
|
279
|
+
expect.objectContaining({ contextId: "turn-cached", text: "already final" }),
|
|
280
|
+
]);
|
|
281
|
+
vi.useRealTimers();
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
it("completes a committed take_turn from the latest interim when no final STT ever arrives", async () => {
|
|
285
|
+
vi.useFakeTimers();
|
|
286
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "take_turn", confidence: 1 }]);
|
|
287
|
+
const finalizeRequests: string[] = [];
|
|
288
|
+
const completions: EndOfSpeechPacket[] = [];
|
|
289
|
+
bus.on("stt.finalize", (pkt) => {
|
|
290
|
+
finalizeRequests.push(pkt.contextId);
|
|
291
|
+
});
|
|
292
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
293
|
+
completions.push(pkt as EndOfSpeechPacket);
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
coordinator.observe({
|
|
297
|
+
kind: "vad_speech_ended",
|
|
298
|
+
contextId: "turn-interim",
|
|
299
|
+
timestampMs: 1000,
|
|
300
|
+
hasActiveTts: false,
|
|
301
|
+
});
|
|
302
|
+
// Only an interim arrives — the provider never emits a final stt.result.
|
|
303
|
+
bus.push(Route.Main, {
|
|
304
|
+
kind: "stt.interim",
|
|
305
|
+
contextId: "turn-interim",
|
|
306
|
+
timestampMs: 1100,
|
|
307
|
+
text: "i wanted to ask",
|
|
308
|
+
});
|
|
309
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
310
|
+
expect(finalizeRequests).toEqual(["turn-interim"]);
|
|
311
|
+
expect(completions).toEqual([]);
|
|
312
|
+
|
|
313
|
+
// High confidence → 150ms wait; the committed turn must NOT be dropped.
|
|
314
|
+
await vi.advanceTimersByTimeAsync(150);
|
|
315
|
+
expect(completions).toHaveLength(1);
|
|
316
|
+
expect(completions[0]).toMatchObject({ contextId: "turn-interim", text: "i wanted to ask" });
|
|
317
|
+
expect(completions[0]!.transcripts).toHaveLength(1);
|
|
318
|
+
vi.useRealTimers();
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
it("does not complete a take_turn when neither a final nor an interim ever arrives", async () => {
|
|
322
|
+
vi.useFakeTimers();
|
|
323
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "take_turn", confidence: 1 }]);
|
|
324
|
+
const completions: string[] = [];
|
|
325
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
326
|
+
completions.push(pkt.contextId);
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
coordinator.observe({
|
|
330
|
+
kind: "vad_speech_ended",
|
|
331
|
+
contextId: "turn-empty",
|
|
332
|
+
timestampMs: 1000,
|
|
333
|
+
hasActiveTts: false,
|
|
334
|
+
});
|
|
335
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
336
|
+
expect(completions).toEqual([]);
|
|
337
|
+
vi.useRealTimers();
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
it("revokes a pending take_turn on hold", async () => {
|
|
341
|
+
vi.useFakeTimers();
|
|
342
|
+
const bus = new PipelineBusImpl();
|
|
343
|
+
void bus.start();
|
|
344
|
+
const coordinator = new InteractionCoordinator({
|
|
345
|
+
bus,
|
|
346
|
+
policy: new StatefulPolicy([
|
|
347
|
+
[{ kind: "take_turn", confidence: 0.2 }],
|
|
348
|
+
[{ kind: "hold" }],
|
|
349
|
+
]),
|
|
350
|
+
executor: new TurnArbiter({
|
|
351
|
+
bus,
|
|
352
|
+
primarySpeakerGate: new PrimarySpeakerGate(),
|
|
353
|
+
ttsPlayout: new TtsPlayoutClock(),
|
|
354
|
+
minInterruptionMs: 280,
|
|
355
|
+
}),
|
|
356
|
+
scheduler: new TimerScheduler(),
|
|
357
|
+
caps: {},
|
|
358
|
+
});
|
|
359
|
+
coordinator.initialize();
|
|
360
|
+
|
|
361
|
+
const completions: string[] = [];
|
|
362
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
363
|
+
completions.push(pkt.contextId);
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
coordinator.observe({
|
|
367
|
+
kind: "vad_speech_ended",
|
|
368
|
+
contextId: "turn-hold",
|
|
369
|
+
timestampMs: 1000,
|
|
370
|
+
hasActiveTts: false,
|
|
371
|
+
});
|
|
372
|
+
bus.push(Route.Main, {
|
|
373
|
+
kind: "stt.result",
|
|
374
|
+
contextId: "turn-hold",
|
|
375
|
+
timestampMs: 1100,
|
|
376
|
+
text: "wait",
|
|
377
|
+
confidence: 0.9,
|
|
378
|
+
language: "en-US",
|
|
379
|
+
});
|
|
380
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
381
|
+
coordinator.observe({
|
|
382
|
+
kind: "vad_speech_activity",
|
|
383
|
+
contextId: "turn-hold",
|
|
384
|
+
timestampMs: 1200,
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
388
|
+
expect(completions).toEqual([]);
|
|
389
|
+
vi.useRealTimers();
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
it("revokes a pending take_turn when speech resumes", async () => {
|
|
393
|
+
vi.useFakeTimers();
|
|
394
|
+
const { bus, coordinator } = await createCoordinator([{ kind: "take_turn", confidence: 0.5 }]);
|
|
395
|
+
const completions: string[] = [];
|
|
396
|
+
bus.on("eos.turn_complete", (pkt) => {
|
|
397
|
+
completions.push(pkt.contextId);
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
coordinator.observe({
|
|
401
|
+
kind: "vad_speech_ended",
|
|
402
|
+
contextId: "turn-resume",
|
|
403
|
+
timestampMs: 1000,
|
|
404
|
+
hasActiveTts: false,
|
|
405
|
+
});
|
|
406
|
+
bus.push(Route.Main, {
|
|
407
|
+
kind: "stt.result",
|
|
408
|
+
contextId: "turn-resume",
|
|
409
|
+
timestampMs: 1100,
|
|
410
|
+
text: "partial",
|
|
411
|
+
confidence: 0.9,
|
|
412
|
+
language: "en-US",
|
|
413
|
+
});
|
|
414
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
415
|
+
bus.push(Route.Main, {
|
|
416
|
+
kind: "vad.speech_started",
|
|
417
|
+
contextId: "turn-resume",
|
|
418
|
+
timestampMs: 1200,
|
|
419
|
+
confidence: 0.9,
|
|
420
|
+
});
|
|
421
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
422
|
+
expect(completions).toEqual([]);
|
|
423
|
+
vi.useRealTimers();
|
|
424
|
+
});
|
|
425
|
+
});
|