@kuralle-syrinx/core 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/package.json +22 -3
- package/src/audio/alaw.ts +56 -0
- package/src/audio/g722.ts +329 -0
- package/src/audio/index.ts +12 -0
- package/src/audio/loudness.ts +87 -0
- package/src/confidence-to-wait.ts +30 -0
- package/src/idle-timeout.ts +1 -0
- package/src/incremental-unit.ts +19 -0
- package/src/index.ts +101 -1
- package/src/interaction-coordinator.ts +240 -0
- package/src/interaction-policy.ts +116 -0
- package/src/iu-ledger.ts +110 -0
- package/src/observability-observer.ts +8 -4
- package/src/observability.ts +30 -1
- package/src/packet-factories.ts +137 -2
- package/src/packets.ts +140 -4
- package/src/plugin-contract.ts +41 -0
- package/src/policies/defer.ts +15 -0
- package/src/policies/rule-based.ts +155 -0
- package/src/pricing.ts +137 -0
- package/src/primary-speaker-gate.ts +23 -3
- package/src/reasoner-hedge.ts +216 -0
- package/src/reasoner-route.ts +113 -0
- package/src/reasoner.ts +28 -1
- package/src/spend-cap.ts +52 -0
- package/src/turn-arbiter.ts +45 -4
- package/src/voice-agent-session.ts +682 -53
- package/src/voice-text.ts +69 -0
- package/src/audio/audio.test.ts +0 -285
- package/src/audio-envelope.test.ts +0 -167
- package/src/error-handler.test.ts +0 -56
- package/src/latency-filler.test.ts +0 -62
- package/src/observability-observer.test.ts +0 -245
- package/src/observability.test.ts +0 -85
- package/src/packet-factories.test.ts +0 -34
- package/src/pipeline-bus.g10.test.ts +0 -145
- package/src/pipeline-bus.test.ts +0 -211
- package/src/primary-speaker-gate.test.ts +0 -150
- package/src/provider-fallback.test.ts +0 -87
- package/src/reasoner.test.ts +0 -69
- package/src/retry.test.ts +0 -83
- package/src/tts-playout-clock.test.ts +0 -125
- package/src/turn-arbiter.characterization.test.ts +0 -477
- package/src/turn-arbiter.test.ts +0 -567
- package/src/voice-agent-session.test.ts +0 -2879
- package/src/voice-text.test.ts +0 -94
- package/tsconfig.json +0 -21
package/src/index.ts
CHANGED
|
@@ -30,8 +30,10 @@ export {
|
|
|
30
30
|
type VadSpeechActivityPacket,
|
|
31
31
|
type SpeechToTextAudioPacket,
|
|
32
32
|
type SttInterimPacket,
|
|
33
|
+
type SttPartialPacket,
|
|
33
34
|
type SttResultPacket,
|
|
34
35
|
type FinalizeSttPacket,
|
|
36
|
+
type SttReconfigurePacket,
|
|
35
37
|
type SttErrorPacket,
|
|
36
38
|
type EndOfSpeechAudioPacket,
|
|
37
39
|
type EndOfSpeechPacket,
|
|
@@ -49,6 +51,16 @@ export {
|
|
|
49
51
|
type TurnChangePacket,
|
|
50
52
|
} from "./packets.js";
|
|
51
53
|
|
|
54
|
+
// Telephony control packets (DTMF / transfer)
|
|
55
|
+
export {
|
|
56
|
+
type DtmfDigit,
|
|
57
|
+
type DtmfReceivedPacket,
|
|
58
|
+
type DtmfSendPacket,
|
|
59
|
+
type CallTransferMode,
|
|
60
|
+
type CallTransferPacket,
|
|
61
|
+
} from "./packets.js";
|
|
62
|
+
export { parseDtmfDigit, dtmfReceived, dtmfSend, callTransfer } from "./packet-factories.js";
|
|
63
|
+
|
|
52
64
|
// Pipeline packets — LLM
|
|
53
65
|
export {
|
|
54
66
|
type LlmDeltaPacket,
|
|
@@ -78,6 +90,18 @@ export {
|
|
|
78
90
|
type ReasonerSessionStore,
|
|
79
91
|
} from "./reasoner-session-store.js";
|
|
80
92
|
|
|
93
|
+
// Incremental-Unit ledger (IU substrate — dormant until C2)
|
|
94
|
+
export {
|
|
95
|
+
type IuState,
|
|
96
|
+
type IncrementalUnitId,
|
|
97
|
+
type IncrementalUnit,
|
|
98
|
+
} from "./incremental-unit.js";
|
|
99
|
+
export {
|
|
100
|
+
type IuLedger,
|
|
101
|
+
type IuLedgerAnomaly,
|
|
102
|
+
InMemoryIuLedger,
|
|
103
|
+
} from "./iu-ledger.js";
|
|
104
|
+
|
|
81
105
|
// Pipeline packets — TTS
|
|
82
106
|
export {
|
|
83
107
|
type TextToSpeechTextPacket,
|
|
@@ -113,16 +137,41 @@ export {
|
|
|
113
137
|
export {
|
|
114
138
|
type MessageCreatePacket,
|
|
115
139
|
type ConversationMetricPacket,
|
|
140
|
+
type AcousticSignalPacket,
|
|
141
|
+
type AcousticSignal,
|
|
142
|
+
type TurnLocalizationPacket,
|
|
143
|
+
type UsageStage,
|
|
144
|
+
type UsageRecordedPacket,
|
|
116
145
|
type TurnBoundaryKind,
|
|
117
146
|
type TurnBoundaryEventPacket,
|
|
118
147
|
type ObservabilityPacket,
|
|
119
148
|
type PipelineErrorPacket,
|
|
120
149
|
} from "./packets.js";
|
|
121
150
|
|
|
151
|
+
// Metering: price catalog + spend-cap guard (standalone; session wires later)
|
|
152
|
+
export {
|
|
153
|
+
type SttPrice,
|
|
154
|
+
type LlmPrice,
|
|
155
|
+
type TtsPrice,
|
|
156
|
+
type PriceCatalog,
|
|
157
|
+
type CostResult,
|
|
158
|
+
DEFAULT_PRICE_CATALOG,
|
|
159
|
+
costOf,
|
|
160
|
+
} from "./pricing.js";
|
|
161
|
+
export {
|
|
162
|
+
SpendCapGuard,
|
|
163
|
+
type SpendCapConfig,
|
|
164
|
+
type SpendCapCheck,
|
|
165
|
+
} from "./spend-cap.js";
|
|
166
|
+
|
|
122
167
|
// Observability backbone (VE-07)
|
|
123
168
|
export {
|
|
124
169
|
monotonicNowMs,
|
|
125
170
|
type MetricTags,
|
|
171
|
+
type ObservabilityLayer,
|
|
172
|
+
type TurnLocalizationVerdict,
|
|
173
|
+
type TurnLocalizationSignals,
|
|
174
|
+
localizeTurn,
|
|
126
175
|
type SpanHandle,
|
|
127
176
|
type MetricsExporter,
|
|
128
177
|
noopMetricsExporter,
|
|
@@ -149,6 +198,8 @@ export {
|
|
|
149
198
|
type PluginConfig,
|
|
150
199
|
type EndpointingOwner,
|
|
151
200
|
type EndpointingCapability,
|
|
201
|
+
type SttReconfigure,
|
|
202
|
+
type SttReconfigurePartial,
|
|
152
203
|
requireStringConfig,
|
|
153
204
|
optionalStringConfig,
|
|
154
205
|
} from "./plugin-contract.js";
|
|
@@ -187,8 +238,42 @@ export {
|
|
|
187
238
|
type SyrinxAudioEnvelopeHeader,
|
|
188
239
|
} from "./audio-envelope.js";
|
|
189
240
|
|
|
241
|
+
export {
|
|
242
|
+
StreamingPcm16Resampler,
|
|
243
|
+
createLoudnessState,
|
|
244
|
+
normalizeLoudness,
|
|
245
|
+
type LoudnessConfig,
|
|
246
|
+
type LoudnessState,
|
|
247
|
+
} from "./audio/index.js";
|
|
248
|
+
|
|
249
|
+
// Interaction policy seam (IP-C1)
|
|
250
|
+
export {
|
|
251
|
+
type InteractionPolicy,
|
|
252
|
+
type LifecycleInteractionPolicy,
|
|
253
|
+
isLifecycleInteractionPolicy,
|
|
254
|
+
type InteractionObservation,
|
|
255
|
+
type InteractionDecision,
|
|
256
|
+
type AcousticSignalObservation,
|
|
257
|
+
type AcousticSignalSink,
|
|
258
|
+
type WordTiming,
|
|
259
|
+
} from "./interaction-policy.js";
|
|
260
|
+
export { confidenceToWaitMs, type ConfidenceToWaitConfig } from "./confidence-to-wait.js";
|
|
261
|
+
export { RuleBasedInteractionPolicy } from "./policies/rule-based.js";
|
|
262
|
+
export { DeferInteractionPolicy } from "./policies/defer.js";
|
|
263
|
+
export { InteractionCoordinator, type InteractionCaps } from "./interaction-coordinator.js";
|
|
264
|
+
export {
|
|
265
|
+
type InteractionBackchannelPacket,
|
|
266
|
+
type InteractionDuckPacket,
|
|
267
|
+
type InteractionResumePacket,
|
|
268
|
+
} from "./packets.js";
|
|
269
|
+
|
|
190
270
|
// VoiceAgentSession
|
|
191
|
-
export {
|
|
271
|
+
export {
|
|
272
|
+
VoiceAgentSession,
|
|
273
|
+
type VoiceAgentSessionConfig,
|
|
274
|
+
type VoiceAgentSessionEvents,
|
|
275
|
+
type SessionStageUsage,
|
|
276
|
+
} from "./voice-agent-session.js";
|
|
192
277
|
|
|
193
278
|
// Primary-speaker barge-in gate (VE-02)
|
|
194
279
|
export {
|
|
@@ -196,6 +281,7 @@ export {
|
|
|
196
281
|
extractSpeakerFingerprint,
|
|
197
282
|
fingerprintSimilarity,
|
|
198
283
|
type SpeakerFingerprint,
|
|
284
|
+
type PrimarySpeakerGateDecision,
|
|
199
285
|
type PrimarySpeakerGateConfig,
|
|
200
286
|
} from "./primary-speaker-gate.js";
|
|
201
287
|
export {
|
|
@@ -221,10 +307,24 @@ export {
|
|
|
221
307
|
type LatencyFillerFixture,
|
|
222
308
|
} from "./latency-filler-fixtures.js";
|
|
223
309
|
|
|
310
|
+
// Voice text: markdown/formatting normalization + leaked-tool-call guard before TTS
|
|
311
|
+
export { normalizeForSpeech, stripLeakedToolCalls } from "./voice-text.js";
|
|
312
|
+
|
|
224
313
|
// Reasoner seam (RFC §4.2)
|
|
225
314
|
export {
|
|
226
315
|
type Reasoner,
|
|
227
316
|
type ReasonerTurn,
|
|
228
317
|
type ReasonerMessage,
|
|
318
|
+
type ReasonerUsage,
|
|
229
319
|
type ReasoningPart,
|
|
230
320
|
} from "./reasoner.js";
|
|
321
|
+
|
|
322
|
+
// Hedged reasoner (Lever C — reasoner-latency RFC)
|
|
323
|
+
export { HedgedReasoner, type HedgedReasonerOptions } from "./reasoner-hedge.js";
|
|
324
|
+
|
|
325
|
+
// Routing reasoner (Lever B — reasoner-latency RFC)
|
|
326
|
+
export {
|
|
327
|
+
RoutingReasoner,
|
|
328
|
+
type RoutingReasonerOptions,
|
|
329
|
+
type ReasonerRoute,
|
|
330
|
+
} from "./reasoner-route.js";
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import type { VadAudioPacket } from "./packets.js";
|
|
4
|
+
import type { SttInterimPacket, SttResultPacket } from "./packets.js";
|
|
5
|
+
import { Route } from "./pipeline-bus.js";
|
|
6
|
+
import type { PipelineBus } from "./pipeline-bus.js";
|
|
7
|
+
import { confidenceToWaitMs } from "./confidence-to-wait.js";
|
|
8
|
+
import type { InteractionDecision, InteractionObservation, InteractionPolicy } from "./interaction-policy.js";
|
|
9
|
+
import type { TurnArbiter } from "./turn-arbiter.js";
|
|
10
|
+
import { RuleBasedInteractionPolicy } from "./policies/rule-based.js";
|
|
11
|
+
import { TimerScheduler, type Scheduler } from "./scheduler.js";
|
|
12
|
+
import * as make from "./packet-factories.js";
|
|
13
|
+
|
|
14
|
+
export interface InteractionCaps {
|
|
15
|
+
readonly emitsBackchannel?: boolean;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
interface PendingTakeTurn {
|
|
19
|
+
confidence: number;
|
|
20
|
+
waitMs?: number;
|
|
21
|
+
transcripts: SttResultPacket[];
|
|
22
|
+
latestInterim: string;
|
|
23
|
+
finalizeRequested: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export class InteractionCoordinator {
|
|
27
|
+
private readonly disposers: Array<() => void> = [];
|
|
28
|
+
private readonly pendingTakeTurns = new Map<string, PendingTakeTurn>();
|
|
29
|
+
private readonly transcriptsByContext = new Map<string, SttResultPacket[]>();
|
|
30
|
+
private readonly interimByContext = new Map<string, string>();
|
|
31
|
+
private readonly scheduler: Scheduler;
|
|
32
|
+
|
|
33
|
+
constructor(
|
|
34
|
+
private readonly deps: {
|
|
35
|
+
bus: PipelineBus;
|
|
36
|
+
policy: InteractionPolicy;
|
|
37
|
+
executor: TurnArbiter;
|
|
38
|
+
caps: InteractionCaps;
|
|
39
|
+
scheduler?: Scheduler;
|
|
40
|
+
isUserSpeaking?: () => boolean;
|
|
41
|
+
isTtsActive?: () => boolean;
|
|
42
|
+
hasCueAsset?: (cueId: string) => boolean;
|
|
43
|
+
onBackchannelEmitted?: (contextId: string) => void;
|
|
44
|
+
},
|
|
45
|
+
) {
|
|
46
|
+
this.scheduler = deps.scheduler ?? new TimerScheduler();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
initialize(): void {
|
|
50
|
+
const { bus } = this.deps;
|
|
51
|
+
this.disposers.push(
|
|
52
|
+
bus.on("stt.result", (pkt) => {
|
|
53
|
+
this.handleSttResult(pkt as SttResultPacket);
|
|
54
|
+
}),
|
|
55
|
+
bus.on("stt.interim", (pkt) => {
|
|
56
|
+
this.handleSttInterim(pkt as SttInterimPacket);
|
|
57
|
+
}),
|
|
58
|
+
bus.on("vad.speech_started", (pkt) => {
|
|
59
|
+
this.revokePendingTakeTurn((pkt as { contextId: string }).contextId);
|
|
60
|
+
}),
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
dispose(): void {
|
|
65
|
+
for (const contextId of this.pendingTakeTurns.keys()) {
|
|
66
|
+
this.revokePendingTakeTurn(contextId);
|
|
67
|
+
}
|
|
68
|
+
for (const dispose of this.disposers.splice(0)) {
|
|
69
|
+
dispose();
|
|
70
|
+
}
|
|
71
|
+
this.transcriptsByContext.clear();
|
|
72
|
+
this.interimByContext.clear();
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
observe(obs: InteractionObservation): void {
|
|
76
|
+
for (const d of this.deps.policy.observe(obs)) {
|
|
77
|
+
this.apply(d, obs);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
observeBargeInAudio(pkt: VadAudioPacket): boolean {
|
|
82
|
+
this.observe({
|
|
83
|
+
kind: "vad_barge_in_audio",
|
|
84
|
+
contextId: pkt.contextId,
|
|
85
|
+
timestampMs: pkt.timestampMs,
|
|
86
|
+
audio: pkt.audio,
|
|
87
|
+
});
|
|
88
|
+
if (this.deps.policy instanceof RuleBasedInteractionPolicy) {
|
|
89
|
+
return this.deps.policy.takeBargeInAudioConsumed();
|
|
90
|
+
}
|
|
91
|
+
return this.deps.executor.observeBargeInAudio(pkt);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private apply(d: InteractionDecision, obs: InteractionObservation): void {
|
|
95
|
+
switch (d.kind) {
|
|
96
|
+
case "take_turn":
|
|
97
|
+
this.armTakeTurn(obs.contextId, d.confidence, d.waitMs, obs.timestampMs);
|
|
98
|
+
break;
|
|
99
|
+
case "hold":
|
|
100
|
+
case "keep_listening":
|
|
101
|
+
this.revokePendingTakeTurn(obs.contextId);
|
|
102
|
+
break;
|
|
103
|
+
case "interrupt":
|
|
104
|
+
this.deps.executor.emitInterruptDetected(d.interruptedContextId);
|
|
105
|
+
break;
|
|
106
|
+
case "backchannel":
|
|
107
|
+
this.applyBackchannel(d, obs);
|
|
108
|
+
break;
|
|
109
|
+
default:
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private armTakeTurn(contextId: string, confidence: number, waitMs: number | undefined, timestampMs: number): void {
|
|
115
|
+
const existing = this.pendingTakeTurns.get(contextId);
|
|
116
|
+
const pending: PendingTakeTurn = existing ?? {
|
|
117
|
+
confidence,
|
|
118
|
+
transcripts: [...(this.transcriptsByContext.get(contextId) ?? [])],
|
|
119
|
+
latestInterim: this.interimByContext.get(contextId) ?? "",
|
|
120
|
+
finalizeRequested: false,
|
|
121
|
+
};
|
|
122
|
+
pending.confidence = confidence;
|
|
123
|
+
pending.waitMs = waitMs;
|
|
124
|
+
this.pendingTakeTurns.set(contextId, pending);
|
|
125
|
+
|
|
126
|
+
if (!pending.finalizeRequested) {
|
|
127
|
+
pending.finalizeRequested = true;
|
|
128
|
+
this.deps.bus.push(Route.Critical, make.finalizeStt(contextId, timestampMs));
|
|
129
|
+
}
|
|
130
|
+
this.tryScheduleTurnComplete(contextId);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private handleSttInterim(pkt: SttInterimPacket): void {
|
|
134
|
+
const text = pkt.text.trim();
|
|
135
|
+
if (!text) return;
|
|
136
|
+
this.interimByContext.set(pkt.contextId, text);
|
|
137
|
+
const pending = this.pendingTakeTurns.get(pkt.contextId);
|
|
138
|
+
if (pending) pending.latestInterim = text;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private handleSttResult(pkt: SttResultPacket): void {
|
|
142
|
+
const text = pkt.text.trim();
|
|
143
|
+
if (!text) return;
|
|
144
|
+
const transcripts = this.transcriptsByContext.get(pkt.contextId) ?? [];
|
|
145
|
+
if (transcripts.at(-1)?.text.trim() !== text) {
|
|
146
|
+
transcripts.push(pkt);
|
|
147
|
+
this.transcriptsByContext.set(pkt.contextId, transcripts);
|
|
148
|
+
}
|
|
149
|
+
this.interimByContext.delete(pkt.contextId);
|
|
150
|
+
|
|
151
|
+
const pending = this.pendingTakeTurns.get(pkt.contextId);
|
|
152
|
+
if (!pending) return;
|
|
153
|
+
pending.transcripts = [...transcripts];
|
|
154
|
+
pending.latestInterim = "";
|
|
155
|
+
this.tryScheduleTurnComplete(pkt.contextId);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private tryScheduleTurnComplete(contextId: string): void {
|
|
159
|
+
const pending = this.pendingTakeTurns.get(contextId);
|
|
160
|
+
if (!pending) return;
|
|
161
|
+
|
|
162
|
+
// Anchor the finalize timer at the policy's commitment even before any text
|
|
163
|
+
// exists — the callback reads live transcript/interim state at fire time and
|
|
164
|
+
// is the sole gate on whether the turn actually completes.
|
|
165
|
+
const delayMs = pending.waitMs ?? confidenceToWaitMs(pending.confidence);
|
|
166
|
+
const timerKey = takeTurnTimerKey(contextId);
|
|
167
|
+
this.scheduler.cancel(timerKey);
|
|
168
|
+
this.scheduler.schedule(timerKey, delayMs, () => {
|
|
169
|
+
const live = this.pendingTakeTurns.get(contextId);
|
|
170
|
+
if (!live) return;
|
|
171
|
+
|
|
172
|
+
const finalText = joinTranscript(live.transcripts, live.latestInterim);
|
|
173
|
+
// The policy committed to this turn. Prefer real STT finals; if none ever
|
|
174
|
+
// arrived, fall back to the latest interim so a committed turn is never
|
|
175
|
+
// silently dropped. Only bail when there is genuinely nothing to commit.
|
|
176
|
+
if (!finalText) return;
|
|
177
|
+
|
|
178
|
+
this.pendingTakeTurns.delete(contextId);
|
|
179
|
+
const transcripts: SttResultPacket[] =
|
|
180
|
+
live.transcripts.length > 0
|
|
181
|
+
? live.transcripts
|
|
182
|
+
: [{ kind: "stt.result", contextId, timestampMs: Date.now(), text: finalText, confidence: 0 }];
|
|
183
|
+
this.deps.bus.push(Route.Main, make.eosTurnComplete(contextId, Date.now(), finalText, transcripts));
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private revokePendingTakeTurn(contextId: string): void {
|
|
188
|
+
if (!this.pendingTakeTurns.has(contextId)) return;
|
|
189
|
+
this.scheduler.cancel(takeTurnTimerKey(contextId));
|
|
190
|
+
this.pendingTakeTurns.delete(contextId);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
private applyBackchannel(
|
|
194
|
+
d: Extract<InteractionDecision, { kind: "backchannel" }>,
|
|
195
|
+
obs: InteractionObservation,
|
|
196
|
+
): void {
|
|
197
|
+
const contextId = obs.contextId;
|
|
198
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.candidate", d.cue));
|
|
199
|
+
|
|
200
|
+
if (this.deps.caps.emitsBackchannel) {
|
|
201
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.suppressed_caps", d.cue));
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
if (this.deps.isTtsActive?.()) {
|
|
205
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.suppressed_tts_active", d.cue));
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (this.deps.isUserSpeaking?.()) {
|
|
209
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.suppressed_user_speaking", d.cue));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (this.deps.hasCueAsset && !this.deps.hasCueAsset(d.cue)) {
|
|
213
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.suppressed_missing_asset", d.cue));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
this.deps.bus.push(Route.Main, make.interactionBackchannel(contextId, Date.now(), d.cue));
|
|
218
|
+
this.deps.bus.push(Route.Background, make.metric(contextId, "backchannel.emitted", d.cue));
|
|
219
|
+
this.deps.onBackchannelEmitted?.(contextId);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
reset(contextId: string): void {
|
|
223
|
+
this.revokePendingTakeTurn(contextId);
|
|
224
|
+
this.transcriptsByContext.delete(contextId);
|
|
225
|
+
this.interimByContext.delete(contextId);
|
|
226
|
+
this.deps.policy.reset(contextId);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function takeTurnTimerKey(contextId: string): string {
|
|
231
|
+
return `interaction.take_turn:${contextId}`;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function joinTranscript(transcripts: readonly SttResultPacket[], interim: string): string {
|
|
235
|
+
const segments = [
|
|
236
|
+
...transcripts.map((t) => t.text.trim()),
|
|
237
|
+
...(interim ? [interim.trim()] : []),
|
|
238
|
+
].filter(Boolean);
|
|
239
|
+
return segments.join(" ").replace(/\s+/g, " ").trim();
|
|
240
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import type { AcousticSignal } from "./packets.js";
|
|
4
|
+
|
|
5
|
+
export interface WordTiming {
|
|
6
|
+
readonly word: string;
|
|
7
|
+
readonly startMs: number;
|
|
8
|
+
readonly endMs: number;
|
|
9
|
+
readonly confidence: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export type InteractionObservation =
|
|
13
|
+
| {
|
|
14
|
+
readonly kind: "vad_speech_started";
|
|
15
|
+
readonly contextId: string;
|
|
16
|
+
readonly timestampMs: number;
|
|
17
|
+
readonly confidence: number;
|
|
18
|
+
readonly interruptedContextId?: string;
|
|
19
|
+
}
|
|
20
|
+
| {
|
|
21
|
+
readonly kind: "vad_speech_activity";
|
|
22
|
+
readonly contextId: string;
|
|
23
|
+
readonly timestampMs: number;
|
|
24
|
+
}
|
|
25
|
+
| {
|
|
26
|
+
readonly kind: "vad_speech_ended";
|
|
27
|
+
readonly contextId: string;
|
|
28
|
+
readonly timestampMs: number;
|
|
29
|
+
readonly hasActiveTts: boolean;
|
|
30
|
+
}
|
|
31
|
+
| {
|
|
32
|
+
readonly kind: "vad_barge_in_audio";
|
|
33
|
+
readonly contextId: string;
|
|
34
|
+
readonly timestampMs: number;
|
|
35
|
+
readonly audio: Uint8Array;
|
|
36
|
+
}
|
|
37
|
+
| {
|
|
38
|
+
readonly kind: "stt_partial";
|
|
39
|
+
readonly contextId: string;
|
|
40
|
+
readonly timestampMs: number;
|
|
41
|
+
readonly text: string;
|
|
42
|
+
readonly confidence?: number;
|
|
43
|
+
readonly interruptedContextId?: string;
|
|
44
|
+
readonly wordTimings?: readonly WordTiming[];
|
|
45
|
+
}
|
|
46
|
+
| {
|
|
47
|
+
readonly kind: "stt_final";
|
|
48
|
+
readonly contextId: string;
|
|
49
|
+
readonly timestampMs: number;
|
|
50
|
+
readonly text: string;
|
|
51
|
+
readonly confidence?: number;
|
|
52
|
+
readonly interruptedContextId?: string;
|
|
53
|
+
readonly wordTimings?: readonly WordTiming[];
|
|
54
|
+
}
|
|
55
|
+
| {
|
|
56
|
+
readonly kind: "audio_frame";
|
|
57
|
+
readonly contextId: string;
|
|
58
|
+
readonly timestampMs: number;
|
|
59
|
+
readonly audio?: Int16Array;
|
|
60
|
+
readonly sampleRateHz?: number;
|
|
61
|
+
readonly wordTimings?: readonly WordTiming[];
|
|
62
|
+
readonly prosody?: Float32Array;
|
|
63
|
+
}
|
|
64
|
+
| {
|
|
65
|
+
readonly kind: "playout_tick";
|
|
66
|
+
readonly contextId: string;
|
|
67
|
+
readonly timestampMs: number;
|
|
68
|
+
readonly playedOutMs?: number;
|
|
69
|
+
readonly ttsActive?: boolean;
|
|
70
|
+
/** PCM queued for assistant playout. Present on audio-bearing ticks. */
|
|
71
|
+
readonly audio?: Int16Array;
|
|
72
|
+
readonly sampleRateHz?: number;
|
|
73
|
+
}
|
|
74
|
+
| {
|
|
75
|
+
readonly kind: "delegate_state";
|
|
76
|
+
readonly contextId: string;
|
|
77
|
+
readonly timestampMs: number;
|
|
78
|
+
readonly delegateInFlight?: boolean;
|
|
79
|
+
readonly toolCallPhase?: "started" | "delayed" | "complete" | "failed";
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
export type InteractionDecision =
|
|
83
|
+
| { readonly kind: "keep_listening" }
|
|
84
|
+
| { readonly kind: "take_turn"; readonly confidence: number; readonly waitMs?: number }
|
|
85
|
+
/** `cue` is a stable pre-cached asset id (e.g. `mm_hmm`), not free-form text. */
|
|
86
|
+
| { readonly kind: "backchannel"; readonly cue: string }
|
|
87
|
+
| { readonly kind: "hold" }
|
|
88
|
+
| { readonly kind: "interrupt"; readonly interruptedContextId: string };
|
|
89
|
+
|
|
90
|
+
export interface AcousticSignalObservation {
|
|
91
|
+
readonly contextId: string;
|
|
92
|
+
readonly timestampMs: number;
|
|
93
|
+
readonly signal: AcousticSignal;
|
|
94
|
+
readonly payload?: Readonly<Record<string, unknown>>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export type AcousticSignalSink = (signal: AcousticSignalObservation) => void;
|
|
98
|
+
|
|
99
|
+
export interface InteractionPolicy {
|
|
100
|
+
observe(obs: InteractionObservation): readonly InteractionDecision[];
|
|
101
|
+
reset(contextId: string): void;
|
|
102
|
+
setAcousticSignalSink?(sink: AcousticSignalSink): void;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Optional lifecycle for externally supplied model policies (session-owned). */
|
|
106
|
+
export interface LifecycleInteractionPolicy extends InteractionPolicy {
|
|
107
|
+
initialize(config: Record<string, unknown>): Promise<void>;
|
|
108
|
+
close(): Promise<void>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function isLifecycleInteractionPolicy(
|
|
112
|
+
policy: InteractionPolicy,
|
|
113
|
+
): policy is LifecycleInteractionPolicy {
|
|
114
|
+
const candidate = policy as LifecycleInteractionPolicy;
|
|
115
|
+
return typeof candidate.initialize === "function" && typeof candidate.close === "function";
|
|
116
|
+
}
|
package/src/iu-ledger.ts
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Incremental-Unit ledger (IU substrate, RFC incremental-unit-substrate)
|
|
4
|
+
|
|
5
|
+
import type { IncrementalUnit, IncrementalUnitId, IuState } from "./incremental-unit.js";
|
|
6
|
+
|
|
7
|
+
export interface IuLedger {
|
|
8
|
+
add(iu: IncrementalUnit): void;
|
|
9
|
+
commit(id: IncrementalUnitId, prefix?: { chars?: number; ms?: number }): void;
|
|
10
|
+
revoke(id: IncrementalUnitId): void;
|
|
11
|
+
get(id: IncrementalUnitId): IncrementalUnit | undefined;
|
|
12
|
+
latest(contextId: string, kind: IncrementalUnit["kind"]): IncrementalUnit | undefined;
|
|
13
|
+
clear(contextId: string): void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type IuLedgerAnomaly =
|
|
17
|
+
| { readonly kind: "terminal_op"; readonly op: "commit" | "revoke"; readonly id: IncrementalUnitId; readonly state: IuState }
|
|
18
|
+
| { readonly kind: "unknown_iu"; readonly op: "commit" | "revoke"; readonly id: IncrementalUnitId };
|
|
19
|
+
|
|
20
|
+
export class InMemoryIuLedger implements IuLedger {
|
|
21
|
+
private readonly byCtx = new Map<string, Map<string, IncrementalUnit>>();
|
|
22
|
+
|
|
23
|
+
constructor(
|
|
24
|
+
private readonly onEvent: (a: IuLedgerAnomaly) => void = () => {},
|
|
25
|
+
private readonly maxContexts: number = 256,
|
|
26
|
+
) {}
|
|
27
|
+
|
|
28
|
+
add(iu: IncrementalUnit): void {
|
|
29
|
+
const ctx = iu.id.contextId;
|
|
30
|
+
if (!this.byCtx.has(ctx) && this.byCtx.size >= this.maxContexts) {
|
|
31
|
+
const oldest = this.byCtx.keys().next().value;
|
|
32
|
+
if (oldest !== undefined) this.byCtx.delete(oldest);
|
|
33
|
+
}
|
|
34
|
+
this.ctxMap(ctx).set(iu.id.iuId, iu);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
commit(id: IncrementalUnitId, prefix?: { chars?: number; ms?: number }): void {
|
|
38
|
+
const ctxMap = this.byCtx.get(id.contextId);
|
|
39
|
+
if (!ctxMap) {
|
|
40
|
+
this.onEvent({ kind: "unknown_iu", op: "commit", id });
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const iu = ctxMap.get(id.iuId);
|
|
44
|
+
if (!iu) {
|
|
45
|
+
this.onEvent({ kind: "unknown_iu", op: "commit", id });
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (iu.state !== "hypothesized") {
|
|
49
|
+
this.onEvent({ kind: "terminal_op", op: "commit", id, state: iu.state });
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
iu.state = "committed";
|
|
53
|
+
if (prefix) {
|
|
54
|
+
iu.committedPrefix = prefix;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
revoke(id: IncrementalUnitId): void {
|
|
59
|
+
const ctxMap = this.byCtx.get(id.contextId);
|
|
60
|
+
if (!ctxMap) {
|
|
61
|
+
this.onEvent({ kind: "unknown_iu", op: "revoke", id });
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const iu = ctxMap.get(id.iuId);
|
|
65
|
+
if (!iu) {
|
|
66
|
+
this.onEvent({ kind: "unknown_iu", op: "revoke", id });
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
if (iu.state !== "hypothesized") {
|
|
70
|
+
this.onEvent({ kind: "terminal_op", op: "revoke", id, state: iu.state });
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
iu.state = "revoked";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
get(id: IncrementalUnitId): IncrementalUnit | undefined {
|
|
77
|
+
const ctxMap = this.byCtx.get(id.contextId);
|
|
78
|
+
if (!ctxMap) {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
return ctxMap.get(id.iuId);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
latest(contextId: string, kind: IncrementalUnit["kind"]): IncrementalUnit | undefined {
|
|
85
|
+
const ctxMap = this.byCtx.get(contextId);
|
|
86
|
+
if (!ctxMap) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
let found: IncrementalUnit | undefined;
|
|
90
|
+
for (const iu of ctxMap.values()) {
|
|
91
|
+
if (iu.kind === kind) {
|
|
92
|
+
found = iu;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return found;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
clear(contextId: string): void {
|
|
99
|
+
this.byCtx.delete(contextId);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private ctxMap(contextId: string): Map<string, IncrementalUnit> {
|
|
103
|
+
let map = this.byCtx.get(contextId);
|
|
104
|
+
if (!map) {
|
|
105
|
+
map = new Map();
|
|
106
|
+
this.byCtx.set(contextId, map);
|
|
107
|
+
}
|
|
108
|
+
return map;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
|
|
3
3
|
import { Route, type PipelineBus } from "./pipeline-bus.js";
|
|
4
|
-
import { monotonicNowMs, type MetricsExporter } from "./observability.js";
|
|
4
|
+
import { monotonicNowMs, type MetricsExporter, type ObservabilityLayer } from "./observability.js";
|
|
5
5
|
import type {
|
|
6
6
|
TurnBoundaryKind,
|
|
7
7
|
VadSpeechStartedPacket,
|
|
@@ -17,6 +17,7 @@ export interface ObservabilityDims {
|
|
|
17
17
|
readonly provider: string;
|
|
18
18
|
readonly model: string;
|
|
19
19
|
readonly region: string;
|
|
20
|
+
readonly layer?: ObservabilityLayer;
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
export interface ObservabilityObserverDeps {
|
|
@@ -124,12 +125,11 @@ export class ObservabilityObserver {
|
|
|
124
125
|
|
|
125
126
|
const dims = this.dimsFor(speechId, cancelled);
|
|
126
127
|
const tags = {
|
|
127
|
-
sessionId: this.deps.sessionId,
|
|
128
|
-
speechId,
|
|
129
128
|
provider: dims.provider,
|
|
130
129
|
model: dims.model,
|
|
131
130
|
region: dims.region,
|
|
132
131
|
cancelled: dims.cancelled ? "true" : "false",
|
|
132
|
+
layer: dims.layer ?? "conversation",
|
|
133
133
|
};
|
|
134
134
|
|
|
135
135
|
if (boundary === "agent_started_speaking") {
|
|
@@ -179,12 +179,16 @@ export class ObservabilityObserver {
|
|
|
179
179
|
});
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
-
private dimsFor(
|
|
182
|
+
private dimsFor(
|
|
183
|
+
speechId: string,
|
|
184
|
+
cancelled: boolean,
|
|
185
|
+
): Required<ObservabilityDims> & { cancelled: boolean } {
|
|
183
186
|
const stage = this.stageDims.get(speechId) ?? {};
|
|
184
187
|
return {
|
|
185
188
|
provider: stage.provider ?? this.deps.dims.provider,
|
|
186
189
|
model: stage.model ?? this.deps.dims.model,
|
|
187
190
|
region: stage.region ?? this.deps.dims.region,
|
|
191
|
+
layer: stage.layer ?? this.deps.dims.layer ?? "conversation",
|
|
188
192
|
cancelled: cancelled || stage.cancelled === true,
|
|
189
193
|
};
|
|
190
194
|
}
|