@kuralle-syrinx/core 4.1.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +4 -0
  2. package/package.json +22 -3
  3. package/src/audio/alaw.ts +56 -0
  4. package/src/audio/g722.ts +329 -0
  5. package/src/audio/index.ts +12 -0
  6. package/src/audio/loudness.ts +87 -0
  7. package/src/confidence-to-wait.ts +30 -0
  8. package/src/idle-timeout.ts +1 -0
  9. package/src/incremental-unit.ts +19 -0
  10. package/src/index.ts +101 -1
  11. package/src/interaction-coordinator.ts +240 -0
  12. package/src/interaction-policy.ts +116 -0
  13. package/src/iu-ledger.ts +110 -0
  14. package/src/observability-observer.ts +8 -4
  15. package/src/observability.ts +30 -1
  16. package/src/packet-factories.ts +137 -2
  17. package/src/packets.ts +140 -4
  18. package/src/plugin-contract.ts +41 -0
  19. package/src/policies/defer.ts +15 -0
  20. package/src/policies/rule-based.ts +155 -0
  21. package/src/pricing.ts +137 -0
  22. package/src/primary-speaker-gate.ts +23 -3
  23. package/src/reasoner-hedge.ts +216 -0
  24. package/src/reasoner-route.ts +113 -0
  25. package/src/reasoner.ts +28 -1
  26. package/src/spend-cap.ts +52 -0
  27. package/src/turn-arbiter.ts +45 -4
  28. package/src/voice-agent-session.ts +682 -53
  29. package/src/voice-text.ts +69 -0
  30. package/src/audio/audio.test.ts +0 -285
  31. package/src/audio-envelope.test.ts +0 -167
  32. package/src/error-handler.test.ts +0 -56
  33. package/src/latency-filler.test.ts +0 -62
  34. package/src/observability-observer.test.ts +0 -245
  35. package/src/observability.test.ts +0 -85
  36. package/src/packet-factories.test.ts +0 -34
  37. package/src/pipeline-bus.g10.test.ts +0 -145
  38. package/src/pipeline-bus.test.ts +0 -211
  39. package/src/primary-speaker-gate.test.ts +0 -150
  40. package/src/provider-fallback.test.ts +0 -87
  41. package/src/reasoner.test.ts +0 -69
  42. package/src/retry.test.ts +0 -83
  43. package/src/tts-playout-clock.test.ts +0 -125
  44. package/src/turn-arbiter.characterization.test.ts +0 -477
  45. package/src/turn-arbiter.test.ts +0 -567
  46. package/src/voice-agent-session.test.ts +0 -2879
  47. package/src/voice-text.test.ts +0 -94
  48. package/tsconfig.json +0 -21
@@ -0,0 +1,155 @@
1
+ // SPDX-License-Identifier: MIT
2
+
3
+ import type { PipelineBus } from "../pipeline-bus.js";
4
+ import type { InteractionDecision, InteractionObservation, InteractionPolicy } from "../interaction-policy.js";
5
+ import { PrimarySpeakerGate } from "../primary-speaker-gate.js";
6
+ import { TtsPlayoutClock } from "../tts-playout-clock.js";
7
+ import { TurnArbiter } from "../turn-arbiter.js";
8
+
9
+ export interface RuleBasedInteractionPolicyDeps {
10
+ readonly bus: PipelineBus;
11
+ readonly primarySpeakerGate: PrimarySpeakerGate;
12
+ readonly ttsPlayout: TtsPlayoutClock;
13
+ readonly minInterruptionMs: number;
14
+ /** When true, the underlying TurnArbiter emits duck/resume signals during barge-in evaluation. */
15
+ readonly pauseThenResolveBargeIn?: boolean;
16
+ /** Tenant-level backchannel knobs. Default (undefined) = current behavior (`mm_hmm`). */
17
+ readonly backchannel?: {
18
+ readonly enabled?: boolean;
19
+ readonly cues?: readonly string[];
20
+ };
21
+ }
22
+
23
+ export class RuleBasedInteractionPolicy implements InteractionPolicy {
24
+ readonly arbiter: TurnArbiter;
25
+ private readonly ttsPlayout: TtsPlayoutClock;
26
+ private readonly backchannelConfig: RuleBasedInteractionPolicyDeps["backchannel"];
27
+ private pending: InteractionDecision[] = [];
28
+ private bargeInAudioConsumed = false;
29
+ private delegateGapOpen = false;
30
+ private delegateCuePlayed = false;
31
+ private userSpeaking = false;
32
+
33
+ constructor(deps: RuleBasedInteractionPolicyDeps) {
34
+ this.ttsPlayout = deps.ttsPlayout;
35
+ this.backchannelConfig = deps.backchannel;
36
+ this.arbiter = new TurnArbiter({
37
+ bus: deps.bus,
38
+ primarySpeakerGate: deps.primarySpeakerGate,
39
+ ttsPlayout: deps.ttsPlayout,
40
+ minInterruptionMs: deps.minInterruptionMs,
41
+ pauseThenResolveBargeIn: deps.pauseThenResolveBargeIn,
42
+ onInterrupt: (id) => this.pending.push({ kind: "interrupt", interruptedContextId: id }),
43
+ });
44
+ }
45
+
46
+ observe(obs: InteractionObservation): readonly InteractionDecision[] {
47
+ switch (obs.kind) {
48
+ case "vad_speech_started":
49
+ this.userSpeaking = true;
50
+ if (obs.interruptedContextId) this.arbiter.onSpeechStarted(
51
+ {
52
+ kind: "vad.speech_started",
53
+ contextId: obs.contextId,
54
+ timestampMs: obs.timestampMs,
55
+ confidence: obs.confidence,
56
+ },
57
+ obs.interruptedContextId,
58
+ );
59
+ break;
60
+ case "vad_speech_activity":
61
+ this.arbiter.onSpeechActivity({
62
+ kind: "vad.speech_activity",
63
+ contextId: obs.contextId,
64
+ timestampMs: obs.timestampMs,
65
+ isAsync: true,
66
+ });
67
+ break;
68
+ case "vad_speech_ended":
69
+ this.userSpeaking = false;
70
+ this.arbiter.onSpeechEnded(
71
+ {
72
+ kind: "vad.speech_ended",
73
+ contextId: obs.contextId,
74
+ timestampMs: obs.timestampMs,
75
+ },
76
+ obs.hasActiveTts,
77
+ );
78
+ break;
79
+ case "vad_barge_in_audio":
80
+ this.bargeInAudioConsumed = this.arbiter.observeBargeInAudio({
81
+ kind: "vad.audio",
82
+ contextId: obs.contextId,
83
+ timestampMs: obs.timestampMs,
84
+ audio: obs.audio,
85
+ });
86
+ break;
87
+ case "stt_partial":
88
+ this.arbiter.noteInterimEvidence(obs.text);
89
+ if (obs.interruptedContextId) {
90
+ this.arbiter.onProviderSttEvidence(obs.contextId, obs.timestampMs, obs.interruptedContextId);
91
+ }
92
+ break;
93
+ case "stt_final":
94
+ this.arbiter.noteInterimEvidence(obs.text, obs.confidence);
95
+ if (obs.interruptedContextId) {
96
+ this.arbiter.onProviderSttEvidence(obs.contextId, obs.timestampMs, obs.interruptedContextId);
97
+ }
98
+ break;
99
+ case "delegate_state":
100
+ this.pending.push(...this.handleDelegateState(obs));
101
+ break;
102
+ default:
103
+ break;
104
+ }
105
+ if (this.pending.length === 0) return [];
106
+ const out = this.pending;
107
+ this.pending = [];
108
+ return out;
109
+ }
110
+
111
+ private handleDelegateState(obs: InteractionObservation & { kind: "delegate_state" }): InteractionDecision[] {
112
+ const phase = obs.toolCallPhase;
113
+ if (!phase) return [];
114
+
115
+ switch (phase) {
116
+ case "started":
117
+ this.delegateGapOpen = true;
118
+ this.delegateCuePlayed = false;
119
+ return [];
120
+ case "delayed": {
121
+ if (!this.delegateGapOpen || this.delegateCuePlayed) return [];
122
+ if (this.depsTtsActive()) return [];
123
+ if (this.userSpeaking) return [];
124
+ this.delegateCuePlayed = true;
125
+ if (this.backchannelConfig?.enabled === false) return [];
126
+ const cue = this.backchannelConfig?.cues?.[0] ?? "mm_hmm";
127
+ return [{ kind: "backchannel", cue }];
128
+ }
129
+ case "complete":
130
+ case "failed":
131
+ this.delegateGapOpen = false;
132
+ this.delegateCuePlayed = false;
133
+ return [];
134
+ default:
135
+ return [];
136
+ }
137
+ }
138
+
139
+ private depsTtsActive(): boolean {
140
+ return this.ttsPlayout.activeContexts().length > 0;
141
+ }
142
+
143
+ reset(_contextId: string): void {
144
+ this.delegateGapOpen = false;
145
+ this.delegateCuePlayed = false;
146
+ this.userSpeaking = false;
147
+ this.arbiter.clear();
148
+ }
149
+
150
+ takeBargeInAudioConsumed(): boolean {
151
+ const consumed = this.bargeInAudioConsumed;
152
+ this.bargeInAudioConsumed = false;
153
+ return consumed;
154
+ }
155
+ }
package/src/pricing.ts ADDED
@@ -0,0 +1,137 @@
1
+ // SPDX-License-Identifier: MIT
2
+ //
3
+ // Versioned price catalog: usage.recorded quantities → USD.
4
+ // Unknown provider/model keys yield unpriced (never silent 0 for real providers).
5
+ // Declared local/self-hosted models are explicitly zero-cost.
6
+
7
+ import type { UsageRecordedPacket } from "./packets.js";
8
+
9
+ export interface SttPrice {
10
+ readonly usdPerAudioSecond: number;
11
+ }
12
+
13
+ export interface LlmPrice {
14
+ readonly usdPer1MInputTokens: number;
15
+ readonly usdPer1MOutputTokens: number;
16
+ readonly usdPer1MCachedInputTokens?: number;
17
+ }
18
+
19
+ export interface TtsPrice {
20
+ readonly usdPer1MCharacters: number;
21
+ }
22
+
23
+ export interface PriceCatalog {
24
+ readonly source: string;
25
+ readonly version: string;
26
+ /** Keys: `provider/model` (e.g. `deepgram/nova-3`). */
27
+ readonly stt: Readonly<Record<string, SttPrice>>;
28
+ readonly llm: Readonly<Record<string, LlmPrice>>;
29
+ readonly tts: Readonly<Record<string, TtsPrice>>;
30
+ }
31
+
32
+ export type CostResult =
33
+ | { readonly usd: number; readonly unpriced?: undefined }
34
+ | { readonly usd: null; readonly unpriced: true };
35
+
36
+ function catalogKey(provider: string | undefined, model: string | undefined): string | null {
37
+ if (!provider || !model) return null;
38
+ return `${provider}/${model}`;
39
+ }
40
+
41
+ /**
42
+ * Public list prices stamped into DEFAULT_PRICE_CATALOG.source.
43
+ * STT $/audio-second from $/min ÷ 60. TTS is $/1M characters. LLM is $/1M tokens.
44
+ */
45
+ export const DEFAULT_PRICE_CATALOG: PriceCatalog = {
46
+ source:
47
+ "voice-prices@3 | deepgram.com/pricing PAYG Nova-3 streaming monolingual $0.0077/min → $0.000128333/s; " +
48
+ "deepgram.com/pricing Aura-2 TTS $0.030/1k chars → $30/1M; " +
49
+ "cartesia.ai/pricing + docs.cartesia.ai/pricing ~1 credit/char, PAYG ~$50/1M chars (cloudtalk.io cartesia-pricing 2026); " +
50
+ "openai tts-1 $15/1M chars (openai.com api pricing); gpt-4o-mini-tts text $0.60/1M (community/azure listed text input rate); " +
51
+ "openai GPT-4.1-mini $0.40/$1.60 per 1M in/out (developers.openai.com/api/docs/pricing; Azure GPT-4.1-mini-2025-04-14 Global); " +
52
+ "cloud.google.com/speech-to-text/pricing V2 standard recognition $0.016/min → $0.000266667/s (latest_long); " +
53
+ "ai.google.dev/pricing Gemini 2.5 Flash TTS preview text $0.50/1M chars (Gemini API TTS text input rate, 2026); " +
54
+ "elevenlabs.io/pricing/api Flash/Turbo TTS $0.05/1k chars → $50/1M; Multilingual v2/v3 $0.10/1k → $100/1M; " +
55
+ "Scribe v2 Realtime $0.39/hour → $0.000108333/s; " +
56
+ "grok/stt + grok/* TTS + epsilon/epsilon-tts intentionally ABSENT (no public list price → costOf unpriced, never silent $0)",
57
+ version: "3",
58
+ stt: {
59
+ "deepgram/nova-3": { usdPerAudioSecond: 0.0077 / 60 },
60
+ "deepgram/nova-2": { usdPerAudioSecond: 0.0077 / 60 },
61
+ "deepgram/flux-general-en": { usdPerAudioSecond: 0.0077 / 60 },
62
+ // Google Cloud Speech-to-Text V2 standard streaming recognition list rate.
63
+ "google/latest_long": { usdPerAudioSecond: 0.016 / 60 },
64
+ "google/latest_short": { usdPerAudioSecond: 0.016 / 60 },
65
+ "google/chirp_2": { usdPerAudioSecond: 0.016 / 60 },
66
+ // ElevenLabs Scribe v2 Realtime — public list $0.39/hour.
67
+ "elevenlabs/scribe_v2_realtime": { usdPerAudioSecond: 0.39 / 3600 },
68
+ "local/whisper": { usdPerAudioSecond: 0 },
69
+ // grok/stt: no public list price — omit so costOf → unpriced
70
+ },
71
+ llm: {
72
+ "openai/gpt-4.1-mini": {
73
+ usdPer1MInputTokens: 0.4,
74
+ usdPer1MOutputTokens: 1.6,
75
+ usdPer1MCachedInputTokens: 0.1,
76
+ },
77
+ "local/llm": {
78
+ usdPer1MInputTokens: 0,
79
+ usdPer1MOutputTokens: 0,
80
+ },
81
+ },
82
+ tts: {
83
+ "cartesia/sonic-3": { usdPer1MCharacters: 50 },
84
+ "openai/gpt-4o-mini-tts": { usdPer1MCharacters: 0.6 },
85
+ "openai/tts-1": { usdPer1MCharacters: 15 },
86
+ "deepgram/aura-2-thalia-en": { usdPer1MCharacters: 30 },
87
+ // Gemini TTS preview models — Gemini API TTS text input list rate.
88
+ "gemini/gemini-3.1-flash-tts-preview": { usdPer1MCharacters: 0.5 },
89
+ "gemini/gemini-2.5-flash-preview-tts": { usdPer1MCharacters: 0.5 },
90
+ "gemini/gemini-2.5-pro-preview-tts": { usdPer1MCharacters: 0.5 },
91
+ // ElevenLabs public list — Flash/Turbo $0.05/1k, Multilingual $0.10/1k.
92
+ "elevenlabs/eleven_flash_v2_5": { usdPer1MCharacters: 50 },
93
+ "elevenlabs/eleven_flash_v2": { usdPer1MCharacters: 50 },
94
+ "elevenlabs/eleven_turbo_v2_5": { usdPer1MCharacters: 50 },
95
+ "elevenlabs/eleven_turbo_v2": { usdPer1MCharacters: 50 },
96
+ "elevenlabs/eleven_multilingual_v2": { usdPer1MCharacters: 100 },
97
+ "local/tts": { usdPer1MCharacters: 0 },
98
+ // grok/* voice keys + epsilon/epsilon-tts: no public list — omit → unpriced
99
+ },
100
+ };
101
+
102
+ export function costOf(usage: UsageRecordedPacket, catalog: PriceCatalog): CostResult {
103
+ const key = catalogKey(usage.provider, usage.model);
104
+ if (!key) return { usd: null, unpriced: true };
105
+
106
+ switch (usage.stage) {
107
+ case "stt": {
108
+ const price = catalog.stt[key];
109
+ if (!price) return { usd: null, unpriced: true };
110
+ const seconds = usage.audioSeconds ?? 0;
111
+ return { usd: seconds * price.usdPerAudioSecond };
112
+ }
113
+ case "tts": {
114
+ const price = catalog.tts[key];
115
+ if (!price) return { usd: null, unpriced: true };
116
+ const characters = usage.characters ?? 0;
117
+ return { usd: (characters / 1_000_000) * price.usdPer1MCharacters };
118
+ }
119
+ case "llm": {
120
+ const price = catalog.llm[key];
121
+ if (!price) return { usd: null, unpriced: true };
122
+ const input = usage.inputTokens ?? 0;
123
+ const output = usage.outputTokens ?? 0;
124
+ const cached = usage.cachedInputTokens ?? 0;
125
+ const cachedRate = price.usdPer1MCachedInputTokens ?? price.usdPer1MInputTokens;
126
+ // Prefer charging cached tokens at the cached rate when present; non-cached input is the remainder.
127
+ const nonCachedInput = Math.max(0, input - cached);
128
+ const usd =
129
+ (nonCachedInput / 1_000_000) * price.usdPer1MInputTokens +
130
+ (cached / 1_000_000) * cachedRate +
131
+ (output / 1_000_000) * price.usdPer1MOutputTokens;
132
+ return { usd };
133
+ }
134
+ default:
135
+ return { usd: null, unpriced: true };
136
+ }
137
+ }
@@ -19,12 +19,23 @@ export interface PrimarySpeakerGateConfig {
19
19
  readonly enabled?: boolean;
20
20
  readonly similarityThreshold?: number;
21
21
  readonly echoDominanceMargin?: number;
22
+ readonly onDecision?: (decision: PrimarySpeakerGateDecision) => void;
23
+ }
24
+
25
+ export interface PrimarySpeakerGateDecision {
26
+ readonly contextId: string;
27
+ readonly signal: "primary_speaker" | "echo_rejected";
28
+ readonly accepted: boolean;
29
+ readonly frameCount: number;
30
+ readonly primaryHits: number;
31
+ readonly echoRejectedFrames: number;
22
32
  }
23
33
 
24
34
  export class PrimarySpeakerGate {
25
35
  private readonly enabled: boolean;
26
36
  private readonly similarityThreshold: number;
27
37
  private readonly echoDominanceMargin: number;
38
+ private readonly onDecision?: PrimarySpeakerGateConfig["onDecision"];
28
39
  private profile: SpeakerFingerprint | null = null;
29
40
  private enrollFrames: SpeakerFingerprint[] = [];
30
41
  private bargeInFrames: SpeakerFingerprint[] = [];
@@ -34,6 +45,7 @@ export class PrimarySpeakerGate {
34
45
  this.enabled = config.enabled !== false;
35
46
  this.similarityThreshold = config.similarityThreshold ?? 0.72;
36
47
  this.echoDominanceMargin = config.echoDominanceMargin ?? 0.12;
48
+ this.onDecision = config.onDecision;
37
49
  }
38
50
 
39
51
  isEnabled(): boolean {
@@ -73,22 +85,30 @@ export class PrimarySpeakerGate {
73
85
  this.assistantProfile = frame;
74
86
  }
75
87
 
76
- shouldCommitBargeIn(): boolean {
88
+ shouldCommitBargeIn(contextId = ""): boolean {
77
89
  if (!this.enabled || this.profile === null) return true;
78
90
  if (this.bargeInFrames.length === 0) return false;
79
91
 
80
92
  let primaryHits = 0;
93
+ let echoRejectedFrames = 0;
81
94
  for (const frame of this.bargeInFrames) {
82
95
  const primarySim = fingerprintSimilarity(frame, this.profile);
83
96
  if (this.assistantProfile !== null) {
84
97
  const echoSim = fingerprintSimilarity(frame, this.assistantProfile);
85
- if (echoSim >= primarySim + this.echoDominanceMargin) continue;
98
+ if (echoSim >= primarySim + this.echoDominanceMargin) {
99
+ echoRejectedFrames += 1;
100
+ continue;
101
+ }
86
102
  }
87
103
  if (primarySim >= this.similarityThreshold) primaryHits += 1;
88
104
  }
89
105
 
90
106
  const required = Math.max(1, Math.ceil(this.bargeInFrames.length * 0.45));
91
- return primaryHits >= required;
107
+ const accepted = primaryHits >= required;
108
+ const decision = { contextId, accepted, frameCount: this.bargeInFrames.length, primaryHits, echoRejectedFrames };
109
+ if (echoRejectedFrames > 0) this.onDecision?.({ signal: "echo_rejected", ...decision });
110
+ this.onDecision?.({ signal: "primary_speaker", ...decision });
111
+ return accepted;
92
112
  }
93
113
 
94
114
  resetBargeInWindow(): void {
@@ -0,0 +1,216 @@
1
+ // SPDX-License-Identifier: MIT
2
+
3
+ import { Route, type PipelineBus } from "./pipeline-bus.js";
4
+ import * as make from "./packet-factories.js";
5
+ import type { Reasoner, ReasonerTurn, ReasoningPart } from "./reasoner.js";
6
+ import { TimerScheduler, type Scheduler } from "./scheduler.js";
7
+
8
+ export interface HedgedReasonerOptions {
9
+ readonly primary: Reasoner;
10
+ readonly backup: Reasoner;
11
+ readonly hedgeAfterMs: number;
12
+ readonly bus?: PipelineBus;
13
+ readonly contextId?: string;
14
+ readonly scheduler?: Scheduler;
15
+ }
16
+
17
+ type Backend = "primary" | "backup";
18
+
19
+ type RacerResult = { who: Backend; result: IteratorResult<ReasoningPart> };
20
+
21
+ const asRacer = (who: Backend, next: Promise<IteratorResult<ReasoningPart>>): Promise<RacerResult> =>
22
+ next
23
+ .then((result) => ({ who, result }))
24
+ .catch((err): RacerResult => ({
25
+ who,
26
+ result: {
27
+ done: false,
28
+ value: {
29
+ type: "error",
30
+ cause: err instanceof Error ? err : new Error(String(err)),
31
+ recoverable: true,
32
+ },
33
+ },
34
+ }));
35
+
36
+ type CommitResult =
37
+ | { readonly ok: true; readonly winner: Backend; readonly first: ReasoningPart; readonly iter: AsyncIterator<ReasoningPart> }
38
+ | { readonly ok: false; readonly error: ReasoningPart };
39
+
40
+ export class HedgedReasoner implements Reasoner {
41
+ private readonly scheduler: Scheduler;
42
+
43
+ constructor(private readonly opts: HedgedReasonerOptions) {
44
+ this.scheduler = opts.scheduler ?? new TimerScheduler();
45
+ }
46
+
47
+ stream(turn: ReasonerTurn): AsyncIterable<ReasoningPart> {
48
+ return this.doStream(turn);
49
+ }
50
+
51
+ private async *doStream(turn: ReasonerTurn): AsyncGenerator<ReasoningPart> {
52
+ const commit = await this.raceToCommit(turn);
53
+ if (!commit.ok) {
54
+ yield commit.error;
55
+ return;
56
+ }
57
+
58
+ yield commit.first;
59
+ let tail = await commit.iter.next();
60
+ while (!tail.done) {
61
+ yield tail.value;
62
+ tail = await commit.iter.next();
63
+ }
64
+ }
65
+
66
+ private async raceToCommit(turn: ReasonerTurn): Promise<CommitResult> {
67
+ const pc = new AbortController();
68
+ const bc = new AbortController();
69
+
70
+ if (turn.signal.aborted) {
71
+ pc.abort();
72
+ bc.abort();
73
+ return { ok: false, error: abortedError() };
74
+ }
75
+
76
+ turn.signal.addEventListener(
77
+ "abort",
78
+ () => {
79
+ pc.abort();
80
+ bc.abort();
81
+ },
82
+ { once: true },
83
+ );
84
+
85
+ let committed = false;
86
+ let primaryExhausted = false;
87
+ let backupExhausted = false;
88
+ let lastError: ReasoningPart | null = null;
89
+
90
+ const primaryIter = this.opts.primary.stream({ ...turn, signal: pc.signal })[Symbol.asyncIterator]();
91
+ let primaryNext: Promise<IteratorResult<ReasoningPart>> = primaryIter.next();
92
+
93
+ let backupIter: AsyncIterator<ReasoningPart> | null = null;
94
+ let backupNext: Promise<IteratorResult<ReasoningPart>> | null = null;
95
+ let repollRace: (() => void) | null = null;
96
+
97
+ const ensureBackup = (): void => {
98
+ if (backupIter) return;
99
+ backupIter = this.opts.backup.stream({ ...turn, signal: bc.signal })[Symbol.asyncIterator]();
100
+ backupNext = backupIter.next();
101
+ this.metric("hedge.fired", "1");
102
+ repollRace?.();
103
+ };
104
+
105
+ this.scheduler.schedule("hedge", this.opts.hedgeAfterMs, () => {
106
+ if (!committed) ensureBackup();
107
+ });
108
+
109
+ while (!committed) {
110
+ const raced = await new Promise<{ who: Backend; result: IteratorResult<ReasoningPart> } | "repoll">(
111
+ (resolve) => {
112
+ const racers: Array<Promise<RacerResult>> = [];
113
+
114
+ if (!primaryExhausted) {
115
+ racers.push(asRacer("primary", primaryNext));
116
+ }
117
+ if (backupNext && !backupExhausted) {
118
+ racers.push(asRacer("backup", backupNext));
119
+ }
120
+
121
+ if (racers.length === 0) {
122
+ repollRace = null;
123
+ resolve("repoll");
124
+ return;
125
+ }
126
+
127
+ repollRace = () => resolve("repoll");
128
+ void Promise.race(racers).then((winner) => {
129
+ repollRace = null;
130
+ resolve(winner);
131
+ });
132
+ },
133
+ );
134
+
135
+ if (raced === "repoll") {
136
+ if (primaryExhausted && (backupExhausted || !backupNext)) {
137
+ return { ok: false, error: lastError ?? abortedError() };
138
+ }
139
+ continue;
140
+ }
141
+
142
+ const { who, result } = raced;
143
+
144
+ if (result.done) {
145
+ if (who === "primary") {
146
+ primaryExhausted = true;
147
+ if (!backupIter) {
148
+ this.scheduler.cancel("hedge");
149
+ ensureBackup();
150
+ } else if (backupExhausted) {
151
+ return { ok: false, error: lastError ?? abortedError() };
152
+ }
153
+ } else {
154
+ backupExhausted = true;
155
+ if (primaryExhausted) {
156
+ return { ok: false, error: lastError ?? abortedError() };
157
+ }
158
+ }
159
+ continue;
160
+ }
161
+
162
+ const part = result.value;
163
+
164
+ if (part.type === "error") {
165
+ lastError = part;
166
+ if (who === "primary") {
167
+ primaryExhausted = true;
168
+ if (!backupIter) {
169
+ this.scheduler.cancel("hedge");
170
+ ensureBackup();
171
+ }
172
+ if (backupExhausted) {
173
+ return { ok: false, error: part };
174
+ }
175
+ } else {
176
+ backupExhausted = true;
177
+ if (primaryExhausted) {
178
+ return { ok: false, error: part };
179
+ }
180
+ }
181
+ continue;
182
+ }
183
+
184
+ committed = true;
185
+ this.scheduler.cancel("hedge");
186
+
187
+ if (who === "primary") {
188
+ bc.abort();
189
+ releaseIterator(backupIter);
190
+ this.metric("hedge.committed_to", "primary");
191
+ return { ok: true, winner: "primary", first: part, iter: primaryIter };
192
+ }
193
+
194
+ pc.abort();
195
+ releaseIterator(primaryIter);
196
+ this.metric("hedge.committed_to", "backup");
197
+ return { ok: true, winner: "backup", first: part, iter: backupIter! };
198
+ }
199
+
200
+ return { ok: false, error: lastError ?? abortedError() };
201
+ }
202
+
203
+ private metric(name: string, value: string): void {
204
+ this.opts.bus?.push(Route.Background, make.metric(this.opts.contextId ?? "", name, value));
205
+ }
206
+ }
207
+
208
+ function releaseIterator(iter: AsyncIterator<ReasoningPart> | null): void {
209
+ if (!iter) return;
210
+ const release = iter.return;
211
+ if (release) void release.call(iter, undefined);
212
+ }
213
+
214
+ function abortedError(): ReasoningPart {
215
+ return { type: "error", cause: new Error("aborted"), recoverable: true };
216
+ }
@@ -0,0 +1,113 @@
1
+ // SPDX-License-Identifier: MIT
2
+
3
+ import { Route, type PipelineBus } from "./pipeline-bus.js";
4
+ import * as make from "./packet-factories.js";
5
+ import type { Reasoner, ReasonerTurn, ReasoningPart } from "./reasoner.js";
6
+
7
+ export interface ReasonerRoute {
8
+ readonly id: string;
9
+ readonly reasoner: Reasoner;
10
+ }
11
+
12
+ export interface RoutingReasonerOptions {
13
+ readonly routes: readonly ReasonerRoute[];
14
+ readonly classify: (turn: ReasonerTurn) => string | Promise<string>;
15
+ readonly speculateRouteId?: string;
16
+ readonly bus?: PipelineBus;
17
+ readonly contextId?: string;
18
+ }
19
+
20
+ export class RoutingReasoner implements Reasoner {
21
+ constructor(private readonly opts: RoutingReasonerOptions) {}
22
+
23
+ stream(turn: ReasonerTurn): AsyncIterable<ReasoningPart> {
24
+ return this.doStream(turn);
25
+ }
26
+
27
+ private async *doStream(turn: ReasonerTurn): AsyncGenerator<ReasoningPart> {
28
+ if (turn.signal.aborted) return;
29
+
30
+ if (this.opts.speculateRouteId !== undefined) {
31
+ yield* this.streamWithSpeculation(turn);
32
+ return;
33
+ }
34
+
35
+ const routeId = await this.opts.classify(turn);
36
+ const route = this.resolveRoute(routeId);
37
+ this.metric("route.selected", routeId);
38
+ const iter = route.reasoner.stream({ ...turn, signal: turn.signal })[Symbol.asyncIterator]();
39
+ yield* this.forwardRoute(iter, undefined, turn.signal);
40
+ }
41
+
42
+ private async *streamWithSpeculation(turn: ReasonerTurn): AsyncGenerator<ReasoningPart> {
43
+ const specId = this.opts.speculateRouteId!;
44
+ const specRoute = this.resolveRoute(specId);
45
+
46
+ const child = new AbortController();
47
+ if (turn.signal.aborted) {
48
+ child.abort();
49
+ return;
50
+ }
51
+ turn.signal.addEventListener("abort", () => child.abort(), { once: true });
52
+
53
+ const specIter = specRoute.reasoner.stream({ ...turn, signal: child.signal })[Symbol.asyncIterator]();
54
+ const specNext = specIter.next();
55
+
56
+ const classifiedId = await this.opts.classify(turn);
57
+
58
+ if (classifiedId === specId) {
59
+ this.metric("route.selected", classifiedId);
60
+ yield* this.forwardRoute(specIter, specNext, turn.signal);
61
+ return;
62
+ }
63
+
64
+ child.abort();
65
+ releaseIterator(specIter);
66
+ void specNext.catch(() => undefined);
67
+ this.metric("route.mispredict", "1");
68
+
69
+ const route = this.resolveRoute(classifiedId);
70
+ this.metric("route.selected", classifiedId);
71
+ const iter = route.reasoner.stream({ ...turn, signal: turn.signal })[Symbol.asyncIterator]();
72
+ yield* this.forwardRoute(iter, undefined, turn.signal);
73
+ }
74
+
75
+ private async *forwardRoute(
76
+ iter: AsyncIterator<ReasoningPart>,
77
+ first: Promise<IteratorResult<ReasoningPart>> | undefined,
78
+ signal: AbortSignal,
79
+ ): AsyncGenerator<ReasoningPart> {
80
+ try {
81
+ let next = first ? await first : await iter.next();
82
+ while (!next.done) {
83
+ if (signal.aborted) return;
84
+ yield next.value;
85
+ next = await iter.next();
86
+ }
87
+ } catch (err) {
88
+ yield {
89
+ type: "error",
90
+ cause: err instanceof Error ? err : new Error(String(err)),
91
+ recoverable: true,
92
+ };
93
+ }
94
+ }
95
+
96
+ private resolveRoute(id: string): ReasonerRoute {
97
+ const route = this.opts.routes.find((r) => r.id === id);
98
+ if (!route) {
99
+ throw new Error(`RoutingReasoner: unknown route id "${id}"`);
100
+ }
101
+ return route;
102
+ }
103
+
104
+ private metric(name: string, value: string): void {
105
+ this.opts.bus?.push(Route.Background, make.metric(this.opts.contextId ?? "", name, value));
106
+ }
107
+ }
108
+
109
+ function releaseIterator(iter: AsyncIterator<ReasoningPart> | null): void {
110
+ if (!iter) return;
111
+ const release = iter.return;
112
+ if (release) void release.call(iter, undefined);
113
+ }