@juspay/neurolink 11.25.4 → 11.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -4
- package/dist/browser/neurolink.min.js +380 -380
- package/dist/core/baseProvider.d.ts +5 -0
- package/dist/core/baseProvider.js +99 -54
- package/dist/factories/providerDescriptors.js +16 -0
- package/dist/models/modelResolver.js +28 -6
- package/dist/neurolink.d.ts +7 -20
- package/dist/neurolink.js +99 -85
- package/dist/routing/classifierRouter.js +18 -4
- package/dist/types/providers.d.ts +18 -0
- package/dist/types/stream.d.ts +44 -6
- package/dist/types/tts.d.ts +9 -0
- package/dist/types/tts.js +6 -0
- package/dist/utils/providerHealth.js +15 -20
- package/dist/utils/retryHandler.d.ts +0 -11
- package/dist/utils/retryHandler.js +6 -21
- package/dist/utils/ttsProcessor.d.ts +20 -1
- package/dist/utils/ttsProcessor.js +167 -0
- package/dist/utils/ttsStream.d.ts +15 -0
- package/dist/utils/ttsStream.js +225 -0
- package/package.json +3 -2
package/dist/neurolink.js
CHANGED
|
@@ -73,6 +73,7 @@ import { getConversationMessages, storeConversationTurn, } from "./utils/convers
|
|
|
73
73
|
// Enhanced error handling imports
|
|
74
74
|
import { CircuitBreaker, ERROR_CODES, ErrorFactory, isAbortError, isRetriableError, logStructuredError, NeuroLinkError, withRetry, withTimeout, } from "./utils/errorHandling.js";
|
|
75
75
|
import { hasLifecycleErrorFired, markLifecycleErrorFired, } from "./utils/lifecycleCallbacks.js";
|
|
76
|
+
import { interleaveTTSStream } from "./utils/ttsStream.js";
|
|
76
77
|
import { resolveLifecycleTimeoutMs } from "./utils/lifecycleTimeout.js";
|
|
77
78
|
import { cloneOptionsForCallIsolation } from "./utils/cloneOptions.js";
|
|
78
79
|
import { coerceJsonToSchema, recoverScalarRoot, schemaAccepts, } from "./utils/json/coerce.js";
|
|
@@ -6376,7 +6377,9 @@ Current user's request: ${currentInput}`;
|
|
|
6376
6377
|
*
|
|
6377
6378
|
* // Consume the stream
|
|
6378
6379
|
* for await (const chunk of result.stream) {
|
|
6379
|
-
*
|
|
6380
|
+
* if ("content" in chunk) {
|
|
6381
|
+
* process.stdout.write(chunk.content);
|
|
6382
|
+
* }
|
|
6380
6383
|
* }
|
|
6381
6384
|
*
|
|
6382
6385
|
* // Advanced streaming with options
|
|
@@ -6684,15 +6687,17 @@ Current user's request: ${currentInput}`;
|
|
|
6684
6687
|
(!!(options.tools && Object.keys(options.tools).length > 0) ||
|
|
6685
6688
|
this.getCustomTools().size > 0), !!(options.input?.images && options.input.images.length > 0), options
|
|
6686
6689
|
.thinking?.thinkingLevel);
|
|
6687
|
-
// TTS
|
|
6688
|
-
//
|
|
6689
|
-
//
|
|
6690
|
+
// Streaming TTS deferred: stream() always synthesizes the streamed AI
|
|
6691
|
+
// response when TTS is enabled, regardless of generate()'s
|
|
6692
|
+
// input-vs-response `useAiResponse` switch. It resolves
|
|
6693
|
+
// `streamResult.audio` with the aggregate TTSResult. The resolver is
|
|
6690
6694
|
// plumbed explicitly through the params bag (M11: previously a
|
|
6691
6695
|
// `_streamTtsResolve` cast on the caller's options object — fragile if
|
|
6692
6696
|
// the same options object was reused across concurrent stream() calls).
|
|
6693
6697
|
const ttsOptions = options.tts;
|
|
6694
|
-
const wantsStreamTtsMode2 =
|
|
6698
|
+
const wantsStreamTtsMode2 = ttsOptions?.enabled === true;
|
|
6695
6699
|
let resolveStreamTtsAudio;
|
|
6700
|
+
let streamTtsMetadata;
|
|
6696
6701
|
const streamTtsAudioPromise = wantsStreamTtsMode2
|
|
6697
6702
|
? new Promise((resolve) => {
|
|
6698
6703
|
resolveStreamTtsAudio = resolve;
|
|
@@ -6707,12 +6712,16 @@ Current user's request: ${currentInput}`;
|
|
|
6707
6712
|
streamId,
|
|
6708
6713
|
originalPrompt,
|
|
6709
6714
|
ttsResolver: resolveStreamTtsAudio,
|
|
6715
|
+
ttsMetadataSink: (metadata) => {
|
|
6716
|
+
streamTtsMetadata = metadata;
|
|
6717
|
+
},
|
|
6710
6718
|
})));
|
|
6711
6719
|
if (streamSttTranscription) {
|
|
6712
6720
|
streamResult.transcription = streamSttTranscription;
|
|
6713
6721
|
}
|
|
6714
6722
|
if (streamTtsAudioPromise) {
|
|
6715
6723
|
streamResult.audio = streamTtsAudioPromise;
|
|
6724
|
+
streamResult.ttsMetadata = streamTtsMetadata;
|
|
6716
6725
|
}
|
|
6717
6726
|
return streamResult;
|
|
6718
6727
|
}
|
|
@@ -7243,7 +7252,7 @@ Current user's request: ${currentInput}`;
|
|
|
7243
7252
|
return result;
|
|
7244
7253
|
}
|
|
7245
7254
|
async runStandardStreamRequest(params) {
|
|
7246
|
-
const { options, streamSpan, spanStartTime, startTime, hrTimeStart, streamId, originalPrompt, ttsResolver, } = params;
|
|
7255
|
+
const { options, streamSpan, spanStartTime, startTime, hrTimeStart, streamId, originalPrompt, ttsResolver, ttsMetadataSink, } = params;
|
|
7247
7256
|
logger.debug("[NeuroLink] Running standard stream request", {
|
|
7248
7257
|
streamId,
|
|
7249
7258
|
provider: options.provider,
|
|
@@ -7265,6 +7274,17 @@ Current user's request: ${currentInput}`;
|
|
|
7265
7274
|
sessionId: enhancedOptions.context?.sessionId,
|
|
7266
7275
|
});
|
|
7267
7276
|
const { stream: mcpStream, provider: providerName, usage: streamUsage, model: streamModel, finishReason: streamFinishReason, toolCalls: streamToolCalls, toolResults: streamToolResults, analytics: streamAnalytics, metadata: providerStreamMetadata, } = await this.createMCPStream(enhancedOptions);
|
|
7277
|
+
let streamedTTSResult;
|
|
7278
|
+
const { stream: incrementalStream, ttsMetadata: streamTtsMetadata } = await this.createIncrementalTTSStream({
|
|
7279
|
+
stream: mcpStream,
|
|
7280
|
+
ttsOptions: enhancedOptions.tts,
|
|
7281
|
+
providerName,
|
|
7282
|
+
fallbackProvider: enhancedOptions.provider,
|
|
7283
|
+
onComplete: (result) => {
|
|
7284
|
+
streamedTTSResult = result;
|
|
7285
|
+
},
|
|
7286
|
+
});
|
|
7287
|
+
ttsMetadataSink?.(streamTtsMetadata);
|
|
7268
7288
|
const streamState = {
|
|
7269
7289
|
finishReason: streamFinishReason ?? "stop",
|
|
7270
7290
|
toolCalls: streamToolCalls,
|
|
@@ -7313,7 +7333,7 @@ Current user's request: ${currentInput}`;
|
|
|
7313
7333
|
// NoOutputGeneratedError, and we want fallback to fire there.
|
|
7314
7334
|
let realOutputChunks = 0;
|
|
7315
7335
|
try {
|
|
7316
|
-
for await (const chunk of
|
|
7336
|
+
for await (const chunk of incrementalStream) {
|
|
7317
7337
|
chunkCount++;
|
|
7318
7338
|
const isNoOutputSentinel = chunk !== null &&
|
|
7319
7339
|
typeof chunk === "object" &&
|
|
@@ -7369,26 +7389,22 @@ Current user's request: ${currentInput}`;
|
|
|
7369
7389
|
!enhancedOptions.disableInternalFallback &&
|
|
7370
7390
|
streamState.toolCalls.length === 0 &&
|
|
7371
7391
|
streamState.toolResults.length === 0) {
|
|
7372
|
-
|
|
7392
|
+
const fallbackStream = self.handleStreamFallback(metadata, streamState, originalPrompt, enhancedOptions, providerName, (content) => {
|
|
7373
7393
|
accumulatedContent += content;
|
|
7374
7394
|
});
|
|
7395
|
+
const { stream: incrementalFallback } = await self.createIncrementalTTSStream({
|
|
7396
|
+
stream: fallbackStream,
|
|
7397
|
+
ttsOptions: enhancedOptions.tts,
|
|
7398
|
+
providerName,
|
|
7399
|
+
fallbackProvider: enhancedOptions.provider,
|
|
7400
|
+
ttsMetadata: streamTtsMetadata,
|
|
7401
|
+
onComplete: (result) => {
|
|
7402
|
+
streamedTTSResult = result;
|
|
7403
|
+
},
|
|
7404
|
+
});
|
|
7405
|
+
yield* incrementalFallback;
|
|
7375
7406
|
}
|
|
7376
|
-
|
|
7377
|
-
// and yield ONE final audio chunk so callers iterating the stream
|
|
7378
|
-
// get the audio inline; also resolve `streamResult.audio` so the
|
|
7379
|
-
// ergonomic `await result.audio` pattern works post-iteration.
|
|
7380
|
-
// m5: synthesis logic lives in a dedicated helper to keep this
|
|
7381
|
-
// generator under the max-lines-per-function lint budget.
|
|
7382
|
-
const ttsModeResult = await self.synthesizeStreamModeTwo({
|
|
7383
|
-
ttsOptions: enhancedOptions.tts,
|
|
7384
|
-
providerName,
|
|
7385
|
-
fallbackProvider: enhancedOptions.provider,
|
|
7386
|
-
accumulatedContent,
|
|
7387
|
-
ttsResolver,
|
|
7388
|
-
});
|
|
7389
|
-
if (ttsModeResult.audioChunk) {
|
|
7390
|
-
yield ttsModeResult.audioChunk;
|
|
7391
|
-
}
|
|
7407
|
+
ttsResolver?.(streamedTTSResult);
|
|
7392
7408
|
resolvedUsage = streamUsage;
|
|
7393
7409
|
if (!resolvedUsage && streamAnalytics) {
|
|
7394
7410
|
try {
|
|
@@ -7625,72 +7641,70 @@ Current user's request: ${currentInput}`;
|
|
|
7625
7641
|
});
|
|
7626
7642
|
}
|
|
7627
7643
|
catch (error) {
|
|
7644
|
+
ttsResolver?.(undefined);
|
|
7628
7645
|
if (options.disableInternalFallback) {
|
|
7629
7646
|
throw error;
|
|
7630
7647
|
}
|
|
7631
7648
|
return this.handleStreamError(error, options, startTime, streamId, undefined, undefined);
|
|
7632
7649
|
}
|
|
7633
7650
|
}
|
|
7634
|
-
/**
|
|
7635
|
-
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7644
|
-
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
const
|
|
7654
|
-
|
|
7655
|
-
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
}
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
return {
|
|
7675
|
-
audioChunk: {
|
|
7676
|
-
type: "tts_audio",
|
|
7677
|
-
audio: {
|
|
7678
|
-
data: ttsResult.buffer,
|
|
7679
|
-
format: ttsResult.format,
|
|
7680
|
-
index: 0,
|
|
7681
|
-
isFinal: true,
|
|
7682
|
-
cumulativeSize: ttsResult.size,
|
|
7683
|
-
voice: ttsResult.voice,
|
|
7684
|
-
sampleRate: ttsResult.sampleRate,
|
|
7685
|
-
},
|
|
7686
|
-
},
|
|
7687
|
-
};
|
|
7651
|
+
/** Wrap one provider stream with incremental TTS synthesis. */
|
|
7652
|
+
async createIncrementalTTSStream(params) {
|
|
7653
|
+
const { stream, ttsOptions, providerName, fallbackProvider, onComplete } = params;
|
|
7654
|
+
if (!ttsOptions?.enabled) {
|
|
7655
|
+
onComplete(undefined);
|
|
7656
|
+
return { stream };
|
|
7657
|
+
}
|
|
7658
|
+
const { TTSProcessor } = await import("./utils/ttsProcessor.js");
|
|
7659
|
+
const concreteFallback = fallbackProvider === "auto" ? undefined : fallbackProvider;
|
|
7660
|
+
const candidate = ttsOptions.provider ?? concreteFallback ?? providerName;
|
|
7661
|
+
const ttsProvider = candidate && TTSProcessor.supports(candidate) ? candidate : undefined;
|
|
7662
|
+
const ttsMetadata = params.ttsMetadata ?? {
|
|
7663
|
+
attempted: ttsProvider !== undefined,
|
|
7664
|
+
success: false,
|
|
7665
|
+
};
|
|
7666
|
+
ttsMetadata.attempted = ttsProvider !== undefined;
|
|
7667
|
+
ttsMetadata.success = false;
|
|
7668
|
+
delete ttsMetadata.error;
|
|
7669
|
+
delete ttsMetadata.latency;
|
|
7670
|
+
const ttsStartedAt = Date.now();
|
|
7671
|
+
let completionRecorded = false;
|
|
7672
|
+
const recordCompletion = (result, error) => {
|
|
7673
|
+
if (completionRecorded) {
|
|
7674
|
+
return;
|
|
7675
|
+
}
|
|
7676
|
+
completionRecorded = true;
|
|
7677
|
+
ttsMetadata.success = error === undefined && result !== undefined;
|
|
7678
|
+
if (error) {
|
|
7679
|
+
ttsMetadata.error = error;
|
|
7680
|
+
}
|
|
7681
|
+
else {
|
|
7682
|
+
delete ttsMetadata.error;
|
|
7683
|
+
}
|
|
7684
|
+
ttsMetadata.latency = Date.now() - ttsStartedAt;
|
|
7685
|
+
onComplete(result);
|
|
7686
|
+
};
|
|
7687
|
+
if (!ttsProvider) {
|
|
7688
|
+
logger.warn(`[NeuroLink.stream] No TTS provider resolved for incremental streaming (set tts.provider explicitly — chat provider "${candidate ?? "<unset>"}" is not a registered TTS handler)`);
|
|
7689
|
+
recordCompletion(undefined);
|
|
7690
|
+
return { stream, ttsMetadata };
|
|
7688
7691
|
}
|
|
7689
|
-
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
|
|
7692
|
+
return {
|
|
7693
|
+
stream: interleaveTTSStream({
|
|
7694
|
+
stream,
|
|
7695
|
+
provider: ttsProvider,
|
|
7696
|
+
options: ttsOptions,
|
|
7697
|
+
onComplete: recordCompletion,
|
|
7698
|
+
}),
|
|
7699
|
+
ttsMetadata,
|
|
7700
|
+
};
|
|
7701
|
+
}
|
|
7702
|
+
/** Prevent provider fallback streams from duplicating outer streaming TTS. */
|
|
7703
|
+
deferProviderStreamTTS(options) {
|
|
7704
|
+
if (options.tts?.enabled) {
|
|
7705
|
+
return { ...options, tts: undefined };
|
|
7693
7706
|
}
|
|
7707
|
+
return options;
|
|
7694
7708
|
}
|
|
7695
7709
|
/**
|
|
7696
7710
|
* Prepare stream options: initialize memory, MCP, retrieval, orchestration,
|
|
@@ -7966,7 +7980,7 @@ Current user's request: ${currentInput}`;
|
|
|
7966
7980
|
context: enhancedOptions.context,
|
|
7967
7981
|
});
|
|
7968
7982
|
const fallbackResult = await fallbackProvider.stream({
|
|
7969
|
-
...enhancedOptions,
|
|
7983
|
+
...this.deferProviderStreamTTS(enhancedOptions),
|
|
7970
7984
|
model: fallbackRoute.model,
|
|
7971
7985
|
conversationMessages,
|
|
7972
7986
|
});
|
|
@@ -8395,7 +8409,7 @@ Current user's request: ${currentInput}`;
|
|
|
8395
8409
|
}),
|
|
8396
8410
|
}, "NeuroLink.createMCPStream");
|
|
8397
8411
|
const poolStreamResult = await poolStreamProvider.stream({
|
|
8398
|
-
...options,
|
|
8412
|
+
...this.deferProviderStreamTTS(options),
|
|
8399
8413
|
provider: poolStreamProviderName,
|
|
8400
8414
|
model: poolStreamModel,
|
|
8401
8415
|
region: poolStreamRegion,
|
|
@@ -8467,7 +8481,7 @@ Current user's request: ${currentInput}`;
|
|
|
8467
8481
|
// 🔧 FIX: Pass enhanced system prompt to real streaming
|
|
8468
8482
|
// Tools will be accessed through the streamText call in executeStream
|
|
8469
8483
|
const streamResult = await provider.stream({
|
|
8470
|
-
...options,
|
|
8484
|
+
...this.deferProviderStreamTTS(options),
|
|
8471
8485
|
systemPrompt: enhancedSystemPrompt, // Use enhanced prompt with tool descriptions
|
|
8472
8486
|
conversationMessages,
|
|
8473
8487
|
});
|
|
@@ -160,7 +160,13 @@ export class ClassifierRouter {
|
|
|
160
160
|
const mode = DIFFICULTY_RANK_MODE[difficulty];
|
|
161
161
|
const originalIndex = new Map(members.map((m, i) => [m, i]));
|
|
162
162
|
const num = (v) => (typeof v === "number" ? v : NEUTRAL);
|
|
163
|
-
|
|
163
|
+
const isFullyUnmeasured = (m) => {
|
|
164
|
+
const meta = this.metaFor(m);
|
|
165
|
+
return meta.cost === undefined && meta.quality === undefined;
|
|
166
|
+
};
|
|
167
|
+
const measured = members.filter((m) => !isFullyUnmeasured(m));
|
|
168
|
+
const unmeasured = members.filter(isFullyUnmeasured);
|
|
169
|
+
const sortMeasured = (pool) => [...pool].sort((a, b) => {
|
|
164
170
|
const ma = this.metaFor(a);
|
|
165
171
|
const mb = this.metaFor(b);
|
|
166
172
|
let delta;
|
|
@@ -185,6 +191,11 @@ export class ClassifierRouter {
|
|
|
185
191
|
// Stable: preserve declared pool order on a tie.
|
|
186
192
|
return (originalIndex.get(a) ?? 0) - (originalIndex.get(b) ?? 0);
|
|
187
193
|
});
|
|
194
|
+
// The unmeasured bucket ranks after every measured member, but WITHIN the
|
|
195
|
+
// bucket the same comparator still applies: with cost/quality both
|
|
196
|
+
// NEUTRAL it falls through to the weight tie-break, preserving the
|
|
197
|
+
// documented weight contract that a plain append would silently drop.
|
|
198
|
+
return [...sortMeasured(measured), ...sortMeasured(unmeasured)];
|
|
188
199
|
}
|
|
189
200
|
/** Tool narrowing: per-difficulty directive, then classifier hints. */
|
|
190
201
|
selectTools(decision) {
|
|
@@ -223,7 +234,10 @@ export class ClassifierRouter {
|
|
|
223
234
|
if (needsEnrichment && member.model) {
|
|
224
235
|
try {
|
|
225
236
|
const info = ModelResolver.resolveModel(member.model);
|
|
226
|
-
if (info) {
|
|
237
|
+
if (!info) {
|
|
238
|
+
this.deps.logger?.debug?.(`[ClassifierRouter] metaFor: no registry match for ${member.provider}/${member.model}`);
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
227
241
|
if (cost === undefined) {
|
|
228
242
|
cost = info.pricing.inputCostPer1K + info.pricing.outputCostPer1K;
|
|
229
243
|
}
|
|
@@ -251,8 +265,8 @@ export class ClassifierRouter {
|
|
|
251
265
|
}
|
|
252
266
|
}
|
|
253
267
|
}
|
|
254
|
-
catch {
|
|
255
|
-
|
|
268
|
+
catch (err) {
|
|
269
|
+
this.deps.logger?.warn?.(`[ClassifierRouter] metaFor: registry lookup threw for ${member.provider}/${member.model}`, { error: err instanceof Error ? err.message : String(err) });
|
|
256
270
|
}
|
|
257
271
|
}
|
|
258
272
|
const meta = { cost, quality, capabilities };
|
|
@@ -1885,6 +1885,24 @@ export type ProviderDescriptor = {
|
|
|
1885
1885
|
localRuntime: boolean;
|
|
1886
1886
|
/** How ProviderHealthChecker should verify this provider is reachable. */
|
|
1887
1887
|
healthCheck: "env-only" | "models-probe" | "live-generate";
|
|
1888
|
+
/**
|
|
1889
|
+
* Membership + order in the default health sweep
|
|
1890
|
+
* (`ProviderHealthChecker.checkAllProvidersHealth` with no explicit
|
|
1891
|
+
* list). Lower number = checked and reported first; the sweep's array
|
|
1892
|
+
* order is behaviour for its first-healthy fallback consumers. Absent =
|
|
1893
|
+
* not part of the default sweep. Replaces the hand-maintained 8-provider
|
|
1894
|
+
* array that lived in providerHealth.ts.
|
|
1895
|
+
*/
|
|
1896
|
+
defaultHealthSweepPriority?: number;
|
|
1897
|
+
/**
|
|
1898
|
+
* Preference rank for `getBestHealthyProvider`'s default auto-selection
|
|
1899
|
+
* (lower = tried first). Deliberately a SEPARATE ordering from the sweep:
|
|
1900
|
+
* auto-select prefers local/cheap runtimes (litellm, ollama) before cloud
|
|
1901
|
+
* providers, while the sweep reports the majors first. Absent = not in
|
|
1902
|
+
* the default preference list. Replaces the second hand-maintained array
|
|
1903
|
+
* that lived inline as getBestHealthyProvider's default parameter.
|
|
1904
|
+
*/
|
|
1905
|
+
autoSelectPreference?: number;
|
|
1888
1906
|
setupUrl?: string;
|
|
1889
1907
|
timeouts?: {
|
|
1890
1908
|
generateMs?: number;
|
package/dist/types/stream.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { JsonValue, UnknownRecord } from "./common.js";
|
|
|
10
10
|
import type { Content, ImageWithAltText } from "./content.js";
|
|
11
11
|
import type { ChatMessage } from "./conversation.js";
|
|
12
12
|
import type { StreamNoOutputSentinel } from "./noOutputSentinel.js";
|
|
13
|
-
import type { AdditionalMemoryUser, GenerateStopReason, ToolExecutionCaptureOptions } from "./generate.js";
|
|
13
|
+
import type { AdditionalMemoryUser, GenerateStopReason, TTSMetadata, ToolExecutionCaptureOptions } from "./generate.js";
|
|
14
14
|
import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js";
|
|
15
15
|
import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
|
|
16
16
|
import type { STTOptions, STTResult } from "./stt.js";
|
|
@@ -169,6 +169,21 @@ export type StreamChunk = {
|
|
|
169
169
|
/** TTS audio chunk data */
|
|
170
170
|
audio: TTSChunk;
|
|
171
171
|
};
|
|
172
|
+
/** Provider-level chunks accepted by NeuroLink's core streaming pipeline. */
|
|
173
|
+
export type ProviderStreamChunk = {
|
|
174
|
+
content: string;
|
|
175
|
+
} | {
|
|
176
|
+
type: "audio";
|
|
177
|
+
audio: AudioChunk;
|
|
178
|
+
} | {
|
|
179
|
+
type: "tts_audio";
|
|
180
|
+
audio: TTSChunk;
|
|
181
|
+
} | {
|
|
182
|
+
type: "image";
|
|
183
|
+
imageOutput: {
|
|
184
|
+
base64: string;
|
|
185
|
+
};
|
|
186
|
+
};
|
|
172
187
|
export type StreamOptions = {
|
|
173
188
|
/**
|
|
174
189
|
* Opt this stream call into the knowledge grounding configured on the
|
|
@@ -699,13 +714,36 @@ export type StreamResult = {
|
|
|
699
714
|
/** STT transcription result (when stt option is used) */
|
|
700
715
|
transcription?: STTResult;
|
|
701
716
|
/**
|
|
702
|
-
* TTS
|
|
703
|
-
*
|
|
704
|
-
*
|
|
705
|
-
*
|
|
706
|
-
*
|
|
717
|
+
* Streaming TTS result (when `tts.enabled`). `stream()` synthesizes the AI
|
|
718
|
+
* response incrementally; `useAiResponse` continues to select input vs
|
|
719
|
+
* response synthesis for non-streaming generation.
|
|
720
|
+
* Resolves with the synthesized audio after the caller drains `stream` to
|
|
721
|
+
* completion; like other stream-final fields, it remains pending while the
|
|
722
|
+
* lazy stream is unconsumed. It resolves with the aggregate of whatever
|
|
723
|
+
* segments were synthesized: a synthesis failure part-way through still
|
|
724
|
+
* resolves with the earlier segments rather than discarding them. It resolves
|
|
725
|
+
* to undefined only when no segment was produced — TTS was not enabled, no
|
|
726
|
+
* handler resolved for the requested provider, the model stream errored, every
|
|
727
|
+
* synthesis failed, or the caller stopped draining `stream` before it ended
|
|
728
|
+
* (an abandoned stream settles undefined rather than a partial aggregate).
|
|
729
|
+
* Audio is also yielded incrementally as ordered
|
|
730
|
+
* `tts_audio` chunks. Each chunk, including the final one, contains only its
|
|
731
|
+
* own buffered segment. The aggregate is a byte concatenation of those
|
|
732
|
+
* independently synthesized segments, so what it is depends on the format's
|
|
733
|
+
* framing: for frame- or sample-stream formats (`mp3`, `mpeg`, `mpga`,
|
|
734
|
+
* `pcm16`) it is one playable stream; for header-bearing container formats
|
|
735
|
+
* (`wav`, `flac`, `m4a`, `mp4`, `webm`) it is not a valid file, because each
|
|
736
|
+
* segment carries its own header; for `ogg`/`opus` it is a chained stream that
|
|
737
|
+
* some decoders read only through its first segment. Use the individual chunk
|
|
738
|
+
* buffers when each segment must be a valid container file.
|
|
707
739
|
*/
|
|
708
740
|
audio?: Promise<TTSResult | undefined>;
|
|
741
|
+
/**
|
|
742
|
+
* Outcome metadata for streaming TTS synthesis. This is a mutable reference
|
|
743
|
+
* whose success and latency fields are finalized asynchronously; read it
|
|
744
|
+
* after draining `stream`.
|
|
745
|
+
*/
|
|
746
|
+
ttsMetadata?: TTSMetadata;
|
|
709
747
|
};
|
|
710
748
|
/**
|
|
711
749
|
* Enhanced provider type with stream method
|
package/dist/types/tts.d.ts
CHANGED
|
@@ -34,6 +34,9 @@ export type TTSOptions = {
|
|
|
34
34
|
/**
|
|
35
35
|
* Use the AI-generated response for TTS instead of the input text
|
|
36
36
|
*
|
|
37
|
+
* This switch applies to non-streaming generation. `stream()` always
|
|
38
|
+
* synthesizes the streamed AI response incrementally when TTS is enabled.
|
|
39
|
+
*
|
|
37
40
|
* When false or undefined (default): TTS will synthesize the input text/prompt directly without calling AI generation
|
|
38
41
|
* When true: TTS will synthesize the AI-generated response after generation completes
|
|
39
42
|
*
|
|
@@ -78,6 +81,12 @@ export type TTSOptions = {
|
|
|
78
81
|
play?: boolean;
|
|
79
82
|
/** Override TTS provider (e.g., "elevenlabs", "openai-tts", "azure-tts") */
|
|
80
83
|
provider?: TTSProviderName;
|
|
84
|
+
/**
|
|
85
|
+
* Minimum buffered text length before incremental stream synthesis flushes
|
|
86
|
+
* at a sentence boundary. The provider's maximum text length remains a hard
|
|
87
|
+
* upper bound. Defaults to 120 characters.
|
|
88
|
+
*/
|
|
89
|
+
streamingBufferSize?: number;
|
|
81
90
|
};
|
|
82
91
|
/**
|
|
83
92
|
* TTS audio result returned from generation
|
package/dist/types/tts.js
CHANGED
|
@@ -60,5 +60,11 @@ export function isValidTTSOptions(options) {
|
|
|
60
60
|
return false;
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
|
+
if (opts.streamingBufferSize !== undefined) {
|
|
64
|
+
if (!Number.isInteger(opts.streamingBufferSize) ||
|
|
65
|
+
opts.streamingBufferSize < 1) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
63
69
|
return true;
|
|
64
70
|
}
|
|
@@ -9,6 +9,7 @@ import { basename } from "path";
|
|
|
9
9
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
10
10
|
import { DEFAULT_OLLAMA_MODEL } from "../providers/ollama/constants.js";
|
|
11
11
|
import { ProviderFactory } from "../factories/providerFactory.js";
|
|
12
|
+
import { PROVIDER_DESCRIPTORS } from "../factories/providerDescriptors.js";
|
|
12
13
|
export class ProviderHealthChecker {
|
|
13
14
|
static healthCache = new Map();
|
|
14
15
|
static DEFAULT_TIMEOUT = 5000; // 5 seconds
|
|
@@ -1382,16 +1383,13 @@ export class ProviderHealthChecker {
|
|
|
1382
1383
|
* Prioritizes healthy providers over configured but unhealthy ones
|
|
1383
1384
|
* Uses fast, cached health checks to avoid blocking initialization
|
|
1384
1385
|
*/
|
|
1385
|
-
static async getBestHealthyProvider(
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
"bedrock",
|
|
1393
|
-
"azure",
|
|
1394
|
-
]) {
|
|
1386
|
+
static async getBestHealthyProvider(
|
|
1387
|
+
// Auto-select preference comes from the descriptors too — a SEPARATE
|
|
1388
|
+
// ordering from the sweep (local/cheap runtimes first), carried by
|
|
1389
|
+
// autoSelectPreference rather than defaultHealthSweepPriority.
|
|
1390
|
+
preferredProviders = PROVIDER_DESCRIPTORS.filter((d) => d.autoSelectPreference !== undefined)
|
|
1391
|
+
.sort((a, b) => (a.autoSelectPreference ?? 0) - (b.autoSelectPreference ?? 0))
|
|
1392
|
+
.map((d) => d.name)) {
|
|
1395
1393
|
const healthStatuses = await this.checkAllProvidersHealth({
|
|
1396
1394
|
includeConnectivityTest: false, // Quick config check only
|
|
1397
1395
|
cacheResults: true,
|
|
@@ -1424,16 +1422,13 @@ export class ProviderHealthChecker {
|
|
|
1424
1422
|
* Get health status for all registered providers
|
|
1425
1423
|
*/
|
|
1426
1424
|
static async checkAllProvidersHealth(options = {}) {
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
AIProviderName.LITELLM,
|
|
1435
|
-
AIProviderName.OLLAMA,
|
|
1436
|
-
];
|
|
1425
|
+
// Sweep membership and ORDER come from the descriptors. Order is
|
|
1426
|
+
// behaviour: auto-select takes the first healthy provider, so the
|
|
1427
|
+
// priority field, not the descriptor array's layout, decides preference.
|
|
1428
|
+
const providers = PROVIDER_DESCRIPTORS.filter((d) => d.defaultHealthSweepPriority !== undefined)
|
|
1429
|
+
.sort((a, b) => (a.defaultHealthSweepPriority ?? 0) -
|
|
1430
|
+
(b.defaultHealthSweepPriority ?? 0))
|
|
1431
|
+
.map((d) => d.name);
|
|
1437
1432
|
const healthChecks = providers.map((provider) => this.checkProviderHealth(provider, options));
|
|
1438
1433
|
const results = await Promise.allSettled(healthChecks);
|
|
1439
1434
|
return results.map((result, index) => {
|
|
@@ -13,17 +13,6 @@ import type { RetryOptions } from "../types/index.js";
|
|
|
13
13
|
* @returns Calculated delay in milliseconds
|
|
14
14
|
*/
|
|
15
15
|
export declare function calculateBackoffDelay(attempt: number, initialDelay?: number, multiplier?: number, maxDelay?: number, addJitter?: boolean): number;
|
|
16
|
-
/**
|
|
17
|
-
* Error types that are typically retryable
|
|
18
|
-
*/
|
|
19
|
-
export declare class NetworkError extends Error {
|
|
20
|
-
readonly cause?: Error | undefined;
|
|
21
|
-
constructor(message: string, cause?: Error | undefined);
|
|
22
|
-
}
|
|
23
|
-
export declare class TemporaryError extends Error {
|
|
24
|
-
readonly cause?: Error | undefined;
|
|
25
|
-
constructor(message: string, cause?: Error | undefined);
|
|
26
|
-
}
|
|
27
16
|
/**
|
|
28
17
|
* Default retry configuration
|
|
29
18
|
*/
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { logger } from "./logger.js";
|
|
6
6
|
import { SYSTEM_LIMITS } from "../core/constants.js";
|
|
7
|
+
import { NetworkError } from "../types/index.js";
|
|
7
8
|
/**
|
|
8
9
|
* Calculate exponential backoff delay with jitter
|
|
9
10
|
* @param attempt - Current attempt number (1-based)
|
|
@@ -24,25 +25,6 @@ export function calculateBackoffDelay(attempt, initialDelay = SYSTEM_LIMITS.DEFA
|
|
|
24
25
|
: 0;
|
|
25
26
|
return cappedDelay + jitter;
|
|
26
27
|
}
|
|
27
|
-
/**
|
|
28
|
-
* Error types that are typically retryable
|
|
29
|
-
*/
|
|
30
|
-
export class NetworkError extends Error {
|
|
31
|
-
cause;
|
|
32
|
-
constructor(message, cause) {
|
|
33
|
-
super(message);
|
|
34
|
-
this.cause = cause;
|
|
35
|
-
this.name = "NetworkError";
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
export class TemporaryError extends Error {
|
|
39
|
-
cause;
|
|
40
|
-
constructor(message, cause) {
|
|
41
|
-
super(message);
|
|
42
|
-
this.cause = cause;
|
|
43
|
-
this.name = "TemporaryError";
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
28
|
/**
|
|
47
29
|
* Default retry configuration
|
|
48
30
|
*/
|
|
@@ -52,8 +34,11 @@ export const DEFAULT_RETRY_CONFIG = {
|
|
|
52
34
|
maxDelay: SYSTEM_LIMITS.DEFAULT_MAX_DELAY,
|
|
53
35
|
backoffMultiplier: SYSTEM_LIMITS.DEFAULT_BACKOFF_MULTIPLIER,
|
|
54
36
|
retryCondition: (error) => {
|
|
55
|
-
// Retry on network errors, timeouts, and specific HTTP errors
|
|
56
|
-
|
|
37
|
+
// Retry on network errors, timeouts, and specific HTTP errors. The
|
|
38
|
+
// instanceof now matches the canonical types/errors.js NetworkError —
|
|
39
|
+
// the local shadow class this file used to declare matched nothing the
|
|
40
|
+
// rest of the SDK ever threw.
|
|
41
|
+
if (error instanceof NetworkError) {
|
|
57
42
|
return true;
|
|
58
43
|
}
|
|
59
44
|
// Retry on timeout errors
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module utils/ttsProcessor
|
|
8
8
|
*/
|
|
9
|
-
import type { TTSOptions, TTSResult, TTSHandler } from "../types/index.js";
|
|
9
|
+
import type { TTSChunk, TTSOptions, TTSResult, TTSHandler } from "../types/index.js";
|
|
10
10
|
import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
|
|
11
11
|
import { NeuroLinkError } from "./errorHandling.js";
|
|
12
12
|
/**
|
|
@@ -20,6 +20,12 @@ export declare const TTS_ERROR_CODES: {
|
|
|
20
20
|
readonly SYNTHESIS_FAILED: "TTS_SYNTHESIS_FAILED";
|
|
21
21
|
readonly INVALID_INPUT: "TTS_INVALID_INPUT";
|
|
22
22
|
};
|
|
23
|
+
/** Internal signal raised after all buffered segments have been attempted. */
|
|
24
|
+
export declare class IncrementalTTSSynthesisError extends Error {
|
|
25
|
+
readonly firstError: unknown;
|
|
26
|
+
readonly failedSegments: readonly number[];
|
|
27
|
+
constructor(firstError: unknown, failedSegments: number[]);
|
|
28
|
+
}
|
|
23
29
|
/**
|
|
24
30
|
* TTS Error class for text-to-speech specific errors
|
|
25
31
|
*/
|
|
@@ -156,4 +162,17 @@ export declare class TTSProcessor {
|
|
|
156
162
|
* ```
|
|
157
163
|
*/
|
|
158
164
|
static synthesize(text: string, provider: string, options: TTSOptions): Promise<TTSResult>;
|
|
165
|
+
/**
|
|
166
|
+
* Incrementally synthesize sentence-buffered text chunks.
|
|
167
|
+
*
|
|
168
|
+
* Text is flushed at a sentence boundary after `streamingBufferSize`
|
|
169
|
+
* characters, or hard-split before the provider's maximum text length.
|
|
170
|
+
* Each segment goes through `synthesize()`, preserving the existing handler
|
|
171
|
+
* registry, validation, error normalization, and telemetry seam.
|
|
172
|
+
*
|
|
173
|
+
* The most recent successful audio chunk is held until another succeeds or
|
|
174
|
+
* the input ends, so exactly one real audio chunk carries `isFinal: true`
|
|
175
|
+
* without emitting a separate empty terminator chunk.
|
|
176
|
+
*/
|
|
177
|
+
static synthesizeStream(textChunks: AsyncIterable<string>, provider: string, options: TTSOptions, shouldStop?: () => boolean): AsyncGenerator<TTSChunk>;
|
|
159
178
|
}
|