@juspay/neurolink 12.47.13 → 12.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
- package/dist/browser/neurolink.min.js +578 -565
- package/dist/cli/factories/commandFactory.d.ts +7 -0
- package/dist/cli/factories/commandFactory.js +26 -0
- package/dist/cli/loop/optionsSchema.js +9 -0
- package/dist/context/budgetChecker.js +54 -2
- package/dist/context/contextDecision.js +2 -0
- package/dist/context/toolPairRepair.d.ts +12 -0
- package/dist/context/toolPairRepair.js +29 -3
- package/dist/core/baseProvider.d.ts +92 -3
- package/dist/core/baseProvider.js +254 -10
- package/dist/core/conversationMemoryManager.js +17 -2
- package/dist/core/loopEngine.js +1 -1
- package/dist/core/modules/MessageBuilder.js +11 -3
- package/dist/core/modules/TelemetryHandler.d.ts +2 -2
- package/dist/core/modules/ToolsManager.d.ts +35 -0
- package/dist/core/modules/ToolsManager.js +159 -55
- package/dist/core/nativeGenerateLoop.d.ts +58 -1
- package/dist/core/nativeGenerateLoop.js +277 -3
- package/dist/core/redisConversationMemoryManager.js +10 -3
- package/dist/core/resolveRequestKind.d.ts +24 -4
- package/dist/core/resolveRequestKind.js +31 -4
- package/dist/core/toolExecutionRecorder.d.ts +6 -0
- package/dist/core/toolExecutionRecorder.js +24 -2
- package/dist/core/toolRoutingDecision.js +1 -0
- package/dist/factories/mediaHandlerCatalog.js +5 -0
- package/dist/factories/providerDescriptors.js +15 -0
- package/dist/files/fileTools.d.ts +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.js +9 -2
- package/dist/memory/condensationPrompt.d.ts +8 -0
- package/dist/memory/condensationPrompt.js +29 -0
- package/dist/memory/hippocampusInitializer.d.ts +14 -2
- package/dist/memory/hippocampusInitializer.js +64 -3
- package/dist/neurolink.d.ts +252 -3
- package/dist/neurolink.js +898 -43
- package/dist/providers/amazonBedrock/client.js +6 -0
- package/dist/providers/amazonSagemaker.js +14 -1
- package/dist/providers/anthropic/client.js +148 -19
- package/dist/providers/anthropic/loopAdapter.js +2 -2
- package/dist/providers/googleAiStudio/client.js +29 -14
- package/dist/providers/openaiChatCompletionsBase.js +103 -23
- package/dist/providers/systemOneDecision.d.ts +6 -0
- package/dist/providers/systemOneDecision.js +14 -63
- package/dist/providers/typesafe.d.ts +6 -0
- package/dist/providers/typesafe.js +6 -0
- package/dist/rag/retrieval/searchDecision.js +1 -0
- package/dist/routing/classifierRouter.js +6 -5
- package/dist/routing/classifierStrategies.js +8 -1
- package/dist/types/classifierRouter.d.ts +16 -3
- package/dist/types/common.d.ts +2 -0
- package/dist/types/config.d.ts +9 -0
- package/dist/types/context.d.ts +8 -1
- package/dist/types/conversation.d.ts +87 -0
- package/dist/types/decision.d.ts +210 -1
- package/dist/types/decision.js +9 -1
- package/dist/types/dispatch.d.ts +11 -4
- package/dist/types/generate.d.ts +118 -54
- package/dist/types/loopEngine.d.ts +7 -2
- package/dist/types/memory.d.ts +177 -9
- package/dist/types/openaiCompatible.d.ts +11 -3
- package/dist/types/stream.d.ts +13 -16
- package/dist/types/stt.d.ts +58 -1
- package/dist/types/tools.d.ts +14 -0
- package/dist/types/tools.js +8 -0
- package/dist/types/tts.d.ts +108 -0
- package/dist/types/voice.d.ts +41 -4
- package/dist/utils/decisionAnswers.d.ts +31 -1
- package/dist/utils/decisionAnswers.js +114 -0
- package/dist/utils/decisionLimits.d.ts +39 -0
- package/dist/utils/decisionLimits.js +116 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.js +9 -0
- package/dist/utils/messageBuilder.d.ts +45 -1
- package/dist/utils/messageBuilder.js +393 -16
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/utils/speechText.d.ts +41 -0
- package/dist/utils/speechText.js +364 -0
- package/dist/utils/toolResultStatus.d.ts +26 -0
- package/dist/utils/toolResultStatus.js +73 -0
- package/dist/utils/ttsProcessor.d.ts +29 -0
- package/dist/utils/ttsProcessor.js +84 -2
- package/dist/voice/index.d.ts +1 -0
- package/dist/voice/index.js +3 -0
- package/dist/voice/providers/ElevenLabsSTT.d.ts +42 -0
- package/dist/voice/providers/ElevenLabsSTT.js +351 -0
- package/dist/voice/providers/ElevenLabsTTS.d.ts +81 -1
- package/dist/voice/providers/ElevenLabsTTS.js +415 -74
- package/docs-site/static/search-index.json +48 -33
- package/package.json +5 -2
- package/dist/utils/toolEndEmitter.d.ts +0 -25
- package/dist/utils/toolEndEmitter.js +0 -64
|
@@ -5,7 +5,7 @@ import { defaultProviderFor } from "../factories/mediaHandlerCatalog.js";
|
|
|
5
5
|
import { DECISION_PROVIDERS, PROVIDER_DESCRIPTORS_BY_NAME, } from "../factories/providerDescriptors.js";
|
|
6
6
|
import { MiddlewareFactory } from "../middleware/factory.js";
|
|
7
7
|
import { modelSupports } from "../models/modelRegistry.js";
|
|
8
|
-
import { resolveRequestKind } from "./resolveRequestKind.js";
|
|
8
|
+
import { resolveRequestKind, resolveTTSMode } from "./resolveRequestKind.js";
|
|
9
9
|
import { ATTR, tracers } from "../telemetry/index.js";
|
|
10
10
|
import { ERROR_CODES, isAbortError, NeuroLinkError, } from "../utils/errorHandling.js";
|
|
11
11
|
import { InvalidModelError, ProviderError } from "../types/index.js";
|
|
@@ -34,7 +34,8 @@ import { composeAbortSignals, composeAbortSignalsScoped, createTimeoutController
|
|
|
34
34
|
import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
|
|
35
35
|
import { getKeyCount, getKeysAsString, transformToolExecutions, } from "../utils/transformationUtils.js";
|
|
36
36
|
import { ToolExecutionRecorder } from "./toolExecutionRecorder.js";
|
|
37
|
-
import { TTS_ERROR_CODES, TTSProcessor } from "../utils/ttsProcessor.js";
|
|
37
|
+
import { resolveTTSHandlerBudgetMs, TTS_ERROR_CODES, TTSProcessor, } from "../utils/ttsProcessor.js";
|
|
38
|
+
import { prepareTextForSpeech, resolveSpeechSanitizeOptions, } from "../utils/speechText.js";
|
|
38
39
|
import { executeVideoAnalysis, hasVideoFrames, } from "../utils/videoAnalysisProcessor.js";
|
|
39
40
|
import { dedupeTools } from "./toolDedup.js";
|
|
40
41
|
import { resolveToolPolicy, toolNameMatcher } from "../tools/toolPolicy.js";
|
|
@@ -1074,6 +1075,7 @@ export class BaseProvider {
|
|
|
1074
1075
|
? {
|
|
1075
1076
|
attempted: TTSProcessor.supports(ttsProvider),
|
|
1076
1077
|
success: false,
|
|
1078
|
+
mode: "response",
|
|
1077
1079
|
}
|
|
1078
1080
|
: result?.ttsMetadata;
|
|
1079
1081
|
const onTTSComplete = incrementalTTS
|
|
@@ -1348,10 +1350,13 @@ export class BaseProvider {
|
|
|
1348
1350
|
async prepareGenerationContext(options) {
|
|
1349
1351
|
const shouldUseTools = this.shouldUseTools(options, true);
|
|
1350
1352
|
const baseTools = shouldUseTools ? await this.getAllTools() : {};
|
|
1353
|
+
// Registered and MCP tools arrive from ToolsManager already emitting
|
|
1354
|
+
// tool:start / tool:end; per-call tools get the same wrapper here so the
|
|
1355
|
+
// event bus sees every execution exactly once.
|
|
1351
1356
|
let tools = shouldUseTools
|
|
1352
1357
|
? {
|
|
1353
1358
|
...baseTools,
|
|
1354
|
-
...(options.tools || {}),
|
|
1359
|
+
...this.toolsManager.wrapExternalToolsWithEvents((options.tools || {})),
|
|
1355
1360
|
}
|
|
1356
1361
|
: {};
|
|
1357
1362
|
// Apply per-call tool filtering (whitelist/blacklist)
|
|
@@ -1390,7 +1395,7 @@ export class BaseProvider {
|
|
|
1390
1395
|
return {};
|
|
1391
1396
|
}
|
|
1392
1397
|
const baseTools = await this.getAllTools();
|
|
1393
|
-
const externalTools = (options.tools || {});
|
|
1398
|
+
const externalTools = this.toolsManager.wrapExternalToolsWithEvents((options.tools || {}));
|
|
1394
1399
|
let merged = { ...baseTools, ...externalTools };
|
|
1395
1400
|
// Apply per-call tool filtering (whitelist/blacklist)
|
|
1396
1401
|
merged = this.applyToolFiltering(merged, options);
|
|
@@ -1430,8 +1435,44 @@ export class BaseProvider {
|
|
|
1430
1435
|
* Build messages array for generation - delegated to MessageBuilder
|
|
1431
1436
|
*/
|
|
1432
1437
|
async buildMessages(options) {
|
|
1438
|
+
this.applyToolReplayDefault(options);
|
|
1433
1439
|
return this.messageBuilder.buildMessages(options);
|
|
1434
1440
|
}
|
|
1441
|
+
/**
|
|
1442
|
+
* Fold the instance's `conversationMemory.replayToolSteps` into the request
|
|
1443
|
+
* when the caller set none, so every path into the message builder — text
|
|
1444
|
+
* or multimodal, generate or stream — reads one resolved value. The
|
|
1445
|
+
* builder itself defaults to `"marker"` when neither is set.
|
|
1446
|
+
*
|
|
1447
|
+
* Then degrade `"full"` to `"marker"` for a request that declares no tools:
|
|
1448
|
+
* Anthropic rejects a prompt carrying tool_use / tool_result blocks without
|
|
1449
|
+
* a `tools` field ("Requests which include tool_use or tool_result blocks
|
|
1450
|
+
* must define tools"), so a tools-off call — `disableTools`, or a filter
|
|
1451
|
+
* that removed every tool — in a session that already ran one would fail
|
|
1452
|
+
* on every later turn. Runs after the provider has merged `options.tools`,
|
|
1453
|
+
* which is where every native path calls the builder from.
|
|
1454
|
+
*/
|
|
1455
|
+
applyToolReplayDefault(options) {
|
|
1456
|
+
if (options.replayToolSteps === undefined) {
|
|
1457
|
+
const configured = this.neurolink?.conversationMemory?.config?.replayToolSteps;
|
|
1458
|
+
if (configured !== undefined) {
|
|
1459
|
+
options.replayToolSteps = configured;
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
if (options.replayToolSteps !== "full") {
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
const declaredTools = options.tools;
|
|
1466
|
+
const declaresNoTools = options.disableTools === true ||
|
|
1467
|
+
(declaredTools !== undefined &&
|
|
1468
|
+
declaredTools !== null &&
|
|
1469
|
+
typeof declaredTools === "object" &&
|
|
1470
|
+
Object.keys(declaredTools).length === 0);
|
|
1471
|
+
if (declaresNoTools) {
|
|
1472
|
+
logger.debug(`[${this.providerName}] replayToolSteps "full" degraded to "marker": the request declares no tools`);
|
|
1473
|
+
options.replayToolSteps = "marker";
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1435
1476
|
/**
|
|
1436
1477
|
* Build messages array for streaming operations - delegated to MessageBuilder
|
|
1437
1478
|
* This is a protected helper method that providers can use to build messages
|
|
@@ -1441,6 +1482,7 @@ export class BaseProvider {
|
|
|
1441
1482
|
* @returns Promise resolving to ModelMessage array ready for AI SDK
|
|
1442
1483
|
*/
|
|
1443
1484
|
async buildMessagesForStream(options) {
|
|
1485
|
+
this.applyToolReplayDefault(options);
|
|
1444
1486
|
return this.messageBuilder.buildMessagesForStream(options);
|
|
1445
1487
|
}
|
|
1446
1488
|
/**
|
|
@@ -1674,6 +1716,15 @@ export class BaseProvider {
|
|
|
1674
1716
|
}
|
|
1675
1717
|
}
|
|
1676
1718
|
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Mode 1 — synthesize the input text itself, with no LLM call.
|
|
1721
|
+
*
|
|
1722
|
+
* This is the documented default for `generate({ tts: { enabled: true } })`
|
|
1723
|
+
* and what every "speak this text" caller relies on, so a request that
|
|
1724
|
+
* looks like it wanted the model's reply (tools, a system prompt, message
|
|
1725
|
+
* history, a schema, conversation memory) is warned about, never rejected:
|
|
1726
|
+
* those options have no effect here, and the fix is `tts.mode: "response"`.
|
|
1727
|
+
*/
|
|
1677
1728
|
async handleDirectTTSSynthesis(options, startTime) {
|
|
1678
1729
|
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
1679
1730
|
const baseResult = {
|
|
@@ -1686,14 +1737,17 @@ export class BaseProvider {
|
|
|
1686
1737
|
if (!ttsOptions) {
|
|
1687
1738
|
return this.enhanceResult(baseResult, options, startTime);
|
|
1688
1739
|
}
|
|
1740
|
+
this.warnIfDirectTTSLooksLikeResponseRequest(options);
|
|
1689
1741
|
const ttsStartTime = Date.now();
|
|
1690
1742
|
const ttsProvider = ttsOptions.provider ?? options.provider ?? this.providerName;
|
|
1691
|
-
const ttsTimeout = this.
|
|
1743
|
+
const ttsTimeout = this.resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider);
|
|
1744
|
+
const speechText = this.prepareTTSInput(textToSynthesize, ttsOptions);
|
|
1692
1745
|
try {
|
|
1693
|
-
baseResult.audio = await
|
|
1746
|
+
baseResult.audio = await this.runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, options.abortSignal);
|
|
1694
1747
|
baseResult.ttsMetadata = {
|
|
1695
1748
|
attempted: true,
|
|
1696
1749
|
success: true,
|
|
1750
|
+
mode: "direct",
|
|
1697
1751
|
latency: Date.now() - ttsStartTime,
|
|
1698
1752
|
};
|
|
1699
1753
|
}
|
|
@@ -1703,6 +1757,7 @@ export class BaseProvider {
|
|
|
1703
1757
|
baseResult.ttsMetadata = {
|
|
1704
1758
|
attempted: true,
|
|
1705
1759
|
success: false,
|
|
1760
|
+
mode: "direct",
|
|
1706
1761
|
error,
|
|
1707
1762
|
latency,
|
|
1708
1763
|
};
|
|
@@ -1711,6 +1766,105 @@ export class BaseProvider {
|
|
|
1711
1766
|
}
|
|
1712
1767
|
return this.enhanceResult(baseResult, options, startTime);
|
|
1713
1768
|
}
|
|
1769
|
+
/**
|
|
1770
|
+
* One warning when a direct-synthesis request carries options that only a
|
|
1771
|
+
* model call would honour and the caller never chose a mode. Silence is
|
|
1772
|
+
* the wrong answer here — the caller gets `usage` of zero and `content`
|
|
1773
|
+
* echoing their prompt, and would otherwise have to guess why.
|
|
1774
|
+
*
|
|
1775
|
+
* Only options the CALLER supplied count. `conversationMemoryConfig` is
|
|
1776
|
+
* attached by NeuroLink to every request on a memory-enabled instance, and
|
|
1777
|
+
* on such an instance `conversationMessages` is the stored history
|
|
1778
|
+
* NeuroLink injected itself — neither is something the caller set, and a
|
|
1779
|
+
* voice bot with memory used to see this warning on every turn after the
|
|
1780
|
+
* first for options it never passed.
|
|
1781
|
+
*/
|
|
1782
|
+
warnIfDirectTTSLooksLikeResponseRequest(options) {
|
|
1783
|
+
const tts = options.tts;
|
|
1784
|
+
if (!tts || tts.mode !== undefined || tts.useAiResponse !== undefined) {
|
|
1785
|
+
return;
|
|
1786
|
+
}
|
|
1787
|
+
const memoryAttached = options.conversationMemoryConfig !== undefined;
|
|
1788
|
+
const llmShaped = [];
|
|
1789
|
+
if (options.tools && Object.keys(options.tools).length > 0) {
|
|
1790
|
+
llmShaped.push("tools");
|
|
1791
|
+
}
|
|
1792
|
+
if (options.systemPrompt) {
|
|
1793
|
+
llmShaped.push("systemPrompt");
|
|
1794
|
+
}
|
|
1795
|
+
if (!memoryAttached &&
|
|
1796
|
+
options.conversationMessages &&
|
|
1797
|
+
options.conversationMessages.length > 0) {
|
|
1798
|
+
llmShaped.push("conversationMessages");
|
|
1799
|
+
}
|
|
1800
|
+
if (options.conversationHistory && options.conversationHistory.length > 0) {
|
|
1801
|
+
llmShaped.push("conversationHistory");
|
|
1802
|
+
}
|
|
1803
|
+
if (options.schema) {
|
|
1804
|
+
llmShaped.push("schema");
|
|
1805
|
+
}
|
|
1806
|
+
if (llmShaped.length === 0) {
|
|
1807
|
+
return;
|
|
1808
|
+
}
|
|
1809
|
+
logger.warn(`[${this.providerName}] tts.enabled without tts.mode synthesizes the INPUT text and makes no model call, so ${llmShaped.join(", ")} will be ignored. Set tts.mode: "response" to speak the model's reply, or tts.mode: "direct" to silence this warning.`);
|
|
1810
|
+
}
|
|
1811
|
+
/**
|
|
1812
|
+
* Outer bound on one synthesis call: the provider's request timeout, raised
|
|
1813
|
+
* to whatever the handler itself may legitimately spend — `timeoutMs × (1 +
|
|
1814
|
+
* retries)` plus retry pauses, with the handler defaults (30 s, one retry)
|
|
1815
|
+
* standing in for a knob the caller left unset. Without the raise, a
|
|
1816
|
+
* handler's own retry loop could be cut off from outside one attempt in:
|
|
1817
|
+
* with the 60 s provider default, a first attempt that hung to its 30 s
|
|
1818
|
+
* abort left no room for the documented default retry.
|
|
1819
|
+
*/
|
|
1820
|
+
resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider) {
|
|
1821
|
+
return Math.max(this.getTimeout(options), resolveTTSHandlerBudgetMs(ttsOptions, ttsProvider));
|
|
1822
|
+
}
|
|
1823
|
+
/**
|
|
1824
|
+
* Apply `tts.sanitize` (off by default — existing callers see byte-identical
|
|
1825
|
+
* input). Always runs when configured, regardless of the raw text's
|
|
1826
|
+
* length against the provider's cap: sanitizing can shrink text a great
|
|
1827
|
+
* deal (the default `codeBlocks: "phrase"` replaces a whole fenced block
|
|
1828
|
+
* with one short phrase, `urls: "hostname"` shortens every link), so a
|
|
1829
|
+
* reply that is over the cap raw can still fit after sanitizing — a coding
|
|
1830
|
+
* answer with a large code block is exactly the reply this option is for.
|
|
1831
|
+
* A reply still over the cap afterwards is rejected downstream by
|
|
1832
|
+
* `TTSProcessor.synthesize` as `TTS_TEXT_TOO_LONG`, same as today.
|
|
1833
|
+
*/
|
|
1834
|
+
prepareTTSInput(text, ttsOptions) {
|
|
1835
|
+
const sanitizeOptions = resolveSpeechSanitizeOptions(ttsOptions.sanitize);
|
|
1836
|
+
if (!sanitizeOptions) {
|
|
1837
|
+
return text;
|
|
1838
|
+
}
|
|
1839
|
+
return prepareTextForSpeech(text, sanitizeOptions);
|
|
1840
|
+
}
|
|
1841
|
+
/**
|
|
1842
|
+
* One synthesis call under the outer budget. The handler receives a signal
|
|
1843
|
+
* that fires when that budget lapses or the caller's own `abortSignal`
|
|
1844
|
+
* does, so a handler with a retry loop (ElevenLabs) stops rather than
|
|
1845
|
+
* pausing and issuing a further billable request after the caller has
|
|
1846
|
+
* already been told the synthesis failed.
|
|
1847
|
+
*/
|
|
1848
|
+
async runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, abortSignal) {
|
|
1849
|
+
const cancel = new AbortController();
|
|
1850
|
+
const signals = [cancel.signal];
|
|
1851
|
+
if (abortSignal) {
|
|
1852
|
+
signals.push(abortSignal);
|
|
1853
|
+
}
|
|
1854
|
+
if (ttsOptions.signal) {
|
|
1855
|
+
signals.push(ttsOptions.signal);
|
|
1856
|
+
}
|
|
1857
|
+
const signal = AbortSignal.any(signals);
|
|
1858
|
+
try {
|
|
1859
|
+
return await withTimeoutFn(() => TTSProcessor.synthesize(speechText, ttsProvider, {
|
|
1860
|
+
...ttsOptions,
|
|
1861
|
+
signal,
|
|
1862
|
+
}), ttsTimeout, `TTS synthesis timed out after ${ttsTimeout}ms for provider "${ttsProvider}"`);
|
|
1863
|
+
}
|
|
1864
|
+
finally {
|
|
1865
|
+
cancel.abort();
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1714
1868
|
async handleVideoFrameGeneration(options, messages, model, startTime) {
|
|
1715
1869
|
if (!hasVideoFrames(messages)) {
|
|
1716
1870
|
return null;
|
|
@@ -1795,15 +1949,93 @@ export class BaseProvider {
|
|
|
1795
1949
|
* synthesis so telemetry sees the model's own usage, and `enhanceResult`
|
|
1796
1950
|
* runs last so analytics and evaluation observe the final content.
|
|
1797
1951
|
*/
|
|
1798
|
-
async finalizeNativeGenerate(result, options, startTime) {
|
|
1952
|
+
async finalizeNativeGenerate(result, options, startTime, toolExecutionSummaries) {
|
|
1799
1953
|
// onFinish is NOT fired here. `applyGenerateLifecycleMiddleware` folds it
|
|
1800
1954
|
// into `options.middleware.middlewareConfig.lifecycle`, and the native
|
|
1801
1955
|
// paths now wrap their model, so the lifecycle middleware fires it.
|
|
1802
1956
|
// Firing it here too would deliver every callback twice.
|
|
1803
1957
|
await this.recordPerformanceMetrics(result.usage, Date.now() - startTime);
|
|
1958
|
+
await this.persistGenerateToolSteps(options, toolExecutionSummaries);
|
|
1804
1959
|
const synthesized = await this.synthesizeAIResponseIfNeeded(result, options);
|
|
1805
1960
|
return this.enhanceResult(synthesized, options, startTime);
|
|
1806
1961
|
}
|
|
1962
|
+
/**
|
|
1963
|
+
* Persist a generate() turn's tool steps into conversation memory.
|
|
1964
|
+
*
|
|
1965
|
+
* The stream loops store each step's calls and results as they run;
|
|
1966
|
+
* generate() never did, so the same session held tool rows for streamed
|
|
1967
|
+
* turns and none for generated ones, and a later turn could not replay
|
|
1968
|
+
* what generate() had called. Every native generate path closes out
|
|
1969
|
+
* through finalizeNativeGenerate, so this is the one place to do it.
|
|
1970
|
+
*
|
|
1971
|
+
* Gated on a real session id: the storage helper otherwise invents a
|
|
1972
|
+
* `session-<nanoid>` key, and on the in-memory backend that creates a
|
|
1973
|
+
* throwaway session per tool-using call — evicting real ones once
|
|
1974
|
+
* `maxSessions` is reached. A turn with no session has no history to
|
|
1975
|
+
* replay into, so there is nothing to persist.
|
|
1976
|
+
*/
|
|
1977
|
+
async persistGenerateToolSteps(options, summaries) {
|
|
1978
|
+
if (!summaries || summaries.length === 0) {
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
const sessionId = options.context?.sessionId ??
|
|
1982
|
+
options.sessionId;
|
|
1983
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) {
|
|
1984
|
+
return;
|
|
1985
|
+
}
|
|
1986
|
+
// One storage call per loop step, in step order. The stream loops store
|
|
1987
|
+
// as each step finishes, so a step's calls and results sit together;
|
|
1988
|
+
// storing a whole generate() turn as one batch replayed a sequential
|
|
1989
|
+
// lookup-then-send as a single parallel step, with the second call's
|
|
1990
|
+
// arguments appearing before the output they were built from.
|
|
1991
|
+
//
|
|
1992
|
+
// Summaries arrive in execution order, so a new batch starts whenever
|
|
1993
|
+
// the step changes rather than grouping by stepIndex value: a provider
|
|
1994
|
+
// can run the native loop more than once against the same summaries
|
|
1995
|
+
// array (a response_format rejection retry, a schema-validation retry),
|
|
1996
|
+
// and each run numbers its own steps from 0 — grouping by value would
|
|
1997
|
+
// merge run 1's step 0 with run 2's step 0 into one parallel batch.
|
|
1998
|
+
const batches = [];
|
|
1999
|
+
for (const summary of summaries) {
|
|
2000
|
+
const last = batches[batches.length - 1];
|
|
2001
|
+
if (last && last[0].stepIndex === summary.stepIndex) {
|
|
2002
|
+
last.push(summary);
|
|
2003
|
+
}
|
|
2004
|
+
else {
|
|
2005
|
+
batches.push([summary]);
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
for (const stepSummaries of batches) {
|
|
2009
|
+
const toolCalls = stepSummaries.map((s) => ({
|
|
2010
|
+
toolCallId: s.toolCallId,
|
|
2011
|
+
toolName: s.toolName,
|
|
2012
|
+
args: s.input !== null &&
|
|
2013
|
+
typeof s.input === "object" &&
|
|
2014
|
+
!Array.isArray(s.input)
|
|
2015
|
+
? s.input
|
|
2016
|
+
: {},
|
|
2017
|
+
...(s.stepIndex !== undefined ? { stepIndex: s.stepIndex } : {}),
|
|
2018
|
+
timestamp: s.startTime,
|
|
2019
|
+
}));
|
|
2020
|
+
const toolResults = stepSummaries.map((s) => ({
|
|
2021
|
+
toolCallId: s.toolCallId,
|
|
2022
|
+
toolName: s.toolName,
|
|
2023
|
+
output: s.output,
|
|
2024
|
+
...(s.error ? { error: s.error } : {}),
|
|
2025
|
+
...(s.stepIndex !== undefined ? { stepIndex: s.stepIndex } : {}),
|
|
2026
|
+
timestamp: s.endTime,
|
|
2027
|
+
}));
|
|
2028
|
+
try {
|
|
2029
|
+
await this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date());
|
|
2030
|
+
}
|
|
2031
|
+
catch (error) {
|
|
2032
|
+
logger.warn(`[${this.providerName}] Failed to store tool executions`, {
|
|
2033
|
+
provider: this.providerName,
|
|
2034
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2035
|
+
});
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
1807
2039
|
/**
|
|
1808
2040
|
* Invoke a caller's `onFinish` for a native turn.
|
|
1809
2041
|
*
|
|
@@ -1838,7 +2070,7 @@ export class BaseProvider {
|
|
|
1838
2070
|
}
|
|
1839
2071
|
}
|
|
1840
2072
|
async synthesizeAIResponseIfNeeded(enhancedResult, options) {
|
|
1841
|
-
if (!options.tts?.enabled ||
|
|
2073
|
+
if (!options.tts?.enabled || resolveTTSMode(options.tts) !== "response") {
|
|
1842
2074
|
return enhancedResult;
|
|
1843
2075
|
}
|
|
1844
2076
|
const ttsOptions = options.tts;
|
|
@@ -1852,6 +2084,7 @@ export class BaseProvider {
|
|
|
1852
2084
|
hasProvider: !!ttsProvider,
|
|
1853
2085
|
ttsConfig: {
|
|
1854
2086
|
enabled: options.tts?.enabled,
|
|
2087
|
+
mode: options.tts?.mode,
|
|
1855
2088
|
useAiResponse: options.tts?.useAiResponse,
|
|
1856
2089
|
},
|
|
1857
2090
|
reason: !aiResponse
|
|
@@ -1863,19 +2096,22 @@ export class BaseProvider {
|
|
|
1863
2096
|
ttsMetadata: {
|
|
1864
2097
|
attempted: false,
|
|
1865
2098
|
success: false,
|
|
2099
|
+
mode: "response",
|
|
1866
2100
|
},
|
|
1867
2101
|
};
|
|
1868
2102
|
}
|
|
1869
2103
|
const ttsStartTime = Date.now();
|
|
1870
|
-
const ttsTimeout = this.
|
|
2104
|
+
const ttsTimeout = this.resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider);
|
|
2105
|
+
const speechText = this.prepareTTSInput(aiResponse, ttsOptions);
|
|
1871
2106
|
try {
|
|
1872
|
-
const ttsResult = await
|
|
2107
|
+
const ttsResult = await this.runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, options.abortSignal);
|
|
1873
2108
|
return {
|
|
1874
2109
|
...enhancedResult,
|
|
1875
2110
|
audio: ttsResult,
|
|
1876
2111
|
ttsMetadata: {
|
|
1877
2112
|
attempted: true,
|
|
1878
2113
|
success: true,
|
|
2114
|
+
mode: "response",
|
|
1879
2115
|
latency: Date.now() - ttsStartTime,
|
|
1880
2116
|
},
|
|
1881
2117
|
};
|
|
@@ -1890,6 +2126,7 @@ export class BaseProvider {
|
|
|
1890
2126
|
ttsMetadata: {
|
|
1891
2127
|
attempted: true,
|
|
1892
2128
|
success: false,
|
|
2129
|
+
mode: "response",
|
|
1893
2130
|
error,
|
|
1894
2131
|
latency,
|
|
1895
2132
|
},
|
|
@@ -2385,6 +2622,13 @@ export class BaseProvider {
|
|
|
2385
2622
|
this.fileToolRootsBound = true;
|
|
2386
2623
|
this.toolsManager.setDirectTools(policy.roots === null ? this.directTools : createDirectAgentTools(policy));
|
|
2387
2624
|
}
|
|
2625
|
+
/**
|
|
2626
|
+
* Emit the start/end pair for a tool call a loop rejected before execute
|
|
2627
|
+
* (unknown tool, schema rejection). See ToolsManager.emitRejectedToolCall.
|
|
2628
|
+
*/
|
|
2629
|
+
emitRejectedToolCall(toolName, error, toolCallId) {
|
|
2630
|
+
this.toolsManager.emitRejectedToolCall(toolName, error, toolCallId);
|
|
2631
|
+
}
|
|
2388
2632
|
/**
|
|
2389
2633
|
* Resolve roots for a provider called directly, without NeuroLink. A
|
|
2390
2634
|
* policy NeuroLink already bound is left alone unless these options carry
|
|
@@ -11,6 +11,7 @@ import { tracers, withSpan } from "../telemetry/index.js";
|
|
|
11
11
|
import { ConversationMemoryError } from "../types/index.js";
|
|
12
12
|
import { buildContextFromPointer, getEffectiveTokenThreshold, } from "../utils/conversationMemory.js";
|
|
13
13
|
import { logger } from "../utils/logger.js";
|
|
14
|
+
import { extractIsErrorText } from "../utils/toolResultStatus.js";
|
|
14
15
|
export class ConversationMemoryManager {
|
|
15
16
|
sessions = new Map();
|
|
16
17
|
config;
|
|
@@ -421,6 +422,7 @@ export class ConversationMemoryManager {
|
|
|
421
422
|
if (toolCallId) {
|
|
422
423
|
toolNameById.set(toolCallId, toolName);
|
|
423
424
|
}
|
|
425
|
+
const callStepIndex = toolCall.stepIndex;
|
|
424
426
|
session.messages.push({
|
|
425
427
|
id: randomUUID(),
|
|
426
428
|
role: "tool_call",
|
|
@@ -428,6 +430,10 @@ export class ConversationMemoryManager {
|
|
|
428
430
|
tool: toolName,
|
|
429
431
|
...(toolCallId ? { toolCallId } : {}),
|
|
430
432
|
args: (toolCall.args ?? {}),
|
|
433
|
+
// Zero-based loop step, so replay can keep sequential steps apart.
|
|
434
|
+
...(typeof callStepIndex === "number"
|
|
435
|
+
? { metadata: { stepIndex: callStepIndex } }
|
|
436
|
+
: {}),
|
|
431
437
|
timestamp,
|
|
432
438
|
});
|
|
433
439
|
}
|
|
@@ -448,6 +454,14 @@ export class ConversationMemoryManager {
|
|
|
448
454
|
content = `[Serialization failed: ${error instanceof Error ? error.message : String(error)}]`;
|
|
449
455
|
}
|
|
450
456
|
}
|
|
457
|
+
// A result the tool RETURNED with `isError: true` (a thrown error that
|
|
458
|
+
// neurolink.executeTool converted, a breaker refusal, an MCP failure)
|
|
459
|
+
// is a failure too; recording it as success made the next turn's
|
|
460
|
+
// replay say "→ ok" for a call that failed.
|
|
461
|
+
const error = toolResult.error !== undefined && toolResult.error !== null
|
|
462
|
+
? String(toolResult.error)
|
|
463
|
+
: extractIsErrorText(rawOutput);
|
|
464
|
+
const stepIndex = toolResult.stepIndex;
|
|
451
465
|
session.messages.push({
|
|
452
466
|
id: randomUUID(),
|
|
453
467
|
role: "tool_result",
|
|
@@ -455,9 +469,10 @@ export class ConversationMemoryManager {
|
|
|
455
469
|
tool: toolName,
|
|
456
470
|
...(toolCallId ? { toolCallId } : {}),
|
|
457
471
|
result: {
|
|
458
|
-
success:
|
|
459
|
-
...(
|
|
472
|
+
success: error === undefined,
|
|
473
|
+
...(error !== undefined ? { error } : {}),
|
|
460
474
|
},
|
|
475
|
+
...(typeof stepIndex === "number" ? { metadata: { stepIndex } } : {}),
|
|
461
476
|
timestamp,
|
|
462
477
|
});
|
|
463
478
|
}
|
package/dist/core/loopEngine.js
CHANGED
|
@@ -324,7 +324,7 @@ export function runAgenticLoop(adapter, initialConversation, options) {
|
|
|
324
324
|
break;
|
|
325
325
|
}
|
|
326
326
|
}
|
|
327
|
-
const request = adapter.buildStepRequest(conversation, step);
|
|
327
|
+
const request = await adapter.buildStepRequest(conversation, step);
|
|
328
328
|
// A tool that just became callable starts clean. Its TOOL_NOT_FOUND
|
|
329
329
|
// strikes were recorded against a name that genuinely did not resolve
|
|
330
330
|
// yet, and the breaker is consulted before the lookup, so without this
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { tracers, ATTR, withSpan } from "../../telemetry/index.js";
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
|
-
import { buildMessagesArray,
|
|
3
|
+
import { buildMessagesArray, buildMultimodalModelMessages, } from "../../utils/messageBuilder.js";
|
|
4
4
|
/**
|
|
5
5
|
* Compute total content length across all messages for span attributes.
|
|
6
6
|
*/
|
|
@@ -127,11 +127,12 @@ export class MessageBuilder {
|
|
|
127
127
|
enableEvaluation: options.enableEvaluation,
|
|
128
128
|
context: options.context,
|
|
129
129
|
conversationHistory: options.conversationMessages,
|
|
130
|
+
replayToolSteps: options.replayToolSteps,
|
|
130
131
|
schema: options.schema,
|
|
131
132
|
output: options.output,
|
|
132
133
|
fileRegistry: options.fileRegistry,
|
|
133
134
|
};
|
|
134
|
-
messages = await
|
|
135
|
+
messages = await buildMultimodalModelMessages(multimodalOptions, this.providerName, this.modelName);
|
|
135
136
|
// Propagate any systemPrompt augmentation (e.g. inline-file
|
|
136
137
|
// handling guidance from processUnifiedFilesArray) back to the
|
|
137
138
|
// caller's options. Providers like GoogleVertex's native
|
|
@@ -163,6 +164,9 @@ export class MessageBuilder {
|
|
|
163
164
|
}
|
|
164
165
|
else {
|
|
165
166
|
return {
|
|
167
|
+
// `tool` rides through untouched: a replayed tool step
|
|
168
|
+
// (replayToolSteps: "full") is a tool turn of tool-result parts,
|
|
169
|
+
// and its tool-call parts fall to the passthrough branch below.
|
|
166
170
|
role: msg.role,
|
|
167
171
|
content: msg.content.map((item) => {
|
|
168
172
|
const itemProviderOptions = item.providerOptions;
|
|
@@ -272,11 +276,12 @@ export class MessageBuilder {
|
|
|
272
276
|
context: options.context,
|
|
273
277
|
conversationHistory: options
|
|
274
278
|
.conversationMessages,
|
|
279
|
+
replayToolSteps: options.replayToolSteps,
|
|
275
280
|
schema: options.schema,
|
|
276
281
|
output: options.output,
|
|
277
282
|
fileRegistry: options.fileRegistry,
|
|
278
283
|
};
|
|
279
|
-
messages = await
|
|
284
|
+
messages = await buildMultimodalModelMessages(multimodalOptions, this.providerName, this.modelName);
|
|
280
285
|
// Propagate any systemPrompt augmentation (e.g. inline-file
|
|
281
286
|
// handling guidance from processUnifiedFilesArray) back to the
|
|
282
287
|
// caller's options. Providers like GoogleVertex's native
|
|
@@ -308,6 +313,9 @@ export class MessageBuilder {
|
|
|
308
313
|
}
|
|
309
314
|
else {
|
|
310
315
|
return {
|
|
316
|
+
// `tool` rides through untouched: a replayed tool step
|
|
317
|
+
// (replayToolSteps: "full") is a tool turn of tool-result parts,
|
|
318
|
+
// and its tool-call parts fall to the passthrough branch below.
|
|
311
319
|
role: msg.role,
|
|
312
320
|
content: msg.content.map((item) => {
|
|
313
321
|
const itemProviderOptions = item.providerOptions;
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* @module core/modules/TelemetryHandler
|
|
15
15
|
*/
|
|
16
16
|
import type { NeuroLink } from "../../neurolink.js";
|
|
17
|
-
import type { StreamOptions, AIProviderName, AnalyticsData, EnhancedGenerateResult, EvaluationData, RawUsageObject, TextGenerationOptions, TokenUsage, TTSMetadata } from "../../types/index.js";
|
|
17
|
+
import type { StreamOptions, AIProviderName, AnalyticsData, EnhancedGenerateResult, EvaluationData, MemoryToolCallRecord, MemoryToolResultRecord, RawUsageObject, TextGenerationOptions, TokenUsage, TTSMetadata } from "../../types/index.js";
|
|
18
18
|
/**
|
|
19
19
|
* TelemetryHandler class - Handles analytics and telemetry for AI providers
|
|
20
20
|
*/
|
|
@@ -66,5 +66,5 @@ export declare class TelemetryHandler {
|
|
|
66
66
|
/**
|
|
67
67
|
* Handle tool execution storage if available
|
|
68
68
|
*/
|
|
69
|
-
handleToolExecutionStorage(toolCalls:
|
|
69
|
+
handleToolExecutionStorage(toolCalls: MemoryToolCallRecord[], toolResults: MemoryToolResultRecord[], options: TextGenerationOptions | StreamOptions, currentTime: Date): Promise<void>;
|
|
70
70
|
}
|
|
@@ -28,6 +28,7 @@ export declare class ToolsManager {
|
|
|
28
28
|
protected toolExecutor?: (toolName: string, params: unknown, options?: Record<string, unknown>) => Promise<unknown>;
|
|
29
29
|
protected sessionId?: string;
|
|
30
30
|
protected userId?: string;
|
|
31
|
+
private readonly eventWrappedExecutes;
|
|
31
32
|
constructor(providerName: AIProviderName, directTools: Record<string, unknown>, neurolink?: NeuroLink | undefined, utilities?: ToolUtilities | undefined);
|
|
32
33
|
/**
|
|
33
34
|
* BZ-666: Wrap tool execute with output truncation to prevent
|
|
@@ -93,7 +94,41 @@ export declare class ToolsManager {
|
|
|
93
94
|
* Set session context for MCP tools
|
|
94
95
|
*/
|
|
95
96
|
setSessionContext(sessionId?: string, userId?: string): void;
|
|
97
|
+
/**
|
|
98
|
+
* Emit one tool event. A listener that throws must never reach the tool
|
|
99
|
+
* loop: it would turn a successful execution into a failed one (the model
|
|
100
|
+
* then retries a side effect that already happened) and, from inside the
|
|
101
|
+
* wrapper's catch, produce a second `tool:end` for the same call. Listener
|
|
102
|
+
* failures are logged and swallowed here instead.
|
|
103
|
+
*/
|
|
96
104
|
private emitToolEvent;
|
|
105
|
+
/**
|
|
106
|
+
* Wrap an execute function so one `tool:start` and exactly one `tool:end`
|
|
107
|
+
* (with the real duration) fire per invocation. Idempotent: a function this
|
|
108
|
+
* manager already wrapped is returned as is, so a tool that re-enters
|
|
109
|
+
* resolution (the stream → generate fallback re-resolves `options.tools`)
|
|
110
|
+
* never emits twice.
|
|
111
|
+
*
|
|
112
|
+
* A result the tool RETURNED with `isError: true` (MCP failure, breaker
|
|
113
|
+
* refusal, a thrown error already converted by `neurolink.executeTool`) is
|
|
114
|
+
* a failure: `tool:end` carries `success: false` and the extracted text,
|
|
115
|
+
* as the step-finish emitter this wrapper replaced did.
|
|
116
|
+
*/
|
|
117
|
+
private wrapExecuteWithEvents;
|
|
118
|
+
/**
|
|
119
|
+
* Report a tool call the loop rejected BEFORE any execute ran — an unknown
|
|
120
|
+
* tool name, arguments the schema rejected, a breaker skip. Nothing else
|
|
121
|
+
* emits for such a call, so listeners (memory event capture, tool spans,
|
|
122
|
+
* error-rate metrics) saw zero attempts while the result list held the
|
|
123
|
+
* failures. One `tool:start` / `tool:end(success: false)` pair.
|
|
124
|
+
*/
|
|
125
|
+
emitRejectedToolCall(toolName: string, error: string, toolCallId?: string): void;
|
|
126
|
+
/**
|
|
127
|
+
* Give per-call tools (`options.tools`) the same `tool:start` / `tool:end`
|
|
128
|
+
* emission that registered and MCP tools get from their executors. Tools
|
|
129
|
+
* without an `execute` pass through untouched.
|
|
130
|
+
*/
|
|
131
|
+
wrapExternalToolsWithEvents(tools: Record<string, Tool>): Record<string, Tool>;
|
|
97
132
|
/**
|
|
98
133
|
* Set up tool executor for a provider to enable actual tool execution
|
|
99
134
|
* @param sdk - The NeuroLinkSDK instance for tool execution
|