@juspay/neurolink 12.47.13 → 12.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/CHANGELOG.md +3 -3
  2. package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
  3. package/dist/browser/neurolink.min.js +578 -565
  4. package/dist/cli/factories/commandFactory.d.ts +7 -0
  5. package/dist/cli/factories/commandFactory.js +26 -0
  6. package/dist/cli/loop/optionsSchema.js +9 -0
  7. package/dist/context/budgetChecker.js +54 -2
  8. package/dist/context/contextDecision.js +2 -0
  9. package/dist/context/toolPairRepair.d.ts +12 -0
  10. package/dist/context/toolPairRepair.js +29 -3
  11. package/dist/core/baseProvider.d.ts +92 -3
  12. package/dist/core/baseProvider.js +254 -10
  13. package/dist/core/conversationMemoryManager.js +17 -2
  14. package/dist/core/loopEngine.js +1 -1
  15. package/dist/core/modules/MessageBuilder.js +11 -3
  16. package/dist/core/modules/TelemetryHandler.d.ts +2 -2
  17. package/dist/core/modules/ToolsManager.d.ts +35 -0
  18. package/dist/core/modules/ToolsManager.js +159 -55
  19. package/dist/core/nativeGenerateLoop.d.ts +58 -1
  20. package/dist/core/nativeGenerateLoop.js +277 -3
  21. package/dist/core/redisConversationMemoryManager.js +10 -3
  22. package/dist/core/resolveRequestKind.d.ts +24 -4
  23. package/dist/core/resolveRequestKind.js +31 -4
  24. package/dist/core/toolExecutionRecorder.d.ts +6 -0
  25. package/dist/core/toolExecutionRecorder.js +24 -2
  26. package/dist/core/toolRoutingDecision.js +1 -0
  27. package/dist/factories/mediaHandlerCatalog.js +5 -0
  28. package/dist/factories/providerDescriptors.js +15 -0
  29. package/dist/files/fileTools.d.ts +1 -1
  30. package/dist/index.d.ts +5 -2
  31. package/dist/index.js +9 -2
  32. package/dist/memory/condensationPrompt.d.ts +8 -0
  33. package/dist/memory/condensationPrompt.js +29 -0
  34. package/dist/memory/hippocampusInitializer.d.ts +14 -2
  35. package/dist/memory/hippocampusInitializer.js +64 -3
  36. package/dist/neurolink.d.ts +252 -3
  37. package/dist/neurolink.js +898 -43
  38. package/dist/providers/amazonBedrock/client.js +6 -0
  39. package/dist/providers/amazonSagemaker.js +14 -1
  40. package/dist/providers/anthropic/client.js +148 -19
  41. package/dist/providers/anthropic/loopAdapter.js +2 -2
  42. package/dist/providers/googleAiStudio/client.js +29 -14
  43. package/dist/providers/openaiChatCompletionsBase.js +103 -23
  44. package/dist/providers/systemOneDecision.d.ts +6 -0
  45. package/dist/providers/systemOneDecision.js +14 -63
  46. package/dist/providers/typesafe.d.ts +6 -0
  47. package/dist/providers/typesafe.js +6 -0
  48. package/dist/rag/retrieval/searchDecision.js +1 -0
  49. package/dist/routing/classifierRouter.js +6 -5
  50. package/dist/routing/classifierStrategies.js +8 -1
  51. package/dist/types/classifierRouter.d.ts +16 -3
  52. package/dist/types/common.d.ts +2 -0
  53. package/dist/types/config.d.ts +9 -0
  54. package/dist/types/context.d.ts +8 -1
  55. package/dist/types/conversation.d.ts +87 -0
  56. package/dist/types/decision.d.ts +210 -1
  57. package/dist/types/decision.js +9 -1
  58. package/dist/types/dispatch.d.ts +11 -4
  59. package/dist/types/generate.d.ts +118 -54
  60. package/dist/types/loopEngine.d.ts +7 -2
  61. package/dist/types/memory.d.ts +177 -9
  62. package/dist/types/openaiCompatible.d.ts +11 -3
  63. package/dist/types/stream.d.ts +13 -16
  64. package/dist/types/stt.d.ts +58 -1
  65. package/dist/types/tools.d.ts +14 -0
  66. package/dist/types/tools.js +8 -0
  67. package/dist/types/tts.d.ts +108 -0
  68. package/dist/types/voice.d.ts +41 -4
  69. package/dist/utils/decisionAnswers.d.ts +31 -1
  70. package/dist/utils/decisionAnswers.js +114 -0
  71. package/dist/utils/decisionLimits.d.ts +39 -0
  72. package/dist/utils/decisionLimits.js +116 -0
  73. package/dist/utils/logger.d.ts +7 -0
  74. package/dist/utils/logger.js +9 -0
  75. package/dist/utils/messageBuilder.d.ts +45 -1
  76. package/dist/utils/messageBuilder.js +393 -16
  77. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
  78. package/dist/utils/multimodalOptionsBuilder.js +1 -0
  79. package/dist/utils/speechText.d.ts +41 -0
  80. package/dist/utils/speechText.js +364 -0
  81. package/dist/utils/toolResultStatus.d.ts +26 -0
  82. package/dist/utils/toolResultStatus.js +73 -0
  83. package/dist/utils/ttsProcessor.d.ts +29 -0
  84. package/dist/utils/ttsProcessor.js +84 -2
  85. package/dist/voice/index.d.ts +1 -0
  86. package/dist/voice/index.js +3 -0
  87. package/dist/voice/providers/ElevenLabsSTT.d.ts +42 -0
  88. package/dist/voice/providers/ElevenLabsSTT.js +351 -0
  89. package/dist/voice/providers/ElevenLabsTTS.d.ts +81 -1
  90. package/dist/voice/providers/ElevenLabsTTS.js +415 -74
  91. package/docs-site/static/search-index.json +48 -33
  92. package/package.json +5 -2
  93. package/dist/utils/toolEndEmitter.d.ts +0 -25
  94. package/dist/utils/toolEndEmitter.js +0 -64
@@ -5,7 +5,7 @@ import { defaultProviderFor } from "../factories/mediaHandlerCatalog.js";
5
5
  import { DECISION_PROVIDERS, PROVIDER_DESCRIPTORS_BY_NAME, } from "../factories/providerDescriptors.js";
6
6
  import { MiddlewareFactory } from "../middleware/factory.js";
7
7
  import { modelSupports } from "../models/modelRegistry.js";
8
- import { resolveRequestKind } from "./resolveRequestKind.js";
8
+ import { resolveRequestKind, resolveTTSMode } from "./resolveRequestKind.js";
9
9
  import { ATTR, tracers } from "../telemetry/index.js";
10
10
  import { ERROR_CODES, isAbortError, NeuroLinkError, } from "../utils/errorHandling.js";
11
11
  import { InvalidModelError, ProviderError } from "../types/index.js";
@@ -34,7 +34,8 @@ import { composeAbortSignals, composeAbortSignalsScoped, createTimeoutController
34
34
  import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
35
35
  import { getKeyCount, getKeysAsString, transformToolExecutions, } from "../utils/transformationUtils.js";
36
36
  import { ToolExecutionRecorder } from "./toolExecutionRecorder.js";
37
- import { TTS_ERROR_CODES, TTSProcessor } from "../utils/ttsProcessor.js";
37
+ import { resolveTTSHandlerBudgetMs, TTS_ERROR_CODES, TTSProcessor, } from "../utils/ttsProcessor.js";
38
+ import { prepareTextForSpeech, resolveSpeechSanitizeOptions, } from "../utils/speechText.js";
38
39
  import { executeVideoAnalysis, hasVideoFrames, } from "../utils/videoAnalysisProcessor.js";
39
40
  import { dedupeTools } from "./toolDedup.js";
40
41
  import { resolveToolPolicy, toolNameMatcher } from "../tools/toolPolicy.js";
@@ -1074,6 +1075,7 @@ export class BaseProvider {
1074
1075
  ? {
1075
1076
  attempted: TTSProcessor.supports(ttsProvider),
1076
1077
  success: false,
1078
+ mode: "response",
1077
1079
  }
1078
1080
  : result?.ttsMetadata;
1079
1081
  const onTTSComplete = incrementalTTS
@@ -1348,10 +1350,13 @@ export class BaseProvider {
1348
1350
  async prepareGenerationContext(options) {
1349
1351
  const shouldUseTools = this.shouldUseTools(options, true);
1350
1352
  const baseTools = shouldUseTools ? await this.getAllTools() : {};
1353
+ // Registered and MCP tools arrive from ToolsManager already emitting
1354
+ // tool:start / tool:end; per-call tools get the same wrapper here so the
1355
+ // event bus sees every execution exactly once.
1351
1356
  let tools = shouldUseTools
1352
1357
  ? {
1353
1358
  ...baseTools,
1354
- ...(options.tools || {}),
1359
+ ...this.toolsManager.wrapExternalToolsWithEvents((options.tools || {})),
1355
1360
  }
1356
1361
  : {};
1357
1362
  // Apply per-call tool filtering (whitelist/blacklist)
@@ -1390,7 +1395,7 @@ export class BaseProvider {
1390
1395
  return {};
1391
1396
  }
1392
1397
  const baseTools = await this.getAllTools();
1393
- const externalTools = (options.tools || {});
1398
+ const externalTools = this.toolsManager.wrapExternalToolsWithEvents((options.tools || {}));
1394
1399
  let merged = { ...baseTools, ...externalTools };
1395
1400
  // Apply per-call tool filtering (whitelist/blacklist)
1396
1401
  merged = this.applyToolFiltering(merged, options);
@@ -1430,8 +1435,44 @@ export class BaseProvider {
1430
1435
  * Build messages array for generation - delegated to MessageBuilder
1431
1436
  */
1432
1437
  async buildMessages(options) {
1438
+ this.applyToolReplayDefault(options);
1433
1439
  return this.messageBuilder.buildMessages(options);
1434
1440
  }
1441
+ /**
1442
+ * Fold the instance's `conversationMemory.replayToolSteps` into the request
1443
+ * when the caller set none, so every path into the message builder — text
1444
+ * or multimodal, generate or stream — reads one resolved value. The
1445
+ * builder itself defaults to `"marker"` when neither is set.
1446
+ *
1447
+ * Then degrade `"full"` to `"marker"` for a request that declares no tools:
1448
+ * Anthropic rejects a prompt carrying tool_use / tool_result blocks without
1449
+ * a `tools` field ("Requests which include tool_use or tool_result blocks
1450
+ * must define tools"), so a tools-off call — `disableTools`, or a filter
1451
+ * that removed every tool — in a session that already ran one would fail
1452
+ * on every later turn. Runs after the provider has merged `options.tools`,
1453
+ * which is where every native path calls the builder from.
1454
+ */
1455
+ applyToolReplayDefault(options) {
1456
+ if (options.replayToolSteps === undefined) {
1457
+ const configured = this.neurolink?.conversationMemory?.config?.replayToolSteps;
1458
+ if (configured !== undefined) {
1459
+ options.replayToolSteps = configured;
1460
+ }
1461
+ }
1462
+ if (options.replayToolSteps !== "full") {
1463
+ return;
1464
+ }
1465
+ const declaredTools = options.tools;
1466
+ const declaresNoTools = options.disableTools === true ||
1467
+ (declaredTools !== undefined &&
1468
+ declaredTools !== null &&
1469
+ typeof declaredTools === "object" &&
1470
+ Object.keys(declaredTools).length === 0);
1471
+ if (declaresNoTools) {
1472
+ logger.debug(`[${this.providerName}] replayToolSteps "full" degraded to "marker": the request declares no tools`);
1473
+ options.replayToolSteps = "marker";
1474
+ }
1475
+ }
1435
1476
  /**
1436
1477
  * Build messages array for streaming operations - delegated to MessageBuilder
1437
1478
  * This is a protected helper method that providers can use to build messages
@@ -1441,6 +1482,7 @@ export class BaseProvider {
1441
1482
  * @returns Promise resolving to ModelMessage array ready for AI SDK
1442
1483
  */
1443
1484
  async buildMessagesForStream(options) {
1485
+ this.applyToolReplayDefault(options);
1444
1486
  return this.messageBuilder.buildMessagesForStream(options);
1445
1487
  }
1446
1488
  /**
@@ -1674,6 +1716,15 @@ export class BaseProvider {
1674
1716
  }
1675
1717
  }
1676
1718
  }
1719
+ /**
1720
+ * Mode 1 — synthesize the input text itself, with no LLM call.
1721
+ *
1722
+ * This is the documented default for `generate({ tts: { enabled: true } })`
1723
+ * and what every "speak this text" caller relies on, so a request that
1724
+ * looks like it wanted the model's reply (tools, a system prompt, message
1725
+ * history, a schema, conversation memory) is warned about, never rejected:
1726
+ * those options have no effect here, and the fix is `tts.mode: "response"`.
1727
+ */
1677
1728
  async handleDirectTTSSynthesis(options, startTime) {
1678
1729
  const textToSynthesize = options.prompt ?? options.input?.text ?? "";
1679
1730
  const baseResult = {
@@ -1686,14 +1737,17 @@ export class BaseProvider {
1686
1737
  if (!ttsOptions) {
1687
1738
  return this.enhanceResult(baseResult, options, startTime);
1688
1739
  }
1740
+ this.warnIfDirectTTSLooksLikeResponseRequest(options);
1689
1741
  const ttsStartTime = Date.now();
1690
1742
  const ttsProvider = ttsOptions.provider ?? options.provider ?? this.providerName;
1691
- const ttsTimeout = this.getTimeout(options);
1743
+ const ttsTimeout = this.resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider);
1744
+ const speechText = this.prepareTTSInput(textToSynthesize, ttsOptions);
1692
1745
  try {
1693
- baseResult.audio = await withTimeoutFn(() => TTSProcessor.synthesize(textToSynthesize, ttsProvider, ttsOptions), ttsTimeout, `TTS synthesis timed out after ${ttsTimeout}ms for provider "${ttsProvider}"`);
1746
+ baseResult.audio = await this.runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, options.abortSignal);
1694
1747
  baseResult.ttsMetadata = {
1695
1748
  attempted: true,
1696
1749
  success: true,
1750
+ mode: "direct",
1697
1751
  latency: Date.now() - ttsStartTime,
1698
1752
  };
1699
1753
  }
@@ -1703,6 +1757,7 @@ export class BaseProvider {
1703
1757
  baseResult.ttsMetadata = {
1704
1758
  attempted: true,
1705
1759
  success: false,
1760
+ mode: "direct",
1706
1761
  error,
1707
1762
  latency,
1708
1763
  };
@@ -1711,6 +1766,105 @@ export class BaseProvider {
1711
1766
  }
1712
1767
  return this.enhanceResult(baseResult, options, startTime);
1713
1768
  }
1769
+ /**
1770
+ * One warning when a direct-synthesis request carries options that only a
1771
+ * model call would honour and the caller never chose a mode. Silence is
1772
+ * the wrong answer here — the caller gets `usage` of zero and `content`
1773
+ * echoing their prompt, and would otherwise have to guess why.
1774
+ *
1775
+ * Only options the CALLER supplied count. `conversationMemoryConfig` is
1776
+ * attached by NeuroLink to every request on a memory-enabled instance, and
1777
+ * on such an instance `conversationMessages` is the stored history
1778
+ * NeuroLink injected itself — neither is something the caller set, and a
1779
+ * voice bot with memory used to see this warning on every turn after the
1780
+ * first for options it never passed.
1781
+ */
1782
+ warnIfDirectTTSLooksLikeResponseRequest(options) {
1783
+ const tts = options.tts;
1784
+ if (!tts || tts.mode !== undefined || tts.useAiResponse !== undefined) {
1785
+ return;
1786
+ }
1787
+ const memoryAttached = options.conversationMemoryConfig !== undefined;
1788
+ const llmShaped = [];
1789
+ if (options.tools && Object.keys(options.tools).length > 0) {
1790
+ llmShaped.push("tools");
1791
+ }
1792
+ if (options.systemPrompt) {
1793
+ llmShaped.push("systemPrompt");
1794
+ }
1795
+ if (!memoryAttached &&
1796
+ options.conversationMessages &&
1797
+ options.conversationMessages.length > 0) {
1798
+ llmShaped.push("conversationMessages");
1799
+ }
1800
+ if (options.conversationHistory && options.conversationHistory.length > 0) {
1801
+ llmShaped.push("conversationHistory");
1802
+ }
1803
+ if (options.schema) {
1804
+ llmShaped.push("schema");
1805
+ }
1806
+ if (llmShaped.length === 0) {
1807
+ return;
1808
+ }
1809
+ logger.warn(`[${this.providerName}] tts.enabled without tts.mode synthesizes the INPUT text and makes no model call, so ${llmShaped.join(", ")} will be ignored. Set tts.mode: "response" to speak the model's reply, or tts.mode: "direct" to silence this warning.`);
1810
+ }
1811
+ /**
1812
+ * Outer bound on one synthesis call: the provider's request timeout, raised
1813
+ * to whatever the handler itself may legitimately spend — `timeoutMs × (1 +
1814
+ * retries)` plus retry pauses, with the handler defaults (30 s, one retry)
1815
+ * standing in for a knob the caller left unset. Without the raise, a
1816
+ * handler's own retry loop could be cut off from outside one attempt in:
1817
+ * with the 60 s provider default, a first attempt that hung to its 30 s
1818
+ * abort left no room for the documented default retry.
1819
+ */
1820
+ resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider) {
1821
+ return Math.max(this.getTimeout(options), resolveTTSHandlerBudgetMs(ttsOptions, ttsProvider));
1822
+ }
1823
+ /**
1824
+ * Apply `tts.sanitize` (off by default — existing callers see byte-identical
1825
+ * input). Always runs when configured, regardless of the raw text's
1826
+ * length against the provider's cap: sanitizing can shrink text a great
1827
+ * deal (the default `codeBlocks: "phrase"` replaces a whole fenced block
1828
+ * with one short phrase, `urls: "hostname"` shortens every link), so a
1829
+ * reply that is over the cap raw can still fit after sanitizing — a coding
1830
+ * answer with a large code block is exactly the reply this option is for.
1831
+ * A reply still over the cap afterwards is rejected downstream by
1832
+ * `TTSProcessor.synthesize` as `TTS_TEXT_TOO_LONG`, same as today.
1833
+ */
1834
+ prepareTTSInput(text, ttsOptions) {
1835
+ const sanitizeOptions = resolveSpeechSanitizeOptions(ttsOptions.sanitize);
1836
+ if (!sanitizeOptions) {
1837
+ return text;
1838
+ }
1839
+ return prepareTextForSpeech(text, sanitizeOptions);
1840
+ }
1841
+ /**
1842
+ * One synthesis call under the outer budget. The handler receives a signal
1843
+ * that fires when that budget lapses or the caller's own `abortSignal`
1844
+ * does, so a handler with a retry loop (ElevenLabs) stops rather than
1845
+ * pausing and issuing a further billable request after the caller has
1846
+ * already been told the synthesis failed.
1847
+ */
1848
+ async runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, abortSignal) {
1849
+ const cancel = new AbortController();
1850
+ const signals = [cancel.signal];
1851
+ if (abortSignal) {
1852
+ signals.push(abortSignal);
1853
+ }
1854
+ if (ttsOptions.signal) {
1855
+ signals.push(ttsOptions.signal);
1856
+ }
1857
+ const signal = AbortSignal.any(signals);
1858
+ try {
1859
+ return await withTimeoutFn(() => TTSProcessor.synthesize(speechText, ttsProvider, {
1860
+ ...ttsOptions,
1861
+ signal,
1862
+ }), ttsTimeout, `TTS synthesis timed out after ${ttsTimeout}ms for provider "${ttsProvider}"`);
1863
+ }
1864
+ finally {
1865
+ cancel.abort();
1866
+ }
1867
+ }
1714
1868
  async handleVideoFrameGeneration(options, messages, model, startTime) {
1715
1869
  if (!hasVideoFrames(messages)) {
1716
1870
  return null;
@@ -1795,15 +1949,93 @@ export class BaseProvider {
1795
1949
  * synthesis so telemetry sees the model's own usage, and `enhanceResult`
1796
1950
  * runs last so analytics and evaluation observe the final content.
1797
1951
  */
1798
- async finalizeNativeGenerate(result, options, startTime) {
1952
+ async finalizeNativeGenerate(result, options, startTime, toolExecutionSummaries) {
1799
1953
  // onFinish is NOT fired here. `applyGenerateLifecycleMiddleware` folds it
1800
1954
  // into `options.middleware.middlewareConfig.lifecycle`, and the native
1801
1955
  // paths now wrap their model, so the lifecycle middleware fires it.
1802
1956
  // Firing it here too would deliver every callback twice.
1803
1957
  await this.recordPerformanceMetrics(result.usage, Date.now() - startTime);
1958
+ await this.persistGenerateToolSteps(options, toolExecutionSummaries);
1804
1959
  const synthesized = await this.synthesizeAIResponseIfNeeded(result, options);
1805
1960
  return this.enhanceResult(synthesized, options, startTime);
1806
1961
  }
1962
+ /**
1963
+ * Persist a generate() turn's tool steps into conversation memory.
1964
+ *
1965
+ * The stream loops store each step's calls and results as they run;
1966
+ * generate() never did, so the same session held tool rows for streamed
1967
+ * turns and none for generated ones, and a later turn could not replay
1968
+ * what generate() had called. Every native generate path closes out
1969
+ * through finalizeNativeGenerate, so this is the one place to do it.
1970
+ *
1971
+ * Gated on a real session id: the storage helper otherwise invents a
1972
+ * `session-<nanoid>` key, and on the in-memory backend that creates a
1973
+ * throwaway session per tool-using call — evicting real ones once
1974
+ * `maxSessions` is reached. A turn with no session has no history to
1975
+ * replay into, so there is nothing to persist.
1976
+ */
1977
+ async persistGenerateToolSteps(options, summaries) {
1978
+ if (!summaries || summaries.length === 0) {
1979
+ return;
1980
+ }
1981
+ const sessionId = options.context?.sessionId ??
1982
+ options.sessionId;
1983
+ if (typeof sessionId !== "string" || sessionId.length === 0) {
1984
+ return;
1985
+ }
1986
+ // One storage call per loop step, in step order. The stream loops store
1987
+ // as each step finishes, so a step's calls and results sit together;
1988
+ // storing a whole generate() turn as one batch replayed a sequential
1989
+ // lookup-then-send as a single parallel step, with the second call's
1990
+ // arguments appearing before the output they were built from.
1991
+ //
1992
+ // Summaries arrive in execution order, so a new batch starts whenever
1993
+ // the step changes rather than grouping by stepIndex value: a provider
1994
+ // can run the native loop more than once against the same summaries
1995
+ // array (a response_format rejection retry, a schema-validation retry),
1996
+ // and each run numbers its own steps from 0 — grouping by value would
1997
+ // merge run 1's step 0 with run 2's step 0 into one parallel batch.
1998
+ const batches = [];
1999
+ for (const summary of summaries) {
2000
+ const last = batches[batches.length - 1];
2001
+ if (last && last[0].stepIndex === summary.stepIndex) {
2002
+ last.push(summary);
2003
+ }
2004
+ else {
2005
+ batches.push([summary]);
2006
+ }
2007
+ }
2008
+ for (const stepSummaries of batches) {
2009
+ const toolCalls = stepSummaries.map((s) => ({
2010
+ toolCallId: s.toolCallId,
2011
+ toolName: s.toolName,
2012
+ args: s.input !== null &&
2013
+ typeof s.input === "object" &&
2014
+ !Array.isArray(s.input)
2015
+ ? s.input
2016
+ : {},
2017
+ ...(s.stepIndex !== undefined ? { stepIndex: s.stepIndex } : {}),
2018
+ timestamp: s.startTime,
2019
+ }));
2020
+ const toolResults = stepSummaries.map((s) => ({
2021
+ toolCallId: s.toolCallId,
2022
+ toolName: s.toolName,
2023
+ output: s.output,
2024
+ ...(s.error ? { error: s.error } : {}),
2025
+ ...(s.stepIndex !== undefined ? { stepIndex: s.stepIndex } : {}),
2026
+ timestamp: s.endTime,
2027
+ }));
2028
+ try {
2029
+ await this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date());
2030
+ }
2031
+ catch (error) {
2032
+ logger.warn(`[${this.providerName}] Failed to store tool executions`, {
2033
+ provider: this.providerName,
2034
+ error: error instanceof Error ? error.message : String(error),
2035
+ });
2036
+ }
2037
+ }
2038
+ }
1807
2039
  /**
1808
2040
  * Invoke a caller's `onFinish` for a native turn.
1809
2041
  *
@@ -1838,7 +2070,7 @@ export class BaseProvider {
1838
2070
  }
1839
2071
  }
1840
2072
  async synthesizeAIResponseIfNeeded(enhancedResult, options) {
1841
- if (!options.tts?.enabled || !options.tts?.useAiResponse) {
2073
+ if (!options.tts?.enabled || resolveTTSMode(options.tts) !== "response") {
1842
2074
  return enhancedResult;
1843
2075
  }
1844
2076
  const ttsOptions = options.tts;
@@ -1852,6 +2084,7 @@ export class BaseProvider {
1852
2084
  hasProvider: !!ttsProvider,
1853
2085
  ttsConfig: {
1854
2086
  enabled: options.tts?.enabled,
2087
+ mode: options.tts?.mode,
1855
2088
  useAiResponse: options.tts?.useAiResponse,
1856
2089
  },
1857
2090
  reason: !aiResponse
@@ -1863,19 +2096,22 @@ export class BaseProvider {
1863
2096
  ttsMetadata: {
1864
2097
  attempted: false,
1865
2098
  success: false,
2099
+ mode: "response",
1866
2100
  },
1867
2101
  };
1868
2102
  }
1869
2103
  const ttsStartTime = Date.now();
1870
- const ttsTimeout = this.getTimeout(options);
2104
+ const ttsTimeout = this.resolveTTSSynthesisTimeout(options, ttsOptions, ttsProvider);
2105
+ const speechText = this.prepareTTSInput(aiResponse, ttsOptions);
1871
2106
  try {
1872
- const ttsResult = await withTimeoutFn(() => TTSProcessor.synthesize(aiResponse, ttsProvider, ttsOptions), ttsTimeout, `TTS synthesis timed out after ${ttsTimeout}ms for provider "${ttsProvider}"`);
2107
+ const ttsResult = await this.runTTSSynthesis(speechText, ttsProvider, ttsOptions, ttsTimeout, options.abortSignal);
1873
2108
  return {
1874
2109
  ...enhancedResult,
1875
2110
  audio: ttsResult,
1876
2111
  ttsMetadata: {
1877
2112
  attempted: true,
1878
2113
  success: true,
2114
+ mode: "response",
1879
2115
  latency: Date.now() - ttsStartTime,
1880
2116
  },
1881
2117
  };
@@ -1890,6 +2126,7 @@ export class BaseProvider {
1890
2126
  ttsMetadata: {
1891
2127
  attempted: true,
1892
2128
  success: false,
2129
+ mode: "response",
1893
2130
  error,
1894
2131
  latency,
1895
2132
  },
@@ -2385,6 +2622,13 @@ export class BaseProvider {
2385
2622
  this.fileToolRootsBound = true;
2386
2623
  this.toolsManager.setDirectTools(policy.roots === null ? this.directTools : createDirectAgentTools(policy));
2387
2624
  }
2625
+ /**
2626
+ * Emit the start/end pair for a tool call a loop rejected before execute
2627
+ * (unknown tool, schema rejection). See ToolsManager.emitRejectedToolCall.
2628
+ */
2629
+ emitRejectedToolCall(toolName, error, toolCallId) {
2630
+ this.toolsManager.emitRejectedToolCall(toolName, error, toolCallId);
2631
+ }
2388
2632
  /**
2389
2633
  * Resolve roots for a provider called directly, without NeuroLink. A
2390
2634
  * policy NeuroLink already bound is left alone unless these options carry
@@ -11,6 +11,7 @@ import { tracers, withSpan } from "../telemetry/index.js";
11
11
  import { ConversationMemoryError } from "../types/index.js";
12
12
  import { buildContextFromPointer, getEffectiveTokenThreshold, } from "../utils/conversationMemory.js";
13
13
  import { logger } from "../utils/logger.js";
14
+ import { extractIsErrorText } from "../utils/toolResultStatus.js";
14
15
  export class ConversationMemoryManager {
15
16
  sessions = new Map();
16
17
  config;
@@ -421,6 +422,7 @@ export class ConversationMemoryManager {
421
422
  if (toolCallId) {
422
423
  toolNameById.set(toolCallId, toolName);
423
424
  }
425
+ const callStepIndex = toolCall.stepIndex;
424
426
  session.messages.push({
425
427
  id: randomUUID(),
426
428
  role: "tool_call",
@@ -428,6 +430,10 @@ export class ConversationMemoryManager {
428
430
  tool: toolName,
429
431
  ...(toolCallId ? { toolCallId } : {}),
430
432
  args: (toolCall.args ?? {}),
433
+ // Zero-based loop step, so replay can keep sequential steps apart.
434
+ ...(typeof callStepIndex === "number"
435
+ ? { metadata: { stepIndex: callStepIndex } }
436
+ : {}),
431
437
  timestamp,
432
438
  });
433
439
  }
@@ -448,6 +454,14 @@ export class ConversationMemoryManager {
448
454
  content = `[Serialization failed: ${error instanceof Error ? error.message : String(error)}]`;
449
455
  }
450
456
  }
457
+ // A result the tool RETURNED with `isError: true` (a thrown error that
458
+ // neurolink.executeTool converted, a breaker refusal, an MCP failure)
459
+ // is a failure too; recording it as success made the next turn's
460
+ // replay say "→ ok" for a call that failed.
461
+ const error = toolResult.error !== undefined && toolResult.error !== null
462
+ ? String(toolResult.error)
463
+ : extractIsErrorText(rawOutput);
464
+ const stepIndex = toolResult.stepIndex;
451
465
  session.messages.push({
452
466
  id: randomUUID(),
453
467
  role: "tool_result",
@@ -455,9 +469,10 @@ export class ConversationMemoryManager {
455
469
  tool: toolName,
456
470
  ...(toolCallId ? { toolCallId } : {}),
457
471
  result: {
458
- success: !toolResult.error,
459
- ...(toolResult.error ? { error: String(toolResult.error) } : {}),
472
+ success: error === undefined,
473
+ ...(error !== undefined ? { error } : {}),
460
474
  },
475
+ ...(typeof stepIndex === "number" ? { metadata: { stepIndex } } : {}),
461
476
  timestamp,
462
477
  });
463
478
  }
@@ -324,7 +324,7 @@ export function runAgenticLoop(adapter, initialConversation, options) {
324
324
  break;
325
325
  }
326
326
  }
327
- const request = adapter.buildStepRequest(conversation, step);
327
+ const request = await adapter.buildStepRequest(conversation, step);
328
328
  // A tool that just became callable starts clean. Its TOOL_NOT_FOUND
329
329
  // strikes were recorded against a name that genuinely did not resolve
330
330
  // yet, and the breaker is consulted before the lookup, so without this
@@ -1,6 +1,6 @@
1
1
  import { tracers, ATTR, withSpan } from "../../telemetry/index.js";
2
2
  import { logger } from "../../utils/logger.js";
3
- import { buildMessagesArray, buildMultimodalMessagesArray, } from "../../utils/messageBuilder.js";
3
+ import { buildMessagesArray, buildMultimodalModelMessages, } from "../../utils/messageBuilder.js";
4
4
  /**
5
5
  * Compute total content length across all messages for span attributes.
6
6
  */
@@ -127,11 +127,12 @@ export class MessageBuilder {
127
127
  enableEvaluation: options.enableEvaluation,
128
128
  context: options.context,
129
129
  conversationHistory: options.conversationMessages,
130
+ replayToolSteps: options.replayToolSteps,
130
131
  schema: options.schema,
131
132
  output: options.output,
132
133
  fileRegistry: options.fileRegistry,
133
134
  };
134
- messages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
135
+ messages = await buildMultimodalModelMessages(multimodalOptions, this.providerName, this.modelName);
135
136
  // Propagate any systemPrompt augmentation (e.g. inline-file
136
137
  // handling guidance from processUnifiedFilesArray) back to the
137
138
  // caller's options. Providers like GoogleVertex's native
@@ -163,6 +164,9 @@ export class MessageBuilder {
163
164
  }
164
165
  else {
165
166
  return {
167
+ // `tool` rides through untouched: a replayed tool step
168
+ // (replayToolSteps: "full") is a tool turn of tool-result parts,
169
+ // and its tool-call parts fall to the passthrough branch below.
166
170
  role: msg.role,
167
171
  content: msg.content.map((item) => {
168
172
  const itemProviderOptions = item.providerOptions;
@@ -272,11 +276,12 @@ export class MessageBuilder {
272
276
  context: options.context,
273
277
  conversationHistory: options
274
278
  .conversationMessages,
279
+ replayToolSteps: options.replayToolSteps,
275
280
  schema: options.schema,
276
281
  output: options.output,
277
282
  fileRegistry: options.fileRegistry,
278
283
  };
279
- messages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
284
+ messages = await buildMultimodalModelMessages(multimodalOptions, this.providerName, this.modelName);
280
285
  // Propagate any systemPrompt augmentation (e.g. inline-file
281
286
  // handling guidance from processUnifiedFilesArray) back to the
282
287
  // caller's options. Providers like GoogleVertex's native
@@ -308,6 +313,9 @@ export class MessageBuilder {
308
313
  }
309
314
  else {
310
315
  return {
316
+ // `tool` rides through untouched: a replayed tool step
317
+ // (replayToolSteps: "full") is a tool turn of tool-result parts,
318
+ // and its tool-call parts fall to the passthrough branch below.
311
319
  role: msg.role,
312
320
  content: msg.content.map((item) => {
313
321
  const itemProviderOptions = item.providerOptions;
@@ -14,7 +14,7 @@
14
14
  * @module core/modules/TelemetryHandler
15
15
  */
16
16
  import type { NeuroLink } from "../../neurolink.js";
17
- import type { StreamOptions, AIProviderName, AnalyticsData, EnhancedGenerateResult, EvaluationData, RawUsageObject, TextGenerationOptions, TokenUsage, TTSMetadata } from "../../types/index.js";
17
+ import type { StreamOptions, AIProviderName, AnalyticsData, EnhancedGenerateResult, EvaluationData, MemoryToolCallRecord, MemoryToolResultRecord, RawUsageObject, TextGenerationOptions, TokenUsage, TTSMetadata } from "../../types/index.js";
18
18
  /**
19
19
  * TelemetryHandler class - Handles analytics and telemetry for AI providers
20
20
  */
@@ -66,5 +66,5 @@ export declare class TelemetryHandler {
66
66
  /**
67
67
  * Handle tool execution storage if available
68
68
  */
69
- handleToolExecutionStorage(toolCalls: unknown[], toolResults: unknown[], options: TextGenerationOptions | StreamOptions, currentTime: Date): Promise<void>;
69
+ handleToolExecutionStorage(toolCalls: MemoryToolCallRecord[], toolResults: MemoryToolResultRecord[], options: TextGenerationOptions | StreamOptions, currentTime: Date): Promise<void>;
70
70
  }
@@ -28,6 +28,7 @@ export declare class ToolsManager {
28
28
  protected toolExecutor?: (toolName: string, params: unknown, options?: Record<string, unknown>) => Promise<unknown>;
29
29
  protected sessionId?: string;
30
30
  protected userId?: string;
31
+ private readonly eventWrappedExecutes;
31
32
  constructor(providerName: AIProviderName, directTools: Record<string, unknown>, neurolink?: NeuroLink | undefined, utilities?: ToolUtilities | undefined);
32
33
  /**
33
34
  * BZ-666: Wrap tool execute with output truncation to prevent
@@ -93,7 +94,41 @@ export declare class ToolsManager {
93
94
  * Set session context for MCP tools
94
95
  */
95
96
  setSessionContext(sessionId?: string, userId?: string): void;
97
+ /**
98
+ * Emit one tool event. A listener that throws must never reach the tool
99
+ * loop: it would turn a successful execution into a failed one (the model
100
+ * then retries a side effect that already happened) and, from inside the
101
+ * wrapper's catch, produce a second `tool:end` for the same call. Listener
102
+ * failures are logged and swallowed here instead.
103
+ */
96
104
  private emitToolEvent;
105
+ /**
106
+ * Wrap an execute function so one `tool:start` and exactly one `tool:end`
107
+ * (with the real duration) fire per invocation. Idempotent: a function this
108
+ * manager already wrapped is returned as is, so a tool that re-enters
109
+ * resolution (the stream → generate fallback re-resolves `options.tools`)
110
+ * never emits twice.
111
+ *
112
+ * A result the tool RETURNED with `isError: true` (MCP failure, breaker
113
+ * refusal, a thrown error already converted by `neurolink.executeTool`) is
114
+ * a failure: `tool:end` carries `success: false` and the extracted text,
115
+ * as the step-finish emitter this wrapper replaced did.
116
+ */
117
+ private wrapExecuteWithEvents;
118
+ /**
119
+ * Report a tool call the loop rejected BEFORE any execute ran — an unknown
120
+ * tool name, arguments the schema rejected, a breaker skip. Nothing else
121
+ * emits for such a call, so listeners (memory event capture, tool spans,
122
+ * error-rate metrics) saw zero attempts while the result list held the
123
+ * failures. One `tool:start` / `tool:end(success: false)` pair.
124
+ */
125
+ emitRejectedToolCall(toolName: string, error: string, toolCallId?: string): void;
126
+ /**
127
+ * Give per-call tools (`options.tools`) the same `tool:start` / `tool:end`
128
+ * emission that registered and MCP tools get from their executors. Tools
129
+ * without an `execute` pass through untouched.
130
+ */
131
+ wrapExternalToolsWithEvents(tools: Record<string, Tool>): Record<string, Tool>;
97
132
  /**
98
133
  * Set up tool executor for a provider to enable actual tool execution
99
134
  * @param sdk - The NeuroLinkSDK instance for tool execution