@juspay/neurolink 12.47.14 → 12.48.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
- package/dist/browser/neurolink.min.js +578 -565
- package/dist/cli/factories/commandFactory.d.ts +7 -0
- package/dist/cli/factories/commandFactory.js +26 -0
- package/dist/cli/loop/optionsSchema.js +9 -0
- package/dist/context/budgetChecker.js +54 -2
- package/dist/context/contextDecision.js +2 -0
- package/dist/context/toolPairRepair.d.ts +12 -0
- package/dist/context/toolPairRepair.js +29 -3
- package/dist/core/baseProvider.d.ts +92 -3
- package/dist/core/baseProvider.js +254 -10
- package/dist/core/conversationMemoryManager.js +17 -2
- package/dist/core/loopEngine.js +1 -1
- package/dist/core/modules/MessageBuilder.js +11 -3
- package/dist/core/modules/TelemetryHandler.d.ts +2 -2
- package/dist/core/modules/ToolsManager.d.ts +35 -0
- package/dist/core/modules/ToolsManager.js +159 -55
- package/dist/core/nativeGenerateLoop.d.ts +58 -1
- package/dist/core/nativeGenerateLoop.js +277 -3
- package/dist/core/redisConversationMemoryManager.js +10 -3
- package/dist/core/resolveRequestKind.d.ts +24 -4
- package/dist/core/resolveRequestKind.js +31 -4
- package/dist/core/toolExecutionRecorder.d.ts +6 -0
- package/dist/core/toolExecutionRecorder.js +24 -2
- package/dist/core/toolRoutingDecision.js +1 -0
- package/dist/factories/mediaHandlerCatalog.js +5 -0
- package/dist/factories/providerDescriptors.js +15 -0
- package/dist/files/fileTools.d.ts +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.js +9 -2
- package/dist/memory/condensationPrompt.d.ts +8 -0
- package/dist/memory/condensationPrompt.js +29 -0
- package/dist/memory/hippocampusInitializer.d.ts +14 -2
- package/dist/memory/hippocampusInitializer.js +64 -3
- package/dist/neurolink.d.ts +252 -3
- package/dist/neurolink.js +898 -43
- package/dist/providers/amazonBedrock/client.js +6 -0
- package/dist/providers/amazonSagemaker.js +14 -1
- package/dist/providers/anthropic/client.js +148 -19
- package/dist/providers/anthropic/loopAdapter.js +2 -2
- package/dist/providers/googleAiStudio/client.js +2 -1
- package/dist/providers/openaiChatCompletionsBase.js +103 -23
- package/dist/providers/systemOneDecision.d.ts +6 -0
- package/dist/providers/systemOneDecision.js +14 -63
- package/dist/providers/typesafe.d.ts +6 -0
- package/dist/providers/typesafe.js +6 -0
- package/dist/rag/retrieval/searchDecision.js +1 -0
- package/dist/routing/classifierRouter.js +6 -5
- package/dist/routing/classifierStrategies.js +8 -1
- package/dist/types/classifierRouter.d.ts +16 -3
- package/dist/types/common.d.ts +2 -0
- package/dist/types/config.d.ts +9 -0
- package/dist/types/context.d.ts +8 -1
- package/dist/types/conversation.d.ts +87 -0
- package/dist/types/decision.d.ts +210 -1
- package/dist/types/decision.js +9 -1
- package/dist/types/dispatch.d.ts +11 -4
- package/dist/types/generate.d.ts +118 -54
- package/dist/types/loopEngine.d.ts +7 -2
- package/dist/types/memory.d.ts +177 -9
- package/dist/types/openaiCompatible.d.ts +11 -3
- package/dist/types/stream.d.ts +13 -16
- package/dist/types/stt.d.ts +58 -1
- package/dist/types/tools.d.ts +14 -0
- package/dist/types/tools.js +8 -0
- package/dist/types/tts.d.ts +108 -0
- package/dist/types/voice.d.ts +41 -4
- package/dist/utils/decisionAnswers.d.ts +31 -1
- package/dist/utils/decisionAnswers.js +114 -0
- package/dist/utils/decisionLimits.d.ts +39 -0
- package/dist/utils/decisionLimits.js +116 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.js +9 -0
- package/dist/utils/messageBuilder.d.ts +45 -1
- package/dist/utils/messageBuilder.js +393 -16
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/utils/speechText.d.ts +41 -0
- package/dist/utils/speechText.js +364 -0
- package/dist/utils/toolResultStatus.d.ts +26 -0
- package/dist/utils/toolResultStatus.js +73 -0
- package/dist/utils/ttsProcessor.d.ts +29 -0
- package/dist/utils/ttsProcessor.js +84 -2
- package/dist/voice/index.d.ts +1 -0
- package/dist/voice/index.js +3 -0
- package/dist/voice/providers/ElevenLabsSTT.d.ts +42 -0
- package/dist/voice/providers/ElevenLabsSTT.js +351 -0
- package/dist/voice/providers/ElevenLabsTTS.d.ts +81 -1
- package/dist/voice/providers/ElevenLabsTTS.js +415 -74
- package/docs-site/static/search-index.json +48 -33
- package/package.json +5 -2
- package/dist/utils/toolEndEmitter.d.ts +0 -25
- package/dist/utils/toolEndEmitter.js +0 -64
package/dist/utils/logger.d.ts
CHANGED
|
@@ -30,6 +30,12 @@ declare class NeuroLinkLogger {
|
|
|
30
30
|
* @param emitter - The event emitter instance
|
|
31
31
|
*/
|
|
32
32
|
setEventEmitter(emitter: LogEventEmitter): void;
|
|
33
|
+
/**
|
|
34
|
+
* The process-wide sink currently receiving log events, or undefined when
|
|
35
|
+
* none is set. Lets code that must construct an instance without changing
|
|
36
|
+
* the sink capture it beforehand and put it back afterwards.
|
|
37
|
+
*/
|
|
38
|
+
getEventEmitter(): LogEventEmitter | undefined;
|
|
33
39
|
/**
|
|
34
40
|
* Clears the event emitter reference.
|
|
35
41
|
* Should be called when a NeuroLink instance is disposed to prevent memory
|
|
@@ -279,6 +285,7 @@ export declare const logger: {
|
|
|
279
285
|
getLogs: (level?: LogLevel) => LogEntry[];
|
|
280
286
|
clearLogs: () => void;
|
|
281
287
|
setEventEmitter: (emitter: LogEventEmitter) => void;
|
|
288
|
+
getEventEmitter: () => LogEventEmitter | undefined;
|
|
282
289
|
clearEventEmitter: (ifEmitter?: LogEventEmitter) => void;
|
|
283
290
|
runInInstanceScope: <T>(instanceId: string, fn: () => T) => T;
|
|
284
291
|
getInstanceScope: () => string | undefined;
|
package/dist/utils/logger.js
CHANGED
|
@@ -113,6 +113,14 @@ class NeuroLinkLogger {
|
|
|
113
113
|
setEventEmitter(emitter) {
|
|
114
114
|
this.eventEmitter = emitter;
|
|
115
115
|
}
|
|
116
|
+
/**
|
|
117
|
+
* The process-wide sink currently receiving log events, or undefined when
|
|
118
|
+
* none is set. Lets code that must construct an instance without changing
|
|
119
|
+
* the sink capture it beforehand and put it back afterwards.
|
|
120
|
+
*/
|
|
121
|
+
getEventEmitter() {
|
|
122
|
+
return this.eventEmitter;
|
|
123
|
+
}
|
|
116
124
|
/**
|
|
117
125
|
* Clears the event emitter reference.
|
|
118
126
|
* Should be called when a NeuroLink instance is disposed to prevent memory
|
|
@@ -634,6 +642,7 @@ export const logger = {
|
|
|
634
642
|
getLogs: (level) => neuroLinkLogger.getLogs(level),
|
|
635
643
|
clearLogs: () => neuroLinkLogger.clearLogs(),
|
|
636
644
|
setEventEmitter: (emitter) => neuroLinkLogger.setEventEmitter(emitter),
|
|
645
|
+
getEventEmitter: () => neuroLinkLogger.getEventEmitter(),
|
|
637
646
|
clearEventEmitter: (ifEmitter) => neuroLinkLogger.clearEventEmitter(ifEmitter),
|
|
638
647
|
// Per-instance routing (see NeuroLinkLogger.runInInstanceScope)
|
|
639
648
|
runInInstanceScope: (instanceId, fn) => neuroLinkLogger.runInInstanceScope(instanceId, fn),
|
|
@@ -1,10 +1,47 @@
|
|
|
1
1
|
import type { GenerateOptions, MultimodalChatMessage, StreamOptions, TextGenerationOptions, VisionImageOutputFormat } from "../types/index.js";
|
|
2
|
-
import type { ModelMessage } from "../types/index.js";
|
|
2
|
+
import type { ModelMessage, ToolHistoryRow, ToolReplayMode } from "../types/index.js";
|
|
3
3
|
/**
|
|
4
4
|
* Type-safe conversion from MultimodalChatMessage[] to ModelMessage[]
|
|
5
5
|
* Filters out invalid content and ensures strict ModelMessage contract compliance
|
|
6
6
|
*/
|
|
7
7
|
export declare function convertToModelMessages(messages: MultimodalChatMessage[]): ModelMessage[];
|
|
8
|
+
/**
|
|
9
|
+
* Resolve the replay mode for a request: the per-request option, else the
|
|
10
|
+
* default. The instance-level `conversationMemory.replayToolSteps` is folded
|
|
11
|
+
* into the option by BaseProvider before the builder runs.
|
|
12
|
+
*/
|
|
13
|
+
export declare const resolveToolReplayMode: (options: {
|
|
14
|
+
replayToolSteps?: ToolReplayMode;
|
|
15
|
+
} | undefined) => ToolReplayMode;
|
|
16
|
+
/**
|
|
17
|
+
* Replay a session's tool rows into the prompt while the caller converts the
|
|
18
|
+
* regular rows its own way.
|
|
19
|
+
*
|
|
20
|
+
* Usage: normalise the history with `rows()`, then for each row call
|
|
21
|
+
* `consume(row)`; a `true` return means the row was a tool row and is now
|
|
22
|
+
* held. Before converting a non-tool row (and once after the last row) call
|
|
23
|
+
* `flush(messages, nextRole)`, which appends whatever the held tool rows
|
|
24
|
+
* amount to under `mode`:
|
|
25
|
+
*
|
|
26
|
+
* - `"full"`: paired tool-call / tool-result turns, one pair per step.
|
|
27
|
+
* - `"marker"`: markers are folded into the assistant text — returned for
|
|
28
|
+
* the caller to prepend when the next row is the assistant that answered
|
|
29
|
+
* after those tools; otherwise appended to the assistant turn just
|
|
30
|
+
* emitted, or emitted as a marker-only assistant turn when there is none.
|
|
31
|
+
* - `"off"`: dropped.
|
|
32
|
+
*
|
|
33
|
+
* The caller owns the conversion of regular rows so that the text and the
|
|
34
|
+
* multimodal builders keep their own (different) handling of them.
|
|
35
|
+
*/
|
|
36
|
+
export declare const createToolStepReplay: (mode: ToolReplayMode) => {
|
|
37
|
+
rows: <TRow extends ToolHistoryRow>(rows: ReadonlyArray<TRow>) => TRow[];
|
|
38
|
+
consume: (row: ToolHistoryRow) => boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Append the held steps to `messages`. Returns marker text the caller must
|
|
41
|
+
* prepend to the assistant row it is about to convert, or undefined.
|
|
42
|
+
*/
|
|
43
|
+
flush: (messages: Array<ModelMessage | MultimodalChatMessage>, nextRole: string | undefined) => string | undefined;
|
|
44
|
+
};
|
|
8
45
|
/**
|
|
9
46
|
* Build a properly formatted message array for AI providers
|
|
10
47
|
* Combines system prompt, conversation history, and current user prompt
|
|
@@ -43,6 +80,13 @@ export declare function processUnifiedFilesArray(options: GenerateOptions, maxSi
|
|
|
43
80
|
* Detects when images are present and routes through provider adapter
|
|
44
81
|
*/
|
|
45
82
|
export declare function buildMultimodalMessagesArray(options: GenerateOptions, provider: string, model: string): Promise<MultimodalChatMessage[]>;
|
|
83
|
+
/**
|
|
84
|
+
* The multimodal builder proper. Returns model messages so a session's tool
|
|
85
|
+
* steps can be replayed as real tool turns (`replayToolSteps: "full"`) on a
|
|
86
|
+
* turn that also carries files or images; `buildMultimodalMessagesArray`
|
|
87
|
+
* wraps this for callers on the older `MultimodalChatMessage[]` shape.
|
|
88
|
+
*/
|
|
89
|
+
export declare function buildMultimodalModelMessages(options: GenerateOptions, provider: string, model: string, replayMode?: ToolReplayMode): Promise<Array<MultimodalChatMessage | ModelMessage>>;
|
|
46
90
|
/**
|
|
47
91
|
* Transcode any image in `input.images` that no vision provider accepts,
|
|
48
92
|
* rewriting the entry in place as a PNG data URI.
|
|
@@ -24,6 +24,8 @@ import { redactUrlForError } from "./logSanitize.js";
|
|
|
24
24
|
import { PDFImageConverter, PDFProcessor } from "./pdfProcessor.js";
|
|
25
25
|
import { urlDownloadRateLimiter } from "./rateLimiter.js";
|
|
26
26
|
import { estimateTokens } from "./tokenEstimation.js";
|
|
27
|
+
import { extractIsErrorTextFromContent } from "./toolResultStatus.js";
|
|
28
|
+
import { isRepairPlaceholder } from "../context/toolPairRepair.js";
|
|
27
29
|
// ---------------------------------------------------------------------------
|
|
28
30
|
// SDK-7: Lightweight file-type inference helpers for budget estimation
|
|
29
31
|
// These avoid calling the full FileDetector pipeline — they only need to
|
|
@@ -383,8 +385,337 @@ function toModelMessage(message) {
|
|
|
383
385
|
content: message.content,
|
|
384
386
|
};
|
|
385
387
|
}
|
|
386
|
-
return null; //
|
|
388
|
+
return null; // tool_call / tool_result rows are replayed by createToolStepReplay
|
|
387
389
|
}
|
|
390
|
+
const isToolRow = (row) => row.role === "tool_call" || row.role === "tool_result";
|
|
391
|
+
const DEFAULT_TOOL_REPLAY_MODE = "marker";
|
|
392
|
+
/**
|
|
393
|
+
* Resolve the replay mode for a request: the per-request option, else the
|
|
394
|
+
* default. The instance-level `conversationMemory.replayToolSteps` is folded
|
|
395
|
+
* into the option by BaseProvider before the builder runs.
|
|
396
|
+
*/
|
|
397
|
+
export const resolveToolReplayMode = (options) => options?.replayToolSteps ?? DEFAULT_TOOL_REPLAY_MODE;
|
|
398
|
+
/**
|
|
399
|
+
* Put a tool batch after the user turn it belongs to.
|
|
400
|
+
*
|
|
401
|
+
* The in-memory memory backend appends `tool_call` / `tool_result` rows the
|
|
402
|
+
* moment a step finishes, but the turn's own user and assistant rows only
|
|
403
|
+
* when the turn ends — so a streamed or generated tool turn is stored as
|
|
404
|
+
* `tool_call, tool_result, user, assistant`. The Redis backend buffers the
|
|
405
|
+
* rows and writes `user, tool_call, tool_result, assistant`. Replayed as
|
|
406
|
+
* stored, the in-memory order would put the assistant's tool call BEFORE
|
|
407
|
+
* the question that prompted it (and, at the start of a session, before any
|
|
408
|
+
* user turn at all — which Anthropic rejects).
|
|
409
|
+
*
|
|
410
|
+
* The tell is unambiguous: a batch that was written before its user row is
|
|
411
|
+
* immediately followed by a `user` row and is NOT immediately preceded by
|
|
412
|
+
* one. A batch in the Redis order always follows its user row (or the
|
|
413
|
+
* previous step of the same turn). An interrupted Redis turn —
|
|
414
|
+
* `user, tool_call, tool_result, user` — is preceded by a user row and is
|
|
415
|
+
* left where it is.
|
|
416
|
+
*/
|
|
417
|
+
/**
|
|
418
|
+
* A row that is the user's own turn — not a summary the memory layer emits
|
|
419
|
+
* under the user role, nor a pinned skill activation. Only a genuine user turn
|
|
420
|
+
* can be the question a tool batch answered.
|
|
421
|
+
*/
|
|
422
|
+
const isGenuineUserTurn = (row) => row?.role === "user" &&
|
|
423
|
+
row.metadata?.isSummary !== true &&
|
|
424
|
+
row.metadata?.isSkill !== true;
|
|
425
|
+
const normalizeToolRowOrder = (rows) => {
|
|
426
|
+
const out = [];
|
|
427
|
+
let i = 0;
|
|
428
|
+
while (i < rows.length) {
|
|
429
|
+
if (!isToolRow(rows[i])) {
|
|
430
|
+
out.push(rows[i]);
|
|
431
|
+
i++;
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
let j = i;
|
|
435
|
+
while (j < rows.length && isToolRow(rows[j])) {
|
|
436
|
+
j++;
|
|
437
|
+
}
|
|
438
|
+
const batch = rows.slice(i, j);
|
|
439
|
+
const previous = out[out.length - 1];
|
|
440
|
+
const next = rows[j];
|
|
441
|
+
if (isGenuineUserTurn(next) && !isGenuineUserTurn(previous)) {
|
|
442
|
+
out.push(next, ...batch);
|
|
443
|
+
i = j + 1;
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
out.push(...batch);
|
|
447
|
+
i = j;
|
|
448
|
+
}
|
|
449
|
+
return out;
|
|
450
|
+
};
|
|
451
|
+
/**
|
|
452
|
+
* The id alphabet every provider accepts for a replayed tool-call id.
|
|
453
|
+
* Anthropic enforces `^[a-zA-Z0-9_-]+$`; an OpenAI-compatible backend may
|
|
454
|
+
* emit ids like `functions.x:0` that a later provider in the same session
|
|
455
|
+
* would reject, so those are re-keyed to synthetic ids.
|
|
456
|
+
*/
|
|
457
|
+
const REPLAY_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
|
|
458
|
+
/**
|
|
459
|
+
* Hands out tool-call ids for one replay of one history. Synthetic ids are
|
|
460
|
+
* unique across the WHOLE history (a per-turn counter restarted at `hist_0_0`
|
|
461
|
+
* on every turn, so two id-less turns replayed the same id twice and
|
|
462
|
+
* Anthropic rejected the request), a recorded id is kept the first time it
|
|
463
|
+
* is seen and re-keyed after that, and an id outside the accepted alphabet is
|
|
464
|
+
* re-keyed too. Call and result always share the id they are given.
|
|
465
|
+
*/
|
|
466
|
+
const createReplayIdAllocator = () => {
|
|
467
|
+
const seen = new Set();
|
|
468
|
+
let counter = 0;
|
|
469
|
+
const fresh = () => {
|
|
470
|
+
let id = `hist_${counter++}`;
|
|
471
|
+
while (seen.has(id)) {
|
|
472
|
+
id = `hist_${counter++}`;
|
|
473
|
+
}
|
|
474
|
+
seen.add(id);
|
|
475
|
+
return id;
|
|
476
|
+
};
|
|
477
|
+
return (recorded) => {
|
|
478
|
+
if (recorded && REPLAY_ID_PATTERN.test(recorded) && !seen.has(recorded)) {
|
|
479
|
+
seen.add(recorded);
|
|
480
|
+
return recorded;
|
|
481
|
+
}
|
|
482
|
+
return fresh();
|
|
483
|
+
};
|
|
484
|
+
};
|
|
485
|
+
/**
|
|
486
|
+
* Classify a stored result. `result.success === false` / `result.error` is
|
|
487
|
+
* what the stores write for a thrown or rejected call; an `isError: true`
|
|
488
|
+
* payload in the content covers rows written before the stores learned to
|
|
489
|
+
* flag those, and MCP failures a tool returned rather than threw.
|
|
490
|
+
*/
|
|
491
|
+
const resolveReplayOutcome = (result) => {
|
|
492
|
+
if (isRepairPlaceholder(result)) {
|
|
493
|
+
return { outcome: "unknown" };
|
|
494
|
+
}
|
|
495
|
+
if (result.result?.success === false || result.result?.error) {
|
|
496
|
+
return { outcome: "error", error: result.result?.error };
|
|
497
|
+
}
|
|
498
|
+
const isErrorText = extractIsErrorTextFromContent(result.content);
|
|
499
|
+
if (isErrorText !== undefined) {
|
|
500
|
+
return { outcome: "error", error: isErrorText };
|
|
501
|
+
}
|
|
502
|
+
return { outcome: "ok" };
|
|
503
|
+
};
|
|
504
|
+
/**
|
|
505
|
+
* Group a run of tool rows into steps and pair every call with its result.
|
|
506
|
+
*
|
|
507
|
+
* A batch is a maximal run of `tool_call` rows followed by the `tool_result`
|
|
508
|
+
* rows that answer them — a step with parallel calls is persisted as every
|
|
509
|
+
* call, then every result, so adjacency is not pairing. Pairing is by
|
|
510
|
+
* `toolCallId`; rows written before that field existed pair by position
|
|
511
|
+
* within the batch. A batch whose rows carry `metadata.stepIndex` (the loops
|
|
512
|
+
* record it; generate() persisted a whole turn as one batch) is then split
|
|
513
|
+
* into one step per index, so sequential calls are not replayed as a
|
|
514
|
+
* parallel one. Whatever is left unpaired — a call whose result never
|
|
515
|
+
* arrived, or the reverse — is dropped: both Anthropic and OpenAI reject an
|
|
516
|
+
* orphaned tool_use / tool_result with a 400. A call `repairToolPairs`
|
|
517
|
+
* invented for an orphaned result is dropped too: the model never made it.
|
|
518
|
+
*/
|
|
519
|
+
const pairToolSteps = (rows, allocateId) => {
|
|
520
|
+
const steps = [];
|
|
521
|
+
let calls = [];
|
|
522
|
+
let results = [];
|
|
523
|
+
const flush = () => {
|
|
524
|
+
if (calls.length === 0 && results.length === 0) {
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
const paired = [];
|
|
528
|
+
// Each result pairs at most once: a duplicated toolCallId, or a batch
|
|
529
|
+
// that mixes rows with and without ids, must never replay the same
|
|
530
|
+
// result twice — Anthropic rejects duplicate tool_result blocks.
|
|
531
|
+
const used = new Set();
|
|
532
|
+
const unmatchedResults = results.filter((r) => !r.toolCallId);
|
|
533
|
+
for (const call of calls) {
|
|
534
|
+
const result = call.toolCallId
|
|
535
|
+
? results.find((r) => !used.has(r) && r.toolCallId === call.toolCallId)
|
|
536
|
+
: unmatchedResults.shift();
|
|
537
|
+
if (!result) {
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
used.add(result);
|
|
541
|
+
if (isRepairPlaceholder(call)) {
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
const output = typeof result.content === "string" && result.content.length > 0
|
|
545
|
+
? result.content
|
|
546
|
+
: "(no output)";
|
|
547
|
+
const { outcome, error } = resolveReplayOutcome(result);
|
|
548
|
+
// A failed step's output is often `null` — the error lives only in
|
|
549
|
+
// `result.error` — so the model saw "null" and could not tell why the
|
|
550
|
+
// call failed, or that it had. Carry the error text into the output.
|
|
551
|
+
const replayedOutput = outcome === "error" && error && !output.includes(error)
|
|
552
|
+
? output === "null" || output === "(no output)"
|
|
553
|
+
? JSON.stringify({ error })
|
|
554
|
+
: `${output}\n[error: ${error}]`
|
|
555
|
+
: output;
|
|
556
|
+
paired.push({
|
|
557
|
+
step: {
|
|
558
|
+
callId: allocateId(call.toolCallId),
|
|
559
|
+
toolName: call.tool ?? result.tool ?? "unknown",
|
|
560
|
+
args: call.args ?? {},
|
|
561
|
+
output: replayedOutput,
|
|
562
|
+
outcome,
|
|
563
|
+
},
|
|
564
|
+
stepIndex: call.metadata?.stepIndex,
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
if (paired.length > 0) {
|
|
568
|
+
// Split by recorded step; pairs without one form a single step.
|
|
569
|
+
const byStep = new Map();
|
|
570
|
+
for (const entry of paired) {
|
|
571
|
+
const key = typeof entry.stepIndex === "number" ? entry.stepIndex : undefined;
|
|
572
|
+
const bucket = byStep.get(key);
|
|
573
|
+
if (bucket) {
|
|
574
|
+
bucket.push(entry.step);
|
|
575
|
+
}
|
|
576
|
+
else {
|
|
577
|
+
byStep.set(key, [entry.step]);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
const ordered = [...byStep.entries()].sort(([a], [b]) => {
|
|
581
|
+
if (a === undefined) {
|
|
582
|
+
return b === undefined ? 0 : -1;
|
|
583
|
+
}
|
|
584
|
+
return b === undefined ? 1 : a - b;
|
|
585
|
+
});
|
|
586
|
+
for (const [, group] of ordered) {
|
|
587
|
+
steps.push(group);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
calls = [];
|
|
591
|
+
results = [];
|
|
592
|
+
};
|
|
593
|
+
for (const row of rows) {
|
|
594
|
+
if (row.role === "tool_call") {
|
|
595
|
+
// A call after results closes the previous step.
|
|
596
|
+
if (results.length > 0) {
|
|
597
|
+
flush();
|
|
598
|
+
}
|
|
599
|
+
calls.push(row);
|
|
600
|
+
}
|
|
601
|
+
else if (row.role === "tool_result") {
|
|
602
|
+
results.push(row);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
flush();
|
|
606
|
+
return steps;
|
|
607
|
+
};
|
|
608
|
+
/**
|
|
609
|
+
* The compact text form of one replayed tool call. `→ unknown` is what a
|
|
610
|
+
* call whose result was lost to compaction gets: never `ok`, since nothing
|
|
611
|
+
* says it succeeded.
|
|
612
|
+
*/
|
|
613
|
+
const toolStepMarker = (step) => `[called ${step.toolName} → ${step.outcome}]`;
|
|
614
|
+
/**
|
|
615
|
+
* Text of an assistant turn whose tool steps are replayed as markers: the
|
|
616
|
+
* marker lines, then the turn's own text.
|
|
617
|
+
*/
|
|
618
|
+
const prependMarkers = (markers, text) => text.length > 0 ? `${markers}\n${text}` : markers;
|
|
619
|
+
/**
|
|
620
|
+
* A message the legacy `MultimodalChatMessage[]` shape can carry: not a tool
|
|
621
|
+
* turn, and not an assistant turn made of tool-call parts. Both come only from
|
|
622
|
+
* `"full"` replay, which that shape cannot express.
|
|
623
|
+
*/
|
|
624
|
+
const isMultimodalChatMessage = (message) => message.role !== "tool" &&
|
|
625
|
+
!(Array.isArray(message.content) &&
|
|
626
|
+
message.content.some((part) => part.type === "tool-call"));
|
|
627
|
+
/**
|
|
628
|
+
* Full replay of one step: an assistant turn of tool-call parts followed by a
|
|
629
|
+
* tool turn of tool-result parts, in the shape every native converter already
|
|
630
|
+
* round-trips mid-turn.
|
|
631
|
+
*/
|
|
632
|
+
const toolStepMessages = (step) => {
|
|
633
|
+
const callParts = step.map((s) => ({
|
|
634
|
+
type: "tool-call",
|
|
635
|
+
toolCallId: s.callId,
|
|
636
|
+
toolName: s.toolName,
|
|
637
|
+
input: s.args,
|
|
638
|
+
}));
|
|
639
|
+
const resultParts = step.map((s) => ({
|
|
640
|
+
type: "tool-result",
|
|
641
|
+
toolCallId: s.callId,
|
|
642
|
+
toolName: s.toolName,
|
|
643
|
+
output: s.output,
|
|
644
|
+
}));
|
|
645
|
+
const assistant = {
|
|
646
|
+
role: "assistant",
|
|
647
|
+
content: callParts,
|
|
648
|
+
};
|
|
649
|
+
const tool = { role: "tool", content: resultParts };
|
|
650
|
+
return [assistant, tool];
|
|
651
|
+
};
|
|
652
|
+
/**
|
|
653
|
+
* Replay a session's tool rows into the prompt while the caller converts the
|
|
654
|
+
* regular rows its own way.
|
|
655
|
+
*
|
|
656
|
+
* Usage: normalise the history with `rows()`, then for each row call
|
|
657
|
+
* `consume(row)`; a `true` return means the row was a tool row and is now
|
|
658
|
+
* held. Before converting a non-tool row (and once after the last row) call
|
|
659
|
+
* `flush(messages, nextRole)`, which appends whatever the held tool rows
|
|
660
|
+
* amount to under `mode`:
|
|
661
|
+
*
|
|
662
|
+
* - `"full"`: paired tool-call / tool-result turns, one pair per step.
|
|
663
|
+
* - `"marker"`: markers are folded into the assistant text — returned for
|
|
664
|
+
* the caller to prepend when the next row is the assistant that answered
|
|
665
|
+
* after those tools; otherwise appended to the assistant turn just
|
|
666
|
+
* emitted, or emitted as a marker-only assistant turn when there is none.
|
|
667
|
+
* - `"off"`: dropped.
|
|
668
|
+
*
|
|
669
|
+
* The caller owns the conversion of regular rows so that the text and the
|
|
670
|
+
* multimodal builders keep their own (different) handling of them.
|
|
671
|
+
*/
|
|
672
|
+
export const createToolStepReplay = (mode) => {
|
|
673
|
+
let held = [];
|
|
674
|
+
// One allocator per replay: ids must be unique across the whole history,
|
|
675
|
+
// not merely within one flush.
|
|
676
|
+
const allocateId = createReplayIdAllocator();
|
|
677
|
+
return {
|
|
678
|
+
rows: (rows) => (mode === "off" ? [...rows] : normalizeToolRowOrder(rows)),
|
|
679
|
+
consume: (row) => {
|
|
680
|
+
if (!isToolRow(row)) {
|
|
681
|
+
return false;
|
|
682
|
+
}
|
|
683
|
+
if (mode !== "off") {
|
|
684
|
+
held.push(row);
|
|
685
|
+
}
|
|
686
|
+
return true;
|
|
687
|
+
},
|
|
688
|
+
flush: (messages, nextRole) => {
|
|
689
|
+
if (held.length === 0) {
|
|
690
|
+
return undefined;
|
|
691
|
+
}
|
|
692
|
+
const steps = pairToolSteps(held, allocateId);
|
|
693
|
+
held = [];
|
|
694
|
+
if (steps.length === 0) {
|
|
695
|
+
return undefined;
|
|
696
|
+
}
|
|
697
|
+
if (mode === "full") {
|
|
698
|
+
for (const step of steps) {
|
|
699
|
+
messages.push(...toolStepMessages(step));
|
|
700
|
+
}
|
|
701
|
+
return undefined;
|
|
702
|
+
}
|
|
703
|
+
const markers = steps.flat().map(toolStepMarker);
|
|
704
|
+
if (nextRole === "assistant") {
|
|
705
|
+
return markers.join("\n");
|
|
706
|
+
}
|
|
707
|
+
const last = messages[messages.length - 1];
|
|
708
|
+
if (last &&
|
|
709
|
+
last.role === "assistant" &&
|
|
710
|
+
typeof last.content === "string") {
|
|
711
|
+
last.content = `${last.content}\n${markers.join("\n")}`;
|
|
712
|
+
return undefined;
|
|
713
|
+
}
|
|
714
|
+
messages.push({ role: "assistant", content: markers.join("\n") });
|
|
715
|
+
return undefined;
|
|
716
|
+
},
|
|
717
|
+
};
|
|
718
|
+
};
|
|
388
719
|
/**
|
|
389
720
|
* Format CSV metadata for LLM consumption
|
|
390
721
|
*/
|
|
@@ -469,15 +800,27 @@ export async function buildMessagesArray(options) {
|
|
|
469
800
|
},
|
|
470
801
|
});
|
|
471
802
|
}
|
|
472
|
-
// Add conversation history if available
|
|
473
|
-
//
|
|
803
|
+
// Add conversation history if available. Regular rows convert one-to-one;
|
|
804
|
+
// tool_call / tool_result rows are replayed per `replayToolSteps`.
|
|
474
805
|
if (hasConversationHistory && options.conversationMessages) {
|
|
475
|
-
|
|
806
|
+
const replay = createToolStepReplay(resolveToolReplayMode(options));
|
|
807
|
+
for (const chatMessage of replay.rows(options.conversationMessages)) {
|
|
808
|
+
if (replay.consume(chatMessage)) {
|
|
809
|
+
continue;
|
|
810
|
+
}
|
|
811
|
+
const markers = replay.flush(messages, chatMessage.role);
|
|
476
812
|
const coreMessage = toModelMessage(chatMessage);
|
|
477
813
|
if (coreMessage) {
|
|
814
|
+
if (markers && coreMessage.role === "assistant") {
|
|
815
|
+
coreMessage.content = prependMarkers(markers, coreMessage.content);
|
|
816
|
+
}
|
|
478
817
|
messages.push(coreMessage);
|
|
479
818
|
}
|
|
819
|
+
else if (markers) {
|
|
820
|
+
messages.push({ role: "assistant", content: markers });
|
|
821
|
+
}
|
|
480
822
|
}
|
|
823
|
+
replay.flush(messages, undefined);
|
|
481
824
|
}
|
|
482
825
|
// Add current user prompt (required)
|
|
483
826
|
// Handle both TextGenerationOptions (prompt field) and StreamOptions (input.text field)
|
|
@@ -1403,6 +1746,20 @@ function buildMultimodalSystemPrompt(options, hasPDFFiles) {
|
|
|
1403
1746
|
* Detects when images are present and routes through provider adapter
|
|
1404
1747
|
*/
|
|
1405
1748
|
export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
1749
|
+
// The legacy shape has no tool role, so a `"full"` replay degrades to
|
|
1750
|
+
// markers here; callers that can carry tool turns (the core MessageBuilder)
|
|
1751
|
+
// use buildMultimodalModelMessages directly.
|
|
1752
|
+
const mode = resolveToolReplayMode(options);
|
|
1753
|
+
const messages = await buildMultimodalModelMessages(options, provider, model, mode === "full" ? "marker" : mode);
|
|
1754
|
+
return messages.filter(isMultimodalChatMessage);
|
|
1755
|
+
}
|
|
1756
|
+
/**
|
|
1757
|
+
* The multimodal builder proper. Returns model messages so a session's tool
|
|
1758
|
+
* steps can be replayed as real tool turns (`replayToolSteps: "full"`) on a
|
|
1759
|
+
* turn that also carries files or images; `buildMultimodalMessagesArray`
|
|
1760
|
+
* wraps this for callers on the older `MultimodalChatMessage[]` shape.
|
|
1761
|
+
*/
|
|
1762
|
+
export async function buildMultimodalModelMessages(options, provider, model, replayMode = resolveToolReplayMode(options)) {
|
|
1406
1763
|
// Media-only callers (avatar / music / video) may omit `input` entirely.
|
|
1407
1764
|
// Normalise to an empty object so all sub-functions can access input.*
|
|
1408
1765
|
// without defensive null checks on every field access.
|
|
@@ -1487,15 +1844,11 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
1487
1844
|
if (inp.files) {
|
|
1488
1845
|
inp.files = [];
|
|
1489
1846
|
}
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
role: msg.role,
|
|
1496
|
-
content: msg.content,
|
|
1497
|
-
...(msgProviderOptions && { providerOptions: msgProviderOptions }),
|
|
1498
|
-
};
|
|
1847
|
+
// Already model messages, providerOptions included; the replay mode
|
|
1848
|
+
// decided here has to travel with them.
|
|
1849
|
+
return buildMessagesArray({
|
|
1850
|
+
...options,
|
|
1851
|
+
replayToolSteps: replayMode,
|
|
1499
1852
|
});
|
|
1500
1853
|
}
|
|
1501
1854
|
// Validate provider supports vision
|
|
@@ -1518,11 +1871,16 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
1518
1871
|
},
|
|
1519
1872
|
});
|
|
1520
1873
|
}
|
|
1521
|
-
// Add conversation history if available
|
|
1874
|
+
// Add conversation history if available. tool_call / tool_result rows are
|
|
1875
|
+
// replayed per `replayMode`; the rest convert as before.
|
|
1522
1876
|
const hasConversationHistory = options.conversationHistory && options.conversationHistory.length > 0;
|
|
1523
1877
|
if (hasConversationHistory && options.conversationHistory) {
|
|
1524
|
-
|
|
1525
|
-
|
|
1878
|
+
const replay = createToolStepReplay(replayMode);
|
|
1879
|
+
for (const msg of replay.rows(options.conversationHistory)) {
|
|
1880
|
+
if (replay.consume(msg)) {
|
|
1881
|
+
continue;
|
|
1882
|
+
}
|
|
1883
|
+
const markers = replay.flush(messages, msg.role);
|
|
1526
1884
|
if (msg.role === "user" ||
|
|
1527
1885
|
msg.role === "assistant" ||
|
|
1528
1886
|
msg.role === "system") {
|
|
@@ -1540,6 +1898,9 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
1540
1898
|
typeof item.text === "string");
|
|
1541
1899
|
if (textParts.length === 0) {
|
|
1542
1900
|
// All content was tool_use/tool_result/non-text — skip message
|
|
1901
|
+
if (markers) {
|
|
1902
|
+
messages.push({ role: "assistant", content: markers });
|
|
1903
|
+
}
|
|
1543
1904
|
continue;
|
|
1544
1905
|
}
|
|
1545
1906
|
// Check if any retained text part carries providerOptions
|
|
@@ -1560,15 +1921,31 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
1560
1921
|
}
|
|
1561
1922
|
// Skip empty string content to avoid Claude API rejection
|
|
1562
1923
|
if (sanitizedContent === "") {
|
|
1924
|
+
if (markers) {
|
|
1925
|
+
messages.push({ role: "assistant", content: markers });
|
|
1926
|
+
}
|
|
1563
1927
|
continue;
|
|
1564
1928
|
}
|
|
1929
|
+
if (markers && msg.role === "assistant") {
|
|
1930
|
+
sanitizedContent =
|
|
1931
|
+
typeof sanitizedContent === "string"
|
|
1932
|
+
? prependMarkers(markers, sanitizedContent)
|
|
1933
|
+
: [
|
|
1934
|
+
{ type: "text", text: markers },
|
|
1935
|
+
...sanitizedContent,
|
|
1936
|
+
];
|
|
1937
|
+
}
|
|
1565
1938
|
messages.push({
|
|
1566
1939
|
role: msg.role,
|
|
1567
1940
|
content: sanitizedContent,
|
|
1568
1941
|
...(providerOptions && { providerOptions }),
|
|
1569
1942
|
});
|
|
1570
1943
|
}
|
|
1944
|
+
else if (markers) {
|
|
1945
|
+
messages.push({ role: "assistant", content: markers });
|
|
1946
|
+
}
|
|
1571
1947
|
}
|
|
1948
|
+
replay.flush(messages, undefined);
|
|
1572
1949
|
}
|
|
1573
1950
|
// Handle multimodal content
|
|
1574
1951
|
try {
|
|
@@ -71,6 +71,7 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
71
71
|
officeOptions: import("../index.js").OfficeProcessorOptions | undefined;
|
|
72
72
|
systemPrompt: string | undefined;
|
|
73
73
|
conversationHistory: import("../index.js").ChatMessage[] | undefined;
|
|
74
|
+
replayToolSteps: import("../index.js").ToolReplayMode | undefined;
|
|
74
75
|
provider: string;
|
|
75
76
|
model: string;
|
|
76
77
|
temperature: number | undefined;
|
|
@@ -75,6 +75,7 @@ export function buildMultimodalOptions(options, providerName, modelName) {
|
|
|
75
75
|
officeOptions: options.officeOptions,
|
|
76
76
|
systemPrompt: options.systemPrompt,
|
|
77
77
|
conversationHistory: options.conversationMessages,
|
|
78
|
+
replayToolSteps: options.replayToolSteps,
|
|
78
79
|
provider: providerName,
|
|
79
80
|
model: modelName,
|
|
80
81
|
temperature: options.temperature,
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text preparation for speech synthesis.
|
|
3
|
+
*
|
|
4
|
+
* Model output is written for a screen: markdown structure, URLs, emoji.
|
|
5
|
+
* Read aloud, every one of those is noise — "asterisk asterisk", a 90-character
|
|
6
|
+
* URL spelled letter by letter, "grinning face with smiling eyes". This module
|
|
7
|
+
* rewrites such text into what a voice should actually say.
|
|
8
|
+
*
|
|
9
|
+
* Every regular expression here is linear-time by construction: no nested
|
|
10
|
+
* quantifiers, no lazy scan to a closing token, and no unbounded greedy class
|
|
11
|
+
* followed by a required literal (the `[ \t]+\n` shape that goes quadratic on
|
|
12
|
+
* a long run of spaces). Where a closing token is required, the scan between
|
|
13
|
+
* the two is bounded AND excludes the opener's own character, so an unclosed
|
|
14
|
+
* opener fails at the next opener rather than walking the bound — a run of
|
|
15
|
+
* `[` costs one step per character, not one thousand. Block-level structure
|
|
16
|
+
* (fenced code, lists, tables, headings, blockquotes) is a line pass that
|
|
17
|
+
* toggles state, so an unterminated fence costs one scan rather than a
|
|
18
|
+
* backtracking search.
|
|
19
|
+
*
|
|
20
|
+
* @module utils/speechText
|
|
21
|
+
*/
|
|
22
|
+
import type { SpeechSanitizeOptions } from "../types/index.js";
|
|
23
|
+
/**
|
|
24
|
+
* Resolve a `TTSOptions.sanitize` value into options, or `undefined` when the
|
|
25
|
+
* pass is off. `true` means the defaults; an object tunes them.
|
|
26
|
+
*/
|
|
27
|
+
export declare function resolveSpeechSanitizeOptions(sanitize: boolean | SpeechSanitizeOptions | undefined): SpeechSanitizeOptions | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* Rewrite `text` into what a TTS voice should say.
|
|
30
|
+
*
|
|
31
|
+
* Pure and deterministic: the same input and options always yield the same
|
|
32
|
+
* output, and nothing here reads the environment. Runs in time linear in the
|
|
33
|
+
* length of `text`.
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* ```typescript
|
|
37
|
+
* prepareTextForSpeech("## Title\n\nSee **this** at https://docs.example.com/x 🎉");
|
|
38
|
+
* // → "Title\n\nSee this at docs.example.com"
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
export declare function prepareTextForSpeech(text: string, opts?: SpeechSanitizeOptions): string;
|