@juspay/neurolink 9.5.3 → 9.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +29 -25
- package/dist/agent/directTools.d.ts +5 -5
- package/dist/cli/commands/config.d.ts +9 -9
- package/dist/cli/commands/serve.d.ts +37 -0
- package/dist/cli/commands/serve.js +302 -229
- package/dist/cli/commands/setup-anthropic.d.ts +2 -2
- package/dist/cli/commands/setup-azure.d.ts +2 -2
- package/dist/cli/commands/setup-bedrock.d.ts +2 -2
- package/dist/cli/commands/setup-gcp.d.ts +2 -2
- package/dist/cli/commands/setup-google-ai.d.ts +2 -2
- package/dist/cli/commands/setup-huggingface.d.ts +2 -2
- package/dist/cli/commands/setup-mistral.d.ts +2 -2
- package/dist/cli/commands/setup-openai.d.ts +2 -2
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/factories/commandFactory.js +16 -2
- package/dist/cli/loop/optionsSchema.d.ts +2 -2
- package/dist/cli/loop/session.d.ts +4 -0
- package/dist/cli/loop/session.js +49 -4
- package/dist/cli/utils/interactiveSetup.d.ts +4 -4
- package/dist/config/conversationMemory.d.ts +2 -0
- package/dist/config/conversationMemory.js +5 -5
- package/dist/constants/contextWindows.d.ts +46 -0
- package/dist/constants/contextWindows.js +156 -0
- package/dist/context/budgetChecker.d.ts +18 -0
- package/dist/context/budgetChecker.js +71 -0
- package/dist/context/contextCompactor.d.ts +22 -0
- package/dist/context/contextCompactor.js +106 -0
- package/dist/context/effectiveHistory.d.ts +52 -0
- package/dist/context/effectiveHistory.js +105 -0
- package/dist/context/errorDetection.d.ts +14 -0
- package/dist/context/errorDetection.js +124 -0
- package/dist/context/fileSummarizationService.d.ts +54 -0
- package/dist/context/fileSummarizationService.js +255 -0
- package/dist/context/fileSummarizer.d.ts +56 -0
- package/dist/context/fileSummarizer.js +145 -0
- package/dist/context/fileTokenBudget.d.ts +53 -0
- package/dist/context/fileTokenBudget.js +127 -0
- package/dist/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/context/prompts/summarizationPrompt.js +110 -0
- package/dist/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/context/stages/fileReadDeduplicator.js +66 -0
- package/dist/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/context/stages/slidingWindowTruncator.js +42 -0
- package/dist/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/context/stages/structuredSummarizer.js +49 -0
- package/dist/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/context/stages/toolOutputPruner.js +52 -0
- package/dist/context/summarizationEngine.d.ts +45 -0
- package/dist/context/summarizationEngine.js +110 -0
- package/dist/context/toolOutputLimits.d.ts +17 -0
- package/dist/context/toolOutputLimits.js +84 -0
- package/dist/context/toolPairRepair.d.ts +16 -0
- package/dist/context/toolPairRepair.js +66 -0
- package/dist/core/conversationMemoryManager.d.ts +5 -15
- package/dist/core/conversationMemoryManager.js +15 -75
- package/dist/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/core/modules/MessageBuilder.js +2 -0
- package/dist/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/core/modules/TelemetryHandler.js +3 -3
- package/dist/core/modules/ToolsManager.d.ts +2 -2
- package/dist/core/redisConversationMemoryManager.d.ts +8 -14
- package/dist/core/redisConversationMemoryManager.js +69 -78
- package/dist/factories/providerFactory.d.ts +2 -2
- package/dist/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/files/fileReferenceRegistry.js +1543 -0
- package/dist/files/fileTools.d.ts +423 -0
- package/dist/files/fileTools.js +449 -0
- package/dist/files/index.d.ts +14 -0
- package/dist/files/index.js +13 -0
- package/dist/files/streamingReader.d.ts +93 -0
- package/dist/files/streamingReader.js +321 -0
- package/dist/files/types.d.ts +23 -0
- package/dist/files/types.js +23 -0
- package/dist/image-gen/imageGenTools.d.ts +2 -2
- package/dist/image-gen/types.d.ts +12 -12
- package/dist/lib/agent/directTools.d.ts +7 -7
- package/dist/lib/config/conversationMemory.d.ts +2 -0
- package/dist/lib/config/conversationMemory.js +5 -5
- package/dist/lib/constants/contextWindows.d.ts +46 -0
- package/dist/lib/constants/contextWindows.js +157 -0
- package/dist/lib/context/budgetChecker.d.ts +18 -0
- package/dist/lib/context/budgetChecker.js +72 -0
- package/dist/lib/context/contextCompactor.d.ts +22 -0
- package/dist/lib/context/contextCompactor.js +107 -0
- package/dist/lib/context/effectiveHistory.d.ts +52 -0
- package/dist/lib/context/effectiveHistory.js +106 -0
- package/dist/lib/context/errorDetection.d.ts +14 -0
- package/dist/lib/context/errorDetection.js +125 -0
- package/dist/lib/context/fileSummarizationService.d.ts +54 -0
- package/dist/lib/context/fileSummarizationService.js +256 -0
- package/dist/lib/context/fileSummarizer.d.ts +56 -0
- package/dist/lib/context/fileSummarizer.js +146 -0
- package/dist/lib/context/fileTokenBudget.d.ts +53 -0
- package/dist/lib/context/fileTokenBudget.js +128 -0
- package/dist/lib/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/lib/context/prompts/summarizationPrompt.js +111 -0
- package/dist/lib/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/lib/context/stages/fileReadDeduplicator.js +67 -0
- package/dist/lib/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/lib/context/stages/slidingWindowTruncator.js +43 -0
- package/dist/lib/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/lib/context/stages/structuredSummarizer.js +50 -0
- package/dist/lib/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/lib/context/stages/toolOutputPruner.js +53 -0
- package/dist/lib/context/summarizationEngine.d.ts +45 -0
- package/dist/lib/context/summarizationEngine.js +111 -0
- package/dist/lib/context/toolOutputLimits.d.ts +17 -0
- package/dist/lib/context/toolOutputLimits.js +85 -0
- package/dist/lib/context/toolPairRepair.d.ts +16 -0
- package/dist/lib/context/toolPairRepair.js +67 -0
- package/dist/lib/core/conversationMemoryManager.d.ts +5 -15
- package/dist/lib/core/conversationMemoryManager.js +15 -75
- package/dist/lib/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/lib/core/modules/MessageBuilder.js +2 -0
- package/dist/lib/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/lib/core/modules/TelemetryHandler.js +3 -3
- package/dist/lib/core/modules/ToolsManager.d.ts +2 -2
- package/dist/lib/core/redisConversationMemoryManager.d.ts +8 -14
- package/dist/lib/core/redisConversationMemoryManager.js +69 -78
- package/dist/lib/factories/providerFactory.d.ts +2 -2
- package/dist/lib/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/lib/files/fileReferenceRegistry.js +1544 -0
- package/dist/lib/files/fileTools.d.ts +423 -0
- package/dist/lib/files/fileTools.js +450 -0
- package/dist/lib/files/index.d.ts +14 -0
- package/dist/lib/files/index.js +14 -0
- package/dist/lib/files/streamingReader.d.ts +93 -0
- package/dist/lib/files/streamingReader.js +322 -0
- package/dist/lib/files/types.d.ts +23 -0
- package/dist/lib/files/types.js +24 -0
- package/dist/lib/image-gen/imageGenTools.d.ts +2 -2
- package/dist/lib/image-gen/types.d.ts +12 -12
- package/dist/lib/memory/mem0Initializer.d.ts +2 -2
- package/dist/lib/neurolink.d.ts +61 -2
- package/dist/lib/neurolink.js +619 -307
- package/dist/lib/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/lib/processors/archive/ArchiveProcessor.js +1309 -0
- package/dist/lib/processors/archive/index.d.ts +33 -0
- package/dist/lib/processors/archive/index.js +43 -0
- package/dist/lib/processors/base/types.d.ts +70 -64
- package/dist/lib/processors/base/types.js +6 -0
- package/dist/lib/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/lib/processors/cli/fileProcessorCli.js +5 -5
- package/dist/lib/processors/config/mimeTypes.js +25 -0
- package/dist/lib/processors/config/sizeLimits.d.ts +52 -40
- package/dist/lib/processors/config/sizeLimits.js +56 -44
- package/dist/lib/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/lib/processors/document/ExcelProcessor.js +72 -1
- package/dist/lib/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/lib/processors/document/PptxProcessor.js +158 -0
- package/dist/lib/processors/document/index.d.ts +1 -0
- package/dist/lib/processors/document/index.js +6 -0
- package/dist/lib/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/lib/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/lib/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/lib/processors/index.d.ts +8 -2
- package/dist/lib/processors/index.js +5 -2
- package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/lib/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/lib/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/lib/processors/media/AudioProcessor.js +708 -0
- package/dist/lib/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/lib/processors/media/VideoProcessor.js +992 -0
- package/dist/lib/processors/media/index.d.ts +27 -0
- package/dist/lib/processors/media/index.js +37 -0
- package/dist/lib/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/lib/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/lib/processors/registry/index.d.ts +1 -1
- package/dist/lib/processors/registry/index.js +1 -1
- package/dist/lib/processors/registry/types.d.ts +2 -2
- package/dist/lib/providers/googleAiStudio.d.ts +34 -0
- package/dist/lib/providers/googleAiStudio.js +267 -397
- package/dist/lib/providers/googleVertex.d.ts +55 -1
- package/dist/lib/providers/googleVertex.js +452 -719
- package/dist/lib/providers/sagemaker/detection.d.ts +6 -6
- package/dist/lib/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/lib/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/lib/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/lib/rag/document/loaders.d.ts +6 -71
- package/dist/lib/rag/document/loaders.js +5 -5
- package/dist/lib/rag/graphRag/graphRAG.js +26 -9
- package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/lib/rag/metadata/metadataExtractor.js +6 -3
- package/dist/lib/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/lib/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/lib/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/lib/rag/pipeline/contextAssembly.js +6 -3
- package/dist/lib/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/lib/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/lib/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/lib/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/lib/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/lib/rag/types.d.ts +3 -3
- package/dist/lib/sdk/toolRegistration.d.ts +2 -2
- package/dist/lib/server/middleware/cache.d.ts +2 -2
- package/dist/lib/server/middleware/rateLimit.d.ts +2 -2
- package/dist/lib/server/routes/mcpRoutes.js +277 -249
- package/dist/lib/server/routes/memoryRoutes.js +287 -281
- package/dist/lib/server/utils/validation.d.ts +10 -10
- package/dist/lib/session/globalSessionState.d.ts +2 -2
- package/dist/lib/telemetry/telemetryService.d.ts +2 -2
- package/dist/lib/types/common.d.ts +39 -0
- package/dist/lib/types/contextTypes.d.ts +255 -0
- package/dist/lib/types/contextTypes.js +0 -2
- package/dist/lib/types/conversation.d.ts +62 -0
- package/dist/lib/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/lib/types/conversationMemoryInterface.js +7 -0
- package/dist/lib/types/fileReferenceTypes.d.ts +222 -0
- package/dist/lib/types/fileReferenceTypes.js +9 -0
- package/dist/lib/types/fileTypes.d.ts +26 -3
- package/dist/lib/types/generateTypes.d.ts +22 -1
- package/dist/lib/types/index.d.ts +4 -5
- package/dist/lib/types/index.js +8 -10
- package/dist/lib/types/modelTypes.d.ts +2 -2
- package/dist/lib/types/processorTypes.d.ts +597 -0
- package/dist/lib/types/processorTypes.js +91 -0
- package/dist/lib/types/ragTypes.d.ts +481 -0
- package/dist/lib/types/ragTypes.js +8 -0
- package/dist/lib/types/sdkTypes.d.ts +17 -18
- package/dist/lib/types/streamTypes.d.ts +11 -1
- package/dist/lib/utils/async/retry.d.ts +2 -2
- package/dist/lib/utils/async/withTimeout.js +3 -1
- package/dist/lib/utils/conversationMemory.d.ts +12 -6
- package/dist/lib/utils/conversationMemory.js +76 -36
- package/dist/lib/utils/fileDetector.d.ts +62 -0
- package/dist/lib/utils/fileDetector.js +1014 -14
- package/dist/lib/utils/json/safeParse.d.ts +2 -2
- package/dist/lib/utils/messageBuilder.js +806 -153
- package/dist/lib/utils/modelChoices.d.ts +2 -2
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/lib/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/lib/utils/rateLimiter.d.ts +2 -2
- package/dist/lib/utils/sanitizers/filename.d.ts +4 -4
- package/dist/lib/utils/sanitizers/svg.d.ts +2 -2
- package/dist/lib/utils/thinkingConfig.d.ts +6 -6
- package/dist/lib/utils/tokenEstimation.d.ts +68 -0
- package/dist/lib/utils/tokenEstimation.js +113 -0
- package/dist/lib/utils/tokenUtils.d.ts +4 -4
- package/dist/lib/utils/ttsProcessor.d.ts +2 -2
- package/dist/lib/workflow/config.d.ts +150 -150
- package/dist/memory/mem0Initializer.d.ts +2 -2
- package/dist/neurolink.d.ts +61 -2
- package/dist/neurolink.js +619 -307
- package/dist/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/processors/archive/ArchiveProcessor.js +1308 -0
- package/dist/processors/archive/index.d.ts +33 -0
- package/dist/processors/archive/index.js +42 -0
- package/dist/processors/base/types.d.ts +70 -64
- package/dist/processors/base/types.js +6 -0
- package/dist/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/processors/cli/fileProcessorCli.js +5 -5
- package/dist/processors/config/mimeTypes.js +25 -0
- package/dist/processors/config/sizeLimits.d.ts +52 -40
- package/dist/processors/config/sizeLimits.js +56 -44
- package/dist/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/processors/document/ExcelProcessor.js +72 -1
- package/dist/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/processors/document/PptxProcessor.js +157 -0
- package/dist/processors/document/index.d.ts +1 -0
- package/dist/processors/document/index.js +6 -0
- package/dist/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/processors/index.d.ts +8 -2
- package/dist/processors/index.js +5 -2
- package/dist/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/processors/media/AudioProcessor.js +707 -0
- package/dist/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/processors/media/VideoProcessor.js +991 -0
- package/dist/processors/media/ffprobe-static.d.ts +4 -0
- package/dist/processors/media/index.d.ts +27 -0
- package/dist/processors/media/index.js +36 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/processors/registry/index.d.ts +1 -1
- package/dist/processors/registry/index.js +1 -1
- package/dist/processors/registry/types.d.ts +2 -2
- package/dist/providers/googleAiStudio.d.ts +34 -0
- package/dist/providers/googleAiStudio.js +267 -397
- package/dist/providers/googleVertex.d.ts +55 -1
- package/dist/providers/googleVertex.js +452 -719
- package/dist/providers/sagemaker/detection.d.ts +6 -6
- package/dist/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/rag/document/loaders.d.ts +6 -71
- package/dist/rag/document/loaders.js +5 -5
- package/dist/rag/graphRag/graphRAG.js +26 -9
- package/dist/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/rag/metadata/metadataExtractor.js +6 -3
- package/dist/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/rag/pipeline/contextAssembly.js +6 -3
- package/dist/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/rag/types.d.ts +3 -3
- package/dist/sdk/toolRegistration.d.ts +2 -2
- package/dist/server/middleware/cache.d.ts +2 -2
- package/dist/server/middleware/rateLimit.d.ts +2 -2
- package/dist/server/routes/mcpRoutes.js +277 -249
- package/dist/server/routes/memoryRoutes.js +287 -281
- package/dist/server/utils/validation.d.ts +4 -4
- package/dist/session/globalSessionState.d.ts +2 -2
- package/dist/telemetry/telemetryService.d.ts +2 -2
- package/dist/types/common.d.ts +39 -0
- package/dist/types/contextTypes.d.ts +255 -0
- package/dist/types/contextTypes.js +0 -2
- package/dist/types/conversation.d.ts +62 -0
- package/dist/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/types/conversationMemoryInterface.js +6 -0
- package/dist/types/fileReferenceTypes.d.ts +222 -0
- package/dist/types/fileReferenceTypes.js +8 -0
- package/dist/types/fileTypes.d.ts +26 -3
- package/dist/types/generateTypes.d.ts +22 -1
- package/dist/types/index.d.ts +4 -5
- package/dist/types/index.js +8 -10
- package/dist/types/processorTypes.d.ts +597 -0
- package/dist/types/processorTypes.js +90 -0
- package/dist/types/ragTypes.d.ts +481 -0
- package/dist/types/ragTypes.js +7 -0
- package/dist/types/sdkTypes.d.ts +17 -18
- package/dist/types/streamTypes.d.ts +11 -1
- package/dist/utils/async/retry.d.ts +2 -2
- package/dist/utils/async/withTimeout.js +3 -1
- package/dist/utils/conversationMemory.d.ts +12 -6
- package/dist/utils/conversationMemory.js +76 -36
- package/dist/utils/fileDetector.d.ts +62 -0
- package/dist/utils/fileDetector.js +1014 -14
- package/dist/utils/json/safeParse.d.ts +2 -2
- package/dist/utils/messageBuilder.js +806 -153
- package/dist/utils/modelChoices.d.ts +2 -2
- package/dist/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/utils/rateLimiter.d.ts +2 -2
- package/dist/utils/sanitizers/filename.d.ts +4 -4
- package/dist/utils/sanitizers/svg.d.ts +2 -2
- package/dist/utils/thinkingConfig.d.ts +6 -6
- package/dist/utils/tokenEstimation.d.ts +68 -0
- package/dist/utils/tokenEstimation.js +112 -0
- package/dist/utils/tokenUtils.d.ts +4 -4
- package/dist/utils/ttsProcessor.d.ts +2 -2
- package/dist/workflow/config.d.ts +104 -104
- package/package.json +18 -6
- package/dist/lib/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/lib/utils/conversationMemoryUtils.js +0 -138
- package/dist/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/utils/conversationMemoryUtils.js +0 -137
|
@@ -6,11 +6,11 @@ import { AIProviderName } from "../constants/enums.js";
|
|
|
6
6
|
/**
|
|
7
7
|
* Model choice for CLI prompts (inquirer format)
|
|
8
8
|
*/
|
|
9
|
-
export
|
|
9
|
+
export type ModelChoice = {
|
|
10
10
|
name: string;
|
|
11
11
|
value: string;
|
|
12
12
|
description?: string;
|
|
13
|
-
}
|
|
13
|
+
};
|
|
14
14
|
/**
|
|
15
15
|
* Default models per provider (first choice/recommended)
|
|
16
16
|
*/
|
|
@@ -46,7 +46,7 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
46
46
|
text: string;
|
|
47
47
|
images: (string | Buffer<ArrayBufferLike> | import("../index.js").ImageWithAltText)[] | undefined;
|
|
48
48
|
content: import("../index.js").Content[] | undefined;
|
|
49
|
-
files: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
49
|
+
files: (string | Buffer<ArrayBufferLike> | import("../index.js").FileWithMetadata)[] | undefined;
|
|
50
50
|
csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
51
51
|
pdfFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
52
52
|
};
|
|
@@ -64,4 +64,5 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
64
64
|
enableAnalytics: boolean | undefined;
|
|
65
65
|
enableEvaluation: boolean | undefined;
|
|
66
66
|
context: import("../index.js").UnknownRecord | undefined;
|
|
67
|
+
fileRegistry: unknown;
|
|
67
68
|
};
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
/**
|
|
10
10
|
* Configuration options for the rate limiter
|
|
11
11
|
*/
|
|
12
|
-
export
|
|
12
|
+
export type RateLimiterConfig = {
|
|
13
13
|
/** Maximum tokens (downloads) allowed per interval */
|
|
14
14
|
maxTokens: number;
|
|
15
15
|
/** Refill interval in milliseconds */
|
|
@@ -20,7 +20,7 @@ export interface RateLimiterConfig {
|
|
|
20
20
|
maxQueueSize: number;
|
|
21
21
|
/** Timeout for queued requests in milliseconds */
|
|
22
22
|
queueTimeoutMs: number;
|
|
23
|
-
}
|
|
23
|
+
};
|
|
24
24
|
/**
|
|
25
25
|
* Token Bucket Rate Limiter
|
|
26
26
|
*
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
/**
|
|
13
13
|
* Options for filename sanitization.
|
|
14
14
|
*/
|
|
15
|
-
export
|
|
15
|
+
export type SanitizeFileNameOptions = {
|
|
16
16
|
/** Maximum length for the filename (default: 255) */
|
|
17
17
|
maxLength?: number;
|
|
18
18
|
/** Replacement character for invalid chars (default: '_') */
|
|
@@ -21,16 +21,16 @@ export interface SanitizeFileNameOptions {
|
|
|
21
21
|
blockDangerousExtensions?: boolean;
|
|
22
22
|
/** Whether to allow hidden files starting with dot (default: false) */
|
|
23
23
|
allowHiddenFiles?: boolean;
|
|
24
|
-
}
|
|
24
|
+
};
|
|
25
25
|
/**
|
|
26
26
|
* Options for display name sanitization.
|
|
27
27
|
*/
|
|
28
|
-
export
|
|
28
|
+
export type SanitizeDisplayNameOptions = {
|
|
29
29
|
/** Maximum length for the name (default: 100) */
|
|
30
30
|
maxLength?: number;
|
|
31
31
|
/** Whether to allow unicode characters (default: true) */
|
|
32
32
|
allowUnicode?: boolean;
|
|
33
|
-
}
|
|
33
|
+
};
|
|
34
34
|
/**
|
|
35
35
|
* Sanitize a filename for safe filesystem storage.
|
|
36
36
|
* Removes characters that are invalid on various operating systems.
|
|
@@ -19,14 +19,14 @@
|
|
|
19
19
|
/**
|
|
20
20
|
* Result of SVG sanitization including metadata about removed content
|
|
21
21
|
*/
|
|
22
|
-
export
|
|
22
|
+
export type SvgSanitizationResult = {
|
|
23
23
|
/** Sanitized SVG content */
|
|
24
24
|
content: string;
|
|
25
25
|
/** Items that were removed during sanitization */
|
|
26
26
|
removedItems: string[];
|
|
27
27
|
/** Whether any content was modified */
|
|
28
28
|
wasModified: boolean;
|
|
29
|
-
}
|
|
29
|
+
};
|
|
30
30
|
/**
|
|
31
31
|
* Sanitize SVG content by removing dangerous elements and attributes.
|
|
32
32
|
* Uses OWASP-compliant allowlist approach with regex-based parsing.
|
|
@@ -11,32 +11,32 @@ export type ThinkingLevel = "minimal" | "low" | "medium" | "high";
|
|
|
11
11
|
/**
|
|
12
12
|
* ThinkingConfig interface matching the SDK's expected structure
|
|
13
13
|
*/
|
|
14
|
-
export
|
|
14
|
+
export type ThinkingConfig = {
|
|
15
15
|
enabled?: boolean;
|
|
16
16
|
type?: "enabled" | "disabled";
|
|
17
17
|
/** Token budget for thinking (Anthropic models: 5000-100000) */
|
|
18
18
|
budgetTokens?: number;
|
|
19
19
|
/** Thinking level for Gemini 3 models */
|
|
20
20
|
thinkingLevel?: ThinkingLevel;
|
|
21
|
-
}
|
|
21
|
+
};
|
|
22
22
|
/**
|
|
23
23
|
* Options for creating a thinkingConfig from CLI-style options
|
|
24
24
|
*/
|
|
25
|
-
export
|
|
25
|
+
export type CreateThinkingConfigOptions = {
|
|
26
26
|
/** Enable thinking mode */
|
|
27
27
|
thinking?: boolean;
|
|
28
28
|
/** Token budget for thinking (defaults to 10000) */
|
|
29
29
|
thinkingBudget?: number;
|
|
30
30
|
/** Thinking level for Gemini 3 models */
|
|
31
31
|
thinkingLevel?: ThinkingLevel;
|
|
32
|
-
}
|
|
32
|
+
};
|
|
33
33
|
/**
|
|
34
34
|
* Native SDK thinkingConfig structure for Gemini native SDK.
|
|
35
35
|
*/
|
|
36
|
-
export
|
|
36
|
+
export type NativeThinkingConfig = {
|
|
37
37
|
includeThoughts: boolean;
|
|
38
38
|
thinkingLevel: ThinkingLevel;
|
|
39
|
-
}
|
|
39
|
+
};
|
|
40
40
|
/**
|
|
41
41
|
* Default token budget for thinking operations
|
|
42
42
|
*/
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token Estimation Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides character-based token estimation with per-provider adjustment
|
|
5
|
+
* multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
|
|
6
|
+
* + provider multipliers) but without requiring a tokenizer dependency.
|
|
7
|
+
*
|
|
8
|
+
* Multiplier sources: Continue project's getAdjustedTokenCount.ts
|
|
9
|
+
* - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
|
|
10
|
+
* - Google (AI Studio / Vertex): 1.18x
|
|
11
|
+
* - Mistral/Codestral: 1.26x
|
|
12
|
+
* - OpenAI/GPT: 1.0x (baseline)
|
|
13
|
+
*/
|
|
14
|
+
import type { ChatMessage } from "../types/conversation.js";
|
|
15
|
+
/** Characters per token for English text */
|
|
16
|
+
export declare const CHARS_PER_TOKEN = 4;
|
|
17
|
+
/** Characters per token for code */
|
|
18
|
+
export declare const CODE_CHARS_PER_TOKEN = 3;
|
|
19
|
+
/** Safety margin to avoid underestimation */
|
|
20
|
+
export declare const TOKEN_SAFETY_MARGIN = 1.15;
|
|
21
|
+
/** Message framing overhead in tokens (role + delimiters) */
|
|
22
|
+
export declare const TOKENS_PER_MESSAGE = 4;
|
|
23
|
+
/** Conversation-level overhead in tokens */
|
|
24
|
+
export declare const TOKENS_PER_CONVERSATION = 24;
|
|
25
|
+
/** Image token estimate (flat) */
|
|
26
|
+
export declare const IMAGE_TOKEN_ESTIMATE = 1024;
|
|
27
|
+
/**
|
|
28
|
+
* Get the token multiplier for a given provider.
|
|
29
|
+
*/
|
|
30
|
+
export declare function getProviderMultiplier(provider?: string): number;
|
|
31
|
+
/**
|
|
32
|
+
* Estimate token count for a string.
|
|
33
|
+
*
|
|
34
|
+
* @param text - Input text
|
|
35
|
+
* @param provider - Optional provider for multiplier adjustment
|
|
36
|
+
* @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
|
|
37
|
+
* @returns Estimated token count
|
|
38
|
+
*/
|
|
39
|
+
export declare function estimateTokens(text: string, provider?: string, isCode?: boolean): number;
|
|
40
|
+
/**
|
|
41
|
+
* Estimate token count for a single ChatMessage.
|
|
42
|
+
* Includes message framing overhead.
|
|
43
|
+
*/
|
|
44
|
+
export declare function estimateMessageTokens(message: ChatMessage | {
|
|
45
|
+
role: string;
|
|
46
|
+
content: string;
|
|
47
|
+
}, provider?: string): number;
|
|
48
|
+
/**
|
|
49
|
+
* Estimate total token count for an array of messages.
|
|
50
|
+
* Includes conversation-level overhead.
|
|
51
|
+
*/
|
|
52
|
+
export declare function estimateMessagesTokens(messages: Array<ChatMessage | {
|
|
53
|
+
role: string;
|
|
54
|
+
content: string;
|
|
55
|
+
}>, provider?: string): number;
|
|
56
|
+
/**
|
|
57
|
+
* Truncate text to fit within a token budget.
|
|
58
|
+
* Tries to cut at sentence or word boundaries.
|
|
59
|
+
*
|
|
60
|
+
* @param text - Input text
|
|
61
|
+
* @param maxTokens - Maximum tokens allowed
|
|
62
|
+
* @param provider - Optional provider for multiplier
|
|
63
|
+
* @returns Truncated text with "..." suffix if truncated
|
|
64
|
+
*/
|
|
65
|
+
export declare function truncateToTokenBudget(text: string, maxTokens: number, provider?: string): {
|
|
66
|
+
text: string;
|
|
67
|
+
truncated: boolean;
|
|
68
|
+
};
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token Estimation Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides character-based token estimation with per-provider adjustment
|
|
5
|
+
* multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
|
|
6
|
+
* + provider multipliers) but without requiring a tokenizer dependency.
|
|
7
|
+
*
|
|
8
|
+
* Multiplier sources: Continue project's getAdjustedTokenCount.ts
|
|
9
|
+
* - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
|
|
10
|
+
* - Google (AI Studio / Vertex): 1.18x
|
|
11
|
+
* - Mistral/Codestral: 1.26x
|
|
12
|
+
* - OpenAI/GPT: 1.0x (baseline)
|
|
13
|
+
*/
|
|
14
|
+
/** Characters per token for English text */
|
|
15
|
+
export const CHARS_PER_TOKEN = 4;
|
|
16
|
+
/** Characters per token for code */
|
|
17
|
+
export const CODE_CHARS_PER_TOKEN = 3;
|
|
18
|
+
/** Safety margin to avoid underestimation */
|
|
19
|
+
export const TOKEN_SAFETY_MARGIN = 1.15;
|
|
20
|
+
/** Message framing overhead in tokens (role + delimiters) */
|
|
21
|
+
export const TOKENS_PER_MESSAGE = 4;
|
|
22
|
+
/** Conversation-level overhead in tokens */
|
|
23
|
+
export const TOKENS_PER_CONVERSATION = 24;
|
|
24
|
+
/** Image token estimate (flat) */
|
|
25
|
+
export const IMAGE_TOKEN_ESTIMATE = 1_024;
|
|
26
|
+
/**
|
|
27
|
+
* Per-provider token multipliers.
|
|
28
|
+
* Applied on top of the base GPT-style character estimate.
|
|
29
|
+
*/
|
|
30
|
+
const PROVIDER_MULTIPLIERS = {
|
|
31
|
+
anthropic: 1.23,
|
|
32
|
+
"google-ai": 1.18,
|
|
33
|
+
vertex: 1.18,
|
|
34
|
+
mistral: 1.26,
|
|
35
|
+
openai: 1.0,
|
|
36
|
+
azure: 1.0,
|
|
37
|
+
bedrock: 1.23, // Bedrock is mostly Anthropic models
|
|
38
|
+
ollama: 1.0,
|
|
39
|
+
litellm: 1.0,
|
|
40
|
+
huggingface: 1.0,
|
|
41
|
+
sagemaker: 1.0,
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Get the token multiplier for a given provider.
|
|
45
|
+
*/
|
|
46
|
+
export function getProviderMultiplier(provider) {
|
|
47
|
+
if (!provider) {
|
|
48
|
+
return 1.0;
|
|
49
|
+
}
|
|
50
|
+
return PROVIDER_MULTIPLIERS[provider] ?? 1.0;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Estimate token count for a string.
|
|
54
|
+
*
|
|
55
|
+
* @param text - Input text
|
|
56
|
+
* @param provider - Optional provider for multiplier adjustment
|
|
57
|
+
* @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
|
|
58
|
+
* @returns Estimated token count
|
|
59
|
+
*/
|
|
60
|
+
export function estimateTokens(text, provider, isCode) {
|
|
61
|
+
if (!text || text.length === 0) {
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
|
64
|
+
const charsPerToken = isCode ? CODE_CHARS_PER_TOKEN : CHARS_PER_TOKEN;
|
|
65
|
+
const baseTokens = Math.ceil(text.length / charsPerToken);
|
|
66
|
+
const multiplier = getProviderMultiplier(provider);
|
|
67
|
+
return Math.ceil(baseTokens * multiplier * TOKEN_SAFETY_MARGIN);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Estimate token count for a single ChatMessage.
|
|
71
|
+
* Includes message framing overhead.
|
|
72
|
+
*/
|
|
73
|
+
export function estimateMessageTokens(message, provider) {
|
|
74
|
+
const contentTokens = estimateTokens(message.content || "", provider);
|
|
75
|
+
return contentTokens + TOKENS_PER_MESSAGE;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Estimate total token count for an array of messages.
|
|
79
|
+
* Includes conversation-level overhead.
|
|
80
|
+
*/
|
|
81
|
+
export function estimateMessagesTokens(messages, provider) {
|
|
82
|
+
if (!messages || messages.length === 0) {
|
|
83
|
+
return 0;
|
|
84
|
+
}
|
|
85
|
+
const messageTokens = messages.reduce((sum, msg) => sum + estimateMessageTokens(msg, provider), 0);
|
|
86
|
+
return messageTokens + TOKENS_PER_CONVERSATION;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Truncate text to fit within a token budget.
|
|
90
|
+
* Tries to cut at sentence or word boundaries.
|
|
91
|
+
*
|
|
92
|
+
* @param text - Input text
|
|
93
|
+
* @param maxTokens - Maximum tokens allowed
|
|
94
|
+
* @param provider - Optional provider for multiplier
|
|
95
|
+
* @returns Truncated text with "..." suffix if truncated
|
|
96
|
+
*/
|
|
97
|
+
export function truncateToTokenBudget(text, maxTokens, provider) {
|
|
98
|
+
if (estimateTokens(text, provider) <= maxTokens) {
|
|
99
|
+
return { text, truncated: false };
|
|
100
|
+
}
|
|
101
|
+
const multiplier = getProviderMultiplier(provider);
|
|
102
|
+
const safetyFactor = multiplier * TOKEN_SAFETY_MARGIN;
|
|
103
|
+
const maxChars = Math.floor((maxTokens / safetyFactor) * CHARS_PER_TOKEN);
|
|
104
|
+
if (maxChars <= 0) {
|
|
105
|
+
return { text: "", truncated: true };
|
|
106
|
+
}
|
|
107
|
+
// Try to cut at sentence boundary
|
|
108
|
+
const candidate = text.slice(0, maxChars);
|
|
109
|
+
const lastSentence = Math.max(candidate.lastIndexOf(". "), candidate.lastIndexOf("! "), candidate.lastIndexOf("? "), candidate.lastIndexOf("\n"));
|
|
110
|
+
const cutPoint = lastSentence > maxChars * 0.5 ? lastSentence + 1 : maxChars;
|
|
111
|
+
return { text: text.slice(0, cutPoint) + "...", truncated: true };
|
|
112
|
+
}
|
|
@@ -12,7 +12,7 @@ import type { TokenUsage } from "../types/analytics.js";
|
|
|
12
12
|
* Raw usage object that may come from various AI providers
|
|
13
13
|
* Supports multiple naming conventions and nested structures
|
|
14
14
|
*/
|
|
15
|
-
export
|
|
15
|
+
export type RawUsageObject = {
|
|
16
16
|
input?: number;
|
|
17
17
|
output?: number;
|
|
18
18
|
total?: number;
|
|
@@ -30,11 +30,11 @@ export interface RawUsageObject {
|
|
|
30
30
|
reasoning_tokens?: number;
|
|
31
31
|
thinkingTokens?: number;
|
|
32
32
|
usage?: RawUsageObject;
|
|
33
|
-
}
|
|
33
|
+
};
|
|
34
34
|
/**
|
|
35
35
|
* Options for token extraction behavior
|
|
36
36
|
*/
|
|
37
|
-
export
|
|
37
|
+
export type TokenExtractionOptions = {
|
|
38
38
|
/**
|
|
39
39
|
* Whether to calculate cache savings percentage
|
|
40
40
|
* @default true
|
|
@@ -46,7 +46,7 @@ export interface TokenExtractionOptions {
|
|
|
46
46
|
* - "undefined": Return undefined for missing optional fields (default)
|
|
47
47
|
*/
|
|
48
48
|
missingOptionalBehavior?: "zero" | "undefined";
|
|
49
|
-
}
|
|
49
|
+
};
|
|
50
50
|
/**
|
|
51
51
|
* Extract input token count from various provider formats
|
|
52
52
|
* Priority: input > inputTokens > promptTokens
|
|
@@ -67,7 +67,7 @@ export declare class TTSError extends NeuroLinkError {
|
|
|
67
67
|
* }
|
|
68
68
|
* ```
|
|
69
69
|
*/
|
|
70
|
-
export
|
|
70
|
+
export type TTSHandler = {
|
|
71
71
|
/**
|
|
72
72
|
* Generate audio from text using provider-specific TTS API
|
|
73
73
|
*
|
|
@@ -101,7 +101,7 @@ export interface TTSHandler {
|
|
|
101
101
|
* @default 3000 if not specified
|
|
102
102
|
*/
|
|
103
103
|
maxTextLength?: number;
|
|
104
|
-
}
|
|
104
|
+
};
|
|
105
105
|
/**
|
|
106
106
|
* TTS processor class for orchestrating text-to-speech operations
|
|
107
107
|
*
|