@juspay/neurolink 9.5.2 → 9.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +29 -25
- package/dist/agent/directTools.d.ts +5 -5
- package/dist/cli/commands/config.d.ts +9 -9
- package/dist/cli/commands/serve.d.ts +37 -0
- package/dist/cli/commands/serve.js +302 -229
- package/dist/cli/commands/setup-anthropic.d.ts +2 -2
- package/dist/cli/commands/setup-azure.d.ts +2 -2
- package/dist/cli/commands/setup-bedrock.d.ts +2 -2
- package/dist/cli/commands/setup-gcp.d.ts +2 -2
- package/dist/cli/commands/setup-google-ai.d.ts +2 -2
- package/dist/cli/commands/setup-huggingface.d.ts +2 -2
- package/dist/cli/commands/setup-mistral.d.ts +2 -2
- package/dist/cli/commands/setup-openai.d.ts +2 -2
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/factories/commandFactory.js +16 -2
- package/dist/cli/loop/optionsSchema.d.ts +2 -2
- package/dist/cli/loop/session.d.ts +4 -0
- package/dist/cli/loop/session.js +49 -4
- package/dist/cli/utils/interactiveSetup.d.ts +4 -4
- package/dist/config/conversationMemory.d.ts +2 -0
- package/dist/config/conversationMemory.js +5 -5
- package/dist/constants/contextWindows.d.ts +46 -0
- package/dist/constants/contextWindows.js +156 -0
- package/dist/context/budgetChecker.d.ts +18 -0
- package/dist/context/budgetChecker.js +71 -0
- package/dist/context/contextCompactor.d.ts +22 -0
- package/dist/context/contextCompactor.js +106 -0
- package/dist/context/effectiveHistory.d.ts +52 -0
- package/dist/context/effectiveHistory.js +105 -0
- package/dist/context/errorDetection.d.ts +14 -0
- package/dist/context/errorDetection.js +124 -0
- package/dist/context/fileSummarizationService.d.ts +54 -0
- package/dist/context/fileSummarizationService.js +255 -0
- package/dist/context/fileSummarizer.d.ts +56 -0
- package/dist/context/fileSummarizer.js +145 -0
- package/dist/context/fileTokenBudget.d.ts +53 -0
- package/dist/context/fileTokenBudget.js +127 -0
- package/dist/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/context/prompts/summarizationPrompt.js +110 -0
- package/dist/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/context/stages/fileReadDeduplicator.js +66 -0
- package/dist/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/context/stages/slidingWindowTruncator.js +42 -0
- package/dist/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/context/stages/structuredSummarizer.js +49 -0
- package/dist/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/context/stages/toolOutputPruner.js +52 -0
- package/dist/context/summarizationEngine.d.ts +45 -0
- package/dist/context/summarizationEngine.js +110 -0
- package/dist/context/toolOutputLimits.d.ts +17 -0
- package/dist/context/toolOutputLimits.js +84 -0
- package/dist/context/toolPairRepair.d.ts +16 -0
- package/dist/context/toolPairRepair.js +66 -0
- package/dist/core/conversationMemoryManager.d.ts +5 -15
- package/dist/core/conversationMemoryManager.js +15 -75
- package/dist/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/core/modules/MessageBuilder.js +2 -0
- package/dist/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/core/modules/TelemetryHandler.js +3 -3
- package/dist/core/modules/ToolsManager.d.ts +2 -2
- package/dist/core/redisConversationMemoryManager.d.ts +8 -14
- package/dist/core/redisConversationMemoryManager.js +69 -78
- package/dist/factories/providerFactory.d.ts +2 -2
- package/dist/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/files/fileReferenceRegistry.js +1543 -0
- package/dist/files/fileTools.d.ts +423 -0
- package/dist/files/fileTools.js +449 -0
- package/dist/files/index.d.ts +14 -0
- package/dist/files/index.js +13 -0
- package/dist/files/streamingReader.d.ts +93 -0
- package/dist/files/streamingReader.js +321 -0
- package/dist/files/types.d.ts +23 -0
- package/dist/files/types.js +23 -0
- package/dist/image-gen/imageGenTools.d.ts +2 -2
- package/dist/image-gen/types.d.ts +12 -12
- package/dist/lib/agent/directTools.d.ts +7 -7
- package/dist/lib/config/conversationMemory.d.ts +2 -0
- package/dist/lib/config/conversationMemory.js +5 -5
- package/dist/lib/constants/contextWindows.d.ts +46 -0
- package/dist/lib/constants/contextWindows.js +157 -0
- package/dist/lib/context/budgetChecker.d.ts +18 -0
- package/dist/lib/context/budgetChecker.js +72 -0
- package/dist/lib/context/contextCompactor.d.ts +22 -0
- package/dist/lib/context/contextCompactor.js +107 -0
- package/dist/lib/context/effectiveHistory.d.ts +52 -0
- package/dist/lib/context/effectiveHistory.js +106 -0
- package/dist/lib/context/errorDetection.d.ts +14 -0
- package/dist/lib/context/errorDetection.js +125 -0
- package/dist/lib/context/fileSummarizationService.d.ts +54 -0
- package/dist/lib/context/fileSummarizationService.js +256 -0
- package/dist/lib/context/fileSummarizer.d.ts +56 -0
- package/dist/lib/context/fileSummarizer.js +146 -0
- package/dist/lib/context/fileTokenBudget.d.ts +53 -0
- package/dist/lib/context/fileTokenBudget.js +128 -0
- package/dist/lib/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/lib/context/prompts/summarizationPrompt.js +111 -0
- package/dist/lib/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/lib/context/stages/fileReadDeduplicator.js +67 -0
- package/dist/lib/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/lib/context/stages/slidingWindowTruncator.js +43 -0
- package/dist/lib/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/lib/context/stages/structuredSummarizer.js +50 -0
- package/dist/lib/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/lib/context/stages/toolOutputPruner.js +53 -0
- package/dist/lib/context/summarizationEngine.d.ts +45 -0
- package/dist/lib/context/summarizationEngine.js +111 -0
- package/dist/lib/context/toolOutputLimits.d.ts +17 -0
- package/dist/lib/context/toolOutputLimits.js +85 -0
- package/dist/lib/context/toolPairRepair.d.ts +16 -0
- package/dist/lib/context/toolPairRepair.js +67 -0
- package/dist/lib/core/conversationMemoryManager.d.ts +5 -15
- package/dist/lib/core/conversationMemoryManager.js +15 -75
- package/dist/lib/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/lib/core/modules/MessageBuilder.js +2 -0
- package/dist/lib/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/lib/core/modules/TelemetryHandler.js +3 -3
- package/dist/lib/core/modules/ToolsManager.d.ts +2 -2
- package/dist/lib/core/redisConversationMemoryManager.d.ts +8 -14
- package/dist/lib/core/redisConversationMemoryManager.js +69 -78
- package/dist/lib/factories/providerFactory.d.ts +2 -2
- package/dist/lib/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/lib/files/fileReferenceRegistry.js +1544 -0
- package/dist/lib/files/fileTools.d.ts +423 -0
- package/dist/lib/files/fileTools.js +450 -0
- package/dist/lib/files/index.d.ts +14 -0
- package/dist/lib/files/index.js +14 -0
- package/dist/lib/files/streamingReader.d.ts +93 -0
- package/dist/lib/files/streamingReader.js +322 -0
- package/dist/lib/files/types.d.ts +23 -0
- package/dist/lib/files/types.js +24 -0
- package/dist/lib/image-gen/imageGenTools.d.ts +2 -2
- package/dist/lib/image-gen/types.d.ts +12 -12
- package/dist/lib/memory/mem0Initializer.d.ts +2 -2
- package/dist/lib/neurolink.d.ts +61 -2
- package/dist/lib/neurolink.js +619 -307
- package/dist/lib/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/lib/processors/archive/ArchiveProcessor.js +1309 -0
- package/dist/lib/processors/archive/index.d.ts +33 -0
- package/dist/lib/processors/archive/index.js +43 -0
- package/dist/lib/processors/base/types.d.ts +70 -64
- package/dist/lib/processors/base/types.js +6 -0
- package/dist/lib/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/lib/processors/cli/fileProcessorCli.js +5 -5
- package/dist/lib/processors/config/mimeTypes.js +25 -0
- package/dist/lib/processors/config/sizeLimits.d.ts +52 -40
- package/dist/lib/processors/config/sizeLimits.js +56 -44
- package/dist/lib/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/lib/processors/document/ExcelProcessor.js +72 -1
- package/dist/lib/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/lib/processors/document/PptxProcessor.js +158 -0
- package/dist/lib/processors/document/index.d.ts +1 -0
- package/dist/lib/processors/document/index.js +6 -0
- package/dist/lib/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/lib/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/lib/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/lib/processors/index.d.ts +8 -2
- package/dist/lib/processors/index.js +5 -2
- package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/lib/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/lib/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/lib/processors/media/AudioProcessor.js +708 -0
- package/dist/lib/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/lib/processors/media/VideoProcessor.js +992 -0
- package/dist/lib/processors/media/index.d.ts +27 -0
- package/dist/lib/processors/media/index.js +37 -0
- package/dist/lib/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/lib/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/lib/processors/registry/index.d.ts +1 -1
- package/dist/lib/processors/registry/index.js +1 -1
- package/dist/lib/processors/registry/types.d.ts +2 -2
- package/dist/lib/providers/googleAiStudio.d.ts +34 -0
- package/dist/lib/providers/googleAiStudio.js +267 -397
- package/dist/lib/providers/googleVertex.d.ts +55 -1
- package/dist/lib/providers/googleVertex.js +452 -719
- package/dist/lib/providers/sagemaker/detection.d.ts +6 -6
- package/dist/lib/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/lib/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/lib/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/lib/rag/document/loaders.d.ts +6 -71
- package/dist/lib/rag/document/loaders.js +5 -5
- package/dist/lib/rag/graphRag/graphRAG.js +26 -9
- package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/lib/rag/metadata/metadataExtractor.js +6 -3
- package/dist/lib/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/lib/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/lib/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/lib/rag/pipeline/contextAssembly.js +6 -3
- package/dist/lib/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/lib/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/lib/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/lib/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/lib/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/lib/rag/types.d.ts +3 -3
- package/dist/lib/sdk/toolRegistration.d.ts +2 -2
- package/dist/lib/server/middleware/cache.d.ts +2 -2
- package/dist/lib/server/middleware/rateLimit.d.ts +2 -2
- package/dist/lib/server/routes/mcpRoutes.js +277 -249
- package/dist/lib/server/routes/memoryRoutes.js +287 -281
- package/dist/lib/server/utils/validation.d.ts +10 -10
- package/dist/lib/session/globalSessionState.d.ts +2 -2
- package/dist/lib/telemetry/telemetryService.d.ts +2 -2
- package/dist/lib/types/common.d.ts +39 -0
- package/dist/lib/types/contextTypes.d.ts +255 -0
- package/dist/lib/types/contextTypes.js +0 -2
- package/dist/lib/types/conversation.d.ts +62 -0
- package/dist/lib/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/lib/types/conversationMemoryInterface.js +7 -0
- package/dist/lib/types/fileReferenceTypes.d.ts +222 -0
- package/dist/lib/types/fileReferenceTypes.js +9 -0
- package/dist/lib/types/fileTypes.d.ts +26 -3
- package/dist/lib/types/generateTypes.d.ts +22 -1
- package/dist/lib/types/index.d.ts +4 -5
- package/dist/lib/types/index.js +8 -10
- package/dist/lib/types/modelTypes.d.ts +2 -2
- package/dist/lib/types/processorTypes.d.ts +597 -0
- package/dist/lib/types/processorTypes.js +91 -0
- package/dist/lib/types/ragTypes.d.ts +481 -0
- package/dist/lib/types/ragTypes.js +8 -0
- package/dist/lib/types/sdkTypes.d.ts +17 -18
- package/dist/lib/types/streamTypes.d.ts +11 -1
- package/dist/lib/utils/async/retry.d.ts +2 -2
- package/dist/lib/utils/async/withTimeout.js +3 -1
- package/dist/lib/utils/conversationMemory.d.ts +12 -6
- package/dist/lib/utils/conversationMemory.js +76 -36
- package/dist/lib/utils/fileDetector.d.ts +62 -0
- package/dist/lib/utils/fileDetector.js +1014 -14
- package/dist/lib/utils/json/safeParse.d.ts +2 -2
- package/dist/lib/utils/messageBuilder.js +806 -153
- package/dist/lib/utils/modelChoices.d.ts +2 -2
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/lib/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/lib/utils/rateLimiter.d.ts +2 -2
- package/dist/lib/utils/sanitizers/filename.d.ts +4 -4
- package/dist/lib/utils/sanitizers/svg.d.ts +2 -2
- package/dist/lib/utils/thinkingConfig.d.ts +6 -6
- package/dist/lib/utils/tokenEstimation.d.ts +68 -0
- package/dist/lib/utils/tokenEstimation.js +113 -0
- package/dist/lib/utils/tokenUtils.d.ts +4 -4
- package/dist/lib/utils/ttsProcessor.d.ts +2 -2
- package/dist/lib/workflow/config.d.ts +150 -150
- package/dist/memory/mem0Initializer.d.ts +2 -2
- package/dist/neurolink.d.ts +61 -2
- package/dist/neurolink.js +619 -307
- package/dist/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/processors/archive/ArchiveProcessor.js +1308 -0
- package/dist/processors/archive/index.d.ts +33 -0
- package/dist/processors/archive/index.js +42 -0
- package/dist/processors/base/types.d.ts +70 -64
- package/dist/processors/base/types.js +6 -0
- package/dist/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/processors/cli/fileProcessorCli.js +5 -5
- package/dist/processors/config/mimeTypes.js +25 -0
- package/dist/processors/config/sizeLimits.d.ts +52 -40
- package/dist/processors/config/sizeLimits.js +56 -44
- package/dist/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/processors/document/ExcelProcessor.js +72 -1
- package/dist/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/processors/document/PptxProcessor.js +157 -0
- package/dist/processors/document/index.d.ts +1 -0
- package/dist/processors/document/index.js +6 -0
- package/dist/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/processors/index.d.ts +8 -2
- package/dist/processors/index.js +5 -2
- package/dist/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/processors/media/AudioProcessor.js +707 -0
- package/dist/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/processors/media/VideoProcessor.js +991 -0
- package/dist/processors/media/ffprobe-static.d.ts +4 -0
- package/dist/processors/media/index.d.ts +27 -0
- package/dist/processors/media/index.js +36 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/processors/registry/index.d.ts +1 -1
- package/dist/processors/registry/index.js +1 -1
- package/dist/processors/registry/types.d.ts +2 -2
- package/dist/providers/googleAiStudio.d.ts +34 -0
- package/dist/providers/googleAiStudio.js +267 -397
- package/dist/providers/googleVertex.d.ts +55 -1
- package/dist/providers/googleVertex.js +452 -719
- package/dist/providers/sagemaker/detection.d.ts +6 -6
- package/dist/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/rag/document/loaders.d.ts +6 -71
- package/dist/rag/document/loaders.js +5 -5
- package/dist/rag/graphRag/graphRAG.js +26 -9
- package/dist/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/rag/metadata/metadataExtractor.js +6 -3
- package/dist/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/rag/pipeline/contextAssembly.js +6 -3
- package/dist/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/rag/types.d.ts +3 -3
- package/dist/sdk/toolRegistration.d.ts +2 -2
- package/dist/server/middleware/cache.d.ts +2 -2
- package/dist/server/middleware/rateLimit.d.ts +2 -2
- package/dist/server/routes/mcpRoutes.js +277 -249
- package/dist/server/routes/memoryRoutes.js +287 -281
- package/dist/server/utils/validation.d.ts +4 -4
- package/dist/session/globalSessionState.d.ts +2 -2
- package/dist/telemetry/telemetryService.d.ts +2 -2
- package/dist/types/common.d.ts +39 -0
- package/dist/types/contextTypes.d.ts +255 -0
- package/dist/types/contextTypes.js +0 -2
- package/dist/types/conversation.d.ts +62 -0
- package/dist/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/types/conversationMemoryInterface.js +6 -0
- package/dist/types/fileReferenceTypes.d.ts +222 -0
- package/dist/types/fileReferenceTypes.js +8 -0
- package/dist/types/fileTypes.d.ts +26 -3
- package/dist/types/generateTypes.d.ts +22 -1
- package/dist/types/index.d.ts +4 -5
- package/dist/types/index.js +8 -10
- package/dist/types/processorTypes.d.ts +597 -0
- package/dist/types/processorTypes.js +90 -0
- package/dist/types/ragTypes.d.ts +481 -0
- package/dist/types/ragTypes.js +7 -0
- package/dist/types/sdkTypes.d.ts +17 -18
- package/dist/types/streamTypes.d.ts +11 -1
- package/dist/utils/async/retry.d.ts +2 -2
- package/dist/utils/async/withTimeout.js +3 -1
- package/dist/utils/conversationMemory.d.ts +12 -6
- package/dist/utils/conversationMemory.js +76 -36
- package/dist/utils/fileDetector.d.ts +62 -0
- package/dist/utils/fileDetector.js +1014 -14
- package/dist/utils/json/safeParse.d.ts +2 -2
- package/dist/utils/messageBuilder.js +806 -153
- package/dist/utils/modelChoices.d.ts +2 -2
- package/dist/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/utils/rateLimiter.d.ts +2 -2
- package/dist/utils/sanitizers/filename.d.ts +4 -4
- package/dist/utils/sanitizers/svg.d.ts +2 -2
- package/dist/utils/thinkingConfig.d.ts +6 -6
- package/dist/utils/tokenEstimation.d.ts +68 -0
- package/dist/utils/tokenEstimation.js +112 -0
- package/dist/utils/tokenUtils.d.ts +4 -4
- package/dist/utils/ttsProcessor.d.ts +2 -2
- package/dist/workflow/config.d.ts +104 -104
- package/package.json +18 -6
- package/dist/lib/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/lib/utils/conversationMemoryUtils.js +0 -138
- package/dist/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/utils/conversationMemoryUtils.js +0 -137
|
@@ -8,7 +8,7 @@ import type { SageMakerConfig, SageMakerModelConfig } from "../../types/provider
|
|
|
8
8
|
/**
|
|
9
9
|
* Streaming capability information for an endpoint
|
|
10
10
|
*/
|
|
11
|
-
export
|
|
11
|
+
export type StreamingCapability = {
|
|
12
12
|
/** Whether streaming is supported */
|
|
13
13
|
supported: boolean;
|
|
14
14
|
/** Detected streaming protocol */
|
|
@@ -28,11 +28,11 @@ export interface StreamingCapability {
|
|
|
28
28
|
version?: string;
|
|
29
29
|
tags?: string[];
|
|
30
30
|
};
|
|
31
|
-
}
|
|
31
|
+
};
|
|
32
32
|
/**
|
|
33
33
|
* Model type detection result
|
|
34
34
|
*/
|
|
35
|
-
export
|
|
35
|
+
export type ModelDetectionResult = {
|
|
36
36
|
/** Primary model type */
|
|
37
37
|
type: StreamingCapability["modelType"];
|
|
38
38
|
/** Detection confidence (0-1) */
|
|
@@ -41,11 +41,11 @@ export interface ModelDetectionResult {
|
|
|
41
41
|
evidence: string[];
|
|
42
42
|
/** Suggested configuration */
|
|
43
43
|
suggestedConfig?: Partial<SageMakerModelConfig>;
|
|
44
|
-
}
|
|
44
|
+
};
|
|
45
45
|
/**
|
|
46
46
|
* Endpoint health and metadata information
|
|
47
47
|
*/
|
|
48
|
-
export
|
|
48
|
+
export type EndpointHealth = {
|
|
49
49
|
/** Health status */
|
|
50
50
|
status: "healthy" | "unhealthy" | "unknown";
|
|
51
51
|
/** Response time in milliseconds */
|
|
@@ -59,7 +59,7 @@ export interface EndpointHealth {
|
|
|
59
59
|
framework?: string;
|
|
60
60
|
architecture?: string;
|
|
61
61
|
};
|
|
62
|
-
}
|
|
62
|
+
};
|
|
63
63
|
/**
|
|
64
64
|
* SageMaker Model Detection and Capability Discovery Service
|
|
65
65
|
*/
|
|
@@ -6,18 +6,18 @@
|
|
|
6
6
|
/**
|
|
7
7
|
* Simple diagnostic result interface
|
|
8
8
|
*/
|
|
9
|
-
export
|
|
9
|
+
export type DiagnosticResult = {
|
|
10
10
|
name: string;
|
|
11
11
|
category: "configuration" | "connectivity" | "streaming";
|
|
12
12
|
status: "pass" | "fail" | "warning";
|
|
13
13
|
message: string;
|
|
14
14
|
details?: string;
|
|
15
15
|
recommendation?: string;
|
|
16
|
-
}
|
|
16
|
+
};
|
|
17
17
|
/**
|
|
18
18
|
* Diagnostic report interface
|
|
19
19
|
*/
|
|
20
|
-
export
|
|
20
|
+
export type DiagnosticReport = {
|
|
21
21
|
overallStatus: "healthy" | "issues" | "critical";
|
|
22
22
|
results: DiagnosticResult[];
|
|
23
23
|
summary: {
|
|
@@ -26,7 +26,7 @@ export interface DiagnosticReport {
|
|
|
26
26
|
failed: number;
|
|
27
27
|
warnings: number;
|
|
28
28
|
};
|
|
29
|
-
}
|
|
29
|
+
};
|
|
30
30
|
/**
|
|
31
31
|
* Run quick diagnostics for SageMaker configuration
|
|
32
32
|
*/
|
|
@@ -10,12 +10,12 @@ import { type StructuredOutputParser } from "./structured-parser.js";
|
|
|
10
10
|
* Shared bracket counting state and utilities
|
|
11
11
|
* Used by both validateJSONCompleteness and StructuredOutputParser
|
|
12
12
|
*/
|
|
13
|
-
export
|
|
13
|
+
export type BracketCountingState = {
|
|
14
14
|
braceCount: number;
|
|
15
15
|
bracketCount: number;
|
|
16
16
|
inString: boolean;
|
|
17
17
|
escapeNext: boolean;
|
|
18
|
-
}
|
|
18
|
+
};
|
|
19
19
|
/**
|
|
20
20
|
* Process a single character for bracket counting logic
|
|
21
21
|
* Shared utility to avoid code duplication between parsers
|
|
@@ -44,7 +44,7 @@ export declare function parseToolCallArguments(args: string): {
|
|
|
44
44
|
/**
|
|
45
45
|
* Base interface for streaming response parsers
|
|
46
46
|
*/
|
|
47
|
-
export
|
|
47
|
+
export type StreamingParser = {
|
|
48
48
|
/** Parse a chunk of streaming data */
|
|
49
49
|
parse(chunk: Uint8Array): SageMakerStreamChunk[];
|
|
50
50
|
/** Check if a chunk indicates completion */
|
|
@@ -55,7 +55,7 @@ export interface StreamingParser {
|
|
|
55
55
|
getName(): string;
|
|
56
56
|
/** Reset parser state for new stream */
|
|
57
57
|
reset(): void;
|
|
58
|
-
}
|
|
58
|
+
};
|
|
59
59
|
/**
|
|
60
60
|
* Abstract base parser with common functionality
|
|
61
61
|
*/
|
|
@@ -127,8 +127,8 @@ export class RecursiveChunker extends BaseChunker {
|
|
|
127
127
|
for (let i = 0; i < chunks.length; i++) {
|
|
128
128
|
let chunk = chunks[i] ?? "";
|
|
129
129
|
// Add overlap from previous chunk
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
const prevChunk = i > 0 ? chunks[i - 1] : undefined;
|
|
131
|
+
if (prevChunk) {
|
|
132
132
|
const overlapText = prevChunk.slice(-Math.min(overlap, prevChunk.length));
|
|
133
133
|
chunk = overlapText + chunk;
|
|
134
134
|
}
|
|
@@ -24,77 +24,12 @@
|
|
|
24
24
|
* ```
|
|
25
25
|
*/
|
|
26
26
|
import { MDocument } from "./MDocument.js";
|
|
27
|
-
import type {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
export
|
|
32
|
-
|
|
33
|
-
metadata?: Record<string, unknown>;
|
|
34
|
-
/** Encoding for text files */
|
|
35
|
-
encoding?: BufferEncoding;
|
|
36
|
-
/** Document type override */
|
|
37
|
-
type?: DocumentType;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Web loader options
|
|
41
|
-
*/
|
|
42
|
-
export interface WebLoaderOptions extends LoaderOptions {
|
|
43
|
-
/** Request timeout in milliseconds */
|
|
44
|
-
timeout?: number;
|
|
45
|
-
/** Custom headers for request */
|
|
46
|
-
headers?: Record<string, string>;
|
|
47
|
-
/** Extract only main content (remove navigation, ads, etc.) */
|
|
48
|
-
extractMainContent?: boolean;
|
|
49
|
-
/** Selector for main content (CSS selector) */
|
|
50
|
-
contentSelector?: string;
|
|
51
|
-
/** User agent string */
|
|
52
|
-
userAgent?: string;
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* PDF loader options
|
|
56
|
-
*/
|
|
57
|
-
export interface PDFLoaderOptions extends LoaderOptions {
|
|
58
|
-
/** Page range to extract (e.g., "1-5" or "1,3,5") */
|
|
59
|
-
pageRange?: string;
|
|
60
|
-
/** Extract images as base64 */
|
|
61
|
-
extractImages?: boolean;
|
|
62
|
-
/** OCR for scanned documents */
|
|
63
|
-
enableOCR?: boolean;
|
|
64
|
-
/** Preserve layout formatting */
|
|
65
|
-
preserveLayout?: boolean;
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* CSV loader options
|
|
69
|
-
*/
|
|
70
|
-
export interface CSVLoaderOptions extends LoaderOptions {
|
|
71
|
-
/** Delimiter character */
|
|
72
|
-
delimiter?: string;
|
|
73
|
-
/** Whether first row is header */
|
|
74
|
-
hasHeader?: boolean;
|
|
75
|
-
/** Column names (if no header) */
|
|
76
|
-
columns?: string[];
|
|
77
|
-
/** Output format */
|
|
78
|
-
outputFormat?: "text" | "json" | "markdown";
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Abstract document loader interface
|
|
82
|
-
*/
|
|
83
|
-
export interface DocumentLoader {
|
|
84
|
-
/**
|
|
85
|
-
* Load document from source
|
|
86
|
-
* @param source - File path, URL, or content
|
|
87
|
-
* @param options - Loader options
|
|
88
|
-
* @returns Promise resolving to MDocument
|
|
89
|
-
*/
|
|
90
|
-
load(source: string, options?: LoaderOptions): Promise<MDocument>;
|
|
91
|
-
/**
|
|
92
|
-
* Check if loader can handle the source
|
|
93
|
-
* @param source - File path, URL, or content
|
|
94
|
-
* @returns True if loader can handle the source
|
|
95
|
-
*/
|
|
96
|
-
canHandle(source: string): boolean;
|
|
97
|
-
}
|
|
27
|
+
import type { LoaderOptions, WebLoaderOptions, PDFLoaderOptions, CSVLoaderOptions, DocumentLoader } from "../../types/ragTypes.js";
|
|
28
|
+
export type { LoaderOptions } from "../../types/ragTypes.js";
|
|
29
|
+
export type { WebLoaderOptions } from "../../types/ragTypes.js";
|
|
30
|
+
export type { PDFLoaderOptions } from "../../types/ragTypes.js";
|
|
31
|
+
export type { CSVLoaderOptions } from "../../types/ragTypes.js";
|
|
32
|
+
export type { DocumentLoader } from "../../types/ragTypes.js";
|
|
98
33
|
/**
|
|
99
34
|
* Text file loader
|
|
100
35
|
*/
|
|
@@ -23,11 +23,11 @@
|
|
|
23
23
|
* const pdfDoc = await PDFLoader.load('/path/to/document.pdf');
|
|
24
24
|
* ```
|
|
25
25
|
*/
|
|
26
|
-
import { readFile } from "fs/promises";
|
|
27
26
|
import { existsSync } from "fs";
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
27
|
+
import { readFile } from "fs/promises";
|
|
28
|
+
import { basename, extname } from "path";
|
|
30
29
|
import { logger } from "../../utils/logger.js";
|
|
30
|
+
import { MDocument } from "./MDocument.js";
|
|
31
31
|
/**
|
|
32
32
|
* Text file loader
|
|
33
33
|
*/
|
|
@@ -244,8 +244,8 @@ export class PDFLoader {
|
|
|
244
244
|
}
|
|
245
245
|
async loadPdfParser() {
|
|
246
246
|
try {
|
|
247
|
-
//
|
|
248
|
-
const pdfParse = await import("pdf-parse");
|
|
247
|
+
// pdf-parse is an optional dependency - use dynamic import with type assertion
|
|
248
|
+
const pdfParse = (await import("pdf-parse"));
|
|
249
249
|
return pdfParse.default || pdfParse;
|
|
250
250
|
}
|
|
251
251
|
catch {
|
|
@@ -52,11 +52,17 @@ export class GraphRAG {
|
|
|
52
52
|
for (let i = 0; i < nodeIds.length; i++) {
|
|
53
53
|
const edges = [];
|
|
54
54
|
const nodeA = this.nodes.get(nodeIds[i]);
|
|
55
|
+
if (!nodeA?.embedding) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
55
58
|
for (let j = 0; j < nodeIds.length; j++) {
|
|
56
59
|
if (i === j) {
|
|
57
60
|
continue;
|
|
58
61
|
}
|
|
59
62
|
const nodeB = this.nodes.get(nodeIds[j]);
|
|
63
|
+
if (!nodeB?.embedding) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
60
66
|
const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
|
|
61
67
|
if (similarity >= this.threshold) {
|
|
62
68
|
edges.push({
|
|
@@ -137,14 +143,19 @@ export class GraphRAG {
|
|
|
137
143
|
const rankedNodes = Array.from(scores.entries())
|
|
138
144
|
.sort((a, b) => b[1] - a[1])
|
|
139
145
|
.slice(0, topK)
|
|
140
|
-
.
|
|
146
|
+
.flatMap(([id, score]) => {
|
|
141
147
|
const node = this.nodes.get(id);
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
+
if (!node) {
|
|
149
|
+
return [];
|
|
150
|
+
}
|
|
151
|
+
return [
|
|
152
|
+
{
|
|
153
|
+
id,
|
|
154
|
+
content: node.content,
|
|
155
|
+
metadata: node.metadata,
|
|
156
|
+
score,
|
|
157
|
+
},
|
|
158
|
+
];
|
|
148
159
|
});
|
|
149
160
|
logger.debug("[GraphRAG] Query completed", {
|
|
150
161
|
topK,
|
|
@@ -174,6 +185,9 @@ export class GraphRAG {
|
|
|
174
185
|
if (existingId === id) {
|
|
175
186
|
continue;
|
|
176
187
|
}
|
|
188
|
+
if (!existingNode.embedding) {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
177
191
|
const similarity = this.cosineSimilarity(embedding.vector, existingNode.embedding);
|
|
178
192
|
if (similarity >= this.threshold) {
|
|
179
193
|
edges.push({
|
|
@@ -259,7 +273,7 @@ export class GraphRAG {
|
|
|
259
273
|
const queue = [nodeId];
|
|
260
274
|
while (queue.length > 0) {
|
|
261
275
|
const current = queue.shift();
|
|
262
|
-
if (visited.has(current)) {
|
|
276
|
+
if (current === undefined || visited.has(current)) {
|
|
263
277
|
continue;
|
|
264
278
|
}
|
|
265
279
|
visited.add(current);
|
|
@@ -327,12 +341,15 @@ export class GraphRAG {
|
|
|
327
341
|
for (let i = 0; i < nodeIds.length; i++) {
|
|
328
342
|
const edges = [];
|
|
329
343
|
const nodeA = this.nodes.get(nodeIds[i]);
|
|
344
|
+
if (!nodeA?.embedding) {
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
330
347
|
for (let j = 0; j < nodeIds.length; j++) {
|
|
331
348
|
if (i === j) {
|
|
332
349
|
continue;
|
|
333
350
|
}
|
|
334
351
|
const nodeB = this.nodes.get(nodeIds[j]);
|
|
335
|
-
if (!
|
|
352
|
+
if (!nodeB?.embedding) {
|
|
336
353
|
continue;
|
|
337
354
|
}
|
|
338
355
|
const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
|
|
@@ -5,61 +5,11 @@
|
|
|
5
5
|
* Follows the BaseFactory pattern for consistent lifecycle management.
|
|
6
6
|
*/
|
|
7
7
|
import { BaseFactory } from "../../core/infrastructure/index.js";
|
|
8
|
-
import type {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
export type
|
|
13
|
-
/**
|
|
14
|
-
* Metadata Extractor interface - all extractors implement this
|
|
15
|
-
*/
|
|
16
|
-
export interface MetadataExtractor {
|
|
17
|
-
/** Extractor type identifier */
|
|
18
|
-
readonly type: MetadataExtractorType;
|
|
19
|
-
/**
|
|
20
|
-
* Extract metadata from chunks
|
|
21
|
-
* @param chunks - Array of chunks to extract metadata from
|
|
22
|
-
* @param params - Extraction parameters
|
|
23
|
-
* @returns Array of extraction results
|
|
24
|
-
*/
|
|
25
|
-
extract(chunks: Chunk[], params?: ExtractParams): Promise<ExtractionResult[]>;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Metadata extractor configuration
|
|
29
|
-
*/
|
|
30
|
-
export interface MetadataExtractorConfig {
|
|
31
|
-
/** Extractor type */
|
|
32
|
-
type: MetadataExtractorType;
|
|
33
|
-
/** Language model provider */
|
|
34
|
-
provider?: string;
|
|
35
|
-
/** Model name for LLM-based extraction */
|
|
36
|
-
modelName?: string;
|
|
37
|
-
/** Custom prompt template */
|
|
38
|
-
promptTemplate?: string;
|
|
39
|
-
/** Maximum tokens for LLM response */
|
|
40
|
-
maxTokens?: number;
|
|
41
|
-
/** Temperature for LLM generation */
|
|
42
|
-
temperature?: number;
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Metadata extractor metadata for discovery and documentation
|
|
46
|
-
*/
|
|
47
|
-
export interface MetadataExtractorMetadata {
|
|
48
|
-
/** Human-readable description */
|
|
49
|
-
description: string;
|
|
50
|
-
/** Default configuration */
|
|
51
|
-
defaultConfig: Partial<MetadataExtractorConfig>;
|
|
52
|
-
/** Supported configuration options */
|
|
53
|
-
supportedOptions: string[];
|
|
54
|
-
/** Recommended use cases */
|
|
55
|
-
useCases: string[];
|
|
56
|
-
/** Alternative names for this extractor */
|
|
57
|
-
aliases: string[];
|
|
58
|
-
/** Whether this extractor requires an AI model */
|
|
59
|
-
requiresModel: boolean;
|
|
60
|
-
/** Extraction types this extractor can produce */
|
|
61
|
-
extractionTypes: string[];
|
|
62
|
-
}
|
|
8
|
+
import type { MetadataExtractor, MetadataExtractorConfig, MetadataExtractorMetadata, MetadataExtractorType } from "../../types/ragTypes.js";
|
|
9
|
+
export type { MetadataExtractorType } from "../../types/ragTypes.js";
|
|
10
|
+
export type { MetadataExtractor } from "../../types/ragTypes.js";
|
|
11
|
+
export type { MetadataExtractorConfig } from "../../types/ragTypes.js";
|
|
12
|
+
export type { MetadataExtractorMetadata } from "../../types/ragTypes.js";
|
|
63
13
|
/**
|
|
64
14
|
* Metadata Extractor Factory
|
|
65
15
|
*
|
|
@@ -113,10 +113,13 @@ export class LLMMetadataExtractor {
|
|
|
113
113
|
const groups = new Map();
|
|
114
114
|
for (const chunk of chunks) {
|
|
115
115
|
const docId = chunk.metadata.documentId;
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
const group = groups.get(docId);
|
|
117
|
+
if (group) {
|
|
118
|
+
group.push(chunk);
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
groups.set(docId, [chunk]);
|
|
118
122
|
}
|
|
119
|
-
groups.get(docId).push(chunk);
|
|
120
123
|
}
|
|
121
124
|
return groups;
|
|
122
125
|
}
|
|
@@ -25,133 +25,15 @@
|
|
|
25
25
|
* console.log(response.answer, response.sources);
|
|
26
26
|
* ```
|
|
27
27
|
*/
|
|
28
|
-
import type { ChunkingStrategy } from "../types.js";
|
|
29
28
|
import { MDocument } from "../document/MDocument.js";
|
|
30
|
-
import {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
export
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Generation model configuration
|
|
41
|
-
*/
|
|
42
|
-
export interface GenerationModelConfig {
|
|
43
|
-
provider: string;
|
|
44
|
-
modelName: string;
|
|
45
|
-
temperature?: number;
|
|
46
|
-
maxTokens?: number;
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* RAG pipeline configuration
|
|
50
|
-
*/
|
|
51
|
-
export interface RAGPipelineConfig {
|
|
52
|
-
/** Pipeline identifier */
|
|
53
|
-
id?: string;
|
|
54
|
-
/** Vector store instance (defaults to in-memory) */
|
|
55
|
-
vectorStore?: VectorStore;
|
|
56
|
-
/** BM25 index for hybrid search (defaults to in-memory) */
|
|
57
|
-
bm25Index?: BM25Index;
|
|
58
|
-
/** Index name for vector store */
|
|
59
|
-
indexName?: string;
|
|
60
|
-
/** Embedding model configuration */
|
|
61
|
-
embeddingModel: EmbeddingModelConfig;
|
|
62
|
-
/** Generation model configuration (for RAG responses) */
|
|
63
|
-
generationModel?: GenerationModelConfig;
|
|
64
|
-
/** Default chunking strategy */
|
|
65
|
-
defaultChunkingStrategy?: ChunkingStrategy;
|
|
66
|
-
/** Default chunk size */
|
|
67
|
-
defaultChunkSize?: number;
|
|
68
|
-
/** Default chunk overlap */
|
|
69
|
-
defaultChunkOverlap?: number;
|
|
70
|
-
/** Enable hybrid search (vector + BM25) */
|
|
71
|
-
enableHybridSearch?: boolean;
|
|
72
|
-
/** Enable Graph RAG */
|
|
73
|
-
enableGraphRAG?: boolean;
|
|
74
|
-
/** Graph RAG similarity threshold */
|
|
75
|
-
graphThreshold?: number;
|
|
76
|
-
/** Default number of results to retrieve */
|
|
77
|
-
defaultTopK?: number;
|
|
78
|
-
/** Enable reranking */
|
|
79
|
-
enableReranking?: boolean;
|
|
80
|
-
/** Reranking model configuration */
|
|
81
|
-
rerankingModel?: EmbeddingModelConfig;
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Ingestion options
|
|
85
|
-
*/
|
|
86
|
-
export interface IngestOptions {
|
|
87
|
-
/** Chunking strategy override */
|
|
88
|
-
strategy?: ChunkingStrategy;
|
|
89
|
-
/** Chunk size override */
|
|
90
|
-
chunkSize?: number;
|
|
91
|
-
/** Chunk overlap override */
|
|
92
|
-
chunkOverlap?: number;
|
|
93
|
-
/** Custom metadata to add */
|
|
94
|
-
metadata?: Record<string, unknown>;
|
|
95
|
-
/** Extract metadata using LLM */
|
|
96
|
-
extractMetadata?: boolean;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Query options
|
|
100
|
-
*/
|
|
101
|
-
export interface QueryOptions {
|
|
102
|
-
/** Number of chunks to retrieve */
|
|
103
|
-
topK?: number;
|
|
104
|
-
/** Use hybrid search */
|
|
105
|
-
hybrid?: boolean;
|
|
106
|
-
/** Use Graph RAG */
|
|
107
|
-
graph?: boolean;
|
|
108
|
-
/** Enable reranking */
|
|
109
|
-
rerank?: boolean;
|
|
110
|
-
/** Metadata filter */
|
|
111
|
-
filter?: Record<string, unknown>;
|
|
112
|
-
/** Include sources in response */
|
|
113
|
-
includeSources?: boolean;
|
|
114
|
-
/** Generate response (vs just retrieve) */
|
|
115
|
-
generate?: boolean;
|
|
116
|
-
/** Custom system prompt for generation */
|
|
117
|
-
systemPrompt?: string;
|
|
118
|
-
/** Temperature for generation */
|
|
119
|
-
temperature?: number;
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Query response
|
|
123
|
-
*/
|
|
124
|
-
export interface RAGResponse {
|
|
125
|
-
/** Generated answer (if generate=true) */
|
|
126
|
-
answer?: string;
|
|
127
|
-
/** Retrieved context chunks */
|
|
128
|
-
context: string;
|
|
129
|
-
/** Source documents/chunks */
|
|
130
|
-
sources: Array<{
|
|
131
|
-
id: string;
|
|
132
|
-
text: string;
|
|
133
|
-
score: number;
|
|
134
|
-
metadata?: Record<string, unknown>;
|
|
135
|
-
}>;
|
|
136
|
-
/** Query metadata */
|
|
137
|
-
metadata: {
|
|
138
|
-
queryTime: number;
|
|
139
|
-
retrievalMethod: string;
|
|
140
|
-
chunksRetrieved: number;
|
|
141
|
-
reranked: boolean;
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
/**
|
|
145
|
-
* Pipeline statistics
|
|
146
|
-
*/
|
|
147
|
-
export interface PipelineStats {
|
|
148
|
-
totalDocuments: number;
|
|
149
|
-
totalChunks: number;
|
|
150
|
-
indexName: string;
|
|
151
|
-
embeddingDimension?: number;
|
|
152
|
-
hybridSearchEnabled: boolean;
|
|
153
|
-
graphRAGEnabled: boolean;
|
|
154
|
-
}
|
|
29
|
+
import type { RAGPipelineConfig, IngestOptions, QueryOptions, RAGResponse, PipelineStats } from "../../types/ragTypes.js";
|
|
30
|
+
export type { EmbeddingModelConfig } from "../../types/ragTypes.js";
|
|
31
|
+
export type { GenerationModelConfig } from "../../types/ragTypes.js";
|
|
32
|
+
export type { RAGPipelineConfig } from "../../types/ragTypes.js";
|
|
33
|
+
export type { IngestOptions } from "../../types/ragTypes.js";
|
|
34
|
+
export type { QueryOptions } from "../../types/ragTypes.js";
|
|
35
|
+
export type { RAGResponse } from "../../types/ragTypes.js";
|
|
36
|
+
export type { PipelineStats } from "../../types/ragTypes.js";
|
|
155
37
|
/**
|
|
156
38
|
* RAG Pipeline Orchestrator
|
|
157
39
|
*
|
|
@@ -89,7 +89,7 @@ export class RAGPipeline {
|
|
|
89
89
|
this.hybridSearch = createHybridSearch({
|
|
90
90
|
vectorStore: this.vectorStore,
|
|
91
91
|
bm25Index: this.bm25Index,
|
|
92
|
-
indexName: this.config.indexName,
|
|
92
|
+
indexName: this.config.indexName ?? "default",
|
|
93
93
|
embeddingModel: this.config.embeddingModel,
|
|
94
94
|
});
|
|
95
95
|
}
|
|
@@ -103,9 +103,9 @@ export class RAGPipeline {
|
|
|
103
103
|
*/
|
|
104
104
|
async ingest(sources, options) {
|
|
105
105
|
await this.ensureInitialized();
|
|
106
|
-
const strategy = options?.strategy || this.config.defaultChunkingStrategy;
|
|
107
|
-
const chunkSize = options?.chunkSize || this.config.defaultChunkSize;
|
|
108
|
-
const chunkOverlap = options?.chunkOverlap || this.config.defaultChunkOverlap;
|
|
106
|
+
const strategy = options?.strategy || this.config.defaultChunkingStrategy || "recursive";
|
|
107
|
+
const chunkSize = options?.chunkSize || this.config.defaultChunkSize || 1000;
|
|
108
|
+
const chunkOverlap = options?.chunkOverlap || this.config.defaultChunkOverlap || 200;
|
|
109
109
|
let documentsProcessed = 0;
|
|
110
110
|
let chunksCreated = 0;
|
|
111
111
|
for (const source of sources) {
|
|
@@ -137,13 +137,13 @@ export class RAGPipeline {
|
|
|
137
137
|
const embeddings = doc.getEmbeddings();
|
|
138
138
|
// Store in vector store
|
|
139
139
|
await this.vectorStore.query({
|
|
140
|
-
indexName: this.config.indexName,
|
|
140
|
+
indexName: this.config.indexName ?? "default",
|
|
141
141
|
queryVector: embeddings[0],
|
|
142
142
|
topK: 1,
|
|
143
143
|
}); // Warm up
|
|
144
144
|
// Upsert into vector store
|
|
145
145
|
if ("upsert" in this.vectorStore) {
|
|
146
|
-
await this.vectorStore.upsert(this.config.indexName, chunks.map((chunk, i) => ({
|
|
146
|
+
await this.vectorStore.upsert(this.config.indexName ?? "default", chunks.map((chunk, i) => ({
|
|
147
147
|
id: chunk.id,
|
|
148
148
|
vector: embeddings[i],
|
|
149
149
|
metadata: { ...chunk.metadata, text: chunk.text },
|
|
@@ -197,7 +197,7 @@ export class RAGPipeline {
|
|
|
197
197
|
async query(query, options) {
|
|
198
198
|
await this.ensureInitialized();
|
|
199
199
|
const startTime = Date.now();
|
|
200
|
-
const topK = options?.topK || this.config.defaultTopK;
|
|
200
|
+
const topK = options?.topK || this.config.defaultTopK || 5;
|
|
201
201
|
const useHybrid = options?.hybrid ?? this.config.enableHybridSearch;
|
|
202
202
|
const useGraph = options?.graph ?? this.config.enableGraphRAG;
|
|
203
203
|
const useRerank = options?.rerank ?? this.config.enableReranking;
|
|
@@ -233,7 +233,7 @@ export class RAGPipeline {
|
|
|
233
233
|
else {
|
|
234
234
|
// Vector search
|
|
235
235
|
results = await this.vectorStore.query({
|
|
236
|
-
indexName: this.config.indexName,
|
|
236
|
+
indexName: this.config.indexName ?? "default",
|
|
237
237
|
queryVector: queryEmbedding,
|
|
238
238
|
topK: topK * 2,
|
|
239
239
|
filter: options?.filter,
|
|
@@ -293,10 +293,10 @@ export class RAGPipeline {
|
|
|
293
293
|
return {
|
|
294
294
|
totalDocuments: this.documents.size,
|
|
295
295
|
totalChunks: this.allChunks.length,
|
|
296
|
-
indexName: this.config.indexName,
|
|
296
|
+
indexName: this.config.indexName ?? "default",
|
|
297
297
|
embeddingDimension: this.allChunks[0]?.embedding?.length,
|
|
298
|
-
hybridSearchEnabled: this.config.enableHybridSearch,
|
|
299
|
-
graphRAGEnabled: this.config.enableGraphRAG,
|
|
298
|
+
hybridSearchEnabled: this.config.enableHybridSearch ?? false,
|
|
299
|
+
graphRAGEnabled: this.config.enableGraphRAG ?? false,
|
|
300
300
|
};
|
|
301
301
|
}
|
|
302
302
|
/**
|
|
@@ -12,52 +12,13 @@
|
|
|
12
12
|
* - Context summarization
|
|
13
13
|
*/
|
|
14
14
|
import type { Chunk, VectorQueryResult } from "../types.js";
|
|
15
|
+
import type { ContextAssemblyOptions, ContextWindow } from "../../types/ragTypes.js";
|
|
15
16
|
/**
|
|
16
17
|
* Citation format options
|
|
17
18
|
*/
|
|
18
19
|
export type CitationFormat = "inline" | "footnote" | "numbered" | "none";
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
*/
|
|
22
|
-
export interface ContextAssemblyOptions {
|
|
23
|
-
/** Maximum characters in assembled context */
|
|
24
|
-
maxChars?: number;
|
|
25
|
-
/** Maximum tokens (approximate, 4 chars/token) */
|
|
26
|
-
maxTokens?: number;
|
|
27
|
-
/** Citation format to use */
|
|
28
|
-
citationFormat?: CitationFormat;
|
|
29
|
-
/** Separator between chunks */
|
|
30
|
-
separator?: string;
|
|
31
|
-
/** Include chunk metadata in context */
|
|
32
|
-
includeMetadata?: boolean;
|
|
33
|
-
/** Deduplicate overlapping content */
|
|
34
|
-
deduplicate?: boolean;
|
|
35
|
-
/** Similarity threshold for deduplication (0-1) */
|
|
36
|
-
dedupeThreshold?: number;
|
|
37
|
-
/** Order by relevance score */
|
|
38
|
-
orderByRelevance?: boolean;
|
|
39
|
-
/** Include section headers */
|
|
40
|
-
includeSectionHeaders?: boolean;
|
|
41
|
-
/** Header template (use {index}, {source}, {score} placeholders) */
|
|
42
|
-
headerTemplate?: string;
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Context window representation
|
|
46
|
-
*/
|
|
47
|
-
export interface ContextWindow {
|
|
48
|
-
/** Assembled context text */
|
|
49
|
-
text: string;
|
|
50
|
-
/** Number of chunks included */
|
|
51
|
-
chunkCount: number;
|
|
52
|
-
/** Total character count */
|
|
53
|
-
charCount: number;
|
|
54
|
-
/** Estimated token count */
|
|
55
|
-
tokenCount: number;
|
|
56
|
-
/** Chunks that were truncated/excluded */
|
|
57
|
-
truncatedChunks: number;
|
|
58
|
-
/** Citation map (id -> citation text) */
|
|
59
|
-
citations: Map<string, string>;
|
|
60
|
-
}
|
|
20
|
+
export type { ContextAssemblyOptions } from "../../types/ragTypes.js";
|
|
21
|
+
export type { ContextWindow } from "../../types/ragTypes.js";
|
|
61
22
|
/**
|
|
62
23
|
* Assemble context from retrieved results
|
|
63
24
|
*
|