@framers/agentos 0.1.120 → 0.1.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/api/agency.d.ts.map +1 -1
- package/dist/api/agency.js +227 -84
- package/dist/api/agency.js.map +1 -1
- package/dist/api/analyzeVideo.d.ts +127 -0
- package/dist/api/analyzeVideo.d.ts.map +1 -0
- package/dist/api/analyzeVideo.js +136 -0
- package/dist/api/analyzeVideo.js.map +1 -0
- package/dist/api/detectScenes.d.ts +82 -0
- package/dist/api/detectScenes.d.ts.map +1 -0
- package/dist/api/detectScenes.js +67 -0
- package/dist/api/detectScenes.js.map +1 -0
- package/dist/api/generateImage.d.ts +7 -0
- package/dist/api/generateImage.d.ts.map +1 -1
- package/dist/api/generateImage.js +133 -9
- package/dist/api/generateImage.js.map +1 -1
- package/dist/api/generateMusic.d.ts +98 -0
- package/dist/api/generateMusic.d.ts.map +1 -0
- package/dist/api/generateMusic.js +319 -0
- package/dist/api/generateMusic.js.map +1 -0
- package/dist/api/generateSFX.d.ts +96 -0
- package/dist/api/generateSFX.d.ts.map +1 -0
- package/dist/api/generateSFX.js +317 -0
- package/dist/api/generateSFX.js.map +1 -0
- package/dist/api/generateVideo.d.ts +113 -0
- package/dist/api/generateVideo.d.ts.map +1 -0
- package/dist/api/generateVideo.js +342 -0
- package/dist/api/generateVideo.js.map +1 -0
- package/dist/api/model.d.ts.map +1 -1
- package/dist/api/model.js +8 -4
- package/dist/api/model.js.map +1 -1
- package/dist/api/performOCR.d.ts +169 -0
- package/dist/api/performOCR.d.ts.map +1 -0
- package/dist/api/performOCR.js +198 -0
- package/dist/api/performOCR.js.map +1 -0
- package/dist/api/provider-defaults.d.ts +7 -5
- package/dist/api/provider-defaults.d.ts.map +1 -1
- package/dist/api/provider-defaults.js +32 -10
- package/dist/api/provider-defaults.js.map +1 -1
- package/dist/api/strategies/debate.d.ts.map +1 -1
- package/dist/api/strategies/debate.js +1 -0
- package/dist/api/strategies/debate.js.map +1 -1
- package/dist/api/strategies/graph.d.ts.map +1 -1
- package/dist/api/strategies/graph.js +69 -13
- package/dist/api/strategies/graph.js.map +1 -1
- package/dist/api/strategies/hierarchical.d.ts.map +1 -1
- package/dist/api/strategies/hierarchical.js +1 -0
- package/dist/api/strategies/hierarchical.js.map +1 -1
- package/dist/api/strategies/parallel.d.ts.map +1 -1
- package/dist/api/strategies/parallel.js +1 -0
- package/dist/api/strategies/parallel.js.map +1 -1
- package/dist/api/strategies/review-loop.d.ts.map +1 -1
- package/dist/api/strategies/review-loop.js +1 -0
- package/dist/api/strategies/review-loop.js.map +1 -1
- package/dist/api/strategies/sequential.d.ts.map +1 -1
- package/dist/api/strategies/sequential.js +54 -48
- package/dist/api/strategies/sequential.js.map +1 -1
- package/dist/api/streamBuffer.d.ts +20 -0
- package/dist/api/streamBuffer.d.ts.map +1 -0
- package/dist/api/streamBuffer.js +81 -0
- package/dist/api/streamBuffer.js.map +1 -0
- package/dist/api/types.d.ts +145 -5
- package/dist/api/types.d.ts.map +1 -1
- package/dist/api/types.js.map +1 -1
- package/dist/channels/adapters/RedditChannelAdapter.js.map +1 -1
- package/dist/core/audio/AudioProcessor.d.ts.map +1 -1
- package/dist/core/audio/AudioProcessor.js +1 -0
- package/dist/core/audio/AudioProcessor.js.map +1 -1
- package/dist/core/audio/EnvironmentalCalibrator.d.ts.map +1 -1
- package/dist/core/audio/EnvironmentalCalibrator.js +1 -0
- package/dist/core/audio/EnvironmentalCalibrator.js.map +1 -1
- package/dist/core/audio/FallbackAudioProxy.d.ts +169 -0
- package/dist/core/audio/FallbackAudioProxy.d.ts.map +1 -0
- package/dist/core/audio/FallbackAudioProxy.js +236 -0
- package/dist/core/audio/FallbackAudioProxy.js.map +1 -0
- package/dist/core/audio/IAudioGenerator.d.ts +103 -0
- package/dist/core/audio/IAudioGenerator.d.ts.map +1 -0
- package/dist/core/audio/IAudioGenerator.js +24 -0
- package/dist/core/audio/IAudioGenerator.js.map +1 -0
- package/dist/core/audio/index.d.ts +54 -0
- package/dist/core/audio/index.d.ts.map +1 -1
- package/dist/core/audio/index.js +93 -0
- package/dist/core/audio/index.js.map +1 -1
- package/dist/core/audio/providers/AudioGenLocalProvider.d.ts +136 -0
- package/dist/core/audio/providers/AudioGenLocalProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/AudioGenLocalProvider.js +235 -0
- package/dist/core/audio/providers/AudioGenLocalProvider.js.map +1 -0
- package/dist/core/audio/providers/ElevenLabsSFXProvider.d.ts +107 -0
- package/dist/core/audio/providers/ElevenLabsSFXProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/ElevenLabsSFXProvider.js +154 -0
- package/dist/core/audio/providers/ElevenLabsSFXProvider.js.map +1 -0
- package/dist/core/audio/providers/FalAudioProvider.d.ts +207 -0
- package/dist/core/audio/providers/FalAudioProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/FalAudioProvider.js +315 -0
- package/dist/core/audio/providers/FalAudioProvider.js.map +1 -0
- package/dist/core/audio/providers/MusicGenLocalProvider.d.ts +136 -0
- package/dist/core/audio/providers/MusicGenLocalProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/MusicGenLocalProvider.js +235 -0
- package/dist/core/audio/providers/MusicGenLocalProvider.js.map +1 -0
- package/dist/core/audio/providers/ReplicateAudioProvider.d.ts +200 -0
- package/dist/core/audio/providers/ReplicateAudioProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/ReplicateAudioProvider.js +346 -0
- package/dist/core/audio/providers/ReplicateAudioProvider.js.map +1 -0
- package/dist/core/audio/providers/StableAudioProvider.d.ts +138 -0
- package/dist/core/audio/providers/StableAudioProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/StableAudioProvider.js +192 -0
- package/dist/core/audio/providers/StableAudioProvider.js.map +1 -0
- package/dist/core/audio/providers/SunoProvider.d.ts +182 -0
- package/dist/core/audio/providers/SunoProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/SunoProvider.js +312 -0
- package/dist/core/audio/providers/SunoProvider.js.map +1 -0
- package/dist/core/audio/providers/UdioProvider.d.ts +177 -0
- package/dist/core/audio/providers/UdioProvider.d.ts.map +1 -0
- package/dist/core/audio/providers/UdioProvider.js +305 -0
- package/dist/core/audio/providers/UdioProvider.js.map +1 -0
- package/dist/core/audio/types.d.ts +257 -0
- package/dist/core/audio/types.d.ts.map +1 -0
- package/dist/core/audio/types.js +21 -0
- package/dist/core/audio/types.js.map +1 -0
- package/dist/core/images/FallbackImageProxy.d.ts +183 -0
- package/dist/core/images/FallbackImageProxy.d.ts.map +1 -0
- package/dist/core/images/FallbackImageProxy.js +283 -0
- package/dist/core/images/FallbackImageProxy.js.map +1 -0
- package/dist/core/images/IImageProvider.d.ts +1 -1
- package/dist/core/images/IImageProvider.d.ts.map +1 -1
- package/dist/core/images/index.d.ts +1 -0
- package/dist/core/images/index.d.ts.map +1 -1
- package/dist/core/images/index.js +1 -0
- package/dist/core/images/index.js.map +1 -1
- package/dist/core/llm/providers/AIModelProviderManager.d.ts +3 -1
- package/dist/core/llm/providers/AIModelProviderManager.d.ts.map +1 -1
- package/dist/core/llm/providers/AIModelProviderManager.js +8 -0
- package/dist/core/llm/providers/AIModelProviderManager.js.map +1 -1
- package/dist/core/llm/providers/errors/ClaudeCodeProviderError.d.ts +52 -0
- package/dist/core/llm/providers/errors/ClaudeCodeProviderError.d.ts.map +1 -0
- package/dist/core/llm/providers/errors/ClaudeCodeProviderError.js +36 -0
- package/dist/core/llm/providers/errors/ClaudeCodeProviderError.js.map +1 -0
- package/dist/core/llm/providers/errors/GeminiCLIProviderError.d.ts +32 -0
- package/dist/core/llm/providers/errors/GeminiCLIProviderError.d.ts.map +1 -0
- package/dist/core/llm/providers/errors/GeminiCLIProviderError.js +27 -0
- package/dist/core/llm/providers/errors/GeminiCLIProviderError.js.map +1 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeCLIBridge.d.ts +38 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeCLIBridge.d.ts.map +1 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeCLIBridge.js +128 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeCLIBridge.js.map +1 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeProvider.d.ts +107 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeProvider.d.ts.map +1 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeProvider.js +504 -0
- package/dist/core/llm/providers/implementations/ClaudeCodeProvider.js.map +1 -0
- package/dist/core/llm/providers/implementations/GeminiCLIBridge.d.ts +60 -0
- package/dist/core/llm/providers/implementations/GeminiCLIBridge.d.ts.map +1 -0
- package/dist/core/llm/providers/implementations/GeminiCLIBridge.js +177 -0
- package/dist/core/llm/providers/implementations/GeminiCLIBridge.js.map +1 -0
- package/dist/core/llm/providers/implementations/GeminiCLIProvider.d.ts +55 -0
- package/dist/core/llm/providers/implementations/GeminiCLIProvider.d.ts.map +1 -0
- package/dist/core/llm/providers/implementations/GeminiCLIProvider.js +447 -0
- package/dist/core/llm/providers/implementations/GeminiCLIProvider.js.map +1 -0
- package/dist/core/media/ProviderPreferences.d.ts +158 -0
- package/dist/core/media/ProviderPreferences.d.ts.map +1 -0
- package/dist/core/media/ProviderPreferences.js +183 -0
- package/dist/core/media/ProviderPreferences.js.map +1 -0
- package/dist/core/subprocess/CLIRegistry.d.ts +71 -0
- package/dist/core/subprocess/CLIRegistry.d.ts.map +1 -0
- package/dist/core/subprocess/CLIRegistry.js +210 -0
- package/dist/core/subprocess/CLIRegistry.js.map +1 -0
- package/dist/core/subprocess/CLISubprocessBridge.d.ts +117 -0
- package/dist/core/subprocess/CLISubprocessBridge.d.ts.map +1 -0
- package/dist/core/subprocess/CLISubprocessBridge.js +199 -0
- package/dist/core/subprocess/CLISubprocessBridge.js.map +1 -0
- package/dist/core/subprocess/errors.d.ts +76 -0
- package/dist/core/subprocess/errors.d.ts.map +1 -0
- package/dist/core/subprocess/errors.js +75 -0
- package/dist/core/subprocess/errors.js.map +1 -0
- package/dist/core/subprocess/index.d.ts +11 -0
- package/dist/core/subprocess/index.d.ts.map +1 -0
- package/dist/core/subprocess/index.js +10 -0
- package/dist/core/subprocess/index.js.map +1 -0
- package/dist/core/subprocess/types.d.ts +100 -0
- package/dist/core/subprocess/types.d.ts.map +1 -0
- package/dist/core/subprocess/types.js +9 -0
- package/dist/core/subprocess/types.js.map +1 -0
- package/dist/core/video/FallbackVideoProxy.d.ts +166 -0
- package/dist/core/video/FallbackVideoProxy.d.ts.map +1 -0
- package/dist/core/video/FallbackVideoProxy.js +228 -0
- package/dist/core/video/FallbackVideoProxy.js.map +1 -0
- package/dist/core/video/IVideoAnalyzer.d.ts +29 -0
- package/dist/core/video/IVideoAnalyzer.d.ts.map +1 -0
- package/dist/core/video/IVideoAnalyzer.js +12 -0
- package/dist/core/video/IVideoAnalyzer.js.map +1 -0
- package/dist/core/video/IVideoGenerator.d.ts +76 -0
- package/dist/core/video/IVideoGenerator.d.ts.map +1 -0
- package/dist/core/video/IVideoGenerator.js +13 -0
- package/dist/core/video/IVideoGenerator.js.map +1 -0
- package/dist/core/video/VideoAnalyzer.d.ts +278 -0
- package/dist/core/video/VideoAnalyzer.d.ts.map +1 -0
- package/dist/core/video/VideoAnalyzer.js +648 -0
- package/dist/core/video/VideoAnalyzer.js.map +1 -0
- package/dist/core/video/index.d.ts +55 -0
- package/dist/core/video/index.d.ts.map +1 -0
- package/dist/core/video/index.js +78 -0
- package/dist/core/video/index.js.map +1 -0
- package/dist/core/video/providers/FalVideoProvider.d.ts +195 -0
- package/dist/core/video/providers/FalVideoProvider.d.ts.map +1 -0
- package/dist/core/video/providers/FalVideoProvider.js +322 -0
- package/dist/core/video/providers/FalVideoProvider.js.map +1 -0
- package/dist/core/video/providers/ReplicateVideoProvider.d.ts +194 -0
- package/dist/core/video/providers/ReplicateVideoProvider.d.ts.map +1 -0
- package/dist/core/video/providers/ReplicateVideoProvider.js +356 -0
- package/dist/core/video/providers/ReplicateVideoProvider.js.map +1 -0
- package/dist/core/video/providers/RunwayVideoProvider.d.ts +175 -0
- package/dist/core/video/providers/RunwayVideoProvider.d.ts.map +1 -0
- package/dist/core/video/providers/RunwayVideoProvider.js +293 -0
- package/dist/core/video/providers/RunwayVideoProvider.js.map +1 -0
- package/dist/core/video/types.d.ts +441 -0
- package/dist/core/video/types.d.ts.map +1 -0
- package/dist/core/video/types.js +10 -0
- package/dist/core/video/types.js.map +1 -0
- package/dist/core/vision/SceneDetector.d.ts +180 -0
- package/dist/core/vision/SceneDetector.d.ts.map +1 -0
- package/dist/core/vision/SceneDetector.js +366 -0
- package/dist/core/vision/SceneDetector.js.map +1 -0
- package/dist/core/vision/index.d.ts +2 -1
- package/dist/core/vision/index.d.ts.map +1 -1
- package/dist/core/vision/index.js +1 -0
- package/dist/core/vision/index.js.map +1 -1
- package/dist/core/vision/types.d.ts +125 -0
- package/dist/core/vision/types.d.ts.map +1 -1
- package/dist/discovery/CapabilityDiscoveryEngine.d.ts +32 -0
- package/dist/discovery/CapabilityDiscoveryEngine.d.ts.map +1 -1
- package/dist/discovery/CapabilityDiscoveryEngine.js +46 -0
- package/dist/discovery/CapabilityDiscoveryEngine.js.map +1 -1
- package/dist/emergent/AdaptPersonalityTool.d.ts +158 -0
- package/dist/emergent/AdaptPersonalityTool.d.ts.map +1 -0
- package/dist/emergent/AdaptPersonalityTool.js +190 -0
- package/dist/emergent/AdaptPersonalityTool.js.map +1 -0
- package/dist/emergent/CreateWorkflowTool.d.ts +181 -0
- package/dist/emergent/CreateWorkflowTool.d.ts.map +1 -0
- package/dist/emergent/CreateWorkflowTool.js +307 -0
- package/dist/emergent/CreateWorkflowTool.js.map +1 -0
- package/dist/emergent/ManageSkillsTool.d.ts +154 -0
- package/dist/emergent/ManageSkillsTool.d.ts.map +1 -0
- package/dist/emergent/ManageSkillsTool.js +241 -0
- package/dist/emergent/ManageSkillsTool.js.map +1 -0
- package/dist/emergent/PersonalityMutationStore.d.ts +183 -0
- package/dist/emergent/PersonalityMutationStore.d.ts.map +1 -0
- package/dist/emergent/PersonalityMutationStore.js +222 -0
- package/dist/emergent/PersonalityMutationStore.js.map +1 -0
- package/dist/emergent/SelfEvaluateTool.d.ts +189 -0
- package/dist/emergent/SelfEvaluateTool.d.ts.map +1 -0
- package/dist/emergent/SelfEvaluateTool.js +353 -0
- package/dist/emergent/SelfEvaluateTool.js.map +1 -0
- package/dist/emergent/SelfImprovementConfig.d.ts +166 -0
- package/dist/emergent/SelfImprovementConfig.d.ts.map +1 -0
- package/dist/emergent/SelfImprovementConfig.js +43 -0
- package/dist/emergent/SelfImprovementConfig.js.map +1 -0
- package/dist/emergent/index.d.ts +1 -0
- package/dist/emergent/index.d.ts.map +1 -1
- package/dist/emergent/index.js +1 -0
- package/dist/emergent/index.js.map +1 -1
- package/dist/emergent/types.d.ts +9 -0
- package/dist/emergent/types.d.ts.map +1 -1
- package/dist/emergent/types.js.map +1 -1
- package/dist/extensions/MultiRegistryLoader.js.map +1 -1
- package/dist/index.d.ts +17 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -1
- package/dist/memory/CognitiveMemoryManager.d.ts +40 -0
- package/dist/memory/CognitiveMemoryManager.d.ts.map +1 -1
- package/dist/memory/CognitiveMemoryManager.js +54 -1
- package/dist/memory/CognitiveMemoryManager.js.map +1 -1
- package/dist/memory/facade/Memory.d.ts +4 -0
- package/dist/memory/facade/Memory.d.ts.map +1 -1
- package/dist/memory/facade/Memory.js +140 -4
- package/dist/memory/facade/Memory.js.map +1 -1
- package/dist/memory/facade/types.d.ts +30 -2
- package/dist/memory/facade/types.d.ts.map +1 -1
- package/dist/memory/index.d.ts +1 -0
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/index.js +1 -0
- package/dist/memory/index.js.map +1 -1
- package/dist/memory/types.d.ts +15 -0
- package/dist/memory/types.d.ts.map +1 -1
- package/dist/query-router/QueryClassifier.d.ts +192 -21
- package/dist/query-router/QueryClassifier.d.ts.map +1 -1
- package/dist/query-router/QueryClassifier.js +604 -23
- package/dist/query-router/QueryClassifier.js.map +1 -1
- package/dist/query-router/QueryDispatcher.d.ts +106 -8
- package/dist/query-router/QueryDispatcher.d.ts.map +1 -1
- package/dist/query-router/QueryDispatcher.js +387 -8
- package/dist/query-router/QueryDispatcher.js.map +1 -1
- package/dist/query-router/QueryRouter.d.ts +198 -14
- package/dist/query-router/QueryRouter.d.ts.map +1 -1
- package/dist/query-router/QueryRouter.js +738 -50
- package/dist/query-router/QueryRouter.js.map +1 -1
- package/dist/query-router/index.d.ts +1 -1
- package/dist/query-router/index.d.ts.map +1 -1
- package/dist/query-router/index.js +1 -1
- package/dist/query-router/index.js.map +1 -1
- package/dist/query-router/types.d.ts +396 -3
- package/dist/query-router/types.d.ts.map +1 -1
- package/dist/query-router/types.js +35 -0
- package/dist/query-router/types.js.map +1 -1
- package/dist/rag/HydeRetriever.d.ts +108 -0
- package/dist/rag/HydeRetriever.d.ts.map +1 -1
- package/dist/rag/HydeRetriever.js +184 -0
- package/dist/rag/HydeRetriever.js.map +1 -1
- package/dist/rag/IRetrievalAugmentor.d.ts +15 -0
- package/dist/rag/IRetrievalAugmentor.d.ts.map +1 -1
- package/dist/rag/RetrievalAugmentor.d.ts +58 -0
- package/dist/rag/RetrievalAugmentor.d.ts.map +1 -1
- package/dist/rag/RetrievalAugmentor.js +200 -32
- package/dist/rag/RetrievalAugmentor.js.map +1 -1
- package/dist/rag/VectorStoreManager.js +1 -1
- package/dist/rag/audit/RAGAuditCollector.d.ts +7 -0
- package/dist/rag/audit/RAGAuditCollector.d.ts.map +1 -1
- package/dist/rag/audit/RAGAuditCollector.js +10 -0
- package/dist/rag/audit/RAGAuditCollector.js.map +1 -1
- package/dist/rag/audit/RAGAuditTypes.d.ts +10 -1
- package/dist/rag/audit/RAGAuditTypes.d.ts.map +1 -1
- package/dist/rag/chunking/SemanticChunker.d.ts +210 -0
- package/dist/rag/chunking/SemanticChunker.d.ts.map +1 -0
- package/dist/rag/chunking/SemanticChunker.js +460 -0
- package/dist/rag/chunking/SemanticChunker.js.map +1 -0
- package/dist/rag/chunking/index.d.ts +10 -0
- package/dist/rag/chunking/index.d.ts.map +1 -0
- package/dist/rag/chunking/index.js +10 -0
- package/dist/rag/chunking/index.js.map +1 -0
- package/dist/rag/implementations/vector_stores/PineconeVectorStore.d.ts +103 -0
- package/dist/rag/implementations/vector_stores/PineconeVectorStore.d.ts.map +1 -0
- package/dist/rag/implementations/vector_stores/PineconeVectorStore.js +315 -0
- package/dist/rag/implementations/vector_stores/PineconeVectorStore.js.map +1 -0
- package/dist/rag/implementations/vector_stores/PostgresVectorStore.d.ts +107 -0
- package/dist/rag/implementations/vector_stores/PostgresVectorStore.d.ts.map +1 -0
- package/dist/rag/implementations/vector_stores/PostgresVectorStore.js +438 -0
- package/dist/rag/implementations/vector_stores/PostgresVectorStore.js.map +1 -0
- package/dist/rag/index.d.ts +15 -1
- package/dist/rag/index.d.ts.map +1 -1
- package/dist/rag/index.js +32 -0
- package/dist/rag/index.js.map +1 -1
- package/dist/rag/migration/MigrationEngine.d.ts +47 -0
- package/dist/rag/migration/MigrationEngine.d.ts.map +1 -0
- package/dist/rag/migration/MigrationEngine.js +168 -0
- package/dist/rag/migration/MigrationEngine.js.map +1 -0
- package/dist/rag/migration/adapters/PineconeSourceAdapter.d.ts +23 -0
- package/dist/rag/migration/adapters/PineconeSourceAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/PineconeSourceAdapter.js +63 -0
- package/dist/rag/migration/adapters/PineconeSourceAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/PostgresSourceAdapter.d.ts +30 -0
- package/dist/rag/migration/adapters/PostgresSourceAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/PostgresSourceAdapter.js +71 -0
- package/dist/rag/migration/adapters/PostgresSourceAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/PostgresTargetAdapter.d.ts +38 -0
- package/dist/rag/migration/adapters/PostgresTargetAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/PostgresTargetAdapter.js +114 -0
- package/dist/rag/migration/adapters/PostgresTargetAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/QdrantSourceAdapter.d.ts +36 -0
- package/dist/rag/migration/adapters/QdrantSourceAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/QdrantSourceAdapter.js +109 -0
- package/dist/rag/migration/adapters/QdrantSourceAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/QdrantTargetAdapter.d.ts +35 -0
- package/dist/rag/migration/adapters/QdrantTargetAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/QdrantTargetAdapter.js +110 -0
- package/dist/rag/migration/adapters/QdrantTargetAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/SqliteSourceAdapter.d.ts +37 -0
- package/dist/rag/migration/adapters/SqliteSourceAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/SqliteSourceAdapter.js +72 -0
- package/dist/rag/migration/adapters/SqliteSourceAdapter.js.map +1 -0
- package/dist/rag/migration/adapters/SqliteTargetAdapter.d.ts +47 -0
- package/dist/rag/migration/adapters/SqliteTargetAdapter.d.ts.map +1 -0
- package/dist/rag/migration/adapters/SqliteTargetAdapter.js +93 -0
- package/dist/rag/migration/adapters/SqliteTargetAdapter.js.map +1 -0
- package/dist/rag/migration/types.d.ts +108 -0
- package/dist/rag/migration/types.d.ts.map +1 -0
- package/dist/rag/migration/types.js +11 -0
- package/dist/rag/migration/types.js.map +1 -0
- package/dist/rag/multimodal/MultimodalIndexer.d.ts +35 -0
- package/dist/rag/multimodal/MultimodalIndexer.d.ts.map +1 -1
- package/dist/rag/multimodal/MultimodalIndexer.js +66 -1
- package/dist/rag/multimodal/MultimodalIndexer.js.map +1 -1
- package/dist/rag/multimodal/types.d.ts +24 -0
- package/dist/rag/multimodal/types.d.ts.map +1 -1
- package/dist/rag/raptor/RaptorTree.d.ts +268 -0
- package/dist/rag/raptor/RaptorTree.d.ts.map +1 -0
- package/dist/rag/raptor/RaptorTree.js +443 -0
- package/dist/rag/raptor/RaptorTree.js.map +1 -0
- package/dist/rag/raptor/index.d.ts +11 -0
- package/dist/rag/raptor/index.d.ts.map +1 -0
- package/dist/rag/raptor/index.js +11 -0
- package/dist/rag/raptor/index.js.map +1 -0
- package/dist/rag/reranking/providers/CohereReranker.js.map +1 -1
- package/dist/rag/search/BM25Index.d.ts +282 -0
- package/dist/rag/search/BM25Index.d.ts.map +1 -0
- package/dist/rag/search/BM25Index.js +344 -0
- package/dist/rag/search/BM25Index.js.map +1 -0
- package/dist/rag/search/HybridSearcher.d.ts +198 -0
- package/dist/rag/search/HybridSearcher.d.ts.map +1 -0
- package/dist/rag/search/HybridSearcher.js +316 -0
- package/dist/rag/search/HybridSearcher.js.map +1 -0
- package/dist/rag/search/index.d.ts +12 -0
- package/dist/rag/search/index.d.ts.map +1 -0
- package/dist/rag/search/index.js +12 -0
- package/dist/rag/search/index.js.map +1 -0
- package/dist/rag/setup/DockerDetector.d.ts +67 -0
- package/dist/rag/setup/DockerDetector.d.ts.map +1 -0
- package/dist/rag/setup/DockerDetector.js +125 -0
- package/dist/rag/setup/DockerDetector.js.map +1 -0
- package/dist/rag/setup/PostgresSetup.d.ts +20 -0
- package/dist/rag/setup/PostgresSetup.d.ts.map +1 -0
- package/dist/rag/setup/PostgresSetup.js +133 -0
- package/dist/rag/setup/PostgresSetup.js.map +1 -0
- package/dist/rag/setup/QdrantSetup.d.ts +26 -0
- package/dist/rag/setup/QdrantSetup.d.ts.map +1 -0
- package/dist/rag/setup/QdrantSetup.js +96 -0
- package/dist/rag/setup/QdrantSetup.js.map +1 -0
- package/dist/rag/setup/types.d.ts +55 -0
- package/dist/rag/setup/types.d.ts.map +1 -0
- package/dist/rag/setup/types.js +6 -0
- package/dist/rag/setup/types.js.map +1 -0
- package/dist/rag/unified/UnifiedRetriever.d.ts +472 -0
- package/dist/rag/unified/UnifiedRetriever.d.ts.map +1 -0
- package/dist/rag/unified/UnifiedRetriever.js +887 -0
- package/dist/rag/unified/UnifiedRetriever.js.map +1 -0
- package/dist/rag/unified/index.d.ts +24 -0
- package/dist/rag/unified/index.d.ts.map +1 -0
- package/dist/rag/unified/index.js +23 -0
- package/dist/rag/unified/index.js.map +1 -0
- package/dist/rag/unified/types.d.ts +546 -0
- package/dist/rag/unified/types.d.ts.map +1 -0
- package/dist/rag/unified/types.js +177 -0
- package/dist/rag/unified/types.js.map +1 -0
- package/dist/speech/providers/AssemblyAISTTProvider.js.map +1 -1
- package/dist/speech/providers/AzureSpeechSTTProvider.js.map +1 -1
- package/dist/speech/providers/BuiltInAdaptiveVadProvider.d.ts +1 -1
- package/dist/speech/providers/DeepgramBatchSTTProvider.js.map +1 -1
- package/package.json +5 -2
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Qdrant source adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/QdrantSourceAdapter
|
|
4
|
+
*
|
|
5
|
+
* Reads vectors from Qdrant collections using the scroll API.
|
|
6
|
+
* Non-vector data (knowledge graph, documents, etc.) is read from
|
|
7
|
+
* the sidecar SQLite file that accompanies Qdrant deployments.
|
|
8
|
+
*/
|
|
9
|
+
/** Tables stored as Qdrant collections (vector data). */
|
|
10
|
+
const QDRANT_COLLECTIONS = ['memory_traces', 'document_chunks'];
|
|
11
|
+
/** Tables stored in the sidecar SQLite file (non-vector data). */
|
|
12
|
+
const SIDECAR_TABLES = [
|
|
13
|
+
'brain_meta', 'knowledge_nodes', 'knowledge_edges',
|
|
14
|
+
'documents', 'document_images', 'consolidation_log',
|
|
15
|
+
'retrieval_feedback', 'conversations', 'messages',
|
|
16
|
+
];
|
|
17
|
+
export class QdrantSourceAdapter {
|
|
18
|
+
/**
|
|
19
|
+
* @param url - Qdrant base URL (e.g. 'http://localhost:6333').
|
|
20
|
+
* @param apiKey - Optional API key for cloud instances.
|
|
21
|
+
*/
|
|
22
|
+
constructor(url, apiKey) {
|
|
23
|
+
this.url = url;
|
|
24
|
+
this.apiKey = apiKey;
|
|
25
|
+
this.sidecarDb = null; // better-sqlite3 Database for non-vector tables
|
|
26
|
+
}
|
|
27
|
+
/** Build fetch headers with optional API key. */
|
|
28
|
+
_headers() {
|
|
29
|
+
const h = { 'Content-Type': 'application/json' };
|
|
30
|
+
if (this.apiKey)
|
|
31
|
+
h['api-key'] = this.apiKey;
|
|
32
|
+
return h;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* List available tables/collections.
|
|
36
|
+
* Combines Qdrant collections and sidecar SQLite tables.
|
|
37
|
+
*/
|
|
38
|
+
async listTables() {
|
|
39
|
+
const tables = [];
|
|
40
|
+
// Check which Qdrant collections exist.
|
|
41
|
+
try {
|
|
42
|
+
const res = await fetch(`${this.url}/collections`, { headers: this._headers() });
|
|
43
|
+
if (res.ok) {
|
|
44
|
+
const data = await res.json();
|
|
45
|
+
const names = new Set(data.result.collections.map(c => c.name));
|
|
46
|
+
for (const t of QDRANT_COLLECTIONS) {
|
|
47
|
+
if (names.has(t))
|
|
48
|
+
tables.push(t);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// Qdrant not reachable — skip vector tables.
|
|
54
|
+
}
|
|
55
|
+
// Check sidecar SQLite tables.
|
|
56
|
+
// Sidecar path convention: same directory as Qdrant data, file 'sidecar.sqlite'.
|
|
57
|
+
// For source adapter, this is typically not needed (data comes from the API).
|
|
58
|
+
// We include sidecar tables only if the sidecar DB path is available via env.
|
|
59
|
+
// TODO: Accept sidecar path in constructor for full migration support.
|
|
60
|
+
return tables;
|
|
61
|
+
}
|
|
62
|
+
/** Count points in a Qdrant collection. */
|
|
63
|
+
async countRows(table) {
|
|
64
|
+
if (QDRANT_COLLECTIONS.includes(table)) {
|
|
65
|
+
const res = await fetch(`${this.url}/collections/${table}`, { headers: this._headers() });
|
|
66
|
+
if (res.ok) {
|
|
67
|
+
const data = await res.json();
|
|
68
|
+
return data.result.points_count;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Read a batch of points from a Qdrant collection using the scroll API.
|
|
75
|
+
* Converts Qdrant point format to flat row objects.
|
|
76
|
+
*/
|
|
77
|
+
async readBatch(table, offset, limit) {
|
|
78
|
+
if (!QDRANT_COLLECTIONS.includes(table))
|
|
79
|
+
return [];
|
|
80
|
+
const body = JSON.stringify({
|
|
81
|
+
limit,
|
|
82
|
+
offset,
|
|
83
|
+
with_payload: true,
|
|
84
|
+
with_vector: true,
|
|
85
|
+
});
|
|
86
|
+
const res = await fetch(`${this.url}/collections/${table}/points/scroll`, {
|
|
87
|
+
method: 'POST',
|
|
88
|
+
headers: this._headers(),
|
|
89
|
+
body,
|
|
90
|
+
});
|
|
91
|
+
if (!res.ok)
|
|
92
|
+
return [];
|
|
93
|
+
const data = await res.json();
|
|
94
|
+
// Flatten Qdrant point structure into row objects.
|
|
95
|
+
return data.result.points.map(pt => ({
|
|
96
|
+
id: String(pt.id),
|
|
97
|
+
embedding: pt.vector,
|
|
98
|
+
...pt.payload,
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
/** Close connections. */
|
|
102
|
+
async close() {
|
|
103
|
+
if (this.sidecarDb) {
|
|
104
|
+
this.sidecarDb.close();
|
|
105
|
+
this.sidecarDb = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=QdrantSourceAdapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QdrantSourceAdapter.js","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/QdrantSourceAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,yDAAyD;AACzD,MAAM,kBAAkB,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;AAEhE,kEAAkE;AAClE,MAAM,cAAc,GAAG;IACrB,YAAY,EAAE,iBAAiB,EAAE,iBAAiB;IAClD,WAAW,EAAE,iBAAiB,EAAE,mBAAmB;IACnD,oBAAoB,EAAE,eAAe,EAAE,UAAU;CAClD,CAAC;AAEF,MAAM,OAAO,mBAAmB;IAG9B;;;OAGG;IACH,YACmB,GAAW,EACX,MAAe;QADf,QAAG,GAAH,GAAG,CAAQ;QACX,WAAM,GAAN,MAAM,CAAS;QAR1B,cAAS,GAAQ,IAAI,CAAC,CAAC,gDAAgD;IAS5E,CAAC;IAEJ,iDAAiD;IACzC,QAAQ;QACd,MAAM,CAAC,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;QACzE,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5C,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjF,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAqD,CAAC;gBACjF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChE,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;oBACnC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;wBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,6CAA6C;QAC/C,CAAC;QAED,+BAA+B;QAC/B,iFAAiF;QACjF,8EAA8E;QAC9E,8EAA8E;QAC9E,uEAAuE;QAEvE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,2CAA2C;IAC3C,KAAK,CAAC,SAAS,CAAC,KAAa;QAC3B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,gBAAgB,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC1F,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA0C,CAAC;gBACtE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC1D,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEnD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YAC1B,KAAK;YACL,MAAM;YACN,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,gBAAgB,KAAK,gBAAgB,EAAE;YACxE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;YACxB,IAAI;SACL,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAEvB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAE1B,CAAC;QAEF,mDAAmD;QACnD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,GAAG,EAAE,CAAC,OAAO;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,yBAAyB;IACzB,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Qdrant target adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/QdrantTargetAdapter
|
|
4
|
+
*
|
|
5
|
+
* Writes vector data to Qdrant collections and non-vector data
|
|
6
|
+
* to a sidecar SQLite file. Creates collections with appropriate
|
|
7
|
+
* vector configuration on first write.
|
|
8
|
+
*/
|
|
9
|
+
import type { IMigrationTarget } from '../types.js';
|
|
10
|
+
export declare class QdrantTargetAdapter implements IMigrationTarget {
|
|
11
|
+
private readonly url;
|
|
12
|
+
private readonly apiKey?;
|
|
13
|
+
private createdCollections;
|
|
14
|
+
private sidecarDb;
|
|
15
|
+
/**
|
|
16
|
+
* @param url - Qdrant base URL.
|
|
17
|
+
* @param apiKey - Optional API key for cloud instances.
|
|
18
|
+
*/
|
|
19
|
+
constructor(url: string, apiKey?: string | undefined);
|
|
20
|
+
/** Build fetch headers. */
|
|
21
|
+
private _headers;
|
|
22
|
+
/**
|
|
23
|
+
* Ensure a Qdrant collection exists with the correct vector configuration.
|
|
24
|
+
* For non-vector tables, ensures the sidecar SQLite file has the table.
|
|
25
|
+
*/
|
|
26
|
+
ensureTable(table: string, sampleRow: Record<string, unknown>): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Write a batch of rows as Qdrant points.
|
|
29
|
+
* Extracts `id` and `embedding` fields; everything else becomes payload.
|
|
30
|
+
*/
|
|
31
|
+
writeBatch(table: string, rows: Record<string, unknown>[]): Promise<number>;
|
|
32
|
+
/** Close connections. */
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=QdrantTargetAdapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QdrantTargetAdapter.d.ts","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/QdrantTargetAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAKpD,qBAAa,mBAAoB,YAAW,gBAAgB;IASxD,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAT1B,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,SAAS,CAAa;IAE9B;;;OAGG;gBAEgB,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,MAAM,YAAA;IAGlC,2BAA2B;IAC3B,OAAO,CAAC,QAAQ;IAMhB;;;OAGG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCnF;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwCjF,yBAAyB;IACnB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Qdrant target adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/QdrantTargetAdapter
|
|
4
|
+
*
|
|
5
|
+
* Writes vector data to Qdrant collections and non-vector data
|
|
6
|
+
* to a sidecar SQLite file. Creates collections with appropriate
|
|
7
|
+
* vector configuration on first write.
|
|
8
|
+
*/
|
|
9
|
+
/** Tables that go into Qdrant as point collections. */
|
|
10
|
+
const QDRANT_COLLECTIONS = new Set(['memory_traces', 'document_chunks']);
|
|
11
|
+
export class QdrantTargetAdapter {
|
|
12
|
+
/**
|
|
13
|
+
* @param url - Qdrant base URL.
|
|
14
|
+
* @param apiKey - Optional API key for cloud instances.
|
|
15
|
+
*/
|
|
16
|
+
constructor(url, apiKey) {
|
|
17
|
+
this.url = url;
|
|
18
|
+
this.apiKey = apiKey;
|
|
19
|
+
this.createdCollections = new Set();
|
|
20
|
+
this.sidecarDb = null; // For non-vector tables
|
|
21
|
+
}
|
|
22
|
+
/** Build fetch headers. */
|
|
23
|
+
_headers() {
|
|
24
|
+
const h = { 'Content-Type': 'application/json' };
|
|
25
|
+
if (this.apiKey)
|
|
26
|
+
h['api-key'] = this.apiKey;
|
|
27
|
+
return h;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Ensure a Qdrant collection exists with the correct vector configuration.
|
|
31
|
+
* For non-vector tables, ensures the sidecar SQLite file has the table.
|
|
32
|
+
*/
|
|
33
|
+
async ensureTable(table, sampleRow) {
|
|
34
|
+
if (this.createdCollections.has(table))
|
|
35
|
+
return;
|
|
36
|
+
if (QDRANT_COLLECTIONS.has(table)) {
|
|
37
|
+
// Determine vector dimensions from sample row.
|
|
38
|
+
const embedding = sampleRow['embedding'];
|
|
39
|
+
const dim = Array.isArray(embedding)
|
|
40
|
+
? embedding.length
|
|
41
|
+
: (embedding instanceof Buffer ? embedding.byteLength / 4 : 1536);
|
|
42
|
+
// Check if collection already exists.
|
|
43
|
+
const checkRes = await fetch(`${this.url}/collections/${table}`, {
|
|
44
|
+
headers: this._headers(),
|
|
45
|
+
});
|
|
46
|
+
if (!checkRes.ok) {
|
|
47
|
+
// Create the collection with cosine distance.
|
|
48
|
+
await fetch(`${this.url}/collections/${table}`, {
|
|
49
|
+
method: 'PUT',
|
|
50
|
+
headers: this._headers(),
|
|
51
|
+
body: JSON.stringify({
|
|
52
|
+
vectors: { size: dim, distance: 'Cosine' },
|
|
53
|
+
}),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
this.createdCollections.add(table);
|
|
57
|
+
}
|
|
58
|
+
// Non-vector tables would go to sidecar SQLite — not implemented here.
|
|
59
|
+
// TODO: Wire sidecar SQLite for graph/document metadata tables.
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Write a batch of rows as Qdrant points.
|
|
63
|
+
* Extracts `id` and `embedding` fields; everything else becomes payload.
|
|
64
|
+
*/
|
|
65
|
+
async writeBatch(table, rows) {
|
|
66
|
+
if (rows.length === 0)
|
|
67
|
+
return 0;
|
|
68
|
+
if (QDRANT_COLLECTIONS.has(table)) {
|
|
69
|
+
// Convert rows to Qdrant point format.
|
|
70
|
+
const points = rows.map(row => {
|
|
71
|
+
const { id, embedding, ...payload } = row;
|
|
72
|
+
// Convert Buffer embedding to number[] if needed.
|
|
73
|
+
let vector;
|
|
74
|
+
if (embedding instanceof Buffer) {
|
|
75
|
+
const f32 = new Float32Array(embedding.buffer, embedding.byteOffset, embedding.byteLength / 4);
|
|
76
|
+
vector = Array.from(f32);
|
|
77
|
+
}
|
|
78
|
+
else if (Array.isArray(embedding)) {
|
|
79
|
+
vector = embedding;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
vector = []; // Fallback — shouldn't happen.
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
id: String(id),
|
|
86
|
+
vector,
|
|
87
|
+
payload,
|
|
88
|
+
};
|
|
89
|
+
});
|
|
90
|
+
// Upsert points in a single batch request.
|
|
91
|
+
const res = await fetch(`${this.url}/collections/${table}/points`, {
|
|
92
|
+
method: 'PUT',
|
|
93
|
+
headers: this._headers(),
|
|
94
|
+
body: JSON.stringify({ points }),
|
|
95
|
+
});
|
|
96
|
+
return res.ok ? points.length : 0;
|
|
97
|
+
}
|
|
98
|
+
// Non-vector tables: would write to sidecar SQLite.
|
|
99
|
+
// TODO: Implement sidecar write for graph/metadata tables.
|
|
100
|
+
return 0;
|
|
101
|
+
}
|
|
102
|
+
/** Close connections. */
|
|
103
|
+
async close() {
|
|
104
|
+
if (this.sidecarDb) {
|
|
105
|
+
this.sidecarDb.close();
|
|
106
|
+
this.sidecarDb = null;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=QdrantTargetAdapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QdrantTargetAdapter.js","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/QdrantTargetAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAI9B;;;OAGG;IACH,YACmB,GAAW,EACX,MAAe;QADf,QAAG,GAAH,GAAG,CAAQ;QACX,WAAM,GAAN,MAAM,CAAS;QAT1B,uBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,cAAS,GAAQ,IAAI,CAAC,CAAC,wBAAwB;IASpD,CAAC;IAEJ,2BAA2B;IACnB,QAAQ;QACd,MAAM,CAAC,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;QACzE,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5C,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,SAAkC;QACjE,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QAE/C,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,+CAA+C;YAC/C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;gBAClC,CAAC,CAAC,SAAS,CAAC,MAAM;gBAClB,CAAC,CAAC,CAAC,SAAS,YAAY,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEpE,sCAAsC;YACtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,gBAAgB,KAAK,EAAE,EAAE;gBAC/D,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;aACzB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,8CAA8C;gBAC9C,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,gBAAgB,KAAK,EAAE,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACxB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE;qBAC3C,CAAC;iBACH,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QACD,uEAAuE;QACvE,gEAAgE;IAClE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,IAA+B;QAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,uCAAuC;YACvC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC;gBAC1C,kDAAkD;gBAClD,IAAI,MAAgB,CAAC;gBACrB,IAAI,SAAS,YAAY,MAAM,EAAE,CAAC;oBAChC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;oBAC/F,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;qBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;oBACpC,MAAM,GAAG,SAAqB,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,EAAE,CAAC,CAAC,+BAA+B;gBAC9C,CAAC;gBAED,OAAO;oBACL,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;oBACd,MAAM;oBACN,OAAO;iBACR,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,2CAA2C;YAC3C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,gBAAgB,KAAK,SAAS,EAAE;gBACjE,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACxB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;aACjC,CAAC,CAAC;YAEH,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;QAED,oDAAoD;QACpD,2DAA2D;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IAED,yBAAyB;IACzB,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite source adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/SqliteSourceAdapter
|
|
4
|
+
*
|
|
5
|
+
* Reads data from a brain.sqlite file in streaming batches using
|
|
6
|
+
* better-sqlite3's synchronous API (fastest for local reads).
|
|
7
|
+
*/
|
|
8
|
+
import type { IMigrationSource } from '../types.js';
|
|
9
|
+
export declare class SqliteSourceAdapter implements IMigrationSource {
|
|
10
|
+
private db;
|
|
11
|
+
/**
|
|
12
|
+
* @param path - Path to the source brain.sqlite file. Opened read-only.
|
|
13
|
+
*/
|
|
14
|
+
constructor(path: string);
|
|
15
|
+
/**
|
|
16
|
+
* List tables that exist in the source database AND are part of
|
|
17
|
+
* the AgentOS memory schema. Unknown tables are skipped.
|
|
18
|
+
*/
|
|
19
|
+
listTables(): Promise<string[]>;
|
|
20
|
+
/**
|
|
21
|
+
* Count rows in a table.
|
|
22
|
+
* @param table - Table name.
|
|
23
|
+
*/
|
|
24
|
+
countRows(table: string): Promise<number>;
|
|
25
|
+
/**
|
|
26
|
+
* Read a batch of rows from a table using LIMIT/OFFSET pagination.
|
|
27
|
+
*
|
|
28
|
+
* @param table - Table name.
|
|
29
|
+
* @param offset - Number of rows to skip.
|
|
30
|
+
* @param limit - Maximum rows to return.
|
|
31
|
+
* @returns Array of row objects with column name → value mappings.
|
|
32
|
+
*/
|
|
33
|
+
readBatch(table: string, offset: number, limit: number): Promise<Record<string, unknown>[]>;
|
|
34
|
+
/** Close the read-only database connection. */
|
|
35
|
+
close(): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=SqliteSourceAdapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteSourceAdapter.d.ts","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/SqliteSourceAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAqBpD,qBAAa,mBAAoB,YAAW,gBAAgB;IAC1D,OAAO,CAAC,EAAE,CAAoB;IAE9B;;OAEG;gBACS,IAAI,EAAE,MAAM;IAIxB;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IASrC;;;OAGG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK/C;;;;;;;OAOG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAMjG,+CAA+C;IACzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite source adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/SqliteSourceAdapter
|
|
4
|
+
*
|
|
5
|
+
* Reads data from a brain.sqlite file in streaming batches using
|
|
6
|
+
* better-sqlite3's synchronous API (fastest for local reads).
|
|
7
|
+
*/
|
|
8
|
+
import Database from 'better-sqlite3';
|
|
9
|
+
/**
|
|
10
|
+
* Tables that are part of the AgentOS memory schema and should
|
|
11
|
+
* be included in migrations. Order matters — foreign key dependencies
|
|
12
|
+
* require parent tables to be migrated before children.
|
|
13
|
+
*/
|
|
14
|
+
const MIGRATION_TABLES = [
|
|
15
|
+
'brain_meta',
|
|
16
|
+
'memory_traces',
|
|
17
|
+
'knowledge_nodes',
|
|
18
|
+
'knowledge_edges',
|
|
19
|
+
'documents',
|
|
20
|
+
'document_chunks',
|
|
21
|
+
'document_images',
|
|
22
|
+
'consolidation_log',
|
|
23
|
+
'retrieval_feedback',
|
|
24
|
+
'conversations',
|
|
25
|
+
'messages',
|
|
26
|
+
];
|
|
27
|
+
export class SqliteSourceAdapter {
|
|
28
|
+
/**
|
|
29
|
+
* @param path - Path to the source brain.sqlite file. Opened read-only.
|
|
30
|
+
*/
|
|
31
|
+
constructor(path) {
|
|
32
|
+
this.db = new Database(path, { readonly: true });
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* List tables that exist in the source database AND are part of
|
|
36
|
+
* the AgentOS memory schema. Unknown tables are skipped.
|
|
37
|
+
*/
|
|
38
|
+
async listTables() {
|
|
39
|
+
const rows = this.db
|
|
40
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
|
41
|
+
.all();
|
|
42
|
+
const existing = new Set(rows.map(r => r.name));
|
|
43
|
+
// Return only known tables that actually exist, in dependency order.
|
|
44
|
+
return MIGRATION_TABLES.filter(t => existing.has(t));
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Count rows in a table.
|
|
48
|
+
* @param table - Table name.
|
|
49
|
+
*/
|
|
50
|
+
async countRows(table) {
|
|
51
|
+
const row = this.db.prepare(`SELECT COUNT(*) as c FROM "${table}"`).get();
|
|
52
|
+
return row.c;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Read a batch of rows from a table using LIMIT/OFFSET pagination.
|
|
56
|
+
*
|
|
57
|
+
* @param table - Table name.
|
|
58
|
+
* @param offset - Number of rows to skip.
|
|
59
|
+
* @param limit - Maximum rows to return.
|
|
60
|
+
* @returns Array of row objects with column name → value mappings.
|
|
61
|
+
*/
|
|
62
|
+
async readBatch(table, offset, limit) {
|
|
63
|
+
return this.db
|
|
64
|
+
.prepare(`SELECT * FROM "${table}" LIMIT ? OFFSET ?`)
|
|
65
|
+
.all(limit, offset);
|
|
66
|
+
}
|
|
67
|
+
/** Close the read-only database connection. */
|
|
68
|
+
async close() {
|
|
69
|
+
this.db.close();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=SqliteSourceAdapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteSourceAdapter.js","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/SqliteSourceAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC;;;;GAIG;AACH,MAAM,gBAAgB,GAAG;IACvB,YAAY;IACZ,eAAe;IACf,iBAAiB;IACjB,iBAAiB;IACjB,WAAW;IACX,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,oBAAoB;IACpB,eAAe;IACf,UAAU;CACX,CAAC;AAEF,MAAM,OAAO,mBAAmB;IAG9B;;OAEG;IACH,YAAY,IAAY;QACtB,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,gFAAgF,CAAC;aACzF,GAAG,EAAwB,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAChD,qEAAqE;QACrE,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAa;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8BAA8B,KAAK,GAAG,CAAC,CAAC,GAAG,EAAmB,CAAC;QAC3F,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC1D,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CAAC,kBAAkB,KAAK,oBAAoB,CAAC;aACpD,GAAG,CAAC,KAAK,EAAE,MAAM,CAA8B,CAAC;IACrD,CAAC;IAED,+CAA+C;IAC/C,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite target adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/SqliteTargetAdapter
|
|
4
|
+
*
|
|
5
|
+
* Writes migration data to a new or existing brain.sqlite file.
|
|
6
|
+
* Uses WAL mode for concurrent read safety and wraps each batch
|
|
7
|
+
* in a transaction for atomicity and performance.
|
|
8
|
+
*/
|
|
9
|
+
import type { IMigrationTarget } from '../types.js';
|
|
10
|
+
export declare class SqliteTargetAdapter implements IMigrationTarget {
|
|
11
|
+
private db;
|
|
12
|
+
/** Track which tables we've already created to avoid redundant DDL. */
|
|
13
|
+
private createdTables;
|
|
14
|
+
/**
|
|
15
|
+
* @param path - Path to the target brain.sqlite file. Created if it doesn't exist.
|
|
16
|
+
*/
|
|
17
|
+
constructor(path: string);
|
|
18
|
+
/**
|
|
19
|
+
* Ensure the target table exists. Infers column types from a sample row:
|
|
20
|
+
* - Buffer → BLOB
|
|
21
|
+
* - number (integer) → INTEGER
|
|
22
|
+
* - number (float) → REAL
|
|
23
|
+
* - everything else → TEXT
|
|
24
|
+
*
|
|
25
|
+
* Uses CREATE TABLE IF NOT EXISTS so it's safe to call multiple times.
|
|
26
|
+
*
|
|
27
|
+
* @param table - Table name to create.
|
|
28
|
+
* @param sampleRow - A sample row to derive column types from.
|
|
29
|
+
*/
|
|
30
|
+
ensureTable(table: string, sampleRow: Record<string, unknown>): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Write a batch of rows to the target table.
|
|
33
|
+
* Uses INSERT OR REPLACE to handle duplicates (requires a PRIMARY KEY
|
|
34
|
+
* or UNIQUE constraint — if none exists, rows are simply inserted).
|
|
35
|
+
*
|
|
36
|
+
* Wraps the entire batch in a single transaction for atomicity and
|
|
37
|
+
* dramatically better write performance (avoids per-row fsync).
|
|
38
|
+
*
|
|
39
|
+
* @param table - Table name to write to.
|
|
40
|
+
* @param rows - Array of row objects.
|
|
41
|
+
* @returns Number of rows written.
|
|
42
|
+
*/
|
|
43
|
+
writeBatch(table: string, rows: Record<string, unknown>[]): Promise<number>;
|
|
44
|
+
/** Close the database connection. */
|
|
45
|
+
close(): Promise<void>;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=SqliteTargetAdapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteTargetAdapter.d.ts","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/SqliteTargetAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD,qBAAa,mBAAoB,YAAW,gBAAgB;IAC1D,OAAO,CAAC,EAAE,CAAoB;IAC9B,uEAAuE;IACvE,OAAO,CAAC,aAAa,CAAqB;IAE1C;;OAEG;gBACS,IAAI,EAAE,MAAM;IAMxB;;;;;;;;;;;OAWG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBnF;;;;;;;;;;;OAWG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IA8BjF,qCAAqC;IAC/B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite target adapter for the migration engine.
|
|
3
|
+
* @module rag/migration/adapters/SqliteTargetAdapter
|
|
4
|
+
*
|
|
5
|
+
* Writes migration data to a new or existing brain.sqlite file.
|
|
6
|
+
* Uses WAL mode for concurrent read safety and wraps each batch
|
|
7
|
+
* in a transaction for atomicity and performance.
|
|
8
|
+
*/
|
|
9
|
+
import Database from 'better-sqlite3';
|
|
10
|
+
export class SqliteTargetAdapter {
|
|
11
|
+
/**
|
|
12
|
+
* @param path - Path to the target brain.sqlite file. Created if it doesn't exist.
|
|
13
|
+
*/
|
|
14
|
+
constructor(path) {
|
|
15
|
+
/** Track which tables we've already created to avoid redundant DDL. */
|
|
16
|
+
this.createdTables = new Set();
|
|
17
|
+
this.db = new Database(path);
|
|
18
|
+
this.db.pragma('journal_mode = WAL');
|
|
19
|
+
this.db.pragma('foreign_keys = ON');
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Ensure the target table exists. Infers column types from a sample row:
|
|
23
|
+
* - Buffer → BLOB
|
|
24
|
+
* - number (integer) → INTEGER
|
|
25
|
+
* - number (float) → REAL
|
|
26
|
+
* - everything else → TEXT
|
|
27
|
+
*
|
|
28
|
+
* Uses CREATE TABLE IF NOT EXISTS so it's safe to call multiple times.
|
|
29
|
+
*
|
|
30
|
+
* @param table - Table name to create.
|
|
31
|
+
* @param sampleRow - A sample row to derive column types from.
|
|
32
|
+
*/
|
|
33
|
+
async ensureTable(table, sampleRow) {
|
|
34
|
+
if (this.createdTables.has(table))
|
|
35
|
+
return;
|
|
36
|
+
const columns = Object.keys(sampleRow);
|
|
37
|
+
const colDefs = columns.map(col => {
|
|
38
|
+
const val = sampleRow[col];
|
|
39
|
+
// Infer SQLite column type from the sample value.
|
|
40
|
+
if (val instanceof Buffer || val instanceof Uint8Array)
|
|
41
|
+
return `"${col}" BLOB`;
|
|
42
|
+
if (typeof val === 'number')
|
|
43
|
+
return `"${col}" ${Number.isInteger(val) ? 'INTEGER' : 'REAL'}`;
|
|
44
|
+
if (typeof val === 'boolean')
|
|
45
|
+
return `"${col}" INTEGER`; // SQLite stores booleans as 0/1
|
|
46
|
+
return `"${col}" TEXT`;
|
|
47
|
+
}).join(', ');
|
|
48
|
+
this.db.exec(`CREATE TABLE IF NOT EXISTS "${table}" (${colDefs})`);
|
|
49
|
+
this.createdTables.add(table);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Write a batch of rows to the target table.
|
|
53
|
+
* Uses INSERT OR REPLACE to handle duplicates (requires a PRIMARY KEY
|
|
54
|
+
* or UNIQUE constraint — if none exists, rows are simply inserted).
|
|
55
|
+
*
|
|
56
|
+
* Wraps the entire batch in a single transaction for atomicity and
|
|
57
|
+
* dramatically better write performance (avoids per-row fsync).
|
|
58
|
+
*
|
|
59
|
+
* @param table - Table name to write to.
|
|
60
|
+
* @param rows - Array of row objects.
|
|
61
|
+
* @returns Number of rows written.
|
|
62
|
+
*/
|
|
63
|
+
async writeBatch(table, rows) {
|
|
64
|
+
if (rows.length === 0)
|
|
65
|
+
return 0;
|
|
66
|
+
const columns = Object.keys(rows[0]);
|
|
67
|
+
const quotedCols = columns.map(c => `"${c}"`).join(', ');
|
|
68
|
+
const placeholders = columns.map(() => '?').join(', ');
|
|
69
|
+
const stmt = this.db.prepare(`INSERT OR REPLACE INTO "${table}" (${quotedCols}) VALUES (${placeholders})`);
|
|
70
|
+
// Transaction wrapper: single fsync for the entire batch.
|
|
71
|
+
const tx = this.db.transaction((batch) => {
|
|
72
|
+
let count = 0;
|
|
73
|
+
for (const row of batch) {
|
|
74
|
+
const values = columns.map(c => {
|
|
75
|
+
const v = row[c];
|
|
76
|
+
// Convert booleans to integers for SQLite compatibility.
|
|
77
|
+
if (typeof v === 'boolean')
|
|
78
|
+
return v ? 1 : 0;
|
|
79
|
+
return v;
|
|
80
|
+
});
|
|
81
|
+
stmt.run(...values);
|
|
82
|
+
count++;
|
|
83
|
+
}
|
|
84
|
+
return count;
|
|
85
|
+
});
|
|
86
|
+
return tx(rows);
|
|
87
|
+
}
|
|
88
|
+
/** Close the database connection. */
|
|
89
|
+
async close() {
|
|
90
|
+
this.db.close();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=SqliteTargetAdapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteTargetAdapter.js","sourceRoot":"","sources":["../../../../src/rag/migration/adapters/SqliteTargetAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,MAAM,OAAO,mBAAmB;IAK9B;;OAEG;IACH,YAAY,IAAY;QANxB,uEAAuE;QAC/D,kBAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QAMxC,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,SAAkC;QACjE,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QAE1C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC3B,kDAAkD;YAClD,IAAI,GAAG,YAAY,MAAM,IAAI,GAAG,YAAY,UAAU;gBAAE,OAAO,IAAI,GAAG,QAAQ,CAAC;YAC/E,IAAI,OAAO,GAAG,KAAK,QAAQ;gBAAE,OAAO,IAAI,GAAG,KAAK,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7F,IAAI,OAAO,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,GAAG,WAAW,CAAC,CAAC,gCAAgC;YACzF,OAAO,IAAI,GAAG,QAAQ,CAAC;QACzB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,+BAA+B,KAAK,MAAM,OAAO,GAAG,CAAC,CAAC;QACnE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,IAA+B;QAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC1B,2BAA2B,KAAK,MAAM,UAAU,aAAa,YAAY,GAAG,CAC7E,CAAC;QAEF,0DAA0D;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAgC,EAAE,EAAE;YAClE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAC7B,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;oBACjB,yDAAyD;oBACzD,IAAI,OAAO,CAAC,KAAK,SAAS;wBAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7C,OAAO,CAAC,CAAC;gBACX,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;gBACpB,KAAK,EAAE,CAAC;YACV,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,qCAAqC;IACrC,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Types for the universal vector store migration engine.
|
|
3
|
+
* @module rag/migration/types
|
|
4
|
+
*
|
|
5
|
+
* Defines the interfaces for streaming data between any pair of
|
|
6
|
+
* vector store backends (SQLite, Postgres, Qdrant). The migration
|
|
7
|
+
* engine reads batches from an IMigrationSource and writes them
|
|
8
|
+
* to an IMigrationTarget, never loading the entire dataset into memory.
|
|
9
|
+
*/
|
|
10
|
+
/** Supported backend types for migration. */
|
|
11
|
+
export type BackendType = 'sqlite' | 'postgres' | 'qdrant' | 'pinecone';
|
|
12
|
+
/**
|
|
13
|
+
* Configuration for a migration source or target backend.
|
|
14
|
+
* Only the fields relevant to the chosen `type` need to be provided.
|
|
15
|
+
*/
|
|
16
|
+
export interface BackendConfig {
|
|
17
|
+
/** Which backend type to connect to. */
|
|
18
|
+
type: BackendType;
|
|
19
|
+
/** SQLite file path. Required when type='sqlite'. */
|
|
20
|
+
path?: string;
|
|
21
|
+
/** Postgres connection string. Required when type='postgres'. */
|
|
22
|
+
connectionString?: string;
|
|
23
|
+
/** Qdrant base URL (e.g. 'http://localhost:6333'). Required when type='qdrant'. */
|
|
24
|
+
url?: string;
|
|
25
|
+
/** Qdrant API key for cloud instances. Optional. */
|
|
26
|
+
apiKey?: string;
|
|
27
|
+
/** Qdrant collection name prefix. @default 'wunderland' */
|
|
28
|
+
collectionPrefix?: string;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Options for a migration operation.
|
|
32
|
+
*/
|
|
33
|
+
export interface MigrationOptions {
|
|
34
|
+
/** Source backend configuration. */
|
|
35
|
+
from: BackendConfig;
|
|
36
|
+
/** Target backend configuration. */
|
|
37
|
+
to: BackendConfig;
|
|
38
|
+
/** Rows per batch for streaming reads/writes. @default 1000 */
|
|
39
|
+
batchSize?: number;
|
|
40
|
+
/**
|
|
41
|
+
* Progress callback fired after each batch write.
|
|
42
|
+
* @param done - Number of rows written so far for the current table.
|
|
43
|
+
* @param total - Total row count for the current table.
|
|
44
|
+
* @param table - Name of the table currently being migrated.
|
|
45
|
+
*/
|
|
46
|
+
onProgress?: (done: number, total: number, table: string) => void;
|
|
47
|
+
/** If true, counts rows but does not write to target. @default false */
|
|
48
|
+
dryRun?: boolean;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Result returned after a migration completes.
|
|
52
|
+
*/
|
|
53
|
+
export interface MigrationResult {
|
|
54
|
+
/** Names of tables that were processed. */
|
|
55
|
+
tablesProcessed: string[];
|
|
56
|
+
/** Total rows migrated across all tables. */
|
|
57
|
+
totalRows: number;
|
|
58
|
+
/** Wall-clock duration in milliseconds. */
|
|
59
|
+
durationMs: number;
|
|
60
|
+
/** True if post-migration verification passed. */
|
|
61
|
+
verified: boolean;
|
|
62
|
+
/** Any errors encountered (non-fatal errors are collected, not thrown). */
|
|
63
|
+
errors: string[];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Adapter for reading data from a migration source backend.
|
|
67
|
+
* Implementations exist for SQLite, Postgres, and Qdrant.
|
|
68
|
+
*/
|
|
69
|
+
export interface IMigrationSource {
|
|
70
|
+
/** List table/collection names available for migration. */
|
|
71
|
+
listTables(): Promise<string[]>;
|
|
72
|
+
/** Count rows/points in a table. */
|
|
73
|
+
countRows(table: string): Promise<number>;
|
|
74
|
+
/**
|
|
75
|
+
* Read a batch of rows starting at offset.
|
|
76
|
+
* @param table - Table name to read from.
|
|
77
|
+
* @param offset - Number of rows to skip.
|
|
78
|
+
* @param limit - Maximum rows to return.
|
|
79
|
+
* @returns Array of row objects with column values.
|
|
80
|
+
*/
|
|
81
|
+
readBatch(table: string, offset: number, limit: number): Promise<Record<string, unknown>[]>;
|
|
82
|
+
/** Close the connection and release resources. */
|
|
83
|
+
close(): Promise<void>;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Adapter for writing data to a migration target backend.
|
|
87
|
+
* Implementations exist for SQLite, Postgres, and Qdrant.
|
|
88
|
+
*/
|
|
89
|
+
export interface IMigrationTarget {
|
|
90
|
+
/**
|
|
91
|
+
* Ensure the target schema/collection exists for a table.
|
|
92
|
+
* Creates it if it doesn't exist, using a sample row to infer column types.
|
|
93
|
+
* @param table - Table name to create.
|
|
94
|
+
* @param sampleRow - A sample row to derive schema from.
|
|
95
|
+
*/
|
|
96
|
+
ensureTable(table: string, sampleRow: Record<string, unknown>): Promise<void>;
|
|
97
|
+
/**
|
|
98
|
+
* Write a batch of rows to the target.
|
|
99
|
+
* Uses INSERT OR REPLACE / upsert semantics to handle duplicates.
|
|
100
|
+
* @param table - Table name to write to.
|
|
101
|
+
* @param rows - Array of row objects.
|
|
102
|
+
* @returns Number of rows successfully written.
|
|
103
|
+
*/
|
|
104
|
+
writeBatch(table: string, rows: Record<string, unknown>[]): Promise<number>;
|
|
105
|
+
/** Close the connection and release resources. */
|
|
106
|
+
close(): Promise<void>;
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/rag/migration/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,6CAA6C;AAC7C,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,CAAC;AAMxE;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,IAAI,EAAE,WAAW,CAAC;IAClB,qDAAqD;IACrD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oCAAoC;IACpC,IAAI,EAAE,aAAa,CAAC;IACpB,oCAAoC;IACpC,EAAE,EAAE,aAAa,CAAC;IAClB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClE,wEAAwE;IACxE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,QAAQ,EAAE,OAAO,CAAC;IAClB,2EAA2E;IAC3E,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAMD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2DAA2D;IAC3D,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChC,oCAAoC;IACpC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C;;;;;;OAMG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5F,kDAAkD;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9E;;;;;;OAMG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,kDAAkD;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}
|