lmgrep 0.1.17 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -63
- package/completions/_lmgrep +49 -40
- package/dist/application/Lmgrep.d.ts +107 -0
- package/dist/application/Lmgrep.js +99 -0
- package/dist/application/Lmgrep.js.map +1 -0
- package/dist/application/LmgrepFactory.d.ts +40 -0
- package/dist/application/LmgrepFactory.js +155 -0
- package/dist/application/LmgrepFactory.js.map +1 -0
- package/dist/application/indexing/BranchBootstrapper.d.ts +21 -0
- package/dist/application/indexing/BranchBootstrapper.js +52 -0
- package/dist/application/indexing/BranchBootstrapper.js.map +1 -0
- package/dist/application/indexing/BranchManifestSweeper.d.ts +17 -0
- package/dist/application/indexing/BranchManifestSweeper.js +35 -0
- package/dist/application/indexing/BranchManifestSweeper.js.map +1 -0
- package/dist/application/indexing/Duration.d.ts +9 -0
- package/dist/application/indexing/Duration.js +25 -0
- package/dist/application/indexing/Duration.js.map +1 -0
- package/dist/application/indexing/EmbeddingAbortError.d.ts +11 -0
- package/dist/application/indexing/EmbeddingAbortError.js +20 -0
- package/dist/application/indexing/EmbeddingAbortError.js.map +1 -0
- package/dist/application/indexing/EmbeddingPipeline.d.ts +59 -0
- package/dist/application/indexing/EmbeddingPipeline.js +140 -0
- package/dist/application/indexing/EmbeddingPipeline.js.map +1 -0
- package/dist/application/indexing/IndexBuilder.d.ts +112 -0
- package/dist/application/indexing/IndexBuilder.js +370 -0
- package/dist/application/indexing/IndexBuilder.js.map +1 -0
- package/dist/application/indexing/IndexingProgress.d.ts +34 -0
- package/dist/application/indexing/IndexingProgress.js +2 -0
- package/dist/application/indexing/IndexingProgress.js.map +1 -0
- package/dist/application/operations/Deadline.d.ts +13 -0
- package/dist/application/operations/Deadline.js +29 -0
- package/dist/application/operations/Deadline.js.map +1 -0
- package/dist/application/operations/HealthMonitor.d.ts +58 -0
- package/dist/application/operations/HealthMonitor.js +128 -0
- package/dist/application/operations/HealthMonitor.js.map +1 -0
- package/dist/application/operations/IndexAlternatives.d.ts +35 -0
- package/dist/application/operations/IndexAlternatives.js +61 -0
- package/dist/application/operations/IndexAlternatives.js.map +1 -0
- package/dist/application/operations/IndexInventory.d.ts +69 -0
- package/dist/application/operations/IndexInventory.js +99 -0
- package/dist/application/operations/IndexInventory.js.map +1 -0
- package/dist/application/operations/StatusService.d.ts +90 -0
- package/dist/application/operations/StatusService.js +171 -0
- package/dist/application/operations/StatusService.js.map +1 -0
- package/dist/application/operations/WatchService.d.ts +67 -0
- package/dist/application/operations/WatchService.js +145 -0
- package/dist/application/operations/WatchService.js.map +1 -0
- package/dist/application/operations/WorkingTreeDiff.d.ts +19 -0
- package/dist/application/operations/WorkingTreeDiff.js +36 -0
- package/dist/application/operations/WorkingTreeDiff.js.map +1 -0
- package/dist/application/research/ResearchAgent.d.ts +68 -0
- package/dist/application/research/ResearchAgent.js +201 -0
- package/dist/application/research/ResearchAgent.js.map +1 -0
- package/dist/application/research/ResearchPrompts.d.ts +16 -0
- package/dist/application/research/ResearchPrompts.js +34 -0
- package/dist/application/research/ResearchPrompts.js.map +1 -0
- package/dist/application/search/SearchCriteria.d.ts +37 -0
- package/dist/application/search/SearchCriteria.js +48 -0
- package/dist/application/search/SearchCriteria.js.map +1 -0
- package/dist/application/search/SearchService.d.ts +36 -0
- package/dist/application/search/SearchService.js +108 -0
- package/dist/application/search/SearchService.js.map +1 -0
- package/dist/application/search/SearchTargetResolver.d.ts +56 -0
- package/dist/application/search/SearchTargetResolver.js +104 -0
- package/dist/application/search/SearchTargetResolver.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +5 -754
- package/dist/cli.js.map +1 -1
- package/dist/domain/config/ConfigSource.d.ts +16 -0
- package/dist/domain/config/ConfigSource.js +2 -0
- package/dist/domain/config/ConfigSource.js.map +1 -0
- package/dist/domain/config/LmgrepConfig.d.ts +53 -0
- package/dist/domain/config/LmgrepConfig.js +2 -0
- package/dist/domain/config/LmgrepConfig.js.map +1 -0
- package/dist/domain/corpus/Chunk.d.ts +49 -0
- package/dist/domain/corpus/Chunk.js +60 -0
- package/dist/domain/corpus/Chunk.js.map +1 -0
- package/dist/domain/corpus/CodeLocation.d.ts +18 -0
- package/dist/domain/corpus/CodeLocation.js +34 -0
- package/dist/domain/corpus/CodeLocation.js.map +1 -0
- package/dist/domain/corpus/ContentHash.d.ts +15 -0
- package/dist/domain/corpus/ContentHash.js +30 -0
- package/dist/domain/corpus/ContentHash.js.map +1 -0
- package/dist/domain/corpus/FileVersion.d.ts +31 -0
- package/dist/domain/corpus/FileVersion.js +47 -0
- package/dist/domain/corpus/FileVersion.js.map +1 -0
- package/dist/domain/corpus/SourceFile.d.ts +26 -0
- package/dist/domain/corpus/SourceFile.js +47 -0
- package/dist/domain/corpus/SourceFile.js.map +1 -0
- package/dist/domain/corpus/Vector.d.ts +37 -0
- package/dist/domain/corpus/Vector.js +96 -0
- package/dist/domain/corpus/Vector.js.map +1 -0
- package/dist/domain/ports/ChatModelPort.d.ts +57 -0
- package/dist/domain/ports/ChatModelPort.js +2 -0
- package/dist/domain/ports/ChatModelPort.js.map +1 -0
- package/dist/domain/ports/ChunkRepositoryPort.d.ts +42 -0
- package/dist/domain/ports/ChunkRepositoryPort.js +2 -0
- package/dist/domain/ports/ChunkRepositoryPort.js.map +1 -0
- package/dist/domain/ports/ChunkerPort.d.ts +5 -0
- package/dist/domain/ports/ChunkerPort.js +2 -0
- package/dist/domain/ports/ChunkerPort.js.map +1 -0
- package/dist/domain/ports/DatabaseSessionPort.d.ts +10 -0
- package/dist/domain/ports/DatabaseSessionPort.js +2 -0
- package/dist/domain/ports/DatabaseSessionPort.js.map +1 -0
- package/dist/domain/ports/EmbedderPort.d.ts +14 -0
- package/dist/domain/ports/EmbedderPort.js +2 -0
- package/dist/domain/ports/EmbedderPort.js.map +1 -0
- package/dist/domain/ports/FileManifestRepositoryPort.d.ts +32 -0
- package/dist/domain/ports/FileManifestRepositoryPort.js +2 -0
- package/dist/domain/ports/FileManifestRepositoryPort.js.map +1 -0
- package/dist/domain/ports/GitPort.d.ts +24 -0
- package/dist/domain/ports/GitPort.js +2 -0
- package/dist/domain/ports/GitPort.js.map +1 -0
- package/dist/domain/ports/IndexMaintenancePort.d.ts +58 -0
- package/dist/domain/ports/IndexMaintenancePort.js +2 -0
- package/dist/domain/ports/IndexMaintenancePort.js.map +1 -0
- package/dist/domain/ports/IndexMetadataPort.d.ts +23 -0
- package/dist/domain/ports/IndexMetadataPort.js +2 -0
- package/dist/domain/ports/IndexMetadataPort.js.map +1 -0
- package/dist/domain/ports/LockPort.d.ts +17 -0
- package/dist/domain/ports/LockPort.js +2 -0
- package/dist/domain/ports/LockPort.js.map +1 -0
- package/dist/domain/ports/LoggerPort.d.ts +5 -0
- package/dist/domain/ports/LoggerPort.js +2 -0
- package/dist/domain/ports/LoggerPort.js.map +1 -0
- package/dist/domain/ports/ProjectIndexesPort.d.ts +20 -0
- package/dist/domain/ports/ProjectIndexesPort.js +2 -0
- package/dist/domain/ports/ProjectIndexesPort.js.map +1 -0
- package/dist/domain/ports/ProjectRegistryPort.d.ts +42 -0
- package/dist/domain/ports/ProjectRegistryPort.js +2 -0
- package/dist/domain/ports/ProjectRegistryPort.js.map +1 -0
- package/dist/domain/ports/StateDirectoryPort.d.ts +29 -0
- package/dist/domain/ports/StateDirectoryPort.js +2 -0
- package/dist/domain/ports/StateDirectoryPort.js.map +1 -0
- package/dist/domain/ports/WorkspacePort.d.ts +38 -0
- package/dist/domain/ports/WorkspacePort.js +2 -0
- package/dist/domain/ports/WorkspacePort.js.map +1 -0
- package/dist/domain/project/Branch.d.ts +20 -0
- package/dist/domain/project/Branch.js +33 -0
- package/dist/domain/project/Branch.js.map +1 -0
- package/dist/domain/project/DatabaseLocation.d.ts +32 -0
- package/dist/domain/project/DatabaseLocation.js +35 -0
- package/dist/domain/project/DatabaseLocation.js.map +1 -0
- package/dist/domain/project/IndexMetadata.d.ts +17 -0
- package/dist/domain/project/IndexMetadata.js +2 -0
- package/dist/domain/project/IndexMetadata.js.map +1 -0
- package/dist/domain/project/ModelIdentity.d.ts +58 -0
- package/dist/domain/project/ModelIdentity.js +97 -0
- package/dist/domain/project/ModelIdentity.js.map +1 -0
- package/dist/domain/project/Project.d.ts +17 -0
- package/dist/domain/project/Project.js +21 -0
- package/dist/domain/project/Project.js.map +1 -0
- package/dist/domain/project/ProjectId.d.ts +28 -0
- package/dist/domain/project/ProjectId.js +58 -0
- package/dist/domain/project/ProjectId.js.map +1 -0
- package/dist/domain/project/ProjectLocator.d.ts +112 -0
- package/dist/domain/project/ProjectLocator.js +178 -0
- package/dist/domain/project/ProjectLocator.js.map +1 -0
- package/dist/domain/research/Citation.d.ts +27 -0
- package/dist/domain/research/Citation.js +43 -0
- package/dist/domain/research/Citation.js.map +1 -0
- package/dist/domain/research/EvidenceLedger.d.ts +47 -0
- package/dist/domain/research/EvidenceLedger.js +111 -0
- package/dist/domain/research/EvidenceLedger.js.map +1 -0
- package/dist/domain/research/ResearchTrace.d.ts +19 -0
- package/dist/domain/research/ResearchTrace.js +22 -0
- package/dist/domain/research/ResearchTrace.js.map +1 -0
- package/dist/domain/retrieval/Hit.d.ts +19 -0
- package/dist/domain/retrieval/Hit.js +28 -0
- package/dist/domain/retrieval/Hit.js.map +1 -0
- package/dist/domain/retrieval/HitList.d.ts +35 -0
- package/dist/domain/retrieval/HitList.js +81 -0
- package/dist/domain/retrieval/HitList.js.map +1 -0
- package/dist/domain/retrieval/MissingIndexError.d.ts +11 -0
- package/dist/domain/retrieval/MissingIndexError.js +15 -0
- package/dist/domain/retrieval/MissingIndexError.js.map +1 -0
- package/dist/index.d.ts +43 -33
- package/dist/index.js +32 -215
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/ai/AiSdkChatModel.d.ts +31 -0
- package/dist/infrastructure/ai/AiSdkChatModel.js +137 -0
- package/dist/infrastructure/ai/AiSdkChatModel.js.map +1 -0
- package/dist/infrastructure/ai/AiSdkEmbedder.d.ts +28 -0
- package/dist/infrastructure/ai/AiSdkEmbedder.js +79 -0
- package/dist/infrastructure/ai/AiSdkEmbedder.js.map +1 -0
- package/dist/infrastructure/ai/EmbeddingPrefixes.d.ts +21 -0
- package/dist/infrastructure/ai/EmbeddingPrefixes.js +40 -0
- package/dist/infrastructure/ai/EmbeddingPrefixes.js.map +1 -0
- package/dist/infrastructure/ai/LmStudioProbe.d.ts +18 -0
- package/dist/infrastructure/ai/LmStudioProbe.js +49 -0
- package/dist/infrastructure/ai/LmStudioProbe.js.map +1 -0
- package/dist/infrastructure/ai/LocalModelReloader.d.ts +21 -0
- package/dist/infrastructure/ai/LocalModelReloader.js +61 -0
- package/dist/infrastructure/ai/LocalModelReloader.js.map +1 -0
- package/dist/infrastructure/ai/LocalRuntimeDetector.d.ts +36 -0
- package/dist/infrastructure/ai/LocalRuntimeDetector.js +61 -0
- package/dist/infrastructure/ai/LocalRuntimeDetector.js.map +1 -0
- package/dist/infrastructure/ai/ModelRuntime.d.ts +32 -0
- package/dist/infrastructure/ai/ModelRuntime.js +2 -0
- package/dist/infrastructure/ai/ModelRuntime.js.map +1 -0
- package/dist/infrastructure/ai/OllamaProbe.d.ts +21 -0
- package/dist/infrastructure/ai/OllamaProbe.js +69 -0
- package/dist/infrastructure/ai/OllamaProbe.js.map +1 -0
- package/dist/infrastructure/ai/ProviderFailure.d.ts +16 -0
- package/dist/infrastructure/ai/ProviderFailure.js +35 -0
- package/dist/infrastructure/ai/ProviderFailure.js.map +1 -0
- package/dist/infrastructure/ai/ProviderModuleLoader.d.ts +13 -0
- package/dist/infrastructure/ai/ProviderModuleLoader.js +49 -0
- package/dist/infrastructure/ai/ProviderModuleLoader.js.map +1 -0
- package/dist/infrastructure/ai/ProviderRegistry.d.ts +37 -0
- package/dist/infrastructure/ai/ProviderRegistry.js +55 -0
- package/dist/infrastructure/ai/ProviderRegistry.js.map +1 -0
- package/dist/infrastructure/fs/ConfigLoader.d.ts +59 -0
- package/dist/infrastructure/fs/ConfigLoader.js +194 -0
- package/dist/infrastructure/fs/ConfigLoader.js.map +1 -0
- package/dist/infrastructure/fs/DatabaseLocks.d.ts +52 -0
- package/dist/infrastructure/fs/DatabaseLocks.js +92 -0
- package/dist/infrastructure/fs/DatabaseLocks.js.map +1 -0
- package/dist/infrastructure/fs/DiskUsage.d.ts +6 -0
- package/dist/infrastructure/fs/DiskUsage.js +41 -0
- package/dist/infrastructure/fs/DiskUsage.js.map +1 -0
- package/dist/infrastructure/fs/IndexableFileRules.d.ts +33 -0
- package/dist/infrastructure/fs/IndexableFileRules.js +177 -0
- package/dist/infrastructure/fs/IndexableFileRules.js.map +1 -0
- package/dist/infrastructure/fs/Loggers.d.ts +14 -0
- package/dist/infrastructure/fs/Loggers.js +18 -0
- package/dist/infrastructure/fs/Loggers.js.map +1 -0
- package/dist/infrastructure/fs/PidFileLock.d.ts +40 -0
- package/dist/infrastructure/fs/PidFileLock.js +97 -0
- package/dist/infrastructure/fs/PidFileLock.js.map +1 -0
- package/dist/infrastructure/fs/ProjectIndexes.d.ts +8 -0
- package/dist/infrastructure/fs/ProjectIndexes.js +35 -0
- package/dist/infrastructure/fs/ProjectIndexes.js.map +1 -0
- package/dist/infrastructure/fs/ProjectMetadataStore.d.ts +42 -0
- package/dist/infrastructure/fs/ProjectMetadataStore.js +86 -0
- package/dist/infrastructure/fs/ProjectMetadataStore.js.map +1 -0
- package/dist/infrastructure/fs/ProjectRegistry.d.ts +25 -0
- package/dist/infrastructure/fs/ProjectRegistry.js +83 -0
- package/dist/infrastructure/fs/ProjectRegistry.js.map +1 -0
- package/dist/infrastructure/fs/StateDirectory.d.ts +28 -0
- package/dist/infrastructure/fs/StateDirectory.js +71 -0
- package/dist/infrastructure/fs/StateDirectory.js.map +1 -0
- package/dist/infrastructure/fs/Workspace.d.ts +25 -0
- package/dist/infrastructure/fs/Workspace.js +90 -0
- package/dist/infrastructure/fs/Workspace.js.map +1 -0
- package/dist/infrastructure/git/GitClient.d.ts +34 -0
- package/dist/infrastructure/git/GitClient.js +74 -0
- package/dist/infrastructure/git/GitClient.js.map +1 -0
- package/dist/infrastructure/lancedb/ChunkRepository.d.ts +38 -0
- package/dist/infrastructure/lancedb/ChunkRepository.js +166 -0
- package/dist/infrastructure/lancedb/ChunkRepository.js.map +1 -0
- package/dist/infrastructure/lancedb/DatabaseImporter.d.ts +22 -0
- package/dist/infrastructure/lancedb/DatabaseImporter.js +74 -0
- package/dist/infrastructure/lancedb/DatabaseImporter.js.map +1 -0
- package/dist/infrastructure/lancedb/FileManifestRepository.d.ts +34 -0
- package/dist/infrastructure/lancedb/FileManifestRepository.js +137 -0
- package/dist/infrastructure/lancedb/FileManifestRepository.js.map +1 -0
- package/dist/infrastructure/lancedb/IndexMaintenance.d.ts +53 -0
- package/dist/infrastructure/lancedb/IndexMaintenance.js +227 -0
- package/dist/infrastructure/lancedb/IndexMaintenance.js.map +1 -0
- package/dist/infrastructure/lancedb/LanceTables.d.ts +67 -0
- package/dist/infrastructure/lancedb/LanceTables.js +159 -0
- package/dist/infrastructure/lancedb/LanceTables.js.map +1 -0
- package/dist/infrastructure/lancedb/NativeTuning.d.ts +1 -0
- package/dist/infrastructure/lancedb/NativeTuning.js +9 -0
- package/dist/infrastructure/lancedb/NativeTuning.js.map +1 -0
- package/dist/infrastructure/lancedb/RowReplication.d.ts +27 -0
- package/dist/infrastructure/lancedb/RowReplication.js +82 -0
- package/dist/infrastructure/lancedb/RowReplication.js.map +1 -0
- package/dist/infrastructure/lancedb/VectorIndexPolicy.d.ts +60 -0
- package/dist/infrastructure/lancedb/VectorIndexPolicy.js +72 -0
- package/dist/infrastructure/lancedb/VectorIndexPolicy.js.map +1 -0
- package/dist/infrastructure/p2p/IndexShare.d.ts +45 -0
- package/dist/infrastructure/p2p/IndexShare.js +160 -0
- package/dist/infrastructure/p2p/IndexShare.js.map +1 -0
- package/dist/infrastructure/p2p/PeerSwarm.d.ts +16 -0
- package/dist/infrastructure/p2p/PeerSwarm.js +41 -0
- package/dist/infrastructure/p2p/PeerSwarm.js.map +1 -0
- package/dist/infrastructure/p2p/SecureChannel.d.ts +60 -0
- package/dist/infrastructure/p2p/SecureChannel.js +71 -0
- package/dist/infrastructure/p2p/SecureChannel.js.map +1 -0
- package/dist/infrastructure/p2p/ShareCode.d.ts +25 -0
- package/dist/infrastructure/p2p/ShareCode.js +304 -0
- package/dist/infrastructure/p2p/ShareCode.js.map +1 -0
- package/dist/infrastructure/process/ProcessRegistry.d.ts +50 -0
- package/dist/infrastructure/process/ProcessRegistry.js +109 -0
- package/dist/infrastructure/process/ProcessRegistry.js.map +1 -0
- package/dist/infrastructure/treesitter/ChunkContextBuilder.d.ts +45 -0
- package/dist/infrastructure/treesitter/ChunkContextBuilder.js +159 -0
- package/dist/infrastructure/treesitter/ChunkContextBuilder.js.map +1 -0
- package/dist/infrastructure/treesitter/LanguageCatalog.d.ts +30 -0
- package/dist/{lib/chunker/languages.js → infrastructure/treesitter/LanguageCatalog.js} +65 -41
- package/dist/infrastructure/treesitter/LanguageCatalog.js.map +1 -0
- package/dist/infrastructure/treesitter/SlidingWindowChunker.d.ts +15 -0
- package/dist/infrastructure/treesitter/SlidingWindowChunker.js +42 -0
- package/dist/infrastructure/treesitter/SlidingWindowChunker.js.map +1 -0
- package/dist/infrastructure/treesitter/TreeSitterChunker.d.ts +42 -0
- package/dist/infrastructure/treesitter/TreeSitterChunker.js +163 -0
- package/dist/infrastructure/treesitter/TreeSitterChunker.js.map +1 -0
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +15 -61
- package/dist/mcp.js.map +1 -1
- package/dist/presentation/cli/Cli.d.ts +31 -0
- package/dist/presentation/cli/Cli.js +120 -0
- package/dist/presentation/cli/Cli.js.map +1 -0
- package/dist/presentation/cli/CliOptions.d.ts +18 -0
- package/dist/presentation/cli/CliOptions.js +25 -0
- package/dist/presentation/cli/CliOptions.js.map +1 -0
- package/dist/presentation/cli/CommandContext.d.ts +50 -0
- package/dist/presentation/cli/CommandContext.js +88 -0
- package/dist/presentation/cli/CommandContext.js.map +1 -0
- package/dist/presentation/cli/ConfigTemplate.d.ts +31 -0
- package/dist/presentation/cli/ConfigTemplate.js +83 -0
- package/dist/presentation/cli/ConfigTemplate.js.map +1 -0
- package/dist/presentation/cli/Renderer.d.ts +90 -0
- package/dist/presentation/cli/Renderer.js +290 -0
- package/dist/presentation/cli/Renderer.js.map +1 -0
- package/dist/presentation/cli/commands/AskCommand.d.ts +14 -0
- package/dist/presentation/cli/commands/AskCommand.js +48 -0
- package/dist/presentation/cli/commands/AskCommand.js.map +1 -0
- package/dist/presentation/cli/commands/CompletionsCommand.d.ts +18 -0
- package/dist/presentation/cli/commands/CompletionsCommand.js +96 -0
- package/dist/presentation/cli/commands/CompletionsCommand.js.map +1 -0
- package/dist/presentation/cli/commands/ConfigCommands.d.ts +55 -0
- package/dist/presentation/cli/commands/ConfigCommands.js +204 -0
- package/dist/presentation/cli/commands/ConfigCommands.js.map +1 -0
- package/dist/presentation/cli/commands/IndexCommand.d.ts +18 -0
- package/dist/presentation/cli/commands/IndexCommand.js +54 -0
- package/dist/presentation/cli/commands/IndexCommand.js.map +1 -0
- package/dist/presentation/cli/commands/ProjectsCommand.d.ts +48 -0
- package/dist/presentation/cli/commands/ProjectsCommand.js +240 -0
- package/dist/presentation/cli/commands/ProjectsCommand.js.map +1 -0
- package/dist/presentation/cli/commands/SearchCommand.d.ts +19 -0
- package/dist/presentation/cli/commands/SearchCommand.js +55 -0
- package/dist/presentation/cli/commands/SearchCommand.js.map +1 -0
- package/dist/presentation/cli/commands/ServerCommands.d.ts +13 -0
- package/dist/presentation/cli/commands/ServerCommands.js +40 -0
- package/dist/presentation/cli/commands/ServerCommands.js.map +1 -0
- package/dist/presentation/cli/commands/ShareCommands.d.ts +22 -0
- package/dist/presentation/cli/commands/ShareCommands.js +134 -0
- package/dist/presentation/cli/commands/ShareCommands.js.map +1 -0
- package/dist/presentation/cli/commands/StatusCommand.d.ts +15 -0
- package/dist/presentation/cli/commands/StatusCommand.js +53 -0
- package/dist/presentation/cli/commands/StatusCommand.js.map +1 -0
- package/dist/presentation/mcp/HitFormatter.d.ts +17 -0
- package/dist/presentation/mcp/HitFormatter.js +42 -0
- package/dist/presentation/mcp/HitFormatter.js.map +1 -0
- package/dist/presentation/mcp/IndexWatchController.d.ts +17 -0
- package/dist/presentation/mcp/IndexWatchController.js +31 -0
- package/dist/presentation/mcp/IndexWatchController.js.map +1 -0
- package/dist/presentation/mcp/LmgrepCore.d.ts +66 -0
- package/dist/presentation/mcp/LmgrepCore.js +178 -0
- package/dist/presentation/mcp/LmgrepCore.js.map +1 -0
- package/dist/presentation/mcp/McpServer.d.ts +18 -0
- package/dist/presentation/mcp/McpServer.js +63 -0
- package/dist/presentation/mcp/McpServer.js.map +1 -0
- package/dist/presentation/mcp/ToolDescriptions.d.ts +40 -0
- package/dist/presentation/mcp/ToolDescriptions.js +85 -0
- package/dist/presentation/mcp/ToolDescriptions.js.map +1 -0
- package/package.json +84 -63
- package/dist/lib/build.d.ts +0 -7
- package/dist/lib/build.js +0 -284
- package/dist/lib/build.js.map +0 -1
- package/dist/lib/chunker/context.d.ts +0 -21
- package/dist/lib/chunker/context.js +0 -131
- package/dist/lib/chunker/context.js.map +0 -1
- package/dist/lib/chunker/index.d.ts +0 -7
- package/dist/lib/chunker/index.js +0 -150
- package/dist/lib/chunker/index.js.map +0 -1
- package/dist/lib/chunker/languages.d.ts +0 -15
- package/dist/lib/chunker/languages.js.map +0 -1
- package/dist/lib/config.d.ts +0 -12
- package/dist/lib/config.js +0 -83
- package/dist/lib/config.js.map +0 -1
- package/dist/lib/embedder.d.ts +0 -55
- package/dist/lib/embedder.js +0 -221
- package/dist/lib/embedder.js.map +0 -1
- package/dist/lib/p2p.d.ts +0 -32
- package/dist/lib/p2p.js +0 -281
- package/dist/lib/p2p.js.map +0 -1
- package/dist/lib/providers.d.ts +0 -1
- package/dist/lib/providers.js +0 -41
- package/dist/lib/providers.js.map +0 -1
- package/dist/lib/repair.d.ts +0 -10
- package/dist/lib/repair.js +0 -57
- package/dist/lib/repair.js.map +0 -1
- package/dist/lib/scanner.d.ts +0 -34
- package/dist/lib/scanner.js +0 -242
- package/dist/lib/scanner.js.map +0 -1
- package/dist/lib/search-tool.d.ts +0 -47
- package/dist/lib/search-tool.js +0 -239
- package/dist/lib/search-tool.js.map +0 -1
- package/dist/lib/serve.d.ts +0 -11
- package/dist/lib/serve.js +0 -127
- package/dist/lib/serve.js.map +0 -1
- package/dist/lib/store.d.ts +0 -177
- package/dist/lib/store.js +0 -825
- package/dist/lib/store.js.map +0 -1
- package/dist/lib/types.d.ts +0 -129
- package/dist/lib/types.js +0 -13
- package/dist/lib/types.js.map +0 -1
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { connect } from "@lancedb/lancedb";
|
|
3
|
+
/**
|
|
4
|
+
* Table names inside a LanceDB database directory.
|
|
5
|
+
*
|
|
6
|
+
* A const object rather than a `const enum`: const enums are erased at emit,
|
|
7
|
+
* so importing one across modules compiles but can vanish at runtime.
|
|
8
|
+
*/
|
|
9
|
+
export const TableName = {
|
|
10
|
+
Chunks: "chunks",
|
|
11
|
+
Files: "files",
|
|
12
|
+
/**
|
|
13
|
+
* Written by the removed faceting feature. Nothing reads it; it is listed
|
|
14
|
+
* so maintenance can drop it and give the disk back.
|
|
15
|
+
*/
|
|
16
|
+
LegacyVocab: "vocab",
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Shared access to one LanceDB database: connecting, opening tables lazily,
|
|
20
|
+
* and running the schema backfills legacy databases need.
|
|
21
|
+
*
|
|
22
|
+
* The repositories share an instance so they share a connection — LanceDB
|
|
23
|
+
* connections are per-directory and opening several would multiply the native
|
|
24
|
+
* runtime's memory.
|
|
25
|
+
*/
|
|
26
|
+
export class LanceTables {
|
|
27
|
+
path;
|
|
28
|
+
branch;
|
|
29
|
+
/**
|
|
30
|
+
* Predicates are built by string interpolation, so batches are kept small
|
|
31
|
+
* enough that no single filter grows unreasonably long.
|
|
32
|
+
*/
|
|
33
|
+
static FILTER_BATCH_SIZE = 50;
|
|
34
|
+
db;
|
|
35
|
+
open = new Map();
|
|
36
|
+
constructor(path, branch) {
|
|
37
|
+
this.path = path;
|
|
38
|
+
this.branch = branch;
|
|
39
|
+
}
|
|
40
|
+
async connection() {
|
|
41
|
+
if (this.db)
|
|
42
|
+
return this.db;
|
|
43
|
+
mkdirSync(this.path, { recursive: true });
|
|
44
|
+
this.db = await connect(this.path);
|
|
45
|
+
return this.db;
|
|
46
|
+
}
|
|
47
|
+
/** Open a table, or undefined when the database has never held it. */
|
|
48
|
+
async table(name) {
|
|
49
|
+
const cached = this.open.get(name);
|
|
50
|
+
if (cached)
|
|
51
|
+
return cached;
|
|
52
|
+
const conn = await this.connection();
|
|
53
|
+
const names = await conn.tableNames();
|
|
54
|
+
if (!names.includes(name))
|
|
55
|
+
return undefined;
|
|
56
|
+
const table = await conn.openTable(name);
|
|
57
|
+
await this.migrate(name, table);
|
|
58
|
+
this.open.set(name, table);
|
|
59
|
+
return table;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Open a table, creating it from `seed` rows when it does not exist.
|
|
63
|
+
* `seeded` says whether creation consumed the rows, so the caller knows not
|
|
64
|
+
* to insert them a second time.
|
|
65
|
+
*/
|
|
66
|
+
async tableOrCreate(name, seed) {
|
|
67
|
+
const existing = await this.table(name);
|
|
68
|
+
if (existing)
|
|
69
|
+
return { table: existing, seeded: false };
|
|
70
|
+
const conn = await this.connection();
|
|
71
|
+
const created = await conn.createTable(name, seed);
|
|
72
|
+
this.open.set(name, created);
|
|
73
|
+
return { table: created, seeded: true };
|
|
74
|
+
}
|
|
75
|
+
async tableNames() {
|
|
76
|
+
return (await this.connection()).tableNames();
|
|
77
|
+
}
|
|
78
|
+
async dropTable(name) {
|
|
79
|
+
const conn = await this.connection();
|
|
80
|
+
const names = await conn.tableNames();
|
|
81
|
+
if (names.includes(name))
|
|
82
|
+
await conn.dropTable(name);
|
|
83
|
+
this.open.delete(name);
|
|
84
|
+
}
|
|
85
|
+
/** Forget cached handles so the next access re-opens and re-migrates. */
|
|
86
|
+
forget(name) {
|
|
87
|
+
if (name)
|
|
88
|
+
this.open.delete(name);
|
|
89
|
+
else
|
|
90
|
+
this.open.clear();
|
|
91
|
+
}
|
|
92
|
+
close() {
|
|
93
|
+
this.open.clear();
|
|
94
|
+
this.db = undefined;
|
|
95
|
+
}
|
|
96
|
+
/** `column IN ('a', 'b')`, with values single-quote escaped. */
|
|
97
|
+
static inFilter(column, values) {
|
|
98
|
+
const escaped = values.map((v) => `'${v.replace(/'/g, "''")}'`);
|
|
99
|
+
return `${column} IN (${escaped.join(", ")})`;
|
|
100
|
+
}
|
|
101
|
+
static quote(value) {
|
|
102
|
+
return value.replace(/'/g, "''");
|
|
103
|
+
}
|
|
104
|
+
/** Run `column IN (...)` deletes in batches small enough for one predicate. */
|
|
105
|
+
static async deleteIn(table, column, values) {
|
|
106
|
+
for (let i = 0; i < values.length; i += LanceTables.FILTER_BATCH_SIZE) {
|
|
107
|
+
const batch = values.slice(i, i + LanceTables.FILTER_BATCH_SIZE);
|
|
108
|
+
await table.delete(LanceTables.inFilter(column, batch));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** Run a batched `column IN (...)` select, collecting one column's values. */
|
|
112
|
+
static async selectIn(table, column, values) {
|
|
113
|
+
const found = new Set();
|
|
114
|
+
for (let i = 0; i < values.length; i += LanceTables.FILTER_BATCH_SIZE) {
|
|
115
|
+
const batch = values.slice(i, i + LanceTables.FILTER_BATCH_SIZE);
|
|
116
|
+
const rows = await table
|
|
117
|
+
.query()
|
|
118
|
+
.where(LanceTables.inFilter(column, batch))
|
|
119
|
+
.select([column])
|
|
120
|
+
.toArray();
|
|
121
|
+
for (const r of rows)
|
|
122
|
+
found.add(r[column]);
|
|
123
|
+
}
|
|
124
|
+
return found;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Backfill columns that legacy databases predate. Both are additive and
|
|
128
|
+
* idempotent — a table that already has the column is left alone.
|
|
129
|
+
*/
|
|
130
|
+
async migrate(name, table) {
|
|
131
|
+
if (name === TableName.Files) {
|
|
132
|
+
// Pre branch-scoping `files` tables have no `branch` column. Fill it
|
|
133
|
+
// with the current branch so branch-filtered queries keep working.
|
|
134
|
+
const schema = await table.schema();
|
|
135
|
+
if (schema.fields.some((f) => f.name === "branch"))
|
|
136
|
+
return;
|
|
137
|
+
await table.addColumns([
|
|
138
|
+
{
|
|
139
|
+
name: "branch",
|
|
140
|
+
valueSql: `CAST('${LanceTables.quote(this.branch.toString())}' AS STRING)`,
|
|
141
|
+
},
|
|
142
|
+
]);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (name === TableName.Chunks) {
|
|
146
|
+
// Pre version-scoping `chunks` tables have no `fileHash`. Fill it
|
|
147
|
+
// with "" — treated as a wildcard at search time, so legacy chunks
|
|
148
|
+
// keep appearing until their file is re-indexed and gets a real
|
|
149
|
+
// version hash.
|
|
150
|
+
const schema = await table.schema();
|
|
151
|
+
if (schema.fields.some((f) => f.name === "fileHash"))
|
|
152
|
+
return;
|
|
153
|
+
await table.addColumns([
|
|
154
|
+
{ name: "fileHash", valueSql: `CAST('' AS STRING)` },
|
|
155
|
+
]);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=LanceTables.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LanceTables.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/LanceTables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAmB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAGxE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG;IACxB,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,OAAO;IACd;;;OAGG;IACH,WAAW,EAAE,OAAO;CACX,CAAC;AAIX;;;;;;;GAOG;AACH,MAAM,OAAO,WAAW;IAWb;IACQ;IAXlB;;;OAGG;IACH,MAAM,CAAU,iBAAiB,GAAG,EAAE,CAAC;IAE/B,EAAE,CAAyB;IAClB,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEpD,YACU,IAAY,EACJ,MAAc;QADtB,SAAI,GAAJ,IAAI,CAAQ;QACJ,WAAM,GAAN,MAAM,CAAQ;IAC7B,CAAC;IAEI,KAAK,CAAC,UAAU;QACvB,IAAI,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,EAAE,CAAC;IAChB,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,KAAK,CAAC,IAAe;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAClB,IAAe,EACf,IAA+B;QAE/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAExD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,UAAU;QACf,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAe;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAED,yEAAyE;IACzE,MAAM,CAAC,IAAgB;QACtB,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;;YAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC;IACrB,CAAC;IAED,gEAAgE;IAChE,MAAM,CAAC,QAAQ,CAAC,MAAc,EAAE,MAAyB;QACxD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAChE,OAAO,GAAG,MAAM,QAAQ,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAa;QACzB,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,+EAA+E;IAC/E,MAAM,CAAC,KAAK,CAAC,QAAQ,CACpB,KAAY,EACZ,MAAc,EACd,MAAyB;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC;YACjE,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC;IACF,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,KAAK,CAAC,QAAQ,CACpB,KAAY,EACZ,MAAc,EACd,MAAyB;QAEzB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC;YACjE,MAAM,IAAI,GAAG,MAAM,KAAK;iBACtB,KAAK,EAAE;iBACP,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;iBAC1C,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;iBAChB,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAW,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,OAAO,CAAC,IAAe,EAAE,KAAY;QAClD,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC3D,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtB;oBACC,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,SAAS,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,cAAc;iBAC1E;aACD,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC/B,kEAAkE;YAClE,mEAAmE;YACnE,gEAAgE;YAChE,gBAAgB;YAChB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;gBAAE,OAAO;YAC7D,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtB,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,oBAAoB,EAAE;aACpD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Set before any native module (LanceDB, libuv) initializes its thread pool.
|
|
2
|
+
// This module is imported first by entry points to ensure the env is in
|
|
3
|
+
// place before downstream imports load native bindings.
|
|
4
|
+
process.env.TOKIO_WORKER_THREADS ??= "4";
|
|
5
|
+
process.env.UV_THREADPOOL_SIZE ??= "4";
|
|
6
|
+
process.env.RAYON_NUM_THREADS ??= "4";
|
|
7
|
+
process.env.MATMUL_NUM_THREADS ??= "4";
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=NativeTuning.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"NativeTuning.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/NativeTuning.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,wEAAwE;AACxE,wDAAwD;AACxD,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,GAAG,CAAC;AACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC;AACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,GAAG,CAAC;AACtC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Branch } from "../../domain/project/Branch.js";
|
|
2
|
+
import { type LanceTables } from "./LanceTables.js";
|
|
3
|
+
/** A manifest row as it travels between peers. */
|
|
4
|
+
export interface ManifestRow {
|
|
5
|
+
filePath: string;
|
|
6
|
+
fileHash: string;
|
|
7
|
+
branch: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Row-level access to the tables, for replication between databases.
|
|
11
|
+
*
|
|
12
|
+
* Peer sharing copies rows verbatim — embeddings included, since avoiding
|
|
13
|
+
* re-embedding is the entire point — so it works below the domain model rather
|
|
14
|
+
* than through it. Keeping that access behind this one class means the rest of
|
|
15
|
+
* the system never sees raw rows.
|
|
16
|
+
*/
|
|
17
|
+
export declare class RowReplication {
|
|
18
|
+
private readonly tables;
|
|
19
|
+
private readonly branch;
|
|
20
|
+
constructor(tables: LanceTables, branch: Branch);
|
|
21
|
+
chunkCount(): Promise<number>;
|
|
22
|
+
/** Stream chunk rows in pages, so a large index never loads at once. */
|
|
23
|
+
streamChunkRows(batchSize: number): AsyncGenerator<Record<string, unknown>[]>;
|
|
24
|
+
allManifestRows(): Promise<ManifestRow[]>;
|
|
25
|
+
addChunkRows(rows: Record<string, unknown>[]): Promise<void>;
|
|
26
|
+
addManifestRows(rows: ManifestRow[]): Promise<void>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { TableName } from "./LanceTables.js";
|
|
2
|
+
/**
|
|
3
|
+
* Row-level access to the tables, for replication between databases.
|
|
4
|
+
*
|
|
5
|
+
* Peer sharing copies rows verbatim — embeddings included, since avoiding
|
|
6
|
+
* re-embedding is the entire point — so it works below the domain model rather
|
|
7
|
+
* than through it. Keeping that access behind this one class means the rest of
|
|
8
|
+
* the system never sees raw rows.
|
|
9
|
+
*/
|
|
10
|
+
export class RowReplication {
|
|
11
|
+
tables;
|
|
12
|
+
branch;
|
|
13
|
+
constructor(tables, branch) {
|
|
14
|
+
this.tables = tables;
|
|
15
|
+
this.branch = branch;
|
|
16
|
+
}
|
|
17
|
+
async chunkCount() {
|
|
18
|
+
const table = await this.tables.table(TableName.Chunks);
|
|
19
|
+
return table ? table.countRows() : 0;
|
|
20
|
+
}
|
|
21
|
+
/** Stream chunk rows in pages, so a large index never loads at once. */
|
|
22
|
+
async *streamChunkRows(batchSize) {
|
|
23
|
+
const table = await this.tables.table(TableName.Chunks);
|
|
24
|
+
if (!table)
|
|
25
|
+
return;
|
|
26
|
+
const total = await table.countRows();
|
|
27
|
+
for (let offset = 0; offset < total; offset += batchSize) {
|
|
28
|
+
const rows = await table
|
|
29
|
+
.query()
|
|
30
|
+
.limit(batchSize)
|
|
31
|
+
.offset(offset)
|
|
32
|
+
.toArray();
|
|
33
|
+
if (rows.length === 0)
|
|
34
|
+
break;
|
|
35
|
+
yield rows.map((r) => ({
|
|
36
|
+
id: r.id,
|
|
37
|
+
filePath: r.filePath,
|
|
38
|
+
startLine: r.startLine,
|
|
39
|
+
endLine: r.endLine,
|
|
40
|
+
type: r.type,
|
|
41
|
+
name: r.name,
|
|
42
|
+
content: r.content,
|
|
43
|
+
context: r.context,
|
|
44
|
+
hash: r.hash,
|
|
45
|
+
fileHash: r.fileHash ?? "",
|
|
46
|
+
// Arrow typed arrays do not survive JSON; send plain numbers.
|
|
47
|
+
vector: Array.from(r.vector),
|
|
48
|
+
}));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async allManifestRows() {
|
|
52
|
+
const table = await this.tables.table(TableName.Files);
|
|
53
|
+
if (!table)
|
|
54
|
+
return [];
|
|
55
|
+
const rows = await table.query().toArray();
|
|
56
|
+
return rows.map((r) => ({
|
|
57
|
+
filePath: r.filePath,
|
|
58
|
+
fileHash: r.fileHash,
|
|
59
|
+
branch: r.branch ?? this.branch.toString(),
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
async addChunkRows(rows) {
|
|
63
|
+
if (rows.length === 0)
|
|
64
|
+
return;
|
|
65
|
+
const { table, seeded } = await this.tables.tableOrCreate(TableName.Chunks, rows);
|
|
66
|
+
if (!seeded)
|
|
67
|
+
await table.add(rows);
|
|
68
|
+
}
|
|
69
|
+
async addManifestRows(rows) {
|
|
70
|
+
if (rows.length === 0)
|
|
71
|
+
return;
|
|
72
|
+
const records = rows.map((r) => ({
|
|
73
|
+
filePath: r.filePath,
|
|
74
|
+
fileHash: r.fileHash,
|
|
75
|
+
branch: r.branch ?? this.branch.toString(),
|
|
76
|
+
}));
|
|
77
|
+
const { table, seeded } = await this.tables.tableOrCreate(TableName.Files, records);
|
|
78
|
+
if (!seeded)
|
|
79
|
+
await table.add(records);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=RowReplication.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RowReplication.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/RowReplication.ts"],"names":[],"mappings":"AACA,OAAO,EAAoB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAS/D;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAER;IACA;IAFlB,YACkB,MAAmB,EACnB,MAAc;QADd,WAAM,GAAN,MAAM,CAAa;QACnB,WAAM,GAAN,MAAM,CAAQ;IAC7B,CAAC;IAEJ,KAAK,CAAC,UAAU;QACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,CAAC,eAAe,CACrB,SAAiB;QAEjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,MAAM,KAAK;iBACtB,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,CAAC;iBAChB,MAAM,CAAC,MAAM,CAAC;iBACd,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAG,CAAC,CAAC,QAAmB,IAAI,EAAE;gBACtC,8DAA8D;gBAC9D,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAA0B,CAAC;aAChD,CAAC,CAAC,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,eAAe;QACpB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,QAAQ,EAAE,CAAC,CAAC,QAAkB;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAkB;YAC9B,MAAM,EAAG,CAAC,CAAC,MAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;SACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAA+B;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACxD,SAAS,CAAC,MAAM,EAChB,IAAI,CACJ,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAmB;QACxC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;SAC1C,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACxD,SAAS,CAAC,KAAK,EACf,OAAO,CACP,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACD"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tuning for the LanceDB vector index. The numbers here were measured on a
|
|
3
|
+
* 43k-chunk / 3584-dimension index; the comments record what they bought so a
|
|
4
|
+
* later change can tell a tuned value from an arbitrary one.
|
|
5
|
+
*/
|
|
6
|
+
export declare const VectorIndexPolicy: {
|
|
7
|
+
/**
|
|
8
|
+
* Below this row count a brute-force scan is cheap enough that training an
|
|
9
|
+
* index costs more than it saves, and IVF-PQ has too few vectors to cluster
|
|
10
|
+
* well.
|
|
11
|
+
*/
|
|
12
|
+
readonly MIN_ROWS_FOR_ANN: 5000;
|
|
13
|
+
/**
|
|
14
|
+
* Re-absorb new rows into the index once the unindexed tail passes this
|
|
15
|
+
* share of the indexed body. Rows outside the index are searched by flat
|
|
16
|
+
* scan, so the tail is exactly what needs bounding.
|
|
17
|
+
*/
|
|
18
|
+
readonly UNINDEXED_REINDEX_RATIO: 0.2;
|
|
19
|
+
/** ...but always tolerate a small tail, so a 3-chunk edit never retrains. */
|
|
20
|
+
readonly UNINDEXED_REINDEX_FLOOR: 2000;
|
|
21
|
+
/**
|
|
22
|
+
* Vector searches run without an explicit distance type, which LanceDB
|
|
23
|
+
* resolves to L2. The index has its own default, so it is pinned: an index
|
|
24
|
+
* built on a different metric would silently reorder every result.
|
|
25
|
+
*/
|
|
26
|
+
readonly DISTANCE_TYPE: "l2";
|
|
27
|
+
/**
|
|
28
|
+
* IVF-PQ stores compressed vectors, so its raw distances approximate the
|
|
29
|
+
* real ones. `refineFactor` makes LanceDB pull the uncompressed vectors for
|
|
30
|
+
* the top `limit * factor` candidates and re-score them exactly.
|
|
31
|
+
*
|
|
32
|
+
* It is not optional. Measured mean recall@20 against an exact scan:
|
|
33
|
+
*
|
|
34
|
+
* off -> 53% 1 -> 75% 3 -> 86% 10 -> 91% 20 -> 94% 50 -> 94%
|
|
35
|
+
*
|
|
36
|
+
* and without it `_distance` is wrong by ~0.07, which matters because score
|
|
37
|
+
* is derived from it and users filter on `--min-score`. With refinement the
|
|
38
|
+
* scores match an exact scan to the digit.
|
|
39
|
+
*
|
|
40
|
+
* 20 sits at the knee: ~94% recall at ~11ms/query versus ~115ms for an
|
|
41
|
+
* exact scan. Beyond it recall flattens (the probed partitions run out of
|
|
42
|
+
* candidates) while latency keeps climbing. The refined set is bounded by
|
|
43
|
+
* the query limit, not by table size, so this costs a fixed few MB.
|
|
44
|
+
*
|
|
45
|
+
* Harmless on tables with no index: a flat scan already computes exact
|
|
46
|
+
* distances and returns identical rows.
|
|
47
|
+
*/
|
|
48
|
+
readonly REFINE_FACTOR: 20;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Columns a search reads off a result row.
|
|
52
|
+
*
|
|
53
|
+
* Selecting explicitly keeps the embedding out of the result set — without it
|
|
54
|
+
* LanceDB ships every over-fetched row's full vector back into JS for nothing.
|
|
55
|
+
*
|
|
56
|
+
* `_distance` is listed on purpose: LanceDB currently auto-projects it into any
|
|
57
|
+
* scoring query but warns that it will stop, so naming it pins the behaviour
|
|
58
|
+
* rather than relying on a deprecated default.
|
|
59
|
+
*/
|
|
60
|
+
export declare const SEARCH_COLUMNS: readonly ["id", "filePath", "startLine", "endLine", "type", "name", "content", "context", "fileHash", "_distance"];
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tuning for the LanceDB vector index. The numbers here were measured on a
|
|
3
|
+
* 43k-chunk / 3584-dimension index; the comments record what they bought so a
|
|
4
|
+
* later change can tell a tuned value from an arbitrary one.
|
|
5
|
+
*/
|
|
6
|
+
export const VectorIndexPolicy = {
|
|
7
|
+
/**
|
|
8
|
+
* Below this row count a brute-force scan is cheap enough that training an
|
|
9
|
+
* index costs more than it saves, and IVF-PQ has too few vectors to cluster
|
|
10
|
+
* well.
|
|
11
|
+
*/
|
|
12
|
+
MIN_ROWS_FOR_ANN: 5_000,
|
|
13
|
+
/**
|
|
14
|
+
* Re-absorb new rows into the index once the unindexed tail passes this
|
|
15
|
+
* share of the indexed body. Rows outside the index are searched by flat
|
|
16
|
+
* scan, so the tail is exactly what needs bounding.
|
|
17
|
+
*/
|
|
18
|
+
UNINDEXED_REINDEX_RATIO: 0.2,
|
|
19
|
+
/** ...but always tolerate a small tail, so a 3-chunk edit never retrains. */
|
|
20
|
+
UNINDEXED_REINDEX_FLOOR: 2_000,
|
|
21
|
+
/**
|
|
22
|
+
* Vector searches run without an explicit distance type, which LanceDB
|
|
23
|
+
* resolves to L2. The index has its own default, so it is pinned: an index
|
|
24
|
+
* built on a different metric would silently reorder every result.
|
|
25
|
+
*/
|
|
26
|
+
DISTANCE_TYPE: "l2",
|
|
27
|
+
/**
|
|
28
|
+
* IVF-PQ stores compressed vectors, so its raw distances approximate the
|
|
29
|
+
* real ones. `refineFactor` makes LanceDB pull the uncompressed vectors for
|
|
30
|
+
* the top `limit * factor` candidates and re-score them exactly.
|
|
31
|
+
*
|
|
32
|
+
* It is not optional. Measured mean recall@20 against an exact scan:
|
|
33
|
+
*
|
|
34
|
+
* off -> 53% 1 -> 75% 3 -> 86% 10 -> 91% 20 -> 94% 50 -> 94%
|
|
35
|
+
*
|
|
36
|
+
* and without it `_distance` is wrong by ~0.07, which matters because score
|
|
37
|
+
* is derived from it and users filter on `--min-score`. With refinement the
|
|
38
|
+
* scores match an exact scan to the digit.
|
|
39
|
+
*
|
|
40
|
+
* 20 sits at the knee: ~94% recall at ~11ms/query versus ~115ms for an
|
|
41
|
+
* exact scan. Beyond it recall flattens (the probed partitions run out of
|
|
42
|
+
* candidates) while latency keeps climbing. The refined set is bounded by
|
|
43
|
+
* the query limit, not by table size, so this costs a fixed few MB.
|
|
44
|
+
*
|
|
45
|
+
* Harmless on tables with no index: a flat scan already computes exact
|
|
46
|
+
* distances and returns identical rows.
|
|
47
|
+
*/
|
|
48
|
+
REFINE_FACTOR: 20,
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Columns a search reads off a result row.
|
|
52
|
+
*
|
|
53
|
+
* Selecting explicitly keeps the embedding out of the result set — without it
|
|
54
|
+
* LanceDB ships every over-fetched row's full vector back into JS for nothing.
|
|
55
|
+
*
|
|
56
|
+
* `_distance` is listed on purpose: LanceDB currently auto-projects it into any
|
|
57
|
+
* scoring query but warns that it will stop, so naming it pins the behaviour
|
|
58
|
+
* rather than relying on a deprecated default.
|
|
59
|
+
*/
|
|
60
|
+
export const SEARCH_COLUMNS = [
|
|
61
|
+
"id",
|
|
62
|
+
"filePath",
|
|
63
|
+
"startLine",
|
|
64
|
+
"endLine",
|
|
65
|
+
"type",
|
|
66
|
+
"name",
|
|
67
|
+
"content",
|
|
68
|
+
"context",
|
|
69
|
+
"fileHash",
|
|
70
|
+
"_distance",
|
|
71
|
+
];
|
|
72
|
+
//# sourceMappingURL=VectorIndexPolicy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VectorIndexPolicy.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/VectorIndexPolicy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAChC;;;;OAIG;IACH,gBAAgB,EAAE,KAAK;IAEvB;;;;OAIG;IACH,uBAAuB,EAAE,GAAG;IAE5B,6EAA6E;IAC7E,uBAAuB,EAAE,KAAK;IAE9B;;;;OAIG;IACH,aAAa,EAAE,IAAI;IAEnB;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,aAAa,EAAE,EAAE;CACR,CAAC;AAEX;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC7B,IAAI;IACJ,UAAU;IACV,WAAW;IACX,SAAS;IACT,MAAM;IACN,MAAM;IACN,SAAS;IACT,SAAS;IACT,UAAU;IACV,WAAW;CACF,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { IndexMetadata } from "../../domain/project/IndexMetadata.js";
|
|
2
|
+
import type { RowReplication } from "../lancedb/RowReplication.js";
|
|
3
|
+
import { ShareCode } from "./ShareCode.js";
|
|
4
|
+
export interface ShareProgress {
|
|
5
|
+
onProgress?: (done: number, total: number) => void;
|
|
6
|
+
}
|
|
7
|
+
export interface ReceiveOptions extends ShareProgress {
|
|
8
|
+
onMeta?: (meta: {
|
|
9
|
+
model?: string;
|
|
10
|
+
dimensions?: number;
|
|
11
|
+
chunkCount: number;
|
|
12
|
+
remote?: string;
|
|
13
|
+
}) => void;
|
|
14
|
+
onWarning?: (message: string) => void;
|
|
15
|
+
}
|
|
16
|
+
export interface ReceiveResult {
|
|
17
|
+
chunks: number;
|
|
18
|
+
files: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Sends and receives a whole index between two machines.
|
|
22
|
+
*
|
|
23
|
+
* The transfer copies rows verbatim, embeddings included: re-embedding a large
|
|
24
|
+
* repository is the expensive thing, and avoiding it is the entire reason to
|
|
25
|
+
* share. That makes model compatibility the receiver's responsibility to check
|
|
26
|
+
* — vectors from a different model are silently meaningless, so a mismatch is
|
|
27
|
+
* warned about explicitly.
|
|
28
|
+
*/
|
|
29
|
+
export declare class IndexShare {
|
|
30
|
+
private readonly rows;
|
|
31
|
+
private static readonly BATCH_SIZE;
|
|
32
|
+
private static readonly PEER_TIMEOUT_MS;
|
|
33
|
+
constructor(rows: RowReplication);
|
|
34
|
+
/**
|
|
35
|
+
* Publish the index and return the code a peer needs. `done` resolves once
|
|
36
|
+
* a peer has taken the whole transfer.
|
|
37
|
+
*/
|
|
38
|
+
send(metadata: IndexMetadata | undefined, options?: ShareProgress): Promise<{
|
|
39
|
+
code: string;
|
|
40
|
+
done: Promise<void>;
|
|
41
|
+
}>;
|
|
42
|
+
/** Connect to a sharer and write everything it sends into this database. */
|
|
43
|
+
receive(code: ShareCode, localMetadata: IndexMetadata | undefined, options?: ReceiveOptions): Promise<ReceiveResult>;
|
|
44
|
+
private warnOnModelMismatch;
|
|
45
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { ModelIdentity } from "../../domain/project/ModelIdentity.js";
|
|
2
|
+
import { PeerSwarm } from "./PeerSwarm.js";
|
|
3
|
+
import { SecureChannel, } from "./SecureChannel.js";
|
|
4
|
+
import { ShareCode } from "./ShareCode.js";
|
|
5
|
+
/**
|
|
6
|
+
* Sends and receives a whole index between two machines.
|
|
7
|
+
*
|
|
8
|
+
* The transfer copies rows verbatim, embeddings included: re-embedding a large
|
|
9
|
+
* repository is the expensive thing, and avoiding it is the entire reason to
|
|
10
|
+
* share. That makes model compatibility the receiver's responsibility to check
|
|
11
|
+
* — vectors from a different model are silently meaningless, so a mismatch is
|
|
12
|
+
* warned about explicitly.
|
|
13
|
+
*/
|
|
14
|
+
export class IndexShare {
|
|
15
|
+
rows;
|
|
16
|
+
static BATCH_SIZE = 200;
|
|
17
|
+
static PEER_TIMEOUT_MS = 5 * 60 * 1000;
|
|
18
|
+
constructor(rows) {
|
|
19
|
+
this.rows = rows;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Publish the index and return the code a peer needs. `done` resolves once
|
|
23
|
+
* a peer has taken the whole transfer.
|
|
24
|
+
*/
|
|
25
|
+
async send(metadata, options = {}) {
|
|
26
|
+
const swarm = await PeerSwarm.create();
|
|
27
|
+
const code = ShareCode.generate();
|
|
28
|
+
const channel = new SecureChannel(code.toEncryptionKey());
|
|
29
|
+
const chunkCount = await this.rows.chunkCount();
|
|
30
|
+
const done = new Promise((resolve, reject) => {
|
|
31
|
+
const timeout = setTimeout(() => {
|
|
32
|
+
void swarm.destroy();
|
|
33
|
+
reject(new Error("Timed out waiting for peer (5 minutes)."));
|
|
34
|
+
}, IndexShare.PEER_TIMEOUT_MS);
|
|
35
|
+
swarm.onConnection(async (socket) => {
|
|
36
|
+
clearTimeout(timeout);
|
|
37
|
+
try {
|
|
38
|
+
channel.send(socket, {
|
|
39
|
+
type: "meta",
|
|
40
|
+
model: metadata?.model,
|
|
41
|
+
dimensions: metadata?.dimensions,
|
|
42
|
+
chunkCount,
|
|
43
|
+
branch: metadata?.branch ?? "main",
|
|
44
|
+
remote: metadata?.remote,
|
|
45
|
+
});
|
|
46
|
+
// Wait for the receiver to accept before streaming — it may
|
|
47
|
+
// refuse on an incompatible model.
|
|
48
|
+
await channel.await(socket, "ready");
|
|
49
|
+
let sent = 0;
|
|
50
|
+
for await (const batch of this.rows.streamChunkRows(IndexShare.BATCH_SIZE)) {
|
|
51
|
+
channel.send(socket, { type: "chunks", batch });
|
|
52
|
+
sent += batch.length;
|
|
53
|
+
options.onProgress?.(sent, chunkCount);
|
|
54
|
+
}
|
|
55
|
+
const files = await this.rows.allManifestRows();
|
|
56
|
+
if (files.length > 0) {
|
|
57
|
+
channel.send(socket, { type: "files", batch: files });
|
|
58
|
+
}
|
|
59
|
+
channel.send(socket, { type: "done" });
|
|
60
|
+
// Wait for the ack so the socket is not torn down before
|
|
61
|
+
// the receiver has committed.
|
|
62
|
+
await channel.await(socket, "ack");
|
|
63
|
+
socket.end();
|
|
64
|
+
await swarm.destroy();
|
|
65
|
+
resolve();
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
await swarm.destroy();
|
|
69
|
+
reject(err);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
swarm.join(code.toTopic(), "server").catch(reject);
|
|
73
|
+
});
|
|
74
|
+
return { code: code.toString(), done };
|
|
75
|
+
}
|
|
76
|
+
/** Connect to a sharer and write everything it sends into this database. */
|
|
77
|
+
async receive(code, localMetadata, options = {}) {
|
|
78
|
+
const swarm = await PeerSwarm.create();
|
|
79
|
+
const channel = new SecureChannel(code.toEncryptionKey());
|
|
80
|
+
return new Promise((resolve, reject) => {
|
|
81
|
+
const timeout = setTimeout(() => {
|
|
82
|
+
void swarm.destroy();
|
|
83
|
+
reject(new Error("Timed out waiting for peer (5 minutes)."));
|
|
84
|
+
}, IndexShare.PEER_TIMEOUT_MS);
|
|
85
|
+
const fail = (err) => {
|
|
86
|
+
void swarm.destroy().then(() => reject(err));
|
|
87
|
+
};
|
|
88
|
+
swarm.onConnection((socket) => {
|
|
89
|
+
clearTimeout(timeout);
|
|
90
|
+
let expected = 0;
|
|
91
|
+
let chunks = 0;
|
|
92
|
+
let files = 0;
|
|
93
|
+
let accepted = false;
|
|
94
|
+
const handle = (message) => {
|
|
95
|
+
switch (message.type) {
|
|
96
|
+
case "meta": {
|
|
97
|
+
expected = message.chunkCount;
|
|
98
|
+
options.onMeta?.({
|
|
99
|
+
model: message.model,
|
|
100
|
+
dimensions: message.dimensions,
|
|
101
|
+
chunkCount: message.chunkCount,
|
|
102
|
+
remote: message.remote,
|
|
103
|
+
});
|
|
104
|
+
this.warnOnModelMismatch(localMetadata, message, options.onWarning);
|
|
105
|
+
channel.send(socket, { type: "ready" });
|
|
106
|
+
accepted = true;
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
case "chunks": {
|
|
110
|
+
// Ignore anything arriving before the handshake.
|
|
111
|
+
if (!accepted)
|
|
112
|
+
break;
|
|
113
|
+
chunks += message.batch.length;
|
|
114
|
+
options.onProgress?.(chunks, expected);
|
|
115
|
+
this.rows.addChunkRows(message.batch).catch(fail);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
case "files": {
|
|
119
|
+
if (!accepted)
|
|
120
|
+
break;
|
|
121
|
+
files += message.batch.length;
|
|
122
|
+
this.rows.addManifestRows(message.batch).catch(fail);
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
case "done": {
|
|
126
|
+
channel.send(socket, { type: "ack" });
|
|
127
|
+
void swarm.destroy().then(() => resolve({ chunks, files }));
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
const read = channel.reader((message) => {
|
|
133
|
+
try {
|
|
134
|
+
handle(message);
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
fail(new Error(`Protocol error: ${err}`));
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
socket.on("data", read);
|
|
141
|
+
socket.on("error", fail);
|
|
142
|
+
});
|
|
143
|
+
swarm.join(code.toTopic(), "client").catch(reject);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
warnOnModelMismatch(local, remote, onWarning) {
|
|
147
|
+
if (!local?.model || !remote.model)
|
|
148
|
+
return;
|
|
149
|
+
const localFamily = ModelIdentity.of(local.model).family;
|
|
150
|
+
const remoteFamily = ModelIdentity.of(remote.model).family;
|
|
151
|
+
if (localFamily === remoteFamily &&
|
|
152
|
+
local.dimensions === remote.dimensions) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
onWarning?.(`Warning: source uses "${remote.model}" (${remote.dimensions} dims) ` +
|
|
156
|
+
`but local index uses "${local.model}" (${local.dimensions} dims). ` +
|
|
157
|
+
"Imported vectors may not be compatible.");
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
//# sourceMappingURL=IndexShare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IndexShare.js","sourceRoot":"","sources":["../../../src/infrastructure/p2p/IndexShare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAEtE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAEN,aAAa,GAEb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAqB3C;;;;;;;;GAQG;AACH,MAAM,OAAO,UAAU;IAIO;IAHrB,MAAM,CAAU,UAAU,GAAG,GAAG,CAAC;IACjC,MAAM,CAAU,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IAExD,YAA6B,IAAoB;QAApB,SAAI,GAAJ,IAAI,CAAgB;IAAG,CAAC;IAErD;;;OAGG;IACH,KAAK,CAAC,IAAI,CACT,QAAmC,EACnC,UAAyB,EAAE;QAE3B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAEhD,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAClD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC,CAAC;YAC9D,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC;YAE/B,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACnC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,IAAI,CAAC;oBACJ,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE;wBACpB,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,QAAQ,EAAE,KAAK;wBACtB,UAAU,EAAE,QAAQ,EAAE,UAAU;wBAChC,UAAU;wBACV,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,MAAM;wBAClC,MAAM,EAAE,QAAQ,EAAE,MAAM;qBACxB,CAAC,CAAC;oBACH,4DAA4D;oBAC5D,mCAAmC;oBACnC,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAErC,IAAI,IAAI,GAAG,CAAC,CAAC;oBACb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,CAClD,UAAU,CAAC,UAAU,CACrB,EAAE,CAAC;wBACH,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;wBAChD,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;wBACrB,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;oBACxC,CAAC;oBAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAChD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACtB,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvD,CAAC;oBAED,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;oBACvC,yDAAyD;oBACzD,8BAA8B;oBAC9B,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBAEnC,MAAM,CAAC,GAAG,EAAE,CAAC;oBACb,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;oBACtB,OAAO,EAAE,CAAC;gBACX,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;oBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC;YACF,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,4EAA4E;IAC5E,KAAK,CAAC,OAAO,CACZ,IAAe,EACf,aAAwC,EACxC,UAA0B,EAAE;QAE5B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE1D,OAAO,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC,CAAC;YAC9D,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC;YAE/B,MAAM,IAAI,GAAG,CAAC,GAAY,EAAQ,EAAE;gBACnC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,CAAC,CAAC;YAEF,KAAK,CAAC,YAAY,CAAC,CAAC,MAAoB,EAAE,EAAE;gBAC3C,YAAY,CAAC,OAAO,CAAC,CAAC;gBAEtB,IAAI,QAAQ,GAAG,CAAC,CAAC;gBACjB,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,QAAQ,GAAG,KAAK,CAAC;gBAErB,MAAM,MAAM,GAAG,CAAC,OAAqB,EAAQ,EAAE;oBAC9C,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;wBACtB,KAAK,MAAM,CAAC,CAAC,CAAC;4BACb,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;4BAC9B,OAAO,CAAC,MAAM,EAAE,CAAC;gCAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gCACpB,UAAU,EAAE,OAAO,CAAC,UAAU;gCAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;gCAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;6BACtB,CAAC,CAAC;4BACH,IAAI,CAAC,mBAAmB,CACvB,aAAa,EACb,OAAO,EACP,OAAO,CAAC,SAAS,CACjB,CAAC;4BACF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;4BACxC,QAAQ,GAAG,IAAI,CAAC;4BAChB,MAAM;wBACP,CAAC;wBACD,KAAK,QAAQ,CAAC,CAAC,CAAC;4BACf,iDAAiD;4BACjD,IAAI,CAAC,QAAQ;gCAAE,MAAM;4BACrB,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;4BAC/B,OAAO,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;4BACvC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;4BAClD,MAAM;wBACP,CAAC;wBACD,KAAK,OAAO,CAAC,CAAC,CAAC;4BACd,IAAI,CAAC,QAAQ;gCAAE,MAAM;4BACrB,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;4BAC9B,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;4BACrD,MAAM;wBACP,CAAC;wBACD,KAAK,MAAM,CAAC,CAAC,CAAC;4BACb,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;4BACtC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;4BAC5D,MAAM;wBACP,CAAC;oBACF,CAAC;gBACF,CAAC,CAAC;gBAEF,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;oBACvC,IAAI,CAAC;wBACJ,MAAM,CAAC,OAAO,CAAC,CAAC;oBACjB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACd,IAAI,CAAC,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC;oBAC3C,CAAC;gBACF,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACxB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,mBAAmB,CAC1B,KAAgC,EAChC,MAA+C,EAC/C,SAAqC;QAErC,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO;QAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACzD,MAAM,YAAY,GAAG,aAAa,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QAC3D,IACC,WAAW,KAAK,YAAY;YAC5B,KAAK,CAAC,UAAU,KAAK,MAAM,CAAC,UAAU,EACrC,CAAC;YACF,OAAO;QACR,CAAC;QACD,SAAS,EAAE,CACV,yBAAyB,MAAM,CAAC,KAAK,MAAM,MAAM,CAAC,UAAU,SAAS;YACpE,yBAAyB,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,UAAU,UAAU;YACpE,yCAAyC,CAC1C,CAAC;IACH,CAAC"}
|