codesearch-mcp 0.2.3-alpha → 0.4.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -5
- package/LICENSE +661 -21
- package/README.md +76 -40
- package/dist/app.js +17 -38
- package/dist/src/ast/CallExtractor.d.ts +8 -0
- package/dist/src/ast/CallExtractor.js +68 -0
- package/dist/src/ast/GraphqlNameExtractor.d.ts +22 -0
- package/dist/src/ast/GraphqlNameExtractor.js +240 -0
- package/dist/src/ast/ImportEdgeExtractor.d.ts +11 -0
- package/dist/src/ast/ImportEdgeExtractor.js +129 -0
- package/dist/src/ast/IndexedNodeResolver.d.ts +14 -0
- package/dist/src/ast/IndexedNodeResolver.js +217 -0
- package/dist/src/ast/MemberExtractor.d.ts +8 -0
- package/dist/src/ast/MemberExtractor.js +129 -0
- package/dist/src/ast/TopLevelSymbolExtractor.d.ts +5 -0
- package/dist/src/ast/TopLevelSymbolExtractor.js +86 -0
- package/dist/src/ast/graphql-grammar.d.ts +24 -0
- package/dist/src/ast/graphql-grammar.js +93 -0
- package/dist/src/ast/node-helpers.d.ts +11 -0
- package/dist/src/ast/node-helpers.js +55 -0
- package/dist/src/chunking/AstChunker.d.ts +12 -0
- package/dist/src/chunking/AstChunker.js +30 -0
- package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
- package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
- package/dist/src/chunking/FragmentMerger.d.ts +11 -0
- package/dist/src/chunking/FragmentMerger.js +61 -0
- package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
- package/dist/src/chunking/ImportOnlyFilter.js +14 -0
- package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
- package/dist/src/chunking/LangchainTextSplitter.js +133 -0
- package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
- package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
- package/dist/src/chunking/RunPacker.d.ts +12 -0
- package/dist/src/chunking/RunPacker.js +84 -0
- package/dist/src/chunking/create-chunker.d.ts +3 -0
- package/dist/src/chunking/create-chunker.js +15 -0
- package/dist/src/chunking/interfaces.d.ts +27 -0
- package/dist/src/commands/ConfigCommand.js +1 -9
- package/dist/src/commands/EvalCommand.d.ts +10 -5
- package/dist/src/commands/EvalCommand.js +60 -149
- package/dist/src/commands/IndexCommand.d.ts +10 -4
- package/dist/src/commands/IndexCommand.js +52 -146
- package/dist/src/commands/McpCommand.d.ts +6 -3
- package/dist/src/commands/McpCommand.js +19 -20
- package/dist/src/commands/WebCommand.d.ts +9 -3
- package/dist/src/commands/WebCommand.js +17 -11
- package/dist/src/commands/eval/baseline-comparison.d.ts +6 -0
- package/dist/src/commands/eval/baseline-comparison.js +54 -0
- package/dist/src/commands/eval/eval-args.d.ts +6 -0
- package/dist/src/commands/eval/eval-args.js +71 -0
- package/dist/src/commands/eval/eval-corpora.d.ts +6 -0
- package/dist/src/commands/eval/eval-corpora.js +76 -0
- package/dist/src/commands/eval/eval-metadata.d.ts +17 -0
- package/dist/src/commands/eval/eval-metadata.js +75 -0
- package/dist/src/commands/eval/index-inspector.d.ts +3 -0
- package/dist/src/commands/eval/index-inspector.js +21 -0
- package/dist/src/commands/eval/staged-search.d.ts +3 -0
- package/dist/src/commands/eval/staged-search.js +20 -0
- package/dist/src/commands/eval/suite-loader.d.ts +8 -0
- package/dist/src/commands/eval/suite-loader.js +52 -0
- package/dist/src/commands/index/backend-probe.d.ts +2 -0
- package/dist/src/commands/index/backend-probe.js +15 -0
- package/dist/src/commands/index/embedding-setup.d.ts +3 -0
- package/dist/src/commands/index/embedding-setup.js +16 -0
- package/dist/src/commands/index/index-summary.d.ts +2 -0
- package/dist/src/commands/index/index-summary.js +22 -0
- package/dist/src/commands/index/legacy-ollama-guard.d.ts +3 -0
- package/dist/src/commands/index/legacy-ollama-guard.js +23 -0
- package/dist/src/commands/index/prune-step.d.ts +3 -0
- package/dist/src/commands/index/prune-step.js +22 -0
- package/dist/src/commands/index/scan-targets.d.ts +8 -0
- package/dist/src/commands/index/scan-targets.js +107 -0
- package/dist/src/config-error.d.ts +0 -4
- package/dist/src/config-error.js +0 -4
- package/dist/src/config-store.d.ts +1 -55
- package/dist/src/config-store.js +11 -73
- package/dist/src/config.d.ts +9 -18
- package/dist/src/config.js +36 -35
- package/dist/src/constants.d.ts +17 -28
- package/dist/src/constants.js +35 -29
- package/dist/src/container.d.ts +10 -2
- package/dist/src/container.js +19 -8
- package/dist/src/db/drizzle-types.d.ts +7 -0
- package/dist/src/db/drizzle-types.js +2 -0
- package/dist/src/db/index.d.ts +1 -0
- package/dist/src/db/schema-steps.d.ts +8 -0
- package/dist/src/db/schema-steps.js +105 -0
- package/dist/src/db/schema.d.ts +17 -0
- package/dist/src/db/schema.js +1 -6
- package/dist/src/declarations/DeclarationTree.d.ts +19 -0
- package/dist/src/declarations/DeclarationTree.js +66 -0
- package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
- package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
- package/dist/src/declarations/SourceLines.d.ts +37 -0
- package/dist/src/declarations/SourceLines.js +415 -0
- package/dist/src/dependency-traversal.d.ts +0 -6
- package/dist/src/dependency-traversal.js +0 -6
- package/dist/src/enriched-prompt.d.ts +0 -6
- package/dist/src/enriched-prompt.js +0 -6
- package/dist/src/error-record.d.ts +12 -0
- package/dist/src/error-record.js +248 -0
- package/dist/src/eval-corpus.d.ts +16 -0
- package/dist/src/eval-corpus.js +169 -0
- package/dist/src/eval-results.d.ts +6 -3
- package/dist/src/eval-results.js +323 -26
- package/dist/src/eval-vector-index.d.ts +5 -0
- package/dist/src/eval-vector-index.js +60 -0
- package/dist/src/file-utils.d.ts +0 -5
- package/dist/src/file-utils.js +0 -5
- package/dist/src/graph.d.ts +0 -25
- package/dist/src/graph.js +0 -28
- package/dist/src/help.js +9 -1
- package/dist/src/hnsw-settings.d.ts +3 -0
- package/dist/src/hnsw-settings.js +24 -0
- package/dist/src/index-args.d.ts +13 -0
- package/dist/src/index-args.js +48 -0
- package/dist/src/indexing/AnalysisBatcher.d.ts +12 -0
- package/dist/src/indexing/AnalysisBatcher.js +28 -0
- package/dist/src/indexing/ChunkEmbedder.d.ts +24 -0
- package/dist/src/indexing/ChunkEmbedder.js +37 -0
- package/dist/src/indexing/ConsoleScanReporter.d.ts +10 -0
- package/dist/src/indexing/ConsoleScanReporter.js +73 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.d.ts +14 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.js +66 -0
- package/dist/src/indexing/FileIngestor.d.ts +22 -0
- package/dist/src/indexing/FileIngestor.js +116 -0
- package/dist/src/indexing/IndexStore.d.ts +24 -0
- package/dist/src/indexing/IndexStore.js +69 -0
- package/dist/src/indexing/StaleFilePruner.d.ts +8 -0
- package/dist/src/indexing/StaleFilePruner.js +55 -0
- package/dist/src/indexing/analysis-rows.d.ts +4 -0
- package/dist/src/indexing/analysis-rows.js +65 -0
- package/dist/src/indexing/project-notices.d.ts +2 -0
- package/dist/src/indexing/project-notices.js +63 -0
- package/dist/src/interfaces/IAstParser.d.ts +4 -40
- package/dist/src/interfaces/ICallGraph.d.ts +28 -0
- package/dist/src/interfaces/ICallGraph.js +2 -0
- package/dist/src/interfaces/IChunkSearch.d.ts +42 -0
- package/dist/src/interfaces/IChunkSearch.js +2 -0
- package/dist/src/interfaces/IChunker.d.ts +3 -12
- package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
- package/dist/src/interfaces/ICorpusInspector.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +8 -24
- package/dist/src/interfaces/IDependencyGraph.d.ts +24 -0
- package/dist/src/interfaces/IDependencyGraph.js +2 -0
- package/dist/src/interfaces/IEmbedder.d.ts +0 -9
- package/dist/src/interfaces/IIndexIdentityGuard.d.ts +3 -0
- package/dist/src/interfaces/IIndexIdentityGuard.js +2 -0
- package/dist/src/interfaces/IIndexedNodeResolver.d.ts +10 -0
- package/dist/src/interfaces/IIndexedNodeResolver.js +2 -0
- package/dist/src/interfaces/ILogger.d.ts +1 -11
- package/dist/src/interfaces/IMcpReadiness.d.ts +12 -0
- package/dist/src/interfaces/IMcpReadiness.js +2 -0
- package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
- package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
- package/dist/src/interfaces/IProjectCatalog.d.ts +13 -0
- package/dist/src/interfaces/IProjectCatalog.js +2 -0
- package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
- package/dist/src/interfaces/IProjectRoots.d.ts +3 -0
- package/dist/src/interfaces/IProjectRoots.js +2 -0
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +153 -12
- package/dist/src/interfaces/IScanReporter.d.ts +13 -0
- package/dist/src/interfaces/IScanReporter.js +2 -0
- package/dist/src/interfaces/ISymbolLookup.d.ts +49 -0
- package/dist/src/interfaces/ISymbolLookup.js +2 -0
- package/dist/src/interfaces/IToolPaths.d.ts +62 -0
- package/dist/src/interfaces/IToolPaths.js +2 -0
- package/dist/src/interfaces/IWorkspaceFiles.d.ts +28 -0
- package/dist/src/interfaces/IWorkspaceFiles.js +2 -0
- package/dist/src/interfaces/SymbolKind.d.ts +1 -0
- package/dist/src/interfaces/SymbolKind.js +2 -0
- package/dist/src/interfaces/index.d.ts +12 -1
- package/dist/src/interrupt.js +0 -2
- package/dist/src/owner-path.d.ts +9 -10
- package/dist/src/owner-path.js +11 -23
- package/dist/src/project-scope.d.ts +1 -12
- package/dist/src/project-scope.js +5 -12
- package/dist/src/project-selector.d.ts +0 -5
- package/dist/src/project-selector.js +0 -11
- package/dist/src/retrieval-eval.d.ts +15 -6
- package/dist/src/retrieval-eval.js +388 -59
- package/dist/src/search/CallGraph.d.ts +9 -0
- package/dist/src/search/CallGraph.js +84 -0
- package/dist/src/search/ChunkSearch.d.ts +23 -0
- package/dist/src/search/ChunkSearch.js +115 -0
- package/dist/src/search/DependencyGraph.d.ts +19 -0
- package/dist/src/search/DependencyGraph.js +180 -0
- package/dist/src/search/IndexIdentityGuard.d.ts +11 -0
- package/dist/src/search/IndexIdentityGuard.js +37 -0
- package/dist/src/search/ProjectCatalog.d.ts +11 -0
- package/dist/src/search/ProjectCatalog.js +38 -0
- package/dist/src/search/SymbolLookup.d.ts +24 -0
- package/dist/src/search/SymbolLookup.js +97 -0
- package/dist/src/search/chunks/CandidateFetcher.d.ts +19 -0
- package/dist/src/search/chunks/CandidateFetcher.js +40 -0
- package/dist/src/search/chunks/IterativeScanGate.d.ts +10 -0
- package/dist/src/search/chunks/IterativeScanGate.js +42 -0
- package/dist/src/search/chunks/RerankStage.d.ts +15 -0
- package/dist/src/search/chunks/RerankStage.js +42 -0
- package/dist/src/search/chunks/search-metrics.d.ts +18 -0
- package/dist/src/search/chunks/search-metrics.js +32 -0
- package/dist/src/search/chunks/search-plan.d.ts +18 -0
- package/dist/src/search/chunks/search-plan.js +19 -0
- package/dist/src/search/create-search-services.d.ts +26 -0
- package/dist/src/search/create-search-services.js +29 -0
- package/dist/src/search/index-paths.d.ts +1 -0
- package/dist/src/search/index-paths.js +5 -0
- package/dist/src/search/result-shaping.d.ts +4 -0
- package/dist/src/search/result-shaping.js +19 -0
- package/dist/src/search/symbol-sql.d.ts +13 -0
- package/dist/src/search/symbol-sql.js +66 -0
- package/dist/src/search/symbols/SymbolGrepFallback.d.ts +19 -0
- package/dist/src/search/symbols/SymbolGrepFallback.js +106 -0
- package/dist/src/search/symbols/SymbolIndexQuery.d.ts +14 -0
- package/dist/src/search/symbols/SymbolIndexQuery.js +52 -0
- package/dist/src/search/symbols/SymbolTextSearch.d.ts +13 -0
- package/dist/src/search/symbols/SymbolTextSearch.js +84 -0
- package/dist/src/search/symbols/TargetColumnGate.d.ts +7 -0
- package/dist/src/search/symbols/TargetColumnGate.js +23 -0
- package/dist/src/search/symbols/find-symbol-constants.d.ts +4 -0
- package/dist/src/search/symbols/find-symbol-constants.js +8 -0
- package/dist/src/search/symbols/grep-args.d.ts +3 -0
- package/dist/src/search/symbols/grep-args.js +7 -0
- package/dist/src/search/symbols/grep-patterns.d.ts +4 -0
- package/dist/src/search/symbols/grep-patterns.js +60 -0
- package/dist/src/services/AstParserService.d.ts +10 -106
- package/dist/src/services/AstParserService.js +37 -1068
- package/dist/src/services/DatabaseService.d.ts +10 -17
- package/dist/src/services/DatabaseService.js +71 -115
- package/dist/src/services/EmbedderFactory.d.ts +0 -4
- package/dist/src/services/EmbedderFactory.js +0 -5
- package/dist/src/services/IndexerService.d.ts +7 -26
- package/dist/src/services/IndexerService.js +38 -286
- package/dist/src/services/InfinityEmbedder.d.ts +0 -21
- package/dist/src/services/InfinityEmbedder.js +0 -15
- package/dist/src/services/LoggerService.d.ts +7 -5
- package/dist/src/services/LoggerService.js +37 -41
- package/dist/src/services/McpReadiness.d.ts +18 -0
- package/dist/src/services/McpReadiness.js +73 -0
- package/dist/src/services/McpServer.d.ts +50 -19
- package/dist/src/services/McpServer.js +132 -62
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
- package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
- package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
- package/dist/src/services/NxMonorepoResolver.js +5 -56
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
- package/dist/src/services/OpenAICompatibleEmbedder.js +0 -14
- package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
- package/dist/src/services/ProjectDiscoveryService.js +0 -4
- package/dist/src/services/VoyageEmbedder.d.ts +0 -6
- package/dist/src/services/VoyageEmbedder.js +0 -6
- package/dist/src/services/WebServerService.d.ts +5 -56
- package/dist/src/services/WebServerService.js +25 -63
- package/dist/src/services/http-retry.d.ts +0 -7
- package/dist/src/services/http-retry.js +1 -10
- package/dist/src/services/infinity-provisioning.d.ts +0 -74
- package/dist/src/services/infinity-provisioning.js +5 -88
- package/dist/src/services/legacy-ollama-index.d.ts +0 -16
- package/dist/src/services/legacy-ollama-index.js +0 -23
- package/dist/src/services/reranker.d.ts +0 -9
- package/dist/src/services/reranker.js +46 -12
- package/dist/src/setup.d.ts +24 -24
- package/dist/src/setup.js +205 -103
- package/dist/src/symbol-name.d.ts +1 -12
- package/dist/src/symbol-name.js +16 -25
- package/dist/src/symbol-rows.d.ts +0 -14
- package/dist/src/symbol-rows.js +0 -14
- package/dist/src/tool-call-log.d.ts +2 -0
- package/dist/src/tool-call-log.js +24 -0
- package/dist/src/tools/FindSymbolTool.d.ts +17 -7
- package/dist/src/tools/FindSymbolTool.js +54 -35
- package/dist/src/tools/GetCallersTool.d.ts +16 -8
- package/dist/src/tools/GetCallersTool.js +65 -35
- package/dist/src/tools/GetDependenciesTool.d.ts +16 -8
- package/dist/src/tools/GetDependenciesTool.js +36 -29
- package/dist/src/tools/GetDependentsTool.d.ts +20 -8
- package/dist/src/tools/GetDependentsTool.js +48 -34
- package/dist/src/tools/ListFilesTool.d.ts +16 -8
- package/dist/src/tools/ListFilesTool.js +20 -26
- package/dist/src/tools/ListProjectsTool.d.ts +11 -7
- package/dist/src/tools/ListProjectsTool.js +13 -11
- package/dist/src/tools/McpTool.d.ts +17 -0
- package/dist/src/tools/McpTool.js +28 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +16 -8
- package/dist/src/tools/ReadFileRangeTool.js +23 -25
- package/dist/src/tools/ReadFileTool.d.ts +15 -8
- package/dist/src/tools/ReadFileTool.js +19 -24
- package/dist/src/tools/SearchCodebaseTool.d.ts +22 -10
- package/dist/src/tools/SearchCodebaseTool.js +71 -50
- package/dist/src/tools/format-helpers.d.ts +9 -9
- package/dist/src/tools/format-helpers.js +146 -28
- package/dist/src/tools/tool-args.d.ts +38 -0
- package/dist/src/tools/tool-args.js +95 -0
- package/dist/src/web/dashboard.d.ts +0 -8
- package/dist/src/web/dashboard.js +7 -10
- package/dist/src/workspace/GrepRunner.d.ts +15 -0
- package/dist/src/workspace/GrepRunner.js +104 -0
- package/dist/src/workspace/ProjectRoots.d.ts +22 -0
- package/dist/src/workspace/ProjectRoots.js +102 -0
- package/dist/src/workspace/ToolPaths.d.ts +31 -0
- package/dist/src/workspace/ToolPaths.js +312 -0
- package/dist/src/workspace/WorkspaceFiles.d.ts +19 -0
- package/dist/src/workspace/WorkspaceFiles.js +285 -0
- package/dist/src/workspace/fs-access.d.ts +9 -0
- package/dist/src/workspace/fs-access.js +47 -0
- package/dist/src/workspace/path-helpers.d.ts +3 -0
- package/dist/src/workspace/path-helpers.js +43 -0
- package/package.json +5 -4
- package/dist/src/interfaces/ISearcher.d.ts +0 -35
- package/dist/src/services/AstChunker.d.ts +0 -42
- package/dist/src/services/AstChunker.js +0 -158
- package/dist/src/services/SearcherService.d.ts +0 -342
- package/dist/src/services/SearcherService.js +0 -1199
- package/docs/web-dashboard.md +0 -206
- /package/dist/src/{interfaces/ISearcher.js → chunking/interfaces.js} +0 -0
|
@@ -1,11 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RetrievalEvaluator = exports.EvalSuiteError = void 0;
|
|
3
|
+
exports.RetrievalEvaluator = exports.LIVE_GROUP = exports.EvalSuiteError = exports.STAGE_ORDER = void 0;
|
|
4
4
|
exports.parseSuite = parseSuite;
|
|
5
|
+
exports.corporaOf = corporaOf;
|
|
6
|
+
exports.isUnderRoot = isUnderRoot;
|
|
7
|
+
exports.checkCaseCorpora = checkCaseCorpora;
|
|
5
8
|
exports.mergeSuites = mergeSuites;
|
|
9
|
+
exports.signTest = signTest;
|
|
10
|
+
exports.pairedChange = pairedChange;
|
|
11
|
+
exports.formatThresholdConditions = formatThresholdConditions;
|
|
6
12
|
exports.compare = compare;
|
|
7
13
|
const zod_1 = require("zod");
|
|
8
|
-
|
|
14
|
+
const constants_1 = require("./constants");
|
|
15
|
+
const eval_vector_index_1 = require("./eval-vector-index");
|
|
16
|
+
const CATEGORIES = ['identifier', 'behaviour', 'concept'];
|
|
17
|
+
exports.STAGE_ORDER = ['vector', 'reranked'];
|
|
18
|
+
const STAGE_SOURCE = { reranked: 'vector' };
|
|
19
|
+
const UNRANKED = 11;
|
|
9
20
|
class EvalSuiteError extends Error {
|
|
10
21
|
caseId;
|
|
11
22
|
field;
|
|
@@ -29,11 +40,15 @@ const CaseSchema = zod_1.z.object({
|
|
|
29
40
|
expect: zod_1.z.array(ExpectedAnswerSchema).min(1),
|
|
30
41
|
threshold: zod_1.z.number().min(0).max(1).optional(),
|
|
31
42
|
});
|
|
43
|
+
const CorpusSchema = zod_1.z.object({
|
|
44
|
+
root: zod_1.z.string().min(1).refine(r => !r.startsWith('/') && !r.split('/').includes('..') && !r.endsWith('/'), 'must be a workspace-relative directory'),
|
|
45
|
+
commit: zod_1.z.string().regex(/^[0-9a-f]{40}$/, 'must be a full 40-hex commit SHA'),
|
|
46
|
+
}).strict();
|
|
32
47
|
const SuiteSchema = zod_1.z.object({
|
|
33
48
|
version: zod_1.z.literal(1),
|
|
49
|
+
corpus: CorpusSchema.optional(),
|
|
34
50
|
cases: zod_1.z.array(CaseSchema),
|
|
35
51
|
});
|
|
36
|
-
/** Best-effort id of the case at `cases[index]` in a raw (possibly invalid) object, for error messages. */
|
|
37
52
|
function rawCaseId(raw, index) {
|
|
38
53
|
if (typeof raw !== 'object' || raw === null)
|
|
39
54
|
return null;
|
|
@@ -46,7 +61,6 @@ function rawCaseId(raw, index) {
|
|
|
46
61
|
const id = entry.id;
|
|
47
62
|
return typeof id === 'string' ? id : null;
|
|
48
63
|
}
|
|
49
|
-
/** Validates a loaded suite object against the schema; rejects before any search runs. */
|
|
50
64
|
function parseSuite(raw) {
|
|
51
65
|
const result = SuiteSchema.safeParse(raw);
|
|
52
66
|
if (!result.success) {
|
|
@@ -64,10 +78,40 @@ function parseSuite(raw) {
|
|
|
64
78
|
}
|
|
65
79
|
seen.add(c.id);
|
|
66
80
|
}
|
|
67
|
-
|
|
81
|
+
const corpus = result.data.corpus?.root ?? null;
|
|
82
|
+
return { ...result.data, cases: result.data.cases.map(c => ({ ...c, corpus })) };
|
|
83
|
+
}
|
|
84
|
+
function corporaOf(suites) {
|
|
85
|
+
const byRoot = new Map();
|
|
86
|
+
for (const suite of suites) {
|
|
87
|
+
if (!suite.corpus)
|
|
88
|
+
continue;
|
|
89
|
+
const prev = byRoot.get(suite.corpus.root);
|
|
90
|
+
if (prev && prev.commit !== suite.corpus.commit) {
|
|
91
|
+
throw new EvalSuiteError(`Corpus ${suite.corpus.root} is declared at two commits: ${prev.commit} and ${suite.corpus.commit}`, null, 'corpus');
|
|
92
|
+
}
|
|
93
|
+
byRoot.set(suite.corpus.root, { root: suite.corpus.root, commit: suite.corpus.commit });
|
|
94
|
+
}
|
|
95
|
+
return [...byRoot.values()].sort((a, b) => (a.root < b.root ? -1 : a.root > b.root ? 1 : 0));
|
|
96
|
+
}
|
|
97
|
+
function isUnderRoot(dir, root) {
|
|
98
|
+
return dir === root || dir.startsWith(root + '/');
|
|
99
|
+
}
|
|
100
|
+
async function checkCaseCorpora(cases, projectDir) {
|
|
101
|
+
for (const c of cases) {
|
|
102
|
+
if (!c.corpus)
|
|
103
|
+
continue;
|
|
104
|
+
const dir = await projectDir(c.project);
|
|
105
|
+
if (dir === null) {
|
|
106
|
+
throw new EvalSuiteError(`Project ${c.project} is not indexed, so it cannot be checked against corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
|
|
107
|
+
}
|
|
108
|
+
if (!isUnderRoot(dir, c.corpus)) {
|
|
109
|
+
throw new EvalSuiteError(`Project ${c.project} (${dir}) is outside corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
68
112
|
}
|
|
69
|
-
/** Merges several suites (already parsed) into one; case ids must be unique across all of them. */
|
|
70
113
|
function mergeSuites(suites) {
|
|
114
|
+
corporaOf(suites);
|
|
71
115
|
const cases = [];
|
|
72
116
|
const seen = new Set();
|
|
73
117
|
for (const suite of suites) {
|
|
@@ -80,7 +124,6 @@ function mergeSuites(suites) {
|
|
|
80
124
|
}
|
|
81
125
|
return { version: 1, cases };
|
|
82
126
|
}
|
|
83
|
-
/** `owner` matches exactly, or as a prefix at a `.` boundary (`RepairsService` accepts any member of the class). */
|
|
84
127
|
function ownerMatches(ownerPath, expectedOwner) {
|
|
85
128
|
return ownerPath === expectedOwner || ownerPath.startsWith(expectedOwner + '.');
|
|
86
129
|
}
|
|
@@ -117,45 +160,219 @@ function rankOf(hits, valid) {
|
|
|
117
160
|
function emptyMetrics() {
|
|
118
161
|
return { count: 0, hit1: 0, hit5: 0, mrr10: 0 };
|
|
119
162
|
}
|
|
120
|
-
function accumulate(
|
|
121
|
-
const n =
|
|
163
|
+
function accumulate(ranks) {
|
|
164
|
+
const n = ranks.length;
|
|
122
165
|
if (n === 0)
|
|
123
166
|
return emptyMetrics();
|
|
124
167
|
let hit1 = 0, hit5 = 0, mrrSum = 0;
|
|
125
|
-
for (const
|
|
126
|
-
if (
|
|
168
|
+
for (const rank of ranks) {
|
|
169
|
+
if (rank === 1)
|
|
127
170
|
hit1++;
|
|
128
|
-
if (
|
|
171
|
+
if (rank !== null && rank <= 5)
|
|
129
172
|
hit5++;
|
|
130
|
-
mrrSum +=
|
|
173
|
+
mrrSum += rank !== null ? 1 / rank : 0;
|
|
131
174
|
}
|
|
132
175
|
return { count: n, hit1: hit1 / n, hit5: hit5 / n, mrr10: mrrSum / n };
|
|
133
176
|
}
|
|
177
|
+
function stageRank(c, stage) {
|
|
178
|
+
return c.stages[stage]?.rank ?? null;
|
|
179
|
+
}
|
|
180
|
+
function stageMetrics(cases, stage) {
|
|
181
|
+
const byCategory = {};
|
|
182
|
+
for (const category of CATEGORIES) {
|
|
183
|
+
const subset = cases.filter(c => c.category === category);
|
|
184
|
+
if (subset.length > 0)
|
|
185
|
+
byCategory[category] = accumulate(subset.map(c => stageRank(c, stage)));
|
|
186
|
+
}
|
|
187
|
+
return { overall: accumulate(cases.map(c => stageRank(c, stage))), byCategory };
|
|
188
|
+
}
|
|
189
|
+
function poolMetricsOf(cases) {
|
|
190
|
+
const count = cases.length;
|
|
191
|
+
return {
|
|
192
|
+
count,
|
|
193
|
+
recall: count === 0 ? 0 : cases.filter(c => c.poolRank !== null).length / count,
|
|
194
|
+
short: cases.filter(c => c.short).length,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
function binomialCdfHalf(k, n) {
|
|
198
|
+
let pmf = Math.pow(0.5, n);
|
|
199
|
+
let cdf = pmf;
|
|
200
|
+
for (let i = 1; i <= k; i++) {
|
|
201
|
+
pmf = pmf * (n - i + 1) / i;
|
|
202
|
+
cdf += pmf;
|
|
203
|
+
}
|
|
204
|
+
return cdf;
|
|
205
|
+
}
|
|
206
|
+
function signTest(improved, worsened) {
|
|
207
|
+
const n = improved + worsened;
|
|
208
|
+
if (n === 0)
|
|
209
|
+
return 1;
|
|
210
|
+
return Math.min(1, 2 * binomialCdfHalf(Math.min(improved, worsened), n));
|
|
211
|
+
}
|
|
212
|
+
function pairCounts(pairs) {
|
|
213
|
+
let improved = 0, worsened = 0, unchanged = 0;
|
|
214
|
+
for (const p of pairs) {
|
|
215
|
+
if (p.to < p.from)
|
|
216
|
+
improved++;
|
|
217
|
+
else if (p.to > p.from)
|
|
218
|
+
worsened++;
|
|
219
|
+
else
|
|
220
|
+
unchanged++;
|
|
221
|
+
}
|
|
222
|
+
return { improved, worsened, unchanged, signTestP: signTest(improved, worsened) };
|
|
223
|
+
}
|
|
224
|
+
function pairedChange(from, to) {
|
|
225
|
+
const fromById = new Map(from.map(c => [c.id, c.rank]));
|
|
226
|
+
const pairs = [];
|
|
227
|
+
for (const c of to) {
|
|
228
|
+
if (!fromById.has(c.id))
|
|
229
|
+
continue;
|
|
230
|
+
pairs.push({ id: c.id, category: c.category, from: fromById.get(c.id) ?? UNRANKED, to: c.rank ?? UNRANKED });
|
|
231
|
+
}
|
|
232
|
+
const byCategory = {};
|
|
233
|
+
for (const category of CATEGORIES) {
|
|
234
|
+
const subset = pairs.filter(p => p.category === category);
|
|
235
|
+
if (subset.length > 0)
|
|
236
|
+
byCategory[category] = pairCounts(subset);
|
|
237
|
+
}
|
|
238
|
+
return {
|
|
239
|
+
overall: pairCounts(pairs),
|
|
240
|
+
byCategory,
|
|
241
|
+
hit5Gained: pairs.filter(p => p.from > 5 && p.to <= 5).map(p => p.id),
|
|
242
|
+
hit5Lost: pairs.filter(p => p.from <= 5 && p.to > 5).map(p => p.id),
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
function rankedCases(cases, stage) {
|
|
246
|
+
return cases.map(c => ({ id: c.id, category: c.category, rank: stageRank(c, stage) }));
|
|
247
|
+
}
|
|
248
|
+
function stageChangesOf(cases, stages) {
|
|
249
|
+
const changes = [];
|
|
250
|
+
for (const to of stages) {
|
|
251
|
+
const from = STAGE_SOURCE[to];
|
|
252
|
+
if (from && stages.includes(from)) {
|
|
253
|
+
changes.push({ from, to, change: pairedChange(rankedCases(cases, from), rankedCases(cases, to)) });
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return changes;
|
|
257
|
+
}
|
|
258
|
+
function reportedStages(stages) {
|
|
259
|
+
return exports.STAGE_ORDER.filter(s => stages[s] !== undefined);
|
|
260
|
+
}
|
|
261
|
+
function sortedDistinct(values) {
|
|
262
|
+
return [...new Set(values)].sort((a, b) => {
|
|
263
|
+
if (a === b)
|
|
264
|
+
return 0;
|
|
265
|
+
if (a === null)
|
|
266
|
+
return -1;
|
|
267
|
+
if (b === null)
|
|
268
|
+
return 1;
|
|
269
|
+
return a < b ? -1 : 1;
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
exports.LIVE_GROUP = 'live';
|
|
273
|
+
function runFigures(cases, stages, shortPool) {
|
|
274
|
+
const metrics = {};
|
|
275
|
+
for (const stage of stages)
|
|
276
|
+
metrics[stage] = stageMetrics(cases, stage);
|
|
277
|
+
const poolInput = (subset) => subset.map(c => ({ poolRank: c.poolRank, short: shortPool.get(c.id) }));
|
|
278
|
+
const poolByCategory = {};
|
|
279
|
+
for (const category of CATEGORIES) {
|
|
280
|
+
const subset = cases.filter(c => c.category === category);
|
|
281
|
+
if (subset.length > 0)
|
|
282
|
+
poolByCategory[category] = poolMetricsOf(poolInput(subset));
|
|
283
|
+
}
|
|
284
|
+
return {
|
|
285
|
+
cases: cases.length,
|
|
286
|
+
metrics,
|
|
287
|
+
pool: { overall: poolMetricsOf(poolInput(cases)), byCategory: poolByCategory },
|
|
288
|
+
stageChanges: stageChangesOf(cases, stages),
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function distinctThresholds(pairs) {
|
|
292
|
+
const byKey = new Map(pairs.map(p => [`${p.value}|${p.source}`, p]));
|
|
293
|
+
return [...byKey.values()].sort((a, b) => b.value - a.value || a.source.localeCompare(b.source));
|
|
294
|
+
}
|
|
295
|
+
function formatThresholdConditions(pairs) {
|
|
296
|
+
return pairs.map(p => `${(0, constants_1.formatThreshold)(p.value)} (${p.source})`).join(', ') || '—';
|
|
297
|
+
}
|
|
134
298
|
class RetrievalEvaluator {
|
|
135
|
-
async evaluate(suite, search, inspect) {
|
|
299
|
+
async evaluate(suite, search, inspect, onCase) {
|
|
300
|
+
await checkCaseCorpora(suite.cases, p => inspect.projectDir(p));
|
|
136
301
|
const cases = [];
|
|
302
|
+
const shortPool = new Map();
|
|
137
303
|
const invalid = [];
|
|
138
304
|
const warnings = [];
|
|
139
|
-
|
|
305
|
+
const fetchLimits = [];
|
|
306
|
+
const efSearches = [];
|
|
307
|
+
const iterativeScans = [];
|
|
308
|
+
const thresholds = [];
|
|
309
|
+
let stages = null;
|
|
310
|
+
const total = suite.cases.length;
|
|
311
|
+
for (const [i, c] of suite.cases.entries()) {
|
|
312
|
+
const started = Date.now();
|
|
313
|
+
const report = (result, inv) => onCase?.({ index: i + 1, total, id: c.id, result, invalid: inv, ms: Date.now() - started });
|
|
140
314
|
const { valid, invalid: invalidExpects } = await validateCase(c, inspect);
|
|
141
315
|
if (valid.length === 0) {
|
|
142
|
-
|
|
316
|
+
const inv = { id: c.id, reason: 'no valid expected answer', invalidExpects };
|
|
317
|
+
invalid.push(inv);
|
|
318
|
+
report(null, inv);
|
|
143
319
|
continue;
|
|
144
320
|
}
|
|
145
321
|
if (invalidExpects.length > 0) {
|
|
146
322
|
warnings.push({ id: c.id, droppedExpects: invalidExpects });
|
|
147
323
|
}
|
|
148
324
|
const hits = await search(c.query, 10, c.project, c.threshold);
|
|
149
|
-
const
|
|
150
|
-
|
|
325
|
+
const caseStages = reportedStages(hits.stages);
|
|
326
|
+
if (stages === null) {
|
|
327
|
+
stages = caseStages;
|
|
328
|
+
}
|
|
329
|
+
else if (caseStages.join(',') !== stages.join(',')) {
|
|
330
|
+
throw new Error(`Case ${c.id} reported stages [${caseStages.join(', ')}], but the run's stages are [${stages.join(', ')}]`);
|
|
331
|
+
}
|
|
332
|
+
const ranks = {};
|
|
333
|
+
for (const stage of caseStages)
|
|
334
|
+
ranks[stage] = rankOf(hits.stages[stage], valid);
|
|
335
|
+
const result = {
|
|
336
|
+
id: c.id, project: c.project, category: c.category, query: c.query,
|
|
337
|
+
stages: ranks,
|
|
338
|
+
poolRank: rankOf(hits.pool, valid).rank,
|
|
339
|
+
poolSize: hits.pool.length,
|
|
340
|
+
rerankFailed: hits.rerankFailed,
|
|
341
|
+
corpus: c.corpus ?? null,
|
|
342
|
+
};
|
|
343
|
+
cases.push(result);
|
|
344
|
+
shortPool.set(c.id, hits.pool.length < hits.fetchLimit);
|
|
345
|
+
fetchLimits.push(hits.fetchLimit);
|
|
346
|
+
efSearches.push(hits.indexParams.efSearch);
|
|
347
|
+
iterativeScans.push(hits.indexParams.iterativeScan);
|
|
348
|
+
thresholds.push(hits.threshold);
|
|
349
|
+
report(result, null);
|
|
151
350
|
}
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
351
|
+
const runStages = stages ?? ['vector'];
|
|
352
|
+
const figures = (subset) => runFigures(subset, runStages, shortPool);
|
|
353
|
+
const byCorpus = {};
|
|
354
|
+
for (const key of [...new Set(cases.map(c => c.corpus ?? exports.LIVE_GROUP))].sort()) {
|
|
355
|
+
byCorpus[key] = figures(cases.filter(c => (c.corpus ?? exports.LIVE_GROUP) === key));
|
|
157
356
|
}
|
|
158
|
-
|
|
357
|
+
const all = figures(cases);
|
|
358
|
+
const observed = {
|
|
359
|
+
fetchLimit: sortedDistinct(fetchLimits),
|
|
360
|
+
efSearch: sortedDistinct(efSearches),
|
|
361
|
+
iterativeScan: sortedDistinct(iterativeScans),
|
|
362
|
+
thresholds: distinctThresholds(thresholds),
|
|
363
|
+
};
|
|
364
|
+
return {
|
|
365
|
+
run: {
|
|
366
|
+
cases, invalid, warnings,
|
|
367
|
+
stages: runStages,
|
|
368
|
+
metrics: all.metrics,
|
|
369
|
+
pool: all.pool,
|
|
370
|
+
stageChanges: all.stageChanges,
|
|
371
|
+
rerankFailures: cases.filter(c => c.rerankFailed).length,
|
|
372
|
+
byCorpus,
|
|
373
|
+
},
|
|
374
|
+
observed,
|
|
375
|
+
};
|
|
159
376
|
}
|
|
160
377
|
}
|
|
161
378
|
exports.RetrievalEvaluator = RetrievalEvaluator;
|
|
@@ -170,45 +387,157 @@ function metricsDelta(baseline, current) {
|
|
|
170
387
|
mrr10: metricDelta(baseline.mrr10, current.mrr10),
|
|
171
388
|
};
|
|
172
389
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
390
|
+
function metricsDeltaByCategory(baseline, current) {
|
|
391
|
+
const out = {};
|
|
392
|
+
for (const category of CATEGORIES) {
|
|
393
|
+
const b = baseline[category];
|
|
394
|
+
const cur = current[category];
|
|
395
|
+
if (b && cur)
|
|
396
|
+
out[category] = metricsDelta(b, cur);
|
|
397
|
+
}
|
|
398
|
+
return out;
|
|
399
|
+
}
|
|
400
|
+
function poolValueDelta(baseline, current) {
|
|
401
|
+
return { baseline, current, delta: baseline === null ? null : current - baseline };
|
|
402
|
+
}
|
|
403
|
+
function poolDelta(baseline, current) {
|
|
404
|
+
return {
|
|
405
|
+
recall: poolValueDelta(baseline?.recall ?? null, current.recall),
|
|
406
|
+
short: poolValueDelta(baseline?.short ?? null, current.short),
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
function isThresholdCondition(v) {
|
|
410
|
+
return typeof v === 'object' && v !== null && 'value' in v && 'source' in v;
|
|
411
|
+
}
|
|
412
|
+
function formatCondition(value) {
|
|
413
|
+
if (!Array.isArray(value))
|
|
414
|
+
return String(value);
|
|
415
|
+
if (value.length > 0 && value.every(isThresholdCondition))
|
|
416
|
+
return `[${formatThresholdConditions(value)}]`;
|
|
417
|
+
return `[${value.map(v => (v === null ? 'not sent' : String(v))).join(', ')}]`;
|
|
418
|
+
}
|
|
419
|
+
function conditionKey(value) {
|
|
420
|
+
return JSON.stringify(Array.isArray(value) ? value.map(v => JSON.stringify(v)).sort() : value);
|
|
421
|
+
}
|
|
422
|
+
function corpusWarningsOf(b, cur) {
|
|
423
|
+
const warnings = [];
|
|
424
|
+
if (b.corpora.length === 0)
|
|
425
|
+
warnings.push('baseline records no corpus');
|
|
426
|
+
const baseByRoot = new Map(b.corpora.map(c => [c.root, c]));
|
|
427
|
+
const curByRoot = new Map(cur.corpora.map(c => [c.root, c]));
|
|
428
|
+
for (const root of [...new Set([...baseByRoot.keys(), ...curByRoot.keys()])].sort()) {
|
|
429
|
+
const bc = baseByRoot.get(root), cc = curByRoot.get(root);
|
|
430
|
+
if (!bc) {
|
|
431
|
+
warnings.push(`corpus ${root} only in the current run`);
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
if (!cc) {
|
|
435
|
+
warnings.push(`corpus ${root} only in the baseline`);
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
if (bc.commit !== cc.commit) {
|
|
439
|
+
warnings.push(`corpus ${root} commit differs: baseline ${bc.commit} vs current ${cc.commit}`);
|
|
440
|
+
}
|
|
441
|
+
else if (bc.fingerprint !== cc.fingerprint) {
|
|
442
|
+
warnings.push(`corpus ${root} was searched on a different index build: fingerprint ${bc.fingerprint} (${bc.chunks} chunks) vs ${cc.fingerprint} (${cc.chunks} chunks)`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const live = (side, n) => `${side} includes ${n} live-corpus case(s); differences may come from code changes`;
|
|
446
|
+
if (b.liveCases > 0)
|
|
447
|
+
warnings.push(live('baseline', b.liveCases));
|
|
448
|
+
if (cur.liveCases > 0)
|
|
449
|
+
warnings.push(live('current run', cur.liveCases));
|
|
450
|
+
return warnings;
|
|
451
|
+
}
|
|
452
|
+
function vectorIndexWarningsOf(b, cur) {
|
|
453
|
+
if (!b.vectorIndex || !cur.vectorIndex) {
|
|
454
|
+
return [...(b.vectorIndex ? [] : ['baseline records no vector index']), ...(cur.vectorIndex ? [] : ['current run records no vector index'])];
|
|
178
455
|
}
|
|
179
|
-
|
|
180
|
-
|
|
456
|
+
const warnings = [];
|
|
457
|
+
if (b.vectorIndex.pgvector !== cur.vectorIndex.pgvector) {
|
|
458
|
+
warnings.push(`pgvector differs: baseline ${b.vectorIndex.pgvector ?? 'not installed'} vs current ${cur.vectorIndex.pgvector ?? 'not installed'}`);
|
|
181
459
|
}
|
|
182
|
-
if (
|
|
183
|
-
|
|
460
|
+
if (JSON.stringify(b.vectorIndex.indexes) !== JSON.stringify(cur.vectorIndex.indexes)) {
|
|
461
|
+
warnings.push(`vector index differs: baseline ${(0, eval_vector_index_1.formatIndexes)(b.vectorIndex)} vs current ${(0, eval_vector_index_1.formatIndexes)(cur.vectorIndex)}`);
|
|
184
462
|
}
|
|
185
|
-
const
|
|
186
|
-
for (const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
const
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
463
|
+
const settings = [['hnsw.max_scan_tuples', 'maxScanTuples'], ['hnsw.scan_mem_multiplier', 'scanMemMultiplier']];
|
|
464
|
+
for (const [name, key] of settings) {
|
|
465
|
+
if (b.vectorIndex[key] !== cur.vectorIndex[key]) {
|
|
466
|
+
warnings.push(`${name} differs: baseline ${b.vectorIndex[key] ?? 'not defined'} vs current ${cur.vectorIndex[key] ?? 'not defined'}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return warnings;
|
|
470
|
+
}
|
|
471
|
+
function conditionWarningsOf(baseline, current) {
|
|
472
|
+
const warnings = [];
|
|
473
|
+
const b = baseline.metadata, cur = current.metadata;
|
|
474
|
+
if (b.suiteHash !== cur.suiteHash) {
|
|
475
|
+
warnings.push(`suite differs: baseline ${b.suiteHash} vs current ${cur.suiteHash}`);
|
|
476
|
+
}
|
|
477
|
+
if (b.embeddingModel !== cur.embeddingModel) {
|
|
478
|
+
warnings.push(`embedding model differs: baseline ${b.embeddingModel} vs current ${cur.embeddingModel}`);
|
|
479
|
+
}
|
|
480
|
+
if (b.rerankerModel !== cur.rerankerModel) {
|
|
481
|
+
warnings.push(`reranker differs: baseline ${b.rerankerModel ?? 'none'} vs current ${cur.rerankerModel ?? 'none'}`);
|
|
482
|
+
}
|
|
483
|
+
if (!b.conditions)
|
|
484
|
+
warnings.push('baseline records no search conditions');
|
|
485
|
+
if (!cur.conditions)
|
|
486
|
+
warnings.push('current run records no search conditions');
|
|
487
|
+
if (b.conditions && cur.conditions) {
|
|
488
|
+
const bc = b.conditions;
|
|
489
|
+
const cc = cur.conditions;
|
|
490
|
+
for (const key of [...new Set([...Object.keys(bc), ...Object.keys(cc)])]) {
|
|
491
|
+
if (!(key in bc))
|
|
492
|
+
warnings.push(`search condition ${key} recorded only by the current run (${formatCondition(cc[key])})`);
|
|
493
|
+
else if (!(key in cc))
|
|
494
|
+
warnings.push(`search condition ${key} recorded only by the baseline (${formatCondition(bc[key])})`);
|
|
495
|
+
else if (conditionKey(bc[key]) !== conditionKey(cc[key])) {
|
|
496
|
+
warnings.push(`${key} differs: baseline ${formatCondition(bc[key])} vs current ${formatCondition(cc[key])}`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
warnings.push(...vectorIndexWarningsOf(b, cur));
|
|
501
|
+
warnings.push(...corpusWarningsOf(b, cur));
|
|
502
|
+
if (baseline.run.rerankFailures > 0)
|
|
503
|
+
warnings.push(`baseline has ${baseline.run.rerankFailures} rerank failure(s)`);
|
|
504
|
+
if (current.run.rerankFailures > 0)
|
|
505
|
+
warnings.push(`current run has ${current.run.rerankFailures} rerank failure(s)`);
|
|
506
|
+
return warnings;
|
|
507
|
+
}
|
|
508
|
+
function compare(baseline, current) {
|
|
509
|
+
const stages = [];
|
|
510
|
+
for (const stage of current.run.stages) {
|
|
511
|
+
const b = baseline.run.metrics[stage];
|
|
512
|
+
const cur = current.run.metrics[stage];
|
|
513
|
+
if (!baseline.run.stages.includes(stage) || !b || !cur)
|
|
514
|
+
continue;
|
|
515
|
+
stages.push({
|
|
516
|
+
stage,
|
|
517
|
+
overall: metricsDelta(b.overall, cur.overall),
|
|
518
|
+
byCategory: metricsDeltaByCategory(b.byCategory, cur.byCategory),
|
|
519
|
+
change: pairedChange(rankedCases(baseline.run.cases, stage), rankedCases(current.run.cases, stage)),
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
const unmatchedStages = [
|
|
523
|
+
...baseline.run.stages.filter(s => !current.run.stages.includes(s)).map(stage => ({ stage, side: 'baseline' })),
|
|
524
|
+
...current.run.stages.filter(s => !baseline.run.stages.includes(s)).map(stage => ({ stage, side: 'current' })),
|
|
525
|
+
];
|
|
526
|
+
const basePool = baseline.legacy ? null : baseline.run.pool;
|
|
527
|
+
const poolByCategory = {};
|
|
528
|
+
for (const category of CATEGORIES) {
|
|
529
|
+
const cur = current.run.pool.byCategory[category];
|
|
530
|
+
if (!cur)
|
|
531
|
+
continue;
|
|
532
|
+
if (basePool === null)
|
|
533
|
+
poolByCategory[category] = poolDelta(null, cur);
|
|
534
|
+
else if (basePool.byCategory[category])
|
|
535
|
+
poolByCategory[category] = poolDelta(basePool.byCategory[category], cur);
|
|
206
536
|
}
|
|
207
537
|
return {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
conditionWarnings,
|
|
538
|
+
stages,
|
|
539
|
+
unmatchedStages,
|
|
540
|
+
pool: { overall: poolDelta(basePool?.overall ?? null, current.run.pool.overall), byCategory: poolByCategory },
|
|
541
|
+
conditionWarnings: conditionWarningsOf(baseline, current),
|
|
213
542
|
};
|
|
214
543
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { IDatabase, ILogger } from '../interfaces';
|
|
2
|
+
import type { CallerRow, ICallGraph, NoCallersCause } from '../interfaces/ICallGraph';
|
|
3
|
+
export declare class CallGraph implements ICallGraph {
|
|
4
|
+
private db;
|
|
5
|
+
private logger;
|
|
6
|
+
constructor(db: IDatabase, logger: ILogger);
|
|
7
|
+
getCallers(symbol: string, project: string, calleeFile?: string | null): Promise<CallerRow[]>;
|
|
8
|
+
explainNoCallers(symbol: string, project: string): Promise<NoCallersCause>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CallGraph = void 0;
|
|
4
|
+
const symbol_name_1 = require("../symbol-name");
|
|
5
|
+
const symbol_sql_1 = require("./symbol-sql");
|
|
6
|
+
const NO_CALLERS_LIST_LIMIT = 5;
|
|
7
|
+
class CallGraph {
|
|
8
|
+
db;
|
|
9
|
+
logger;
|
|
10
|
+
constructor(db, logger) {
|
|
11
|
+
this.db = db;
|
|
12
|
+
this.logger = logger;
|
|
13
|
+
}
|
|
14
|
+
async getCallers(symbol, project, calleeFile = null) {
|
|
15
|
+
const t0 = Date.now();
|
|
16
|
+
this.logger.debug('CallGraph: getCallers', { symbol, project, calleeFile });
|
|
17
|
+
const parsed = (0, symbol_name_1.parseCallerSymbol)(symbol);
|
|
18
|
+
if (!parsed)
|
|
19
|
+
throw new Error('Invalid symbol name');
|
|
20
|
+
const params = [project];
|
|
21
|
+
let nameFilter;
|
|
22
|
+
if (parsed.form === 'bare') {
|
|
23
|
+
params.push(parsed.name);
|
|
24
|
+
nameFilter = `"calleeName" = $${params.length}`;
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
params.push(`${parsed.container}.${parsed.name}`);
|
|
28
|
+
const exactParam = params.length;
|
|
29
|
+
params.push(`%.${(0, symbol_name_1.escapeLike)(parsed.container)}.${(0, symbol_name_1.escapeLike)(parsed.name)}`);
|
|
30
|
+
const suffixParam = params.length;
|
|
31
|
+
nameFilter = `("calleeSymbol" = $${exactParam} OR "calleeSymbol" LIKE $${suffixParam} ESCAPE '\\')`;
|
|
32
|
+
}
|
|
33
|
+
let fileFilter = '';
|
|
34
|
+
if (calleeFile) {
|
|
35
|
+
params.push(calleeFile);
|
|
36
|
+
fileFilter = `AND "calleeFile" = $${params.length}`;
|
|
37
|
+
}
|
|
38
|
+
const rows = await this.db.query(`SELECT "fromFile", "fromSymbol", "calleeName", "calleeFile", line
|
|
39
|
+
FROM code_call
|
|
40
|
+
WHERE project = $1 AND ${nameFilter}
|
|
41
|
+
${fileFilter}
|
|
42
|
+
ORDER BY "fromFile", line
|
|
43
|
+
LIMIT 200`, params);
|
|
44
|
+
this.logger.debug('CallGraph: getCallers result', { count: rows.length, ms: Date.now() - t0 });
|
|
45
|
+
return rows;
|
|
46
|
+
}
|
|
47
|
+
async explainNoCallers(symbol, project) {
|
|
48
|
+
const t0 = Date.now();
|
|
49
|
+
const parsed = (0, symbol_name_1.parseCallerSymbol)(symbol);
|
|
50
|
+
if (!parsed)
|
|
51
|
+
throw new Error('Invalid symbol name');
|
|
52
|
+
const container = parsed.form === 'qualified' ? parsed.container : null;
|
|
53
|
+
try {
|
|
54
|
+
const calls = await this.db.query(`SELECT EXISTS (SELECT 1 FROM code_call WHERE project = $1) AS present`, [project]);
|
|
55
|
+
if (!calls[0]?.present)
|
|
56
|
+
return { kind: 'no-call-graph' };
|
|
57
|
+
const params = [project];
|
|
58
|
+
const match = (0, symbol_sql_1.symbolMatch)(parsed.name, container, params);
|
|
59
|
+
params.push(NO_CALLERS_LIST_LIMIT + 1);
|
|
60
|
+
const definitions = await this.db.query(`SELECT "filePath", line, qualified
|
|
61
|
+
FROM (
|
|
62
|
+
SELECT DISTINCT "filePath", line, ${symbol_sql_1.QUALIFIED_SQL} AS qualified${match.exactRank ? `, ${match.exactRank} AS exact_rank` : ''}
|
|
63
|
+
FROM code_symbol
|
|
64
|
+
WHERE project = $1 AND ${match.filter}
|
|
65
|
+
) AS definitions
|
|
66
|
+
ORDER BY ${match.exactRank ? 'exact_rank, ' : ''}"filePath", line, qualified COLLATE "C"
|
|
67
|
+
LIMIT $${params.length}`, params);
|
|
68
|
+
if (definitions.length > 0) {
|
|
69
|
+
return {
|
|
70
|
+
kind: 'no-recorded-calls',
|
|
71
|
+
definitions: definitions.slice(0, NO_CALLERS_LIST_LIMIT).map(({ qualified, filePath, line }) => ({ qualified, filePath, line })),
|
|
72
|
+
truncated: definitions.length > NO_CALLERS_LIST_LIMIT,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const { names, truncated } = await (0, symbol_sql_1.similarSymbolNames)(this.db, parsed.name, project, false);
|
|
76
|
+
return { kind: 'unknown-symbol', suggestions: names, truncated };
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
this.logger.warn('CallGraph: explainNoCallers lookup failed', { err, symbol, project, ms: Date.now() - t0 });
|
|
80
|
+
return { kind: 'lookup-failed' };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
exports.CallGraph = CallGraph;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IDatabase, IEmbedder, ILogger } from '../interfaces';
|
|
3
|
+
import type { IChunkSearch, NeighborResult, NoResultsCause, SearchResult, StagedSearchResult } from '../interfaces/IChunkSearch';
|
|
4
|
+
import type { IndexIdentityGuard } from './IndexIdentityGuard';
|
|
5
|
+
export declare class ChunkSearch implements IChunkSearch {
|
|
6
|
+
private db;
|
|
7
|
+
private embedder;
|
|
8
|
+
private identityGuard;
|
|
9
|
+
private config;
|
|
10
|
+
private logger;
|
|
11
|
+
private readonly gate;
|
|
12
|
+
private readonly fetcher;
|
|
13
|
+
private readonly rerankStage;
|
|
14
|
+
constructor(db: IDatabase, embedder: IEmbedder, identityGuard: IndexIdentityGuard, config: Config, logger: ILogger);
|
|
15
|
+
prepareQuery(raw: string): Promise<{
|
|
16
|
+
query: string;
|
|
17
|
+
translated: boolean;
|
|
18
|
+
}>;
|
|
19
|
+
searchChunks(query: string, limit: number, project: string | null, threshold?: number, rerankOverride?: boolean, perFile?: number): Promise<SearchResult[]>;
|
|
20
|
+
searchChunksWithStages(query: string, limit: number, project: string | null, threshold?: number, rerankOverride?: boolean, perFile?: number): Promise<StagedSearchResult>;
|
|
21
|
+
fetchNeighbors(results: SearchResult[]): Promise<NeighborResult[]>;
|
|
22
|
+
explainNoResults(project: string | null): Promise<NoResultsCause>;
|
|
23
|
+
}
|