codesearch-mcp 0.3.0-alpha → 0.4.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -6
- package/LICENSE +661 -21
- package/README.md +33 -17
- package/dist/app.js +13 -20
- package/dist/src/ast/CallExtractor.d.ts +8 -0
- package/dist/src/ast/CallExtractor.js +68 -0
- package/dist/src/ast/GraphqlNameExtractor.d.ts +22 -0
- package/dist/src/ast/GraphqlNameExtractor.js +240 -0
- package/dist/src/ast/ImportEdgeExtractor.d.ts +11 -0
- package/dist/src/ast/ImportEdgeExtractor.js +129 -0
- package/dist/src/ast/IndexedNodeResolver.d.ts +14 -0
- package/dist/src/ast/IndexedNodeResolver.js +217 -0
- package/dist/src/ast/MemberExtractor.d.ts +8 -0
- package/dist/src/ast/MemberExtractor.js +129 -0
- package/dist/src/ast/TopLevelSymbolExtractor.d.ts +5 -0
- package/dist/src/ast/TopLevelSymbolExtractor.js +86 -0
- package/dist/src/ast/graphql-grammar.d.ts +24 -0
- package/dist/src/ast/graphql-grammar.js +93 -0
- package/dist/src/ast/node-helpers.d.ts +11 -0
- package/dist/src/ast/node-helpers.js +55 -0
- package/dist/src/commands/EvalCommand.d.ts +7 -3
- package/dist/src/commands/EvalCommand.js +36 -158
- package/dist/src/commands/IndexCommand.d.ts +10 -4
- package/dist/src/commands/IndexCommand.js +52 -124
- package/dist/src/commands/McpCommand.d.ts +6 -3
- package/dist/src/commands/McpCommand.js +19 -14
- package/dist/src/commands/WebCommand.d.ts +9 -3
- package/dist/src/commands/WebCommand.js +17 -8
- package/dist/src/commands/eval/baseline-comparison.d.ts +6 -0
- package/dist/src/commands/eval/baseline-comparison.js +54 -0
- package/dist/src/commands/eval/eval-args.d.ts +6 -0
- package/dist/src/commands/eval/eval-args.js +71 -0
- package/dist/src/commands/eval/eval-corpora.d.ts +6 -0
- package/dist/src/commands/eval/eval-corpora.js +76 -0
- package/dist/src/commands/eval/eval-metadata.d.ts +17 -0
- package/dist/src/commands/eval/eval-metadata.js +75 -0
- package/dist/src/commands/eval/index-inspector.d.ts +3 -0
- package/dist/src/commands/eval/index-inspector.js +21 -0
- package/dist/src/commands/eval/staged-search.d.ts +3 -0
- package/dist/src/commands/eval/staged-search.js +20 -0
- package/dist/src/commands/eval/suite-loader.d.ts +8 -0
- package/dist/src/commands/eval/suite-loader.js +52 -0
- package/dist/src/commands/index/backend-probe.d.ts +2 -0
- package/dist/src/commands/index/backend-probe.js +15 -0
- package/dist/src/commands/index/embedding-setup.d.ts +3 -0
- package/dist/src/commands/index/embedding-setup.js +16 -0
- package/dist/src/commands/index/index-summary.d.ts +2 -0
- package/dist/src/commands/index/index-summary.js +22 -0
- package/dist/src/commands/index/legacy-ollama-guard.d.ts +3 -0
- package/dist/src/commands/index/legacy-ollama-guard.js +23 -0
- package/dist/src/commands/index/prune-step.d.ts +3 -0
- package/dist/src/commands/index/prune-step.js +22 -0
- package/dist/src/commands/index/scan-targets.d.ts +8 -0
- package/dist/src/commands/index/scan-targets.js +107 -0
- package/dist/src/config-store.d.ts +0 -1
- package/dist/src/config-store.js +3 -4
- package/dist/src/config.d.ts +1 -1
- package/dist/src/config.js +1 -1
- package/dist/src/constants.d.ts +15 -0
- package/dist/src/constants.js +34 -6
- package/dist/src/container.d.ts +4 -2
- package/dist/src/container.js +7 -5
- package/dist/src/db/drizzle-types.d.ts +7 -0
- package/dist/src/db/index.d.ts +1 -0
- package/dist/src/db/schema-steps.d.ts +8 -0
- package/dist/src/db/schema-steps.js +105 -0
- package/dist/src/declarations/DeclarationTree.d.ts +1 -1
- package/dist/src/eval-results.js +16 -2
- package/dist/src/eval-vector-index.d.ts +5 -0
- package/dist/src/eval-vector-index.js +60 -0
- package/dist/src/help.js +9 -1
- package/dist/src/index-args.d.ts +13 -0
- package/dist/src/index-args.js +48 -0
- package/dist/src/indexing/AnalysisBatcher.d.ts +12 -0
- package/dist/src/indexing/AnalysisBatcher.js +28 -0
- package/dist/src/indexing/ChunkEmbedder.d.ts +24 -0
- package/dist/src/indexing/ChunkEmbedder.js +37 -0
- package/dist/src/indexing/ConsoleScanReporter.d.ts +10 -0
- package/dist/src/indexing/ConsoleScanReporter.js +73 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.d.ts +14 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.js +66 -0
- package/dist/src/indexing/FileIngestor.d.ts +22 -0
- package/dist/src/indexing/FileIngestor.js +116 -0
- package/dist/src/indexing/IndexStore.d.ts +24 -0
- package/dist/src/indexing/IndexStore.js +69 -0
- package/dist/src/indexing/StaleFilePruner.d.ts +8 -0
- package/dist/src/indexing/StaleFilePruner.js +55 -0
- package/dist/src/indexing/analysis-rows.d.ts +4 -0
- package/dist/src/indexing/analysis-rows.js +65 -0
- package/dist/src/indexing/project-notices.d.ts +2 -0
- package/dist/src/indexing/project-notices.js +63 -0
- package/dist/src/interfaces/IAstParser.d.ts +2 -1
- package/dist/src/interfaces/ICallGraph.d.ts +28 -0
- package/dist/src/interfaces/ICallGraph.js +2 -0
- package/dist/src/interfaces/IChunkSearch.d.ts +42 -0
- package/dist/src/interfaces/IChunkSearch.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +6 -2
- package/dist/src/interfaces/IDependencyGraph.d.ts +24 -0
- package/dist/src/interfaces/IDependencyGraph.js +2 -0
- package/dist/src/interfaces/IIndexIdentityGuard.d.ts +3 -0
- package/dist/src/interfaces/IIndexIdentityGuard.js +2 -0
- package/dist/src/interfaces/IIndexedNodeResolver.d.ts +10 -0
- package/dist/src/interfaces/IIndexedNodeResolver.js +2 -0
- package/dist/src/interfaces/IMcpReadiness.d.ts +12 -0
- package/dist/src/interfaces/IMcpReadiness.js +2 -0
- package/dist/src/interfaces/IProjectCatalog.d.ts +13 -0
- package/dist/src/interfaces/IProjectCatalog.js +2 -0
- package/dist/src/interfaces/IProjectRoots.d.ts +3 -0
- package/dist/src/interfaces/IProjectRoots.js +2 -0
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +21 -2
- package/dist/src/interfaces/IScanReporter.d.ts +13 -0
- package/dist/src/interfaces/IScanReporter.js +2 -0
- package/dist/src/interfaces/ISymbolLookup.d.ts +49 -0
- package/dist/src/interfaces/ISymbolLookup.js +2 -0
- package/dist/src/interfaces/IToolPaths.d.ts +62 -0
- package/dist/src/interfaces/IToolPaths.js +2 -0
- package/dist/src/interfaces/IWorkspaceFiles.d.ts +28 -0
- package/dist/src/interfaces/IWorkspaceFiles.js +2 -0
- package/dist/src/interfaces/SymbolKind.d.ts +1 -0
- package/dist/src/interfaces/SymbolKind.js +2 -0
- package/dist/src/interfaces/index.d.ts +12 -1
- package/dist/src/project-scope.d.ts +1 -0
- package/dist/src/project-scope.js +5 -0
- package/dist/src/retrieval-eval.d.ts +2 -1
- package/dist/src/retrieval-eval.js +45 -2
- package/dist/src/search/CallGraph.d.ts +9 -0
- package/dist/src/search/CallGraph.js +84 -0
- package/dist/src/search/ChunkSearch.d.ts +23 -0
- package/dist/src/search/ChunkSearch.js +115 -0
- package/dist/src/search/DependencyGraph.d.ts +19 -0
- package/dist/src/search/DependencyGraph.js +180 -0
- package/dist/src/search/IndexIdentityGuard.d.ts +11 -0
- package/dist/src/search/IndexIdentityGuard.js +37 -0
- package/dist/src/search/ProjectCatalog.d.ts +11 -0
- package/dist/src/search/ProjectCatalog.js +38 -0
- package/dist/src/search/SymbolLookup.d.ts +24 -0
- package/dist/src/search/SymbolLookup.js +97 -0
- package/dist/src/search/chunks/CandidateFetcher.d.ts +19 -0
- package/dist/src/search/chunks/CandidateFetcher.js +40 -0
- package/dist/src/search/chunks/IterativeScanGate.d.ts +10 -0
- package/dist/src/search/chunks/IterativeScanGate.js +42 -0
- package/dist/src/search/chunks/RerankStage.d.ts +15 -0
- package/dist/src/search/chunks/RerankStage.js +42 -0
- package/dist/src/search/chunks/search-metrics.d.ts +18 -0
- package/dist/src/search/chunks/search-metrics.js +32 -0
- package/dist/src/search/chunks/search-plan.d.ts +18 -0
- package/dist/src/search/chunks/search-plan.js +19 -0
- package/dist/src/search/create-search-services.d.ts +26 -0
- package/dist/src/search/create-search-services.js +29 -0
- package/dist/src/search/index-paths.d.ts +1 -0
- package/dist/src/search/index-paths.js +5 -0
- package/dist/src/search/symbol-sql.d.ts +13 -0
- package/dist/src/search/symbol-sql.js +66 -0
- package/dist/src/search/symbols/SymbolGrepFallback.d.ts +19 -0
- package/dist/src/search/symbols/SymbolGrepFallback.js +106 -0
- package/dist/src/search/symbols/SymbolIndexQuery.d.ts +14 -0
- package/dist/src/search/symbols/SymbolIndexQuery.js +52 -0
- package/dist/src/search/symbols/SymbolTextSearch.d.ts +13 -0
- package/dist/src/search/symbols/SymbolTextSearch.js +84 -0
- package/dist/src/search/symbols/TargetColumnGate.d.ts +7 -0
- package/dist/src/search/symbols/TargetColumnGate.js +23 -0
- package/dist/src/search/symbols/find-symbol-constants.d.ts +4 -0
- package/dist/src/search/symbols/find-symbol-constants.js +8 -0
- package/dist/src/search/symbols/grep-args.d.ts +3 -0
- package/dist/src/search/symbols/grep-args.js +7 -0
- package/dist/src/search/symbols/grep-patterns.d.ts +4 -0
- package/dist/src/search/symbols/grep-patterns.js +60 -0
- package/dist/src/services/AstParserService.d.ts +8 -44
- package/dist/src/services/AstParserService.js +19 -938
- package/dist/src/services/DatabaseService.d.ts +8 -10
- package/dist/src/services/DatabaseService.js +29 -86
- package/dist/src/services/IndexerService.d.ts +7 -10
- package/dist/src/services/IndexerService.js +36 -255
- package/dist/src/services/McpReadiness.d.ts +18 -0
- package/dist/src/services/McpReadiness.js +73 -0
- package/dist/src/services/McpServer.d.ts +32 -9
- package/dist/src/services/McpServer.js +78 -41
- package/dist/src/services/WebServerService.d.ts +4 -3
- package/dist/src/services/WebServerService.js +16 -8
- package/dist/src/setup.d.ts +16 -4
- package/dist/src/setup.js +115 -40
- package/dist/src/symbol-name.d.ts +1 -0
- package/dist/src/symbol-name.js +16 -12
- package/dist/src/tools/FindSymbolTool.d.ts +17 -7
- package/dist/src/tools/FindSymbolTool.js +54 -35
- package/dist/src/tools/GetCallersTool.d.ts +16 -8
- package/dist/src/tools/GetCallersTool.js +65 -35
- package/dist/src/tools/GetDependenciesTool.d.ts +16 -8
- package/dist/src/tools/GetDependenciesTool.js +35 -28
- package/dist/src/tools/GetDependentsTool.d.ts +20 -8
- package/dist/src/tools/GetDependentsTool.js +47 -33
- package/dist/src/tools/ListFilesTool.d.ts +16 -8
- package/dist/src/tools/ListFilesTool.js +20 -26
- package/dist/src/tools/ListProjectsTool.d.ts +11 -7
- package/dist/src/tools/ListProjectsTool.js +13 -11
- package/dist/src/tools/McpTool.d.ts +17 -0
- package/dist/src/tools/McpTool.js +28 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +16 -8
- package/dist/src/tools/ReadFileRangeTool.js +23 -25
- package/dist/src/tools/ReadFileTool.d.ts +15 -8
- package/dist/src/tools/ReadFileTool.js +19 -24
- package/dist/src/tools/SearchCodebaseTool.d.ts +22 -10
- package/dist/src/tools/SearchCodebaseTool.js +69 -51
- package/dist/src/tools/format-helpers.d.ts +9 -3
- package/dist/src/tools/format-helpers.js +59 -14
- package/dist/src/tools/tool-args.d.ts +38 -0
- package/dist/src/tools/tool-args.js +95 -0
- package/dist/src/workspace/GrepRunner.d.ts +15 -0
- package/dist/src/workspace/GrepRunner.js +104 -0
- package/dist/src/workspace/ProjectRoots.d.ts +22 -0
- package/dist/src/workspace/ProjectRoots.js +102 -0
- package/dist/src/workspace/ToolPaths.d.ts +31 -0
- package/dist/src/workspace/ToolPaths.js +312 -0
- package/dist/src/workspace/WorkspaceFiles.d.ts +19 -0
- package/dist/src/workspace/WorkspaceFiles.js +285 -0
- package/dist/src/workspace/fs-access.d.ts +9 -0
- package/dist/src/workspace/fs-access.js +47 -0
- package/dist/src/workspace/path-helpers.d.ts +3 -0
- package/dist/src/workspace/path-helpers.js +43 -0
- package/package.json +4 -3
- package/dist/src/interfaces/ISearcher.d.ts +0 -46
- package/dist/src/services/SearcherService.d.ts +0 -235
- package/dist/src/services/SearcherService.js +0 -1118
- /package/dist/src/{interfaces/ISearcher.js → db/drizzle-types.js} +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.refusesLegacyOllamaIndex = refusesLegacyOllamaIndex;
|
|
4
|
+
const legacy_ollama_index_1 = require("../../services/legacy-ollama-index");
|
|
5
|
+
async function refusesLegacyOllamaIndex(db, config) {
|
|
6
|
+
let identity;
|
|
7
|
+
try {
|
|
8
|
+
identity = await (0, legacy_ollama_index_1.resolveIndexedIdentity)(db);
|
|
9
|
+
}
|
|
10
|
+
catch (err) {
|
|
11
|
+
console.error(`\n${err.message}`);
|
|
12
|
+
process.exitCode = 1;
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
if (identity?.provider === 'ollama') {
|
|
16
|
+
console.error(`\nThe existing index was built with ${identity.provider}/${identity.model}, but "ollama" is no ` +
|
|
17
|
+
`longer a supported embedding provider (configured: ${config.embedding.provider}/${config.embedding.model}). ` +
|
|
18
|
+
`Run \`rag-index --reindex\` to confirm rebuilding the index under the configured provider.`);
|
|
19
|
+
process.exitCode = 1;
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.pruneUnselected = pruneUnselected;
|
|
4
|
+
async function pruneUnselected(indexer, projects) {
|
|
5
|
+
let pruneResult;
|
|
6
|
+
try {
|
|
7
|
+
pruneResult = await indexer.pruneProjects(projects.map(p => p.name));
|
|
8
|
+
}
|
|
9
|
+
catch (err) {
|
|
10
|
+
console.error(`\nPruning failed: ${err.message}`);
|
|
11
|
+
process.exitCode = 1;
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
const { removed, failed } = pruneResult;
|
|
15
|
+
for (const name of removed) {
|
|
16
|
+
console.log(`Removed project ${name} (not selected)`);
|
|
17
|
+
}
|
|
18
|
+
if (failed.length > 0) {
|
|
19
|
+
process.exitCode = 1;
|
|
20
|
+
}
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Config } from '../../config';
|
|
2
|
+
import type { IndexArgs } from '../../index-args';
|
|
3
|
+
import type { DiscoveredProject, IProjectDiscovery } from '../../interfaces';
|
|
4
|
+
export interface ScanTargets {
|
|
5
|
+
discovered: DiscoveredProject[];
|
|
6
|
+
scanTargets?: DiscoveredProject[];
|
|
7
|
+
}
|
|
8
|
+
export declare function resolveScanTargets(discovery: IProjectDiscovery, config: Config, args: IndexArgs): Promise<ScanTargets | null>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.resolveScanTargets = resolveScanTargets;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const project_selector_1 = require("../../project-selector");
|
|
40
|
+
const project_scope_1 = require("../../project-scope");
|
|
41
|
+
async function resolveScanTargets(discovery, config, args) {
|
|
42
|
+
const { projectFlag, requireCwd } = args;
|
|
43
|
+
let discovered;
|
|
44
|
+
try {
|
|
45
|
+
discovered = discovery.discover(config.workspaceRoot);
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
console.error(`\n${err.message}`);
|
|
49
|
+
process.exitCode = 1;
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
if (requireCwd && projectFlag.name === undefined) {
|
|
53
|
+
console.error('\n--require-cwd needs --project <name>.');
|
|
54
|
+
process.exitCode = 1;
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
let scanTargets;
|
|
58
|
+
if (projectFlag.present) {
|
|
59
|
+
let name = projectFlag.name;
|
|
60
|
+
if (name === undefined) {
|
|
61
|
+
if (discovered.length === 0) {
|
|
62
|
+
console.error('\nNo projects available to select.');
|
|
63
|
+
process.exitCode = 1;
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
name = await (0, project_selector_1.selectSingleProject)(discovered);
|
|
67
|
+
if (name === undefined) {
|
|
68
|
+
process.exitCode = 1;
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const { targets, notFound } = (0, project_scope_1.resolveProjectNames)([name], discovered);
|
|
73
|
+
if (notFound.length > 0) {
|
|
74
|
+
const available = (0, project_scope_1.listAvailableNames)(discovered).join(', ');
|
|
75
|
+
console.error(`\nProject '${notFound[0]}' not found. Available projects: ${available}`);
|
|
76
|
+
process.exitCode = 1;
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
if (requireCwd && !isInProjectDir(config, name, discovered)) {
|
|
80
|
+
process.exitCode = 1;
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
scanTargets = targets;
|
|
84
|
+
}
|
|
85
|
+
return { discovered, scanTargets };
|
|
86
|
+
}
|
|
87
|
+
function isInProjectDir(config, name, discovered) {
|
|
88
|
+
const { workspaceRoot } = config;
|
|
89
|
+
if (!path.isAbsolute(workspaceRoot)) {
|
|
90
|
+
console.error(`\nworkspaceRoot '${workspaceRoot}' is relative, so it moves with the current directory and --require-cwd cannot tell checkouts apart.`);
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
let cwd;
|
|
94
|
+
let projectDir;
|
|
95
|
+
try {
|
|
96
|
+
cwd = fs.realpathSync.native(process.cwd());
|
|
97
|
+
projectDir = fs.realpathSync.native((0, project_scope_1.matchedProjectDir)(name, discovered));
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
console.error(`\n${err.message}`);
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
if (cwd === projectDir)
|
|
104
|
+
return true;
|
|
105
|
+
console.error(`\nCurrent directory ${cwd} is not the directory of project '${name}' (${projectDir}).`);
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
@@ -75,7 +75,6 @@ export declare const CONFIG_DEFAULTS: {
|
|
|
75
75
|
readonly 'indexing.embedBatchSize': 5;
|
|
76
76
|
readonly 'indexing.astBatchSize': 3000;
|
|
77
77
|
readonly 'indexing.chunkStrategy': "ast";
|
|
78
|
-
readonly 'search.similarityThreshold': 0.6;
|
|
79
78
|
readonly 'search.defaultLimit': 5;
|
|
80
79
|
readonly 'search.maxLimit': 20;
|
|
81
80
|
readonly 'reranker.enabled': true;
|
package/dist/src/config-store.js
CHANGED
|
@@ -79,7 +79,7 @@ const jsonSchema = zod_1.z.object({
|
|
|
79
79
|
chunkStrategy: zod_1.z.enum(['ast', 'character']).describe('How TS/JS files are chunked: ast = symbol-aligned boundaries, character = fixed-size recursive splitter (default ast)').optional(),
|
|
80
80
|
}).optional(),
|
|
81
81
|
search: zod_1.z.object({
|
|
82
|
-
similarityThreshold: zod_1.z.number().min(0).max(1).describe('Max cosine distance 0–1
|
|
82
|
+
similarityThreshold: zod_1.z.number().min(0).max(1).describe('Max cosine distance 0–1, lower = stricter. Unset: recommended value for the embedding model').optional(),
|
|
83
83
|
defaultLimit: zod_1.z.number().int().describe('Default result count (default 5)').optional(),
|
|
84
84
|
maxLimit: zod_1.z.number().int().describe('Hard cap on results (default 20)').optional(),
|
|
85
85
|
}).optional(),
|
|
@@ -90,8 +90,8 @@ const jsonSchema = zod_1.z.object({
|
|
|
90
90
|
apiKey: zod_1.z.string().describe('API key, or env:VAR_NAME. Empty for local servers; required for voyage.').optional(),
|
|
91
91
|
model: zod_1.z.string().describe('Reranker model name (default mixedbread-ai/mxbai-rerank-xsmall-v1 for openai-compatible, rerank-3 for voyage)').optional(),
|
|
92
92
|
url: zod_1.z.string().describe('Full URL of the rerank endpoint (default derived from provider + baseURL)').optional(),
|
|
93
|
-
overfetch: zod_1.z.number().int().min(1).max(20).describe('Candidate multiplier before reranking (limit × N, default 5)').optional(),
|
|
94
|
-
candidateFloor: zod_1.z.number().int().min(1).describe('Minimum candidate pool fed to the reranker regardless of limit; fetchLimit = min(max(limit × overfetch, candidateFloor), 200) (default 80)').optional(),
|
|
93
|
+
overfetch: zod_1.z.number().int().min(1).max(20).describe('Candidate multiplier before reranking (limit × N, default 5); also sizes the pool of a search without reranking whose per-file limit is below limit').optional(),
|
|
94
|
+
candidateFloor: zod_1.z.number().int().min(1).describe('Minimum candidate pool fed to the reranker regardless of limit; fetchLimit = min(max(limit × overfetch, candidateFloor), 200) (default 80); the same pool serves a search without reranking whose per-file limit is below limit').optional(),
|
|
95
95
|
}).optional(),
|
|
96
96
|
logging: zod_1.z.object({
|
|
97
97
|
level: zod_1.z.enum(['debug', 'info', 'warn', 'error']).describe('File log level (default error)').optional(),
|
|
@@ -143,7 +143,6 @@ exports.CONFIG_DEFAULTS = {
|
|
|
143
143
|
'indexing.embedBatchSize': 5,
|
|
144
144
|
'indexing.astBatchSize': 3000,
|
|
145
145
|
'indexing.chunkStrategy': 'ast',
|
|
146
|
-
'search.similarityThreshold': 0.6,
|
|
147
146
|
'search.defaultLimit': 5,
|
|
148
147
|
'search.maxLimit': 20,
|
|
149
148
|
'reranker.enabled': true,
|
package/dist/src/config.d.ts
CHANGED
|
@@ -43,9 +43,9 @@ export declare class Config {
|
|
|
43
43
|
chunkStrategy: "ast" | "character";
|
|
44
44
|
};
|
|
45
45
|
get search(): {
|
|
46
|
-
similarityThreshold: number;
|
|
47
46
|
defaultLimit: number;
|
|
48
47
|
maxLimit: number;
|
|
48
|
+
similarityThreshold?: number | undefined;
|
|
49
49
|
};
|
|
50
50
|
get logging(): {
|
|
51
51
|
level: "error" | "debug" | "info" | "warn";
|
package/dist/src/config.js
CHANGED
|
@@ -111,7 +111,7 @@ const resolvedSchema = zod_1.z.object({
|
|
|
111
111
|
chunkStrategy: zod_1.z.enum(['ast', 'character']).default(config_store_1.CONFIG_DEFAULTS['indexing.chunkStrategy']),
|
|
112
112
|
}),
|
|
113
113
|
search: zod_1.z.object({
|
|
114
|
-
similarityThreshold: zod_1.z.number().min(0).max(1).
|
|
114
|
+
similarityThreshold: zod_1.z.number().min(0).max(1).optional(),
|
|
115
115
|
defaultLimit: zod_1.z.number().int().positive().default(config_store_1.CONFIG_DEFAULTS['search.defaultLimit']),
|
|
116
116
|
maxLimit: zod_1.z.number().int().positive().default(config_store_1.CONFIG_DEFAULTS['search.maxLimit']),
|
|
117
117
|
}),
|
package/dist/src/constants.d.ts
CHANGED
|
@@ -8,9 +8,24 @@ export interface ModelInfo {
|
|
|
8
8
|
dimConfigurable?: boolean;
|
|
9
9
|
maxIndexable?: number;
|
|
10
10
|
maxDim?: number;
|
|
11
|
+
defaultThreshold?: number;
|
|
11
12
|
}
|
|
12
13
|
export declare const MODEL_REGISTRY: Record<string, ModelInfo>;
|
|
13
14
|
export declare const DEFAULT_EMBED_MODEL = "jinaai/jina-embeddings-v2-base-code";
|
|
14
15
|
export declare const FORMER_OLLAMA_MODELS: readonly ["bge-m3", "nomic-embed-text", "mxbai-embed-large"];
|
|
15
16
|
export declare function registryDimension(model: string): number | undefined;
|
|
17
|
+
export type ThresholdSource = 'call' | 'config' | 'model' | 'fallback';
|
|
18
|
+
export interface ResolvedThreshold {
|
|
19
|
+
value: number;
|
|
20
|
+
source: ThresholdSource;
|
|
21
|
+
}
|
|
22
|
+
export declare const FALLBACK_THRESHOLD = 1;
|
|
23
|
+
export declare function resolveThreshold({ callValue, configValue, model, registry }: {
|
|
24
|
+
callValue?: number;
|
|
25
|
+
configValue?: number;
|
|
26
|
+
model: string;
|
|
27
|
+
registry?: Record<string, ModelInfo>;
|
|
28
|
+
}): ResolvedThreshold;
|
|
29
|
+
export declare function formatThreshold(value: number): string;
|
|
30
|
+
export declare function thresholdMismatchWarning(configValue: number | undefined, model: string, registry?: Record<string, ModelInfo>): string | null;
|
|
16
31
|
export declare function resolveDimension(override: number | null | undefined, model: string, probeFn: () => Promise<number>, registry?: Record<string, ModelInfo>): Promise<number>;
|
package/dist/src/constants.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.FORMER_OLLAMA_MODELS = exports.DEFAULT_EMBED_MODEL = exports.MODEL_REGISTRY = exports.MAX_EMBED_DIM = exports.VOYAGE_RERANK_MODELS = void 0;
|
|
3
|
+
exports.FALLBACK_THRESHOLD = exports.FORMER_OLLAMA_MODELS = exports.DEFAULT_EMBED_MODEL = exports.MODEL_REGISTRY = exports.MAX_EMBED_DIM = exports.VOYAGE_RERANK_MODELS = void 0;
|
|
4
4
|
exports.registryDimension = registryDimension;
|
|
5
|
+
exports.resolveThreshold = resolveThreshold;
|
|
6
|
+
exports.formatThreshold = formatThreshold;
|
|
7
|
+
exports.thresholdMismatchWarning = thresholdMismatchWarning;
|
|
5
8
|
exports.resolveDimension = resolveDimension;
|
|
6
9
|
exports.VOYAGE_RERANK_MODELS = ['rerank-3', 'rerank-3-lite'];
|
|
7
10
|
exports.MAX_EMBED_DIM = 2000;
|
|
@@ -9,18 +12,43 @@ exports.MODEL_REGISTRY = {
|
|
|
9
12
|
'text-embedding-3-small': { provider: 'openai-compatible', dimensions: 1536, dimConfigurable: true },
|
|
10
13
|
'text-embedding-3-large': { provider: 'openai-compatible', dimensions: 3072, dimConfigurable: true, maxIndexable: 2000 },
|
|
11
14
|
'text-embedding-ada-002': { provider: 'openai-compatible', dimensions: 1536 },
|
|
12
|
-
'voyage-code-4': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048 },
|
|
13
|
-
'voyage-4-large': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048 },
|
|
14
|
-
'voyage-4': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048 },
|
|
15
|
-
'voyage-4-lite': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048 },
|
|
15
|
+
'voyage-code-4': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048, defaultThreshold: 0.95 },
|
|
16
|
+
'voyage-4-large': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048, defaultThreshold: 0.95 },
|
|
17
|
+
'voyage-4': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048, defaultThreshold: 0.95 },
|
|
18
|
+
'voyage-4-lite': { provider: 'voyage', dimensions: 1024, dimConfigurable: true, maxDim: 2048, defaultThreshold: 0.95 },
|
|
16
19
|
'BAAI/bge-m3': { provider: 'infinity', dimensions: 1024 },
|
|
17
|
-
'jinaai/jina-embeddings-v2-base-code': { provider: 'infinity', dimensions: 768 },
|
|
20
|
+
'jinaai/jina-embeddings-v2-base-code': { provider: 'infinity', dimensions: 768, defaultThreshold: 0.6 },
|
|
18
21
|
};
|
|
19
22
|
exports.DEFAULT_EMBED_MODEL = 'jinaai/jina-embeddings-v2-base-code';
|
|
20
23
|
exports.FORMER_OLLAMA_MODELS = ['bge-m3', 'nomic-embed-text', 'mxbai-embed-large'];
|
|
21
24
|
function registryDimension(model) {
|
|
22
25
|
return exports.MODEL_REGISTRY[model]?.dimensions;
|
|
23
26
|
}
|
|
27
|
+
exports.FALLBACK_THRESHOLD = 1.0;
|
|
28
|
+
const THRESHOLD_MISMATCH_TOLERANCE = 0.1;
|
|
29
|
+
const FLOAT_EPSILON = 1e-9;
|
|
30
|
+
function resolveThreshold({ callValue, configValue, model, registry = exports.MODEL_REGISTRY }) {
|
|
31
|
+
if (callValue !== undefined)
|
|
32
|
+
return { value: callValue, source: 'call' };
|
|
33
|
+
if (configValue !== undefined)
|
|
34
|
+
return { value: configValue, source: 'config' };
|
|
35
|
+
const recommended = registry[model]?.defaultThreshold;
|
|
36
|
+
if (recommended !== undefined)
|
|
37
|
+
return { value: recommended, source: 'model' };
|
|
38
|
+
return { value: exports.FALLBACK_THRESHOLD, source: 'fallback' };
|
|
39
|
+
}
|
|
40
|
+
function formatThreshold(value) {
|
|
41
|
+
return Number.isInteger(value) ? value.toFixed(1) : String(value);
|
|
42
|
+
}
|
|
43
|
+
function thresholdMismatchWarning(configValue, model, registry = exports.MODEL_REGISTRY) {
|
|
44
|
+
const recommended = registry[model]?.defaultThreshold;
|
|
45
|
+
if (configValue === undefined || recommended === undefined)
|
|
46
|
+
return null;
|
|
47
|
+
if (Math.abs(configValue - recommended) <= THRESHOLD_MISMATCH_TOLERANCE + FLOAT_EPSILON)
|
|
48
|
+
return null;
|
|
49
|
+
return `search.similarityThreshold is ${formatThreshold(configValue)}, but the recommended value for ${model} is ${formatThreshold(recommended)}. `
|
|
50
|
+
+ 'Searches may miss relevant code or return weak matches. Remove the key to follow the model, or retune it (rag-config edit).';
|
|
51
|
+
}
|
|
24
52
|
async function resolveDimension(override, model, probeFn, registry = exports.MODEL_REGISTRY) {
|
|
25
53
|
let dim;
|
|
26
54
|
if (override != null) {
|
package/dist/src/container.d.ts
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import { Config } from './config';
|
|
2
2
|
import { IndexerService } from './services/IndexerService';
|
|
3
|
+
import { type SearchServices } from './search/create-search-services';
|
|
3
4
|
import { McpServer } from './services/McpServer';
|
|
4
5
|
import { LoggerService } from './services/LoggerService';
|
|
5
6
|
import type { IDatabase } from './interfaces';
|
|
6
7
|
import type { IEmbedder } from './interfaces';
|
|
7
|
-
import type { ISearcher } from './interfaces';
|
|
8
8
|
import type { ILogger } from './interfaces';
|
|
9
9
|
import type { IProjectDiscovery } from './interfaces';
|
|
10
|
+
import type { IMcpReadiness } from './interfaces';
|
|
10
11
|
export interface Container {
|
|
11
12
|
config: Config;
|
|
12
13
|
db: IDatabase;
|
|
13
14
|
embedder: IEmbedder;
|
|
14
15
|
indexer: IndexerService;
|
|
15
|
-
|
|
16
|
+
search: SearchServices;
|
|
17
|
+
readiness: IMcpReadiness;
|
|
16
18
|
mcp: McpServer;
|
|
17
19
|
logger: ILogger;
|
|
18
20
|
discovery: IProjectDiscovery;
|
package/dist/src/container.js
CHANGED
|
@@ -10,8 +10,9 @@ const EmbedderFactory_1 = require("./services/EmbedderFactory");
|
|
|
10
10
|
const AstParserService_1 = require("./services/AstParserService");
|
|
11
11
|
const create_chunker_1 = require("./chunking/create-chunker");
|
|
12
12
|
const IndexerService_1 = require("./services/IndexerService");
|
|
13
|
-
const
|
|
13
|
+
const create_search_services_1 = require("./search/create-search-services");
|
|
14
14
|
const McpServer_1 = require("./services/McpServer");
|
|
15
|
+
const McpReadiness_1 = require("./services/McpReadiness");
|
|
15
16
|
const LoggerService_1 = require("./services/LoggerService");
|
|
16
17
|
const MonorepoAwareDiscoveryService_1 = require("./services/MonorepoAwareDiscoveryService");
|
|
17
18
|
const NxMonorepoResolver_1 = require("./services/NxMonorepoResolver");
|
|
@@ -24,13 +25,14 @@ function createLogger(cfg, loggerName) {
|
|
|
24
25
|
function createContainer(loggerName) {
|
|
25
26
|
const config = new config_1.Config();
|
|
26
27
|
const logger = createLogger(config.logging, loggerName);
|
|
27
|
-
const db = new DatabaseService_1.DatabaseService(config);
|
|
28
|
+
const db = new DatabaseService_1.DatabaseService(config, logger);
|
|
28
29
|
const embedder = (0, EmbedderFactory_1.createEmbedder)(config, logger);
|
|
29
30
|
const ast = new AstParserService_1.AstParserService(logger);
|
|
30
31
|
const chunker = (0, create_chunker_1.createChunker)(config);
|
|
31
32
|
const discovery = new MonorepoAwareDiscoveryService_1.MonorepoAwareDiscoveryService([new NxMonorepoResolver_1.NxMonorepoResolver(logger)], logger);
|
|
32
33
|
const indexer = new IndexerService_1.IndexerService(db, embedder, ast, chunker, config, logger);
|
|
33
|
-
const
|
|
34
|
-
const
|
|
35
|
-
|
|
34
|
+
const search = (0, create_search_services_1.createSearchServices)(db, embedder, discovery, config, logger);
|
|
35
|
+
const readiness = new McpReadiness_1.McpReadiness(db, search.identity, logger);
|
|
36
|
+
const mcp = new McpServer_1.McpServer(search, readiness, config, logger);
|
|
37
|
+
return { config, db, embedder, indexer, search, readiness, mcp, logger, discovery };
|
|
36
38
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { NodePgDatabase, NodePgTransaction } from 'drizzle-orm/node-postgres';
|
|
2
|
+
import type { ExtractTablesWithRelations } from 'drizzle-orm/relations';
|
|
3
|
+
import * as schema from './schema';
|
|
4
|
+
type Schema = typeof schema;
|
|
5
|
+
export type DrizzleDb = NodePgDatabase<Schema>;
|
|
6
|
+
export type DrizzleTx = NodePgTransaction<Schema, ExtractTablesWithRelations<Schema>>;
|
|
7
|
+
export {};
|
package/dist/src/db/index.d.ts
CHANGED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
2
|
+
export type SchemaDb = Pick<IDatabase, 'query' | 'transaction' | 'ensureChunkTable'>;
|
|
3
|
+
export interface SchemaStep {
|
|
4
|
+
readonly name: string;
|
|
5
|
+
run(db: SchemaDb): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export declare const SCHEMA_STEPS: readonly SchemaStep[];
|
|
8
|
+
export declare function runSchemaSteps(db: SchemaDb, steps?: readonly SchemaStep[]): Promise<void>;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SCHEMA_STEPS = void 0;
|
|
4
|
+
exports.runSchemaSteps = runSchemaSteps;
|
|
5
|
+
async function codeChunkTable(db) {
|
|
6
|
+
await db.ensureChunkTable(1024);
|
|
7
|
+
}
|
|
8
|
+
async function indexMetaTable(db) {
|
|
9
|
+
await db.query(`
|
|
10
|
+
CREATE TABLE IF NOT EXISTS index_meta (
|
|
11
|
+
id BOOLEAN PRIMARY KEY DEFAULT true,
|
|
12
|
+
provider TEXT NOT NULL,
|
|
13
|
+
model TEXT NOT NULL,
|
|
14
|
+
dimensions INT NOT NULL,
|
|
15
|
+
"indexedAt" TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
16
|
+
)
|
|
17
|
+
`);
|
|
18
|
+
}
|
|
19
|
+
async function codeDependencyTable(db) {
|
|
20
|
+
const [{ existsDep }] = await db.query(`
|
|
21
|
+
SELECT EXISTS (
|
|
22
|
+
SELECT 1 FROM information_schema.tables
|
|
23
|
+
WHERE table_schema = 'public' AND table_name = 'code_dependency'
|
|
24
|
+
) AS "existsDep"
|
|
25
|
+
`);
|
|
26
|
+
if (!existsDep) {
|
|
27
|
+
await db.query(`
|
|
28
|
+
CREATE TABLE code_dependency (
|
|
29
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
30
|
+
project TEXT NOT NULL,
|
|
31
|
+
"fromFile" TEXT NOT NULL,
|
|
32
|
+
"fromSymbol" TEXT,
|
|
33
|
+
"toFile" TEXT NOT NULL,
|
|
34
|
+
"toSymbol" TEXT,
|
|
35
|
+
"depType" TEXT NOT NULL
|
|
36
|
+
)
|
|
37
|
+
`);
|
|
38
|
+
}
|
|
39
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
40
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
41
|
+
await db.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
42
|
+
await db.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
43
|
+
await db.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
44
|
+
}
|
|
45
|
+
async function codeSymbolTable(db) {
|
|
46
|
+
const [{ existsSym }] = await db.query(`
|
|
47
|
+
SELECT EXISTS (
|
|
48
|
+
SELECT 1 FROM information_schema.tables
|
|
49
|
+
WHERE table_schema = 'public' AND table_name = 'code_symbol'
|
|
50
|
+
) AS "existsSym"
|
|
51
|
+
`);
|
|
52
|
+
if (!existsSym) {
|
|
53
|
+
await db.query(`
|
|
54
|
+
CREATE TABLE code_symbol (
|
|
55
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
56
|
+
project TEXT NOT NULL,
|
|
57
|
+
"filePath" TEXT NOT NULL,
|
|
58
|
+
name TEXT NOT NULL,
|
|
59
|
+
kind TEXT NOT NULL,
|
|
60
|
+
signature TEXT,
|
|
61
|
+
line INT,
|
|
62
|
+
col INT
|
|
63
|
+
)
|
|
64
|
+
`);
|
|
65
|
+
}
|
|
66
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
|
|
67
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
|
|
68
|
+
await db.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS container TEXT`);
|
|
69
|
+
await db.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "isStatic" BOOLEAN NOT NULL DEFAULT false`);
|
|
70
|
+
await db.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "target" TEXT`);
|
|
71
|
+
await db.transaction(async (tx) => {
|
|
72
|
+
await tx.execute('DROP INDEX IF EXISTS idx_symbol_unique');
|
|
73
|
+
await tx.execute('CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique_v2 ' +
|
|
74
|
+
'ON code_symbol (project, "filePath", container, name, kind, line) NULLS NOT DISTINCT');
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
async function codeCallTable(db) {
|
|
78
|
+
await db.query(`
|
|
79
|
+
CREATE TABLE IF NOT EXISTS code_call (
|
|
80
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
81
|
+
project TEXT NOT NULL,
|
|
82
|
+
"fromFile" TEXT NOT NULL,
|
|
83
|
+
"fromSymbol" TEXT,
|
|
84
|
+
"calleeName" TEXT NOT NULL,
|
|
85
|
+
"calleeFile" TEXT NOT NULL,
|
|
86
|
+
"calleeSymbol" TEXT,
|
|
87
|
+
line INT,
|
|
88
|
+
col INT
|
|
89
|
+
)
|
|
90
|
+
`);
|
|
91
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_call_callee ON code_call (project, "calleeName")`);
|
|
92
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_call_callee_file ON code_call (project, "calleeFile")`);
|
|
93
|
+
await db.query(`CREATE INDEX IF NOT EXISTS idx_call_from ON code_call (project, "fromFile")`);
|
|
94
|
+
}
|
|
95
|
+
exports.SCHEMA_STEPS = [
|
|
96
|
+
{ name: 'codeChunkTable', run: codeChunkTable },
|
|
97
|
+
{ name: 'indexMetaTable', run: indexMetaTable },
|
|
98
|
+
{ name: 'codeDependencyTable', run: codeDependencyTable },
|
|
99
|
+
{ name: 'codeSymbolTable', run: codeSymbolTable },
|
|
100
|
+
{ name: 'codeCallTable', run: codeCallTable },
|
|
101
|
+
];
|
|
102
|
+
async function runSchemaSteps(db, steps = exports.SCHEMA_STEPS) {
|
|
103
|
+
for (const step of steps)
|
|
104
|
+
await step.run(db);
|
|
105
|
+
}
|
package/dist/src/eval-results.js
CHANGED
|
@@ -43,6 +43,8 @@ exports.normaliseResultFile = normaliseResultFile;
|
|
|
43
43
|
exports.readResultFile = readResultFile;
|
|
44
44
|
const fs = __importStar(require("fs"));
|
|
45
45
|
const path = __importStar(require("path"));
|
|
46
|
+
const retrieval_eval_1 = require("./retrieval-eval");
|
|
47
|
+
const eval_vector_index_1 = require("./eval-vector-index");
|
|
46
48
|
const CATEGORIES = ['identifier', 'behaviour', 'concept'];
|
|
47
49
|
const STAGE_LABEL = {
|
|
48
50
|
vector: 'vector (reranker input order, not a rerank-off search)',
|
|
@@ -147,11 +149,12 @@ function renderMarkdown(result, comparison) {
|
|
|
147
149
|
if (metadata.indexMeta) {
|
|
148
150
|
lines.push(`**Index:** ${metadata.indexMeta.provider}/${metadata.indexMeta.model} (${metadata.indexMeta.dimensions} dims)`);
|
|
149
151
|
}
|
|
152
|
+
lines.push(`**Vector index:** ${(0, eval_vector_index_1.formatVectorIndex)(metadata.vectorIndex)}`);
|
|
150
153
|
const corpora = corpusLines(metadata);
|
|
151
154
|
lines.push(corpora.length === 1 ? `**Corpus:** ${corpora[0]}` : `**Corpora:** ${corpora.join('; ')}`);
|
|
152
155
|
const c = metadata.conditions;
|
|
153
156
|
if (c) {
|
|
154
|
-
lines.push(`**Search conditions:** threshold ${c.
|
|
157
|
+
lines.push(`**Search conditions:** threshold ${(0, retrieval_eval_1.formatThresholdConditions)(c.thresholds)} | overfetch ${c.overfetch} | candidateFloor ${c.candidateFloor} | fetchLimit ${formatList(c.fetchLimit)} | ef_search ${formatList(c.efSearch)} | iterative scan ${formatList(c.iterativeScan)}`);
|
|
155
158
|
}
|
|
156
159
|
else {
|
|
157
160
|
lines.push('**Search conditions:** not recorded');
|
|
@@ -238,6 +241,7 @@ function renderSummary(result) {
|
|
|
238
241
|
lines.push('', `Evaluated ${run.cases.length} case(s) (${run.invalid.length} invalid) against ${metadata.embeddingProvider}/${metadata.embeddingModel}`);
|
|
239
242
|
for (const line of corpusLines(metadata))
|
|
240
243
|
lines.push(` corpus: ${line}`);
|
|
244
|
+
lines.push(` vector index: ${(0, eval_vector_index_1.formatVectorIndex)(metadata.vectorIndex)}`);
|
|
241
245
|
for (const stage of run.stages) {
|
|
242
246
|
const m = run.metrics[stage];
|
|
243
247
|
if (!m)
|
|
@@ -357,7 +361,7 @@ const EMPTY_POOL = { count: 0, recall: 0, short: 0 };
|
|
|
357
361
|
function normaliseResultFile(raw) {
|
|
358
362
|
const run = raw?.run;
|
|
359
363
|
if (!run || run.stages !== undefined || run.metrics?.overall === undefined)
|
|
360
|
-
return withCorpusMetadata(raw);
|
|
364
|
+
return withThresholdConditions(withCorpusMetadata(raw));
|
|
361
365
|
const stage = raw.metadata?.rerankerModel != null ? 'reranked' : 'vector';
|
|
362
366
|
const cases = run.cases.map(c => ({
|
|
363
367
|
id: c.id, project: c.project, category: c.category, query: c.query,
|
|
@@ -392,6 +396,16 @@ function withCorpusMetadata(file) {
|
|
|
392
396
|
run: { ...file.run, cases: file.run.cases.map(c => ({ ...c, corpus: c.corpus ?? null })), byCorpus: file.run.byCorpus ?? {} },
|
|
393
397
|
};
|
|
394
398
|
}
|
|
399
|
+
function withThresholdConditions(file) {
|
|
400
|
+
const conditions = file?.metadata?.conditions;
|
|
401
|
+
if (conditions?.similarityThreshold === undefined)
|
|
402
|
+
return file;
|
|
403
|
+
const { similarityThreshold, ...rest } = conditions;
|
|
404
|
+
return {
|
|
405
|
+
...file,
|
|
406
|
+
metadata: { ...file.metadata, conditions: { ...rest, thresholds: [{ value: similarityThreshold, source: 'not recorded' }] } },
|
|
407
|
+
};
|
|
408
|
+
}
|
|
395
409
|
function readResultFile(filePath) {
|
|
396
410
|
return normaliseResultFile(JSON.parse(fs.readFileSync(filePath, 'utf-8')));
|
|
397
411
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { IDatabase } from './interfaces/IDatabase';
|
|
2
|
+
import type { VectorIndexRecord } from './interfaces/IRetrievalEvaluator';
|
|
3
|
+
export declare function describeVectorIndex(db: Pick<IDatabase, 'query' | 'pgvectorVersion'>): Promise<VectorIndexRecord>;
|
|
4
|
+
export declare function formatIndexes(record: VectorIndexRecord): string;
|
|
5
|
+
export declare function formatVectorIndex(record: VectorIndexRecord | null): string;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.describeVectorIndex = describeVectorIndex;
|
|
4
|
+
exports.formatIndexes = formatIndexes;
|
|
5
|
+
exports.formatVectorIndex = formatVectorIndex;
|
|
6
|
+
const BUILD_DEFAULTS = {
|
|
7
|
+
hnsw: { m: '16', ef_construction: '64' },
|
|
8
|
+
ivfflat: { lists: '100' },
|
|
9
|
+
};
|
|
10
|
+
const INDEX_SQL = `
|
|
11
|
+
SELECT c.relname AS name, am.amname AS method, oc.opcname AS opclass, c.reloptions AS options
|
|
12
|
+
FROM pg_index i
|
|
13
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
14
|
+
JOIN pg_am am ON am.oid = c.relam
|
|
15
|
+
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0]
|
|
16
|
+
JOIN pg_opclass oc ON oc.oid = i.indclass[0]
|
|
17
|
+
WHERE i.indrelid = to_regclass('code_chunk') AND a.attname = 'embedding' AND i.indisvalid
|
|
18
|
+
ORDER BY c.relname`;
|
|
19
|
+
const SCAN_SETTINGS_SQL = `
|
|
20
|
+
SELECT current_setting('hnsw.max_scan_tuples', true) AS max_scan_tuples,
|
|
21
|
+
current_setting('hnsw.scan_mem_multiplier', true) AS scan_mem_multiplier
|
|
22
|
+
FROM (SELECT '[1]'::vector) AS loaded`;
|
|
23
|
+
function numericSetting(value) {
|
|
24
|
+
if (value == null || value === '')
|
|
25
|
+
return null;
|
|
26
|
+
const n = Number(value);
|
|
27
|
+
return Number.isFinite(n) ? n : null;
|
|
28
|
+
}
|
|
29
|
+
function parseOptions(options) {
|
|
30
|
+
const params = {};
|
|
31
|
+
for (const option of options ?? []) {
|
|
32
|
+
const eq = option.indexOf('=');
|
|
33
|
+
if (eq > 0)
|
|
34
|
+
params[option.slice(0, eq)] = option.slice(eq + 1);
|
|
35
|
+
}
|
|
36
|
+
return params;
|
|
37
|
+
}
|
|
38
|
+
async function describeVectorIndex(db) {
|
|
39
|
+
const rows = await db.query(INDEX_SQL);
|
|
40
|
+
const [settings] = await db.query(SCAN_SETTINGS_SQL);
|
|
41
|
+
return {
|
|
42
|
+
pgvector: await db.pgvectorVersion(),
|
|
43
|
+
indexes: rows.map(r => ({
|
|
44
|
+
name: r.name,
|
|
45
|
+
method: r.method,
|
|
46
|
+
opclass: r.opclass,
|
|
47
|
+
params: { ...BUILD_DEFAULTS[r.method], ...parseOptions(r.options) },
|
|
48
|
+
})),
|
|
49
|
+
maxScanTuples: numericSetting(settings?.max_scan_tuples),
|
|
50
|
+
scanMemMultiplier: numericSetting(settings?.scan_mem_multiplier),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
function formatIndexes(record) {
|
|
54
|
+
return record.indexes.map(i => `${i.name} ${i.method} (${[i.opclass, ...Object.entries(i.params).map(([k, v]) => `${k}=${v}`)].join(', ')})`).join('; ') || 'none';
|
|
55
|
+
}
|
|
56
|
+
function formatVectorIndex(record) {
|
|
57
|
+
if (!record)
|
|
58
|
+
return 'not recorded';
|
|
59
|
+
return `${formatIndexes(record)} | pgvector ${record.pgvector ?? 'not installed'} | hnsw.max_scan_tuples ${record.maxScanTuples ?? 'not defined'} | hnsw.scan_mem_multiplier ${record.scanMemMultiplier ?? 'not defined'}`;
|
|
60
|
+
}
|