codesearch-mcp 0.0.6-alpha → 0.0.8-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/app.js +93 -24
- package/dist/src/ast.d.ts +1 -1
- package/dist/src/ast.js +64 -14
- package/dist/src/config.js +2 -1
- package/dist/src/constants.d.ts +1 -0
- package/dist/src/constants.js +7 -0
- package/dist/src/db/init.js +16 -4
- package/dist/src/help.d.ts +1 -0
- package/dist/src/help.js +52 -0
- package/dist/src/indexer.d.ts +5 -1
- package/dist/src/indexer.js +57 -33
- package/dist/src/mcp.js +71 -8
- package/dist/src/ollama.d.ts +2 -0
- package/dist/src/ollama.js +6 -0
- package/dist/src/project-selector.js +10 -3
- package/dist/src/searcher.d.ts +10 -4
- package/dist/src/searcher.js +92 -21
- package/dist/src/setup.js +14 -5
- package/docker/docker-compose.yml +2 -0
- package/package.json +2 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 aleks
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/dist/app.js
CHANGED
|
@@ -36,6 +36,14 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
36
36
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
37
37
|
require("./src/env-loader");
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
+
const help_1 = require("./src/help");
|
|
40
|
+
function formatEta(sec) {
|
|
41
|
+
if (sec < 60)
|
|
42
|
+
return `${sec}s`;
|
|
43
|
+
const m = Math.floor(sec / 60);
|
|
44
|
+
const s = sec % 60;
|
|
45
|
+
return s > 0 ? `${m}m ${s}s` : `${m}m`;
|
|
46
|
+
}
|
|
39
47
|
function printSummary(allStats) {
|
|
40
48
|
const cols = { name: 30, num: 8 };
|
|
41
49
|
const line = '─'.repeat(cols.name + cols.num * 5 + 5);
|
|
@@ -54,13 +62,29 @@ function printSummary(allStats) {
|
|
|
54
62
|
elapsedSec: acc.elapsedSec + s.elapsedSec,
|
|
55
63
|
}), { indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0 });
|
|
56
64
|
console.log('TOTAL'.padEnd(cols.name), String(total.indexed).padStart(cols.num), String(total.skipped).padStart(cols.num), String(total.failed).padStart(cols.num), String(total.chunks).padStart(cols.num), String(total.elapsedSec.toFixed(1)).padStart(cols.num));
|
|
65
|
+
// Reprint warnings at the end — progress bar activity scrolls per-project warnings out of
|
|
66
|
+
// view, so we collect them on IndexStats and surface them here as an explicit section.
|
|
67
|
+
const projectsWithWarnings = allStats.filter(s => s.warnings.length > 0);
|
|
68
|
+
if (projectsWithWarnings.length > 0) {
|
|
69
|
+
console.log('\n=== Warnings ===');
|
|
70
|
+
for (const s of projectsWithWarnings) {
|
|
71
|
+
console.log(`\n${s.serviceName}:`);
|
|
72
|
+
for (const w of s.warnings)
|
|
73
|
+
console.log(` ⚠ ${w}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
57
76
|
}
|
|
58
77
|
async function main() {
|
|
59
78
|
const argv1 = path.basename(process.argv[1] ?? '');
|
|
60
79
|
const isIndex = argv1 === 'rag-index' || process.argv[2] === 'index';
|
|
61
80
|
const isMcp = argv1 === 'rag-mcp' || process.argv[2] === 'mcp';
|
|
62
81
|
const isSetup = argv1 === 'rag-setup' || process.argv[2] === 'setup';
|
|
63
|
-
const
|
|
82
|
+
const isBackfill = argv1 === 'rag-index' && process.argv[2] === 'backfill';
|
|
83
|
+
const mode = isBackfill ? 'backfill' : isIndex ? 'index' : isMcp ? 'mcp' : isSetup ? 'setup' : (process.argv[2] ?? '');
|
|
84
|
+
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
85
|
+
(0, help_1.printHelp)(argv1 !== 'app.js' ? argv1 : mode);
|
|
86
|
+
process.exit(0);
|
|
87
|
+
}
|
|
64
88
|
if (mode === 'index') {
|
|
65
89
|
const { initDbWithSchema } = await Promise.resolve().then(() => __importStar(require('./src/db')));
|
|
66
90
|
const { discoverProjects, scanAndIndex } = await Promise.resolve().then(() => __importStar(require('./src/indexer')));
|
|
@@ -81,27 +105,41 @@ async function main() {
|
|
|
81
105
|
process.env.WORKSPACE_ROOT ??= workspacePath;
|
|
82
106
|
await initDbWithSchema();
|
|
83
107
|
setupInterrupt();
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
console.log(
|
|
93
|
-
|
|
108
|
+
try {
|
|
109
|
+
const { collectFiles, loadIgnore } = await Promise.resolve().then(() => __importStar(require('./src/file-utils')));
|
|
110
|
+
const discovered = discoverProjects(workspacePath);
|
|
111
|
+
const projects = await selectProjects(discovered, workspacePath, forceSelect);
|
|
112
|
+
if (projects.length === 0)
|
|
113
|
+
process.exit(0);
|
|
114
|
+
const filesPerProject = projects.map(p => collectFiles(p.path, loadIgnore(p.path), p.path));
|
|
115
|
+
const totalFiles = filesPerProject.reduce((sum, f) => sum + f.length, 0);
|
|
116
|
+
console.log(`\n${projects.length} projects — ${totalFiles} files total`);
|
|
117
|
+
if (force)
|
|
118
|
+
console.log('Reindex mode: all files will be re-indexed.');
|
|
119
|
+
const allStats = [];
|
|
120
|
+
for (let i = 0; i < projects.length; i++) {
|
|
121
|
+
if (signal.aborted)
|
|
122
|
+
break;
|
|
123
|
+
const stats = await scanAndIndex(projects[i].path, projects[i].name, force, signal, filesPerProject[i]);
|
|
124
|
+
allStats.push(stats);
|
|
125
|
+
const remaining = projects.length - allStats.length;
|
|
126
|
+
if (remaining > 0 && !signal.aborted) {
|
|
127
|
+
const doneFiles = allStats.reduce((s, r) => s + r.indexed + r.skipped + r.failed, 0);
|
|
128
|
+
const elapsedSec = allStats.reduce((s, r) => s + r.elapsedSec, 0);
|
|
129
|
+
const leftFiles = filesPerProject.slice(allStats.length).reduce((sum, f) => sum + f.length, 0);
|
|
130
|
+
const eta = doneFiles > 0 ? formatEta(Math.round(elapsedSec / doneFiles * leftFiles)) : '?';
|
|
131
|
+
console.log(` ${allStats.length}/${projects.length} projects done — ${remaining} left (~${leftFiles} files) — ETA ~${eta}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
94
134
|
if (signal.aborted)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
135
|
+
console.log('\nIndexing interrupted.');
|
|
136
|
+
printSummary(allStats);
|
|
137
|
+
if (!signal.aborted)
|
|
138
|
+
generateProjectGraph(projects, workspacePath);
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
teardownInterrupt();
|
|
98
142
|
}
|
|
99
|
-
teardownInterrupt();
|
|
100
|
-
if (signal.aborted)
|
|
101
|
-
console.log('\nIndexing interrupted.');
|
|
102
|
-
printSummary(allStats);
|
|
103
|
-
if (!signal.aborted)
|
|
104
|
-
generateProjectGraph(projects, workspacePath);
|
|
105
143
|
process.exit(0);
|
|
106
144
|
}
|
|
107
145
|
else if (mode === 'mcp') {
|
|
@@ -123,11 +161,42 @@ async function main() {
|
|
|
123
161
|
const { runSetup } = await Promise.resolve().then(() => __importStar(require('./src/setup')));
|
|
124
162
|
await runSetup();
|
|
125
163
|
}
|
|
164
|
+
else if (mode === 'backfill') {
|
|
165
|
+
const { initDbWithSchema, db } = await Promise.resolve().then(() => __importStar(require('./src/db')));
|
|
166
|
+
const { collectDepEdges, writeDepEdges } = await Promise.resolve().then(() => __importStar(require('./src/indexer')));
|
|
167
|
+
const { config } = await Promise.resolve().then(() => __importStar(require('./src/config')));
|
|
168
|
+
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
|
|
169
|
+
const path = await Promise.resolve().then(() => __importStar(require('path')));
|
|
170
|
+
await initDbWithSchema();
|
|
171
|
+
const rows = await db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
|
|
172
|
+
console.log(`Backfilling dependency edges for ${rows.length} indexed files...\n`);
|
|
173
|
+
let processed = 0, skipped = 0, totalEdges = 0;
|
|
174
|
+
for (const { serviceName, filePath } of rows) {
|
|
175
|
+
const absPath = path.join(config.workspaceRoot, filePath);
|
|
176
|
+
const projectRoot = path.join(config.workspaceRoot, serviceName);
|
|
177
|
+
if (!fs.existsSync(absPath)) {
|
|
178
|
+
skipped++;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const edges = await collectDepEdges(absPath, serviceName, projectRoot);
|
|
182
|
+
await writeDepEdges(serviceName, filePath, edges);
|
|
183
|
+
totalEdges += edges.length;
|
|
184
|
+
processed++;
|
|
185
|
+
if (processed % 50 === 0)
|
|
186
|
+
process.stdout.write(` ${processed}/${rows.length} processed, ${totalEdges} edges...\r`);
|
|
187
|
+
}
|
|
188
|
+
console.log(`\nDone. Processed: ${processed}, Skipped (not on disk): ${skipped}, Total edges: ${totalEdges}`);
|
|
189
|
+
await db.destroy();
|
|
190
|
+
process.exit(0);
|
|
191
|
+
}
|
|
126
192
|
else {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
console.log
|
|
130
|
-
|
|
193
|
+
// No command → show help and exit cleanly. Unknown command → error + help + exit 1.
|
|
194
|
+
const hasUnknownArg = mode !== '';
|
|
195
|
+
const out = hasUnknownArg ? console.error : console.log;
|
|
196
|
+
if (hasUnknownArg)
|
|
197
|
+
out(`Unknown command: ${mode}\n`);
|
|
198
|
+
(0, help_1.printHelp)('default');
|
|
199
|
+
process.exit(hasUnknownArg ? 1 : 0);
|
|
131
200
|
}
|
|
132
201
|
}
|
|
133
202
|
main().catch(err => { console.error(err); process.exit(1); });
|
package/dist/src/ast.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import type { CodeDependency } from './db';
|
|
2
|
-
export declare function extractImportEdges(filePath: string, serviceName: string): Promise<Omit<CodeDependency, 'id'>[]>;
|
|
2
|
+
export declare function extractImportEdges(filePath: string, serviceName: string, projectRoot?: string): Promise<Omit<CodeDependency, 'id'>[]>;
|
package/dist/src/ast.js
CHANGED
|
@@ -1,21 +1,75 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.extractImportEdges = extractImportEdges;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
4
39
|
const ts_morph_1 = require("ts-morph");
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
40
|
+
// Per-project ts-morph instances keyed by projectRoot. Each project gets its own instance so that
|
|
41
|
+
// projects without a tsconfig (key previously shared as '') don't leak files into each other.
|
|
42
|
+
// Loading the tsconfig gives ts-morph the compilerOptions.paths needed to resolve aliases like @core/*.
|
|
43
|
+
const morphProjects = new Map();
|
|
44
|
+
const FALLBACK_OPTS = { skipAddingFilesFromTsConfig: true, skipFileDependencyResolution: false, compilerOptions: { allowJs: true, noEmit: true } };
|
|
45
|
+
function getMorphProject(projectRoot) {
|
|
46
|
+
if (!morphProjects.has(projectRoot)) {
|
|
47
|
+
const tsconfigPath = path.join(projectRoot, 'tsconfig.json');
|
|
48
|
+
const hasTsconfig = fs.existsSync(tsconfigPath);
|
|
49
|
+
let project;
|
|
50
|
+
try {
|
|
51
|
+
const opts = hasTsconfig
|
|
52
|
+
? { tsConfigFilePath: tsconfigPath, skipAddingFilesFromTsConfig: true, skipFileDependencyResolution: false }
|
|
53
|
+
: FALLBACK_OPTS;
|
|
54
|
+
project = new ts_morph_1.Project(opts);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
// tsconfig is invalid — fall back so other files in the project can still be processed
|
|
58
|
+
project = new ts_morph_1.Project(FALLBACK_OPTS);
|
|
59
|
+
}
|
|
60
|
+
morphProjects.set(projectRoot, project);
|
|
61
|
+
}
|
|
62
|
+
return morphProjects.get(projectRoot);
|
|
63
|
+
}
|
|
12
64
|
// Extract import edges from a TS/JS file using ts-morph.
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
|
|
65
|
+
// projectRoot must be the project root (where tsconfig.json lives) for alias resolution to work.
|
|
66
|
+
// Defaults to the file's directory — callers that need alias resolution must pass the project root explicitly.
|
|
67
|
+
// Removes the source file from the morph project after parsing to prevent memory accumulation.
|
|
68
|
+
async function extractImportEdges(filePath, serviceName, projectRoot = path.dirname(filePath)) {
|
|
16
69
|
// Only process JS/TS files; skip declaration files
|
|
17
70
|
if (!/\.(ts|tsx|js|jsx)$/.test(filePath) || /\.d\.ts$/.test(filePath))
|
|
18
71
|
return [];
|
|
72
|
+
const morphProject = getMorphProject(projectRoot);
|
|
19
73
|
// Remove stale cached version before re-adding
|
|
20
74
|
const existing = morphProject.getSourceFile(filePath);
|
|
21
75
|
if (existing)
|
|
@@ -49,10 +103,6 @@ async function extractImportEdges(filePath, serviceName) {
|
|
|
49
103
|
else if (namespaceImport) {
|
|
50
104
|
edges.push({ project: serviceName, fromFile: filePath, fromSymbol: null, toFile, toSymbol: null, depType: 'import-namespace' });
|
|
51
105
|
}
|
|
52
|
-
else {
|
|
53
|
-
// Side-effect import: import './foo'
|
|
54
|
-
edges.push({ project: serviceName, fromFile: filePath, fromSymbol: null, toFile, toSymbol: null, depType: 'import' });
|
|
55
|
-
}
|
|
56
106
|
}
|
|
57
107
|
// Remove from memory — prevents accumulation across large workspaces
|
|
58
108
|
morphProject.removeSourceFile(sourceFile);
|
package/dist/src/config.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.config = void 0;
|
|
4
4
|
const zod_1 = require("zod");
|
|
5
|
+
const constants_1 = require("./constants");
|
|
5
6
|
const schema = zod_1.z.object({
|
|
6
7
|
// required — no defaults, process exits if missing
|
|
7
8
|
DB_HOST: zod_1.z.string(),
|
|
@@ -11,7 +12,7 @@ const schema = zod_1.z.object({
|
|
|
11
12
|
DB_NAME: zod_1.z.string(),
|
|
12
13
|
// optional with defaults
|
|
13
14
|
OLLAMA_HOST: zod_1.z.string().url().default('http://localhost:11434'),
|
|
14
|
-
OLLAMA_MODEL: zod_1.z.string().default(
|
|
15
|
+
OLLAMA_MODEL: zod_1.z.string().default(constants_1.DEFAULT_EMBED_MODEL),
|
|
15
16
|
CHUNK_SIZE: zod_1.z.coerce.number().default(1500),
|
|
16
17
|
CHUNK_OVERLAP: zod_1.z.coerce.number().default(200),
|
|
17
18
|
EMBED_BATCH_SIZE: zod_1.z.coerce.number().default(5),
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const DEFAULT_EMBED_MODEL = "bge-m3";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_EMBED_MODEL = void 0;
|
|
4
|
+
// Embedding model is tied to the vector column dimension (bge-m3 → 1024).
|
|
5
|
+
// Change both together: update this constant, drop/recreate the `embedding` column,
|
|
6
|
+
// and re-run indexing.
|
|
7
|
+
exports.DEFAULT_EMBED_MODEL = 'bge-m3';
|
package/dist/src/db/init.js
CHANGED
|
@@ -7,7 +7,8 @@ const data_source_1 = require("./data-source");
|
|
|
7
7
|
async function initDb() {
|
|
8
8
|
await data_source_1.db.initialize();
|
|
9
9
|
}
|
|
10
|
-
// For indexer — connect + create schema only if tables don't exist yet
|
|
10
|
+
// For indexer — connect + create schema only if tables don't exist yet.
|
|
11
|
+
// All secondary indexes use IF NOT EXISTS so existing installs pick up new indexes on re-run.
|
|
11
12
|
async function initDbWithSchema() {
|
|
12
13
|
await data_source_1.db.initialize();
|
|
13
14
|
// code_chunk table
|
|
@@ -24,6 +25,9 @@ async function initDbWithSchema() {
|
|
|
24
25
|
ON code_chunk USING hnsw (embedding vector_cosine_ops)
|
|
25
26
|
`);
|
|
26
27
|
}
|
|
28
|
+
// Composite btree for all per-service lookups (scanAndIndex stale-sweep, ingest upsert, search filter).
|
|
29
|
+
// Applied idempotently so existing installs pick it up on next indexing run.
|
|
30
|
+
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
27
31
|
// code_dependency table
|
|
28
32
|
const [{ existsDep }] = await data_source_1.db.query(`
|
|
29
33
|
SELECT EXISTS (
|
|
@@ -43,8 +47,16 @@ async function initDbWithSchema() {
|
|
|
43
47
|
"depType" TEXT NOT NULL
|
|
44
48
|
)
|
|
45
49
|
`);
|
|
46
|
-
await data_source_1.db.query(`CREATE INDEX idx_dep_from ON code_dependency ("fromFile")`);
|
|
47
|
-
await data_source_1.db.query(`CREATE INDEX idx_dep_to ON code_dependency ("toFile")`);
|
|
48
|
-
await data_source_1.db.query(`CREATE INDEX idx_dep_project ON code_dependency (project)`);
|
|
49
50
|
}
|
|
51
|
+
// Composite indexes for the two query shapes used everywhere:
|
|
52
|
+
// WHERE project = $1 AND "fromFile" = $2 (get_dependencies, ingest delete-by-from)
|
|
53
|
+
// WHERE project = $1 AND "toFile" = $2 (get_dependents, stale-file sweep)
|
|
54
|
+
// IF NOT EXISTS lets existing installs upgrade without a manual migration.
|
|
55
|
+
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
56
|
+
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
57
|
+
// Drop redundant legacy single-column indexes (covered by the composites above).
|
|
58
|
+
// Safe on fresh installs too — DROP IF EXISTS is a no-op when absent.
|
|
59
|
+
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
60
|
+
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
61
|
+
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
50
62
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function printHelp(command: string): void;
|
package/dist/src/help.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.printHelp = printHelp;
|
|
4
|
+
function printHelp(command) {
|
|
5
|
+
switch (command) {
|
|
6
|
+
case 'rag-index':
|
|
7
|
+
case 'index':
|
|
8
|
+
console.log('Usage: rag-index [path] [options]');
|
|
9
|
+
console.log(' rag-index backfill');
|
|
10
|
+
console.log('');
|
|
11
|
+
console.log('Index a workspace of JS/TS projects into the vector database.');
|
|
12
|
+
console.log('');
|
|
13
|
+
console.log('Arguments:');
|
|
14
|
+
console.log(' path Path to workspace (defaults to WORKSPACE_ROOT env var)');
|
|
15
|
+
console.log('');
|
|
16
|
+
console.log('Options:');
|
|
17
|
+
console.log(' --reindex Force re-index all files (ignores cached mtimes)');
|
|
18
|
+
console.log(' --select Interactively select which projects to index');
|
|
19
|
+
console.log(' -h, --help Show this help message');
|
|
20
|
+
console.log('');
|
|
21
|
+
console.log('Subcommands:');
|
|
22
|
+
console.log(' backfill Populate dependency graph for already-indexed files');
|
|
23
|
+
break;
|
|
24
|
+
case 'rag-mcp':
|
|
25
|
+
case 'mcp':
|
|
26
|
+
console.log('Usage: rag-mcp');
|
|
27
|
+
console.log('');
|
|
28
|
+
console.log('Start the MCP server for semantic code search.');
|
|
29
|
+
console.log('Requires WORKSPACE_ROOT to be set (run rag-setup first).');
|
|
30
|
+
console.log('');
|
|
31
|
+
console.log('Options:');
|
|
32
|
+
console.log(' -h, --help Show this help message');
|
|
33
|
+
break;
|
|
34
|
+
case 'rag-setup':
|
|
35
|
+
case 'setup':
|
|
36
|
+
console.log('Usage: rag-setup');
|
|
37
|
+
console.log('');
|
|
38
|
+
console.log('Interactive setup: configure DB, Ollama, and WORKSPACE_ROOT.');
|
|
39
|
+
console.log('Run once before using rag-index or rag-mcp.');
|
|
40
|
+
console.log('');
|
|
41
|
+
console.log('Options:');
|
|
42
|
+
console.log(' -h, --help Show this help message');
|
|
43
|
+
break;
|
|
44
|
+
default:
|
|
45
|
+
console.log('Usage: rag-index [path] [--reindex] [--select]');
|
|
46
|
+
console.log(' rag-index backfill');
|
|
47
|
+
console.log(' rag-mcp');
|
|
48
|
+
console.log(' rag-setup');
|
|
49
|
+
console.log('');
|
|
50
|
+
console.log('Run any command with -h or --help for details.');
|
|
51
|
+
}
|
|
52
|
+
}
|
package/dist/src/indexer.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CodeDependency } from './db';
|
|
1
2
|
export interface IndexStats {
|
|
2
3
|
serviceName: string;
|
|
3
4
|
indexed: number;
|
|
@@ -5,9 +6,12 @@ export interface IndexStats {
|
|
|
5
6
|
failed: number;
|
|
6
7
|
chunks: number;
|
|
7
8
|
elapsedSec: number;
|
|
9
|
+
warnings: string[];
|
|
8
10
|
}
|
|
9
11
|
export declare function discoverProjects(rootDir: string): {
|
|
10
12
|
name: string;
|
|
11
13
|
path: string;
|
|
12
14
|
}[];
|
|
13
|
-
export declare function
|
|
15
|
+
export declare function collectDepEdges(absFilePath: string, serviceName: string, projectRoot: string): Promise<Omit<CodeDependency, 'id'>[]>;
|
|
16
|
+
export declare function writeDepEdges(serviceName: string, relFilePath: string, edges: Omit<CodeDependency, 'id'>[], manager?: import('typeorm').EntityManager): Promise<void>;
|
|
17
|
+
export declare function scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[]): Promise<IndexStats>;
|
package/dist/src/indexer.js
CHANGED
|
@@ -34,17 +34,18 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.discoverProjects = discoverProjects;
|
|
37
|
+
exports.collectDepEdges = collectDepEdges;
|
|
38
|
+
exports.writeDepEdges = writeDepEdges;
|
|
37
39
|
exports.scanAndIndex = scanAndIndex;
|
|
38
40
|
const fs = __importStar(require("fs"));
|
|
39
41
|
const path = __importStar(require("path"));
|
|
40
42
|
const textsplitters_1 = require("@langchain/textsplitters");
|
|
41
|
-
const ollama_1 = require("ollama");
|
|
42
43
|
const cli_progress_1 = require("cli-progress");
|
|
43
44
|
const db_1 = require("./db");
|
|
44
45
|
const ast_1 = require("./ast");
|
|
45
46
|
const config_1 = require("./config");
|
|
47
|
+
const ollama_1 = require("./ollama");
|
|
46
48
|
const file_utils_1 = require("./file-utils");
|
|
47
|
-
const ollama = new ollama_1.Ollama({ host: config_1.config.ollama.host });
|
|
48
49
|
// JS/TS splitter uses language-aware separators (functions, classes, blocks)
|
|
49
50
|
const jsSplitter = textsplitters_1.RecursiveCharacterTextSplitter.fromLanguage('js', {
|
|
50
51
|
chunkSize: config_1.config.indexer.chunkSize,
|
|
@@ -69,12 +70,39 @@ function discoverProjects(rootDir) {
|
|
|
69
70
|
}
|
|
70
71
|
// Embed a pre-formatted prompt (caller is responsible for prefix and context)
|
|
71
72
|
async function embedText(prompt) {
|
|
72
|
-
const { embedding } = await ollama.embeddings({
|
|
73
|
+
const { embedding } = await ollama_1.ollama.embeddings({
|
|
73
74
|
model: config_1.config.ollama.model,
|
|
74
75
|
prompt,
|
|
75
76
|
});
|
|
76
77
|
return embedding;
|
|
77
78
|
}
|
|
79
|
+
// Shared helpers for dependency edge writes — used by ingestFile (incremental) and backfill CLI.
|
|
80
|
+
// Extract edges via ts-morph (disk I/O) and normalize to workspace-relative paths.
|
|
81
|
+
// Call this OUTSIDE any DB transaction — disk/AST work must not hold a DB connection open.
|
|
82
|
+
async function collectDepEdges(absFilePath, serviceName, projectRoot) {
|
|
83
|
+
const rawEdges = await (0, ast_1.extractImportEdges)(absFilePath, serviceName, projectRoot);
|
|
84
|
+
return rawEdges
|
|
85
|
+
.filter(e => e.toFile.startsWith(config_1.config.workspaceRoot))
|
|
86
|
+
.map(e => ({
|
|
87
|
+
...e,
|
|
88
|
+
fromFile: path.relative(config_1.config.workspaceRoot, e.fromFile),
|
|
89
|
+
toFile: path.relative(config_1.config.workspaceRoot, e.toFile),
|
|
90
|
+
}));
|
|
91
|
+
}
|
|
92
|
+
// Replace edge rows for a file. When `manager` is provided, runs inside that transaction;
|
|
93
|
+
// otherwise opens its own. Keeps delete + insert atomic either way.
|
|
94
|
+
async function writeDepEdges(serviceName, relFilePath, edges, manager) {
|
|
95
|
+
const run = async (mgr) => {
|
|
96
|
+
await mgr.delete(db_1.CodeDependency, { project: serviceName, fromFile: relFilePath });
|
|
97
|
+
if (edges.length > 0) {
|
|
98
|
+
await mgr.createQueryBuilder().insert().into(db_1.CodeDependency).values(edges).execute();
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
if (manager)
|
|
102
|
+
await run(manager);
|
|
103
|
+
else
|
|
104
|
+
await db_1.db.transaction(run);
|
|
105
|
+
}
|
|
78
106
|
// Ingest a single file: skip if unchanged, otherwise delete old chunks and re-index.
|
|
79
107
|
// Returns status and chunk count. Never throws — failures are caught and reported.
|
|
80
108
|
async function ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
|
|
@@ -90,14 +118,7 @@ async function ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
|
|
|
90
118
|
return { status: 'skipped', chunks: 0 };
|
|
91
119
|
}
|
|
92
120
|
// Extract import edges before the transaction — disk I/O must not hold a DB connection open
|
|
93
|
-
const
|
|
94
|
-
const edges = rawEdges
|
|
95
|
-
.filter(e => e.toFile.startsWith(config_1.config.workspaceRoot))
|
|
96
|
-
.map(e => ({
|
|
97
|
-
...e,
|
|
98
|
-
fromFile: path.relative(config_1.config.workspaceRoot, e.fromFile),
|
|
99
|
-
toFile: path.relative(config_1.config.workspaceRoot, e.toFile),
|
|
100
|
-
}));
|
|
121
|
+
const edges = await collectDepEdges(filePath, serviceName, projectRoot);
|
|
101
122
|
// Read and embed outside the transaction — Ollama calls must not hold a DB transaction open
|
|
102
123
|
const code = fs.readFileSync(filePath, 'utf-8');
|
|
103
124
|
const chunks = await getSplitter(filePath).createDocuments([code]);
|
|
@@ -129,14 +150,7 @@ async function ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
|
|
|
129
150
|
.values(records.slice(i, i + config_1.config.indexer.batchSize))
|
|
130
151
|
.execute();
|
|
131
152
|
}
|
|
132
|
-
await
|
|
133
|
-
if (edges.length > 0) {
|
|
134
|
-
await manager.createQueryBuilder()
|
|
135
|
-
.insert()
|
|
136
|
-
.into(db_1.CodeDependency)
|
|
137
|
-
.values(edges)
|
|
138
|
-
.execute();
|
|
139
|
-
}
|
|
153
|
+
await writeDepEdges(serviceName, relFilePath, edges, manager);
|
|
140
154
|
});
|
|
141
155
|
return { status: 'indexed', chunks: chunks.length };
|
|
142
156
|
}
|
|
@@ -149,48 +163,58 @@ async function ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
|
|
|
149
163
|
// Removes chunks for deleted files, skips unchanged files, re-indexes modified ones.
|
|
150
164
|
// Pass force=true to invalidate all cached mtimes and force full re-index (e.g. after embedding strategy change).
|
|
151
165
|
// Pass signal to support graceful interruption — stops between files, never mid-transaction.
|
|
152
|
-
async function scanAndIndex(dir, serviceName, force = false, signal) {
|
|
166
|
+
async function scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles) {
|
|
153
167
|
const startTime = Date.now();
|
|
154
168
|
const repo = db_1.db.getRepository(db_1.CodeChunk);
|
|
155
|
-
//
|
|
169
|
+
// Force mode: wipe lastModified so all files are treated as changed
|
|
170
|
+
if (force) {
|
|
171
|
+
await db_1.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
172
|
+
}
|
|
173
|
+
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
174
|
+
// Clean up chunks for files no longer in the collected set — covers both deleted files and
|
|
175
|
+
// files newly excluded by .ragignore (they exist on disk but should no longer be indexed).
|
|
176
|
+
const collectedRelPaths = new Set(files.map(f => path.relative(config_1.config.workspaceRoot, f)));
|
|
156
177
|
const dbPaths = await db_1.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
|
|
157
178
|
for (const { filePath } of dbPaths) {
|
|
158
|
-
if (!
|
|
179
|
+
if (!collectedRelPaths.has(filePath)) {
|
|
159
180
|
await repo.delete({ serviceName, filePath });
|
|
160
|
-
// Remove dependency edges on both sides — this file no longer exists as importer or target
|
|
161
181
|
await db_1.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
|
|
162
182
|
await db_1.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
|
|
163
183
|
console.log(` Removed stale: ${path.basename(filePath)}`);
|
|
164
184
|
}
|
|
165
185
|
}
|
|
166
|
-
// Force mode: wipe lastModified so all files are treated as changed
|
|
167
|
-
if (force) {
|
|
168
|
-
await db_1.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
169
|
-
}
|
|
170
|
-
const ig = (0, file_utils_1.loadIgnore)(dir);
|
|
171
|
-
const files = (0, file_utils_1.collectFiles)(dir, ig, dir);
|
|
172
186
|
const bar = new cli_progress_1.SingleBar({
|
|
173
|
-
format: `{serviceName} {bar} {percentage}% | {value}/{total} | {filename}`,
|
|
187
|
+
format: `{serviceName} {bar} {percentage}% | {value}/{total} | ETA {eta_formatted} | {filename}`,
|
|
174
188
|
barCompleteChar: '█',
|
|
175
189
|
barIncompleteChar: '░',
|
|
176
190
|
hideCursor: true,
|
|
177
191
|
clearOnComplete: false,
|
|
178
192
|
}, cli_progress_1.Presets.shades_grey);
|
|
193
|
+
const warnings = [];
|
|
179
194
|
console.log(`\nIndexing ${serviceName}: ${files.length} files`);
|
|
180
195
|
const meta = (0, file_utils_1.readRagMeta)(dir);
|
|
181
196
|
if (meta === null && fs.existsSync(path.join(dir, '.ragmeta.json'))) {
|
|
182
|
-
|
|
197
|
+
const msg = `.ragmeta.json found but invalid JSON`;
|
|
198
|
+
console.log(` ⚠ ${msg}`);
|
|
199
|
+
warnings.push(msg);
|
|
183
200
|
}
|
|
184
201
|
else if (meta) {
|
|
185
202
|
const parts = [meta.description, meta.stack && `[${meta.stack}]`].filter(Boolean).join(' ');
|
|
186
203
|
console.log(` ✓ .ragmeta.json: ${parts || '(no fields)'}`);
|
|
187
204
|
}
|
|
188
205
|
else {
|
|
189
|
-
|
|
206
|
+
const msg = `No .ragmeta.json — add {"description":"...","stack":"...","notes":"..."} so list_projects can describe this project`;
|
|
207
|
+
console.log(` ⚠ ${msg}`);
|
|
208
|
+
warnings.push(msg);
|
|
209
|
+
}
|
|
210
|
+
if (!fs.existsSync(path.join(dir, '.ragignore'))) {
|
|
211
|
+
const msg = `No .ragignore — create one to exclude generated/codegen files (e.g. src/gql/graphql.ts, **/__snapshots__/**)`;
|
|
212
|
+
console.log(` ⚠ ${msg}`);
|
|
213
|
+
warnings.push(msg);
|
|
190
214
|
}
|
|
191
215
|
bar.start(files.length, 0, { serviceName, filename: '' });
|
|
192
216
|
const stats = {
|
|
193
|
-
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0,
|
|
217
|
+
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
|
|
194
218
|
};
|
|
195
219
|
for (const filePath of files) {
|
|
196
220
|
if (signal?.aborted) {
|
package/dist/src/mcp.js
CHANGED
|
@@ -1,11 +1,54 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.startMCP = startMCP;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
4
39
|
const index_js_1 = require("@modelcontextprotocol/sdk/server/index.js");
|
|
5
40
|
const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
6
41
|
const types_js_1 = require("@modelcontextprotocol/sdk/types.js");
|
|
7
42
|
const config_1 = require("./config");
|
|
8
43
|
const searcher_1 = require("./searcher");
|
|
44
|
+
const PKG_VERSION = (() => {
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf-8')).version ?? '0.0.0';
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return '0.0.0';
|
|
50
|
+
}
|
|
51
|
+
})();
|
|
9
52
|
const SERVER_INSTRUCTIONS = `
|
|
10
53
|
You have access to a local semantic code search server. Use these tools to navigate unfamiliar codebases.
|
|
11
54
|
|
|
@@ -14,6 +57,8 @@ You have access to a local semantic code search server. Use these tools to navig
|
|
|
14
57
|
- **list_projects** — call this first to see what projects are indexed and their exact names.
|
|
15
58
|
- **search_codebase** — semantic search across indexed source files. Returns file path, line number, and a code chunk.
|
|
16
59
|
- **read_file_range** — read a specific line range from any indexed file. Use this to expand context around a search result.
|
|
60
|
+
- **get_dependencies** — show what files a given file imports (direct or transitive). Use to understand what a file depends on.
|
|
61
|
+
- **get_dependents** — show which files import a given file (reverse graph). Use before refactoring to find all callers.
|
|
17
62
|
|
|
18
63
|
## Recommended workflow
|
|
19
64
|
|
|
@@ -79,7 +124,7 @@ function formatChunk(r, label) {
|
|
|
79
124
|
return `${label ? `*(${label})*\n` : ''}**File:** ${r.filePath}\n**Project:** ${r.serviceName}\n**Line:** ${r.startLine}\n\`\`\`\n${r.content}\n\`\`\``;
|
|
80
125
|
}
|
|
81
126
|
async function startMCP() {
|
|
82
|
-
const server = new index_js_1.Server({ name: 'local-rag', version:
|
|
127
|
+
const server = new index_js_1.Server({ name: 'local-rag', version: PKG_VERSION }, { capabilities: { tools: {} }, instructions: SERVER_INSTRUCTIONS });
|
|
83
128
|
server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => ({
|
|
84
129
|
tools: [
|
|
85
130
|
{
|
|
@@ -136,7 +181,7 @@ async function startMCP() {
|
|
|
136
181
|
properties: {
|
|
137
182
|
file_path: { type: 'string', description: 'File path as returned by search_codebase' },
|
|
138
183
|
project: { type: 'string', description: 'Project name (use list_projects to find it)' },
|
|
139
|
-
symbol: { type: 'string', description: '
|
|
184
|
+
symbol: { type: 'string', description: 'Exported symbol name. Two behaviors: (1) if graph dependents exist, narrows them to imports of this symbol; (2) if the import graph returns zero results, triggers a whole-word text-search fallback (grep -F -w) across all .ts/.tsx/.js/.jsx files in the project to find runtime usages — DI containers, singletons, dynamic lookups, string references. Capped at 200 matches.' },
|
|
140
185
|
transitive: { type: 'boolean', description: 'Include transitive (indirect) dependents. Default: false.' },
|
|
141
186
|
},
|
|
142
187
|
required: ['file_path', 'project'],
|
|
@@ -203,9 +248,9 @@ async function startMCP() {
|
|
|
203
248
|
const endLine = Number(req.params.arguments?.end_line) || startLine + 99;
|
|
204
249
|
const range = await (0, searcher_1.readFileRange)(filePath, startLine, endLine);
|
|
205
250
|
if (!range.ok) {
|
|
206
|
-
const message = range.reason === '
|
|
207
|
-
? `File is not indexed: ${filePath}. Run indexing first.`
|
|
208
|
-
|
|
251
|
+
const message = range.reason === 'invalid_path' ? `Invalid file path (outside workspace): ${filePath}` :
|
|
252
|
+
range.reason === 'not_indexed' ? `File is not indexed: ${filePath}. Run indexing first.` :
|
|
253
|
+
`File was deleted from disk after indexing: ${filePath}. Re-run indexing to update.`;
|
|
209
254
|
return { content: [{ type: 'text', text: message }] };
|
|
210
255
|
}
|
|
211
256
|
const numbered = range.lines.map((line, i) => `${startLine + i}: ${line}`).join('\n');
|
|
@@ -233,10 +278,28 @@ async function startMCP() {
|
|
|
233
278
|
const symbol = req.params.arguments?.symbol ? String(req.params.arguments.symbol) : null;
|
|
234
279
|
const transitive = Boolean(req.params.arguments?.transitive);
|
|
235
280
|
const rows = await (0, searcher_1.getDependents)(filePath, project, symbol, transitive);
|
|
236
|
-
if (rows.length
|
|
237
|
-
return { content: [{ type: 'text', text:
|
|
281
|
+
if (rows.length > 0) {
|
|
282
|
+
return { content: [{ type: 'text', text: formatDependents(filePath, rows, transitive) }] };
|
|
283
|
+
}
|
|
284
|
+
const header = `**Dependents of** \`${filePath}\`\n\n**Direct:** No direct import dependents found.`;
|
|
285
|
+
if (!symbol) {
|
|
286
|
+
return { content: [{ type: 'text', text: `${header}\n\nTip: pass \`symbol\` to trigger a text-search fallback for runtime usages (e.g. via singleton or container).` }] };
|
|
287
|
+
}
|
|
288
|
+
const { matches, truncated } = (0, searcher_1.grepSymbol)(symbol, project, filePath);
|
|
289
|
+
if (matches.length === 0) {
|
|
290
|
+
return { content: [{ type: 'text', text: `${header}\n\n**Fallback \`[text search]\` for \`${symbol}\`:** No results.` }] };
|
|
291
|
+
}
|
|
292
|
+
const grouped = new Map();
|
|
293
|
+
for (const m of matches) {
|
|
294
|
+
if (!grouped.has(m.filePath))
|
|
295
|
+
grouped.set(m.filePath, []);
|
|
296
|
+
grouped.get(m.filePath).push(m.line);
|
|
297
|
+
}
|
|
298
|
+
const fallbackLines = [`\n**Fallback \`[text search]\` for \`${symbol}\`:**${truncated ? ' *(truncated to 200 results)*' : ''}`];
|
|
299
|
+
for (const [fp, lines] of grouped) {
|
|
300
|
+
fallbackLines.push(` ~ \`${fp}\`:${lines.join(', ')}`);
|
|
238
301
|
}
|
|
239
|
-
return { content: [{ type: 'text', text:
|
|
302
|
+
return { content: [{ type: 'text', text: header + fallbackLines.join('\n') }] };
|
|
240
303
|
}
|
|
241
304
|
throw new Error(`Unknown tool: ${req.params.name}`);
|
|
242
305
|
}
|
|
@@ -45,8 +45,16 @@ function loadLines(filePath) {
|
|
|
45
45
|
.map(l => l.trim())
|
|
46
46
|
.filter(l => l && !l.startsWith('#')));
|
|
47
47
|
}
|
|
48
|
+
// Write the set if non-empty, otherwise delete the file so state is deterministic.
|
|
49
|
+
// Previously .ragskip could be left stale (last entry removed but file not rewritten);
|
|
50
|
+
// treating both files the same way avoids that drift.
|
|
48
51
|
function saveLines(filePath, names) {
|
|
49
|
-
|
|
52
|
+
if (names.size > 0) {
|
|
53
|
+
fs.writeFileSync(filePath, [...names].sort().join('\n') + '\n');
|
|
54
|
+
}
|
|
55
|
+
else if (fs.existsSync(filePath)) {
|
|
56
|
+
fs.unlinkSync(filePath);
|
|
57
|
+
}
|
|
50
58
|
}
|
|
51
59
|
function loadConfig(workspaceDir) {
|
|
52
60
|
return {
|
|
@@ -56,8 +64,7 @@ function loadConfig(workspaceDir) {
|
|
|
56
64
|
}
|
|
57
65
|
function saveConfig(workspaceDir, config) {
|
|
58
66
|
saveLines(path.join(workspaceDir, '.ragprojects'), config.included);
|
|
59
|
-
|
|
60
|
-
saveLines(path.join(workspaceDir, '.ragskip'), config.skipped);
|
|
67
|
+
saveLines(path.join(workspaceDir, '.ragskip'), config.skipped);
|
|
61
68
|
}
|
|
62
69
|
async function selectProjects(allProjects, workspaceDir, forceSelect) {
|
|
63
70
|
const config = loadConfig(workspaceDir);
|
package/dist/src/searcher.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
export type { ProjectMeta } from './file-utils';
|
|
1
|
+
import { type ProjectMeta } from './file-utils';
|
|
3
2
|
export interface ChunkRow {
|
|
4
3
|
serviceName: string;
|
|
5
4
|
filePath: string;
|
|
@@ -22,7 +21,7 @@ export type FileRangeResult = {
|
|
|
22
21
|
lines: string[];
|
|
23
22
|
} | {
|
|
24
23
|
ok: false;
|
|
25
|
-
reason: 'not_indexed' | 'not_on_disk';
|
|
24
|
+
reason: 'not_indexed' | 'not_on_disk' | 'invalid_path';
|
|
26
25
|
};
|
|
27
26
|
export interface DependencyRow {
|
|
28
27
|
fromFile: string;
|
|
@@ -37,7 +36,6 @@ export declare function prepareQuery(raw: string): Promise<{
|
|
|
37
36
|
query: string;
|
|
38
37
|
translated: boolean;
|
|
39
38
|
}>;
|
|
40
|
-
import type { ProjectMeta } from './file-utils';
|
|
41
39
|
export interface ProjectRow extends ProjectMeta {
|
|
42
40
|
serviceName: string;
|
|
43
41
|
chunks: number;
|
|
@@ -47,4 +45,12 @@ export declare function searchChunks(query: string, limit: number, project: stri
|
|
|
47
45
|
export declare function fetchNeighbors(results: SearchResult[]): Promise<NeighborResult[]>;
|
|
48
46
|
export declare function getDependencies(filePath: string, project: string, symbol?: string | null, transitive?: boolean): Promise<DependencyRow[]>;
|
|
49
47
|
export declare function getDependents(filePath: string, project: string, symbol?: string | null, transitive?: boolean): Promise<DependencyRow[]>;
|
|
48
|
+
export interface GrepMatch {
|
|
49
|
+
filePath: string;
|
|
50
|
+
line: number;
|
|
51
|
+
}
|
|
52
|
+
export declare function grepSymbol(symbol: string, project: string, excludeFile: string): {
|
|
53
|
+
matches: GrepMatch[];
|
|
54
|
+
truncated: boolean;
|
|
55
|
+
};
|
|
50
56
|
export declare function readFileRange(filePath: string, startLine: number, endLine: number): Promise<FileRangeResult>;
|
package/dist/src/searcher.js
CHANGED
|
@@ -39,21 +39,23 @@ exports.searchChunks = searchChunks;
|
|
|
39
39
|
exports.fetchNeighbors = fetchNeighbors;
|
|
40
40
|
exports.getDependencies = getDependencies;
|
|
41
41
|
exports.getDependents = getDependents;
|
|
42
|
+
exports.grepSymbol = grepSymbol;
|
|
42
43
|
exports.readFileRange = readFileRange;
|
|
43
44
|
const fs = __importStar(require("fs"));
|
|
44
45
|
const path = __importStar(require("path"));
|
|
45
|
-
const
|
|
46
|
+
const readline = __importStar(require("readline"));
|
|
47
|
+
const child_process_1 = require("child_process");
|
|
46
48
|
const db_1 = require("./db");
|
|
47
49
|
const config_1 = require("./config");
|
|
50
|
+
const ollama_1 = require("./ollama");
|
|
48
51
|
const file_utils_1 = require("./file-utils");
|
|
49
|
-
const ollama = new ollama_1.Ollama({ host: config_1.config.ollama.host });
|
|
50
52
|
// Detect non-English input by presence of non-ASCII characters (Cyrillic, etc.)
|
|
51
53
|
function isNonEnglish(text) {
|
|
52
54
|
return /[^\x00-\x7F]/.test(text);
|
|
53
55
|
}
|
|
54
56
|
// Translate query to English using a local Ollama model
|
|
55
57
|
async function translateToEnglish(text) {
|
|
56
|
-
const response = await ollama.chat({
|
|
58
|
+
const response = await ollama_1.ollama.chat({
|
|
57
59
|
model: config_1.config.search.translateModel,
|
|
58
60
|
messages: [{ role: 'user', content: `Translate the following text to English. Return only the translation, no explanations:\n${text}` }],
|
|
59
61
|
});
|
|
@@ -79,7 +81,7 @@ async function listProjects() {
|
|
|
79
81
|
}
|
|
80
82
|
// Embed query and run vector similarity search
|
|
81
83
|
async function searchChunks(query, limit, project) {
|
|
82
|
-
const { embedding } = await ollama.embeddings({ model: config_1.config.ollama.model, prompt: query });
|
|
84
|
+
const { embedding } = await ollama_1.ollama.embeddings({ model: config_1.config.ollama.model, prompt: query });
|
|
83
85
|
const params = [`[${embedding}]`, config_1.config.search.similarityThreshold, limit];
|
|
84
86
|
let projectFilter = '';
|
|
85
87
|
if (project) {
|
|
@@ -95,20 +97,32 @@ async function searchChunks(query, limit, project) {
|
|
|
95
97
|
ORDER BY embedding <=> $1
|
|
96
98
|
LIMIT $3`, params);
|
|
97
99
|
}
|
|
98
|
-
// Fetch the chunk immediately before and after each result in the same file (Small-to-Big)
|
|
100
|
+
// Fetch the chunk immediately before and after each result in the same file (Small-to-Big).
|
|
101
|
+
// Single query with LATERAL joins — avoids 2N round-trips.
|
|
99
102
|
async function fetchNeighbors(results) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
103
|
+
if (results.length === 0)
|
|
104
|
+
return [];
|
|
105
|
+
const values = results.map((_, i) => `(${i}, $${i * 2 + 1}, $${i * 2 + 2}::int)`).join(',');
|
|
106
|
+
const params = results.flatMap(r => [r.filePath, r.startLine]);
|
|
107
|
+
const rows = await db_1.db.query(`SELECT r.idx,
|
|
108
|
+
prev."serviceName" AS prev_sn, prev."filePath" AS prev_fp, prev."startLine" AS prev_sl, prev.content AS prev_c,
|
|
109
|
+
next."serviceName" AS next_sn, next."filePath" AS next_fp, next."startLine" AS next_sl, next.content AS next_c
|
|
110
|
+
FROM (VALUES ${values}) r(idx, fp, sl)
|
|
111
|
+
LEFT JOIN LATERAL (
|
|
112
|
+
SELECT "serviceName", "filePath", "startLine", content
|
|
113
|
+
FROM code_chunk WHERE "filePath" = r.fp AND "startLine" < r.sl
|
|
114
|
+
ORDER BY "startLine" DESC LIMIT 1
|
|
115
|
+
) prev ON true
|
|
116
|
+
LEFT JOIN LATERAL (
|
|
117
|
+
SELECT "serviceName", "filePath", "startLine", content
|
|
118
|
+
FROM code_chunk WHERE "filePath" = r.fp AND "startLine" > r.sl
|
|
119
|
+
ORDER BY "startLine" ASC LIMIT 1
|
|
120
|
+
) next ON true
|
|
121
|
+
ORDER BY r.idx`, params);
|
|
122
|
+
return rows.map((row, i) => ({
|
|
123
|
+
result: results[i],
|
|
124
|
+
prevChunk: row.prev_fp ? { serviceName: row.prev_sn, filePath: row.prev_fp, startLine: row.prev_sl, content: row.prev_c } : null,
|
|
125
|
+
nextChunk: row.next_fp ? { serviceName: row.next_sn, filePath: row.next_fp, startLine: row.next_sl, content: row.next_c } : null,
|
|
112
126
|
}));
|
|
113
127
|
}
|
|
114
128
|
// Return all files imported by filePath (direct or transitive)
|
|
@@ -191,15 +205,72 @@ async function getDependents(filePath, project, symbol = null, transitive = fals
|
|
|
191
205
|
ORDER BY MIN(depth), "fromFile"`, [filePath, project]);
|
|
192
206
|
return [...direct, ...indirect];
|
|
193
207
|
}
|
|
208
|
+
const GREP_MAX_RESULTS = 200;
|
|
209
|
+
// Grep stops after this many raw lines; generous multiple of GREP_MAX_RESULTS to account for
|
|
210
|
+
// the declaring file's own matches being excluded from the final count.
|
|
211
|
+
const GREP_RAW_LIMIT = GREP_MAX_RESULTS * 5;
|
|
212
|
+
const GREP_EXCLUDE_DIRS = [...file_utils_1.IGNORE_DIRS].map(d => `--exclude-dir=${d}`);
|
|
213
|
+
// Text search for a symbol across all TS/JS files in the project directory.
|
|
214
|
+
// Uses grep -F -w (literal whole-word match) via spawnSync to avoid shell injection.
|
|
215
|
+
// Excludes the declaring file itself. Uses IGNORE_DIRS for consistency with the indexer.
|
|
216
|
+
function grepSymbol(symbol, project, excludeFile) {
|
|
217
|
+
const projectDir = path.join(config_1.config.workspaceRoot, project);
|
|
218
|
+
const result = (0, child_process_1.spawnSync)('grep', [
|
|
219
|
+
'-rn', '-F', '-w',
|
|
220
|
+
'-m', String(GREP_RAW_LIMIT),
|
|
221
|
+
'--include=*.ts', '--include=*.tsx', '--include=*.js', '--include=*.jsx',
|
|
222
|
+
...GREP_EXCLUDE_DIRS,
|
|
223
|
+
symbol,
|
|
224
|
+
projectDir,
|
|
225
|
+
], { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
|
|
226
|
+
if (result.error || !result.stdout) {
|
|
227
|
+
if (result.error)
|
|
228
|
+
console.error(`[grepSymbol] grep failed: ${result.error.message}`);
|
|
229
|
+
return { matches: [], truncated: false };
|
|
230
|
+
}
|
|
231
|
+
const excludeAbsPath = path.join(config_1.config.workspaceRoot, excludeFile);
|
|
232
|
+
const all = result.stdout.split('\n').flatMap(line => {
|
|
233
|
+
const match = line.match(/^(.+?):(\d+):/);
|
|
234
|
+
if (!match)
|
|
235
|
+
return [];
|
|
236
|
+
const [, absFile, lineNum] = match;
|
|
237
|
+
if (absFile === excludeAbsPath)
|
|
238
|
+
return [];
|
|
239
|
+
return [{ filePath: path.relative(config_1.config.workspaceRoot, absFile), line: parseInt(lineNum, 10) }];
|
|
240
|
+
});
|
|
241
|
+
return { matches: all.slice(0, GREP_MAX_RESULTS), truncated: all.length > GREP_MAX_RESULTS };
|
|
242
|
+
}
|
|
194
243
|
// Read a line range from a file that is present in the index
|
|
195
244
|
async function readFileRange(filePath, startLine, endLine) {
|
|
245
|
+
// Defensive: reject paths that escape the workspace root (e.g. "../../etc/passwd")
|
|
246
|
+
const resolvedRoot = path.resolve(config_1.config.workspaceRoot);
|
|
247
|
+
const absPath = path.resolve(resolvedRoot, filePath);
|
|
248
|
+
if (absPath !== resolvedRoot && !absPath.startsWith(resolvedRoot + path.sep)) {
|
|
249
|
+
return { ok: false, reason: 'invalid_path' };
|
|
250
|
+
}
|
|
196
251
|
const [{ exists }] = await db_1.db.query(`SELECT EXISTS (SELECT 1 FROM code_chunk WHERE "filePath" = $1) AS exists`, [filePath]);
|
|
197
252
|
if (!exists)
|
|
198
253
|
return { ok: false, reason: 'not_indexed' };
|
|
199
|
-
const absPath = path.join(config_1.config.workspaceRoot, filePath);
|
|
200
254
|
if (!fs.existsSync(absPath))
|
|
201
255
|
return { ok: false, reason: 'not_on_disk' };
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
256
|
+
// Stream line-by-line — avoids loading entire file into memory.
|
|
257
|
+
// Stops reading as soon as we pass endLine.
|
|
258
|
+
const lines = [];
|
|
259
|
+
const stream = fs.createReadStream(absPath, { encoding: 'utf-8' });
|
|
260
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
261
|
+
let lineNo = 0;
|
|
262
|
+
let lastRead = 0;
|
|
263
|
+
for await (const line of rl) {
|
|
264
|
+
lineNo++;
|
|
265
|
+
lastRead = lineNo;
|
|
266
|
+
if (lineNo >= startLine && lineNo <= endLine)
|
|
267
|
+
lines.push(line);
|
|
268
|
+
if (lineNo >= endLine) {
|
|
269
|
+
rl.close();
|
|
270
|
+
stream.destroy();
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
// If we broke out early, lastRead is endLine. If file was shorter, lastRead is file length.
|
|
275
|
+
return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines };
|
|
205
276
|
}
|
package/dist/src/setup.js
CHANGED
|
@@ -39,6 +39,7 @@ const fs = __importStar(require("fs"));
|
|
|
39
39
|
const path = __importStar(require("path"));
|
|
40
40
|
const os = __importStar(require("os"));
|
|
41
41
|
const child_process_1 = require("child_process");
|
|
42
|
+
const constants_1 = require("./constants");
|
|
42
43
|
const CONFIG_DIR = path.join(os.homedir(), '.config', 'codesearch-mcp');
|
|
43
44
|
const CONFIG_FILE = path.join(CONFIG_DIR, '.env');
|
|
44
45
|
const DOCKER_COMPOSE_SRC = path.join(__dirname, '..', '..', 'docker', 'docker-compose.yml');
|
|
@@ -133,8 +134,18 @@ async function setupLocalDocker(ollamaHost) {
|
|
|
133
134
|
}
|
|
134
135
|
await waitForDb('localhost', 5433, 'dev', 'dev', 'rag');
|
|
135
136
|
await waitForOllama(ollamaHost);
|
|
136
|
-
|
|
137
|
-
|
|
137
|
+
// Resolve the ollama container ID via compose — compose project name (and thus the
|
|
138
|
+
// default container name) depends on the parent directory, so hardcoding it breaks when
|
|
139
|
+
// users install via npm -g or move the compose file around.
|
|
140
|
+
const ollamaId = (0, child_process_1.spawnSync)('docker', ['compose', '-f', DOCKER_COMPOSE_DST, 'ps', '-q', 'ollama'], { encoding: 'utf-8' })
|
|
141
|
+
.stdout?.trim();
|
|
142
|
+
if (!ollamaId) {
|
|
143
|
+
console.error('Could not resolve ollama container ID from compose. Pull the model manually:');
|
|
144
|
+
console.error(` docker exec <ollama-container> ollama pull ${constants_1.DEFAULT_EMBED_MODEL}`);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
}
|
|
147
|
+
console.log(`Pulling ${constants_1.DEFAULT_EMBED_MODEL} model (this may take a while)...`);
|
|
148
|
+
(0, child_process_1.spawnSync)('docker', ['exec', ollamaId, 'ollama', 'pull', constants_1.DEFAULT_EMBED_MODEL], { stdio: 'inherit' });
|
|
138
149
|
return {
|
|
139
150
|
DB_HOST: 'localhost',
|
|
140
151
|
DB_PORT: '5433',
|
|
@@ -166,13 +177,12 @@ async function setupRemoteDb(rl) {
|
|
|
166
177
|
async function runSetup() {
|
|
167
178
|
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
168
179
|
console.log('=== codesearch-mcp setup ===\n');
|
|
169
|
-
const dbChoice = await ask(rl, '
|
|
180
|
+
const dbChoice = await ask(rl, 'Infrastructure:\n 1. Local Docker (auto-start PostgreSQL + Ollama)\n 2. Remote (Supabase, Railway, self-hosted DB + local Ollama)\n\nEnter choice (1): ');
|
|
170
181
|
const useDocker = (dbChoice.trim() || '1') === '1';
|
|
171
182
|
const defaultWorkspace = path.join(os.homedir(), 'Projects');
|
|
172
183
|
const workspaceInput = await ask(rl, `\nWorkspace root — directory that contains all your projects\n Example: ${os.homedir()}/WebstormProjects\n Path (${defaultWorkspace}): `);
|
|
173
184
|
const workspaceRoot = expandHome(workspaceInput.trim() || defaultWorkspace);
|
|
174
185
|
const ollamaHost = await prompt(rl, 'Ollama API URL', 'http://localhost:11434');
|
|
175
|
-
const ollamaModel = await prompt(rl, 'Embedding model', 'bge-m3');
|
|
176
186
|
const dbVars = useDocker
|
|
177
187
|
? await setupLocalDocker(ollamaHost)
|
|
178
188
|
: await setupRemoteDb(rl);
|
|
@@ -180,7 +190,6 @@ async function runSetup() {
|
|
|
180
190
|
...dbVars,
|
|
181
191
|
WORKSPACE_ROOT: workspaceRoot,
|
|
182
192
|
OLLAMA_HOST: ollamaHost,
|
|
183
|
-
OLLAMA_MODEL: ollamaModel,
|
|
184
193
|
});
|
|
185
194
|
rl.close();
|
|
186
195
|
console.log('\nInitializing database schema...');
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codesearch-mcp",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8-alpha",
|
|
4
4
|
"description": "Local semantic codebase search via MCP — indexes projects into pgvector, exposes search to Claude",
|
|
5
|
+
"license": "MIT",
|
|
5
6
|
"main": "dist/app.js",
|
|
6
7
|
"bin": {
|
|
7
8
|
"rag-index": "dist/app.js",
|