codesearch-mcp 0.0.18-alpha → 0.1.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +75 -6
- package/dist/app.js +36 -0
- package/dist/src/commands/IndexCommand.js +14 -1
- package/dist/src/commands/McpCommand.js +6 -2
- package/dist/src/config-store.d.ts +49 -11
- package/dist/src/config-store.js +79 -19
- package/dist/src/config.d.ts +32 -8
- package/dist/src/config.js +84 -20
- package/dist/src/constants.d.ts +36 -2
- package/dist/src/constants.js +58 -7
- package/dist/src/container.js +5 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/help.js +5 -4
- package/dist/src/interfaces/IAstParser.d.ts +14 -0
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +14 -0
- package/dist/src/interfaces/IEmbedder.d.ts +14 -2
- package/dist/src/interfaces/ISearcher.d.ts +1 -0
- package/dist/src/interfaces/index.d.ts +2 -1
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/services/AstChunker.d.ts +31 -0
- package/dist/src/services/AstChunker.js +114 -0
- package/dist/src/services/AstParserService.d.ts +6 -0
- package/dist/src/services/AstParserService.js +31 -2
- package/dist/src/services/DatabaseService.d.ts +13 -1
- package/dist/src/services/DatabaseService.js +70 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +29 -0
- package/dist/src/services/IndexerService.d.ts +17 -5
- package/dist/src/services/IndexerService.js +101 -38
- package/dist/src/services/InfinityEmbedder.d.ts +79 -0
- package/dist/src/services/InfinityEmbedder.js +199 -0
- package/dist/src/services/OllamaEmbedder.d.ts +16 -5
- package/dist/src/services/OllamaEmbedder.js +84 -36
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/SearcherService.d.ts +7 -1
- package/dist/src/services/SearcherService.js +35 -16
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +16 -11
- package/dist/src/services/WebServerService.js +44 -24
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +6 -0
- package/dist/src/setup.js +243 -25
- package/dist/src/tools/SearchCodebaseTool.js +1 -1
- package/dist/src/tools/format-helpers.js +2 -1
- package/dist/src/web/dashboard.js +5 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +25 -0
- package/package.json +21 -2
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.d.ts +0 -7
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/env-loader.js +0 -41
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.CodeChunk = void 0;
|
|
13
|
-
const typeorm_1 = require("typeorm");
|
|
14
|
-
let CodeChunk = class CodeChunk {
|
|
15
|
-
id;
|
|
16
|
-
serviceName;
|
|
17
|
-
filePath;
|
|
18
|
-
startLine;
|
|
19
|
-
content;
|
|
20
|
-
embedding;
|
|
21
|
-
lastModified;
|
|
22
|
-
};
|
|
23
|
-
exports.CodeChunk = CodeChunk;
|
|
24
|
-
__decorate([
|
|
25
|
-
(0, typeorm_1.PrimaryGeneratedColumn)('uuid'),
|
|
26
|
-
__metadata("design:type", String)
|
|
27
|
-
], CodeChunk.prototype, "id", void 0);
|
|
28
|
-
__decorate([
|
|
29
|
-
(0, typeorm_1.Column)(),
|
|
30
|
-
__metadata("design:type", String)
|
|
31
|
-
], CodeChunk.prototype, "serviceName", void 0);
|
|
32
|
-
__decorate([
|
|
33
|
-
(0, typeorm_1.Column)(),
|
|
34
|
-
__metadata("design:type", String)
|
|
35
|
-
], CodeChunk.prototype, "filePath", void 0);
|
|
36
|
-
__decorate([
|
|
37
|
-
(0, typeorm_1.Column)({ type: 'int', nullable: true }),
|
|
38
|
-
__metadata("design:type", Object)
|
|
39
|
-
], CodeChunk.prototype, "startLine", void 0);
|
|
40
|
-
__decorate([
|
|
41
|
-
(0, typeorm_1.Column)('text'),
|
|
42
|
-
__metadata("design:type", String)
|
|
43
|
-
], CodeChunk.prototype, "content", void 0);
|
|
44
|
-
__decorate([
|
|
45
|
-
(0, typeorm_1.Column)({ type: 'vector', length: 1024 }),
|
|
46
|
-
__metadata("design:type", String)
|
|
47
|
-
], CodeChunk.prototype, "embedding", void 0);
|
|
48
|
-
__decorate([
|
|
49
|
-
(0, typeorm_1.Column)({ type: 'double precision', nullable: true }),
|
|
50
|
-
__metadata("design:type", Number)
|
|
51
|
-
], CodeChunk.prototype, "lastModified", void 0);
|
|
52
|
-
exports.CodeChunk = CodeChunk = __decorate([
|
|
53
|
-
(0, typeorm_1.Entity)()
|
|
54
|
-
], CodeChunk);
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.CodeDependency = void 0;
|
|
13
|
-
const typeorm_1 = require("typeorm");
|
|
14
|
-
let CodeDependency = class CodeDependency {
|
|
15
|
-
id;
|
|
16
|
-
project;
|
|
17
|
-
fromFile;
|
|
18
|
-
fromSymbol;
|
|
19
|
-
toFile;
|
|
20
|
-
toSymbol;
|
|
21
|
-
depType; // 'import' | 'import-default' | 'import-namespace'
|
|
22
|
-
};
|
|
23
|
-
exports.CodeDependency = CodeDependency;
|
|
24
|
-
__decorate([
|
|
25
|
-
(0, typeorm_1.PrimaryGeneratedColumn)('uuid'),
|
|
26
|
-
__metadata("design:type", String)
|
|
27
|
-
], CodeDependency.prototype, "id", void 0);
|
|
28
|
-
__decorate([
|
|
29
|
-
(0, typeorm_1.Column)(),
|
|
30
|
-
__metadata("design:type", String)
|
|
31
|
-
], CodeDependency.prototype, "project", void 0);
|
|
32
|
-
__decorate([
|
|
33
|
-
(0, typeorm_1.Column)(),
|
|
34
|
-
__metadata("design:type", String)
|
|
35
|
-
], CodeDependency.prototype, "fromFile", void 0);
|
|
36
|
-
__decorate([
|
|
37
|
-
(0, typeorm_1.Column)({ type: 'text', nullable: true }),
|
|
38
|
-
__metadata("design:type", Object)
|
|
39
|
-
], CodeDependency.prototype, "fromSymbol", void 0);
|
|
40
|
-
__decorate([
|
|
41
|
-
(0, typeorm_1.Column)(),
|
|
42
|
-
__metadata("design:type", String)
|
|
43
|
-
], CodeDependency.prototype, "toFile", void 0);
|
|
44
|
-
__decorate([
|
|
45
|
-
(0, typeorm_1.Column)({ type: 'text', nullable: true }),
|
|
46
|
-
__metadata("design:type", Object)
|
|
47
|
-
], CodeDependency.prototype, "toSymbol", void 0);
|
|
48
|
-
__decorate([
|
|
49
|
-
(0, typeorm_1.Column)(),
|
|
50
|
-
__metadata("design:type", String)
|
|
51
|
-
], CodeDependency.prototype, "depType", void 0);
|
|
52
|
-
exports.CodeDependency = CodeDependency = __decorate([
|
|
53
|
-
(0, typeorm_1.Entity)()
|
|
54
|
-
], CodeDependency);
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.CodeSymbol = void 0;
|
|
13
|
-
const typeorm_1 = require("typeorm");
|
|
14
|
-
let CodeSymbol = class CodeSymbol {
|
|
15
|
-
id;
|
|
16
|
-
project;
|
|
17
|
-
filePath;
|
|
18
|
-
name;
|
|
19
|
-
kind;
|
|
20
|
-
signature;
|
|
21
|
-
line;
|
|
22
|
-
col;
|
|
23
|
-
};
|
|
24
|
-
exports.CodeSymbol = CodeSymbol;
|
|
25
|
-
__decorate([
|
|
26
|
-
(0, typeorm_1.PrimaryGeneratedColumn)('uuid'),
|
|
27
|
-
__metadata("design:type", String)
|
|
28
|
-
], CodeSymbol.prototype, "id", void 0);
|
|
29
|
-
__decorate([
|
|
30
|
-
(0, typeorm_1.Column)(),
|
|
31
|
-
__metadata("design:type", String)
|
|
32
|
-
], CodeSymbol.prototype, "project", void 0);
|
|
33
|
-
__decorate([
|
|
34
|
-
(0, typeorm_1.Column)(),
|
|
35
|
-
__metadata("design:type", String)
|
|
36
|
-
], CodeSymbol.prototype, "filePath", void 0);
|
|
37
|
-
__decorate([
|
|
38
|
-
(0, typeorm_1.Column)(),
|
|
39
|
-
__metadata("design:type", String)
|
|
40
|
-
], CodeSymbol.prototype, "name", void 0);
|
|
41
|
-
__decorate([
|
|
42
|
-
(0, typeorm_1.Column)(),
|
|
43
|
-
__metadata("design:type", String)
|
|
44
|
-
], CodeSymbol.prototype, "kind", void 0);
|
|
45
|
-
__decorate([
|
|
46
|
-
(0, typeorm_1.Column)({ type: 'text', nullable: true }),
|
|
47
|
-
__metadata("design:type", Object)
|
|
48
|
-
], CodeSymbol.prototype, "signature", void 0);
|
|
49
|
-
__decorate([
|
|
50
|
-
(0, typeorm_1.Column)({ type: 'int', nullable: true }),
|
|
51
|
-
__metadata("design:type", Object)
|
|
52
|
-
], CodeSymbol.prototype, "line", void 0);
|
|
53
|
-
__decorate([
|
|
54
|
-
(0, typeorm_1.Column)({ type: 'int', nullable: true }),
|
|
55
|
-
__metadata("design:type", Object)
|
|
56
|
-
], CodeSymbol.prototype, "col", void 0);
|
|
57
|
-
exports.CodeSymbol = CodeSymbol = __decorate([
|
|
58
|
-
(0, typeorm_1.Entity)()
|
|
59
|
-
], CodeSymbol);
|
package/dist/src/db/init.d.ts
DELETED
package/dist/src/db/init.js
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.initDb = initDb;
|
|
4
|
-
exports.initDbWithSchema = initDbWithSchema;
|
|
5
|
-
const data_source_1 = require("./data-source");
|
|
6
|
-
// For MCP server — just connect, no schema changes
|
|
7
|
-
async function initDb() {
|
|
8
|
-
await data_source_1.db.initialize();
|
|
9
|
-
}
|
|
10
|
-
// For indexer — connect + create schema only if tables don't exist yet.
|
|
11
|
-
// All secondary indexes use IF NOT EXISTS so existing installs pick up new indexes on re-run.
|
|
12
|
-
async function initDbWithSchema() {
|
|
13
|
-
await data_source_1.db.initialize();
|
|
14
|
-
// code_chunk table
|
|
15
|
-
const [{ exists }] = await data_source_1.db.query(`
|
|
16
|
-
SELECT EXISTS (
|
|
17
|
-
SELECT 1 FROM information_schema.tables
|
|
18
|
-
WHERE table_schema = 'public' AND table_name = 'code_chunk'
|
|
19
|
-
)
|
|
20
|
-
`);
|
|
21
|
-
if (!exists) {
|
|
22
|
-
await data_source_1.db.synchronize();
|
|
23
|
-
await data_source_1.db.query(`
|
|
24
|
-
CREATE INDEX idx_code_chunk_embedding
|
|
25
|
-
ON code_chunk USING hnsw (embedding vector_cosine_ops)
|
|
26
|
-
`);
|
|
27
|
-
}
|
|
28
|
-
// Composite btree for all per-service lookups (scanAndIndex stale-sweep, ingest upsert, search filter).
|
|
29
|
-
// Applied idempotently so existing installs pick it up on next indexing run.
|
|
30
|
-
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
31
|
-
// code_dependency table
|
|
32
|
-
const [{ existsDep }] = await data_source_1.db.query(`
|
|
33
|
-
SELECT EXISTS (
|
|
34
|
-
SELECT 1 FROM information_schema.tables
|
|
35
|
-
WHERE table_schema = 'public' AND table_name = 'code_dependency'
|
|
36
|
-
) AS "existsDep"
|
|
37
|
-
`);
|
|
38
|
-
if (!existsDep) {
|
|
39
|
-
await data_source_1.db.query(`
|
|
40
|
-
CREATE TABLE code_dependency (
|
|
41
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
42
|
-
project TEXT NOT NULL,
|
|
43
|
-
"fromFile" TEXT NOT NULL,
|
|
44
|
-
"fromSymbol" TEXT,
|
|
45
|
-
"toFile" TEXT NOT NULL,
|
|
46
|
-
"toSymbol" TEXT,
|
|
47
|
-
"depType" TEXT NOT NULL
|
|
48
|
-
)
|
|
49
|
-
`);
|
|
50
|
-
}
|
|
51
|
-
// Composite indexes for the two query shapes used everywhere:
|
|
52
|
-
// WHERE project = $1 AND "fromFile" = $2 (get_dependencies, ingest delete-by-from)
|
|
53
|
-
// WHERE project = $1 AND "toFile" = $2 (get_dependents, stale-file sweep)
|
|
54
|
-
// IF NOT EXISTS lets existing installs upgrade without a manual migration.
|
|
55
|
-
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
56
|
-
await data_source_1.db.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
57
|
-
// Drop redundant legacy single-column indexes (covered by the composites above).
|
|
58
|
-
// Safe on fresh installs too — DROP IF EXISTS is a no-op when absent.
|
|
59
|
-
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
60
|
-
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
61
|
-
await data_source_1.db.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
62
|
-
}
|
package/dist/src/env-loader.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
package/dist/src/env-loader.js
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
const dotenv = __importStar(require("dotenv"));
|
|
37
|
-
const path = __importStar(require("path"));
|
|
38
|
-
const os = __importStar(require("os"));
|
|
39
|
-
// Global config first, then local .env overrides (for development)
|
|
40
|
-
dotenv.config({ path: path.join(os.homedir(), '.config', 'codesearch-mcp', '.env') });
|
|
41
|
-
dotenv.config();
|
package/dist/src/indexer.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { CodeDependency } from './db';
|
|
2
|
-
export interface IndexStats {
|
|
3
|
-
serviceName: string;
|
|
4
|
-
indexed: number;
|
|
5
|
-
skipped: number;
|
|
6
|
-
failed: number;
|
|
7
|
-
chunks: number;
|
|
8
|
-
elapsedSec: number;
|
|
9
|
-
warnings: string[];
|
|
10
|
-
}
|
|
11
|
-
export declare function discoverProjects(rootDir: string): {
|
|
12
|
-
name: string;
|
|
13
|
-
path: string;
|
|
14
|
-
}[];
|
|
15
|
-
export declare function collectDepEdges(absFilePath: string, serviceName: string, projectRoot: string): Promise<Omit<CodeDependency, 'id'>[]>;
|
|
16
|
-
export declare function writeDepEdges(serviceName: string, relFilePath: string, edges: Omit<CodeDependency, 'id'>[], manager?: import('typeorm').EntityManager): Promise<void>;
|
|
17
|
-
export declare function scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[]): Promise<IndexStats>;
|
package/dist/src/indexer.js
DELETED
|
@@ -1,242 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.discoverProjects = discoverProjects;
|
|
37
|
-
exports.collectDepEdges = collectDepEdges;
|
|
38
|
-
exports.writeDepEdges = writeDepEdges;
|
|
39
|
-
exports.scanAndIndex = scanAndIndex;
|
|
40
|
-
const fs = __importStar(require("fs"));
|
|
41
|
-
const path = __importStar(require("path"));
|
|
42
|
-
const textsplitters_1 = require("@langchain/textsplitters");
|
|
43
|
-
const cli_progress_1 = require("cli-progress");
|
|
44
|
-
const db_1 = require("./db");
|
|
45
|
-
const ast_1 = require("./ast");
|
|
46
|
-
const config_1 = require("./config");
|
|
47
|
-
const ollama_1 = require("./ollama");
|
|
48
|
-
const file_utils_1 = require("./file-utils");
|
|
49
|
-
// JS/TS splitter uses language-aware separators (functions, classes, blocks)
|
|
50
|
-
const jsSplitter = textsplitters_1.RecursiveCharacterTextSplitter.fromLanguage('js', {
|
|
51
|
-
chunkSize: config_1.config.indexer.chunkSize,
|
|
52
|
-
chunkOverlap: config_1.config.indexer.chunkOverlap,
|
|
53
|
-
});
|
|
54
|
-
// Schema splitter uses generic separators — better for Prisma/GraphQL block structure
|
|
55
|
-
const schemaSplitter = new textsplitters_1.RecursiveCharacterTextSplitter({
|
|
56
|
-
chunkSize: config_1.config.indexer.chunkSize,
|
|
57
|
-
chunkOverlap: config_1.config.indexer.chunkOverlap,
|
|
58
|
-
separators: ['\n\n', '\n', ' '],
|
|
59
|
-
});
|
|
60
|
-
const SCHEMA_EXTENSIONS = /\.(prisma|graphql|gql)$/;
|
|
61
|
-
function getSplitter(filePath) {
|
|
62
|
-
return SCHEMA_EXTENSIONS.test(filePath) ? schemaSplitter : jsSplitter;
|
|
63
|
-
}
|
|
64
|
-
// Detect JS/TS projects by the presence of package.json in their root
|
|
65
|
-
function discoverProjects(rootDir) {
|
|
66
|
-
return fs.readdirSync(rootDir, { withFileTypes: true })
|
|
67
|
-
.filter(d => d.isDirectory() && !file_utils_1.IGNORE_DIRS.has(d.name))
|
|
68
|
-
.map(d => ({ name: d.name, path: path.join(rootDir, d.name) }))
|
|
69
|
-
.filter(p => fs.existsSync(path.join(p.path, 'package.json')));
|
|
70
|
-
}
|
|
71
|
-
// Embed a pre-formatted prompt (caller is responsible for prefix and context)
|
|
72
|
-
async function embedText(prompt) {
|
|
73
|
-
const { embedding } = await ollama_1.ollama.embeddings({
|
|
74
|
-
model: config_1.config.ollama.model,
|
|
75
|
-
prompt,
|
|
76
|
-
});
|
|
77
|
-
return embedding;
|
|
78
|
-
}
|
|
79
|
-
// Shared helpers for dependency edge writes — used by ingestFile (incremental) and backfill CLI.
|
|
80
|
-
// Extract edges via ts-morph (disk I/O) and normalize to workspace-relative paths.
|
|
81
|
-
// Call this OUTSIDE any DB transaction — disk/AST work must not hold a DB connection open.
|
|
82
|
-
async function collectDepEdges(absFilePath, serviceName, projectRoot) {
|
|
83
|
-
const rawEdges = await (0, ast_1.extractImportEdges)(absFilePath, serviceName, projectRoot);
|
|
84
|
-
return rawEdges
|
|
85
|
-
.filter(e => e.toFile.startsWith(config_1.config.workspaceRoot))
|
|
86
|
-
.map(e => ({
|
|
87
|
-
...e,
|
|
88
|
-
fromFile: path.relative(config_1.config.workspaceRoot, e.fromFile),
|
|
89
|
-
toFile: path.relative(config_1.config.workspaceRoot, e.toFile),
|
|
90
|
-
}));
|
|
91
|
-
}
|
|
92
|
-
// Replace edge rows for a file. When `manager` is provided, runs inside that transaction;
|
|
93
|
-
// otherwise opens its own. Keeps delete + insert atomic either way.
|
|
94
|
-
async function writeDepEdges(serviceName, relFilePath, edges, manager) {
|
|
95
|
-
const run = async (mgr) => {
|
|
96
|
-
await mgr.delete(db_1.CodeDependency, { project: serviceName, fromFile: relFilePath });
|
|
97
|
-
if (edges.length > 0) {
|
|
98
|
-
await mgr.createQueryBuilder().insert().into(db_1.CodeDependency).values(edges).execute();
|
|
99
|
-
}
|
|
100
|
-
};
|
|
101
|
-
if (manager)
|
|
102
|
-
await run(manager);
|
|
103
|
-
else
|
|
104
|
-
await db_1.db.transaction(run);
|
|
105
|
-
}
|
|
106
|
-
// Ingest a single file: skip if unchanged, otherwise delete old chunks and re-index.
|
|
107
|
-
// Returns status and chunk count. Never throws — failures are caught and reported.
|
|
108
|
-
async function ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
|
|
109
|
-
try {
|
|
110
|
-
const repo = db_1.db.getRepository(db_1.CodeChunk);
|
|
111
|
-
const relFilePath = path.relative(config_1.config.workspaceRoot, filePath);
|
|
112
|
-
// Incremental check: if mtime matches what's stored, the file hasn't changed
|
|
113
|
-
const existing = await repo.findOne({
|
|
114
|
-
select: ['id', 'lastModified'],
|
|
115
|
-
where: { serviceName, filePath: relFilePath },
|
|
116
|
-
});
|
|
117
|
-
if (existing?.lastModified === mtimeMs) {
|
|
118
|
-
return { status: 'skipped', chunks: 0 };
|
|
119
|
-
}
|
|
120
|
-
// Extract import edges before the transaction — disk I/O must not hold a DB connection open
|
|
121
|
-
const edges = await collectDepEdges(filePath, serviceName, projectRoot);
|
|
122
|
-
// Read and embed outside the transaction — Ollama calls must not hold a DB transaction open
|
|
123
|
-
const code = fs.readFileSync(filePath, 'utf-8');
|
|
124
|
-
const chunks = await getSplitter(filePath).createDocuments([code]);
|
|
125
|
-
const relPath = path.relative(projectRoot, filePath); // project-relative for embedding header
|
|
126
|
-
const records = [];
|
|
127
|
-
for (let i = 0; i < chunks.length; i += config_1.config.indexer.batchSize) {
|
|
128
|
-
const batch = chunks.slice(i, i + config_1.config.indexer.batchSize);
|
|
129
|
-
const embedded = await Promise.all(batch.map(async (chunk) => {
|
|
130
|
-
const startLine = (chunk.metadata?.loc?.lines?.from ?? 1);
|
|
131
|
-
const prompt = `Project: ${serviceName} | Path: ${relPath}\n${chunk.pageContent}`;
|
|
132
|
-
const embedding = await embedText(prompt);
|
|
133
|
-
return repo.create({
|
|
134
|
-
serviceName, filePath: relFilePath, startLine,
|
|
135
|
-
content: chunk.pageContent,
|
|
136
|
-
embedding: `[${embedding}]`,
|
|
137
|
-
lastModified: mtimeMs,
|
|
138
|
-
});
|
|
139
|
-
}));
|
|
140
|
-
records.push(...embedded);
|
|
141
|
-
}
|
|
142
|
-
// Atomic replace: delete stale chunks + dependency edges, insert new ones in a single transaction.
|
|
143
|
-
// Prevents partial index if the process crashes mid-write.
|
|
144
|
-
await db_1.db.transaction(async (manager) => {
|
|
145
|
-
await manager.delete(db_1.CodeChunk, { serviceName, filePath: relFilePath });
|
|
146
|
-
for (let i = 0; i < records.length; i += config_1.config.indexer.batchSize) {
|
|
147
|
-
await manager.createQueryBuilder()
|
|
148
|
-
.insert()
|
|
149
|
-
.into(db_1.CodeChunk)
|
|
150
|
-
.values(records.slice(i, i + config_1.config.indexer.batchSize))
|
|
151
|
-
.execute();
|
|
152
|
-
}
|
|
153
|
-
await writeDepEdges(serviceName, relFilePath, edges, manager);
|
|
154
|
-
});
|
|
155
|
-
return { status: 'indexed', chunks: chunks.length };
|
|
156
|
-
}
|
|
157
|
-
catch (err) {
|
|
158
|
-
console.error(`\n[indexer] Failed: ${filePath} — ${err.message}`);
|
|
159
|
-
return { status: 'failed', chunks: 0 };
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
// Index a project directory incrementally.
|
|
163
|
-
// Removes chunks for deleted files, skips unchanged files, re-indexes modified ones.
|
|
164
|
-
// Pass force=true to invalidate all cached mtimes and force full re-index (e.g. after embedding strategy change).
|
|
165
|
-
// Pass signal to support graceful interruption — stops between files, never mid-transaction.
|
|
166
|
-
async function scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles) {
|
|
167
|
-
const startTime = Date.now();
|
|
168
|
-
const repo = db_1.db.getRepository(db_1.CodeChunk);
|
|
169
|
-
// Force mode: wipe lastModified so all files are treated as changed
|
|
170
|
-
if (force) {
|
|
171
|
-
await db_1.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
172
|
-
}
|
|
173
|
-
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
174
|
-
// Clean up chunks for files no longer in the collected set — covers both deleted files and
|
|
175
|
-
// files newly excluded by .ragignore (they exist on disk but should no longer be indexed).
|
|
176
|
-
const collectedRelPaths = new Set(files.map(f => path.relative(config_1.config.workspaceRoot, f)));
|
|
177
|
-
const dbPaths = await db_1.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
|
|
178
|
-
for (const { filePath } of dbPaths) {
|
|
179
|
-
if (!collectedRelPaths.has(filePath)) {
|
|
180
|
-
await repo.delete({ serviceName, filePath });
|
|
181
|
-
await db_1.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
|
|
182
|
-
await db_1.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
|
|
183
|
-
console.log(` Removed stale: ${path.basename(filePath)}`);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
const bar = new cli_progress_1.SingleBar({
|
|
187
|
-
format: `{serviceName} {bar} {percentage}% | {value}/{total} | ETA {eta_formatted} | {filename}`,
|
|
188
|
-
barCompleteChar: '█',
|
|
189
|
-
barIncompleteChar: '░',
|
|
190
|
-
hideCursor: true,
|
|
191
|
-
clearOnComplete: false,
|
|
192
|
-
}, cli_progress_1.Presets.shades_grey);
|
|
193
|
-
const warnings = [];
|
|
194
|
-
console.log(`\nIndexing ${serviceName}: ${files.length} files`);
|
|
195
|
-
const meta = (0, file_utils_1.readRagMeta)(dir);
|
|
196
|
-
if (meta === null && fs.existsSync(path.join(dir, '.ragmeta.json'))) {
|
|
197
|
-
const msg = `.ragmeta.json found but invalid JSON`;
|
|
198
|
-
console.log(` ⚠ ${msg}`);
|
|
199
|
-
warnings.push(msg);
|
|
200
|
-
}
|
|
201
|
-
else if (meta) {
|
|
202
|
-
const parts = [meta.description, meta.stack && `[${meta.stack}]`].filter(Boolean).join(' ');
|
|
203
|
-
console.log(` ✓ .ragmeta.json: ${parts || '(no fields)'}`);
|
|
204
|
-
}
|
|
205
|
-
else {
|
|
206
|
-
const msg = `No .ragmeta.json — add {"description":"...","stack":"...","notes":"..."} so list_projects can describe this project`;
|
|
207
|
-
console.log(` ⚠ ${msg}`);
|
|
208
|
-
warnings.push(msg);
|
|
209
|
-
}
|
|
210
|
-
if (!fs.existsSync(path.join(dir, '.ragignore'))) {
|
|
211
|
-
const msg = `No .ragignore — create one to exclude generated/codegen files (e.g. src/gql/graphql.ts, **/__snapshots__/**)`;
|
|
212
|
-
console.log(` ⚠ ${msg}`);
|
|
213
|
-
warnings.push(msg);
|
|
214
|
-
}
|
|
215
|
-
bar.start(files.length, 0, { serviceName, filename: '' });
|
|
216
|
-
const stats = {
|
|
217
|
-
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
|
|
218
|
-
};
|
|
219
|
-
for (const filePath of files) {
|
|
220
|
-
if (signal?.aborted) {
|
|
221
|
-
bar.stop();
|
|
222
|
-
console.log(`\n Interrupted — ${serviceName}`);
|
|
223
|
-
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
224
|
-
return stats;
|
|
225
|
-
}
|
|
226
|
-
bar.update({ filename: path.basename(filePath) });
|
|
227
|
-
const mtimeMs = fs.statSync(filePath).mtimeMs;
|
|
228
|
-
const result = await ingestFile(filePath, serviceName, mtimeMs, dir);
|
|
229
|
-
if (result.status === 'indexed')
|
|
230
|
-
stats.indexed++;
|
|
231
|
-
else if (result.status === 'skipped')
|
|
232
|
-
stats.skipped++;
|
|
233
|
-
else
|
|
234
|
-
stats.failed++;
|
|
235
|
-
stats.chunks += result.chunks;
|
|
236
|
-
bar.increment();
|
|
237
|
-
}
|
|
238
|
-
bar.stop();
|
|
239
|
-
process.stdout.write('\x1B[J'); // erase any blank lines cli-progress left below the bar
|
|
240
|
-
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
241
|
-
return stats;
|
|
242
|
-
}
|
package/dist/src/mcp.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function startMCP(): Promise<void>;
|