@ixxrllg/localmind 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +290 -0
- package/dist/ask/answer.d.ts +9 -0
- package/dist/ask/answer.d.ts.map +1 -0
- package/dist/ask/answer.js +183 -0
- package/dist/ask/answer.js.map +1 -0
- package/dist/bin/localmind.d.ts +3 -0
- package/dist/bin/localmind.d.ts.map +1 -0
- package/dist/bin/localmind.js +11 -0
- package/dist/bin/localmind.js.map +1 -0
- package/dist/cli/format.d.ts +7 -0
- package/dist/cli/format.d.ts.map +1 -0
- package/dist/cli/format.js +67 -0
- package/dist/cli/format.js.map +1 -0
- package/dist/cli/program.d.ts +4 -0
- package/dist/cli/program.d.ts.map +1 -0
- package/dist/cli/program.js +228 -0
- package/dist/cli/program.js.map +1 -0
- package/dist/core/config.d.ts +17 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +55 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/db.d.ts +50 -0
- package/dist/core/db.d.ts.map +1 -0
- package/dist/core/db.js +180 -0
- package/dist/core/db.js.map +1 -0
- package/dist/core/hash.d.ts +3 -0
- package/dist/core/hash.d.ts.map +1 -0
- package/dist/core/hash.js +8 -0
- package/dist/core/hash.js.map +1 -0
- package/dist/core/logger.d.ts +12 -0
- package/dist/core/logger.d.ts.map +1 -0
- package/dist/core/logger.js +45 -0
- package/dist/core/logger.js.map +1 -0
- package/dist/core/paths.d.ts +16 -0
- package/dist/core/paths.d.ts.map +1 -0
- package/dist/core/paths.js +43 -0
- package/dist/core/paths.js.map +1 -0
- package/dist/graph/related.d.ts +11 -0
- package/dist/graph/related.d.ts.map +1 -0
- package/dist/graph/related.js +144 -0
- package/dist/graph/related.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/chunker.d.ts +16 -0
- package/dist/indexer/chunker.d.ts.map +1 -0
- package/dist/indexer/chunker.js +76 -0
- package/dist/indexer/chunker.js.map +1 -0
- package/dist/indexer/ignore.d.ts +6 -0
- package/dist/indexer/ignore.d.ts.map +1 -0
- package/dist/indexer/ignore.js +71 -0
- package/dist/indexer/ignore.js.map +1 -0
- package/dist/indexer/indexer.d.ts +11 -0
- package/dist/indexer/indexer.d.ts.map +1 -0
- package/dist/indexer/indexer.js +182 -0
- package/dist/indexer/indexer.js.map +1 -0
- package/dist/indexer/languages.d.ts +5 -0
- package/dist/indexer/languages.d.ts.map +1 -0
- package/dist/indexer/languages.js +144 -0
- package/dist/indexer/languages.js.map +1 -0
- package/dist/indexer/relations.d.ts +9 -0
- package/dist/indexer/relations.d.ts.map +1 -0
- package/dist/indexer/relations.js +81 -0
- package/dist/indexer/relations.js.map +1 -0
- package/dist/indexer/walker.d.ts +21 -0
- package/dist/indexer/walker.d.ts.map +1 -0
- package/dist/indexer/walker.js +86 -0
- package/dist/indexer/walker.js.map +1 -0
- package/dist/mcp/server.d.ts +5 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +291 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/search/bm25.d.ts +9 -0
- package/dist/search/bm25.d.ts.map +1 -0
- package/dist/search/bm25.js +109 -0
- package/dist/search/bm25.js.map +1 -0
- package/dist/search/embeddings.d.ts +15 -0
- package/dist/search/embeddings.d.ts.map +1 -0
- package/dist/search/embeddings.js +193 -0
- package/dist/search/embeddings.js.map +1 -0
- package/dist/search/search.d.ts +9 -0
- package/dist/search/search.d.ts.map +1 -0
- package/dist/search/search.js +47 -0
- package/dist/search/search.js.map +1 -0
- package/dist/search/tokenize.d.ts +6 -0
- package/dist/search/tokenize.d.ts.map +1 -0
- package/dist/search/tokenize.js +163 -0
- package/dist/search/tokenize.js.map +1 -0
- package/dist/status.d.ts +4 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +33 -0
- package/dist/status.js.map +1 -0
- package/dist/types.d.ts +105 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +11 -0
- package/dist/types.js.map +1 -0
- package/package.json +72 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { clearChunksForFile, closeDb, deleteFileByPath, getFileByPath, insertChunk, insertEdge, insertPosting, getOrCreateTermId, listAllFilePaths, openDb, pruneOrphanTerms, upsertFile, withTransaction, } from "../core/db.js";
|
|
3
|
+
import { sha256 } from "../core/hash.js";
|
|
4
|
+
import { log } from "../core/logger.js";
|
|
5
|
+
import { chunkText } from "./chunker.js";
|
|
6
|
+
import { buildIgnoreFilter } from "./ignore.js";
|
|
7
|
+
import { detectLanguage } from "./languages.js";
|
|
8
|
+
import { extractRelations } from "./relations.js";
|
|
9
|
+
import { looksBinary, walkFiles } from "./walker.js";
|
|
10
|
+
import { termFrequencies, tokenize } from "../search/tokenize.js";
|
|
11
|
+
import { createEmbedder, storeEmbeddingsForChunks, } from "../search/embeddings.js";
|
|
12
|
+
export async function runIndex(config, options = {}) {
|
|
13
|
+
const started = Date.now();
|
|
14
|
+
const db = openDb(config.root);
|
|
15
|
+
const full = options.full ?? !config.incremental;
|
|
16
|
+
const ig = buildIgnoreFilter(config.root, config.exclude);
|
|
17
|
+
const walked = walkFiles({
|
|
18
|
+
root: config.root,
|
|
19
|
+
ig,
|
|
20
|
+
include: config.include,
|
|
21
|
+
maxFileBytes: config.maxFileBytes,
|
|
22
|
+
});
|
|
23
|
+
let filesIndexed = 0;
|
|
24
|
+
let filesSkipped = 0;
|
|
25
|
+
let chunksCreated = 0;
|
|
26
|
+
let filesRemoved = 0;
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
const embeddingQueue = [];
|
|
29
|
+
for (const file of walked) {
|
|
30
|
+
seen.add(file.relPath);
|
|
31
|
+
let content;
|
|
32
|
+
try {
|
|
33
|
+
const buf = fs.readFileSync(file.absPath);
|
|
34
|
+
if (looksBinary(buf)) {
|
|
35
|
+
filesSkipped++;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
content = buf.toString("utf8");
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
filesSkipped++;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
// Skip if content has too many replacement characters / non-text
|
|
45
|
+
if ((content.match(/\uFFFD/g) ?? []).length > 10) {
|
|
46
|
+
filesSkipped++;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const contentHash = sha256(content);
|
|
50
|
+
const existing = getFileByPath(db, file.relPath);
|
|
51
|
+
if (!full && existing && existing.content_hash === contentHash) {
|
|
52
|
+
filesSkipped++;
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const language = detectLanguage(file.relPath);
|
|
56
|
+
const lineCount = content.split(/\r?\n/).length;
|
|
57
|
+
const chunks = chunkText(content, config.chunkSize, config.chunkOverlap);
|
|
58
|
+
const relations = extractRelations(content, language);
|
|
59
|
+
const indexedAt = new Date().toISOString();
|
|
60
|
+
withTransaction(db, () => {
|
|
61
|
+
const fileId = upsertFile(db, {
|
|
62
|
+
path: file.relPath,
|
|
63
|
+
absPath: file.absPath,
|
|
64
|
+
language,
|
|
65
|
+
size: file.size,
|
|
66
|
+
mtimeMs: file.mtimeMs,
|
|
67
|
+
contentHash,
|
|
68
|
+
lineCount,
|
|
69
|
+
indexedAt,
|
|
70
|
+
});
|
|
71
|
+
clearChunksForFile(db, fileId);
|
|
72
|
+
for (const chunk of chunks) {
|
|
73
|
+
const chunkId = insertChunk(db, {
|
|
74
|
+
fileId,
|
|
75
|
+
chunkIndex: chunk.chunkIndex,
|
|
76
|
+
startLine: chunk.startLine,
|
|
77
|
+
endLine: chunk.endLine,
|
|
78
|
+
content: chunk.content,
|
|
79
|
+
contentHash: chunk.contentHash,
|
|
80
|
+
tokenCount: chunk.tokenCount,
|
|
81
|
+
});
|
|
82
|
+
chunksCreated++;
|
|
83
|
+
const tf = termFrequencies(tokenize(chunk.content));
|
|
84
|
+
for (const [term, freq] of tf) {
|
|
85
|
+
const termId = getOrCreateTermId(db, term);
|
|
86
|
+
insertPosting(db, termId, chunkId, freq);
|
|
87
|
+
}
|
|
88
|
+
embeddingQueue.push({ chunkId, content: chunk.content });
|
|
89
|
+
}
|
|
90
|
+
for (const rel of relations) {
|
|
91
|
+
insertEdge(db, fileId, rel.target, rel.relation);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
filesIndexed++;
|
|
95
|
+
if (!options.quiet && filesIndexed % 25 === 0) {
|
|
96
|
+
log.info(`Indexed ${filesIndexed} files…`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
// Remove deleted files from index
|
|
100
|
+
const known = listAllFilePaths(db);
|
|
101
|
+
for (const path of known) {
|
|
102
|
+
if (!seen.has(path)) {
|
|
103
|
+
deleteFileByPath(db, path);
|
|
104
|
+
filesRemoved++;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
pruneOrphanTerms(db);
|
|
108
|
+
let embeddingsStored = 0;
|
|
109
|
+
if (options.withEmbeddings !== false) {
|
|
110
|
+
const embedder = createEmbedder(config.provider);
|
|
111
|
+
if (embedder && embeddingQueue.length) {
|
|
112
|
+
if (!options.quiet) {
|
|
113
|
+
log.info(`Computing embeddings with ${embedder.provider}/${embedder.model}…`);
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
// Only embed newly processed chunks
|
|
117
|
+
embeddingsStored = await storeEmbeddingsForChunks(db, embedder, embeddingQueue);
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
log.warn(`Embedding step failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
closeDb(db);
|
|
125
|
+
const stats = {
|
|
126
|
+
filesScanned: walked.length,
|
|
127
|
+
filesIndexed,
|
|
128
|
+
filesSkipped,
|
|
129
|
+
filesRemoved,
|
|
130
|
+
chunksCreated,
|
|
131
|
+
durationMs: Date.now() - started,
|
|
132
|
+
};
|
|
133
|
+
if (!options.quiet && embeddingsStored > 0) {
|
|
134
|
+
log.info(`Stored ${embeddingsStored} embeddings`);
|
|
135
|
+
}
|
|
136
|
+
return stats;
|
|
137
|
+
}
|
|
138
|
+
/** Index a single relative path (used by tests / future watch mode). */
|
|
139
|
+
export function indexSingleFile(db, config, absPath, relPath) {
|
|
140
|
+
const buf = fs.readFileSync(absPath);
|
|
141
|
+
if (looksBinary(buf))
|
|
142
|
+
return 0;
|
|
143
|
+
const content = buf.toString("utf8");
|
|
144
|
+
const language = detectLanguage(relPath);
|
|
145
|
+
const stat = fs.statSync(absPath);
|
|
146
|
+
const chunks = chunkText(content, config.chunkSize, config.chunkOverlap);
|
|
147
|
+
let count = 0;
|
|
148
|
+
withTransaction(db, () => {
|
|
149
|
+
const fileId = upsertFile(db, {
|
|
150
|
+
path: relPath,
|
|
151
|
+
absPath,
|
|
152
|
+
language,
|
|
153
|
+
size: stat.size,
|
|
154
|
+
mtimeMs: Math.floor(stat.mtimeMs),
|
|
155
|
+
contentHash: sha256(content),
|
|
156
|
+
lineCount: content.split(/\r?\n/).length,
|
|
157
|
+
indexedAt: new Date().toISOString(),
|
|
158
|
+
});
|
|
159
|
+
clearChunksForFile(db, fileId);
|
|
160
|
+
for (const chunk of chunks) {
|
|
161
|
+
const chunkId = insertChunk(db, {
|
|
162
|
+
fileId,
|
|
163
|
+
chunkIndex: chunk.chunkIndex,
|
|
164
|
+
startLine: chunk.startLine,
|
|
165
|
+
endLine: chunk.endLine,
|
|
166
|
+
content: chunk.content,
|
|
167
|
+
contentHash: chunk.contentHash,
|
|
168
|
+
tokenCount: chunk.tokenCount,
|
|
169
|
+
});
|
|
170
|
+
count++;
|
|
171
|
+
const tf = termFrequencies(tokenize(chunk.content));
|
|
172
|
+
for (const [term, freq] of tf) {
|
|
173
|
+
insertPosting(db, getOrCreateTermId(db, term), chunkId, freq);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
for (const rel of extractRelations(content, language)) {
|
|
177
|
+
insertEdge(db, fileId, rel.target, rel.relation);
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
return count;
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,OAAO,EACP,gBAAgB,EAChB,aAAa,EACb,WAAW,EACX,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,MAAM,EACN,gBAAgB,EAChB,UAAU,EACV,eAAe,GAEhB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EACL,cAAc,EACd,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AAQjC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAuB,EACvB,UAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;IAEjD,MAAM,EAAE,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,EAAE;QACF,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;KAClC,CAAC,CAAC;IAEH,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,cAAc,GAAgD,EAAE,CAAC;IAEvE,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,YAAY,EAAE,CAAC;gBACf,SAAS;YACX,CAAC;YACD,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACjD,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjD,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,QAAQ,CAAC,YAAY,KAAK,WAAW,EAAE,CAAC;YAC/D,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAChD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,eAAe,CAAC,EAAE,EAAE,GAAG,EAAE;YACvB,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,EAAE;gBAC5B,IAAI,EAAE,IAAI,CAAC,OAAO;gBAClB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ;gBACR,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,WAAW;gBACX,SAAS;gBACT,SAAS;aACV,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,EAAE;oBAC9B,MAAM;oBACN,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;iBAC7B,CAAC,CAAC;gBACH,aAAa,EAAE,CAAC;gBAEhB,MAAM,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC9B,MAAM,MAAM,GAAG,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBAC3C,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC3C,CAAC;gBAED,cAAc,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,YAAY,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,YAAY,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,WAAW,YAAY,SAAS,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,MAAM,KAAK,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAC3B,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAErB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,OAAO,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,QAAQ,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,GAAG,CAAC,IAAI,CACN,6BAA6B,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,GAAG,CACpE,CAAC;YACJ,CAAC;YACD,IAAI,CAAC;gBACH,oCAAoC;gBACpC,gBAAgB,GAAG,MAAM,wBAAwB,CAC/C,EAAE,EACF,QAAQ,EACR,cAAc,CACf,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CACN,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC7E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,EAAE,CAAC,CAAC;IAEZ,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,YAAY;QACZ,YAAY;QACZ,YAAY;QACZ,aAAa;QACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KACjC,CAAC;IAEF,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CAAC,UAAU,gBAAgB,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,eAAe,CAC7B,EAAM,EACN,MAAuB,EACvB,OAAe,EACf,OAAe;IAEf,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,WAAW,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACzE,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,eAAe,CAAC,EAAE,EAAE,GAAG,EAAE;QACvB,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,EAAE;YAC5B,IAAI,EAAE,OAAO;YACb,OAAO;YACP,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;YACjC,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC;YAC5B,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM;YACxC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QACH,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,EAAE;gBAC9B,MAAM;gBACN,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;aAC7B,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACpD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,aAAa,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;YACtD,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare function detectLanguage(filePath: string): string;
|
|
2
|
+
export declare function isIndexablePath(filePath: string): boolean;
|
|
3
|
+
/** Binary / junk extensions we never open. */
|
|
4
|
+
export declare const BINARY_EXTENSIONS: Set<string>;
|
|
5
|
+
//# sourceMappingURL=languages.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"languages.d.ts","sourceRoot":"","sources":["../../src/indexer/languages.ts"],"names":[],"mappings":"AAmFA,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CASvD;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAOzD;AAED,8CAA8C;AAC9C,eAAO,MAAM,iBAAiB,aAsC5B,CAAC"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
const EXT_LANG = {
|
|
3
|
+
".ts": "typescript",
|
|
4
|
+
".tsx": "typescript",
|
|
5
|
+
".mts": "typescript",
|
|
6
|
+
".cts": "typescript",
|
|
7
|
+
".js": "javascript",
|
|
8
|
+
".jsx": "javascript",
|
|
9
|
+
".mjs": "javascript",
|
|
10
|
+
".cjs": "javascript",
|
|
11
|
+
".py": "python",
|
|
12
|
+
".rs": "rust",
|
|
13
|
+
".go": "go",
|
|
14
|
+
".java": "java",
|
|
15
|
+
".kt": "kotlin",
|
|
16
|
+
".kts": "kotlin",
|
|
17
|
+
".cs": "csharp",
|
|
18
|
+
".cpp": "cpp",
|
|
19
|
+
".cc": "cpp",
|
|
20
|
+
".cxx": "cpp",
|
|
21
|
+
".c": "c",
|
|
22
|
+
".h": "c",
|
|
23
|
+
".hpp": "cpp",
|
|
24
|
+
".rb": "ruby",
|
|
25
|
+
".php": "php",
|
|
26
|
+
".swift": "swift",
|
|
27
|
+
".scala": "scala",
|
|
28
|
+
".sh": "shell",
|
|
29
|
+
".bash": "shell",
|
|
30
|
+
".zsh": "shell",
|
|
31
|
+
".ps1": "powershell",
|
|
32
|
+
".sql": "sql",
|
|
33
|
+
".md": "markdown",
|
|
34
|
+
".mdx": "markdown",
|
|
35
|
+
".json": "json",
|
|
36
|
+
".jsonc": "json",
|
|
37
|
+
".yaml": "yaml",
|
|
38
|
+
".yml": "yaml",
|
|
39
|
+
".toml": "toml",
|
|
40
|
+
".ini": "ini",
|
|
41
|
+
".env": "dotenv",
|
|
42
|
+
".html": "html",
|
|
43
|
+
".css": "css",
|
|
44
|
+
".scss": "scss",
|
|
45
|
+
".less": "less",
|
|
46
|
+
".vue": "vue",
|
|
47
|
+
".svelte": "svelte",
|
|
48
|
+
".xml": "xml",
|
|
49
|
+
".graphql": "graphql",
|
|
50
|
+
".gql": "graphql",
|
|
51
|
+
".proto": "protobuf",
|
|
52
|
+
".dockerfile": "dockerfile",
|
|
53
|
+
".txt": "text",
|
|
54
|
+
".csv": "csv",
|
|
55
|
+
".r": "r",
|
|
56
|
+
".lua": "lua",
|
|
57
|
+
".zig": "zig",
|
|
58
|
+
".ex": "elixir",
|
|
59
|
+
".exs": "elixir",
|
|
60
|
+
".erl": "erlang",
|
|
61
|
+
".hs": "haskell",
|
|
62
|
+
".clj": "clojure",
|
|
63
|
+
".dart": "dart",
|
|
64
|
+
};
|
|
65
|
+
const INDEXABLE_EXT = new Set(Object.keys(EXT_LANG));
|
|
66
|
+
/** Special filenames without useful extensions. */
|
|
67
|
+
const SPECIAL_NAMES = {
|
|
68
|
+
dockerfile: "dockerfile",
|
|
69
|
+
makefile: "makefile",
|
|
70
|
+
gemfile: "ruby",
|
|
71
|
+
rakefile: "ruby",
|
|
72
|
+
procfile: "text",
|
|
73
|
+
license: "text",
|
|
74
|
+
readme: "markdown",
|
|
75
|
+
changelog: "markdown",
|
|
76
|
+
contributing: "markdown",
|
|
77
|
+
authors: "text",
|
|
78
|
+
"cmakelists.txt": "cmake",
|
|
79
|
+
};
|
|
80
|
+
export function detectLanguage(filePath) {
|
|
81
|
+
const base = path.basename(filePath).toLowerCase();
|
|
82
|
+
if (SPECIAL_NAMES[base])
|
|
83
|
+
return SPECIAL_NAMES[base];
|
|
84
|
+
// README.md etc. already handled by extension
|
|
85
|
+
const ext = path.extname(base).toLowerCase();
|
|
86
|
+
if (EXT_LANG[ext])
|
|
87
|
+
return EXT_LANG[ext];
|
|
88
|
+
// e.g. .eslintrc.json already covered; bare .gitignore
|
|
89
|
+
if (base.startsWith(".") && !ext)
|
|
90
|
+
return "config";
|
|
91
|
+
return "text";
|
|
92
|
+
}
|
|
93
|
+
export function isIndexablePath(filePath) {
|
|
94
|
+
const base = path.basename(filePath).toLowerCase();
|
|
95
|
+
if (SPECIAL_NAMES[base])
|
|
96
|
+
return true;
|
|
97
|
+
if (base === "dockerfile" || base.startsWith("dockerfile."))
|
|
98
|
+
return true;
|
|
99
|
+
if (base.endsWith(".md") || base.endsWith(".mdx"))
|
|
100
|
+
return true;
|
|
101
|
+
const ext = path.extname(base).toLowerCase();
|
|
102
|
+
return INDEXABLE_EXT.has(ext);
|
|
103
|
+
}
|
|
104
|
+
/** Binary / junk extensions we never open. */
|
|
105
|
+
export const BINARY_EXTENSIONS = new Set([
|
|
106
|
+
".png",
|
|
107
|
+
".jpg",
|
|
108
|
+
".jpeg",
|
|
109
|
+
".gif",
|
|
110
|
+
".webp",
|
|
111
|
+
".ico",
|
|
112
|
+
".bmp",
|
|
113
|
+
".svg",
|
|
114
|
+
".pdf",
|
|
115
|
+
".zip",
|
|
116
|
+
".gz",
|
|
117
|
+
".tar",
|
|
118
|
+
".7z",
|
|
119
|
+
".rar",
|
|
120
|
+
".exe",
|
|
121
|
+
".dll",
|
|
122
|
+
".so",
|
|
123
|
+
".dylib",
|
|
124
|
+
".bin",
|
|
125
|
+
".o",
|
|
126
|
+
".a",
|
|
127
|
+
".class",
|
|
128
|
+
".jar",
|
|
129
|
+
".war",
|
|
130
|
+
".pyc",
|
|
131
|
+
".pyo",
|
|
132
|
+
".wasm",
|
|
133
|
+
".woff",
|
|
134
|
+
".woff2",
|
|
135
|
+
".ttf",
|
|
136
|
+
".eot",
|
|
137
|
+
".mp3",
|
|
138
|
+
".mp4",
|
|
139
|
+
".avi",
|
|
140
|
+
".mov",
|
|
141
|
+
".webm",
|
|
142
|
+
".lock",
|
|
143
|
+
]);
|
|
144
|
+
//# sourceMappingURL=languages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"languages.js","sourceRoot":"","sources":["../../src/indexer/languages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,QAAQ,GAA2B;IACvC,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,OAAO;IACd,OAAO,EAAE,OAAO;IAChB,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,MAAM;IAChB,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,QAAQ;IAChB,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,SAAS;IACrB,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,UAAU;IACpB,aAAa,EAAE,YAAY;IAC3B,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAErD,mDAAmD;AACnD,MAAM,aAAa,GAA2B;IAC5C,UAAU,EAAE,YAAY;IACxB,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,MAAM;IAChB,QAAQ,EAAE,MAAM;IAChB,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,UAAU;IAClB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,UAAU;IACxB,OAAO,EAAE,MAAM;IACf,gBAAgB,EAAE,OAAO;CAC1B,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;IACpD,8CAA8C;IAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,IAAI,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;IACxC,uDAAuD;IACvD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAClD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IACzE,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,OAAO,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,8CAA8C;AAC9C,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACvC,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;CACR,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract lightweight dependency edges from source (imports / requires).
|
|
3
|
+
* Best-effort, language-agnostic regexes — good enough for a knowledge graph seed.
|
|
4
|
+
*/
|
|
5
|
+
export declare function extractRelations(content: string, language: string): Array<{
|
|
6
|
+
target: string;
|
|
7
|
+
relation: string;
|
|
8
|
+
}>;
|
|
9
|
+
//# sourceMappingURL=relations.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relations.d.ts","sourceRoot":"","sources":["../../src/indexer/relations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAiE7C"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract lightweight dependency edges from source (imports / requires).
|
|
3
|
+
* Best-effort, language-agnostic regexes — good enough for a knowledge graph seed.
|
|
4
|
+
*/
|
|
5
|
+
export function extractRelations(content, language) {
|
|
6
|
+
const edges = [];
|
|
7
|
+
const seen = new Set();
|
|
8
|
+
const add = (target, relation) => {
|
|
9
|
+
const t = normalizeTarget(target);
|
|
10
|
+
if (!t)
|
|
11
|
+
return;
|
|
12
|
+
const key = `${relation}:${t}`;
|
|
13
|
+
if (seen.has(key))
|
|
14
|
+
return;
|
|
15
|
+
seen.add(key);
|
|
16
|
+
edges.push({ target: t, relation });
|
|
17
|
+
};
|
|
18
|
+
// JS/TS: import x from 'y' / require('y') / export ... from 'y'
|
|
19
|
+
if (language === "javascript" || language === "typescript" || language === "vue" || language === "svelte") {
|
|
20
|
+
const fromRe = /(?:import|export)\s+(?:type\s+)?(?:[\s\S]*?\s+from\s+|)\s*['"]([^'"]+)['"]/g;
|
|
21
|
+
const requireRe = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
|
|
22
|
+
const dynamicRe = /import\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
|
|
23
|
+
let m;
|
|
24
|
+
while ((m = fromRe.exec(content)))
|
|
25
|
+
add(m[1], "import");
|
|
26
|
+
while ((m = requireRe.exec(content)))
|
|
27
|
+
add(m[1], "import");
|
|
28
|
+
while ((m = dynamicRe.exec(content)))
|
|
29
|
+
add(m[1], "import");
|
|
30
|
+
}
|
|
31
|
+
// Python
|
|
32
|
+
if (language === "python") {
|
|
33
|
+
const importRe = /(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))/g;
|
|
34
|
+
let m;
|
|
35
|
+
while ((m = importRe.exec(content))) {
|
|
36
|
+
add((m[1] || m[2]), "import");
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Go
|
|
40
|
+
if (language === "go") {
|
|
41
|
+
const goRe = /import\s+(?:\w+\s+)?["']([^"']+)["']/g;
|
|
42
|
+
let m;
|
|
43
|
+
while ((m = goRe.exec(content)))
|
|
44
|
+
add(m[1], "import");
|
|
45
|
+
// multi-import blocks roughly handled by same pattern on each line
|
|
46
|
+
}
|
|
47
|
+
// Rust
|
|
48
|
+
if (language === "rust") {
|
|
49
|
+
const useRe = /use\s+((?:crate|super|self)?(?:::\w+)+)/g;
|
|
50
|
+
let m;
|
|
51
|
+
while ((m = useRe.exec(content)))
|
|
52
|
+
add(m[1], "import");
|
|
53
|
+
const modRe = /mod\s+(\w+)\s*;/g;
|
|
54
|
+
while ((m = modRe.exec(content)))
|
|
55
|
+
add(m[1], "mod");
|
|
56
|
+
}
|
|
57
|
+
// Markdown links to local files
|
|
58
|
+
if (language === "markdown") {
|
|
59
|
+
const mdRe = /\[[^\]]*]\(([^)]+)\)/g;
|
|
60
|
+
let m;
|
|
61
|
+
while ((m = mdRe.exec(content))) {
|
|
62
|
+
const target = m[1];
|
|
63
|
+
if (target.startsWith("http") || target.startsWith("#") || target.startsWith("mailto:")) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
add(target.split("#")[0], "link");
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return edges;
|
|
70
|
+
}
|
|
71
|
+
function normalizeTarget(raw) {
|
|
72
|
+
let t = raw.trim().replace(/\\/g, "/");
|
|
73
|
+
if (!t || t.length > 260)
|
|
74
|
+
return null;
|
|
75
|
+
// strip query/hash
|
|
76
|
+
t = t.split("?")[0]?.split("#")[0] ?? t;
|
|
77
|
+
if (!t)
|
|
78
|
+
return null;
|
|
79
|
+
return t;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=relations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relations.js","sourceRoot":"","sources":["../../src/indexer/relations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,QAAgB;IAEhB,MAAM,KAAK,GAAgD,EAAE,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,QAAgB,EAAE,EAAE;QAC/C,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,CAAC,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtC,CAAC,CAAC;IAEF,gEAAgE;IAChE,IAAI,QAAQ,KAAK,YAAY,IAAI,QAAQ,KAAK,YAAY,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1G,MAAM,MAAM,GACV,6EAA6E,CAAC;QAChF,MAAM,SAAS,GAAG,uCAAuC,CAAC;QAC1D,MAAM,SAAS,GAAG,sCAAsC,CAAC;QACzD,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;QACxD,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAED,SAAS;IACT,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,iDAAiD,CAAC;QACnE,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK;IACL,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,uCAAuC,CAAC;QACrD,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;QACtD,mEAAmE;IACrE,CAAC;IAED,OAAO;IACP,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,0CAA0C,CAAC;QACzD,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,kBAAkB,CAAC;QACjC,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAED,gCAAgC;IAChC,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,uBAAuB,CAAC;QACrC,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;YACrB,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxF,SAAS;YACX,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,EAAE,MAAM,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IACtC,mBAAmB;IACnB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Ignore } from "ignore";
|
|
2
|
+
export interface WalkedFile {
|
|
3
|
+
absPath: string;
|
|
4
|
+
relPath: string;
|
|
5
|
+
size: number;
|
|
6
|
+
mtimeMs: number;
|
|
7
|
+
}
|
|
8
|
+
export interface WalkOptions {
|
|
9
|
+
root: string;
|
|
10
|
+
ig: Ignore;
|
|
11
|
+
/** If set, only files under these relative prefixes are kept. */
|
|
12
|
+
include: string[];
|
|
13
|
+
maxFileBytes: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Depth-first walk of project files eligible for indexing.
|
|
17
|
+
*/
|
|
18
|
+
export declare function walkFiles(opts: WalkOptions): WalkedFile[];
|
|
19
|
+
/** Detect likely binary content (null bytes in sample). */
|
|
20
|
+
export declare function looksBinary(buf: Buffer): boolean;
|
|
21
|
+
//# sourceMappingURL=walker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"walker.d.ts","sourceRoot":"","sources":["../../src/indexer/walker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKrC,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,iEAAiE;IACjE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAUD;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,WAAW,GAAG,UAAU,EAAE,CAwDzD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAMhD"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { relativePosix, toPosix } from "../core/paths.js";
|
|
4
|
+
import { BINARY_EXTENSIONS, isIndexablePath } from "./languages.js";
|
|
5
|
+
import { shouldIgnore } from "./ignore.js";
|
|
6
|
+
function matchesInclude(relPath, include) {
|
|
7
|
+
if (!include.length)
|
|
8
|
+
return true;
|
|
9
|
+
return include.some((prefix) => {
|
|
10
|
+
const p = toPosix(prefix).replace(/^\.\//, "").replace(/\/$/, "");
|
|
11
|
+
return relPath === p || relPath.startsWith(p + "/");
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Depth-first walk of project files eligible for indexing.
|
|
16
|
+
*/
|
|
17
|
+
export function walkFiles(opts) {
|
|
18
|
+
const { root, ig, include, maxFileBytes } = opts;
|
|
19
|
+
const results = [];
|
|
20
|
+
function visit(dirAbs) {
|
|
21
|
+
let entries;
|
|
22
|
+
try {
|
|
23
|
+
entries = fs.readdirSync(dirAbs, { withFileTypes: true });
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
for (const entry of entries) {
|
|
29
|
+
const abs = path.join(dirAbs, entry.name);
|
|
30
|
+
const rel = relativePosix(root, abs);
|
|
31
|
+
if (!rel)
|
|
32
|
+
continue;
|
|
33
|
+
if (entry.isDirectory()) {
|
|
34
|
+
if (shouldIgnore(ig, rel, true))
|
|
35
|
+
continue;
|
|
36
|
+
// Skip known huge dirs even if gitignore missing
|
|
37
|
+
if (entry.name === "node_modules" || entry.name === ".git" || entry.name === ".localmind") {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
visit(abs);
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (!entry.isFile())
|
|
44
|
+
continue;
|
|
45
|
+
if (shouldIgnore(ig, rel, false))
|
|
46
|
+
continue;
|
|
47
|
+
if (!matchesInclude(rel, include))
|
|
48
|
+
continue;
|
|
49
|
+
const ext = path.extname(entry.name).toLowerCase();
|
|
50
|
+
if (BINARY_EXTENSIONS.has(ext))
|
|
51
|
+
continue;
|
|
52
|
+
if (!isIndexablePath(abs))
|
|
53
|
+
continue;
|
|
54
|
+
let stat;
|
|
55
|
+
try {
|
|
56
|
+
stat = fs.statSync(abs);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (stat.size > maxFileBytes)
|
|
62
|
+
continue;
|
|
63
|
+
if (stat.size === 0)
|
|
64
|
+
continue;
|
|
65
|
+
results.push({
|
|
66
|
+
absPath: abs,
|
|
67
|
+
relPath: rel,
|
|
68
|
+
size: stat.size,
|
|
69
|
+
mtimeMs: Math.floor(stat.mtimeMs),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
visit(root);
|
|
74
|
+
results.sort((a, b) => a.relPath.localeCompare(b.relPath));
|
|
75
|
+
return results;
|
|
76
|
+
}
|
|
77
|
+
/** Detect likely binary content (null bytes in sample). */
|
|
78
|
+
export function looksBinary(buf) {
|
|
79
|
+
const sample = buf.subarray(0, Math.min(buf.length, 8000));
|
|
80
|
+
for (let i = 0; i < sample.length; i++) {
|
|
81
|
+
if (sample[i] === 0)
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=walker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"walker.js","sourceRoot":"","sources":["../../src/indexer/walker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAiB3C,SAAS,cAAc,CAAC,OAAe,EAAE,OAAiB;IACxD,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QAC7B,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAClE,OAAO,OAAO,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAiB;IACzC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACjD,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,SAAS,KAAK,CAAC,MAAc;QAC3B,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,YAAY,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC;oBAAE,SAAS;gBAC1C,iDAAiD;gBACjD,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAC1F,SAAS;gBACX,CAAC;gBACD,KAAK,CAAC,GAAG,CAAC,CAAC;gBACX,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;gBAAE,SAAS;YAC9B,IAAI,YAAY,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC;gBAAE,SAAS;YAC3C,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,SAAS;YAE5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YACnD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACzC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEpC,IAAI,IAAc,CAAC;YACnB,IAAI,CAAC;gBACH,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,GAAG,YAAY;gBAAE,SAAS;YACvC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YAE9B,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,GAAG;gBACZ,OAAO,EAAE,GAAG;gBACZ,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,CAAC;IACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;IAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAqCA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,cAAc,CAAC,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAySlF"}
|