@jafreck/lore 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -196
- package/dist/cli.js +133 -12
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/indexer/db.d.ts.map +1 -1
- package/dist/indexer/db.js +96 -2
- package/dist/indexer/db.js.map +1 -1
- package/dist/indexer/docs.d.ts +42 -0
- package/dist/indexer/docs.d.ts.map +1 -0
- package/dist/indexer/docs.js +214 -0
- package/dist/indexer/docs.js.map +1 -0
- package/dist/indexer/embedder.d.ts +7 -0
- package/dist/indexer/embedder.d.ts.map +1 -1
- package/dist/indexer/embedder.js +10 -0
- package/dist/indexer/embedder.js.map +1 -1
- package/dist/indexer/extractors/types.d.ts +22 -0
- package/dist/indexer/extractors/types.d.ts.map +1 -1
- package/dist/indexer/extractors/types.js +12 -0
- package/dist/indexer/extractors/types.js.map +1 -1
- package/dist/indexer/extractors/typescript.d.ts +1 -1
- package/dist/indexer/extractors/typescript.d.ts.map +1 -1
- package/dist/indexer/extractors/typescript.js +38 -8
- package/dist/indexer/extractors/typescript.js.map +1 -1
- package/dist/indexer/git-hooks.d.ts +1 -0
- package/dist/indexer/git-hooks.d.ts.map +1 -1
- package/dist/indexer/git-hooks.js +3 -2
- package/dist/indexer/git-hooks.js.map +1 -1
- package/dist/indexer/index.d.ts +32 -7
- package/dist/indexer/index.d.ts.map +1 -1
- package/dist/indexer/index.js +427 -15
- package/dist/indexer/index.js.map +1 -1
- package/dist/indexer/lsp/client.d.ts +61 -0
- package/dist/indexer/lsp/client.d.ts.map +1 -0
- package/dist/indexer/lsp/client.js +217 -0
- package/dist/indexer/lsp/client.js.map +1 -0
- package/dist/indexer/lsp/config.d.ts +16 -0
- package/dist/indexer/lsp/config.d.ts.map +1 -0
- package/dist/indexer/lsp/config.js +78 -0
- package/dist/indexer/lsp/config.js.map +1 -0
- package/dist/indexer/lsp/enrichment.d.ts +55 -0
- package/dist/indexer/lsp/enrichment.d.ts.map +1 -0
- package/dist/indexer/lsp/enrichment.js +211 -0
- package/dist/indexer/lsp/enrichment.js.map +1 -0
- package/dist/indexer/lsp/registry.d.ts +19 -0
- package/dist/indexer/lsp/registry.d.ts.map +1 -0
- package/dist/indexer/lsp/registry.js +118 -0
- package/dist/indexer/lsp/registry.js.map +1 -0
- package/dist/indexer/parser.d.ts +7 -0
- package/dist/indexer/parser.d.ts.map +1 -1
- package/dist/indexer/parser.js +3 -1
- package/dist/indexer/parser.js.map +1 -1
- package/dist/indexer/poller.d.ts +7 -0
- package/dist/indexer/poller.d.ts.map +1 -1
- package/dist/indexer/poller.js +6 -0
- package/dist/indexer/poller.js.map +1 -1
- package/dist/indexer/walker.d.ts +22 -0
- package/dist/indexer/walker.d.ts.map +1 -1
- package/dist/indexer/walker.js +15 -0
- package/dist/indexer/walker.js.map +1 -1
- package/dist/indexer/watcher.d.ts +7 -0
- package/dist/indexer/watcher.d.ts.map +1 -1
- package/dist/indexer/watcher.js +6 -0
- package/dist/indexer/watcher.js.map +1 -1
- package/dist/kb-server/db.d.ts +82 -0
- package/dist/kb-server/db.d.ts.map +1 -1
- package/dist/kb-server/db.js +239 -0
- package/dist/kb-server/db.js.map +1 -1
- package/dist/kb-server/server.d.ts +11 -3
- package/dist/kb-server/server.d.ts.map +1 -1
- package/dist/kb-server/server.js +28 -6
- package/dist/kb-server/server.js.map +1 -1
- package/dist/kb-server/tools/architecture.d.ts +7 -0
- package/dist/kb-server/tools/architecture.d.ts.map +1 -1
- package/dist/kb-server/tools/architecture.js +35 -0
- package/dist/kb-server/tools/architecture.js.map +1 -1
- package/dist/kb-server/tools/docs.d.ts +78 -0
- package/dist/kb-server/tools/docs.d.ts.map +1 -0
- package/dist/kb-server/tools/docs.js +136 -0
- package/dist/kb-server/tools/docs.js.map +1 -0
- package/dist/kb-server/tools/lookup.d.ts.map +1 -1
- package/dist/kb-server/tools/lookup.js +5 -4
- package/dist/kb-server/tools/lookup.js.map +1 -1
- package/dist/kb-server/tools/notes.d.ts +1 -1
- package/dist/kb-server/tools/notes.d.ts.map +1 -1
- package/dist/kb-server/tools/notes.js +35 -0
- package/dist/kb-server/tools/notes.js.map +1 -1
- package/dist/kb-server/tools/search.d.ts +54 -11
- package/dist/kb-server/tools/search.d.ts.map +1 -1
- package/dist/kb-server/tools/search.js +138 -34
- package/dist/kb-server/tools/search.js.map +1 -1
- package/package.json +16 -7
package/dist/indexer/index.d.ts
CHANGED
|
@@ -10,6 +10,17 @@
|
|
|
10
10
|
import type { WalkerConfig } from './walker.js';
|
|
11
11
|
import type { EmbeddingProvider } from './embedder.js';
|
|
12
12
|
import { type CoverageFormat } from './coverage.js';
|
|
13
|
+
import type { EffectiveLspSettings } from './lsp/config.js';
|
|
14
|
+
interface IndexBuilderOptions {
|
|
15
|
+
history?: boolean | {
|
|
16
|
+
depth?: number;
|
|
17
|
+
all?: boolean;
|
|
18
|
+
};
|
|
19
|
+
embeddingModel?: string;
|
|
20
|
+
docsAutoNotes?: boolean;
|
|
21
|
+
indexDependencies?: boolean;
|
|
22
|
+
lsp?: EffectiveLspSettings;
|
|
23
|
+
}
|
|
13
24
|
/**
|
|
14
25
|
* Orchestrates the full M1 indexing pipeline.
|
|
15
26
|
*
|
|
@@ -26,14 +37,11 @@ export declare class IndexBuilder {
|
|
|
26
37
|
private readonly resolver;
|
|
27
38
|
private readonly embedder;
|
|
28
39
|
private readonly history;
|
|
40
|
+
private readonly indexDependencies;
|
|
29
41
|
private readonly embeddingModel;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
all?: boolean;
|
|
34
|
-
};
|
|
35
|
-
embeddingModel?: string;
|
|
36
|
-
});
|
|
42
|
+
private readonly docsAutoNotes;
|
|
43
|
+
private readonly lspSettings;
|
|
44
|
+
constructor(dbPath: string, walkerConfig: WalkerConfig, embedder?: EmbeddingProvider, embeddingModelOrOptions?: string | IndexBuilderOptions);
|
|
37
45
|
/**
|
|
38
46
|
* Performs a full build: walks all files, parses them, extracts
|
|
39
47
|
* symbols/imports/callRefs, resolves imports, and persists everything to
|
|
@@ -60,15 +68,30 @@ export declare class IndexBuilder {
|
|
|
60
68
|
ingestCoverage(reportPath: string, format: CoverageFormat, commitSha?: string): Promise<void>;
|
|
61
69
|
/** Parse one file, extract symbols/imports/callRefs, and insert into the DB. */
|
|
62
70
|
private processFile;
|
|
71
|
+
private processDocumentationFile;
|
|
72
|
+
private upsertSeededDocumentationNote;
|
|
73
|
+
private removeStaleDocumentation;
|
|
74
|
+
private deleteDocumentationByPath;
|
|
75
|
+
private deleteDocumentationById;
|
|
76
|
+
private deleteDocSectionEmbeddings;
|
|
63
77
|
/**
|
|
64
78
|
* Second pass: resolve raw_import strings to file IDs in the
|
|
65
79
|
* `file_imports.resolved_id` column. Also populates `external_deps` for
|
|
66
80
|
* any import that resolves to an external package.
|
|
67
81
|
*/
|
|
68
82
|
private resolveImports;
|
|
83
|
+
private indexDependencyDeclarations;
|
|
84
|
+
private createLspEnrichmentCoordinator;
|
|
85
|
+
private enrichProjectSymbolsAndCallRefs;
|
|
69
86
|
private resolveBranch;
|
|
70
87
|
private saveLastKnownHead;
|
|
88
|
+
private saveDocsAutoNotesSetting;
|
|
71
89
|
private readGitValue;
|
|
90
|
+
private loadDirectDependencies;
|
|
91
|
+
private readInstalledPackageVersion;
|
|
92
|
+
private collectDeclarationFiles;
|
|
93
|
+
private shouldIndexDependencySymbol;
|
|
94
|
+
private hasImplementationBody;
|
|
72
95
|
private loadBuildCheckpoint;
|
|
73
96
|
private saveBuildCheckpoint;
|
|
74
97
|
/**
|
|
@@ -79,5 +102,7 @@ export declare class IndexBuilder {
|
|
|
79
102
|
* creates the vec0 tables if they don't exist yet.
|
|
80
103
|
*/
|
|
81
104
|
private embedStructural;
|
|
105
|
+
private embedDocumentation;
|
|
82
106
|
}
|
|
107
|
+
export {};
|
|
83
108
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAoBH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAuChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AA4D5D,UAAU,mBAAmB;IAC3B,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,GAAG,CAAC,EAAE,oBAAoB,CAAC;CAC5B;AAID;;;;;;;;GAQG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2B;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IACtE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;gBAGxD,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,uBAAuB,CAAC,EAAE,MAAM,GAAG,mBAAmB;IA4BxD;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD5B;;;;;OAKG;IACG,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyFnD;;;;;;;;OAQG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,SAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBlF,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnG,gFAAgF;IAChF,OAAO,CAAC,WAAW;IA8HnB,OAAO,CAAC,wBAAwB;IA6EhC,OAAO,CAAC,6BAA6B;IA8BrC,OAAO,CAAC,wBAAwB;IAShC,OAAO,CAAC,yBAAyB;IAQjC,OAAO,CAAC,uBAAuB;IAM/B,OAAO,CAAC,0BAA0B;IAWlC;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAyCR,2BAA2B;IA8EzC,OAAO,CAAC,8BAA8B;YAOxB,+BAA+B;IA+G7C,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,YAAY;IAapB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,2BAA2B;IASnC,OAAO,CAAC,uBAAuB;IA2B/B,OAAO,CAAC,2BAA2B;IAMnC,OAAO,CAAC,qBAAqB;IAyB7B,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,mBAAmB;IAW3B;;;;;;OAMG;YACW,eAAe;YAiDf,kBAAkB;CAmCjC"}
|
package/dist/indexer/index.js
CHANGED
|
@@ -9,14 +9,18 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import * as fs from 'node:fs';
|
|
11
11
|
import * as crypto from 'node:crypto';
|
|
12
|
+
import * as path from 'node:path';
|
|
12
13
|
import { execFileSync } from 'node:child_process';
|
|
13
14
|
import { openDb, setKbMeta, getKbMeta, createVec0Tables, KB_META_INDEX_CHECKPOINT, KB_META_LAST_HEAD_SHA, KB_META_COVERAGE_LAST_SOURCE_PATH, KB_META_COVERAGE_LAST_SOURCE_MTIME, } from './db.js';
|
|
14
15
|
import { walkFiles } from './walker.js';
|
|
15
16
|
import { detectLanguageForPath } from './walker.js';
|
|
17
|
+
import { walkDocumentationFiles } from './walker.js';
|
|
18
|
+
import { inferSeededDocNoteKey, buildDocNoteScope } from './docs.js';
|
|
16
19
|
import { ingestGitHistory } from './git-history.js';
|
|
17
20
|
import { ParserPool } from './parser.js';
|
|
18
21
|
import { ImportResolver } from './resolver.js';
|
|
19
22
|
import { buildCallGraph } from './call-graph.js';
|
|
23
|
+
import { isPublicDeclarationSurfaceSymbol, } from './extractors/types.js';
|
|
20
24
|
import { CExtractor } from './extractors/c.js';
|
|
21
25
|
import { RustExtractor } from './extractors/rust.js';
|
|
22
26
|
import { PythonExtractor } from './extractors/python.js';
|
|
@@ -41,9 +45,10 @@ import { HaskellExtractor } from './extractors/haskell.js';
|
|
|
41
45
|
import { JuliaExtractor } from './extractors/julia.js';
|
|
42
46
|
import { ElmExtractor } from './extractors/elm.js';
|
|
43
47
|
import { ObjcExtractor } from './extractors/objc.js';
|
|
44
|
-
import { DEFAULT_EMBEDDING_MODEL } from './embedder.js';
|
|
48
|
+
import { DEFAULT_EMBEDDING_MODEL, buildStructuralEmbeddingText } from './embedder.js';
|
|
45
49
|
import { ingestCoverageReport } from './coverage.js';
|
|
46
50
|
import { refreshTestMappings } from './test-mapper.js';
|
|
51
|
+
import { LspEnrichmentCoordinator } from './lsp/enrichment.js';
|
|
47
52
|
// ─── Extractor registry ───────────────────────────────────────────────────────
|
|
48
53
|
const EXTRACTORS = {
|
|
49
54
|
c: new CExtractor(),
|
|
@@ -90,7 +95,10 @@ export class IndexBuilder {
|
|
|
90
95
|
resolver;
|
|
91
96
|
embedder;
|
|
92
97
|
history;
|
|
98
|
+
indexDependencies;
|
|
93
99
|
embeddingModel;
|
|
100
|
+
docsAutoNotes;
|
|
101
|
+
lspSettings;
|
|
94
102
|
constructor(dbPath, walkerConfig, embedder, embeddingModelOrOptions) {
|
|
95
103
|
this.dbPath = dbPath;
|
|
96
104
|
this.walkerConfig = walkerConfig;
|
|
@@ -108,6 +116,9 @@ export class IndexBuilder {
|
|
|
108
116
|
this.embedder = null;
|
|
109
117
|
}
|
|
110
118
|
this.history = opts.history ?? false;
|
|
119
|
+
this.docsAutoNotes = opts.docsAutoNotes ?? true;
|
|
120
|
+
this.indexDependencies = opts.indexDependencies ?? false;
|
|
121
|
+
this.lspSettings = opts.lsp ?? null;
|
|
111
122
|
}
|
|
112
123
|
// ─── Public API ──────────────────────────────────────────────────────────
|
|
113
124
|
/**
|
|
@@ -118,8 +129,17 @@ export class IndexBuilder {
|
|
|
118
129
|
async build() {
|
|
119
130
|
const db = openDb(this.dbPath);
|
|
120
131
|
const branch = this.resolveBranch();
|
|
132
|
+
const lspCoordinator = this.createLspEnrichmentCoordinator();
|
|
121
133
|
try {
|
|
134
|
+
this.saveDocsAutoNotesSetting(db);
|
|
122
135
|
const files = await walkFiles(this.walkerConfig);
|
|
136
|
+
const docs = await walkDocumentationFiles(this.walkerConfig);
|
|
137
|
+
if (lspCoordinator) {
|
|
138
|
+
const languages = new Set(files.map((file) => file.language));
|
|
139
|
+
if (this.indexDependencies)
|
|
140
|
+
languages.add('typescript');
|
|
141
|
+
await lspCoordinator.start(languages);
|
|
142
|
+
}
|
|
123
143
|
const resumeAt = this.loadBuildCheckpoint(db, branch, files.length);
|
|
124
144
|
db.transaction(() => {
|
|
125
145
|
for (let i = resumeAt; i < files.length; i++) {
|
|
@@ -129,15 +149,25 @@ export class IndexBuilder {
|
|
|
129
149
|
this.processFile(db, file.path, file.language, branch);
|
|
130
150
|
this.saveBuildCheckpoint(db, branch, i + 1, files.length);
|
|
131
151
|
}
|
|
152
|
+
const seenDocPaths = new Set();
|
|
153
|
+
for (const doc of docs) {
|
|
154
|
+
seenDocPaths.add(doc.path);
|
|
155
|
+
this.processDocumentationFile(db, doc, branch);
|
|
156
|
+
this.upsertSeededDocumentationNote(db, doc, branch);
|
|
157
|
+
}
|
|
158
|
+
this.removeStaleDocumentation(db, branch, seenDocPaths);
|
|
132
159
|
})();
|
|
133
160
|
this.saveBuildCheckpoint(db, branch, files.length, files.length);
|
|
134
161
|
this.resolveImports(db, branch);
|
|
162
|
+
await this.indexDependencyDeclarations(db, lspCoordinator);
|
|
163
|
+
await this.enrichProjectSymbolsAndCallRefs(db, branch, files, lspCoordinator);
|
|
135
164
|
refreshTestMappings(db, branch);
|
|
136
165
|
buildCallGraph(db);
|
|
137
166
|
this.saveLastKnownHead(db);
|
|
138
167
|
if (this.embedder) {
|
|
139
168
|
await this.embedder.init();
|
|
140
169
|
await this.embedStructural(db);
|
|
170
|
+
await this.embedDocumentation(db);
|
|
141
171
|
}
|
|
142
172
|
if (this.history) {
|
|
143
173
|
const historyOptions = typeof this.history === 'object' ? this.history : undefined;
|
|
@@ -145,6 +175,9 @@ export class IndexBuilder {
|
|
|
145
175
|
}
|
|
146
176
|
}
|
|
147
177
|
finally {
|
|
178
|
+
if (lspCoordinator) {
|
|
179
|
+
await lspCoordinator.dispose();
|
|
180
|
+
}
|
|
148
181
|
db.close();
|
|
149
182
|
}
|
|
150
183
|
}
|
|
@@ -157,7 +190,25 @@ export class IndexBuilder {
|
|
|
157
190
|
async update(changedFiles) {
|
|
158
191
|
const db = openDb(this.dbPath);
|
|
159
192
|
const branch = this.resolveBranch();
|
|
193
|
+
const lspCoordinator = this.createLspEnrichmentCoordinator();
|
|
194
|
+
const enrichedFiles = [];
|
|
160
195
|
try {
|
|
196
|
+
this.saveDocsAutoNotesSetting(db);
|
|
197
|
+
const docs = await walkDocumentationFiles(this.walkerConfig);
|
|
198
|
+
const docsByPath = new Map(docs.map(doc => [doc.path, doc]));
|
|
199
|
+
if (lspCoordinator) {
|
|
200
|
+
const languages = new Set();
|
|
201
|
+
for (const filePath of changedFiles) {
|
|
202
|
+
if (!fs.existsSync(filePath))
|
|
203
|
+
continue;
|
|
204
|
+
const language = detectLanguageForPath(filePath, this.walkerConfig);
|
|
205
|
+
if (language)
|
|
206
|
+
languages.add(language);
|
|
207
|
+
}
|
|
208
|
+
if (this.indexDependencies)
|
|
209
|
+
languages.add('typescript');
|
|
210
|
+
await lspCoordinator.start(languages);
|
|
211
|
+
}
|
|
161
212
|
db.transaction(() => {
|
|
162
213
|
for (const filePath of changedFiles) {
|
|
163
214
|
// If the file no longer exists, remove it from the DB
|
|
@@ -169,23 +220,35 @@ export class IndexBuilder {
|
|
|
169
220
|
db.prepare('DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_id = ?)').run(row.id);
|
|
170
221
|
db.prepare('DELETE FROM files WHERE id = ?').run(row.id);
|
|
171
222
|
}
|
|
223
|
+
this.deleteDocumentationByPath(db, filePath, branch);
|
|
172
224
|
continue;
|
|
173
225
|
}
|
|
174
226
|
const language = detectLanguageForPath(filePath, this.walkerConfig);
|
|
175
|
-
if (
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
227
|
+
if (language) {
|
|
228
|
+
enrichedFiles.push({ path: filePath, language });
|
|
229
|
+
// Null out resolved_id references pointing to this file before deletion
|
|
230
|
+
const existingRow = db.prepare('SELECT id FROM files WHERE path = ? AND branch = ?').get(filePath, branch);
|
|
231
|
+
if (existingRow) {
|
|
232
|
+
db.prepare('UPDATE file_imports SET resolved_id = NULL WHERE resolved_id = ?').run(existingRow.id);
|
|
233
|
+
db.prepare('DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_id = ?)').run(existingRow.id);
|
|
234
|
+
}
|
|
235
|
+
// Delete existing rows for this file (cascade handles symbols/imports)
|
|
236
|
+
db.prepare('DELETE FROM files WHERE path = ? AND branch = ?').run(filePath, branch);
|
|
237
|
+
this.processFile(db, filePath, language, branch);
|
|
238
|
+
}
|
|
239
|
+
const changedDoc = docsByPath.get(filePath);
|
|
240
|
+
if (changedDoc) {
|
|
241
|
+
this.processDocumentationFile(db, changedDoc, branch);
|
|
242
|
+
this.upsertSeededDocumentationNote(db, changedDoc, branch);
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
this.deleteDocumentationByPath(db, filePath, branch);
|
|
182
246
|
}
|
|
183
|
-
// Delete existing rows for this file (cascade handles symbols/imports)
|
|
184
|
-
db.prepare('DELETE FROM files WHERE path = ? AND branch = ?').run(filePath, branch);
|
|
185
|
-
this.processFile(db, filePath, language, branch);
|
|
186
247
|
}
|
|
187
248
|
})();
|
|
188
249
|
this.resolveImports(db, branch);
|
|
250
|
+
await this.indexDependencyDeclarations(db, lspCoordinator);
|
|
251
|
+
await this.enrichProjectSymbolsAndCallRefs(db, branch, enrichedFiles, lspCoordinator);
|
|
189
252
|
refreshTestMappings(db, branch);
|
|
190
253
|
if (this.history) {
|
|
191
254
|
const historyOptions = typeof this.history === 'object' ? this.history : undefined;
|
|
@@ -194,11 +257,15 @@ export class IndexBuilder {
|
|
|
194
257
|
if (this.embedder) {
|
|
195
258
|
await this.embedder.init();
|
|
196
259
|
await this.embedStructural(db);
|
|
260
|
+
await this.embedDocumentation(db);
|
|
197
261
|
}
|
|
198
262
|
buildCallGraph(db);
|
|
199
263
|
this.saveLastKnownHead(db);
|
|
200
264
|
}
|
|
201
265
|
finally {
|
|
266
|
+
if (lspCoordinator) {
|
|
267
|
+
await lspCoordinator.dispose();
|
|
268
|
+
}
|
|
202
269
|
db.close();
|
|
203
270
|
}
|
|
204
271
|
}
|
|
@@ -299,7 +366,10 @@ export class IndexBuilder {
|
|
|
299
366
|
const info = insertSymbol.run(fileId, sym.name, sym.kind, sym.startLine, sym.endLine, sym.signature ?? null, sym.docComment ?? null);
|
|
300
367
|
const symId = Number(info.lastInsertRowid);
|
|
301
368
|
symbolIdMap.set(sym.name, symId);
|
|
302
|
-
insertFts.run(symId, sym.name,
|
|
369
|
+
insertFts.run(symId, sym.name, buildStructuralEmbeddingText({
|
|
370
|
+
name: sym.name,
|
|
371
|
+
signature: sym.signature ?? null,
|
|
372
|
+
}), sym.kind);
|
|
303
373
|
}
|
|
304
374
|
const insertRoute = db.prepare(`INSERT INTO api_routes (file_id, method, path, handler_id, handler_name, framework, line, middleware)
|
|
305
375
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
@@ -321,6 +391,95 @@ export class IndexBuilder {
|
|
|
321
391
|
}
|
|
322
392
|
}
|
|
323
393
|
}
|
|
394
|
+
processDocumentationFile(db, doc, branch) {
|
|
395
|
+
const existing = db.prepare('SELECT id, content_hash FROM docs WHERE path = ? AND branch = ?').get(doc.path, branch);
|
|
396
|
+
if (existing?.content_hash === doc.hash) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
let docId;
|
|
400
|
+
if (existing) {
|
|
401
|
+
db.prepare(`UPDATE docs
|
|
402
|
+
SET kind = ?, title = ?, content = ?, content_hash = ?, indexed_at = unixepoch()
|
|
403
|
+
WHERE id = ?`).run(doc.kind, doc.title, doc.content, doc.hash, existing.id);
|
|
404
|
+
docId = existing.id;
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
const info = db.prepare(`INSERT INTO docs (path, branch, kind, title, content, content_hash)
|
|
408
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(doc.path, branch, doc.kind, doc.title, doc.content, doc.hash);
|
|
409
|
+
docId = Number(info.lastInsertRowid);
|
|
410
|
+
}
|
|
411
|
+
const existingSections = db.prepare('SELECT id, section_index FROM doc_sections WHERE doc_id = ?').all(docId);
|
|
412
|
+
const insertSection = db.prepare(`INSERT INTO doc_sections (
|
|
413
|
+
doc_id, section_index, title, depth, heading_path, line_start, line_end, content, content_hash
|
|
414
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
415
|
+
ON CONFLICT(doc_id, section_index) DO UPDATE SET
|
|
416
|
+
title = excluded.title,
|
|
417
|
+
depth = excluded.depth,
|
|
418
|
+
heading_path = excluded.heading_path,
|
|
419
|
+
line_start = excluded.line_start,
|
|
420
|
+
line_end = excluded.line_end,
|
|
421
|
+
content = excluded.content,
|
|
422
|
+
content_hash = excluded.content_hash`);
|
|
423
|
+
const activeSectionIndexes = new Set();
|
|
424
|
+
for (const chunk of doc.chunks) {
|
|
425
|
+
activeSectionIndexes.add(chunk.sectionIndex);
|
|
426
|
+
insertSection.run(docId, chunk.sectionIndex, chunk.title, chunk.depth, JSON.stringify(chunk.headingPath), chunk.lineStart, chunk.lineEnd, chunk.content, chunk.hash);
|
|
427
|
+
}
|
|
428
|
+
const staleSectionIds = existingSections
|
|
429
|
+
.filter(section => !activeSectionIndexes.has(section.section_index))
|
|
430
|
+
.map(section => section.id);
|
|
431
|
+
this.deleteDocSectionEmbeddings(db, staleSectionIds);
|
|
432
|
+
if (staleSectionIds.length > 0) {
|
|
433
|
+
db.prepare(`DELETE FROM doc_sections
|
|
434
|
+
WHERE id IN (${staleSectionIds.map(() => '?').join(', ')})`).run(...staleSectionIds);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
upsertSeededDocumentationNote(db, doc, branch) {
|
|
438
|
+
if (!this.docsAutoNotes)
|
|
439
|
+
return;
|
|
440
|
+
const key = inferSeededDocNoteKey(doc);
|
|
441
|
+
if (!key)
|
|
442
|
+
return;
|
|
443
|
+
const scope = buildDocNoteScope(doc.path, branch);
|
|
444
|
+
const existing = db.prepare('SELECT content, source_hash FROM notes WHERE key = ? AND scope = ?').get(key, scope);
|
|
445
|
+
if (existing?.content === doc.content && existing.source_hash === doc.hash) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
db.prepare(`INSERT INTO notes (key, scope, content, model, source_hash, created_at, updated_at)
|
|
449
|
+
VALUES (?, ?, ?, ?, ?, unixepoch(), unixepoch())
|
|
450
|
+
ON CONFLICT(key, scope) DO UPDATE SET
|
|
451
|
+
content = excluded.content,
|
|
452
|
+
model = excluded.model,
|
|
453
|
+
source_hash = excluded.source_hash,
|
|
454
|
+
updated_at = unixepoch()`).run(key, scope, doc.content, 'system:auto-doc-seed', doc.hash);
|
|
455
|
+
}
|
|
456
|
+
removeStaleDocumentation(db, branch, retainedPaths) {
|
|
457
|
+
const docs = db.prepare('SELECT id, path FROM docs WHERE branch = ?').all(branch);
|
|
458
|
+
for (const doc of docs) {
|
|
459
|
+
if (!retainedPaths.has(doc.path)) {
|
|
460
|
+
this.deleteDocumentationById(db, doc.id);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
deleteDocumentationByPath(db, docPath, branch) {
|
|
465
|
+
const row = db.prepare('SELECT id FROM docs WHERE path = ? AND branch = ?').get(docPath, branch);
|
|
466
|
+
if (!row)
|
|
467
|
+
return;
|
|
468
|
+
this.deleteDocumentationById(db, row.id);
|
|
469
|
+
}
|
|
470
|
+
deleteDocumentationById(db, docId) {
|
|
471
|
+
const sectionIds = db.prepare('SELECT id FROM doc_sections WHERE doc_id = ?').all(docId);
|
|
472
|
+
this.deleteDocSectionEmbeddings(db, sectionIds.map(row => row.id));
|
|
473
|
+
db.prepare('DELETE FROM docs WHERE id = ?').run(docId);
|
|
474
|
+
}
|
|
475
|
+
deleteDocSectionEmbeddings(db, sectionIds) {
|
|
476
|
+
if (sectionIds.length === 0)
|
|
477
|
+
return;
|
|
478
|
+
const hasEmbeddingsTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type IN ('table', 'virtual table') AND name = 'doc_section_embeddings'").get();
|
|
479
|
+
if (!hasEmbeddingsTable)
|
|
480
|
+
return;
|
|
481
|
+
db.prepare(`DELETE FROM doc_section_embeddings WHERE rowid IN (${sectionIds.map(() => '?').join(', ')})`).run(...sectionIds);
|
|
482
|
+
}
|
|
324
483
|
/**
|
|
325
484
|
* Second pass: resolve raw_import strings to file IDs in the
|
|
326
485
|
* `file_imports.resolved_id` column. Also populates `external_deps` for
|
|
@@ -352,6 +511,146 @@ export class IndexBuilder {
|
|
|
352
511
|
}
|
|
353
512
|
}
|
|
354
513
|
}
|
|
514
|
+
async indexDependencyDeclarations(db, lspCoordinator) {
|
|
515
|
+
db.prepare('DELETE FROM external_symbols').run();
|
|
516
|
+
if (!this.indexDependencies)
|
|
517
|
+
return;
|
|
518
|
+
const directDependencies = this.loadDirectDependencies();
|
|
519
|
+
if (directDependencies.size === 0)
|
|
520
|
+
return;
|
|
521
|
+
const extractor = EXTRACTORS.typescript;
|
|
522
|
+
if (!extractor)
|
|
523
|
+
return;
|
|
524
|
+
const insertExternalSymbol = db.prepare(`INSERT OR IGNORE INTO external_symbols
|
|
525
|
+
(
|
|
526
|
+
package_name,
|
|
527
|
+
package_version,
|
|
528
|
+
source_ref,
|
|
529
|
+
symbol_name,
|
|
530
|
+
symbol_kind,
|
|
531
|
+
signature,
|
|
532
|
+
doc_comment,
|
|
533
|
+
resolved_type_signature,
|
|
534
|
+
resolved_return_type,
|
|
535
|
+
definition_uri,
|
|
536
|
+
definition_path
|
|
537
|
+
)
|
|
538
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
539
|
+
for (const [packageName, declaredVersion] of directDependencies) {
|
|
540
|
+
const packageDir = path.join(this.walkerConfig.rootDir, 'node_modules', packageName);
|
|
541
|
+
if (!fs.existsSync(packageDir) || !fs.statSync(packageDir).isDirectory())
|
|
542
|
+
continue;
|
|
543
|
+
const packageVersion = this.readInstalledPackageVersion(packageDir) ?? declaredVersion ?? null;
|
|
544
|
+
const declarationFiles = this.collectDeclarationFiles(packageDir);
|
|
545
|
+
for (const declarationFile of declarationFiles) {
|
|
546
|
+
const source = fs.readFileSync(declarationFile, 'utf8');
|
|
547
|
+
const tree = this.pool.parse('typescript', source);
|
|
548
|
+
if (!tree)
|
|
549
|
+
continue;
|
|
550
|
+
const result = extractor.extract(tree, source, declarationFile);
|
|
551
|
+
const declarationSymbols = result.symbols.filter((symbol) => this.shouldIndexDependencySymbol(symbol));
|
|
552
|
+
const enrichmentRows = lspCoordinator
|
|
553
|
+
? await lspCoordinator.enrich({
|
|
554
|
+
filePath: declarationFile,
|
|
555
|
+
language: 'typescript',
|
|
556
|
+
source,
|
|
557
|
+
targets: declarationSymbols.map((symbol) => ({
|
|
558
|
+
line: symbol.startLine,
|
|
559
|
+
character: symbol.startCharacter ?? 0,
|
|
560
|
+
})),
|
|
561
|
+
})
|
|
562
|
+
: declarationSymbols.map(() => null);
|
|
563
|
+
for (let i = 0; i < declarationSymbols.length; i++) {
|
|
564
|
+
const symbol = declarationSymbols[i];
|
|
565
|
+
if (!symbol)
|
|
566
|
+
continue;
|
|
567
|
+
const metadata = enrichmentRows[i];
|
|
568
|
+
insertExternalSymbol.run(packageName, packageVersion, declarationFile, symbol.name, symbol.kind, symbol.signature, symbol.docComment ?? null, metadata?.resolvedTypeSignature ?? null, metadata?.resolvedReturnType ?? null, metadata?.definitionUri ?? null, metadata?.definitionPath ?? null);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
createLspEnrichmentCoordinator() {
|
|
574
|
+
if (!this.lspSettings?.enabled) {
|
|
575
|
+
return null;
|
|
576
|
+
}
|
|
577
|
+
return new LspEnrichmentCoordinator(this.lspSettings, this.walkerConfig.rootDir);
|
|
578
|
+
}
|
|
579
|
+
async enrichProjectSymbolsAndCallRefs(db, branch, files, lspCoordinator) {
|
|
580
|
+
if (!lspCoordinator || files.length === 0)
|
|
581
|
+
return;
|
|
582
|
+
const selectSymbols = db.prepare(`SELECT s.id, s.name, s.signature, s.start_line
|
|
583
|
+
FROM symbols s
|
|
584
|
+
JOIN files f ON f.id = s.file_id
|
|
585
|
+
WHERE f.path = ? AND f.branch = ?
|
|
586
|
+
ORDER BY s.id`);
|
|
587
|
+
const selectCallRefs = db.prepare(`SELECT sr.id, sr.call_line
|
|
588
|
+
FROM symbol_refs sr
|
|
589
|
+
JOIN symbols s ON s.id = sr.caller_id
|
|
590
|
+
JOIN files f ON f.id = s.file_id
|
|
591
|
+
WHERE f.path = ? AND f.branch = ?
|
|
592
|
+
ORDER BY sr.id`);
|
|
593
|
+
const updateSymbol = db.prepare(`UPDATE symbols
|
|
594
|
+
SET resolved_type_signature = ?, resolved_return_type = ?, definition_uri = ?, definition_path = ?
|
|
595
|
+
WHERE id = ?`);
|
|
596
|
+
const updateSymbolFts = db.prepare('UPDATE symbols_fts SET signature = ? WHERE rowid = ?');
|
|
597
|
+
const updateCallRef = db.prepare(`UPDATE symbol_refs
|
|
598
|
+
SET resolved_type_signature = ?, resolved_return_type = ?, definition_uri = ?, definition_path = ?
|
|
599
|
+
WHERE id = ?`);
|
|
600
|
+
for (const file of files) {
|
|
601
|
+
if (!file || !fs.existsSync(file.path))
|
|
602
|
+
continue;
|
|
603
|
+
let source;
|
|
604
|
+
try {
|
|
605
|
+
source = fs.readFileSync(file.path, 'utf8');
|
|
606
|
+
}
|
|
607
|
+
catch {
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
const symbols = selectSymbols.all(file.path, branch);
|
|
611
|
+
if (symbols.length > 0) {
|
|
612
|
+
const symbolMetadata = await lspCoordinator.enrich({
|
|
613
|
+
filePath: file.path,
|
|
614
|
+
language: file.language,
|
|
615
|
+
source,
|
|
616
|
+
targets: symbols.map((symbol) => ({ line: symbol.start_line, character: 0 })),
|
|
617
|
+
});
|
|
618
|
+
for (let i = 0; i < symbols.length; i++) {
|
|
619
|
+
const symbol = symbols[i];
|
|
620
|
+
if (!symbol)
|
|
621
|
+
continue;
|
|
622
|
+
const metadata = symbolMetadata[i];
|
|
623
|
+
if (!metadata)
|
|
624
|
+
continue;
|
|
625
|
+
updateSymbol.run(metadata.resolvedTypeSignature, metadata.resolvedReturnType, metadata.definitionUri, metadata.definitionPath, symbol.id);
|
|
626
|
+
updateSymbolFts.run(buildStructuralEmbeddingText({
|
|
627
|
+
name: symbol.name,
|
|
628
|
+
signature: symbol.signature,
|
|
629
|
+
resolvedTypeSignature: metadata.resolvedTypeSignature,
|
|
630
|
+
resolvedReturnType: metadata.resolvedReturnType,
|
|
631
|
+
}), symbol.id);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
const callRefs = selectCallRefs.all(file.path, branch);
|
|
635
|
+
if (callRefs.length > 0) {
|
|
636
|
+
const callRefMetadata = await lspCoordinator.enrich({
|
|
637
|
+
filePath: file.path,
|
|
638
|
+
language: file.language,
|
|
639
|
+
source,
|
|
640
|
+
targets: callRefs.map((callRef) => ({ line: callRef.call_line, character: 0 })),
|
|
641
|
+
});
|
|
642
|
+
for (let i = 0; i < callRefs.length; i++) {
|
|
643
|
+
const callRef = callRefs[i];
|
|
644
|
+
if (!callRef)
|
|
645
|
+
continue;
|
|
646
|
+
const metadata = callRefMetadata[i];
|
|
647
|
+
if (!metadata)
|
|
648
|
+
continue;
|
|
649
|
+
updateCallRef.run(metadata.resolvedTypeSignature, metadata.resolvedReturnType, metadata.definitionUri, metadata.definitionPath, callRef.id);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
}
|
|
355
654
|
resolveBranch() {
|
|
356
655
|
if (this.walkerConfig.branch)
|
|
357
656
|
return this.walkerConfig.branch;
|
|
@@ -363,6 +662,9 @@ export class IndexBuilder {
|
|
|
363
662
|
setKbMeta(db, KB_META_LAST_HEAD_SHA, headSha);
|
|
364
663
|
}
|
|
365
664
|
}
|
|
665
|
+
saveDocsAutoNotesSetting(db) {
|
|
666
|
+
setKbMeta(db, 'docs_auto_notes', this.docsAutoNotes ? '1' : '0');
|
|
667
|
+
}
|
|
366
668
|
readGitValue(args) {
|
|
367
669
|
try {
|
|
368
670
|
const value = execFileSync('git', ['-C', this.walkerConfig.rootDir, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
@@ -372,6 +674,80 @@ export class IndexBuilder {
|
|
|
372
674
|
return undefined;
|
|
373
675
|
}
|
|
374
676
|
}
|
|
677
|
+
loadDirectDependencies() {
|
|
678
|
+
const packageJsonPath = path.join(this.walkerConfig.rootDir, 'package.json');
|
|
679
|
+
if (!fs.existsSync(packageJsonPath))
|
|
680
|
+
return new Map();
|
|
681
|
+
const raw = fs.readFileSync(packageJsonPath, 'utf8');
|
|
682
|
+
const pkg = JSON.parse(raw);
|
|
683
|
+
const deps = new Map();
|
|
684
|
+
for (const section of [pkg.dependencies, pkg.devDependencies, pkg.peerDependencies]) {
|
|
685
|
+
if (!section)
|
|
686
|
+
continue;
|
|
687
|
+
for (const [name, version] of Object.entries(section)) {
|
|
688
|
+
if (!deps.has(name))
|
|
689
|
+
deps.set(name, version);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
return deps;
|
|
693
|
+
}
|
|
694
|
+
readInstalledPackageVersion(packageDir) {
|
|
695
|
+
const packageJsonPath = path.join(packageDir, 'package.json');
|
|
696
|
+
if (!fs.existsSync(packageJsonPath))
|
|
697
|
+
return undefined;
|
|
698
|
+
const raw = fs.readFileSync(packageJsonPath, 'utf8');
|
|
699
|
+
const pkg = JSON.parse(raw);
|
|
700
|
+
return pkg.version;
|
|
701
|
+
}
|
|
702
|
+
collectDeclarationFiles(packageDir) {
|
|
703
|
+
const declarations = [];
|
|
704
|
+
const stack = [packageDir];
|
|
705
|
+
while (stack.length > 0) {
|
|
706
|
+
const currentDir = stack.pop();
|
|
707
|
+
if (!currentDir)
|
|
708
|
+
continue;
|
|
709
|
+
const entries = fs.readdirSync(currentDir, { withFileTypes: true });
|
|
710
|
+
for (const entry of entries) {
|
|
711
|
+
if (entry.name === 'node_modules')
|
|
712
|
+
continue;
|
|
713
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
714
|
+
if (entry.isDirectory()) {
|
|
715
|
+
stack.push(fullPath);
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
if (entry.isFile() && fullPath.endsWith('.d.ts')) {
|
|
719
|
+
declarations.push(fullPath);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return declarations;
|
|
724
|
+
}
|
|
725
|
+
shouldIndexDependencySymbol(symbol) {
|
|
726
|
+
if (!isPublicDeclarationSurfaceSymbol(symbol))
|
|
727
|
+
return false;
|
|
728
|
+
if (symbol.declarationSurface)
|
|
729
|
+
return true;
|
|
730
|
+
return !this.hasImplementationBody(symbol);
|
|
731
|
+
}
|
|
732
|
+
hasImplementationBody(symbol) {
|
|
733
|
+
const node = symbol.astNode;
|
|
734
|
+
if (!node)
|
|
735
|
+
return false;
|
|
736
|
+
if (node.type === 'arrow_function' ||
|
|
737
|
+
node.type === 'function_expression' ||
|
|
738
|
+
node.type === 'generator_function') {
|
|
739
|
+
return true;
|
|
740
|
+
}
|
|
741
|
+
if (node.type === 'class_declaration' ||
|
|
742
|
+
node.type === 'interface_declaration' ||
|
|
743
|
+
node.type === 'type_alias_declaration') {
|
|
744
|
+
return false;
|
|
745
|
+
}
|
|
746
|
+
const bodyNode = node.childForFieldName('body');
|
|
747
|
+
if (!bodyNode)
|
|
748
|
+
return false;
|
|
749
|
+
return bodyNode.namedChildCount > 0 || bodyNode.text.trim() !== '';
|
|
750
|
+
}
|
|
375
751
|
loadBuildCheckpoint(db, branch, totalFiles) {
|
|
376
752
|
const raw = getKbMeta(db, KB_META_INDEX_CHECKPOINT);
|
|
377
753
|
if (!raw)
|
|
@@ -409,14 +785,23 @@ export class IndexBuilder {
|
|
|
409
785
|
setKbMeta(db, 'embedding_model', embedder.modelName);
|
|
410
786
|
setKbMeta(db, 'embedding_dims', String(embedder.dims));
|
|
411
787
|
createVec0Tables(db, embedder.dims);
|
|
412
|
-
// Fetch all symbols that have
|
|
788
|
+
// Fetch all symbols that have structural text to embed.
|
|
413
789
|
const symbols = db
|
|
414
|
-
.prepare(
|
|
790
|
+
.prepare(`SELECT id, name, signature, resolved_type_signature, resolved_return_type
|
|
791
|
+
FROM symbols
|
|
792
|
+
WHERE signature IS NOT NULL
|
|
793
|
+
OR resolved_type_signature IS NOT NULL
|
|
794
|
+
OR resolved_return_type IS NOT NULL`)
|
|
415
795
|
.all();
|
|
416
796
|
const insertEmbed = db.prepare('INSERT OR REPLACE INTO symbol_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), json(?))');
|
|
417
797
|
for (let i = 0; i < symbols.length; i += EMBED_BATCH_SIZE) {
|
|
418
798
|
const batch = symbols.slice(i, i + EMBED_BATCH_SIZE);
|
|
419
|
-
const texts = batch.map(
|
|
799
|
+
const texts = batch.map((symbol) => buildStructuralEmbeddingText({
|
|
800
|
+
name: symbol.name,
|
|
801
|
+
signature: symbol.signature,
|
|
802
|
+
resolvedTypeSignature: symbol.resolved_type_signature,
|
|
803
|
+
resolvedReturnType: symbol.resolved_return_type,
|
|
804
|
+
}));
|
|
420
805
|
const embeddings = await embedder.embed(texts);
|
|
421
806
|
db.transaction(() => {
|
|
422
807
|
for (let j = 0; j < batch.length; j++) {
|
|
@@ -427,5 +812,32 @@ export class IndexBuilder {
|
|
|
427
812
|
})();
|
|
428
813
|
}
|
|
429
814
|
}
|
|
815
|
+
async embedDocumentation(db) {
|
|
816
|
+
const embedder = this.embedder;
|
|
817
|
+
db.exec(`
|
|
818
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS doc_section_embeddings USING vec0(
|
|
819
|
+
embedding FLOAT[${embedder.dims}]
|
|
820
|
+
);
|
|
821
|
+
`);
|
|
822
|
+
const sections = db.prepare(`SELECT id, title, content
|
|
823
|
+
FROM doc_sections
|
|
824
|
+
ORDER BY id`).all();
|
|
825
|
+
if (sections.length === 0)
|
|
826
|
+
return;
|
|
827
|
+
const insertEmbed = db.prepare('INSERT OR REPLACE INTO doc_section_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), json(?))');
|
|
828
|
+
for (let i = 0; i < sections.length; i += EMBED_BATCH_SIZE) {
|
|
829
|
+
const batch = sections.slice(i, i + EMBED_BATCH_SIZE);
|
|
830
|
+
const texts = batch.map(section => section.content || section.title);
|
|
831
|
+
const embeddings = await embedder.embed(texts);
|
|
832
|
+
db.transaction(() => {
|
|
833
|
+
for (let j = 0; j < batch.length; j++) {
|
|
834
|
+
const section = batch[j];
|
|
835
|
+
if (section) {
|
|
836
|
+
insertEmbed.run(section.id, JSON.stringify(embeddings[j]));
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
})();
|
|
840
|
+
}
|
|
841
|
+
}
|
|
430
842
|
}
|
|
431
843
|
//# sourceMappingURL=index.js.map
|