@jafreck/lore 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +345 -196
  2. package/dist/cli.js +133 -12
  3. package/dist/cli.js.map +1 -1
  4. package/dist/index.d.ts +3 -0
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js.map +1 -1
  7. package/dist/indexer/db.d.ts.map +1 -1
  8. package/dist/indexer/db.js +96 -2
  9. package/dist/indexer/db.js.map +1 -1
  10. package/dist/indexer/docs.d.ts +42 -0
  11. package/dist/indexer/docs.d.ts.map +1 -0
  12. package/dist/indexer/docs.js +214 -0
  13. package/dist/indexer/docs.js.map +1 -0
  14. package/dist/indexer/embedder.d.ts +7 -0
  15. package/dist/indexer/embedder.d.ts.map +1 -1
  16. package/dist/indexer/embedder.js +10 -0
  17. package/dist/indexer/embedder.js.map +1 -1
  18. package/dist/indexer/extractors/types.d.ts +22 -0
  19. package/dist/indexer/extractors/types.d.ts.map +1 -1
  20. package/dist/indexer/extractors/types.js +12 -0
  21. package/dist/indexer/extractors/types.js.map +1 -1
  22. package/dist/indexer/extractors/typescript.d.ts +1 -1
  23. package/dist/indexer/extractors/typescript.d.ts.map +1 -1
  24. package/dist/indexer/extractors/typescript.js +38 -8
  25. package/dist/indexer/extractors/typescript.js.map +1 -1
  26. package/dist/indexer/git-hooks.d.ts +1 -0
  27. package/dist/indexer/git-hooks.d.ts.map +1 -1
  28. package/dist/indexer/git-hooks.js +3 -2
  29. package/dist/indexer/git-hooks.js.map +1 -1
  30. package/dist/indexer/index.d.ts +32 -7
  31. package/dist/indexer/index.d.ts.map +1 -1
  32. package/dist/indexer/index.js +427 -15
  33. package/dist/indexer/index.js.map +1 -1
  34. package/dist/indexer/lsp/client.d.ts +61 -0
  35. package/dist/indexer/lsp/client.d.ts.map +1 -0
  36. package/dist/indexer/lsp/client.js +217 -0
  37. package/dist/indexer/lsp/client.js.map +1 -0
  38. package/dist/indexer/lsp/config.d.ts +16 -0
  39. package/dist/indexer/lsp/config.d.ts.map +1 -0
  40. package/dist/indexer/lsp/config.js +78 -0
  41. package/dist/indexer/lsp/config.js.map +1 -0
  42. package/dist/indexer/lsp/enrichment.d.ts +55 -0
  43. package/dist/indexer/lsp/enrichment.d.ts.map +1 -0
  44. package/dist/indexer/lsp/enrichment.js +211 -0
  45. package/dist/indexer/lsp/enrichment.js.map +1 -0
  46. package/dist/indexer/lsp/registry.d.ts +19 -0
  47. package/dist/indexer/lsp/registry.d.ts.map +1 -0
  48. package/dist/indexer/lsp/registry.js +118 -0
  49. package/dist/indexer/lsp/registry.js.map +1 -0
  50. package/dist/indexer/parser.d.ts +7 -0
  51. package/dist/indexer/parser.d.ts.map +1 -1
  52. package/dist/indexer/parser.js +3 -1
  53. package/dist/indexer/parser.js.map +1 -1
  54. package/dist/indexer/poller.d.ts +7 -0
  55. package/dist/indexer/poller.d.ts.map +1 -1
  56. package/dist/indexer/poller.js +6 -0
  57. package/dist/indexer/poller.js.map +1 -1
  58. package/dist/indexer/walker.d.ts +22 -0
  59. package/dist/indexer/walker.d.ts.map +1 -1
  60. package/dist/indexer/walker.js +15 -0
  61. package/dist/indexer/walker.js.map +1 -1
  62. package/dist/indexer/watcher.d.ts +7 -0
  63. package/dist/indexer/watcher.d.ts.map +1 -1
  64. package/dist/indexer/watcher.js +6 -0
  65. package/dist/indexer/watcher.js.map +1 -1
  66. package/dist/kb-server/db.d.ts +82 -0
  67. package/dist/kb-server/db.d.ts.map +1 -1
  68. package/dist/kb-server/db.js +239 -0
  69. package/dist/kb-server/db.js.map +1 -1
  70. package/dist/kb-server/server.d.ts +11 -3
  71. package/dist/kb-server/server.d.ts.map +1 -1
  72. package/dist/kb-server/server.js +28 -6
  73. package/dist/kb-server/server.js.map +1 -1
  74. package/dist/kb-server/tools/architecture.d.ts +7 -0
  75. package/dist/kb-server/tools/architecture.d.ts.map +1 -1
  76. package/dist/kb-server/tools/architecture.js +35 -0
  77. package/dist/kb-server/tools/architecture.js.map +1 -1
  78. package/dist/kb-server/tools/docs.d.ts +78 -0
  79. package/dist/kb-server/tools/docs.d.ts.map +1 -0
  80. package/dist/kb-server/tools/docs.js +136 -0
  81. package/dist/kb-server/tools/docs.js.map +1 -0
  82. package/dist/kb-server/tools/lookup.d.ts.map +1 -1
  83. package/dist/kb-server/tools/lookup.js +5 -4
  84. package/dist/kb-server/tools/lookup.js.map +1 -1
  85. package/dist/kb-server/tools/notes.d.ts +1 -1
  86. package/dist/kb-server/tools/notes.d.ts.map +1 -1
  87. package/dist/kb-server/tools/notes.js +35 -0
  88. package/dist/kb-server/tools/notes.js.map +1 -1
  89. package/dist/kb-server/tools/search.d.ts +54 -11
  90. package/dist/kb-server/tools/search.d.ts.map +1 -1
  91. package/dist/kb-server/tools/search.js +138 -34
  92. package/dist/kb-server/tools/search.js.map +1 -1
  93. package/package.json +16 -7
@@ -10,6 +10,17 @@
10
10
  import type { WalkerConfig } from './walker.js';
11
11
  import type { EmbeddingProvider } from './embedder.js';
12
12
  import { type CoverageFormat } from './coverage.js';
13
+ import type { EffectiveLspSettings } from './lsp/config.js';
14
+ interface IndexBuilderOptions {
15
+ history?: boolean | {
16
+ depth?: number;
17
+ all?: boolean;
18
+ };
19
+ embeddingModel?: string;
20
+ docsAutoNotes?: boolean;
21
+ indexDependencies?: boolean;
22
+ lsp?: EffectiveLspSettings;
23
+ }
13
24
  /**
14
25
  * Orchestrates the full M1 indexing pipeline.
15
26
  *
@@ -26,14 +37,11 @@ export declare class IndexBuilder {
26
37
  private readonly resolver;
27
38
  private readonly embedder;
28
39
  private readonly history;
40
+ private readonly indexDependencies;
29
41
  private readonly embeddingModel;
30
- constructor(dbPath: string, walkerConfig: WalkerConfig, embedder?: EmbeddingProvider, embeddingModelOrOptions?: string | {
31
- history?: boolean | {
32
- depth?: number;
33
- all?: boolean;
34
- };
35
- embeddingModel?: string;
36
- });
42
+ private readonly docsAutoNotes;
43
+ private readonly lspSettings;
44
+ constructor(dbPath: string, walkerConfig: WalkerConfig, embedder?: EmbeddingProvider, embeddingModelOrOptions?: string | IndexBuilderOptions);
37
45
  /**
38
46
  * Performs a full build: walks all files, parses them, extracts
39
47
  * symbols/imports/callRefs, resolves imports, and persists everything to
@@ -60,15 +68,30 @@ export declare class IndexBuilder {
60
68
  ingestCoverage(reportPath: string, format: CoverageFormat, commitSha?: string): Promise<void>;
61
69
  /** Parse one file, extract symbols/imports/callRefs, and insert into the DB. */
62
70
  private processFile;
71
+ private processDocumentationFile;
72
+ private upsertSeededDocumentationNote;
73
+ private removeStaleDocumentation;
74
+ private deleteDocumentationByPath;
75
+ private deleteDocumentationById;
76
+ private deleteDocSectionEmbeddings;
63
77
  /**
64
78
  * Second pass: resolve raw_import strings to file IDs in the
65
79
  * `file_imports.resolved_id` column. Also populates `external_deps` for
66
80
  * any import that resolves to an external package.
67
81
  */
68
82
  private resolveImports;
83
+ private indexDependencyDeclarations;
84
+ private createLspEnrichmentCoordinator;
85
+ private enrichProjectSymbolsAndCallRefs;
69
86
  private resolveBranch;
70
87
  private saveLastKnownHead;
88
+ private saveDocsAutoNotesSetting;
71
89
  private readGitValue;
90
+ private loadDirectDependencies;
91
+ private readInstalledPackageVersion;
92
+ private collectDeclarationFiles;
93
+ private shouldIndexDependencySymbol;
94
+ private hasImplementationBody;
72
95
  private loadBuildCheckpoint;
73
96
  private saveBuildCheckpoint;
74
97
  /**
@@ -79,5 +102,7 @@ export declare class IndexBuilder {
79
102
  * creates the vec0 tables if they don't exist yet.
80
103
  */
81
104
  private embedStructural;
105
+ private embedDocumentation;
82
106
  }
107
+ export {};
83
108
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkBH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA+BhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAoD1E;;;;;;;;GAQG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2B;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IACtE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAGtC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,uBAAuB,CAAC,EAAE,MAAM,GAAG;QAAE,OAAO,CAAC,EAAE,OAAO,GAAG;YAAE,KAAK,CAAC,EAAE,MAAM,CAAC;YAAC,GAAG,CAAC,EAAE,OAAO,CAAA;SAAE,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE;IAyBvH;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC5B;;;;;OAKG;IACG,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDnD;;;;;;;;OAQG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,SAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBlF,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnG,gFAAgF;IAChF,OAAO,CAAC,WAAW;IAsHnB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAyCtB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,YAAY;IAapB,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,mBAAmB;IAW3B;;;;;;OAMG;YACW,eAAe;CA6B9B"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAoBH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAuChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AA4D5D,UAAU,mBAAmB;IAC3B,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,GAAG,CAAC,EAAE,oBAAoB,CAAC;CAC5B;AAID;;;;;;;;GAQG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2B;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IACtE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;gBAGxD,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,uBAAuB,CAAC,EAAE,MAAM,GAAG,mBAAmB;IA4BxD;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD5B;;;;;OAKG;IACG,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyFnD;;;;;;;;OAQG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,SAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBlF,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnG,gFAAgF;IAChF,OAAO,CAAC,WAAW;IA8HnB,OAAO,CAAC,wBAAwB;IA6EhC,OAAO,CAAC,6BAA6B;IA8BrC,OAAO,CAAC,wBAAwB;IAShC,OAAO,CAAC,yBAAyB;IAQjC,OAAO,CAAC,uBAAuB;IAM/B,OAAO,CAAC,0BAA0B;IAWlC;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAyCR,2BAA2B;IA8EzC,OAAO,CAAC,8BAA8B;YAOxB,+BAA+B;IA+G7C,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,YAAY;IAapB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,2BAA2B;IASnC,OAAO,CAAC,uBAAuB;IA2B/B,OAAO,CAAC,2BAA2B;IAMnC,OAAO,CAAC,qBAAqB;IAyB7B,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,mBAAmB;IAW3B;;;;;;OAMG;YACW,eAAe;YAiDf,kBAAkB;CAmCjC"}
@@ -9,14 +9,18 @@
9
9
  */
10
10
  import * as fs from 'node:fs';
11
11
  import * as crypto from 'node:crypto';
12
+ import * as path from 'node:path';
12
13
  import { execFileSync } from 'node:child_process';
13
14
  import { openDb, setKbMeta, getKbMeta, createVec0Tables, KB_META_INDEX_CHECKPOINT, KB_META_LAST_HEAD_SHA, KB_META_COVERAGE_LAST_SOURCE_PATH, KB_META_COVERAGE_LAST_SOURCE_MTIME, } from './db.js';
14
15
  import { walkFiles } from './walker.js';
15
16
  import { detectLanguageForPath } from './walker.js';
17
+ import { walkDocumentationFiles } from './walker.js';
18
+ import { inferSeededDocNoteKey, buildDocNoteScope } from './docs.js';
16
19
  import { ingestGitHistory } from './git-history.js';
17
20
  import { ParserPool } from './parser.js';
18
21
  import { ImportResolver } from './resolver.js';
19
22
  import { buildCallGraph } from './call-graph.js';
23
+ import { isPublicDeclarationSurfaceSymbol, } from './extractors/types.js';
20
24
  import { CExtractor } from './extractors/c.js';
21
25
  import { RustExtractor } from './extractors/rust.js';
22
26
  import { PythonExtractor } from './extractors/python.js';
@@ -41,9 +45,10 @@ import { HaskellExtractor } from './extractors/haskell.js';
41
45
  import { JuliaExtractor } from './extractors/julia.js';
42
46
  import { ElmExtractor } from './extractors/elm.js';
43
47
  import { ObjcExtractor } from './extractors/objc.js';
44
- import { DEFAULT_EMBEDDING_MODEL } from './embedder.js';
48
+ import { DEFAULT_EMBEDDING_MODEL, buildStructuralEmbeddingText } from './embedder.js';
45
49
  import { ingestCoverageReport } from './coverage.js';
46
50
  import { refreshTestMappings } from './test-mapper.js';
51
+ import { LspEnrichmentCoordinator } from './lsp/enrichment.js';
47
52
  // ─── Extractor registry ───────────────────────────────────────────────────────
48
53
  const EXTRACTORS = {
49
54
  c: new CExtractor(),
@@ -90,7 +95,10 @@ export class IndexBuilder {
90
95
  resolver;
91
96
  embedder;
92
97
  history;
98
+ indexDependencies;
93
99
  embeddingModel;
100
+ docsAutoNotes;
101
+ lspSettings;
94
102
  constructor(dbPath, walkerConfig, embedder, embeddingModelOrOptions) {
95
103
  this.dbPath = dbPath;
96
104
  this.walkerConfig = walkerConfig;
@@ -108,6 +116,9 @@ export class IndexBuilder {
108
116
  this.embedder = null;
109
117
  }
110
118
  this.history = opts.history ?? false;
119
+ this.docsAutoNotes = opts.docsAutoNotes ?? true;
120
+ this.indexDependencies = opts.indexDependencies ?? false;
121
+ this.lspSettings = opts.lsp ?? null;
111
122
  }
112
123
  // ─── Public API ──────────────────────────────────────────────────────────
113
124
  /**
@@ -118,8 +129,17 @@ export class IndexBuilder {
118
129
  async build() {
119
130
  const db = openDb(this.dbPath);
120
131
  const branch = this.resolveBranch();
132
+ const lspCoordinator = this.createLspEnrichmentCoordinator();
121
133
  try {
134
+ this.saveDocsAutoNotesSetting(db);
122
135
  const files = await walkFiles(this.walkerConfig);
136
+ const docs = await walkDocumentationFiles(this.walkerConfig);
137
+ if (lspCoordinator) {
138
+ const languages = new Set(files.map((file) => file.language));
139
+ if (this.indexDependencies)
140
+ languages.add('typescript');
141
+ await lspCoordinator.start(languages);
142
+ }
123
143
  const resumeAt = this.loadBuildCheckpoint(db, branch, files.length);
124
144
  db.transaction(() => {
125
145
  for (let i = resumeAt; i < files.length; i++) {
@@ -129,15 +149,25 @@ export class IndexBuilder {
129
149
  this.processFile(db, file.path, file.language, branch);
130
150
  this.saveBuildCheckpoint(db, branch, i + 1, files.length);
131
151
  }
152
+ const seenDocPaths = new Set();
153
+ for (const doc of docs) {
154
+ seenDocPaths.add(doc.path);
155
+ this.processDocumentationFile(db, doc, branch);
156
+ this.upsertSeededDocumentationNote(db, doc, branch);
157
+ }
158
+ this.removeStaleDocumentation(db, branch, seenDocPaths);
132
159
  })();
133
160
  this.saveBuildCheckpoint(db, branch, files.length, files.length);
134
161
  this.resolveImports(db, branch);
162
+ await this.indexDependencyDeclarations(db, lspCoordinator);
163
+ await this.enrichProjectSymbolsAndCallRefs(db, branch, files, lspCoordinator);
135
164
  refreshTestMappings(db, branch);
136
165
  buildCallGraph(db);
137
166
  this.saveLastKnownHead(db);
138
167
  if (this.embedder) {
139
168
  await this.embedder.init();
140
169
  await this.embedStructural(db);
170
+ await this.embedDocumentation(db);
141
171
  }
142
172
  if (this.history) {
143
173
  const historyOptions = typeof this.history === 'object' ? this.history : undefined;
@@ -145,6 +175,9 @@ export class IndexBuilder {
145
175
  }
146
176
  }
147
177
  finally {
178
+ if (lspCoordinator) {
179
+ await lspCoordinator.dispose();
180
+ }
148
181
  db.close();
149
182
  }
150
183
  }
@@ -157,7 +190,25 @@ export class IndexBuilder {
157
190
  async update(changedFiles) {
158
191
  const db = openDb(this.dbPath);
159
192
  const branch = this.resolveBranch();
193
+ const lspCoordinator = this.createLspEnrichmentCoordinator();
194
+ const enrichedFiles = [];
160
195
  try {
196
+ this.saveDocsAutoNotesSetting(db);
197
+ const docs = await walkDocumentationFiles(this.walkerConfig);
198
+ const docsByPath = new Map(docs.map(doc => [doc.path, doc]));
199
+ if (lspCoordinator) {
200
+ const languages = new Set();
201
+ for (const filePath of changedFiles) {
202
+ if (!fs.existsSync(filePath))
203
+ continue;
204
+ const language = detectLanguageForPath(filePath, this.walkerConfig);
205
+ if (language)
206
+ languages.add(language);
207
+ }
208
+ if (this.indexDependencies)
209
+ languages.add('typescript');
210
+ await lspCoordinator.start(languages);
211
+ }
161
212
  db.transaction(() => {
162
213
  for (const filePath of changedFiles) {
163
214
  // If the file no longer exists, remove it from the DB
@@ -169,23 +220,35 @@ export class IndexBuilder {
169
220
  db.prepare('DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_id = ?)').run(row.id);
170
221
  db.prepare('DELETE FROM files WHERE id = ?').run(row.id);
171
222
  }
223
+ this.deleteDocumentationByPath(db, filePath, branch);
172
224
  continue;
173
225
  }
174
226
  const language = detectLanguageForPath(filePath, this.walkerConfig);
175
- if (!language)
176
- continue;
177
- // Null out resolved_id references pointing to this file before deletion
178
- const existingRow = db.prepare('SELECT id FROM files WHERE path = ? AND branch = ?').get(filePath, branch);
179
- if (existingRow) {
180
- db.prepare('UPDATE file_imports SET resolved_id = NULL WHERE resolved_id = ?').run(existingRow.id);
181
- db.prepare('DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_id = ?)').run(existingRow.id);
227
+ if (language) {
228
+ enrichedFiles.push({ path: filePath, language });
229
+ // Null out resolved_id references pointing to this file before deletion
230
+ const existingRow = db.prepare('SELECT id FROM files WHERE path = ? AND branch = ?').get(filePath, branch);
231
+ if (existingRow) {
232
+ db.prepare('UPDATE file_imports SET resolved_id = NULL WHERE resolved_id = ?').run(existingRow.id);
233
+ db.prepare('DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_id = ?)').run(existingRow.id);
234
+ }
235
+ // Delete existing rows for this file (cascade handles symbols/imports)
236
+ db.prepare('DELETE FROM files WHERE path = ? AND branch = ?').run(filePath, branch);
237
+ this.processFile(db, filePath, language, branch);
238
+ }
239
+ const changedDoc = docsByPath.get(filePath);
240
+ if (changedDoc) {
241
+ this.processDocumentationFile(db, changedDoc, branch);
242
+ this.upsertSeededDocumentationNote(db, changedDoc, branch);
243
+ }
244
+ else {
245
+ this.deleteDocumentationByPath(db, filePath, branch);
182
246
  }
183
- // Delete existing rows for this file (cascade handles symbols/imports)
184
- db.prepare('DELETE FROM files WHERE path = ? AND branch = ?').run(filePath, branch);
185
- this.processFile(db, filePath, language, branch);
186
247
  }
187
248
  })();
188
249
  this.resolveImports(db, branch);
250
+ await this.indexDependencyDeclarations(db, lspCoordinator);
251
+ await this.enrichProjectSymbolsAndCallRefs(db, branch, enrichedFiles, lspCoordinator);
189
252
  refreshTestMappings(db, branch);
190
253
  if (this.history) {
191
254
  const historyOptions = typeof this.history === 'object' ? this.history : undefined;
@@ -194,11 +257,15 @@ export class IndexBuilder {
194
257
  if (this.embedder) {
195
258
  await this.embedder.init();
196
259
  await this.embedStructural(db);
260
+ await this.embedDocumentation(db);
197
261
  }
198
262
  buildCallGraph(db);
199
263
  this.saveLastKnownHead(db);
200
264
  }
201
265
  finally {
266
+ if (lspCoordinator) {
267
+ await lspCoordinator.dispose();
268
+ }
202
269
  db.close();
203
270
  }
204
271
  }
@@ -299,7 +366,10 @@ export class IndexBuilder {
299
366
  const info = insertSymbol.run(fileId, sym.name, sym.kind, sym.startLine, sym.endLine, sym.signature ?? null, sym.docComment ?? null);
300
367
  const symId = Number(info.lastInsertRowid);
301
368
  symbolIdMap.set(sym.name, symId);
302
- insertFts.run(symId, sym.name, sym.signature ?? '', sym.kind);
369
+ insertFts.run(symId, sym.name, buildStructuralEmbeddingText({
370
+ name: sym.name,
371
+ signature: sym.signature ?? null,
372
+ }), sym.kind);
303
373
  }
304
374
  const insertRoute = db.prepare(`INSERT INTO api_routes (file_id, method, path, handler_id, handler_name, framework, line, middleware)
305
375
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
@@ -321,6 +391,95 @@ export class IndexBuilder {
321
391
  }
322
392
  }
323
393
  }
394
+ processDocumentationFile(db, doc, branch) {
395
+ const existing = db.prepare('SELECT id, content_hash FROM docs WHERE path = ? AND branch = ?').get(doc.path, branch);
396
+ if (existing?.content_hash === doc.hash) {
397
+ return;
398
+ }
399
+ let docId;
400
+ if (existing) {
401
+ db.prepare(`UPDATE docs
402
+ SET kind = ?, title = ?, content = ?, content_hash = ?, indexed_at = unixepoch()
403
+ WHERE id = ?`).run(doc.kind, doc.title, doc.content, doc.hash, existing.id);
404
+ docId = existing.id;
405
+ }
406
+ else {
407
+ const info = db.prepare(`INSERT INTO docs (path, branch, kind, title, content, content_hash)
408
+ VALUES (?, ?, ?, ?, ?, ?)`).run(doc.path, branch, doc.kind, doc.title, doc.content, doc.hash);
409
+ docId = Number(info.lastInsertRowid);
410
+ }
411
+ const existingSections = db.prepare('SELECT id, section_index FROM doc_sections WHERE doc_id = ?').all(docId);
412
+ const insertSection = db.prepare(`INSERT INTO doc_sections (
413
+ doc_id, section_index, title, depth, heading_path, line_start, line_end, content, content_hash
414
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
415
+ ON CONFLICT(doc_id, section_index) DO UPDATE SET
416
+ title = excluded.title,
417
+ depth = excluded.depth,
418
+ heading_path = excluded.heading_path,
419
+ line_start = excluded.line_start,
420
+ line_end = excluded.line_end,
421
+ content = excluded.content,
422
+ content_hash = excluded.content_hash`);
423
+ const activeSectionIndexes = new Set();
424
+ for (const chunk of doc.chunks) {
425
+ activeSectionIndexes.add(chunk.sectionIndex);
426
+ insertSection.run(docId, chunk.sectionIndex, chunk.title, chunk.depth, JSON.stringify(chunk.headingPath), chunk.lineStart, chunk.lineEnd, chunk.content, chunk.hash);
427
+ }
428
+ const staleSectionIds = existingSections
429
+ .filter(section => !activeSectionIndexes.has(section.section_index))
430
+ .map(section => section.id);
431
+ this.deleteDocSectionEmbeddings(db, staleSectionIds);
432
+ if (staleSectionIds.length > 0) {
433
+ db.prepare(`DELETE FROM doc_sections
434
+ WHERE id IN (${staleSectionIds.map(() => '?').join(', ')})`).run(...staleSectionIds);
435
+ }
436
+ }
437
+ upsertSeededDocumentationNote(db, doc, branch) {
438
+ if (!this.docsAutoNotes)
439
+ return;
440
+ const key = inferSeededDocNoteKey(doc);
441
+ if (!key)
442
+ return;
443
+ const scope = buildDocNoteScope(doc.path, branch);
444
+ const existing = db.prepare('SELECT content, source_hash FROM notes WHERE key = ? AND scope = ?').get(key, scope);
445
+ if (existing?.content === doc.content && existing.source_hash === doc.hash) {
446
+ return;
447
+ }
448
+ db.prepare(`INSERT INTO notes (key, scope, content, model, source_hash, created_at, updated_at)
449
+ VALUES (?, ?, ?, ?, ?, unixepoch(), unixepoch())
450
+ ON CONFLICT(key, scope) DO UPDATE SET
451
+ content = excluded.content,
452
+ model = excluded.model,
453
+ source_hash = excluded.source_hash,
454
+ updated_at = unixepoch()`).run(key, scope, doc.content, 'system:auto-doc-seed', doc.hash);
455
+ }
456
+ removeStaleDocumentation(db, branch, retainedPaths) {
457
+ const docs = db.prepare('SELECT id, path FROM docs WHERE branch = ?').all(branch);
458
+ for (const doc of docs) {
459
+ if (!retainedPaths.has(doc.path)) {
460
+ this.deleteDocumentationById(db, doc.id);
461
+ }
462
+ }
463
+ }
464
+ deleteDocumentationByPath(db, docPath, branch) {
465
+ const row = db.prepare('SELECT id FROM docs WHERE path = ? AND branch = ?').get(docPath, branch);
466
+ if (!row)
467
+ return;
468
+ this.deleteDocumentationById(db, row.id);
469
+ }
470
+ deleteDocumentationById(db, docId) {
471
+ const sectionIds = db.prepare('SELECT id FROM doc_sections WHERE doc_id = ?').all(docId);
472
+ this.deleteDocSectionEmbeddings(db, sectionIds.map(row => row.id));
473
+ db.prepare('DELETE FROM docs WHERE id = ?').run(docId);
474
+ }
475
+ deleteDocSectionEmbeddings(db, sectionIds) {
476
+ if (sectionIds.length === 0)
477
+ return;
478
+ const hasEmbeddingsTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type IN ('table', 'virtual table') AND name = 'doc_section_embeddings'").get();
479
+ if (!hasEmbeddingsTable)
480
+ return;
481
+ db.prepare(`DELETE FROM doc_section_embeddings WHERE rowid IN (${sectionIds.map(() => '?').join(', ')})`).run(...sectionIds);
482
+ }
324
483
  /**
325
484
  * Second pass: resolve raw_import strings to file IDs in the
326
485
  * `file_imports.resolved_id` column. Also populates `external_deps` for
@@ -352,6 +511,146 @@ export class IndexBuilder {
352
511
  }
353
512
  }
354
513
  }
514
+ async indexDependencyDeclarations(db, lspCoordinator) {
515
+ db.prepare('DELETE FROM external_symbols').run();
516
+ if (!this.indexDependencies)
517
+ return;
518
+ const directDependencies = this.loadDirectDependencies();
519
+ if (directDependencies.size === 0)
520
+ return;
521
+ const extractor = EXTRACTORS.typescript;
522
+ if (!extractor)
523
+ return;
524
+ const insertExternalSymbol = db.prepare(`INSERT OR IGNORE INTO external_symbols
525
+ (
526
+ package_name,
527
+ package_version,
528
+ source_ref,
529
+ symbol_name,
530
+ symbol_kind,
531
+ signature,
532
+ doc_comment,
533
+ resolved_type_signature,
534
+ resolved_return_type,
535
+ definition_uri,
536
+ definition_path
537
+ )
538
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
539
+ for (const [packageName, declaredVersion] of directDependencies) {
540
+ const packageDir = path.join(this.walkerConfig.rootDir, 'node_modules', packageName);
541
+ if (!fs.existsSync(packageDir) || !fs.statSync(packageDir).isDirectory())
542
+ continue;
543
+ const packageVersion = this.readInstalledPackageVersion(packageDir) ?? declaredVersion ?? null;
544
+ const declarationFiles = this.collectDeclarationFiles(packageDir);
545
+ for (const declarationFile of declarationFiles) {
546
+ const source = fs.readFileSync(declarationFile, 'utf8');
547
+ const tree = this.pool.parse('typescript', source);
548
+ if (!tree)
549
+ continue;
550
+ const result = extractor.extract(tree, source, declarationFile);
551
+ const declarationSymbols = result.symbols.filter((symbol) => this.shouldIndexDependencySymbol(symbol));
552
+ const enrichmentRows = lspCoordinator
553
+ ? await lspCoordinator.enrich({
554
+ filePath: declarationFile,
555
+ language: 'typescript',
556
+ source,
557
+ targets: declarationSymbols.map((symbol) => ({
558
+ line: symbol.startLine,
559
+ character: symbol.startCharacter ?? 0,
560
+ })),
561
+ })
562
+ : declarationSymbols.map(() => null);
563
+ for (let i = 0; i < declarationSymbols.length; i++) {
564
+ const symbol = declarationSymbols[i];
565
+ if (!symbol)
566
+ continue;
567
+ const metadata = enrichmentRows[i];
568
+ insertExternalSymbol.run(packageName, packageVersion, declarationFile, symbol.name, symbol.kind, symbol.signature, symbol.docComment ?? null, metadata?.resolvedTypeSignature ?? null, metadata?.resolvedReturnType ?? null, metadata?.definitionUri ?? null, metadata?.definitionPath ?? null);
569
+ }
570
+ }
571
+ }
572
+ }
573
+ createLspEnrichmentCoordinator() {
574
+ if (!this.lspSettings?.enabled) {
575
+ return null;
576
+ }
577
+ return new LspEnrichmentCoordinator(this.lspSettings, this.walkerConfig.rootDir);
578
+ }
579
+ async enrichProjectSymbolsAndCallRefs(db, branch, files, lspCoordinator) {
580
+ if (!lspCoordinator || files.length === 0)
581
+ return;
582
+ const selectSymbols = db.prepare(`SELECT s.id, s.name, s.signature, s.start_line
583
+ FROM symbols s
584
+ JOIN files f ON f.id = s.file_id
585
+ WHERE f.path = ? AND f.branch = ?
586
+ ORDER BY s.id`);
587
+ const selectCallRefs = db.prepare(`SELECT sr.id, sr.call_line
588
+ FROM symbol_refs sr
589
+ JOIN symbols s ON s.id = sr.caller_id
590
+ JOIN files f ON f.id = s.file_id
591
+ WHERE f.path = ? AND f.branch = ?
592
+ ORDER BY sr.id`);
593
+ const updateSymbol = db.prepare(`UPDATE symbols
594
+ SET resolved_type_signature = ?, resolved_return_type = ?, definition_uri = ?, definition_path = ?
595
+ WHERE id = ?`);
596
+ const updateSymbolFts = db.prepare('UPDATE symbols_fts SET signature = ? WHERE rowid = ?');
597
+ const updateCallRef = db.prepare(`UPDATE symbol_refs
598
+ SET resolved_type_signature = ?, resolved_return_type = ?, definition_uri = ?, definition_path = ?
599
+ WHERE id = ?`);
600
+ for (const file of files) {
601
+ if (!file || !fs.existsSync(file.path))
602
+ continue;
603
+ let source;
604
+ try {
605
+ source = fs.readFileSync(file.path, 'utf8');
606
+ }
607
+ catch {
608
+ continue;
609
+ }
610
+ const symbols = selectSymbols.all(file.path, branch);
611
+ if (symbols.length > 0) {
612
+ const symbolMetadata = await lspCoordinator.enrich({
613
+ filePath: file.path,
614
+ language: file.language,
615
+ source,
616
+ targets: symbols.map((symbol) => ({ line: symbol.start_line, character: 0 })),
617
+ });
618
+ for (let i = 0; i < symbols.length; i++) {
619
+ const symbol = symbols[i];
620
+ if (!symbol)
621
+ continue;
622
+ const metadata = symbolMetadata[i];
623
+ if (!metadata)
624
+ continue;
625
+ updateSymbol.run(metadata.resolvedTypeSignature, metadata.resolvedReturnType, metadata.definitionUri, metadata.definitionPath, symbol.id);
626
+ updateSymbolFts.run(buildStructuralEmbeddingText({
627
+ name: symbol.name,
628
+ signature: symbol.signature,
629
+ resolvedTypeSignature: metadata.resolvedTypeSignature,
630
+ resolvedReturnType: metadata.resolvedReturnType,
631
+ }), symbol.id);
632
+ }
633
+ }
634
+ const callRefs = selectCallRefs.all(file.path, branch);
635
+ if (callRefs.length > 0) {
636
+ const callRefMetadata = await lspCoordinator.enrich({
637
+ filePath: file.path,
638
+ language: file.language,
639
+ source,
640
+ targets: callRefs.map((callRef) => ({ line: callRef.call_line, character: 0 })),
641
+ });
642
+ for (let i = 0; i < callRefs.length; i++) {
643
+ const callRef = callRefs[i];
644
+ if (!callRef)
645
+ continue;
646
+ const metadata = callRefMetadata[i];
647
+ if (!metadata)
648
+ continue;
649
+ updateCallRef.run(metadata.resolvedTypeSignature, metadata.resolvedReturnType, metadata.definitionUri, metadata.definitionPath, callRef.id);
650
+ }
651
+ }
652
+ }
653
+ }
355
654
  resolveBranch() {
356
655
  if (this.walkerConfig.branch)
357
656
  return this.walkerConfig.branch;
@@ -363,6 +662,9 @@ export class IndexBuilder {
363
662
  setKbMeta(db, KB_META_LAST_HEAD_SHA, headSha);
364
663
  }
365
664
  }
665
+ saveDocsAutoNotesSetting(db) {
666
+ setKbMeta(db, 'docs_auto_notes', this.docsAutoNotes ? '1' : '0');
667
+ }
366
668
  readGitValue(args) {
367
669
  try {
368
670
  const value = execFileSync('git', ['-C', this.walkerConfig.rootDir, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
@@ -372,6 +674,80 @@ export class IndexBuilder {
372
674
  return undefined;
373
675
  }
374
676
  }
677
+ loadDirectDependencies() {
678
+ const packageJsonPath = path.join(this.walkerConfig.rootDir, 'package.json');
679
+ if (!fs.existsSync(packageJsonPath))
680
+ return new Map();
681
+ const raw = fs.readFileSync(packageJsonPath, 'utf8');
682
+ const pkg = JSON.parse(raw);
683
+ const deps = new Map();
684
+ for (const section of [pkg.dependencies, pkg.devDependencies, pkg.peerDependencies]) {
685
+ if (!section)
686
+ continue;
687
+ for (const [name, version] of Object.entries(section)) {
688
+ if (!deps.has(name))
689
+ deps.set(name, version);
690
+ }
691
+ }
692
+ return deps;
693
+ }
694
+ readInstalledPackageVersion(packageDir) {
695
+ const packageJsonPath = path.join(packageDir, 'package.json');
696
+ if (!fs.existsSync(packageJsonPath))
697
+ return undefined;
698
+ const raw = fs.readFileSync(packageJsonPath, 'utf8');
699
+ const pkg = JSON.parse(raw);
700
+ return pkg.version;
701
+ }
702
+ collectDeclarationFiles(packageDir) {
703
+ const declarations = [];
704
+ const stack = [packageDir];
705
+ while (stack.length > 0) {
706
+ const currentDir = stack.pop();
707
+ if (!currentDir)
708
+ continue;
709
+ const entries = fs.readdirSync(currentDir, { withFileTypes: true });
710
+ for (const entry of entries) {
711
+ if (entry.name === 'node_modules')
712
+ continue;
713
+ const fullPath = path.join(currentDir, entry.name);
714
+ if (entry.isDirectory()) {
715
+ stack.push(fullPath);
716
+ continue;
717
+ }
718
+ if (entry.isFile() && fullPath.endsWith('.d.ts')) {
719
+ declarations.push(fullPath);
720
+ }
721
+ }
722
+ }
723
+ return declarations;
724
+ }
725
+ shouldIndexDependencySymbol(symbol) {
726
+ if (!isPublicDeclarationSurfaceSymbol(symbol))
727
+ return false;
728
+ if (symbol.declarationSurface)
729
+ return true;
730
+ return !this.hasImplementationBody(symbol);
731
+ }
732
+ hasImplementationBody(symbol) {
733
+ const node = symbol.astNode;
734
+ if (!node)
735
+ return false;
736
+ if (node.type === 'arrow_function' ||
737
+ node.type === 'function_expression' ||
738
+ node.type === 'generator_function') {
739
+ return true;
740
+ }
741
+ if (node.type === 'class_declaration' ||
742
+ node.type === 'interface_declaration' ||
743
+ node.type === 'type_alias_declaration') {
744
+ return false;
745
+ }
746
+ const bodyNode = node.childForFieldName('body');
747
+ if (!bodyNode)
748
+ return false;
749
+ return bodyNode.namedChildCount > 0 || bodyNode.text.trim() !== '';
750
+ }
375
751
  loadBuildCheckpoint(db, branch, totalFiles) {
376
752
  const raw = getKbMeta(db, KB_META_INDEX_CHECKPOINT);
377
753
  if (!raw)
@@ -409,14 +785,23 @@ export class IndexBuilder {
409
785
  setKbMeta(db, 'embedding_model', embedder.modelName);
410
786
  setKbMeta(db, 'embedding_dims', String(embedder.dims));
411
787
  createVec0Tables(db, embedder.dims);
412
- // Fetch all symbols that have a signature to embed.
788
+ // Fetch all symbols that have structural text to embed.
413
789
  const symbols = db
414
- .prepare('SELECT id, name, signature FROM symbols WHERE signature IS NOT NULL')
790
+ .prepare(`SELECT id, name, signature, resolved_type_signature, resolved_return_type
791
+ FROM symbols
792
+ WHERE signature IS NOT NULL
793
+ OR resolved_type_signature IS NOT NULL
794
+ OR resolved_return_type IS NOT NULL`)
415
795
  .all();
416
796
  const insertEmbed = db.prepare('INSERT OR REPLACE INTO symbol_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), json(?))');
417
797
  for (let i = 0; i < symbols.length; i += EMBED_BATCH_SIZE) {
418
798
  const batch = symbols.slice(i, i + EMBED_BATCH_SIZE);
419
- const texts = batch.map(s => s.signature || s.name);
799
+ const texts = batch.map((symbol) => buildStructuralEmbeddingText({
800
+ name: symbol.name,
801
+ signature: symbol.signature,
802
+ resolvedTypeSignature: symbol.resolved_type_signature,
803
+ resolvedReturnType: symbol.resolved_return_type,
804
+ }));
420
805
  const embeddings = await embedder.embed(texts);
421
806
  db.transaction(() => {
422
807
  for (let j = 0; j < batch.length; j++) {
@@ -427,5 +812,32 @@ export class IndexBuilder {
427
812
  })();
428
813
  }
429
814
  }
815
+ async embedDocumentation(db) {
816
+ const embedder = this.embedder;
817
+ db.exec(`
818
+ CREATE VIRTUAL TABLE IF NOT EXISTS doc_section_embeddings USING vec0(
819
+ embedding FLOAT[${embedder.dims}]
820
+ );
821
+ `);
822
+ const sections = db.prepare(`SELECT id, title, content
823
+ FROM doc_sections
824
+ ORDER BY id`).all();
825
+ if (sections.length === 0)
826
+ return;
827
+ const insertEmbed = db.prepare('INSERT OR REPLACE INTO doc_section_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), json(?))');
828
+ for (let i = 0; i < sections.length; i += EMBED_BATCH_SIZE) {
829
+ const batch = sections.slice(i, i + EMBED_BATCH_SIZE);
830
+ const texts = batch.map(section => section.content || section.title);
831
+ const embeddings = await embedder.embed(texts);
832
+ db.transaction(() => {
833
+ for (let j = 0; j < batch.length; j++) {
834
+ const section = batch[j];
835
+ if (section) {
836
+ insertEmbed.run(section.id, JSON.stringify(embeddings[j]));
837
+ }
838
+ }
839
+ })();
840
+ }
841
+ }
430
842
  }
431
843
  //# sourceMappingURL=index.js.map