@danypops/lector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +57 -0
  3. package/package.json +48 -0
  4. package/src/adapters/directory-size.ts +19 -0
  5. package/src/adapters/find-source-files.ts +35 -0
  6. package/src/adapters/git-repo-fetcher.ts +146 -0
  7. package/src/adapters/in-memory-content-cache.ts +19 -0
  8. package/src/adapters/in-memory-search-cache.ts +32 -0
  9. package/src/adapters/in-memory-symbol-graph.ts +80 -0
  10. package/src/adapters/in-memory-workspace.ts +28 -0
  11. package/src/adapters/local-filesystem-workspace.ts +96 -0
  12. package/src/adapters/local-git.ts +73 -0
  13. package/src/adapters/lsp/discover-seed-file.ts +130 -0
  14. package/src/adapters/lsp/json-rpc-stream.ts +67 -0
  15. package/src/adapters/lsp/language-server-process.ts +184 -0
  16. package/src/adapters/lsp/lsp-symbol-index.ts +470 -0
  17. package/src/adapters/lsp/process-resource-usage.ts +61 -0
  18. package/src/adapters/lsp/typescript-project-files.ts +51 -0
  19. package/src/adapters/read-only-workspace.ts +23 -0
  20. package/src/adapters/ripgrep-text-search.ts +97 -0
  21. package/src/adapters/source-manifest.ts +44 -0
  22. package/src/adapters/sqlite-content-cache.ts +67 -0
  23. package/src/adapters/sqlite-search-cache.ts +60 -0
  24. package/src/adapters/sqlite-symbol-graph.ts +154 -0
  25. package/src/adapters/tiered-search-cache.ts +29 -0
  26. package/src/adapters/tree-sitter/typescript-tree-sitter-symbol-index.ts +155 -0
  27. package/src/cli.ts +777 -0
  28. package/src/client.ts +63 -0
  29. package/src/constants.ts +16 -0
  30. package/src/daemon.ts +166 -0
  31. package/src/domain/assert-safe-git-argument.ts +19 -0
  32. package/src/domain/assert-safe-path-segment.ts +17 -0
  33. package/src/domain/assert-safe-repo-reference.ts +20 -0
  34. package/src/domain/assert-safe-search-query.ts +17 -0
  35. package/src/domain/bounded-job-executor.ts +219 -0
  36. package/src/domain/call-hierarchy.ts +23 -0
  37. package/src/domain/code-range.ts +11 -0
  38. package/src/domain/content-hash.ts +14 -0
  39. package/src/domain/diagnostic.ts +12 -0
  40. package/src/domain/diagnostics.ts +7 -0
  41. package/src/domain/document-symbol.ts +19 -0
  42. package/src/domain/document-symbols.ts +7 -0
  43. package/src/domain/exact-edit.ts +43 -0
  44. package/src/domain/find-references.ts +7 -0
  45. package/src/domain/find-workspace-symbols.ts +7 -0
  46. package/src/domain/git-diff-result.ts +5 -0
  47. package/src/domain/git-log-entry.ts +9 -0
  48. package/src/domain/git-status.ts +17 -0
  49. package/src/domain/go-to-definition.ts +7 -0
  50. package/src/domain/go-to-implementation.ts +7 -0
  51. package/src/domain/hover-at.ts +8 -0
  52. package/src/domain/hover.ts +7 -0
  53. package/src/domain/incoming-calls.ts +8 -0
  54. package/src/domain/language-server-descriptor.ts +122 -0
  55. package/src/domain/outgoing-calls.ts +8 -0
  56. package/src/domain/populate-symbol-graph.ts +91 -0
  57. package/src/domain/prepare-call-hierarchy.ts +8 -0
  58. package/src/domain/race-workspace-query.ts +39 -0
  59. package/src/domain/raw-read.ts +24 -0
  60. package/src/domain/reachable-symbols-from.ts +13 -0
  61. package/src/domain/repo-fetch-result.ts +18 -0
  62. package/src/domain/repo-reference.ts +7 -0
  63. package/src/domain/search-cache-key.ts +16 -0
  64. package/src/domain/search-text.ts +29 -0
  65. package/src/domain/symbol-edges-from.ts +9 -0
  66. package/src/domain/symbol-edges-to.ts +9 -0
  67. package/src/domain/symbol-graph-generation.ts +14 -0
  68. package/src/domain/symbol-node-id.ts +13 -0
  69. package/src/domain/text-search-result.ts +15 -0
  70. package/src/domain/workspace-query-outcome.ts +24 -0
  71. package/src/domain/workspace-symbol.ts +14 -0
  72. package/src/index.ts +121 -0
  73. package/src/ports/code-intelligence-port.ts +38 -0
  74. package/src/ports/content-cache-port.ts +52 -0
  75. package/src/ports/git-port.ts +23 -0
  76. package/src/ports/repo-fetcher-port.ts +11 -0
  77. package/src/ports/search-cache-port.ts +15 -0
  78. package/src/ports/symbol-graph-port.ts +35 -0
  79. package/src/ports/symbol-index-port.ts +11 -0
  80. package/src/ports/text-search-port.ts +12 -0
  81. package/src/ports/workspace-port.ts +35 -0
  82. package/src/service.ts +961 -0
  83. package/src/version.ts +6 -0
@@ -0,0 +1,97 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createInterface } from "node:readline";
3
+ import { assertSafeSearchQuery } from "../domain/assert-safe-search-query.ts";
4
+ import type { TextSearchMatch, TextSearchResult } from "../domain/text-search-result.ts";
5
+ import type { TextSearchOptions, TextSearchPort } from "../ports/text-search-port.ts";
6
+ import { SKIP_DIRECTORY_NAMES } from "./find-source-files.ts";
7
+
8
+ // ripgrep only skips a directory automatically when a real .gitignore names it -- verified
9
+ // empirically (a fixture with no .gitignore let a bare `rg` search node_modules freely).
10
+ // Explicit globs make the same bound findSourceFiles already enforces hold here too, regardless
11
+ // of whether the target repo's own .gitignore happens to list these directories.
12
+ const EXCLUDE_GLOBS = Array.from(SKIP_DIRECTORY_NAMES, (name) => ["--glob", `!${name}/`]).flat();
13
+
14
+ interface RipgrepMatchEvent {
15
+ readonly type: "match";
16
+ readonly data: {
17
+ readonly path: { readonly text: string };
18
+ readonly lines: { readonly text: string };
19
+ readonly line_number: number;
20
+ readonly submatches: readonly { readonly start: number; readonly end: number }[];
21
+ };
22
+ }
23
+
24
+ function isMatchEvent(value: unknown): value is RipgrepMatchEvent {
25
+ return typeof value === "object" && value !== null && (value as { type?: unknown }).type === "match";
26
+ }
27
+
28
+ /**
29
+ * TextSearchPort backed by real ripgrep (`rg --json`), streamed line-by-line via readline
30
+ * rather than buffered wholesale (execFile's default maxBuffer would either truncate or throw
31
+ * on a search producing megabytes of matches). The moment a bound is hit the child process is
32
+ * killed outright, not just stopped-reading-from -- a search over a huge monorepo must not keep
33
+ * scanning to completion just because this adapter already has enough matches.
34
+ *
35
+ * ripgrep already respects a real .gitignore and skips hidden/binary files by default, but a
36
+ * repo with no .gitignore listing node_modules gets searched into it freely -- explicit
37
+ * --glob exclusions (the same skip-list findSourceFiles uses) make that bound unconditional.
38
+ */
39
+ export class RipgrepTextSearch implements TextSearchPort {
40
+ async search(rootPath: string, query: string, options: TextSearchOptions): Promise<TextSearchResult> {
41
+ assertSafeSearchQuery(query);
42
+ // `--` marks the end of flags -- defense in depth beyond assertSafeSearchQuery, the same
43
+ // two-layer approach local-git.ts already uses for git's diff ref.
44
+ const child = spawn("rg", ["--json", ...EXCLUDE_GLOBS, "--", query], { cwd: rootPath, stdio: ["ignore", "pipe", "pipe"] });
45
+
46
+ // Registered before draining stdout, not after: an 'exit' event fired while this adapter
47
+ // was still reading matches would otherwise be missed entirely, hanging the process-exit
48
+ // wait forever.
49
+ const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
50
+ child.once("exit", (code, signal) => resolve({ code, signal }));
51
+ child.once("error", reject);
52
+ });
53
+
54
+ const matches: TextSearchMatch[] = [];
55
+ let bytesUsed = 0;
56
+ let truncated = false;
57
+
58
+ const rl = createInterface({ input: child.stdout });
59
+ for await (const line of rl) {
60
+ let event: unknown;
61
+ try {
62
+ event = JSON.parse(line);
63
+ } catch {
64
+ continue;
65
+ }
66
+ if (!isMatchEvent(event)) continue;
67
+
68
+ const lineText = event.data.lines.text;
69
+ const submatch = event.data.submatches[0];
70
+ matches.push({
71
+ path: event.data.path.text.replace(/^\.\//, ""),
72
+ lineNumber: event.data.line_number,
73
+ line: lineText,
74
+ matchStart: submatch?.start ?? 0,
75
+ matchEnd: submatch?.end ?? 0,
76
+ });
77
+ bytesUsed += Buffer.byteLength(lineText, "utf8");
78
+
79
+ if (matches.length >= options.maxMatches || bytesUsed >= options.maxBytes) {
80
+ truncated = true;
81
+ child.kill();
82
+ break;
83
+ }
84
+ }
85
+ rl.close();
86
+
87
+ const { code } = await exited;
88
+ // rg's own exit codes: 0 = matches found, 1 = no matches, 2 = a real error (e.g. an
89
+ // invalid regex). A kill this adapter itself issued is expected and never an error,
90
+ // whatever exit code/signal it produced.
91
+ if (!truncated && code !== 0 && code !== 1) {
92
+ throw new Error(`rg exited with code ${code}`);
93
+ }
94
+
95
+ return { matches, truncated };
96
+ }
97
+ }
@@ -0,0 +1,44 @@
1
+ import { createHash } from "node:crypto";
2
+ import { createReadStream } from "node:fs";
3
+ import { stat } from "node:fs/promises";
4
+ import { join } from "node:path";
5
+ import { findSourceFiles } from "./find-source-files.ts";
6
+
7
+ export interface SourceManifest {
8
+ readonly fingerprint: string;
9
+ readonly absoluteFiles: readonly string[];
10
+ }
11
+
12
+ export class SourceManifestLimitExceeded extends Error {
13
+ constructor(readonly maxBytes: number) {
14
+ super(`source manifest exceeds the ${maxBytes}-byte hashing bound`);
15
+ this.name = "SourceManifestLimitExceeded";
16
+ }
17
+ }
18
+
19
+ /** Hashes the same bounded, sorted source-file set population will consume. */
20
+ export async function deriveSourceManifest(rootPath: string, extensions: readonly string[], maxFiles: number, maxBytes: number): Promise<SourceManifest> {
21
+ if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new RangeError("maxBytes must be a positive safe integer");
22
+ const relativeFiles = findSourceFiles(rootPath, (extension) => extensions.includes(extension), maxFiles);
23
+ const hash = createHash("sha256");
24
+ const absoluteFiles: string[] = [];
25
+ let bytesHashed = 0;
26
+ for (const relativePath of relativeFiles) {
27
+ const absolutePath = join(rootPath, relativePath);
28
+ const expectedSize = (await stat(absolutePath)).size;
29
+ if (bytesHashed + expectedSize > maxBytes) throw new SourceManifestLimitExceeded(maxBytes);
30
+ hash.update(String(Buffer.byteLength(relativePath)));
31
+ hash.update(":");
32
+ hash.update(relativePath);
33
+ hash.update(":");
34
+ hash.update(String(expectedSize));
35
+ hash.update(":");
36
+ for await (const chunk of createReadStream(absolutePath)) {
37
+ bytesHashed += chunk.byteLength;
38
+ if (bytesHashed > maxBytes) throw new SourceManifestLimitExceeded(maxBytes);
39
+ hash.update(chunk);
40
+ }
41
+ absoluteFiles.push(absolutePath);
42
+ }
43
+ return { fingerprint: hash.digest("hex"), absoluteFiles };
44
+ }
@@ -0,0 +1,67 @@
1
+ import type { Database } from "bun:sqlite";
2
+ import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
3
+ import type { ContentHash } from "../domain/content-hash.ts";
4
+ import type { ContentCacheEntry, ContentCachePort, ContentSymbol } from "../ports/content-cache-port.ts";
5
+
6
+ const MIGRATIONS: Migration[] = [
7
+ {
8
+ version: 1,
9
+ up: (db) => {
10
+ db.exec("CREATE TABLE content_cache (hash TEXT PRIMARY KEY, raw_content TEXT, symbols_json TEXT)");
11
+ },
12
+ },
13
+ ];
14
+
15
+ interface ContentCacheRow {
16
+ raw_content: string | null;
17
+ symbols_json: string | null;
18
+ }
19
+
20
+ /**
21
+ * SQLite-backed ContentCachePort (via daemon-kit's migration-runner
22
+ * bootstrap -- same pragmas/versioning as every other @danypops daemon,
23
+ * not a bespoke SQLite setup). One row per ContentHash; a put for one lens
24
+ * only ever updates that lens's column (SQLite upsert with a targeted SET
25
+ * clause), never clobbering a lens already recorded for the same hash --
26
+ * this is what makes the single-row-per-hash design actually deliver "one
27
+ * shared store," not two columns that happen to live in the same table but
28
+ * still get overwritten independently.
29
+ *
30
+ * This is Lector's first genuinely durable state: unlike everything else
31
+ * built so far (in-memory registries, live LSP/tree-sitter queries), a
32
+ * cache entry written here is still present after the daemon restarts,
33
+ * pointed at the same database file (test: content-cache-port-conformance
34
+ * runs this exact "survives reopen" case against both adapters).
35
+ */
36
+ export class SqliteContentCache implements ContentCachePort {
37
+ private readonly db: Database;
38
+
39
+ constructor(path: string) {
40
+ this.db = openSqliteWithPragmas(path, { migrations: MIGRATIONS });
41
+ }
42
+
43
+ async get(hash: ContentHash): Promise<ContentCacheEntry | undefined> {
44
+ const row = this.db.query("SELECT raw_content, symbols_json FROM content_cache WHERE hash = ?").get(hash) as ContentCacheRow | null;
45
+ if (!row) return undefined;
46
+ const entry: { rawContent?: string; symbols?: readonly ContentSymbol[] } = {};
47
+ if (row.raw_content !== null) entry.rawContent = row.raw_content;
48
+ if (row.symbols_json !== null) entry.symbols = JSON.parse(row.symbols_json) as ContentSymbol[];
49
+ return entry;
50
+ }
51
+
52
+ async putRawContent(hash: ContentHash, content: string): Promise<void> {
53
+ this.db
54
+ .query("INSERT INTO content_cache (hash, raw_content) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET raw_content = excluded.raw_content")
55
+ .run(hash, content);
56
+ }
57
+
58
+ async putSymbols(hash: ContentHash, symbols: readonly ContentSymbol[]): Promise<void> {
59
+ this.db
60
+ .query("INSERT INTO content_cache (hash, symbols_json) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET symbols_json = excluded.symbols_json")
61
+ .run(hash, JSON.stringify(symbols));
62
+ }
63
+
64
+ close(): void {
65
+ this.db.close();
66
+ }
67
+ }
@@ -0,0 +1,60 @@
1
+ import type { Database } from "bun:sqlite";
2
+ import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
3
+ import { deriveSearchCacheKey, type SearchCacheKey } from "../domain/search-cache-key.ts";
4
+ import type { TextSearchResult } from "../domain/text-search-result.ts";
5
+ import type { SearchCachePort } from "../ports/search-cache-port.ts";
6
+
7
+ const MIGRATIONS: Migration[] = [
8
+ {
9
+ version: 1,
10
+ up: (db) => {
11
+ db.exec("CREATE TABLE search_cache (key TEXT PRIMARY KEY, result_json TEXT NOT NULL, expires_at INTEGER NOT NULL)");
12
+ },
13
+ },
14
+ ];
15
+
16
+ const DEFAULT_TTL_MS = 5 * 60 * 1000;
17
+
18
+ interface SearchCacheRow {
19
+ result_json: string;
20
+ expires_at: number;
21
+ }
22
+
23
+ export interface SqliteSearchCacheOptions {
24
+ readonly ttlMs?: number;
25
+ }
26
+
27
+ /**
28
+ * SQLite-backed SearchCachePort -- survives a daemon restart, unlike InMemorySearchCache.
29
+ * TTL is enforced by storing an explicit expires_at and filtering on read; there is no
30
+ * background sweep of expired rows (this is a cache, not a durability guarantee -- a stale row
31
+ * sitting unread on disk costs nothing until something actually asks for that exact key again).
32
+ */
33
+ export class SqliteSearchCache implements SearchCachePort {
34
+ private readonly db: Database;
35
+ private readonly ttlMs: number;
36
+
37
+ constructor(path: string, options: SqliteSearchCacheOptions = {}) {
38
+ this.db = openSqliteWithPragmas(path, { migrations: MIGRATIONS });
39
+ this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
40
+ }
41
+
42
+ async get(key: SearchCacheKey): Promise<TextSearchResult | undefined> {
43
+ const row = this.db.query("SELECT result_json, expires_at FROM search_cache WHERE key = ?").get(deriveSearchCacheKey(key)) as SearchCacheRow | null;
44
+ if (!row) return undefined;
45
+ if (row.expires_at <= Date.now()) return undefined;
46
+ return JSON.parse(row.result_json) as TextSearchResult;
47
+ }
48
+
49
+ async set(key: SearchCacheKey, result: TextSearchResult): Promise<void> {
50
+ this.db
51
+ .query(
52
+ "INSERT INTO search_cache (key, result_json, expires_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET result_json = excluded.result_json, expires_at = excluded.expires_at",
53
+ )
54
+ .run(deriveSearchCacheKey(key), JSON.stringify(result), Date.now() + this.ttlMs);
55
+ }
56
+
57
+ close(): void {
58
+ this.db.close();
59
+ }
60
+ }
@@ -0,0 +1,154 @@
1
+ import type { Database } from "bun:sqlite";
2
+ import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
3
+ import type { SymbolGraphGeneration } from "../domain/symbol-graph-generation.ts";
4
+ import type { SymbolNodeId } from "../domain/symbol-node-id.ts";
5
+ import type { SymbolEdgeKind, SymbolGraphPort, SymbolNode } from "../ports/symbol-graph-port.ts";
6
+
7
+ const MIGRATIONS: Migration[] = [
8
+ {
9
+ version: 1,
10
+ up: (db) => {
11
+ db.exec(
12
+ "CREATE TABLE symbol_nodes (id TEXT PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL, path TEXT NOT NULL, line INTEGER NOT NULL, character INTEGER NOT NULL)",
13
+ );
14
+ db.exec("CREATE TABLE symbol_edges (from_id TEXT NOT NULL, to_id TEXT NOT NULL, kind TEXT NOT NULL, PRIMARY KEY (from_id, to_id, kind))");
15
+ db.exec("CREATE INDEX symbol_edges_to_idx ON symbol_edges (to_id, kind)");
16
+ },
17
+ },
18
+ {
19
+ version: 2,
20
+ up: (db) => {
21
+ db.exec(
22
+ "CREATE TABLE symbol_graph_generation (singleton INTEGER PRIMARY KEY CHECK (singleton = 1), source_fingerprint TEXT NOT NULL, max_files INTEGER NOT NULL, max_symbols_per_file INTEGER NOT NULL, completed_at INTEGER NOT NULL, files_processed INTEGER NOT NULL, symbols_processed INTEGER NOT NULL, nodes_added INTEGER NOT NULL, edges_added INTEGER NOT NULL)",
23
+ );
24
+ },
25
+ },
26
+ ];
27
+
28
+ interface NodeRow {
29
+ name: string;
30
+ kind: string;
31
+ path: string;
32
+ line: number;
33
+ character: number;
34
+ }
35
+
36
+ interface GenerationRow {
37
+ source_fingerprint: string;
38
+ max_files: number;
39
+ max_symbols_per_file: number;
40
+ completed_at: number;
41
+ files_processed: number;
42
+ symbols_processed: number;
43
+ nodes_added: number;
44
+ edges_added: number;
45
+ }
46
+
47
+ /**
48
+ * SQLite-backed SymbolGraphPort; survives a daemon restart pointed at the
49
+ * same database file. reachableFrom uses `WITH RECURSIVE` rather than an
50
+ * application-level loop issuing one query per hop -- one round trip
51
+ * regardless of maxDepth.
52
+ */
53
+ export class SqliteSymbolGraph implements SymbolGraphPort {
54
+ private readonly db: Database;
55
+
56
+ constructor(path: string) {
57
+ this.db = openSqliteWithPragmas(path, { migrations: MIGRATIONS });
58
+ }
59
+
60
+ async addNode(node: SymbolNode): Promise<void> {
61
+ this.db
62
+ .query(
63
+ "INSERT INTO symbol_nodes (id, name, kind, path, line, character) VALUES (?, ?, ?, ?, ?, ?) " +
64
+ "ON CONFLICT(id) DO UPDATE SET name = excluded.name, kind = excluded.kind, path = excluded.path, line = excluded.line, character = excluded.character",
65
+ )
66
+ .run(node.id, node.name, node.kind, node.location.path, node.location.line, node.location.character);
67
+ }
68
+
69
+ async getNode(id: SymbolNodeId): Promise<SymbolNode | undefined> {
70
+ const row = this.db.query("SELECT name, kind, path, line, character FROM symbol_nodes WHERE id = ?").get(id) as NodeRow | null;
71
+ if (!row) return undefined;
72
+ return { id, name: row.name, kind: row.kind, location: { path: row.path, line: row.line, character: row.character } };
73
+ }
74
+
75
+ async addEdge(from: SymbolNodeId, to: SymbolNodeId, kind: SymbolEdgeKind): Promise<void> {
76
+ this.db.query("INSERT OR IGNORE INTO symbol_edges (from_id, to_id, kind) VALUES (?, ?, ?)").run(from, to, kind);
77
+ }
78
+
79
+ async edgesFrom(id: SymbolNodeId, kind?: SymbolEdgeKind): Promise<readonly SymbolNodeId[]> {
80
+ const rows = kind
81
+ ? (this.db.query("SELECT to_id FROM symbol_edges WHERE from_id = ? AND kind = ?").all(id, kind) as { to_id: string }[])
82
+ : (this.db.query("SELECT to_id FROM symbol_edges WHERE from_id = ?").all(id) as { to_id: string }[]);
83
+ return rows.map((row) => row.to_id);
84
+ }
85
+
86
+ async edgesTo(id: SymbolNodeId, kind?: SymbolEdgeKind): Promise<readonly SymbolNodeId[]> {
87
+ const rows = kind
88
+ ? (this.db.query("SELECT from_id FROM symbol_edges WHERE to_id = ? AND kind = ?").all(id, kind) as { from_id: string }[])
89
+ : (this.db.query("SELECT from_id FROM symbol_edges WHERE to_id = ?").all(id) as { from_id: string }[]);
90
+ return rows.map((row) => row.from_id);
91
+ }
92
+
93
+ async reachableFrom(id: SymbolNodeId, options: { maxDepth: number; kind?: SymbolEdgeKind }): Promise<readonly SymbolNodeId[]> {
94
+ if (options.maxDepth < 1) return [];
95
+ const kindClause = options.kind ? "AND e.kind = $kind" : "";
96
+ const sql = `
97
+ WITH RECURSIVE reachable(id, depth) AS (
98
+ SELECT to_id, 1 FROM symbol_edges e WHERE e.from_id = $id ${kindClause}
99
+ UNION
100
+ SELECT e.to_id, r.depth + 1
101
+ FROM symbol_edges e
102
+ JOIN reachable r ON e.from_id = r.id
103
+ WHERE r.depth < $maxDepth ${kindClause}
104
+ )
105
+ SELECT DISTINCT id FROM reachable
106
+ `;
107
+ const params: Record<string, string | number> = { $id: id, $maxDepth: options.maxDepth };
108
+ if (options.kind) params["$kind"] = options.kind;
109
+ const rows = this.db.query(sql).all(params) as { id: string }[];
110
+ return rows.map((row) => row.id).filter((reachedId) => reachedId !== id);
111
+ }
112
+
113
+ async getGeneration(): Promise<SymbolGraphGeneration | undefined> {
114
+ const row = this.db
115
+ .query(
116
+ "SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added FROM symbol_graph_generation WHERE singleton = 1",
117
+ )
118
+ .get() as GenerationRow | null;
119
+ if (!row) return undefined;
120
+ return {
121
+ sourceFingerprint: row.source_fingerprint,
122
+ maxFiles: row.max_files,
123
+ maxSymbolsPerFile: row.max_symbols_per_file,
124
+ completedAt: row.completed_at,
125
+ result: {
126
+ filesProcessed: row.files_processed,
127
+ symbolsProcessed: row.symbols_processed,
128
+ nodesAdded: row.nodes_added,
129
+ edgesAdded: row.edges_added,
130
+ },
131
+ };
132
+ }
133
+
134
+ async setGeneration(generation: SymbolGraphGeneration): Promise<void> {
135
+ this.db
136
+ .query(
137
+ "INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added",
138
+ )
139
+ .run(
140
+ generation.sourceFingerprint,
141
+ generation.maxFiles,
142
+ generation.maxSymbolsPerFile,
143
+ generation.completedAt,
144
+ generation.result.filesProcessed,
145
+ generation.result.symbolsProcessed,
146
+ generation.result.nodesAdded,
147
+ generation.result.edgesAdded,
148
+ );
149
+ }
150
+
151
+ async close(): Promise<void> {
152
+ this.db.close();
153
+ }
154
+ }
@@ -0,0 +1,29 @@
1
+ import type { SearchCacheKey } from "../domain/search-cache-key.ts";
2
+ import type { TextSearchResult } from "../domain/text-search-result.ts";
3
+ import type { SearchCachePort } from "../ports/search-cache-port.ts";
4
+
5
+ /**
6
+ * Composes a fast (in-memory) and a durable (disk-backed) SearchCachePort into one: a read
7
+ * checks fast first, falls through to durable on a miss and warms fast with what it finds; a
8
+ * write goes to both. This is what actually delivers the "in-memory tier plus a disk-backed
9
+ * tier" design -- a single SearchCachePort implementation can only be one or the other.
10
+ */
11
+ export class TieredSearchCache implements SearchCachePort {
12
+ constructor(
13
+ private readonly fast: SearchCachePort,
14
+ private readonly durable: SearchCachePort,
15
+ ) {}
16
+
17
+ async get(key: SearchCacheKey): Promise<TextSearchResult | undefined> {
18
+ const fastHit = await this.fast.get(key);
19
+ if (fastHit) return fastHit;
20
+
21
+ const durableHit = await this.durable.get(key);
22
+ if (durableHit) await this.fast.set(key, durableHit);
23
+ return durableHit;
24
+ }
25
+
26
+ async set(key: SearchCacheKey, result: TextSearchResult): Promise<void> {
27
+ await Promise.all([this.fast.set(key, result), this.durable.set(key, result)]);
28
+ }
29
+ }
@@ -0,0 +1,155 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { extname, join } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import Parser from "web-tree-sitter";
5
+ import { contentHashOf } from "../../domain/content-hash.ts";
6
+ import type { WorkspaceSymbol } from "../../domain/workspace-symbol.ts";
7
+ import type { ContentCachePort, ContentSymbol } from "../../ports/content-cache-port.ts";
8
+ import type { SymbolIndexPort } from "../../ports/symbol-index-port.ts";
9
+ import { findSourceFiles } from "../find-source-files.ts";
10
+ import { InMemoryContentCache } from "../in-memory-content-cache.ts";
11
+
12
+ const MAX_FILES_SCANNED = 5_000;
13
+
14
+ interface DeclarationKind {
15
+ readonly nodeType: string;
16
+ readonly kind: string;
17
+ }
18
+
19
+ /** Top-level and class-member declaration shapes in tree-sitter's TypeScript/JavaScript grammars. */
20
+ const DECLARATION_KINDS: readonly DeclarationKind[] = [
21
+ { nodeType: "function_declaration", kind: "function" },
22
+ { nodeType: "class_declaration", kind: "class" },
23
+ { nodeType: "interface_declaration", kind: "interface" },
24
+ { nodeType: "type_alias_declaration", kind: "type-alias" },
25
+ { nodeType: "enum_declaration", kind: "enum" },
26
+ { nodeType: "method_definition", kind: "method" },
27
+ ];
28
+
29
+ let parserInitialization: Promise<void> | undefined;
30
+ function ensureParserInitialized(): Promise<void> {
31
+ if (!parserInitialization) parserInitialization = Parser.init();
32
+ return parserInitialization;
33
+ }
34
+
35
+ function wasmPathFor(extension: string): string | undefined {
36
+ const specifier =
37
+ extension === ".ts"
38
+ ? "tree-sitter-wasms/out/tree-sitter-typescript.wasm"
39
+ : extension === ".tsx"
40
+ ? "tree-sitter-wasms/out/tree-sitter-tsx.wasm"
41
+ : extension === ".js" || extension === ".jsx" || extension === ".mjs" || extension === ".cjs"
42
+ ? "tree-sitter-wasms/out/tree-sitter-javascript.wasm"
43
+ : undefined;
44
+ return specifier ? fileURLToPath(import.meta.resolve(specifier)) : undefined;
45
+ }
46
+
47
+ /** Content-derived only -- no path, so the extraction result is valid caching material regardless of which file currently holds this content. */
48
+ function extractContentSymbols(root: Parser.SyntaxNode): ContentSymbol[] {
49
+ const results: ContentSymbol[] = [];
50
+ for (const spec of DECLARATION_KINDS) {
51
+ for (const node of root.descendantsOfType(spec.nodeType)) {
52
+ const nameNode = node.childForFieldName("name");
53
+ if (!nameNode) continue;
54
+ results.push({ name: nameNode.text, kind: spec.kind, line: node.startPosition.row + 1, character: node.startPosition.column + 1 });
55
+ }
56
+ }
57
+ return results;
58
+ }
59
+
60
+ function toWorkspaceSymbols(symbols: readonly ContentSymbol[], relativePath: string): WorkspaceSymbol[] {
61
+ return symbols.map((symbol) => ({
62
+ name: symbol.name,
63
+ kind: symbol.kind,
64
+ location: { path: relativePath, line: symbol.line, character: symbol.character },
65
+ ...(symbol.containerName !== undefined ? { containerName: symbol.containerName } : {}),
66
+ }));
67
+ }
68
+
69
+ /**
70
+ * SymbolIndexPort backed by tree-sitter, via web-tree-sitter's WASM runtime
71
+ * (the native `tree-sitter` binding needs node-gyp, unavailable here).
72
+ *
73
+ * No subprocess or warm server: every call parses whatever files currently
74
+ * exist under the workspace root, so results are always current, at the
75
+ * cost of re-parsing on every query -- mitigated by the ContentHash-keyed
76
+ * cache below.
77
+ *
78
+ * `web-tree-sitter` is pinned to 0.20.8, matching the WASM ABI
79
+ * `tree-sitter-wasms`' prebuilt grammars were compiled against; 0.26.x
80
+ * fails to load them.
81
+ *
82
+ * Per-file results are cached, keyed by ContentHash, via an injected
83
+ * ContentCachePort (default in-memory; pass SqliteContentCache for
84
+ * durability). A cache hit skips parsing entirely, and also warms the
85
+ * cache's rawContent lens for that hash, since the content was already
86
+ * read to compute it.
87
+ */
88
+ export class TreeSitterSymbolIndex implements SymbolIndexPort {
89
+ private readonly rootPath: string;
90
+ private readonly contentCache: ContentCachePort;
91
+ private readonly parsersByWasmPath = new Map<string, Parser>();
92
+
93
+ constructor(rootPath: string, contentCache: ContentCachePort = new InMemoryContentCache()) {
94
+ this.rootPath = rootPath;
95
+ this.contentCache = contentCache;
96
+ }
97
+
98
+ private async parserFor(wasmPath: string): Promise<Parser> {
99
+ const cached = this.parsersByWasmPath.get(wasmPath);
100
+ if (cached) return cached;
101
+ await ensureParserInitialized();
102
+ const language = await Parser.Language.load(wasmPath);
103
+ const parser = new Parser();
104
+ parser.setLanguage(language);
105
+ this.parsersByWasmPath.set(wasmPath, parser);
106
+ return parser;
107
+ }
108
+
109
+ private async contentSymbolsFor(relativePath: string, wasmPath: string, content: string): Promise<ContentSymbol[]> {
110
+ const hash = contentHashOf(content);
111
+
112
+ // Reading the file already required reading its content -- warm the fs lens for this
113
+ // hash regardless of whether the symbols lens below is a hit or a miss. Awaited, not
114
+ // fire-and-forget: a caller reading this hash's rawContent right after findSymbols
115
+ // resolves must see it, not race an in-flight write.
116
+ await this.contentCache.putRawContent(hash, content);
117
+
118
+ const cached = await this.contentCache.get(hash);
119
+ if (cached?.symbols) return [...cached.symbols];
120
+
121
+ const parser = await this.parserFor(wasmPath);
122
+ const tree = parser.parse(content);
123
+ const symbols = tree ? extractContentSymbols(tree.rootNode) : [];
124
+ await this.contentCache.putSymbols(hash, symbols);
125
+ return symbols;
126
+ }
127
+
128
+ async findSymbols(query: string): Promise<WorkspaceSymbol[]> {
129
+ const lowerQuery = query.toLowerCase();
130
+ const results: WorkspaceSymbol[] = [];
131
+
132
+ for (const relativePath of findSourceFiles(this.rootPath, (extension) => wasmPathFor(extension) !== undefined, MAX_FILES_SCANNED)) {
133
+ const wasmPath = wasmPathFor(extname(relativePath));
134
+ if (!wasmPath) continue;
135
+
136
+ let content: string;
137
+ try {
138
+ content = readFileSync(join(this.rootPath, relativePath), "utf-8");
139
+ } catch {
140
+ continue;
141
+ }
142
+
143
+ const contentSymbols = await this.contentSymbolsFor(relativePath, wasmPath, content);
144
+ for (const symbol of toWorkspaceSymbols(contentSymbols, relativePath)) {
145
+ if (symbol.name.toLowerCase().includes(lowerQuery)) results.push(symbol);
146
+ }
147
+ }
148
+
149
+ return results;
150
+ }
151
+
152
+ async close(): Promise<void> {
153
+ // In-process, synchronous parsing -- no subprocess or handle to release.
154
+ }
155
+ }