@danypops/lector 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/package.json +5 -1
- package/src/adapters/fallback-code-intelligence-index.ts +73 -0
- package/src/adapters/find-source-files.ts +1 -1
- package/src/adapters/git-repo-fetcher.ts +154 -37
- package/src/adapters/lsp/discover-seed-file.ts +17 -9
- package/src/adapters/lsp/json-rpc-stream.ts +52 -2
- package/src/adapters/lsp/language-server-process.ts +54 -10
- package/src/adapters/lsp/lsp-symbol-index.ts +167 -41
- package/src/adapters/normalize-npm-repository.ts +77 -0
- package/src/adapters/npm-lockfile-version-resolver.ts +519 -0
- package/src/adapters/npm-package-source-resolver.ts +322 -0
- package/src/adapters/npm-registry-client.ts +304 -0
- package/src/adapters/polyglot-code-intelligence-index.ts +135 -0
- package/src/adapters/sqlite-symbol-graph.ts +68 -3
- package/src/adapters/tree-sitter/typescript-tree-sitter-symbol-index.ts +61 -11
- package/src/adapters/typescript-compiler-symbol-index.ts +129 -0
- package/src/cli.ts +97 -23
- package/src/daemon.ts +4 -0
- package/src/domain/find-workspace-symbols.ts +4 -3
- package/src/domain/installed-package-version.ts +66 -0
- package/src/domain/intelligence-provenance.ts +27 -0
- package/src/domain/language-server-descriptor.ts +22 -0
- package/src/domain/npm-package-metadata.ts +26 -0
- package/src/domain/package-source.ts +143 -0
- package/src/domain/repo-fetch-result.ts +33 -1
- package/src/domain/resolve-package-source.ts +204 -0
- package/src/domain/symbol-graph-generation.ts +5 -0
- package/src/domain/symbol-query.ts +16 -0
- package/src/domain/workspace-symbol.ts +13 -0
- package/src/index.ts +68 -4
- package/src/ports/installed-package-version-resolver-port.ts +5 -0
- package/src/ports/npm-registry-port.ts +5 -0
- package/src/ports/package-source-resolver-port.ts +5 -0
- package/src/ports/repo-fetcher-port.ts +2 -2
- package/src/ports/symbol-index-port.ts +4 -2
- package/src/service.ts +152 -43
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
import type { CallHierarchyEntry, IncomingCall, OutgoingCall } from "../domain/call-hierarchy.ts";
|
|
3
|
+
import type { Diagnostic } from "../domain/diagnostic.ts";
|
|
4
|
+
import type { DocumentSymbolEntry } from "../domain/document-symbol.ts";
|
|
5
|
+
import type { Hover } from "../domain/hover.ts";
|
|
6
|
+
import type { IntelligenceProvenance, IntelligenceSourceOutcome, SymbolSearchBounds } from "../domain/intelligence-provenance.ts";
|
|
7
|
+
import type { LanguageServerDescriptor } from "../domain/language-server-descriptor.ts";
|
|
8
|
+
import type { SymbolSearchResult, WorkspaceLocation, WorkspaceSymbol } from "../domain/workspace-symbol.ts";
|
|
9
|
+
import type { CodeIntelligencePort } from "../ports/code-intelligence-port.ts";
|
|
10
|
+
import type { SymbolIndexPort } from "../ports/symbol-index-port.ts";
|
|
11
|
+
|
|
12
|
+
const MAX_SOURCE_ERROR_LENGTH = 500;
|
|
13
|
+
const MAX_LANGUAGE_INDEXES = 16;
|
|
14
|
+
const DEFAULT_MAX_RESULTS = 1_000;
|
|
15
|
+
|
|
16
|
+
export interface PolyglotIndexEntry {
|
|
17
|
+
readonly descriptor: LanguageServerDescriptor;
|
|
18
|
+
readonly index: SymbolIndexPort;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function boundedError(error: unknown): { code: string; message: string } {
|
|
22
|
+
const code = error instanceof Error && error.name ? error.name : "CodeIntelligenceError";
|
|
23
|
+
const message = (error instanceof Error ? error.message : String(error)).slice(0, MAX_SOURCE_ERROR_LENGTH);
|
|
24
|
+
return { code, message };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function supportsCodeIntelligence(index: SymbolIndexPort): index is SymbolIndexPort & CodeIntelligencePort {
|
|
28
|
+
return "goToDefinition" in index && typeof index.goToDefinition === "function";
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function compareSymbols(left: WorkspaceSymbol, right: WorkspaceSymbol): number {
|
|
32
|
+
return (
|
|
33
|
+
left.location.path.localeCompare(right.location.path) ||
|
|
34
|
+
left.location.line - right.location.line ||
|
|
35
|
+
left.location.character - right.location.character ||
|
|
36
|
+
left.name.localeCompare(right.name) ||
|
|
37
|
+
left.kind.localeCompare(right.kind)
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class PolyglotCodeIntelligenceIndex implements SymbolIndexPort, CodeIntelligencePort {
|
|
42
|
+
readonly provenance: IntelligenceProvenance = {
|
|
43
|
+
fidelity: "semantic",
|
|
44
|
+
backend: "polyglot-language-servers",
|
|
45
|
+
languageId: "polyglot",
|
|
46
|
+
authority: "language-server",
|
|
47
|
+
freshness: "live-process",
|
|
48
|
+
limitations: [],
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
constructor(private readonly entries: readonly PolyglotIndexEntry[]) {
|
|
52
|
+
if (entries.length === 0 || entries.length > MAX_LANGUAGE_INDEXES) {
|
|
53
|
+
throw new TypeError(`polyglot index requires between 1 and ${MAX_LANGUAGE_INDEXES} language indexes`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private indexForPath(path: string): SymbolIndexPort & CodeIntelligencePort {
|
|
58
|
+
const extension = extname(path);
|
|
59
|
+
const entry = this.entries.find((candidate) => candidate.descriptor.extensions.includes(extension));
|
|
60
|
+
if (!entry) throw new Error(`no language index available for extension "${extension}"`);
|
|
61
|
+
if (!supportsCodeIntelligence(entry.index)) throw new Error(`code intelligence unavailable for ${entry.descriptor.languageId}`);
|
|
62
|
+
return entry.index;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async findSymbols(query: string, bounds: SymbolSearchBounds = { maxResults: DEFAULT_MAX_RESULTS }): Promise<SymbolSearchResult> {
|
|
66
|
+
if (!Number.isSafeInteger(bounds.maxResults) || bounds.maxResults < 1) throw new TypeError("maxResults must be a positive safe integer");
|
|
67
|
+
const settled = await Promise.allSettled(this.entries.map(({ index }) => index.findSymbols(query, bounds)));
|
|
68
|
+
const symbols: WorkspaceSymbol[] = [];
|
|
69
|
+
const sources: IntelligenceSourceOutcome[] = [];
|
|
70
|
+
let sourceTruncated = false;
|
|
71
|
+
|
|
72
|
+
for (const [position, outcome] of settled.entries()) {
|
|
73
|
+
const entry = this.entries[position];
|
|
74
|
+
if (!entry) continue;
|
|
75
|
+
if (outcome.status === "rejected") {
|
|
76
|
+
sources.push({ provenance: entry.index.provenance, status: "failed", symbolCount: 0, error: boundedError(outcome.reason) });
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
sources.push({
|
|
80
|
+
provenance: outcome.value.provenance,
|
|
81
|
+
status: "ready",
|
|
82
|
+
symbolCount: outcome.value.symbols.length,
|
|
83
|
+
truncated: outcome.value.truncated,
|
|
84
|
+
});
|
|
85
|
+
sourceTruncated ||= outcome.value.truncated;
|
|
86
|
+
for (const symbol of outcome.value.symbols) symbols.push({ ...symbol, provenance: outcome.value.provenance });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
symbols.sort(compareSymbols);
|
|
90
|
+
const truncated = sourceTruncated || symbols.length > bounds.maxResults;
|
|
91
|
+
return {
|
|
92
|
+
symbols: symbols.slice(0, bounds.maxResults),
|
|
93
|
+
truncated,
|
|
94
|
+
provenance: this.provenance,
|
|
95
|
+
completeness: sources.every((source) => source.status === "ready") ? "complete" : "partial",
|
|
96
|
+
sources,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
goToDefinition(at: WorkspaceLocation) {
|
|
101
|
+
return this.indexForPath(at.path).goToDefinition(at);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
goToImplementation(at: WorkspaceLocation) {
|
|
105
|
+
return this.indexForPath(at.path).goToImplementation(at);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
findReferences(at: WorkspaceLocation, includeDeclaration: boolean) {
|
|
109
|
+
return this.indexForPath(at.path).findReferences(at, includeDeclaration);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
hover(at: WorkspaceLocation): Promise<Hover | undefined> {
|
|
113
|
+
return this.indexForPath(at.path).hover(at);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
documentSymbols(path: string): Promise<DocumentSymbolEntry[]> {
|
|
117
|
+
return this.indexForPath(path).documentSymbols(path);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
diagnostics(path: string): Promise<Diagnostic[]> {
|
|
121
|
+
return this.indexForPath(path).diagnostics(path);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
prepareCallHierarchy(at: WorkspaceLocation): Promise<CallHierarchyEntry[]> {
|
|
125
|
+
return this.indexForPath(at.path).prepareCallHierarchy(at);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
incomingCalls(at: WorkspaceLocation): Promise<IncomingCall[]> {
|
|
129
|
+
return this.indexForPath(at.path).incomingCalls(at);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
outgoingCalls(at: WorkspaceLocation): Promise<OutgoingCall[]> {
|
|
133
|
+
return this.indexForPath(at.path).outgoingCalls(at);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
|
|
3
|
+
import type { IntelligenceProvenance } from "../domain/intelligence-provenance.ts";
|
|
3
4
|
import type { SymbolGraphGeneration } from "../domain/symbol-graph-generation.ts";
|
|
4
5
|
import type { SymbolNodeId } from "../domain/symbol-node-id.ts";
|
|
5
6
|
import type { SymbolEdgeKind, SymbolGraphPort, SymbolNode } from "../ports/symbol-graph-port.ts";
|
|
@@ -23,6 +24,14 @@ const MIGRATIONS: Migration[] = [
|
|
|
23
24
|
);
|
|
24
25
|
},
|
|
25
26
|
},
|
|
27
|
+
{
|
|
28
|
+
version: 3,
|
|
29
|
+
up: (db) => db.exec("ALTER TABLE symbol_graph_generation ADD COLUMN provenance_json TEXT"),
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
version: 4,
|
|
33
|
+
up: (db) => db.exec("ALTER TABLE symbol_graph_generation ADD COLUMN sources_json TEXT"),
|
|
34
|
+
},
|
|
26
35
|
];
|
|
27
36
|
|
|
28
37
|
interface NodeRow {
|
|
@@ -42,6 +51,56 @@ interface GenerationRow {
|
|
|
42
51
|
symbols_processed: number;
|
|
43
52
|
nodes_added: number;
|
|
44
53
|
edges_added: number;
|
|
54
|
+
provenance_json: string | null;
|
|
55
|
+
sources_json: string | null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
59
|
+
return typeof value === "object" && value !== null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function asProvenance(value: unknown): IntelligenceProvenance | undefined {
|
|
63
|
+
if (!isRecord(value)) return undefined;
|
|
64
|
+
const candidate = value;
|
|
65
|
+
if (
|
|
66
|
+
(candidate.fidelity !== "semantic" && candidate.fidelity !== "structural") ||
|
|
67
|
+
typeof candidate.backend !== "string" ||
|
|
68
|
+
typeof candidate.languageId !== "string" ||
|
|
69
|
+
(candidate.authority !== "language-server" && candidate.authority !== "parser" && candidate.authority !== "compiler") ||
|
|
70
|
+
(candidate.freshness !== "live-process" && candidate.freshness !== "content-hash" && candidate.freshness !== "filesystem-snapshot") ||
|
|
71
|
+
!Array.isArray(candidate.limitations) ||
|
|
72
|
+
!candidate.limitations.every((item) => typeof item === "string")
|
|
73
|
+
) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
return {
|
|
77
|
+
fidelity: candidate.fidelity,
|
|
78
|
+
backend: candidate.backend,
|
|
79
|
+
languageId: candidate.languageId,
|
|
80
|
+
authority: candidate.authority,
|
|
81
|
+
freshness: candidate.freshness,
|
|
82
|
+
limitations: candidate.limitations,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function parseJson(json: string | null): unknown {
|
|
87
|
+
if (!json) return undefined;
|
|
88
|
+
try {
|
|
89
|
+
return JSON.parse(json);
|
|
90
|
+
} catch {
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function parseProvenance(json: string | null): IntelligenceProvenance | undefined {
|
|
96
|
+
return asProvenance(parseJson(json));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function parseSources(json: string | null): readonly IntelligenceProvenance[] | undefined {
|
|
100
|
+
const value = parseJson(json);
|
|
101
|
+
if (!Array.isArray(value)) return undefined;
|
|
102
|
+
const sources = value.map(asProvenance);
|
|
103
|
+
return sources.every((source) => source !== undefined) ? sources : undefined;
|
|
45
104
|
}
|
|
46
105
|
|
|
47
106
|
/**
|
|
@@ -105,7 +164,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
105
164
|
SELECT DISTINCT id FROM reachable
|
|
106
165
|
`;
|
|
107
166
|
const params: Record<string, string | number> = { $id: id, $maxDepth: options.maxDepth };
|
|
108
|
-
if (options.kind) params
|
|
167
|
+
if (options.kind) params.$kind = options.kind;
|
|
109
168
|
const rows = this.db.query(sql).all(params) as { id: string }[];
|
|
110
169
|
return rows.map((row) => row.id).filter((reachedId) => reachedId !== id);
|
|
111
170
|
}
|
|
@@ -113,15 +172,19 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
113
172
|
async getGeneration(): Promise<SymbolGraphGeneration | undefined> {
|
|
114
173
|
const row = this.db
|
|
115
174
|
.query(
|
|
116
|
-
"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added FROM symbol_graph_generation WHERE singleton = 1",
|
|
175
|
+
"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json FROM symbol_graph_generation WHERE singleton = 1",
|
|
117
176
|
)
|
|
118
177
|
.get() as GenerationRow | null;
|
|
119
178
|
if (!row) return undefined;
|
|
179
|
+
const provenance = parseProvenance(row.provenance_json);
|
|
180
|
+
const sources = parseSources(row.sources_json);
|
|
120
181
|
return {
|
|
121
182
|
sourceFingerprint: row.source_fingerprint,
|
|
122
183
|
maxFiles: row.max_files,
|
|
123
184
|
maxSymbolsPerFile: row.max_symbols_per_file,
|
|
124
185
|
completedAt: row.completed_at,
|
|
186
|
+
...(provenance ? { provenance } : {}),
|
|
187
|
+
...(sources ? { sources } : {}),
|
|
125
188
|
result: {
|
|
126
189
|
filesProcessed: row.files_processed,
|
|
127
190
|
symbolsProcessed: row.symbols_processed,
|
|
@@ -134,7 +197,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
134
197
|
async setGeneration(generation: SymbolGraphGeneration): Promise<void> {
|
|
135
198
|
this.db
|
|
136
199
|
.query(
|
|
137
|
-
"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added",
|
|
200
|
+
"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added, provenance_json = excluded.provenance_json, sources_json = excluded.sources_json",
|
|
138
201
|
)
|
|
139
202
|
.run(
|
|
140
203
|
generation.sourceFingerprint,
|
|
@@ -145,6 +208,8 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
145
208
|
generation.result.symbolsProcessed,
|
|
146
209
|
generation.result.nodesAdded,
|
|
147
210
|
generation.result.edgesAdded,
|
|
211
|
+
generation.provenance ? JSON.stringify(generation.provenance) : null,
|
|
212
|
+
generation.sources ? JSON.stringify(generation.sources) : null,
|
|
148
213
|
);
|
|
149
214
|
}
|
|
150
215
|
|
|
@@ -1,15 +1,32 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
2
|
import { extname, join } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import Parser from "web-tree-sitter";
|
|
5
5
|
import { contentHashOf } from "../../domain/content-hash.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { IntelligenceProvenance, SymbolSearchBounds } from "../../domain/intelligence-provenance.ts";
|
|
7
|
+
import type { SymbolSearchResult, WorkspaceSymbol } from "../../domain/workspace-symbol.ts";
|
|
7
8
|
import type { ContentCachePort, ContentSymbol } from "../../ports/content-cache-port.ts";
|
|
8
9
|
import type { SymbolIndexPort } from "../../ports/symbol-index-port.ts";
|
|
9
10
|
import { findSourceFiles } from "../find-source-files.ts";
|
|
10
11
|
import { InMemoryContentCache } from "../in-memory-content-cache.ts";
|
|
11
12
|
|
|
12
|
-
const
|
|
13
|
+
const DEFAULT_MAX_FILES = 5_000;
|
|
14
|
+
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024;
|
|
15
|
+
const DEFAULT_MAX_TOTAL_BYTES = 50 * 1024 * 1024;
|
|
16
|
+
const DEFAULT_MAX_RESULTS = 1_000;
|
|
17
|
+
|
|
18
|
+
export interface TreeSitterSymbolIndexOptions {
|
|
19
|
+
readonly maxFiles?: number;
|
|
20
|
+
readonly maxFileBytes?: number;
|
|
21
|
+
readonly maxTotalBytes?: number;
|
|
22
|
+
readonly maxResults?: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function positiveLimit(value: number | undefined, fallback: number, field: string): number {
|
|
26
|
+
const result = value ?? fallback;
|
|
27
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new TypeError(`${field} must be a positive safe integer`);
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
13
30
|
|
|
14
31
|
interface DeclarationKind {
|
|
15
32
|
readonly nodeType: string;
|
|
@@ -86,13 +103,29 @@ function toWorkspaceSymbols(symbols: readonly ContentSymbol[], relativePath: str
|
|
|
86
103
|
* read to compute it.
|
|
87
104
|
*/
|
|
88
105
|
export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
106
|
+
readonly provenance: IntelligenceProvenance = {
|
|
107
|
+
fidelity: "structural",
|
|
108
|
+
backend: "tree-sitter-typescript-javascript",
|
|
109
|
+
languageId: "typescript-javascript",
|
|
110
|
+
authority: "parser",
|
|
111
|
+
freshness: "content-hash",
|
|
112
|
+
limitations: ["no cross-file identity", "no type information", "syntax recovery may include malformed declarations"],
|
|
113
|
+
};
|
|
89
114
|
private readonly rootPath: string;
|
|
90
115
|
private readonly contentCache: ContentCachePort;
|
|
116
|
+
private readonly maxFiles: number;
|
|
117
|
+
private readonly maxFileBytes: number;
|
|
118
|
+
private readonly maxTotalBytes: number;
|
|
119
|
+
private readonly maxResults: number;
|
|
91
120
|
private readonly parsersByWasmPath = new Map<string, Parser>();
|
|
92
121
|
|
|
93
|
-
constructor(rootPath: string, contentCache: ContentCachePort = new InMemoryContentCache()) {
|
|
122
|
+
constructor(rootPath: string, contentCache: ContentCachePort = new InMemoryContentCache(), options: TreeSitterSymbolIndexOptions = {}) {
|
|
94
123
|
this.rootPath = rootPath;
|
|
95
124
|
this.contentCache = contentCache;
|
|
125
|
+
this.maxFiles = positiveLimit(options.maxFiles, DEFAULT_MAX_FILES, "maxFiles");
|
|
126
|
+
this.maxFileBytes = positiveLimit(options.maxFileBytes, DEFAULT_MAX_FILE_BYTES, "maxFileBytes");
|
|
127
|
+
this.maxTotalBytes = positiveLimit(options.maxTotalBytes, DEFAULT_MAX_TOTAL_BYTES, "maxTotalBytes");
|
|
128
|
+
this.maxResults = positiveLimit(options.maxResults, DEFAULT_MAX_RESULTS, "maxResults");
|
|
96
129
|
}
|
|
97
130
|
|
|
98
131
|
private async parserFor(wasmPath: string): Promise<Parser> {
|
|
@@ -106,7 +139,7 @@ export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
|
106
139
|
return parser;
|
|
107
140
|
}
|
|
108
141
|
|
|
109
|
-
private async contentSymbolsFor(
|
|
142
|
+
private async contentSymbolsFor(wasmPath: string, content: string): Promise<ContentSymbol[]> {
|
|
110
143
|
const hash = contentHashOf(content);
|
|
111
144
|
|
|
112
145
|
// Reading the file already required reading its content -- warm the fs lens for this
|
|
@@ -125,28 +158,45 @@ export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
|
125
158
|
return symbols;
|
|
126
159
|
}
|
|
127
160
|
|
|
128
|
-
async findSymbols(query: string): Promise<
|
|
161
|
+
async findSymbols(query: string, bounds: SymbolSearchBounds = { maxResults: this.maxResults }): Promise<SymbolSearchResult> {
|
|
162
|
+
const maxResults = Math.min(positiveLimit(bounds.maxResults, this.maxResults, "maxResults"), this.maxResults);
|
|
129
163
|
const lowerQuery = query.toLowerCase();
|
|
130
164
|
const results: WorkspaceSymbol[] = [];
|
|
165
|
+
const files = findSourceFiles(this.rootPath, (extension) => wasmPathFor(extension) !== undefined, this.maxFiles);
|
|
166
|
+
let totalBytes = 0;
|
|
167
|
+
let truncated = files.length === this.maxFiles;
|
|
131
168
|
|
|
132
|
-
for (const relativePath of
|
|
169
|
+
for (const relativePath of files) {
|
|
133
170
|
const wasmPath = wasmPathFor(extname(relativePath));
|
|
134
171
|
if (!wasmPath) continue;
|
|
135
172
|
|
|
136
173
|
let content: string;
|
|
137
174
|
try {
|
|
138
|
-
|
|
175
|
+
const absolutePath = join(this.rootPath, relativePath);
|
|
176
|
+
const size = statSync(absolutePath).size;
|
|
177
|
+
if (size > this.maxFileBytes || totalBytes + size > this.maxTotalBytes) {
|
|
178
|
+
truncated = true;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
totalBytes += size;
|
|
182
|
+
content = readFileSync(absolutePath, "utf-8");
|
|
139
183
|
} catch {
|
|
140
184
|
continue;
|
|
141
185
|
}
|
|
142
186
|
|
|
143
|
-
const contentSymbols = await this.contentSymbolsFor(
|
|
187
|
+
const contentSymbols = await this.contentSymbolsFor(wasmPath, content);
|
|
144
188
|
for (const symbol of toWorkspaceSymbols(contentSymbols, relativePath)) {
|
|
145
|
-
if (symbol.name.toLowerCase().includes(lowerQuery))
|
|
189
|
+
if (!symbol.name.toLowerCase().includes(lowerQuery)) continue;
|
|
190
|
+
if (results.length === maxResults) {
|
|
191
|
+
truncated = true;
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
results.push(symbol);
|
|
146
195
|
}
|
|
196
|
+
if (results.length === maxResults) break;
|
|
147
197
|
}
|
|
148
198
|
|
|
149
|
-
return results;
|
|
199
|
+
return { symbols: results, truncated, provenance: this.provenance };
|
|
150
200
|
}
|
|
151
201
|
|
|
152
202
|
async close(): Promise<void> {
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { extname, join } from "node:path";
|
|
3
|
+
import ts from "typescript";
|
|
4
|
+
import type { IntelligenceProvenance, SymbolSearchBounds } from "../domain/intelligence-provenance.ts";
|
|
5
|
+
import type { SymbolSearchResult, WorkspaceSymbol } from "../domain/workspace-symbol.ts";
|
|
6
|
+
import type { SymbolIndexPort } from "../ports/symbol-index-port.ts";
|
|
7
|
+
import { findSourceFiles } from "./find-source-files.ts";
|
|
8
|
+
|
|
9
|
+
const SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
|
|
10
|
+
|
|
11
|
+
export interface TypeScriptCompilerSymbolIndexOptions {
|
|
12
|
+
readonly maxFiles?: number;
|
|
13
|
+
readonly maxFileBytes?: number;
|
|
14
|
+
readonly maxTotalBytes?: number;
|
|
15
|
+
readonly maxResults?: number;
|
|
16
|
+
readonly maxNodesPerFile?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function positiveLimit(value: number | undefined, fallback: number, field: string): number {
|
|
20
|
+
const result = value ?? fallback;
|
|
21
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new TypeError(`${field} must be a positive safe integer`);
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function scriptKind(path: string): ts.ScriptKind {
|
|
26
|
+
switch (extname(path)) {
|
|
27
|
+
case ".tsx":
|
|
28
|
+
return ts.ScriptKind.TSX;
|
|
29
|
+
case ".jsx":
|
|
30
|
+
return ts.ScriptKind.JSX;
|
|
31
|
+
case ".js":
|
|
32
|
+
case ".mjs":
|
|
33
|
+
case ".cjs":
|
|
34
|
+
return ts.ScriptKind.JS;
|
|
35
|
+
default:
|
|
36
|
+
return ts.ScriptKind.TS;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function declaration(node: ts.Node): { name: ts.Node; kind: string } | undefined {
|
|
41
|
+
if (ts.isFunctionDeclaration(node) && node.name) return { name: node.name, kind: "function" };
|
|
42
|
+
if (ts.isClassDeclaration(node) && node.name) return { name: node.name, kind: "class" };
|
|
43
|
+
if (ts.isInterfaceDeclaration(node)) return { name: node.name, kind: "interface" };
|
|
44
|
+
if (ts.isTypeAliasDeclaration(node)) return { name: node.name, kind: "type-alias" };
|
|
45
|
+
if (ts.isEnumDeclaration(node)) return { name: node.name, kind: "enum" };
|
|
46
|
+
if (ts.isMethodDeclaration(node)) return { name: node.name, kind: "method" };
|
|
47
|
+
if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) return { name: node.name, kind: "variable" };
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class TypeScriptCompilerSymbolIndex implements SymbolIndexPort {
|
|
52
|
+
readonly provenance: IntelligenceProvenance = {
|
|
53
|
+
fidelity: "structural",
|
|
54
|
+
backend: "typescript-compiler",
|
|
55
|
+
languageId: "typescript-javascript",
|
|
56
|
+
authority: "compiler",
|
|
57
|
+
freshness: "filesystem-snapshot",
|
|
58
|
+
limitations: ["syntax declarations only", "no cross-file identity", "no language-server project state"],
|
|
59
|
+
};
|
|
60
|
+
private readonly maxFiles: number;
|
|
61
|
+
private readonly maxFileBytes: number;
|
|
62
|
+
private readonly maxTotalBytes: number;
|
|
63
|
+
private readonly maxResults: number;
|
|
64
|
+
private readonly maxNodesPerFile: number;
|
|
65
|
+
|
|
66
|
+
constructor(
|
|
67
|
+
private readonly rootPath: string,
|
|
68
|
+
options: TypeScriptCompilerSymbolIndexOptions = {},
|
|
69
|
+
) {
|
|
70
|
+
this.maxFiles = positiveLimit(options.maxFiles, 5_000, "maxFiles");
|
|
71
|
+
this.maxFileBytes = positiveLimit(options.maxFileBytes, 2 * 1024 * 1024, "maxFileBytes");
|
|
72
|
+
this.maxTotalBytes = positiveLimit(options.maxTotalBytes, 50 * 1024 * 1024, "maxTotalBytes");
|
|
73
|
+
this.maxResults = positiveLimit(options.maxResults, 1_000, "maxResults");
|
|
74
|
+
this.maxNodesPerFile = positiveLimit(options.maxNodesPerFile, 100_000, "maxNodesPerFile");
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
findSymbols(query: string, bounds: SymbolSearchBounds = { maxResults: this.maxResults }): Promise<SymbolSearchResult> {
|
|
78
|
+
const maxResults = Math.min(positiveLimit(bounds.maxResults, this.maxResults, "maxResults"), this.maxResults);
|
|
79
|
+
const files = findSourceFiles(this.rootPath, (extension) => SOURCE_EXTENSIONS.has(extension), this.maxFiles);
|
|
80
|
+
const symbols: WorkspaceSymbol[] = [];
|
|
81
|
+
const lowerQuery = query.toLowerCase();
|
|
82
|
+
let totalBytes = 0;
|
|
83
|
+
let truncated = files.length === this.maxFiles;
|
|
84
|
+
|
|
85
|
+
for (const relativePath of files) {
|
|
86
|
+
const absolutePath = join(this.rootPath, relativePath);
|
|
87
|
+
let sourceText: string;
|
|
88
|
+
try {
|
|
89
|
+
const size = statSync(absolutePath).size;
|
|
90
|
+
if (size > this.maxFileBytes || totalBytes + size > this.maxTotalBytes) {
|
|
91
|
+
truncated = true;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
totalBytes += size;
|
|
95
|
+
sourceText = readFileSync(absolutePath, "utf-8");
|
|
96
|
+
} catch {
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const sourceFile = ts.createSourceFile(relativePath, sourceText, ts.ScriptTarget.Latest, true, scriptKind(relativePath));
|
|
101
|
+
let visited = 0;
|
|
102
|
+
const visit = (node: ts.Node): void => {
|
|
103
|
+
if (visited++ >= this.maxNodesPerFile || symbols.length >= maxResults) {
|
|
104
|
+
truncated = true;
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const found = declaration(node);
|
|
108
|
+
if (found) {
|
|
109
|
+
const name = found.name.getText(sourceFile);
|
|
110
|
+
if (name.toLowerCase().includes(lowerQuery)) {
|
|
111
|
+
const position = sourceFile.getLineAndCharacterOfPosition(found.name.getStart(sourceFile));
|
|
112
|
+
symbols.push({ name, kind: found.kind, location: { path: relativePath, line: position.line + 1, character: position.character + 1 } });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (visited < this.maxNodesPerFile && symbols.length < maxResults) ts.forEachChild(node, visit);
|
|
116
|
+
else if (node.getChildCount(sourceFile) > 0) truncated = true;
|
|
117
|
+
};
|
|
118
|
+
visit(sourceFile);
|
|
119
|
+
if (symbols.length >= maxResults) {
|
|
120
|
+
truncated = true;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return Promise.resolve({ symbols, truncated, provenance: this.provenance });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async close(): Promise<void> {}
|
|
129
|
+
}
|