@danypops/lector 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -7
- package/package.json +5 -1
- package/src/adapters/fallback-code-intelligence-index.ts +73 -0
- package/src/adapters/find-source-files.ts +1 -1
- package/src/adapters/git-repo-fetcher.ts +154 -37
- package/src/adapters/lsp/json-rpc-stream.ts +52 -2
- package/src/adapters/lsp/language-server-process.ts +54 -10
- package/src/adapters/lsp/lsp-symbol-index.ts +129 -30
- package/src/adapters/normalize-npm-repository.ts +77 -0
- package/src/adapters/npm-lockfile-version-resolver.ts +519 -0
- package/src/adapters/npm-package-source-resolver.ts +322 -0
- package/src/adapters/npm-registry-client.ts +304 -0
- package/src/adapters/sqlite-symbol-graph.ts +47 -3
- package/src/adapters/tree-sitter/typescript-tree-sitter-symbol-index.ts +61 -11
- package/src/adapters/typescript-compiler-symbol-index.ts +129 -0
- package/src/cli.ts +87 -23
- package/src/daemon.ts +4 -0
- package/src/domain/find-workspace-symbols.ts +4 -3
- package/src/domain/installed-package-version.ts +66 -0
- package/src/domain/intelligence-provenance.ts +19 -0
- package/src/domain/language-server-descriptor.ts +18 -0
- package/src/domain/npm-package-metadata.ts +26 -0
- package/src/domain/package-source.ts +143 -0
- package/src/domain/repo-fetch-result.ts +33 -1
- package/src/domain/resolve-package-source.ts +204 -0
- package/src/domain/symbol-graph-generation.ts +3 -0
- package/src/domain/symbol-query.ts +16 -0
- package/src/domain/workspace-symbol.ts +8 -0
- package/src/index.ts +61 -4
- package/src/ports/installed-package-version-resolver-port.ts +5 -0
- package/src/ports/npm-registry-port.ts +5 -0
- package/src/ports/package-source-resolver-port.ts +5 -0
- package/src/ports/repo-fetcher-port.ts +2 -2
- package/src/ports/symbol-index-port.ts +4 -2
- package/src/service.ts +89 -25
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
|
|
3
|
+
import type { IntelligenceProvenance } from "../domain/intelligence-provenance.ts";
|
|
3
4
|
import type { SymbolGraphGeneration } from "../domain/symbol-graph-generation.ts";
|
|
4
5
|
import type { SymbolNodeId } from "../domain/symbol-node-id.ts";
|
|
5
6
|
import type { SymbolEdgeKind, SymbolGraphPort, SymbolNode } from "../ports/symbol-graph-port.ts";
|
|
@@ -23,6 +24,10 @@ const MIGRATIONS: Migration[] = [
|
|
|
23
24
|
);
|
|
24
25
|
},
|
|
25
26
|
},
|
|
27
|
+
{
|
|
28
|
+
version: 3,
|
|
29
|
+
up: (db) => db.exec("ALTER TABLE symbol_graph_generation ADD COLUMN provenance_json TEXT"),
|
|
30
|
+
},
|
|
26
31
|
];
|
|
27
32
|
|
|
28
33
|
interface NodeRow {
|
|
@@ -42,6 +47,42 @@ interface GenerationRow {
|
|
|
42
47
|
symbols_processed: number;
|
|
43
48
|
nodes_added: number;
|
|
44
49
|
edges_added: number;
|
|
50
|
+
provenance_json: string | null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
54
|
+
return typeof value === "object" && value !== null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function parseProvenance(json: string | null): IntelligenceProvenance | undefined {
|
|
58
|
+
if (!json) return undefined;
|
|
59
|
+
let value: unknown;
|
|
60
|
+
try {
|
|
61
|
+
value = JSON.parse(json);
|
|
62
|
+
} catch {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
if (!isRecord(value)) return undefined;
|
|
66
|
+
const candidate = value;
|
|
67
|
+
if (
|
|
68
|
+
(candidate.fidelity !== "semantic" && candidate.fidelity !== "structural") ||
|
|
69
|
+
typeof candidate.backend !== "string" ||
|
|
70
|
+
typeof candidate.languageId !== "string" ||
|
|
71
|
+
(candidate.authority !== "language-server" && candidate.authority !== "parser" && candidate.authority !== "compiler") ||
|
|
72
|
+
(candidate.freshness !== "live-process" && candidate.freshness !== "content-hash" && candidate.freshness !== "filesystem-snapshot") ||
|
|
73
|
+
!Array.isArray(candidate.limitations) ||
|
|
74
|
+
!candidate.limitations.every((item) => typeof item === "string")
|
|
75
|
+
) {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
fidelity: candidate.fidelity,
|
|
80
|
+
backend: candidate.backend,
|
|
81
|
+
languageId: candidate.languageId,
|
|
82
|
+
authority: candidate.authority,
|
|
83
|
+
freshness: candidate.freshness,
|
|
84
|
+
limitations: candidate.limitations,
|
|
85
|
+
};
|
|
45
86
|
}
|
|
46
87
|
|
|
47
88
|
/**
|
|
@@ -105,7 +146,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
105
146
|
SELECT DISTINCT id FROM reachable
|
|
106
147
|
`;
|
|
107
148
|
const params: Record<string, string | number> = { $id: id, $maxDepth: options.maxDepth };
|
|
108
|
-
if (options.kind) params
|
|
149
|
+
if (options.kind) params.$kind = options.kind;
|
|
109
150
|
const rows = this.db.query(sql).all(params) as { id: string }[];
|
|
110
151
|
return rows.map((row) => row.id).filter((reachedId) => reachedId !== id);
|
|
111
152
|
}
|
|
@@ -113,15 +154,17 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
113
154
|
async getGeneration(): Promise<SymbolGraphGeneration | undefined> {
|
|
114
155
|
const row = this.db
|
|
115
156
|
.query(
|
|
116
|
-
"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added FROM symbol_graph_generation WHERE singleton = 1",
|
|
157
|
+
"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json FROM symbol_graph_generation WHERE singleton = 1",
|
|
117
158
|
)
|
|
118
159
|
.get() as GenerationRow | null;
|
|
119
160
|
if (!row) return undefined;
|
|
161
|
+
const provenance = parseProvenance(row.provenance_json);
|
|
120
162
|
return {
|
|
121
163
|
sourceFingerprint: row.source_fingerprint,
|
|
122
164
|
maxFiles: row.max_files,
|
|
123
165
|
maxSymbolsPerFile: row.max_symbols_per_file,
|
|
124
166
|
completedAt: row.completed_at,
|
|
167
|
+
...(provenance ? { provenance } : {}),
|
|
125
168
|
result: {
|
|
126
169
|
filesProcessed: row.files_processed,
|
|
127
170
|
symbolsProcessed: row.symbols_processed,
|
|
@@ -134,7 +177,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
134
177
|
async setGeneration(generation: SymbolGraphGeneration): Promise<void> {
|
|
135
178
|
this.db
|
|
136
179
|
.query(
|
|
137
|
-
"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added",
|
|
180
|
+
"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added, provenance_json = excluded.provenance_json",
|
|
138
181
|
)
|
|
139
182
|
.run(
|
|
140
183
|
generation.sourceFingerprint,
|
|
@@ -145,6 +188,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
|
|
|
145
188
|
generation.result.symbolsProcessed,
|
|
146
189
|
generation.result.nodesAdded,
|
|
147
190
|
generation.result.edgesAdded,
|
|
191
|
+
generation.provenance ? JSON.stringify(generation.provenance) : null,
|
|
148
192
|
);
|
|
149
193
|
}
|
|
150
194
|
|
|
@@ -1,15 +1,32 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
2
|
import { extname, join } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import Parser from "web-tree-sitter";
|
|
5
5
|
import { contentHashOf } from "../../domain/content-hash.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { IntelligenceProvenance, SymbolSearchBounds } from "../../domain/intelligence-provenance.ts";
|
|
7
|
+
import type { SymbolSearchResult, WorkspaceSymbol } from "../../domain/workspace-symbol.ts";
|
|
7
8
|
import type { ContentCachePort, ContentSymbol } from "../../ports/content-cache-port.ts";
|
|
8
9
|
import type { SymbolIndexPort } from "../../ports/symbol-index-port.ts";
|
|
9
10
|
import { findSourceFiles } from "../find-source-files.ts";
|
|
10
11
|
import { InMemoryContentCache } from "../in-memory-content-cache.ts";
|
|
11
12
|
|
|
12
|
-
const
|
|
13
|
+
const DEFAULT_MAX_FILES = 5_000;
|
|
14
|
+
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024;
|
|
15
|
+
const DEFAULT_MAX_TOTAL_BYTES = 50 * 1024 * 1024;
|
|
16
|
+
const DEFAULT_MAX_RESULTS = 1_000;
|
|
17
|
+
|
|
18
|
+
export interface TreeSitterSymbolIndexOptions {
|
|
19
|
+
readonly maxFiles?: number;
|
|
20
|
+
readonly maxFileBytes?: number;
|
|
21
|
+
readonly maxTotalBytes?: number;
|
|
22
|
+
readonly maxResults?: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function positiveLimit(value: number | undefined, fallback: number, field: string): number {
|
|
26
|
+
const result = value ?? fallback;
|
|
27
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new TypeError(`${field} must be a positive safe integer`);
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
13
30
|
|
|
14
31
|
interface DeclarationKind {
|
|
15
32
|
readonly nodeType: string;
|
|
@@ -86,13 +103,29 @@ function toWorkspaceSymbols(symbols: readonly ContentSymbol[], relativePath: str
|
|
|
86
103
|
* read to compute it.
|
|
87
104
|
*/
|
|
88
105
|
export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
106
|
+
readonly provenance: IntelligenceProvenance = {
|
|
107
|
+
fidelity: "structural",
|
|
108
|
+
backend: "tree-sitter-typescript-javascript",
|
|
109
|
+
languageId: "typescript-javascript",
|
|
110
|
+
authority: "parser",
|
|
111
|
+
freshness: "content-hash",
|
|
112
|
+
limitations: ["no cross-file identity", "no type information", "syntax recovery may include malformed declarations"],
|
|
113
|
+
};
|
|
89
114
|
private readonly rootPath: string;
|
|
90
115
|
private readonly contentCache: ContentCachePort;
|
|
116
|
+
private readonly maxFiles: number;
|
|
117
|
+
private readonly maxFileBytes: number;
|
|
118
|
+
private readonly maxTotalBytes: number;
|
|
119
|
+
private readonly maxResults: number;
|
|
91
120
|
private readonly parsersByWasmPath = new Map<string, Parser>();
|
|
92
121
|
|
|
93
|
-
constructor(rootPath: string, contentCache: ContentCachePort = new InMemoryContentCache()) {
|
|
122
|
+
constructor(rootPath: string, contentCache: ContentCachePort = new InMemoryContentCache(), options: TreeSitterSymbolIndexOptions = {}) {
|
|
94
123
|
this.rootPath = rootPath;
|
|
95
124
|
this.contentCache = contentCache;
|
|
125
|
+
this.maxFiles = positiveLimit(options.maxFiles, DEFAULT_MAX_FILES, "maxFiles");
|
|
126
|
+
this.maxFileBytes = positiveLimit(options.maxFileBytes, DEFAULT_MAX_FILE_BYTES, "maxFileBytes");
|
|
127
|
+
this.maxTotalBytes = positiveLimit(options.maxTotalBytes, DEFAULT_MAX_TOTAL_BYTES, "maxTotalBytes");
|
|
128
|
+
this.maxResults = positiveLimit(options.maxResults, DEFAULT_MAX_RESULTS, "maxResults");
|
|
96
129
|
}
|
|
97
130
|
|
|
98
131
|
private async parserFor(wasmPath: string): Promise<Parser> {
|
|
@@ -106,7 +139,7 @@ export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
|
106
139
|
return parser;
|
|
107
140
|
}
|
|
108
141
|
|
|
109
|
-
private async contentSymbolsFor(
|
|
142
|
+
private async contentSymbolsFor(wasmPath: string, content: string): Promise<ContentSymbol[]> {
|
|
110
143
|
const hash = contentHashOf(content);
|
|
111
144
|
|
|
112
145
|
// Reading the file already required reading its content -- warm the fs lens for this
|
|
@@ -125,28 +158,45 @@ export class TreeSitterSymbolIndex implements SymbolIndexPort {
|
|
|
125
158
|
return symbols;
|
|
126
159
|
}
|
|
127
160
|
|
|
128
|
-
async findSymbols(query: string): Promise<
|
|
161
|
+
async findSymbols(query: string, bounds: SymbolSearchBounds = { maxResults: this.maxResults }): Promise<SymbolSearchResult> {
|
|
162
|
+
const maxResults = Math.min(positiveLimit(bounds.maxResults, this.maxResults, "maxResults"), this.maxResults);
|
|
129
163
|
const lowerQuery = query.toLowerCase();
|
|
130
164
|
const results: WorkspaceSymbol[] = [];
|
|
165
|
+
const files = findSourceFiles(this.rootPath, (extension) => wasmPathFor(extension) !== undefined, this.maxFiles);
|
|
166
|
+
let totalBytes = 0;
|
|
167
|
+
let truncated = files.length === this.maxFiles;
|
|
131
168
|
|
|
132
|
-
for (const relativePath of
|
|
169
|
+
for (const relativePath of files) {
|
|
133
170
|
const wasmPath = wasmPathFor(extname(relativePath));
|
|
134
171
|
if (!wasmPath) continue;
|
|
135
172
|
|
|
136
173
|
let content: string;
|
|
137
174
|
try {
|
|
138
|
-
|
|
175
|
+
const absolutePath = join(this.rootPath, relativePath);
|
|
176
|
+
const size = statSync(absolutePath).size;
|
|
177
|
+
if (size > this.maxFileBytes || totalBytes + size > this.maxTotalBytes) {
|
|
178
|
+
truncated = true;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
totalBytes += size;
|
|
182
|
+
content = readFileSync(absolutePath, "utf-8");
|
|
139
183
|
} catch {
|
|
140
184
|
continue;
|
|
141
185
|
}
|
|
142
186
|
|
|
143
|
-
const contentSymbols = await this.contentSymbolsFor(
|
|
187
|
+
const contentSymbols = await this.contentSymbolsFor(wasmPath, content);
|
|
144
188
|
for (const symbol of toWorkspaceSymbols(contentSymbols, relativePath)) {
|
|
145
|
-
if (symbol.name.toLowerCase().includes(lowerQuery))
|
|
189
|
+
if (!symbol.name.toLowerCase().includes(lowerQuery)) continue;
|
|
190
|
+
if (results.length === maxResults) {
|
|
191
|
+
truncated = true;
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
results.push(symbol);
|
|
146
195
|
}
|
|
196
|
+
if (results.length === maxResults) break;
|
|
147
197
|
}
|
|
148
198
|
|
|
149
|
-
return results;
|
|
199
|
+
return { symbols: results, truncated, provenance: this.provenance };
|
|
150
200
|
}
|
|
151
201
|
|
|
152
202
|
async close(): Promise<void> {
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { extname, join } from "node:path";
|
|
3
|
+
import ts from "typescript";
|
|
4
|
+
import type { IntelligenceProvenance, SymbolSearchBounds } from "../domain/intelligence-provenance.ts";
|
|
5
|
+
import type { SymbolSearchResult, WorkspaceSymbol } from "../domain/workspace-symbol.ts";
|
|
6
|
+
import type { SymbolIndexPort } from "../ports/symbol-index-port.ts";
|
|
7
|
+
import { findSourceFiles } from "./find-source-files.ts";
|
|
8
|
+
|
|
9
|
+
const SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
|
|
10
|
+
|
|
11
|
+
export interface TypeScriptCompilerSymbolIndexOptions {
|
|
12
|
+
readonly maxFiles?: number;
|
|
13
|
+
readonly maxFileBytes?: number;
|
|
14
|
+
readonly maxTotalBytes?: number;
|
|
15
|
+
readonly maxResults?: number;
|
|
16
|
+
readonly maxNodesPerFile?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function positiveLimit(value: number | undefined, fallback: number, field: string): number {
|
|
20
|
+
const result = value ?? fallback;
|
|
21
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new TypeError(`${field} must be a positive safe integer`);
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function scriptKind(path: string): ts.ScriptKind {
|
|
26
|
+
switch (extname(path)) {
|
|
27
|
+
case ".tsx":
|
|
28
|
+
return ts.ScriptKind.TSX;
|
|
29
|
+
case ".jsx":
|
|
30
|
+
return ts.ScriptKind.JSX;
|
|
31
|
+
case ".js":
|
|
32
|
+
case ".mjs":
|
|
33
|
+
case ".cjs":
|
|
34
|
+
return ts.ScriptKind.JS;
|
|
35
|
+
default:
|
|
36
|
+
return ts.ScriptKind.TS;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function declaration(node: ts.Node): { name: ts.Node; kind: string } | undefined {
|
|
41
|
+
if (ts.isFunctionDeclaration(node) && node.name) return { name: node.name, kind: "function" };
|
|
42
|
+
if (ts.isClassDeclaration(node) && node.name) return { name: node.name, kind: "class" };
|
|
43
|
+
if (ts.isInterfaceDeclaration(node)) return { name: node.name, kind: "interface" };
|
|
44
|
+
if (ts.isTypeAliasDeclaration(node)) return { name: node.name, kind: "type-alias" };
|
|
45
|
+
if (ts.isEnumDeclaration(node)) return { name: node.name, kind: "enum" };
|
|
46
|
+
if (ts.isMethodDeclaration(node)) return { name: node.name, kind: "method" };
|
|
47
|
+
if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) return { name: node.name, kind: "variable" };
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class TypeScriptCompilerSymbolIndex implements SymbolIndexPort {
|
|
52
|
+
readonly provenance: IntelligenceProvenance = {
|
|
53
|
+
fidelity: "structural",
|
|
54
|
+
backend: "typescript-compiler",
|
|
55
|
+
languageId: "typescript-javascript",
|
|
56
|
+
authority: "compiler",
|
|
57
|
+
freshness: "filesystem-snapshot",
|
|
58
|
+
limitations: ["syntax declarations only", "no cross-file identity", "no language-server project state"],
|
|
59
|
+
};
|
|
60
|
+
private readonly maxFiles: number;
|
|
61
|
+
private readonly maxFileBytes: number;
|
|
62
|
+
private readonly maxTotalBytes: number;
|
|
63
|
+
private readonly maxResults: number;
|
|
64
|
+
private readonly maxNodesPerFile: number;
|
|
65
|
+
|
|
66
|
+
constructor(
|
|
67
|
+
private readonly rootPath: string,
|
|
68
|
+
options: TypeScriptCompilerSymbolIndexOptions = {},
|
|
69
|
+
) {
|
|
70
|
+
this.maxFiles = positiveLimit(options.maxFiles, 5_000, "maxFiles");
|
|
71
|
+
this.maxFileBytes = positiveLimit(options.maxFileBytes, 2 * 1024 * 1024, "maxFileBytes");
|
|
72
|
+
this.maxTotalBytes = positiveLimit(options.maxTotalBytes, 50 * 1024 * 1024, "maxTotalBytes");
|
|
73
|
+
this.maxResults = positiveLimit(options.maxResults, 1_000, "maxResults");
|
|
74
|
+
this.maxNodesPerFile = positiveLimit(options.maxNodesPerFile, 100_000, "maxNodesPerFile");
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
findSymbols(query: string, bounds: SymbolSearchBounds = { maxResults: this.maxResults }): Promise<SymbolSearchResult> {
|
|
78
|
+
const maxResults = Math.min(positiveLimit(bounds.maxResults, this.maxResults, "maxResults"), this.maxResults);
|
|
79
|
+
const files = findSourceFiles(this.rootPath, (extension) => SOURCE_EXTENSIONS.has(extension), this.maxFiles);
|
|
80
|
+
const symbols: WorkspaceSymbol[] = [];
|
|
81
|
+
const lowerQuery = query.toLowerCase();
|
|
82
|
+
let totalBytes = 0;
|
|
83
|
+
let truncated = files.length === this.maxFiles;
|
|
84
|
+
|
|
85
|
+
for (const relativePath of files) {
|
|
86
|
+
const absolutePath = join(this.rootPath, relativePath);
|
|
87
|
+
let sourceText: string;
|
|
88
|
+
try {
|
|
89
|
+
const size = statSync(absolutePath).size;
|
|
90
|
+
if (size > this.maxFileBytes || totalBytes + size > this.maxTotalBytes) {
|
|
91
|
+
truncated = true;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
totalBytes += size;
|
|
95
|
+
sourceText = readFileSync(absolutePath, "utf-8");
|
|
96
|
+
} catch {
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const sourceFile = ts.createSourceFile(relativePath, sourceText, ts.ScriptTarget.Latest, true, scriptKind(relativePath));
|
|
101
|
+
let visited = 0;
|
|
102
|
+
const visit = (node: ts.Node): void => {
|
|
103
|
+
if (visited++ >= this.maxNodesPerFile || symbols.length >= maxResults) {
|
|
104
|
+
truncated = true;
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const found = declaration(node);
|
|
108
|
+
if (found) {
|
|
109
|
+
const name = found.name.getText(sourceFile);
|
|
110
|
+
if (name.toLowerCase().includes(lowerQuery)) {
|
|
111
|
+
const position = sourceFile.getLineAndCharacterOfPosition(found.name.getStart(sourceFile));
|
|
112
|
+
symbols.push({ name, kind: found.kind, location: { path: relativePath, line: position.line + 1, character: position.character + 1 } });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (visited < this.maxNodesPerFile && symbols.length < maxResults) ts.forEachChild(node, visit);
|
|
116
|
+
else if (node.getChildCount(sourceFile) > 0) truncated = true;
|
|
117
|
+
};
|
|
118
|
+
visit(sourceFile);
|
|
119
|
+
if (symbols.length >= maxResults) {
|
|
120
|
+
truncated = true;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return Promise.resolve({ symbols, truncated, provenance: this.provenance });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async close(): Promise<void> {}
|
|
129
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { execFileSync } from "node:child_process";
|
|
3
3
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
|
-
import { dirname, join } from "node:path";
|
|
5
|
+
import { dirname, join, resolve } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { InMemoryWorkspace } from "./adapters/in-memory-workspace.ts";
|
|
8
8
|
import { LocalFilesystemWorkspace } from "./adapters/local-filesystem-workspace.ts";
|
|
@@ -11,6 +11,7 @@ import { LECTOR_PATH_NAMES } from "./constants.ts";
|
|
|
11
11
|
import { serveMain } from "./daemon.ts";
|
|
12
12
|
import type { JobSnapshot } from "./domain/bounded-job-executor.ts";
|
|
13
13
|
import type { ContentHash } from "./domain/content-hash.ts";
|
|
14
|
+
import { DEFAULT_PACKAGE_SOURCE_BOUNDS, type PackageSourceOperationResult } from "./domain/package-source.ts";
|
|
14
15
|
import type { WorkspacePort } from "./ports/workspace-port.ts";
|
|
15
16
|
import type { WorkspaceId } from "./service.ts";
|
|
16
17
|
|
|
@@ -51,6 +52,8 @@ const USAGE = `Usage:
|
|
|
51
52
|
lector workspace git-diff <workspace-id> [--ref <ref>] --max-bytes <n> [--json]
|
|
52
53
|
lector workspace repo-fetch <owner>/<repo>[@ref] [--host <host>] [--json]
|
|
53
54
|
shallow-clones an external repo into a disk-bounded cache and registers it read-only
|
|
55
|
+
lector package source <project-dir> <package-name> [--version <exact-version>] [--registry <url>] [--json]
|
|
56
|
+
resolves an installed npm package to verified exact repository source and registers it read-only
|
|
54
57
|
lector workspace search-text <workspace-id> <query> --max-matches <n> --max-bytes <n> [--json]
|
|
55
58
|
lector search symbols <query> [--workspace <id>]... [--timeout-ms <n>] [--json]
|
|
56
59
|
lector search text <query> --max-matches <n> --max-bytes <n> [--workspace <id>]... [--timeout-ms <n>] [--json]
|
|
@@ -131,11 +134,13 @@ async function runWorkspaceSymbols(workspaceId: string | undefined, query: strin
|
|
|
131
134
|
if (!workspaceId || !query) fail(USAGE);
|
|
132
135
|
const seedFile = flagValue(flags, "--seed-file"); // omit to auto-discover one
|
|
133
136
|
const client = await connectLectorClient();
|
|
134
|
-
const
|
|
137
|
+
const result = await client.call("workspace.findSymbols", { workspaceId, query, seedFile });
|
|
135
138
|
if (hasFlag(flags, "--json")) {
|
|
136
|
-
console.log(JSON.stringify(
|
|
139
|
+
console.log(JSON.stringify(result));
|
|
137
140
|
return;
|
|
138
141
|
}
|
|
142
|
+
const { symbols, provenance, truncated } = result;
|
|
143
|
+
console.log(`${provenance.fidelity} via ${provenance.backend}${truncated ? " (truncated)" : ""}`);
|
|
139
144
|
if (symbols.length === 0) {
|
|
140
145
|
console.log(`no symbols matched "${query}"`);
|
|
141
146
|
return;
|
|
@@ -154,16 +159,22 @@ function parsePosition(line: string | undefined, character: string | undefined):
|
|
|
154
159
|
return { line: parsedLine, character: parsedCharacter };
|
|
155
160
|
}
|
|
156
161
|
|
|
162
|
+
function formatIntelligenceSource(provenance: { fidelity: string; backend: string }): string {
|
|
163
|
+
return `${provenance.fidelity} via ${provenance.backend}`;
|
|
164
|
+
}
|
|
165
|
+
|
|
157
166
|
async function runWorkspaceDefinition(workspaceId: string | undefined, path: string | undefined, rest: string[]): Promise<void> {
|
|
158
167
|
if (!workspaceId || !path) fail(USAGE);
|
|
159
168
|
const [lineArg, characterArg, ...flags] = rest;
|
|
160
169
|
const { line, character } = parsePosition(lineArg, characterArg);
|
|
161
170
|
const client = await connectLectorClient();
|
|
162
|
-
const
|
|
171
|
+
const result = await client.call("workspace.goToDefinition", { workspaceId, path, line, character });
|
|
163
172
|
if (hasFlag(flags, "--json")) {
|
|
164
|
-
console.log(JSON.stringify(
|
|
173
|
+
console.log(JSON.stringify(result));
|
|
165
174
|
return;
|
|
166
175
|
}
|
|
176
|
+
const { locations, provenance } = result;
|
|
177
|
+
console.log(formatIntelligenceSource(provenance));
|
|
167
178
|
if (locations.length === 0) {
|
|
168
179
|
console.log("no definition found");
|
|
169
180
|
return;
|
|
@@ -176,11 +187,13 @@ async function runWorkspaceImplementation(workspaceId: string | undefined, path:
|
|
|
176
187
|
const [lineArg, characterArg, ...flags] = rest;
|
|
177
188
|
const { line, character } = parsePosition(lineArg, characterArg);
|
|
178
189
|
const client = await connectLectorClient();
|
|
179
|
-
const
|
|
190
|
+
const result = await client.call("workspace.goToImplementation", { workspaceId, path, line, character });
|
|
180
191
|
if (hasFlag(flags, "--json")) {
|
|
181
|
-
console.log(JSON.stringify(
|
|
192
|
+
console.log(JSON.stringify(result));
|
|
182
193
|
return;
|
|
183
194
|
}
|
|
195
|
+
const { locations, provenance } = result;
|
|
196
|
+
console.log(formatIntelligenceSource(provenance));
|
|
184
197
|
if (locations.length === 0) {
|
|
185
198
|
console.log("no implementation found");
|
|
186
199
|
return;
|
|
@@ -194,11 +207,13 @@ async function runWorkspaceReferences(workspaceId: string | undefined, path: str
|
|
|
194
207
|
const { line, character } = parsePosition(lineArg, characterArg);
|
|
195
208
|
const includeDeclaration = hasFlag(flags, "--include-declaration");
|
|
196
209
|
const client = await connectLectorClient();
|
|
197
|
-
const
|
|
210
|
+
const result = await client.call("workspace.findReferences", { workspaceId, path, line, character, includeDeclaration });
|
|
198
211
|
if (hasFlag(flags, "--json")) {
|
|
199
|
-
console.log(JSON.stringify(
|
|
212
|
+
console.log(JSON.stringify(result));
|
|
200
213
|
return;
|
|
201
214
|
}
|
|
215
|
+
const { locations, provenance } = result;
|
|
216
|
+
console.log(formatIntelligenceSource(provenance));
|
|
202
217
|
if (locations.length === 0) {
|
|
203
218
|
console.log("no references found");
|
|
204
219
|
return;
|
|
@@ -211,22 +226,25 @@ async function runWorkspaceHover(workspaceId: string | undefined, path: string |
|
|
|
211
226
|
const [lineArg, characterArg, ...flags] = rest;
|
|
212
227
|
const { line, character } = parsePosition(lineArg, characterArg);
|
|
213
228
|
const client = await connectLectorClient();
|
|
214
|
-
const
|
|
229
|
+
const result = await client.call("workspace.hover", { workspaceId, path, line, character });
|
|
215
230
|
if (hasFlag(flags, "--json")) {
|
|
216
|
-
console.log(JSON.stringify(
|
|
231
|
+
console.log(JSON.stringify(result));
|
|
217
232
|
return;
|
|
218
233
|
}
|
|
219
|
-
console.log(
|
|
234
|
+
console.log(formatIntelligenceSource(result.provenance));
|
|
235
|
+
console.log(result.hover ? result.hover.contents : "no hover information available");
|
|
220
236
|
}
|
|
221
237
|
|
|
222
238
|
async function runWorkspaceDocumentSymbols(workspaceId: string | undefined, path: string | undefined, flags: string[]): Promise<void> {
|
|
223
239
|
if (!workspaceId || !path) fail(USAGE);
|
|
224
240
|
const client = await connectLectorClient();
|
|
225
|
-
const
|
|
241
|
+
const result = await client.call("workspace.documentSymbols", { workspaceId, path });
|
|
226
242
|
if (hasFlag(flags, "--json")) {
|
|
227
|
-
console.log(JSON.stringify(
|
|
243
|
+
console.log(JSON.stringify(result));
|
|
228
244
|
return;
|
|
229
245
|
}
|
|
246
|
+
const { symbols, provenance } = result;
|
|
247
|
+
console.log(formatIntelligenceSource(provenance));
|
|
230
248
|
if (symbols.length === 0) {
|
|
231
249
|
console.log("no symbols found");
|
|
232
250
|
return;
|
|
@@ -241,11 +259,13 @@ async function runWorkspaceDocumentSymbols(workspaceId: string | undefined, path
|
|
|
241
259
|
async function runWorkspaceDiagnostics(workspaceId: string | undefined, path: string | undefined, flags: string[]): Promise<void> {
|
|
242
260
|
if (!workspaceId || !path) fail(USAGE);
|
|
243
261
|
const client = await connectLectorClient();
|
|
244
|
-
const
|
|
262
|
+
const result = await client.call("workspace.diagnostics", { workspaceId, path });
|
|
245
263
|
if (hasFlag(flags, "--json")) {
|
|
246
|
-
console.log(JSON.stringify(
|
|
264
|
+
console.log(JSON.stringify(result));
|
|
247
265
|
return;
|
|
248
266
|
}
|
|
267
|
+
const { diagnostics, provenance } = result;
|
|
268
|
+
console.log(formatIntelligenceSource(provenance));
|
|
249
269
|
if (diagnostics.length === 0) {
|
|
250
270
|
console.log("no diagnostics");
|
|
251
271
|
return;
|
|
@@ -273,11 +293,13 @@ async function runWorkspaceCallHierarchy(
|
|
|
273
293
|
const client = await connectLectorClient();
|
|
274
294
|
|
|
275
295
|
if (subcommand === "prepare") {
|
|
276
|
-
const
|
|
296
|
+
const result = await client.call("workspace.prepareCallHierarchy", { workspaceId, path, line, character });
|
|
277
297
|
if (hasFlag(flags, "--json")) {
|
|
278
|
-
console.log(JSON.stringify(
|
|
298
|
+
console.log(JSON.stringify(result));
|
|
279
299
|
return;
|
|
280
300
|
}
|
|
301
|
+
const { items, provenance } = result;
|
|
302
|
+
console.log(formatIntelligenceSource(provenance));
|
|
281
303
|
if (items.length === 0) {
|
|
282
304
|
console.log("no call-hierarchy root at this position");
|
|
283
305
|
return;
|
|
@@ -286,11 +308,13 @@ async function runWorkspaceCallHierarchy(
|
|
|
286
308
|
return;
|
|
287
309
|
}
|
|
288
310
|
if (subcommand === "incoming") {
|
|
289
|
-
const
|
|
311
|
+
const result = await client.call("workspace.incomingCalls", { workspaceId, path, line, character });
|
|
290
312
|
if (hasFlag(flags, "--json")) {
|
|
291
|
-
console.log(JSON.stringify(
|
|
313
|
+
console.log(JSON.stringify(result));
|
|
292
314
|
return;
|
|
293
315
|
}
|
|
316
|
+
const { calls, provenance } = result;
|
|
317
|
+
console.log(formatIntelligenceSource(provenance));
|
|
294
318
|
if (calls.length === 0) {
|
|
295
319
|
console.log("no incoming calls found");
|
|
296
320
|
return;
|
|
@@ -299,11 +323,13 @@ async function runWorkspaceCallHierarchy(
|
|
|
299
323
|
return;
|
|
300
324
|
}
|
|
301
325
|
if (subcommand === "outgoing") {
|
|
302
|
-
const
|
|
326
|
+
const result = await client.call("workspace.outgoingCalls", { workspaceId, path, line, character });
|
|
303
327
|
if (hasFlag(flags, "--json")) {
|
|
304
|
-
console.log(JSON.stringify(
|
|
328
|
+
console.log(JSON.stringify(result));
|
|
305
329
|
return;
|
|
306
330
|
}
|
|
331
|
+
const { calls, provenance } = result;
|
|
332
|
+
console.log(formatIntelligenceSource(provenance));
|
|
307
333
|
if (calls.length === 0) {
|
|
308
334
|
console.log("no outgoing calls found");
|
|
309
335
|
return;
|
|
@@ -454,6 +480,38 @@ async function runWorkspaceRepoFetch(spec: string | undefined, flags: string[]):
|
|
|
454
480
|
if (result.refFallbackOccurred) console.log(`note: requested ref not found, fell back to the default branch (resolved: ${result.resolvedRef})`);
|
|
455
481
|
}
|
|
456
482
|
|
|
483
|
+
function formatPackageSourceResult(result: PackageSourceOperationResult): string {
|
|
484
|
+
const { outcome } = result;
|
|
485
|
+
if (outcome.status === "verified") {
|
|
486
|
+
return `${result.workspaceId ?? "unregistered"} ${outcome.coordinate.name}@${outcome.coordinate.resolvedVersion} -- ${outcome.workspace.cachePath}\n${outcome.repository.url ?? "local source"}@${outcome.repository.resolvedRef ?? "local"} ${outcome.repository.commit ?? outcome.verification.integrity}`;
|
|
487
|
+
}
|
|
488
|
+
if (outcome.status === "ambiguous") {
|
|
489
|
+
return `ambiguous [${outcome.code}] -- ${outcome.candidates.map((candidate) => `${candidate.version} (${candidate.source})`).join(", ")}${outcome.truncated ? ", …" : ""}`;
|
|
490
|
+
}
|
|
491
|
+
if (outcome.status === "unauthenticated") return `unauthenticated [${outcome.code}] -- configure ${outcome.requiredCredentialNames.join(", ")}`;
|
|
492
|
+
if (outcome.status === "oversized") return `oversized [${outcome.code}] -- ${outcome.resource} exceeded ${outcome.limit}`;
|
|
493
|
+
if (outcome.status === "mismatched") return `mismatched [${outcome.code}] -- expected ${outcome.expected}, got ${outcome.actual}`;
|
|
494
|
+
return `unavailable [${outcome.code}]`;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function runPackageSource(projectDir: string | undefined, packageName: string | undefined, flags: string[]): Promise<void> {
|
|
498
|
+
if (!projectDir || !packageName) fail(USAGE);
|
|
499
|
+
const client = await connectLectorClient();
|
|
500
|
+
const result = await client.call("package.resolveSource", {
|
|
501
|
+
request: {
|
|
502
|
+
projectRoot: resolve(projectDir),
|
|
503
|
+
coordinate: {
|
|
504
|
+
ecosystem: "npm",
|
|
505
|
+
registry: flagValue(flags, "--registry") ?? null,
|
|
506
|
+
name: packageName,
|
|
507
|
+
requestedVersion: flagValue(flags, "--version") ?? null,
|
|
508
|
+
},
|
|
509
|
+
},
|
|
510
|
+
bounds: DEFAULT_PACKAGE_SOURCE_BOUNDS,
|
|
511
|
+
});
|
|
512
|
+
console.log(hasFlag(flags, "--json") ? JSON.stringify(result) : formatPackageSourceResult(result));
|
|
513
|
+
}
|
|
514
|
+
|
|
457
515
|
async function runWorkspaceSearchText(workspaceId: string | undefined, query: string | undefined, flags: string[]): Promise<void> {
|
|
458
516
|
if (!workspaceId || !query) fail(USAGE);
|
|
459
517
|
const maxMatches = requiredIntFlag(flags, "--max-matches");
|
|
@@ -655,7 +713,7 @@ WantedBy=default.target
|
|
|
655
713
|
}
|
|
656
714
|
|
|
657
715
|
function unitPath(): string {
|
|
658
|
-
const configHome = process.env
|
|
716
|
+
const configHome = process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config");
|
|
659
717
|
return join(configHome, "systemd", "user", LECTOR_PATH_NAMES.systemdUnitName);
|
|
660
718
|
}
|
|
661
719
|
|
|
@@ -713,6 +771,12 @@ async function main(): Promise<void> {
|
|
|
713
771
|
fail(USAGE);
|
|
714
772
|
}
|
|
715
773
|
|
|
774
|
+
if (command === "package") {
|
|
775
|
+
const [action, projectDir, packageName, ...packageFlags] = rest;
|
|
776
|
+
if (action === "source") return runPackageSource(projectDir, packageName, packageFlags);
|
|
777
|
+
fail(USAGE);
|
|
778
|
+
}
|
|
779
|
+
|
|
716
780
|
if (command === "workspace") {
|
|
717
781
|
const [action, ...actionArgs] = rest;
|
|
718
782
|
if (action === "register") {
|
package/src/daemon.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { SqliteSearchCache } from "./adapters/sqlite-search-cache.ts";
|
|
|
9
9
|
import { SqliteSymbolGraph } from "./adapters/sqlite-symbol-graph.ts";
|
|
10
10
|
import { TieredSearchCache } from "./adapters/tiered-search-cache.ts";
|
|
11
11
|
import { resolveLectorPaths } from "./constants.ts";
|
|
12
|
+
import type { PackageSourceResolverPort } from "./ports/package-source-resolver-port.ts";
|
|
12
13
|
import type { RepoFetcherPort } from "./ports/repo-fetcher-port.ts";
|
|
13
14
|
import type { WorkspacePort } from "./ports/workspace-port.ts";
|
|
14
15
|
import { createLectorService, type LectorService, type OperationName, type WorkspaceId } from "./service.ts";
|
|
@@ -84,6 +85,8 @@ export interface LectorDaemonOptions {
|
|
|
84
85
|
symbolIndexReapIntervalMs?: number;
|
|
85
86
|
/** Override the repo.fetch backend (tests inject a GitRepoFetcher pointed at a local fixture repo, avoiding live network). Defaults to a real GitRepoFetcher under this daemon's own data directory. */
|
|
86
87
|
createRepoFetcher?: () => RepoFetcherPort;
|
|
88
|
+
/** Override package source resolution while retaining the authenticated daemon/client seam. */
|
|
89
|
+
createPackageSourceResolver?: () => PackageSourceResolverPort;
|
|
87
90
|
}
|
|
88
91
|
|
|
89
92
|
function prepare(options: LectorDaemonOptions): {
|
|
@@ -110,6 +113,7 @@ function prepare(options: LectorDaemonOptions): {
|
|
|
110
113
|
allowDynamicOnly: options.allowDynamicOnly,
|
|
111
114
|
createSymbolGraph: (workspaceId) => new SqliteSymbolGraph(join(symbolGraphDirectory, `${workspaceId}.db`)),
|
|
112
115
|
createRepoFetcher: options.createRepoFetcher ?? (() => new GitRepoFetcher(reposDirectory)),
|
|
116
|
+
createPackageSourceResolver: options.createPackageSourceResolver,
|
|
113
117
|
// The real production shape the SearchCachePort design was for: an in-memory tier for
|
|
114
118
|
// speed plus a disk-backed tier so repeated searches survive a daemon restart -- a single
|
|
115
119
|
// SearchCachePort adapter can only be one or the other, service.ts's own safe default is
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { SymbolIndexPort } from "../ports/symbol-index-port.ts";
|
|
2
|
-
import type {
|
|
2
|
+
import type { SymbolSearchBounds } from "./intelligence-provenance.ts";
|
|
3
|
+
import type { SymbolSearchResult } from "./workspace-symbol.ts";
|
|
3
4
|
|
|
4
5
|
/** Find workspace symbols matching a fuzzy query string. */
|
|
5
|
-
export async function findWorkspaceSymbols(index: SymbolIndexPort, query: string): Promise<
|
|
6
|
-
return index.findSymbols(query);
|
|
6
|
+
export async function findWorkspaceSymbols(index: SymbolIndexPort, query: string, bounds?: SymbolSearchBounds): Promise<SymbolSearchResult> {
|
|
7
|
+
return index.findSymbols(query, bounds);
|
|
7
8
|
}
|