@wrongstack/tools 0.236.0 → 0.250.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.js +1 -0
- package/dist/audit.js.map +1 -1
- package/dist/background-indexer-DwJsyAB0.d.ts +373 -0
- package/dist/bash.js +5 -0
- package/dist/bash.js.map +1 -1
- package/dist/builtin.js +865 -327
- package/dist/builtin.js.map +1 -1
- package/dist/codebase-index/index.d.ts +53 -2
- package/dist/codebase-index/index.js +854 -364
- package/dist/codebase-index/index.js.map +1 -1
- package/dist/codebase-index/worker.d.ts +2 -0
- package/dist/codebase-index/worker.js +2321 -0
- package/dist/codebase-index/worker.js.map +1 -0
- package/dist/diff.js +2 -1
- package/dist/diff.js.map +1 -1
- package/dist/exec.js +1 -0
- package/dist/exec.js.map +1 -1
- package/dist/format.js +1 -0
- package/dist/format.js.map +1 -1
- package/dist/git.js +2 -1
- package/dist/git.js.map +1 -1
- package/dist/grep.js +2 -2
- package/dist/grep.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +886 -386
- package/dist/index.js.map +1 -1
- package/dist/install.js +1 -0
- package/dist/install.js.map +1 -1
- package/dist/lint.js +1 -0
- package/dist/lint.js.map +1 -1
- package/dist/logs.js +1 -1
- package/dist/logs.js.map +1 -1
- package/dist/outdated.js +1 -1
- package/dist/outdated.js.map +1 -1
- package/dist/pack.js +865 -327
- package/dist/pack.js.map +1 -1
- package/dist/patch.js +1 -1
- package/dist/patch.js.map +1 -1
- package/dist/replace.js +3 -2
- package/dist/replace.js.map +1 -1
- package/dist/test.js +1 -0
- package/dist/test.js.map +1 -1
- package/dist/typecheck.js +1 -0
- package/dist/typecheck.js.map +1 -1
- package/package.json +2 -2
- package/dist/background-indexer-CtbgPExj.d.ts +0 -228
|
@@ -0,0 +1,2321 @@
|
|
|
1
|
+
import { parentPort } from 'node:worker_threads';
|
|
2
|
+
import { expectDefined, resolveWstackPaths, compileGlob, truncate } from '@wrongstack/core';
|
|
3
|
+
import * as fs3 from 'node:fs/promises';
|
|
4
|
+
import * as path4 from 'node:path';
|
|
5
|
+
import { createRequire } from 'node:module';
|
|
6
|
+
import * as fs from 'node:fs';
|
|
7
|
+
import { writeFileSync, mkdirSync } from 'node:fs';
|
|
8
|
+
import * as ts from 'typescript';
|
|
9
|
+
import { execFileSync, spawnSync } from 'node:child_process';
|
|
10
|
+
import * as os from 'node:os';
|
|
11
|
+
|
|
12
|
+
// src/codebase-index/worker.ts
|
|
13
|
+
|
|
14
|
+
// src/codebase-index/circuit-breaker.ts
|
|
15
|
+
var LockError = class extends Error {
|
|
16
|
+
name = "LockError";
|
|
17
|
+
};
|
|
18
|
+
var IndexCircuitBreaker = class {
|
|
19
|
+
failureThreshold;
|
|
20
|
+
cooldownMs;
|
|
21
|
+
now;
|
|
22
|
+
state = "closed";
|
|
23
|
+
consecutiveFailures = 0;
|
|
24
|
+
openedAt = 0;
|
|
25
|
+
lastFailure = null;
|
|
26
|
+
probeInFlight = false;
|
|
27
|
+
constructor(opts = {}) {
|
|
28
|
+
this.failureThreshold = opts.failureThreshold ?? 3;
|
|
29
|
+
this.cooldownMs = opts.cooldownMs ?? 6e4;
|
|
30
|
+
this.now = opts.now ?? Date.now;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* True when a run may proceed. An open circuit transitions to half-open once
|
|
34
|
+
* the cooldown has elapsed, admitting exactly one probe; further requests
|
|
35
|
+
* are rejected until that probe settles via recordSuccess/recordFailure.
|
|
36
|
+
*/
|
|
37
|
+
allowRequest() {
|
|
38
|
+
if (this.state === "closed") return true;
|
|
39
|
+
if (this.state === "open") {
|
|
40
|
+
if (this.now() - this.openedAt < this.cooldownMs) return false;
|
|
41
|
+
this.state = "half-open";
|
|
42
|
+
this.probeInFlight = true;
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
if (this.probeInFlight) return false;
|
|
46
|
+
this.probeInFlight = true;
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
recordSuccess() {
|
|
50
|
+
this.state = "closed";
|
|
51
|
+
this.consecutiveFailures = 0;
|
|
52
|
+
this.lastFailure = null;
|
|
53
|
+
this.probeInFlight = false;
|
|
54
|
+
}
|
|
55
|
+
recordFailure(err) {
|
|
56
|
+
if (err instanceof LockError) {
|
|
57
|
+
this.lastFailure = `[transient/lock] ${err.message}`;
|
|
58
|
+
this.probeInFlight = false;
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
this.lastFailure = err instanceof Error ? err.message : String(err);
|
|
62
|
+
this.probeInFlight = false;
|
|
63
|
+
this.consecutiveFailures++;
|
|
64
|
+
if (this.state === "half-open" || this.consecutiveFailures >= this.failureThreshold) {
|
|
65
|
+
this.state = "open";
|
|
66
|
+
this.openedAt = this.now();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** Force-close the circuit (manual recovery: `/codebase-reindex`). */
|
|
70
|
+
reset() {
|
|
71
|
+
this.state = "closed";
|
|
72
|
+
this.consecutiveFailures = 0;
|
|
73
|
+
this.lastFailure = null;
|
|
74
|
+
this.probeInFlight = false;
|
|
75
|
+
this.openedAt = 0;
|
|
76
|
+
}
|
|
77
|
+
snapshot() {
|
|
78
|
+
return {
|
|
79
|
+
state: this.state,
|
|
80
|
+
consecutiveFailures: this.consecutiveFailures,
|
|
81
|
+
lastFailure: this.lastFailure,
|
|
82
|
+
cooldownRemainingMs: this.state === "open" ? Math.max(0, this.cooldownMs - (this.now() - this.openedAt)) : 0
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
new IndexCircuitBreaker();
|
|
87
|
+
|
|
88
|
+
// src/codebase-index/schema.ts
|
|
89
|
+
var SCHEMA_VERSION = 2;
|
|
90
|
+
|
|
91
|
+
// src/codebase-index/lsp-kind.ts
|
|
92
|
+
function lspKindToInternalKind(k) {
|
|
93
|
+
switch (k) {
|
|
94
|
+
case 5 /* Class */:
|
|
95
|
+
return "class";
|
|
96
|
+
case 6 /* Method */:
|
|
97
|
+
return "method";
|
|
98
|
+
case 7 /* Property */:
|
|
99
|
+
case 8 /* Field */:
|
|
100
|
+
return "property";
|
|
101
|
+
case 9 /* Constructor */:
|
|
102
|
+
return "class";
|
|
103
|
+
case 10 /* Enum */:
|
|
104
|
+
return "enum";
|
|
105
|
+
case 11 /* Interface */:
|
|
106
|
+
return "interface";
|
|
107
|
+
case 12 /* Function */:
|
|
108
|
+
return "function";
|
|
109
|
+
case 13 /* Variable */:
|
|
110
|
+
return "var";
|
|
111
|
+
case 14 /* Constant */:
|
|
112
|
+
return "const";
|
|
113
|
+
case 22 /* EnumMember */:
|
|
114
|
+
return "enum";
|
|
115
|
+
case 26 /* TypeParameter */:
|
|
116
|
+
return "type";
|
|
117
|
+
case 3 /* Namespace */:
|
|
118
|
+
return "namespace";
|
|
119
|
+
default:
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// src/codebase-index/bm25.ts
|
|
125
|
+
var K1 = 1.5;
|
|
126
|
+
var B = 0.75;
|
|
127
|
+
function tokenise(text) {
|
|
128
|
+
const sanitised = text.replace(/[^\p{L}\p{N}$'_]/gu, " ").replace(/_/g, " ");
|
|
129
|
+
return sanitised.toLowerCase().split(" ").filter(Boolean);
|
|
130
|
+
}
|
|
131
|
+
function splitName(name) {
|
|
132
|
+
return name.replace(/([a-z])([A-Z])/g, "$1 $2").replace(/[_-]+/g, " ").trim();
|
|
133
|
+
}
|
|
134
|
+
function buildIndexableText(name, signature, docComment) {
|
|
135
|
+
return [splitName(name), name, signature, docComment].filter(Boolean).join(" ");
|
|
136
|
+
}
|
|
137
|
+
function buildBm25Index(docs) {
|
|
138
|
+
const documents = docs.map((d) => {
|
|
139
|
+
const tokens = tokenise(d.text);
|
|
140
|
+
return { id: d.id, tokens, raw: d.text, len: tokens.length };
|
|
141
|
+
});
|
|
142
|
+
const df = {};
|
|
143
|
+
for (const doc of documents) {
|
|
144
|
+
const seen = /* @__PURE__ */ new Set();
|
|
145
|
+
for (const t of doc.tokens) {
|
|
146
|
+
if (!seen.has(t)) {
|
|
147
|
+
df[t] = (df[t] ?? 0) + 1;
|
|
148
|
+
seen.add(t);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const N = documents.length;
|
|
153
|
+
const totalLen = documents.reduce((sum, d) => sum + d.len, 0);
|
|
154
|
+
const avgLen = N === 0 ? 0 : totalLen / N;
|
|
155
|
+
return new Bm25Index(documents, df, N, avgLen);
|
|
156
|
+
}
|
|
157
|
+
var Bm25Index = class {
|
|
158
|
+
constructor(documents, df, N, avgLen) {
|
|
159
|
+
this.documents = documents;
|
|
160
|
+
this.df = df;
|
|
161
|
+
this.N = N;
|
|
162
|
+
this.safeAvgLen = avgLen === 0 ? 1 : avgLen;
|
|
163
|
+
}
|
|
164
|
+
documents;
|
|
165
|
+
df;
|
|
166
|
+
N;
|
|
167
|
+
safeAvgLen;
|
|
168
|
+
score(query, filter) {
|
|
169
|
+
const qTokens = tokenise(query);
|
|
170
|
+
if (qTokens.length === 0) return [];
|
|
171
|
+
const results = [];
|
|
172
|
+
for (const doc of this.documents) {
|
|
173
|
+
if (filter && !filter(doc.id)) continue;
|
|
174
|
+
let docScore = 0;
|
|
175
|
+
for (const qTerm of qTokens) {
|
|
176
|
+
let tf = 0;
|
|
177
|
+
for (const t of doc.tokens) {
|
|
178
|
+
if (t === qTerm) tf++;
|
|
179
|
+
}
|
|
180
|
+
if (tf === 0) continue;
|
|
181
|
+
const dfVal = this.df[qTerm] ?? 0;
|
|
182
|
+
if (dfVal === 0) continue;
|
|
183
|
+
const idf = Math.log((this.N - dfVal + 0.5) / (dfVal + 0.5) + 1);
|
|
184
|
+
const lenRatio = B * (doc.len / this.safeAvgLen);
|
|
185
|
+
const tfComponent = tf * (K1 + 1) / (tf + K1 * (1 - B + lenRatio));
|
|
186
|
+
docScore += idf * tfComponent;
|
|
187
|
+
}
|
|
188
|
+
if (docScore > 0) results.push({ id: doc.id, score: docScore });
|
|
189
|
+
}
|
|
190
|
+
return results;
|
|
191
|
+
}
|
|
192
|
+
getDoc(id) {
|
|
193
|
+
return this.documents.find((d) => d.id === id);
|
|
194
|
+
}
|
|
195
|
+
extractSnippet(docId, queryTokens, radius = 40) {
|
|
196
|
+
const doc = this.getDoc(docId);
|
|
197
|
+
if (!doc) return "";
|
|
198
|
+
for (const tok of queryTokens) {
|
|
199
|
+
const idx = doc.raw.toLowerCase().indexOf(tok);
|
|
200
|
+
if (idx !== -1) {
|
|
201
|
+
const start = Math.max(0, idx - radius);
|
|
202
|
+
const end = Math.min(doc.raw.length, idx + tok.length + radius);
|
|
203
|
+
const excerpt = doc.raw.slice(start, end);
|
|
204
|
+
const ellipsis = "\u2026";
|
|
205
|
+
return (start > 0 ? ellipsis : "") + excerpt + (end < doc.raw.length ? ellipsis : "");
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return doc.raw.slice(0, radius * 2) + (doc.raw.length > radius * 2 ? "\u2026" : "");
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
// src/codebase-index/writer.ts
|
|
213
|
+
var DB_FILE = "index.db";
|
|
214
|
+
function resolveIndexDir(projectRoot, override) {
|
|
215
|
+
return override ?? resolveWstackPaths({ projectRoot }).projectCodebaseIndex;
|
|
216
|
+
}
|
|
217
|
+
var warningSilenced = false;
|
|
218
|
+
function silenceSqliteExperimentalWarning() {
|
|
219
|
+
if (warningSilenced) return;
|
|
220
|
+
warningSilenced = true;
|
|
221
|
+
const original = process.emitWarning.bind(process);
|
|
222
|
+
process.emitWarning = ((warning, ...rest) => {
|
|
223
|
+
const msg = typeof warning === "string" ? warning : warning?.message ?? "";
|
|
224
|
+
const name = typeof warning === "string" ? String(rest[0] ?? "") : warning?.name ?? "";
|
|
225
|
+
if (/sqlite/i.test(msg) && /experimental/i.test(`${name} ${msg}`)) return;
|
|
226
|
+
original(warning, ...rest);
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
var DatabaseSyncCtor;
|
|
230
|
+
function loadDatabaseSync() {
|
|
231
|
+
if (DatabaseSyncCtor) return DatabaseSyncCtor;
|
|
232
|
+
silenceSqliteExperimentalWarning();
|
|
233
|
+
try {
|
|
234
|
+
const req = createRequire(import.meta.url);
|
|
235
|
+
DatabaseSyncCtor = req("node:sqlite").DatabaseSync;
|
|
236
|
+
} catch (err) {
|
|
237
|
+
throw new Error(
|
|
238
|
+
`The codebase index needs Node's built-in SQLite (node:sqlite), available since Node 22.5. This runtime doesn't provide it: ${err instanceof Error ? err.message : String(err)}`
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
return DatabaseSyncCtor;
|
|
242
|
+
}
|
|
243
|
+
var MAX_LOCK_RETRIES = 3;
|
|
244
|
+
var LOCK_RETRY_BASE_DELAY_MS = 50;
|
|
245
|
+
var LOCK_RETRY_MAX_DELAY_MS = 500;
|
|
246
|
+
function isLockError(err) {
|
|
247
|
+
if (!(err instanceof Error)) return false;
|
|
248
|
+
const e = err;
|
|
249
|
+
const code = e.code ?? e.sqliteCode;
|
|
250
|
+
if (typeof code === "string" && /SQLITE_(BUSY|LOCKED)/.test(code)) return true;
|
|
251
|
+
if (typeof code === "number" && (code === 5 || code === 6)) return true;
|
|
252
|
+
if (/SQLITE_(BUSY|LOCKED)/.test(err.message)) return true;
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
function sleepSync(ms) {
|
|
256
|
+
try {
|
|
257
|
+
const sab = new SharedArrayBuffer(4);
|
|
258
|
+
const view = new Int32Array(sab);
|
|
259
|
+
Atomics.wait(view, 0, 0, ms);
|
|
260
|
+
} catch {
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
var IndexStore = class {
|
|
264
|
+
db;
|
|
265
|
+
/** Absolute path to this project's index directory. */
|
|
266
|
+
indexDir;
|
|
267
|
+
/**
|
|
268
|
+
* True when the SQLite build provides FTS5 (Node's bundled SQLite does).
|
|
269
|
+
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
270
|
+
*/
|
|
271
|
+
ftsAvailable = false;
|
|
272
|
+
/**
|
|
273
|
+
* Execute a SQLite write operation with automatic retry on lock conflicts.
|
|
274
|
+
*
|
|
275
|
+
* When another wstack process is holding the write lock the statement first
|
|
276
|
+
* waits up to `busy_timeout` ms, then throws SQLITE_BUSY. This wrapper catches
|
|
277
|
+
* that error and retries (up to MAX_LOCK_RETRIES) with exponential backoff,
|
|
278
|
+
* giving the competing writer time to finish and release the lock.
|
|
279
|
+
*
|
|
280
|
+
* @param fn The write operation to execute. Can return a value which is
|
|
281
|
+
* returned to the caller on success.
|
|
282
|
+
* @throws {@link LockError} when all retries are exhausted on a lock conflict
|
|
283
|
+
* (non-lock errors always propagate on the first attempt).
|
|
284
|
+
*/
|
|
285
|
+
runWithRetry(fn) {
|
|
286
|
+
let lastError;
|
|
287
|
+
for (let attempt = 0; attempt <= MAX_LOCK_RETRIES; attempt++) {
|
|
288
|
+
try {
|
|
289
|
+
return fn();
|
|
290
|
+
} catch (err) {
|
|
291
|
+
lastError = err;
|
|
292
|
+
if (!isLockError(err)) throw err;
|
|
293
|
+
if (attempt === MAX_LOCK_RETRIES) {
|
|
294
|
+
const msg = lastError instanceof Error ? lastError.message : String(lastError);
|
|
295
|
+
throw new LockError(`SQLite lock conflict after ${MAX_LOCK_RETRIES} retries: ${msg}`);
|
|
296
|
+
}
|
|
297
|
+
const delay = Math.min(
|
|
298
|
+
LOCK_RETRY_BASE_DELAY_MS * Math.pow(2, attempt),
|
|
299
|
+
LOCK_RETRY_MAX_DELAY_MS
|
|
300
|
+
);
|
|
301
|
+
sleepSync(delay);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
throw lastError;
|
|
305
|
+
}
|
|
306
|
+
constructor(projectRoot, opts = {}) {
|
|
307
|
+
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
308
|
+
fs.mkdirSync(this.indexDir, { recursive: true });
|
|
309
|
+
const Database = loadDatabaseSync();
|
|
310
|
+
this.db = new Database(path4.join(this.indexDir, DB_FILE));
|
|
311
|
+
try {
|
|
312
|
+
this.db.exec("PRAGMA journal_mode = WAL");
|
|
313
|
+
this.db.exec("PRAGMA busy_timeout = 5000");
|
|
314
|
+
} catch {
|
|
315
|
+
}
|
|
316
|
+
this.initSchema();
|
|
317
|
+
}
|
|
318
|
+
initSchema() {
|
|
319
|
+
this.db.exec(`
|
|
320
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
321
|
+
key TEXT PRIMARY KEY,
|
|
322
|
+
value TEXT NOT NULL
|
|
323
|
+
);
|
|
324
|
+
`);
|
|
325
|
+
const storedRows = this.db.prepare("SELECT value FROM metadata WHERE key = ?").all("version");
|
|
326
|
+
const storedVersion = storedRows.length ? Number(storedRows[0]?.value) : null;
|
|
327
|
+
if (storedVersion !== null && storedVersion !== SCHEMA_VERSION) {
|
|
328
|
+
this.db.exec(`
|
|
329
|
+
DROP TABLE IF EXISTS symbols;
|
|
330
|
+
DROP TABLE IF EXISTS files;
|
|
331
|
+
DROP TABLE IF EXISTS refs;
|
|
332
|
+
`);
|
|
333
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
334
|
+
this.db.prepare("UPDATE metadata SET value = ? WHERE key = ?").run(String(SCHEMA_VERSION), "version");
|
|
335
|
+
} else if (storedVersion === null) {
|
|
336
|
+
this.db.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)").run("version", String(SCHEMA_VERSION));
|
|
337
|
+
}
|
|
338
|
+
this.db.exec(`
|
|
339
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
340
|
+
file TEXT PRIMARY KEY,
|
|
341
|
+
lang TEXT NOT NULL,
|
|
342
|
+
mtime_ms INTEGER NOT NULL,
|
|
343
|
+
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
344
|
+
last_indexed INTEGER NOT NULL
|
|
345
|
+
);
|
|
346
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
347
|
+
id INTEGER PRIMARY KEY,
|
|
348
|
+
lang TEXT NOT NULL,
|
|
349
|
+
kind TEXT NOT NULL,
|
|
350
|
+
name TEXT NOT NULL,
|
|
351
|
+
file TEXT NOT NULL,
|
|
352
|
+
line INTEGER NOT NULL,
|
|
353
|
+
col INTEGER NOT NULL,
|
|
354
|
+
signature TEXT NOT NULL DEFAULT '',
|
|
355
|
+
doc_comment TEXT NOT NULL DEFAULT '',
|
|
356
|
+
scope TEXT NOT NULL DEFAULT '',
|
|
357
|
+
text TEXT NOT NULL DEFAULT '',
|
|
358
|
+
file_fk TEXT NOT NULL
|
|
359
|
+
);
|
|
360
|
+
`);
|
|
361
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_name ON symbols(name)");
|
|
362
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_kind ON symbols(kind)");
|
|
363
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_lang ON symbols(lang)");
|
|
364
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_file ON symbols(file)");
|
|
365
|
+
this.db.exec(`
|
|
366
|
+
CREATE TABLE IF NOT EXISTS refs (
|
|
367
|
+
id INTEGER PRIMARY KEY,
|
|
368
|
+
from_id INTEGER NOT NULL,
|
|
369
|
+
to_name TEXT NOT NULL,
|
|
370
|
+
to_id INTEGER,
|
|
371
|
+
call_type TEXT NOT NULL,
|
|
372
|
+
line INTEGER NOT NULL
|
|
373
|
+
);
|
|
374
|
+
`);
|
|
375
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_r_from ON refs(from_id)");
|
|
376
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_r_to_id ON refs(to_id)");
|
|
377
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_r_to_name ON refs(to_name)");
|
|
378
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_r_call_type ON refs(call_type)");
|
|
379
|
+
try {
|
|
380
|
+
this.db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'unicode61')");
|
|
381
|
+
this.ftsAvailable = true;
|
|
382
|
+
} catch {
|
|
383
|
+
this.ftsAvailable = false;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
// ─── Symbol CRUD ─────────────────────────────────────────────────────────────
|
|
387
|
+
insertSymbols(symbols, nextId) {
|
|
388
|
+
return this.runWithRetry(() => {
|
|
389
|
+
const stmt = this.db.prepare(
|
|
390
|
+
`INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text, file_fk)
|
|
391
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
392
|
+
);
|
|
393
|
+
const ftsStmt = this.ftsAvailable ? this.db.prepare("INSERT INTO symbols_fts(rowid, text) VALUES (?, ?)") : null;
|
|
394
|
+
let id = nextId;
|
|
395
|
+
for (const s of symbols) {
|
|
396
|
+
stmt.run(
|
|
397
|
+
id,
|
|
398
|
+
s.lang,
|
|
399
|
+
s.kind,
|
|
400
|
+
s.name,
|
|
401
|
+
s.file,
|
|
402
|
+
s.line,
|
|
403
|
+
s.col,
|
|
404
|
+
s.signature,
|
|
405
|
+
s.docComment,
|
|
406
|
+
s.scope,
|
|
407
|
+
s.text,
|
|
408
|
+
s.file
|
|
409
|
+
);
|
|
410
|
+
ftsStmt?.run(id, buildIndexableText(s.name, s.signature, s.docComment));
|
|
411
|
+
id++;
|
|
412
|
+
}
|
|
413
|
+
return id;
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
deleteSymbolsForFile(file) {
|
|
417
|
+
this.runWithRetry(() => {
|
|
418
|
+
if (this.ftsAvailable) {
|
|
419
|
+
this.db.prepare("DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)").run(file);
|
|
420
|
+
}
|
|
421
|
+
this.db.prepare("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Remove every trace of a file (refs, symbols, FTS rows, file meta). Used
|
|
426
|
+
* when a source file disappears between index runs — previously this only
|
|
427
|
+
* dropped the `files` row, leaving its symbols orphaned but still searchable.
|
|
428
|
+
*/
|
|
429
|
+
deleteFile(file) {
|
|
430
|
+
this.runWithRetry(() => {
|
|
431
|
+
this.deleteRefsForFile(file);
|
|
432
|
+
this.deleteSymbolsForFile(file);
|
|
433
|
+
this.db.prepare("DELETE FROM files WHERE file = ?").run(file);
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
// ─── File metadata ──────────────────────────────────────────────────────────
|
|
437
|
+
upsertFile(meta) {
|
|
438
|
+
this.runWithRetry(() => {
|
|
439
|
+
this.db.prepare(
|
|
440
|
+
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
441
|
+
VALUES (?, ?, ?, ?, ?)
|
|
442
|
+
ON CONFLICT(file) DO UPDATE SET
|
|
443
|
+
lang = excluded.lang,
|
|
444
|
+
mtime_ms = excluded.mtime_ms,
|
|
445
|
+
symbol_count = excluded.symbol_count,
|
|
446
|
+
last_indexed = excluded.last_indexed`
|
|
447
|
+
).run(meta.file, meta.lang, meta.mtimeMs, meta.symbolCount, meta.lastIndexed);
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
getFileMeta(file) {
|
|
451
|
+
const rows = this.db.prepare(
|
|
452
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
453
|
+
).all(file);
|
|
454
|
+
if (!rows.length) return null;
|
|
455
|
+
const r = expectDefined(rows[0]);
|
|
456
|
+
return { file: r.file, lang: r.lang, mtimeMs: r.mtime_ms, symbolCount: r.symbol_count, lastIndexed: r.last_indexed };
|
|
457
|
+
}
|
|
458
|
+
getAllFileMetas() {
|
|
459
|
+
return this.db.prepare(
|
|
460
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files"
|
|
461
|
+
).all().map(
|
|
462
|
+
(r) => ({ file: r.file, lang: r.lang, mtimeMs: r.mtime_ms, symbolCount: r.symbol_count, lastIndexed: r.last_indexed })
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
// ─── Search ──────────────────────────────────────────────────────────────────
|
|
466
|
+
search(query, filter) {
|
|
467
|
+
const conditions = [];
|
|
468
|
+
const values = [];
|
|
469
|
+
let effectiveKind = filter?.kind;
|
|
470
|
+
if (filter?.lspKind !== void 0) {
|
|
471
|
+
const mapped = lspKindToInternalKind(filter.lspKind);
|
|
472
|
+
if (mapped !== null) {
|
|
473
|
+
effectiveKind = mapped;
|
|
474
|
+
} else {
|
|
475
|
+
return [];
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if (effectiveKind) {
|
|
479
|
+
conditions.push("kind = ?");
|
|
480
|
+
values.push(effectiveKind);
|
|
481
|
+
}
|
|
482
|
+
if (filter?.lang) {
|
|
483
|
+
conditions.push("lang = ?");
|
|
484
|
+
values.push(filter.lang);
|
|
485
|
+
}
|
|
486
|
+
if (filter?.file) {
|
|
487
|
+
conditions.push("file LIKE ?");
|
|
488
|
+
values.push(`%${filter.file}%`);
|
|
489
|
+
}
|
|
490
|
+
if (query.trim()) {
|
|
491
|
+
const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
492
|
+
const tokenConds = tokens.map(() => "text LIKE ?");
|
|
493
|
+
conditions.push(`(${tokenConds.join(" OR ")})`);
|
|
494
|
+
for (const t of tokens) values.push(`%${t}%`);
|
|
495
|
+
}
|
|
496
|
+
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
497
|
+
const sql = `SELECT id, lang, kind, name, file, line, col, signature, doc_comment, text FROM symbols ${where}`;
|
|
498
|
+
const stmt = this.db.prepare(sql);
|
|
499
|
+
const rows = stmt.all(...values);
|
|
500
|
+
return rows.map((r) => ({
|
|
501
|
+
id: r.id,
|
|
502
|
+
lang: r.lang,
|
|
503
|
+
kind: r.kind,
|
|
504
|
+
name: r.name,
|
|
505
|
+
file: r.file,
|
|
506
|
+
line: r.line,
|
|
507
|
+
col: r.col,
|
|
508
|
+
signature: r.signature,
|
|
509
|
+
docComment: r.doc_comment,
|
|
510
|
+
score: 0,
|
|
511
|
+
snippet: "",
|
|
512
|
+
lspKind: filter?.lspKind
|
|
513
|
+
}));
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* Ranked search — the one-stop query the codebase-search tool and plug-lsp
|
|
517
|
+
* use. With FTS5 this is a single indexed `MATCH` ranked by SQLite's native
|
|
518
|
+
* `bm25()` with a built-in `snippet()`; without FTS5 it falls back to the
|
|
519
|
+
* legacy LIKE scan + in-process BM25 (identical semantics, slower).
|
|
520
|
+
*
|
|
521
|
+
* Tokens are matched as prefixes (`"tok"*`), mirroring the old
|
|
522
|
+
* `LIKE '%tok%'` recall for the common symbol-search shapes ("user" finds
|
|
523
|
+
* "users", camelCase-split text makes "complex" find "complexOperation").
|
|
524
|
+
*/
|
|
525
|
+
searchRanked(query, filter, limit) {
|
|
526
|
+
const tokens = tokenise(query);
|
|
527
|
+
if (tokens.length === 0 || !this.ftsAvailable) {
|
|
528
|
+
return this.searchRankedFallback(query, filter, limit);
|
|
529
|
+
}
|
|
530
|
+
let effectiveKind = filter?.kind;
|
|
531
|
+
if (filter?.lspKind !== void 0) {
|
|
532
|
+
const mapped = lspKindToInternalKind(filter.lspKind);
|
|
533
|
+
if (mapped === null) return { results: [], total: 0 };
|
|
534
|
+
effectiveKind = mapped;
|
|
535
|
+
}
|
|
536
|
+
const match = tokens.map((t) => `"${t.replaceAll('"', "")}"*`).join(" OR ");
|
|
537
|
+
const conditions = ["symbols_fts MATCH ?"];
|
|
538
|
+
const values = [match];
|
|
539
|
+
if (effectiveKind) {
|
|
540
|
+
conditions.push("s.kind = ?");
|
|
541
|
+
values.push(effectiveKind);
|
|
542
|
+
}
|
|
543
|
+
if (filter?.lang) {
|
|
544
|
+
conditions.push("s.lang = ?");
|
|
545
|
+
values.push(filter.lang);
|
|
546
|
+
}
|
|
547
|
+
if (filter?.file) {
|
|
548
|
+
conditions.push("s.file LIKE ?");
|
|
549
|
+
values.push(`%${filter.file}%`);
|
|
550
|
+
}
|
|
551
|
+
const where = conditions.join(" AND ");
|
|
552
|
+
const countRows = this.db.prepare(`SELECT COUNT(*) AS n FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE ${where}`).all(...values);
|
|
553
|
+
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
554
|
+
if (total === 0) return { results: [], total: 0 };
|
|
555
|
+
const rows = this.db.prepare(
|
|
556
|
+
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
557
|
+
-bm25(symbols_fts) AS score,
|
|
558
|
+
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
559
|
+
FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid
|
|
560
|
+
WHERE ${where}
|
|
561
|
+
ORDER BY bm25(symbols_fts)
|
|
562
|
+
LIMIT ?`
|
|
563
|
+
).all(...values, limit);
|
|
564
|
+
return {
|
|
565
|
+
results: rows.map((r) => ({
|
|
566
|
+
id: r.id,
|
|
567
|
+
lang: r.lang,
|
|
568
|
+
kind: r.kind,
|
|
569
|
+
name: r.name,
|
|
570
|
+
file: r.file,
|
|
571
|
+
line: r.line,
|
|
572
|
+
col: r.col,
|
|
573
|
+
signature: r.signature,
|
|
574
|
+
docComment: r.doc_comment,
|
|
575
|
+
// bm25() is negative-is-better; negate so callers keep "higher is
|
|
576
|
+
// better" and clamp so a match never reports a zero score.
|
|
577
|
+
score: Math.max(1e-4, r.score),
|
|
578
|
+
snippet: r.snippet,
|
|
579
|
+
lspKind: filter?.lspKind
|
|
580
|
+
})),
|
|
581
|
+
total
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
/** Legacy ranked path: LIKE candidates + in-process BM25 + JS snippets. */
|
|
585
|
+
searchRankedFallback(query, filter, limit) {
|
|
586
|
+
const candidates = this.search(query, filter);
|
|
587
|
+
if (candidates.length === 0) return { results: [], total: 0 };
|
|
588
|
+
if (!query.trim()) {
|
|
589
|
+
return { results: candidates.slice(0, limit), total: candidates.length };
|
|
590
|
+
}
|
|
591
|
+
const bm25 = buildBm25Index(
|
|
592
|
+
candidates.map((c) => ({ id: c.id, text: buildIndexableText(c.name, c.signature, c.docComment) }))
|
|
593
|
+
);
|
|
594
|
+
const scored = bm25.score(query, (id) => candidates.some((c) => c.id === id));
|
|
595
|
+
scored.sort((a, b) => b.score - a.score);
|
|
596
|
+
const qTokens = tokenise(query);
|
|
597
|
+
const results = scored.slice(0, limit).map(({ id, score }) => {
|
|
598
|
+
const c = expectDefined(candidates.find((cand) => cand.id === id));
|
|
599
|
+
return { ...c, score, snippet: bm25.extractSnippet(id, qTokens) };
|
|
600
|
+
});
|
|
601
|
+
return { results, total: candidates.length };
|
|
602
|
+
}
|
|
603
|
+
getAllIndexable() {
|
|
604
|
+
return this.db.prepare("SELECT id, text FROM symbols").all().map(
|
|
605
|
+
({ id, text }) => ({ id, text })
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Largest symbol id currently in the table (0 when empty). New ids must be
|
|
610
|
+
* allocated from this, NOT from `COUNT(*)`: incremental reindexes delete a
|
|
611
|
+
* changed file's rows, so the row count drops below the max id and a
|
|
612
|
+
* count-based id would collide with a surviving row (UNIQUE constraint on
|
|
613
|
+
* `symbols.id`). Ids may have gaps — that is fine.
|
|
614
|
+
*/
|
|
615
|
+
getMaxSymbolId() {
|
|
616
|
+
const rows = this.db.prepare("SELECT MAX(id) AS m FROM symbols").all();
|
|
617
|
+
return rows[0]?.m ?? 0;
|
|
618
|
+
}
|
|
619
|
+
// ─── Stats ───────────────────────────────────────────────────────────────────
|
|
620
|
+
getStats() {
|
|
621
|
+
const sizeBytes = this.sizeBytes();
|
|
622
|
+
const lastRows = this.db.prepare(
|
|
623
|
+
"SELECT value FROM metadata WHERE key = 'last_indexed'"
|
|
624
|
+
).all();
|
|
625
|
+
const lastIndexed = lastRows.length ? Number(lastRows[0]?.value) : null;
|
|
626
|
+
const totalRows = this.db.prepare("SELECT COUNT(*) FROM symbols").all();
|
|
627
|
+
const totalSymbols = totalRows[0] ? Number(totalRows[0]["COUNT(*)"]) : 0;
|
|
628
|
+
const fileRows = this.db.prepare("SELECT COUNT(*) FROM files").all();
|
|
629
|
+
const totalFiles = fileRows[0] ? Number(fileRows[0]["COUNT(*)"]) : 0;
|
|
630
|
+
const langRows = this.db.prepare(
|
|
631
|
+
"SELECT lang, COUNT(*) FROM symbols GROUP BY lang"
|
|
632
|
+
).all();
|
|
633
|
+
const byLang = {};
|
|
634
|
+
for (const row of langRows) byLang[row.lang] = Number(row["COUNT(*)"]);
|
|
635
|
+
const kindRows = this.db.prepare(
|
|
636
|
+
"SELECT kind, COUNT(*) FROM symbols GROUP BY kind"
|
|
637
|
+
).all();
|
|
638
|
+
const byKind = {};
|
|
639
|
+
for (const row of kindRows) byKind[row.kind] = Number(row["COUNT(*)"]);
|
|
640
|
+
return {
|
|
641
|
+
totalSymbols,
|
|
642
|
+
totalFiles,
|
|
643
|
+
byLang,
|
|
644
|
+
byKind,
|
|
645
|
+
indexPath: this.indexDir,
|
|
646
|
+
lastIndexed,
|
|
647
|
+
sizeBytes,
|
|
648
|
+
version: SCHEMA_VERSION
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
setLastIndexed(ts2) {
|
|
652
|
+
this.runWithRetry(() => {
|
|
653
|
+
this.db.prepare(
|
|
654
|
+
"INSERT OR REPLACE INTO metadata(key, value) VALUES('last_indexed', ?)"
|
|
655
|
+
).run(String(ts2));
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
clearAll() {
|
|
659
|
+
this.runWithRetry(() => {
|
|
660
|
+
this.db.exec("DELETE FROM symbols");
|
|
661
|
+
this.db.exec("DELETE FROM files");
|
|
662
|
+
this.db.exec("DELETE FROM refs");
|
|
663
|
+
if (this.ftsAvailable) this.db.exec("DELETE FROM symbols_fts");
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
// ─── Ref CRUD ────────────────────────────────────────────────────────────────
|
|
667
|
+
/**
|
|
668
|
+
* Insert cross-references for a given source symbol id.
|
|
669
|
+
* Replaces any existing refs from the same source (idempotent on re-index).
|
|
670
|
+
*/
|
|
671
|
+
insertRefs(fromId, refs) {
|
|
672
|
+
this.runWithRetry(() => {
|
|
673
|
+
this.db.prepare("DELETE FROM refs WHERE from_id = ?").run(fromId);
|
|
674
|
+
if (refs.length === 0) return;
|
|
675
|
+
const stmt = this.db.prepare(
|
|
676
|
+
`INSERT INTO refs(from_id, to_name, to_id, call_type, line)
|
|
677
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
678
|
+
);
|
|
679
|
+
for (const ref of refs) {
|
|
680
|
+
stmt.run(fromId, ref.toName, ref.toId ?? null, ref.callType, ref.line);
|
|
681
|
+
}
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Delete all refs whose source symbols are in a given file.
|
|
686
|
+
* Used when re-indexing a file to clear stale refs.
|
|
687
|
+
*/
|
|
688
|
+
deleteRefsForFile(file) {
|
|
689
|
+
this.runWithRetry(() => {
|
|
690
|
+
const ids = this.db.prepare(
|
|
691
|
+
"SELECT id FROM symbols WHERE file = ?"
|
|
692
|
+
).all(file);
|
|
693
|
+
if (!ids.length) return;
|
|
694
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
695
|
+
this.db.prepare(`DELETE FROM refs WHERE from_id IN (${placeholders})`).run(...ids.map((r) => r.id));
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
/**
|
|
699
|
+
* Resolve `to_name` → `to_id` for all refs that have a name but no id.
|
|
700
|
+
* Call this after all symbols have been inserted to fill in cross-references.
|
|
701
|
+
*/
|
|
702
|
+
resolveRefs() {
|
|
703
|
+
return this.runWithRetry(() => {
|
|
704
|
+
const unresolved = this.db.prepare(
|
|
705
|
+
"SELECT id, to_name FROM refs WHERE to_id IS NULL AND to_name IS NOT NULL"
|
|
706
|
+
).all();
|
|
707
|
+
let resolved = 0;
|
|
708
|
+
for (const row of unresolved) {
|
|
709
|
+
const target = this.db.prepare("SELECT id FROM symbols WHERE name = ? LIMIT 1").all(row.to_name);
|
|
710
|
+
const first = target[0];
|
|
711
|
+
if (first) {
|
|
712
|
+
this.db.prepare("UPDATE refs SET to_id = ? WHERE id = ?").run(first.id, row.id);
|
|
713
|
+
resolved++;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return resolved;
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
721
|
+
*/
|
|
722
|
+
findRefsTo(symbolId) {
|
|
723
|
+
return this.db.prepare(
|
|
724
|
+
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
725
|
+
).all(symbolId, symbolId).map((r) => ({
|
|
726
|
+
id: r.id,
|
|
727
|
+
fromId: r.from_id,
|
|
728
|
+
toName: r.to_name,
|
|
729
|
+
toId: r.to_id ?? void 0,
|
|
730
|
+
callType: r.call_type,
|
|
731
|
+
line: r.line
|
|
732
|
+
}));
|
|
733
|
+
}
|
|
734
|
+
/**
|
|
735
|
+
* Find all references FROM a given symbol (what does this symbol call/use?).
|
|
736
|
+
*/
|
|
737
|
+
findRefsFrom(symbolId) {
|
|
738
|
+
return this.db.prepare(
|
|
739
|
+
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE from_id = ?"
|
|
740
|
+
).all(symbolId).map((r) => ({
|
|
741
|
+
id: r.id,
|
|
742
|
+
fromId: r.from_id,
|
|
743
|
+
toName: r.to_name,
|
|
744
|
+
toId: r.to_id ?? void 0,
|
|
745
|
+
callType: r.call_type,
|
|
746
|
+
line: r.line
|
|
747
|
+
}));
|
|
748
|
+
}
|
|
749
|
+
sizeBytes() {
|
|
750
|
+
const dbPath = path4.join(this.indexDir, DB_FILE);
|
|
751
|
+
try {
|
|
752
|
+
return fs.statSync(dbPath).size;
|
|
753
|
+
} catch {
|
|
754
|
+
return 0;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
close() {
|
|
758
|
+
try {
|
|
759
|
+
this.db.close();
|
|
760
|
+
} catch {
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
};
|
|
764
|
+
var KIND_MAP = {
|
|
765
|
+
[ts.SyntaxKind.ClassDeclaration]: "class",
|
|
766
|
+
[ts.SyntaxKind.InterfaceDeclaration]: "interface",
|
|
767
|
+
[ts.SyntaxKind.EnumDeclaration]: "enum",
|
|
768
|
+
[ts.SyntaxKind.TypeAliasDeclaration]: "type",
|
|
769
|
+
[ts.SyntaxKind.FunctionDeclaration]: "function",
|
|
770
|
+
[ts.SyntaxKind.MethodDeclaration]: "method",
|
|
771
|
+
[ts.SyntaxKind.GetAccessor]: "property",
|
|
772
|
+
[ts.SyntaxKind.SetAccessor]: "property",
|
|
773
|
+
[ts.SyntaxKind.PropertyDeclaration]: "property",
|
|
774
|
+
[ts.SyntaxKind.Parameter]: "parameter",
|
|
775
|
+
[ts.SyntaxKind.NamespaceExportDeclaration]: "namespace"
|
|
776
|
+
};
|
|
777
|
+
function kindOf(node) {
|
|
778
|
+
if (ts.isVariableDeclaration(node)) {
|
|
779
|
+
const parent = node.parent;
|
|
780
|
+
if (ts.isVariableDeclarationList(parent)) {
|
|
781
|
+
const flags = parent.flags;
|
|
782
|
+
if (flags & ts.NodeFlags.Let) return "let";
|
|
783
|
+
if (flags & ts.NodeFlags.Const) return "const";
|
|
784
|
+
return "var";
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
if (ts.isModuleDeclaration(node)) return "namespace";
|
|
788
|
+
return KIND_MAP[node.kind] ?? null;
|
|
789
|
+
}
|
|
790
|
+
function extToLang(ext) {
|
|
791
|
+
switch (ext) {
|
|
792
|
+
case ".ts":
|
|
793
|
+
return "ts";
|
|
794
|
+
case ".tsx":
|
|
795
|
+
return "tsx";
|
|
796
|
+
case ".js":
|
|
797
|
+
return "js";
|
|
798
|
+
case ".jsx":
|
|
799
|
+
return "jsx";
|
|
800
|
+
case ".go":
|
|
801
|
+
return "go";
|
|
802
|
+
case ".py":
|
|
803
|
+
return "py";
|
|
804
|
+
case ".rs":
|
|
805
|
+
return "rs";
|
|
806
|
+
case ".json":
|
|
807
|
+
return "json";
|
|
808
|
+
case ".yaml":
|
|
809
|
+
return "yaml";
|
|
810
|
+
case ".yml":
|
|
811
|
+
return "yaml";
|
|
812
|
+
default:
|
|
813
|
+
return null;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
function getSignature(node, sourceFile) {
|
|
817
|
+
const printer = ts.createPrinter({});
|
|
818
|
+
const raw = printer.printNode(ts.EmitHint.Unspecified, node, sourceFile);
|
|
819
|
+
return raw.replace(/\s+/g, " ").slice(0, 500);
|
|
820
|
+
}
|
|
821
|
+
function getJsDoc(node, sourceFile) {
|
|
822
|
+
const fullText = sourceFile.getFullText();
|
|
823
|
+
const nodePos = node.getFullWidth();
|
|
824
|
+
const comments = ts.getLeadingCommentRanges(fullText, nodePos);
|
|
825
|
+
if (!comments) return "";
|
|
826
|
+
for (const range of comments) {
|
|
827
|
+
const commentText = fullText.slice(range.pos, range.end);
|
|
828
|
+
const trimmed = commentText.trim();
|
|
829
|
+
if (trimmed.startsWith("/**") && trimmed.endsWith("*/")) {
|
|
830
|
+
const inner = trimmed.slice(3, -2).replace(/^[ \t]*\*[ ]?/gm, "").trim();
|
|
831
|
+
return inner.split("\n")[0]?.trim().slice(0, 200) ?? "";
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
return "";
|
|
835
|
+
}
|
|
836
|
+
function buildScope(node) {
|
|
837
|
+
const parts = [];
|
|
838
|
+
let current = node.parent;
|
|
839
|
+
while (current) {
|
|
840
|
+
if (ts.isClassDeclaration(current) || ts.isInterfaceDeclaration(current) || ts.isEnumDeclaration(current) || ts.isTypeAliasDeclaration(current)) {
|
|
841
|
+
parts.unshift(current.name?.text ?? "Anon");
|
|
842
|
+
} else if (ts.isMethodDeclaration(current) || ts.isGetAccessor(current) || ts.isSetAccessor(current) || ts.isPropertyDeclaration(current) || ts.isFunctionDeclaration(current)) {
|
|
843
|
+
if (current.name && ts.isIdentifier(current.name)) {
|
|
844
|
+
parts.unshift(current.name.text);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
current = current.parent;
|
|
848
|
+
}
|
|
849
|
+
return parts.join(".");
|
|
850
|
+
}
|
|
851
|
+
function parseSymbols(opts) {
|
|
852
|
+
const { file, content, lang } = opts;
|
|
853
|
+
let sourceFile;
|
|
854
|
+
try {
|
|
855
|
+
sourceFile = ts.createSourceFile(file, content, ts.ScriptTarget.Latest, true);
|
|
856
|
+
} catch {
|
|
857
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
858
|
+
}
|
|
859
|
+
const symbols = [];
|
|
860
|
+
function visit(node) {
|
|
861
|
+
const kind = kindOf(node);
|
|
862
|
+
if (kind) {
|
|
863
|
+
const nameNode = node.name;
|
|
864
|
+
if (!nameNode || !ts.isIdentifier(nameNode)) return;
|
|
865
|
+
const name = nameNode.text;
|
|
866
|
+
const pos = nameNode.getStart(sourceFile);
|
|
867
|
+
const { line, character } = sourceFile.getLineAndCharacterOfPosition(pos);
|
|
868
|
+
const scope = buildScope(node);
|
|
869
|
+
const signature = getSignature(node, sourceFile);
|
|
870
|
+
const docComment = getJsDoc(node, sourceFile);
|
|
871
|
+
const text = [name, signature, docComment].filter(Boolean).join(" | ");
|
|
872
|
+
symbols.push({
|
|
873
|
+
id: 0,
|
|
874
|
+
lang,
|
|
875
|
+
kind,
|
|
876
|
+
name,
|
|
877
|
+
file,
|
|
878
|
+
line: line + 1,
|
|
879
|
+
col: character,
|
|
880
|
+
signature,
|
|
881
|
+
docComment,
|
|
882
|
+
scope,
|
|
883
|
+
text
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
ts.forEachChild(node, visit);
|
|
887
|
+
}
|
|
888
|
+
visit(sourceFile);
|
|
889
|
+
const refs = extractRefs(sourceFile);
|
|
890
|
+
return { file, lang, symbols, refs, mtimeMs: Date.now() };
|
|
891
|
+
}
|
|
892
|
+
function extractRefs(sourceFile) {
|
|
893
|
+
const refs = [];
|
|
894
|
+
function visit(node) {
|
|
895
|
+
const pos = node.getStart(sourceFile);
|
|
896
|
+
const { line } = sourceFile.getLineAndCharacterOfPosition(pos);
|
|
897
|
+
const lineNum = line + 1;
|
|
898
|
+
if (ts.isCallExpression(node)) {
|
|
899
|
+
const expr = node.expression;
|
|
900
|
+
if (ts.isIdentifier(expr)) {
|
|
901
|
+
refs.push({ fromId: 0, toName: expr.text, callType: "call", line: lineNum });
|
|
902
|
+
}
|
|
903
|
+
} else if (ts.isPropertyAccessExpression(node)) {
|
|
904
|
+
if (ts.isIdentifier(node.expression)) {
|
|
905
|
+
refs.push({ fromId: 0, toName: node.expression.text, callType: "call", line: lineNum });
|
|
906
|
+
}
|
|
907
|
+
} else if (ts.isTypeReferenceNode(node)) {
|
|
908
|
+
const name = getTypeName(node.typeName);
|
|
909
|
+
if (name) refs.push({ fromId: 0, toName: name, callType: "type_ref", line: lineNum });
|
|
910
|
+
} else if (ts.isHeritageClause(node)) {
|
|
911
|
+
for (const t of node.types) {
|
|
912
|
+
const name = getTypeName(t.expression);
|
|
913
|
+
if (name) refs.push({ fromId: 0, toName: name, callType: node.token === ts.SyntaxKind.ExtendsKeyword ? "inherit" : "implement", line: lineNum });
|
|
914
|
+
}
|
|
915
|
+
} else if (ts.isImportDeclaration(node)) {
|
|
916
|
+
const moduleName = getModuleName(node);
|
|
917
|
+
if (moduleName) refs.push({ fromId: 0, toName: moduleName, callType: "import", line: lineNum });
|
|
918
|
+
}
|
|
919
|
+
ts.forEachChild(node, visit);
|
|
920
|
+
}
|
|
921
|
+
visit(sourceFile);
|
|
922
|
+
return deduplicateRefs(refs);
|
|
923
|
+
}
|
|
924
|
+
function getTypeName(name) {
|
|
925
|
+
if (ts.isIdentifier(name)) return name.text;
|
|
926
|
+
if (ts.isQualifiedName(name)) return `${getTypeName(name.left)}.${name.right.text}`;
|
|
927
|
+
return "";
|
|
928
|
+
}
|
|
929
|
+
function getModuleName(node) {
|
|
930
|
+
const moduleSpecifier = node.moduleSpecifier;
|
|
931
|
+
if (ts.isStringLiteral(moduleSpecifier)) return moduleSpecifier.text;
|
|
932
|
+
return "";
|
|
933
|
+
}
|
|
934
|
+
function deduplicateRefs(refs) {
|
|
935
|
+
const seen = /* @__PURE__ */ new Set();
|
|
936
|
+
return refs.filter((r) => {
|
|
937
|
+
const key = `${r.toName}:${r.callType}:${r.line}`;
|
|
938
|
+
if (seen.has(key)) return false;
|
|
939
|
+
seen.add(key);
|
|
940
|
+
return true;
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
function detectLang(file) {
|
|
944
|
+
const idx = file.lastIndexOf(".");
|
|
945
|
+
if (idx < 0) return null;
|
|
946
|
+
return extToLang(file.slice(idx));
|
|
947
|
+
}
|
|
948
|
+
function parseSymbols2(opts) {
|
|
949
|
+
const { file, content, lang } = opts;
|
|
950
|
+
try {
|
|
951
|
+
return syncGoParse(file, content, lang);
|
|
952
|
+
} catch {
|
|
953
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
var GO_PARSE_SCRIPT = `
|
|
957
|
+
package main
|
|
958
|
+
|
|
959
|
+
import (
|
|
960
|
+
"encoding/json"
|
|
961
|
+
"fmt"
|
|
962
|
+
"go/ast"
|
|
963
|
+
"go/parser"
|
|
964
|
+
"go/token"
|
|
965
|
+
"io"
|
|
966
|
+
"os"
|
|
967
|
+
"strings"
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
type Sym struct {
|
|
971
|
+
Name string \`json:"name"\`
|
|
972
|
+
Kind string \`json:"kind"\`
|
|
973
|
+
Line int \`json:"line"\`
|
|
974
|
+
Col int \`json:"col"\`
|
|
975
|
+
Signature string \`json:"signature"\`
|
|
976
|
+
Scope string \`json:"scope"\`
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
func main() {
|
|
980
|
+
src, err := io.ReadAll(os.Stdin)
|
|
981
|
+
if err != nil {
|
|
982
|
+
fmt.Print("[]")
|
|
983
|
+
return
|
|
984
|
+
}
|
|
985
|
+
fset := token.NewFileSet()
|
|
986
|
+
node, err := parser.ParseFile(fset, "src.go", src, 0)
|
|
987
|
+
if err != nil {
|
|
988
|
+
fmt.Print("[]")
|
|
989
|
+
return
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
var syms []Sym
|
|
993
|
+
|
|
994
|
+
// Package-level scope
|
|
995
|
+
pkgScope := node.Name.Name
|
|
996
|
+
|
|
997
|
+
// Collect all top-level declarations
|
|
998
|
+
for _, decl := range node.Decls {
|
|
999
|
+
switch d := decl.(type) {
|
|
1000
|
+
case *ast.FuncDecl:
|
|
1001
|
+
name := d.Name.Name
|
|
1002
|
+
kind := "function"
|
|
1003
|
+
scope := pkgScope
|
|
1004
|
+
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
1005
|
+
scope = pkgScope + "." + recvTypeName(d.Recv.List[0].Type) + "." + name
|
|
1006
|
+
kind = "method"
|
|
1007
|
+
} else {
|
|
1008
|
+
scope = pkgScope + "." + name
|
|
1009
|
+
}
|
|
1010
|
+
pos := fset.Position(d.Pos())
|
|
1011
|
+
sig := formatFuncSig(d)
|
|
1012
|
+
syms = append(syms, Sym{Name: name, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: scope})
|
|
1013
|
+
|
|
1014
|
+
case *ast.GenDecl:
|
|
1015
|
+
for _, spec := range d.Specs {
|
|
1016
|
+
switch s := spec.(type) {
|
|
1017
|
+
case *ast.TypeSpec:
|
|
1018
|
+
name := s.Name.Name
|
|
1019
|
+
pos := fset.Position(s.Pos())
|
|
1020
|
+
sig := "type " + name
|
|
1021
|
+
if s.TypeParams != nil {
|
|
1022
|
+
sig += formatTypeParams(s.TypeParams)
|
|
1023
|
+
}
|
|
1024
|
+
if st, ok := s.Type.(*ast.StructType); ok {
|
|
1025
|
+
sig += " = struct { " + formatFields(st.Fields.List) + " }"
|
|
1026
|
+
} else if it, ok := s.Type.(*ast.InterfaceType); ok {
|
|
1027
|
+
sig += " = interface { " + formatMethods(it.Methods.List) + " }"
|
|
1028
|
+
} else {
|
|
1029
|
+
sig += " = " + formatType(s.Type)
|
|
1030
|
+
}
|
|
1031
|
+
syms = append(syms, Sym{Name: name, Kind: "type", Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
1032
|
+
|
|
1033
|
+
case *ast.ValueSpec:
|
|
1034
|
+
for _, n := range s.Names {
|
|
1035
|
+
name := n.Name
|
|
1036
|
+
pos := fset.Position(n.Pos())
|
|
1037
|
+
kind := "var"
|
|
1038
|
+
if d.Tok == token.CONST {
|
|
1039
|
+
kind = "const"
|
|
1040
|
+
}
|
|
1041
|
+
sig := kind + " " + name
|
|
1042
|
+
if s.Type != nil {
|
|
1043
|
+
sig += " " + formatType(s.Type)
|
|
1044
|
+
}
|
|
1045
|
+
syms = append(syms, Sym{Name: name, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
data, err := json.Marshal(syms)
|
|
1053
|
+
if err != nil {
|
|
1054
|
+
fmt.Print("[]")
|
|
1055
|
+
return
|
|
1056
|
+
}
|
|
1057
|
+
fmt.Print(string(data))
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
func recvTypeName(t ast.Expr) string {
|
|
1061
|
+
switch v := t.(type) {
|
|
1062
|
+
case *ast.Ident:
|
|
1063
|
+
return v.Name
|
|
1064
|
+
case *ast.StarExpr:
|
|
1065
|
+
return recvTypeName(v.X)
|
|
1066
|
+
default:
|
|
1067
|
+
return "?"
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
func formatFuncSig(d *ast.FuncDecl) string {
|
|
1072
|
+
scope := ""
|
|
1073
|
+
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
1074
|
+
scope = "(" + formatFieldList(d.Recv.List) + ") "
|
|
1075
|
+
}
|
|
1076
|
+
scope += formatFuncType(d.Type)
|
|
1077
|
+
return "func " + scope
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
func formatFuncType(f *ast.FuncType) string {
|
|
1081
|
+
params := formatFieldList(f.Params.List)
|
|
1082
|
+
results := ""
|
|
1083
|
+
if f.Results != nil {
|
|
1084
|
+
results = " -> " + formatFieldList(f.Results.List)
|
|
1085
|
+
}
|
|
1086
|
+
return params + results
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
func formatFieldList(fields []*ast.Field) string {
|
|
1090
|
+
if len(fields) == 0 {
|
|
1091
|
+
return "()"
|
|
1092
|
+
}
|
|
1093
|
+
names := make([]string, 0, len(fields))
|
|
1094
|
+
for _, f := range fields {
|
|
1095
|
+
name := ""
|
|
1096
|
+
if len(f.Names) > 0 {
|
|
1097
|
+
name = f.Names[0].Name
|
|
1098
|
+
}
|
|
1099
|
+
t := formatType(f.Type)
|
|
1100
|
+
if name != "" {
|
|
1101
|
+
names = append(names, name+" "+t)
|
|
1102
|
+
} else {
|
|
1103
|
+
names = append(names, t)
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
return "(" + strings.Join(names, ", ") + ")"
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
func formatFields(fields []*ast.Field) string {
|
|
1110
|
+
lines := make([]string, 0)
|
|
1111
|
+
for _, f := range fields {
|
|
1112
|
+
name := ""
|
|
1113
|
+
if len(f.Names) > 0 {
|
|
1114
|
+
name = f.Names[0].Name
|
|
1115
|
+
}
|
|
1116
|
+
t := formatType(f.Type)
|
|
1117
|
+
if name != "" {
|
|
1118
|
+
lines = append(lines, name+" "+t)
|
|
1119
|
+
} else {
|
|
1120
|
+
lines = append(lines, t)
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
return strings.Join(lines, "; ")
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
func formatMethods(fields []*ast.Field) string {
|
|
1127
|
+
return formatFields(fields)
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
func formatTypeParams(tp *ast.FieldList) string {
|
|
1131
|
+
if tp == nil || len(tp.List) == 0 {
|
|
1132
|
+
return ""
|
|
1133
|
+
}
|
|
1134
|
+
params := make([]string, len(tp.List))
|
|
1135
|
+
for i, p := range tp.List {
|
|
1136
|
+
if len(p.Names) > 0 {
|
|
1137
|
+
params[i] = p.Names[0].Name
|
|
1138
|
+
} else {
|
|
1139
|
+
params[i] = "T"
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
return "[" + strings.Join(params, ", ") + "]"
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
func formatType(t ast.Expr) string {
|
|
1146
|
+
if t == nil {
|
|
1147
|
+
return "?"
|
|
1148
|
+
}
|
|
1149
|
+
switch v := t.(type) {
|
|
1150
|
+
case *ast.Ident:
|
|
1151
|
+
return v.Name
|
|
1152
|
+
case *ast.SelectorExpr:
|
|
1153
|
+
return formatType(v.X) + "." + v.Sel.Name
|
|
1154
|
+
case *ast.StarExpr:
|
|
1155
|
+
return "*" + formatType(v.X)
|
|
1156
|
+
case *ast.ArrayType:
|
|
1157
|
+
if v.Len == nil {
|
|
1158
|
+
return "[]" + formatType(v.Elt)
|
|
1159
|
+
}
|
|
1160
|
+
return "[...]" + formatType(v.Elt)
|
|
1161
|
+
case *ast.MapType:
|
|
1162
|
+
return "map[" + formatType(v.Key) + "]" + formatType(v.Value)
|
|
1163
|
+
case *ast.InterfaceType:
|
|
1164
|
+
return "interface{}"
|
|
1165
|
+
case *ast.StructType:
|
|
1166
|
+
return "struct{}"
|
|
1167
|
+
case *ast.FuncType:
|
|
1168
|
+
return formatFuncType(v)
|
|
1169
|
+
case *ast.ChanType:
|
|
1170
|
+
return "chan " + formatType(v.Value)
|
|
1171
|
+
case *ast.BasicLit:
|
|
1172
|
+
return v.Value
|
|
1173
|
+
case *ast.IndexExpr:
|
|
1174
|
+
// Generic instantiation with one type arg, e.g. Logger[int].
|
|
1175
|
+
return formatType(v.X) + "[" + formatType(v.Index) + "]"
|
|
1176
|
+
case *ast.IndexListExpr:
|
|
1177
|
+
// Generic instantiation with multiple type args, e.g. Map[K, V].
|
|
1178
|
+
args := make([]string, len(v.Indices))
|
|
1179
|
+
for i, idx := range v.Indices {
|
|
1180
|
+
args[i] = formatType(idx)
|
|
1181
|
+
}
|
|
1182
|
+
return formatType(v.X) + "[" + strings.Join(args, ", ") + "]"
|
|
1183
|
+
default:
|
|
1184
|
+
return "?"
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
`;
|
|
1188
|
+
function syncGoParse(filePath, content, lang) {
|
|
1189
|
+
const tmpDir = path4.join(os.tmpdir(), "ws-go-parse");
|
|
1190
|
+
try {
|
|
1191
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
1192
|
+
const scriptPath = path4.join(tmpDir, "parse.go");
|
|
1193
|
+
writeFileSync(scriptPath, GO_PARSE_SCRIPT, "utf8");
|
|
1194
|
+
const stdout = execFileSync("go", ["run", scriptPath], {
|
|
1195
|
+
input: content,
|
|
1196
|
+
timeout: 15e3,
|
|
1197
|
+
encoding: "utf8",
|
|
1198
|
+
windowsHide: true
|
|
1199
|
+
});
|
|
1200
|
+
if (!stdout.trim()) {
|
|
1201
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
1202
|
+
}
|
|
1203
|
+
const raw = JSON.parse(stdout.trim());
|
|
1204
|
+
const symbols = raw.map((s) => ({
|
|
1205
|
+
id: 0,
|
|
1206
|
+
lang,
|
|
1207
|
+
kind: s.kind,
|
|
1208
|
+
name: s.name,
|
|
1209
|
+
file: filePath,
|
|
1210
|
+
line: s.line,
|
|
1211
|
+
col: s.col,
|
|
1212
|
+
signature: s.signature ?? "",
|
|
1213
|
+
docComment: "",
|
|
1214
|
+
scope: s.scope ?? "",
|
|
1215
|
+
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
1216
|
+
}));
|
|
1217
|
+
return { file: filePath, lang, symbols, mtimeMs: Date.now() };
|
|
1218
|
+
} catch {
|
|
1219
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
function parseSymbols3(opts) {
|
|
1223
|
+
const { file, lang } = opts;
|
|
1224
|
+
try {
|
|
1225
|
+
return syncPyParse(file, lang);
|
|
1226
|
+
} catch {
|
|
1227
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
var PY_PARSE_SCRIPT = `import ast, json, sys, os
|
|
1231
|
+
|
|
1232
|
+
def get_name(node):
|
|
1233
|
+
if isinstance(node, ast.Name):
|
|
1234
|
+
return node.id
|
|
1235
|
+
elif isinstance(node, ast.Attribute):
|
|
1236
|
+
return get_name(node.value) + "." + node.attr
|
|
1237
|
+
elif isinstance(node, ast.Subscript):
|
|
1238
|
+
return get_name(node.value)
|
|
1239
|
+
elif isinstance(node, ast.Call):
|
|
1240
|
+
return get_name(node.func)
|
|
1241
|
+
elif isinstance(node, ast.Constant):
|
|
1242
|
+
return str(node.value)
|
|
1243
|
+
return ""
|
|
1244
|
+
|
|
1245
|
+
def get_decorators(node):
|
|
1246
|
+
decs = []
|
|
1247
|
+
for dec in node.decorator_list:
|
|
1248
|
+
decs.append(get_name(dec))
|
|
1249
|
+
return decs
|
|
1250
|
+
|
|
1251
|
+
def get_bases(node):
|
|
1252
|
+
bases = []
|
|
1253
|
+
for base in node.bases:
|
|
1254
|
+
bases.append(get_name(base))
|
|
1255
|
+
return bases
|
|
1256
|
+
|
|
1257
|
+
def get_args(args):
|
|
1258
|
+
parts = []
|
|
1259
|
+
for arg in args.args:
|
|
1260
|
+
parts.append(arg.arg)
|
|
1261
|
+
return ", ".join(parts)
|
|
1262
|
+
|
|
1263
|
+
def get_returns(node):
|
|
1264
|
+
if node.returns is None:
|
|
1265
|
+
return ""
|
|
1266
|
+
return get_name(node.returns)
|
|
1267
|
+
|
|
1268
|
+
class Sym:
|
|
1269
|
+
def __init__(self, name, kind, line, col, signature, scope):
|
|
1270
|
+
self.name = name
|
|
1271
|
+
self.kind = kind
|
|
1272
|
+
self.line = line
|
|
1273
|
+
self.col = col
|
|
1274
|
+
self.signature = signature
|
|
1275
|
+
self.scope = scope
|
|
1276
|
+
def to_dict(self):
|
|
1277
|
+
return {
|
|
1278
|
+
"name": self.name,
|
|
1279
|
+
"kind": self.kind,
|
|
1280
|
+
"line": self.line,
|
|
1281
|
+
"col": self.col,
|
|
1282
|
+
"signature": self.signature,
|
|
1283
|
+
"scope": self.scope,
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
def is_private(name):
|
|
1287
|
+
return name.startswith("__") and not name.endswith("__")
|
|
1288
|
+
|
|
1289
|
+
syms = []
|
|
1290
|
+
errors = []
|
|
1291
|
+
|
|
1292
|
+
try:
|
|
1293
|
+
with open(sys.argv[1], "r", encoding="utf-8") as f:
|
|
1294
|
+
source = f.read()
|
|
1295
|
+
tree = ast.parse(source, filename=sys.argv[1])
|
|
1296
|
+
except Exception as e:
|
|
1297
|
+
errors.append(str(e))
|
|
1298
|
+
print("[]")
|
|
1299
|
+
sys.exit(0)
|
|
1300
|
+
|
|
1301
|
+
# Module-level scope
|
|
1302
|
+
module_scope = os.path.basename(sys.argv[1])[:-3] # strip .py
|
|
1303
|
+
|
|
1304
|
+
class ModuleVisitor(ast.NodeVisitor):
|
|
1305
|
+
def __init__(self):
|
|
1306
|
+
self.scope_stack = [module_scope]
|
|
1307
|
+
|
|
1308
|
+
def visit_ClassDef(self, node):
|
|
1309
|
+
bases = get_bases(node)
|
|
1310
|
+
decs = get_decorators(node)
|
|
1311
|
+
sig = "class " + node.name
|
|
1312
|
+
if bases:
|
|
1313
|
+
sig += "(" + ", ".join(bases) + ")"
|
|
1314
|
+
sig += ": ..."
|
|
1315
|
+
syms.append(Sym(
|
|
1316
|
+
name=node.name,
|
|
1317
|
+
kind="class",
|
|
1318
|
+
line=node.lineno,
|
|
1319
|
+
col=node.col_offset,
|
|
1320
|
+
signature=sig,
|
|
1321
|
+
scope=".".join(self.scope_stack) + "." + node.name,
|
|
1322
|
+
))
|
|
1323
|
+
self.scope_stack.append(node.name)
|
|
1324
|
+
self.generic_visit(node)
|
|
1325
|
+
self.scope_stack.pop()
|
|
1326
|
+
|
|
1327
|
+
def visit_FunctionDef(self, node):
|
|
1328
|
+
decs = get_decorators(node)
|
|
1329
|
+
args = get_args(node.args)
|
|
1330
|
+
returns = get_returns(node)
|
|
1331
|
+
is_async = isinstance(node, ast.AsyncFunctionDef)
|
|
1332
|
+
|
|
1333
|
+
kind = "function"
|
|
1334
|
+
prefix = "def "
|
|
1335
|
+
if decs:
|
|
1336
|
+
for d in decs:
|
|
1337
|
+
if d.endswith(".staticmethod"):
|
|
1338
|
+
kind = "staticmethod"
|
|
1339
|
+
elif d.endswith(".classmethod"):
|
|
1340
|
+
kind = "classmethod"
|
|
1341
|
+
elif d == "property":
|
|
1342
|
+
kind = "property"
|
|
1343
|
+
|
|
1344
|
+
if is_async:
|
|
1345
|
+
kind = "async_" + kind
|
|
1346
|
+
|
|
1347
|
+
sig = f"{prefix}{node.name}({args})"
|
|
1348
|
+
if returns:
|
|
1349
|
+
sig += f" -> {returns}"
|
|
1350
|
+
scope = ".".join(self.scope_stack) + "." + node.name
|
|
1351
|
+
|
|
1352
|
+
syms.append(Sym(
|
|
1353
|
+
name=node.name,
|
|
1354
|
+
kind=kind,
|
|
1355
|
+
line=node.lineno,
|
|
1356
|
+
col=node.col_offset,
|
|
1357
|
+
signature=sig,
|
|
1358
|
+
scope=scope,
|
|
1359
|
+
))
|
|
1360
|
+
# Don't descend into function bodies to avoid local symbols
|
|
1361
|
+
# self.generic_visit(node)
|
|
1362
|
+
|
|
1363
|
+
def visit_AsyncFunctionDef(self, node):
|
|
1364
|
+
# Treat as function
|
|
1365
|
+
self.visit_FunctionDef(node)
|
|
1366
|
+
|
|
1367
|
+
def visit_Assign(self, node):
|
|
1368
|
+
for target in node.targets:
|
|
1369
|
+
if isinstance(target, ast.Name):
|
|
1370
|
+
name = target.id
|
|
1371
|
+
if is_private(name):
|
|
1372
|
+
continue
|
|
1373
|
+
# Infer constness from UPPER_CASE naming
|
|
1374
|
+
kind = "const" if name.isupper() else "var"
|
|
1375
|
+
col = target.col_offset if hasattr(target, 'col_offset') else 0
|
|
1376
|
+
syms.append(Sym(
|
|
1377
|
+
name=name,
|
|
1378
|
+
kind=kind,
|
|
1379
|
+
line=node.lineno,
|
|
1380
|
+
col=col,
|
|
1381
|
+
signature=f"{name} = ...",
|
|
1382
|
+
scope=".".join(self.scope_stack),
|
|
1383
|
+
))
|
|
1384
|
+
|
|
1385
|
+
def visit_AnnAssign(self, node):
|
|
1386
|
+
if isinstance(node.target, ast.Name):
|
|
1387
|
+
name = node.target.id
|
|
1388
|
+
if is_private(name):
|
|
1389
|
+
return
|
|
1390
|
+
kind = "const" if name.isupper() else "var"
|
|
1391
|
+
col = node.target.col_offset if hasattr(node.target, 'col_offset') else 0
|
|
1392
|
+
sig = f"{name}: {get_name(node.annotation)}"
|
|
1393
|
+
if node.value:
|
|
1394
|
+
sig += " = ..."
|
|
1395
|
+
syms.append(Sym(
|
|
1396
|
+
name=name,
|
|
1397
|
+
kind=kind,
|
|
1398
|
+
line=node.lineno,
|
|
1399
|
+
col=col,
|
|
1400
|
+
signature=sig,
|
|
1401
|
+
scope=".".join(self.scope_stack),
|
|
1402
|
+
))
|
|
1403
|
+
|
|
1404
|
+
def visit_Import(self, node):
|
|
1405
|
+
for alias in node.names:
|
|
1406
|
+
name = alias.asname or alias.name
|
|
1407
|
+
syms.append(Sym(
|
|
1408
|
+
name=name,
|
|
1409
|
+
kind="import",
|
|
1410
|
+
line=node.lineno,
|
|
1411
|
+
col=node.col_offset,
|
|
1412
|
+
signature=f"import {alias.name}",
|
|
1413
|
+
scope=".".join(self.scope_stack),
|
|
1414
|
+
))
|
|
1415
|
+
|
|
1416
|
+
def visit_ImportFrom(self, node):
|
|
1417
|
+
module = node.module or ""
|
|
1418
|
+
for alias in node.names:
|
|
1419
|
+
name = alias.asname or alias.name
|
|
1420
|
+
syms.append(Sym(
|
|
1421
|
+
name=name,
|
|
1422
|
+
kind="import",
|
|
1423
|
+
line=node.lineno,
|
|
1424
|
+
col=node.col_offset,
|
|
1425
|
+
signature=f"from {module} import {alias.name}",
|
|
1426
|
+
scope=".".join(self.scope_stack),
|
|
1427
|
+
))
|
|
1428
|
+
|
|
1429
|
+
visitor = ModuleVisitor()
|
|
1430
|
+
visitor.visit(tree)
|
|
1431
|
+
|
|
1432
|
+
print(json.dumps([s.to_dict() for s in syms]))
|
|
1433
|
+
`;
|
|
1434
|
+
function syncPyParse(filePath, lang) {
|
|
1435
|
+
try {
|
|
1436
|
+
const tmpDir = path4.join(os.tmpdir(), "ws-py-parse");
|
|
1437
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
1438
|
+
const scriptPath = path4.join(tmpDir, "parse.py");
|
|
1439
|
+
writeFileSync(scriptPath, PY_PARSE_SCRIPT, "utf8");
|
|
1440
|
+
const stdout = execFileSync("python", [scriptPath, filePath], {
|
|
1441
|
+
timeout: 15e3,
|
|
1442
|
+
encoding: "utf8",
|
|
1443
|
+
windowsHide: true
|
|
1444
|
+
});
|
|
1445
|
+
if (!stdout.trim()) {
|
|
1446
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
1447
|
+
}
|
|
1448
|
+
const raw = JSON.parse(stdout.trim());
|
|
1449
|
+
const symbols = raw.map((s) => ({
|
|
1450
|
+
id: 0,
|
|
1451
|
+
lang,
|
|
1452
|
+
kind: s.kind,
|
|
1453
|
+
name: s.name,
|
|
1454
|
+
file: filePath,
|
|
1455
|
+
line: s.line,
|
|
1456
|
+
col: s.col,
|
|
1457
|
+
signature: s.signature ?? "",
|
|
1458
|
+
docComment: "",
|
|
1459
|
+
scope: s.scope ?? "",
|
|
1460
|
+
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
1461
|
+
}));
|
|
1462
|
+
return { file: filePath, lang, symbols, mtimeMs: Date.now() };
|
|
1463
|
+
} catch {
|
|
1464
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
function parseSymbols4(opts) {
|
|
1468
|
+
const { file, content, lang } = opts;
|
|
1469
|
+
const nativeAvailable = checkNativeParser();
|
|
1470
|
+
if (nativeAvailable) {
|
|
1471
|
+
const result = tryNativeParse(file, content);
|
|
1472
|
+
if (result) return result;
|
|
1473
|
+
}
|
|
1474
|
+
return regexParse({ file, content, lang });
|
|
1475
|
+
}
|
|
1476
|
+
function checkNativeParser() {
|
|
1477
|
+
try {
|
|
1478
|
+
execFileSync("rustc", ["--version"], { stdio: "pipe", windowsHide: true });
|
|
1479
|
+
const toolsDir = path4.join(process.cwd(), "tools");
|
|
1480
|
+
try {
|
|
1481
|
+
execFileSync(
|
|
1482
|
+
"cargo",
|
|
1483
|
+
[
|
|
1484
|
+
"metadata",
|
|
1485
|
+
"--no-deps",
|
|
1486
|
+
"--format-version",
|
|
1487
|
+
"1",
|
|
1488
|
+
"--manifest-path",
|
|
1489
|
+
path4.join(toolsDir, "Cargo.toml")
|
|
1490
|
+
],
|
|
1491
|
+
{ stdio: "pipe", windowsHide: true }
|
|
1492
|
+
);
|
|
1493
|
+
return true;
|
|
1494
|
+
} catch {
|
|
1495
|
+
return false;
|
|
1496
|
+
}
|
|
1497
|
+
} catch {
|
|
1498
|
+
return false;
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
function tryNativeParse(file, content) {
|
|
1502
|
+
try {
|
|
1503
|
+
const toolsDir = path4.join(process.cwd(), "tools");
|
|
1504
|
+
const crateDir = path4.join(toolsDir, "syn-parser");
|
|
1505
|
+
const tmpFile = path4.join(crateDir, "src", "input.rs");
|
|
1506
|
+
writeFileSync(tmpFile, content, "utf8");
|
|
1507
|
+
const result = spawnSync(
|
|
1508
|
+
"cargo",
|
|
1509
|
+
["run", "--manifest-path", path4.join(toolsDir, "Cargo.toml")],
|
|
1510
|
+
{
|
|
1511
|
+
cwd: process.cwd(),
|
|
1512
|
+
encoding: "utf8",
|
|
1513
|
+
timeout: 15e3,
|
|
1514
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1515
|
+
windowsHide: true
|
|
1516
|
+
}
|
|
1517
|
+
);
|
|
1518
|
+
if (result.status === 0 && result.stdout) {
|
|
1519
|
+
const symbols = JSON.parse(result.stdout);
|
|
1520
|
+
return {
|
|
1521
|
+
file,
|
|
1522
|
+
lang: "rs",
|
|
1523
|
+
symbols: symbols.map((s) => ({ ...s, id: 0, lang: "rs" })),
|
|
1524
|
+
mtimeMs: Date.now()
|
|
1525
|
+
};
|
|
1526
|
+
}
|
|
1527
|
+
} catch {
|
|
1528
|
+
}
|
|
1529
|
+
return null;
|
|
1530
|
+
}
|
|
1531
|
+
var RS_PATTERNS = [
|
|
1532
|
+
{ regex: /fn\s+(\w+)\s*\([^)]*\)/g, kind: "function" },
|
|
1533
|
+
{ regex: /struct\s+(\w+)/g, kind: "struct" },
|
|
1534
|
+
{ regex: /enum\s+(\w+)/g, kind: "enum" },
|
|
1535
|
+
{ regex: /trait\s+(\w+)/g, kind: "trait" },
|
|
1536
|
+
{ regex: /impl\s+(?:<[^>]+>)?(\w+)/g, kind: "impl" },
|
|
1537
|
+
{ regex: /type\s+(\w+)\s*=/g, kind: "type" },
|
|
1538
|
+
{ regex: /const\s+(\w+)/g, kind: "const" },
|
|
1539
|
+
{ regex: /static\s+(\w+)/g, kind: "static" },
|
|
1540
|
+
{ regex: /mod\s+(\w+)/g, kind: "mod" }
|
|
1541
|
+
];
|
|
1542
|
+
function regexParse(opts) {
|
|
1543
|
+
const { file, content, lang } = opts;
|
|
1544
|
+
const symbols = [];
|
|
1545
|
+
const lines = content.split("\n");
|
|
1546
|
+
const lineOffsets = [0];
|
|
1547
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1548
|
+
lineOffsets.push((lineOffsets[i] ?? 0) + (lines[i]?.length ?? 0) + 1);
|
|
1549
|
+
}
|
|
1550
|
+
function lineFromOffset(offset) {
|
|
1551
|
+
let lo = 0;
|
|
1552
|
+
let hi = lineOffsets.length - 1;
|
|
1553
|
+
while (lo < hi) {
|
|
1554
|
+
const mid = lo + hi + 1 >>> 1;
|
|
1555
|
+
if (expectDefined(lineOffsets[mid]) <= offset) lo = mid;
|
|
1556
|
+
else hi = mid - 1;
|
|
1557
|
+
}
|
|
1558
|
+
return lo + 1;
|
|
1559
|
+
}
|
|
1560
|
+
function extractDeclaration(lineIdx, _match) {
|
|
1561
|
+
const line = lines[lineIdx] ?? "";
|
|
1562
|
+
return line.trim().slice(0, 500);
|
|
1563
|
+
}
|
|
1564
|
+
for (const pattern of RS_PATTERNS) {
|
|
1565
|
+
pattern.regex.lastIndex = 0;
|
|
1566
|
+
for (let match = pattern.regex.exec(content); match !== null; match = pattern.regex.exec(content)) {
|
|
1567
|
+
const name = expectDefined(match[1]);
|
|
1568
|
+
const offset = match.index ?? 0;
|
|
1569
|
+
const line = lineFromOffset(offset);
|
|
1570
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1571
|
+
const lineIdx = line - 1;
|
|
1572
|
+
const signature = extractDeclaration(lineIdx);
|
|
1573
|
+
symbols.push({
|
|
1574
|
+
id: 0,
|
|
1575
|
+
lang,
|
|
1576
|
+
kind: pattern.kind,
|
|
1577
|
+
name,
|
|
1578
|
+
file,
|
|
1579
|
+
line,
|
|
1580
|
+
col,
|
|
1581
|
+
signature,
|
|
1582
|
+
docComment: "",
|
|
1583
|
+
scope: "",
|
|
1584
|
+
text: `${name} ${signature}`.trim()
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1589
|
+
const deduped = symbols.filter((s) => {
|
|
1590
|
+
const key = `${s.name}:${s.line}`;
|
|
1591
|
+
if (seen.has(key)) return false;
|
|
1592
|
+
seen.add(key);
|
|
1593
|
+
return true;
|
|
1594
|
+
});
|
|
1595
|
+
return { file, lang, symbols: deduped, mtimeMs: Date.now() };
|
|
1596
|
+
}
|
|
1597
|
+
function parseSymbols5(opts) {
|
|
1598
|
+
const { file, content, lang } = opts;
|
|
1599
|
+
try {
|
|
1600
|
+
return regexParse2({ file, content, lang });
|
|
1601
|
+
} catch {
|
|
1602
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
function regexParse2(opts) {
|
|
1606
|
+
const { file, content, lang } = opts;
|
|
1607
|
+
const symbols = [];
|
|
1608
|
+
const basename2 = path4.basename(file).toLowerCase();
|
|
1609
|
+
const isPackageJson = basename2 === "package.json";
|
|
1610
|
+
const isTsconfig = basename2 === "tsconfig.json" || basename2 === "tsconfig.build.json";
|
|
1611
|
+
const isJsonSchema = content.includes("$schema") || content.includes("$id") || content.includes("$ref");
|
|
1612
|
+
const isOpenApi = content.includes("openapi") || content.includes("swagger");
|
|
1613
|
+
const lines = content.split("\n");
|
|
1614
|
+
const lineOffsets = [0];
|
|
1615
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1616
|
+
lineOffsets.push((lineOffsets[i] ?? 0) + (lines[i]?.length ?? 0) + 1);
|
|
1617
|
+
}
|
|
1618
|
+
function lineFromOffset(offset) {
|
|
1619
|
+
let lo = 0;
|
|
1620
|
+
let hi = lineOffsets.length - 1;
|
|
1621
|
+
while (lo < hi) {
|
|
1622
|
+
const mid = lo + hi + 1 >>> 1;
|
|
1623
|
+
if (expectDefined(lineOffsets[mid]) <= offset) lo = mid;
|
|
1624
|
+
else hi = mid - 1;
|
|
1625
|
+
}
|
|
1626
|
+
return lo + 1;
|
|
1627
|
+
}
|
|
1628
|
+
const rootMatch = content.match(/^\s*\{/m);
|
|
1629
|
+
if (rootMatch) {
|
|
1630
|
+
const offset = expectDefined(rootMatch.index);
|
|
1631
|
+
const line = lineFromOffset(offset);
|
|
1632
|
+
symbols.push(
|
|
1633
|
+
makeSymbol({
|
|
1634
|
+
name: path4.basename(file),
|
|
1635
|
+
kind: "object",
|
|
1636
|
+
line,
|
|
1637
|
+
col: 0,
|
|
1638
|
+
signature: `"${path4.basename(file)}" = { ... }`,
|
|
1639
|
+
file,
|
|
1640
|
+
lang
|
|
1641
|
+
})
|
|
1642
|
+
);
|
|
1643
|
+
}
|
|
1644
|
+
const topLevelKeyRegex = /^\s*"([^"]+)"\s*:/gm;
|
|
1645
|
+
for (let match = topLevelKeyRegex.exec(content); match !== null; match = topLevelKeyRegex.exec(content)) {
|
|
1646
|
+
const key = expectDefined(match[1]);
|
|
1647
|
+
const offset = match.index ?? 0;
|
|
1648
|
+
const line = lineFromOffset(offset);
|
|
1649
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1650
|
+
let kind = "property";
|
|
1651
|
+
let signature = `"${key}": ..."`;
|
|
1652
|
+
if (isPackageJson) {
|
|
1653
|
+
if (key === "scripts" || key === "dependencies" || key === "devDependencies" || key === "peerDependencies" || key === "optionalDependencies") {
|
|
1654
|
+
kind = "const";
|
|
1655
|
+
signature = `"${key}": { ... }`;
|
|
1656
|
+
}
|
|
1657
|
+
} else if (isTsconfig) {
|
|
1658
|
+
if (key === "compilerOptions") {
|
|
1659
|
+
kind = "property";
|
|
1660
|
+
signature = `"compilerOptions": { ... }`;
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
if (isJsonSchema || isOpenApi) {
|
|
1664
|
+
if (key === "$schema" || key === "$id") {
|
|
1665
|
+
kind = "schema";
|
|
1666
|
+
signature = `"${key}": "..."`;
|
|
1667
|
+
} else if (key === "$ref") {
|
|
1668
|
+
kind = "schema";
|
|
1669
|
+
signature = `"$ref": "..."`;
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
symbols.push(
|
|
1673
|
+
makeSymbol({
|
|
1674
|
+
name: key,
|
|
1675
|
+
kind,
|
|
1676
|
+
line,
|
|
1677
|
+
col,
|
|
1678
|
+
signature,
|
|
1679
|
+
file,
|
|
1680
|
+
lang
|
|
1681
|
+
})
|
|
1682
|
+
);
|
|
1683
|
+
if (isPackageJson && key === "scripts") {
|
|
1684
|
+
extractPackageScripts(content, symbols, file, lang, lineOffsets, lineFromOffset);
|
|
1685
|
+
}
|
|
1686
|
+
if (isTsconfig && key === "compilerOptions") {
|
|
1687
|
+
extractCompilerOptions(content, symbols, file, lang, lineOffsets, line, lineFromOffset);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
const defsRegex = /"\$defs"\s*:|"\$defs"\s*:/g;
|
|
1691
|
+
const defsMatch = defsRegex.exec(content);
|
|
1692
|
+
if (defsMatch !== null) {
|
|
1693
|
+
const offset = expectDefined(defsMatch.index);
|
|
1694
|
+
const line = lineFromOffset(offset);
|
|
1695
|
+
symbols.push(
|
|
1696
|
+
makeSymbol({
|
|
1697
|
+
name: "$defs",
|
|
1698
|
+
kind: "property",
|
|
1699
|
+
line,
|
|
1700
|
+
col: offset - (lineOffsets[line - 1] ?? 0),
|
|
1701
|
+
signature: '"$defs": { ... }',
|
|
1702
|
+
file,
|
|
1703
|
+
lang
|
|
1704
|
+
})
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
const defsPatterns = [
|
|
1708
|
+
/"\$defs"\s*:/g,
|
|
1709
|
+
/"definitions"\s*:/g,
|
|
1710
|
+
/"components"\s*:/g,
|
|
1711
|
+
/"schemas"\s*:/g
|
|
1712
|
+
];
|
|
1713
|
+
for (const pat of defsPatterns) {
|
|
1714
|
+
pat.lastIndex = 0;
|
|
1715
|
+
for (let match = pat.exec(content); match !== null; match = pat.exec(content)) {
|
|
1716
|
+
const offset = match.index ?? 0;
|
|
1717
|
+
const line = lineFromOffset(offset);
|
|
1718
|
+
const key = match[0]?.match(/"([^"]+)"/)?.[1] ?? expectDefined(match[0]);
|
|
1719
|
+
symbols.push(
|
|
1720
|
+
makeSymbol({
|
|
1721
|
+
name: key,
|
|
1722
|
+
kind: "property",
|
|
1723
|
+
line,
|
|
1724
|
+
col: offset - (lineOffsets[line - 1] ?? 0),
|
|
1725
|
+
signature: `"${key}": { ... }`,
|
|
1726
|
+
file,
|
|
1727
|
+
lang
|
|
1728
|
+
})
|
|
1729
|
+
);
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
return { file, lang, symbols, mtimeMs: Date.now() };
|
|
1733
|
+
}
|
|
1734
|
+
function extractPackageScripts(content, symbols, file, lang, lineOffsets, lineFromOffset) {
|
|
1735
|
+
const scriptsBlockRegex = /"scripts"\s*:\s*\{([^}]+)\}/g;
|
|
1736
|
+
for (let match = scriptsBlockRegex.exec(content); match !== null; match = scriptsBlockRegex.exec(content)) {
|
|
1737
|
+
const blockContent = expectDefined(match[0]);
|
|
1738
|
+
const blockOffset = match.index ?? 0;
|
|
1739
|
+
const scriptKeyRegex = /"(\w[\w-]*)"\s*:/g;
|
|
1740
|
+
for (let scriptMatch = scriptKeyRegex.exec(blockContent); scriptMatch !== null; scriptMatch = scriptKeyRegex.exec(blockContent)) {
|
|
1741
|
+
const key = expectDefined(scriptMatch[1]);
|
|
1742
|
+
const keyOffset = blockOffset + expectDefined(scriptMatch.index);
|
|
1743
|
+
const line = lineFromOffset(keyOffset);
|
|
1744
|
+
symbols.push(
|
|
1745
|
+
makeSymbol({
|
|
1746
|
+
name: key,
|
|
1747
|
+
kind: "function",
|
|
1748
|
+
line,
|
|
1749
|
+
col: keyOffset - (lineOffsets[line - 1] ?? 0),
|
|
1750
|
+
signature: `"${key}": "..."`,
|
|
1751
|
+
file,
|
|
1752
|
+
lang
|
|
1753
|
+
})
|
|
1754
|
+
);
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
function extractCompilerOptions(content, symbols, file, lang, lineOffsets, parentLine, lineFromOffset) {
|
|
1759
|
+
const optsBlockRegex = /"compilerOptions"\s*:\s*\{([^}]+)\}/g;
|
|
1760
|
+
for (let match = optsBlockRegex.exec(content); match !== null; match = optsBlockRegex.exec(content)) {
|
|
1761
|
+
const blockContent = expectDefined(match[0]);
|
|
1762
|
+
const blockOffset = match.index ?? 0;
|
|
1763
|
+
const optKeyRegex = /"(\w[\w]*)"\s*:/g;
|
|
1764
|
+
for (let optMatch = optKeyRegex.exec(blockContent); optMatch !== null; optMatch = optKeyRegex.exec(blockContent)) {
|
|
1765
|
+
const key = expectDefined(optMatch[1]);
|
|
1766
|
+
const keyOffset = blockOffset + expectDefined(optMatch.index);
|
|
1767
|
+
const line = lineFromOffset(keyOffset);
|
|
1768
|
+
if (line <= parentLine) continue;
|
|
1769
|
+
symbols.push(
|
|
1770
|
+
makeSymbol({
|
|
1771
|
+
name: key,
|
|
1772
|
+
kind: "property",
|
|
1773
|
+
line,
|
|
1774
|
+
col: keyOffset - (lineOffsets[line - 1] ?? 0),
|
|
1775
|
+
signature: `"${key}": ...`,
|
|
1776
|
+
file,
|
|
1777
|
+
lang
|
|
1778
|
+
})
|
|
1779
|
+
);
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
function makeSymbol(opts) {
|
|
1784
|
+
return {
|
|
1785
|
+
id: 0,
|
|
1786
|
+
lang: opts.lang,
|
|
1787
|
+
kind: opts.kind,
|
|
1788
|
+
name: opts.name,
|
|
1789
|
+
file: opts.file,
|
|
1790
|
+
line: opts.line,
|
|
1791
|
+
col: opts.col,
|
|
1792
|
+
signature: opts.signature,
|
|
1793
|
+
docComment: "",
|
|
1794
|
+
scope: "",
|
|
1795
|
+
text: `${opts.name} ${opts.signature}`.trim()
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
function parseSymbols6(opts) {
|
|
1799
|
+
const { file, content, lang } = opts;
|
|
1800
|
+
try {
|
|
1801
|
+
return regexParse3({ file, content, lang });
|
|
1802
|
+
} catch {
|
|
1803
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
function regexParse3(opts) {
|
|
1807
|
+
const { file, content, lang } = opts;
|
|
1808
|
+
const symbols = [];
|
|
1809
|
+
const lines = content.split("\n");
|
|
1810
|
+
const lineOffsets = [0];
|
|
1811
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1812
|
+
lineOffsets.push((lineOffsets[i] ?? 0) + (lines[i]?.length ?? 0) + 1);
|
|
1813
|
+
}
|
|
1814
|
+
function lineFromOffset(offset) {
|
|
1815
|
+
let lo = 0;
|
|
1816
|
+
let hi = lineOffsets.length - 1;
|
|
1817
|
+
while (lo < hi) {
|
|
1818
|
+
const mid = lo + hi + 1 >>> 1;
|
|
1819
|
+
if (expectDefined(lineOffsets[mid]) <= offset) lo = mid;
|
|
1820
|
+
else hi = mid - 1;
|
|
1821
|
+
}
|
|
1822
|
+
return lo + 1;
|
|
1823
|
+
}
|
|
1824
|
+
const anchorRegex = /&(\w[\w-]*)/g;
|
|
1825
|
+
for (let match = anchorRegex.exec(content); match !== null; match = anchorRegex.exec(content)) {
|
|
1826
|
+
const name = expectDefined(match[1]);
|
|
1827
|
+
const offset = match.index ?? 0;
|
|
1828
|
+
const line = lineFromOffset(offset);
|
|
1829
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1830
|
+
symbols.push(
|
|
1831
|
+
makeSymbol2({
|
|
1832
|
+
name,
|
|
1833
|
+
kind: "const",
|
|
1834
|
+
line,
|
|
1835
|
+
col,
|
|
1836
|
+
signature: `&${name}`,
|
|
1837
|
+
file,
|
|
1838
|
+
lang
|
|
1839
|
+
})
|
|
1840
|
+
);
|
|
1841
|
+
}
|
|
1842
|
+
const aliasRegex = /\*(\w[\w-]*)/g;
|
|
1843
|
+
for (let match = aliasRegex.exec(content); match !== null; match = aliasRegex.exec(content)) {
|
|
1844
|
+
const name = expectDefined(match[1]);
|
|
1845
|
+
const offset = match.index ?? 0;
|
|
1846
|
+
const line = lineFromOffset(offset);
|
|
1847
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1848
|
+
symbols.push(
|
|
1849
|
+
makeSymbol2({
|
|
1850
|
+
name,
|
|
1851
|
+
kind: "const",
|
|
1852
|
+
line,
|
|
1853
|
+
col,
|
|
1854
|
+
signature: `*${name}`,
|
|
1855
|
+
file,
|
|
1856
|
+
lang
|
|
1857
|
+
})
|
|
1858
|
+
);
|
|
1859
|
+
}
|
|
1860
|
+
const kvRegex = /^(\s*)([^:#\s][^:#\s]*)\s*:/gm;
|
|
1861
|
+
for (let match = kvRegex.exec(content); match !== null; match = kvRegex.exec(content)) {
|
|
1862
|
+
const indent = match[1]?.length ?? 0;
|
|
1863
|
+
const key = match[2];
|
|
1864
|
+
if (!key) continue;
|
|
1865
|
+
const offset = match.index ?? 0;
|
|
1866
|
+
const line = lineFromOffset(offset);
|
|
1867
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1868
|
+
const lineContent = lines[line - 1] ?? "";
|
|
1869
|
+
if (/^[|&>]/.test(lineContent.trim())) continue;
|
|
1870
|
+
if (key === "---" || key === "...") continue;
|
|
1871
|
+
if (indent > 12) continue;
|
|
1872
|
+
const value = extractValue(content, match.index ?? 0);
|
|
1873
|
+
const kind = isScalar(value) ? "literal" : "property";
|
|
1874
|
+
const signature = `${key}: ${truncate(value, 60)}`;
|
|
1875
|
+
symbols.push(makeSymbol2({ name: key, kind, line, col, signature, file, lang }));
|
|
1876
|
+
}
|
|
1877
|
+
const listItemRegex = /^-(\s+)([^:#\s][^:#\s]*)\s*:/gm;
|
|
1878
|
+
for (let match = listItemRegex.exec(content); match !== null; match = listItemRegex.exec(content)) {
|
|
1879
|
+
const key = expectDefined(match[2]);
|
|
1880
|
+
const offset = match.index ?? 0;
|
|
1881
|
+
const line = lineFromOffset(offset);
|
|
1882
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1883
|
+
const value = extractValue(content, offset + match[0]?.length);
|
|
1884
|
+
const kind = isScalar(value) ? "literal" : "property";
|
|
1885
|
+
symbols.push(
|
|
1886
|
+
makeSymbol2({
|
|
1887
|
+
name: key,
|
|
1888
|
+
kind,
|
|
1889
|
+
line,
|
|
1890
|
+
col,
|
|
1891
|
+
signature: `- ${key}: ${truncate(value, 60)}`,
|
|
1892
|
+
file,
|
|
1893
|
+
lang
|
|
1894
|
+
})
|
|
1895
|
+
);
|
|
1896
|
+
}
|
|
1897
|
+
const blockScalarRegex = /^(\s*)([^:#\s][^:#\s]*)\s*:\s*[|>](\s|$)/gm;
|
|
1898
|
+
for (let match = blockScalarRegex.exec(content); match !== null; match = blockScalarRegex.exec(content)) {
|
|
1899
|
+
const key = expectDefined(match[2]);
|
|
1900
|
+
const offset = match.index ?? 0;
|
|
1901
|
+
const line = lineFromOffset(offset);
|
|
1902
|
+
const col = offset - (lineOffsets[line - 1] ?? 0);
|
|
1903
|
+
symbols.push(
|
|
1904
|
+
makeSymbol2({
|
|
1905
|
+
name: key,
|
|
1906
|
+
kind: "property",
|
|
1907
|
+
line,
|
|
1908
|
+
col,
|
|
1909
|
+
signature: `${key}: | ...`,
|
|
1910
|
+
file,
|
|
1911
|
+
lang
|
|
1912
|
+
})
|
|
1913
|
+
);
|
|
1914
|
+
}
|
|
1915
|
+
return { file, lang, symbols, mtimeMs: Date.now() };
|
|
1916
|
+
}
|
|
1917
|
+
function extractValue(content, afterColonOffset) {
|
|
1918
|
+
const lineEnd = content.indexOf("\n", afterColonOffset);
|
|
1919
|
+
const rest = content.slice(afterColonOffset, lineEnd < 0 ? void 0 : lineEnd);
|
|
1920
|
+
return rest.trim();
|
|
1921
|
+
}
|
|
1922
|
+
function isScalar(value) {
|
|
1923
|
+
if (!value) return false;
|
|
1924
|
+
if (/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(value)) return true;
|
|
1925
|
+
if (/^(true|false|null|undefined)$/i.test(value)) return true;
|
|
1926
|
+
if (/^'[^']*'$/.test(value) || /^"[^"]*"$/.test(value)) return true;
|
|
1927
|
+
return false;
|
|
1928
|
+
}
|
|
1929
|
+
function makeSymbol2(opts) {
|
|
1930
|
+
return {
|
|
1931
|
+
id: 0,
|
|
1932
|
+
lang: opts.lang,
|
|
1933
|
+
kind: opts.kind,
|
|
1934
|
+
name: opts.name,
|
|
1935
|
+
file: opts.file,
|
|
1936
|
+
line: opts.line,
|
|
1937
|
+
col: opts.col,
|
|
1938
|
+
signature: opts.signature,
|
|
1939
|
+
docComment: "",
|
|
1940
|
+
scope: "",
|
|
1941
|
+
text: `${opts.name} ${opts.signature}`.trim()
|
|
1942
|
+
};
|
|
1943
|
+
}
|
|
1944
|
+
function globBody(glob) {
|
|
1945
|
+
return compileGlob(glob).source.replace(/^\^/, "").replace(/\$$/, "");
|
|
1946
|
+
}
|
|
1947
|
+
function compileGitignore(lines) {
|
|
1948
|
+
const rules = [];
|
|
1949
|
+
for (const raw of lines) {
|
|
1950
|
+
let line = raw.replace(/\r$/, "");
|
|
1951
|
+
if (!line.trim() || line.trimStart().startsWith("#")) continue;
|
|
1952
|
+
line = line.trim();
|
|
1953
|
+
let negated = false;
|
|
1954
|
+
if (line.startsWith("!")) {
|
|
1955
|
+
negated = true;
|
|
1956
|
+
line = line.slice(1);
|
|
1957
|
+
}
|
|
1958
|
+
let dirOnly = false;
|
|
1959
|
+
if (line.endsWith("/")) {
|
|
1960
|
+
dirOnly = true;
|
|
1961
|
+
line = line.slice(0, -1);
|
|
1962
|
+
}
|
|
1963
|
+
if (!line) continue;
|
|
1964
|
+
const anchored = line.startsWith("/") || line.includes("/");
|
|
1965
|
+
if (line.startsWith("/")) line = line.slice(1);
|
|
1966
|
+
const body = globBody(line);
|
|
1967
|
+
const prefix = anchored ? "^" : "(?:^|.*/)";
|
|
1968
|
+
rules.push({
|
|
1969
|
+
eqOrUnder: new RegExp(`${prefix}${body}(?:/.*)?$`),
|
|
1970
|
+
under: new RegExp(`${prefix}${body}/.*$`),
|
|
1971
|
+
negated,
|
|
1972
|
+
dirOnly
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
return (relPath, isDir) => {
|
|
1976
|
+
const p = relPath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
1977
|
+
let ignored = false;
|
|
1978
|
+
for (const r of rules) {
|
|
1979
|
+
const re = r.dirOnly && !isDir ? r.under : r.eqOrUnder;
|
|
1980
|
+
if (re.test(p)) ignored = !r.negated;
|
|
1981
|
+
}
|
|
1982
|
+
return ignored;
|
|
1983
|
+
};
|
|
1984
|
+
}
|
|
1985
|
+
async function loadGitignoreMatcher(projectRoot) {
|
|
1986
|
+
let lines = [];
|
|
1987
|
+
try {
|
|
1988
|
+
const raw = await fs3.readFile(path4.join(projectRoot, ".gitignore"), "utf8");
|
|
1989
|
+
lines = raw.split("\n");
|
|
1990
|
+
} catch {
|
|
1991
|
+
}
|
|
1992
|
+
return compileGitignore(lines);
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
// src/codebase-index/indexer.ts
|
|
1996
|
+
var YIELD_EVERY_N = 50;
|
|
1997
|
+
function yieldEventLoop() {
|
|
1998
|
+
return new Promise((resolve2) => setImmediate(resolve2));
|
|
1999
|
+
}
|
|
2000
|
+
function throwIfAborted(signal) {
|
|
2001
|
+
if (!signal?.aborted) return;
|
|
2002
|
+
if (signal.reason instanceof Error) throw signal.reason;
|
|
2003
|
+
throw new Error(
|
|
2004
|
+
typeof signal.reason === "string" ? signal.reason : "Indexing cancelled"
|
|
2005
|
+
);
|
|
2006
|
+
}
|
|
2007
|
+
function isAbortError(err) {
|
|
2008
|
+
return err instanceof DOMException && err.name === "AbortError";
|
|
2009
|
+
}
|
|
2010
|
+
var DEFAULT_IGNORE = [
|
|
2011
|
+
"node_modules",
|
|
2012
|
+
".git",
|
|
2013
|
+
"dist",
|
|
2014
|
+
"build",
|
|
2015
|
+
".next",
|
|
2016
|
+
"coverage",
|
|
2017
|
+
".turbo",
|
|
2018
|
+
"__snapshots__",
|
|
2019
|
+
".nyc_output"
|
|
2020
|
+
];
|
|
2021
|
+
async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
2022
|
+
const results = [];
|
|
2023
|
+
const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
|
|
2024
|
+
const globs = [
|
|
2025
|
+
{ ext: ".ts", pat: compileGlob("**/*.ts") },
|
|
2026
|
+
{ ext: ".tsx", pat: compileGlob("**/*.tsx") },
|
|
2027
|
+
{ ext: ".js", pat: compileGlob("**/*.js") },
|
|
2028
|
+
{ ext: ".jsx", pat: compileGlob("**/*.jsx") },
|
|
2029
|
+
{ ext: ".go", pat: compileGlob("**/*.go") },
|
|
2030
|
+
{ ext: ".py", pat: compileGlob("**/*.py") },
|
|
2031
|
+
{ ext: ".rs", pat: compileGlob("**/*.rs") },
|
|
2032
|
+
{ ext: ".json", pat: compileGlob("**/*.json") },
|
|
2033
|
+
{ ext: ".yaml", pat: compileGlob("**/*.yaml") },
|
|
2034
|
+
{ ext: ".yml", pat: compileGlob("**/*.yml") }
|
|
2035
|
+
];
|
|
2036
|
+
let dirCount = 0;
|
|
2037
|
+
const walk = async (dir) => {
|
|
2038
|
+
throwIfAborted(signal);
|
|
2039
|
+
if (dirCount > 0 && dirCount % YIELD_EVERY_N === 0) {
|
|
2040
|
+
await yieldEventLoop();
|
|
2041
|
+
throwIfAborted(signal);
|
|
2042
|
+
}
|
|
2043
|
+
let entries;
|
|
2044
|
+
try {
|
|
2045
|
+
entries = await fs3.readdir(dir, { withFileTypes: true });
|
|
2046
|
+
} catch {
|
|
2047
|
+
return;
|
|
2048
|
+
}
|
|
2049
|
+
dirCount++;
|
|
2050
|
+
for (const e of entries) {
|
|
2051
|
+
if (ignoreSet.has(e.name)) continue;
|
|
2052
|
+
const full = path4.join(dir, e.name);
|
|
2053
|
+
const rel = path4.relative(projectRoot, full).replace(/\\/g, "/");
|
|
2054
|
+
if (e.isDirectory()) {
|
|
2055
|
+
if (isGitIgnored(rel, true)) continue;
|
|
2056
|
+
await walk(full);
|
|
2057
|
+
} else if (e.isFile()) {
|
|
2058
|
+
if (isGitIgnored(rel, false)) continue;
|
|
2059
|
+
const ext = path4.extname(e.name);
|
|
2060
|
+
for (const { ext: extName, pat } of globs) {
|
|
2061
|
+
if (ext === extName && (pat.test(rel) || pat.test(e.name))) {
|
|
2062
|
+
results.push(full);
|
|
2063
|
+
break;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
};
|
|
2069
|
+
await walk(projectRoot);
|
|
2070
|
+
return results;
|
|
2071
|
+
}
|
|
2072
|
+
async function parseFile(file, content, lang) {
|
|
2073
|
+
switch (lang) {
|
|
2074
|
+
case "ts":
|
|
2075
|
+
case "tsx":
|
|
2076
|
+
case "js":
|
|
2077
|
+
case "jsx":
|
|
2078
|
+
return parseSymbols({ file, content, lang });
|
|
2079
|
+
case "go":
|
|
2080
|
+
return parseSymbols2({ file, content, lang: "go" });
|
|
2081
|
+
case "py":
|
|
2082
|
+
return parseSymbols3({ file, lang: "py" });
|
|
2083
|
+
case "rs":
|
|
2084
|
+
return parseSymbols4({ file, content, lang: "rs" });
|
|
2085
|
+
case "json":
|
|
2086
|
+
return parseSymbols5({ file, content, lang: "json" });
|
|
2087
|
+
case "yaml":
|
|
2088
|
+
return parseSymbols6({ file, content, lang: "yaml" });
|
|
2089
|
+
default:
|
|
2090
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
async function runIndexer(_ctx, opts) {
|
|
2094
|
+
const store = new IndexStore(opts.projectRoot, { indexDir: opts.indexDir });
|
|
2095
|
+
try {
|
|
2096
|
+
return await runIndexerWithStore(store, opts);
|
|
2097
|
+
} finally {
|
|
2098
|
+
try {
|
|
2099
|
+
store.close();
|
|
2100
|
+
} catch {
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
async function runIndexerWithStore(store, opts) {
|
|
2105
|
+
const { projectRoot, force = false, langs, ignore = [], signal } = opts;
|
|
2106
|
+
const startMs = Date.now();
|
|
2107
|
+
const errors = [];
|
|
2108
|
+
const langStats = {};
|
|
2109
|
+
let filesIndexed = 0;
|
|
2110
|
+
let symbolsIndexed = 0;
|
|
2111
|
+
const isGitIgnored = await loadGitignoreMatcher(projectRoot);
|
|
2112
|
+
let files;
|
|
2113
|
+
if (opts.files && opts.files.length > 0) {
|
|
2114
|
+
files = opts.files.map((f) => path4.resolve(projectRoot, f)).filter((f) => !isGitIgnored(path4.relative(projectRoot, f).replace(/\\/g, "/"), false));
|
|
2115
|
+
} else {
|
|
2116
|
+
files = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
2117
|
+
}
|
|
2118
|
+
if (langs && langs.length > 0) {
|
|
2119
|
+
const langSet = new Set(langs);
|
|
2120
|
+
files = files.filter((f) => {
|
|
2121
|
+
const lang = detectLang(f);
|
|
2122
|
+
return lang ? langSet.has(lang) : false;
|
|
2123
|
+
});
|
|
2124
|
+
}
|
|
2125
|
+
if (force) store.clearAll();
|
|
2126
|
+
const existingMeta = /* @__PURE__ */ new Map();
|
|
2127
|
+
if (!force) {
|
|
2128
|
+
for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
|
|
2129
|
+
}
|
|
2130
|
+
for (let fi = 0; fi < files.length; fi++) {
|
|
2131
|
+
const file = expectDefined(files[fi]);
|
|
2132
|
+
opts.onProgress?.(fi + 1, files.length);
|
|
2133
|
+
if (fi > 0 && fi % YIELD_EVERY_N === 0) {
|
|
2134
|
+
await yieldEventLoop();
|
|
2135
|
+
throwIfAborted(signal);
|
|
2136
|
+
}
|
|
2137
|
+
let stat2;
|
|
2138
|
+
try {
|
|
2139
|
+
const statOpts = signal ? { signal } : {};
|
|
2140
|
+
stat2 = await fs3.stat(file, statOpts);
|
|
2141
|
+
} catch (e) {
|
|
2142
|
+
if (isAbortError(e)) throw e;
|
|
2143
|
+
store.deleteFile(file);
|
|
2144
|
+
continue;
|
|
2145
|
+
}
|
|
2146
|
+
if (!stat2.isFile()) continue;
|
|
2147
|
+
const lang = detectLang(file);
|
|
2148
|
+
if (!lang) continue;
|
|
2149
|
+
const meta = existingMeta.get(file);
|
|
2150
|
+
if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
|
|
2151
|
+
langStats[lang] = (langStats[lang] ?? 0) + meta.symbolCount;
|
|
2152
|
+
symbolsIndexed += meta.symbolCount;
|
|
2153
|
+
filesIndexed++;
|
|
2154
|
+
continue;
|
|
2155
|
+
}
|
|
2156
|
+
store.deleteRefsForFile(file);
|
|
2157
|
+
store.deleteSymbolsForFile(file);
|
|
2158
|
+
let content;
|
|
2159
|
+
try {
|
|
2160
|
+
content = await fs3.readFile(file, { encoding: "utf8", signal });
|
|
2161
|
+
} catch (e) {
|
|
2162
|
+
if (isAbortError(e)) throw e;
|
|
2163
|
+
errors.push(`read error: ${file}: ${e instanceof Error ? e.message : String(e)}`);
|
|
2164
|
+
continue;
|
|
2165
|
+
}
|
|
2166
|
+
let parsed;
|
|
2167
|
+
try {
|
|
2168
|
+
parsed = await parseFile(file, content, lang);
|
|
2169
|
+
} catch (e) {
|
|
2170
|
+
errors.push(`parse error: ${file}: ${e instanceof Error ? e.message : String(e)}`);
|
|
2171
|
+
continue;
|
|
2172
|
+
}
|
|
2173
|
+
if (parsed.symbols.length === 0) {
|
|
2174
|
+
store.upsertFile({
|
|
2175
|
+
file,
|
|
2176
|
+
lang,
|
|
2177
|
+
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
2178
|
+
symbolCount: 0,
|
|
2179
|
+
lastIndexed: Date.now()
|
|
2180
|
+
});
|
|
2181
|
+
filesIndexed++;
|
|
2182
|
+
continue;
|
|
2183
|
+
}
|
|
2184
|
+
const nextId = store.getMaxSymbolId() + 1;
|
|
2185
|
+
const symbolsWithIds = parsed.symbols.map((s, i) => ({ ...s, id: nextId + i }));
|
|
2186
|
+
store.insertSymbols(symbolsWithIds, nextId);
|
|
2187
|
+
const count = symbolsWithIds.length;
|
|
2188
|
+
symbolsIndexed += count;
|
|
2189
|
+
langStats[lang] = (langStats[lang] ?? 0) + count;
|
|
2190
|
+
if (parsed.refs && parsed.refs.length > 0) {
|
|
2191
|
+
for (let i = 0; i < symbolsWithIds.length; i++) {
|
|
2192
|
+
const sym = expectDefined(symbolsWithIds[i]);
|
|
2193
|
+
const symRefs = parsed.refs.filter((r) => r.line === sym.line);
|
|
2194
|
+
if (symRefs.length > 0) {
|
|
2195
|
+
const refsWithFromId = symRefs.map((r) => ({ ...r, fromId: sym.id }));
|
|
2196
|
+
store.insertRefs(sym.id, refsWithFromId);
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
store.upsertFile({
|
|
2201
|
+
file,
|
|
2202
|
+
lang,
|
|
2203
|
+
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
2204
|
+
symbolCount: count,
|
|
2205
|
+
lastIndexed: Date.now()
|
|
2206
|
+
});
|
|
2207
|
+
filesIndexed++;
|
|
2208
|
+
}
|
|
2209
|
+
for (const [file_] of existingMeta) {
|
|
2210
|
+
try {
|
|
2211
|
+
await fs3.stat(file_);
|
|
2212
|
+
} catch {
|
|
2213
|
+
store.deleteFile(file_);
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
const durationMs = Date.now() - startMs;
|
|
2217
|
+
store.setLastIndexed(Date.now());
|
|
2218
|
+
return {
|
|
2219
|
+
filesIndexed,
|
|
2220
|
+
symbolsIndexed,
|
|
2221
|
+
langStats,
|
|
2222
|
+
durationMs,
|
|
2223
|
+
errors
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
// src/codebase-index/index-service.ts
|
|
2228
|
+
function stubCtx(projectRoot) {
|
|
2229
|
+
return {
|
|
2230
|
+
projectRoot,
|
|
2231
|
+
cwd: projectRoot,
|
|
2232
|
+
messages: [],
|
|
2233
|
+
todos: [],
|
|
2234
|
+
readFiles: /* @__PURE__ */ new Set(),
|
|
2235
|
+
fileMtimes: /* @__PURE__ */ new Map()
|
|
2236
|
+
};
|
|
2237
|
+
}
|
|
2238
|
+
async function indexService(args, hooks = {}) {
|
|
2239
|
+
return runIndexer(stubCtx(args.projectRoot), {
|
|
2240
|
+
projectRoot: args.projectRoot,
|
|
2241
|
+
indexDir: args.indexDir,
|
|
2242
|
+
files: args.files,
|
|
2243
|
+
force: args.force,
|
|
2244
|
+
langs: args.langs,
|
|
2245
|
+
ignore: args.ignore,
|
|
2246
|
+
signal: hooks.signal,
|
|
2247
|
+
onProgress: hooks.onProgress
|
|
2248
|
+
});
|
|
2249
|
+
}
|
|
2250
|
+
function searchService(args) {
|
|
2251
|
+
const store = new IndexStore(args.projectRoot, { indexDir: args.indexDir });
|
|
2252
|
+
try {
|
|
2253
|
+
return store.searchRanked(
|
|
2254
|
+
args.query,
|
|
2255
|
+
{
|
|
2256
|
+
kind: args.kind,
|
|
2257
|
+
lang: args.lang,
|
|
2258
|
+
file: args.file,
|
|
2259
|
+
lspKind: args.lspKind
|
|
2260
|
+
},
|
|
2261
|
+
args.limit
|
|
2262
|
+
);
|
|
2263
|
+
} finally {
|
|
2264
|
+
store.close();
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
function statsService(args) {
|
|
2268
|
+
const store = new IndexStore(args.projectRoot, { indexDir: args.indexDir });
|
|
2269
|
+
try {
|
|
2270
|
+
return store.getStats();
|
|
2271
|
+
} finally {
|
|
2272
|
+
store.close();
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
// src/codebase-index/worker.ts
|
|
2277
|
+
if (!parentPort) throw new Error("codebase-index worker must be started as a worker thread");
|
|
2278
|
+
var port = parentPort;
|
|
2279
|
+
var inFlight = /* @__PURE__ */ new Map();
|
|
2280
|
+
function post(msg) {
|
|
2281
|
+
port.postMessage(msg);
|
|
2282
|
+
}
|
|
2283
|
+
async function dispatch(msg) {
|
|
2284
|
+
switch (msg.op) {
|
|
2285
|
+
case "index": {
|
|
2286
|
+
const ac = new AbortController();
|
|
2287
|
+
inFlight.set(msg.id, ac);
|
|
2288
|
+
try {
|
|
2289
|
+
return await indexService(msg.args, {
|
|
2290
|
+
signal: ac.signal,
|
|
2291
|
+
onProgress: (current, total) => post({ type: "progress", id: msg.id, current, total })
|
|
2292
|
+
});
|
|
2293
|
+
} finally {
|
|
2294
|
+
inFlight.delete(msg.id);
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
case "search":
|
|
2298
|
+
return searchService(msg.args);
|
|
2299
|
+
case "stats":
|
|
2300
|
+
return statsService(msg.args);
|
|
2301
|
+
default:
|
|
2302
|
+
throw new Error(`unknown index op: ${msg.op}`);
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
2305
|
+
port.on("message", (msg) => {
|
|
2306
|
+
if (msg.type === "cancel") {
|
|
2307
|
+
inFlight.get(msg.id)?.abort(new Error("Indexing cancelled"));
|
|
2308
|
+
return;
|
|
2309
|
+
}
|
|
2310
|
+
void dispatch(msg).then(
|
|
2311
|
+
(result) => post({ type: "response", id: msg.id, ok: true, result }),
|
|
2312
|
+
(err) => post({
|
|
2313
|
+
type: "response",
|
|
2314
|
+
id: msg.id,
|
|
2315
|
+
ok: false,
|
|
2316
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2317
|
+
})
|
|
2318
|
+
);
|
|
2319
|
+
});
|
|
2320
|
+
//# sourceMappingURL=worker.js.map
|
|
2321
|
+
//# sourceMappingURL=worker.js.map
|