@mnemonik/shared 6.50.0 → 6.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ast/astChunker.d.ts +124 -0
- package/dist/ast/astChunker.d.ts.map +1 -0
- package/dist/ast/astChunker.js +559 -0
- package/dist/ast/astChunker.js.map +1 -0
- package/dist/ast/grammars.d.ts +170 -0
- package/dist/ast/grammars.d.ts.map +1 -0
- package/dist/ast/grammars.js +411 -0
- package/dist/ast/grammars.js.map +1 -0
- package/dist/codeScanner.d.ts +59 -0
- package/dist/codeScanner.d.ts.map +1 -1
- package/dist/codeScanner.js +259 -2
- package/dist/codeScanner.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/package.json +3 -2
- package/queries/bash.tags.scm +7 -0
- package/queries/groovy.tags.scm +15 -0
- package/queries/powershell.tags.scm +14 -0
- package/scripts/vendor-grammars.ts +317 -0
- package/src/ast/astChunker.ts +661 -0
- package/src/ast/grammars.ts +490 -0
- package/src/codeScanner.ts +273 -3
- package/src/index.ts +22 -0
- package/wasm/bash.tags.scm +7 -0
- package/wasm/bash.wasm +0 -0
- package/wasm/c-sharp.tags.scm +23 -0
- package/wasm/c-sharp.wasm +0 -0
- package/wasm/c.tags.scm +9 -0
- package/wasm/c.wasm +0 -0
- package/wasm/cpp.tags.scm +15 -0
- package/wasm/cpp.wasm +0 -0
- package/wasm/elixir.tags.scm +54 -0
- package/wasm/elixir.wasm +0 -0
- package/wasm/go.tags.scm +42 -0
- package/wasm/go.wasm +0 -0
- package/wasm/groovy.tags.scm +15 -0
- package/wasm/groovy.wasm +0 -0
- package/wasm/java.tags.scm +20 -0
- package/wasm/java.wasm +0 -0
- package/wasm/javascript.tags.scm +99 -0
- package/wasm/javascript.wasm +0 -0
- package/wasm/php.tags.scm +40 -0
- package/wasm/php.wasm +0 -0
- package/wasm/powershell.tags.scm +14 -0
- package/wasm/powershell.wasm +0 -0
- package/wasm/python.tags.scm +14 -0
- package/wasm/python.wasm +0 -0
- package/wasm/ruby.tags.scm +64 -0
- package/wasm/ruby.wasm +0 -0
- package/wasm/rust.tags.scm +60 -0
- package/wasm/rust.wasm +0 -0
- package/wasm/scala.tags.scm +66 -0
- package/wasm/scala.wasm +0 -0
- package/wasm/solidity.tags.scm +43 -0
- package/wasm/solidity.wasm +0 -0
- package/wasm/tsx.wasm +0 -0
- package/wasm/typescript.tags.scm +23 -0
- package/wasm/typescript.wasm +0 -0
package/src/codeScanner.ts
CHANGED
|
@@ -6,9 +6,11 @@ import { readdir, readFile, stat, lstat, realpath } from 'fs/promises';
|
|
|
6
6
|
import { join, relative, extname, sep } from 'path';
|
|
7
7
|
import { createHash } from 'crypto';
|
|
8
8
|
import ignore, { type Ignore } from 'ignore';
|
|
9
|
-
import { debug as logDebug, warn as logWarn } from './logger.js';
|
|
9
|
+
import { debug as logDebug, info as logInfo, warn as logWarn } from './logger.js';
|
|
10
10
|
import { withTimeout } from './asyncUtils.js';
|
|
11
11
|
import { scrubSecrets } from './secretPatterns.js';
|
|
12
|
+
import { MAX_AST_PARSE_BYTES, chunkWithAst, type AstChunk } from './ast/astChunker.js';
|
|
13
|
+
import { astArtifactReport, resolveAstLanguage } from './ast/grammars.js';
|
|
12
14
|
|
|
13
15
|
/**
|
|
14
16
|
* File operation timeout (5 seconds) to prevent hanging on slow/unresponsive filesystems
|
|
@@ -51,6 +53,11 @@ export interface CodeChunk {
|
|
|
51
53
|
size: number;
|
|
52
54
|
signature?: string; // Function/class signature (e.g. "function foo(bar: string): number")
|
|
53
55
|
symbolName?: string; // Symbol name (e.g. "foo")
|
|
56
|
+
// The AST chunker's finer classification, verbatim from the tag capture
|
|
57
|
+
// ('method', 'interface', 'struct', ...). `chunkType` above is the coarse
|
|
58
|
+
// wire enum and cannot express it; only the AST path sets this, so its
|
|
59
|
+
// absence is exactly the signal "this chunk came from the heuristic path".
|
|
60
|
+
symbolKind?: string;
|
|
54
61
|
};
|
|
55
62
|
}
|
|
56
63
|
|
|
@@ -59,6 +66,13 @@ export interface ScanOptions {
|
|
|
59
66
|
minChunkSize?: number;
|
|
60
67
|
ignorePatterns?: string[];
|
|
61
68
|
includeExtensions?: string[];
|
|
69
|
+
/**
|
|
70
|
+
* Byte ceiling above which a file is chunked heuristically instead of parsed.
|
|
71
|
+
* Defaults to `MAX_AST_PARSE_BYTES`; exposed so a caller (and the routing
|
|
72
|
+
* test) can drive the degradation path through production code rather than a
|
|
73
|
+
* mock.
|
|
74
|
+
*/
|
|
75
|
+
maxAstParseBytes?: number;
|
|
62
76
|
}
|
|
63
77
|
|
|
64
78
|
/**
|
|
@@ -501,6 +515,7 @@ const DEFAULT_OPTIONS: Required<ScanOptions> = {
|
|
|
501
515
|
minChunkSize: 100,
|
|
502
516
|
ignorePatterns: [...BUILT_IN_IGNORE_DIRS, ...BUILT_IN_IGNORE_FILE_PATTERNS],
|
|
503
517
|
includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
|
|
518
|
+
maxAstParseBytes: MAX_AST_PARSE_BYTES,
|
|
504
519
|
};
|
|
505
520
|
|
|
506
521
|
/**
|
|
@@ -613,6 +628,65 @@ interface IgnoreLayer {
|
|
|
613
628
|
}
|
|
614
629
|
type IgnoreStack = IgnoreLayer[];
|
|
615
630
|
|
|
631
|
+
/**
|
|
632
|
+
* Latch for `logAstCapabilityOnce`. A promise, not a boolean: two concurrent
|
|
633
|
+
* callers must both wait on the same report rather than the second returning
|
|
634
|
+
* before the first has logged.
|
|
635
|
+
*/
|
|
636
|
+
let astCapabilityLog: Promise<void> | null = null;
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* `reason:grammar` pairs already warned about, for the reasons that are
|
|
640
|
+
* process-global rather than file-specific.
|
|
641
|
+
*
|
|
642
|
+
* Only `grammar_unavailable` qualifies: `loadGrammar` caches its failure, so the
|
|
643
|
+
* answer is identical for every file of that language and warning per file
|
|
644
|
+
* printed 1,037 lines in one scan of this repo. `file_too_large` and
|
|
645
|
+
* `parse_failed` are properties of one file and stay per file.
|
|
646
|
+
*/
|
|
647
|
+
const reportedGrammarFallbacks = new Set<string>();
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Log which grammars this install ships — ONCE per process, at daemon start,
|
|
651
|
+
* never per file.
|
|
652
|
+
*
|
|
653
|
+
* A missing grammar silently degrades every file of that language to the
|
|
654
|
+
* heuristic chunker. Per-file warnings would say so 30,000 times and drown the
|
|
655
|
+
* log; saying nothing is how a half-broken install looks healthy. One startup
|
|
656
|
+
* line naming the vendored grammars, and a WARNING when an artifact is missing,
|
|
657
|
+
* is the whole contract.
|
|
658
|
+
*
|
|
659
|
+
* Deliberately a `statSync` of the artifacts (`astArtifactReport`) and not a
|
|
660
|
+
* load of them (`astCapabilityReport`). Loading all 18 to print this line costs
|
|
661
|
+
* ~690 ms and ~75 MB of RSS that is never returned — web-tree-sitter exposes no
|
|
662
|
+
* `Language.delete` — which is a permanent tax on a daemon watching a pure
|
|
663
|
+
* TypeScript repo, paid to pre-answer a question about seventeen languages it
|
|
664
|
+
* will never see. Grammars load lazily instead, on the first file of a language,
|
|
665
|
+
* and the two failure modes only a load can detect (`wasm_load_failed`,
|
|
666
|
+
* `query_compile_failed`) are warned there, once per language, by the
|
|
667
|
+
* `grammar_unavailable` branch in `chunkFile`.
|
|
668
|
+
*
|
|
669
|
+
* Cheap to call repeatedly: this latches, and the report instantiates nothing.
|
|
670
|
+
*/
|
|
671
|
+
export function logAstCapabilityOnce(): Promise<void> {
|
|
672
|
+
astCapabilityLog ??= (async () => {
|
|
673
|
+
const report = astArtifactReport();
|
|
674
|
+
const detail = {
|
|
675
|
+
grammars: report.vendored.length,
|
|
676
|
+
languages: report.vendored.join(' '),
|
|
677
|
+
};
|
|
678
|
+
if (report.missing.length > 0) {
|
|
679
|
+
logWarn('AST chunking: some vendored grammar artifacts are missing', {
|
|
680
|
+
...detail,
|
|
681
|
+
missing: report.missing.map((m) => `${m.id}(${m.reason}: ${m.detail})`).join('; '),
|
|
682
|
+
});
|
|
683
|
+
} else {
|
|
684
|
+
logInfo('AST chunking ready (grammars load lazily, per language)', detail);
|
|
685
|
+
}
|
|
686
|
+
})();
|
|
687
|
+
return astCapabilityLog;
|
|
688
|
+
}
|
|
689
|
+
|
|
616
690
|
export class CodeScanner {
|
|
617
691
|
private options: Required<ScanOptions>;
|
|
618
692
|
|
|
@@ -1144,14 +1218,131 @@ export class CodeScanner {
|
|
|
1144
1218
|
return this.chunkMarkdown(content, relativePath, stats.size);
|
|
1145
1219
|
}
|
|
1146
1220
|
|
|
1147
|
-
const structuredChunks = this.extractStructuredChunks(content, language);
|
|
1148
|
-
|
|
1149
1221
|
const fileMetadata = {
|
|
1150
1222
|
fileName: filePath.split('/').pop() || '',
|
|
1151
1223
|
extension: extname(filePath),
|
|
1152
1224
|
size: stats.size,
|
|
1153
1225
|
};
|
|
1154
1226
|
|
|
1227
|
+
// AST first for the languages with a vendored grammar. `null` means "no
|
|
1228
|
+
// grammar for this" — the honest majority of the allowlist — and the
|
|
1229
|
+
// heuristic path below takes over with no log, because that is not a
|
|
1230
|
+
// degradation.
|
|
1231
|
+
const astLanguage = resolveAstLanguage(fileMetadata.extension, language);
|
|
1232
|
+
if (astLanguage) {
|
|
1233
|
+
const ast = await chunkWithAst(content, astLanguage, {
|
|
1234
|
+
maxParseBytes: this.options.maxAstParseBytes,
|
|
1235
|
+
});
|
|
1236
|
+
if ('chunks' in ast) {
|
|
1237
|
+
// A definition the chunker could not place reaches no chunk, so its
|
|
1238
|
+
// name reaches no index and a citation to it resolves as
|
|
1239
|
+
// `unresolved_symbol`. It cannot be emitted (two chunks over one line
|
|
1240
|
+
// range collide on the `filePath:startLine-endLine` staleness key), but
|
|
1241
|
+
// a scan that quietly loses symbols must not look like a healthy one.
|
|
1242
|
+
if (ast.droppedDefinitions > 0) {
|
|
1243
|
+
logWarn('AST chunker dropped definitions that share a line range', {
|
|
1244
|
+
filePath: relativePath,
|
|
1245
|
+
grammar: astLanguage,
|
|
1246
|
+
droppedDefinitions: ast.droppedDefinitions,
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
// `errorNodes` is handed back precisely so this degradation is not
|
|
1250
|
+
// silent, and discarding it made it silent. One ERROR node can swallow
|
|
1251
|
+
// the rest of a file — `typeof import(...)` in a type argument does
|
|
1252
|
+
// exactly that to tree-sitter-typescript — after which no definition is
|
|
1253
|
+
// captured and the file arrives as one unnamed whole-file chunk. Naming
|
|
1254
|
+
// the file, the error count and how many definitions survived is what
|
|
1255
|
+
// makes that visible in a scan log.
|
|
1256
|
+
//
|
|
1257
|
+
// Deliberately NOT a fallback to the heuristic path: measured on the
|
|
1258
|
+
// real construct (tests/AgentInjectionDeliveryWiring.test.ts), the
|
|
1259
|
+
// heuristic path names nothing either (its name pattern needs a
|
|
1260
|
+
// leading `function`/`class`/`const`) AND covers less — it emitted 3
|
|
1261
|
+
// chunks starting at line 18 and dropped lines 1-17, because the
|
|
1262
|
+
// uncovered-region supplement only fires above 50 lines. Trading
|
|
1263
|
+
// total coverage for no gain is worse than one honest blob, so the
|
|
1264
|
+
// AST result stands and the log says so.
|
|
1265
|
+
if (ast.errorNodes > 0) {
|
|
1266
|
+
logWarn('AST parse errors; extents and symbol names degraded for this file', {
|
|
1267
|
+
filePath: relativePath,
|
|
1268
|
+
grammar: astLanguage,
|
|
1269
|
+
errorNodes: ast.errorNodes,
|
|
1270
|
+
definitions: ast.chunks.filter((chunk) => chunk.symbolKind).length,
|
|
1271
|
+
chunks: ast.chunks.length,
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
// The wire cap (`scanChunkSchema` allows 500,000 chars) and the
|
|
1275
|
+
// embedding cap (8191 tokens per input) are both downstream of
|
|
1276
|
+
// `maxChunkSize`, and the AST chunker only bounds its uncovered spans:
|
|
1277
|
+
// a definition chunk is whatever the definition is. `boundAstChunks`
|
|
1278
|
+
// makes the cap unreachable by construction, or answers null when the
|
|
1279
|
+
// file cannot be bounded on line boundaries at all.
|
|
1280
|
+
const bounded = this.boundAstChunks(ast.chunks);
|
|
1281
|
+
if (bounded) {
|
|
1282
|
+
// Zero chunks here means the file has no non-whitespace line — the AST
|
|
1283
|
+
// chunker covers every other line by construction — so returning
|
|
1284
|
+
// nothing is the correct answer, not a lost file.
|
|
1285
|
+
return bounded.map(({ symbolName, symbolKind, signature, ...chunk }) => ({
|
|
1286
|
+
...chunk,
|
|
1287
|
+
filePath: relativePath,
|
|
1288
|
+
language,
|
|
1289
|
+
contentHash: this.hash(chunk.content),
|
|
1290
|
+
metadata: {
|
|
1291
|
+
...fileMetadata,
|
|
1292
|
+
...(signature ? { signature } : {}),
|
|
1293
|
+
...(symbolName ? { symbolName } : {}),
|
|
1294
|
+
...(symbolKind ? { symbolKind } : {}),
|
|
1295
|
+
},
|
|
1296
|
+
}));
|
|
1297
|
+
}
|
|
1298
|
+
// One line longer than `maxChunkSize` — a generated `*_pb2.py` carries
|
|
1299
|
+
// the whole serialized descriptor on one. Splitting it would have to
|
|
1300
|
+
// cut mid-line, and two chunks over one line range collide on the
|
|
1301
|
+
// `filePath:startLine-endLine` staleness key, so the whole file goes to
|
|
1302
|
+
// the heuristic chunker, which force-splits long lines by character
|
|
1303
|
+
// count. Loud, because a file that loses its symbol names must not look
|
|
1304
|
+
// like a healthy one.
|
|
1305
|
+
logWarn('AST chunk exceeds maxChunkSize on a single line; heuristic chunking instead', {
|
|
1306
|
+
filePath: relativePath,
|
|
1307
|
+
grammar: astLanguage,
|
|
1308
|
+
maxChunkSize: this.options.maxChunkSize,
|
|
1309
|
+
longestLine: content
|
|
1310
|
+
.split('\n')
|
|
1311
|
+
.reduce((longest, line) => Math.max(longest, line.length), 0),
|
|
1312
|
+
});
|
|
1313
|
+
} else if (ast.unsupported === 'grammar_unavailable') {
|
|
1314
|
+
// Process-global, not per file: `loadGrammar` caches its failures, so
|
|
1315
|
+
// every file of this language degrades for the same reason. Warned per
|
|
1316
|
+
// file, this printed 1,037 identical lines in one scan of this repo —
|
|
1317
|
+
// the log flood `logAstCapabilityOnce` exists to avoid. Once per
|
|
1318
|
+
// (reason, grammar); the file-specific reasons below stay per file.
|
|
1319
|
+
const key = `${ast.unsupported}:${astLanguage}`;
|
|
1320
|
+
if (!reportedGrammarFallbacks.has(key)) {
|
|
1321
|
+
reportedGrammarFallbacks.add(key);
|
|
1322
|
+
logWarn('AST chunking unavailable for a whole language; heuristic chunking instead', {
|
|
1323
|
+
grammar: astLanguage,
|
|
1324
|
+
reason: ast.unsupported,
|
|
1325
|
+
detail: ast.detail,
|
|
1326
|
+
firstFile: relativePath,
|
|
1327
|
+
note: 'logged once per grammar per process; every file of this language degrades',
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
} else {
|
|
1331
|
+
// Not a silent no-op: name the file AND the reason, then degrade to the
|
|
1332
|
+
// heuristic path. A degraded scan indistinguishable from a healthy one
|
|
1333
|
+
// is the defect class this routing exists to avoid — and degrading to
|
|
1334
|
+
// zero chunks would delete the file from the index instead.
|
|
1335
|
+
logWarn('AST chunking unavailable; falling back to heuristic chunking', {
|
|
1336
|
+
filePath: relativePath,
|
|
1337
|
+
grammar: astLanguage,
|
|
1338
|
+
reason: ast.unsupported,
|
|
1339
|
+
detail: ast.detail,
|
|
1340
|
+
});
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
const structuredChunks = this.extractStructuredChunks(content, language);
|
|
1345
|
+
|
|
1155
1346
|
if (structuredChunks.length > 0) {
|
|
1156
1347
|
const mapped = structuredChunks.map(({ signature, symbolName, ...chunk }) => ({
|
|
1157
1348
|
...chunk,
|
|
@@ -1201,6 +1392,85 @@ export class CodeScanner {
|
|
|
1201
1392
|
}
|
|
1202
1393
|
}
|
|
1203
1394
|
|
|
1395
|
+
/**
|
|
1396
|
+
* Hold every AST chunk at or under `maxChunkSize`, or answer `null` when this
|
|
1397
|
+
* file cannot be held there on line boundaries.
|
|
1398
|
+
*
|
|
1399
|
+
* The AST chunker bounds only the spans it invents (`MAX_RAW_SPAN_CHARS`); a
|
|
1400
|
+
* chunk that came from a definition is exactly as big as the definition, and a
|
|
1401
|
+
* span that is one enormous line is left whole on purpose. Both used to reach
|
|
1402
|
+
* the wire verbatim, where `scanChunkSchema` caps content at 500,000 chars and
|
|
1403
|
+
* rejects the WHOLE push if any chunk is over — so one generated file stopped
|
|
1404
|
+
* all scanning for that push — and where anything past 8191 tokens is only
|
|
1405
|
+
* partially embedded.
|
|
1406
|
+
*
|
|
1407
|
+
* The invariants, all of which the split preserves:
|
|
1408
|
+
* - content stays verbatim contiguous source: `lines[startLine-1..endLine-1]`
|
|
1409
|
+
* joined by '\n', never a join of disjoint regions and never an elision;
|
|
1410
|
+
* - total coverage is unchanged: the pieces tile the original range, in order,
|
|
1411
|
+
* with no gap and no overlap;
|
|
1412
|
+
* - the definition's identity rides on the piece that carries its signature —
|
|
1413
|
+
* the first one — and the continuations are plain `raw` spans, because a
|
|
1414
|
+
* continuation is not the definition and must not claim its name.
|
|
1415
|
+
*
|
|
1416
|
+
* `null` (the caller degrades the whole file to the heuristic chunker) is
|
|
1417
|
+
* reserved for the one shape a line-boundary split cannot fix: a single line
|
|
1418
|
+
* longer than the cap, as every generated `*_pb2.py` has. Cutting mid-line
|
|
1419
|
+
* would give two chunks the same `filePath:startLine-endLine` staleness key,
|
|
1420
|
+
* which is the collision the chunker refuses everywhere else.
|
|
1421
|
+
*/
|
|
1422
|
+
private boundAstChunks(chunks: readonly AstChunk[]): AstChunk[] | null {
|
|
1423
|
+
const max = this.options.maxChunkSize;
|
|
1424
|
+
if (chunks.every((chunk) => chunk.content.length <= max)) return [...chunks];
|
|
1425
|
+
|
|
1426
|
+
const bounded: AstChunk[] = [];
|
|
1427
|
+
for (const chunk of chunks) {
|
|
1428
|
+
if (chunk.content.length <= max) {
|
|
1429
|
+
bounded.push(chunk);
|
|
1430
|
+
continue;
|
|
1431
|
+
}
|
|
1432
|
+
// `content` is exactly `startLine..endLine` joined by '\n', so this split
|
|
1433
|
+
// recovers the file's own lines for that range.
|
|
1434
|
+
const lines = chunk.content.split('\n');
|
|
1435
|
+
if (lines.some((line) => line.length > max)) return null;
|
|
1436
|
+
|
|
1437
|
+
/** `from`/`to` are 0-based offsets into `lines`, inclusive. */
|
|
1438
|
+
const emit = (from: number, to: number): void => {
|
|
1439
|
+
const content = lines.slice(from, to + 1).join('\n');
|
|
1440
|
+
// A piece of nothing but blank lines is owed no chunk (the chunker's own
|
|
1441
|
+
// rule) and `scanChunkSchema` requires content.min(1) anyway.
|
|
1442
|
+
if (content.trim() === '') return;
|
|
1443
|
+
const isFirst = from === 0;
|
|
1444
|
+
bounded.push({
|
|
1445
|
+
content,
|
|
1446
|
+
startLine: chunk.startLine + from,
|
|
1447
|
+
endLine: chunk.startLine + to,
|
|
1448
|
+
chunkType: isFirst ? chunk.chunkType : 'raw',
|
|
1449
|
+
...(isFirst && chunk.symbolName ? { symbolName: chunk.symbolName } : {}),
|
|
1450
|
+
...(isFirst && chunk.symbolKind ? { symbolKind: chunk.symbolKind } : {}),
|
|
1451
|
+
...(isFirst && chunk.signature ? { signature: chunk.signature } : {}),
|
|
1452
|
+
});
|
|
1453
|
+
};
|
|
1454
|
+
|
|
1455
|
+
let pieceStart = 0;
|
|
1456
|
+
let pieceLength = 0;
|
|
1457
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1458
|
+
const lineLength = (lines[i] ?? '').length;
|
|
1459
|
+
// The '\n' the join will put back is part of what has to fit.
|
|
1460
|
+
const withLine = i === pieceStart ? lineLength : pieceLength + 1 + lineLength;
|
|
1461
|
+
if (i > pieceStart && withLine > max) {
|
|
1462
|
+
emit(pieceStart, i - 1);
|
|
1463
|
+
pieceStart = i;
|
|
1464
|
+
pieceLength = lineLength;
|
|
1465
|
+
} else {
|
|
1466
|
+
pieceLength = withLine;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
emit(pieceStart, lines.length - 1);
|
|
1470
|
+
}
|
|
1471
|
+
return bounded;
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1204
1474
|
/**
|
|
1205
1475
|
* Detect language from file extension
|
|
1206
1476
|
*/
|
package/src/index.ts
CHANGED
|
@@ -20,9 +20,31 @@ export {
|
|
|
20
20
|
isFixturePath,
|
|
21
21
|
isSecretFile,
|
|
22
22
|
isAuthorityOnlyPath,
|
|
23
|
+
logAstCapabilityOnce,
|
|
23
24
|
type CodeChunk,
|
|
24
25
|
type ScanOptions,
|
|
25
26
|
} from './codeScanner.js';
|
|
27
|
+
export {
|
|
28
|
+
AST_LANGUAGE_IDS,
|
|
29
|
+
QUERY_CHAIN,
|
|
30
|
+
VENDORED_ARTIFACT_DIR,
|
|
31
|
+
astArtifactReport,
|
|
32
|
+
astCapabilityReport,
|
|
33
|
+
grammarArtifactBasename,
|
|
34
|
+
loadGrammar,
|
|
35
|
+
loadedGrammarIds,
|
|
36
|
+
resolveAstLanguage,
|
|
37
|
+
type AstLanguageId,
|
|
38
|
+
type GrammarBinding,
|
|
39
|
+
type GrammarLoad,
|
|
40
|
+
type GrammarUnavailableReason,
|
|
41
|
+
} from './ast/grammars.js';
|
|
42
|
+
export {
|
|
43
|
+
MAX_AST_PARSE_BYTES,
|
|
44
|
+
chunkWithAst,
|
|
45
|
+
type AstChunk,
|
|
46
|
+
type AstChunkResult,
|
|
47
|
+
} from './ast/astChunker.js';
|
|
26
48
|
export {
|
|
27
49
|
SECRET_PATTERNS,
|
|
28
50
|
SECRET_REDACTION_PLACEHOLDER,
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
; Hand-written: tree-sitter-bash ships no queries/tags.scm.
|
|
2
|
+
;
|
|
3
|
+
; Node names verified by parsing a sample with the vendored bash.wasm rather than
|
|
4
|
+
; taken from documentation — `function_definition` carries its name in a `name`
|
|
5
|
+
; field typed `word`, covering both `foo() { }` and `function foo { }`.
|
|
6
|
+
(function_definition
|
|
7
|
+
name: (word) @name) @definition.function
|
package/wasm/bash.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
(class_declaration name: (identifier) @name) @definition.class
|
|
2
|
+
|
|
3
|
+
(class_declaration (base_list (_) @name)) @reference.class
|
|
4
|
+
|
|
5
|
+
(interface_declaration name: (identifier) @name) @definition.interface
|
|
6
|
+
|
|
7
|
+
(interface_declaration (base_list (_) @name)) @reference.interface
|
|
8
|
+
|
|
9
|
+
(method_declaration name: (identifier) @name) @definition.method
|
|
10
|
+
|
|
11
|
+
(object_creation_expression type: (identifier) @name) @reference.class
|
|
12
|
+
|
|
13
|
+
(type_parameter_constraints_clause (identifier) @name) @reference.class
|
|
14
|
+
|
|
15
|
+
(type_parameter_constraint (type type: (identifier) @name)) @reference.class
|
|
16
|
+
|
|
17
|
+
(variable_declaration type: (identifier) @name) @reference.class
|
|
18
|
+
|
|
19
|
+
(invocation_expression function: (member_access_expression name: (identifier) @name)) @reference.send
|
|
20
|
+
|
|
21
|
+
(namespace_declaration name: (identifier) @name) @definition.module
|
|
22
|
+
|
|
23
|
+
(namespace_declaration name: (identifier) @name) @module
|
|
Binary file
|
package/wasm/c.tags.scm
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
|
|
2
|
+
|
|
3
|
+
(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
|
|
4
|
+
|
|
5
|
+
(function_declarator declarator: (identifier) @name) @definition.function
|
|
6
|
+
|
|
7
|
+
(type_definition declarator: (type_identifier) @name) @definition.type
|
|
8
|
+
|
|
9
|
+
(enum_specifier name: (type_identifier) @name) @definition.type
|
package/wasm/c.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
|
|
2
|
+
|
|
3
|
+
(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
|
|
4
|
+
|
|
5
|
+
(function_declarator declarator: (identifier) @name) @definition.function
|
|
6
|
+
|
|
7
|
+
(function_declarator declarator: (field_identifier) @name) @definition.function
|
|
8
|
+
|
|
9
|
+
(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @local.scope name: (identifier) @name)) @definition.method
|
|
10
|
+
|
|
11
|
+
(type_definition declarator: (type_identifier) @name) @definition.type
|
|
12
|
+
|
|
13
|
+
(enum_specifier name: (type_identifier) @name) @definition.type
|
|
14
|
+
|
|
15
|
+
(class_specifier name: (type_identifier) @name) @definition.class
|
package/wasm/cpp.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
; Definitions
|
|
2
|
+
|
|
3
|
+
; * modules and protocols
|
|
4
|
+
(call
|
|
5
|
+
target: (identifier) @ignore
|
|
6
|
+
(arguments (alias) @name)
|
|
7
|
+
(#any-of? @ignore "defmodule" "defprotocol")) @definition.module
|
|
8
|
+
|
|
9
|
+
; * functions/macros
|
|
10
|
+
(call
|
|
11
|
+
target: (identifier) @ignore
|
|
12
|
+
(arguments
|
|
13
|
+
[
|
|
14
|
+
; zero-arity functions with no parentheses
|
|
15
|
+
(identifier) @name
|
|
16
|
+
; regular function clause
|
|
17
|
+
(call target: (identifier) @name)
|
|
18
|
+
; function clause with a guard clause
|
|
19
|
+
(binary_operator
|
|
20
|
+
left: (call target: (identifier) @name)
|
|
21
|
+
operator: "when")
|
|
22
|
+
])
|
|
23
|
+
(#any-of? @ignore "def" "defp" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp")) @definition.function
|
|
24
|
+
|
|
25
|
+
; References
|
|
26
|
+
|
|
27
|
+
; ignore calls to kernel/special-forms keywords
|
|
28
|
+
(call
|
|
29
|
+
target: (identifier) @ignore
|
|
30
|
+
(#any-of? @ignore "def" "defp" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp" "defmodule" "defprotocol" "defimpl" "defstruct" "defexception" "defoverridable" "alias" "case" "cond" "else" "for" "if" "import" "quote" "raise" "receive" "require" "reraise" "super" "throw" "try" "unless" "unquote" "unquote_splicing" "use" "with"))
|
|
31
|
+
|
|
32
|
+
; ignore module attributes
|
|
33
|
+
(unary_operator
|
|
34
|
+
operator: "@"
|
|
35
|
+
operand: (call
|
|
36
|
+
target: (identifier) @ignore))
|
|
37
|
+
|
|
38
|
+
; * function call
|
|
39
|
+
(call
|
|
40
|
+
target: [
|
|
41
|
+
; local
|
|
42
|
+
(identifier) @name
|
|
43
|
+
; remote
|
|
44
|
+
(dot
|
|
45
|
+
right: (identifier) @name)
|
|
46
|
+
]) @reference.call
|
|
47
|
+
|
|
48
|
+
; * pipe into function call
|
|
49
|
+
(binary_operator
|
|
50
|
+
operator: "|>"
|
|
51
|
+
right: (identifier) @name) @reference.call
|
|
52
|
+
|
|
53
|
+
; * modules
|
|
54
|
+
(alias) @name @reference.module
|
package/wasm/elixir.wasm
ADDED
|
Binary file
|
package/wasm/go.tags.scm
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
(
|
|
2
|
+
(comment)* @doc
|
|
3
|
+
.
|
|
4
|
+
(function_declaration
|
|
5
|
+
name: (identifier) @name) @definition.function
|
|
6
|
+
(#strip! @doc "^//\\s*")
|
|
7
|
+
(#set-adjacent! @doc @definition.function)
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
(
|
|
11
|
+
(comment)* @doc
|
|
12
|
+
.
|
|
13
|
+
(method_declaration
|
|
14
|
+
name: (field_identifier) @name) @definition.method
|
|
15
|
+
(#strip! @doc "^//\\s*")
|
|
16
|
+
(#set-adjacent! @doc @definition.method)
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
(call_expression
|
|
20
|
+
function: [
|
|
21
|
+
(identifier) @name
|
|
22
|
+
(parenthesized_expression (identifier) @name)
|
|
23
|
+
(selector_expression field: (field_identifier) @name)
|
|
24
|
+
(parenthesized_expression (selector_expression field: (field_identifier) @name))
|
|
25
|
+
]) @reference.call
|
|
26
|
+
|
|
27
|
+
(type_spec
|
|
28
|
+
name: (type_identifier) @name) @definition.type
|
|
29
|
+
|
|
30
|
+
(type_identifier) @name @reference.type
|
|
31
|
+
|
|
32
|
+
(package_clause "package" (package_identifier) @name)
|
|
33
|
+
|
|
34
|
+
(type_declaration (type_spec name: (type_identifier) @name type: (interface_type)))
|
|
35
|
+
|
|
36
|
+
(type_declaration (type_spec name: (type_identifier) @name type: (struct_type)))
|
|
37
|
+
|
|
38
|
+
(import_declaration (import_spec) @name)
|
|
39
|
+
|
|
40
|
+
(var_declaration (var_spec name: (identifier) @name))
|
|
41
|
+
|
|
42
|
+
(const_declaration (const_spec name: (identifier) @name))
|
package/wasm/go.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
; Hand-written: tree-sitter-groovy ships no queries/tags.scm.
|
|
2
|
+
;
|
|
3
|
+
; Node names verified against the vendored groovy.wasm. Groovy distinguishes a
|
|
4
|
+
; top-level `function_definition` from a `method_declaration` inside a
|
|
5
|
+
; `class_body`, and both name via an `identifier` field — so methods get
|
|
6
|
+
; definition.method and keep their own name, which is the defect this plan
|
|
7
|
+
; exists to fix for every language.
|
|
8
|
+
(class_declaration
|
|
9
|
+
name: (identifier) @name) @definition.class
|
|
10
|
+
|
|
11
|
+
(method_declaration
|
|
12
|
+
name: (identifier) @name) @definition.method
|
|
13
|
+
|
|
14
|
+
(function_definition
|
|
15
|
+
name: (identifier) @name) @definition.function
|
package/wasm/groovy.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
(class_declaration
|
|
2
|
+
name: (identifier) @name) @definition.class
|
|
3
|
+
|
|
4
|
+
(method_declaration
|
|
5
|
+
name: (identifier) @name) @definition.method
|
|
6
|
+
|
|
7
|
+
(method_invocation
|
|
8
|
+
name: (identifier) @name
|
|
9
|
+
arguments: (argument_list) @reference.call)
|
|
10
|
+
|
|
11
|
+
(interface_declaration
|
|
12
|
+
name: (identifier) @name) @definition.interface
|
|
13
|
+
|
|
14
|
+
(type_list
|
|
15
|
+
(type_identifier) @name) @reference.implementation
|
|
16
|
+
|
|
17
|
+
(object_creation_expression
|
|
18
|
+
type: (type_identifier) @name) @reference.class
|
|
19
|
+
|
|
20
|
+
(superclass (type_identifier) @name) @reference.class
|
package/wasm/java.wasm
ADDED
|
Binary file
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
(
|
|
2
|
+
(comment)* @doc
|
|
3
|
+
.
|
|
4
|
+
(method_definition
|
|
5
|
+
name: (property_identifier) @name) @definition.method
|
|
6
|
+
(#not-eq? @name "constructor")
|
|
7
|
+
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
|
|
8
|
+
(#select-adjacent! @doc @definition.method)
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
(
|
|
12
|
+
(comment)* @doc
|
|
13
|
+
.
|
|
14
|
+
[
|
|
15
|
+
(class
|
|
16
|
+
name: (_) @name)
|
|
17
|
+
(class_declaration
|
|
18
|
+
name: (_) @name)
|
|
19
|
+
] @definition.class
|
|
20
|
+
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
|
|
21
|
+
(#select-adjacent! @doc @definition.class)
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
(
|
|
25
|
+
(comment)* @doc
|
|
26
|
+
.
|
|
27
|
+
[
|
|
28
|
+
(function_expression
|
|
29
|
+
name: (identifier) @name)
|
|
30
|
+
(function_declaration
|
|
31
|
+
name: (identifier) @name)
|
|
32
|
+
(generator_function
|
|
33
|
+
name: (identifier) @name)
|
|
34
|
+
(generator_function_declaration
|
|
35
|
+
name: (identifier) @name)
|
|
36
|
+
] @definition.function
|
|
37
|
+
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
|
|
38
|
+
(#select-adjacent! @doc @definition.function)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
(
|
|
42
|
+
(comment)* @doc
|
|
43
|
+
.
|
|
44
|
+
(lexical_declaration
|
|
45
|
+
(variable_declarator
|
|
46
|
+
name: (identifier) @name
|
|
47
|
+
value: [(arrow_function) (function_expression)]) @definition.function)
|
|
48
|
+
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
|
|
49
|
+
(#select-adjacent! @doc @definition.function)
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
(
|
|
53
|
+
(comment)* @doc
|
|
54
|
+
.
|
|
55
|
+
(variable_declaration
|
|
56
|
+
(variable_declarator
|
|
57
|
+
name: (identifier) @name
|
|
58
|
+
value: [(arrow_function) (function_expression)]) @definition.function)
|
|
59
|
+
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
|
|
60
|
+
(#select-adjacent! @doc @definition.function)
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
(assignment_expression
|
|
64
|
+
left: [
|
|
65
|
+
(identifier) @name
|
|
66
|
+
(member_expression
|
|
67
|
+
property: (property_identifier) @name)
|
|
68
|
+
]
|
|
69
|
+
right: [(arrow_function) (function_expression)]
|
|
70
|
+
) @definition.function
|
|
71
|
+
|
|
72
|
+
(pair
|
|
73
|
+
key: (property_identifier) @name
|
|
74
|
+
value: [(arrow_function) (function_expression)]) @definition.function
|
|
75
|
+
|
|
76
|
+
(
|
|
77
|
+
(call_expression
|
|
78
|
+
function: (identifier) @name) @reference.call
|
|
79
|
+
(#not-match? @name "^(require)$")
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
(call_expression
|
|
83
|
+
function: (member_expression
|
|
84
|
+
property: (property_identifier) @name)
|
|
85
|
+
arguments: (_) @reference.call)
|
|
86
|
+
|
|
87
|
+
(new_expression
|
|
88
|
+
constructor: (_) @name) @reference.class
|
|
89
|
+
|
|
90
|
+
(export_statement value: (assignment_expression left: (identifier) @name right: ([
|
|
91
|
+
(number)
|
|
92
|
+
(string)
|
|
93
|
+
(identifier)
|
|
94
|
+
(undefined)
|
|
95
|
+
(null)
|
|
96
|
+
(new_expression)
|
|
97
|
+
(binary_expression)
|
|
98
|
+
(call_expression)
|
|
99
|
+
]))) @definition.constant
|
|
Binary file
|