@quolu/lattice 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/sensor/NOTICE +8 -19
- package/sensor/dist/bin/lattice-sensor.js +31 -0
- package/sensor/dist/db/index.d.ts +114 -0
- package/sensor/dist/db/index.d.ts.map +1 -1
- package/sensor/dist/db/index.js +235 -7
- package/sensor/dist/db/index.js.map +1 -1
- package/sensor/dist/db/migrations.js +1 -1
- package/sensor/dist/db/migrations.js.map +1 -1
- package/sensor/dist/db/queries.d.ts +46 -8
- package/sensor/dist/db/queries.d.ts.map +1 -1
- package/sensor/dist/db/queries.js +99 -16
- package/sensor/dist/db/queries.js.map +1 -1
- package/sensor/dist/db/wal-valve.d.ts +15 -2
- package/sensor/dist/db/wal-valve.d.ts.map +1 -1
- package/sensor/dist/db/wal-valve.js +88 -6
- package/sensor/dist/db/wal-valve.js.map +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts.map +1 -1
- package/sensor/dist/extraction/extraction-version.js +5 -1
- package/sensor/dist/extraction/extraction-version.js.map +1 -1
- package/sensor/dist/extraction/function-ref.d.ts.map +1 -1
- package/sensor/dist/extraction/function-ref.js +6 -0
- package/sensor/dist/extraction/function-ref.js.map +1 -1
- package/sensor/dist/extraction/grammars.d.ts.map +1 -1
- package/sensor/dist/extraction/grammars.js +46 -0
- package/sensor/dist/extraction/grammars.js.map +1 -1
- package/sensor/dist/extraction/index.d.ts +11 -1
- package/sensor/dist/extraction/index.d.ts.map +1 -1
- package/sensor/dist/extraction/index.js +68 -16
- package/sensor/dist/extraction/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/decode.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/decode.js +9 -3
- package/sensor/dist/extraction/kernel/decode.js.map +1 -1
- package/sensor/dist/extraction/kernel/index.d.ts +2 -0
- package/sensor/dist/extraction/kernel/index.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/index.js +122 -8
- package/sensor/dist/extraction/kernel/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/layout.d.ts +10 -1
- package/sensor/dist/extraction/kernel/layout.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/layout.js +11 -2
- package/sensor/dist/extraction/kernel/layout.js.map +1 -1
- package/sensor/dist/extraction/kernel/loader.d.ts +42 -0
- package/sensor/dist/extraction/kernel/loader.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/loader.js.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.d.ts +13 -0
- package/sensor/dist/extraction/languages/c-cpp.d.ts.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.js +809 -13
- package/sensor/dist/extraction/languages/c-cpp.js.map +1 -1
- package/sensor/dist/extraction/store-worker.js +29 -0
- package/sensor/dist/extraction/store-worker.js.map +1 -1
- package/sensor/dist/extraction/tree-sitter.d.ts +4 -1
- package/sensor/dist/extraction/tree-sitter.d.ts.map +1 -1
- package/sensor/dist/extraction/tree-sitter.js +20 -4
- package/sensor/dist/extraction/tree-sitter.js.map +1 -1
- package/sensor/dist/extraction/wasm/tree-sitter-c.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-cpp.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-dart.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-kotlin.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-php.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-ruby.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-rust.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-swift.wasm +0 -0
- package/sensor/dist/index.d.ts +12 -1
- package/sensor/dist/index.d.ts.map +1 -1
- package/sensor/dist/index.js +42 -7
- package/sensor/dist/index.js.map +1 -1
- package/sensor/dist/mcp/daemon-registry.d.ts +5 -2
- package/sensor/dist/mcp/daemon-registry.d.ts.map +1 -1
- package/sensor/dist/mcp/daemon-registry.js +5 -2
- package/sensor/dist/mcp/daemon-registry.js.map +1 -1
- package/sensor/dist/mcp/index.d.ts +24 -0
- package/sensor/dist/mcp/index.d.ts.map +1 -1
- package/sensor/dist/mcp/index.js +51 -2
- package/sensor/dist/mcp/index.js.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.d.ts.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.js +3 -1
- package/sensor/dist/mcp/liveness-watchdog.js.map +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts.map +1 -1
- package/sensor/dist/mcp/server-instructions.js +1 -0
- package/sensor/dist/mcp/server-instructions.js.map +1 -1
- package/sensor/dist/mcp/tools.d.ts +48 -0
- package/sensor/dist/mcp/tools.d.ts.map +1 -1
- package/sensor/dist/mcp/tools.js +231 -4
- package/sensor/dist/mcp/tools.js.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.js +561 -127
- package/sensor/dist/resolution/c-fnptr-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.js +80 -11
- package/sensor/dist/resolution/callback-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/import-resolver.d.ts.map +1 -1
- package/sensor/dist/resolution/import-resolver.js +34 -2
- package/sensor/dist/resolution/import-resolver.js.map +1 -1
- package/sensor/dist/resolution/index.d.ts +34 -0
- package/sensor/dist/resolution/index.d.ts.map +1 -1
- package/sensor/dist/resolution/index.js +393 -51
- package/sensor/dist/resolution/index.js.map +1 -1
- package/sensor/dist/resolution/memory-budget.d.ts +48 -0
- package/sensor/dist/resolution/memory-budget.d.ts.map +1 -0
- package/sensor/dist/resolution/memory-budget.js +162 -0
- package/sensor/dist/resolution/memory-budget.js.map +1 -0
- package/sensor/dist/resolution/name-matcher.d.ts +4 -7
- package/sensor/dist/resolution/name-matcher.d.ts.map +1 -1
- package/sensor/dist/resolution/name-matcher.js +256 -91
- package/sensor/dist/resolution/name-matcher.js.map +1 -1
- package/sensor/dist/resolution/resolver-pool.d.ts +41 -1
- package/sensor/dist/resolution/resolver-pool.d.ts.map +1 -1
- package/sensor/dist/resolution/resolver-pool.js +103 -3
- package/sensor/dist/resolution/resolver-pool.js.map +1 -1
- package/sensor/dist/resolution/resolver-worker.js +27 -0
- package/sensor/dist/resolution/resolver-worker.js.map +1 -1
- package/sensor/dist/sync/watcher.d.ts +8 -1
- package/sensor/dist/sync/watcher.d.ts.map +1 -1
- package/sensor/dist/sync/watcher.js +44 -2
- package/sensor/dist/sync/watcher.js.map +1 -1
- package/sensor/dist/types.d.ts +7 -3
- package/sensor/dist/types.d.ts.map +1 -1
- package/src/cli-help.mjs +16 -0
- package/src/sensor-adapter.mjs +4 -0
- package/src/sensor-cli.mjs +37 -5
- package/src/sensor-diff.mjs +661 -0
|
@@ -8,10 +8,23 @@ exports.blankCppInlineMacros = blankCppInlineMacros;
|
|
|
8
8
|
exports.recoverMangledCppName = recoverMangledCppName;
|
|
9
9
|
exports.blankMetalAttributes = blankMetalAttributes;
|
|
10
10
|
exports.blankCppAnnotationMacroCalls = blankCppAnnotationMacroCalls;
|
|
11
|
+
exports.blankLoneMacroLines = blankLoneMacroLines;
|
|
11
12
|
exports.blankCppApiPrefixMacros = blankCppApiPrefixMacros;
|
|
12
13
|
exports.blankCppInlineAnnotationMacros = blankCppInlineAnnotationMacros;
|
|
13
14
|
exports.blankCudaConstructs = blankCudaConstructs;
|
|
14
15
|
exports.blankCLeadingAttrMacros = blankCLeadingAttrMacros;
|
|
16
|
+
exports.blankCCplusplusGuardBodies = blankCCplusplusGuardBodies;
|
|
17
|
+
exports.blankCStatementMacroCalls = blankCStatementMacroCalls;
|
|
18
|
+
exports.blankCSandwichedAnnotations = blankCSandwichedAnnotations;
|
|
19
|
+
exports.blankCAutoInference = blankCAutoInference;
|
|
20
|
+
exports.blankCTrailingParamAttrMacros = blankCTrailingParamAttrMacros;
|
|
21
|
+
exports.blankCKernelAnnotations = blankCKernelAnnotations;
|
|
22
|
+
exports.blankCParameterizedAnnotationMacros = blankCParameterizedAnnotationMacros;
|
|
23
|
+
exports.blankCTypeKeywordArgs = blankCTypeKeywordArgs;
|
|
24
|
+
exports.blankCFileScopePrefixedDeclMacros = blankCFileScopePrefixedDeclMacros;
|
|
25
|
+
exports.rewriteCPrefixedDeclMacroInitializers = rewriteCPrefixedDeclMacroInitializers;
|
|
26
|
+
exports.blankCVaArgQualifiedTypeArgs = blankCVaArgQualifiedTypeArgs;
|
|
27
|
+
exports.blankCNamedVariadicDefineDots = blankCNamedVariadicDefineDots;
|
|
15
28
|
const tree_sitter_helpers_1 = require("../tree-sitter-helpers");
|
|
16
29
|
/**
|
|
17
30
|
* Find the function NAME's `qualified_identifier` (`Foo::bar`) inside a
|
|
@@ -554,6 +567,69 @@ function blankCppAnnotationMacroCalls(source) {
|
|
|
554
567
|
}
|
|
555
568
|
return chars.join('');
|
|
556
569
|
}
|
|
570
|
+
/**
|
|
571
|
+
* Blank a macro that is the ONLY token on its line — no parens, no semicolon:
|
|
572
|
+
* namespace-management macros (`FMT_BEGIN_NAMESPACE`, `FMT_END_EXPORT`,
|
|
573
|
+
* `JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS`), Qt's `Q_OBJECT`, and friends. A
|
|
574
|
+
* bare identifier is not a statement or declaration in C or C++, so
|
|
575
|
+
* tree-sitter drops into error recovery at every one — and since the kernel
|
|
576
|
+
* path defers ANY erroring file to wasm, this single idiom deferred 13/73 fmt
|
|
577
|
+
* files and a comparable share of jemalloc, forfeiting the native-parse win
|
|
578
|
+
* on exactly the header-heavy trees it targets (the wasm path also mis-nests
|
|
579
|
+
* scopes around them today). Replacing the token with equal-length spaces
|
|
580
|
+
* preserves every byte offset and the surrounding declarations parse clean.
|
|
581
|
+
*
|
|
582
|
+
* Matched tightly so a real identifier can never be touched — ALL of:
|
|
583
|
+
* - the line consists of ONE ALL-CAPS token (≥4 chars, with `_`), optionally
|
|
584
|
+
* followed by a same-line comment — a lone lowercase identifier or any
|
|
585
|
+
* second token disqualifies;
|
|
586
|
+
* - the PREVIOUS non-blank line does not end in a continuation character
|
|
587
|
+
* (`=`, an operator, `,`, `(`, `?`, `:`, or a `\` macro-definition
|
|
588
|
+
* continuation) — so an ALL-CAPS operand split onto its own line inside a
|
|
589
|
+
* multi-line expression (`int x =\n SOME_CONST\n | OTHER;`) is left
|
|
590
|
+
* alone; and
|
|
591
|
+
* - the NEXT non-blank line starts like a declaration/scope token
|
|
592
|
+
* (letter, `_`, `#`, `{`, `}`, or `~`) or the file ends — an operator,
|
|
593
|
+
* string literal, or `;` continuation rejects the match.
|
|
594
|
+
* Shared by C and C++ (the idiom is identical in both).
|
|
595
|
+
*/
|
|
596
|
+
const LONE_MACRO_LINE_RE = /^[ \t]*([A-Z][A-Z0-9_]{3,})[ \t]*(?:\/\/[^\n\r]*|\/\*[^\n\r]*\*\/[ \t]*)?\r?$/;
|
|
597
|
+
const LONE_MACRO_CONTINUATION_END_RE = /[=+\-*/%&|^<>?:,(\\]$/;
|
|
598
|
+
function blankLoneMacroLines(source) {
|
|
599
|
+
if (!/^[ \t]*[A-Z][A-Z0-9_]{3,}[ \t]*\r?$/m.test(source))
|
|
600
|
+
return source;
|
|
601
|
+
const lines = source.split('\n');
|
|
602
|
+
const content = (l) => l.replace(/\r$/, '').trim();
|
|
603
|
+
let changed = false;
|
|
604
|
+
for (let i = 0; i < lines.length; i++) {
|
|
605
|
+
const line = lines[i];
|
|
606
|
+
const m = LONE_MACRO_LINE_RE.exec(line);
|
|
607
|
+
if (!m)
|
|
608
|
+
continue;
|
|
609
|
+
// Underscore requirement rides the macro convention (FMT_BEGIN_NAMESPACE,
|
|
610
|
+
// Q_OBJECT); a solid all-caps word (`NDEBUG`-style) alone is too risky.
|
|
611
|
+
if (!m[1].includes('_'))
|
|
612
|
+
continue;
|
|
613
|
+
let prev = i - 1;
|
|
614
|
+
while (prev >= 0 && content(lines[prev]) === '')
|
|
615
|
+
prev--;
|
|
616
|
+
if (prev >= 0 && LONE_MACRO_CONTINUATION_END_RE.test(content(lines[prev])))
|
|
617
|
+
continue;
|
|
618
|
+
let next = i + 1;
|
|
619
|
+
while (next < lines.length && content(lines[next]) === '')
|
|
620
|
+
next++;
|
|
621
|
+
if (next < lines.length) {
|
|
622
|
+
const first = content(lines[next])[0];
|
|
623
|
+
if (!first || !/[A-Za-z_#{}~]/.test(first))
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
const start = line.indexOf(m[1]);
|
|
627
|
+
lines[i] =
|
|
628
|
+
line.slice(0, start) + ' '.repeat(m[1].length) + line.slice(start + m[1].length);
|
|
629
|
+
changed = true;
|
|
630
|
+
}
|
|
631
|
+
return changed ? lines.join('\n') : source;
|
|
632
|
+
}
|
|
557
633
|
/**
|
|
558
634
|
* Blank an export/visibility macro sitting in front of a *member* or *method*
|
|
559
635
|
* declaration inside a class/namespace (`ENGINE_API virtual void Tick(…)`,
|
|
@@ -727,21 +803,57 @@ function looksLikeCudaSource(source) {
|
|
|
727
803
|
source.indexOf('__constant__') !== -1 ||
|
|
728
804
|
source.indexOf('cudaStream_t') !== -1);
|
|
729
805
|
}
|
|
806
|
+
/**
|
|
807
|
+
* Restore preprocessor-directive lines to their original bytes after the
|
|
808
|
+
* blanking passes ran. The token-level blanks match on shape, not context, so
|
|
809
|
+
* a macro name that happens to sit inside a DIRECTIVE gets blanked too — and
|
|
810
|
+
* blanking the name position of `#define FMT_API FMT_VISIBILITY("default")`
|
|
811
|
+
* leaves a nameless `# define FMT_VISIBILITY(…)`, which is a parse
|
|
812
|
+
* ERROR (fmt's base.h carries several). Inside a directive the blanks were
|
|
813
|
+
* never useful anyway: tree-sitter stores `#define` bodies as raw
|
|
814
|
+
* preproc_arg text it doesn't parse, so blanking there can only ever break
|
|
815
|
+
* the directive itself. Copying the original directive lines back (including
|
|
816
|
+
* `\`-continuation lines of multi-line defines) is offset-preserving by
|
|
817
|
+
* construction and strictly reduces parse errors on both extraction arms.
|
|
818
|
+
*/
|
|
819
|
+
function restoreDirectiveLines(original, blanked) {
|
|
820
|
+
if (blanked === original || original.indexOf('#') === -1)
|
|
821
|
+
return blanked;
|
|
822
|
+
const o = original.split('\n');
|
|
823
|
+
const b = blanked.split('\n');
|
|
824
|
+
let changed = false;
|
|
825
|
+
let continuation = false;
|
|
826
|
+
for (let i = 0; i < o.length && i < b.length; i++) {
|
|
827
|
+
const line = o[i];
|
|
828
|
+
const isDirective = continuation || /^[ \t]*#/.test(line);
|
|
829
|
+
if (isDirective && b[i] !== line) {
|
|
830
|
+
b[i] = line;
|
|
831
|
+
changed = true;
|
|
832
|
+
}
|
|
833
|
+
continuation = isDirective && /\\\s*$/.test(line.replace(/\r$/, ''));
|
|
834
|
+
}
|
|
835
|
+
return changed ? b.join('\n') : blanked;
|
|
836
|
+
}
|
|
730
837
|
/** C/C++ source pre-processing before tree-sitter: recover macro-annotated class
|
|
731
838
|
* definitions, macro-prefixed function definitions, macro-prefixed members, and
|
|
732
839
|
* macro-decorated members (Unreal-Engine reflection markup) — plus the non-C++
|
|
733
840
|
* surface of the dialects parsed with the C++ grammar: `.metal` MSL attribute
|
|
734
841
|
* annotations, and CUDA specifiers + launch syntax (by `.cu`/`.cuh` extension
|
|
735
|
-
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving
|
|
842
|
+
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving;
|
|
843
|
+
* directive lines are restored at the end (see restoreDirectiveLines). */
|
|
736
844
|
function preParseCppSource(source, filePath) {
|
|
737
|
-
|
|
845
|
+
// blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked
|
|
846
|
+
// `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape
|
|
847
|
+
// it matches (the _API token is already spaces by then).
|
|
848
|
+
let blanked = blankLoneMacroLines(blankCLeadingAttrMacros(blankCppAnnotationMacroCalls(blankCppInlineAnnotationMacros(blankCppApiPrefixMacros(blankCppInlineMacros(blankCppExportMacros(source)))))));
|
|
738
849
|
const lower = filePath ? filePath.toLowerCase() : '';
|
|
739
|
-
if (lower.endsWith('.metal'))
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
850
|
+
if (lower.endsWith('.metal')) {
|
|
851
|
+
blanked = blankMetalAttributes(blanked);
|
|
852
|
+
}
|
|
853
|
+
else if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
|
|
854
|
+
blanked = blankCudaConstructs(blanked);
|
|
743
855
|
}
|
|
744
|
-
return blanked;
|
|
856
|
+
return restoreDirectiveLines(source, blanked);
|
|
745
857
|
}
|
|
746
858
|
/**
|
|
747
859
|
* Blank an unknown attribute macro sitting in front of a C function
|
|
@@ -772,13 +884,697 @@ const C_LEADING_ATTR_MACRO_RE = /^([ \t]*)([A-Z][A-Z0-9_]{2,})(?=\s+[A-Za-z_]\w*
|
|
|
772
884
|
function blankCLeadingAttrMacros(source) {
|
|
773
885
|
return source.replace(C_LEADING_ATTR_MACRO_RE, (_m, ws, macro) => ws + ' '.repeat(macro.length));
|
|
774
886
|
}
|
|
775
|
-
/**
|
|
776
|
-
*
|
|
777
|
-
*
|
|
778
|
-
*
|
|
887
|
+
/**
|
|
888
|
+
* Blank the body of `#ifdef __cplusplus … #endif` guard regions in C sources.
|
|
889
|
+
* The ubiquitous C-header compatibility idiom
|
|
890
|
+
*
|
|
891
|
+
* #ifdef __cplusplus
|
|
892
|
+
* extern "C" {
|
|
893
|
+
* #endif
|
|
894
|
+
*
|
|
895
|
+
* is NOT valid C — `extern "C" {` (and any other C++-only line under the
|
|
896
|
+
* guard) drops tree-sitter-c into error recovery, so effectively every public
|
|
897
|
+
* C header carries parse errors. The wasm path shrugs (recovery keeps the
|
|
898
|
+
* rest); the kernel path defers EVERY erroring file to wasm by policy — so
|
|
899
|
+
* this one idiom pushed C-header deferral to ~32% on redis (vs the <10%
|
|
900
|
+
* gate) and forfeited the native-parse win exactly where C repos have the
|
|
901
|
+
* most files. A C compiler never sees the guarded lines (`__cplusplus` is
|
|
902
|
+
* only defined for C++), so blanking the region BODY mirrors the
|
|
903
|
+
* preprocessor's own view of the file.
|
|
904
|
+
*
|
|
905
|
+
* Matched conservatively, line-based and offset-preserving:
|
|
906
|
+
* - the opener must be `#ifdef __cplusplus` / `#if defined(__cplusplus)`;
|
|
907
|
+
* - the body may contain NO other preprocessor directive (a nested `#if`,
|
|
908
|
+
* `#else`, or `#define` bails the whole region — those need real
|
|
909
|
+
* preprocessing, so the file keeps its current behavior);
|
|
910
|
+
* - the region must close with `#endif` within a few lines (guards are
|
|
911
|
+
* tiny; a giant region is something else).
|
|
912
|
+
* The `#ifdef`/`#endif` directive lines themselves are kept — an empty
|
|
913
|
+
* preproc_ifdef parses clean — and every blanked byte becomes a space with
|
|
914
|
+
* `\r` preserved, so offsets, lines, and columns survive on CRLF checkouts.
|
|
915
|
+
*/
|
|
916
|
+
const C_CPLUSPLUS_GUARD_OPEN_RE = /^[ \t]*#[ \t]*(?:ifdef[ \t]+__cplusplus\b|if[ \t]+defined[ \t]*\(?[ \t]*__cplusplus[ \t]*\)?)/;
|
|
917
|
+
const C_PREPROC_DIRECTIVE_RE = /^[ \t]*#/;
|
|
918
|
+
const C_PREPROC_ENDIF_RE = /^[ \t]*#[ \t]*endif\b/;
|
|
919
|
+
const C_CPLUSPLUS_GUARD_MAX_BODY_LINES = 40;
|
|
920
|
+
function blankCCplusplusGuardBodies(source) {
|
|
921
|
+
if (source.indexOf('__cplusplus') === -1)
|
|
922
|
+
return source;
|
|
923
|
+
const lines = source.split('\n');
|
|
924
|
+
const stripCr = (l) => (l.endsWith('\r') ? l.slice(0, -1) : l);
|
|
925
|
+
let changed = false;
|
|
926
|
+
for (let i = 0; i < lines.length; i++) {
|
|
927
|
+
if (!C_CPLUSPLUS_GUARD_OPEN_RE.test(stripCr(lines[i])))
|
|
928
|
+
continue;
|
|
929
|
+
let end = -1;
|
|
930
|
+
for (let j = i + 1; j < lines.length && j - i - 1 <= C_CPLUSPLUS_GUARD_MAX_BODY_LINES; j++) {
|
|
931
|
+
const line = stripCr(lines[j]);
|
|
932
|
+
if (C_PREPROC_ENDIF_RE.test(line)) {
|
|
933
|
+
end = j;
|
|
934
|
+
break;
|
|
935
|
+
}
|
|
936
|
+
if (C_PREPROC_DIRECTIVE_RE.test(line))
|
|
937
|
+
break; // nested directive — bail
|
|
938
|
+
}
|
|
939
|
+
if (end < 0)
|
|
940
|
+
continue;
|
|
941
|
+
for (let k = i + 1; k < end; k++) {
|
|
942
|
+
lines[k] = lines[k].replace(/[^\r]/g, ' ');
|
|
943
|
+
}
|
|
944
|
+
changed = true;
|
|
945
|
+
i = end;
|
|
946
|
+
}
|
|
947
|
+
return changed ? lines.join('\n') : source;
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* Blank a C iterator-macro call in STATEMENT position — `ql_foreach(iter,
|
|
951
|
+
* &arena->tcache_ql, link) { … }` (jemalloc), `for_each_string_list_item(item,
|
|
952
|
+
* &list) { … }` (git), `list_for_each_entry(pos, head, member) { … }` (the
|
|
953
|
+
* Linux kernel's core iteration idiom). A call followed by a brace block is
|
|
954
|
+
* not a C statement, so tree-sitter-c drops into error recovery at every use —
|
|
955
|
+
* these macros are the single largest source of parse errors in macro-heavy C
|
|
956
|
+
* trees (git: ~39% of files error; the kernel path defers each one to wasm).
|
|
957
|
+
* Blanking JUST the macro call leaves the brace block as a bare compound
|
|
958
|
+
* statement — valid C — so the body's calls/locals extract normally on both
|
|
959
|
+
* arms instead of riding error recovery.
|
|
960
|
+
*
|
|
961
|
+
* C-ONLY, and matched tightly:
|
|
962
|
+
* - the call must be INDENTED (statement position; file-scope definitions
|
|
963
|
+
* start at column 0, and an unbraced file-scope `name(args) { }` is a
|
|
964
|
+
* valid implicit-int function definition that must not be touched);
|
|
965
|
+
* - lowercase-led identifier (iterator macros are lowercase by convention;
|
|
966
|
+
* this also excludes constructors if the file is really C++) that is not a
|
|
967
|
+
* control keyword;
|
|
968
|
+
* - the parens balance ON the line (string literals skipped), and after
|
|
969
|
+
* them only `{` or end-of-line may follow — a `;` (a real call statement),
|
|
970
|
+
* an operator, or any other token disqualifies;
|
|
971
|
+
* - when the line ends at `)`, the NEXT non-blank line must begin with `{`.
|
|
972
|
+
* C++ deliberately does NOT get this pass: an indented snake_case
|
|
973
|
+
* constructor (`basic_string_view(const Char* s) : … {`) is exactly this
|
|
974
|
+
* shape, and blanking it would corrupt every STL-style class.
|
|
975
|
+
*/
|
|
976
|
+
const C_STMT_MACRO_KEYWORDS = new Set([
|
|
977
|
+
'if', 'while', 'for', 'switch', 'return', 'do', 'else', 'sizeof',
|
|
978
|
+
]);
|
|
979
|
+
function blankCStatementMacroCalls(source) {
|
|
980
|
+
const lines = source.split('\n');
|
|
981
|
+
let changed = false;
|
|
982
|
+
const content = (l) => l.replace(/\r$/, '').trim();
|
|
983
|
+
for (let i = 0; i < lines.length; i++) {
|
|
984
|
+
const line = lines[i];
|
|
985
|
+
const m = /^[ \t]+([a-z_][a-z0-9_]*)[ \t]*\(/.exec(line);
|
|
986
|
+
if (!m || C_STMT_MACRO_KEYWORDS.has(m[1]))
|
|
987
|
+
continue;
|
|
988
|
+
const open = line.indexOf('(', m[0].length - 1);
|
|
989
|
+
let depth = 0;
|
|
990
|
+
let close = -1;
|
|
991
|
+
for (let k = open; k < line.length; k++) {
|
|
992
|
+
const ch = line[k];
|
|
993
|
+
if (ch === '"' || ch === "'") {
|
|
994
|
+
const quote = ch;
|
|
995
|
+
k++;
|
|
996
|
+
while (k < line.length && line[k] !== quote) {
|
|
997
|
+
if (line[k] === '\\')
|
|
998
|
+
k++;
|
|
999
|
+
k++;
|
|
1000
|
+
}
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
if (ch === '(')
|
|
1004
|
+
depth++;
|
|
1005
|
+
else if (ch === ')') {
|
|
1006
|
+
depth--;
|
|
1007
|
+
if (depth === 0) {
|
|
1008
|
+
close = k;
|
|
1009
|
+
break;
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
let endLine = i;
|
|
1014
|
+
if (close < 0) {
|
|
1015
|
+
// Parens don't balance on the head line — the kernel wraps iterator
|
|
1016
|
+
// macros (`hlist_for_each_entry_rcu(p, head, hlist,\n\t\t
|
|
1017
|
+
// lockdep_is_held(&kprobe_mutex)) {`). Continue the same
|
|
1018
|
+
// string-skipping paren scan over a few continuation lines. A `;` or
|
|
1019
|
+
// a brace anywhere in the span means a real statement or compound
|
|
1020
|
+
// literal — bail (missing a blank is safe; corrupting one is not).
|
|
1021
|
+
if (line.indexOf(';') !== -1)
|
|
1022
|
+
continue;
|
|
1023
|
+
for (let j = i + 1; j <= i + 5 && j < lines.length && close < 0; j++) {
|
|
1024
|
+
const cont = lines[j];
|
|
1025
|
+
let bail = false;
|
|
1026
|
+
for (let k = 0; k < cont.length; k++) {
|
|
1027
|
+
const ch = cont[k];
|
|
1028
|
+
if (ch === '"' || ch === "'") {
|
|
1029
|
+
const quote = ch;
|
|
1030
|
+
k++;
|
|
1031
|
+
while (k < cont.length && cont[k] !== quote) {
|
|
1032
|
+
if (cont[k] === '\\')
|
|
1033
|
+
k++;
|
|
1034
|
+
k++;
|
|
1035
|
+
}
|
|
1036
|
+
continue;
|
|
1037
|
+
}
|
|
1038
|
+
if (ch === ';' || ch === '{' || ch === '}') {
|
|
1039
|
+
bail = true;
|
|
1040
|
+
break;
|
|
1041
|
+
}
|
|
1042
|
+
if (ch === '(')
|
|
1043
|
+
depth++;
|
|
1044
|
+
else if (ch === ')') {
|
|
1045
|
+
depth--;
|
|
1046
|
+
if (depth === 0) {
|
|
1047
|
+
close = k;
|
|
1048
|
+
endLine = j;
|
|
1049
|
+
break;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
if (bail)
|
|
1054
|
+
break;
|
|
1055
|
+
}
|
|
1056
|
+
if (close < 0)
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
1059
|
+
const endLineStr = lines[endLine];
|
|
1060
|
+
const after = endLineStr.slice(close + 1).replace(/\r$/, '').trim();
|
|
1061
|
+
if (after === '') {
|
|
1062
|
+
// Brace on the next line (`ql_foreach(…)\n{`) or a brace-less
|
|
1063
|
+
// single-statement body (`for_each_subsys(ss, i)\n\tstmt;` — blanking
|
|
1064
|
+
// leaves the bare statement, valid C). A next line starting with an
|
|
1065
|
+
// operator/string/`;` is an expression continuation — bail.
|
|
1066
|
+
let next = endLine + 1;
|
|
1067
|
+
while (next < lines.length && content(lines[next]) === '')
|
|
1068
|
+
next++;
|
|
1069
|
+
if (next >= lines.length)
|
|
1070
|
+
continue;
|
|
1071
|
+
const first = content(lines[next])[0];
|
|
1072
|
+
if (!first || !/[A-Za-z_{]/.test(first))
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
else if (after !== '{') {
|
|
1076
|
+
continue;
|
|
1077
|
+
}
|
|
1078
|
+
const identStart = line.indexOf(m[1]);
|
|
1079
|
+
if (endLine === i) {
|
|
1080
|
+
lines[i] =
|
|
1081
|
+
line.slice(0, identStart) +
|
|
1082
|
+
' '.repeat(close + 1 - identStart) +
|
|
1083
|
+
line.slice(close + 1);
|
|
1084
|
+
}
|
|
1085
|
+
else {
|
|
1086
|
+
lines[i] = line.slice(0, identStart) + line.slice(identStart).replace(/[^\r]/g, ' ');
|
|
1087
|
+
for (let j = i + 1; j < endLine; j++) {
|
|
1088
|
+
lines[j] = lines[j].replace(/[^\r]/g, ' ');
|
|
1089
|
+
}
|
|
1090
|
+
lines[endLine] =
|
|
1091
|
+
endLineStr.slice(0, close + 1).replace(/[^\r]/g, ' ') + endLineStr.slice(close + 1);
|
|
1092
|
+
i = endLine; // the blanked span can't host another head
|
|
1093
|
+
}
|
|
1094
|
+
changed = true;
|
|
1095
|
+
}
|
|
1096
|
+
return changed ? lines.join('\n') : source;
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Blank a lowercase compiler-annotation word SANDWICHED between a storage
|
|
1100
|
+
* class and the rest of a declaration — `static notrace void tick(…)`,
|
|
1101
|
+
* `static nokprobe_inline void arm(…)` (kernel compiler.h markers). The
|
|
1102
|
+
* dunder word-list can't carry these bare-word forms: `notrace` is a
|
|
1103
|
+
* plausible identifier. The sandwich IS the guard — the token counts only
|
|
1104
|
+
* when directly preceded by `static`/`extern`/`inline` AND followed by
|
|
1105
|
+
* another word, a position where it cannot be a variable name (an archaic
|
|
1106
|
+
* implicit-int `static notrace = 1;` fails the following-word requirement).
|
|
1107
|
+
* C-only.
|
|
1108
|
+
*/
|
|
1109
|
+
const C_SANDWICHED_ANNOTATIONS = [
|
|
1110
|
+
'noinline_for_stack', 'nokprobe_inline', 'noinline', 'notrace', 'noinstr',
|
|
1111
|
+
];
|
|
1112
|
+
const C_SANDWICH_RE = new RegExp(`\\b(static|extern|inline)([ \\t]+)(${C_SANDWICHED_ANNOTATIONS.join('|')})\\b(?=[ \\t]+[A-Za-z_])`, 'g');
|
|
1113
|
+
function blankCSandwichedAnnotations(source) {
|
|
1114
|
+
C_SANDWICH_RE.lastIndex = 0;
|
|
1115
|
+
if (!C_SANDWICH_RE.test(source))
|
|
1116
|
+
return source;
|
|
1117
|
+
C_SANDWICH_RE.lastIndex = 0;
|
|
1118
|
+
return source.replace(C_SANDWICH_RE, (_m, storage, ws, ann) => storage + ws + ' '.repeat(ann.length));
|
|
1119
|
+
}
|
|
1120
|
+
/**
|
|
1121
|
+
* Blank a C23 `auto` type-inference keyword — `auto hb = hbr.hb;` (the
|
|
1122
|
+
* futex code; tree-sitter-c predates C23 auto and errors the enclosing
|
|
1123
|
+
* function). The old storage-class reading (`auto int x = 1;`) has a TYPE
|
|
1124
|
+
* between `auto` and the `=` and is untouched — the match requires
|
|
1125
|
+
* `auto IDENT =` directly, which only the C23 form exhibits. Blanking
|
|
1126
|
+
* leaves a plain assignment statement. C-only.
|
|
1127
|
+
*/
|
|
1128
|
+
const C_AUTO_INFER_RE = /\bauto(?=[ \t]+[A-Za-z_]\w*[ \t]*=)/g;
|
|
1129
|
+
function blankCAutoInference(source) {
|
|
1130
|
+
C_AUTO_INFER_RE.lastIndex = 0;
|
|
1131
|
+
if (!C_AUTO_INFER_RE.test(source))
|
|
1132
|
+
return source;
|
|
1133
|
+
C_AUTO_INFER_RE.lastIndex = 0;
|
|
1134
|
+
return source.replace(C_AUTO_INFER_RE, () => ' ');
|
|
1135
|
+
}
|
|
1136
|
+
/**
|
|
1137
|
+
* Blank a trailing parameter-attribute macro — `int argc UNUSED,` /
|
|
1138
|
+
* `struct repository *repo UNUSED)` — git's house style for
|
|
1139
|
+
* `__attribute__((unused))` on nearly every callback parameter (and the same
|
|
1140
|
+
* shape as `MAYBE_UNUSED`/`G_GNUC_UNUSED` elsewhere). tree-sitter-c can't
|
|
1141
|
+
* parse a second identifier after the parameter name, so every such
|
|
1142
|
+
* SIGNATURE drops into error recovery — the single largest deferral bucket
|
|
1143
|
+
* on git (~150 files). Blanking the macro leaves an ordinary parameter.
|
|
1144
|
+
*
|
|
1145
|
+
* Matched tightly: an identifier, whitespace, then an ALL-CAPS ≥3-char token
|
|
1146
|
+
* immediately before `,` or `)`. Two juxtaposed identifiers in that position
|
|
1147
|
+
* have no other valid-C reading — in a CALL the would-be macro is preceded
|
|
1148
|
+
* by `,`/`(`, an operator, or a literal, never by a bare identifier. C-only:
|
|
1149
|
+
* C++ grammars accept more juxtapositions (user-defined suffixes, macro'd
|
|
1150
|
+
* `final`/`override`), so cpp keeps its existing recovery there.
|
|
1151
|
+
*/
|
|
1152
|
+
const C_TRAILING_PARAM_ATTR_RE = /\b([A-Za-z_]\w*)([ \t]+)([A-Z][A-Z0-9_]{2,})(?=[ \t]*[,)])/g;
|
|
1153
|
+
function blankCTrailingParamAttrMacros(source) {
|
|
1154
|
+
if (!C_TRAILING_PARAM_ATTR_RE.test(source)) {
|
|
1155
|
+
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
|
|
1156
|
+
return source;
|
|
1157
|
+
}
|
|
1158
|
+
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
|
|
1159
|
+
return source.replace(C_TRAILING_PARAM_ATTR_RE, (_m, name, ws, macro) => name + ws + ' '.repeat(macro.length));
|
|
1160
|
+
}
|
|
1161
|
+
/**
|
|
1162
|
+
* Blank the Linux-kernel/sparse declaration-annotation macros — `static int
|
|
1163
|
+
* __init audit_init(void)`, `void __user *buf`, `__bpf_kfunc void f(…)`,
|
|
1164
|
+
* `int x __ro_after_init;`. These lowercase double-underscore annotations sit
|
|
1165
|
+
* between storage/type tokens and the declarator, a position tree-sitter-c
|
|
1166
|
+
* can't reconcile, and they blanket the Linux tree: measured on the kernel's
|
|
1167
|
+
* own `kernel/` + `mm/` subtrees, they are the largest single deferral cause
|
|
1168
|
+
* (the `__init` family alone heads ~37% of erroring files).
|
|
1169
|
+
*
|
|
1170
|
+
* A structural match is IMPOSSIBLE here: `__u32 count` (a real typedef) and
|
|
1171
|
+
* `__init foo` (an annotation) are byte-shape identical — so unlike the
|
|
1172
|
+
* shape-keyed blanks above, this is a CURATED list (the CPP_INLINE_MACROS
|
|
1173
|
+
* precedent) of well-known sparse/section/compiler annotations that are
|
|
1174
|
+
* reserved-namespace macros in every codebase that spells them. Whole-word,
|
|
1175
|
+
* equal-length spaces, C-only (the C++ grammar's kernel exposure is
|
|
1176
|
+
* negligible and cpp keeps its narrower blank set).
|
|
1177
|
+
*/
|
|
1178
|
+
const C_KERNEL_ANNOTATIONS = [
|
|
1179
|
+
'__init', '__exit', '__initdata', '__initconst', '__exitdata',
|
|
1180
|
+
'__devinit', '__devexit', '__cpuinit', '__meminit', '__meminitdata',
|
|
1181
|
+
'__net_init', '__net_exit', '__net_initdata', '__init_or_module',
|
|
1182
|
+
'__user', '__kernel', '__iomem', '__percpu', '__rcu', '__force', '__nocast',
|
|
1183
|
+
'__must_check', '__maybe_unused', '__always_unused', '__used', '__cold',
|
|
1184
|
+
'__hot', '__weak', '__pure', '__sched', '__malloc', '__visible',
|
|
1185
|
+
'__deprecated', '__ro_after_init', '__read_mostly', '__refdata',
|
|
1186
|
+
'__latent_entropy', '__randomize_layout', '__no_randomize_layout',
|
|
1187
|
+
'__bpf_kfunc', '__function_aligned', '__always_inline', '__noreturn',
|
|
1188
|
+
// Round-2 additions, each measured heading first-error lines on the
|
|
1189
|
+
// v7.2-rc2 kernel/+mm/ census (2026-07-17): cacheline placement, sparse
|
|
1190
|
+
// lock/section markers, and bpf/typecheck annotations. Real dunder TYPES
|
|
1191
|
+
// (`__u32`, `__s64`) and operators (`__alignof__`) are deliberately absent.
|
|
1192
|
+
'__cacheline_aligned_in_smp', '__cacheline_aligned',
|
|
1193
|
+
'__cacheline_internodealigned_in_smp', '____cacheline_aligned_in_smp',
|
|
1194
|
+
'____cacheline_aligned', '____cacheline_internodealigned_in_smp',
|
|
1195
|
+
'__noclone', '__lockfunc', '__ref', '__private', '__bitwise',
|
|
1196
|
+
'__nosavedata', '__no_kcsan', '__cpuidle', '__ksym',
|
|
1197
|
+
'__initdata_memblock', '__initdata_or_meminfo',
|
|
1198
|
+
];
|
|
1199
|
+
// `(?!\s*\()` keeps the parameterized annotations (`__printf(1, 2)`,
|
|
1200
|
+
// `__aligned(8)`, `__section("x")`) intact — blanking just their name would
|
|
1201
|
+
// strand the argument list as a floating parenthesis and CREATE an error.
|
|
1202
|
+
const C_KERNEL_ANNOTATION_RE = new RegExp(`\\b(${[...C_KERNEL_ANNOTATIONS].sort((a, b) => b.length - a.length).join('|')})\\b(?!\\s*\\()`, 'g');
|
|
1203
|
+
function blankCKernelAnnotations(source) {
|
|
1204
|
+
if (source.indexOf('__') === -1)
|
|
1205
|
+
return source;
|
|
1206
|
+
C_KERNEL_ANNOTATION_RE.lastIndex = 0;
|
|
1207
|
+
if (!C_KERNEL_ANNOTATION_RE.test(source))
|
|
1208
|
+
return source;
|
|
1209
|
+
let out = source.replace(C_KERNEL_ANNOTATION_RE, (m) => ' '.repeat(m.length));
|
|
1210
|
+
// `container_of(ptr, struct T, member)` — the type-keyword argument is the
|
|
1211
|
+
// one call shape tree-sitter-c cannot read (a macro taking a TYPE), and it
|
|
1212
|
+
// is pervasive across the Linux tree. Blanking just the `struct`/`union`
|
|
1213
|
+
// keyword leaves `container_of(ptr, T, member)` — a plain
|
|
1214
|
+
// identifier argument the grammar parses natively. Keyed to the macro name
|
|
1215
|
+
// so no other `struct` keyword anywhere is ever touched.
|
|
1216
|
+
if (out.indexOf('container_of') !== -1) {
|
|
1217
|
+
out = out.replace(/(\bcontainer_of\s*\([^;()]*?,\s*)(struct|union)(\s+)/g, (_m, head, kw, ws) => head + ' '.repeat(kw.length) + ws);
|
|
1218
|
+
}
|
|
1219
|
+
return out;
|
|
1220
|
+
}
|
|
1221
|
+
/**
|
|
1222
|
+
* Blank the PARAMETERIZED sparse/compiler annotations whole — name AND
|
|
1223
|
+
* argument list — `struct file *f __free(fput) = NULL;`, `static void
|
|
1224
|
+
* __printf(4, 0) log_it(…)`, `} owners[] __counted_by(count);`,
|
|
1225
|
+
* `__bpf_md_ptr(struct bpf_iter_meta *, meta);`. The word-list blank above
|
|
1226
|
+
* deliberately skips these via `(?!\s*\()` because blanking just the NAME
|
|
1227
|
+
* strands `(args)` as a floating parenthesis — so every such file kept
|
|
1228
|
+
* deferring (the v7.2-rc2 kernel/+mm/ census puts `__free`/`__printf`/
|
|
1229
|
+
* `__counted_by`/`__aligned` at the head of 39 first-error lines). Blanking
|
|
1230
|
+
* the whole `__name(args)` span leaves an ordinary declaration.
|
|
1231
|
+
*
|
|
1232
|
+
* CURATED for the same reason as the word list: `__aligned(8)` (annotation)
|
|
1233
|
+
* and `__hash(key)` (a real static helper call) are byte-shape identical, so
|
|
1234
|
+
* only reserved-namespace names that are annotations in every codebase that
|
|
1235
|
+
* spells them are listed. One nesting level of parens is allowed
|
|
1236
|
+
* (`__aligned(sizeof(struct x))`); newlines inside the span survive so byte
|
|
1237
|
+
* offsets are preserved. C-only.
|
|
1238
|
+
*/
|
|
1239
|
+
const C_PARAMETERIZED_ANNOTATIONS = [
|
|
1240
|
+
'__free', '__printf', '__scanf', '__counted_by', '__counted_by_le',
|
|
1241
|
+
'__counted_by_be', '__guarded_by', '__pt_guarded_by', '__acquires',
|
|
1242
|
+
'__releases', '__must_hold', '__cleanup', '__aligned', '__section',
|
|
1243
|
+
'__bpf_md_ptr', '__assume_aligned',
|
|
1244
|
+
];
|
|
1245
|
+
const C_PARAM_ANNOTATION_RE = new RegExp(`\\b(${[...C_PARAMETERIZED_ANNOTATIONS].sort((a, b) => b.length - a.length).join('|')})[ \\t]*\\((?:[^()]|\\([^()]*\\))*\\)`, 'g');
|
|
1246
|
+
function blankCParameterizedAnnotationMacros(source) {
|
|
1247
|
+
if (source.indexOf('__') === -1)
|
|
1248
|
+
return source;
|
|
1249
|
+
C_PARAM_ANNOTATION_RE.lastIndex = 0;
|
|
1250
|
+
if (!C_PARAM_ANNOTATION_RE.test(source))
|
|
1251
|
+
return source;
|
|
1252
|
+
C_PARAM_ANNOTATION_RE.lastIndex = 0;
|
|
1253
|
+
let result = '';
|
|
1254
|
+
let last = 0;
|
|
1255
|
+
let m;
|
|
1256
|
+
while ((m = C_PARAM_ANNOTATION_RE.exec(source)) !== null) {
|
|
1257
|
+
const start = m.index;
|
|
1258
|
+
let end = start + m[0].length;
|
|
1259
|
+
// Field-position form — `__bpf_md_ptr(struct bpf_iter_meta *, meta);` as
|
|
1260
|
+
// the whole line: a lone `;` field is ITSELF a parse error (measured;
|
|
1261
|
+
// an empty struct body is fine), so the semicolon blanks with it. A
|
|
1262
|
+
// mid-line match (`… __free(fput) = NULL;`) keeps its statement tail.
|
|
1263
|
+
const lineStart = source.lastIndexOf('\n', start - 1) + 1;
|
|
1264
|
+
if (/^[ \t]*$/.test(source.slice(lineStart, start))) {
|
|
1265
|
+
const after = /^[ \t]*;/.exec(source.slice(end));
|
|
1266
|
+
if (after)
|
|
1267
|
+
end += after[0].length;
|
|
1268
|
+
}
|
|
1269
|
+
result += source.slice(last, start) + source.slice(start, end).replace(/[^\n\r]/g, ' ');
|
|
1270
|
+
last = end;
|
|
1271
|
+
}
|
|
1272
|
+
return result + source.slice(last);
|
|
1273
|
+
}
|
|
1274
|
+
/**
|
|
1275
|
+
* Blank a bare `struct`/`union`/`enum` TYPE-keyword argument inside a macro
|
|
1276
|
+
* call — `kzalloc_obj(struct bpf_mount_opts)`, `alloc_percpu(struct irqstat)`,
|
|
1277
|
+
* `list_first_entry(&pending,\n\t\tstruct async_entry, domain_list)` — the
|
|
1278
|
+
* `container_of` disease generalized (that blank stays as the keyed
|
|
1279
|
+
* precedent). A bare type is never a valid C expression argument, so
|
|
1280
|
+
* tree-sitter drops into error recovery at every such call; removing just the
|
|
1281
|
+
* keyword leaves a plain identifier argument the grammar parses natively.
|
|
1282
|
+
* This single shape heads ~35 first-error lines on the kernel/+mm/ census
|
|
1283
|
+
* (kzalloc_obj/list_entry/alloc_percpu + multi-line continuations).
|
|
1284
|
+
*
|
|
1285
|
+
* Guarded three ways, because `f(struct T)` has VALID look-alikes:
|
|
1286
|
+
* - head exclusions: `sizeof(struct T)` / `offsetof(struct T, m)` /
|
|
1287
|
+
* `_Generic(x, struct T *: …)` all parse natively (probed) and must keep
|
|
1288
|
+
* their keyword; `va_arg` gets its own dedicated blank below.
|
|
1289
|
+
* - call-vs-declaration: in `int wf(struct a, struct b);` (a wrapped
|
|
1290
|
+
* prototype — valid C) the head is preceded by a TYPE, so any identifier
|
|
1291
|
+
* or `*` before the head rejects the match — except the word `return`,
|
|
1292
|
+
* which precedes real calls. Newline/`=`/`,`/`(`/`;`/`{`/`>` etc. accept.
|
|
1293
|
+
* - the keyword must OPEN a top-level argument and the argument must be
|
|
1294
|
+
* exactly `struct T` (optionally `struct T *`, whose stars blank too —
|
|
1295
|
+
* `DEFINE_PER_CPU(struct task_struct *, ksoftirqd)` leaves two plain
|
|
1296
|
+
* identifier arguments): a cast (`f((struct T *)p)`) opens a nested
|
|
1297
|
+
* group, and a declaration argument (`TP_PROTO(struct foo *bar)`) trails
|
|
1298
|
+
* an extra identifier — neither matches.
|
|
1299
|
+
* A hand-rolled bounded scanner rather than a nested-alternation regex:
|
|
1300
|
+
* preceding args may themselves contain calls
|
|
1301
|
+
* (`hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&d->h)),\n
|
|
1302
|
+
* struct bpf_dtab_netdev, index_hlist)`), and lazy nested regex groups over
|
|
1303
|
+
* untrusted repo text invite catastrophic backtracking. C-only.
|
|
1304
|
+
*/
|
|
1305
|
+
const C_TYPE_ARG_HEAD_EXCLUSIONS = new Set([
|
|
1306
|
+
'sizeof', 'alignof', '_Alignof', 'typeof', '__typeof__', '__typeof',
|
|
1307
|
+
'offsetof', '_Generic', 'va_arg', 'if', 'while', 'for', 'switch', 'case',
|
|
1308
|
+
]);
|
|
1309
|
+
const C_TYPE_ARG_HEAD_RE = /\b([A-Za-z_]\w*)[ \t]*\(/g;
|
|
1310
|
+
const C_TYPE_ARG_OPENER_RE = /^(struct|union|enum)([ \t\r\n]+)([A-Za-z_]\w*)([ \t\r\n]*\*+)?(?=[ \t\r\n]*[,)])/;
|
|
1311
|
+
const C_TYPE_ARG_SCAN_CAP = 600;
|
|
1312
|
+
function blankCTypeKeywordArgs(source) {
|
|
1313
|
+
if (!/\b(?:struct|union|enum)[ \t\r\n]/.test(source))
|
|
1314
|
+
return source;
|
|
1315
|
+
let chars = null;
|
|
1316
|
+
C_TYPE_ARG_HEAD_RE.lastIndex = 0;
|
|
1317
|
+
let m;
|
|
1318
|
+
while ((m = C_TYPE_ARG_HEAD_RE.exec(source)) !== null) {
|
|
1319
|
+
const head = m[1];
|
|
1320
|
+
if (C_TYPE_ARG_HEAD_EXCLUSIONS.has(head))
|
|
1321
|
+
continue;
|
|
1322
|
+
// Reject declaration shapes: an identifier or `*` (a return/field type)
|
|
1323
|
+
// immediately before the head — unless that word is `return`.
|
|
1324
|
+
let j = m.index - 1;
|
|
1325
|
+
while (j >= 0 && (source[j] === ' ' || source[j] === '\t'))
|
|
1326
|
+
j--;
|
|
1327
|
+
const prev = j >= 0 ? source[j] : '';
|
|
1328
|
+
if (/[\w*]/.test(prev)) {
|
|
1329
|
+
let w = j;
|
|
1330
|
+
while (w >= 0 && /\w/.test(source[w]))
|
|
1331
|
+
w--;
|
|
1332
|
+
if (source.slice(w + 1, j + 1) !== 'return')
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1335
|
+
const open = m.index + m[0].length - 1;
|
|
1336
|
+
let depth = 1;
|
|
1337
|
+
let atArgStart = true;
|
|
1338
|
+
for (let k = open + 1; k < source.length && k - open <= C_TYPE_ARG_SCAN_CAP; k++) {
|
|
1339
|
+
const ch = source[k];
|
|
1340
|
+
if (ch === '"' || ch === "'") {
|
|
1341
|
+
const quote = ch;
|
|
1342
|
+
k++;
|
|
1343
|
+
while (k < source.length && source[k] !== quote) {
|
|
1344
|
+
if (source[k] === '\\')
|
|
1345
|
+
k++;
|
|
1346
|
+
k++;
|
|
1347
|
+
}
|
|
1348
|
+
atArgStart = false;
|
|
1349
|
+
continue;
|
|
1350
|
+
}
|
|
1351
|
+
if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r')
|
|
1352
|
+
continue;
|
|
1353
|
+
if (ch === '(') {
|
|
1354
|
+
depth++;
|
|
1355
|
+
atArgStart = false;
|
|
1356
|
+
continue;
|
|
1357
|
+
}
|
|
1358
|
+
if (ch === ')') {
|
|
1359
|
+
depth--;
|
|
1360
|
+
if (depth === 0)
|
|
1361
|
+
break;
|
|
1362
|
+
atArgStart = false;
|
|
1363
|
+
continue;
|
|
1364
|
+
}
|
|
1365
|
+
if (ch === ';' || ch === '{' || ch === '}')
|
|
1366
|
+
break; // not an argument list
|
|
1367
|
+
if (ch === ',') {
|
|
1368
|
+
if (depth === 1)
|
|
1369
|
+
atArgStart = true;
|
|
1370
|
+
continue;
|
|
1371
|
+
}
|
|
1372
|
+
if (depth === 1 && atArgStart) {
|
|
1373
|
+
const opener = C_TYPE_ARG_OPENER_RE.exec(source.slice(k, k + 160));
|
|
1374
|
+
if (opener) {
|
|
1375
|
+
chars ??= [...source];
|
|
1376
|
+
const kw = opener[1];
|
|
1377
|
+
for (let b = k; b < k + kw.length; b++)
|
|
1378
|
+
chars[b] = ' ';
|
|
1379
|
+
if (opener[4]) {
|
|
1380
|
+
const tail = opener[4];
|
|
1381
|
+
const stars = /\*+$/.exec(tail)[0].length;
|
|
1382
|
+
const starsStart = k + kw.length + opener[2].length + opener[3].length + (tail.length - stars);
|
|
1383
|
+
for (let b = starsStart; b < starsStart + stars; b++)
|
|
1384
|
+
chars[b] = ' ';
|
|
1385
|
+
}
|
|
1386
|
+
k += opener[0].length - 1;
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
atArgStart = false;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
return chars ? chars.join('') : source;
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Blank a file-scope `static`/`extern`-prefixed ALL-CAPS declaration macro —
|
|
1396
|
+
* `static DEFINE_PER_CPU(struct llist_head, rstat_backlog_list);`,
|
|
1397
|
+
* `static DECLARE_WORK(init_free_wq, do_free_init);`,
|
|
1398
|
+
* `static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);` — the single
|
|
1399
|
+
* largest deferral family on the kernel/+mm/ census (~45 first-error lines
|
|
1400
|
+
* across the DEFINE_/DECLARE_ variants). A storage-class specifier followed
|
|
1401
|
+
* by a call expression is never valid C, so every one errors; the whole line
|
|
1402
|
+
* blanks to spaces (the macro-declared variable was invisible to extraction
|
|
1403
|
+
* anyway, and the file's remaining symbols recover).
|
|
1404
|
+
*
|
|
1405
|
+
* The UNPREFIXED form (`EXPORT_SYMBOL(x);`, `DEFINE_MUTEX(lock);` at column
|
|
1406
|
+
* 0) parses natively as a K&R implicit-int declaration (probed) — extraction
|
|
1407
|
+
* ignores declarations, so those lines are left alone; the type-keyword-arg
|
|
1408
|
+
* blank above already recovers the `DEFINE_PER_CPU(struct T, x);` bare form.
|
|
1409
|
+
* Matched tightly: `static`/`extern` first on the line (indentation allowed —
|
|
1410
|
+
* `static DEFINE_RATELIMIT_STATE(…)` appears at BLOCK scope too, and the
|
|
1411
|
+
* storage-class-plus-call shape is invalid at every scope), CAPS macro name,
|
|
1412
|
+
* parens balancing ON the line (string literals skipped), then exactly `;`
|
|
1413
|
+
* to end of line. Initializer forms (`… ) = { …`) are deliberately not
|
|
1414
|
+
* matched — blanking the head would strand the brace block. C-only.
|
|
1415
|
+
*/
|
|
1416
|
+
const C_PREFIXED_DECL_MACRO_RE = /^[ \t]*(?:static|extern)[ \t]+[A-Z][A-Z0-9_]{2,}[ \t]*\(/;
|
|
1417
|
+
function blankCFileScopePrefixedDeclMacros(source) {
|
|
1418
|
+
if (!/^[ \t]*(?:static|extern)[ \t]+[A-Z]/m.test(source))
|
|
1419
|
+
return source;
|
|
1420
|
+
const lines = source.split('\n');
|
|
1421
|
+
let changed = false;
|
|
1422
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1423
|
+
const line = lines[i];
|
|
1424
|
+
const m = C_PREFIXED_DECL_MACRO_RE.exec(line);
|
|
1425
|
+
if (!m)
|
|
1426
|
+
continue;
|
|
1427
|
+
const open = line.indexOf('(', m[0].length - 1);
|
|
1428
|
+
let depth = 0;
|
|
1429
|
+
let close = -1;
|
|
1430
|
+
for (let k = open; k < line.length; k++) {
|
|
1431
|
+
const ch = line[k];
|
|
1432
|
+
if (ch === '"' || ch === "'") {
|
|
1433
|
+
const quote = ch;
|
|
1434
|
+
k++;
|
|
1435
|
+
while (k < line.length && line[k] !== quote) {
|
|
1436
|
+
if (line[k] === '\\')
|
|
1437
|
+
k++;
|
|
1438
|
+
k++;
|
|
1439
|
+
}
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
if (ch === '(')
|
|
1443
|
+
depth++;
|
|
1444
|
+
else if (ch === ')') {
|
|
1445
|
+
depth--;
|
|
1446
|
+
if (depth === 0) {
|
|
1447
|
+
close = k;
|
|
1448
|
+
break;
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
if (close < 0)
|
|
1453
|
+
continue; // spans lines — leave for a future round
|
|
1454
|
+
if (line.slice(close + 1).replace(/\r$/, '').trim() !== ';')
|
|
1455
|
+
continue;
|
|
1456
|
+
lines[i] = line.replace(/[^\n\r]/g, ' ');
|
|
1457
|
+
changed = true;
|
|
1458
|
+
}
|
|
1459
|
+
return changed ? lines.join('\n') : source;
|
|
1460
|
+
}
|
|
1461
|
+
/**
|
|
1462
|
+
* REWRITE (the one non-blank pass in this family — it moves a token) a
|
|
1463
|
+
* 2-argument `static CAPS_MACRO(type, name) = {`-style initialized
|
|
1464
|
+
* declaration macro into the declaration it expands to — `static
|
|
1465
|
+
* DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {` becomes `static
|
|
1466
|
+
* struct cpuhp_cpu_state cpuhp_state) …` → `static struct
|
|
1467
|
+
* cpuhp_cpu_state` + padding + `cpuhp_state` + padding + `= {`. The blank
|
|
1468
|
+
* family can't help here: blanking the head strands the brace block, and
|
|
1469
|
+
* dropping the whole span would discard the initializer's function
|
|
1470
|
+
* references (`.startup.single = bringup_cpu` — real cFnPtr wiring). The
|
|
1471
|
+
* NAME keeps its exact original column and the `= {` tail keeps its exact
|
|
1472
|
+
* offsets (`d`-flag group indices); only the TYPE token sits left of where
|
|
1473
|
+
* the macro name was, and type tokens contribute strings, not positions.
|
|
1474
|
+
* Only the two-argument `(<type-ish>, <ident>)` form with an `=` tail
|
|
1475
|
+
* matches; three-argument macros and `;`-terminated forms (the blank above)
|
|
1476
|
+
* never do. C-only.
|
|
1477
|
+
*/
|
|
1478
|
+
const C_PREFIXED_DECL_MACRO_INIT_RE = /^([ \t]*)static[ \t]+[A-Z][A-Z0-9_]{2,}[ \t]*\(([^();'"]*?),[ \t]*([A-Za-z_]\w*)[ \t]*\)([ \t]*=[ \t]*\{?[ \t]*\r?)$/d;
|
|
1479
|
+
function rewriteCPrefixedDeclMacroInitializers(source) {
|
|
1480
|
+
if (!/^[ \t]*static[ \t]+[A-Z]/m.test(source))
|
|
1481
|
+
return source;
|
|
1482
|
+
const lines = source.split('\n');
|
|
1483
|
+
let changed = false;
|
|
1484
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1485
|
+
const line = lines[i];
|
|
1486
|
+
const m = C_PREFIXED_DECL_MACRO_INIT_RE.exec(line);
|
|
1487
|
+
if (!m)
|
|
1488
|
+
continue;
|
|
1489
|
+
const indices = m.indices;
|
|
1490
|
+
const arg1 = m[2].trim().replace(/[ \t]+/g, ' ');
|
|
1491
|
+
if (!/^[A-Za-z_][\w \t*]*$/.test(arg1))
|
|
1492
|
+
continue; // a type-token run only
|
|
1493
|
+
const name = m[3];
|
|
1494
|
+
const nameStart = indices[3][0];
|
|
1495
|
+
const tailStart = indices[4][0];
|
|
1496
|
+
const prefix = m[1] + 'static ' + arg1;
|
|
1497
|
+
if (prefix.length + 1 > nameStart)
|
|
1498
|
+
continue; // rewrite must fit left of the name
|
|
1499
|
+
lines[i] =
|
|
1500
|
+
prefix +
|
|
1501
|
+
' '.repeat(nameStart - prefix.length) +
|
|
1502
|
+
name +
|
|
1503
|
+
' '.repeat(tailStart - nameStart - name.length) +
|
|
1504
|
+
line.slice(tailStart);
|
|
1505
|
+
changed = true;
|
|
1506
|
+
}
|
|
1507
|
+
return changed ? lines.join('\n') : source;
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* Blank a QUALIFIED/POINTER type argument of `va_arg` — `va_arg(ap, const
|
|
1511
|
+
* char *)`, `va_arg(args, unsigned long)`. tree-sitter-c parses the
|
|
1512
|
+
* single-token forms (`va_arg(ap, int)`, `va_arg(ap, foo_t)`) natively
|
|
1513
|
+
* (probed), but multi-token type descriptors error the whole enclosing
|
|
1514
|
+
* function. Blanking the comma and the entire second argument leaves
|
|
1515
|
+
* `va_arg(ap )` — a one-argument call the grammar accepts.
|
|
1516
|
+
* Function-pointer types (`va_arg(ap, void (*)(int))`) contain parens and
|
|
1517
|
+
* are deliberately unmatched. C-only.
|
|
1518
|
+
*/
|
|
1519
|
+
const C_VA_ARG_RE = /\bva_arg[ \t]*\(([^(),]+)(,[^()]*)\)/g;
|
|
1520
|
+
function blankCVaArgQualifiedTypeArgs(source) {
|
|
1521
|
+
if (source.indexOf('va_arg') === -1)
|
|
1522
|
+
return source;
|
|
1523
|
+
C_VA_ARG_RE.lastIndex = 0;
|
|
1524
|
+
return source.replace(C_VA_ARG_RE, (m, arg1, rest) => {
|
|
1525
|
+
if (/^,[ \t]*[A-Za-z_]\w*[ \t]*$/.test(rest))
|
|
1526
|
+
return m; // single token — parses natively
|
|
1527
|
+
return `va_arg(${arg1}${rest.replace(/[^\n\r]/g, ' ')})`;
|
|
1528
|
+
});
|
|
1529
|
+
}
|
|
1530
|
+
/**
|
|
1531
|
+
* Blank the DOTS of a GNU NAMED-variadic function-like `#define` parameter —
|
|
1532
|
+
* `#define verbose(env, fmt, args...) …` → `#define verbose(env, fmt,
|
|
1533
|
+
* args ) …`. The `ident...` parameter form (unlike standard `...`) errors
|
|
1534
|
+
* the DIRECTIVE itself (measured — and so does trailing whitespace after a
|
|
1535
|
+
* bare `#define NAME`, which is why the tail is NOT blanked wholesale); with
|
|
1536
|
+
* just the dots blanked the params parse as ordinary identifiers and the
|
|
1537
|
+
* body (`##args` included — the preproc body is opaque to the grammar) is
|
|
1538
|
+
* untouched. `restoreDirectiveLines` exists to keep directives out of the
|
|
1539
|
+
* other blanks' blast radius, so this pass runs AFTER the restore and edits
|
|
1540
|
+
* the directive deliberately. Multi-line bodies are fine — the params always
|
|
1541
|
+
* sit on the `#define` line. C-only (the census hits are the kernel's bpf
|
|
1542
|
+
* verifier headers).
|
|
1543
|
+
*/
|
|
1544
|
+
const C_NAMED_VARIADIC_DEFINE_RE = /^([ \t]*#[ \t]*define[ \t]+[A-Za-z_]\w*\([^()\n]*?\b\w+)\.\.\./;
|
|
1545
|
+
function blankCNamedVariadicDefineDots(source) {
|
|
1546
|
+
if (source.indexOf('...') === -1)
|
|
1547
|
+
return source;
|
|
1548
|
+
const lines = source.split('\n');
|
|
1549
|
+
let changed = false;
|
|
1550
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1551
|
+
const line = lines[i];
|
|
1552
|
+
const m = C_NAMED_VARIADIC_DEFINE_RE.exec(line);
|
|
1553
|
+
if (!m)
|
|
1554
|
+
continue;
|
|
1555
|
+
const keep = m[1];
|
|
1556
|
+
lines[i] = keep + ' ' + line.slice(keep.length + 3);
|
|
1557
|
+
changed = true;
|
|
1558
|
+
}
|
|
1559
|
+
return changed ? lines.join('\n') : source;
|
|
1560
|
+
}
|
|
1561
|
+
/** C source pre-processing: neutralize `#ifdef __cplusplus` compat-guard
|
|
1562
|
+
* bodies (invisible to a C compiler; `extern "C" {` otherwise errors every
|
|
1563
|
+
* public header), blank declaration-markup macro calls and lone macro lines
|
|
1564
|
+
* (`REDIS_NO_SANITIZE("bounds")` before a definition, jemalloc's diagnostic
|
|
1565
|
+
* toggles — the same structural shapes the C++ side already blanks), recover
|
|
1566
|
+
* functions hidden behind a leading attribute macro (#1211), then — for
|
|
1567
|
+
* C-detected headers in CUDA projects (llm.c keeps `__device__` helpers and
|
|
1568
|
+
* kernel prototypes in plain `.h`) — the same content-gated CUDA blank as
|
|
1569
|
+
* C++. Offset-preserving. */
|
|
779
1570
|
function preParseCSource(source) {
|
|
780
|
-
const
|
|
781
|
-
|
|
1571
|
+
const inner = blankCKernelAnnotations(blankCCplusplusGuardBodies(source));
|
|
1572
|
+
let blanked = blankCLeadingAttrMacros(blankLoneMacroLines(blankCStatementMacroCalls(blankCTrailingParamAttrMacros(blankCppAnnotationMacroCalls(rewriteCPrefixedDeclMacroInitializers(blankCFileScopePrefixedDeclMacros(blankCVaArgQualifiedTypeArgs(blankCTypeKeywordArgs(blankCParameterizedAnnotationMacros(blankCAutoInference(blankCSandwichedAnnotations(inner))))))))))));
|
|
1573
|
+
if (looksLikeCudaSource(blanked))
|
|
1574
|
+
blanked = blankCudaConstructs(blanked);
|
|
1575
|
+
// The named-variadic `#define` pass runs AFTER the directive restore — it
|
|
1576
|
+
// deliberately edits directive lines (see its doc comment).
|
|
1577
|
+
return blankCNamedVariadicDefineDots(restoreDirectiveLines(source, blanked));
|
|
782
1578
|
}
|
|
783
1579
|
exports.cppExtractor = {
|
|
784
1580
|
// Recover macro-annotated class/struct definitions (`class MYMODULE_API Foo : Base`,
|