@quolu/lattice 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/sensor/NOTICE +8 -19
- package/sensor/dist/bin/lattice-sensor.js +31 -0
- package/sensor/dist/db/index.d.ts +114 -0
- package/sensor/dist/db/index.d.ts.map +1 -1
- package/sensor/dist/db/index.js +235 -7
- package/sensor/dist/db/index.js.map +1 -1
- package/sensor/dist/db/migrations.js +1 -1
- package/sensor/dist/db/migrations.js.map +1 -1
- package/sensor/dist/db/queries.d.ts +46 -8
- package/sensor/dist/db/queries.d.ts.map +1 -1
- package/sensor/dist/db/queries.js +99 -16
- package/sensor/dist/db/queries.js.map +1 -1
- package/sensor/dist/db/wal-valve.d.ts +15 -2
- package/sensor/dist/db/wal-valve.d.ts.map +1 -1
- package/sensor/dist/db/wal-valve.js +88 -6
- package/sensor/dist/db/wal-valve.js.map +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts.map +1 -1
- package/sensor/dist/extraction/extraction-version.js +5 -1
- package/sensor/dist/extraction/extraction-version.js.map +1 -1
- package/sensor/dist/extraction/function-ref.d.ts.map +1 -1
- package/sensor/dist/extraction/function-ref.js +6 -0
- package/sensor/dist/extraction/function-ref.js.map +1 -1
- package/sensor/dist/extraction/grammars.d.ts.map +1 -1
- package/sensor/dist/extraction/grammars.js +46 -0
- package/sensor/dist/extraction/grammars.js.map +1 -1
- package/sensor/dist/extraction/index.d.ts +11 -1
- package/sensor/dist/extraction/index.d.ts.map +1 -1
- package/sensor/dist/extraction/index.js +68 -16
- package/sensor/dist/extraction/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/decode.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/decode.js +9 -3
- package/sensor/dist/extraction/kernel/decode.js.map +1 -1
- package/sensor/dist/extraction/kernel/index.d.ts +2 -0
- package/sensor/dist/extraction/kernel/index.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/index.js +122 -8
- package/sensor/dist/extraction/kernel/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/layout.d.ts +10 -1
- package/sensor/dist/extraction/kernel/layout.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/layout.js +11 -2
- package/sensor/dist/extraction/kernel/layout.js.map +1 -1
- package/sensor/dist/extraction/kernel/loader.d.ts +42 -0
- package/sensor/dist/extraction/kernel/loader.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/loader.js.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.d.ts +13 -0
- package/sensor/dist/extraction/languages/c-cpp.d.ts.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.js +809 -13
- package/sensor/dist/extraction/languages/c-cpp.js.map +1 -1
- package/sensor/dist/extraction/store-worker.js +29 -0
- package/sensor/dist/extraction/store-worker.js.map +1 -1
- package/sensor/dist/extraction/tree-sitter.d.ts +4 -1
- package/sensor/dist/extraction/tree-sitter.d.ts.map +1 -1
- package/sensor/dist/extraction/tree-sitter.js +20 -4
- package/sensor/dist/extraction/tree-sitter.js.map +1 -1
- package/sensor/dist/extraction/wasm/tree-sitter-c.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-cpp.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-dart.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-kotlin.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-php.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-ruby.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-rust.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-swift.wasm +0 -0
- package/sensor/dist/index.d.ts +12 -1
- package/sensor/dist/index.d.ts.map +1 -1
- package/sensor/dist/index.js +42 -7
- package/sensor/dist/index.js.map +1 -1
- package/sensor/dist/mcp/daemon-registry.d.ts +5 -2
- package/sensor/dist/mcp/daemon-registry.d.ts.map +1 -1
- package/sensor/dist/mcp/daemon-registry.js +5 -2
- package/sensor/dist/mcp/daemon-registry.js.map +1 -1
- package/sensor/dist/mcp/index.d.ts +24 -0
- package/sensor/dist/mcp/index.d.ts.map +1 -1
- package/sensor/dist/mcp/index.js +51 -2
- package/sensor/dist/mcp/index.js.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.d.ts.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.js +3 -1
- package/sensor/dist/mcp/liveness-watchdog.js.map +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts.map +1 -1
- package/sensor/dist/mcp/server-instructions.js +1 -0
- package/sensor/dist/mcp/server-instructions.js.map +1 -1
- package/sensor/dist/mcp/tools.d.ts +48 -0
- package/sensor/dist/mcp/tools.d.ts.map +1 -1
- package/sensor/dist/mcp/tools.js +231 -4
- package/sensor/dist/mcp/tools.js.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.js +561 -127
- package/sensor/dist/resolution/c-fnptr-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.js +80 -11
- package/sensor/dist/resolution/callback-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/import-resolver.d.ts.map +1 -1
- package/sensor/dist/resolution/import-resolver.js +34 -2
- package/sensor/dist/resolution/import-resolver.js.map +1 -1
- package/sensor/dist/resolution/index.d.ts +34 -0
- package/sensor/dist/resolution/index.d.ts.map +1 -1
- package/sensor/dist/resolution/index.js +393 -51
- package/sensor/dist/resolution/index.js.map +1 -1
- package/sensor/dist/resolution/memory-budget.d.ts +48 -0
- package/sensor/dist/resolution/memory-budget.d.ts.map +1 -0
- package/sensor/dist/resolution/memory-budget.js +162 -0
- package/sensor/dist/resolution/memory-budget.js.map +1 -0
- package/sensor/dist/resolution/name-matcher.d.ts +4 -7
- package/sensor/dist/resolution/name-matcher.d.ts.map +1 -1
- package/sensor/dist/resolution/name-matcher.js +256 -91
- package/sensor/dist/resolution/name-matcher.js.map +1 -1
- package/sensor/dist/resolution/resolver-pool.d.ts +41 -1
- package/sensor/dist/resolution/resolver-pool.d.ts.map +1 -1
- package/sensor/dist/resolution/resolver-pool.js +103 -3
- package/sensor/dist/resolution/resolver-pool.js.map +1 -1
- package/sensor/dist/resolution/resolver-worker.js +27 -0
- package/sensor/dist/resolution/resolver-worker.js.map +1 -1
- package/sensor/dist/sync/watcher.d.ts +8 -1
- package/sensor/dist/sync/watcher.d.ts.map +1 -1
- package/sensor/dist/sync/watcher.js +44 -2
- package/sensor/dist/sync/watcher.js.map +1 -1
- package/sensor/dist/types.d.ts +7 -3
- package/sensor/dist/types.d.ts.map +1 -1
- package/src/cli-help.mjs +16 -0
- package/src/sensor-adapter.mjs +4 -0
- package/src/sensor-cli.mjs +37 -5
- package/src/sensor-diff.mjs +661 -0
|
@@ -83,17 +83,55 @@ exports.cFnPointerDispatchEdges = cFnPointerDispatchEdges;
|
|
|
83
83
|
* Whole-graph pass after base resolution; all edges are `provenance:'heuristic'`
|
|
84
84
|
* (`synthesizedBy:'fn-pointer-dispatch'`). High precision via the (type, field)
|
|
85
85
|
* key + a real-function gate; a project with no fn-pointer dispatch is a no-op.
|
|
86
|
+
*
|
|
87
|
+
* ## Fuse-then-link architecture (§7a.8, task #5 step 1)
|
|
88
|
+
*
|
|
89
|
+
* The pass used to sweep every file's text FOUR times (typedefs, registrations,
|
|
90
|
+
* propagation, dispatch), and on the Linux kernel the all-or-nothing source
|
|
91
|
+
* cache declines, so each sweep re-read + re-stripped the whole corpus — 4.4
|
|
92
|
+
* strips/file, ~78s of the ~230s kernel-scale wall (§7a.8 calibration). It now
|
|
93
|
+
* runs as ONE extraction sweep plus filtered linking stages:
|
|
94
|
+
*
|
|
95
|
+
* 1. **Extraction sweep** — reads + strips each file ONCE and collects, per
|
|
96
|
+
* file: typedef names, each struct node's field declarations (parsed
|
|
97
|
+
* structurally, fn-pointer classification deferred — the typedef sets
|
|
98
|
+
* aren't complete mid-sweep), the resolved local includes, and cheap
|
|
99
|
+
* SURVIVAL FILTERS for the later stages (distinct initializer type
|
|
100
|
+
* tokens, array element types, inline-struct summaries, field-assignment
|
|
101
|
+
* field pairs, dispatch field / array names — all interned, a few MB even
|
|
102
|
+
* on the kernel).
|
|
103
|
+
* 2. **Struct-layout linking** — classifies the deferred fields against the
|
|
104
|
+
* now-complete typedef sets and registers layouts by replaying the struct
|
|
105
|
+
* kind-scan, so registration order (which decides same-name layout
|
|
106
|
+
* precedence) is byte-identical to the old dedicated pass.
|
|
107
|
+
* 3. **Registration / propagation / dispatch** — the original pass bodies,
|
|
108
|
+
* UNCHANGED, but each file is first checked against its survival filter
|
|
109
|
+
* and only surviving files are re-stripped (LRU-served). The filters only
|
|
110
|
+
* ever over-approximate: a filtered-out file is one where every match
|
|
111
|
+
* would have failed the pass's own gates before any side effect, so
|
|
112
|
+
* skipping it cannot change the edge set. On the kernel only ~16% of
|
|
113
|
+
* files have any dispatch-shaped match at all, so the lazy re-strips are
|
|
114
|
+
* a fraction of a sweep and total strip work drops ~4.4× → ~1.5×.
|
|
115
|
+
*
|
|
116
|
+
* The extraction sweep is also the step-2 boundary: a native per-file extractor
|
|
117
|
+
* can replace the sweep's scans without touching the linking stages.
|
|
86
118
|
*/
|
|
87
119
|
const path = __importStar(require("node:path"));
|
|
120
|
+
const memory_budget_1 = require("./memory-budget");
|
|
88
121
|
const lru_cache_1 = require("./lru-cache");
|
|
89
122
|
const strip_comments_1 = require("./strip-comments");
|
|
123
|
+
const loader_1 = require("../extraction/kernel/loader");
|
|
90
124
|
const C_CPP_EXT = /\.(c|h|cc|cpp|cxx|hpp|hh|hxx|cppm|ipp|inl|tcc)$/i;
|
|
91
125
|
const FN_KINDS = new Set(['function', 'method']);
|
|
92
126
|
const FANOUT_CAP = 300; // a real command table (git ~150) is legitimate fan-out; this only stops pathological cases.
|
|
93
|
-
|
|
127
|
+
/** Slice a node's body from a pre-split line array — the per-file sweeps
|
|
128
|
+
* call this once per NODE, and splitting the whole file per node was an
|
|
129
|
+
* O(nodes × file-size) term (~1.6M full-file splits on the Linux tree,
|
|
130
|
+
* §7a.3 cFnPtr round). Split once per file, slice many times. */
|
|
131
|
+
function sliceLinesPre(lines, startLine, endLine) {
|
|
94
132
|
if (!startLine)
|
|
95
133
|
return '';
|
|
96
|
-
return
|
|
134
|
+
return lines.slice(startLine - 1, endLine ?? startLine).join('\n');
|
|
97
135
|
}
|
|
98
136
|
/** Index of the `}` matching the `{` at `open` (which must point at a `{`). -1 if unbalanced. */
|
|
99
137
|
function matchBrace(src, open) {
|
|
@@ -338,16 +376,66 @@ const C_TYPE_KEYWORDS = new Set([
|
|
|
338
376
|
const INCLUDE_RE = /#[ \t]*include[ \t]+"([^"\n]+)"/g;
|
|
339
377
|
/** Included files worth scanning for registration tables (e.g. a generated `.def`). */
|
|
340
378
|
const INCLUDABLE_EXT = /\.(def|inc|h|hh|hpp|hxx|c|cc|cpp|cxx|ipp|tcc|tbl)$/i;
|
|
379
|
+
/** `#define NAME single_identifier` (possibly `struct`-prefixed) — an
|
|
380
|
+
* object-macro that COULD alias a struct type name (`resolveTypeName`'s exact
|
|
381
|
+
* value shape). The extraction sweep collects every such NAME into a global
|
|
382
|
+
* set: an initializer type token that direct-misses the struct layouts still
|
|
383
|
+
* survives the registration filter when it is alias-SHAPED anywhere, so the
|
|
384
|
+
* per-file macro-env alias resolution (redis' `COMMAND_STRUCT`) keeps working
|
|
385
|
+
* without retaining per-file object-macro tables (6.1M `#define`s on the
|
|
386
|
+
* Linux tree — the amdgpu register headers — rule that out). Numeric values
|
|
387
|
+
* are excluded: `resolveTypeName` would rewrite to a dead-end token that can
|
|
388
|
+
* never name a struct, so skipping them is exact, and it drops the register
|
|
389
|
+
* flood. */
|
|
390
|
+
const OBJ_ALIAS_RE = /^[ \t]*#[ \t]*define[ \t]+(\w+)[ \t]+(?:struct[ \t]+)*[A-Za-z_]\w*[ \t\r]*$/gm;
|
|
391
|
+
/** `(?:struct )?TYPE name[opt] = {` initializers, where TYPE is a struct that
|
|
392
|
+
* has ≥1 fn-pointer field. Handles both single (`= {…}`) and array
|
|
393
|
+
* (`[] = { {…}, {…} }`) forms. Macro calls inside an element are expanded first. */
|
|
394
|
+
const INIT_RE = /(?:^|[;{}])\s*(?:(?:static|const|extern|register|volatile)\s+)*(?:struct\s+)?(\w+)\s+(\w+)\s*(\[[^\]]*\])?\s*=\s*\{/g;
|
|
395
|
+
/** `struct TAG { … } var[opt] [= {…}]` — the struct is defined INLINE with the
|
|
396
|
+
* table (vim's `cmdname`/`nv_cmd`); its layout never became a node, so parse it
|
|
397
|
+
* here and register it before reading the entries. No leading anchor: a
|
|
398
|
+
* `struct TAG {` with a brace body is always a definition (it may be preceded
|
|
399
|
+
* by a `#define …` line ending in a digit, as in vim), and the trailing
|
|
400
|
+
* `var … = {` check below is what distinguishes a TABLE from a plain type. */
|
|
401
|
+
const INLINE_STRUCT_RE = /\bstruct\s+(\w+)\s*\{/g;
|
|
402
|
+
/** `(?:static …)* ELEMTYPE [*] name[…] = { … }` — a bare array of function
|
|
403
|
+
* pointers (no struct wrapper). The optional `*` covers a function-TYPE
|
|
404
|
+
* typedef element (`opcode_t *opcodes[]`); a function-pointer typedef element
|
|
405
|
+
* (`zend_rc_dtor_func_t t[]`) needs none. The typedef-set membership gate
|
|
406
|
+
* is what separates this from a plain data/struct array. */
|
|
407
|
+
const ARRAY_TABLE_RE = /(?:^|[;{}])\s*(?:(?:static|const|extern|register|volatile)\s+)*(\w+)\s+(\*\s*)?(\w+)\s*\[[^\]]*\]\s*=\s*\{/g;
|
|
408
|
+
/** Dispatch sites: `base->…->field(` or `base.…field(` where `field` is a known
|
|
409
|
+
* fn-pointer field. The base may be a chain (`c->cmd->proc`) or carry array
|
|
410
|
+
* subscripts (`cmdnames[i].cmd_func`). An optional `)` before the call covers
|
|
411
|
+
* the parenthesized form `(cmdnames[i].cmd_func)(&ea)` vim uses. */
|
|
412
|
+
const DISPATCH_RE = /((?:\w+(?:\s*\[[^\][]*\])?\s*(?:->|\.)\s*)+)(\w+)\s*\)?\s*\(/g;
|
|
413
|
+
/** Bare-array dispatch: `tbl[i](…)` or the explicit-deref `(*tbl[i])(…)`. The
|
|
414
|
+
* subscript may itself contain a call (`tbl[GC_TYPE(p)](…)`), so the index
|
|
415
|
+
* class excludes only brackets. Precision comes from the `arrayReg` gate —
|
|
416
|
+
* this fires only when `tbl` is a known fn-pointer array. */
|
|
417
|
+
const ARRAY_DISPATCH_RE = /(?:\(\s*\*\s*)?\b(\w+)\s*\[[^\][]*\]\s*\)?\s*\(/g;
|
|
418
|
+
/** Field←field propagation sites: `a->f = b->g`. */
|
|
419
|
+
const FIELD_ASSIGN_RE = /(\w+)\s*(?:->|\.)\s*(\w+)\s*=\s*(\w+)\s*(?:->|\.)\s*(\w+)/g;
|
|
420
|
+
const NO_INCLUDES = [];
|
|
341
421
|
async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
342
422
|
let scannedFiles = 0;
|
|
343
423
|
const files = ctx.getAllFiles().filter((f) => C_CPP_EXT.test(f));
|
|
344
424
|
if (files.length === 0)
|
|
345
425
|
return [];
|
|
426
|
+
// LATTICE_SENSOR_SYNTH_TIMINGS sub-attribution: this pass is 86% of kernel-scale
|
|
427
|
+
// synthesis (306s, §7a.2/§7a.3) — per-stage walls + read/strip accounting
|
|
428
|
+
// name which stage and which cost class owns it. Post-refactor mapping:
|
|
429
|
+
// A = extraction sweep, B = struct-layout linking, C = registration,
|
|
430
|
+
// D = propagation, E = dispatch.
|
|
431
|
+
const prof = process.env.LATTICE_SENSOR_SYNTH_TIMINGS
|
|
432
|
+
? { A: 0, B: 0, C: 0, D: 0, E: 0, readMs: 0, readN: 0, stripMs: 0, stripN: 0, nodesMs: 0, nodesN: 0 }
|
|
433
|
+
: null;
|
|
346
434
|
// Within-pass progress: this is the pass that parks the "Linking dynamic
|
|
347
435
|
// dispatch" bar on C-heavy repos, so it reports a real fraction of its
|
|
348
|
-
// dominant work. `files` is swept once per
|
|
349
|
-
//
|
|
350
|
-
//
|
|
436
|
+
// dominant work. `files` is swept once per stage loop below (extraction,
|
|
437
|
+
// registration, propagation, dispatch), reported at the same per-16-files
|
|
438
|
+
// cadence as the cooperative yield.
|
|
351
439
|
const FILE_SWEEPS = 4;
|
|
352
440
|
const tick = async () => {
|
|
353
441
|
if ((++scannedFiles & 15) === 0) {
|
|
@@ -358,18 +446,35 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
358
446
|
// Cache raw + stripped source per file, LRU-BOUNDED. The old unbounded Maps
|
|
359
447
|
// retained every C/C++ file's raw AND stripped text for the whole pass —
|
|
360
448
|
// multiple GB on the Linux kernel, one of the two OOM culprits in #1212.
|
|
361
|
-
//
|
|
362
|
-
//
|
|
363
|
-
// hits; a miss just re-reads + re-strips.
|
|
364
|
-
|
|
449
|
+
// The extraction sweep reads sequentially; the linking stages re-request
|
|
450
|
+
// only surviving files (plus include units), so access is near-sequential
|
|
451
|
+
// and a small LRU hits; a miss just re-reads + re-strips.
|
|
452
|
+
// Cache sizing is memory-budget-aware AND all-or-nothing (§7a.3 cFnPtr
|
|
453
|
+
// round): a partial LRU is WORSE than useless for cyclic sweeps (a first
|
|
454
|
+
// attempt sized ~61k against 63.8k files thrashed to a ~0% cross-sweep hit
|
|
455
|
+
// rate). Hold every stripped file (~24KB each measured on the Linux tree)
|
|
456
|
+
// only when 40% of the live memory budget covers it; otherwise keep the
|
|
457
|
+
// within-stage-locality 128. When the big cache declines (the kernel), the
|
|
458
|
+
// survival filters keep the linking stages' re-strips to a fraction of a
|
|
459
|
+
// sweep. Slack over files.length: non-indexed includes (.def/.inc, generated
|
|
460
|
+
// headers) join the working set mid-pass. Pass-scoped transient, freed on
|
|
461
|
+
// return.
|
|
462
|
+
const fullCacheCap = Math.ceil(files.length * 1.05) + 512;
|
|
463
|
+
const cacheCap = (0, memory_budget_1.memoryBudgetBytes)() * 0.5 >= fullCacheCap * 24_576 ? fullCacheCap : 128;
|
|
464
|
+
const rawCache = new lru_cache_1.LRUCache(Math.min(cacheCap, 4096));
|
|
365
465
|
const raw = (file) => {
|
|
366
466
|
if (rawCache.has(file))
|
|
367
467
|
return rawCache.get(file);
|
|
468
|
+
const t0 = prof ? Date.now() : 0;
|
|
368
469
|
const r = ctx.readFile(file);
|
|
470
|
+
if (prof) {
|
|
471
|
+
prof.readMs += Date.now() - t0;
|
|
472
|
+
prof.readN++;
|
|
473
|
+
}
|
|
369
474
|
rawCache.set(file, r);
|
|
370
475
|
return r;
|
|
371
476
|
};
|
|
372
|
-
const srcCache = new lru_cache_1.LRUCache(
|
|
477
|
+
const srcCache = new lru_cache_1.LRUCache(cacheCap);
|
|
373
478
|
const src = (file) => {
|
|
374
479
|
// A cached '' (empty or unreadable file) returns '' where the miss path
|
|
375
480
|
// returns null for unreadable — every caller falsy-checks, so the two are
|
|
@@ -378,7 +483,12 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
378
483
|
if (hit !== undefined)
|
|
379
484
|
return hit;
|
|
380
485
|
const r = raw(file);
|
|
486
|
+
const t0 = prof ? Date.now() : 0;
|
|
381
487
|
const s = r == null ? '' : (0, strip_comments_1.stripCommentsForRegex)(r, 'c');
|
|
488
|
+
if (prof) {
|
|
489
|
+
prof.stripMs += Date.now() - t0;
|
|
490
|
+
prof.stripN++;
|
|
491
|
+
}
|
|
382
492
|
srcCache.set(file, s);
|
|
383
493
|
return r == null ? null : s;
|
|
384
494
|
};
|
|
@@ -394,45 +504,40 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
394
504
|
return inc;
|
|
395
505
|
return null;
|
|
396
506
|
};
|
|
397
|
-
//
|
|
507
|
+
// Retained strings are interned through here. Regex captures off a big file
|
|
508
|
+
// string are V8 sliced strings — retaining one pins the whole parent file
|
|
509
|
+
// text, and the facts tables retain captures from EVERY file for the whole
|
|
510
|
+
// pass. The Buffer round-trip forces a flat copy on first sight; repeats
|
|
511
|
+
// (field names recur heavily) then share the one flat instance.
|
|
512
|
+
const interned = new Map();
|
|
513
|
+
const intern = (x) => {
|
|
514
|
+
let f = interned.get(x);
|
|
515
|
+
if (f === undefined) {
|
|
516
|
+
f = Buffer.from(x, 'utf8').toString('utf8');
|
|
517
|
+
interned.set(f, f);
|
|
518
|
+
}
|
|
519
|
+
return f;
|
|
520
|
+
};
|
|
521
|
+
// ---- Global tables the extraction sweep fills ----
|
|
398
522
|
// fn-pointer: typedef RET (*NAME)(…) → a field `NAME f` is a fn ptr
|
|
399
523
|
// fn-type: typedef RET NAME(params) → a field `NAME *f` is a fn ptr
|
|
400
524
|
// The fn-type form is redis' command idiom: `typedef void redisCommandProc(client*)`
|
|
401
525
|
// declared as `redisCommandProc *proc;`. Without this, `proc` reads as data.
|
|
402
526
|
const fnPtrTypedefs = new Set();
|
|
403
527
|
const fnTypeTypedefs = new Set();
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
continue; // pointer form — handled above
|
|
418
|
-
const fm = guts.match(/\b(\w+)\s*\(/); // last identifier before the param list
|
|
419
|
-
if (fm && !C_TYPE_KEYWORDS.has(fm[1]))
|
|
420
|
-
fnTypeTypedefs.add(fm[1]);
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
// ---- Pass B: struct field layouts ----
|
|
424
|
-
// structLayout: struct name → ordered fields, for structs with ≥1 fn-pointer
|
|
425
|
-
// field (drives positional registration + dispatch).
|
|
426
|
-
// allStructFields: EVERY struct name → ALL its field layouts (a name can be
|
|
427
|
-
// reused across files — e.g. redis has two unrelated `client` structs), used
|
|
428
|
-
// to walk a chained receiver's field types (`c->cmd->proc`: client.cmd →
|
|
429
|
-
// redisCommand). The walk searches every same-named layout for the field.
|
|
430
|
-
// fieldToStructs: fn-pointer field name → set of struct names that declare it.
|
|
431
|
-
const structLayout = new Map();
|
|
432
|
-
const allStructFields = new Map();
|
|
433
|
-
const fieldToStructs = new Map();
|
|
434
|
-
// Parse a struct body (the text between its `{` and `}`) into ordered fields.
|
|
435
|
-
const parseStructFields = (inner) => {
|
|
528
|
+
/** Struct node id → its structurally-parsed fields (classified + registered
|
|
529
|
+
* in the linking stage, in kind-scan order). */
|
|
530
|
+
const rawFieldsByNode = new Map();
|
|
531
|
+
const factsByFile = new Map();
|
|
532
|
+
/** Every inline-struct candidate tag anywhere — an over-approximation of the
|
|
533
|
+
* tags the registration stage can add to `structLayout` mid-stage, folded
|
|
534
|
+
* into the registration filter's layout check. */
|
|
535
|
+
const inlineTags = new Set();
|
|
536
|
+
/** Object-macro names with an alias-shaped value anywhere (see OBJ_ALIAS_RE). */
|
|
537
|
+
const aliasNames = new Set();
|
|
538
|
+
// Parse a struct body (the text between its `{` and `}`) into ordered fields,
|
|
539
|
+
// structure only — see RawFieldDecl for why classification is deferred.
|
|
540
|
+
const parseStructFieldsRaw = (inner) => {
|
|
436
541
|
const fields = [];
|
|
437
542
|
let idx = 0;
|
|
438
543
|
for (const rawDecl of splitTopLevel(inner, ';')) {
|
|
@@ -449,11 +554,11 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
449
554
|
const p = parts[pi].trim();
|
|
450
555
|
let name = null;
|
|
451
556
|
let type = '';
|
|
452
|
-
let
|
|
453
|
-
const
|
|
454
|
-
if (
|
|
455
|
-
name =
|
|
456
|
-
|
|
557
|
+
let ptr = false;
|
|
558
|
+
const pm = p.match(FNPTR_DECL_RE);
|
|
559
|
+
if (pm) {
|
|
560
|
+
name = pm[1]; // `… (*name)(…)` — a function pointer
|
|
561
|
+
ptr = true;
|
|
457
562
|
}
|
|
458
563
|
else if (pi === 0) {
|
|
459
564
|
if (firstTyped) {
|
|
@@ -469,17 +574,288 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
469
574
|
type = sharedType;
|
|
470
575
|
}
|
|
471
576
|
}
|
|
472
|
-
if (!ptr && type)
|
|
473
|
-
isFnPtr = fnPtrTypedefs.has(type) || fnTypeTypedefs.has(type);
|
|
474
577
|
// Always advance the positional index. An unparsed field (anonymous
|
|
475
578
|
// union, exotic declarator) still occupies one slot, and macro-expanded
|
|
476
579
|
// positional tables (redis' MAKE_CMD) only align if every field counts.
|
|
477
|
-
fields.push({ name
|
|
580
|
+
fields.push({ name, index: idx, ptr, type });
|
|
478
581
|
idx++;
|
|
479
582
|
}
|
|
480
583
|
}
|
|
481
584
|
return fields;
|
|
482
585
|
};
|
|
586
|
+
// Classify deferred fields against the (now-complete) typedef sets.
|
|
587
|
+
const classifyFields = (rawFields) => rawFields.map((f) => ({
|
|
588
|
+
name: f.name ?? '',
|
|
589
|
+
index: f.index,
|
|
590
|
+
isFnPtr: !!f.name &&
|
|
591
|
+
(f.ptr || (!!f.type && (fnPtrTypedefs.has(f.type) || fnTypeTypedefs.has(f.type)))),
|
|
592
|
+
type: f.type,
|
|
593
|
+
}));
|
|
594
|
+
const parseStructFields = (inner) => classifyFields(parseStructFieldsRaw(inner));
|
|
595
|
+
// Exact per-file include resolution (from RAW source — string contents survive).
|
|
596
|
+
const scanIncludes = (file) => {
|
|
597
|
+
const rawText = raw(file);
|
|
598
|
+
if (!rawText || !rawText.includes('include'))
|
|
599
|
+
return NO_INCLUDES;
|
|
600
|
+
const out = [];
|
|
601
|
+
INCLUDE_RE.lastIndex = 0;
|
|
602
|
+
let im;
|
|
603
|
+
while ((im = INCLUDE_RE.exec(rawText))) {
|
|
604
|
+
if (!INCLUDABLE_EXT.test(im[1]))
|
|
605
|
+
continue;
|
|
606
|
+
const t = resolveInclude(file, im[1]);
|
|
607
|
+
if (t)
|
|
608
|
+
out.push(intern(t));
|
|
609
|
+
}
|
|
610
|
+
return out.length ? out : NO_INCLUDES;
|
|
611
|
+
};
|
|
612
|
+
// Indexed files answer from their facts; non-indexed includes (reached by
|
|
613
|
+
// buildEnv's depth-2 recursion) fall back to a bounded lazy scan.
|
|
614
|
+
const includeCache = new lru_cache_1.LRUCache(1024);
|
|
615
|
+
const localIncludesOf = (file) => {
|
|
616
|
+
const f = factsByFile.get(file);
|
|
617
|
+
if (f)
|
|
618
|
+
return f.includes;
|
|
619
|
+
let out = includeCache.get(file);
|
|
620
|
+
if (out)
|
|
621
|
+
return out;
|
|
622
|
+
out = scanIncludes(file);
|
|
623
|
+
includeCache.set(file, out);
|
|
624
|
+
return out;
|
|
625
|
+
};
|
|
626
|
+
// ---- Stage A: the extraction sweep — ONE read + strip per file ----
|
|
627
|
+
//
|
|
628
|
+
// Two implementations, record-identical by the differential suite:
|
|
629
|
+
// • native (task #5 step 2): the kernel's `cfnptrScanFiles` strips and
|
|
630
|
+
// scans a BATCH of files per NAPI call (lattice-sensor-kernel/src/cfnptr.rs —
|
|
631
|
+
// hand-rolled byte machines replicating the JS regex semantics), and the
|
|
632
|
+
// TS side only reads files, ships batches, and interns the returned
|
|
633
|
+
// facts. Include-path resolution stays here (it needs the filesystem).
|
|
634
|
+
// • JS: the original sweep, kept verbatim — the fallback for platforms
|
|
635
|
+
// without a kernel binary, older binaries (feature detection), the
|
|
636
|
+
// LATTICE_SENSOR_KERNEL=0 kill switch, and LATTICE_SENSOR_KERNEL_CFNPTR=0 (this
|
|
637
|
+
// scanner's own switch).
|
|
638
|
+
const kernel = process.env.LATTICE_SENSOR_KERNEL === '0' || process.env.LATTICE_SENSOR_KERNEL_CFNPTR === '0'
|
|
639
|
+
? null
|
|
640
|
+
: (0, loader_1.getKernel)();
|
|
641
|
+
const nativeSweep = kernel && typeof kernel.cfnptrScanFiles === 'function' ? kernel.cfnptrScanFiles.bind(kernel) : null;
|
|
642
|
+
const mergeNativeFacts = (file, out) => {
|
|
643
|
+
for (const t of out.fnPtrTypedefs)
|
|
644
|
+
fnPtrTypedefs.add(intern(t));
|
|
645
|
+
for (const t of out.fnTypeTypedefs)
|
|
646
|
+
fnTypeTypedefs.add(intern(t));
|
|
647
|
+
for (const so of out.structs) {
|
|
648
|
+
if (!so.parsed)
|
|
649
|
+
continue; // body never parsed — the JS sweep records nothing either
|
|
650
|
+
rawFieldsByNode.set(so.id, so.fields.map((f) => ({ name: f.name || null, index: f.index, ptr: f.ptr, type: f.type })));
|
|
651
|
+
}
|
|
652
|
+
for (const t of out.inlineTags)
|
|
653
|
+
inlineTags.add(intern(t));
|
|
654
|
+
for (const t of out.aliasNames)
|
|
655
|
+
aliasNames.add(intern(t));
|
|
656
|
+
const includes = [];
|
|
657
|
+
for (const cap of out.includes) {
|
|
658
|
+
if (!INCLUDABLE_EXT.test(cap))
|
|
659
|
+
continue;
|
|
660
|
+
const t = resolveInclude(file, cap);
|
|
661
|
+
if (t)
|
|
662
|
+
includes.push(intern(t));
|
|
663
|
+
}
|
|
664
|
+
if (out.initTokens.length || out.arrayElems.length || out.inlinePtr || out.inlineTypes.length ||
|
|
665
|
+
out.dPairs.length || out.dispatchFields.length || out.arrayDispatchNames.length || includes.length) {
|
|
666
|
+
factsByFile.set(file, {
|
|
667
|
+
initTokens: out.initTokens.length ? out.initTokens.map(intern) : null,
|
|
668
|
+
arrayElems: out.arrayElems.length ? out.arrayElems.map(intern) : null,
|
|
669
|
+
inlinePtr: out.inlinePtr,
|
|
670
|
+
inlineTypes: out.inlineTypes.length ? out.inlineTypes.map(intern) : null,
|
|
671
|
+
dPairs: out.dPairs.length ? out.dPairs.map(intern) : null,
|
|
672
|
+
dispatchFields: out.dispatchFields.length ? out.dispatchFields.map(intern) : null,
|
|
673
|
+
arrayDispatchNames: out.arrayDispatchNames.length ? out.arrayDispatchNames.map(intern) : null,
|
|
674
|
+
includes: includes.length ? includes : NO_INCLUDES,
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
let tPass = Date.now();
|
|
679
|
+
if (nativeSweep) {
|
|
680
|
+
// Batch of 16 = the tick/onFraction cadence, so yielding and progress
|
|
681
|
+
// reporting keep their shape while the boundary crossing amortizes.
|
|
682
|
+
const BATCH = 16;
|
|
683
|
+
let batch = [];
|
|
684
|
+
const flush = () => {
|
|
685
|
+
if (batch.length === 0)
|
|
686
|
+
return;
|
|
687
|
+
const outs = nativeSweep(batch.map((b) => b.input));
|
|
688
|
+
for (let bi = 0; bi < batch.length; bi++)
|
|
689
|
+
mergeNativeFacts(batch[bi].file, outs[bi]);
|
|
690
|
+
batch = [];
|
|
691
|
+
};
|
|
692
|
+
for (const file of files) {
|
|
693
|
+
await tick();
|
|
694
|
+
const rawText = raw(file);
|
|
695
|
+
if (!rawText)
|
|
696
|
+
continue; // unreadable or empty — the JS sweep skips these too
|
|
697
|
+
const tN = prof ? Date.now() : 0;
|
|
698
|
+
const fileNodes = ctx.getNodesInFile(file);
|
|
699
|
+
if (prof) {
|
|
700
|
+
prof.nodesMs += Date.now() - tN;
|
|
701
|
+
prof.nodesN++;
|
|
702
|
+
}
|
|
703
|
+
const structs = [];
|
|
704
|
+
for (const st of fileNodes) {
|
|
705
|
+
if (st.kind !== 'struct')
|
|
706
|
+
continue;
|
|
707
|
+
// sliceLinesPre semantics ride along: falsy startLine never parses,
|
|
708
|
+
// and `endLine ?? startLine` is applied here so the kernel sees the
|
|
709
|
+
// exact slice bounds the JS sweep would use.
|
|
710
|
+
structs.push({ id: st.id, startLine: st.startLine ?? 0, endLine: st.endLine ?? st.startLine ?? 0 });
|
|
711
|
+
}
|
|
712
|
+
batch.push({ file, input: { text: rawText, structs } });
|
|
713
|
+
if (batch.length >= BATCH)
|
|
714
|
+
flush();
|
|
715
|
+
}
|
|
716
|
+
flush();
|
|
717
|
+
}
|
|
718
|
+
// JS sweep (fallback path — see the stage comment above).
|
|
719
|
+
if (!nativeSweep)
|
|
720
|
+
for (const file of files) {
|
|
721
|
+
await tick();
|
|
722
|
+
const s = src(file);
|
|
723
|
+
if (!s)
|
|
724
|
+
continue;
|
|
725
|
+
// Typedefs (cross-file).
|
|
726
|
+
if (s.includes('typedef')) {
|
|
727
|
+
FNPTR_TYPEDEF_RE.lastIndex = 0;
|
|
728
|
+
let m;
|
|
729
|
+
while ((m = FNPTR_TYPEDEF_RE.exec(s)))
|
|
730
|
+
fnPtrTypedefs.add(intern(m[1]));
|
|
731
|
+
FNTYPE_TYPEDEF_STMT_RE.lastIndex = 0;
|
|
732
|
+
while ((m = FNTYPE_TYPEDEF_STMT_RE.exec(s))) {
|
|
733
|
+
const guts = m[1];
|
|
734
|
+
if (guts.includes('(*') || guts.includes('( *'))
|
|
735
|
+
continue; // pointer form — handled above
|
|
736
|
+
const fm = guts.match(/\b(\w+)\s*\(/); // last identifier before the param list
|
|
737
|
+
if (fm && !C_TYPE_KEYWORDS.has(fm[1]))
|
|
738
|
+
fnTypeTypedefs.add(intern(fm[1]));
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
// Struct-node field declarations (registered later in kind-scan order).
|
|
742
|
+
const tN = prof ? Date.now() : 0;
|
|
743
|
+
const fileNodes = ctx.getNodesInFile(file);
|
|
744
|
+
if (prof) {
|
|
745
|
+
prof.nodesMs += Date.now() - tN;
|
|
746
|
+
prof.nodesN++;
|
|
747
|
+
}
|
|
748
|
+
let lines = null;
|
|
749
|
+
for (const st of fileNodes) {
|
|
750
|
+
if (st.kind !== 'struct')
|
|
751
|
+
continue;
|
|
752
|
+
lines ??= s.split('\n');
|
|
753
|
+
const body = sliceLinesPre(lines, st.startLine, st.endLine);
|
|
754
|
+
const open = body.indexOf('{');
|
|
755
|
+
const close = open >= 0 ? matchBrace(body, open) : -1;
|
|
756
|
+
if (open < 0 || close < 0)
|
|
757
|
+
continue;
|
|
758
|
+
rawFieldsByNode.set(st.id, parseStructFieldsRaw(body.slice(open + 1, close)));
|
|
759
|
+
}
|
|
760
|
+
// Registration filters. These are full-file, NO-SKIP scans: the original
|
|
761
|
+
// registration pass jumps its scan cursor past a processed initializer
|
|
762
|
+
// body, so a no-skip scan finds a SUPERSET of its matches — exactly the
|
|
763
|
+
// over-approximation the filter needs.
|
|
764
|
+
const initTokens = new Set();
|
|
765
|
+
const arrayElems = new Set();
|
|
766
|
+
const inlineTypes = new Set();
|
|
767
|
+
let inlinePtr = false;
|
|
768
|
+
if (s.includes('{')) {
|
|
769
|
+
INLINE_STRUCT_RE.lastIndex = 0;
|
|
770
|
+
let im;
|
|
771
|
+
while ((im = INLINE_STRUCT_RE.exec(s))) {
|
|
772
|
+
const sOpen = im.index + im[0].length - 1;
|
|
773
|
+
const sClose = matchBrace(s, sOpen);
|
|
774
|
+
if (sClose < 0)
|
|
775
|
+
continue;
|
|
776
|
+
// After `}`, expect `var [opt] [= {…}]` to be a table candidate.
|
|
777
|
+
const vm = s.slice(sClose + 1).match(/^\s*(\w+)\s*(\[[^\]]*\])?\s*(=\s*\{)?/);
|
|
778
|
+
if (!vm || !vm[1])
|
|
779
|
+
continue;
|
|
780
|
+
inlineTags.add(intern(im[1]));
|
|
781
|
+
for (const f of parseStructFieldsRaw(s.slice(sOpen + 1, sClose))) {
|
|
782
|
+
if (!f.name)
|
|
783
|
+
continue;
|
|
784
|
+
if (f.ptr)
|
|
785
|
+
inlinePtr = true;
|
|
786
|
+
else if (f.type)
|
|
787
|
+
inlineTypes.add(intern(f.type));
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
if (s.includes('=')) {
|
|
791
|
+
INIT_RE.lastIndex = 0;
|
|
792
|
+
let m;
|
|
793
|
+
while ((m = INIT_RE.exec(s)))
|
|
794
|
+
initTokens.add(intern(m[1]));
|
|
795
|
+
ARRAY_TABLE_RE.lastIndex = 0;
|
|
796
|
+
while ((m = ARRAY_TABLE_RE.exec(s)))
|
|
797
|
+
arrayElems.add(intern((m[2] ? '*' : '') + m[1]));
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
// Alias-shaped object macros (registration filter support).
|
|
801
|
+
if (s.includes('#define') || s.includes('# define')) {
|
|
802
|
+
const joined = s.replace(/\\\r?\n/g, ' ');
|
|
803
|
+
OBJ_ALIAS_RE.lastIndex = 0;
|
|
804
|
+
let m;
|
|
805
|
+
while ((m = OBJ_ALIAS_RE.exec(joined)))
|
|
806
|
+
aliasNames.add(intern(m[1]));
|
|
807
|
+
}
|
|
808
|
+
// Propagation + dispatch filters (full-file scans ⊇ the per-function-body
|
|
809
|
+
// scans the pass bodies run — a body slice is a substring of the file).
|
|
810
|
+
const dPairs = new Set();
|
|
811
|
+
if (s.includes('=')) {
|
|
812
|
+
FIELD_ASSIGN_RE.lastIndex = 0;
|
|
813
|
+
let m;
|
|
814
|
+
while ((m = FIELD_ASSIGN_RE.exec(s)))
|
|
815
|
+
dPairs.add(intern(m[2] + '\0' + m[4]));
|
|
816
|
+
}
|
|
817
|
+
const dispatchFields = new Set();
|
|
818
|
+
const arrayNames = new Set();
|
|
819
|
+
DISPATCH_RE.lastIndex = 0;
|
|
820
|
+
let dm;
|
|
821
|
+
while ((dm = DISPATCH_RE.exec(s)))
|
|
822
|
+
dispatchFields.add(intern(dm[2]));
|
|
823
|
+
ARRAY_DISPATCH_RE.lastIndex = 0;
|
|
824
|
+
while ((dm = ARRAY_DISPATCH_RE.exec(s)))
|
|
825
|
+
arrayNames.add(intern(dm[1]));
|
|
826
|
+
const includes = scanIncludes(file);
|
|
827
|
+
if (initTokens.size || arrayElems.size || inlinePtr || inlineTypes.size ||
|
|
828
|
+
dPairs.size || dispatchFields.size || arrayNames.size || includes.length) {
|
|
829
|
+
factsByFile.set(file, {
|
|
830
|
+
initTokens: initTokens.size ? [...initTokens] : null,
|
|
831
|
+
arrayElems: arrayElems.size ? [...arrayElems] : null,
|
|
832
|
+
inlinePtr,
|
|
833
|
+
inlineTypes: inlineTypes.size ? [...inlineTypes] : null,
|
|
834
|
+
dPairs: dPairs.size ? [...dPairs] : null,
|
|
835
|
+
dispatchFields: dispatchFields.size ? [...dispatchFields] : null,
|
|
836
|
+
arrayDispatchNames: arrayNames.size ? [...arrayNames] : null,
|
|
837
|
+
includes,
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
if (prof) {
|
|
842
|
+
prof.A = Date.now() - tPass;
|
|
843
|
+
tPass = Date.now();
|
|
844
|
+
}
|
|
845
|
+
// ---- Stage B: struct field layouts (linking — text-free) ----
|
|
846
|
+
// structLayout: struct name → ordered fields, for structs with ≥1 fn-pointer
|
|
847
|
+
// field (drives positional registration + dispatch).
|
|
848
|
+
// allStructFields: EVERY struct name → ALL its field layouts (a name can be
|
|
849
|
+
// reused across files — e.g. redis has two unrelated `client` structs), used
|
|
850
|
+
// to walk a chained receiver's field types (`c->cmd->proc`: client.cmd →
|
|
851
|
+
// redisCommand). The walk searches every same-named layout for the field.
|
|
852
|
+
// fieldToStructs: fn-pointer field name → set of struct names that declare it.
|
|
853
|
+
// Registration REPLAYS the struct kind-scan (rowid order, ≠ the extraction
|
|
854
|
+
// sweep's path order): same-name layout precedence — `structLayout.set`
|
|
855
|
+
// last-wins, `allStructFields` first-match in the chain walk — depends on it.
|
|
856
|
+
const structLayout = new Map();
|
|
857
|
+
const allStructFields = new Map();
|
|
858
|
+
const fieldToStructs = new Map();
|
|
483
859
|
// Register a parsed struct under `name` into the three indexes.
|
|
484
860
|
const registerStructLayout = (name, fields) => {
|
|
485
861
|
if (!allStructFields.has(name))
|
|
@@ -500,22 +876,22 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
500
876
|
await onYield();
|
|
501
877
|
if (!C_CPP_EXT.test(st.filePath))
|
|
502
878
|
continue;
|
|
503
|
-
const
|
|
504
|
-
if (!
|
|
505
|
-
continue;
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
879
|
+
const rawFields = rawFieldsByNode.get(st.id);
|
|
880
|
+
if (!rawFields)
|
|
881
|
+
continue; // file unreadable or body unparsable at sweep time — the old pass skipped it too
|
|
882
|
+
registerStructLayout(st.name, classifyFields(rawFields));
|
|
883
|
+
}
|
|
884
|
+
rawFieldsByNode.clear();
|
|
885
|
+
if (prof) {
|
|
886
|
+
prof.B = Date.now() - tPass;
|
|
887
|
+
tPass = Date.now();
|
|
512
888
|
}
|
|
513
889
|
// NB: no early return on an empty structLayout here — an inline `struct TAG
|
|
514
890
|
// { … } var[]` table whose struct never became a node (vim's `cmdname`, broken
|
|
515
891
|
// up by `#ifdef`) is discovered later during the unit scan. The `reg.size === 0`
|
|
516
892
|
// guard after registration still short-circuits when nothing bridges.
|
|
517
893
|
const fnPtrFieldOf = (struct, field) => !!structLayout.get(struct)?.some((f) => f.name === field && f.isFnPtr);
|
|
518
|
-
// C/C++ function + method nodes are STREAMED per
|
|
894
|
+
// C/C++ function + method nodes are STREAMED per stage (see D/E) —
|
|
519
895
|
// the old materialized `cFns` array held every function node on the repo
|
|
520
896
|
// (O(nodes) memory, part of the #1212 kernel OOM).
|
|
521
897
|
// ---- function-name → node resolution (prefer a function in the same file) ----
|
|
@@ -532,7 +908,7 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
532
908
|
}
|
|
533
909
|
return cands[0];
|
|
534
910
|
};
|
|
535
|
-
// ----
|
|
911
|
+
// ---- Stage C: registrations — Map<"struct.field", Set<funcNodeId>> ----
|
|
536
912
|
// Ids only — retaining the full Node per registration (the old `idToNode`)
|
|
537
913
|
// was write-only dead weight at O(registrations) memory.
|
|
538
914
|
const reg = new Map();
|
|
@@ -634,6 +1010,10 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
634
1010
|
};
|
|
635
1011
|
// Per-file macro + include parsing (any file, indexed or not), cached.
|
|
636
1012
|
// Derived per-file caches, LRU-bounded like the content caches (#1212).
|
|
1013
|
+
// These stay LAZY (recompute-on-miss through `src`): retaining every file's
|
|
1014
|
+
// parsed tables is ruled out by the kernel's 6.1M `#define`s, and the
|
|
1015
|
+
// registration stage below only builds an env for files that survive its
|
|
1016
|
+
// filter or carry local includes, so most files never need one.
|
|
637
1017
|
const fnMacroCache = new lru_cache_1.LRUCache(256);
|
|
638
1018
|
const fileFnMacros = (file) => {
|
|
639
1019
|
let m = fnMacroCache.get(file);
|
|
@@ -661,27 +1041,6 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
661
1041
|
}
|
|
662
1042
|
return d;
|
|
663
1043
|
};
|
|
664
|
-
const includeCache = new lru_cache_1.LRUCache(1024);
|
|
665
|
-
const localIncludesOf = (file) => {
|
|
666
|
-
let out = includeCache.get(file);
|
|
667
|
-
if (out)
|
|
668
|
-
return out;
|
|
669
|
-
out = [];
|
|
670
|
-
const rawText = raw(file);
|
|
671
|
-
if (rawText && rawText.includes('include')) {
|
|
672
|
-
INCLUDE_RE.lastIndex = 0;
|
|
673
|
-
let im;
|
|
674
|
-
while ((im = INCLUDE_RE.exec(rawText))) {
|
|
675
|
-
if (!INCLUDABLE_EXT.test(im[1]))
|
|
676
|
-
continue;
|
|
677
|
-
const t = resolveInclude(file, im[1]);
|
|
678
|
-
if (t)
|
|
679
|
-
out.push(t);
|
|
680
|
-
}
|
|
681
|
-
}
|
|
682
|
-
includeCache.set(file, out);
|
|
683
|
-
return out;
|
|
684
|
-
};
|
|
685
1044
|
// A file's effective macro environment = its own #defines PLUS those of the
|
|
686
1045
|
// headers it #includes (redis' `MAKE_CMD` sits beside the table; sqlite's
|
|
687
1046
|
// `FUNCTION` lives in `sqliteInt.h`, included by the file with the table).
|
|
@@ -727,23 +1086,6 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
727
1086
|
registerStructValue(struct, body, file, env);
|
|
728
1087
|
}
|
|
729
1088
|
};
|
|
730
|
-
// `(?:struct )?TYPE name[opt] = {` initializers, where TYPE is a struct that
|
|
731
|
-
// has ≥1 fn-pointer field. Handles both single (`= {…}`) and array
|
|
732
|
-
// (`[] = { {…}, {…} }`) forms. Macro calls inside an element are expanded first.
|
|
733
|
-
const INIT_RE = /(?:^|[;{}])\s*(?:(?:static|const|extern|register|volatile)\s+)*(?:struct\s+)?(\w+)\s+(\w+)\s*(\[[^\]]*\])?\s*=\s*\{/g;
|
|
734
|
-
// `struct TAG { … } var[opt] [= {…}]` — the struct is defined INLINE with the
|
|
735
|
-
// table (vim's `cmdname`/`nv_cmd`); its layout never became a node, so parse it
|
|
736
|
-
// here and register it before reading the entries. No leading anchor: a
|
|
737
|
-
// `struct TAG {` with a brace body is always a definition (it may be preceded
|
|
738
|
-
// by a `#define …` line ending in a digit, as in vim), and the trailing
|
|
739
|
-
// `var … = {` check below is what distinguishes a TABLE from a plain type.
|
|
740
|
-
const INLINE_STRUCT_RE = /\bstruct\s+(\w+)\s*\{/g;
|
|
741
|
-
// `(?:static …)* ELEMTYPE [*] name[…] = { … }` — a bare array of function
|
|
742
|
-
// pointers (no struct wrapper). The optional `*` covers a function-TYPE
|
|
743
|
-
// typedef element (`opcode_t *opcodes[]`); a function-pointer typedef element
|
|
744
|
-
// (`zend_rc_dtor_func_t t[]`) needs none. The typedef-set membership gate
|
|
745
|
-
// (below) is what separates this from a plain data/struct array.
|
|
746
|
-
const ARRAY_TABLE_RE = /(?:^|[;{}])\s*(?:(?:static|const|extern|register|volatile)\s+)*(\w+)\s+(\*\s*)?(\w+)\s*\[[^\]]*\]\s*=\s*\{/g;
|
|
747
1089
|
// Process ONE unit's text and discard it. The old shape built every unit up
|
|
748
1090
|
// front (`const units: Unit[]`) — the full text of every C file plus its
|
|
749
1091
|
// expanded includes held simultaneously, gigabytes on the kernel (#1212).
|
|
@@ -817,18 +1159,42 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
817
1159
|
ARRAY_TABLE_RE.lastIndex = close;
|
|
818
1160
|
}
|
|
819
1161
|
};
|
|
820
|
-
//
|
|
821
|
-
//
|
|
1162
|
+
// Can this file's OWN unit have any side effect? Every check mirrors a gate
|
|
1163
|
+
// in processUnit, over-approximated to the filter's coarser knowledge:
|
|
1164
|
+
// • inline structs — the fn-ptr-field gate, with per-candidate field types
|
|
1165
|
+
// unioned per file;
|
|
1166
|
+
// • initializers — `structLayout.has` against the layouts' SUPERSET
|
|
1167
|
+
// (kind-scan layouts ∪ every inline tag — structLayout only grows during
|
|
1168
|
+
// this stage), with alias-shaped tokens surviving in place of the
|
|
1169
|
+
// per-file `resolveTypeName` walk;
|
|
1170
|
+
// • bare arrays — the exact typedef-set gate.
|
|
1171
|
+
// A filtered-out file is one where every match fails its gate before any
|
|
1172
|
+
// side effect, so skipping the unit cannot change the outcome.
|
|
1173
|
+
const typedefHit = (t) => fnPtrTypedefs.has(t) || fnTypeTypedefs.has(t);
|
|
1174
|
+
const regSurvives = (f) => f.inlinePtr ||
|
|
1175
|
+
(f.inlineTypes?.some(typedefHit) ?? false) ||
|
|
1176
|
+
(f.initTokens?.some((t) => structLayout.has(t) || inlineTags.has(t) || aliasNames.has(t)) ?? false) ||
|
|
1177
|
+
(f.arrayElems?.some((e) => e.charCodeAt(0) === 42 /* '*' */ ? typedefHit(e.slice(1)) : fnPtrTypedefs.has(e)) ?? false);
|
|
1178
|
+
// ---- Stage C: registrations — stream each surviving file (and every file's
|
|
1179
|
+
// qualifying local includes) through processUnit, one at a time.
|
|
822
1180
|
for (const file of files) {
|
|
823
1181
|
await tick();
|
|
1182
|
+
const facts = factsByFile.get(file);
|
|
1183
|
+
if (!facts)
|
|
1184
|
+
continue; // no facts ⇒ nothing matched at sweep time ⇒ the old pass would no-op here
|
|
1185
|
+
const survives = regSurvives(facts);
|
|
1186
|
+
if (!survives && facts.includes.length === 0)
|
|
1187
|
+
continue;
|
|
824
1188
|
const env = new Map();
|
|
825
1189
|
const objEnv = new Map();
|
|
826
1190
|
const defined = new Set();
|
|
827
1191
|
buildEnv(file, 2, new Set(), env, objEnv, defined);
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
1192
|
+
if (survives) {
|
|
1193
|
+
const s = src(file);
|
|
1194
|
+
if (s)
|
|
1195
|
+
processUnit({ text: s, file, env, objEnv });
|
|
1196
|
+
}
|
|
1197
|
+
for (const target of facts.includes) {
|
|
832
1198
|
if (seenInclude.has(`${file}>${target}`))
|
|
833
1199
|
continue;
|
|
834
1200
|
const incSrc = src(target);
|
|
@@ -857,11 +1223,21 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
857
1223
|
processUnit({ text, file: target, env: incEnv, objEnv: incObjEnv });
|
|
858
1224
|
}
|
|
859
1225
|
}
|
|
1226
|
+
if (prof) {
|
|
1227
|
+
prof.C = Date.now() - tPass;
|
|
1228
|
+
tPass = Date.now();
|
|
1229
|
+
}
|
|
860
1230
|
// ---- receiver-type resolution within a function's source ----
|
|
861
1231
|
// `(?:struct )?TYPE [*]recv` declared in the params or body → TYPE (if a known
|
|
862
1232
|
// fn-pointer-bearing struct).
|
|
1233
|
+
const recvReCache = new Map();
|
|
863
1234
|
const recvTypeIn = (fnSrc, recv) => {
|
|
864
|
-
|
|
1235
|
+
let re = recvReCache.get(recv);
|
|
1236
|
+
if (!re) {
|
|
1237
|
+
re = new RegExp(`(?:struct\\s+)?(\\w+)\\s*\\*?\\s*\\b${recv}\\b\\s*(?:[,)=;]|\\[)`, 'g');
|
|
1238
|
+
recvReCache.set(recv, re);
|
|
1239
|
+
}
|
|
1240
|
+
re.lastIndex = 0;
|
|
865
1241
|
let m;
|
|
866
1242
|
while ((m = re.exec(fnSrc))) {
|
|
867
1243
|
if (structLayout.has(m[1]))
|
|
@@ -873,8 +1249,14 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
873
1249
|
// structs (the base of a chained receiver needn't carry a fn pointer itself).
|
|
874
1250
|
// Falls back to a file-scope table variable (`cmdnames` in `cmdnames[i].fn()`).
|
|
875
1251
|
const escapeRe = (x) => x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
1252
|
+
const varReCache = new Map();
|
|
876
1253
|
const varTypeIn = (fnSrc, v) => {
|
|
877
|
-
|
|
1254
|
+
let re = varReCache.get(v);
|
|
1255
|
+
if (!re) {
|
|
1256
|
+
re = new RegExp(`(?:struct\\s+)?(\\w+)\\s*\\*?\\s*\\b${escapeRe(v)}\\b\\s*(?:[,)=;]|\\[)`, 'g');
|
|
1257
|
+
varReCache.set(v, re);
|
|
1258
|
+
}
|
|
1259
|
+
re.lastIndex = 0;
|
|
878
1260
|
let m;
|
|
879
1261
|
while ((m = re.exec(fnSrc))) {
|
|
880
1262
|
if (!C_TYPE_KEYWORDS.has(m[1]))
|
|
@@ -904,26 +1286,47 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
904
1286
|
}
|
|
905
1287
|
return t;
|
|
906
1288
|
};
|
|
907
|
-
// ----
|
|
1289
|
+
// ---- Stage D: field←field propagation (`a->f = b->g`) ----
|
|
908
1290
|
// Collected as (targetStruct.field ← sourceStruct.field) pairs, then merged to
|
|
909
1291
|
// a fixpoint so a hook slot inherits a registry field's handlers.
|
|
910
|
-
|
|
1292
|
+
// Filter: a file matters only if SOME collected pair has BOTH fields known as
|
|
1293
|
+
// fn-pointer fields — the loop body's own pre-gate. A skipped file's matches
|
|
1294
|
+
// would all `continue` there, so skipping is side-effect-free.
|
|
911
1295
|
const propagations = [];
|
|
912
1296
|
for (const file of files) {
|
|
913
1297
|
await tick();
|
|
1298
|
+
const facts = factsByFile.get(file);
|
|
1299
|
+
if (!facts?.dPairs?.some((p) => {
|
|
1300
|
+
const i = p.indexOf('\0');
|
|
1301
|
+
return fieldToStructs.has(p.slice(0, i)) && fieldToStructs.has(p.slice(i + 1));
|
|
1302
|
+
}))
|
|
1303
|
+
continue;
|
|
914
1304
|
const s = src(file);
|
|
915
1305
|
if (!s || !s.includes('='))
|
|
916
1306
|
continue;
|
|
917
|
-
|
|
1307
|
+
const tN = prof ? Date.now() : 0;
|
|
1308
|
+
const fnsD = ctx.getNodesInFile(file);
|
|
1309
|
+
if (prof) {
|
|
1310
|
+
prof.nodesMs += Date.now() - tN;
|
|
1311
|
+
prof.nodesN++;
|
|
1312
|
+
}
|
|
1313
|
+
const dLines = s.split('\n');
|
|
1314
|
+
for (const fn of fnsD) {
|
|
918
1315
|
if (!FN_KINDS.has(fn.kind))
|
|
919
1316
|
continue;
|
|
920
|
-
const body =
|
|
1317
|
+
const body = sliceLinesPre(dLines, fn.startLine, fn.endLine);
|
|
921
1318
|
if (!body.includes('='))
|
|
922
1319
|
continue;
|
|
923
1320
|
FIELD_ASSIGN_RE.lastIndex = 0;
|
|
924
1321
|
let m;
|
|
925
1322
|
while ((m = FIELD_ASSIGN_RE.exec(body))) {
|
|
926
1323
|
const [, lrecv, lfield, rrecv, rfield] = m;
|
|
1324
|
+
// Pre-gate on field NAMES: `a->f = b->g` matches every struct-field
|
|
1325
|
+
// assignment in the tree (millions on the kernel), but only fields
|
|
1326
|
+
// that are fn-pointer fields of SOME struct can pass fnPtrFieldOf —
|
|
1327
|
+
// skip the two regex type resolutions for the ~99% that can't.
|
|
1328
|
+
if (!fieldToStructs.has(lfield) || !fieldToStructs.has(rfield))
|
|
1329
|
+
continue;
|
|
927
1330
|
const lt = recvTypeIn(body, lrecv);
|
|
928
1331
|
const rt = recvTypeIn(body, rrecv);
|
|
929
1332
|
if (lt && rt && fnPtrFieldOf(lt, lfield) && fnPtrFieldOf(rt, rfield)) {
|
|
@@ -951,33 +1354,57 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
951
1354
|
if (!changed)
|
|
952
1355
|
break;
|
|
953
1356
|
}
|
|
1357
|
+
if (prof) {
|
|
1358
|
+
prof.D = Date.now() - tPass;
|
|
1359
|
+
tPass = Date.now();
|
|
1360
|
+
}
|
|
954
1361
|
if (reg.size === 0 && arrayReg.size === 0)
|
|
955
1362
|
return [];
|
|
956
|
-
// ----
|
|
957
|
-
//
|
|
958
|
-
//
|
|
959
|
-
// (`
|
|
960
|
-
//
|
|
961
|
-
const DISPATCH_RE = /((?:\w+(?:\s*\[[^\][]*\])?\s*(?:->|\.)\s*)+)(\w+)\s*\)?\s*\(/g;
|
|
962
|
-
// Bare-array dispatch: `tbl[i](…)` or the explicit-deref `(*tbl[i])(…)`. The
|
|
963
|
-
// subscript may itself contain a call (`tbl[GC_TYPE(p)](…)`), so the index
|
|
964
|
-
// class excludes only brackets. Precision comes from the `arrayReg` gate below
|
|
965
|
-
// — this fires only when `tbl` is a known fn-pointer array.
|
|
966
|
-
const ARRAY_DISPATCH_RE = /(?:\(\s*\*\s*)?\b(\w+)\s*\[[^\][]*\]\s*\)?\s*\(/g;
|
|
1363
|
+
// ---- Stage E: dispatch sites → edges ----
|
|
1364
|
+
// Filter: a file matters only if some dispatch field is a known fn-pointer
|
|
1365
|
+
// field, or some subscripted name is a registered fn-pointer array — the loop
|
|
1366
|
+
// body's own first gates (`owners` / `entries`), which a skipped file's
|
|
1367
|
+
// matches would all fail before touching `seen`/`added`/`edges`.
|
|
967
1368
|
const edges = [];
|
|
968
1369
|
const seen = new Set();
|
|
969
1370
|
for (const file of files) {
|
|
970
1371
|
await tick();
|
|
1372
|
+
const facts = factsByFile.get(file);
|
|
1373
|
+
if (!facts)
|
|
1374
|
+
continue;
|
|
1375
|
+
const eSurvives = (facts.dispatchFields?.some((f) => fieldToStructs.has(f)) ?? false) ||
|
|
1376
|
+
(arrayReg.size > 0 && (facts.arrayDispatchNames?.some((n) => arrayReg.has(n)) ?? false));
|
|
1377
|
+
if (!eSurvives)
|
|
1378
|
+
continue;
|
|
971
1379
|
const s = src(file);
|
|
972
1380
|
if (!s)
|
|
973
1381
|
continue;
|
|
974
|
-
|
|
1382
|
+
const tN = prof ? Date.now() : 0;
|
|
1383
|
+
const fnsE = ctx.getNodesInFile(file);
|
|
1384
|
+
if (prof) {
|
|
1385
|
+
prof.nodesMs += Date.now() - tN;
|
|
1386
|
+
prof.nodesN++;
|
|
1387
|
+
}
|
|
1388
|
+
const eLines = s.split('\n');
|
|
1389
|
+
for (const fn of fnsE) {
|
|
975
1390
|
if (!FN_KINDS.has(fn.kind))
|
|
976
1391
|
continue;
|
|
977
|
-
const body =
|
|
1392
|
+
const body = sliceLinesPre(eLines, fn.startLine, fn.endLine);
|
|
978
1393
|
DISPATCH_RE.lastIndex = 0;
|
|
979
1394
|
let m;
|
|
980
1395
|
let added = 0;
|
|
1396
|
+
// Incremental line counting: matches arrive in ascending index order, so
|
|
1397
|
+
// count newlines since the previous match instead of re-splitting the
|
|
1398
|
+
// whole body prefix per match (O(body) each — real time on god-files).
|
|
1399
|
+
let lcIdx = 0;
|
|
1400
|
+
let lcLine = fn.startLine;
|
|
1401
|
+
const lineAt = (idx) => {
|
|
1402
|
+
for (let i = lcIdx; i < idx; i++)
|
|
1403
|
+
if (body.charCodeAt(i) === 10)
|
|
1404
|
+
lcLine++;
|
|
1405
|
+
lcIdx = idx;
|
|
1406
|
+
return lcLine;
|
|
1407
|
+
};
|
|
981
1408
|
while ((m = DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
|
|
982
1409
|
const baseChain = m[1].replace(/\s*(?:->|\.)\s*$/, '').trim(); // receiver, minus the trailing arrow
|
|
983
1410
|
const field = m[2];
|
|
@@ -1000,7 +1427,7 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
1000
1427
|
const targets = reg.get(`${struct}.${field}`);
|
|
1001
1428
|
if (!targets)
|
|
1002
1429
|
continue;
|
|
1003
|
-
const line =
|
|
1430
|
+
const line = lineAt(m.index);
|
|
1004
1431
|
for (const tid of targets) {
|
|
1005
1432
|
if (tid === fn.id)
|
|
1006
1433
|
continue;
|
|
@@ -1026,6 +1453,9 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
1026
1453
|
}
|
|
1027
1454
|
// ---- bare array-of-fn-pointers dispatch (`tbl[i](…)`) ----
|
|
1028
1455
|
if (arrayReg.size && added < FANOUT_CAP) {
|
|
1456
|
+
// Fresh scan from the body's start — rewind the line-count cursor too.
|
|
1457
|
+
lcIdx = 0;
|
|
1458
|
+
lcLine = fn.startLine;
|
|
1029
1459
|
ARRAY_DISPATCH_RE.lastIndex = 0;
|
|
1030
1460
|
while ((m = ARRAY_DISPATCH_RE.exec(body)) && added < FANOUT_CAP) {
|
|
1031
1461
|
const entries = arrayReg.get(m[1]);
|
|
@@ -1038,7 +1468,7 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
1038
1468
|
: (entries.find((e) => e.file === fn.filePath)?.ids ?? null);
|
|
1039
1469
|
if (!ids)
|
|
1040
1470
|
continue;
|
|
1041
|
-
const line =
|
|
1471
|
+
const line = lineAt(m.index);
|
|
1042
1472
|
for (const tid of ids) {
|
|
1043
1473
|
if (tid === fn.id)
|
|
1044
1474
|
continue;
|
|
@@ -1065,6 +1495,10 @@ async function cFnPointerDispatchEdges(_queries, ctx, onYield, onFraction) {
|
|
|
1065
1495
|
}
|
|
1066
1496
|
}
|
|
1067
1497
|
}
|
|
1498
|
+
if (prof) {
|
|
1499
|
+
prof.E = Date.now() - tPass;
|
|
1500
|
+
console.error(`[synth-timing] cFnPtr sub: A=${prof.A}ms B=${prof.B}ms C=${prof.C}ms D=${prof.D}ms E=${prof.E}ms | read n=${prof.readN} ${prof.readMs}ms strip n=${prof.stripN} ${prof.stripMs}ms nodesInFile n=${prof.nodesN} ${prof.nodesMs}ms`);
|
|
1501
|
+
}
|
|
1068
1502
|
return edges;
|
|
1069
1503
|
}
|
|
1070
1504
|
//# sourceMappingURL=c-fnptr-synthesizer.js.map
|