@quolu/lattice 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/sensor/NOTICE +8 -19
- package/sensor/dist/bin/lattice-sensor.js +31 -0
- package/sensor/dist/db/index.d.ts +114 -0
- package/sensor/dist/db/index.d.ts.map +1 -1
- package/sensor/dist/db/index.js +235 -7
- package/sensor/dist/db/index.js.map +1 -1
- package/sensor/dist/db/migrations.js +1 -1
- package/sensor/dist/db/migrations.js.map +1 -1
- package/sensor/dist/db/queries.d.ts +46 -8
- package/sensor/dist/db/queries.d.ts.map +1 -1
- package/sensor/dist/db/queries.js +99 -16
- package/sensor/dist/db/queries.js.map +1 -1
- package/sensor/dist/db/wal-valve.d.ts +15 -2
- package/sensor/dist/db/wal-valve.d.ts.map +1 -1
- package/sensor/dist/db/wal-valve.js +88 -6
- package/sensor/dist/db/wal-valve.js.map +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts +1 -1
- package/sensor/dist/extraction/extraction-version.d.ts.map +1 -1
- package/sensor/dist/extraction/extraction-version.js +5 -1
- package/sensor/dist/extraction/extraction-version.js.map +1 -1
- package/sensor/dist/extraction/function-ref.d.ts.map +1 -1
- package/sensor/dist/extraction/function-ref.js +6 -0
- package/sensor/dist/extraction/function-ref.js.map +1 -1
- package/sensor/dist/extraction/grammars.d.ts.map +1 -1
- package/sensor/dist/extraction/grammars.js +46 -0
- package/sensor/dist/extraction/grammars.js.map +1 -1
- package/sensor/dist/extraction/index.d.ts +11 -1
- package/sensor/dist/extraction/index.d.ts.map +1 -1
- package/sensor/dist/extraction/index.js +68 -16
- package/sensor/dist/extraction/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/decode.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/decode.js +9 -3
- package/sensor/dist/extraction/kernel/decode.js.map +1 -1
- package/sensor/dist/extraction/kernel/index.d.ts +2 -0
- package/sensor/dist/extraction/kernel/index.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/index.js +122 -8
- package/sensor/dist/extraction/kernel/index.js.map +1 -1
- package/sensor/dist/extraction/kernel/layout.d.ts +10 -1
- package/sensor/dist/extraction/kernel/layout.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/layout.js +11 -2
- package/sensor/dist/extraction/kernel/layout.js.map +1 -1
- package/sensor/dist/extraction/kernel/loader.d.ts +42 -0
- package/sensor/dist/extraction/kernel/loader.d.ts.map +1 -1
- package/sensor/dist/extraction/kernel/loader.js.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.d.ts +13 -0
- package/sensor/dist/extraction/languages/c-cpp.d.ts.map +1 -1
- package/sensor/dist/extraction/languages/c-cpp.js +809 -13
- package/sensor/dist/extraction/languages/c-cpp.js.map +1 -1
- package/sensor/dist/extraction/store-worker.js +29 -0
- package/sensor/dist/extraction/store-worker.js.map +1 -1
- package/sensor/dist/extraction/tree-sitter.d.ts +4 -1
- package/sensor/dist/extraction/tree-sitter.d.ts.map +1 -1
- package/sensor/dist/extraction/tree-sitter.js +20 -4
- package/sensor/dist/extraction/tree-sitter.js.map +1 -1
- package/sensor/dist/extraction/wasm/tree-sitter-c.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-cpp.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-dart.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-kotlin.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-php.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-ruby.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-rust.wasm +0 -0
- package/sensor/dist/extraction/wasm/tree-sitter-swift.wasm +0 -0
- package/sensor/dist/index.d.ts +12 -1
- package/sensor/dist/index.d.ts.map +1 -1
- package/sensor/dist/index.js +42 -7
- package/sensor/dist/index.js.map +1 -1
- package/sensor/dist/mcp/daemon-registry.d.ts +5 -2
- package/sensor/dist/mcp/daemon-registry.d.ts.map +1 -1
- package/sensor/dist/mcp/daemon-registry.js +5 -2
- package/sensor/dist/mcp/daemon-registry.js.map +1 -1
- package/sensor/dist/mcp/index.d.ts +24 -0
- package/sensor/dist/mcp/index.d.ts.map +1 -1
- package/sensor/dist/mcp/index.js +51 -2
- package/sensor/dist/mcp/index.js.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.d.ts.map +1 -1
- package/sensor/dist/mcp/liveness-watchdog.js +3 -1
- package/sensor/dist/mcp/liveness-watchdog.js.map +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts +1 -1
- package/sensor/dist/mcp/server-instructions.d.ts.map +1 -1
- package/sensor/dist/mcp/server-instructions.js +1 -0
- package/sensor/dist/mcp/server-instructions.js.map +1 -1
- package/sensor/dist/mcp/tools.d.ts +48 -0
- package/sensor/dist/mcp/tools.d.ts.map +1 -1
- package/sensor/dist/mcp/tools.js +231 -4
- package/sensor/dist/mcp/tools.js.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/c-fnptr-synthesizer.js +561 -127
- package/sensor/dist/resolution/c-fnptr-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts +1 -1
- package/sensor/dist/resolution/callback-synthesizer.d.ts.map +1 -1
- package/sensor/dist/resolution/callback-synthesizer.js +80 -11
- package/sensor/dist/resolution/callback-synthesizer.js.map +1 -1
- package/sensor/dist/resolution/import-resolver.d.ts.map +1 -1
- package/sensor/dist/resolution/import-resolver.js +34 -2
- package/sensor/dist/resolution/import-resolver.js.map +1 -1
- package/sensor/dist/resolution/index.d.ts +34 -0
- package/sensor/dist/resolution/index.d.ts.map +1 -1
- package/sensor/dist/resolution/index.js +393 -51
- package/sensor/dist/resolution/index.js.map +1 -1
- package/sensor/dist/resolution/memory-budget.d.ts +48 -0
- package/sensor/dist/resolution/memory-budget.d.ts.map +1 -0
- package/sensor/dist/resolution/memory-budget.js +162 -0
- package/sensor/dist/resolution/memory-budget.js.map +1 -0
- package/sensor/dist/resolution/name-matcher.d.ts +4 -7
- package/sensor/dist/resolution/name-matcher.d.ts.map +1 -1
- package/sensor/dist/resolution/name-matcher.js +256 -91
- package/sensor/dist/resolution/name-matcher.js.map +1 -1
- package/sensor/dist/resolution/resolver-pool.d.ts +41 -1
- package/sensor/dist/resolution/resolver-pool.d.ts.map +1 -1
- package/sensor/dist/resolution/resolver-pool.js +103 -3
- package/sensor/dist/resolution/resolver-pool.js.map +1 -1
- package/sensor/dist/resolution/resolver-worker.js +27 -0
- package/sensor/dist/resolution/resolver-worker.js.map +1 -1
- package/sensor/dist/sync/watcher.d.ts +8 -1
- package/sensor/dist/sync/watcher.d.ts.map +1 -1
- package/sensor/dist/sync/watcher.js +44 -2
- package/sensor/dist/sync/watcher.js.map +1 -1
- package/sensor/dist/types.d.ts +7 -3
- package/sensor/dist/types.d.ts.map +1 -1
- package/src/cli-help.mjs +16 -0
- package/src/sensor-adapter.mjs +4 -0
- package/src/sensor-cli.mjs +37 -5
- package/src/sensor-diff.mjs +661 -0
|
@@ -262,6 +262,32 @@ class ReferenceResolver {
|
|
|
262
262
|
qualifiedNameCache; // qualified_name → nodes cache
|
|
263
263
|
fileLinesCache; // file → split lines cache
|
|
264
264
|
methodMatchCache; // lang\0Type::method → matching method nodes
|
|
265
|
+
// Per-(language, methodName) owner index for getMethodMatches: buckets a
|
|
266
|
+
// method name's candidates by their qualifiedName's last two segments so a
|
|
267
|
+
// (type, method) query is a lookup instead of an O(candidates) filter per
|
|
268
|
+
// methodMatchCache miss. Derived purely from node rows (stable through the
|
|
269
|
+
// resolution loop, same window nameCache relies on); dropped in clearCaches.
|
|
270
|
+
methodOwnerIndexCache = new Map();
|
|
271
|
+
// Generation-tagged memo for getSupertypes. Supertype edges GROW during the
|
|
272
|
+
// resolution loop (batch k persists its implements/extends edges BEFORE
|
|
273
|
+
// batch k+1 fans out — the #1320 ordering), so a plain cache would freeze an
|
|
274
|
+
// early batch's emptier answer and change later batches' outcomes. Within
|
|
275
|
+
// one batch the edge state is fixed by that same ordering, so entries are
|
|
276
|
+
// tagged with a generation that advances at every batch entry point
|
|
277
|
+
// (resolveBatchYielding / resolveListForAdmission) — a stale-gen entry is
|
|
278
|
+
// recomputed, making the memo behavior-identical to no memo at every point
|
|
279
|
+
// in time. On the Swift compiler the unmemoized walk ran 971k times for
|
|
280
|
+
// 565s of combined worker time (~581µs each, recursion-multiplied).
|
|
281
|
+
supertypeGen = 0;
|
|
282
|
+
supertypeMemo = new Map();
|
|
283
|
+
/** Invalidate the getSupertypes memo — call when resolved edges may have advanced. */
|
|
284
|
+
advanceSupertypeGeneration() {
|
|
285
|
+
this.supertypeGen++;
|
|
286
|
+
// Lazy invalidation via the gen tag; bound the map so a long run over many
|
|
287
|
+
// batches doesn't accrete dead entries.
|
|
288
|
+
if (this.supertypeMemo.size > 50_000)
|
|
289
|
+
this.supertypeMemo.clear();
|
|
290
|
+
}
|
|
265
291
|
// Node kinds are a small fixed set (~24), so this is a plain Map, not an LRU.
|
|
266
292
|
// getNodesByKind returns the FULL node list for a kind; it was previously
|
|
267
293
|
// uncached — a per-ref `SELECT * FROM nodes WHERE kind=?` + row-mapping. Called
|
|
@@ -390,14 +416,19 @@ class ReferenceResolver {
|
|
|
390
416
|
this.qualifiedNameCache.clear();
|
|
391
417
|
this.fileLinesCache.clear();
|
|
392
418
|
this.methodMatchCache.clear();
|
|
419
|
+
this.methodOwnerIndexCache.clear();
|
|
420
|
+
this.supertypeMemo.clear();
|
|
421
|
+
this.supertypeGen++;
|
|
393
422
|
this.nodesByKindCache.clear();
|
|
394
423
|
this.knownNames = null;
|
|
395
424
|
this.knownFiles = null;
|
|
396
425
|
this.cachesWarmed = false;
|
|
397
|
-
// The import-resolver's per-context memos assume the
|
|
398
|
-
// as the caches above — drop them together.
|
|
399
|
-
if (this.context)
|
|
426
|
+
// The import-resolver's and name-matcher's per-context memos assume the
|
|
427
|
+
// same stable window as the caches above — drop them together.
|
|
428
|
+
if (this.context) {
|
|
400
429
|
(0, import_resolver_1.clearImportResolverMemos)(this.context);
|
|
430
|
+
(0, name_matcher_1.clearNameMatcherMemos)(this.context);
|
|
431
|
+
}
|
|
401
432
|
}
|
|
402
433
|
/** `readFile` through the LRU content cache (null = read failed, also cached). */
|
|
403
434
|
readFileCached(filePath) {
|
|
@@ -446,15 +477,61 @@ class ReferenceResolver {
|
|
|
446
477
|
this.nameCache.set(methodName, candidates);
|
|
447
478
|
}
|
|
448
479
|
const want = `${typeName}::${methodName}`;
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
480
|
+
let matches;
|
|
481
|
+
if (typeName.includes('::') || methodName.includes(':')) {
|
|
482
|
+
// Legacy linear filter for the shapes the owner index below can't
|
|
483
|
+
// key exactly: a multi-segment typeName (the endsWith test then
|
|
484
|
+
// spans more than two `::` segments) and ObjC selectors (whose
|
|
485
|
+
// single/empty-keyword colons defeat the segment split). Tiny
|
|
486
|
+
// populations; the per-key memo above still amortizes them.
|
|
487
|
+
matches = [];
|
|
488
|
+
for (const m of candidates) {
|
|
489
|
+
if (m.kind !== 'method')
|
|
490
|
+
continue;
|
|
491
|
+
if (m.language !== language)
|
|
492
|
+
continue;
|
|
493
|
+
const qn = m.qualifiedName;
|
|
494
|
+
if (qn === want || qn.endsWith(`::${want}`))
|
|
495
|
+
matches.push(m);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
else {
|
|
499
|
+
// Owner index: the linear filter above is O(all same-named methods)
|
|
500
|
+
// per CACHE MISS, and on overload-heavy landscapes the distinct
|
|
501
|
+
// (type, method) key space is so large the per-key memo never
|
|
502
|
+
// amortizes — Swift's `init` has tens of thousands of candidates
|
|
503
|
+
// and the compiler repo measured 732µs per failing call, most of it
|
|
504
|
+
// this scan (re-entered once per supertype recursion level, too).
|
|
505
|
+
// Bucket each (language, methodName)'s candidates ONCE by the
|
|
506
|
+
// qualifiedName's last two `::` segments — exactly the span the
|
|
507
|
+
// `qn === want || qn.endsWith('::' + want)` predicate tests for a
|
|
508
|
+
// segment-clean typeName — then every query is a map lookup.
|
|
509
|
+
// Bucket insertion follows candidate order, so each bucket is
|
|
510
|
+
// byte-identical to what the linear filter produced.
|
|
511
|
+
const idxKey = `${language} ${methodName}`;
|
|
512
|
+
let ownerIndex = this.methodOwnerIndexCache.get(idxKey);
|
|
513
|
+
if (!ownerIndex) {
|
|
514
|
+
ownerIndex = new Map();
|
|
515
|
+
for (const m of candidates) {
|
|
516
|
+
if (m.kind !== 'method')
|
|
517
|
+
continue;
|
|
518
|
+
if (m.language !== language)
|
|
519
|
+
continue;
|
|
520
|
+
const qn = m.qualifiedName;
|
|
521
|
+
const i2 = qn.lastIndexOf('::');
|
|
522
|
+
if (i2 < 0)
|
|
523
|
+
continue; // single-segment qn can never match `T::m`
|
|
524
|
+
const i1 = qn.lastIndexOf('::', i2 - 1);
|
|
525
|
+
const bucketKey = i1 < 0 ? qn : qn.slice(i1 + 2);
|
|
526
|
+
const bucket = ownerIndex.get(bucketKey);
|
|
527
|
+
if (bucket)
|
|
528
|
+
bucket.push(m);
|
|
529
|
+
else
|
|
530
|
+
ownerIndex.set(bucketKey, [m]);
|
|
531
|
+
}
|
|
532
|
+
this.methodOwnerIndexCache.set(idxKey, ownerIndex);
|
|
533
|
+
}
|
|
534
|
+
matches = ownerIndex.get(want) ?? [];
|
|
458
535
|
}
|
|
459
536
|
this.methodMatchCache.set(key, matches);
|
|
460
537
|
return matches;
|
|
@@ -546,20 +623,34 @@ class ReferenceResolver {
|
|
|
546
623
|
// Matching by simple name (not id) reconciles a type declared in one node
|
|
547
624
|
// (`KF::Builder`) with conformance declared in a separate extension node
|
|
548
625
|
// (`KF.Builder: KFOptionSetter`) — both have name `Builder`.
|
|
626
|
+
// Memoized per batch generation (see supertypeMemo): within a batch the
|
|
627
|
+
// edge state is fixed, and the conformance walk re-queries the same
|
|
628
|
+
// popular supertypes (Swift stdlib protocols especially) thousands of
|
|
629
|
+
// times per batch.
|
|
630
|
+
const memoKey = `${language} ${typeName}`;
|
|
631
|
+
const hit = this.supertypeMemo.get(memoKey);
|
|
632
|
+
if (hit && hit.gen === this.supertypeGen)
|
|
633
|
+
return hit.supers;
|
|
549
634
|
const typeNodes = this.context
|
|
550
635
|
.getNodesByName(typeName)
|
|
551
636
|
.filter((n) => SUPERTYPE_BEARING_KINDS.has(n.kind) && n.language === language);
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
637
|
+
let supers;
|
|
638
|
+
if (typeNodes.length === 0) {
|
|
639
|
+
supers = [];
|
|
640
|
+
}
|
|
641
|
+
else {
|
|
642
|
+
const supertypes = new Set();
|
|
643
|
+
for (const tn of typeNodes) {
|
|
644
|
+
for (const edge of this.queries.getOutgoingEdges(tn.id, ['implements', 'extends'])) {
|
|
645
|
+
const target = this.queries.getNodeById(edge.target);
|
|
646
|
+
if (target?.name && target.name !== typeName)
|
|
647
|
+
supertypes.add(target.name);
|
|
648
|
+
}
|
|
560
649
|
}
|
|
650
|
+
supers = [...supertypes];
|
|
561
651
|
}
|
|
562
|
-
|
|
652
|
+
this.supertypeMemo.set(memoKey, { gen: this.supertypeGen, supers });
|
|
653
|
+
return supers;
|
|
563
654
|
},
|
|
564
655
|
getImportMappings: (filePath, language) => {
|
|
565
656
|
const cacheKey = filePath;
|
|
@@ -645,7 +736,7 @@ class ReferenceResolver {
|
|
|
645
736
|
let lastReportedPercent = -1;
|
|
646
737
|
for (let i = 0; i < refs.length; i++) {
|
|
647
738
|
const ref = refs[i]; // Array index is guaranteed to be in bounds
|
|
648
|
-
const result = this.
|
|
739
|
+
const result = this.resolveOneTimed(ref);
|
|
649
740
|
if (result) {
|
|
650
741
|
resolved.push(result);
|
|
651
742
|
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
|
@@ -807,10 +898,14 @@ class ReferenceResolver {
|
|
|
807
898
|
const existenceName = ref.language === 'arkts' && ref.referenceName.startsWith('.')
|
|
808
899
|
? ref.referenceName.slice(1)
|
|
809
900
|
: ref.referenceName;
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
901
|
+
const tPre = this.profileStages ? process.hrtime.bigint() : 0n;
|
|
902
|
+
const preFilterPass = (0, import_resolver_1.isNixPathImportRef)(ref) ||
|
|
903
|
+
this.hasAnyPossibleMatch(existenceName) ||
|
|
904
|
+
this.matchesAnyImport(ref) ||
|
|
905
|
+
this.frameworks.some((f) => f.claimsReference?.(ref.referenceName));
|
|
906
|
+
if (this.profileStages)
|
|
907
|
+
this.stageAdd('preFilter', ref, preFilterPass, tPre);
|
|
908
|
+
if (!preFilterPass) {
|
|
814
909
|
return null;
|
|
815
910
|
}
|
|
816
911
|
// Function-as-value refs (#756) get a dedicated, strictly-gated path:
|
|
@@ -827,7 +922,13 @@ class ReferenceResolver {
|
|
|
827
922
|
const viaImport = this.gateLanguage((0, import_resolver_1.resolveViaImport)(ref, this.context), ref);
|
|
828
923
|
if (viaImport) {
|
|
829
924
|
const target = this.queries.getNodeById(viaImport.targetNodeId);
|
|
830
|
-
if (target &&
|
|
925
|
+
if (target &&
|
|
926
|
+
(target.kind === 'function' ||
|
|
927
|
+
target.kind === 'method' ||
|
|
928
|
+
// Python (#1478): an imported class used as a value (`return
|
|
929
|
+
// OrgSerializerFull`) resolves through its import like any
|
|
930
|
+
// callback — mirrors matchFunctionRef's bareClassOk.
|
|
931
|
+
(ref.language === 'python' && target.kind === 'class'))) {
|
|
831
932
|
return viaImport;
|
|
832
933
|
}
|
|
833
934
|
}
|
|
@@ -846,7 +947,10 @@ class ReferenceResolver {
|
|
|
846
947
|
// JVM FQN imports skip framework/name-matcher: `import com.example.Bar`
|
|
847
948
|
// resolves directly through the qualifiedName index, which is unambiguous
|
|
848
949
|
// even when several `Bar` classes exist in different packages.
|
|
950
|
+
const tJvm = this.profileStages ? process.hrtime.bigint() : 0n;
|
|
849
951
|
const jvmImport = (0, import_resolver_1.resolveJvmImport)(ref, this.context);
|
|
952
|
+
if (this.profileStages)
|
|
953
|
+
this.stageAdd('jvmImport', ref, !!jvmImport, tJvm);
|
|
850
954
|
if (jvmImport)
|
|
851
955
|
return jvmImport;
|
|
852
956
|
// Razor/Blazor: a markup or `@code` type ref resolves through the file's
|
|
@@ -865,16 +969,27 @@ class ReferenceResolver {
|
|
|
865
969
|
// JS → native `calls`) — `gateFrameworkLanguage` only drops a type/import
|
|
866
970
|
// edge between two KNOWN families (see its doc), never a `calls` bridge or
|
|
867
971
|
// a config↔code edge.
|
|
972
|
+
const tFw = this.profileStages ? process.hrtime.bigint() : 0n;
|
|
973
|
+
let fwEarly = null;
|
|
868
974
|
for (const framework of this.frameworks) {
|
|
869
975
|
const result = this.gateFrameworkLanguage(framework.resolve(ref, this.context), ref);
|
|
870
976
|
if (result) {
|
|
871
|
-
if (result.confidence >= 0.9)
|
|
872
|
-
|
|
977
|
+
if (result.confidence >= 0.9) {
|
|
978
|
+
fwEarly = result; // High confidence, return immediately (below)
|
|
979
|
+
break;
|
|
980
|
+
}
|
|
873
981
|
candidates.push(result);
|
|
874
982
|
}
|
|
875
983
|
}
|
|
984
|
+
if (this.profileStages)
|
|
985
|
+
this.stageAdd('frameworks', ref, fwEarly !== null, tFw);
|
|
986
|
+
if (fwEarly)
|
|
987
|
+
return fwEarly;
|
|
876
988
|
// Strategy 2: Try import-based resolution
|
|
989
|
+
const tImp = this.profileStages ? process.hrtime.bigint() : 0n;
|
|
877
990
|
const importResult = this.gateLanguage((0, import_resolver_1.resolveViaImport)(ref, this.context), ref);
|
|
991
|
+
if (this.profileStages)
|
|
992
|
+
this.stageAdd('viaImport', ref, !!importResult, tImp);
|
|
878
993
|
if (importResult) {
|
|
879
994
|
if (importResult.confidence >= 0.9)
|
|
880
995
|
return importResult;
|
|
@@ -896,7 +1011,10 @@ class ReferenceResolver {
|
|
|
896
1011
|
: null;
|
|
897
1012
|
}
|
|
898
1013
|
// Strategy 3: Try name matching
|
|
1014
|
+
const tName = this.profileStages ? process.hrtime.bigint() : 0n;
|
|
899
1015
|
let nameResult = this.gateLanguage((0, name_matcher_1.matchReference)(ref, this.context), ref);
|
|
1016
|
+
if (this.profileStages)
|
|
1017
|
+
this.stageAdd('nameMatch', ref, !!nameResult, tName);
|
|
900
1018
|
// Nix has no ambient cross-file namespace — a callee binds lexically
|
|
901
1019
|
// (same file) or through explicit import/callPackage wiring (the import
|
|
902
1020
|
// path above). A cross-file name match is wrong by construction: every
|
|
@@ -1221,6 +1339,7 @@ class ReferenceResolver {
|
|
|
1221
1339
|
*/
|
|
1222
1340
|
async resolveBatchYielding(batch, maybeYield) {
|
|
1223
1341
|
this.warmCaches();
|
|
1342
|
+
this.advanceSupertypeGeneration();
|
|
1224
1343
|
const resolved = [];
|
|
1225
1344
|
const unresolved = [];
|
|
1226
1345
|
const byMethod = {};
|
|
@@ -1237,7 +1356,7 @@ class ReferenceResolver {
|
|
|
1237
1356
|
bindingForm: raw.bindingForm,
|
|
1238
1357
|
importedName: raw.importedName,
|
|
1239
1358
|
};
|
|
1240
|
-
const result = this.
|
|
1359
|
+
const result = this.resolveOneTimed(ref);
|
|
1241
1360
|
if (result) {
|
|
1242
1361
|
resolved.push(result);
|
|
1243
1362
|
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
|
@@ -1270,8 +1389,70 @@ class ReferenceResolver {
|
|
|
1270
1389
|
* of resolveBatchYielding, minus the main-thread yields (worker threads have
|
|
1271
1390
|
* no watchdog heartbeat to starve). Results are in input order.
|
|
1272
1391
|
*/
|
|
1392
|
+
/**
|
|
1393
|
+
* LATTICE_SENSOR_RESOLVE_PROFILE=1: per-outcome wall-clock histogram of
|
|
1394
|
+
* resolveOne, keyed by the winning strategy (`resolvedBy`) or
|
|
1395
|
+
* `fail:<referenceKind>` — the §7a.2 "profile the per-ref path" probe. The
|
|
1396
|
+
* kernel-scale batch loop is ~430s and CORE-INVARIANT (835.9s pooled-4-on-8
|
|
1397
|
+
* ≈ 812.5s sequential-on-2 for the whole superphase), so the next lever is
|
|
1398
|
+
* which CLASS of ref the time belongs to, not more parallelism. Off by
|
|
1399
|
+
* default: the hrtime pair costs ~100ns/ref only when the env is set.
|
|
1400
|
+
*/
|
|
1401
|
+
resolveProfile = process.env.LATTICE_SENSOR_RESOLVE_PROFILE ? new Map() : null;
|
|
1402
|
+
/**
|
|
1403
|
+
* LATTICE_SENSOR_RESOLVE_PROFILE=2 additionally attributes time to the
|
|
1404
|
+
* STRATEGIES inside resolveOne (`stage:<name>|<refKind>|hit/miss` rows in
|
|
1405
|
+
* the same histogram) — i.e. WHICH machinery a failing class of refs pays
|
|
1406
|
+
* for, not just that it fails. =1 keeps the per-outcome rows only.
|
|
1407
|
+
*/
|
|
1408
|
+
profileStages = process.env.LATTICE_SENSOR_RESOLVE_PROFILE === '2';
|
|
1409
|
+
stageAdd(stage, ref, hit, t0) {
|
|
1410
|
+
if (!this.resolveProfile)
|
|
1411
|
+
return;
|
|
1412
|
+
const dt = process.hrtime.bigint() - t0;
|
|
1413
|
+
const key = `stage:${stage}|${ref.referenceKind}|${hit ? 'hit' : 'miss'}`;
|
|
1414
|
+
const slot = this.resolveProfile.get(key);
|
|
1415
|
+
if (slot) {
|
|
1416
|
+
slot.n++;
|
|
1417
|
+
slot.ns += dt;
|
|
1418
|
+
}
|
|
1419
|
+
else {
|
|
1420
|
+
this.resolveProfile.set(key, { n: 1, ns: dt });
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
resolveOneTimed(ref) {
|
|
1424
|
+
if (!this.resolveProfile)
|
|
1425
|
+
return this.resolveOne(ref);
|
|
1426
|
+
const t0 = process.hrtime.bigint();
|
|
1427
|
+
const result = this.resolveOne(ref);
|
|
1428
|
+
const dt = process.hrtime.bigint() - t0;
|
|
1429
|
+
const key = result ? result.resolvedBy : `fail:${ref.referenceKind}`;
|
|
1430
|
+
const slot = this.resolveProfile.get(key);
|
|
1431
|
+
if (slot) {
|
|
1432
|
+
slot.n++;
|
|
1433
|
+
slot.ns += dt;
|
|
1434
|
+
}
|
|
1435
|
+
else {
|
|
1436
|
+
this.resolveProfile.set(key, { n: 1, ns: dt });
|
|
1437
|
+
}
|
|
1438
|
+
return result;
|
|
1439
|
+
}
|
|
1440
|
+
/** Dump the LATTICE_SENSOR_RESOLVE_PROFILE histogram to stderr (no-op when off). */
|
|
1441
|
+
dumpResolveProfile(label) {
|
|
1442
|
+
if (!this.resolveProfile || this.resolveProfile.size === 0)
|
|
1443
|
+
return;
|
|
1444
|
+
const rows = [...this.resolveProfile.entries()]
|
|
1445
|
+
.map(([k, v]) => ({ k, n: v.n, ms: Number(v.ns / 1000000n) }))
|
|
1446
|
+
.sort((a, b) => b.ms - a.ms);
|
|
1447
|
+
for (const r of rows) {
|
|
1448
|
+
console.error(`[resolve-profile] ${label} ${r.k}: n=${r.n} total=${(r.ms / 1000).toFixed(1)}s avg=${((r.ms * 1000) / Math.max(1, r.n)).toFixed(0)}µs`);
|
|
1449
|
+
}
|
|
1450
|
+
// =2 only: this thread's matchReference sub-stage table rides along.
|
|
1451
|
+
(0, name_matcher_1.dumpNameMatcherProfile)(label);
|
|
1452
|
+
}
|
|
1273
1453
|
resolveListForAdmission(refs) {
|
|
1274
1454
|
this.warmCaches();
|
|
1455
|
+
this.advanceSupertypeGeneration();
|
|
1275
1456
|
const resolved = [];
|
|
1276
1457
|
const unresolved = [];
|
|
1277
1458
|
const byMethod = {};
|
|
@@ -1288,7 +1469,7 @@ class ReferenceResolver {
|
|
|
1288
1469
|
bindingForm: raw.bindingForm,
|
|
1289
1470
|
importedName: raw.importedName,
|
|
1290
1471
|
};
|
|
1291
|
-
const result = this.
|
|
1472
|
+
const result = this.resolveOneTimed(ref);
|
|
1292
1473
|
if (result) {
|
|
1293
1474
|
resolved.push(result);
|
|
1294
1475
|
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
|
@@ -1333,7 +1514,12 @@ class ReferenceResolver {
|
|
|
1333
1514
|
// Sequential fallback on any pool failure. LATTICE_SENSOR_NO_PARALLEL_RESOLVE=1
|
|
1334
1515
|
// disables entirely. bulkEdgeLoad hooks (when provided) bracket the batch
|
|
1335
1516
|
// loop with drop/recreate of the non-unique edge indexes on big runs —
|
|
1336
|
-
// see DatabaseConnection.beginBulkEdgeLoad.
|
|
1517
|
+
// see DatabaseConnection.beginBulkEdgeLoad. backpressure (when provided)
|
|
1518
|
+
// is the WAL valve's writer-side backstop (WalCheckpointValve.backpressure):
|
|
1519
|
+
// called at pool-idle boundaries so a full backfill can actually complete —
|
|
1520
|
+
// the valve's timer-driven passive passes stay perpetually partial against
|
|
1521
|
+
// the pool's continuous reads, which is how a kernel-scale resolution grew
|
|
1522
|
+
// a 22GB WAL on a 4.6GB DB (migration plan §7a.1).
|
|
1337
1523
|
parallel) {
|
|
1338
1524
|
// Resolution runs on the indexer's MAIN thread, and the #850 liveness
|
|
1339
1525
|
// watchdog SIGKILLs a process whose event loop stalls past its window (60s
|
|
@@ -1342,6 +1528,19 @@ class ReferenceResolver {
|
|
|
1342
1528
|
// (#1091). A shared yielder lets both give the watchdog heartbeat a regular
|
|
1343
1529
|
// window to fire; see ./cooperative-yield.
|
|
1344
1530
|
const maybeYield = (0, cooperative_yield_1.createYielder)();
|
|
1531
|
+
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS) {
|
|
1532
|
+
console.error(`[pool-timing] backpressure hook: ${parallel?.backpressure ? 'present' : 'absent'}`);
|
|
1533
|
+
}
|
|
1534
|
+
// LATTICE_SENSOR_RESOLVE_PROFILE loop-stage attribution: the §7a.2 kernel-scale
|
|
1535
|
+
// histogram showed resolveOne owns only ~93s of the ~436s batch loop —
|
|
1536
|
+
// these counters name where the other ~340s goes (reads, edge build+insert,
|
|
1537
|
+
// deletes/marks, the per-batch count guard).
|
|
1538
|
+
const loopProf = process.env.LATTICE_SENSOR_RESOLVE_PROFILE
|
|
1539
|
+
? { read: 0, settle: 0, backpressure: 0, recycle: 0, createEdges: 0, insertEdges: 0, deletes: 0, marks: 0, countGuard: 0 }
|
|
1540
|
+
: null;
|
|
1541
|
+
const lp = (k, t0) => { if (loopProf)
|
|
1542
|
+
loopProf[k] = (loopProf[k] ?? 0) + (Date.now() - t0); };
|
|
1543
|
+
let tLp = 0;
|
|
1345
1544
|
await this.warmCachesYielding(maybeYield);
|
|
1346
1545
|
const total = this.queries.getUnresolvedReferencesCount();
|
|
1347
1546
|
let processed = 0;
|
|
@@ -1358,21 +1557,51 @@ class ReferenceResolver {
|
|
|
1358
1557
|
// costs zero wall-clock. Any failure downgrades to sequential permanently.
|
|
1359
1558
|
let pool = null;
|
|
1360
1559
|
let poolReady = false;
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1560
|
+
// True once pool creation has been attempted by EITHER engage site (the
|
|
1561
|
+
// up-front ref-count gate or the adaptive projection below) — a pool that
|
|
1562
|
+
// failed or was destroyed must stay down (downgrade is permanent), and
|
|
1563
|
+
// tryCreate's sizing probes shouldn't re-run every batch on hosts that
|
|
1564
|
+
// declined.
|
|
1565
|
+
let poolEngageTried = false;
|
|
1566
|
+
const createPool = (t0, why) => {
|
|
1567
|
+
poolEngageTried = true;
|
|
1568
|
+
if (!parallel)
|
|
1569
|
+
return null;
|
|
1570
|
+
const p = resolver_pool_1.ResolverPool.tryCreate(parallel.dbPath, this.projectRoot);
|
|
1571
|
+
p?.ready().then(() => {
|
|
1365
1572
|
poolReady = true;
|
|
1366
1573
|
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
|
|
1367
|
-
console.error(`[pool-timing] pool ready after ${Date.now() -
|
|
1368
|
-
}, () => {
|
|
1574
|
+
console.error(`[pool-timing] pool ready after ${Date.now() - t0}ms (${why})`);
|
|
1575
|
+
}, () => {
|
|
1576
|
+
void p.destroy().catch(() => undefined);
|
|
1577
|
+
if (pool === p)
|
|
1578
|
+
pool = null;
|
|
1579
|
+
});
|
|
1580
|
+
return p;
|
|
1581
|
+
};
|
|
1582
|
+
if (parallel && total >= (0, resolver_pool_1.minRefsForPool)()) {
|
|
1583
|
+
pool = createPool(Date.now(), 'ref-count');
|
|
1369
1584
|
}
|
|
1585
|
+
// Adaptive engagement bar (see the batch-loop hook): projected remaining
|
|
1586
|
+
// sequential settle above this boots the pool mid-loop. Boot is async and
|
|
1587
|
+
// fan-out waits for ready, so a marginal engage costs background boot
|
|
1588
|
+
// only; the bar just needs to clear the fan-out's own overhead class.
|
|
1589
|
+
const ADAPTIVE_ENGAGE_SETTLE_MS = 400;
|
|
1590
|
+
let adaptiveSeqMs = 0;
|
|
1591
|
+
let adaptiveSeqRefs = 0;
|
|
1370
1592
|
// Process in PIPELINED batches (double-buffer). The enumeration is the
|
|
1371
1593
|
// head of the pending set in rowid order; every ref a persisted batch
|
|
1372
1594
|
// processed leaves the pending set (resolved rows are deleted,
|
|
1373
1595
|
// unresolvable ones flip to status='failed'), shifting the remaining
|
|
1374
1596
|
// pending rows forward.
|
|
1375
1597
|
let prevRemaining = Number.POSITIVE_INFINITY;
|
|
1598
|
+
// Cadence for the worker connection recycling below — ~8 batches
|
|
1599
|
+
// ≈ 40k refs between recycles keeps the WAL shallow at kernel scale
|
|
1600
|
+
// while a small sync never recycles at all. (25 recovered only half
|
|
1601
|
+
// the write tax — the WAL re-deepened between recycles; reopens are
|
|
1602
|
+
// sub-millisecond so the shorter cadence is ~free.)
|
|
1603
|
+
const RECYCLE_EVERY_BATCHES = 8;
|
|
1604
|
+
let batchesSinceRecycle = 0;
|
|
1376
1605
|
// Begin one batch: fan out to the pool when it's ready and the batch is
|
|
1377
1606
|
// big enough — workers then resolve batch k+1 WHILE the main thread
|
|
1378
1607
|
// persists batch k (persist measured at ~58% of resolution wall on a
|
|
@@ -1433,20 +1662,94 @@ class ReferenceResolver {
|
|
|
1433
1662
|
}
|
|
1434
1663
|
catch { /* keep the indexes; inserts just pay the per-row maintenance */ }
|
|
1435
1664
|
}
|
|
1665
|
+
// Same gate for the ref-index window: the loop's deletes stop maintaining
|
|
1666
|
+
// the five sync-path unresolved_refs indexes, and the end-of-loop rebuild
|
|
1667
|
+
// is near-free (only failed refs survive the loop).
|
|
1668
|
+
let bulkRefsActive = false;
|
|
1669
|
+
if (parallel?.refIndexLoad && total >= (0, resolver_pool_1.minRefsForPool)()) {
|
|
1670
|
+
try {
|
|
1671
|
+
parallel.refIndexLoad.begin();
|
|
1672
|
+
bulkRefsActive = true;
|
|
1673
|
+
}
|
|
1674
|
+
catch { /* keep the indexes; deletes just pay the per-row maintenance */ }
|
|
1675
|
+
}
|
|
1436
1676
|
try {
|
|
1437
1677
|
try {
|
|
1438
|
-
|
|
1678
|
+
tLp = Date.now();
|
|
1679
|
+
let batch = this.queries.getUnresolvedReferencesBatchAfter(0, batchSize);
|
|
1680
|
+
lp('read', tLp);
|
|
1439
1681
|
let inFlight = batch.length > 0 ? beginBatch(batch) : null;
|
|
1440
1682
|
while (batch.length > 0 && inFlight) {
|
|
1441
1683
|
// Prefetch the NEXT batch before this one persists: this batch's rows
|
|
1442
1684
|
// are still pending (nothing has mutated the table since they were
|
|
1443
|
-
// read), so
|
|
1444
|
-
// enumeration yields the following batch
|
|
1445
|
-
|
|
1685
|
+
// read), so seeking past this batch's last row id in the same rowid
|
|
1686
|
+
// enumeration yields the following batch (keyset — OFFSET re-walked the
|
|
1687
|
+
// accumulated failed prefix every read, 54.6s at kernel scale, §7a.2).
|
|
1688
|
+
tLp = Date.now();
|
|
1689
|
+
const nextBatch = this.queries.getUnresolvedReferencesBatchAfter(batch[batch.length - 1].rowId, batchSize);
|
|
1690
|
+
lp('read', tLp);
|
|
1446
1691
|
const tBatch = Date.now();
|
|
1447
1692
|
const result = await settleBatch(inFlight, batch);
|
|
1448
1693
|
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
|
|
1449
1694
|
console.error(`[pool-timing] batch ${inFlight.mode}: ${batch.length} refs in ${Date.now() - tBatch}ms`);
|
|
1695
|
+
lp('settle', tBatch);
|
|
1696
|
+
// Adaptive pool engagement: the fixed ref-count gate can't see PER-REF
|
|
1697
|
+
// cost, and settle rates differ ~9× by language (56k Rust refs cost
|
|
1698
|
+
// more sequential settle than 154k Go refs — 36µs vs 4µs measured on
|
|
1699
|
+
// tokio/prometheus). After each sequential batch, project the remaining
|
|
1700
|
+
// settle from the observed rate and boot the pool mid-loop when it
|
|
1701
|
+
// clears the bar. The loop already switches to fan-out only when the
|
|
1702
|
+
// async boot reports ready, admission order is mode-independent, and
|
|
1703
|
+
// 2-core/low-memory hosts still decline inside tryCreate's sizing —
|
|
1704
|
+
// so the switch changes wall-clock, never the graph.
|
|
1705
|
+
if (inFlight.mode === 'seq' && parallel && pool === null && !poolEngageTried) {
|
|
1706
|
+
adaptiveSeqMs += Date.now() - tBatch;
|
|
1707
|
+
adaptiveSeqRefs += batch.length;
|
|
1708
|
+
const remaining = total - processed - batch.length;
|
|
1709
|
+
const projectedMs = (adaptiveSeqMs / Math.max(1, adaptiveSeqRefs)) * Math.max(0, remaining);
|
|
1710
|
+
if (projectedMs >= ADAPTIVE_ENGAGE_SETTLE_MS) {
|
|
1711
|
+
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS) {
|
|
1712
|
+
console.error(`[pool-timing] adaptive engage: projected ${Math.round(projectedMs)}ms sequential settle over ${remaining} remaining refs`);
|
|
1713
|
+
}
|
|
1714
|
+
pool = createPool(Date.now(), 'adaptive');
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
// WAL-valve backstop at the ONE pool-idle boundary of the double-buffer
|
|
1718
|
+
// (this batch settled, the next not yet fanned out): past the hard cap
|
|
1719
|
+
// the writer parks for a full backfill here, where the pool's readers
|
|
1720
|
+
// are all between statements — so the backfill completes, readers
|
|
1721
|
+
// re-enter at SQLite's backfilled mark, and the next persist commit
|
|
1722
|
+
// WRAPS the WAL instead of growing it. No-op (one fstat) under the cap.
|
|
1723
|
+
tLp = Date.now();
|
|
1724
|
+
const bp = parallel?.backpressure?.();
|
|
1725
|
+
if (bp)
|
|
1726
|
+
await bp;
|
|
1727
|
+
lp('backpressure', tLp);
|
|
1728
|
+
// Recycle the workers' read connections periodically at this same
|
|
1729
|
+
// worker-idle boundary (batch k settled, batch k+1 not yet fanned
|
|
1730
|
+
// out): a long-lived reader pins WAL checkpoint progress, and the
|
|
1731
|
+
// deep WAL that accumulates behind it taxes the writer's OWN page
|
|
1732
|
+
// operations — the §7a.6 writes-under-readers finding (deletes
|
|
1733
|
+
// 42.6s → 118.8s from 0 to 4 attached readers; an aggressive valve
|
|
1734
|
+
// recovered the writes but paid +129s in full-park folds). Releasing
|
|
1735
|
+
// the read marks every ~25 batches lets the existing checkpoints
|
|
1736
|
+
// advance instead, at ~milliseconds of reopen cost. A failed recycle
|
|
1737
|
+
// downgrades to sequential permanently, same as a failed fan-out.
|
|
1738
|
+
if (pool && poolReady && ++batchesSinceRecycle >= RECYCLE_EVERY_BATCHES) {
|
|
1739
|
+
batchesSinceRecycle = 0;
|
|
1740
|
+
tLp = Date.now();
|
|
1741
|
+
try {
|
|
1742
|
+
await pool.recycleWorkers();
|
|
1743
|
+
}
|
|
1744
|
+
catch (err) {
|
|
1745
|
+
(0, errors_1.logDebug)('Worker connection recycle failed; falling back to sequential', {
|
|
1746
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1747
|
+
});
|
|
1748
|
+
await pool.destroy().catch(() => undefined);
|
|
1749
|
+
pool = null;
|
|
1750
|
+
}
|
|
1751
|
+
lp('recycle', tLp);
|
|
1752
|
+
}
|
|
1450
1753
|
// Persist in bounded sub-transactions with yields between: a whole
|
|
1451
1754
|
// batch's edge insert / keyed deletes are otherwise one solid
|
|
1452
1755
|
// synchronous span each on a multi-GB index, sitting BETWEEN the
|
|
@@ -1463,11 +1766,15 @@ class ReferenceResolver {
|
|
|
1463
1766
|
// base class if those edges are visible. (Validated on dubbo: fanning
|
|
1464
1767
|
// out first downgraded exactly those supertype-method resolutions from
|
|
1465
1768
|
// the 0.9 typed-receiver path to the 0.65 word-overlap fallback.)
|
|
1769
|
+
tLp = Date.now();
|
|
1466
1770
|
const edges = this.createEdges(result.resolved);
|
|
1771
|
+
lp('createEdges', tLp);
|
|
1772
|
+
tLp = Date.now();
|
|
1467
1773
|
for (let i = 0; i < edges.length; i += PERSIST_CHUNK) {
|
|
1468
1774
|
this.queries.insertEdges(edges.slice(i, i + PERSIST_CHUNK));
|
|
1469
1775
|
await maybeYield();
|
|
1470
1776
|
}
|
|
1777
|
+
lp('insertEdges', tLp);
|
|
1471
1778
|
// NOW fan the next batch out — workers see exactly the edge state the
|
|
1472
1779
|
// sequential baseline would (every batch ≤ this one committed), while
|
|
1473
1780
|
// the main thread spends the REST of the persist (ref deletes + failed
|
|
@@ -1477,28 +1784,33 @@ class ReferenceResolver {
|
|
|
1477
1784
|
// by row id, so a same-key sibling ref in a LATER batch (same caller
|
|
1478
1785
|
// calling the same callee at another line) is left pending for its own
|
|
1479
1786
|
// attempt instead of being swept out with this batch's rows (#1269).
|
|
1787
|
+
tLp = Date.now();
|
|
1788
|
+
let removedThisBatch = 0;
|
|
1480
1789
|
const resolvedCleanup = ReferenceResolver.partitionResolvedCleanup(result.resolved);
|
|
1481
1790
|
for (let i = 0; i < resolvedCleanup.rowIds.length; i += PERSIST_CHUNK) {
|
|
1482
|
-
this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
|
1791
|
+
removedThisBatch += this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
|
1483
1792
|
await maybeYield();
|
|
1484
1793
|
}
|
|
1485
1794
|
for (let i = 0; i < resolvedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
|
1486
|
-
this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
|
1795
|
+
removedThisBatch += this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
|
1487
1796
|
await maybeYield();
|
|
1488
1797
|
}
|
|
1798
|
+
lp('deletes', tLp);
|
|
1489
1799
|
// Park unresolvable refs from this batch as status='failed' so they
|
|
1490
1800
|
// leave the pending set (the batch reader and non-progress guard below
|
|
1491
1801
|
// only see pending rows) but stay retryable when a later sync adds a
|
|
1492
1802
|
// symbol that could satisfy them (#1240).
|
|
1803
|
+
tLp = Date.now();
|
|
1493
1804
|
const failedCleanup = ReferenceResolver.partitionFailedCleanup(result.unresolved);
|
|
1494
1805
|
for (let i = 0; i < failedCleanup.byRowId.length; i += PERSIST_CHUNK) {
|
|
1495
|
-
this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
|
1806
|
+
removedThisBatch += this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
|
1496
1807
|
await maybeYield();
|
|
1497
1808
|
}
|
|
1498
1809
|
for (let i = 0; i < failedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
|
1499
|
-
this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
|
1810
|
+
removedThisBatch += this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
|
1500
1811
|
await maybeYield();
|
|
1501
1812
|
}
|
|
1813
|
+
lp('marks', tLp);
|
|
1502
1814
|
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
|
|
1503
1815
|
console.error(`[pool-timing] batch persist: ${Date.now() - tPersist}ms`);
|
|
1504
1816
|
// Aggregate stats
|
|
@@ -1530,10 +1842,23 @@ class ReferenceResolver {
|
|
|
1530
1842
|
// 1.4 GB before the Go-fallback fix). Stop rather than grow the graph
|
|
1531
1843
|
// without bound. (An in-flight prefetched batch is abandoned unsettled —
|
|
1532
1844
|
// fan-out has no side effects until settleBatch appends its results.)
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1845
|
+
// Non-progress signal, now O(1): `changes` summed across this batch's
|
|
1846
|
+
// deletes + failed-parks is the DIRECT evidence the guard's old count
|
|
1847
|
+
// diff inferred — a resolver returning a mismatched name makes the keyed
|
|
1848
|
+
// cleanup no-op, which shows up here as zero removals. The per-batch
|
|
1849
|
+
// COUNT(*) it replaces walked every remaining pending row — O(N²/batch)
|
|
1850
|
+
// over a run, 93.9s of the kernel-scale batch loop (§7a.2). A REAL count
|
|
1851
|
+
// runs only on the suspicious path (claimed-work batch removed nothing —
|
|
1852
|
+
// e.g. every row was a sibling a legacy-key sweep already consumed),
|
|
1853
|
+
// where it arbitrates stop-vs-continue exactly as before.
|
|
1854
|
+
if (removedThisBatch <= 0 && batch.length > 0) {
|
|
1855
|
+
tLp = Date.now();
|
|
1856
|
+
const remaining = this.queries.getUnresolvedReferencesCount();
|
|
1857
|
+
lp('countGuard', tLp);
|
|
1858
|
+
if (remaining >= prevRemaining)
|
|
1859
|
+
break;
|
|
1860
|
+
prevRemaining = remaining;
|
|
1861
|
+
}
|
|
1537
1862
|
// Advance the pipeline: the prefetched batch (already fanned out when
|
|
1538
1863
|
// the pool is on) becomes the current one.
|
|
1539
1864
|
batch = nextBatch;
|
|
@@ -1544,11 +1869,23 @@ class ReferenceResolver {
|
|
|
1544
1869
|
// Recreate the edge indexes BEFORE synthesis (kind-keyed reads) and on
|
|
1545
1870
|
// any error path. A crash before this line is healed by the next
|
|
1546
1871
|
// DatabaseConnection open (schema.sql re-applies IF NOT EXISTS).
|
|
1872
|
+
if (bulkRefsActive) {
|
|
1873
|
+
const tRef = Date.now();
|
|
1874
|
+
await parallel.refIndexLoad.end();
|
|
1875
|
+
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
|
|
1876
|
+
console.error(`[phase-timing] ref-index-recreate: ${Date.now() - tRef}ms`);
|
|
1877
|
+
}
|
|
1547
1878
|
if (bulkEdgesActive) {
|
|
1548
1879
|
const tIdx = Date.now();
|
|
1549
1880
|
await parallel.bulkEdgeLoad.end();
|
|
1550
1881
|
if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
|
|
1551
1882
|
console.error(`[phase-timing] edge-index-recreate: ${Date.now() - tIdx}ms`);
|
|
1883
|
+
// The recreate just wrote every non-unique edge index into the WAL
|
|
1884
|
+
// (multi-GB at kernel scale) with the pool idle — fold before the
|
|
1885
|
+
// synthesis passes pin readers against it for minutes.
|
|
1886
|
+
const bp = parallel?.backpressure?.();
|
|
1887
|
+
if (bp)
|
|
1888
|
+
await bp;
|
|
1552
1889
|
}
|
|
1553
1890
|
}
|
|
1554
1891
|
// Dynamic-edge synthesis: now that all base `calls` edges are persisted,
|
|
@@ -1560,7 +1897,7 @@ class ReferenceResolver {
|
|
|
1560
1897
|
// loop. See docs/design/callback-edge-synthesis.md.
|
|
1561
1898
|
const tSynth = Date.now();
|
|
1562
1899
|
try {
|
|
1563
|
-
aggregateStats.byMethod['callback-synthesis'] = await (0, callback_synthesizer_1.synthesizeCallbackEdges)(this.queries, this.context, onSynthesisProgress, pool);
|
|
1900
|
+
aggregateStats.byMethod['callback-synthesis'] = await (0, callback_synthesizer_1.synthesizeCallbackEdges)(this.queries, this.context, onSynthesisProgress, pool, parallel?.backpressure);
|
|
1564
1901
|
}
|
|
1565
1902
|
catch {
|
|
1566
1903
|
// synthesis is additive and optional; ignore failures
|
|
@@ -1572,6 +1909,11 @@ class ReferenceResolver {
|
|
|
1572
1909
|
if (pool)
|
|
1573
1910
|
await pool.destroy().catch(() => undefined);
|
|
1574
1911
|
}
|
|
1912
|
+
if (loopProf) {
|
|
1913
|
+
const parts = Object.entries(loopProf).map(([k, v]) => `${k}=${(v / 1000).toFixed(1)}s`).join(' ');
|
|
1914
|
+
console.error(`[resolve-profile] loop-stages ${parts}`);
|
|
1915
|
+
}
|
|
1916
|
+
this.dumpResolveProfile('main');
|
|
1575
1917
|
return {
|
|
1576
1918
|
resolved: [],
|
|
1577
1919
|
unresolved: [],
|