@soulcraft/cor 3.0.11 → 3.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -150,8 +150,12 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
150
150
  * rebuild is NOT redundant, and reporting ready serves empty adjacency.
151
151
  * Bounded O(1): one single-verb canonical page, only on the nothing-durable
152
152
  * path — after the gate-triggered rebuild persists the trees, later boots
153
- * latch edges-ready and never reach the probe. Truly edgeless brains (no
154
- * canonical verbs) stay ready, so the boot-rebuild/502 class stays dead.
153
+ * compute ready from the resident SSTables and never reach the probe.
154
+ * Truly edgeless brains (no canonical verbs) stay ready, so the
155
+ * boot-rebuild/502 class stays dead. The size() guard is honest under
156
+ * partial loads (3.0.13): the native count degrades to memtable-resident
157
+ * when the manifest's claim is not fully loaded, so a damaged durable set
158
+ * can no longer suppress this probe.
155
159
  */
156
160
  private probeCanonicalCoverage;
157
161
  /**
@@ -243,8 +243,12 @@ export class GraphAdjacencyIndex {
243
243
  * rebuild is NOT redundant, and reporting ready serves empty adjacency.
244
244
  * Bounded O(1): one single-verb canonical page, only on the nothing-durable
245
245
  * path — after the gate-triggered rebuild persists the trees, later boots
246
- * latch edges-ready and never reach the probe. Truly edgeless brains (no
247
- * canonical verbs) stay ready, so the boot-rebuild/502 class stays dead.
246
+ * compute ready from the resident SSTables and never reach the probe.
247
+ * Truly edgeless brains (no canonical verbs) stay ready, so the
248
+ * boot-rebuild/502 class stays dead. The size() guard is honest under
249
+ * partial loads (3.0.13): the native count degrades to memtable-resident
250
+ * when the manifest's claim is not fully loaded, so a damaged durable set
251
+ * can no longer suppress this probe.
248
252
  */
249
253
  async probeCanonicalCoverage() {
250
254
  if (this.native.isReady() || this.native.size() > 0)
@@ -566,14 +570,24 @@ export class GraphAdjacencyIndex {
566
570
  // dir): a rebuild is NOT redundant — see {@link probeCanonicalCoverage}.
567
571
  if (this.canonicalUnindexed)
568
572
  return false;
569
- // Cold-load has run but the edges-ready latch is unset. Ready ⇔ there is
570
- // nothing durable to load: no SSTables and empty memtables on either the
571
- // source or target tree. If any durable edges exist but didn't load, this
572
- // is false and brainy correctly rebuilds.
573
573
  const raw = this.native.getStats();
574
574
  const s = typeof raw === 'string' ? JSON.parse(raw) : raw;
575
+ // 3.0.13: the native engine reports manifest-vs-resident truth. A
576
+ // manifest naming SSTables that are not ALL resident means the graph
577
+ // would serve a subset of its edges — never ready (brainy's gate
578
+ // rebuilds; the rebuild persists a truthful manifest and clears this).
579
+ // Absent on pre-3.0.13 binaries (undefined ≠ false → skip).
580
+ if (s?.loadComplete === false)
581
+ return false;
582
+ // Native reported not-ready with a complete (or absent) durable set.
583
+ // Ready ⇔ there is nothing durable to load: no SSTables and empty
584
+ // memtables across ALL FOUR trees (the verbs trees were structurally
585
+ // invisible here before 3.0.13 — a dead verbs-source with a live
586
+ // source tree slipped through as "empty").
575
587
  const durableEmpty = Number(s?.sourceSstableCount ?? 0) === 0 &&
576
588
  Number(s?.targetSstableCount ?? 0) === 0 &&
589
+ Number(s?.verbsSourceSstableCount ?? 0) === 0 &&
590
+ Number(s?.verbsTargetSstableCount ?? 0) === 0 &&
577
591
  Number(s?.sourceMemTableSize ?? 0) === 0 &&
578
592
  Number(s?.targetMemTableSize ?? 0) === 0;
579
593
  return durableEmpty;
@@ -461,6 +461,20 @@ export interface GraphAdjacencyStats {
461
461
  verbIdCountBig: bigint;
462
462
  /** BigInt relationship type count. */
463
463
  relationshipTypeCountBig: bigint;
464
+ /**
465
+ * Verbs-source tree SSTable count (3.0.13 — the readiness fallback was
466
+ * structurally blind to the verbs trees before these fields existed).
467
+ * Absent on pre-3.0.13 binaries.
468
+ */
469
+ verbsSourceSstableCount?: number;
470
+ /** Verbs-target tree SSTable count. Absent on pre-3.0.13 binaries. */
471
+ verbsTargetSstableCount?: number;
472
+ /**
473
+ * Whether every manifest-named SSTable across all four trees is actually
474
+ * resident. `false` = the graph would serve a subset of its durable edges
475
+ * and reports not-ready. Absent on pre-3.0.13 binaries.
476
+ */
477
+ loadComplete?: boolean;
464
478
  }
465
479
  /**
466
480
  * Result from flushing a tree's MemTable to binary .sst format
@@ -1030,8 +1044,33 @@ export interface NativeLsmEngineInstance {
1030
1044
  * data lives in both the log and the persisted SSTable, so the log copy is
1031
1045
  * safe to retire. No-op for in-memory engines (constructed without
1032
1046
  * `openWithDir`).
1047
+ *
1048
+ * Returns the number of records CARRIED FORWARD across the shard-log
1049
+ * rotations — writes that landed between the flush's drain snapshot and
1050
+ * this commit (the wrapper's `await`s on blob + manifest IO let the event
1051
+ * loop run writes in that window). Carried records stay replayable in the
1052
+ * fresh active logs and drain with the next flush; pre-3.0.12 rotations
1053
+ * exiled them into never-replayed archives (CORTEX-RESTART-STRAND).
1054
+ */
1055
+ commitFlushThrough(throughEpoch: bigint): number;
1056
+ /**
1057
+ * Replay accounting from the memtable's cold open, as JSON:
1058
+ * `{ activeRecords, archiveRecovered, archivesUnreadable, tornTailBytes,
1059
+ * liveLogsRecreated, anomalousShards: string[] }`. Logged loudly at init
1060
+ * when any anomaly is non-zero — the forensic record of what cold-open
1061
+ * recovery found, captured BEFORE any rebuild wipes the evidence.
1062
+ * `liveLogsRecreated > 0` or `archivesUnreadable > 0` is a strand verdict
1063
+ * (unflushed records from a missing live log are unknowable).
1064
+ */
1065
+ replayStatsJson(): string;
1066
+ /**
1067
+ * Toggle bulk-load mode on the shard logs. `true` suspends the
1068
+ * per-append msyncs — rebuild-from-canonical ONLY (canonical is the
1069
+ * durability source; a crashed rebuild re-runs). `false` restores them
1070
+ * and performs one full msync per shard: the rebuild's single durability
1071
+ * point. Never used on the live write path.
1033
1072
  */
1034
- commitFlushThrough(throughEpoch: bigint): void;
1073
+ setBulkLoad(on: boolean): void;
1035
1074
  /**
1036
1075
  * Update the budget ceiling. Driven by the wrapper's
1037
1076
  * `ResourceManager` subscriber callback. Pass `0xffffffffffffffffn`
@@ -86,6 +86,18 @@ export declare class NativeColumnStore implements ColumnStoreProvider {
86
86
  * TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
87
87
  */
88
88
  private readonly deletedEntities;
89
+ /**
90
+ * Fields whose persisted state could not be FULLY loaded because a read
91
+ * FAULTED (threw) — as opposed to a blob being cleanly absent. A faulted
92
+ * field's column index may be silently partial (missing segments) or
93
+ * resurrect deleted rows (tombstones unapplied), so serving queries from
94
+ * it would return wrong results with no error. Queries against a faulted
95
+ * field THROW instead ({@link assertFieldServable}) — loud errors, never
96
+ * quiet losses. Cleared by {@link reset} (a rebuild repopulates cleanly)
97
+ * and {@link close}. Ready for brainy's loadBinaryBlob fault-vs-absent
98
+ * tightening (EIO throws instead of returning null).
99
+ */
100
+ private readonly faultedFields;
89
101
  /**
90
102
  * Create a native column store. Call {@link init} before queries that need
91
103
  * storage (loading persisted segments). Writes work immediately — the Rust
@@ -122,6 +134,15 @@ export declare class NativeColumnStore implements ColumnStoreProvider {
122
134
  * @param entityIntId - The entity's u32 id.
123
135
  */
124
136
  removeEntity(entityIntId: number): void;
137
+ /**
138
+ * @description Refuse to serve a field whose persisted column state
139
+ * faulted at load — a partial segment set or unapplied tombstones would
140
+ * return WRONG results silently. Throwing names the field and the heal
141
+ * path; the caller's error surfaces where a silent empty/partial result
142
+ * would have been invisible.
143
+ * @param field - Field about to be served.
144
+ */
145
+ private assertFieldServable;
125
146
  /**
126
147
  * @description Point filter: entity int ids whose `field` equals `value`.
127
148
  * Routes to the numeric/float/string Rust filter by the field's value type.
@@ -105,6 +105,18 @@ export class NativeColumnStore {
105
105
  * TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
106
106
  */
107
107
  deletedEntities = new Map();
108
+ /**
109
+ * Fields whose persisted state could not be FULLY loaded because a read
110
+ * FAULTED (threw) — as opposed to a blob being cleanly absent. A faulted
111
+ * field's column index may be silently partial (missing segments) or
112
+ * resurrect deleted rows (tombstones unapplied), so serving queries from
113
+ * it would return wrong results with no error. Queries against a faulted
114
+ * field THROW instead ({@link assertFieldServable}) — loud errors, never
115
+ * quiet losses. Cleared by {@link reset} (a rebuild repopulates cleanly)
116
+ * and {@link close}. Ready for brainy's loadBinaryBlob fault-vs-absent
117
+ * tightening (EIO throws instead of returning null).
118
+ */
119
+ faultedFields = new Set();
108
120
  /**
109
121
  * Create a native column store. Call {@link init} before queries that need
110
122
  * storage (loading persisted segments). Writes work immediately — the Rust
@@ -164,8 +176,13 @@ export class NativeColumnStore {
164
176
  await this.loadDeletedBitmap(field);
165
177
  }
166
178
  catch (error) {
167
- prodLog.warn(`NativeColumnStore: failed to load column field '${field}' skipping it; ` +
168
- `other fields are unaffected`, error);
179
+ // The field's manifest/segments faulted wholesalemark it faulted
180
+ // so queries THROW instead of silently returning empty (the
181
+ // pre-3.0.13 behavior: fieldTypes never set → filter() returned an
182
+ // empty bitmap with no error).
183
+ this.faultedFields.add(field);
184
+ prodLog.error(`NativeColumnStore: failed to load column field '${field}' — marked ` +
185
+ `unavailable (queries will throw); other fields are unaffected`, error);
169
186
  }
170
187
  }
171
188
  }
@@ -216,6 +233,22 @@ export class NativeColumnStore {
216
233
  deleted.add(entityIntId);
217
234
  }
218
235
  }
236
+ /**
237
+ * @description Refuse to serve a field whose persisted column state
238
+ * faulted at load — a partial segment set or unapplied tombstones would
239
+ * return WRONG results silently. Throwing names the field and the heal
240
+ * path; the caller's error surfaces where a silent empty/partial result
241
+ * would have been invisible.
242
+ * @param field - Field about to be served.
243
+ */
244
+ assertFieldServable(field) {
245
+ if (this.faultedFields.has(field)) {
246
+ throw new Error(`NativeColumnStore: column index for field '${field}' is UNAVAILABLE — ` +
247
+ `a persisted segment or tombstone read FAULTED at load (not merely ` +
248
+ `absent), so serving it would silently return partial/wrong results. ` +
249
+ `Heal: restore storage health and restart, or rebuild the metadata index.`);
250
+ }
251
+ }
219
252
  /**
220
253
  * @description Point filter: entity int ids whose `field` equals `value`.
221
254
  * Routes to the numeric/float/string Rust filter by the field's value type.
@@ -224,6 +257,7 @@ export class NativeColumnStore {
224
257
  * @returns Roaring bitmap of matching entity int ids.
225
258
  */
226
259
  async filter(field, value) {
260
+ this.assertFieldServable(field);
227
261
  const type = this.fieldTypes.get(field);
228
262
  if (type === undefined)
229
263
  return new RoaringBitmap32();
@@ -283,6 +317,7 @@ export class NativeColumnStore {
283
317
  * @returns Roaring bitmap of matching entity int ids.
284
318
  */
285
319
  async rangeQuery(field, min, max) {
320
+ this.assertFieldServable(field);
286
321
  const type = this.fieldTypes.get(field);
287
322
  if (type === undefined)
288
323
  return new RoaringBitmap32();
@@ -323,6 +358,7 @@ export class NativeColumnStore {
323
358
  * @returns Sorted array of entity int ids.
324
359
  */
325
360
  async sortTopK(field, order, k) {
361
+ this.assertFieldServable(field);
326
362
  if (k <= 0)
327
363
  return [];
328
364
  // Cor 3.0 Piece E: napi returns BigInt[]. The wrapper narrows
@@ -342,6 +378,7 @@ export class NativeColumnStore {
342
378
  * @returns Sorted array of entity int ids, all present in `filterBitmap`.
343
379
  */
344
380
  async filteredSortTopK(filterBitmap, field, order, k) {
381
+ this.assertFieldServable(field);
345
382
  if (k <= 0 || filterBitmap.size === 0)
346
383
  return [];
347
384
  // Intersecting only the global top-K could drop matches when the filter is
@@ -371,6 +408,7 @@ export class NativeColumnStore {
371
408
  * @returns Distinct values stringified the way brainy renders them.
372
409
  */
373
410
  async getFilterValues(field) {
411
+ this.assertFieldServable(field);
374
412
  if (!this.native.hasField(field))
375
413
  return [];
376
414
  return this.native.distinctValues(field);
@@ -408,6 +446,7 @@ export class NativeColumnStore {
408
446
  this.manifests.clear();
409
447
  this.fieldTypes.clear();
410
448
  this.deletedEntities.clear();
449
+ this.faultedFields.clear();
411
450
  this.storage = null;
412
451
  }
413
452
  /**
@@ -439,11 +478,14 @@ export class NativeColumnStore {
439
478
  }
440
479
  }
441
480
  // Drop all native state by replacing the engine, then clear TS-side state.
481
+ // Faults clear too: the rebuild that follows repopulates every field
482
+ // cleanly from canonical, so the refusal-to-serve verdict is spent.
442
483
  const bindings = loadNativeModule();
443
484
  this.native = new bindings.NativeColumnStore();
444
485
  this.manifests.clear();
445
486
  this.fieldTypes.clear();
446
487
  this.deletedEntities.clear();
488
+ this.faultedFields.clear();
447
489
  }
448
490
  // =========================================================================
449
491
  // Internal helpers
@@ -596,9 +638,33 @@ export class NativeColumnStore {
596
638
  prodLog.debug(`NativeColumnStore: mmap load failed for ${key}, trying buffer`, error);
597
639
  }
598
640
  }
599
- const buf = fallbackBuffer ?? (await this.storage.loadBinaryBlob(key));
641
+ // Fault-vs-absent discrimination (spine law; ready for brainy's
642
+ // loadBinaryBlob tightening where an IO fault THROWS instead of
643
+ // returning null): a FAULTED read means the segment may exist but be
644
+ // unreadable — serving the field without it would silently return
645
+ // partial results, so the field is marked faulted (queries throw) and
646
+ // the remaining fields/segments continue loading. A clean null stays
647
+ // an absent segment (warn + skip, unchanged semantics).
648
+ let buf;
649
+ try {
650
+ buf = fallbackBuffer ?? (await this.storage.loadBinaryBlob(key));
651
+ }
652
+ catch (error) {
653
+ this.faultedFields.add(field);
654
+ prodLog.error(`NativeColumnStore: segment read FAULTED for ${key} — field ` +
655
+ `'${field}' marked unavailable (queries will throw, not return ` +
656
+ `partial results):`, error);
657
+ return;
658
+ }
600
659
  if (buf) {
601
- this.native.loadSegmentFromBuffer(field, String(segId), buf);
660
+ try {
661
+ this.native.loadSegmentFromBuffer(field, String(segId), buf);
662
+ }
663
+ catch (error) {
664
+ this.faultedFields.add(field);
665
+ prodLog.error(`NativeColumnStore: segment attach FAULTED for ${key} (corrupt ` +
666
+ `bytes?) — field '${field}' marked unavailable:`, error);
667
+ }
602
668
  return;
603
669
  }
604
670
  // No raw blob for this segment. A 2.x→3.0 upgrade does NOT read legacy
@@ -615,7 +681,21 @@ export class NativeColumnStore {
615
681
  async loadDeletedBitmap(field) {
616
682
  if (!this.storage)
617
683
  return;
618
- const buf = await this.storage.loadBinaryBlob(deletedBlobKey(field));
684
+ // Fault-vs-absent: a FAULTED tombstone read leaves deleted rows
685
+ // resurrected in every query — worse than a missing segment, because
686
+ // the results are wrong rather than merely partial. Mark the field
687
+ // faulted (queries throw). A clean null = no tombstones, unchanged.
688
+ let buf;
689
+ try {
690
+ buf = await this.storage.loadBinaryBlob(deletedBlobKey(field));
691
+ }
692
+ catch (error) {
693
+ this.faultedFields.add(field);
694
+ prodLog.error(`NativeColumnStore: tombstone bitmap read FAULTED for field ` +
695
+ `'${field}' — deleted rows would resurrect; field marked ` +
696
+ `unavailable (queries will throw):`, error);
697
+ return;
698
+ }
619
699
  if (!buf)
620
700
  return;
621
701
  const bitmap = RoaringBitmap32.deserialize(buf, 'portable');
@@ -889,6 +889,15 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
889
889
  * @returns the number of entities indexed (those carrying metadata).
890
890
  */
891
891
  private rebuildEntityMetadata;
892
+ /**
893
+ * Rebuild-path verify-read for a single entity's canonical metadata.
894
+ * Retries once on a thrown read, then rethrows — a rebuild must NEVER
895
+ * silently exclude an entity because a storage read faulted (that
896
+ * completes the rebuild under-posted and clears the strand verdict
897
+ * over missing data). A clean `null`/`undefined` result is returned
898
+ * as `null` — a genuinely metadata-less record, the caller counts it.
899
+ */
900
+ private readRebuildMetadataOrThrow;
892
901
  /**
893
902
  * Batch-load metadata for a page of noun/verb ids — prefers the storage
894
903
  * adapter's batch API, falls back to per-id reads.
@@ -487,6 +487,35 @@ export class MetadataIndexManager {
487
487
  engine.registerSstablePath(path);
488
488
  }
489
489
  }
490
+ // Cold-open replay forensics (CORTEX-RESTART-STRAND). The engine's
491
+ // open replayed the active shard logs AND recovered any records a
492
+ // prior session exiled into archives (pre-carry-forward rotations,
493
+ // mid-rotation crashes). Surface every anomaly LOUDLY here — this
494
+ // line is the evidence that survives even if a rebuild later wipes
495
+ // the on-disk state.
496
+ if (metaDir !== null && typeof this.lsmEngine.replayStatsJson === 'function') {
497
+ try {
498
+ const stats = JSON.parse(this.lsmEngine.replayStatsJson());
499
+ if (stats.archiveRecovered > 0 ||
500
+ stats.archivesUnreadable > 0 ||
501
+ stats.tornTailBytes > 0) {
502
+ prodLog.error(`[NativeMetadataIndex] COLD-OPEN RECOVERY: replayed ` +
503
+ `${stats.activeRecords} active-log record(s); RECOVERED ` +
504
+ `${stats.archiveRecovered} record(s) exiled into archives by a ` +
505
+ `prior session; ${stats.archivesUnreadable} unreadable ` +
506
+ `archive(s); ${stats.tornTailBytes} torn-tail byte(s). ` +
507
+ `Recovered records are served and drain with the next flush. ` +
508
+ `Per-shard: ${stats.anomalousShards.join(' | ')}`);
509
+ }
510
+ }
511
+ catch (error) {
512
+ prodLog.warn('[NativeMetadataIndex] replay-forensics read failed (non-fatal):', error);
513
+ }
514
+ }
515
+ else if (metaDir !== null) {
516
+ prodLog.warn('[NativeMetadataIndex] replay forensics unavailable — the loaded ' +
517
+ 'native binary predates 3.0.12; upgrade the paired .node build.');
518
+ }
490
519
  }
491
520
  /**
492
521
  * **LSM durability (steps 3-4) — the sync part.** Register the
@@ -766,7 +795,18 @@ export class MetadataIndexManager {
766
795
  // flushed data is safely in the new SSTable. Warn + continue rather than
767
796
  // abort the rebuild — the next clean flush retires the log.
768
797
  try {
769
- engine.commitFlushThrough(result.lastDurableEpoch);
798
+ const carried = engine.commitFlushThrough(result.lastDurableEpoch);
799
+ // Writes that landed during THIS flush's blob/manifest awaits (the
800
+ // event loop runs between the drain and this commit) were carried
801
+ // into the fresh active logs — they stay replayable and drain with
802
+ // the next flush. Logged for the write-ack audit trail: pre-3.0.12
803
+ // rotations exiled exactly these records into never-replayed
804
+ // archives (CORTEX-RESTART-STRAND).
805
+ if (typeof carried === 'number' && carried > 0) {
806
+ prodLog.info(`flushLsm: carried ${carried} concurrent-write record(s) across ` +
807
+ `the shard-log rotation for SSTable ${id} — they drain with the ` +
808
+ `next flush.`);
809
+ }
770
810
  }
771
811
  catch (error) {
772
812
  prodLog.warn(`flushLsm: commitFlushThrough deferred for SSTable ${id} (it + the ` +
@@ -1197,6 +1237,36 @@ export class MetadataIndexManager {
1197
1237
  async detectDerivedStrand() {
1198
1238
  if (!this.lsmEngine || !this.lsmMetaDir)
1199
1239
  return;
1240
+ // 0. Replay-stats verdict (3.0.12). The native open now RECREATES a
1241
+ // missing live log and recovers what its archives hold, so the
1242
+ // filename census below can no longer see the deleted-live-log
1243
+ // signature — the native layer reports it instead. Either condition
1244
+ // is a strand verdict: a recreated-with-archives shard may have held
1245
+ // UNFLUSHED records that lived only in the missing live log
1246
+ // (unknowable — rebuild is the only honest answer), and an
1247
+ // unreadable archive means potentially-exiled records were
1248
+ // unreachable to recovery.
1249
+ try {
1250
+ const engine = this.lsmEngine;
1251
+ if (typeof engine.replayStatsJson === 'function') {
1252
+ const stats = JSON.parse(engine.replayStatsJson());
1253
+ const recreated = stats.liveLogsRecreated ?? 0;
1254
+ const unreadable = stats.archivesUnreadable ?? 0;
1255
+ if (recreated > 0 || unreadable > 0) {
1256
+ this.strandDetected = true;
1257
+ prodLog.error(`[NativeMetadataIndex] STRAND: cold-open replay reports ` +
1258
+ `${recreated} shard live log(s) MISSING (recreated; archives ` +
1259
+ `recovered what they held, but unflushed records from the ` +
1260
+ `missing logs are unknowable) and ${unreadable} unreadable ` +
1261
+ `archive(s). Reporting not-ready so the cold-open gate ` +
1262
+ `rebuilds from canonical. Per-shard: ` +
1263
+ `${(stats.anomalousShards ?? []).join(' | ')}`);
1264
+ }
1265
+ }
1266
+ }
1267
+ catch {
1268
+ // Verdict probe is advisory — a stats read failure must not block open.
1269
+ }
1200
1270
  // 1. Archive-orphan census over the memtable shard logs.
1201
1271
  try {
1202
1272
  const memtableDir = join(this.lsmMetaDir, 'memtable');
@@ -2584,8 +2654,31 @@ export class MetadataIndexManager {
2584
2654
  // branch loaded the whole corpus in a single getNouns/getVerbs({limit:1e6})
2585
2655
  // call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
2586
2656
  // batch(25) ≈ 250K. (#74 — perf-at-all-scales mandate.)
2587
- const totalNounsProcessed = await this.rebuildEntityMetadata('noun');
2588
- const totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
2657
+ // Bulk-load mode for the re-post walk: canonical is the durability
2658
+ // source during a rebuild (a crash re-runs it), so the per-append
2659
+ // shard-log msyncs — ~2 synchronous disk flushes per posting record,
2660
+ // the mechanism behind memory-vm's MEASURED ~1.3-2 items/s (30 min
2661
+ // for 2,354 entities on e2-standard-2, CORTEX-RESTART-STRAND) — are
2662
+ // suspended and replaced by ONE full msync per shard at the end of
2663
+ // the walk (setBulkLoad(false), inside finally so a failed rebuild
2664
+ // can never leave the live write path in deferred mode).
2665
+ const bulkCapable = this.lsmEngine !== null &&
2666
+ typeof this.lsmEngine.setBulkLoad ===
2667
+ 'function';
2668
+ if (bulkCapable)
2669
+ this.lsmEngine.setBulkLoad(true);
2670
+ let totalNounsProcessed;
2671
+ let totalVerbsProcessed;
2672
+ try {
2673
+ totalNounsProcessed = await this.rebuildEntityMetadata('noun');
2674
+ totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
2675
+ }
2676
+ finally {
2677
+ // Leaving bulk mode performs the walk's single durability point
2678
+ // (one full msync per shard log).
2679
+ if (bulkCapable)
2680
+ this.lsmEngine.setBulkLoad(false);
2681
+ }
2589
2682
  // Final flush
2590
2683
  await this.flushRebuildDirty();
2591
2684
  await this.flush();
@@ -2616,6 +2709,7 @@ export class MetadataIndexManager {
2616
2709
  let hasMore = true;
2617
2710
  let processed = 0;
2618
2711
  let sinceFlush = 0;
2712
+ let noMetadata = 0;
2619
2713
  while (hasMore) {
2620
2714
  const page = kind === 'noun'
2621
2715
  ? await this.storage.getNouns({ pagination: { limit: PAGE, cursor } })
@@ -2626,7 +2720,16 @@ export class MetadataIndexManager {
2626
2720
  const ids = items.map((it) => it.id);
2627
2721
  const md = await this.loadRebuildMetadataBatch(kind, ids);
2628
2722
  for (const it of items) {
2629
- const metadata = md.get(it.id);
2723
+ // An id enumerated by canonical but absent from the batch result
2724
+ // gets an individual verify-read: a batch adapter may return a
2725
+ // partial map on an internal fault, and silently skipping the
2726
+ // entity would complete the rebuild UNDER-POSTED — the exact
2727
+ // damage the strand detector exists to catch, laundered by its
2728
+ // own healer (the 01:31 memory-vm rebuild's failure shape). The
2729
+ // verify-read retries once, then FAILS THE REBUILD loudly; a
2730
+ // clean null (a genuinely metadata-less record) is counted +
2731
+ // reported, never silently dropped.
2732
+ const metadata = md.get(it.id) ?? (await this.readRebuildMetadataOrThrow(kind, it.id));
2630
2733
  if (metadata) {
2631
2734
  await this.addToIndex(it.id, metadata, true, true);
2632
2735
  processed++;
@@ -2635,13 +2738,49 @@ export class MetadataIndexManager {
2635
2738
  sinceFlush = 0;
2636
2739
  }
2637
2740
  }
2741
+ else {
2742
+ noMetadata++;
2743
+ }
2638
2744
  }
2639
2745
  hasMore = page.hasMore;
2640
2746
  cursor = page.nextCursor;
2641
2747
  await this.yieldToEventLoop();
2642
2748
  }
2749
+ if (noMetadata > 0) {
2750
+ prodLog.warn(`[NativeMetadataIndex] rebuild(${kind}): ${noMetadata} enumerated ` +
2751
+ `entit${noMetadata === 1 ? 'y' : 'ies'} carried no canonical ` +
2752
+ `metadata (verified by direct read) and were not indexed — ` +
2753
+ `expected only for metadata-less legacy records.`);
2754
+ }
2643
2755
  return processed;
2644
2756
  }
2757
+ /**
2758
+ * Rebuild-path verify-read for a single entity's canonical metadata.
2759
+ * Retries once on a thrown read, then rethrows — a rebuild must NEVER
2760
+ * silently exclude an entity because a storage read faulted (that
2761
+ * completes the rebuild under-posted and clears the strand verdict
2762
+ * over missing data). A clean `null`/`undefined` result is returned
2763
+ * as `null` — a genuinely metadata-less record, the caller counts it.
2764
+ */
2765
+ async readRebuildMetadataOrThrow(kind, id) {
2766
+ const read = () => kind === 'noun'
2767
+ ? this.storage.getNounMetadata(id)
2768
+ : this.storage.getVerbMetadata(id);
2769
+ try {
2770
+ return (await read()) ?? null;
2771
+ }
2772
+ catch {
2773
+ try {
2774
+ return (await read()) ?? null;
2775
+ }
2776
+ catch (error) {
2777
+ throw new Error(`[NativeMetadataIndex] rebuild(${kind}): canonical metadata read ` +
2778
+ `for '${id}' faulted twice — failing the rebuild rather than ` +
2779
+ `silently excluding the entity (an under-posted rebuild is the ` +
2780
+ `strand bug reintroduced by its own healer). Cause: ${String(error)}`);
2781
+ }
2782
+ }
2783
+ }
2645
2784
  /**
2646
2785
  * Batch-load metadata for a page of noun/verb ids — prefers the storage
2647
2786
  * adapter's batch API, falls back to per-id reads.
@@ -2656,14 +2795,13 @@ export class MetadataIndexManager {
2656
2795
  }
2657
2796
  const m = new Map();
2658
2797
  for (const id of ids) {
2659
- try {
2660
- const md = kind === 'noun'
2661
- ? await this.storage.getNounMetadata(id)
2662
- : await this.storage.getVerbMetadata(id);
2663
- if (md)
2664
- m.set(id, md);
2665
- }
2666
- catch { }
2798
+ // Retry-once-then-throw (readRebuildMetadataOrThrow): a faulted
2799
+ // read must fail the rebuild loudly, never silently exclude the
2800
+ // entity — a swallowed fault here completed the rebuild
2801
+ // under-posted (the strand bug re-created by its own healer).
2802
+ const md = await this.readRebuildMetadataOrThrow(kind, id);
2803
+ if (md)
2804
+ m.set(id, md);
2667
2805
  }
2668
2806
  return m;
2669
2807
  }
@@ -128,7 +128,13 @@ is the reference implementation.
128
128
  tail offset; live writers append past that offset without
129
129
  affecting what the snapshot reader sees. On compaction
130
130
  commit the delta is renamed to a `.archive` sibling and a
131
- fresh empty delta replaces it.
131
+ fresh delta replaces it, carrying forward any record newer
132
+ than the commit watermark — a record appended while the
133
+ commit's durability IO was in flight stays replayable in the
134
+ fresh delta, never exiled to the archive. Opens also scan
135
+ retained archives for records above the watermark and
136
+ recover them, so even a rotation interrupted between its two
137
+ renames loses nothing.
132
138
 
133
139
  3. **Atomic head pointer** (`head`) — a tiny text file named by
134
140
  `PointerFile` (`diskann::pointer_file`). Names the current
package/native/index.d.ts CHANGED
@@ -1169,14 +1169,25 @@ export declare class NativeGraphAdjacencyIndex {
1169
1169
  * empty (a fresh brain with nothing persisted) and brainy should build
1170
1170
  * it.
1171
1171
  *
1172
- * **Edges-loaded semantics — never membership.** Returns `true` iff the
1173
- * graph actually holds edges through a durable or live path: the
1174
- * cold-load latch (set when an SSTable loads from binary/file, a manifest
1175
- * names ≥1 SSTable, or a memtable flushes to a persisted SSTable), or a
1176
- * currently non-empty memtable on any of the four trees. The verb-id
1177
- * membership bitmap whose population is the deferred O(N) fast-follow
1178
- * is deliberately NOT consulted, so a slow membership walk can never make
1179
- * a graph with queryable edges report "not ready".
1172
+ * **Edges-loaded semantics — never membership, never a latch.**
1173
+ * COMPUTED from actual residency (3.0.13; previously a boolean latched
1174
+ * TRUE the moment a manifest merely NAMED an SSTable, before any file
1175
+ * opened a total load failure served ready forever):
1176
+ *
1177
+ * 1. If ANY tree's manifest names an SSTable that is not resident,
1178
+ * the graph is NOT ready it would serve a silent subset of its
1179
+ * edges (e.g. verbs-source dead while getNeighbors works). This
1180
+ * clause dominates: live memtable writes cannot mask a partial
1181
+ * durable load.
1182
+ * 2. Otherwise ready iff durable SSTables exist (all resident by
1183
+ * clause 1) or any memtable holds live edges. A fresh edgeless
1184
+ * brain reports false — the TS wrapper's edgeless-ready +
1185
+ * canonical-coverage logic owns that verdict.
1186
+ *
1187
+ * The verb-id membership bitmap — whose population is the deferred
1188
+ * O(N) fast-follow — is deliberately NOT consulted, so a slow
1189
+ * membership walk can never make a graph with queryable edges report
1190
+ * "not ready".
1180
1191
  */
1181
1192
  isReady(): boolean
1182
1193
  /**
@@ -1260,8 +1271,34 @@ export declare class NativeLsmEngine {
1260
1271
  *
1261
1272
  * No-op for in-memory engines (constructed without
1262
1273
  * `openWithDir`).
1263
- */
1264
- commitFlushThrough(throughEpoch: bigint): void
1274
+ *
1275
+ * Returns the number of records carried forward across the
1276
+ * shard-log rotations — writes that landed between the flush's
1277
+ * drain snapshot and this commit (the wrapper's async
1278
+ * durability-IO window; the JS event loop runs writes there).
1279
+ * They stay replayable in the fresh active logs and drain with
1280
+ * the next flush. The wrapper logs non-zero counts for the
1281
+ * write-ack audit trail.
1282
+ */
1283
+ commitFlushThrough(throughEpoch: bigint): number
1284
+ /**
1285
+ * Toggle bulk-load mode on the shard logs. `true` suspends the
1286
+ * per-append msyncs — for rebuild-from-canonical ONLY (canonical
1287
+ * is the durability source; a crashed rebuild re-runs). `false`
1288
+ * restores them and performs one full msync per shard: the
1289
+ * rebuild's single durability point. Never used on the live
1290
+ * write path, whose ack contract requires the per-append msync.
1291
+ */
1292
+ setBulkLoad(on: boolean): void
1293
+ /**
1294
+ * Replay accounting from the memtable's cold open, as JSON:
1295
+ * `{ activeRecords, archiveRecovered, archivesUnreadable,
1296
+ * tornTailBytes, anomalousShards: string[] }`. The wrapper logs
1297
+ * this at init when any anomaly is non-zero — the forensic
1298
+ * record of what cold-open recovery found, captured BEFORE any
1299
+ * rebuild wipes the evidence (CORTEX-RESTART-STRAND).
1300
+ */
1301
+ replayStatsJson(): string
1265
1302
  /** `"u32"` or `"u64"`. */
1266
1303
  idSpace(): string
1267
1304
  /**
@@ -2726,6 +2763,19 @@ export interface GraphAdjacencyStats {
2726
2763
  verbIdCountBig: bigint
2727
2764
  /** Number of relationship types as a `BigInt`. */
2728
2765
  relationshipTypeCountBig: bigint
2766
+ /**
2767
+ * Verbs-source tree SSTable count (3.0.13 — the readiness fallback
2768
+ * was structurally blind to the verbs trees before these fields).
2769
+ */
2770
+ verbsSourceSstableCount: number
2771
+ /** Verbs-target tree SSTable count. */
2772
+ verbsTargetSstableCount: number
2773
+ /**
2774
+ * Whether every manifest-named SSTable across all four trees is
2775
+ * actually resident. False means the graph is serving (at most) a
2776
+ * subset of its durable edges and `isReady()` reports not-ready.
2777
+ */
2778
+ loadComplete: boolean
2729
2779
  }
2730
2780
 
2731
2781
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@soulcraft/cor",
3
- "version": "3.0.11",
3
+ "version": "3.0.13",
4
4
  "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -82,7 +82,7 @@
82
82
  },
83
83
  "devDependencies": {
84
84
  "@napi-rs/cli": "^3.0.0",
85
- "@soulcraft/brainy": "8.2.2",
85
+ "@soulcraft/brainy": "8.2.6",
86
86
  "@types/node": "^22.0.0",
87
87
  "pg": "^8.21.0",
88
88
  "tsx": "^4.21.0",