@soulcraft/cor 3.0.11 → 3.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +6 -2
- package/dist/graph/NativeGraphAdjacencyIndex.js +20 -6
- package/dist/native/types.d.ts +40 -1
- package/dist/utils/NativeColumnStore.d.ts +21 -0
- package/dist/utils/NativeColumnStore.js +85 -5
- package/dist/utils/NativeMetadataIndex.d.ts +9 -0
- package/dist/utils/NativeMetadataIndex.js +150 -12
- package/docs/snapshot-safety.md +7 -1
- package/native/index.d.ts +60 -10
- package/package.json +2 -2
|
@@ -150,8 +150,12 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
|
|
|
150
150
|
* rebuild is NOT redundant, and reporting ready serves empty adjacency.
|
|
151
151
|
* Bounded O(1): one single-verb canonical page, only on the nothing-durable
|
|
152
152
|
* path — after the gate-triggered rebuild persists the trees, later boots
|
|
153
|
-
*
|
|
154
|
-
* canonical verbs) stay ready, so the
|
|
153
|
+
* compute ready from the resident SSTables and never reach the probe.
|
|
154
|
+
* Truly edgeless brains (no canonical verbs) stay ready, so the
|
|
155
|
+
* boot-rebuild/502 class stays dead. The size() guard is honest under
|
|
156
|
+
* partial loads (3.0.13): the native count degrades to memtable-resident
|
|
157
|
+
* when the manifest's claim is not fully loaded, so a damaged durable set
|
|
158
|
+
* can no longer suppress this probe.
|
|
155
159
|
*/
|
|
156
160
|
private probeCanonicalCoverage;
|
|
157
161
|
/**
|
|
@@ -243,8 +243,12 @@ export class GraphAdjacencyIndex {
|
|
|
243
243
|
* rebuild is NOT redundant, and reporting ready serves empty adjacency.
|
|
244
244
|
* Bounded O(1): one single-verb canonical page, only on the nothing-durable
|
|
245
245
|
* path — after the gate-triggered rebuild persists the trees, later boots
|
|
246
|
-
*
|
|
247
|
-
* canonical verbs) stay ready, so the
|
|
246
|
+
* compute ready from the resident SSTables and never reach the probe.
|
|
247
|
+
* Truly edgeless brains (no canonical verbs) stay ready, so the
|
|
248
|
+
* boot-rebuild/502 class stays dead. The size() guard is honest under
|
|
249
|
+
* partial loads (3.0.13): the native count degrades to memtable-resident
|
|
250
|
+
* when the manifest's claim is not fully loaded, so a damaged durable set
|
|
251
|
+
* can no longer suppress this probe.
|
|
248
252
|
*/
|
|
249
253
|
async probeCanonicalCoverage() {
|
|
250
254
|
if (this.native.isReady() || this.native.size() > 0)
|
|
@@ -566,14 +570,24 @@ export class GraphAdjacencyIndex {
|
|
|
566
570
|
// dir): a rebuild is NOT redundant — see {@link probeCanonicalCoverage}.
|
|
567
571
|
if (this.canonicalUnindexed)
|
|
568
572
|
return false;
|
|
569
|
-
// Cold-load has run but the edges-ready latch is unset. Ready ⇔ there is
|
|
570
|
-
// nothing durable to load: no SSTables and empty memtables on either the
|
|
571
|
-
// source or target tree. If any durable edges exist but didn't load, this
|
|
572
|
-
// is false and brainy correctly rebuilds.
|
|
573
573
|
const raw = this.native.getStats();
|
|
574
574
|
const s = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
|
575
|
+
// 3.0.13: the native engine reports manifest-vs-resident truth. A
|
|
576
|
+
// manifest naming SSTables that are not ALL resident means the graph
|
|
577
|
+
// would serve a subset of its edges — never ready (brainy's gate
|
|
578
|
+
// rebuilds; the rebuild persists a truthful manifest and clears this).
|
|
579
|
+
// Absent on pre-3.0.13 binaries (undefined ≠ false → skip).
|
|
580
|
+
if (s?.loadComplete === false)
|
|
581
|
+
return false;
|
|
582
|
+
// Native reported not-ready with a complete (or absent) durable set.
|
|
583
|
+
// Ready ⇔ there is nothing durable to load: no SSTables and empty
|
|
584
|
+
// memtables across ALL FOUR trees (the verbs trees were structurally
|
|
585
|
+
// invisible here before 3.0.13 — a dead verbs-source with a live
|
|
586
|
+
// source tree slipped through as "empty").
|
|
575
587
|
const durableEmpty = Number(s?.sourceSstableCount ?? 0) === 0 &&
|
|
576
588
|
Number(s?.targetSstableCount ?? 0) === 0 &&
|
|
589
|
+
Number(s?.verbsSourceSstableCount ?? 0) === 0 &&
|
|
590
|
+
Number(s?.verbsTargetSstableCount ?? 0) === 0 &&
|
|
577
591
|
Number(s?.sourceMemTableSize ?? 0) === 0 &&
|
|
578
592
|
Number(s?.targetMemTableSize ?? 0) === 0;
|
|
579
593
|
return durableEmpty;
|
package/dist/native/types.d.ts
CHANGED
|
@@ -461,6 +461,20 @@ export interface GraphAdjacencyStats {
|
|
|
461
461
|
verbIdCountBig: bigint;
|
|
462
462
|
/** BigInt relationship type count. */
|
|
463
463
|
relationshipTypeCountBig: bigint;
|
|
464
|
+
/**
|
|
465
|
+
* Verbs-source tree SSTable count (3.0.13 — the readiness fallback was
|
|
466
|
+
* structurally blind to the verbs trees before these fields existed).
|
|
467
|
+
* Absent on pre-3.0.13 binaries.
|
|
468
|
+
*/
|
|
469
|
+
verbsSourceSstableCount?: number;
|
|
470
|
+
/** Verbs-target tree SSTable count. Absent on pre-3.0.13 binaries. */
|
|
471
|
+
verbsTargetSstableCount?: number;
|
|
472
|
+
/**
|
|
473
|
+
* Whether every manifest-named SSTable across all four trees is actually
|
|
474
|
+
* resident. `false` = the graph would serve a subset of its durable edges
|
|
475
|
+
* and reports not-ready. Absent on pre-3.0.13 binaries.
|
|
476
|
+
*/
|
|
477
|
+
loadComplete?: boolean;
|
|
464
478
|
}
|
|
465
479
|
/**
|
|
466
480
|
* Result from flushing a tree's MemTable to binary .sst format
|
|
@@ -1030,8 +1044,33 @@ export interface NativeLsmEngineInstance {
|
|
|
1030
1044
|
* data lives in both the log and the persisted SSTable, so the log copy is
|
|
1031
1045
|
* safe to retire. No-op for in-memory engines (constructed without
|
|
1032
1046
|
* `openWithDir`).
|
|
1047
|
+
*
|
|
1048
|
+
* Returns the number of records CARRIED FORWARD across the shard-log
|
|
1049
|
+
* rotations — writes that landed between the flush's drain snapshot and
|
|
1050
|
+
* this commit (the wrapper's `await`s on blob + manifest IO let the event
|
|
1051
|
+
* loop run writes in that window). Carried records stay replayable in the
|
|
1052
|
+
* fresh active logs and drain with the next flush; pre-3.0.12 rotations
|
|
1053
|
+
* exiled them into never-replayed archives (CORTEX-RESTART-STRAND).
|
|
1054
|
+
*/
|
|
1055
|
+
commitFlushThrough(throughEpoch: bigint): number;
|
|
1056
|
+
/**
|
|
1057
|
+
* Replay accounting from the memtable's cold open, as JSON:
|
|
1058
|
+
* `{ activeRecords, archiveRecovered, archivesUnreadable, tornTailBytes,
|
|
1059
|
+
* liveLogsRecreated, anomalousShards: string[] }`. Logged loudly at init
|
|
1060
|
+
* when any anomaly is non-zero — the forensic record of what cold-open
|
|
1061
|
+
* recovery found, captured BEFORE any rebuild wipes the evidence.
|
|
1062
|
+
* `liveLogsRecreated > 0` or `archivesUnreadable > 0` is a strand verdict
|
|
1063
|
+
* (unflushed records from a missing live log are unknowable).
|
|
1064
|
+
*/
|
|
1065
|
+
replayStatsJson(): string;
|
|
1066
|
+
/**
|
|
1067
|
+
* Toggle bulk-load mode on the shard logs. `true` suspends the
|
|
1068
|
+
* per-append msyncs — rebuild-from-canonical ONLY (canonical is the
|
|
1069
|
+
* durability source; a crashed rebuild re-runs). `false` restores them
|
|
1070
|
+
* and performs one full msync per shard: the rebuild's single durability
|
|
1071
|
+
* point. Never used on the live write path.
|
|
1033
1072
|
*/
|
|
1034
|
-
|
|
1073
|
+
setBulkLoad(on: boolean): void;
|
|
1035
1074
|
/**
|
|
1036
1075
|
* Update the budget ceiling. Driven by the wrapper's
|
|
1037
1076
|
* `ResourceManager` subscriber callback. Pass `0xffffffffffffffffn`
|
|
@@ -86,6 +86,18 @@ export declare class NativeColumnStore implements ColumnStoreProvider {
|
|
|
86
86
|
* TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
|
|
87
87
|
*/
|
|
88
88
|
private readonly deletedEntities;
|
|
89
|
+
/**
|
|
90
|
+
* Fields whose persisted state could not be FULLY loaded because a read
|
|
91
|
+
* FAULTED (threw) — as opposed to a blob being cleanly absent. A faulted
|
|
92
|
+
* field's column index may be silently partial (missing segments) or
|
|
93
|
+
* resurrect deleted rows (tombstones unapplied), so serving queries from
|
|
94
|
+
* it would return wrong results with no error. Queries against a faulted
|
|
95
|
+
* field THROW instead ({@link assertFieldServable}) — loud errors, never
|
|
96
|
+
* quiet losses. Cleared by {@link reset} (a rebuild repopulates cleanly)
|
|
97
|
+
* and {@link close}. Ready for brainy's loadBinaryBlob fault-vs-absent
|
|
98
|
+
* tightening (EIO throws instead of returning null).
|
|
99
|
+
*/
|
|
100
|
+
private readonly faultedFields;
|
|
89
101
|
/**
|
|
90
102
|
* Create a native column store. Call {@link init} before queries that need
|
|
91
103
|
* storage (loading persisted segments). Writes work immediately — the Rust
|
|
@@ -122,6 +134,15 @@ export declare class NativeColumnStore implements ColumnStoreProvider {
|
|
|
122
134
|
* @param entityIntId - The entity's u32 id.
|
|
123
135
|
*/
|
|
124
136
|
removeEntity(entityIntId: number): void;
|
|
137
|
+
/**
|
|
138
|
+
* @description Refuse to serve a field whose persisted column state
|
|
139
|
+
* faulted at load — a partial segment set or unapplied tombstones would
|
|
140
|
+
* return WRONG results silently. Throwing names the field and the heal
|
|
141
|
+
* path; the caller's error surfaces where a silent empty/partial result
|
|
142
|
+
* would have been invisible.
|
|
143
|
+
* @param field - Field about to be served.
|
|
144
|
+
*/
|
|
145
|
+
private assertFieldServable;
|
|
125
146
|
/**
|
|
126
147
|
* @description Point filter: entity int ids whose `field` equals `value`.
|
|
127
148
|
* Routes to the numeric/float/string Rust filter by the field's value type.
|
|
@@ -105,6 +105,18 @@ export class NativeColumnStore {
|
|
|
105
105
|
* TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
|
|
106
106
|
*/
|
|
107
107
|
deletedEntities = new Map();
|
|
108
|
+
/**
|
|
109
|
+
* Fields whose persisted state could not be FULLY loaded because a read
|
|
110
|
+
* FAULTED (threw) — as opposed to a blob being cleanly absent. A faulted
|
|
111
|
+
* field's column index may be silently partial (missing segments) or
|
|
112
|
+
* resurrect deleted rows (tombstones unapplied), so serving queries from
|
|
113
|
+
* it would return wrong results with no error. Queries against a faulted
|
|
114
|
+
* field THROW instead ({@link assertFieldServable}) — loud errors, never
|
|
115
|
+
* quiet losses. Cleared by {@link reset} (a rebuild repopulates cleanly)
|
|
116
|
+
* and {@link close}. Ready for brainy's loadBinaryBlob fault-vs-absent
|
|
117
|
+
* tightening (EIO throws instead of returning null).
|
|
118
|
+
*/
|
|
119
|
+
faultedFields = new Set();
|
|
108
120
|
/**
|
|
109
121
|
* Create a native column store. Call {@link init} before queries that need
|
|
110
122
|
* storage (loading persisted segments). Writes work immediately — the Rust
|
|
@@ -164,8 +176,13 @@ export class NativeColumnStore {
|
|
|
164
176
|
await this.loadDeletedBitmap(field);
|
|
165
177
|
}
|
|
166
178
|
catch (error) {
|
|
167
|
-
|
|
168
|
-
|
|
179
|
+
// The field's manifest/segments faulted wholesale — mark it faulted
|
|
180
|
+
// so queries THROW instead of silently returning empty (the
|
|
181
|
+
// pre-3.0.13 behavior: fieldTypes never set → filter() returned an
|
|
182
|
+
// empty bitmap with no error).
|
|
183
|
+
this.faultedFields.add(field);
|
|
184
|
+
prodLog.error(`NativeColumnStore: failed to load column field '${field}' — marked ` +
|
|
185
|
+
`unavailable (queries will throw); other fields are unaffected`, error);
|
|
169
186
|
}
|
|
170
187
|
}
|
|
171
188
|
}
|
|
@@ -216,6 +233,22 @@ export class NativeColumnStore {
|
|
|
216
233
|
deleted.add(entityIntId);
|
|
217
234
|
}
|
|
218
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* @description Refuse to serve a field whose persisted column state
|
|
238
|
+
* faulted at load — a partial segment set or unapplied tombstones would
|
|
239
|
+
* return WRONG results silently. Throwing names the field and the heal
|
|
240
|
+
* path; the caller's error surfaces where a silent empty/partial result
|
|
241
|
+
* would have been invisible.
|
|
242
|
+
* @param field - Field about to be served.
|
|
243
|
+
*/
|
|
244
|
+
assertFieldServable(field) {
|
|
245
|
+
if (this.faultedFields.has(field)) {
|
|
246
|
+
throw new Error(`NativeColumnStore: column index for field '${field}' is UNAVAILABLE — ` +
|
|
247
|
+
`a persisted segment or tombstone read FAULTED at load (not merely ` +
|
|
248
|
+
`absent), so serving it would silently return partial/wrong results. ` +
|
|
249
|
+
`Heal: restore storage health and restart, or rebuild the metadata index.`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
219
252
|
/**
|
|
220
253
|
* @description Point filter: entity int ids whose `field` equals `value`.
|
|
221
254
|
* Routes to the numeric/float/string Rust filter by the field's value type.
|
|
@@ -224,6 +257,7 @@ export class NativeColumnStore {
|
|
|
224
257
|
* @returns Roaring bitmap of matching entity int ids.
|
|
225
258
|
*/
|
|
226
259
|
async filter(field, value) {
|
|
260
|
+
this.assertFieldServable(field);
|
|
227
261
|
const type = this.fieldTypes.get(field);
|
|
228
262
|
if (type === undefined)
|
|
229
263
|
return new RoaringBitmap32();
|
|
@@ -283,6 +317,7 @@ export class NativeColumnStore {
|
|
|
283
317
|
* @returns Roaring bitmap of matching entity int ids.
|
|
284
318
|
*/
|
|
285
319
|
async rangeQuery(field, min, max) {
|
|
320
|
+
this.assertFieldServable(field);
|
|
286
321
|
const type = this.fieldTypes.get(field);
|
|
287
322
|
if (type === undefined)
|
|
288
323
|
return new RoaringBitmap32();
|
|
@@ -323,6 +358,7 @@ export class NativeColumnStore {
|
|
|
323
358
|
* @returns Sorted array of entity int ids.
|
|
324
359
|
*/
|
|
325
360
|
async sortTopK(field, order, k) {
|
|
361
|
+
this.assertFieldServable(field);
|
|
326
362
|
if (k <= 0)
|
|
327
363
|
return [];
|
|
328
364
|
// Cor 3.0 Piece E: napi returns BigInt[]. The wrapper narrows
|
|
@@ -342,6 +378,7 @@ export class NativeColumnStore {
|
|
|
342
378
|
* @returns Sorted array of entity int ids, all present in `filterBitmap`.
|
|
343
379
|
*/
|
|
344
380
|
async filteredSortTopK(filterBitmap, field, order, k) {
|
|
381
|
+
this.assertFieldServable(field);
|
|
345
382
|
if (k <= 0 || filterBitmap.size === 0)
|
|
346
383
|
return [];
|
|
347
384
|
// Intersecting only the global top-K could drop matches when the filter is
|
|
@@ -371,6 +408,7 @@ export class NativeColumnStore {
|
|
|
371
408
|
* @returns Distinct values stringified the way brainy renders them.
|
|
372
409
|
*/
|
|
373
410
|
async getFilterValues(field) {
|
|
411
|
+
this.assertFieldServable(field);
|
|
374
412
|
if (!this.native.hasField(field))
|
|
375
413
|
return [];
|
|
376
414
|
return this.native.distinctValues(field);
|
|
@@ -408,6 +446,7 @@ export class NativeColumnStore {
|
|
|
408
446
|
this.manifests.clear();
|
|
409
447
|
this.fieldTypes.clear();
|
|
410
448
|
this.deletedEntities.clear();
|
|
449
|
+
this.faultedFields.clear();
|
|
411
450
|
this.storage = null;
|
|
412
451
|
}
|
|
413
452
|
/**
|
|
@@ -439,11 +478,14 @@ export class NativeColumnStore {
|
|
|
439
478
|
}
|
|
440
479
|
}
|
|
441
480
|
// Drop all native state by replacing the engine, then clear TS-side state.
|
|
481
|
+
// Faults clear too: the rebuild that follows repopulates every field
|
|
482
|
+
// cleanly from canonical, so the refusal-to-serve verdict is spent.
|
|
442
483
|
const bindings = loadNativeModule();
|
|
443
484
|
this.native = new bindings.NativeColumnStore();
|
|
444
485
|
this.manifests.clear();
|
|
445
486
|
this.fieldTypes.clear();
|
|
446
487
|
this.deletedEntities.clear();
|
|
488
|
+
this.faultedFields.clear();
|
|
447
489
|
}
|
|
448
490
|
// =========================================================================
|
|
449
491
|
// Internal helpers
|
|
@@ -596,9 +638,33 @@ export class NativeColumnStore {
|
|
|
596
638
|
prodLog.debug(`NativeColumnStore: mmap load failed for ${key}, trying buffer`, error);
|
|
597
639
|
}
|
|
598
640
|
}
|
|
599
|
-
|
|
641
|
+
// Fault-vs-absent discrimination (spine law; ready for brainy's
|
|
642
|
+
// loadBinaryBlob tightening where an IO fault THROWS instead of
|
|
643
|
+
// returning null): a FAULTED read means the segment may exist but be
|
|
644
|
+
// unreadable — serving the field without it would silently return
|
|
645
|
+
// partial results, so the field is marked faulted (queries throw) and
|
|
646
|
+
// the remaining fields/segments continue loading. A clean null stays
|
|
647
|
+
// an absent segment (warn + skip, unchanged semantics).
|
|
648
|
+
let buf;
|
|
649
|
+
try {
|
|
650
|
+
buf = fallbackBuffer ?? (await this.storage.loadBinaryBlob(key));
|
|
651
|
+
}
|
|
652
|
+
catch (error) {
|
|
653
|
+
this.faultedFields.add(field);
|
|
654
|
+
prodLog.error(`NativeColumnStore: segment read FAULTED for ${key} — field ` +
|
|
655
|
+
`'${field}' marked unavailable (queries will throw, not return ` +
|
|
656
|
+
`partial results):`, error);
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
600
659
|
if (buf) {
|
|
601
|
-
|
|
660
|
+
try {
|
|
661
|
+
this.native.loadSegmentFromBuffer(field, String(segId), buf);
|
|
662
|
+
}
|
|
663
|
+
catch (error) {
|
|
664
|
+
this.faultedFields.add(field);
|
|
665
|
+
prodLog.error(`NativeColumnStore: segment attach FAULTED for ${key} (corrupt ` +
|
|
666
|
+
`bytes?) — field '${field}' marked unavailable:`, error);
|
|
667
|
+
}
|
|
602
668
|
return;
|
|
603
669
|
}
|
|
604
670
|
// No raw blob for this segment. A 2.x→3.0 upgrade does NOT read legacy
|
|
@@ -615,7 +681,21 @@ export class NativeColumnStore {
|
|
|
615
681
|
async loadDeletedBitmap(field) {
|
|
616
682
|
if (!this.storage)
|
|
617
683
|
return;
|
|
618
|
-
|
|
684
|
+
// Fault-vs-absent: a FAULTED tombstone read leaves deleted rows
|
|
685
|
+
// resurrected in every query — worse than a missing segment, because
|
|
686
|
+
// the results are wrong rather than merely partial. Mark the field
|
|
687
|
+
// faulted (queries throw). A clean null = no tombstones, unchanged.
|
|
688
|
+
let buf;
|
|
689
|
+
try {
|
|
690
|
+
buf = await this.storage.loadBinaryBlob(deletedBlobKey(field));
|
|
691
|
+
}
|
|
692
|
+
catch (error) {
|
|
693
|
+
this.faultedFields.add(field);
|
|
694
|
+
prodLog.error(`NativeColumnStore: tombstone bitmap read FAULTED for field ` +
|
|
695
|
+
`'${field}' — deleted rows would resurrect; field marked ` +
|
|
696
|
+
`unavailable (queries will throw):`, error);
|
|
697
|
+
return;
|
|
698
|
+
}
|
|
619
699
|
if (!buf)
|
|
620
700
|
return;
|
|
621
701
|
const bitmap = RoaringBitmap32.deserialize(buf, 'portable');
|
|
@@ -889,6 +889,15 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
|
|
|
889
889
|
* @returns the number of entities indexed (those carrying metadata).
|
|
890
890
|
*/
|
|
891
891
|
private rebuildEntityMetadata;
|
|
892
|
+
/**
|
|
893
|
+
* Rebuild-path verify-read for a single entity's canonical metadata.
|
|
894
|
+
* Retries once on a thrown read, then rethrows — a rebuild must NEVER
|
|
895
|
+
* silently exclude an entity because a storage read faulted (that
|
|
896
|
+
* completes the rebuild under-posted and clears the strand verdict
|
|
897
|
+
* over missing data). A clean `null`/`undefined` result is returned
|
|
898
|
+
* as `null` — a genuinely metadata-less record, the caller counts it.
|
|
899
|
+
*/
|
|
900
|
+
private readRebuildMetadataOrThrow;
|
|
892
901
|
/**
|
|
893
902
|
* Batch-load metadata for a page of noun/verb ids — prefers the storage
|
|
894
903
|
* adapter's batch API, falls back to per-id reads.
|
|
@@ -487,6 +487,35 @@ export class MetadataIndexManager {
|
|
|
487
487
|
engine.registerSstablePath(path);
|
|
488
488
|
}
|
|
489
489
|
}
|
|
490
|
+
// Cold-open replay forensics (CORTEX-RESTART-STRAND). The engine's
|
|
491
|
+
// open replayed the active shard logs AND recovered any records a
|
|
492
|
+
// prior session exiled into archives (pre-carry-forward rotations,
|
|
493
|
+
// mid-rotation crashes). Surface every anomaly LOUDLY here — this
|
|
494
|
+
// line is the evidence that survives even if a rebuild later wipes
|
|
495
|
+
// the on-disk state.
|
|
496
|
+
if (metaDir !== null && typeof this.lsmEngine.replayStatsJson === 'function') {
|
|
497
|
+
try {
|
|
498
|
+
const stats = JSON.parse(this.lsmEngine.replayStatsJson());
|
|
499
|
+
if (stats.archiveRecovered > 0 ||
|
|
500
|
+
stats.archivesUnreadable > 0 ||
|
|
501
|
+
stats.tornTailBytes > 0) {
|
|
502
|
+
prodLog.error(`[NativeMetadataIndex] COLD-OPEN RECOVERY: replayed ` +
|
|
503
|
+
`${stats.activeRecords} active-log record(s); RECOVERED ` +
|
|
504
|
+
`${stats.archiveRecovered} record(s) exiled into archives by a ` +
|
|
505
|
+
`prior session; ${stats.archivesUnreadable} unreadable ` +
|
|
506
|
+
`archive(s); ${stats.tornTailBytes} torn-tail byte(s). ` +
|
|
507
|
+
`Recovered records are served and drain with the next flush. ` +
|
|
508
|
+
`Per-shard: ${stats.anomalousShards.join(' | ')}`);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
catch (error) {
|
|
512
|
+
prodLog.warn('[NativeMetadataIndex] replay-forensics read failed (non-fatal):', error);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
else if (metaDir !== null) {
|
|
516
|
+
prodLog.warn('[NativeMetadataIndex] replay forensics unavailable — the loaded ' +
|
|
517
|
+
'native binary predates 3.0.12; upgrade the paired .node build.');
|
|
518
|
+
}
|
|
490
519
|
}
|
|
491
520
|
/**
|
|
492
521
|
* **LSM durability (steps 3-4) — the sync part.** Register the
|
|
@@ -766,7 +795,18 @@ export class MetadataIndexManager {
|
|
|
766
795
|
// flushed data is safely in the new SSTable. Warn + continue rather than
|
|
767
796
|
// abort the rebuild — the next clean flush retires the log.
|
|
768
797
|
try {
|
|
769
|
-
engine.commitFlushThrough(result.lastDurableEpoch);
|
|
798
|
+
const carried = engine.commitFlushThrough(result.lastDurableEpoch);
|
|
799
|
+
// Writes that landed during THIS flush's blob/manifest awaits (the
|
|
800
|
+
// event loop runs between the drain and this commit) were carried
|
|
801
|
+
// into the fresh active logs — they stay replayable and drain with
|
|
802
|
+
// the next flush. Logged for the write-ack audit trail: pre-3.0.12
|
|
803
|
+
// rotations exiled exactly these records into never-replayed
|
|
804
|
+
// archives (CORTEX-RESTART-STRAND).
|
|
805
|
+
if (typeof carried === 'number' && carried > 0) {
|
|
806
|
+
prodLog.info(`flushLsm: carried ${carried} concurrent-write record(s) across ` +
|
|
807
|
+
`the shard-log rotation for SSTable ${id} — they drain with the ` +
|
|
808
|
+
`next flush.`);
|
|
809
|
+
}
|
|
770
810
|
}
|
|
771
811
|
catch (error) {
|
|
772
812
|
prodLog.warn(`flushLsm: commitFlushThrough deferred for SSTable ${id} (it + the ` +
|
|
@@ -1197,6 +1237,36 @@ export class MetadataIndexManager {
|
|
|
1197
1237
|
async detectDerivedStrand() {
|
|
1198
1238
|
if (!this.lsmEngine || !this.lsmMetaDir)
|
|
1199
1239
|
return;
|
|
1240
|
+
// 0. Replay-stats verdict (3.0.12). The native open now RECREATES a
|
|
1241
|
+
// missing live log and recovers what its archives hold, so the
|
|
1242
|
+
// filename census below can no longer see the deleted-live-log
|
|
1243
|
+
// signature — the native layer reports it instead. Either condition
|
|
1244
|
+
// is a strand verdict: a recreated-with-archives shard may have held
|
|
1245
|
+
// UNFLUSHED records that lived only in the missing live log
|
|
1246
|
+
// (unknowable — rebuild is the only honest answer), and an
|
|
1247
|
+
// unreadable archive means potentially-exiled records were
|
|
1248
|
+
// unreachable to recovery.
|
|
1249
|
+
try {
|
|
1250
|
+
const engine = this.lsmEngine;
|
|
1251
|
+
if (typeof engine.replayStatsJson === 'function') {
|
|
1252
|
+
const stats = JSON.parse(engine.replayStatsJson());
|
|
1253
|
+
const recreated = stats.liveLogsRecreated ?? 0;
|
|
1254
|
+
const unreadable = stats.archivesUnreadable ?? 0;
|
|
1255
|
+
if (recreated > 0 || unreadable > 0) {
|
|
1256
|
+
this.strandDetected = true;
|
|
1257
|
+
prodLog.error(`[NativeMetadataIndex] STRAND: cold-open replay reports ` +
|
|
1258
|
+
`${recreated} shard live log(s) MISSING (recreated; archives ` +
|
|
1259
|
+
`recovered what they held, but unflushed records from the ` +
|
|
1260
|
+
`missing logs are unknowable) and ${unreadable} unreadable ` +
|
|
1261
|
+
`archive(s). Reporting not-ready so the cold-open gate ` +
|
|
1262
|
+
`rebuilds from canonical. Per-shard: ` +
|
|
1263
|
+
`${(stats.anomalousShards ?? []).join(' | ')}`);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
catch {
|
|
1268
|
+
// Verdict probe is advisory — a stats read failure must not block open.
|
|
1269
|
+
}
|
|
1200
1270
|
// 1. Archive-orphan census over the memtable shard logs.
|
|
1201
1271
|
try {
|
|
1202
1272
|
const memtableDir = join(this.lsmMetaDir, 'memtable');
|
|
@@ -2584,8 +2654,31 @@ export class MetadataIndexManager {
|
|
|
2584
2654
|
// branch loaded the whole corpus in a single getNouns/getVerbs({limit:1e6})
|
|
2585
2655
|
// call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
|
|
2586
2656
|
// batch(25) ≈ 250K. (#74 — perf-at-all-scales mandate.)
|
|
2587
|
-
|
|
2588
|
-
|
|
2657
|
+
// Bulk-load mode for the re-post walk: canonical is the durability
|
|
2658
|
+
// source during a rebuild (a crash re-runs it), so the per-append
|
|
2659
|
+
// shard-log msyncs — ~2 synchronous disk flushes per posting record,
|
|
2660
|
+
// the mechanism behind memory-vm's MEASURED ~1.3-2 items/s (30 min
|
|
2661
|
+
// for 2,354 entities on e2-standard-2, CORTEX-RESTART-STRAND) — are
|
|
2662
|
+
// suspended and replaced by ONE full msync per shard at the end of
|
|
2663
|
+
// the walk (setBulkLoad(false), inside finally so a failed rebuild
|
|
2664
|
+
// can never leave the live write path in deferred mode).
|
|
2665
|
+
const bulkCapable = this.lsmEngine !== null &&
|
|
2666
|
+
typeof this.lsmEngine.setBulkLoad ===
|
|
2667
|
+
'function';
|
|
2668
|
+
if (bulkCapable)
|
|
2669
|
+
this.lsmEngine.setBulkLoad(true);
|
|
2670
|
+
let totalNounsProcessed;
|
|
2671
|
+
let totalVerbsProcessed;
|
|
2672
|
+
try {
|
|
2673
|
+
totalNounsProcessed = await this.rebuildEntityMetadata('noun');
|
|
2674
|
+
totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
|
|
2675
|
+
}
|
|
2676
|
+
finally {
|
|
2677
|
+
// Leaving bulk mode performs the walk's single durability point
|
|
2678
|
+
// (one full msync per shard log).
|
|
2679
|
+
if (bulkCapable)
|
|
2680
|
+
this.lsmEngine.setBulkLoad(false);
|
|
2681
|
+
}
|
|
2589
2682
|
// Final flush
|
|
2590
2683
|
await this.flushRebuildDirty();
|
|
2591
2684
|
await this.flush();
|
|
@@ -2616,6 +2709,7 @@ export class MetadataIndexManager {
|
|
|
2616
2709
|
let hasMore = true;
|
|
2617
2710
|
let processed = 0;
|
|
2618
2711
|
let sinceFlush = 0;
|
|
2712
|
+
let noMetadata = 0;
|
|
2619
2713
|
while (hasMore) {
|
|
2620
2714
|
const page = kind === 'noun'
|
|
2621
2715
|
? await this.storage.getNouns({ pagination: { limit: PAGE, cursor } })
|
|
@@ -2626,7 +2720,16 @@ export class MetadataIndexManager {
|
|
|
2626
2720
|
const ids = items.map((it) => it.id);
|
|
2627
2721
|
const md = await this.loadRebuildMetadataBatch(kind, ids);
|
|
2628
2722
|
for (const it of items) {
|
|
2629
|
-
|
|
2723
|
+
// An id enumerated by canonical but absent from the batch result
|
|
2724
|
+
// gets an individual verify-read: a batch adapter may return a
|
|
2725
|
+
// partial map on an internal fault, and silently skipping the
|
|
2726
|
+
// entity would complete the rebuild UNDER-POSTED — the exact
|
|
2727
|
+
// damage the strand detector exists to catch, laundered by its
|
|
2728
|
+
// own healer (the 01:31 memory-vm rebuild's failure shape). The
|
|
2729
|
+
// verify-read retries once, then FAILS THE REBUILD loudly; a
|
|
2730
|
+
// clean null (a genuinely metadata-less record) is counted +
|
|
2731
|
+
// reported, never silently dropped.
|
|
2732
|
+
const metadata = md.get(it.id) ?? (await this.readRebuildMetadataOrThrow(kind, it.id));
|
|
2630
2733
|
if (metadata) {
|
|
2631
2734
|
await this.addToIndex(it.id, metadata, true, true);
|
|
2632
2735
|
processed++;
|
|
@@ -2635,13 +2738,49 @@ export class MetadataIndexManager {
|
|
|
2635
2738
|
sinceFlush = 0;
|
|
2636
2739
|
}
|
|
2637
2740
|
}
|
|
2741
|
+
else {
|
|
2742
|
+
noMetadata++;
|
|
2743
|
+
}
|
|
2638
2744
|
}
|
|
2639
2745
|
hasMore = page.hasMore;
|
|
2640
2746
|
cursor = page.nextCursor;
|
|
2641
2747
|
await this.yieldToEventLoop();
|
|
2642
2748
|
}
|
|
2749
|
+
if (noMetadata > 0) {
|
|
2750
|
+
prodLog.warn(`[NativeMetadataIndex] rebuild(${kind}): ${noMetadata} enumerated ` +
|
|
2751
|
+
`entit${noMetadata === 1 ? 'y' : 'ies'} carried no canonical ` +
|
|
2752
|
+
`metadata (verified by direct read) and were not indexed — ` +
|
|
2753
|
+
`expected only for metadata-less legacy records.`);
|
|
2754
|
+
}
|
|
2643
2755
|
return processed;
|
|
2644
2756
|
}
|
|
2757
|
+
/**
|
|
2758
|
+
* Rebuild-path verify-read for a single entity's canonical metadata.
|
|
2759
|
+
* Retries once on a thrown read, then rethrows — a rebuild must NEVER
|
|
2760
|
+
* silently exclude an entity because a storage read faulted (that
|
|
2761
|
+
* completes the rebuild under-posted and clears the strand verdict
|
|
2762
|
+
* over missing data). A clean `null`/`undefined` result is returned
|
|
2763
|
+
* as `null` — a genuinely metadata-less record, the caller counts it.
|
|
2764
|
+
*/
|
|
2765
|
+
async readRebuildMetadataOrThrow(kind, id) {
|
|
2766
|
+
const read = () => kind === 'noun'
|
|
2767
|
+
? this.storage.getNounMetadata(id)
|
|
2768
|
+
: this.storage.getVerbMetadata(id);
|
|
2769
|
+
try {
|
|
2770
|
+
return (await read()) ?? null;
|
|
2771
|
+
}
|
|
2772
|
+
catch {
|
|
2773
|
+
try {
|
|
2774
|
+
return (await read()) ?? null;
|
|
2775
|
+
}
|
|
2776
|
+
catch (error) {
|
|
2777
|
+
throw new Error(`[NativeMetadataIndex] rebuild(${kind}): canonical metadata read ` +
|
|
2778
|
+
`for '${id}' faulted twice — failing the rebuild rather than ` +
|
|
2779
|
+
`silently excluding the entity (an under-posted rebuild is the ` +
|
|
2780
|
+
`strand bug reintroduced by its own healer). Cause: ${String(error)}`);
|
|
2781
|
+
}
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2645
2784
|
/**
|
|
2646
2785
|
* Batch-load metadata for a page of noun/verb ids — prefers the storage
|
|
2647
2786
|
* adapter's batch API, falls back to per-id reads.
|
|
@@ -2656,14 +2795,13 @@ export class MetadataIndexManager {
|
|
|
2656
2795
|
}
|
|
2657
2796
|
const m = new Map();
|
|
2658
2797
|
for (const id of ids) {
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
catch { }
|
|
2798
|
+
// Retry-once-then-throw (readRebuildMetadataOrThrow): a faulted
|
|
2799
|
+
// read must fail the rebuild loudly, never silently exclude the
|
|
2800
|
+
// entity — a swallowed fault here completed the rebuild
|
|
2801
|
+
// under-posted (the strand bug re-created by its own healer).
|
|
2802
|
+
const md = await this.readRebuildMetadataOrThrow(kind, id);
|
|
2803
|
+
if (md)
|
|
2804
|
+
m.set(id, md);
|
|
2667
2805
|
}
|
|
2668
2806
|
return m;
|
|
2669
2807
|
}
|
package/docs/snapshot-safety.md
CHANGED
|
@@ -128,7 +128,13 @@ is the reference implementation.
|
|
|
128
128
|
tail offset; live writers append past that offset without
|
|
129
129
|
affecting what the snapshot reader sees. On compaction
|
|
130
130
|
commit the delta is renamed to a `.archive` sibling and a
|
|
131
|
-
fresh
|
|
131
|
+
fresh delta replaces it, carrying forward any record newer
|
|
132
|
+
than the commit watermark — a record appended while the
|
|
133
|
+
commit's durability IO was in flight stays replayable in the
|
|
134
|
+
fresh delta, never exiled to the archive. Opens also scan
|
|
135
|
+
retained archives for records above the watermark and
|
|
136
|
+
recover them, so even a rotation interrupted between its two
|
|
137
|
+
renames loses nothing.
|
|
132
138
|
|
|
133
139
|
3. **Atomic head pointer** (`head`) — a tiny text file named by
|
|
134
140
|
`PointerFile` (`diskann::pointer_file`). Names the current
|
package/native/index.d.ts
CHANGED
|
@@ -1169,14 +1169,25 @@ export declare class NativeGraphAdjacencyIndex {
|
|
|
1169
1169
|
* empty (a fresh brain with nothing persisted) and brainy should build
|
|
1170
1170
|
* it.
|
|
1171
1171
|
*
|
|
1172
|
-
* **Edges-loaded semantics — never membership
|
|
1173
|
-
*
|
|
1174
|
-
*
|
|
1175
|
-
*
|
|
1176
|
-
*
|
|
1177
|
-
*
|
|
1178
|
-
* is
|
|
1179
|
-
*
|
|
1172
|
+
* **Edges-loaded semantics — never membership, never a latch.**
|
|
1173
|
+
* COMPUTED from actual residency (3.0.13; previously a boolean latched
|
|
1174
|
+
* TRUE the moment a manifest merely NAMED an SSTable, before any file
|
|
1175
|
+
* opened — a total load failure served ready forever):
|
|
1176
|
+
*
|
|
1177
|
+
* 1. If ANY tree's manifest names an SSTable that is not resident,
|
|
1178
|
+
* the graph is NOT ready — it would serve a silent subset of its
|
|
1179
|
+
* edges (e.g. verbs-source dead while getNeighbors works). This
|
|
1180
|
+
* clause dominates: live memtable writes cannot mask a partial
|
|
1181
|
+
* durable load.
|
|
1182
|
+
* 2. Otherwise ready iff durable SSTables exist (all resident by
|
|
1183
|
+
* clause 1) or any memtable holds live edges. A fresh edgeless
|
|
1184
|
+
* brain reports false — the TS wrapper's edgeless-ready +
|
|
1185
|
+
* canonical-coverage logic owns that verdict.
|
|
1186
|
+
*
|
|
1187
|
+
* The verb-id membership bitmap — whose population is the deferred
|
|
1188
|
+
* O(N) fast-follow — is deliberately NOT consulted, so a slow
|
|
1189
|
+
* membership walk can never make a graph with queryable edges report
|
|
1190
|
+
* "not ready".
|
|
1180
1191
|
*/
|
|
1181
1192
|
isReady(): boolean
|
|
1182
1193
|
/**
|
|
@@ -1260,8 +1271,34 @@ export declare class NativeLsmEngine {
|
|
|
1260
1271
|
*
|
|
1261
1272
|
* No-op for in-memory engines (constructed without
|
|
1262
1273
|
* `openWithDir`).
|
|
1263
|
-
|
|
1264
|
-
|
|
1274
|
+
*
|
|
1275
|
+
* Returns the number of records carried forward across the
|
|
1276
|
+
* shard-log rotations — writes that landed between the flush's
|
|
1277
|
+
* drain snapshot and this commit (the wrapper's async
|
|
1278
|
+
* durability-IO window; the JS event loop runs writes there).
|
|
1279
|
+
* They stay replayable in the fresh active logs and drain with
|
|
1280
|
+
* the next flush. The wrapper logs non-zero counts for the
|
|
1281
|
+
* write-ack audit trail.
|
|
1282
|
+
*/
|
|
1283
|
+
commitFlushThrough(throughEpoch: bigint): number
|
|
1284
|
+
/**
|
|
1285
|
+
* Toggle bulk-load mode on the shard logs. `true` suspends the
|
|
1286
|
+
* per-append msyncs — for rebuild-from-canonical ONLY (canonical
|
|
1287
|
+
* is the durability source; a crashed rebuild re-runs). `false`
|
|
1288
|
+
* restores them and performs one full msync per shard: the
|
|
1289
|
+
* rebuild's single durability point. Never used on the live
|
|
1290
|
+
* write path, whose ack contract requires the per-append msync.
|
|
1291
|
+
*/
|
|
1292
|
+
setBulkLoad(on: boolean): void
|
|
1293
|
+
/**
|
|
1294
|
+
* Replay accounting from the memtable's cold open, as JSON:
|
|
1295
|
+
* `{ activeRecords, archiveRecovered, archivesUnreadable,
|
|
1296
|
+
* tornTailBytes, anomalousShards: string[] }`. The wrapper logs
|
|
1297
|
+
* this at init when any anomaly is non-zero — the forensic
|
|
1298
|
+
* record of what cold-open recovery found, captured BEFORE any
|
|
1299
|
+
* rebuild wipes the evidence (CORTEX-RESTART-STRAND).
|
|
1300
|
+
*/
|
|
1301
|
+
replayStatsJson(): string
|
|
1265
1302
|
/** `"u32"` or `"u64"`. */
|
|
1266
1303
|
idSpace(): string
|
|
1267
1304
|
/**
|
|
@@ -2726,6 +2763,19 @@ export interface GraphAdjacencyStats {
|
|
|
2726
2763
|
verbIdCountBig: bigint
|
|
2727
2764
|
/** Number of relationship types as a `BigInt`. */
|
|
2728
2765
|
relationshipTypeCountBig: bigint
|
|
2766
|
+
/**
|
|
2767
|
+
* Verbs-source tree SSTable count (3.0.13 — the readiness fallback
|
|
2768
|
+
* was structurally blind to the verbs trees before these fields).
|
|
2769
|
+
*/
|
|
2770
|
+
verbsSourceSstableCount: number
|
|
2771
|
+
/** Verbs-target tree SSTable count. */
|
|
2772
|
+
verbsTargetSstableCount: number
|
|
2773
|
+
/**
|
|
2774
|
+
* Whether every manifest-named SSTable across all four trees is
|
|
2775
|
+
* actually resident. False means the graph is serving (at most) a
|
|
2776
|
+
* subset of its durable edges and `isReady()` reports not-ready.
|
|
2777
|
+
*/
|
|
2778
|
+
loadComplete: boolean
|
|
2729
2779
|
}
|
|
2730
2780
|
|
|
2731
2781
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/cor",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.13",
|
|
4
4
|
"description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
},
|
|
83
83
|
"devDependencies": {
|
|
84
84
|
"@napi-rs/cli": "^3.0.0",
|
|
85
|
-
"@soulcraft/brainy": "8.2.
|
|
85
|
+
"@soulcraft/brainy": "8.2.6",
|
|
86
86
|
"@types/node": "^22.0.0",
|
|
87
87
|
"pg": "^8.21.0",
|
|
88
88
|
"tsx": "^4.21.0",
|