@soulcraft/cor 3.0.15 → 3.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +7 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +57 -7
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +41 -1
- package/dist/hnsw/NativeDiskAnnWrapper.js +251 -2
- package/dist/native/types.d.ts +7 -0
- package/dist/utils/NativeColumnStore.d.ts +6 -0
- package/dist/utils/NativeColumnStore.js +61 -11
- package/dist/utils/NativeMetadataIndex.d.ts +30 -0
- package/dist/utils/NativeMetadataIndex.js +248 -8
- package/dist/utils/derivedFamilies.d.ts +48 -0
- package/dist/utils/derivedFamilies.js +128 -0
- package/dist/utils/invariantReport.d.ts +29 -0
- package/dist/utils/invariantReport.js +62 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +23 -1
- package/package.json +2 -2
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import type { StorageAdapter } from '@soulcraft/brainy';
|
|
19
19
|
import { NounType, VerbType } from '@soulcraft/brainy/types/graphTypes';
|
|
20
|
+
import type { ProviderInvariantReport } from '@soulcraft/brainy';
|
|
20
21
|
import { NativeColumnStore } from './NativeColumnStore.js';
|
|
21
22
|
import type { EntityIdMapperLike } from './columnStoreTypes.js';
|
|
22
23
|
import type { MetadataIndexProvider } from '../providerContracts.js';
|
|
@@ -115,6 +116,14 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
|
|
|
115
116
|
* everything from canonical. Cleared by a completed {@link rebuild}.
|
|
116
117
|
*/
|
|
117
118
|
private strandDetected;
|
|
119
|
+
/**
|
|
120
|
+
* SSTable ids actually REGISTERED into the live engine (cold-open
|
|
121
|
+
* registration + post-flush registration). The `manifest-residency`
|
|
122
|
+
* invariant (ADR-004 §6) compares the persisted manifest's claims
|
|
123
|
+
* against this set — the manifest is a claim; registration is the truth.
|
|
124
|
+
* Cleared on the rebuild's fresh re-open.
|
|
125
|
+
*/
|
|
126
|
+
private lsmRegisteredSstableIds;
|
|
118
127
|
/**
|
|
119
128
|
* **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
|
|
120
129
|
* auto-migrate enabled) and read once at the END of {@link init}, which kicks the
|
|
@@ -803,7 +812,28 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
|
|
|
803
812
|
getAllVFSEntityCounts(): Promise<Map<string, number>>;
|
|
804
813
|
getTotalVFSEntityCount(): Promise<number>;
|
|
805
814
|
getCountForCriteria(field: string, value: any): Promise<number>;
|
|
815
|
+
/**
|
|
816
|
+
* ADR-004 §6 — the provider's self-report of its cross-layer invariants.
|
|
817
|
+
* NEVER throws; bounded (residency + O(1) count reads, one limit-1
|
|
818
|
+
* canonical probe); safe on a live brain. Brainy's
|
|
819
|
+
* `validateIndexConsistency` feature-detects and aggregates this;
|
|
820
|
+
* `repairIndex` maps `heal: 'rebuild'` failures to {@link rebuild}.
|
|
821
|
+
*/
|
|
822
|
+
validateInvariants(): Promise<ProviderInvariantReport>;
|
|
823
|
+
/**
|
|
824
|
+
* Serializes every {@link flush} pass. Two flushes interleaving at the
|
|
825
|
+
* awaits is NOT hypothetical — memory-vm runs a periodic flusher AND
|
|
826
|
+
* brainy's shutdown hook flushes on SIGTERM — and the flushLsm pair
|
|
827
|
+
* both read `nextSstableId(manifest)` before either pushes: SAME id →
|
|
828
|
+
* same blob key → the second write OVERWRITES the first SSTable →
|
|
829
|
+
* postings lost with clean logs and NO replay anomaly (the exact
|
|
830
|
+
* acceptance-#1 signature on CORTEX-RESTART-STRAND, 2026-07-13).
|
|
831
|
+
* Chaining gives each caller its own full pass covering everything
|
|
832
|
+
* written before its call — the semantics a shutdown flush needs.
|
|
833
|
+
*/
|
|
834
|
+
private flushChain;
|
|
806
835
|
flush(): Promise<void>;
|
|
836
|
+
private flushOnce;
|
|
807
837
|
/**
|
|
808
838
|
* Persist all state and release the column store's resources. Brainy's
|
|
809
839
|
* shutdown path calls `metadataIndex.close()` when present; flushing here
|
|
@@ -34,6 +34,8 @@ const BUCKETED_INDEX_FIELDS = new Set([
|
|
|
34
34
|
]);
|
|
35
35
|
import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '@soulcraft/brainy/types/graphTypes';
|
|
36
36
|
import { loadNativeModule } from '../native/index.js';
|
|
37
|
+
import { composeInvariantReport } from './invariantReport.js';
|
|
38
|
+
import { declareDerivedFamilies, METADATA_FAMILIES } from './derivedFamilies.js';
|
|
37
39
|
import { openOrCreateBinaryIdMapper } from './binaryIdMapperFactory.js';
|
|
38
40
|
import { NativeColumnStore } from './NativeColumnStore.js';
|
|
39
41
|
import { ResourceManager } from '../resource/ResourceManager.js';
|
|
@@ -189,6 +191,14 @@ export class MetadataIndexManager {
|
|
|
189
191
|
* everything from canonical. Cleared by a completed {@link rebuild}.
|
|
190
192
|
*/
|
|
191
193
|
strandDetected = false;
|
|
194
|
+
/**
|
|
195
|
+
* SSTable ids actually REGISTERED into the live engine (cold-open
|
|
196
|
+
* registration + post-flush registration). The `manifest-residency`
|
|
197
|
+
* invariant (ADR-004 §6) compares the persisted manifest's claims
|
|
198
|
+
* against this set — the manifest is a claim; registration is the truth.
|
|
199
|
+
* Cleared on the rebuild's fresh re-open.
|
|
200
|
+
*/
|
|
201
|
+
lsmRegisteredSstableIds = new Set();
|
|
192
202
|
/**
|
|
193
203
|
* **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
|
|
194
204
|
* auto-migrate enabled) and read once at the END of {@link init}, which kicks the
|
|
@@ -485,6 +495,7 @@ export class MetadataIndexManager {
|
|
|
485
495
|
throw new Error(`openLsmEngine: getBinaryBlobPath returned null for persisted SSTable '${key}'`);
|
|
486
496
|
}
|
|
487
497
|
engine.registerSstablePath(path);
|
|
498
|
+
this.lsmRegisteredSstableIds.add(entry.id);
|
|
488
499
|
}
|
|
489
500
|
}
|
|
490
501
|
// Cold-open replay forensics (CORTEX-RESTART-STRAND). The engine's
|
|
@@ -699,6 +710,7 @@ export class MetadataIndexManager {
|
|
|
699
710
|
});
|
|
700
711
|
this.lsmEngine = engine;
|
|
701
712
|
this.lsmManifest = newEmptyLsmManifest();
|
|
713
|
+
this.lsmRegisteredSstableIds.clear();
|
|
702
714
|
this.native.setLsmEngine(engine);
|
|
703
715
|
// Re-apply the current admission ceiling immediately (the subscriber
|
|
704
716
|
// callback only refreshes it on the next rebalance/pressure tick). The
|
|
@@ -783,6 +795,7 @@ export class MetadataIndexManager {
|
|
|
783
795
|
throw new Error(`flushLsm: getBinaryBlobPath returned null for '${key}'`);
|
|
784
796
|
}
|
|
785
797
|
engine.registerSstablePath(path);
|
|
798
|
+
this.lsmRegisteredSstableIds.add(id);
|
|
786
799
|
// (4) Retire the memtable log below the flushed epoch. The `.sst` bytes
|
|
787
800
|
// AND the manifest referencing them are already durable here, so a failure
|
|
788
801
|
// at THIS step is non-fatal per the ordering invariant above: the records
|
|
@@ -1030,15 +1043,29 @@ export class MetadataIndexManager {
|
|
|
1030
1043
|
getUuid: (intId) => native.intToUuid(intId) ?? undefined,
|
|
1031
1044
|
getInt: (uuid) => native.uuidToInt(uuid) ?? undefined,
|
|
1032
1045
|
getOrAssign: (uuid) => {
|
|
1033
|
-
// The native index assigns ids inside addToIndex
|
|
1034
|
-
//
|
|
1035
|
-
//
|
|
1036
|
-
//
|
|
1046
|
+
// The native index assigns ids inside addToIndex; a mapped UUID
|
|
1047
|
+
// resolves to its existing int. An UNMAPPED uuid here is a
|
|
1048
|
+
// canonical-committed entity whose index ops were lost (the
|
|
1049
|
+
// restart-strand class — memory's boot heal hit exactly this,
|
|
1050
|
+
// CORTEX-RESTART-STRAND 2026-07-13): the cold vector rebuild
|
|
1051
|
+
// walks canonical and must be able to MINT the id through the
|
|
1052
|
+
// SAME allocator addToIndex uses (durable via the mapper's delta
|
|
1053
|
+
// log; a later addToIndex reuses it) instead of aborting the
|
|
1054
|
+
// whole heal. The entity's missing metadata postings stay
|
|
1055
|
+
// visible to posted-count-floor — this masks nothing.
|
|
1037
1056
|
const existing = native.uuidToInt(uuid);
|
|
1038
1057
|
if (existing !== null)
|
|
1039
1058
|
return existing;
|
|
1040
|
-
|
|
1041
|
-
`
|
|
1059
|
+
if (typeof native.getOrAssignInt === 'function') {
|
|
1060
|
+
prodLog.warn(`EntityIdMapper.getOrAssign: '${uuid}' had no native id at resolve ` +
|
|
1061
|
+
`time (canonical-committed entity with lost index ops) — assigned ` +
|
|
1062
|
+
`through the canonical allocator; its metadata postings heal via ` +
|
|
1063
|
+
`the posted-count invariant / rebuild.`);
|
|
1064
|
+
return Number(native.getOrAssignInt(uuid));
|
|
1065
|
+
}
|
|
1066
|
+
throw new Error(`EntityIdMapper.getOrAssign: '${uuid}' has no native id yet and the ` +
|
|
1067
|
+
`loaded native binary predates 3.0.16's getOrAssignInt — upgrade ` +
|
|
1068
|
+
`the paired .node build.`);
|
|
1042
1069
|
},
|
|
1043
1070
|
intsIterableToUuids: (ints) => native.intsToUuids([...ints]),
|
|
1044
1071
|
// Brainy 8.0 `restore()` calls this BEFORE the metadata/vector/graph
|
|
@@ -1073,6 +1100,9 @@ export class MetadataIndexManager {
|
|
|
1073
1100
|
// UUID ↔ entity-int allocator. Must run before columnStore.init (which
|
|
1074
1101
|
// resolves ids through getIdMapper()) and before any mutation.
|
|
1075
1102
|
this.wireIdMapper();
|
|
1103
|
+
// ADR-004 §7: declare this provider's protected families (feature-
|
|
1104
|
+
// detected on brainy ≥8.3.0; idempotent upsert; loud-but-non-blocking).
|
|
1105
|
+
await declareDerivedFamilies(this.storage, METADATA_FAMILIES);
|
|
1076
1106
|
// LSM durability (steps 3-4): open the native LSM engine here (not in the
|
|
1077
1107
|
// constructor) so it can be seeded from the on-disk manifest and cold-load
|
|
1078
1108
|
// every persisted SSTable into its live read set — the entire restore,
|
|
@@ -1114,6 +1144,7 @@ export class MetadataIndexManager {
|
|
|
1114
1144
|
prodLog.info('[NativeMetadataIndex] Field registry missing but the LSM engine ' +
|
|
1115
1145
|
'recovered durable state (SSTables/memtable) — serving from it, ' +
|
|
1116
1146
|
'NO rebuild.');
|
|
1147
|
+
this.readinessInitialized = true;
|
|
1117
1148
|
return;
|
|
1118
1149
|
}
|
|
1119
1150
|
// **Tripwire — a rebuild here on a PREVIOUSLY-HEALTHY brain is a bug,
|
|
@@ -1142,11 +1173,21 @@ export class MetadataIndexManager {
|
|
|
1142
1173
|
}
|
|
1143
1174
|
console.warn(`[NativeMetadataIndex] Field registry missing but ${probe.totalCount ?? 'unknown'} entities exist — rebuilding index`);
|
|
1144
1175
|
await this.rebuild();
|
|
1176
|
+
// A completed rebuild IS the readiness determination for this
|
|
1177
|
+
// open (sweep finding: these early exits previously left
|
|
1178
|
+
// readinessInitialized false — a fresh-rebuilt brain reported
|
|
1179
|
+
// not-ready forever, and brainy's gate re-rebuilt every boot).
|
|
1180
|
+
this.readinessInitialized = true;
|
|
1145
1181
|
return;
|
|
1146
1182
|
}
|
|
1183
|
+
// Probe says genuinely empty — an empty brain is READY by contract.
|
|
1184
|
+
this.readinessInitialized = true;
|
|
1147
1185
|
}
|
|
1148
1186
|
catch {
|
|
1149
|
-
// Storage probe failed —
|
|
1187
|
+
// Storage probe failed — treated as empty per current semantics;
|
|
1188
|
+
// readiness stays UNINITIALIZED (conservative: a faulted probe must
|
|
1189
|
+
// not certify an unverified brain as ready — the fault-vs-absent
|
|
1190
|
+
// discrimination for this probe is tracked ADR-004 §4 M-work).
|
|
1150
1191
|
}
|
|
1151
1192
|
return;
|
|
1152
1193
|
}
|
|
@@ -1946,6 +1987,14 @@ export class MetadataIndexManager {
|
|
|
1946
1987
|
const resultJson = this.native.removeFromIndex(id);
|
|
1947
1988
|
const result = JSON.parse(resultJson);
|
|
1948
1989
|
await this.persistMutationResult(result);
|
|
1990
|
+
// Count symmetry (sweep finding; ADR-004 anti-pattern C): this path
|
|
1991
|
+
// has no metadata to know WHICH type to decrement, and skipping the
|
|
1992
|
+
// decrement permanently inflated totalEntitiesByType /
|
|
1993
|
+
// entityCountsByTypeFixed until restart. Instead of guessing,
|
|
1994
|
+
// RECOMPUTE the counters from native ground truth — bounded
|
|
1995
|
+
// (per-type O(1) bitmap-cardinality reads, no id materialization).
|
|
1996
|
+
await this.lazyLoadCounts();
|
|
1997
|
+
this.syncTypeCountsToFixed();
|
|
1949
1998
|
}
|
|
1950
1999
|
// Tombstone in the column store so the entity drops out of sort/filter/range.
|
|
1951
2000
|
if (intId !== null) {
|
|
@@ -2278,7 +2327,195 @@ export class MetadataIndexManager {
|
|
|
2278
2327
|
// ==========================================================================
|
|
2279
2328
|
// Flush
|
|
2280
2329
|
// ==========================================================================
|
|
2330
|
+
/**
|
|
2331
|
+
* ADR-004 §6 — the provider's self-report of its cross-layer invariants.
|
|
2332
|
+
* NEVER throws; bounded (residency + O(1) count reads, one limit-1
|
|
2333
|
+
* canonical probe); safe on a live brain. Brainy's
|
|
2334
|
+
* `validateIndexConsistency` feature-detects and aggregates this;
|
|
2335
|
+
* `repairIndex` maps `heal: 'rebuild'` failures to {@link rebuild}.
|
|
2336
|
+
*/
|
|
2337
|
+
async validateInvariants() {
|
|
2338
|
+
return composeInvariantReport('metadata', () => this.isReady(), [
|
|
2339
|
+
// manifest-residency: every persisted-manifest-named SSTable is
|
|
2340
|
+
// actually registered in the live engine.
|
|
2341
|
+
() => {
|
|
2342
|
+
const manifest = this.lsmManifest;
|
|
2343
|
+
if (manifest === null || this.lsmMetaDir === null) {
|
|
2344
|
+
return {
|
|
2345
|
+
name: 'manifest-residency',
|
|
2346
|
+
holds: true,
|
|
2347
|
+
detail: 'in-RAM engine or fresh brain — no persisted SSTables to verify',
|
|
2348
|
+
heal: 'none',
|
|
2349
|
+
};
|
|
2350
|
+
}
|
|
2351
|
+
const missing = manifest.sstables
|
|
2352
|
+
.map((e) => e.id)
|
|
2353
|
+
.filter((id) => !this.lsmRegisteredSstableIds.has(id));
|
|
2354
|
+
return {
|
|
2355
|
+
name: 'manifest-residency',
|
|
2356
|
+
holds: missing.length === 0,
|
|
2357
|
+
detail: missing.length === 0
|
|
2358
|
+
? `${manifest.sstables.length} manifest-named SSTable(s), all registered`
|
|
2359
|
+
: `manifest names ${manifest.sstables.length} SSTable(s) but ` +
|
|
2360
|
+
`${missing.length} never registered (ids: ${missing.join(', ')}) — ` +
|
|
2361
|
+
`the index serves a subset of its durable postings`,
|
|
2362
|
+
expected: manifest.sstables.length,
|
|
2363
|
+
actual: manifest.sstables.length - missing.length,
|
|
2364
|
+
heal: 'rebuild',
|
|
2365
|
+
};
|
|
2366
|
+
},
|
|
2367
|
+
// posted-count-floor: Σ posted noun counts ≥ canonical noun count.
|
|
2368
|
+
async () => {
|
|
2369
|
+
const getNouns = this.storage.getNouns;
|
|
2370
|
+
if (typeof getNouns !== 'function') {
|
|
2371
|
+
return {
|
|
2372
|
+
name: 'posted-count-floor',
|
|
2373
|
+
holds: true,
|
|
2374
|
+
detail: 'adapter exposes no getNouns — floor unverifiable, skipped',
|
|
2375
|
+
heal: 'none',
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2378
|
+
let canonical;
|
|
2379
|
+
try {
|
|
2380
|
+
const probe = await getNouns.call(this.storage, { pagination: { limit: 1 } });
|
|
2381
|
+
canonical = probe?.totalCount;
|
|
2382
|
+
}
|
|
2383
|
+
catch (error) {
|
|
2384
|
+
return {
|
|
2385
|
+
name: 'posted-count-floor',
|
|
2386
|
+
holds: false,
|
|
2387
|
+
detail: `canonical count probe FAULTED — divergence unverifiable: ${String(error)}`,
|
|
2388
|
+
heal: 'none',
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
if (typeof canonical !== 'number' || !Number.isFinite(canonical) || canonical >= 0xffff_ffff) {
|
|
2392
|
+
return {
|
|
2393
|
+
name: 'posted-count-floor',
|
|
2394
|
+
holds: true,
|
|
2395
|
+
detail: `canonical count not exactly countable (${String(canonical)}) — floor skipped by design`,
|
|
2396
|
+
heal: 'none',
|
|
2397
|
+
};
|
|
2398
|
+
}
|
|
2399
|
+
const { NounType } = await import('@soulcraft/brainy');
|
|
2400
|
+
let posted = 0;
|
|
2401
|
+
for (const t of Object.values(NounType)) {
|
|
2402
|
+
posted += this.native.getEntityCountByType(t);
|
|
2403
|
+
}
|
|
2404
|
+
return {
|
|
2405
|
+
name: 'posted-count-floor',
|
|
2406
|
+
holds: posted >= canonical,
|
|
2407
|
+
detail: posted >= canonical
|
|
2408
|
+
? `posted ${posted} ≥ canonical ${canonical}`
|
|
2409
|
+
: `canonical holds ${canonical} entities but only ${posted} are ` +
|
|
2410
|
+
`posted — ${canonical - posted} durable-but-unqueryable`,
|
|
2411
|
+
expected: canonical,
|
|
2412
|
+
actual: posted,
|
|
2413
|
+
heal: 'rebuild',
|
|
2414
|
+
};
|
|
2415
|
+
},
|
|
2416
|
+
// replay-clean: the cold open recovered nothing anomalous.
|
|
2417
|
+
() => {
|
|
2418
|
+
const engine = this.lsmEngine;
|
|
2419
|
+
if (!engine || typeof engine.replayStatsJson !== 'function') {
|
|
2420
|
+
return {
|
|
2421
|
+
name: 'replay-clean',
|
|
2422
|
+
holds: true,
|
|
2423
|
+
detail: 'replay stats unavailable (in-RAM engine or pre-3.0.12 binary)',
|
|
2424
|
+
heal: 'none',
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
const s = JSON.parse(engine.replayStatsJson());
|
|
2428
|
+
const unreadable = s.archivesUnreadable ?? 0;
|
|
2429
|
+
const recreated = s.liveLogsRecreated ?? 0;
|
|
2430
|
+
const clean = unreadable === 0 && recreated === 0;
|
|
2431
|
+
return {
|
|
2432
|
+
name: 'replay-clean',
|
|
2433
|
+
holds: clean,
|
|
2434
|
+
detail: clean
|
|
2435
|
+
? 'cold-open replay clean (no unreadable archives, no recreated live logs)'
|
|
2436
|
+
: `cold-open replay anomalies: ${recreated} live log(s) were missing ` +
|
|
2437
|
+
`(recreated), ${unreadable} archive(s) unreadable`,
|
|
2438
|
+
heal: 'rebuild',
|
|
2439
|
+
};
|
|
2440
|
+
},
|
|
2441
|
+
// registry-discoverable: every field this process knows is in the
|
|
2442
|
+
// persisted registry a cold open would discover.
|
|
2443
|
+
async () => {
|
|
2444
|
+
if (this.knownFields.size === 0) {
|
|
2445
|
+
return {
|
|
2446
|
+
name: 'registry-discoverable',
|
|
2447
|
+
holds: true,
|
|
2448
|
+
detail: 'no fields registered yet',
|
|
2449
|
+
heal: 'none',
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
let persisted = [];
|
|
2453
|
+
try {
|
|
2454
|
+
const raw = await this.storage.getMetadata('__metadata_field_registry__');
|
|
2455
|
+
persisted = Array.isArray(raw?.fields)
|
|
2456
|
+
? (raw.fields)
|
|
2457
|
+
: Array.isArray(raw)
|
|
2458
|
+
? raw
|
|
2459
|
+
: [];
|
|
2460
|
+
}
|
|
2461
|
+
catch (error) {
|
|
2462
|
+
return {
|
|
2463
|
+
name: 'registry-discoverable',
|
|
2464
|
+
holds: false,
|
|
2465
|
+
detail: `persisted registry read FAULTED — discoverability unverifiable: ${String(error)}`,
|
|
2466
|
+
heal: 'none',
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
const persistedSet = new Set(persisted);
|
|
2470
|
+
const undiscoverable = [...this.knownFields].filter((f) => !persistedSet.has(f));
|
|
2471
|
+
return {
|
|
2472
|
+
name: 'registry-discoverable',
|
|
2473
|
+
holds: undiscoverable.length === 0,
|
|
2474
|
+
detail: undiscoverable.length === 0
|
|
2475
|
+
? `${this.knownFields.size} known field(s), all in the persisted registry`
|
|
2476
|
+
: `${undiscoverable.length} known field(s) missing from the persisted ` +
|
|
2477
|
+
`registry (would be undiscoverable after restart): ${undiscoverable.join(', ')}`,
|
|
2478
|
+
expected: this.knownFields.size,
|
|
2479
|
+
actual: this.knownFields.size - undiscoverable.length,
|
|
2480
|
+
heal: 'repair',
|
|
2481
|
+
};
|
|
2482
|
+
},
|
|
2483
|
+
]);
|
|
2484
|
+
}
|
|
2485
|
+
/**
|
|
2486
|
+
* Serializes every {@link flush} pass. Two flushes interleaving at the
|
|
2487
|
+
* awaits is NOT hypothetical — memory-vm runs a periodic flusher AND
|
|
2488
|
+
* brainy's shutdown hook flushes on SIGTERM — and the flushLsm pair
|
|
2489
|
+
* both read `nextSstableId(manifest)` before either pushes: SAME id →
|
|
2490
|
+
* same blob key → the second write OVERWRITES the first SSTable →
|
|
2491
|
+
* postings lost with clean logs and NO replay anomaly (the exact
|
|
2492
|
+
* acceptance-#1 signature on CORTEX-RESTART-STRAND, 2026-07-13).
|
|
2493
|
+
* Chaining gives each caller its own full pass covering everything
|
|
2494
|
+
* written before its call — the semantics a shutdown flush needs.
|
|
2495
|
+
*/
|
|
2496
|
+
flushChain = Promise.resolve();
|
|
2281
2497
|
async flush() {
|
|
2498
|
+
// Flush-during-rebuild guard (CORTEX-RESTART-STRAND, memory-vm's
|
|
2499
|
+
// 80-minute heal): a consumer's periodic flush cycle running DURING a
|
|
2500
|
+
// rebuild drains the growing memtable + rewrites field indexes on
|
|
2501
|
+
// every pass — O(current-index-size) IO per pass, interleaved with the
|
|
2502
|
+
// walk (measured live: 572 msync/s + escalating 29s→45s flush passes).
|
|
2503
|
+
// The rebuild's own completion flush owns durability (canonical is the
|
|
2504
|
+
// source of truth throughout), so a concurrent flush only multiplies
|
|
2505
|
+
// IO. Loud no-op, never silent.
|
|
2506
|
+
if (this.isRebuilding) {
|
|
2507
|
+
prodLog.warn('[NativeMetadataIndex] flush() during an active rebuild — deferred ' +
|
|
2508
|
+
'(the rebuild\'s completion flush owns durability; a concurrent ' +
|
|
2509
|
+
'flush would multiply heal IO).');
|
|
2510
|
+
return;
|
|
2511
|
+
}
|
|
2512
|
+
const run = this.flushChain.then(() => this.flushOnce());
|
|
2513
|
+
// Keep the chain alive on failure — the NEXT flush must still run; the
|
|
2514
|
+
// failure itself propagates to THIS caller below.
|
|
2515
|
+
this.flushChain = run.catch(() => { });
|
|
2516
|
+
return run;
|
|
2517
|
+
}
|
|
2518
|
+
async flushOnce() {
|
|
2282
2519
|
// Always save field registry + msync the entity ID mapper — even with
|
|
2283
2520
|
// no dirty fields. The field registry is the critical file init() needs
|
|
2284
2521
|
// to discover persisted indices. Without it, the metadata index appears
|
|
@@ -2679,7 +2916,10 @@ export class MetadataIndexManager {
|
|
|
2679
2916
|
if (bulkCapable)
|
|
2680
2917
|
this.lsmEngine.setBulkLoad(false);
|
|
2681
2918
|
}
|
|
2682
|
-
// Final flush
|
|
2919
|
+
// Final flush. isRebuilding drops FIRST so the completion flush
|
|
2920
|
+
// passes the flush-during-rebuild guard above; readiness is still
|
|
2921
|
+
// gated by strandDetected until the flush lands and clears it below.
|
|
2922
|
+
this.isRebuilding = false;
|
|
2683
2923
|
await this.flushRebuildDirty();
|
|
2684
2924
|
await this.flush();
|
|
2685
2925
|
prodLog.info(`Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/derivedFamilies
|
|
3
|
+
* @description ADR-004 §7 — cor's registered-blob family declarations
|
|
4
|
+
* (adopted 2026-07-13). Each provider declares the artifact families it
|
|
5
|
+
* owns through brainy ≥8.3.0's `registerDerivedFamily` (feature-detected;
|
|
6
|
+
* inert on older adapters). Once declared, the storage layer REFUSES to
|
|
7
|
+
* delete a member (`ProtectedArtifactError` — an in-process GC/sweeper is
|
|
8
|
+
* structurally incapable of removing a load-bearing file), and
|
|
9
|
+
* `checkDerivedFamiliesPresent` names any missing-on-open member (the
|
|
10
|
+
* external-deleter catch).
|
|
11
|
+
*
|
|
12
|
+
* Scope note (deliberate split, per §7): declarations cover
|
|
13
|
+
* KEY-ADDRESSABLE artifacts — members resolve through the same adapter
|
|
14
|
+
* key→path function on both sides, so consistency is by construction.
|
|
15
|
+
* Path-level siblings the native layer writes directly (the vector
|
|
16
|
+
* base's `.slotmap`/`.slotrev` sidecars) are outside the key space; their
|
|
17
|
+
* set-coherence is cor's own `familygen` stamp (the §7 atomic set-swap),
|
|
18
|
+
* not the storage contract. Growing sets use `namespace: true` prefixes.
|
|
19
|
+
*
|
|
20
|
+
* Registration is idempotent (upsert by family name) and runs on every
|
|
21
|
+
* open. A registration failure is LOUD but never blocks an open — the
|
|
22
|
+
* contract is protective, not load-bearing.
|
|
23
|
+
*/
|
|
24
|
+
import type { DerivedFamilyDeclaration } from '@soulcraft/brainy';
|
|
25
|
+
/**
|
|
26
|
+
* Declare families on the adapter when it supports the contract. Never
|
|
27
|
+
* throws; failures are loud. Returns how many were registered (0 when the
|
|
28
|
+
* adapter predates the contract — callers may log that once at debug).
|
|
29
|
+
*/
|
|
30
|
+
export declare function declareDerivedFamilies(storage: unknown, families: DerivedFamilyDeclaration[]): Promise<number>;
|
|
31
|
+
/** The vector index's key-addressable families. */
|
|
32
|
+
export declare const VECTOR_FAMILIES: DerivedFamilyDeclaration[];
|
|
33
|
+
/** The metadata LSM's families (engine dir + SSTables share the key prefix). */
|
|
34
|
+
export declare const METADATA_FAMILIES: DerivedFamilyDeclaration[];
|
|
35
|
+
/** The column store's family (fully key-addressable — API delete-refusal bites here). */
|
|
36
|
+
export declare const COLUMN_FAMILIES: DerivedFamilyDeclaration[];
|
|
37
|
+
/** The graph index's families (LSM SSTables + verb namespace stores). */
|
|
38
|
+
export declare const GRAPH_FAMILIES: DerivedFamilyDeclaration[];
|
|
39
|
+
/**
|
|
40
|
+
* Owner-sanctioned retirement window (the pattern brainy's
|
|
41
|
+
* ProtectedArtifactError names): unregister the family, run the owner's
|
|
42
|
+
* legitimate delete work, re-register in a finally. The unprotected
|
|
43
|
+
* window is small and self-healing — every provider re-declares its
|
|
44
|
+
* families on init, so even a crash inside the window is repaired at the
|
|
45
|
+
* next open. On adapters without the contract, just runs `fn`.
|
|
46
|
+
*/
|
|
47
|
+
export declare function withFamilyRetirement<T>(storage: unknown, family: DerivedFamilyDeclaration, fn: () => Promise<T>): Promise<T>;
|
|
48
|
+
//# sourceMappingURL=derivedFamilies.d.ts.map
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/derivedFamilies
|
|
3
|
+
* @description ADR-004 §7 — cor's registered-blob family declarations
|
|
4
|
+
* (adopted 2026-07-13). Each provider declares the artifact families it
|
|
5
|
+
* owns through brainy ≥8.3.0's `registerDerivedFamily` (feature-detected;
|
|
6
|
+
* inert on older adapters). Once declared, the storage layer REFUSES to
|
|
7
|
+
* delete a member (`ProtectedArtifactError` — an in-process GC/sweeper is
|
|
8
|
+
* structurally incapable of removing a load-bearing file), and
|
|
9
|
+
* `checkDerivedFamiliesPresent` names any missing-on-open member (the
|
|
10
|
+
* external-deleter catch).
|
|
11
|
+
*
|
|
12
|
+
* Scope note (deliberate split, per §7): declarations cover
|
|
13
|
+
* KEY-ADDRESSABLE artifacts — members resolve through the same adapter
|
|
14
|
+
* key→path function on both sides, so consistency is by construction.
|
|
15
|
+
* Path-level siblings the native layer writes directly (the vector
|
|
16
|
+
* base's `.slotmap`/`.slotrev` sidecars) are outside the key space; their
|
|
17
|
+
* set-coherence is cor's own `familygen` stamp (the §7 atomic set-swap),
|
|
18
|
+
* not the storage contract. Growing sets use `namespace: true` prefixes.
|
|
19
|
+
*
|
|
20
|
+
* Registration is idempotent (upsert by family name) and runs on every
|
|
21
|
+
* open. A registration failure is LOUD but never blocks an open — the
|
|
22
|
+
* contract is protective, not load-bearing.
|
|
23
|
+
*/
|
|
24
|
+
import { prodLog } from '@soulcraft/brainy/internals';
|
|
25
|
+
/**
|
|
26
|
+
* Declare families on the adapter when it supports the contract. Never
|
|
27
|
+
* throws; failures are loud. Returns how many were registered (0 when the
|
|
28
|
+
* adapter predates the contract — callers may log that once at debug).
|
|
29
|
+
*/
|
|
30
|
+
export async function declareDerivedFamilies(storage, families) {
|
|
31
|
+
const s = storage;
|
|
32
|
+
if (typeof s?.registerDerivedFamily !== 'function')
|
|
33
|
+
return 0;
|
|
34
|
+
let registered = 0;
|
|
35
|
+
for (const family of families) {
|
|
36
|
+
try {
|
|
37
|
+
await s.registerDerivedFamily(family);
|
|
38
|
+
registered++;
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
prodLog.error(`[cor] registerDerivedFamily('${family.name}') FAILED — its members ` +
|
|
42
|
+
`are NOT delete-protected this session (opens are unaffected):`, error);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return registered;
|
|
46
|
+
}
|
|
47
|
+
/** The vector index's key-addressable families. */
|
|
48
|
+
export const VECTOR_FAMILIES = [
|
|
49
|
+
{
|
|
50
|
+
name: 'vector-base',
|
|
51
|
+
members: ['_system/vector-index/main.dkann'],
|
|
52
|
+
rebuildable: true,
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
name: 'vector-segments',
|
|
56
|
+
members: ['_system/vector-index/seg-'],
|
|
57
|
+
namespace: true,
|
|
58
|
+
rebuildable: true,
|
|
59
|
+
},
|
|
60
|
+
];
|
|
61
|
+
/** The metadata LSM's families (engine dir + SSTables share the key prefix). */
|
|
62
|
+
export const METADATA_FAMILIES = [
|
|
63
|
+
{
|
|
64
|
+
name: 'metadata-lsm',
|
|
65
|
+
members: ['_metadata'],
|
|
66
|
+
namespace: true,
|
|
67
|
+
rebuildable: true,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'id-mapper',
|
|
71
|
+
members: ['_id_mapper/'],
|
|
72
|
+
namespace: true,
|
|
73
|
+
rebuildable: true,
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
/** The column store's family (fully key-addressable — API delete-refusal bites here). */
|
|
77
|
+
export const COLUMN_FAMILIES = [
|
|
78
|
+
{
|
|
79
|
+
name: 'column-index',
|
|
80
|
+
members: ['_column_index/'],
|
|
81
|
+
namespace: true,
|
|
82
|
+
rebuildable: true,
|
|
83
|
+
},
|
|
84
|
+
];
|
|
85
|
+
/** The graph index's families (LSM SSTables + verb namespace stores). */
|
|
86
|
+
export const GRAPH_FAMILIES = [
|
|
87
|
+
{
|
|
88
|
+
name: 'graph-lsm',
|
|
89
|
+
members: ['graph-lsm/'],
|
|
90
|
+
namespace: true,
|
|
91
|
+
rebuildable: true,
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
name: 'graph-verb-namespace',
|
|
95
|
+
members: ['_graph_verb_ns/'],
|
|
96
|
+
namespace: true,
|
|
97
|
+
rebuildable: true,
|
|
98
|
+
},
|
|
99
|
+
];
|
|
100
|
+
/**
|
|
101
|
+
* Owner-sanctioned retirement window (the pattern brainy's
|
|
102
|
+
* ProtectedArtifactError names): unregister the family, run the owner's
|
|
103
|
+
* legitimate delete work, re-register in a finally. The unprotected
|
|
104
|
+
* window is small and self-healing — every provider re-declares its
|
|
105
|
+
* families on init, so even a crash inside the window is repaired at the
|
|
106
|
+
* next open. On adapters without the contract, just runs `fn`.
|
|
107
|
+
*/
|
|
108
|
+
export async function withFamilyRetirement(storage, family, fn) {
|
|
109
|
+
const s = storage;
|
|
110
|
+
if (typeof s?.unregisterDerivedFamily !== 'function' ||
|
|
111
|
+
typeof s?.registerDerivedFamily !== 'function') {
|
|
112
|
+
return fn();
|
|
113
|
+
}
|
|
114
|
+
await s.unregisterDerivedFamily(family.name);
|
|
115
|
+
try {
|
|
116
|
+
return await fn();
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
try {
|
|
120
|
+
await s.registerDerivedFamily(family);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
prodLog.error(`[cor] re-register of family '${family.name}' after retirement ` +
|
|
124
|
+
`FAILED — protection resumes at the next open:`, error);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=derivedFamilies.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/invariantReport
|
|
3
|
+
* @description Shared composer for the ADR-004 §6 `validateInvariants()`
|
|
4
|
+
* hook (adopted 2026-07-13, decision `adr-004-write-index-spine-adoption`).
|
|
5
|
+
* Every cor provider builds its report through this so the contract holds
|
|
6
|
+
* uniformly: the hook NEVER throws (a thrown check is surfaced as a failing
|
|
7
|
+
* invariant naming the throw — never an exception, never swallowed), stays
|
|
8
|
+
* bounded (checks are residency + O(1) counts only; the composer measures
|
|
9
|
+
* and reports duration), and failures carry named, numbered detail.
|
|
10
|
+
*
|
|
11
|
+
* Types are imported from brainy 8.3.0's published surface so both engines
|
|
12
|
+
* share one wire shape (`ProviderInvariantReport` / `InvariantResult` /
|
|
13
|
+
* `InvariantHeal` — brainy's `validateIndexConsistency` feature-detects the
|
|
14
|
+
* hook and aggregates these verbatim; `repairIndex` maps `heal: 'rebuild'`
|
|
15
|
+
* to the provider's own rebuild).
|
|
16
|
+
*/
|
|
17
|
+
import type { InvariantResult, ProviderInvariantReport } from '@soulcraft/brainy';
|
|
18
|
+
/** One invariant check: returns its result, sync or async. A throw is a
|
|
19
|
+
* contract violation the composer converts into a failing invariant. */
|
|
20
|
+
export type InvariantCheck = () => InvariantResult | Promise<InvariantResult>;
|
|
21
|
+
/**
|
|
22
|
+
* Run every check, never throw, stamp timing. A check that throws becomes
|
|
23
|
+
* `holds: false` with `heal: 'none'` and detail naming the throw — the
|
|
24
|
+
* never-throws contract surfaced as unhealthy data rather than an exception
|
|
25
|
+
* (and `heal: 'none'` so a flaky probe can never trigger a spurious
|
|
26
|
+
* rebuild; visibility without overreaction).
|
|
27
|
+
*/
|
|
28
|
+
export declare function composeInvariantReport(provider: string, serving: () => boolean, checks: InvariantCheck[]): Promise<ProviderInvariantReport>;
|
|
29
|
+
//# sourceMappingURL=invariantReport.d.ts.map
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/invariantReport
|
|
3
|
+
* @description Shared composer for the ADR-004 §6 `validateInvariants()`
|
|
4
|
+
* hook (adopted 2026-07-13, decision `adr-004-write-index-spine-adoption`).
|
|
5
|
+
* Every cor provider builds its report through this so the contract holds
|
|
6
|
+
* uniformly: the hook NEVER throws (a thrown check is surfaced as a failing
|
|
7
|
+
* invariant naming the throw — never an exception, never swallowed), stays
|
|
8
|
+
* bounded (checks are residency + O(1) counts only; the composer measures
|
|
9
|
+
* and reports duration), and failures carry named, numbered detail.
|
|
10
|
+
*
|
|
11
|
+
* Types are imported from brainy 8.3.0's published surface so both engines
|
|
12
|
+
* share one wire shape (`ProviderInvariantReport` / `InvariantResult` /
|
|
13
|
+
* `InvariantHeal` — brainy's `validateIndexConsistency` feature-detects the
|
|
14
|
+
* hook and aggregates these verbatim; `repairIndex` maps `heal: 'rebuild'`
|
|
15
|
+
* to the provider's own rebuild).
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Run every check, never throw, stamp timing. A check that throws becomes
|
|
19
|
+
* `holds: false` with `heal: 'none'` and detail naming the throw — the
|
|
20
|
+
* never-throws contract surfaced as unhealthy data rather than an exception
|
|
21
|
+
* (and `heal: 'none'` so a flaky probe can never trigger a spurious
|
|
22
|
+
* rebuild; visibility without overreaction).
|
|
23
|
+
*/
|
|
24
|
+
export async function composeInvariantReport(provider, serving, checks) {
|
|
25
|
+
const startedAt = Date.now();
|
|
26
|
+
const invariants = [];
|
|
27
|
+
for (const check of checks) {
|
|
28
|
+
try {
|
|
29
|
+
invariants.push(await check());
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
invariants.push({
|
|
33
|
+
name: 'check-integrity',
|
|
34
|
+
holds: false,
|
|
35
|
+
detail: `an invariant check THREW (contract violation — checks must ` +
|
|
36
|
+
`return failures as data): ${String(error)}`,
|
|
37
|
+
heal: 'none',
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
let servingNow = false;
|
|
42
|
+
try {
|
|
43
|
+
servingNow = serving();
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
invariants.push({
|
|
47
|
+
name: 'serving-probe',
|
|
48
|
+
holds: false,
|
|
49
|
+
detail: `isReady() itself threw: ${String(error)}`,
|
|
50
|
+
heal: 'none',
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
provider,
|
|
55
|
+
healthy: invariants.every((i) => i.holds),
|
|
56
|
+
serving: servingNow,
|
|
57
|
+
invariants,
|
|
58
|
+
checkedAt: startedAt,
|
|
59
|
+
durationMs: Date.now() - startedAt,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=invariantReport.js.map
|