@soulcraft/cor 3.0.18 → 3.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -86,6 +86,16 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
86
86
  */
87
87
  private canonicalUnindexed;
88
88
  private isRebuilding;
89
+ /** Membership mutated since the last persisted blob (write-side dirty bit). */
90
+ private membershipDirty;
91
+ /** Cached stamp content (minus committedAt) — identical payload = no I/O on idle ticks. */
92
+ private lastMembershipStampBody;
93
+ /** Byte length of the last persisted membership blob (stamp needs it without re-serializing). */
94
+ private lastMembershipBlobBytes;
95
+ /** Monotonic stamp lineage (unified shape); restored at load, bumped at persist. */
96
+ private membershipStampGeneration;
97
+ /** How this boot obtained membership: restored (stamp), healed (walk), or fresh (no verbs). */
98
+ private membershipBootState;
89
99
  private flushTimer?;
90
100
  private rebuildStartTime;
91
101
  private totalRelationshipsIndexed;
@@ -174,6 +184,32 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
174
184
  * queryable edges report "not ready".
175
185
  */
176
186
  private openGraphLsmEngine;
187
+ /**
188
+ * SHA-256 over the tree's durable SSTable identity (sorted `id:level`
189
+ * lines). The stamp records one per tree; a mismatch at open means the
190
+ * trees advanced (or tore) after the last membership persist, so the
191
+ * blob cannot be trusted and the walk heals. Content-relevant fields
192
+ * only — deliberately NOT the raw manifest JSON (timestamps and field
193
+ * ordering must not fail coherence).
194
+ */
195
+ private treeFingerprint;
196
+ /**
197
+ * Persist the membership family: blob (only when membership mutated or
198
+ * `force`) then stamp LAST (§7 — the stamp commits the set). Identical
199
+ * stamp payloads are skipped entirely, so idle auto-flush ticks cost
200
+ * zero I/O. Failures log loudly and leave the previous stamp in place —
201
+ * a stale stamp heals at next open; a silent half-persist would not.
202
+ */
203
+ private persistVerbMembership;
204
+ /**
205
+ * Restore membership from the stamped blob. Returns `true` only when the
206
+ * FULL coherence chain holds: stamp present → blob present with the exact
207
+ * stamped byte length → every tree fingerprint matches the durable state
208
+ * just loaded → the native parser accepts every framed section. Any break
209
+ * returns `false` (absent stamp silently — that is every pre-3.0.19
210
+ * brain; a broken chain with a warn) and the caller's walk heals.
211
+ */
212
+ private loadVerbMembershipFromStorage;
177
213
  private populateVerbIdSetFromStorage;
178
214
  getNeighbors(id: bigint, optionsOrDirection?: {
179
215
  direction?: 'in' | 'out' | 'both';
@@ -13,6 +13,8 @@
13
13
  * since it coordinates cross-subsystem caching and requires async storage access.
14
14
  */
15
15
  import { dirname } from 'node:path';
16
+ import { createHash } from 'node:crypto';
17
+ import { readCommittedGeneration } from '../utils/generationWatermark.js';
16
18
  import { TypeUtils, VerbType } from '@soulcraft/brainy/types/graphTypes';
17
19
  import { loadNativeModule } from '../native/index.js';
18
20
  import { composeInvariantReport } from '../utils/invariantReport.js';
@@ -45,6 +47,38 @@ const VERB_FALLBACK_INDEX = TypeUtils.getVerbIndex(VerbType.RelatedTo) ?? 3;
45
47
  * (`verb-uuid-to-int.mkv`, `verb-int-to-uuid.bin`, `verb-endpoints/`).
46
48
  */
47
49
  const GRAPH_VERB_NS_PROBE_KEY = '_graph_verb_ns/x';
50
+ /**
51
+ * Persisted verb-membership family (3.0.19). The roster / per-type unions /
52
+ * per-(type, subtype) buckets / count families were RAM-only, costing every
53
+ * boot of every verb-bearing brain an O(all-canonical-verbs) walk and
54
+ * silently emptying subtype-filtered traversal after restarts. The blob is
55
+ * the serialized structures; the stamp ties the blob to the exact durable
56
+ * tree state (per-tree SSTable fingerprints) so a torn write is DETECTED at
57
+ * open and heals via the walk — §7 coherent-or-rebuild, same discipline as
58
+ * the vector family.
59
+ */
60
+ const GRAPH_MEMBERSHIP_BLOB_KEY = 'graph-verb-membership';
61
+ const GRAPH_MEMBERSHIP_STAMP_KEY = 'graph-membership-stamp';
62
+ /** Normalized view both wire shapes reduce to; null = unrecognized. */
63
+ function normalizeMembershipStamp(raw) {
64
+ const s = raw;
65
+ if (s?.members?.mode === 'rollup' && s.members.invariants) {
66
+ const inv = s.members.invariants;
67
+ const blobBytes = inv.blobBytes;
68
+ if (typeof blobBytes !== 'number')
69
+ return null;
70
+ const trees = {};
71
+ for (const [k, v] of Object.entries(inv)) {
72
+ if (k.startsWith('tree:') && typeof v === 'string')
73
+ trees[k.slice(5)] = v;
74
+ }
75
+ return { blobBytes, trees, generation: typeof s.generation === 'number' ? s.generation : 0 };
76
+ }
77
+ if (s?.v === 1 && typeof s.blobBytes === 'number' && s.trees) {
78
+ return { blobBytes: s.blobBytes, trees: s.trees, generation: 0 };
79
+ }
80
+ return null;
81
+ }
48
82
  /**
49
83
  * Resolve a verb-type string to its canonical `VerbTypeEnum` index,
50
84
  * falling back to {@link VERB_FALLBACK_INDEX} (`VerbType.RelatedTo`)
@@ -91,6 +125,16 @@ export class GraphAdjacencyIndex {
91
125
  */
92
126
  canonicalUnindexed = false;
93
127
  isRebuilding = false;
128
+ /** Membership mutated since the last persisted blob (write-side dirty bit). */
129
+ membershipDirty = false;
130
+ /** Cached stamp content (minus committedAt) — identical payload = no I/O on idle ticks. */
131
+ lastMembershipStampBody = '';
132
+ /** Byte length of the last persisted membership blob (stamp needs it without re-serializing). */
133
+ lastMembershipBlobBytes = -1;
134
+ /** Monotonic stamp lineage (unified shape); restored at load, bumped at persist. */
135
+ membershipStampGeneration = 0;
136
+ /** How this boot obtained membership: restored (stamp), healed (walk), or fresh (no verbs). */
137
+ membershipBootState = 'fresh';
94
138
  flushTimer;
95
139
  rebuildStartTime = 0;
96
140
  totalRelationshipsIndexed = 0;
@@ -293,13 +337,132 @@ export class GraphAdjacencyIndex {
293
337
  for (const treeName of treeNames) {
294
338
  await this.loadTreeFromStorage(treeName);
295
339
  }
340
+ // Persisted-membership restore (3.0.19): coherent stamp + blob loads the
341
+ // roster/buckets/counts in O(bitmap) — no canonical walk, subtype filters
342
+ // intact. `restored` also covers the legitimately-empty-roster case (all
343
+ // verbs removed) so the walk below can't re-fire on it every boot.
344
+ const restored = await this.loadVerbMembershipFromStorage();
296
345
  // Defensive check: if LSM-trees have data but verbIdSet is empty, populate.
297
- // This is the deferred membership fast-follow it does NOT gate isReady().
346
+ // This is the membership HEAL path (pre-3.0.19 brains, torn/missing
347
+ // stamp) — it does NOT gate isReady().
298
348
  const sourceSize = this.native.size();
299
- if (sourceSize > 0 && this.native.verbIdCount() === 0) {
349
+ if (!restored && sourceSize > 0 && this.native.verbIdCount() === 0) {
300
350
  prodLog.warn(`GraphAdjacencyIndex: LSM-trees have ${sourceSize} relationships but verbIdSet is empty. ` +
301
351
  `Triggering auto-rebuild to restore consistency.`);
302
352
  await this.populateVerbIdSetFromStorage();
353
+ this.membershipBootState = 'healed';
354
+ // Persist immediately so the NEXT boot restores instead of re-walking.
355
+ await this.persistVerbMembership(true);
356
+ }
357
+ else if (!restored && this.native.verbIdCount() > 0) {
358
+ // Verbs arrived without a stamp (e.g. mid-process rebuild) — treat as
359
+ // healed so the invariant reports honestly.
360
+ this.membershipBootState = 'healed';
361
+ }
362
+ }
363
+ /**
364
+ * SHA-256 over the tree's durable SSTable identity (sorted `id:level`
365
+ * lines). The stamp records one per tree; a mismatch at open means the
366
+ * trees advanced (or tore) after the last membership persist, so the
367
+ * blob cannot be trusted and the walk heals. Content-relevant fields
368
+ * only — deliberately NOT the raw manifest JSON (timestamps and field
369
+ * ordering must not fail coherence).
370
+ */
371
+ treeFingerprint(treeName) {
372
+ const ids = this.native.getManifestSstableIds(treeName);
373
+ const lines = ids
374
+ .map((id) => `${id}:${this.native.getManifestSstableLevel(treeName, id) ?? '?'}`)
375
+ .sort();
376
+ return createHash('sha256').update(lines.join('\n')).digest('hex');
377
+ }
378
+ /**
379
+ * Persist the membership family: blob (only when membership mutated or
380
+ * `force`) then stamp LAST (§7 — the stamp commits the set). Identical
381
+ * stamp payloads are skipped entirely, so idle auto-flush ticks cost
382
+ * zero I/O. Failures log loudly and leave the previous stamp in place —
383
+ * a stale stamp heals at next open; a silent half-persist would not.
384
+ */
385
+ async persistVerbMembership(force = false) {
386
+ if (!this.hasBinaryBlobs)
387
+ return;
388
+ try {
389
+ if (this.membershipDirty || force || this.lastMembershipBlobBytes < 0) {
390
+ const blob = this.native.serializeVerbMembership();
391
+ await this.storage.saveBinaryBlob(GRAPH_MEMBERSHIP_BLOB_KEY, Buffer.from(blob));
392
+ this.lastMembershipBlobBytes = blob.length;
393
+ this.membershipDirty = false;
394
+ }
395
+ const invariants = {
396
+ blobBytes: this.lastMembershipBlobBytes,
397
+ };
398
+ for (const treeName of this.native.treeNames()) {
399
+ invariants[`tree:${treeName}`] = this.treeFingerprint(treeName);
400
+ }
401
+ const body = JSON.stringify(invariants);
402
+ if (body === this.lastMembershipStampBody)
403
+ return;
404
+ const stamp = {
405
+ family: 'graph-membership',
406
+ generation: this.membershipStampGeneration + 1,
407
+ committedAt: new Date().toISOString(),
408
+ members: { mode: 'rollup', invariants },
409
+ };
410
+ const sourceGeneration = await readCommittedGeneration(this.storage);
411
+ if (sourceGeneration !== undefined)
412
+ stamp.sourceGeneration = sourceGeneration;
413
+ await this.storage.saveMetadata(GRAPH_MEMBERSHIP_STAMP_KEY, {
414
+ noun: 'thing',
415
+ data: stamp,
416
+ });
417
+ this.membershipStampGeneration = stamp.generation;
418
+ this.lastMembershipStampBody = body;
419
+ }
420
+ catch (error) {
421
+ prodLog.error('GraphAdjacencyIndex: failed to persist verb membership — next open will heal via the canonical walk', error);
422
+ }
423
+ }
424
+ /**
425
+ * Restore membership from the stamped blob. Returns `true` only when the
426
+ * FULL coherence chain holds: stamp present → blob present with the exact
427
+ * stamped byte length → every tree fingerprint matches the durable state
428
+ * just loaded → the native parser accepts every framed section. Any break
429
+ * returns `false` (absent stamp silently — that is every pre-3.0.19
430
+ * brain; a broken chain with a warn) and the caller's walk heals.
431
+ */
432
+ async loadVerbMembershipFromStorage() {
433
+ if (!this.hasBinaryBlobs)
434
+ return false;
435
+ try {
436
+ const meta = await this.storage.getMetadata(GRAPH_MEMBERSHIP_STAMP_KEY);
437
+ const stamp = normalizeMembershipStamp(meta?.data);
438
+ if (!stamp)
439
+ return false;
440
+ const blob = await this.storage.loadBinaryBlob(GRAPH_MEMBERSHIP_BLOB_KEY);
441
+ if (!blob || blob.length !== stamp.blobBytes) {
442
+ prodLog.warn(`GraphAdjacencyIndex: membership blob ${blob ? `is ${blob.length} bytes but the stamp committed ${stamp.blobBytes}` : 'is MISSING under a present stamp'} — healing via the canonical walk`);
443
+ return false;
444
+ }
445
+ for (const treeName of this.native.treeNames()) {
446
+ const now = this.treeFingerprint(treeName);
447
+ if (stamp.trees?.[treeName] !== now) {
448
+ prodLog.warn(`GraphAdjacencyIndex: membership stamp is stale for tree '${treeName}' (durable SSTables changed after the last persist) — healing via the canonical walk`);
449
+ return false;
450
+ }
451
+ }
452
+ this.native.loadVerbMembership(blob);
453
+ this.lastMembershipBlobBytes = blob.length;
454
+ this.membershipStampGeneration = stamp.generation;
455
+ const invariants = { blobBytes: stamp.blobBytes };
456
+ for (const [name, fp] of Object.entries(stamp.trees))
457
+ invariants[`tree:${name}`] = fp;
458
+ this.lastMembershipStampBody = JSON.stringify(invariants);
459
+ this.membershipBootState = 'restored';
460
+ prodLog.info(`GraphAdjacencyIndex: verb membership restored from stamp (${this.native.verbIdCount()} live verbs, no canonical walk)`);
461
+ return true;
462
+ }
463
+ catch (error) {
464
+ prodLog.warn('GraphAdjacencyIndex: membership restore failed — healing via the canonical walk', error);
465
+ return false;
303
466
  }
304
467
  }
305
468
  async populateVerbIdSetFromStorage() {
@@ -313,12 +476,16 @@ export class GraphAdjacencyIndex {
313
476
  pagination: { limit: 10000, cursor }
314
477
  });
315
478
  for (const verb of result.items) {
316
- // Cor 3.0 Piece D: trackVerbIdByIndex was deleted along with
317
- // the other strict-index variants. trackVerbId takes the verb-
318
- // type as a string; unknown strings fall back to RelatedTo —
319
- // acceptable on the rebuild slow path because the membership
320
- // bitmap converges to the same set either way.
321
- this.native.trackVerbId(verb.id, verb.verb);
479
+ // Full-fidelity tracking (3.0.19): the membership-only walk used
480
+ // to drop subtype state, silently emptying findConnectedSubtype
481
+ // after every restart. Canonical carries the subtype (brainy
482
+ // 7.30.0+) a METADATA field on verbs, surfaced top-level on
483
+ // hydrated items; read both shapes. Unknown type strings fall
484
+ // back to RelatedTo — acceptable on the rebuild slow path
485
+ // because the membership bitmap converges either way.
486
+ const v = verb;
487
+ const subtype = v.subtype ?? v.metadata?.subtype ?? '';
488
+ this.native.trackVerbIdFull(verb.id, resolveVerbTypeIndex(verb.verb), subtype);
322
489
  count++;
323
490
  }
324
491
  hasMore = result.hasMore;
@@ -437,6 +604,7 @@ export class GraphAdjacencyIndex {
437
604
  // coalesce absent subtype to '' here.
438
605
  const subtype = verb.subtype ?? '';
439
606
  const result = this.native.addVerbWithEndpointsByIndex(verb.id, sourceInt, targetInt, verbTypeIndex, subtype, generation);
607
+ this.membershipDirty = true;
440
608
  // Flush any trees that hit their threshold
441
609
  const flushPromises = [];
442
610
  if (result.needsFlushSource) {
@@ -465,8 +633,19 @@ export class GraphAdjacencyIndex {
465
633
  await this.ensureInitialized();
466
634
  // Load verb to get type + subtype info
467
635
  const verb = await this.getVerbCached(verbId);
468
- if (!verb)
636
+ if (!verb) {
637
+ // REMOVAL LAW (3.0.19): removal must never require reading the thing
638
+ // being removed. A ghost verb (canonical row unreadable) used to be
639
+ // silently SKIPPED here — its graph rows survived forever, the exact
640
+ // shape of brainy's replace-path defect one layer down. The id-only
641
+ // native path resolves type/subtype from cor-owned state instead.
642
+ const removed = this.native.removeVerbByIdOnly(verbId, generation);
643
+ if (removed) {
644
+ this.membershipDirty = true;
645
+ prodLog.warn(`GraphAdjacencyIndex: removed ghost verb ${verbId} via the id-only path (canonical row unreadable — removal law)`);
646
+ }
469
647
  return;
648
+ }
470
649
  const startTime = performance.now();
471
650
  const verbTypeIndex = resolveVerbTypeIndex(verb.verb);
472
651
  // Native subtype param is `String` with '' = "no subtype" (the empty-string
@@ -475,6 +654,7 @@ export class GraphAdjacencyIndex {
475
654
  // coalesce absent subtype to '' here.
476
655
  const subtype = verb.subtype ?? '';
477
656
  this.native.removeVerb(verbId, verbTypeIndex, subtype, generation);
657
+ this.membershipDirty = true;
478
658
  const elapsed = performance.now() - startTime;
479
659
  if (elapsed > 5.0) {
480
660
  prodLog.warn(`GraphAdjacencyIndex: Slow removeVerb for ${verbId}: ${elapsed.toFixed(2)}ms`);
@@ -600,6 +780,19 @@ export class GraphAdjacencyIndex {
600
780
  : 'no canonical-vs-durable coverage gap detected',
601
781
  heal: 'rebuild',
602
782
  }),
783
+ () => ({
784
+ name: 'membership-stamp',
785
+ // Every boot path lands in an honest state: restored (stamp
786
+ // coherent), healed (walk ran — first boot on 3.0.19, or a torn
787
+ // stamp), or fresh (no verbs anywhere). A verb-bearing graph
788
+ // whose membership is EMPTY while trees hold edges is the only
789
+ // failing shape — subtype filters would silently serve nothing.
790
+ holds: !(this.native.size() > 0 && this.native.verbIdCount() === 0),
791
+ detail: this.native.size() > 0 && this.native.verbIdCount() === 0
792
+ ? 'trees hold edges but the live-verb roster is EMPTY — membership restore AND heal both failed'
793
+ : `membership ${this.membershipBootState} this boot (${this.native.verbIdCount()} live verbs)`,
794
+ heal: 'rebuild',
795
+ }),
603
796
  ]);
604
797
  }
605
798
  isReady() {
@@ -719,8 +912,18 @@ export class GraphAdjacencyIndex {
719
912
  for (const verb of result.items) {
720
913
  totalVerbs++;
721
914
  const verbType = verb.verb ?? 'relatedTo';
915
+ const verbTypeIndex = resolveVerbTypeIndex(verbType);
916
+ // Native subtype param is `String` with '' = "no subtype" (the
917
+ // empty-string escape; graph_adjacency.rs guards on
918
+ // `!subtype.is_empty()`). JS null throws "Null into String" at the
919
+ // napi boundary, so coalesce absent subtype to ''. Subtype is a
920
+ // metadata field on verbs — read both hydration shapes.
921
+ const vv = verb;
922
+ const subtype = vv.subtype ?? vv.metadata?.subtype ?? '';
722
923
  if (!reconstructAdjacency) {
723
- this.native.trackVerbId(verb.id, verbType);
924
+ // Full-fidelity membership (3.0.19): subtype rides the walk so
925
+ // filtered traversal survives the rebuild.
926
+ this.native.trackVerbIdFull(verb.id, verbTypeIndex, subtype);
724
927
  continue;
725
928
  }
726
929
  const sourceId = verb.sourceId;
@@ -730,16 +933,10 @@ export class GraphAdjacencyIndex {
730
933
  if (sourceInt === undefined || targetInt === undefined) {
731
934
  // Endpoint not in the mapper — still record membership so the verb
732
935
  // isn't lost; its adjacency can't be rebuilt without the int.
733
- this.native.trackVerbId(verb.id, verbType);
936
+ this.native.trackVerbIdFull(verb.id, verbTypeIndex, subtype);
734
937
  unresolvedEndpoints++;
735
938
  continue;
736
939
  }
737
- const verbTypeIndex = resolveVerbTypeIndex(verbType);
738
- // Native subtype param is `String` with '' = "no subtype" (the
739
- // empty-string escape; graph_adjacency.rs guards on
740
- // `!subtype.is_empty()`). JS null throws "Null into String" at the
741
- // napi boundary, so coalesce absent subtype to ''.
742
- const subtype = verb.subtype ?? '';
743
940
  const addResult = this.native.addVerbWithEndpointsByIndex(verb.id, BigInt(sourceInt), BigInt(targetInt), verbTypeIndex, subtype, REBUILD_GENERATION);
744
941
  edgesRebuilt++;
745
942
  // Respect memtable flush thresholds mid-walk so RAM stays bounded.
@@ -759,6 +956,15 @@ export class GraphAdjacencyIndex {
759
956
  cursor = result.nextCursor;
760
957
  }
761
958
  this.totalRelationshipsIndexed = totalVerbs;
959
+ // The walk rebuilt membership through the native handle directly (the
960
+ // wrapper's dirty bit never fired) — mark it so the persist below and
961
+ // any later flush write the fresh blob.
962
+ this.membershipDirty = true;
963
+ if (!reconstructAdjacency) {
964
+ // Membership-only rebuild writes no memtables, so no flush runs —
965
+ // persist the family directly or the walk repeats next boot.
966
+ await this.persistVerbMembership(true);
967
+ }
762
968
  if (reconstructAdjacency) {
763
969
  // Persist partially-filled memtables from the final batch.
764
970
  await this.flush();
@@ -800,6 +1006,9 @@ export class GraphAdjacencyIndex {
800
1006
  if (flushPromises.length > 0) {
801
1007
  await Promise.all(flushPromises);
802
1008
  }
1009
+ // Membership family rides every flush cycle: blob only when membership
1010
+ // mutated, stamp only when the payload changed — idle ticks cost nothing.
1011
+ await this.persistVerbMembership();
803
1012
  const elapsed = Date.now() - startTime;
804
1013
  prodLog.debug(`GraphAdjacencyIndex: Flush completed in ${elapsed}ms`);
805
1014
  }
@@ -1031,6 +1240,11 @@ export class GraphAdjacencyIndex {
1031
1240
  });
1032
1241
  }));
1033
1242
  });
1243
+ // Compaction changed the durable SSTable identity, so the
1244
+ // membership stamp's fingerprint for this tree is now stale —
1245
+ // refresh it (stamp-only unless membership also mutated). Without
1246
+ // this, the next open would needlessly heal via the walk.
1247
+ await this.persistVerbMembership();
1034
1248
  const elapsed = Date.now() - startTime;
1035
1249
  prodLog.info(`GraphAdjacencyIndex: Compaction ${treeName} L${level} → L${result.newLevel} complete in ${elapsed}ms`);
1036
1250
  if (result.newLevel < 6 && this.native.needsCompaction(treeName, result.newLevel)) {
@@ -458,6 +458,26 @@ export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
458
458
  getPersistMode(): 'immediate' | 'deferred';
459
459
  /** Absolute path of a segment file (lives next to the base indexPath). */
460
460
  private segPath;
461
+ /**
462
+ * The committed-transaction watermark (M3 PD-2 `sourceGeneration`): read
463
+ * from brainy's generation manifest (`_system/manifest.json`, the same
464
+ * object the generation store treats as committed truth). Read-only and
465
+ * ADVISORY — hosts without the manifest (or a faulted read) stamp no
466
+ * sourceGeneration and verifiers skip the comparison. Never constructs a
467
+ * FactLog (its open() is writer-side reconciliation).
468
+ */
469
+ private readCommittedGeneration;
470
+ /**
471
+ * Move poisoned L0 segments out of the live set into
472
+ * `<indexDir>/quarantine/` — preserved for forensics, NEVER deleted
473
+ * (accord's 545 poisoned segments were the evidence that convicted the
474
+ * June dim bug). Crash-safe order: the manifest stops referencing the
475
+ * segments FIRST (a crash after that leaves orphaned, de-referenced
476
+ * files — a harmless leak), then the files move. The dropped native
477
+ * handles close with their references; nothing reads a de-referenced
478
+ * segment.
479
+ */
480
+ private quarantineSegments;
461
481
  /** Membership across the base + every L0 segment (int → any slot). */
462
482
  private hasIntAnywhere;
463
483
  /**
@@ -75,9 +75,10 @@ import { declareDerivedFamilies, VECTOR_FAMILIES } from '../utils/derivedFamilie
75
75
  import { statSync, readFileSync } from 'node:fs';
76
76
  import { loadNativeModule } from '../native/index.js';
77
77
  import { prodLog } from '@soulcraft/brainy/internals';
78
- import { mkdirSync, existsSync, rmSync } from 'node:fs';
79
- import { dirname, join } from 'node:path';
78
+ import { mkdirSync, existsSync, rmSync, renameSync } from 'node:fs';
79
+ import { dirname, join, basename } from 'node:path';
80
80
  import { autoModeForHeader, selectModeFromResourceManager, } from './AdaptiveDiskAnnModeSelector.js';
81
+ import { readCommittedGeneration } from '../utils/generationWatermark.js';
81
82
  /**
82
83
  * Node-count threshold above which `useMmapAdjacency: 'auto'`
83
84
  * resolves to file-backed. Below this, the in-RAM build adjacency
@@ -1174,13 +1175,50 @@ export class NativeDiskAnnWrapper {
1174
1175
  // into the new base. Their entries are read back through the existing
1175
1176
  // readVectorChunk/slotIdsForSlots surface (each segment is bounded-small
1176
1177
  // by construction, ≤ the delta budget, so this stays bounded RAM).
1177
- const builtL0 = [...this.l0Segments];
1178
- const l0Data = builtL0.map((seg) => ({
1179
- ints: (seg.nodeCount > 0
1180
- ? seg.native.slotIdsForSlots(Array.from({ length: seg.nodeCount }, (_, i) => i))
1181
- : []),
1182
- vecs: seg.native.readVectorChunk(0, seg.nodeCount),
1183
- }));
1178
+ //
1179
+ // SEGMENT QUARANTINE (3.0.19, accord 2026-07-14): a segment whose
1180
+ // read-back throws (June-class dim poisoning — "Range … out of bounds")
1181
+ // used to abort the WHOLE fold, forever: 545 live segments accumulated,
1182
+ // the write brake engaged, and the board refused writes for 20 hours.
1183
+ // A poisoned segment is now dropped from the fold LOUDLY and its files
1184
+ // moved to <indexDir>/quarantine/ (preserved for forensics, never
1185
+ // deleted); the fold completes with the healthy segments and the same
1186
+ // pass restores the lost entries from canonical (hydrate below), so the
1187
+ // rebuilt base is COMPLETE — serving-truth holds without a second heal.
1188
+ const builtL0 = [];
1189
+ const l0Data = [];
1190
+ const poisonedL0 = [];
1191
+ for (const seg of this.l0Segments) {
1192
+ try {
1193
+ const ints = (seg.nodeCount > 0
1194
+ ? seg.native.slotIdsForSlots(Array.from({ length: seg.nodeCount }, (_, i) => i))
1195
+ : []);
1196
+ const vecs = seg.native.readVectorChunk(0, seg.nodeCount);
1197
+ builtL0.push(seg);
1198
+ l0Data.push({ ints, vecs });
1199
+ }
1200
+ catch (error) {
1201
+ poisonedL0.push(seg);
1202
+ prodLog.error(`NativeDiskAnnWrapper: L0 segment ${seg.file} is UNREADABLE ` +
1203
+ `(${error instanceof Error ? error.message : String(error)}) — ` +
1204
+ `QUARANTINED (${seg.nodeCount} entries; restored from canonical in this fold)`);
1205
+ }
1206
+ }
1207
+ if (poisonedL0.length > 0) {
1208
+ await this.quarantineSegments(poisonedL0);
1209
+ if (typeof this.storage?.getNouns === 'function') {
1210
+ // Quarantine fold: the poisoned segments' entries exist ONLY in
1211
+ // canonical now — hydrate the delta BEFORE the dedup sets below are
1212
+ // computed, so newest-wins exclusion sees the full canonical set
1213
+ // (base slots excluded for delta ids, healthy-L0 rows deduped
1214
+ // against the delta — no duplicate ints, complete output).
1215
+ // O(canonical) reads, paid only on an actual corruption event —
1216
+ // the alternative (a base silently missing entities) violates
1217
+ // serving-truth. A scoped missing-ids-only backfill rides the M3
1218
+ // fact-log scan path.
1219
+ await this.hydrateDeltaFromStorage(this.config.dimensions);
1220
+ }
1221
+ }
1184
1222
  // Tombstone ints (used to exclude entries from BOTH the base and the L0
1185
1223
  // read-back). Ids never indexed have no int — nothing to exclude.
1186
1224
  const tombIntKeys = new Set();
@@ -1256,11 +1294,14 @@ export class NativeDiskAnnWrapper {
1256
1294
  let dim = this.config.dimensions ??
1257
1295
  (this.native ? this.native.header().dim : undefined) ??
1258
1296
  builtL0.find((s) => s.nodeCount > 0)?.native.header().dim;
1259
- if (liveOldSlots.length === 0 && typeof this.storage?.getNouns === 'function') {
1297
+ if (liveOldSlots.length === 0 &&
1298
+ poisonedL0.length === 0 &&
1299
+ typeof this.storage?.getNouns === 'function') {
1260
1300
  const probe = await this.storage.getNouns({ pagination: { limit: 1 } });
1261
1301
  if (probe.totalCount === undefined || probe.totalCount > this.delta.size) {
1262
1302
  // Cold/restore rebuild: hydrate also adopts the dim from canonical data
1263
1303
  // when we still don't have it, and returns the width it used.
1304
+ // (A quarantine fold already hydrated ABOVE, before the dedup sets.)
1264
1305
  dim = await this.hydrateDeltaFromStorage(dim);
1265
1306
  }
1266
1307
  }
@@ -1372,6 +1413,7 @@ export class NativeDiskAnnWrapper {
1372
1413
  outputPath: this.config.indexPath,
1373
1414
  cfg,
1374
1415
  slotIds: slotIdsBuf,
1416
+ sourceGeneration: await this.readCommittedGeneration(),
1375
1417
  });
1376
1418
  // Atomic swap. The native engine now owns both off-heap slotmaps
1377
1419
  // (written inside rebuildFromExisting), so there is NO resident
@@ -1461,6 +1503,48 @@ export class NativeDiskAnnWrapper {
1461
1503
  segPath(file) {
1462
1504
  return `${dirname(this.config.indexPath)}/${file}`;
1463
1505
  }
1506
+ /**
1507
+ * The committed-transaction watermark (M3 PD-2 `sourceGeneration`): read
1508
+ * from brainy's generation manifest (`_system/manifest.json`, the same
1509
+ * object the generation store treats as committed truth). Read-only and
1510
+ * ADVISORY — hosts without the manifest (or a faulted read) stamp no
1511
+ * sourceGeneration and verifiers skip the comparison. Never constructs a
1512
+ * FactLog (its open() is writer-side reconciliation).
1513
+ */
1514
+ async readCommittedGeneration() {
1515
+ return readCommittedGeneration(this.storage);
1516
+ }
1517
+ /**
1518
+ * Move poisoned L0 segments out of the live set into
1519
+ * `<indexDir>/quarantine/` — preserved for forensics, NEVER deleted
1520
+ * (accord's 545 poisoned segments were the evidence that convicted the
1521
+ * June dim bug). Crash-safe order: the manifest stops referencing the
1522
+ * segments FIRST (a crash after that leaves orphaned, de-referenced
1523
+ * files — a harmless leak), then the files move. The dropped native
1524
+ * handles close with their references; nothing reads a de-referenced
1525
+ * segment.
1526
+ */
1527
+ async quarantineSegments(poisoned) {
1528
+ const drop = new Set(poisoned.map((s) => s.id));
1529
+ this.l0Segments = this.l0Segments.filter((s) => !drop.has(s.id));
1530
+ await this.saveSegmentManifest();
1531
+ const qDir = join(dirname(this.config.indexPath), 'quarantine');
1532
+ for (const seg of poisoned) {
1533
+ try {
1534
+ mkdirSync(qDir, { recursive: true });
1535
+ const src = this.segPath(seg.file);
1536
+ const family = [src, this.siblingPath(src, 'slotmap'), this.siblingPath(src, 'slotrev')];
1537
+ for (const p of family) {
1538
+ if (existsSync(p))
1539
+ renameSync(p, join(qDir, basename(p)));
1540
+ }
1541
+ }
1542
+ catch (error) {
1543
+ prodLog.error(`NativeDiskAnnWrapper: quarantine move failed for ${seg.file} — ` +
1544
+ `segment stays de-referenced on disk (manifest no longer lists it)`, error);
1545
+ }
1546
+ }
1547
+ }
1464
1548
  /** Membership across the base + every L0 segment (int → any slot). */
1465
1549
  hasIntAnywhere(int) {
1466
1550
  if (this.native && this.native.hasInt(int))
@@ -1675,6 +1759,7 @@ export class NativeDiskAnnWrapper {
1675
1759
  outputPath: this.segPath(file),
1676
1760
  cfg,
1677
1761
  slotIds: Buffer.from(slotIdsArr.buffer, slotIdsArr.byteOffset, slotIdsArr.byteLength),
1762
+ sourceGeneration: await this.readCommittedGeneration(),
1678
1763
  });
1679
1764
  // File (+ sidecars) durably on disk → register → manifest (crash-safe order).
1680
1765
  this.l0Segments.push({ id, file, nodeCount: builtDelta.size, native: newNative });
@@ -1787,10 +1872,17 @@ export class NativeDiskAnnWrapper {
1787
1872
  violations: [`stamp ${stampPath} has no members map — cannot prove family coherence`],
1788
1873
  };
1789
1874
  }
1875
+ // Member sizes, both wire modes (M3 unified stamp, 2026-07-15): the
1876
+ // unified shape carries {mode:'enumerated', files:[{path,bytes}]}; the
1877
+ // legacy cor shape is a flat basename→bytes map. One verifier, one page.
1878
+ const rawMembers = stamp.members;
1879
+ const memberSizes = Array.isArray(rawMembers.files)
1880
+ ? Object.fromEntries(rawMembers.files.map((f) => [f.path, f.bytes]))
1881
+ : stamp.members;
1790
1882
  const dir = dirname(indexPath);
1791
1883
  const violations = [];
1792
1884
  const faults = [];
1793
- for (const [name, expected] of Object.entries(stamp.members)) {
1885
+ for (const [name, expected] of Object.entries(memberSizes)) {
1794
1886
  try {
1795
1887
  const actual = statSync(join(dir, name)).size;
1796
1888
  if (actual !== expected) {
@@ -751,6 +751,31 @@ export interface NativeGraphAdjacencyInstance {
751
751
  * when LSM-trees are already loaded from persisted SSTables.
752
752
  */
753
753
  trackVerbId(verbId: string, verbType: string): void;
754
+ /**
755
+ * Full-fidelity membership tracking for the boot heal walk (3.0.19):
756
+ * roster + per-type union + counts + per-(type, subtype) bucket. The
757
+ * membership-only walk dropped subtype state, silently emptying
758
+ * `findConnectedSubtype` after every restart.
759
+ */
760
+ trackVerbIdFull(verbId: string, verbTypeIndex: number, subtype: string): void;
761
+ /**
762
+ * Serialize every RAM-resident membership structure (roster, per-type
763
+ * unions, per-(type, subtype) buckets, both count families) to one
764
+ * strictly-framed buffer for stamped persistence (3.0.19).
765
+ */
766
+ serializeVerbMembership(): Buffer;
767
+ /**
768
+ * Restore membership from a `serializeVerbMembership` buffer. Throws on
769
+ * ANY malformation (magic/version/id-space/truncation/trailing bytes) —
770
+ * the caller falls back to the canonical heal walk. Idempotent overwrite.
771
+ */
772
+ loadVerbMembership(buf: Buffer): void;
773
+ /**
774
+ * Removal-law removal (3.0.19): removes a verb resolving type/subtype
775
+ * from cor-owned state (endpoint store / membership buckets) — never
776
+ * reads canonical. Returns false when unknown/already dead (idempotent).
777
+ */
778
+ removeVerbByIdOnly(verbId: string, generation: bigint): boolean;
754
779
  /** Clear membership bitmap + relationship counts (rebuild prelude). */
755
780
  clearVerbTracking(): void;
756
781
  /**
@@ -976,6 +1001,15 @@ export interface NativeMetadataIndexInstance {
976
1001
  * Never throws; empty index → `true`. Backs the wrapper's `probeConsistency`.
977
1002
  */
978
1003
  probeConsistency(): boolean;
1004
+ /**
1005
+ * Suspect-surfacing probe (3.0.19): same bounded walk, but returns the
1006
+ * first suspicious `{field, uuid}` as JSON (null = clean) so the wrapper
1007
+ * can adjudicate it with one canonical read instead of guessing from the
1008
+ * sample MIX.
1009
+ */
1010
+ probeConsistencySuspect(): string | null;
1011
+ /** Mark a field learned-multi-valued after canonical adjudication (3.0.19). */
1012
+ markFieldMultiValue(field: string): void;
979
1013
  /**
980
1014
  * Scan for the cross-bucket phantom signature (a query-visible int in >1
981
1015
  * value bucket of a field) and return up to `maxCandidates` (0 = unlimited)
@@ -858,6 +858,24 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
858
858
  */
859
859
  private flushChain;
860
860
  flush(): Promise<void>;
861
+ /**
862
+ * Write the metadata index's family stamp (shared shape/functions from
863
+ * brainy internals): rollup invariant = the posted-entity total the
864
+ * detector already computes; `sourceGeneration` = the committed
865
+ * watermark these postings cover. Advisory end to end — a host without
866
+ * the stamp surface or the watermark simply doesn't stamp, and opens
867
+ * keep their walk-based healing.
868
+ */
869
+ private persistMetadataStamp;
870
+ /**
871
+ * M3 PD-4: incremental catch-up from the generation fact log. Applies
872
+ * exactly the ops in `[fromGeneration, head]` — after-image upserts via
873
+ * the normal add path (idempotent: eviction removes stale buckets),
874
+ * tombstones via the id-only removal path (removal law). Returns false
875
+ * (and logs) on ANY break — no capability, scan gap, apply fault — so
876
+ * the caller falls back to the walk; a half-replay never claims success.
877
+ */
878
+ private replayFactGap;
861
879
  private flushOnce;
862
880
  /**
863
881
  * Persist all state and release the column store's resources. Brainy's
@@ -18,6 +18,7 @@
18
18
  import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
19
19
  import { dirname, join } from 'node:path';
20
20
  import { prodLog, getGlobalCache, FieldTypeInference, resolveEntityField } from '@soulcraft/brainy/internals';
21
+ import { readCommittedGeneration } from './generationWatermark.js';
21
22
  import { compareCodePoints } from './collation.js';
22
23
  /**
23
24
  * Fields whose values are stored in the sparse index as BUCKETED values
@@ -84,6 +85,33 @@ const LSM_MANIFEST_KEY = '__metadata_lsm_manifest__';
84
85
  * CORTEX-RESTART-STRAND 2026-07-13 — four consecutive boots).
85
86
  */
86
87
  const UNPOSTABLE_STAMP_KEY = '__metadata_verified_unpostable__';
88
+ /**
89
+ * M3 PD-2/PD-4 (Stage 1): the metadata index's family stamp — written via
90
+ * brainy's SHARED stamp functions (one verifier, literally one function).
91
+ * `sourceGeneration` records the committed watermark the postings cover;
92
+ * a coherent-but-BEHIND stamp at open triggers an incremental FACT REPLAY
93
+ * of exactly the gap instead of any walk. Path/family per brainy's
94
+ * FAMILY_STAMPS_PREFIX convention.
95
+ */
96
+ const METADATA_FAMILY = 'cor-metadata-lsm';
97
+ const METADATA_STAMP_PATH = '_system/family-stamps/cor-metadata-lsm.json';
98
+ let familyStampFnsCache;
99
+ async function familyStampFns() {
100
+ if (familyStampFnsCache !== undefined)
101
+ return familyStampFnsCache;
102
+ try {
103
+ const internals = (await import('@soulcraft/brainy/internals'));
104
+ familyStampFnsCache =
105
+ typeof internals.readFamilyStamp === 'function' &&
106
+ typeof internals.writeFamilyStamp === 'function'
107
+ ? internals
108
+ : null;
109
+ }
110
+ catch {
111
+ familyStampFnsCache = null;
112
+ }
113
+ return familyStampFnsCache;
114
+ }
87
115
  /** Above this many ids the stamp stores count-only (the boot-time
88
116
  * re-verify would stop being cheap); detection then carries a bounded
89
117
  * staleness caveat, logged when it engages. */
@@ -1371,24 +1399,33 @@ export class MetadataIndexManager {
1371
1399
  async verifiedUnpostableAllowance() {
1372
1400
  try {
1373
1401
  const stamp = (await this.storage.getMetadata(UNPOSTABLE_STAMP_KEY));
1374
- if (!stamp || typeof stamp.nounCount !== 'number' || stamp.nounCount <= 0)
1402
+ if (!stamp)
1375
1403
  return 0;
1404
+ // Completion-gap portion (3.0.19): canonical rows a COMPLETED walk
1405
+ // could not even enumerate (no ids exist to re-verify) — count-only
1406
+ // by nature, refreshed on every completed rebuild. The detector's
1407
+ // "FULLY explained by" line reports it whenever it carries the floor.
1408
+ const gap = typeof stamp.completionGap === 'number' && stamp.completionGap > 0
1409
+ ? stamp.completionGap
1410
+ : 0;
1411
+ if (typeof stamp.nounCount !== 'number' || stamp.nounCount <= 0)
1412
+ return gap;
1376
1413
  if (Array.isArray(stamp.nounIds) && stamp.nounIds.length === stamp.nounCount) {
1377
1414
  const getMeta = this.storage.getNounMetadata;
1378
1415
  if (typeof getMeta !== 'function')
1379
- return stamp.nounCount;
1416
+ return stamp.nounCount + gap;
1380
1417
  let stillUnpostable = 0;
1381
1418
  for (const id of stamp.nounIds) {
1382
1419
  const md = await getMeta.call(this.storage, id).catch(() => undefined);
1383
1420
  if (md == null)
1384
1421
  stillUnpostable++;
1385
1422
  }
1386
- return stillUnpostable;
1423
+ return stillUnpostable + gap;
1387
1424
  }
1388
1425
  prodLog.warn(`[NativeMetadataIndex] unpostable stamp is count-only ` +
1389
1426
  `(${stamp.nounCount} > id cap) — the posted-count floor carries its ` +
1390
1427
  `staleness until the next completed rebuild refreshes it.`);
1391
- return stamp.nounCount;
1428
+ return stamp.nounCount + gap;
1392
1429
  }
1393
1430
  catch {
1394
1431
  return 0;
@@ -1397,6 +1434,29 @@ export class MetadataIndexManager {
1397
1434
  async detectDerivedStrand() {
1398
1435
  if (!this.lsmEngine || !this.lsmMetaDir)
1399
1436
  return;
1437
+ // M3 PD-4 (Stage 1): a coherent-but-BEHIND family stamp means the
1438
+ // postings simply predate the newest commits — replay EXACTLY the gap
1439
+ // from the generation fact log before any strand verdict, so the
1440
+ // checks below see a caught-up index and no walk fires for what was
1441
+ // never damage. Advisory end to end: any break (no 8.5.0 surface, no
1442
+ // watermark, scan fault) falls through and the detector stays
1443
+ // authoritative.
1444
+ try {
1445
+ const fns = await familyStampFns();
1446
+ const head = await readCommittedGeneration(this.storage);
1447
+ if (fns && head !== undefined) {
1448
+ const stamp = (await fns.readFamilyStamp(this.storage, METADATA_STAMP_PATH));
1449
+ const src = stamp?.family === METADATA_FAMILY ? stamp.sourceGeneration : undefined;
1450
+ if (typeof src === 'number' && src < head) {
1451
+ if (await this.replayFactGap(src + 1)) {
1452
+ await this.persistMetadataStamp();
1453
+ }
1454
+ }
1455
+ }
1456
+ }
1457
+ catch {
1458
+ // Detector below stays authoritative.
1459
+ }
1400
1460
  // 0. Replay-stats verdict (3.0.12). The native open now RECREATES a
1401
1461
  // missing live log and recovers what its archives hold, so the
1402
1462
  // filename census below can no longer see the deleted-live-log
@@ -2655,12 +2715,95 @@ export class MetadataIndexManager {
2655
2715
  'flush would multiply heal IO).');
2656
2716
  return;
2657
2717
  }
2658
- const run = this.flushChain.then(() => this.flushOnce());
2718
+ const run = this.flushChain.then(async () => {
2719
+ await this.flushOnce();
2720
+ // M3 Stage 1: the flushed postings durably cover the committed
2721
+ // watermark — stamp it so the next open replays only the gap
2722
+ // (advisory: no-ops without the 8.5.0 stamp surface).
2723
+ await this.persistMetadataStamp();
2724
+ });
2659
2725
  // Keep the chain alive on failure — the NEXT flush must still run; the
2660
2726
  // failure itself propagates to THIS caller below.
2661
2727
  this.flushChain = run.catch(() => { });
2662
2728
  return run;
2663
2729
  }
2730
+ /**
2731
+ * Write the metadata index's family stamp (shared shape/functions from
2732
+ * brainy internals): rollup invariant = the posted-entity total the
2733
+ * detector already computes; `sourceGeneration` = the committed
2734
+ * watermark these postings cover. Advisory end to end — a host without
2735
+ * the stamp surface or the watermark simply doesn't stamp, and opens
2736
+ * keep their walk-based healing.
2737
+ */
2738
+ async persistMetadataStamp() {
2739
+ try {
2740
+ const fns = await familyStampFns();
2741
+ if (!fns)
2742
+ return;
2743
+ const sourceGeneration = await readCommittedGeneration(this.storage);
2744
+ if (sourceGeneration === undefined)
2745
+ return;
2746
+ const { NounType } = await import('@soulcraft/brainy');
2747
+ let posted = 0;
2748
+ for (const t of Object.values(NounType)) {
2749
+ posted += this.native.getEntityCountByType(t);
2750
+ }
2751
+ await fns.writeFamilyStamp(this.storage, METADATA_STAMP_PATH, {
2752
+ family: METADATA_FAMILY,
2753
+ sourceGeneration,
2754
+ members: { mode: 'rollup', invariants: { posted } },
2755
+ });
2756
+ }
2757
+ catch {
2758
+ // Advisory: a failed stamp means the next open heals the old way.
2759
+ }
2760
+ }
2761
+ /**
2762
+ * M3 PD-4: incremental catch-up from the generation fact log. Applies
2763
+ * exactly the ops in `[fromGeneration, head]` — after-image upserts via
2764
+ * the normal add path (idempotent: eviction removes stale buckets),
2765
+ * tombstones via the id-only removal path (removal law). Returns false
2766
+ * (and logs) on ANY break — no capability, scan gap, apply fault — so
2767
+ * the caller falls back to the walk; a half-replay never claims success.
2768
+ */
2769
+ async replayFactGap(fromGeneration) {
2770
+ const scanFn = this.storage.scanFacts;
2771
+ if (typeof scanFn !== 'function')
2772
+ return false;
2773
+ const scan = scanFn.call(this.storage, { fromGeneration, batchSize: 256 });
2774
+ if (!scan)
2775
+ return false;
2776
+ let upserts = 0;
2777
+ let removals = 0;
2778
+ try {
2779
+ for await (const batch of scan.batches()) {
2780
+ for (const fact of batch.facts) {
2781
+ for (const op of fact.ops) {
2782
+ if (op.record === null) {
2783
+ await this.removeFromIndex(op.id);
2784
+ removals++;
2785
+ }
2786
+ else if (op.record.metadata != null) {
2787
+ await this.addToIndex(op.id, op.record.metadata, true, true);
2788
+ upserts++;
2789
+ }
2790
+ // record present with null metadata = vector-only write —
2791
+ // nothing to post here.
2792
+ }
2793
+ }
2794
+ }
2795
+ }
2796
+ catch (error) {
2797
+ opsLine(`fact replay from generation ${fromGeneration} ABORTED (${String(error)}) — ` +
2798
+ `falling back to the walk-based heal path`);
2799
+ return false;
2800
+ }
2801
+ if (upserts + removals > 0) {
2802
+ opsLine(`fact replay: caught up ${upserts} upsert(s) + ${removals} removal(s) ` +
2803
+ `from the generation log — no walk needed`);
2804
+ }
2805
+ return true;
2806
+ }
2664
2807
  async flushOnce() {
2665
2808
  // Always save field registry + msync the entity ID mapper — even with
2666
2809
  // no dirty fields. The field registry is the critical file init() needs
@@ -2847,7 +2990,53 @@ export class MetadataIndexManager {
2847
2990
  await this.ensureFieldLoaded(field);
2848
2991
  loaded++;
2849
2992
  }
2850
- return this.native.probeConsistency();
2993
+ // Adjudicated probe (3.0.19): the native walk surfaces ONE suspect
2994
+ // {field, uuid}; a single canonical read decides it. ≥2 current
2995
+ // values = a mixed-length ARRAY field, not a phantom — learned,
2996
+ // persisted, and the probe re-runs (each pass retires one field
2997
+ // permanently, so this converges in ≤ field count; bounded harder
2998
+ // below). Anything else = a genuine phantom → false, exactly the old
2999
+ // detection strength. This replaces the MIX guess whose verdict
3000
+ // depended on which ints the sample happened to draw (accord's brain:
3001
+ // clean boots 1–2, failing boot 3, identical bytes).
3002
+ if (typeof this.native.probeConsistencySuspect !== 'function') {
3003
+ // Pre-3.0.19 binary: no adjudication surface — use the sampled MIX
3004
+ // verdict. WITHOUT this explicit fallback the call below throws into
3005
+ // the outer catch, which assumes healthy — i.e. a stale binary would
3006
+ // silently DISABLE phantom detection (the 3.0.12–3.0.15 class).
3007
+ return this.native.probeConsistency();
3008
+ }
3009
+ const MAX_ADJUDICATIONS = 8;
3010
+ for (let i = 0; i <= MAX_ADJUDICATIONS; i++) {
3011
+ const suspectJson = this.native.probeConsistencySuspect();
3012
+ if (!suspectJson)
3013
+ return true;
3014
+ if (i === MAX_ADJUDICATIONS)
3015
+ return false;
3016
+ const { field, uuid } = JSON.parse(suspectJson);
3017
+ let currentValues = [];
3018
+ try {
3019
+ const noun = await this.storage.getNoun(uuid);
3020
+ if (noun) {
3021
+ const raw = resolveEntityField(noun, field);
3022
+ if (raw !== undefined && raw !== null) {
3023
+ currentValues = Array.isArray(raw) ? raw : [raw];
3024
+ }
3025
+ }
3026
+ }
3027
+ catch {
3028
+ // Unreadable canonical = cannot exonerate — keep the phantom verdict.
3029
+ }
3030
+ if (currentValues.length < 2)
3031
+ return false;
3032
+ this.native.markFieldMultiValue(field);
3033
+ const json = this.native.saveFieldIndex(field);
3034
+ if (json) {
3035
+ await this.storage.saveMetadata(`__metadata_field_index__field_${field}`, JSON.parse(json));
3036
+ }
3037
+ prodLog.info(`[probeConsistency] field '${field}' adjudicated multi-valued against canonical (${currentValues.length} current values) — learned + persisted, exempt from phantom checks`);
3038
+ }
3039
+ return false;
2851
3040
  }
2852
3041
  catch (error) {
2853
3042
  // A probe must never throw into the read path — assume healthy.
@@ -2943,6 +3132,14 @@ export class MetadataIndexManager {
2943
3132
  purged++;
2944
3133
  repairedFields.add(field);
2945
3134
  }
3135
+ // MULTI-VALUE LEARNING persistence (3.0.19): ≥2 current values
3136
+ // proves the field is array-shaped — the native side just learned
3137
+ // it (probe/detector exemption); persist the flag with the field
3138
+ // index or the next cold boot re-flags the same candidates
3139
+ // forever (accord's 166-loop, second mechanism).
3140
+ if (currentValues.length >= 2) {
3141
+ repairedFields.add(field);
3142
+ }
2946
3143
  }
2947
3144
  catch (error) {
2948
3145
  prodLog.warn(`[purgeCrossBucketPhantoms] repair of ${uuid}.${field} failed: ${error}`);
@@ -3073,12 +3270,52 @@ export class MetadataIndexManager {
3073
3270
  // detector uses the honest floor instead of re-flagging the same
3074
3271
  // cohort forever. Refreshed on every completed rebuild; an empty
3075
3272
  // set clears the stamp.
3273
+ //
3274
+ // COMPLETION GAP (3.0.19, memory's 52): canonical rows the walk
3275
+ // CANNOT enumerate — dangling counter rows with no readable record —
3276
+ // never appear in items, so the per-id stamp stayed EMPTY while the
3277
+ // detector's canonical count still included them: every detector
3278
+ // boot re-healed (~11 min on memory-vm) forever. A COMPLETED walk is
3279
+ // the strongest possible statement of "everything enumerable is
3280
+ // posted", so the residual canonical-minus-walked difference is
3281
+ // stamped as a count-only allowance (invisible rows have no ids).
3282
+ // Bounded staleness: refreshed on every completed walk; brainy's
3283
+ // counter recount (8.3.2 repairIndex) removes the gap permanently.
3284
+ const walked = nounResult.processed + nounResult.noMetadata;
3285
+ let completionGap = 0;
3286
+ let canonicalAtCompletion;
3287
+ try {
3288
+ const getNouns = this.storage.getNouns;
3289
+ if (typeof getNouns === 'function') {
3290
+ const probe = await getNouns.call(this.storage, { pagination: { limit: 1 } });
3291
+ const canonical = probe?.totalCount;
3292
+ if (typeof canonical === 'number' &&
3293
+ Number.isFinite(canonical) &&
3294
+ canonical < 0xffff_ffff) {
3295
+ canonicalAtCompletion = canonical;
3296
+ completionGap = Math.max(0, canonical - walked);
3297
+ }
3298
+ }
3299
+ }
3300
+ catch {
3301
+ // Gap probe is advisory — a failed canonical read stamps no gap
3302
+ // (may over-heal next boot, never under-detects).
3303
+ }
3076
3304
  await this.storage.saveMetadata(UNPOSTABLE_STAMP_KEY, {
3077
3305
  nounCount: nounResult.noMetadata,
3078
3306
  nounIds: nounResult.noMetadataIds,
3079
3307
  verbCount: verbResult.noMetadata,
3308
+ completionGap,
3309
+ canonicalAtCompletion,
3080
3310
  verifiedAt: Date.now(),
3081
3311
  });
3312
+ if (completionGap > 0) {
3313
+ opsLine(`rebuild: canonical reports ${canonicalAtCompletion} nouns but only ` +
3314
+ `${walked} were enumerable by a COMPLETED walk — the ${completionGap}-entity ` +
3315
+ `gap is stamped as a completion-gap allowance (unenumerable canonical ` +
3316
+ `rows, e.g. inflated counters; a brainy counter recount removes them ` +
3317
+ `permanently). Boots stop re-healing over this gap.`);
3318
+ }
3082
3319
  opsLine(`Metadata index rebuild completed! Processed ${nounResult.processed} nouns and ` +
3083
3320
  `${verbResult.processed} verbs` +
3084
3321
  (nounResult.noMetadata > 0
@@ -0,0 +1,15 @@
1
+ /**
2
+ * @module utils/generationWatermark
3
+ * @description The committed-generation watermark (M3 PD-2
4
+ * `sourceGeneration`) — what a projection stamp records as the source
5
+ * truth it reflects. Read CAPABILITY-FIRST: brainy 8.5.0 exposes
6
+ * `storage.committedGeneration()` precisely so providers never parse the
7
+ * store's private manifest format; the direct `_system/manifest.json`
8
+ * read remains as the 8.4.0-host fallback for one release line. Advisory
9
+ * everywhere: `undefined` (no capability, no manifest, faulted read)
10
+ * means the stamp carries no sourceGeneration and verifiers skip the
11
+ * comparison — never a blocked write, never a guessed value.
12
+ */
13
+ /** Best-effort committed watermark; `undefined` when unknowable. */
14
+ export declare function readCommittedGeneration(storage: unknown): Promise<number | undefined>;
15
+ //# sourceMappingURL=generationWatermark.d.ts.map
@@ -0,0 +1,35 @@
1
+ /**
2
+ * @module utils/generationWatermark
3
+ * @description The committed-generation watermark (M3 PD-2
4
+ * `sourceGeneration`) — what a projection stamp records as the source
5
+ * truth it reflects. Read CAPABILITY-FIRST: brainy 8.5.0 exposes
6
+ * `storage.committedGeneration()` precisely so providers never parse the
7
+ * store's private manifest format; the direct `_system/manifest.json`
8
+ * read remains as the 8.4.0-host fallback for one release line. Advisory
9
+ * everywhere: `undefined` (no capability, no manifest, faulted read)
10
+ * means the stamp carries no sourceGeneration and verifiers skip the
11
+ * comparison — never a blocked write, never a guessed value.
12
+ */
13
+ /** Best-effort committed watermark; `undefined` when unknowable. */
14
+ export async function readCommittedGeneration(storage) {
15
+ try {
16
+ const s = storage;
17
+ if (typeof s?.committedGeneration === 'function') {
18
+ const g = s.committedGeneration();
19
+ if (typeof g === 'number' && Number.isFinite(g) && g >= 0)
20
+ return g;
21
+ // Capability present but unwired (bare adapter) — fall through to the
22
+ // manifest read: a stamped brain opened outside a host brain should
23
+ // still stamp truthfully when the manifest exists.
24
+ }
25
+ if (typeof s?.readRawObject !== 'function')
26
+ return undefined;
27
+ const manifest = (await s.readRawObject('_system/manifest.json'));
28
+ const g = manifest?.generation;
29
+ return typeof g === 'number' && Number.isFinite(g) && g >= 0 ? g : undefined;
30
+ }
31
+ catch {
32
+ return undefined;
33
+ }
34
+ }
35
+ //# sourceMappingURL=generationWatermark.js.map
package/dist/version.d.ts CHANGED
@@ -11,5 +11,5 @@
11
11
  * `check:brainy`-style drift between the two fails the release.
12
12
  */
13
13
  /** The @soulcraft/cor version this build was cut from. */
14
- export declare const COR_VERSION = "3.0.18";
14
+ export declare const COR_VERSION = "3.0.20";
15
15
  //# sourceMappingURL=version.d.ts.map
package/dist/version.js CHANGED
@@ -11,5 +11,5 @@
11
11
  * `check:brainy`-style drift between the two fails the release.
12
12
  */
13
13
  /** The @soulcraft/cor version this build was cut from. */
14
- export const COR_VERSION = '3.0.18';
14
+ export const COR_VERSION = '3.0.20';
15
15
  //# sourceMappingURL=version.js.map
Binary file
package/native/index.d.ts CHANGED
@@ -722,6 +722,18 @@ export declare class NativeGraphAdjacencyIndex {
722
722
  * solely for legacy-data remove paths.
723
723
  */
724
724
  removeVerb(verbId: string, verbTypeIndex: number, subtype: string, generation: bigint): void
725
+ /**
726
+ * **Removal law (3.0.19)**: removal must never require reading the
727
+ * thing being removed. The JS remove path read the verb's CANONICAL
728
+ * row to learn (type, subtype) and silently SKIPPED removal when that
729
+ * row was unreadable — a ghost's graph rows survived forever (the
730
+ * third instance of the anti-pattern, this one cor's own). This
731
+ * variant resolves everything from cor-owned state: type from the
732
+ * endpoint store (or the per-type unions for membership-only verbs),
733
+ * subtype by scanning that type's buckets for the interned id.
734
+ * Returns `false` when the id is unknown or already dead (idempotent).
735
+ */
736
+ removeVerbByIdOnly(verbId: string, generation: bigint): boolean
725
737
  /**
726
738
  * **Look up a verb's endpoints by verb-ID.** Returns
727
739
  * `{sourceInt, targetInt}` (as u64 BigInts) when the verb was
@@ -999,6 +1011,43 @@ export declare class NativeGraphAdjacencyIndex {
999
1011
  * counts converge to the same totals).
1000
1012
  */
1001
1013
  trackVerbId(verbId: string, verbType: string): void
1014
+ /**
1015
+ * Membership-only tracking that ALSO restores the per-(type, subtype)
1016
+ * bucket and subtype count — the full-fidelity variant of
1017
+ * [`track_verb_id`](Self::track_verb_id) for the boot heal walk.
1018
+ * The membership-only walk used to drop subtype state entirely, so
1019
+ * `findConnectedSubtype` silently returned nothing after a restart
1020
+ * until a full adjacency rebuild happened to run (3.0.19 find).
1021
+ * Same empty-string escape as the add path: `subtype: ''` tracks
1022
+ * type-level membership only, never an empty-string bucket.
1023
+ */
1024
+ trackVerbIdFull(verbId: string, verbTypeIndex: number, subtype: string): void
1025
+ /**
1026
+ * Serialize EVERY RAM-resident membership structure to one buffer so
1027
+ * the JS wrapper can persist it as a stamped family member (3.0.19:
1028
+ * the roster/buckets/counts were RAM-only, costing every boot an
1029
+ * O(all-canonical-verbs) walk and silently emptying subtype filters).
1030
+ * Format v1, little-endian, strictly length-framed:
1031
+ * magic 'CVMB' u32 | version u8 | idSpace u8 (0=U32, 1=U64) |
1032
+ * roster (u32 len + portable bytes) |
1033
+ * u8 nUnions { u8 typeIdx, u32 len, bytes } |
1034
+ * u32 nSubBuckets { u8 typeIdx, u16 subLen, utf8, u32 len, bytes } |
1035
+ * u8 nCounts { u8 typeIdx, u64 count } |
1036
+ * u32 nSubCounts { u8 typeIdx, u16 subLen, utf8, u64 count }
1037
+ * Subtype keys are emitted sorted so identical state yields identical
1038
+ * bytes (the stamp records the exact byte size).
1039
+ */
1040
+ serializeVerbMembership(): Buffer
1041
+ /**
1042
+ * Restore every membership structure from a
1043
+ * [`serialize_verb_membership`](Self::serialize_verb_membership)
1044
+ * buffer. Strict total parse: ANY malformation (bad magic/version,
1045
+ * id-space mismatch with this index, truncation, trailing bytes,
1046
+ * out-of-range type index) throws so the caller falls back to the
1047
+ * canonical heal walk — a torn blob must never half-load silently.
1048
+ * Clears existing membership state first (idempotent overwrite).
1049
+ */
1050
+ loadVerbMembership(buf: Buffer): void
1002
1051
  /**
1003
1052
  * Clear membership bitmap + relationship counts. Used before
1004
1053
  * rebuild to ensure clean state. The verb-ID namespace's
@@ -1870,6 +1919,23 @@ export declare class NativeMetadataIndex {
1870
1919
  * read path.
1871
1920
  */
1872
1921
  probeConsistency(): boolean
1922
+ /**
1923
+ * The suspect-surfacing probe (3.0.19): same bounded walk as
1924
+ * `probeConsistency`, but instead of a bare verdict it returns the FIRST
1925
+ * suspicious `{field, uuid}` as JSON (None = clean). The TS wrapper
1926
+ * ADJUDICATES the suspect with one canonical read — ≥2 current values
1927
+ * proves the field is array-shaped (learned + persisted, probe re-runs),
1928
+ * anything else is a genuine phantom (probe fails). This replaces the
1929
+ * MIX guess whose verdict depended on WHICH ints the sample drew —
1930
+ * accord's brain passed boots 1–2 and failed boot 3 on identical bytes.
1931
+ */
1932
+ probeConsistencySuspect(): string | null
1933
+ /**
1934
+ * Mark a field as learned-multi-valued (3.0.19). Called by the TS
1935
+ * wrapper after canonical adjudication proves an entity legitimately
1936
+ * holds ≥2 values for the field. The caller persists the field index.
1937
+ */
1938
+ markFieldMultiValue(field: string): void
1873
1939
  /**
1874
1940
  * **Phantom triad — state-level scan (matrix 7.8b).** Find every
1875
1941
  * query-visible int that sits in MORE THAN ONE value bucket of a field —
@@ -2563,6 +2629,13 @@ export interface DiskAnnRebuildFromExistingParams {
2563
2629
  * the slot-only benchmark paths that don't carry entity ids.
2564
2630
  */
2565
2631
  slotIds?: Buffer
2632
+ /**
2633
+ * M3 PD-2: the source-of-truth log generation this publish reflects
2634
+ * (the committed watermark the JS wrapper read at rebuild time).
2635
+ * Omitted on hosts without a generation manifest — the stamp then
2636
+ * carries no `sourceGeneration` and verifiers skip the comparison.
2637
+ */
2638
+ sourceGeneration?: number
2566
2639
  }
2567
2640
 
2568
2641
  /** Vamana graph build knobs exposed to JS. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@soulcraft/cor",
3
- "version": "3.0.18",
3
+ "version": "3.0.20",
4
4
  "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -82,7 +82,7 @@
82
82
  },
83
83
  "devDependencies": {
84
84
  "@napi-rs/cli": "^3.0.0",
85
- "@soulcraft/brainy": "8.3.0",
85
+ "@soulcraft/brainy": "8.5.0",
86
86
  "@types/node": "^22.0.0",
87
87
  "pg": "^8.21.0",
88
88
  "tsx": "^4.21.0",