@soulcraft/cor 3.0.18 → 3.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +36 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +231 -17
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +20 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +103 -11
- package/dist/native/types.d.ts +34 -0
- package/dist/utils/NativeMetadataIndex.d.ts +18 -0
- package/dist/utils/NativeMetadataIndex.js +243 -6
- package/dist/utils/generationWatermark.d.ts +15 -0
- package/dist/utils/generationWatermark.js +35 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +73 -0
- package/package.json +2 -2
|
@@ -86,6 +86,16 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
|
|
|
86
86
|
*/
|
|
87
87
|
private canonicalUnindexed;
|
|
88
88
|
private isRebuilding;
|
|
89
|
+
/** Membership mutated since the last persisted blob (write-side dirty bit). */
|
|
90
|
+
private membershipDirty;
|
|
91
|
+
/** Cached stamp content (minus committedAt) — identical payload = no I/O on idle ticks. */
|
|
92
|
+
private lastMembershipStampBody;
|
|
93
|
+
/** Byte length of the last persisted membership blob (stamp needs it without re-serializing). */
|
|
94
|
+
private lastMembershipBlobBytes;
|
|
95
|
+
/** Monotonic stamp lineage (unified shape); restored at load, bumped at persist. */
|
|
96
|
+
private membershipStampGeneration;
|
|
97
|
+
/** How this boot obtained membership: restored (stamp), healed (walk), or fresh (no verbs). */
|
|
98
|
+
private membershipBootState;
|
|
89
99
|
private flushTimer?;
|
|
90
100
|
private rebuildStartTime;
|
|
91
101
|
private totalRelationshipsIndexed;
|
|
@@ -174,6 +184,32 @@ export declare class GraphAdjacencyIndex implements GraphIndexProvider {
|
|
|
174
184
|
* queryable edges report "not ready".
|
|
175
185
|
*/
|
|
176
186
|
private openGraphLsmEngine;
|
|
187
|
+
/**
|
|
188
|
+
* SHA-256 over the tree's durable SSTable identity (sorted `id:level`
|
|
189
|
+
* lines). The stamp records one per tree; a mismatch at open means the
|
|
190
|
+
* trees advanced (or tore) after the last membership persist, so the
|
|
191
|
+
* blob cannot be trusted and the walk heals. Content-relevant fields
|
|
192
|
+
* only — deliberately NOT the raw manifest JSON (timestamps and field
|
|
193
|
+
* ordering must not fail coherence).
|
|
194
|
+
*/
|
|
195
|
+
private treeFingerprint;
|
|
196
|
+
/**
|
|
197
|
+
* Persist the membership family: blob (only when membership mutated or
|
|
198
|
+
* `force`) then stamp LAST (§7 — the stamp commits the set). Identical
|
|
199
|
+
* stamp payloads are skipped entirely, so idle auto-flush ticks cost
|
|
200
|
+
* zero I/O. Failures log loudly and leave the previous stamp in place —
|
|
201
|
+
* a stale stamp heals at next open; a silent half-persist would not.
|
|
202
|
+
*/
|
|
203
|
+
private persistVerbMembership;
|
|
204
|
+
/**
|
|
205
|
+
* Restore membership from the stamped blob. Returns `true` only when the
|
|
206
|
+
* FULL coherence chain holds: stamp present → blob present with the exact
|
|
207
|
+
* stamped byte length → every tree fingerprint matches the durable state
|
|
208
|
+
* just loaded → the native parser accepts every framed section. Any break
|
|
209
|
+
* returns `false` (absent stamp silently — that is every pre-3.0.19
|
|
210
|
+
* brain; a broken chain with a warn) and the caller's walk heals.
|
|
211
|
+
*/
|
|
212
|
+
private loadVerbMembershipFromStorage;
|
|
177
213
|
private populateVerbIdSetFromStorage;
|
|
178
214
|
getNeighbors(id: bigint, optionsOrDirection?: {
|
|
179
215
|
direction?: 'in' | 'out' | 'both';
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
* since it coordinates cross-subsystem caching and requires async storage access.
|
|
14
14
|
*/
|
|
15
15
|
import { dirname } from 'node:path';
|
|
16
|
+
import { createHash } from 'node:crypto';
|
|
17
|
+
import { readCommittedGeneration } from '../utils/generationWatermark.js';
|
|
16
18
|
import { TypeUtils, VerbType } from '@soulcraft/brainy/types/graphTypes';
|
|
17
19
|
import { loadNativeModule } from '../native/index.js';
|
|
18
20
|
import { composeInvariantReport } from '../utils/invariantReport.js';
|
|
@@ -45,6 +47,38 @@ const VERB_FALLBACK_INDEX = TypeUtils.getVerbIndex(VerbType.RelatedTo) ?? 3;
|
|
|
45
47
|
* (`verb-uuid-to-int.mkv`, `verb-int-to-uuid.bin`, `verb-endpoints/`).
|
|
46
48
|
*/
|
|
47
49
|
const GRAPH_VERB_NS_PROBE_KEY = '_graph_verb_ns/x';
|
|
50
|
+
/**
|
|
51
|
+
* Persisted verb-membership family (3.0.19). The roster / per-type unions /
|
|
52
|
+
* per-(type, subtype) buckets / count families were RAM-only, costing every
|
|
53
|
+
* boot of every verb-bearing brain an O(all-canonical-verbs) walk and
|
|
54
|
+
* silently emptying subtype-filtered traversal after restarts. The blob is
|
|
55
|
+
* the serialized structures; the stamp ties the blob to the exact durable
|
|
56
|
+
* tree state (per-tree SSTable fingerprints) so a torn write is DETECTED at
|
|
57
|
+
* open and heals via the walk — §7 coherent-or-rebuild, same discipline as
|
|
58
|
+
* the vector family.
|
|
59
|
+
*/
|
|
60
|
+
const GRAPH_MEMBERSHIP_BLOB_KEY = 'graph-verb-membership';
|
|
61
|
+
const GRAPH_MEMBERSHIP_STAMP_KEY = 'graph-membership-stamp';
|
|
62
|
+
/** Normalized view both wire shapes reduce to; null = unrecognized. */
|
|
63
|
+
function normalizeMembershipStamp(raw) {
|
|
64
|
+
const s = raw;
|
|
65
|
+
if (s?.members?.mode === 'rollup' && s.members.invariants) {
|
|
66
|
+
const inv = s.members.invariants;
|
|
67
|
+
const blobBytes = inv.blobBytes;
|
|
68
|
+
if (typeof blobBytes !== 'number')
|
|
69
|
+
return null;
|
|
70
|
+
const trees = {};
|
|
71
|
+
for (const [k, v] of Object.entries(inv)) {
|
|
72
|
+
if (k.startsWith('tree:') && typeof v === 'string')
|
|
73
|
+
trees[k.slice(5)] = v;
|
|
74
|
+
}
|
|
75
|
+
return { blobBytes, trees, generation: typeof s.generation === 'number' ? s.generation : 0 };
|
|
76
|
+
}
|
|
77
|
+
if (s?.v === 1 && typeof s.blobBytes === 'number' && s.trees) {
|
|
78
|
+
return { blobBytes: s.blobBytes, trees: s.trees, generation: 0 };
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
48
82
|
/**
|
|
49
83
|
* Resolve a verb-type string to its canonical `VerbTypeEnum` index,
|
|
50
84
|
* falling back to {@link VERB_FALLBACK_INDEX} (`VerbType.RelatedTo`)
|
|
@@ -91,6 +125,16 @@ export class GraphAdjacencyIndex {
|
|
|
91
125
|
*/
|
|
92
126
|
canonicalUnindexed = false;
|
|
93
127
|
isRebuilding = false;
|
|
128
|
+
/** Membership mutated since the last persisted blob (write-side dirty bit). */
|
|
129
|
+
membershipDirty = false;
|
|
130
|
+
/** Cached stamp content (minus committedAt) — identical payload = no I/O on idle ticks. */
|
|
131
|
+
lastMembershipStampBody = '';
|
|
132
|
+
/** Byte length of the last persisted membership blob (stamp needs it without re-serializing). */
|
|
133
|
+
lastMembershipBlobBytes = -1;
|
|
134
|
+
/** Monotonic stamp lineage (unified shape); restored at load, bumped at persist. */
|
|
135
|
+
membershipStampGeneration = 0;
|
|
136
|
+
/** How this boot obtained membership: restored (stamp), healed (walk), or fresh (no verbs). */
|
|
137
|
+
membershipBootState = 'fresh';
|
|
94
138
|
flushTimer;
|
|
95
139
|
rebuildStartTime = 0;
|
|
96
140
|
totalRelationshipsIndexed = 0;
|
|
@@ -293,13 +337,132 @@ export class GraphAdjacencyIndex {
|
|
|
293
337
|
for (const treeName of treeNames) {
|
|
294
338
|
await this.loadTreeFromStorage(treeName);
|
|
295
339
|
}
|
|
340
|
+
// Persisted-membership restore (3.0.19): coherent stamp + blob loads the
|
|
341
|
+
// roster/buckets/counts in O(bitmap) — no canonical walk, subtype filters
|
|
342
|
+
// intact. `restored` also covers the legitimately-empty-roster case (all
|
|
343
|
+
// verbs removed) so the walk below can't re-fire on it every boot.
|
|
344
|
+
const restored = await this.loadVerbMembershipFromStorage();
|
|
296
345
|
// Defensive check: if LSM-trees have data but verbIdSet is empty, populate.
|
|
297
|
-
// This is the
|
|
346
|
+
// This is the membership HEAL path (pre-3.0.19 brains, torn/missing
|
|
347
|
+
// stamp) — it does NOT gate isReady().
|
|
298
348
|
const sourceSize = this.native.size();
|
|
299
|
-
if (sourceSize > 0 && this.native.verbIdCount() === 0) {
|
|
349
|
+
if (!restored && sourceSize > 0 && this.native.verbIdCount() === 0) {
|
|
300
350
|
prodLog.warn(`GraphAdjacencyIndex: LSM-trees have ${sourceSize} relationships but verbIdSet is empty. ` +
|
|
301
351
|
`Triggering auto-rebuild to restore consistency.`);
|
|
302
352
|
await this.populateVerbIdSetFromStorage();
|
|
353
|
+
this.membershipBootState = 'healed';
|
|
354
|
+
// Persist immediately so the NEXT boot restores instead of re-walking.
|
|
355
|
+
await this.persistVerbMembership(true);
|
|
356
|
+
}
|
|
357
|
+
else if (!restored && this.native.verbIdCount() > 0) {
|
|
358
|
+
// Verbs arrived without a stamp (e.g. mid-process rebuild) — treat as
|
|
359
|
+
// healed so the invariant reports honestly.
|
|
360
|
+
this.membershipBootState = 'healed';
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* SHA-256 over the tree's durable SSTable identity (sorted `id:level`
|
|
365
|
+
* lines). The stamp records one per tree; a mismatch at open means the
|
|
366
|
+
* trees advanced (or tore) after the last membership persist, so the
|
|
367
|
+
* blob cannot be trusted and the walk heals. Content-relevant fields
|
|
368
|
+
* only — deliberately NOT the raw manifest JSON (timestamps and field
|
|
369
|
+
* ordering must not fail coherence).
|
|
370
|
+
*/
|
|
371
|
+
treeFingerprint(treeName) {
|
|
372
|
+
const ids = this.native.getManifestSstableIds(treeName);
|
|
373
|
+
const lines = ids
|
|
374
|
+
.map((id) => `${id}:${this.native.getManifestSstableLevel(treeName, id) ?? '?'}`)
|
|
375
|
+
.sort();
|
|
376
|
+
return createHash('sha256').update(lines.join('\n')).digest('hex');
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Persist the membership family: blob (only when membership mutated or
|
|
380
|
+
* `force`) then stamp LAST (§7 — the stamp commits the set). Identical
|
|
381
|
+
* stamp payloads are skipped entirely, so idle auto-flush ticks cost
|
|
382
|
+
* zero I/O. Failures log loudly and leave the previous stamp in place —
|
|
383
|
+
* a stale stamp heals at next open; a silent half-persist would not.
|
|
384
|
+
*/
|
|
385
|
+
async persistVerbMembership(force = false) {
|
|
386
|
+
if (!this.hasBinaryBlobs)
|
|
387
|
+
return;
|
|
388
|
+
try {
|
|
389
|
+
if (this.membershipDirty || force || this.lastMembershipBlobBytes < 0) {
|
|
390
|
+
const blob = this.native.serializeVerbMembership();
|
|
391
|
+
await this.storage.saveBinaryBlob(GRAPH_MEMBERSHIP_BLOB_KEY, Buffer.from(blob));
|
|
392
|
+
this.lastMembershipBlobBytes = blob.length;
|
|
393
|
+
this.membershipDirty = false;
|
|
394
|
+
}
|
|
395
|
+
const invariants = {
|
|
396
|
+
blobBytes: this.lastMembershipBlobBytes,
|
|
397
|
+
};
|
|
398
|
+
for (const treeName of this.native.treeNames()) {
|
|
399
|
+
invariants[`tree:${treeName}`] = this.treeFingerprint(treeName);
|
|
400
|
+
}
|
|
401
|
+
const body = JSON.stringify(invariants);
|
|
402
|
+
if (body === this.lastMembershipStampBody)
|
|
403
|
+
return;
|
|
404
|
+
const stamp = {
|
|
405
|
+
family: 'graph-membership',
|
|
406
|
+
generation: this.membershipStampGeneration + 1,
|
|
407
|
+
committedAt: new Date().toISOString(),
|
|
408
|
+
members: { mode: 'rollup', invariants },
|
|
409
|
+
};
|
|
410
|
+
const sourceGeneration = await readCommittedGeneration(this.storage);
|
|
411
|
+
if (sourceGeneration !== undefined)
|
|
412
|
+
stamp.sourceGeneration = sourceGeneration;
|
|
413
|
+
await this.storage.saveMetadata(GRAPH_MEMBERSHIP_STAMP_KEY, {
|
|
414
|
+
noun: 'thing',
|
|
415
|
+
data: stamp,
|
|
416
|
+
});
|
|
417
|
+
this.membershipStampGeneration = stamp.generation;
|
|
418
|
+
this.lastMembershipStampBody = body;
|
|
419
|
+
}
|
|
420
|
+
catch (error) {
|
|
421
|
+
prodLog.error('GraphAdjacencyIndex: failed to persist verb membership — next open will heal via the canonical walk', error);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Restore membership from the stamped blob. Returns `true` only when the
|
|
426
|
+
* FULL coherence chain holds: stamp present → blob present with the exact
|
|
427
|
+
* stamped byte length → every tree fingerprint matches the durable state
|
|
428
|
+
* just loaded → the native parser accepts every framed section. Any break
|
|
429
|
+
* returns `false` (absent stamp silently — that is every pre-3.0.19
|
|
430
|
+
* brain; a broken chain with a warn) and the caller's walk heals.
|
|
431
|
+
*/
|
|
432
|
+
async loadVerbMembershipFromStorage() {
|
|
433
|
+
if (!this.hasBinaryBlobs)
|
|
434
|
+
return false;
|
|
435
|
+
try {
|
|
436
|
+
const meta = await this.storage.getMetadata(GRAPH_MEMBERSHIP_STAMP_KEY);
|
|
437
|
+
const stamp = normalizeMembershipStamp(meta?.data);
|
|
438
|
+
if (!stamp)
|
|
439
|
+
return false;
|
|
440
|
+
const blob = await this.storage.loadBinaryBlob(GRAPH_MEMBERSHIP_BLOB_KEY);
|
|
441
|
+
if (!blob || blob.length !== stamp.blobBytes) {
|
|
442
|
+
prodLog.warn(`GraphAdjacencyIndex: membership blob ${blob ? `is ${blob.length} bytes but the stamp committed ${stamp.blobBytes}` : 'is MISSING under a present stamp'} — healing via the canonical walk`);
|
|
443
|
+
return false;
|
|
444
|
+
}
|
|
445
|
+
for (const treeName of this.native.treeNames()) {
|
|
446
|
+
const now = this.treeFingerprint(treeName);
|
|
447
|
+
if (stamp.trees?.[treeName] !== now) {
|
|
448
|
+
prodLog.warn(`GraphAdjacencyIndex: membership stamp is stale for tree '${treeName}' (durable SSTables changed after the last persist) — healing via the canonical walk`);
|
|
449
|
+
return false;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
this.native.loadVerbMembership(blob);
|
|
453
|
+
this.lastMembershipBlobBytes = blob.length;
|
|
454
|
+
this.membershipStampGeneration = stamp.generation;
|
|
455
|
+
const invariants = { blobBytes: stamp.blobBytes };
|
|
456
|
+
for (const [name, fp] of Object.entries(stamp.trees))
|
|
457
|
+
invariants[`tree:${name}`] = fp;
|
|
458
|
+
this.lastMembershipStampBody = JSON.stringify(invariants);
|
|
459
|
+
this.membershipBootState = 'restored';
|
|
460
|
+
prodLog.info(`GraphAdjacencyIndex: verb membership restored from stamp (${this.native.verbIdCount()} live verbs, no canonical walk)`);
|
|
461
|
+
return true;
|
|
462
|
+
}
|
|
463
|
+
catch (error) {
|
|
464
|
+
prodLog.warn('GraphAdjacencyIndex: membership restore failed — healing via the canonical walk', error);
|
|
465
|
+
return false;
|
|
303
466
|
}
|
|
304
467
|
}
|
|
305
468
|
async populateVerbIdSetFromStorage() {
|
|
@@ -313,12 +476,16 @@ export class GraphAdjacencyIndex {
|
|
|
313
476
|
pagination: { limit: 10000, cursor }
|
|
314
477
|
});
|
|
315
478
|
for (const verb of result.items) {
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
//
|
|
321
|
-
|
|
479
|
+
// Full-fidelity tracking (3.0.19): the membership-only walk used
|
|
480
|
+
// to drop subtype state, silently emptying findConnectedSubtype
|
|
481
|
+
// after every restart. Canonical carries the subtype (brainy
|
|
482
|
+
// 7.30.0+) — a METADATA field on verbs, surfaced top-level on
|
|
483
|
+
// hydrated items; read both shapes. Unknown type strings fall
|
|
484
|
+
// back to RelatedTo — acceptable on the rebuild slow path
|
|
485
|
+
// because the membership bitmap converges either way.
|
|
486
|
+
const v = verb;
|
|
487
|
+
const subtype = v.subtype ?? v.metadata?.subtype ?? '';
|
|
488
|
+
this.native.trackVerbIdFull(verb.id, resolveVerbTypeIndex(verb.verb), subtype);
|
|
322
489
|
count++;
|
|
323
490
|
}
|
|
324
491
|
hasMore = result.hasMore;
|
|
@@ -437,6 +604,7 @@ export class GraphAdjacencyIndex {
|
|
|
437
604
|
// coalesce absent subtype to '' here.
|
|
438
605
|
const subtype = verb.subtype ?? '';
|
|
439
606
|
const result = this.native.addVerbWithEndpointsByIndex(verb.id, sourceInt, targetInt, verbTypeIndex, subtype, generation);
|
|
607
|
+
this.membershipDirty = true;
|
|
440
608
|
// Flush any trees that hit their threshold
|
|
441
609
|
const flushPromises = [];
|
|
442
610
|
if (result.needsFlushSource) {
|
|
@@ -465,8 +633,19 @@ export class GraphAdjacencyIndex {
|
|
|
465
633
|
await this.ensureInitialized();
|
|
466
634
|
// Load verb to get type + subtype info
|
|
467
635
|
const verb = await this.getVerbCached(verbId);
|
|
468
|
-
if (!verb)
|
|
636
|
+
if (!verb) {
|
|
637
|
+
// REMOVAL LAW (3.0.19): removal must never require reading the thing
|
|
638
|
+
// being removed. A ghost verb (canonical row unreadable) used to be
|
|
639
|
+
// silently SKIPPED here — its graph rows survived forever, the exact
|
|
640
|
+
// shape of brainy's replace-path defect one layer down. The id-only
|
|
641
|
+
// native path resolves type/subtype from cor-owned state instead.
|
|
642
|
+
const removed = this.native.removeVerbByIdOnly(verbId, generation);
|
|
643
|
+
if (removed) {
|
|
644
|
+
this.membershipDirty = true;
|
|
645
|
+
prodLog.warn(`GraphAdjacencyIndex: removed ghost verb ${verbId} via the id-only path (canonical row unreadable — removal law)`);
|
|
646
|
+
}
|
|
469
647
|
return;
|
|
648
|
+
}
|
|
470
649
|
const startTime = performance.now();
|
|
471
650
|
const verbTypeIndex = resolveVerbTypeIndex(verb.verb);
|
|
472
651
|
// Native subtype param is `String` with '' = "no subtype" (the empty-string
|
|
@@ -475,6 +654,7 @@ export class GraphAdjacencyIndex {
|
|
|
475
654
|
// coalesce absent subtype to '' here.
|
|
476
655
|
const subtype = verb.subtype ?? '';
|
|
477
656
|
this.native.removeVerb(verbId, verbTypeIndex, subtype, generation);
|
|
657
|
+
this.membershipDirty = true;
|
|
478
658
|
const elapsed = performance.now() - startTime;
|
|
479
659
|
if (elapsed > 5.0) {
|
|
480
660
|
prodLog.warn(`GraphAdjacencyIndex: Slow removeVerb for ${verbId}: ${elapsed.toFixed(2)}ms`);
|
|
@@ -600,6 +780,19 @@ export class GraphAdjacencyIndex {
|
|
|
600
780
|
: 'no canonical-vs-durable coverage gap detected',
|
|
601
781
|
heal: 'rebuild',
|
|
602
782
|
}),
|
|
783
|
+
() => ({
|
|
784
|
+
name: 'membership-stamp',
|
|
785
|
+
// Every boot path lands in an honest state: restored (stamp
|
|
786
|
+
// coherent), healed (walk ran — first boot on 3.0.19, or a torn
|
|
787
|
+
// stamp), or fresh (no verbs anywhere). A verb-bearing graph
|
|
788
|
+
// whose membership is EMPTY while trees hold edges is the only
|
|
789
|
+
// failing shape — subtype filters would silently serve nothing.
|
|
790
|
+
holds: !(this.native.size() > 0 && this.native.verbIdCount() === 0),
|
|
791
|
+
detail: this.native.size() > 0 && this.native.verbIdCount() === 0
|
|
792
|
+
? 'trees hold edges but the live-verb roster is EMPTY — membership restore AND heal both failed'
|
|
793
|
+
: `membership ${this.membershipBootState} this boot (${this.native.verbIdCount()} live verbs)`,
|
|
794
|
+
heal: 'rebuild',
|
|
795
|
+
}),
|
|
603
796
|
]);
|
|
604
797
|
}
|
|
605
798
|
isReady() {
|
|
@@ -719,8 +912,18 @@ export class GraphAdjacencyIndex {
|
|
|
719
912
|
for (const verb of result.items) {
|
|
720
913
|
totalVerbs++;
|
|
721
914
|
const verbType = verb.verb ?? 'relatedTo';
|
|
915
|
+
const verbTypeIndex = resolveVerbTypeIndex(verbType);
|
|
916
|
+
// Native subtype param is `String` with '' = "no subtype" (the
|
|
917
|
+
// empty-string escape; graph_adjacency.rs guards on
|
|
918
|
+
// `!subtype.is_empty()`). JS null throws "Null into String" at the
|
|
919
|
+
// napi boundary, so coalesce absent subtype to ''. Subtype is a
|
|
920
|
+
// metadata field on verbs — read both hydration shapes.
|
|
921
|
+
const vv = verb;
|
|
922
|
+
const subtype = vv.subtype ?? vv.metadata?.subtype ?? '';
|
|
722
923
|
if (!reconstructAdjacency) {
|
|
723
|
-
|
|
924
|
+
// Full-fidelity membership (3.0.19): subtype rides the walk so
|
|
925
|
+
// filtered traversal survives the rebuild.
|
|
926
|
+
this.native.trackVerbIdFull(verb.id, verbTypeIndex, subtype);
|
|
724
927
|
continue;
|
|
725
928
|
}
|
|
726
929
|
const sourceId = verb.sourceId;
|
|
@@ -730,16 +933,10 @@ export class GraphAdjacencyIndex {
|
|
|
730
933
|
if (sourceInt === undefined || targetInt === undefined) {
|
|
731
934
|
// Endpoint not in the mapper — still record membership so the verb
|
|
732
935
|
// isn't lost; its adjacency can't be rebuilt without the int.
|
|
733
|
-
this.native.
|
|
936
|
+
this.native.trackVerbIdFull(verb.id, verbTypeIndex, subtype);
|
|
734
937
|
unresolvedEndpoints++;
|
|
735
938
|
continue;
|
|
736
939
|
}
|
|
737
|
-
const verbTypeIndex = resolveVerbTypeIndex(verbType);
|
|
738
|
-
// Native subtype param is `String` with '' = "no subtype" (the
|
|
739
|
-
// empty-string escape; graph_adjacency.rs guards on
|
|
740
|
-
// `!subtype.is_empty()`). JS null throws "Null into String" at the
|
|
741
|
-
// napi boundary, so coalesce absent subtype to ''.
|
|
742
|
-
const subtype = verb.subtype ?? '';
|
|
743
940
|
const addResult = this.native.addVerbWithEndpointsByIndex(verb.id, BigInt(sourceInt), BigInt(targetInt), verbTypeIndex, subtype, REBUILD_GENERATION);
|
|
744
941
|
edgesRebuilt++;
|
|
745
942
|
// Respect memtable flush thresholds mid-walk so RAM stays bounded.
|
|
@@ -759,6 +956,15 @@ export class GraphAdjacencyIndex {
|
|
|
759
956
|
cursor = result.nextCursor;
|
|
760
957
|
}
|
|
761
958
|
this.totalRelationshipsIndexed = totalVerbs;
|
|
959
|
+
// The walk rebuilt membership through the native handle directly (the
|
|
960
|
+
// wrapper's dirty bit never fired) — mark it so the persist below and
|
|
961
|
+
// any later flush write the fresh blob.
|
|
962
|
+
this.membershipDirty = true;
|
|
963
|
+
if (!reconstructAdjacency) {
|
|
964
|
+
// Membership-only rebuild writes no memtables, so no flush runs —
|
|
965
|
+
// persist the family directly or the walk repeats next boot.
|
|
966
|
+
await this.persistVerbMembership(true);
|
|
967
|
+
}
|
|
762
968
|
if (reconstructAdjacency) {
|
|
763
969
|
// Persist partially-filled memtables from the final batch.
|
|
764
970
|
await this.flush();
|
|
@@ -800,6 +1006,9 @@ export class GraphAdjacencyIndex {
|
|
|
800
1006
|
if (flushPromises.length > 0) {
|
|
801
1007
|
await Promise.all(flushPromises);
|
|
802
1008
|
}
|
|
1009
|
+
// Membership family rides every flush cycle: blob only when membership
|
|
1010
|
+
// mutated, stamp only when the payload changed — idle ticks cost nothing.
|
|
1011
|
+
await this.persistVerbMembership();
|
|
803
1012
|
const elapsed = Date.now() - startTime;
|
|
804
1013
|
prodLog.debug(`GraphAdjacencyIndex: Flush completed in ${elapsed}ms`);
|
|
805
1014
|
}
|
|
@@ -1031,6 +1240,11 @@ export class GraphAdjacencyIndex {
|
|
|
1031
1240
|
});
|
|
1032
1241
|
}));
|
|
1033
1242
|
});
|
|
1243
|
+
// Compaction changed the durable SSTable identity, so the
|
|
1244
|
+
// membership stamp's fingerprint for this tree is now stale —
|
|
1245
|
+
// refresh it (stamp-only unless membership also mutated). Without
|
|
1246
|
+
// this, the next open would needlessly heal via the walk.
|
|
1247
|
+
await this.persistVerbMembership();
|
|
1034
1248
|
const elapsed = Date.now() - startTime;
|
|
1035
1249
|
prodLog.info(`GraphAdjacencyIndex: Compaction ${treeName} L${level} → L${result.newLevel} complete in ${elapsed}ms`);
|
|
1036
1250
|
if (result.newLevel < 6 && this.native.needsCompaction(treeName, result.newLevel)) {
|
|
@@ -458,6 +458,26 @@ export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
|
|
|
458
458
|
getPersistMode(): 'immediate' | 'deferred';
|
|
459
459
|
/** Absolute path of a segment file (lives next to the base indexPath). */
|
|
460
460
|
private segPath;
|
|
461
|
+
/**
|
|
462
|
+
* The committed-transaction watermark (M3 PD-2 `sourceGeneration`): read
|
|
463
|
+
* from brainy's generation manifest (`_system/manifest.json`, the same
|
|
464
|
+
* object the generation store treats as committed truth). Read-only and
|
|
465
|
+
* ADVISORY — hosts without the manifest (or a faulted read) stamp no
|
|
466
|
+
* sourceGeneration and verifiers skip the comparison. Never constructs a
|
|
467
|
+
* FactLog (its open() is writer-side reconciliation).
|
|
468
|
+
*/
|
|
469
|
+
private readCommittedGeneration;
|
|
470
|
+
/**
|
|
471
|
+
* Move poisoned L0 segments out of the live set into
|
|
472
|
+
* `<indexDir>/quarantine/` — preserved for forensics, NEVER deleted
|
|
473
|
+
* (accord's 545 poisoned segments were the evidence that convicted the
|
|
474
|
+
* June dim bug). Crash-safe order: the manifest stops referencing the
|
|
475
|
+
* segments FIRST (a crash after that leaves orphaned, de-referenced
|
|
476
|
+
* files — a harmless leak), then the files move. The dropped native
|
|
477
|
+
* handles close with their references; nothing reads a de-referenced
|
|
478
|
+
* segment.
|
|
479
|
+
*/
|
|
480
|
+
private quarantineSegments;
|
|
461
481
|
/** Membership across the base + every L0 segment (int → any slot). */
|
|
462
482
|
private hasIntAnywhere;
|
|
463
483
|
/**
|
|
@@ -75,9 +75,10 @@ import { declareDerivedFamilies, VECTOR_FAMILIES } from '../utils/derivedFamilie
|
|
|
75
75
|
import { statSync, readFileSync } from 'node:fs';
|
|
76
76
|
import { loadNativeModule } from '../native/index.js';
|
|
77
77
|
import { prodLog } from '@soulcraft/brainy/internals';
|
|
78
|
-
import { mkdirSync, existsSync, rmSync } from 'node:fs';
|
|
79
|
-
import { dirname, join } from 'node:path';
|
|
78
|
+
import { mkdirSync, existsSync, rmSync, renameSync } from 'node:fs';
|
|
79
|
+
import { dirname, join, basename } from 'node:path';
|
|
80
80
|
import { autoModeForHeader, selectModeFromResourceManager, } from './AdaptiveDiskAnnModeSelector.js';
|
|
81
|
+
import { readCommittedGeneration } from '../utils/generationWatermark.js';
|
|
81
82
|
/**
|
|
82
83
|
* Node-count threshold above which `useMmapAdjacency: 'auto'`
|
|
83
84
|
* resolves to file-backed. Below this, the in-RAM build adjacency
|
|
@@ -1174,13 +1175,50 @@ export class NativeDiskAnnWrapper {
|
|
|
1174
1175
|
// into the new base. Their entries are read back through the existing
|
|
1175
1176
|
// readVectorChunk/slotIdsForSlots surface (each segment is bounded-small
|
|
1176
1177
|
// by construction, ≤ the delta budget, so this stays bounded RAM).
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1178
|
+
//
|
|
1179
|
+
// SEGMENT QUARANTINE (3.0.19, accord 2026-07-14): a segment whose
|
|
1180
|
+
// read-back throws (June-class dim poisoning — "Range … out of bounds")
|
|
1181
|
+
// used to abort the WHOLE fold, forever: 545 live segments accumulated,
|
|
1182
|
+
// the write brake engaged, and the board refused writes for 20 hours.
|
|
1183
|
+
// A poisoned segment is now dropped from the fold LOUDLY and its files
|
|
1184
|
+
// moved to <indexDir>/quarantine/ (preserved for forensics, never
|
|
1185
|
+
// deleted); the fold completes with the healthy segments and the same
|
|
1186
|
+
// pass restores the lost entries from canonical (hydrate below), so the
|
|
1187
|
+
// rebuilt base is COMPLETE — serving-truth holds without a second heal.
|
|
1188
|
+
const builtL0 = [];
|
|
1189
|
+
const l0Data = [];
|
|
1190
|
+
const poisonedL0 = [];
|
|
1191
|
+
for (const seg of this.l0Segments) {
|
|
1192
|
+
try {
|
|
1193
|
+
const ints = (seg.nodeCount > 0
|
|
1194
|
+
? seg.native.slotIdsForSlots(Array.from({ length: seg.nodeCount }, (_, i) => i))
|
|
1195
|
+
: []);
|
|
1196
|
+
const vecs = seg.native.readVectorChunk(0, seg.nodeCount);
|
|
1197
|
+
builtL0.push(seg);
|
|
1198
|
+
l0Data.push({ ints, vecs });
|
|
1199
|
+
}
|
|
1200
|
+
catch (error) {
|
|
1201
|
+
poisonedL0.push(seg);
|
|
1202
|
+
prodLog.error(`NativeDiskAnnWrapper: L0 segment ${seg.file} is UNREADABLE ` +
|
|
1203
|
+
`(${error instanceof Error ? error.message : String(error)}) — ` +
|
|
1204
|
+
`QUARANTINED (${seg.nodeCount} entries; restored from canonical in this fold)`);
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
if (poisonedL0.length > 0) {
|
|
1208
|
+
await this.quarantineSegments(poisonedL0);
|
|
1209
|
+
if (typeof this.storage?.getNouns === 'function') {
|
|
1210
|
+
// Quarantine fold: the poisoned segments' entries exist ONLY in
|
|
1211
|
+
// canonical now — hydrate the delta BEFORE the dedup sets below are
|
|
1212
|
+
// computed, so newest-wins exclusion sees the full canonical set
|
|
1213
|
+
// (base slots excluded for delta ids, healthy-L0 rows deduped
|
|
1214
|
+
// against the delta — no duplicate ints, complete output).
|
|
1215
|
+
// O(canonical) reads, paid only on an actual corruption event —
|
|
1216
|
+
// the alternative (a base silently missing entities) violates
|
|
1217
|
+
// serving-truth. A scoped missing-ids-only backfill rides the M3
|
|
1218
|
+
// fact-log scan path.
|
|
1219
|
+
await this.hydrateDeltaFromStorage(this.config.dimensions);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1184
1222
|
// Tombstone ints (used to exclude entries from BOTH the base and the L0
|
|
1185
1223
|
// read-back). Ids never indexed have no int — nothing to exclude.
|
|
1186
1224
|
const tombIntKeys = new Set();
|
|
@@ -1256,11 +1294,14 @@ export class NativeDiskAnnWrapper {
|
|
|
1256
1294
|
let dim = this.config.dimensions ??
|
|
1257
1295
|
(this.native ? this.native.header().dim : undefined) ??
|
|
1258
1296
|
builtL0.find((s) => s.nodeCount > 0)?.native.header().dim;
|
|
1259
|
-
if (liveOldSlots.length === 0 &&
|
|
1297
|
+
if (liveOldSlots.length === 0 &&
|
|
1298
|
+
poisonedL0.length === 0 &&
|
|
1299
|
+
typeof this.storage?.getNouns === 'function') {
|
|
1260
1300
|
const probe = await this.storage.getNouns({ pagination: { limit: 1 } });
|
|
1261
1301
|
if (probe.totalCount === undefined || probe.totalCount > this.delta.size) {
|
|
1262
1302
|
// Cold/restore rebuild: hydrate also adopts the dim from canonical data
|
|
1263
1303
|
// when we still don't have it, and returns the width it used.
|
|
1304
|
+
// (A quarantine fold already hydrated ABOVE, before the dedup sets.)
|
|
1264
1305
|
dim = await this.hydrateDeltaFromStorage(dim);
|
|
1265
1306
|
}
|
|
1266
1307
|
}
|
|
@@ -1372,6 +1413,7 @@ export class NativeDiskAnnWrapper {
|
|
|
1372
1413
|
outputPath: this.config.indexPath,
|
|
1373
1414
|
cfg,
|
|
1374
1415
|
slotIds: slotIdsBuf,
|
|
1416
|
+
sourceGeneration: await this.readCommittedGeneration(),
|
|
1375
1417
|
});
|
|
1376
1418
|
// Atomic swap. The native engine now owns both off-heap slotmaps
|
|
1377
1419
|
// (written inside rebuildFromExisting), so there is NO resident
|
|
@@ -1461,6 +1503,48 @@ export class NativeDiskAnnWrapper {
|
|
|
1461
1503
|
segPath(file) {
|
|
1462
1504
|
return `${dirname(this.config.indexPath)}/${file}`;
|
|
1463
1505
|
}
|
|
1506
|
+
/**
|
|
1507
|
+
* The committed-transaction watermark (M3 PD-2 `sourceGeneration`): read
|
|
1508
|
+
* from brainy's generation manifest (`_system/manifest.json`, the same
|
|
1509
|
+
* object the generation store treats as committed truth). Read-only and
|
|
1510
|
+
* ADVISORY — hosts without the manifest (or a faulted read) stamp no
|
|
1511
|
+
* sourceGeneration and verifiers skip the comparison. Never constructs a
|
|
1512
|
+
* FactLog (its open() is writer-side reconciliation).
|
|
1513
|
+
*/
|
|
1514
|
+
async readCommittedGeneration() {
|
|
1515
|
+
return readCommittedGeneration(this.storage);
|
|
1516
|
+
}
|
|
1517
|
+
/**
|
|
1518
|
+
* Move poisoned L0 segments out of the live set into
|
|
1519
|
+
* `<indexDir>/quarantine/` — preserved for forensics, NEVER deleted
|
|
1520
|
+
* (accord's 545 poisoned segments were the evidence that convicted the
|
|
1521
|
+
* June dim bug). Crash-safe order: the manifest stops referencing the
|
|
1522
|
+
* segments FIRST (a crash after that leaves orphaned, de-referenced
|
|
1523
|
+
* files — a harmless leak), then the files move. The dropped native
|
|
1524
|
+
* handles close with their references; nothing reads a de-referenced
|
|
1525
|
+
* segment.
|
|
1526
|
+
*/
|
|
1527
|
+
async quarantineSegments(poisoned) {
|
|
1528
|
+
const drop = new Set(poisoned.map((s) => s.id));
|
|
1529
|
+
this.l0Segments = this.l0Segments.filter((s) => !drop.has(s.id));
|
|
1530
|
+
await this.saveSegmentManifest();
|
|
1531
|
+
const qDir = join(dirname(this.config.indexPath), 'quarantine');
|
|
1532
|
+
for (const seg of poisoned) {
|
|
1533
|
+
try {
|
|
1534
|
+
mkdirSync(qDir, { recursive: true });
|
|
1535
|
+
const src = this.segPath(seg.file);
|
|
1536
|
+
const family = [src, this.siblingPath(src, 'slotmap'), this.siblingPath(src, 'slotrev')];
|
|
1537
|
+
for (const p of family) {
|
|
1538
|
+
if (existsSync(p))
|
|
1539
|
+
renameSync(p, join(qDir, basename(p)));
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
catch (error) {
|
|
1543
|
+
prodLog.error(`NativeDiskAnnWrapper: quarantine move failed for ${seg.file} — ` +
|
|
1544
|
+
`segment stays de-referenced on disk (manifest no longer lists it)`, error);
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1464
1548
|
/** Membership across the base + every L0 segment (int → any slot). */
|
|
1465
1549
|
hasIntAnywhere(int) {
|
|
1466
1550
|
if (this.native && this.native.hasInt(int))
|
|
@@ -1675,6 +1759,7 @@ export class NativeDiskAnnWrapper {
|
|
|
1675
1759
|
outputPath: this.segPath(file),
|
|
1676
1760
|
cfg,
|
|
1677
1761
|
slotIds: Buffer.from(slotIdsArr.buffer, slotIdsArr.byteOffset, slotIdsArr.byteLength),
|
|
1762
|
+
sourceGeneration: await this.readCommittedGeneration(),
|
|
1678
1763
|
});
|
|
1679
1764
|
// File (+ sidecars) durably on disk → register → manifest (crash-safe order).
|
|
1680
1765
|
this.l0Segments.push({ id, file, nodeCount: builtDelta.size, native: newNative });
|
|
@@ -1787,10 +1872,17 @@ export class NativeDiskAnnWrapper {
|
|
|
1787
1872
|
violations: [`stamp ${stampPath} has no members map — cannot prove family coherence`],
|
|
1788
1873
|
};
|
|
1789
1874
|
}
|
|
1875
|
+
// Member sizes, both wire modes (M3 unified stamp, 2026-07-15): the
|
|
1876
|
+
// unified shape carries {mode:'enumerated', files:[{path,bytes}]}; the
|
|
1877
|
+
// legacy cor shape is a flat basename→bytes map. One verifier, one page.
|
|
1878
|
+
const rawMembers = stamp.members;
|
|
1879
|
+
const memberSizes = Array.isArray(rawMembers.files)
|
|
1880
|
+
? Object.fromEntries(rawMembers.files.map((f) => [f.path, f.bytes]))
|
|
1881
|
+
: stamp.members;
|
|
1790
1882
|
const dir = dirname(indexPath);
|
|
1791
1883
|
const violations = [];
|
|
1792
1884
|
const faults = [];
|
|
1793
|
-
for (const [name, expected] of Object.entries(
|
|
1885
|
+
for (const [name, expected] of Object.entries(memberSizes)) {
|
|
1794
1886
|
try {
|
|
1795
1887
|
const actual = statSync(join(dir, name)).size;
|
|
1796
1888
|
if (actual !== expected) {
|
package/dist/native/types.d.ts
CHANGED
|
@@ -751,6 +751,31 @@ export interface NativeGraphAdjacencyInstance {
|
|
|
751
751
|
* when LSM-trees are already loaded from persisted SSTables.
|
|
752
752
|
*/
|
|
753
753
|
trackVerbId(verbId: string, verbType: string): void;
|
|
754
|
+
/**
|
|
755
|
+
* Full-fidelity membership tracking for the boot heal walk (3.0.19):
|
|
756
|
+
* roster + per-type union + counts + per-(type, subtype) bucket. The
|
|
757
|
+
* membership-only walk dropped subtype state, silently emptying
|
|
758
|
+
* `findConnectedSubtype` after every restart.
|
|
759
|
+
*/
|
|
760
|
+
trackVerbIdFull(verbId: string, verbTypeIndex: number, subtype: string): void;
|
|
761
|
+
/**
|
|
762
|
+
* Serialize every RAM-resident membership structure (roster, per-type
|
|
763
|
+
* unions, per-(type, subtype) buckets, both count families) to one
|
|
764
|
+
* strictly-framed buffer for stamped persistence (3.0.19).
|
|
765
|
+
*/
|
|
766
|
+
serializeVerbMembership(): Buffer;
|
|
767
|
+
/**
|
|
768
|
+
* Restore membership from a `serializeVerbMembership` buffer. Throws on
|
|
769
|
+
* ANY malformation (magic/version/id-space/truncation/trailing bytes) —
|
|
770
|
+
* the caller falls back to the canonical heal walk. Idempotent overwrite.
|
|
771
|
+
*/
|
|
772
|
+
loadVerbMembership(buf: Buffer): void;
|
|
773
|
+
/**
|
|
774
|
+
* Removal-law removal (3.0.19): removes a verb resolving type/subtype
|
|
775
|
+
* from cor-owned state (endpoint store / membership buckets) — never
|
|
776
|
+
* reads canonical. Returns false when unknown/already dead (idempotent).
|
|
777
|
+
*/
|
|
778
|
+
removeVerbByIdOnly(verbId: string, generation: bigint): boolean;
|
|
754
779
|
/** Clear membership bitmap + relationship counts (rebuild prelude). */
|
|
755
780
|
clearVerbTracking(): void;
|
|
756
781
|
/**
|
|
@@ -976,6 +1001,15 @@ export interface NativeMetadataIndexInstance {
|
|
|
976
1001
|
* Never throws; empty index → `true`. Backs the wrapper's `probeConsistency`.
|
|
977
1002
|
*/
|
|
978
1003
|
probeConsistency(): boolean;
|
|
1004
|
+
/**
|
|
1005
|
+
* Suspect-surfacing probe (3.0.19): same bounded walk, but returns the
|
|
1006
|
+
* first suspicious `{field, uuid}` as JSON (null = clean) so the wrapper
|
|
1007
|
+
* can adjudicate it with one canonical read instead of guessing from the
|
|
1008
|
+
* sample MIX.
|
|
1009
|
+
*/
|
|
1010
|
+
probeConsistencySuspect(): string | null;
|
|
1011
|
+
/** Mark a field learned-multi-valued after canonical adjudication (3.0.19). */
|
|
1012
|
+
markFieldMultiValue(field: string): void;
|
|
979
1013
|
/**
|
|
980
1014
|
* Scan for the cross-bucket phantom signature (a query-visible int in >1
|
|
981
1015
|
* value bucket of a field) and return up to `maxCandidates` (0 = unlimited)
|
|
@@ -858,6 +858,24 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
|
|
|
858
858
|
*/
|
|
859
859
|
private flushChain;
|
|
860
860
|
flush(): Promise<void>;
|
|
861
|
+
/**
|
|
862
|
+
* Write the metadata index's family stamp (shared shape/functions from
|
|
863
|
+
* brainy internals): rollup invariant = the posted-entity total the
|
|
864
|
+
* detector already computes; `sourceGeneration` = the committed
|
|
865
|
+
* watermark these postings cover. Advisory end to end — a host without
|
|
866
|
+
* the stamp surface or the watermark simply doesn't stamp, and opens
|
|
867
|
+
* keep their walk-based healing.
|
|
868
|
+
*/
|
|
869
|
+
private persistMetadataStamp;
|
|
870
|
+
/**
|
|
871
|
+
* M3 PD-4: incremental catch-up from the generation fact log. Applies
|
|
872
|
+
* exactly the ops in `[fromGeneration, head]` — after-image upserts via
|
|
873
|
+
* the normal add path (idempotent: eviction removes stale buckets),
|
|
874
|
+
* tombstones via the id-only removal path (removal law). Returns false
|
|
875
|
+
* (and logs) on ANY break — no capability, scan gap, apply fault — so
|
|
876
|
+
* the caller falls back to the walk; a half-replay never claims success.
|
|
877
|
+
*/
|
|
878
|
+
private replayFactGap;
|
|
861
879
|
private flushOnce;
|
|
862
880
|
/**
|
|
863
881
|
* Persist all state and release the column store's resources. Brainy's
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
|
|
19
19
|
import { dirname, join } from 'node:path';
|
|
20
20
|
import { prodLog, getGlobalCache, FieldTypeInference, resolveEntityField } from '@soulcraft/brainy/internals';
|
|
21
|
+
import { readCommittedGeneration } from './generationWatermark.js';
|
|
21
22
|
import { compareCodePoints } from './collation.js';
|
|
22
23
|
/**
|
|
23
24
|
* Fields whose values are stored in the sparse index as BUCKETED values
|
|
@@ -84,6 +85,33 @@ const LSM_MANIFEST_KEY = '__metadata_lsm_manifest__';
|
|
|
84
85
|
* CORTEX-RESTART-STRAND 2026-07-13 — four consecutive boots).
|
|
85
86
|
*/
|
|
86
87
|
const UNPOSTABLE_STAMP_KEY = '__metadata_verified_unpostable__';
|
|
88
|
+
/**
|
|
89
|
+
* M3 PD-2/PD-4 (Stage 1): the metadata index's family stamp — written via
|
|
90
|
+
* brainy's SHARED stamp functions (one verifier, literally one function).
|
|
91
|
+
* `sourceGeneration` records the committed watermark the postings cover;
|
|
92
|
+
* a coherent-but-BEHIND stamp at open triggers an incremental FACT REPLAY
|
|
93
|
+
* of exactly the gap instead of any walk. Path/family per brainy's
|
|
94
|
+
* FAMILY_STAMPS_PREFIX convention.
|
|
95
|
+
*/
|
|
96
|
+
const METADATA_FAMILY = 'cor-metadata-lsm';
|
|
97
|
+
const METADATA_STAMP_PATH = '_system/family-stamps/cor-metadata-lsm.json';
|
|
98
|
+
let familyStampFnsCache;
|
|
99
|
+
async function familyStampFns() {
|
|
100
|
+
if (familyStampFnsCache !== undefined)
|
|
101
|
+
return familyStampFnsCache;
|
|
102
|
+
try {
|
|
103
|
+
const internals = (await import('@soulcraft/brainy/internals'));
|
|
104
|
+
familyStampFnsCache =
|
|
105
|
+
typeof internals.readFamilyStamp === 'function' &&
|
|
106
|
+
typeof internals.writeFamilyStamp === 'function'
|
|
107
|
+
? internals
|
|
108
|
+
: null;
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
familyStampFnsCache = null;
|
|
112
|
+
}
|
|
113
|
+
return familyStampFnsCache;
|
|
114
|
+
}
|
|
87
115
|
/** Above this many ids the stamp stores count-only (the boot-time
|
|
88
116
|
* re-verify would stop being cheap); detection then carries a bounded
|
|
89
117
|
* staleness caveat, logged when it engages. */
|
|
@@ -1371,24 +1399,33 @@ export class MetadataIndexManager {
|
|
|
1371
1399
|
async verifiedUnpostableAllowance() {
|
|
1372
1400
|
try {
|
|
1373
1401
|
const stamp = (await this.storage.getMetadata(UNPOSTABLE_STAMP_KEY));
|
|
1374
|
-
if (!stamp
|
|
1402
|
+
if (!stamp)
|
|
1375
1403
|
return 0;
|
|
1404
|
+
// Completion-gap portion (3.0.19): canonical rows a COMPLETED walk
|
|
1405
|
+
// could not even enumerate (no ids exist to re-verify) — count-only
|
|
1406
|
+
// by nature, refreshed on every completed rebuild. The detector's
|
|
1407
|
+
// "FULLY explained by" line reports it whenever it carries the floor.
|
|
1408
|
+
const gap = typeof stamp.completionGap === 'number' && stamp.completionGap > 0
|
|
1409
|
+
? stamp.completionGap
|
|
1410
|
+
: 0;
|
|
1411
|
+
if (typeof stamp.nounCount !== 'number' || stamp.nounCount <= 0)
|
|
1412
|
+
return gap;
|
|
1376
1413
|
if (Array.isArray(stamp.nounIds) && stamp.nounIds.length === stamp.nounCount) {
|
|
1377
1414
|
const getMeta = this.storage.getNounMetadata;
|
|
1378
1415
|
if (typeof getMeta !== 'function')
|
|
1379
|
-
return stamp.nounCount;
|
|
1416
|
+
return stamp.nounCount + gap;
|
|
1380
1417
|
let stillUnpostable = 0;
|
|
1381
1418
|
for (const id of stamp.nounIds) {
|
|
1382
1419
|
const md = await getMeta.call(this.storage, id).catch(() => undefined);
|
|
1383
1420
|
if (md == null)
|
|
1384
1421
|
stillUnpostable++;
|
|
1385
1422
|
}
|
|
1386
|
-
return stillUnpostable;
|
|
1423
|
+
return stillUnpostable + gap;
|
|
1387
1424
|
}
|
|
1388
1425
|
prodLog.warn(`[NativeMetadataIndex] unpostable stamp is count-only ` +
|
|
1389
1426
|
`(${stamp.nounCount} > id cap) — the posted-count floor carries its ` +
|
|
1390
1427
|
`staleness until the next completed rebuild refreshes it.`);
|
|
1391
|
-
return stamp.nounCount;
|
|
1428
|
+
return stamp.nounCount + gap;
|
|
1392
1429
|
}
|
|
1393
1430
|
catch {
|
|
1394
1431
|
return 0;
|
|
@@ -1397,6 +1434,29 @@ export class MetadataIndexManager {
|
|
|
1397
1434
|
async detectDerivedStrand() {
|
|
1398
1435
|
if (!this.lsmEngine || !this.lsmMetaDir)
|
|
1399
1436
|
return;
|
|
1437
|
+
// M3 PD-4 (Stage 1): a coherent-but-BEHIND family stamp means the
|
|
1438
|
+
// postings simply predate the newest commits — replay EXACTLY the gap
|
|
1439
|
+
// from the generation fact log before any strand verdict, so the
|
|
1440
|
+
// checks below see a caught-up index and no walk fires for what was
|
|
1441
|
+
// never damage. Advisory end to end: any break (no 8.5.0 surface, no
|
|
1442
|
+
// watermark, scan fault) falls through and the detector stays
|
|
1443
|
+
// authoritative.
|
|
1444
|
+
try {
|
|
1445
|
+
const fns = await familyStampFns();
|
|
1446
|
+
const head = await readCommittedGeneration(this.storage);
|
|
1447
|
+
if (fns && head !== undefined) {
|
|
1448
|
+
const stamp = (await fns.readFamilyStamp(this.storage, METADATA_STAMP_PATH));
|
|
1449
|
+
const src = stamp?.family === METADATA_FAMILY ? stamp.sourceGeneration : undefined;
|
|
1450
|
+
if (typeof src === 'number' && src < head) {
|
|
1451
|
+
if (await this.replayFactGap(src + 1)) {
|
|
1452
|
+
await this.persistMetadataStamp();
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
catch {
|
|
1458
|
+
// Detector below stays authoritative.
|
|
1459
|
+
}
|
|
1400
1460
|
// 0. Replay-stats verdict (3.0.12). The native open now RECREATES a
|
|
1401
1461
|
// missing live log and recovers what its archives hold, so the
|
|
1402
1462
|
// filename census below can no longer see the deleted-live-log
|
|
@@ -2655,12 +2715,95 @@ export class MetadataIndexManager {
|
|
|
2655
2715
|
'flush would multiply heal IO).');
|
|
2656
2716
|
return;
|
|
2657
2717
|
}
|
|
2658
|
-
const run = this.flushChain.then(() =>
|
|
2718
|
+
const run = this.flushChain.then(async () => {
|
|
2719
|
+
await this.flushOnce();
|
|
2720
|
+
// M3 Stage 1: the flushed postings durably cover the committed
|
|
2721
|
+
// watermark — stamp it so the next open replays only the gap
|
|
2722
|
+
// (advisory: no-ops without the 8.5.0 stamp surface).
|
|
2723
|
+
await this.persistMetadataStamp();
|
|
2724
|
+
});
|
|
2659
2725
|
// Keep the chain alive on failure — the NEXT flush must still run; the
|
|
2660
2726
|
// failure itself propagates to THIS caller below.
|
|
2661
2727
|
this.flushChain = run.catch(() => { });
|
|
2662
2728
|
return run;
|
|
2663
2729
|
}
|
|
2730
|
+
/**
|
|
2731
|
+
* Write the metadata index's family stamp (shared shape/functions from
|
|
2732
|
+
* brainy internals): rollup invariant = the posted-entity total the
|
|
2733
|
+
* detector already computes; `sourceGeneration` = the committed
|
|
2734
|
+
* watermark these postings cover. Advisory end to end — a host without
|
|
2735
|
+
* the stamp surface or the watermark simply doesn't stamp, and opens
|
|
2736
|
+
* keep their walk-based healing.
|
|
2737
|
+
*/
|
|
2738
|
+
async persistMetadataStamp() {
|
|
2739
|
+
try {
|
|
2740
|
+
const fns = await familyStampFns();
|
|
2741
|
+
if (!fns)
|
|
2742
|
+
return;
|
|
2743
|
+
const sourceGeneration = await readCommittedGeneration(this.storage);
|
|
2744
|
+
if (sourceGeneration === undefined)
|
|
2745
|
+
return;
|
|
2746
|
+
const { NounType } = await import('@soulcraft/brainy');
|
|
2747
|
+
let posted = 0;
|
|
2748
|
+
for (const t of Object.values(NounType)) {
|
|
2749
|
+
posted += this.native.getEntityCountByType(t);
|
|
2750
|
+
}
|
|
2751
|
+
await fns.writeFamilyStamp(this.storage, METADATA_STAMP_PATH, {
|
|
2752
|
+
family: METADATA_FAMILY,
|
|
2753
|
+
sourceGeneration,
|
|
2754
|
+
members: { mode: 'rollup', invariants: { posted } },
|
|
2755
|
+
});
|
|
2756
|
+
}
|
|
2757
|
+
catch {
|
|
2758
|
+
// Advisory: a failed stamp means the next open heals the old way.
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
/**
|
|
2762
|
+
* M3 PD-4: incremental catch-up from the generation fact log. Applies
|
|
2763
|
+
* exactly the ops in `[fromGeneration, head]` — after-image upserts via
|
|
2764
|
+
* the normal add path (idempotent: eviction removes stale buckets),
|
|
2765
|
+
* tombstones via the id-only removal path (removal law). Returns false
|
|
2766
|
+
* (and logs) on ANY break — no capability, scan gap, apply fault — so
|
|
2767
|
+
* the caller falls back to the walk; a half-replay never claims success.
|
|
2768
|
+
*/
|
|
2769
|
+
async replayFactGap(fromGeneration) {
|
|
2770
|
+
const scanFn = this.storage.scanFacts;
|
|
2771
|
+
if (typeof scanFn !== 'function')
|
|
2772
|
+
return false;
|
|
2773
|
+
const scan = scanFn.call(this.storage, { fromGeneration, batchSize: 256 });
|
|
2774
|
+
if (!scan)
|
|
2775
|
+
return false;
|
|
2776
|
+
let upserts = 0;
|
|
2777
|
+
let removals = 0;
|
|
2778
|
+
try {
|
|
2779
|
+
for await (const batch of scan.batches()) {
|
|
2780
|
+
for (const fact of batch.facts) {
|
|
2781
|
+
for (const op of fact.ops) {
|
|
2782
|
+
if (op.record === null) {
|
|
2783
|
+
await this.removeFromIndex(op.id);
|
|
2784
|
+
removals++;
|
|
2785
|
+
}
|
|
2786
|
+
else if (op.record.metadata != null) {
|
|
2787
|
+
await this.addToIndex(op.id, op.record.metadata, true, true);
|
|
2788
|
+
upserts++;
|
|
2789
|
+
}
|
|
2790
|
+
// record present with null metadata = vector-only write —
|
|
2791
|
+
// nothing to post here.
|
|
2792
|
+
}
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
catch (error) {
|
|
2797
|
+
opsLine(`fact replay from generation ${fromGeneration} ABORTED (${String(error)}) — ` +
|
|
2798
|
+
`falling back to the walk-based heal path`);
|
|
2799
|
+
return false;
|
|
2800
|
+
}
|
|
2801
|
+
if (upserts + removals > 0) {
|
|
2802
|
+
opsLine(`fact replay: caught up ${upserts} upsert(s) + ${removals} removal(s) ` +
|
|
2803
|
+
`from the generation log — no walk needed`);
|
|
2804
|
+
}
|
|
2805
|
+
return true;
|
|
2806
|
+
}
|
|
2664
2807
|
async flushOnce() {
|
|
2665
2808
|
// Always save field registry + msync the entity ID mapper — even with
|
|
2666
2809
|
// no dirty fields. The field registry is the critical file init() needs
|
|
@@ -2847,7 +2990,53 @@ export class MetadataIndexManager {
|
|
|
2847
2990
|
await this.ensureFieldLoaded(field);
|
|
2848
2991
|
loaded++;
|
|
2849
2992
|
}
|
|
2850
|
-
|
|
2993
|
+
// Adjudicated probe (3.0.19): the native walk surfaces ONE suspect
|
|
2994
|
+
// {field, uuid}; a single canonical read decides it. ≥2 current
|
|
2995
|
+
// values = a mixed-length ARRAY field, not a phantom — learned,
|
|
2996
|
+
// persisted, and the probe re-runs (each pass retires one field
|
|
2997
|
+
// permanently, so this converges in ≤ field count; bounded harder
|
|
2998
|
+
// below). Anything else = a genuine phantom → false, exactly the old
|
|
2999
|
+
// detection strength. This replaces the MIX guess whose verdict
|
|
3000
|
+
// depended on which ints the sample happened to draw (accord's brain:
|
|
3001
|
+
// clean boots 1–2, failing boot 3, identical bytes).
|
|
3002
|
+
if (typeof this.native.probeConsistencySuspect !== 'function') {
|
|
3003
|
+
// Pre-3.0.19 binary: no adjudication surface — use the sampled MIX
|
|
3004
|
+
// verdict. WITHOUT this explicit fallback the call below throws into
|
|
3005
|
+
// the outer catch, which assumes healthy — i.e. a stale binary would
|
|
3006
|
+
// silently DISABLE phantom detection (the 3.0.12–3.0.15 class).
|
|
3007
|
+
return this.native.probeConsistency();
|
|
3008
|
+
}
|
|
3009
|
+
const MAX_ADJUDICATIONS = 8;
|
|
3010
|
+
for (let i = 0; i <= MAX_ADJUDICATIONS; i++) {
|
|
3011
|
+
const suspectJson = this.native.probeConsistencySuspect();
|
|
3012
|
+
if (!suspectJson)
|
|
3013
|
+
return true;
|
|
3014
|
+
if (i === MAX_ADJUDICATIONS)
|
|
3015
|
+
return false;
|
|
3016
|
+
const { field, uuid } = JSON.parse(suspectJson);
|
|
3017
|
+
let currentValues = [];
|
|
3018
|
+
try {
|
|
3019
|
+
const noun = await this.storage.getNoun(uuid);
|
|
3020
|
+
if (noun) {
|
|
3021
|
+
const raw = resolveEntityField(noun, field);
|
|
3022
|
+
if (raw !== undefined && raw !== null) {
|
|
3023
|
+
currentValues = Array.isArray(raw) ? raw : [raw];
|
|
3024
|
+
}
|
|
3025
|
+
}
|
|
3026
|
+
}
|
|
3027
|
+
catch {
|
|
3028
|
+
// Unreadable canonical = cannot exonerate — keep the phantom verdict.
|
|
3029
|
+
}
|
|
3030
|
+
if (currentValues.length < 2)
|
|
3031
|
+
return false;
|
|
3032
|
+
this.native.markFieldMultiValue(field);
|
|
3033
|
+
const json = this.native.saveFieldIndex(field);
|
|
3034
|
+
if (json) {
|
|
3035
|
+
await this.storage.saveMetadata(`__metadata_field_index__field_${field}`, JSON.parse(json));
|
|
3036
|
+
}
|
|
3037
|
+
prodLog.info(`[probeConsistency] field '${field}' adjudicated multi-valued against canonical (${currentValues.length} current values) — learned + persisted, exempt from phantom checks`);
|
|
3038
|
+
}
|
|
3039
|
+
return false;
|
|
2851
3040
|
}
|
|
2852
3041
|
catch (error) {
|
|
2853
3042
|
// A probe must never throw into the read path — assume healthy.
|
|
@@ -2943,6 +3132,14 @@ export class MetadataIndexManager {
|
|
|
2943
3132
|
purged++;
|
|
2944
3133
|
repairedFields.add(field);
|
|
2945
3134
|
}
|
|
3135
|
+
// MULTI-VALUE LEARNING persistence (3.0.19): ≥2 current values
|
|
3136
|
+
// proves the field is array-shaped — the native side just learned
|
|
3137
|
+
// it (probe/detector exemption); persist the flag with the field
|
|
3138
|
+
// index or the next cold boot re-flags the same candidates
|
|
3139
|
+
// forever (accord's 166-loop, second mechanism).
|
|
3140
|
+
if (currentValues.length >= 2) {
|
|
3141
|
+
repairedFields.add(field);
|
|
3142
|
+
}
|
|
2946
3143
|
}
|
|
2947
3144
|
catch (error) {
|
|
2948
3145
|
prodLog.warn(`[purgeCrossBucketPhantoms] repair of ${uuid}.${field} failed: ${error}`);
|
|
@@ -3073,12 +3270,52 @@ export class MetadataIndexManager {
|
|
|
3073
3270
|
// detector uses the honest floor instead of re-flagging the same
|
|
3074
3271
|
// cohort forever. Refreshed on every completed rebuild; an empty
|
|
3075
3272
|
// set clears the stamp.
|
|
3273
|
+
//
|
|
3274
|
+
// COMPLETION GAP (3.0.19, memory's 52): canonical rows the walk
|
|
3275
|
+
// CANNOT enumerate — dangling counter rows with no readable record —
|
|
3276
|
+
// never appear in items, so the per-id stamp stayed EMPTY while the
|
|
3277
|
+
// detector's canonical count still included them: every detector
|
|
3278
|
+
// boot re-healed (~11 min on memory-vm) forever. A COMPLETED walk is
|
|
3279
|
+
// the strongest possible statement of "everything enumerable is
|
|
3280
|
+
// posted", so the residual canonical-minus-walked difference is
|
|
3281
|
+
// stamped as a count-only allowance (invisible rows have no ids).
|
|
3282
|
+
// Bounded staleness: refreshed on every completed walk; brainy's
|
|
3283
|
+
// counter recount (8.3.2 repairIndex) removes the gap permanently.
|
|
3284
|
+
const walked = nounResult.processed + nounResult.noMetadata;
|
|
3285
|
+
let completionGap = 0;
|
|
3286
|
+
let canonicalAtCompletion;
|
|
3287
|
+
try {
|
|
3288
|
+
const getNouns = this.storage.getNouns;
|
|
3289
|
+
if (typeof getNouns === 'function') {
|
|
3290
|
+
const probe = await getNouns.call(this.storage, { pagination: { limit: 1 } });
|
|
3291
|
+
const canonical = probe?.totalCount;
|
|
3292
|
+
if (typeof canonical === 'number' &&
|
|
3293
|
+
Number.isFinite(canonical) &&
|
|
3294
|
+
canonical < 0xffff_ffff) {
|
|
3295
|
+
canonicalAtCompletion = canonical;
|
|
3296
|
+
completionGap = Math.max(0, canonical - walked);
|
|
3297
|
+
}
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
catch {
|
|
3301
|
+
// Gap probe is advisory — a failed canonical read stamps no gap
|
|
3302
|
+
// (may over-heal next boot, never under-detects).
|
|
3303
|
+
}
|
|
3076
3304
|
await this.storage.saveMetadata(UNPOSTABLE_STAMP_KEY, {
|
|
3077
3305
|
nounCount: nounResult.noMetadata,
|
|
3078
3306
|
nounIds: nounResult.noMetadataIds,
|
|
3079
3307
|
verbCount: verbResult.noMetadata,
|
|
3308
|
+
completionGap,
|
|
3309
|
+
canonicalAtCompletion,
|
|
3080
3310
|
verifiedAt: Date.now(),
|
|
3081
3311
|
});
|
|
3312
|
+
if (completionGap > 0) {
|
|
3313
|
+
opsLine(`rebuild: canonical reports ${canonicalAtCompletion} nouns but only ` +
|
|
3314
|
+
`${walked} were enumerable by a COMPLETED walk — the ${completionGap}-entity ` +
|
|
3315
|
+
`gap is stamped as a completion-gap allowance (unenumerable canonical ` +
|
|
3316
|
+
`rows, e.g. inflated counters; a brainy counter recount removes them ` +
|
|
3317
|
+
`permanently). Boots stop re-healing over this gap.`);
|
|
3318
|
+
}
|
|
3082
3319
|
opsLine(`Metadata index rebuild completed! Processed ${nounResult.processed} nouns and ` +
|
|
3083
3320
|
`${verbResult.processed} verbs` +
|
|
3084
3321
|
(nounResult.noMetadata > 0
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/generationWatermark
|
|
3
|
+
* @description The committed-generation watermark (M3 PD-2
|
|
4
|
+
* `sourceGeneration`) — what a projection stamp records as the source
|
|
5
|
+
* truth it reflects. Read CAPABILITY-FIRST: brainy 8.5.0 exposes
|
|
6
|
+
* `storage.committedGeneration()` precisely so providers never parse the
|
|
7
|
+
* store's private manifest format; the direct `_system/manifest.json`
|
|
8
|
+
* read remains as the 8.4.0-host fallback for one release line. Advisory
|
|
9
|
+
* everywhere: `undefined` (no capability, no manifest, faulted read)
|
|
10
|
+
* means the stamp carries no sourceGeneration and verifiers skip the
|
|
11
|
+
* comparison — never a blocked write, never a guessed value.
|
|
12
|
+
*/
|
|
13
|
+
/** Best-effort committed watermark; `undefined` when unknowable. */
|
|
14
|
+
export declare function readCommittedGeneration(storage: unknown): Promise<number | undefined>;
|
|
15
|
+
//# sourceMappingURL=generationWatermark.d.ts.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/generationWatermark
|
|
3
|
+
* @description The committed-generation watermark (M3 PD-2
|
|
4
|
+
* `sourceGeneration`) — what a projection stamp records as the source
|
|
5
|
+
* truth it reflects. Read CAPABILITY-FIRST: brainy 8.5.0 exposes
|
|
6
|
+
* `storage.committedGeneration()` precisely so providers never parse the
|
|
7
|
+
* store's private manifest format; the direct `_system/manifest.json`
|
|
8
|
+
* read remains as the 8.4.0-host fallback for one release line. Advisory
|
|
9
|
+
* everywhere: `undefined` (no capability, no manifest, faulted read)
|
|
10
|
+
* means the stamp carries no sourceGeneration and verifiers skip the
|
|
11
|
+
* comparison — never a blocked write, never a guessed value.
|
|
12
|
+
*/
|
|
13
|
+
/** Best-effort committed watermark; `undefined` when unknowable. */
|
|
14
|
+
export async function readCommittedGeneration(storage) {
|
|
15
|
+
try {
|
|
16
|
+
const s = storage;
|
|
17
|
+
if (typeof s?.committedGeneration === 'function') {
|
|
18
|
+
const g = s.committedGeneration();
|
|
19
|
+
if (typeof g === 'number' && Number.isFinite(g) && g >= 0)
|
|
20
|
+
return g;
|
|
21
|
+
// Capability present but unwired (bare adapter) — fall through to the
|
|
22
|
+
// manifest read: a stamped brain opened outside a host brain should
|
|
23
|
+
// still stamp truthfully when the manifest exists.
|
|
24
|
+
}
|
|
25
|
+
if (typeof s?.readRawObject !== 'function')
|
|
26
|
+
return undefined;
|
|
27
|
+
const manifest = (await s.readRawObject('_system/manifest.json'));
|
|
28
|
+
const g = manifest?.generation;
|
|
29
|
+
return typeof g === 'number' && Number.isFinite(g) && g >= 0 ? g : undefined;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=generationWatermark.js.map
|
package/dist/version.d.ts
CHANGED
|
@@ -11,5 +11,5 @@
|
|
|
11
11
|
* `check:brainy`-style drift between the two fails the release.
|
|
12
12
|
*/
|
|
13
13
|
/** The @soulcraft/cor version this build was cut from. */
|
|
14
|
-
export declare const COR_VERSION = "3.0.
|
|
14
|
+
export declare const COR_VERSION = "3.0.20";
|
|
15
15
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/version.js
CHANGED
|
Binary file
|
package/native/index.d.ts
CHANGED
|
@@ -722,6 +722,18 @@ export declare class NativeGraphAdjacencyIndex {
|
|
|
722
722
|
* solely for legacy-data remove paths.
|
|
723
723
|
*/
|
|
724
724
|
removeVerb(verbId: string, verbTypeIndex: number, subtype: string, generation: bigint): void
|
|
725
|
+
/**
|
|
726
|
+
* **Removal law (3.0.19)**: removal must never require reading the
|
|
727
|
+
* thing being removed. The JS remove path read the verb's CANONICAL
|
|
728
|
+
* row to learn (type, subtype) and silently SKIPPED removal when that
|
|
729
|
+
* row was unreadable — a ghost's graph rows survived forever (the
|
|
730
|
+
* third instance of the anti-pattern, this one cor's own). This
|
|
731
|
+
* variant resolves everything from cor-owned state: type from the
|
|
732
|
+
* endpoint store (or the per-type unions for membership-only verbs),
|
|
733
|
+
* subtype by scanning that type's buckets for the interned id.
|
|
734
|
+
* Returns `false` when the id is unknown or already dead (idempotent).
|
|
735
|
+
*/
|
|
736
|
+
removeVerbByIdOnly(verbId: string, generation: bigint): boolean
|
|
725
737
|
/**
|
|
726
738
|
* **Look up a verb's endpoints by verb-ID.** Returns
|
|
727
739
|
* `{sourceInt, targetInt}` (as u64 BigInts) when the verb was
|
|
@@ -999,6 +1011,43 @@ export declare class NativeGraphAdjacencyIndex {
|
|
|
999
1011
|
* counts converge to the same totals).
|
|
1000
1012
|
*/
|
|
1001
1013
|
trackVerbId(verbId: string, verbType: string): void
|
|
1014
|
+
/**
|
|
1015
|
+
* Membership-only tracking that ALSO restores the per-(type, subtype)
|
|
1016
|
+
* bucket and subtype count — the full-fidelity variant of
|
|
1017
|
+
* [`track_verb_id`](Self::track_verb_id) for the boot heal walk.
|
|
1018
|
+
* The membership-only walk used to drop subtype state entirely, so
|
|
1019
|
+
* `findConnectedSubtype` silently returned nothing after a restart
|
|
1020
|
+
* until a full adjacency rebuild happened to run (3.0.19 find).
|
|
1021
|
+
* Same empty-string escape as the add path: `subtype: ''` tracks
|
|
1022
|
+
* type-level membership only, never an empty-string bucket.
|
|
1023
|
+
*/
|
|
1024
|
+
trackVerbIdFull(verbId: string, verbTypeIndex: number, subtype: string): void
|
|
1025
|
+
/**
|
|
1026
|
+
* Serialize EVERY RAM-resident membership structure to one buffer so
|
|
1027
|
+
* the JS wrapper can persist it as a stamped family member (3.0.19:
|
|
1028
|
+
* the roster/buckets/counts were RAM-only, costing every boot an
|
|
1029
|
+
* O(all-canonical-verbs) walk and silently emptying subtype filters).
|
|
1030
|
+
* Format v1, little-endian, strictly length-framed:
|
|
1031
|
+
* magic 'CVMB' u32 | version u8 | idSpace u8 (0=U32, 1=U64) |
|
|
1032
|
+
* roster (u32 len + portable bytes) |
|
|
1033
|
+
* u8 nUnions { u8 typeIdx, u32 len, bytes } |
|
|
1034
|
+
* u32 nSubBuckets { u8 typeIdx, u16 subLen, utf8, u32 len, bytes } |
|
|
1035
|
+
* u8 nCounts { u8 typeIdx, u64 count } |
|
|
1036
|
+
* u32 nSubCounts { u8 typeIdx, u16 subLen, utf8, u64 count }
|
|
1037
|
+
* Subtype keys are emitted sorted so identical state yields identical
|
|
1038
|
+
* bytes (the stamp records the exact byte size).
|
|
1039
|
+
*/
|
|
1040
|
+
serializeVerbMembership(): Buffer
|
|
1041
|
+
/**
|
|
1042
|
+
* Restore every membership structure from a
|
|
1043
|
+
* [`serialize_verb_membership`](Self::serialize_verb_membership)
|
|
1044
|
+
* buffer. Strict total parse: ANY malformation (bad magic/version,
|
|
1045
|
+
* id-space mismatch with this index, truncation, trailing bytes,
|
|
1046
|
+
* out-of-range type index) throws so the caller falls back to the
|
|
1047
|
+
* canonical heal walk — a torn blob must never half-load silently.
|
|
1048
|
+
* Clears existing membership state first (idempotent overwrite).
|
|
1049
|
+
*/
|
|
1050
|
+
loadVerbMembership(buf: Buffer): void
|
|
1002
1051
|
/**
|
|
1003
1052
|
* Clear membership bitmap + relationship counts. Used before
|
|
1004
1053
|
* rebuild to ensure clean state. The verb-ID namespace's
|
|
@@ -1870,6 +1919,23 @@ export declare class NativeMetadataIndex {
|
|
|
1870
1919
|
* read path.
|
|
1871
1920
|
*/
|
|
1872
1921
|
probeConsistency(): boolean
|
|
1922
|
+
/**
|
|
1923
|
+
* The suspect-surfacing probe (3.0.19): same bounded walk as
|
|
1924
|
+
* `probeConsistency`, but instead of a bare verdict it returns the FIRST
|
|
1925
|
+
* suspicious `{field, uuid}` as JSON (None = clean). The TS wrapper
|
|
1926
|
+
* ADJUDICATES the suspect with one canonical read — ≥2 current values
|
|
1927
|
+
* proves the field is array-shaped (learned + persisted, probe re-runs),
|
|
1928
|
+
* anything else is a genuine phantom (probe fails). This replaces the
|
|
1929
|
+
* MIX guess whose verdict depended on WHICH ints the sample drew —
|
|
1930
|
+
* accord's brain passed boots 1–2 and failed boot 3 on identical bytes.
|
|
1931
|
+
*/
|
|
1932
|
+
probeConsistencySuspect(): string | null
|
|
1933
|
+
/**
|
|
1934
|
+
* Mark a field as learned-multi-valued (3.0.19). Called by the TS
|
|
1935
|
+
* wrapper after canonical adjudication proves an entity legitimately
|
|
1936
|
+
* holds ≥2 values for the field. The caller persists the field index.
|
|
1937
|
+
*/
|
|
1938
|
+
markFieldMultiValue(field: string): void
|
|
1873
1939
|
/**
|
|
1874
1940
|
* **Phantom triad — state-level scan (matrix 7.8b).** Find every
|
|
1875
1941
|
* query-visible int that sits in MORE THAN ONE value bucket of a field —
|
|
@@ -2563,6 +2629,13 @@ export interface DiskAnnRebuildFromExistingParams {
|
|
|
2563
2629
|
* the slot-only benchmark paths that don't carry entity ids.
|
|
2564
2630
|
*/
|
|
2565
2631
|
slotIds?: Buffer
|
|
2632
|
+
/**
|
|
2633
|
+
* M3 PD-2: the source-of-truth log generation this publish reflects
|
|
2634
|
+
* (the committed watermark the JS wrapper read at rebuild time).
|
|
2635
|
+
* Omitted on hosts without a generation manifest — the stamp then
|
|
2636
|
+
* carries no `sourceGeneration` and verifiers skip the comparison.
|
|
2637
|
+
*/
|
|
2638
|
+
sourceGeneration?: number
|
|
2566
2639
|
}
|
|
2567
2640
|
|
|
2568
2641
|
/** Vamana graph build knobs exposed to JS. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/cor",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.20",
|
|
4
4
|
"description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
},
|
|
83
83
|
"devDependencies": {
|
|
84
84
|
"@napi-rs/cli": "^3.0.0",
|
|
85
|
-
"@soulcraft/brainy": "8.
|
|
85
|
+
"@soulcraft/brainy": "8.5.0",
|
|
86
86
|
"@types/node": "^22.0.0",
|
|
87
87
|
"pg": "^8.21.0",
|
|
88
88
|
"tsx": "^4.21.0",
|