@soulcraft/cor 3.0.9 → 3.0.11

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -127,6 +127,24 @@ const FLUSH_FAILURE_BACKPRESSURE_THRESHOLD = 4;
  * to this ceiling. A successful consolidation resets to zero.
  */
 const CONSOLIDATION_BACKOFF_MAX_MS = 5 * 60_000;
+/**
+ * Live-L0-segment count that triggers a loud warning: consolidation folds at
+ * {@link L0_COMPACT_THRESHOLD}, so accumulating this many segments means it
+ * has been failing for many cycles. Every query fans out across all live
+ * segments plus the base — memory-vm's incident reached 162 segments and
+ * degraded reads to 25–40 s. The warn is the early alarm; the hard cap below
+ * is the brake.
+ */
+const L0_SEGMENT_WARN_THRESHOLD = 32;
+/**
+ * Live-L0-segment count at which `addItem` REFUSES writes loudly
+ * (CORTEX-CONSOLIDATION-SILENT-DROP hardening). Segments grow past the warn
+ * threshold only while consolidation persistently fails; each one adds query
+ * fan-out and RAM until reads collapse and the process OOMs. Same posture as
+ * the flush-failure brake: a loud bounded stop beats an unbounded quiet
+ * degradation. Clears itself the moment one consolidation folds the segments.
+ */
+const L0_SEGMENT_HARD_CAP = 256;
 /**
  * Storage key of the vector segment manifest (flat key — slashed keys trip
  * the storage adapters' unknown-key warning; same lesson as the metadata
@@ -434,6 +452,18 @@ export class NativeDiskAnnWrapper {
      * `rebuild()` call, which folds the delta into the main index.
      */
     async addItem(item) {
+        // Segment brake (CORTEX-CONSOLIDATION-SILENT-DROP): segments accumulate
+        // past the warn threshold only while consolidation persistently fails —
+        // the delta keeps draining to NEW segments (flush works), so the flush-
+        // failure brake never engages, yet fan-out and RAM grow without bound
+        // (162 segments degraded memory-vm's reads to 25–40 s en route to OOM).
+        // Refuse loudly at the cap; one successful consolidation clears it.
+        if (this.l0Segments.length >= L0_SEGMENT_HARD_CAP) {
+            throw new Error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments — ` +
+                `consolidation is persistently failing; refusing writes to prevent ` +
+                `unbounded segment growth (read collapse / OOM). See the ` +
+                `consolidation-failure log lines for the underlying cause.`);
+        }
         // item.vector.length is the authoritative width for THIS write; fall back to
         // it when the brain is zero-config (config.dimensions undefined). Otherwise
         // bytesPerVector is NaN and every `NaN >= cap` backpressure/fold check is
@@ -1523,13 +1553,33 @@ export class NativeDiskAnnWrapper {
         }
         prodLog.info(`NativeDiskAnnWrapper: flushed ${builtDelta.size} delta entries to L0 segment ${file} ` +
             `(${this.l0Segments.length} live segments)`);
+        // Integrity probe (CORTEX-CONSOLIDATION-SILENT-DROP hardening): one O(1)
+        // stat per flush catches a base deleted out from under the live handle in
+        // SECONDS instead of at the next consolidation — the strand ran for hours
+        // undetected on memory-vm. Detection here; the heal is the consolidation
+        // below (runRebuild's self-heal drops the stale handle and rebuilds from
+        // segments + canonical).
+        const baseMissing = this.native !== null &&
+            this.config.indexPath !== undefined &&
+            !existsSync(this.config.indexPath);
+        if (baseMissing) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — triggering self-heal consolidation`);
+        }
+        if (this.l0Segments.length >= L0_SEGMENT_WARN_THRESHOLD) {
+            prodLog.error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments ` +
+                `(consolidation folds at ${L0_COMPACT_THRESHOLD}) — consolidation is ` +
+                `failing; query fan-out is degrading and the write brake engages at ` +
+                `${L0_SEGMENT_HARD_CAP}`);
+        }
         // Tiered maintenance: enough small segments → one BACKGROUND consolidation
         // (the async rebuild folds base + L0s + delta into a fresh base off-thread).
-        // Guarded by an exponential backoff: a persistently-failing consolidation
-        // must not re-fire on every ~6 s flush (memory-vm hammered it for hours) —
-        // it backs off up to CONSOLIDATION_BACKOFF_MAX_MS, and a success resets it
-        // (see the tail of runRebuild). CORTEX-CONSOLIDATION-SILENT-DROP.
-        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD &&
+        // A missing base skips the segment-count threshold — heal now, not three
+        // flushes from now. Guarded by an exponential backoff: a persistently-
+        // failing consolidation must not re-fire on every ~6 s flush (memory-vm
+        // hammered it for hours) — it backs off up to CONSOLIDATION_BACKOFF_MAX_MS,
+        // and a success resets it (see the tail of runRebuild).
+        if ((baseMissing || this.l0Segments.length >= L0_COMPACT_THRESHOLD) &&
             !this.rebuildInFlight &&
             Date.now() >= this.consolidationCooldownUntil) {
             this.rebuild().catch((error) => {

package/dist/utils/NativeMetadataIndex.d.ts

@@ -103,6 +103,18 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
      * cold-open rebuild gate never skips over an index that has not loaded yet.
      */
     private readinessInitialized;
+    /**
+     * A derived-state strand was detected at cold open (CORTEX-CONSOLIDATION-
+     * SILENT-DROP, memory-vm 2026-07-12): postings lost while canonical is
+     * intact — e.g. memtable log shards deleted out from under the engine (15
+     * of 64 on the incident brain), whose histories silently vanish because a
+     * missing shard log is recreated empty on next touch. Set by
+     * {@link detectDerivedStrand} (archive-orphan census + the
+     * |canonical| == |posted| invariant); forces {@link isReady} to report
+     * `false` so brainy's cold-open gate runs the one rebuild that re-posts
+     * everything from canonical. Cleared by a completed {@link rebuild}.
+     */
+    private strandDetected;
     /**
      * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
      * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
@@ -518,6 +530,31 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
      * @returns `true` when a cold-open rebuild would be redundant.
      */
     isReady(): boolean;
+    /**
+     * Detect a derived-state STRAND at cold open: canonical is the source of
+     * truth, so any loss in the derived posting state — however it happened —
+     * surfaces as one of two signals checked here. Both are cheap (a directory
+     * listing + one bitmap-cardinality read + one canonical count probe), run
+     * once per open, never on a hot path. Advisory on storage adapters that
+     * can't answer (no blob paths / no exact counts): detection skips rather
+     * than false-positives. CORTEX-CONSOLIDATION-SILENT-DROP.
+     *
+     * 1. **Archive-orphan census.** The memtable log rotates
+     *    `memtable-shard-NN.log` → `*.archive` siblings and always recreates
+     *    the live file — so an archive WITHOUT its live log proves the live
+     *    log was deleted (the deleter's signature on memory-vm: 15 of 64
+     *    shards). A missing shard log silently replays nothing and is
+     *    recreated empty on next touch; without this census the loss is
+     *    invisible.
+     * 2. **The |canonical| == |posted| invariant.** Every live entity carries
+     *    exactly one `noun` posting, so the engine's total posted-entity count
+     *    must equal canonical's noun count. `posted < canonical` = stranded
+     *    entities (the incident's ~35 unqueryable writes); `posted > canonical`
+     *    = orphaned postings (egress-guarded at read time, reconciled by the
+     *    same rebuild). Skipped at/above the u32 saturation point of
+     *    `getTotalEntityCount` (a saturated count would make equality lie).
+     */
+    private detectDerivedStrand;
     /**
      * **#72 Phase C.** Build the shared mmap `BinaryIdMapper` from the
      * brain's `_id_mapper/*` storage paths and inject it into the Rust

package/dist/utils/NativeMetadataIndex.js

@@ -15,7 +15,7 @@
  * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
  * Rust operates in-memory, returns serialized state for TS to persist.
  */
-import { existsSync, mkdirSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { prodLog, getGlobalCache, FieldTypeInference, resolveEntityField } from '@soulcraft/brainy/internals';
 import { compareCodePoints } from './collation.js';
@@ -177,6 +177,18 @@ export class MetadataIndexManager {
      * cold-open rebuild gate never skips over an index that has not loaded yet.
      */
     readinessInitialized = false;
+    /**
+     * A derived-state strand was detected at cold open (CORTEX-CONSOLIDATION-
+     * SILENT-DROP, memory-vm 2026-07-12): postings lost while canonical is
+     * intact — e.g. memtable log shards deleted out from under the engine (15
+     * of 64 on the incident brain), whose histories silently vanish because a
+     * missing shard log is recreated empty on next touch. Set by
+     * {@link detectDerivedStrand} (archive-orphan census + the
+     * |canonical| == |posted| invariant); forces {@link isReady} to report
+     * `false` so brainy's cold-open gate runs the one rebuild that re-posts
+     * everything from canonical. Cleared by a completed {@link rebuild}.
+     */
+    strandDetected = false;
     /**
      * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
      * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
@@ -1101,6 +1113,13 @@ export class MetadataIndexManager {
         await this.warmCache();
         await this.lazyLoadCounts();
         this.syncTypeCountsToFixed();
+        // Strand detection (CORTEX-CONSOLIDATION-SILENT-DROP): the LSM can load
+        // "durable state" that is silently MISSING postings — deleted shard logs
+        // replay nothing and are recreated empty on next touch, so the index
+        // serves healthy-while-broken. Detect it here, before the readiness
+        // contract reports ready, so brainy's gate heals via rebuild-from-
+        // canonical instead of trusting a partial index.
+        await this.detectDerivedStrand();
         // **#18 migration kick — LAST, after the index is fully constructed.** If the
         // epoch guard flagged a large stale brain (outcome 4), start the coordinated
         // blocking migration NOW: `start()` flips the lock (`isMigrating()` → true)
@@ -1145,8 +1164,110 @@ export class MetadataIndexManager {
             return false;
         if (this.knownFields.size > 0 && !this.lsmHasDurableState())
             return false;
+        // A detected strand (postings lost while canonical is intact) means a
+        // rebuild is NOT redundant — see {@link detectDerivedStrand}.
+        if (this.strandDetected)
+            return false;
         return true;
     }
+    /**
+     * Detect a derived-state STRAND at cold open: canonical is the source of
+     * truth, so any loss in the derived posting state — however it happened —
+     * surfaces as one of two signals checked here. Both are cheap (a directory
+     * listing + one bitmap-cardinality read + one canonical count probe), run
+     * once per open, never on a hot path. Advisory on storage adapters that
+     * can't answer (no blob paths / no exact counts): detection skips rather
+     * than false-positives. CORTEX-CONSOLIDATION-SILENT-DROP.
+     *
+     * 1. **Archive-orphan census.** The memtable log rotates
+     *    `memtable-shard-NN.log` → `*.archive` siblings and always recreates
+     *    the live file — so an archive WITHOUT its live log proves the live
+     *    log was deleted (the deleter's signature on memory-vm: 15 of 64
+     *    shards). A missing shard log silently replays nothing and is
+     *    recreated empty on next touch; without this census the loss is
+     *    invisible.
+     * 2. **The |canonical| == |posted| invariant.** Every live entity carries
+     *    exactly one `noun` posting, so the engine's total posted-entity count
+     *    must equal canonical's noun count. `posted < canonical` = stranded
+     *    entities (the incident's ~35 unqueryable writes); `posted > canonical`
+     *    = orphaned postings (egress-guarded at read time, reconciled by the
+     *    same rebuild). Skipped at/above the u32 saturation point of
+     *    `getTotalEntityCount` (a saturated count would make equality lie).
+     */
+    async detectDerivedStrand() {
+        if (!this.lsmEngine || !this.lsmMetaDir)
+            return;
+        // 1. Archive-orphan census over the memtable shard logs.
+        try {
+            const memtableDir = join(this.lsmMetaDir, 'memtable');
+            if (existsSync(memtableDir)) {
+                const files = readdirSync(memtableDir);
+                const live = new Set(files.filter((f) => /^memtable-shard-\d{2}\.log$/.test(f)));
+                const orphaned = [];
+                for (const f of files) {
+                    const m = /^(memtable-shard-\d{2}\.log)\..+\.archive$/.exec(f);
+                    if (m && !live.has(m[1]) && !orphaned.includes(m[1])) {
+                        orphaned.push(m[1]);
+                    }
+                }
+                if (orphaned.length > 0) {
+                    this.strandDetected = true;
+                    prodLog.error(`[NativeMetadataIndex] STRAND: ${orphaned.length} memtable shard ` +
+                        `log(s) have archive rotations but the live log is MISSING ` +
+                        `(${orphaned.join(', ')}) — the live logs were deleted and their ` +
+                        `posting history is gone. Reporting not-ready so the cold-open ` +
+                        `gate rebuilds from canonical.`);
+                }
+            }
+        }
+        catch {
+            // Census is advisory — an unreadable dir must not block open.
+        }
+        // 2. |canonical| == |posted| invariant.
+        try {
+            const getNouns = this.storage.getNouns;
+            if (typeof getNouns === 'function') {
+                const probe = await getNouns.call(this.storage, { pagination: { limit: 1 } });
+                const canonical = probe?.totalCount;
+                if (typeof canonical === 'number' &&
+                    Number.isFinite(canonical) &&
+                    canonical < 0xffff_ffff) {
+                    // Posted count = Σ per-type snapshot cardinalities over brainy's
+                    // CLOSED 42-noun vocabulary (wire-stable, validated at the storage
+                    // boundary — no type outside it can exist in canonical).
+                    // getEntityCountByType reads the LSM SNAPSHOT (memtable + SSTables),
+                    // so it is reopen-true; getTotalEntityCount is NOT (it counts the
+                    // in-process field registry, which is empty at cold open). O(42)
+                    // bitmap-cardinality reads, no id materialization — scale-safe.
+                    // Raw (unmapped) cardinality is acceptable here: at cold open there
+                    // is no mapper-eviction churn to inflate it.
+                    const { NounType } = await import('@soulcraft/brainy');
+                    let posted = 0;
+                    for (const t of Object.values(NounType)) {
+                        posted += this.native.getEntityCountByType(t);
+                    }
+                    // LOWER-BOUND invariant, not equality: the adapter's totalCount
+                    // EXCLUDES system-routed entities (e.g. the VFS root) while the
+                    // index posts them, so healthy brains sit at posted ≥ canonical.
+                    // posted < canonical therefore proves lost postings. A strand
+                    // smaller than the (1–2 entity) system skew is masked here and
+                    // falls to the archive census above; the exact invariant (a
+                    // persisted posted-count stamp) is CORTEX-BILLION-SCALE-SPINE work.
+                    if (posted < canonical) {
+                        this.strandDetected = true;
+                        prodLog.error(`[NativeMetadataIndex] STRAND: canonical holds ${canonical} ` +
+                            `entities but the index has postings for only ${posted} — ` +
+                            `${canonical - posted}+ entities are durable in canonical but ` +
+                            `UNQUERYABLE (lost postings). Reporting not-ready so the ` +
+                            `cold-open gate rebuilds from canonical.`);
+                    }
+                }
+            }
+        }
+        catch {
+            // Invariant probe is advisory — a failed canonical read must not block open.
+        }
+    }
     // ==========================================================================
     // Storage I/O helpers
     // ==========================================================================
@@ -2469,6 +2590,10 @@ export class MetadataIndexManager {
             await this.flushRebuildDirty();
             await this.flush();
             prodLog.info(`Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
+            // A completed rebuild re-posted every canonical entity into a freshly
+            // re-opened engine (reopenLsmEngineFresh wiped the stale shard set) —
+            // any cold-open strand verdict is now satisfied.
+            this.strandDetected = false;
         }
         finally {
             this.isRebuilding = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.9",
+  "version": "3.0.11",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",