@soulcraft/cor 3.0.10 → 3.0.11

package/dist/utils/NativeMetadataIndex.d.ts

@@ -103,6 +103,18 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
      * cold-open rebuild gate never skips over an index that has not loaded yet.
      */
     private readinessInitialized;
+    /**
+     * A derived-state strand was detected at cold open (CORTEX-CONSOLIDATION-
+     * SILENT-DROP, memory-vm 2026-07-12): postings lost while canonical is
+     * intact — e.g. memtable log shards deleted out from under the engine (15
+     * of 64 on the incident brain), whose histories silently vanish because a
+     * missing shard log is recreated empty on next touch. Set by
+     * {@link detectDerivedStrand} (archive-orphan census + the
+     * |canonical| == |posted| invariant); forces {@link isReady} to report
+     * `false` so brainy's cold-open gate runs the one rebuild that re-posts
+     * everything from canonical. Cleared by a completed {@link rebuild}.
+     */
+    private strandDetected;
     /**
      * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
      * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
@@ -518,6 +530,31 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
      * @returns `true` when a cold-open rebuild would be redundant.
      */
     isReady(): boolean;
+    /**
+     * Detect a derived-state STRAND at cold open: canonical is the source of
+     * truth, so any loss in the derived posting state — however it happened —
+     * surfaces as one of two signals checked here. Both are cheap (a directory
+     * listing + one bitmap-cardinality read + one canonical count probe), run
+     * once per open, never on a hot path. Advisory on storage adapters that
+     * can't answer (no blob paths / no exact counts): detection skips rather
+     * than false-positives. CORTEX-CONSOLIDATION-SILENT-DROP.
+     *
+     * 1. **Archive-orphan census.** The memtable log rotates
+     *    `memtable-shard-NN.log` → `*.archive` siblings and always recreates
+     *    the live file — so an archive WITHOUT its live log proves the live
+     *    log was deleted (the deleter's signature on memory-vm: 15 of 64
+     *    shards). A missing shard log silently replays nothing and is
+     *    recreated empty on next touch; without this census the loss is
+     *    invisible.
+     * 2. **The |canonical| == |posted| invariant.** Every live entity carries
+     *    exactly one `noun` posting, so the engine's total posted-entity count
+     *    must equal canonical's noun count. `posted < canonical` = stranded
+     *    entities (the incident's ~35 unqueryable writes); `posted > canonical`
+     *    = orphaned postings (egress-guarded at read time, reconciled by the
+     *    same rebuild). Skipped at/above the u32 saturation point of
+     *    `getTotalEntityCount` (a saturated count would make equality lie).
+     */
+    private detectDerivedStrand;
     /**
      * **#72 Phase C.** Build the shared mmap `BinaryIdMapper` from the
      * brain's `_id_mapper/*` storage paths and inject it into the Rust

package/dist/utils/NativeMetadataIndex.js

@@ -15,7 +15,7 @@
  * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
  * Rust operates in-memory, returns serialized state for TS to persist.
  */
-import { existsSync, mkdirSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { prodLog, getGlobalCache, FieldTypeInference, resolveEntityField } from '@soulcraft/brainy/internals';
 import { compareCodePoints } from './collation.js';
@@ -177,6 +177,18 @@ export class MetadataIndexManager {
      * cold-open rebuild gate never skips over an index that has not loaded yet.
      */
     readinessInitialized = false;
+    /**
+     * A derived-state strand was detected at cold open (CORTEX-CONSOLIDATION-
+     * SILENT-DROP, memory-vm 2026-07-12): postings lost while canonical is
+     * intact — e.g. memtable log shards deleted out from under the engine (15
+     * of 64 on the incident brain), whose histories silently vanish because a
+     * missing shard log is recreated empty on next touch. Set by
+     * {@link detectDerivedStrand} (archive-orphan census + the
+     * |canonical| == |posted| invariant); forces {@link isReady} to report
+     * `false` so brainy's cold-open gate runs the one rebuild that re-posts
+     * everything from canonical. Cleared by a completed {@link rebuild}.
+     */
+    strandDetected = false;
     /**
      * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
      * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
@@ -1101,6 +1113,13 @@ export class MetadataIndexManager {
         await this.warmCache();
         await this.lazyLoadCounts();
         this.syncTypeCountsToFixed();
+        // Strand detection (CORTEX-CONSOLIDATION-SILENT-DROP): the LSM can load
+        // "durable state" that is silently MISSING postings — deleted shard logs
+        // replay nothing and are recreated empty on next touch, so the index
+        // serves healthy-while-broken. Detect it here, before the readiness
+        // contract reports ready, so brainy's gate heals via rebuild-from-
+        // canonical instead of trusting a partial index.
+        await this.detectDerivedStrand();
         // **#18 migration kick — LAST, after the index is fully constructed.** If the
         // epoch guard flagged a large stale brain (outcome 4), start the coordinated
         // blocking migration NOW: `start()` flips the lock (`isMigrating()` → true)
@@ -1145,8 +1164,110 @@ export class MetadataIndexManager {
             return false;
         if (this.knownFields.size > 0 && !this.lsmHasDurableState())
             return false;
+        // A detected strand (postings lost while canonical is intact) means a
+        // rebuild is NOT redundant — see {@link detectDerivedStrand}.
+        if (this.strandDetected)
+            return false;
         return true;
     }
+    /**
+     * Detect a derived-state STRAND at cold open: canonical is the source of
+     * truth, so any loss in the derived posting state — however it happened —
+     * surfaces as one of two signals checked here. Both are cheap (a directory
+     * listing + one bitmap-cardinality read + one canonical count probe), run
+     * once per open, never on a hot path. Advisory on storage adapters that
+     * can't answer (no blob paths / no exact counts): detection skips rather
+     * than false-positives. CORTEX-CONSOLIDATION-SILENT-DROP.
+     *
+     * 1. **Archive-orphan census.** The memtable log rotates
+     *    `memtable-shard-NN.log` → `*.archive` siblings and always recreates
+     *    the live file — so an archive WITHOUT its live log proves the live
+     *    log was deleted (the deleter's signature on memory-vm: 15 of 64
+     *    shards). A missing shard log silently replays nothing and is
+     *    recreated empty on next touch; without this census the loss is
+     *    invisible.
+     * 2. **The |canonical| == |posted| invariant.** Every live entity carries
+     *    exactly one `noun` posting, so the engine's total posted-entity count
+     *    must equal canonical's noun count. `posted < canonical` = stranded
+     *    entities (the incident's ~35 unqueryable writes); `posted > canonical`
+     *    = orphaned postings (egress-guarded at read time, reconciled by the
+     *    same rebuild). Skipped at/above the u32 saturation point of
+     *    `getTotalEntityCount` (a saturated count would make equality lie).
+     */
+    async detectDerivedStrand() {
+        if (!this.lsmEngine || !this.lsmMetaDir)
+            return;
+        // 1. Archive-orphan census over the memtable shard logs.
+        try {
+            const memtableDir = join(this.lsmMetaDir, 'memtable');
+            if (existsSync(memtableDir)) {
+                const files = readdirSync(memtableDir);
+                const live = new Set(files.filter((f) => /^memtable-shard-\d{2}\.log$/.test(f)));
+                const orphaned = [];
+                for (const f of files) {
+                    const m = /^(memtable-shard-\d{2}\.log)\..+\.archive$/.exec(f);
+                    if (m && !live.has(m[1]) && !orphaned.includes(m[1])) {
+                        orphaned.push(m[1]);
+                    }
+                }
+                if (orphaned.length > 0) {
+                    this.strandDetected = true;
+                    prodLog.error(`[NativeMetadataIndex] STRAND: ${orphaned.length} memtable shard ` +
+                        `log(s) have archive rotations but the live log is MISSING ` +
+                        `(${orphaned.join(', ')}) — the live logs were deleted and their ` +
+                        `posting history is gone. Reporting not-ready so the cold-open ` +
+                        `gate rebuilds from canonical.`);
+                }
+            }
+        }
+        catch {
+            // Census is advisory — an unreadable dir must not block open.
+        }
+        // 2. |canonical| == |posted| invariant.
+        try {
+            const getNouns = this.storage.getNouns;
+            if (typeof getNouns === 'function') {
+                const probe = await getNouns.call(this.storage, { pagination: { limit: 1 } });
+                const canonical = probe?.totalCount;
+                if (typeof canonical === 'number' &&
+                    Number.isFinite(canonical) &&
+                    canonical < 0xffff_ffff) {
+                    // Posted count = Σ per-type snapshot cardinalities over brainy's
+                    // CLOSED 42-noun vocabulary (wire-stable, validated at the storage
+                    // boundary — no type outside it can exist in canonical).
+                    // getEntityCountByType reads the LSM SNAPSHOT (memtable + SSTables),
+                    // so it is reopen-true; getTotalEntityCount is NOT (it counts the
+                    // in-process field registry, which is empty at cold open). O(42)
+                    // bitmap-cardinality reads, no id materialization — scale-safe.
+                    // Raw (unmapped) cardinality is acceptable here: at cold open there
+                    // is no mapper-eviction churn to inflate it.
+                    const { NounType } = await import('@soulcraft/brainy');
+                    let posted = 0;
+                    for (const t of Object.values(NounType)) {
+                        posted += this.native.getEntityCountByType(t);
+                    }
+                    // LOWER-BOUND invariant, not equality: the adapter's totalCount
+                    // EXCLUDES system-routed entities (e.g. the VFS root) while the
+                    // index posts them, so healthy brains sit at posted ≥ canonical.
+                    // posted < canonical therefore proves lost postings. A strand
+                    // smaller than the (1–2 entity) system skew is masked here and
+                    // falls to the archive census above; the exact invariant (a
+                    // persisted posted-count stamp) is CORTEX-BILLION-SCALE-SPINE work.
+                    if (posted < canonical) {
+                        this.strandDetected = true;
+                        prodLog.error(`[NativeMetadataIndex] STRAND: canonical holds ${canonical} ` +
+                            `entities but the index has postings for only ${posted} — ` +
+                            `${canonical - posted}+ entities are durable in canonical but ` +
+                            `UNQUERYABLE (lost postings). Reporting not-ready so the ` +
+                            `cold-open gate rebuilds from canonical.`);
+                    }
+                }
+            }
+        }
+        catch {
+            // Invariant probe is advisory — a failed canonical read must not block open.
+        }
+    }
     // ==========================================================================
     // Storage I/O helpers
     // ==========================================================================
@@ -2469,6 +2590,10 @@ export class MetadataIndexManager {
             await this.flushRebuildDirty();
             await this.flush();
             prodLog.info(`Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
+            // A completed rebuild re-posted every canonical entity into a freshly
+            // re-opened engine (reopenLsmEngineFresh wiped the stale shard set) —
+            // any cold-open strand verdict is now satisfied.
+            this.strandDetected = false;
         }
         finally {
             this.isRebuilding = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.10",
+  "version": "3.0.11",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",