@soulcraft/cor 3.0.11 → 3.0.12

package/dist/native/types.d.ts

@@ -1030,8 +1030,33 @@ export interface NativeLsmEngineInstance {
      * data lives in both the log and the persisted SSTable, so the log copy is
      * safe to retire. No-op for in-memory engines (constructed without
      * `openWithDir`).
+     *
+     * Returns the number of records CARRIED FORWARD across the shard-log
+     * rotations — writes that landed between the flush's drain snapshot and
+     * this commit (the wrapper's `await`s on blob + manifest IO let the event
+     * loop run writes in that window). Carried records stay replayable in the
+     * fresh active logs and drain with the next flush; pre-3.0.12 rotations
+     * exiled them into never-replayed archives (CORTEX-RESTART-STRAND).
+     */
+    commitFlushThrough(throughEpoch: bigint): number;
+    /**
+     * Replay accounting from the memtable's cold open, as JSON:
+     * `{ activeRecords, archiveRecovered, archivesUnreadable, tornTailBytes,
+     * liveLogsRecreated, anomalousShards: string[] }`. Logged loudly at init
+     * when any anomaly is non-zero — the forensic record of what cold-open
+     * recovery found, captured BEFORE any rebuild wipes the evidence.
+     * `liveLogsRecreated > 0` or `archivesUnreadable > 0` is a strand verdict
+     * (unflushed records from a missing live log are unknowable).
+     */
+    replayStatsJson(): string;
+    /**
+     * Toggle bulk-load mode on the shard logs. `true` suspends the
+     * per-append msyncs — rebuild-from-canonical ONLY (canonical is the
+     * durability source; a crashed rebuild re-runs). `false` restores them
+     * and performs one full msync per shard: the rebuild's single durability
+     * point. Never used on the live write path.
      */
-    commitFlushThrough(throughEpoch: bigint): void;
+    setBulkLoad(on: boolean): void;
     /**
      * Update the budget ceiling. Driven by the wrapper's
      * `ResourceManager` subscriber callback. Pass `0xffffffffffffffffn`

package/dist/utils/NativeMetadataIndex.d.ts

@@ -889,6 +889,15 @@ export declare class MetadataIndexManager implements MetadataIndexProvider {
      * @returns the number of entities indexed (those carrying metadata).
      */
     private rebuildEntityMetadata;
+    /**
+     * Rebuild-path verify-read for a single entity's canonical metadata.
+     * Retries once on a thrown read, then rethrows — a rebuild must NEVER
+     * silently exclude an entity because a storage read faulted (that
+     * completes the rebuild under-posted and clears the strand verdict
+     * over missing data). A clean `null`/`undefined` result is returned
+     * as `null` — a genuinely metadata-less record, the caller counts it.
+     */
+    private readRebuildMetadataOrThrow;
     /**
      * Batch-load metadata for a page of noun/verb ids — prefers the storage
      * adapter's batch API, falls back to per-id reads.

package/dist/utils/NativeMetadataIndex.js

@@ -487,6 +487,35 @@ export class MetadataIndexManager {
                 engine.registerSstablePath(path);
             }
         }
+        // Cold-open replay forensics (CORTEX-RESTART-STRAND). The engine's
+        // open replayed the active shard logs AND recovered any records a
+        // prior session exiled into archives (pre-carry-forward rotations,
+        // mid-rotation crashes). Surface every anomaly LOUDLY here — this
+        // line is the evidence that survives even if a rebuild later wipes
+        // the on-disk state.
+        if (metaDir !== null && typeof this.lsmEngine.replayStatsJson === 'function') {
+            try {
+                const stats = JSON.parse(this.lsmEngine.replayStatsJson());
+                if (stats.archiveRecovered > 0 ||
+                    stats.archivesUnreadable > 0 ||
+                    stats.tornTailBytes > 0) {
+                    prodLog.error(`[NativeMetadataIndex] COLD-OPEN RECOVERY: replayed ` +
+                        `${stats.activeRecords} active-log record(s); RECOVERED ` +
+                        `${stats.archiveRecovered} record(s) exiled into archives by a ` +
+                        `prior session; ${stats.archivesUnreadable} unreadable ` +
+                        `archive(s); ${stats.tornTailBytes} torn-tail byte(s). ` +
+                        `Recovered records are served and drain with the next flush. ` +
+                        `Per-shard: ${stats.anomalousShards.join(' | ')}`);
+                }
+            }
+            catch (error) {
+                prodLog.warn('[NativeMetadataIndex] replay-forensics read failed (non-fatal):', error);
+            }
+        }
+        else if (metaDir !== null) {
+            prodLog.warn('[NativeMetadataIndex] replay forensics unavailable — the loaded ' +
+                'native binary predates 3.0.12; upgrade the paired .node build.');
+        }
     }
     /**
      * **LSM durability (steps 3-4) — the sync part.** Register the
@@ -766,7 +795,18 @@ export class MetadataIndexManager {
         // flushed data is safely in the new SSTable. Warn + continue rather than
         // abort the rebuild — the next clean flush retires the log.
         try {
-            engine.commitFlushThrough(result.lastDurableEpoch);
+            const carried = engine.commitFlushThrough(result.lastDurableEpoch);
+            // Writes that landed during THIS flush's blob/manifest awaits (the
+            // event loop runs between the drain and this commit) were carried
+            // into the fresh active logs — they stay replayable and drain with
+            // the next flush. Logged for the write-ack audit trail: pre-3.0.12
+            // rotations exiled exactly these records into never-replayed
+            // archives (CORTEX-RESTART-STRAND).
+            if (typeof carried === 'number' && carried > 0) {
+                prodLog.info(`flushLsm: carried ${carried} concurrent-write record(s) across ` +
+                    `the shard-log rotation for SSTable ${id} — they drain with the ` +
+                    `next flush.`);
+            }
         }
         catch (error) {
             prodLog.warn(`flushLsm: commitFlushThrough deferred for SSTable ${id} (it + the ` +
@@ -1197,6 +1237,36 @@ export class MetadataIndexManager {
     async detectDerivedStrand() {
         if (!this.lsmEngine || !this.lsmMetaDir)
             return;
+        // 0. Replay-stats verdict (3.0.12). The native open now RECREATES a
+        // missing live log and recovers what its archives hold, so the
+        // filename census below can no longer see the deleted-live-log
+        // signature — the native layer reports it instead. Either condition
+        // is a strand verdict: a recreated-with-archives shard may have held
+        // UNFLUSHED records that lived only in the missing live log
+        // (unknowable — rebuild is the only honest answer), and an
+        // unreadable archive means potentially-exiled records were
+        // unreachable to recovery.
+        try {
+            const engine = this.lsmEngine;
+            if (typeof engine.replayStatsJson === 'function') {
+                const stats = JSON.parse(engine.replayStatsJson());
+                const recreated = stats.liveLogsRecreated ?? 0;
+                const unreadable = stats.archivesUnreadable ?? 0;
+                if (recreated > 0 || unreadable > 0) {
+                    this.strandDetected = true;
+                    prodLog.error(`[NativeMetadataIndex] STRAND: cold-open replay reports ` +
+                        `${recreated} shard live log(s) MISSING (recreated; archives ` +
+                        `recovered what they held, but unflushed records from the ` +
+                        `missing logs are unknowable) and ${unreadable} unreadable ` +
+                        `archive(s). Reporting not-ready so the cold-open gate ` +
+                        `rebuilds from canonical. Per-shard: ` +
+                        `${(stats.anomalousShards ?? []).join(' | ')}`);
+                }
+            }
+        }
+        catch {
+            // Verdict probe is advisory — a stats read failure must not block open.
+        }
         // 1. Archive-orphan census over the memtable shard logs.
         try {
             const memtableDir = join(this.lsmMetaDir, 'memtable');
@@ -2584,8 +2654,31 @@ export class MetadataIndexManager {
             // branch loaded the whole corpus in a single getNouns/getVerbs({limit:1e6})
             // call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
             // batch(25) ≈ 250K. (#74 — perf-at-all-scales mandate.)
-            const totalNounsProcessed = await this.rebuildEntityMetadata('noun');
-            const totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
+            // Bulk-load mode for the re-post walk: canonical is the durability
+            // source during a rebuild (a crash re-runs it), so the per-append
+            // shard-log msyncs — ~2 synchronous disk flushes per posting record,
+            // the mechanism behind memory-vm's MEASURED ~1.3-2 items/s (30 min
+            // for 2,354 entities on e2-standard-2, CORTEX-RESTART-STRAND) — are
+            // suspended and replaced by ONE full msync per shard at the end of
+            // the walk (setBulkLoad(false), inside finally so a failed rebuild
+            // can never leave the live write path in deferred mode).
+            const bulkCapable = this.lsmEngine !== null &&
+                typeof this.lsmEngine.setBulkLoad ===
+                    'function';
+            if (bulkCapable)
+                this.lsmEngine.setBulkLoad(true);
+            let totalNounsProcessed;
+            let totalVerbsProcessed;
+            try {
+                totalNounsProcessed = await this.rebuildEntityMetadata('noun');
+                totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
+            }
+            finally {
+                // Leaving bulk mode performs the walk's single durability point
+                // (one full msync per shard log).
+                if (bulkCapable)
+                    this.lsmEngine.setBulkLoad(false);
+            }
             // Final flush
             await this.flushRebuildDirty();
             await this.flush();
@@ -2616,6 +2709,7 @@ export class MetadataIndexManager {
         let hasMore = true;
         let processed = 0;
         let sinceFlush = 0;
+        let noMetadata = 0;
         while (hasMore) {
             const page = kind === 'noun'
                 ? await this.storage.getNouns({ pagination: { limit: PAGE, cursor } })
@@ -2626,7 +2720,16 @@ export class MetadataIndexManager {
             const ids = items.map((it) => it.id);
             const md = await this.loadRebuildMetadataBatch(kind, ids);
             for (const it of items) {
-                const metadata = md.get(it.id);
+                // An id enumerated by canonical but absent from the batch result
+                // gets an individual verify-read: a batch adapter may return a
+                // partial map on an internal fault, and silently skipping the
+                // entity would complete the rebuild UNDER-POSTED — the exact
+                // damage the strand detector exists to catch, laundered by its
+                // own healer (the 01:31 memory-vm rebuild's failure shape). The
+                // verify-read retries once, then FAILS THE REBUILD loudly; a
+                // clean null (a genuinely metadata-less record) is counted +
+                // reported, never silently dropped.
+                const metadata = md.get(it.id) ?? (await this.readRebuildMetadataOrThrow(kind, it.id));
                 if (metadata) {
                     await this.addToIndex(it.id, metadata, true, true);
                     processed++;
@@ -2635,13 +2738,49 @@ export class MetadataIndexManager {
                         sinceFlush = 0;
                     }
                 }
+                else {
+                    noMetadata++;
+                }
             }
             hasMore = page.hasMore;
             cursor = page.nextCursor;
             await this.yieldToEventLoop();
         }
+        if (noMetadata > 0) {
+            prodLog.warn(`[NativeMetadataIndex] rebuild(${kind}): ${noMetadata} enumerated ` +
+                `entit${noMetadata === 1 ? 'y' : 'ies'} carried no canonical ` +
+                `metadata (verified by direct read) and were not indexed — ` +
+                `expected only for metadata-less legacy records.`);
+        }
         return processed;
     }
+    /**
+     * Rebuild-path verify-read for a single entity's canonical metadata.
+     * Retries once on a thrown read, then rethrows — a rebuild must NEVER
+     * silently exclude an entity because a storage read faulted (that
+     * completes the rebuild under-posted and clears the strand verdict
+     * over missing data). A clean `null`/`undefined` result is returned
+     * as `null` — a genuinely metadata-less record, the caller counts it.
+     */
+    async readRebuildMetadataOrThrow(kind, id) {
+        const read = () => kind === 'noun'
+            ? this.storage.getNounMetadata(id)
+            : this.storage.getVerbMetadata(id);
+        try {
+            return (await read()) ?? null;
+        }
+        catch {
+            try {
+                return (await read()) ?? null;
+            }
+            catch (error) {
+                throw new Error(`[NativeMetadataIndex] rebuild(${kind}): canonical metadata read ` +
+                    `for '${id}' faulted twice — failing the rebuild rather than ` +
+                    `silently excluding the entity (an under-posted rebuild is the ` +
+                    `strand bug reintroduced by its own healer). Cause: ${String(error)}`);
+            }
+        }
+    }
     /**
      * Batch-load metadata for a page of noun/verb ids — prefers the storage
      * adapter's batch API, falls back to per-id reads.
@@ -2656,14 +2795,13 @@ export class MetadataIndexManager {
         }
         const m = new Map();
         for (const id of ids) {
-            try {
-                const md = kind === 'noun'
-                    ? await this.storage.getNounMetadata(id)
-                    : await this.storage.getVerbMetadata(id);
-                if (md)
-                    m.set(id, md);
-            }
-            catch { }
+            // Retry-once-then-throw (readRebuildMetadataOrThrow): a faulted
+            // read must fail the rebuild loudly, never silently exclude the
+            // entity — a swallowed fault here completed the rebuild
+            // under-posted (the strand bug re-created by its own healer).
+            const md = await this.readRebuildMetadataOrThrow(kind, id);
+            if (md)
+                m.set(id, md);
         }
         return m;
     }

package/docs/snapshot-safety.md

@@ -128,7 +128,13 @@ is the reference implementation.
    tail offset; live writers append past that offset without
    affecting what the snapshot reader sees. On compaction
    commit the delta is renamed to a `.archive` sibling and a
-   fresh empty delta replaces it.
+   fresh delta replaces it, carrying forward any record newer
+   than the commit watermark — a record appended while the
+   commit's durability IO was in flight stays replayable in the
+   fresh delta, never exiled to the archive. Opens also scan
+   retained archives for records above the watermark and
+   recover them, so even a rotation interrupted between its two
+   renames loses nothing.
 
 3. **Atomic head pointer** (`head`) — a tiny text file named by
    `PointerFile` (`diskann::pointer_file`). Names the current

package/native/index.d.ts

@@ -1260,8 +1260,34 @@ export declare class NativeLsmEngine {
    *
    * No-op for in-memory engines (constructed without
    * `openWithDir`).
-   */
-  commitFlushThrough(throughEpoch: bigint): void
+   *
+   * Returns the number of records carried forward across the
+   * shard-log rotations — writes that landed between the flush's
+   * drain snapshot and this commit (the wrapper's async
+   * durability-IO window; the JS event loop runs writes there).
+   * They stay replayable in the fresh active logs and drain with
+   * the next flush. The wrapper logs non-zero counts for the
+   * write-ack audit trail.
+   */
+  commitFlushThrough(throughEpoch: bigint): number
+  /**
+   * Toggle bulk-load mode on the shard logs. `true` suspends the
+   * per-append msyncs — for rebuild-from-canonical ONLY (canonical
+   * is the durability source; a crashed rebuild re-runs). `false`
+   * restores them and performs one full msync per shard: the
+   * rebuild's single durability point. Never used on the live
+   * write path, whose ack contract requires the per-append msync.
+   */
+  setBulkLoad(on: boolean): void
+  /**
+   * Replay accounting from the memtable's cold open, as JSON:
+   * `{ activeRecords, archiveRecovered, archivesUnreadable,
+   * tornTailBytes, anomalousShards: string[] }`. The wrapper logs
+   * this at init when any anomaly is non-zero — the forensic
+   * record of what cold-open recovery found, captured BEFORE any
+   * rebuild wipes the evidence (CORTEX-RESTART-STRAND).
+   */
+  replayStatsJson(): string
   /** `"u32"` or `"u64"`. */
   idSpace(): string
   /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.11",
+  "version": "3.0.12",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -82,7 +82,7 @@
   },
   "devDependencies": {
     "@napi-rs/cli": "^3.0.0",
-    "@soulcraft/brainy": "8.2.2",
+    "@soulcraft/brainy": "8.2.5",
     "@types/node": "^22.0.0",
     "pg": "^8.21.0",
     "tsx": "^4.21.0",