@soulcraft/cor 3.0.8 → 3.0.10

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -209,6 +209,14 @@ export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
     private manifestNextId;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     private segmentsLoaded;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    private consecutiveFlushFailures;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    private consolidationFailures;
+    private consolidationCooldownUntil;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -110,6 +110,41 @@ const DELTA_HARD_CAP_BYTES = 256 * 1024 * 1024;
  * consolidation reads them back via `readVectorChunk` — bounded RAM.
  */
 const L0_COMPACT_THRESHOLD = 4;
+/**
+ * Consecutive backpressure-flush failures before `addItem` stops swallowing and
+ * REFUSES writes (CORTEX-CONSOLIDATION-SILENT-DROP). A committed write must not
+ * fail on a transient post-commit blip — but if the flush PERSISTENTLY fails,
+ * the delta can never drain and accepting more writes grows it without bound
+ * until the process OOMs (which is what stranded memory's base). Past this
+ * count the brake engages: refuse loudly so the write-storm halts instead of
+ * crashing. A single successful flush resets the counter.
+ */
+const FLUSH_FAILURE_BACKPRESSURE_THRESHOLD = 4;
+/**
+ * Cap on the exponential backoff between failed background consolidations
+ * (CORTEX-CONSOLIDATION-SILENT-DROP). A persistently-failing consolidation must
+ * not retry on every ~6 s flush (memory-vm hammered it for hours); back off up
+ * to this ceiling. A successful consolidation resets to zero.
+ */
+const CONSOLIDATION_BACKOFF_MAX_MS = 5 * 60_000;
+/**
+ * Live-L0-segment count that triggers a loud warning: consolidation folds at
+ * {@link L0_COMPACT_THRESHOLD}, so accumulating this many segments means it
+ * has been failing for many cycles. Every query fans out across all live
+ * segments plus the base — memory-vm's incident reached 162 segments and
+ * degraded reads to 25–40 s. The warn is the early alarm; the hard cap below
+ * is the brake.
+ */
+const L0_SEGMENT_WARN_THRESHOLD = 32;
+/**
+ * Live-L0-segment count at which `addItem` REFUSES writes loudly
+ * (CORTEX-CONSOLIDATION-SILENT-DROP hardening). Segments grow past the warn
+ * threshold only while consolidation persistently fails; each one adds query
+ * fan-out and RAM until reads collapse and the process OOMs. Same posture as
+ * the flush-failure brake: a loud bounded stop beats an unbounded quiet
+ * degradation. Clears itself the moment one consolidation folds the segments.
+ */
+const L0_SEGMENT_HARD_CAP = 256;
 /**
  * Storage key of the vector segment manifest (flat key — slashed keys trip
  * the storage adapters' unknown-key warning; same lesson as the metadata
@@ -311,6 +346,14 @@ export class NativeDiskAnnWrapper {
     manifestNextId = 1;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     segmentsLoaded = false;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    consecutiveFlushFailures = 0;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    consolidationFailures = 0;
+    consolidationCooldownUntil = 0;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by
@@ -409,6 +452,18 @@ export class NativeDiskAnnWrapper {
      * `rebuild()` call, which folds the delta into the main index.
      */
     async addItem(item) {
+        // Segment brake (CORTEX-CONSOLIDATION-SILENT-DROP): segments accumulate
+        // past the warn threshold only while consolidation persistently fails —
+        // the delta keeps draining to NEW segments (flush works), so the flush-
+        // failure brake never engages, yet fan-out and RAM grow without bound
+        // (162 segments degraded memory-vm's reads to 25–40 s en route to OOM).
+        // Refuse loudly at the cap; one successful consolidation clears it.
+        if (this.l0Segments.length >= L0_SEGMENT_HARD_CAP) {
+            throw new Error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments — ` +
+                `consolidation is persistently failing; refusing writes to prevent ` +
+                `unbounded segment growth (read collapse / OOM). See the ` +
+                `consolidation-failure log lines for the underlying cause.`);
+        }
         // item.vector.length is the authoritative width for THIS write; fall back to
         // it when the brain is zero-config (config.dimensions undefined). Otherwise
         // bytesPerVector is NaN and every `NaN >= cap` backpressure/fold check is
@@ -426,17 +481,29 @@ export class NativeDiskAnnWrapper {
             // its retirement filter and the serialized manifest saves keep the
             // on-disk list consistent).
             //
-            // A committed write must NEVER fail on post-commit index maintenance — the
+            // A committed write must not fail on a TRANSIENT post-commit blip — the
             // caller would retry and double-write (MEMORY-COR-RANGE-ERROR isolation
-            // ask). A backpressure-flush failure is a loud alarm, not a rejected
-            // write: log it, keep the delta for the next trigger, accept the write.
+            // ask), so a one-off flush failure is logged and the write is accepted,
+            // delta retained. BUT this is pre-delta-append backpressure: if the flush
+            // PERSISTENTLY fails the delta can never drain, and swallowing forever
+            // grows it without bound until OOM — which is exactly what stranded
+            // memory-vm's base (CORTEX-CONSOLIDATION-SILENT-DROP). Past the threshold
+            // the brake engages: refuse loudly so the write-storm halts. Recovers
+            // itself once a flush succeeds (self-heal drops a stranded base).
             try {
                 await (this.flushInFlight ?? this.flushDeltaToL0());
+                this.consecutiveFlushFailures = 0;
             }
             catch (error) {
-                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed (write accepted; ` +
-                    `delta retained for retry): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consecutiveFlushFailures++;
+                const msg = error instanceof Error ? error.message : String(error);
+                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed ` +
+                    `(${this.consecutiveFlushFailures}× consecutively): ${msg}`);
+                if (this.consecutiveFlushFailures >= FLUSH_FAILURE_BACKPRESSURE_THRESHOLD) {
+                    throw new Error(`NativeDiskAnnWrapper: vector index flush has failed ` +
+                        `${this.consecutiveFlushFailures}× consecutively — refusing writes to ` +
+                        `prevent unbounded delta growth (OOM). Last cause: ${msg}`);
+                }
             }
         }
         if (this.tombstones.has(item.id)) {
@@ -933,6 +1000,26 @@ export class NativeDiskAnnWrapper {
         if (!NativeDiskANN) {
             throw new Error('NativeDiskANN binding missing — rebuild requires the cor native module');
         }
+        // **Self-heal a stranded base (CORTEX-CONSOLIDATION-SILENT-DROP).** We hold
+        // `this.native` as a live mmap handle to `indexPath`. On Linux that handle
+        // survives its file being unlinked — so if a crash interrupted a prior
+        // consolidation mid-write (native rebuild unlinks the old base to rewrite
+        // it), or an external move took the file, `this.native` stays truthy while
+        // the PATH is dead. Trusting the handle then makes `rebuildFromExistingAsync`
+        // open a path that no longer exists → `open existing index failed: No such
+        // file or directory`, looping every trigger with segments piling up
+        // unbounded. Drop the stale handle here so the fold below rebuilds from the
+        // durable L0 segments + canonical (the source of truth) and re-creates the
+        // base, instead of looping on a ghost. `existsSync` is O(1); this runs only
+        // on the (rare) consolidation path, never per-write.
+        if (this.native && !existsSync(this.config.indexPath)) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — crash-stranded or externally moved. ` +
+                `Rebuilding from segments + canonical instead of looping on the dead path.`);
+            this.native = null;
+            // A rebuild is about to regenerate the base; clear any failed-load flag.
+            this.durableBaseLoadFailed = false;
+        }
         // Build the new logical slot ordering: (live old slots) + (delta).
         // **Critical for billion-scale correctness**: the old vectors stay
         // mmap'd inside the native module — we only pass slot IDs across
@@ -1198,6 +1285,10 @@ export class NativeDiskAnnWrapper {
         // The rebuild's durable output now covers canonical — the cold-open
         // not-ready verdict (migration/restore dir) is satisfied.
         this.canonicalUnindexed = false;
+        // A successful fold clears the consolidation-failure backoff (any caller:
+        // the auto-trigger, brainy's cold-open gate, or a host repairIndex).
+        this.consolidationFailures = 0;
+        this.consolidationCooldownUntil = 0;
     }
     /**
      * Flush the delta buffer to disk. For DiskANN the delta is in-memory
@@ -1462,13 +1553,42 @@ export class NativeDiskAnnWrapper {
         }
         prodLog.info(`NativeDiskAnnWrapper: flushed ${builtDelta.size} delta entries to L0 segment ${file} ` +
             `(${this.l0Segments.length} live segments)`);
+        // Integrity probe (CORTEX-CONSOLIDATION-SILENT-DROP hardening): one O(1)
+        // stat per flush catches a base deleted out from under the live handle in
+        // SECONDS instead of at the next consolidation — the strand ran for hours
+        // undetected on memory-vm. Detection here; the heal is the consolidation
+        // below (runRebuild's self-heal drops the stale handle and rebuilds from
+        // segments + canonical).
+        const baseMissing = this.native !== null &&
+            this.config.indexPath !== undefined &&
+            !existsSync(this.config.indexPath);
+        if (baseMissing) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — triggering self-heal consolidation`);
+        }
+        if (this.l0Segments.length >= L0_SEGMENT_WARN_THRESHOLD) {
+            prodLog.error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments ` +
+                `(consolidation folds at ${L0_COMPACT_THRESHOLD}) — consolidation is ` +
+                `failing; query fan-out is degrading and the write brake engages at ` +
+                `${L0_SEGMENT_HARD_CAP}`);
+        }
         // Tiered maintenance: enough small segments → one BACKGROUND consolidation
         // (the async rebuild folds base + L0s + delta into a fresh base off-thread).
-        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD && !this.rebuildInFlight) {
+        // A missing base skips the segment-count threshold — heal now, not three
+        // flushes from now. Guarded by an exponential backoff: a persistently-
+        // failing consolidation must not re-fire on every ~6 s flush (memory-vm
+        // hammered it for hours) — it backs off up to CONSOLIDATION_BACKOFF_MAX_MS,
+        // and a success resets it (see the tail of runRebuild).
+        if ((baseMissing || this.l0Segments.length >= L0_COMPACT_THRESHOLD) &&
+            !this.rebuildInFlight &&
+            Date.now() >= this.consolidationCooldownUntil) {
             this.rebuild().catch((error) => {
-                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed (segments ` +
-                    `retained; retries at the next trigger): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consolidationFailures++;
+                const backoffMs = Math.min(CONSOLIDATION_BACKOFF_MAX_MS, 1000 * 2 ** Math.min(this.consolidationFailures, 8));
+                this.consolidationCooldownUntil = Date.now() + backoffMs;
+                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed ` +
+                    `(${this.consolidationFailures}× — next attempt in ~${Math.round(backoffMs / 1000)}s; ` +
+                    `segments retained): ${error instanceof Error ? error.message : String(error)}`);
             });
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.8",
+  "version": "3.0.10",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",