@soulcraft/cor 3.0.8 → 3.0.9

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -209,6 +209,14 @@ export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
     private manifestNextId;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     private segmentsLoaded;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    private consecutiveFlushFailures;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    private consolidationFailures;
+    private consolidationCooldownUntil;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -110,6 +110,23 @@ const DELTA_HARD_CAP_BYTES = 256 * 1024 * 1024;
  * consolidation reads them back via `readVectorChunk` — bounded RAM.
  */
 const L0_COMPACT_THRESHOLD = 4;
+/**
+ * Consecutive backpressure-flush failures before `addItem` stops swallowing and
+ * REFUSES writes (CORTEX-CONSOLIDATION-SILENT-DROP). A committed write must not
+ * fail on a transient post-commit blip — but if the flush PERSISTENTLY fails,
+ * the delta can never drain and accepting more writes grows it without bound
+ * until the process OOMs (which is what stranded memory's base). Past this
+ * count the brake engages: refuse loudly so the write-storm halts instead of
+ * crashing. A single successful flush resets the counter.
+ */
+const FLUSH_FAILURE_BACKPRESSURE_THRESHOLD = 4;
+/**
+ * Cap on the exponential backoff between failed background consolidations
+ * (CORTEX-CONSOLIDATION-SILENT-DROP). A persistently-failing consolidation must
+ * not retry on every ~6 s flush (memory-vm hammered it for hours); back off up
+ * to this ceiling. A successful consolidation resets to zero.
+ */
+const CONSOLIDATION_BACKOFF_MAX_MS = 5 * 60_000;
 /**
  * Storage key of the vector segment manifest (flat key — slashed keys trip
  * the storage adapters' unknown-key warning; same lesson as the metadata
@@ -311,6 +328,14 @@ export class NativeDiskAnnWrapper {
     manifestNextId = 1;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     segmentsLoaded = false;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    consecutiveFlushFailures = 0;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    consolidationFailures = 0;
+    consolidationCooldownUntil = 0;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by
@@ -426,17 +451,29 @@ export class NativeDiskAnnWrapper {
             // its retirement filter and the serialized manifest saves keep the
             // on-disk list consistent).
             //
-            // A committed write must NEVER fail on post-commit index maintenance — the
+            // A committed write must not fail on a TRANSIENT post-commit blip — the
             // caller would retry and double-write (MEMORY-COR-RANGE-ERROR isolation
-            // ask). A backpressure-flush failure is a loud alarm, not a rejected
-            // write: log it, keep the delta for the next trigger, accept the write.
+            // ask), so a one-off flush failure is logged and the write is accepted,
+            // delta retained. BUT this is pre-delta-append backpressure: if the flush
+            // PERSISTENTLY fails the delta can never drain, and swallowing forever
+            // grows it without bound until OOM — which is exactly what stranded
+            // memory-vm's base (CORTEX-CONSOLIDATION-SILENT-DROP). Past the threshold
+            // the brake engages: refuse loudly so the write-storm halts. Recovers
+            // itself once a flush succeeds (self-heal drops a stranded base).
             try {
                 await (this.flushInFlight ?? this.flushDeltaToL0());
+                this.consecutiveFlushFailures = 0;
             }
             catch (error) {
-                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed (write accepted; ` +
-                    `delta retained for retry): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consecutiveFlushFailures++;
+                const msg = error instanceof Error ? error.message : String(error);
+                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed ` +
+                    `(${this.consecutiveFlushFailures}× consecutively): ${msg}`);
+                if (this.consecutiveFlushFailures >= FLUSH_FAILURE_BACKPRESSURE_THRESHOLD) {
+                    throw new Error(`NativeDiskAnnWrapper: vector index flush has failed ` +
+                        `${this.consecutiveFlushFailures}× consecutively — refusing writes to ` +
+                        `prevent unbounded delta growth (OOM). Last cause: ${msg}`);
+                }
             }
         }
         if (this.tombstones.has(item.id)) {
@@ -933,6 +970,26 @@ export class NativeDiskAnnWrapper {
         if (!NativeDiskANN) {
             throw new Error('NativeDiskANN binding missing — rebuild requires the cor native module');
         }
+        // **Self-heal a stranded base (CORTEX-CONSOLIDATION-SILENT-DROP).** We hold
+        // `this.native` as a live mmap handle to `indexPath`. On Linux that handle
+        // survives its file being unlinked — so if a crash interrupted a prior
+        // consolidation mid-write (native rebuild unlinks the old base to rewrite
+        // it), or an external move took the file, `this.native` stays truthy while
+        // the PATH is dead. Trusting the handle then makes `rebuildFromExistingAsync`
+        // open a path that no longer exists → `open existing index failed: No such
+        // file or directory`, looping every trigger with segments piling up
+        // unbounded. Drop the stale handle here so the fold below rebuilds from the
+        // durable L0 segments + canonical (the source of truth) and re-creates the
+        // base, instead of looping on a ghost. `existsSync` is O(1); this runs only
+        // on the (rare) consolidation path, never per-write.
+        if (this.native && !existsSync(this.config.indexPath)) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — crash-stranded or externally moved. ` +
+                `Rebuilding from segments + canonical instead of looping on the dead path.`);
+            this.native = null;
+            // A rebuild is about to regenerate the base; clear any failed-load flag.
+            this.durableBaseLoadFailed = false;
+        }
         // Build the new logical slot ordering: (live old slots) + (delta).
         // **Critical for billion-scale correctness**: the old vectors stay
         // mmap'd inside the native module — we only pass slot IDs across
@@ -1198,6 +1255,10 @@ export class NativeDiskAnnWrapper {
         // The rebuild's durable output now covers canonical — the cold-open
         // not-ready verdict (migration/restore dir) is satisfied.
         this.canonicalUnindexed = false;
+        // A successful fold clears the consolidation-failure backoff (any caller:
+        // the auto-trigger, brainy's cold-open gate, or a host repairIndex).
+        this.consolidationFailures = 0;
+        this.consolidationCooldownUntil = 0;
     }
     /**
      * Flush the delta buffer to disk. For DiskANN the delta is in-memory
@@ -1464,11 +1525,20 @@ export class NativeDiskAnnWrapper {
             `(${this.l0Segments.length} live segments)`);
         // Tiered maintenance: enough small segments → one BACKGROUND consolidation
         // (the async rebuild folds base + L0s + delta into a fresh base off-thread).
-        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD && !this.rebuildInFlight) {
+        // Guarded by an exponential backoff: a persistently-failing consolidation
+        // must not re-fire on every ~6 s flush (memory-vm hammered it for hours) —
+        // it backs off up to CONSOLIDATION_BACKOFF_MAX_MS, and a success resets it
+        // (see the tail of runRebuild). CORTEX-CONSOLIDATION-SILENT-DROP.
+        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD &&
+            !this.rebuildInFlight &&
+            Date.now() >= this.consolidationCooldownUntil) {
             this.rebuild().catch((error) => {
-                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed (segments ` +
-                    `retained; retries at the next trigger): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consolidationFailures++;
+                const backoffMs = Math.min(CONSOLIDATION_BACKOFF_MAX_MS, 1000 * 2 ** Math.min(this.consolidationFailures, 8));
+                this.consolidationCooldownUntil = Date.now() + backoffMs;
+                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed ` +
+                    `(${this.consolidationFailures}× — next attempt in ~${Math.round(backoffMs / 1000)}s; ` +
+                    `segments retained): ${error instanceof Error ? error.message : String(error)}`);
             });
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.8",
+  "version": "3.0.9",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",