@soulcraft/cor 3.0.9 → 3.0.10

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -127,6 +127,24 @@ const FLUSH_FAILURE_BACKPRESSURE_THRESHOLD = 4;
  * to this ceiling. A successful consolidation resets to zero.
  */
 const CONSOLIDATION_BACKOFF_MAX_MS = 5 * 60_000;
+/**
+ * Live-L0-segment count that triggers a loud warning: consolidation folds at
+ * {@link L0_COMPACT_THRESHOLD}, so accumulating this many segments means it
+ * has been failing for many cycles. Every query fans out across all live
+ * segments plus the base — memory-vm's incident reached 162 segments and
+ * degraded reads to 25–40 s. The warn is the early alarm; the hard cap below
+ * is the brake.
+ */
+const L0_SEGMENT_WARN_THRESHOLD = 32;
+/**
+ * Live-L0-segment count at which `addItem` REFUSES writes loudly
+ * (CORTEX-CONSOLIDATION-SILENT-DROP hardening). Segments grow past the warn
+ * threshold only while consolidation persistently fails; each one adds query
+ * fan-out and RAM until reads collapse and the process OOMs. Same posture as
+ * the flush-failure brake: a loud bounded stop beats an unbounded quiet
+ * degradation. Clears itself the moment one consolidation folds the segments.
+ */
+const L0_SEGMENT_HARD_CAP = 256;
 /**
  * Storage key of the vector segment manifest (flat key — slashed keys trip
  * the storage adapters' unknown-key warning; same lesson as the metadata
@@ -434,6 +452,18 @@ export class NativeDiskAnnWrapper {
      * `rebuild()` call, which folds the delta into the main index.
      */
     async addItem(item) {
+        // Segment brake (CORTEX-CONSOLIDATION-SILENT-DROP): segments accumulate
+        // past the warn threshold only while consolidation persistently fails —
+        // the delta keeps draining to NEW segments (flush works), so the flush-
+        // failure brake never engages, yet fan-out and RAM grow without bound
+        // (162 segments degraded memory-vm's reads to 25–40 s en route to OOM).
+        // Refuse loudly at the cap; one successful consolidation clears it.
+        if (this.l0Segments.length >= L0_SEGMENT_HARD_CAP) {
+            throw new Error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments — ` +
+                `consolidation is persistently failing; refusing writes to prevent ` +
+                `unbounded segment growth (read collapse / OOM). See the ` +
+                `consolidation-failure log lines for the underlying cause.`);
+        }
         // item.vector.length is the authoritative width for THIS write; fall back to
         // it when the brain is zero-config (config.dimensions undefined). Otherwise
         // bytesPerVector is NaN and every `NaN >= cap` backpressure/fold check is
@@ -1523,13 +1553,33 @@ export class NativeDiskAnnWrapper {
         }
         prodLog.info(`NativeDiskAnnWrapper: flushed ${builtDelta.size} delta entries to L0 segment ${file} ` +
             `(${this.l0Segments.length} live segments)`);
+        // Integrity probe (CORTEX-CONSOLIDATION-SILENT-DROP hardening): one O(1)
+        // stat per flush catches a base deleted out from under the live handle in
+        // SECONDS instead of at the next consolidation — the strand ran for hours
+        // undetected on memory-vm. Detection here; the heal is the consolidation
+        // below (runRebuild's self-heal drops the stale handle and rebuilds from
+        // segments + canonical).
+        const baseMissing = this.native !== null &&
+            this.config.indexPath !== undefined &&
+            !existsSync(this.config.indexPath);
+        if (baseMissing) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — triggering self-heal consolidation`);
+        }
+        if (this.l0Segments.length >= L0_SEGMENT_WARN_THRESHOLD) {
+            prodLog.error(`NativeDiskAnnWrapper: ${this.l0Segments.length} live L0 segments ` +
+                `(consolidation folds at ${L0_COMPACT_THRESHOLD}) — consolidation is ` +
+                `failing; query fan-out is degrading and the write brake engages at ` +
+                `${L0_SEGMENT_HARD_CAP}`);
+        }
         // Tiered maintenance: enough small segments → one BACKGROUND consolidation
         // (the async rebuild folds base + L0s + delta into a fresh base off-thread).
-        // Guarded by an exponential backoff: a persistently-failing consolidation
-        // must not re-fire on every ~6 s flush (memory-vm hammered it for hours) —
-        // it backs off up to CONSOLIDATION_BACKOFF_MAX_MS, and a success resets it
-        // (see the tail of runRebuild). CORTEX-CONSOLIDATION-SILENT-DROP.
-        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD &&
+        // A missing base skips the segment-count threshold — heal now, not three
+        // flushes from now. Guarded by an exponential backoff: a persistently-
+        // failing consolidation must not re-fire on every ~6 s flush (memory-vm
+        // hammered it for hours) — it backs off up to CONSOLIDATION_BACKOFF_MAX_MS,
+        // and a success resets it (see the tail of runRebuild).
+        if ((baseMissing || this.l0Segments.length >= L0_COMPACT_THRESHOLD) &&
             !this.rebuildInFlight &&
             Date.now() >= this.consolidationCooldownUntil) {
             this.rebuild().catch((error) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.9",
+  "version": "3.0.10",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",