@soulcraft/cor 3.0.7 → 3.0.9

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -209,6 +209,14 @@ export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
     private manifestNextId;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     private segmentsLoaded;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    private consecutiveFlushFailures;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    private consolidationFailures;
+    private consolidationCooldownUntil;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -110,6 +110,23 @@ const DELTA_HARD_CAP_BYTES = 256 * 1024 * 1024;
  * consolidation reads them back via `readVectorChunk` — bounded RAM.
  */
 const L0_COMPACT_THRESHOLD = 4;
+/**
+ * Consecutive backpressure-flush failures before `addItem` stops swallowing and
+ * REFUSES writes (CORTEX-CONSOLIDATION-SILENT-DROP). A committed write must not
+ * fail on a transient post-commit blip — but if the flush PERSISTENTLY fails,
+ * the delta can never drain and accepting more writes grows it without bound
+ * until the process OOMs (which is what stranded memory's base). Past this
+ * count the brake engages: refuse loudly so the write-storm halts instead of
+ * crashing. A single successful flush resets the counter.
+ */
+const FLUSH_FAILURE_BACKPRESSURE_THRESHOLD = 4;
+/**
+ * Cap on the exponential backoff between failed background consolidations
+ * (CORTEX-CONSOLIDATION-SILENT-DROP). A persistently-failing consolidation must
+ * not retry on every ~6 s flush (memory-vm hammered it for hours); back off up
+ * to this ceiling. A successful consolidation resets to zero.
+ */
+const CONSOLIDATION_BACKOFF_MAX_MS = 5 * 60_000;
 /**
  * Storage key of the vector segment manifest (flat key — slashed keys trip
  * the storage adapters' unknown-key warning; same lesson as the metadata
@@ -311,6 +328,14 @@ export class NativeDiskAnnWrapper {
     manifestNextId = 1;
     /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
     segmentsLoaded = false;
+    /** Consecutive backpressure-flush failures (see
+     * {@link FLUSH_FAILURE_BACKPRESSURE_THRESHOLD}); reset on a clean flush. */
+    consecutiveFlushFailures = 0;
+    /** Consecutive background-consolidation failures + the earliest time the next
+     * attempt may fire — exponential backoff so a stranded consolidation is not
+     * re-triggered on every flush (CORTEX-CONSOLIDATION-SILENT-DROP). */
+    consolidationFailures = 0;
+    consolidationCooldownUntil = 0;
     /**
      * Canonical storage holds vectored nouns but NO durable vector index exists —
      * the 7.x → 3.0 migration / restore-dir shape. Detected once per cold open by
@@ -426,17 +451,29 @@ export class NativeDiskAnnWrapper {
             // its retirement filter and the serialized manifest saves keep the
             // on-disk list consistent).
             //
-            // A committed write must NEVER fail on post-commit index maintenance — the
+            // A committed write must not fail on a TRANSIENT post-commit blip — the
             // caller would retry and double-write (MEMORY-COR-RANGE-ERROR isolation
-            // ask). A backpressure-flush failure is a loud alarm, not a rejected
-            // write: log it, keep the delta for the next trigger, accept the write.
+            // ask), so a one-off flush failure is logged and the write is accepted,
+            // delta retained. BUT this is pre-delta-append backpressure: if the flush
+            // PERSISTENTLY fails the delta can never drain, and swallowing forever
+            // grows it without bound until OOM — which is exactly what stranded
+            // memory-vm's base (CORTEX-CONSOLIDATION-SILENT-DROP). Past the threshold
+            // the brake engages: refuse loudly so the write-storm halts. Recovers
+            // itself once a flush succeeds (self-heal drops a stranded base).
             try {
                 await (this.flushInFlight ?? this.flushDeltaToL0());
+                this.consecutiveFlushFailures = 0;
             }
             catch (error) {
-                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed (write accepted; ` +
-                    `delta retained for retry): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consecutiveFlushFailures++;
+                const msg = error instanceof Error ? error.message : String(error);
+                prodLog.error(`NativeDiskAnnWrapper: backpressure flush failed ` +
+                    `(${this.consecutiveFlushFailures}× consecutively): ${msg}`);
+                if (this.consecutiveFlushFailures >= FLUSH_FAILURE_BACKPRESSURE_THRESHOLD) {
+                    throw new Error(`NativeDiskAnnWrapper: vector index flush has failed ` +
+                        `${this.consecutiveFlushFailures}× consecutively — refusing writes to ` +
+                        `prevent unbounded delta growth (OOM). Last cause: ${msg}`);
+                }
             }
         }
         if (this.tombstones.has(item.id)) {
@@ -933,6 +970,26 @@ export class NativeDiskAnnWrapper {
         if (!NativeDiskANN) {
             throw new Error('NativeDiskANN binding missing — rebuild requires the cor native module');
         }
+        // **Self-heal a stranded base (CORTEX-CONSOLIDATION-SILENT-DROP).** We hold
+        // `this.native` as a live mmap handle to `indexPath`. On Linux that handle
+        // survives its file being unlinked — so if a crash interrupted a prior
+        // consolidation mid-write (native rebuild unlinks the old base to rewrite
+        // it), or an external move took the file, `this.native` stays truthy while
+        // the PATH is dead. Trusting the handle then makes `rebuildFromExistingAsync`
+        // open a path that no longer exists → `open existing index failed: No such
+        // file or directory`, looping every trigger with segments piling up
+        // unbounded. Drop the stale handle here so the fold below rebuilds from the
+        // durable L0 segments + canonical (the source of truth) and re-creates the
+        // base, instead of looping on a ghost. `existsSync` is O(1); this runs only
+        // on the (rare) consolidation path, never per-write.
+        if (this.native && !existsSync(this.config.indexPath)) {
+            prodLog.error(`NativeDiskAnnWrapper: durable base index missing under a live handle ` +
+                `(${this.config.indexPath}) — crash-stranded or externally moved. ` +
+                `Rebuilding from segments + canonical instead of looping on the dead path.`);
+            this.native = null;
+            // A rebuild is about to regenerate the base; clear any failed-load flag.
+            this.durableBaseLoadFailed = false;
+        }
         // Build the new logical slot ordering: (live old slots) + (delta).
         // **Critical for billion-scale correctness**: the old vectors stay
         // mmap'd inside the native module — we only pass slot IDs across
@@ -1198,6 +1255,10 @@ export class NativeDiskAnnWrapper {
         // The rebuild's durable output now covers canonical — the cold-open
         // not-ready verdict (migration/restore dir) is satisfied.
         this.canonicalUnindexed = false;
+        // A successful fold clears the consolidation-failure backoff (any caller:
+        // the auto-trigger, brainy's cold-open gate, or a host repairIndex).
+        this.consolidationFailures = 0;
+        this.consolidationCooldownUntil = 0;
     }
     /**
      * Flush the delta buffer to disk. For DiskANN the delta is in-memory
@@ -1464,11 +1525,20 @@ export class NativeDiskAnnWrapper {
             `(${this.l0Segments.length} live segments)`);
         // Tiered maintenance: enough small segments → one BACKGROUND consolidation
         // (the async rebuild folds base + L0s + delta into a fresh base off-thread).
-        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD && !this.rebuildInFlight) {
+        // Guarded by an exponential backoff: a persistently-failing consolidation
+        // must not re-fire on every ~6 s flush (memory-vm hammered it for hours) —
+        // it backs off up to CONSOLIDATION_BACKOFF_MAX_MS, and a success resets it
+        // (see the tail of runRebuild). CORTEX-CONSOLIDATION-SILENT-DROP.
+        if (this.l0Segments.length >= L0_COMPACT_THRESHOLD &&
+            !this.rebuildInFlight &&
+            Date.now() >= this.consolidationCooldownUntil) {
             this.rebuild().catch((error) => {
-                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed (segments ` +
-                    `retained; retries at the next trigger): ` +
-                    `${error instanceof Error ? error.message : String(error)}`);
+                this.consolidationFailures++;
+                const backoffMs = Math.min(CONSOLIDATION_BACKOFF_MAX_MS, 1000 * 2 ** Math.min(this.consolidationFailures, 8));
+                this.consolidationCooldownUntil = Date.now() + backoffMs;
+                prodLog.error(`NativeDiskAnnWrapper: background consolidation failed ` +
+                    `(${this.consolidationFailures}× — next attempt in ~${Math.round(backoffMs / 1000)}s; ` +
+                    `segments retained): ${error instanceof Error ? error.message : String(error)}`);
             });
         }
     }

package/dist/native/index.d.ts

@@ -8,6 +8,11 @@
  * Exports the unified NativeBindings interface that all consumers use.
  */
 import type { NativeBindings } from './types.js';
+/**
+ * Diagnostic accessor for the native-module load failure. `null` when the
+ * module loaded successfully or no load has been attempted yet.
+ */
+export declare function getNativeLoadError(): Error | null;
 /**
  * Open-file-limit raise result, captured once when the native module
  * first loads. `null` until then (or if the platform/loader doesn't
@@ -39,7 +44,10 @@ export declare function getOpenFileLimitInfo(): OpenFileLimitInfo | null;
 export declare function loadNativeModule(): NativeBindings;
 /**
  * Check if the native module is available.
- * Does not throw - returns false if loading fails.
+ * Does not throw — returns false if loading fails, and logs the underlying
+ * load error ONCE per process. cor exists to accelerate; a native module
+ * that fails to load must never be a silent no (the dlopen cause — e.g. a
+ * glibc version mismatch — is the only actionable diagnostic).
  */
 export declare function isNativeAvailable(): boolean;
 export type { NativeBindings, NativeEmbeddingEngineInstance, NativeRoaringBitmap32Instance, EmbeddingResult, EngineStats, DeviceInfo, } from './types.js';

package/dist/native/index.js

@@ -11,6 +11,25 @@ import { loadViaNapi } from './napi.js';
 import { loadViaFFI } from './ffi.js';
 // Cached module reference
 let cachedModule = null;
+/**
+ * The error from the most recent failed native-module load, or `null` if the
+ * module loaded (or was never attempted). Carries the underlying dlopen
+ * message — e.g. a glibc version mismatch — which is the only actionable
+ * diagnostic when native acceleration silently fails to engage
+ * (ACC-COR-SILENT-NATIVE-FAIL).
+ */
+let lastLoadError = null;
+/** One-shot latch so the load-failure warning prints once per process, not
+ * once per {@link isNativeAvailable} probe (benchmarks and test gates call
+ * it repeatedly). */
+let loadFailureWarned = false;
+/**
+ * Diagnostic accessor for the native-module load failure. `null` when the
+ * module loaded successfully or no load has been attempted yet.
+ */
+export function getNativeLoadError() {
+    return lastLoadError;
+}
 let fileLimitInfo = null;
 /**
  * Raise the process's soft open-file limit toward its hard cap, once.
@@ -56,6 +75,7 @@ export function loadNativeModule() {
         try {
             cachedModule = loadViaFFI();
             raiseFileLimitOnce(cachedModule);
+            lastLoadError = null;
             return cachedModule;
         }
         catch {
@@ -63,20 +83,35 @@ export function loadNativeModule() {
         }
     }
     // Strategy 2: All platforms (napi-rs)
-    cachedModule = loadViaNapi();
+    try {
+        cachedModule = loadViaNapi();
+    }
+    catch (err) {
+        lastLoadError = err instanceof Error ? err : new Error(String(err));
+        throw err;
+    }
     raiseFileLimitOnce(cachedModule);
+    lastLoadError = null;
     return cachedModule;
 }
 /**
  * Check if the native module is available.
- * Does not throw - returns false if loading fails.
+ * Does not throw — returns false if loading fails, and logs the underlying
+ * load error ONCE per process. cor exists to accelerate; a native module
+ * that fails to load must never be a silent no (the dlopen cause — e.g. a
+ * glibc version mismatch — is the only actionable diagnostic).
  */
 export function isNativeAvailable() {
     try {
         loadNativeModule();
         return true;
     }
-    catch {
+    catch (err) {
+        if (!loadFailureWarned) {
+            loadFailureWarned = true;
+            console.warn(`[cor] native module failed to load — acceleration unavailable: ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
         return false;
     }
 }

package/dist/native/napi.js

@@ -46,30 +46,33 @@ export function loadViaNapi() {
     // This file lives at dist/native/napi.js → package root is ../../
     const __dirname = dirname(fileURLToPath(import.meta.url));
     const nativeDir = join(__dirname, '..', '..', 'native');
+    // Each strategy's real failure is captured and carried in the final throw:
+    // the underlying dlopen error (e.g. "GLIBC_2.39 not found" on an older
+    // distro) is the ONLY actionable diagnostic, and swallowing it produced
+    // native-silently-off deployments (ACC-COR-SILENT-NATIVE-FAIL).
+    const failures = [];
     // Strategy 1: Bundled platform binary (shipped in npm package)
+    const binaryPath = join(nativeDir, `brainy-native.${platformKey}.node`);
     try {
-        const binaryPath = join(nativeDir, `brainy-native.${platformKey}.node`);
         const bindings = require(binaryPath);
         cachedBindings = bindings;
         return bindings;
     }
-    catch {
-        // Fall through to strategy 2
+    catch (err) {
+        failures.push(`  - ${binaryPath}: ${err instanceof Error ? err.message : String(err)}`);
     }
     // Strategy 2: Local dev build (no platform suffix)
+    const devPath = join(nativeDir, 'brainy-native.node');
     try {
-        const devPath = join(nativeDir, 'brainy-native.node');
         const bindings = require(devPath);
         cachedBindings = bindings;
         return bindings;
     }
-    catch {
-        // Fall through to error
+    catch (err) {
+        failures.push(`  - ${devPath}: ${err instanceof Error ? err.message : String(err)}`);
     }
-    throw new Error(`Failed to load Brainy native module for ${key}.\n` +
-        `Looked for:\n` +
-        `  - native/brainy-native.${platformKey}.node (bundled binary)\n` +
-        `  - native/brainy-native.node (local dev build)\n` +
+    throw new Error(`Failed to load Brainy native module for ${key}:\n` +
+        `${failures.join('\n')}\n` +
         `Build locally: cd native && npm run build`);
 }
 /**

package/dist/plugin.js

@@ -42,12 +42,27 @@
  * with a single code path (in-memory / hybrid / on-disk modes
  * selected at brain-open time from observed memory).
  */
-import { loadNativeModule, isNativeAvailable } from './native/index.js';
+import { loadNativeModule, isNativeAvailable, getNativeLoadError } from './native/index.js';
 import { validateLicense, readLicenseToken, readConfig, isShortCode } from './license.js';
 import { MigrationCoordinator } from './migration/MigrationCoordinator.js';
 import { prodLog } from '@soulcraft/brainy/internals';
 import { setEntityCountSource } from './license/onlineValidator.js';
 const PRICING_URL = 'https://soulcraft.com/pricing?focus=pro';
+/**
+ * Resolve the RAW configured license value with the env-then-file precedence
+ * `readLicenseToken()` uses (env `COR_LICENSE_KEY` / legacy `CORTEX_LICENSE_KEY`
+ * wins over `.soulcraft.json`, dual-reading the new `cor` key and the legacy
+ * `cortex` key), WITHOUT exchanging short codes — callers need the raw shape
+ * (is anything configured at all? is it a short code?). `undefined` when no
+ * license is configured anywhere.
+ */
+async function resolveRawLicenseValue() {
+    const envValue = (process.env.COR_LICENSE_KEY ?? process.env.CORTEX_LICENSE_KEY)?.trim();
+    if (envValue)
+        return envValue;
+    const config = await readConfig();
+    return config.cor ?? config.cortex;
+}
 // `brainyRange` engages brainy 8.0's version-coupling gate (handoff LV.3): a
 // brainy outside this range fails loud at init() instead of silently dropping to
 // the JS engine. Typed via intersection until the devDependency moves to brainy
@@ -91,8 +106,21 @@ const corPlugin = {
         // storage. On a cloud adapter the compute providers fail at
         // init with a clear error rather than silently degrading.
         // Brainy continues to work unaccelerated on any adapter.
-        // Gate: native module must be available for compute acceleration
+        // Gate: native module must be available for compute acceleration.
+        // A silent return here with a license configured is the worst failure
+        // shape cor has: the operator PAID for acceleration, the module didn't
+        // load (wrong glibc, missing binary), and every provider quietly ran
+        // brainy-JS with no journal line (ACC-COR-SILENT-NATIVE-FAIL — accord-vm
+        // ran 0/10 native for a day). isNativeAvailable() logs the load error
+        // once; this adds the license-aware alarm.
         if (!isNativeAvailable()) {
+            const configured = await resolveRawLicenseValue();
+            if (configured) {
+                const cause = getNativeLoadError();
+                console.warn(`[cor] a license is configured but the native module FAILED TO LOAD — ` +
+                    `compute acceleration is DISABLED and brainy runs on its JS engines. ` +
+                    `Cause: ${cause?.message ?? 'unknown (no load error recorded)'}`);
+            }
             return true;
         }
         // Gate: license tier determines compute access
@@ -108,23 +136,12 @@ const corPlugin = {
             // The CLI now writes JWTs directly (sc_cor_...), so short codes in .soulcraft.json
             // only appear in old or manually-written files. Exchange at runtime is a fallback path.
             //
-            // Resolve the RAW license value with the same env-then-file precedence
-            // readLicenseToken() uses (env var COR_LICENSE_KEY / legacy CORTEX_LICENSE_KEY
-            // wins over .soulcraft.json). We can't reuse readLicenseToken() directly here
-            // because it exchanges short codes into JWTs (or returns null on a failed
-            // exchange), erasing the short-code shape this branch needs to test. Reading
-            // the file alone would drop offline grace for an env-supplied short code on a
-            // network blip — the exact gap this fixes.
-            const envValue = (process.env.COR_LICENSE_KEY ?? process.env.CORTEX_LICENSE_KEY)?.trim();
-            let licenseValue;
-            if (envValue) {
-                licenseValue = envValue;
-            }
-            else {
-                const config = await readConfig();
-                // Dual-read during the cortex→cor rename: new `cor` key, legacy `cortex`.
-                licenseValue = config.cor ?? config.cortex;
-            }
+            // We can't reuse readLicenseToken() here because it exchanges short codes
+            // into JWTs (or returns null on a failed exchange), erasing the
+            // short-code shape this branch needs to test. Reading the file alone
+            // would drop offline grace for an env-supplied short code on a network
+            // blip — the exact gap this fixes.
+            const licenseValue = await resolveRawLicenseValue();
             if (!licenseValue) {
                 return true; // No license configured — free tier, no warning
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cor",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -22,7 +22,7 @@
     "build:native": "cd native && napi build --release",
     "build:native:dev": "cd native && napi build",
     "prepublishOnly": "npm run build && cd native && napi prepublish -t npm",
-    "test": "vitest run",
+    "test": "vitest run --exclude src/native/lsmPerfGates.test.ts && vitest run --no-file-parallelism src/native/lsmPerfGates.test.ts",
     "bench": "vitest bench",
     "bench:compare": "npx tsx src/benchmarks/comparison.ts",
     "test:ci-native": "node --input-type=module -e \"import { loadNativeModule } from './dist/native/index.js'; const n = loadNativeModule(); console.log('Native module loaded:', Object.keys(n).length, 'exports');\"",
@@ -82,7 +82,7 @@
   },
   "devDependencies": {
     "@napi-rs/cli": "^3.0.0",
-    "@soulcraft/brainy": "8.0.12",
+    "@soulcraft/brainy": "8.2.2",
     "@types/node": "^22.0.0",
     "pg": "^8.21.0",
     "tsx": "^4.21.0",