@soulcraft/cor 3.0.0-rc.1

package/dist/native/types.js

@@ -0,0 +1,16 @@
+/**
+ * Shared types for the native module integration.
+ *
+ * These types are used by both the napi-rs and bun:ffi loaders.
+ * Ported from src/embeddings/wasm/types.ts — keeps the same API surface.
+ */
+/**
+ * Model constants for all-MiniLM-L6-v2
+ */
+export const MODEL_CONSTANTS = {
+    HIDDEN_SIZE: 384,
+    MAX_SEQUENCE_LENGTH: 256,
+    VOCAB_SIZE: 30522,
+    MODEL_NAME: 'all-MiniLM-L6-v2',
+};
+//# sourceMappingURL=types.js.map

package/dist/plugin.d.ts

@@ -0,0 +1,50 @@
+/**
+ * @module plugin
+ * @description `@soulcraft/cor` Brainy plugin entry point.
+ *
+ * Implements the `BrainyPlugin` interface to register every native
+ * provider cor 3.0 ships. Brainy auto-detects this plugin during
+ * `init()` via dynamic import.
+ *
+ * ## Tier gating
+ *
+ * - **Free tier (no license key)**: nothing registered — brainy runs
+ *   on its pure-JS defaults with zero acceleration.
+ * - **Pro / Enterprise / Internal**: every compute provider below
+ *   wired in. Cor is a pure 1:1 accelerator — it registers ONLY
+ *   the providers brainy currently looks up; native capabilities
+ *   that don't yet have a brainy hook stay dormant in the cdylib.
+ *
+ * ## Cor 3.0 surface (Piece F, 2026-06-09)
+ *
+ * | Provider          | Implementation                              |
+ * |-------------------|---------------------------------------------|
+ * | `distance`        | SIMD-accelerated cosine                     |
+ * | `distance:sq8`    | SQ8 quantized cosine (uint8)                |
+ * | `distance:sq4`    | SQ4 quantized cosine (packed nibbles)       |
+ * | `graph:compression` | delta-varint connection codec             |
+ * | `hnsw`            | **Adaptive DiskANN** (`NativeDiskAnnWrapper`) — the vector index. The provider key is brainy's historical name; cor's implementation is DiskANN |
+ * | `diskann`         | Same `NativeDiskAnnWrapper` under brainy's explicit DiskANN key (for callers that set `config.index.type='diskann'`) |
+ * | `cache`           | Native eviction engine (`NativeUnifiedCacheWrapper`) |
+ * | `metadataIndex`   | Native LSM-backed metadata index + column store |
+ * | `graphIndex`      | Native graph adjacency (u64-keyed LSM trees) |
+ * | `embeddings`      | Candle ML (CPU/CUDA/Metal)                  |
+ * | `embedBatch`      | Single-forward-pass batch embeddings        |
+ * | `entityIdMapper`  | mmap binary mapper or JSON fallback         |
+ * | `roaring`         | CRoaring bindings (Roaring32 + Roaring64)   |
+ * | `msgpack`         | Native encode/decode                        |
+ * | `aggregation`     | Rust-accelerated incremental aggregation    |
+ * | `sort:topK`       | `O(N log K)` bounded-heap ranking           |
+ *
+ * The legacy native HNSW implementation (cortex 2.x) and its
+ * companion mmap vector store + BVEC reader were removed in
+ * Piece F. **Adaptive DiskANN** covers the same workload range
+ * with a single code path (in-memory / hybrid / on-disk modes
+ * selected at brain-open time from observed memory).
+ */
+import type { BrainyPlugin } from '@soulcraft/brainy/plugin';
+declare const corPlugin: BrainyPlugin & {
+    brainyRange?: string;
+};
+export default corPlugin;
+//# sourceMappingURL=plugin.d.ts.map

package/dist/plugin.js

@@ -0,0 +1,388 @@
+/**
+ * @module plugin
+ * @description `@soulcraft/cor` Brainy plugin entry point.
+ *
+ * Implements the `BrainyPlugin` interface to register every native
+ * provider cor 3.0 ships. Brainy auto-detects this plugin during
+ * `init()` via dynamic import.
+ *
+ * ## Tier gating
+ *
+ * - **Free tier (no license key)**: nothing registered — brainy runs
+ *   on its pure-JS defaults with zero acceleration.
+ * - **Pro / Enterprise / Internal**: every compute provider below
+ *   wired in. Cor is a pure 1:1 accelerator — it registers ONLY
+ *   the providers brainy currently looks up; native capabilities
+ *   that don't yet have a brainy hook stay dormant in the cdylib.
+ *
+ * ## Cor 3.0 surface (Piece F, 2026-06-09)
+ *
+ * | Provider          | Implementation                              |
+ * |-------------------|---------------------------------------------|
+ * | `distance`        | SIMD-accelerated cosine                     |
+ * | `distance:sq8`    | SQ8 quantized cosine (uint8)                |
+ * | `distance:sq4`    | SQ4 quantized cosine (packed nibbles)       |
+ * | `graph:compression` | delta-varint connection codec             |
+ * | `hnsw`            | **Adaptive DiskANN** (`NativeDiskAnnWrapper`) — the vector index. The provider key is brainy's historical name; cor's implementation is DiskANN |
+ * | `diskann`         | Same `NativeDiskAnnWrapper` under brainy's explicit DiskANN key (for callers that set `config.index.type='diskann'`) |
+ * | `cache`           | Native eviction engine (`NativeUnifiedCacheWrapper`) |
+ * | `metadataIndex`   | Native LSM-backed metadata index + column store |
+ * | `graphIndex`      | Native graph adjacency (u64-keyed LSM trees) |
+ * | `embeddings`      | Candle ML (CPU/CUDA/Metal)                  |
+ * | `embedBatch`      | Single-forward-pass batch embeddings        |
+ * | `entityIdMapper`  | mmap binary mapper or JSON fallback         |
+ * | `roaring`         | CRoaring bindings (Roaring32 + Roaring64)   |
+ * | `msgpack`         | Native encode/decode                        |
+ * | `aggregation`     | Rust-accelerated incremental aggregation    |
+ * | `sort:topK`       | `O(N log K)` bounded-heap ranking           |
+ *
+ * The legacy native HNSW implementation (cortex 2.x) and its
+ * companion mmap vector store + BVEC reader were removed in
+ * Piece F. **Adaptive DiskANN** covers the same workload range
+ * with a single code path (in-memory / hybrid / on-disk modes
+ * selected at brain-open time from observed memory).
+ */
+import { loadNativeModule, isNativeAvailable } from './native/index.js';
+import { validateLicense, readLicenseToken, readConfig, isShortCode } from './license.js';
+import { MigrationCoordinator } from './migration/MigrationCoordinator.js';
+const PRICING_URL = 'https://soulcraft.com/pricing?focus=pro';
+// `brainyRange` engages brainy 8.0's version-coupling gate (handoff LV.3): a
+// brainy outside this range fails loud at init() instead of silently dropping to
+// the JS engine. Typed via intersection until the devDependency moves to brainy
+// 8.0 (whose `BrainyPlugin` declares the field); brainy reads the runtime
+// property regardless of cor's compile-time type. `'vector'` provider key is
+// registered below (the other half of the gate's contract).
+const corPlugin = {
+    name: '@soulcraft/cor',
+    brainyRange: '>=8.0.0 <9.0.0',
+    async activate(context) {
+        // **Safety guard — Cor 3.0 production default.** Refuse to start if
+        // the storage root still contains cortex 2.x on-disk artifacts. The
+        // check is *deferred to the storage-receiving provider factories
+        // below*: brainy 8.0's `activate(context)` does not expose the
+        // brain's storage here (the context carries only `registerProvider`
+        // + `version`), so scanning it now would inspect `undefined` and
+        // silently pass — defeating the safety net. `guardLegacyLayoutOnce`
+        // is called at the top of the metadataIndex / graphIndex /
+        // entityIdMapper factories, which DO receive the real storage, and
+        // fires before any native engine opens files under the root. A loud
+        // "service won't start" failure is recoverable; letting cor 3.0
+        // auto-engagement run alongside 2.x remnants produces a
+        // half-migrated state that's harder to undo. Bypass via env
+        // `COR_LEGACY_LAYOUT_GUARD=skip` (used by the migration script).
+        const { guardLegacyLayoutOnce } = await import('./legacyLayoutGuard.js');
+        // **Piece 19 storage simplification.** Cor does not register
+        // a `storage:filesystem` provider — brainy's built-in
+        // `FileSystemStorage` (which since 7.25.0 ships the binary-blob
+        // primitives + multi-process locking cor used to layer on
+        // top) is the canonical storage adapter. That keeps the
+        // boundary between brainy and cor razor-sharp:
+        //
+        //   - **Brainy** = data + filesystem storage + JS default
+        //     implementations of every operation (MIT open source).
+        //   - **Cor** = drop-in fast replacements for every compute
+        //     hot path (proprietary, license-gated).
+        //
+        // Cor's compute providers (Adaptive DiskANN, metadata index,
+        // graph adjacency, embeddings, etc.) all require local file
+        // paths via `storage.getBinaryBlobPath` — i.e. filesystem
+        // storage. On a cloud adapter the compute providers fail at
+        // init with a clear error rather than silently degrading.
+        // Brainy continues to work unaccelerated on any adapter.
+        // Gate: native module must be available for compute acceleration
+        if (!isNativeAvailable()) {
+            return true;
+        }
+        // Gate: license tier determines compute access
+        const license = await validateLicense();
+        let useOfflineGrace = false;
+        if (license.tier === 'free') {
+            if (license.error) {
+                // 'native module unavailable' — defensive; isNativeAvailable() already gated above
+                console.warn(`[cor] ${license.error}`);
+                return true;
+            }
+            // Distinguish "no key configured" from "short code exchange failed at runtime".
+            // The CLI now writes JWTs directly (sc_cor_...), so short codes in .soulcraft.json
+            // only appear in old or manually-written files. Exchange at runtime is a fallback path.
+            //
+            // Resolve the RAW license value with the same env-then-file precedence
+            // readLicenseToken() uses (env var COR_LICENSE_KEY / legacy CORTEX_LICENSE_KEY
+            // wins over .soulcraft.json). We can't reuse readLicenseToken() directly here
+            // because it exchanges short codes into JWTs (or returns null on a failed
+            // exchange), erasing the short-code shape this branch needs to test. Reading
+            // the file alone would drop offline grace for an env-supplied short code on a
+            // network blip — the exact gap this fixes.
+            const envValue = (process.env.COR_LICENSE_KEY ?? process.env.CORTEX_LICENSE_KEY)?.trim();
+            let licenseValue;
+            if (envValue) {
+                licenseValue = envValue;
+            }
+            else {
+                const config = await readConfig();
+                // Dual-read during the cortex→cor rename: new `cor` key, legacy `cortex`.
+                licenseValue = config.cor ?? config.cortex;
+            }
+            if (!licenseValue) {
+                return true; // No license configured — free tier, no warning
+            }
+            if (isShortCode(licenseValue)) {
+                // Exchange failed. Check heartbeat cache: if the license was previously active,
+                // grant offline grace so a transient network issue doesn't break production.
+                const { readHeartbeatCache } = await import('./license/onlineValidator.js');
+                const cached = await readHeartbeatCache();
+                if (cached?.status === 'active') {
+                    const lastCheck = new Date(cached.lastCheck).toLocaleDateString();
+                    console.warn(`[cor] License server unreachable — offline mode (last verified: ${lastCheck}). ` +
+                        `Ensure soulcraft.com is accessible for periodic renewal.`);
+                    useOfflineGrace = true;
+                }
+                else {
+                    console.warn(`[cor] License exchange failed — cannot verify activation code with soulcraft.com.\n` +
+                        `[cor] Native compute acceleration disabled.\n` +
+                        `[cor] Re-run: npx @soulcraft/cor activate <code> (on a machine with internet access)`);
+                    return true;
+                }
+            }
+            else {
+                return true; // JWT present but resolved to free — valid state, no warning
+            }
+        }
+        if (!useOfflineGrace) {
+            // Invalid key (bad signature, wrong product, expired, etc.)
+            if (!license.valid) {
+                if (license.error) {
+                    console.warn(`[cor] ${license.error}. ` +
+                        `Get a valid license at ${PRICING_URL}`);
+                }
+                return true;
+            }
+            // Check cached heartbeat state BEFORE registering compute providers.
+            // This catches an expired grace period from a previous run without a network call.
+            const { readHeartbeatCache, isGracePeriodExpired } = await import('./license/onlineValidator.js');
+            const cachedState = await readHeartbeatCache();
+            if (cachedState?.gracePeriodStart && isGracePeriodExpired(cachedState.gracePeriodStart)) {
+                console.warn('[cor] License lapsed. Compute acceleration disabled. Renew at soulcraft.com/account');
+                return true;
+            }
+        }
+        // Pro / Enterprise / Internal: register ALL native compute providers
+        const native = loadNativeModule();
+        // Distance: SIMD-accelerated cosine.
+        context.registerProvider('distance', native.cosineDistance);
+        // Quantized distance: SQ8 cosine on uint8 arrays (no
+        // dequantization). Used by brainy's SQ8 rerank path.
+        context.registerProvider('distance:sq8', native.cosineDistanceSq8);
+        // Quantized distance: SQ4 cosine on packed nibbles (2 values
+        // per byte). Used by brainy 7.28.0+ when
+        // `config.hnsw.quantization.bits === 4`. Byte-for-byte
+        // identical to brainy's pure-JS `distanceSQ4Js`;
+        // cross-language parity is asserted by brainy's test suite.
+        context.registerProvider('distance:sq4', native.cosineDistanceSq4);
+        // Graph connection compression: delta-varint encoded
+        // connection lists. Brainy uses this on the JS HNSW path it
+        // still owns for the standalone open-core deployment shape;
+        // cor provides the native codec.
+        context.registerProvider('graph:compression', {
+            encode: native.encodeConnections,
+            decode: native.decodeConnections,
+        });
+        // Metadata index: Native Rust query/mutation engine.
+        // Imported + captured BEFORE the vector factory so the DiskANN
+        // wrapper's lazy mapperResolver can close over `metadataIndexInstance`
+        // (the canonical entity-id mapper). The capture is also handed to the
+        // graphIndex factory below — the SAME mapper brainy resolves verb
+        // endpoints through at write time (`metadataIndex.getIdMapper().getOrAssign`);
+        // restored edges (#68) and DiskANN slot↔int resolution (#72) MUST use
+        // THIS one mapper. The resolver is lazy, so factory-call order between
+        // the providers doesn't matter.
+        const { MetadataIndexManager: NativeMetadataIndex } = await import('./utils/NativeMetadataIndex.js');
+        let metadataIndexInstance;
+        // **Adaptive DiskANN** — the sole vector index in cor 3.0.
+        // Registered against brainy 8.0's canonical `'vector'` provider
+        // key (algorithm-neutral; matches `VectorIndexProvider`). The
+        // wrapper self-selects operating mode (in-memory / hybrid /
+        // on-disk) from observed memory pressure per Piece I — no
+        // user-tunable algorithm choice. #72: the wrapper resolves slot↔uuid
+        // through the off-heap mmap slotmaps + the canonical mapper, so it is
+        // wired with the SAME lazy `getIdMapper()` resolver as the graph index.
+        const { NativeDiskAnnWrapper } = await import('./hnsw/NativeDiskAnnWrapper.js');
+        let vectorInstance;
+        const diskAnnFactory = (config, distanceFn, options) => {
+            // Capture the constructed wrapper so the #18 migration coordinator can reach
+            // the vector provider (brainy calls this factory once for the 'vector' key).
+            vectorInstance = new NativeDiskAnnWrapper(config, distanceFn, {
+                ...options,
+                mapperResolver: () => metadataIndexInstance?.getIdMapper(),
+                migrationCoordinator: getMigrationCoordinator,
+            });
+            return vectorInstance;
+        };
+        context.registerProvider('vector', diskAnnFactory);
+        // Cache: Native Rust eviction engine with JS data storage
+        const { NativeUnifiedCacheWrapper } = await import('./utils/NativeUnifiedCache.js');
+        context.registerProvider('cache', new NativeUnifiedCacheWrapper());
+        context.registerProvider('metadataIndex', (storage) => {
+            guardLegacyLayoutOnce(storage);
+            // The metadata index owns the #18 migration trigger: on a large stale brain
+            // its init() calls coordinator.start(). Create/resolve the one coordinator for
+            // this brain here (the context has no storage; the factories do).
+            ensureMigrationCoordinator(storage);
+            metadataIndexInstance = new NativeMetadataIndex(storage, {}, getMigrationCoordinator);
+            return metadataIndexInstance;
+        });
+        // Graph adjacency: Native 4 LSM-trees with verb tracking
+        const { GraphAdjacencyIndex: NativeGraphAdjacencyIndex } = await import('./graph/NativeGraphAdjacencyIndex.js');
+        // **Phase B graph acceleration (ADR-006) — self-wire.** brainy 8.0's
+        // `activate(context)` has no storage and its provider factories are
+        // called with storage only (not the constructed graphIndex), while the
+        // `graphAcceleration` value is feature-detected RAW (the factory is
+        // never invoked). So we register the adapter here, bound to a resolver
+        // the graphIndex factory fills once it builds the native engine — the
+        // adapter wraps that SAME native handle (never a second index). Until
+        // the engine is built + loaded, the adapter reports `isInitialized:
+        // false` and brainy keeps serving graph ops from TS.
+        let graphEngine;
+        // **#18 migration lock (coordinator).** One per brain, created on the first
+        // storage-receiving factory call (memoized) and shared by all three derived-
+        // index providers via the SAME lazy `() => migrationCoordinator` getter so
+        // factory-call order doesn't matter. It drives the blocking 7.x→8.0 upgrade
+        // (rebuild all 3 indexes from canonical under a lock brainy holds reads+writes
+        // behind) and answers the feature-detected isMigrating / migrationStatus
+        // methods each provider delegates to it.
+        let migrationCoordinator;
+        const getMigrationCoordinator = () => migrationCoordinator;
+        const ensureMigrationCoordinator = (storage) => {
+            if (!migrationCoordinator) {
+                migrationCoordinator = new MigrationCoordinator(storage, {
+                    metadata: () => metadataIndexInstance,
+                    vector: () => vectorInstance,
+                    graph: () => graphEngine,
+                });
+            }
+            return migrationCoordinator;
+        };
+        context.registerProvider('graphIndex', (storage) => {
+            guardLegacyLayoutOnce(storage);
+            ensureMigrationCoordinator(storage);
+            // Pass a lazy resolver to the metadata index's entity-id mapper so
+            // graphIndex.rebuild() (restore path, #68) resolves verb endpoints
+            // through the SAME mapper brainy assigns ints with at write time.
+            graphEngine = new NativeGraphAdjacencyIndex(storage, {}, () => metadataIndexInstance?.getIdMapper(), getMigrationCoordinator);
+            return graphEngine;
+        });
+        const { GraphAccelerationAdapter } = await import('./graph/GraphAccelerationAdapter.js');
+        context.registerProvider('graphAcceleration', new GraphAccelerationAdapter(() => graphEngine));
+        // Embeddings: Candle ML native engine — initialized eagerly so the 88MB model
+        // is loaded at startup, not lazily on first embed call. This eliminates the
+        // ~883ms delay on the first embedding request (e.g. during onboarding seeding).
+        const { NativeEmbeddingEngine } = await import('./native/NativeEmbeddingEngine.js');
+        const engine = NativeEmbeddingEngine.getInstance();
+        if (!engine.isInitialized()) {
+            await engine.initialize();
+        }
+        context.registerProvider('embeddings', async (text) => {
+            if (Array.isArray(text)) {
+                return engine.embedBatch(text);
+            }
+            return engine.embed(text);
+        });
+        // Batch embeddings: single Rust forward pass for multiple texts
+        context.registerProvider('embedBatch', async (texts) => {
+            return engine.embedBatch(texts);
+        });
+        // Entity ID mapper: dispatch on storage capability.
+        //
+        //   • Filesystem-backed storage (exposes `getBinaryBlobPath`) →
+        //     the cor 3.0 mmap-backed binary mapper (billion-scale,
+        //     constant-RAM, crash-safe).
+        //   • Cloud / non-path storage → the JSON-persisted mapper.
+        //
+        // Both implement the same brainy `EntityIdMapperProvider` contract;
+        // the choice is transparent to brainy.
+        //
+        // This is a single dispatch, not a fallback chain. Either the
+        // storage has a local path (binary) or it does not (JSON). A
+        // failure constructing the binary mapper on a path-capable storage
+        // is a real bug and surfaces as an init error rather than silently
+        // degrading to JSON (which would mask corruption / config issues
+        // and quietly break the billion-scale guarantee for that node).
+        const { NativeEntityIdMapperWrapper } = await import('./utils/nativeEntityIdMapper.js');
+        const { NativeBinaryEntityIdMapperWrapper } = await import('./utils/nativeBinaryEntityIdMapper.js');
+        context.registerProvider('entityIdMapper', (storage) => {
+            guardLegacyLayoutOnce(storage);
+            if (typeof storage?.getBinaryBlobPath === 'function') {
+                return new NativeBinaryEntityIdMapperWrapper({ storage });
+            }
+            return new NativeEntityIdMapperWrapper({ storage });
+        });
+        // Roaring bitmaps: CRoaring bindings (binary-compatible with roaring-wasm)
+        const { RoaringBitmap32 } = await import('./native/NativeRoaringBitmap32.js');
+        context.registerProvider('roaring', RoaringBitmap32);
+        // Msgpack: Native encode/decode
+        context.registerProvider('msgpack', {
+            encode: native.msgpackEncode,
+            decode: native.msgpackDecode
+        });
+        // Aggregation: Rust-accelerated incremental aggregation.
+        // Precise MIN/MAX + exact percentile/distinctCount
+        // (value-multiset, never stale after deletes), HAVING,
+        // array-unnest groupBy, Rayon parallel rebuild. Registering
+        // the `'aggregation'` provider makes brainy's `AggregationIndex`
+        // delegate all metric computation to Rust.
+        const { NativeAggregationProvider } = await import('./aggregation/NativeAggregationEngine.js');
+        context.registerProvider('aggregation', new NativeAggregationProvider());
+        // Top-K result ranking: `O(N log K)` bounded-heap selection.
+        // Returns the indices of the best K scores in ranked order
+        // so brainy can rank a large candidate set (e.g. a 1M-result
+        // search with `limit=10`) without an `O(N log N)` full sort.
+        context.registerProvider('sort:topK', native.sortTopKScores);
+        // Resource Manager: adaptive memory budgeting for multi-instance deployments.
+        // Tracks active brainy instances, observes system resources (RAM, cgroups),
+        // and dynamically adjusts cache sizing. Zero-config — auto-detects VM size.
+        const { ResourceManager } = await import('./resource/ResourceManager.js');
+        const { getGlobalCache } = await import('@soulcraft/brainy/internals');
+        const resourceManager = ResourceManager.getInstance();
+        // Connect ResourceManager → UnifiedCache: when instance count changes,
+        // ResourceManager recalculates the cache budget and resizes dynamically.
+        const globalCache = getGlobalCache();
+        if (typeof globalCache.setMaxSize === 'function') {
+            resourceManager.setCacheResizeCallback((newSize) => {
+                globalCache.setMaxSize(newSize);
+            });
+        }
+        // ResourceManager runs internally (adaptive cache sizing, wired above) — it is
+        // not registered as a provider since brainy has no consumer for it.
+        resourceManager.startRebalancing();
+        // Background heartbeat with forceCheck — always checks Portal on startup.
+        // Updates the cache so the next startup sees current subscription state.
+        // Runs after providers are registered (non-blocking for the user).
+        // Skipped in offline grace mode — we have no JWT to send.
+        if (!useOfflineGrace) {
+            const token = await readLicenseToken();
+            if (token) {
+                import('./license/onlineValidator.js')
+                    .then(({ performHeartbeat }) => performHeartbeat(token, { forceCheck: true }))
+                    .then(result => {
+                    if (result.degrade) {
+                        const msg = result.warning ?? 'License lapsed. Compute acceleration disabled. Renew at soulcraft.com/account';
+                        console.warn(`[cor] ${msg}`);
+                    }
+                    else if (result.warning) {
+                        console.warn(`[cor] ${result.warning}`);
+                    }
+                })
+                    .catch(() => {
+                    // Silently ignore heartbeat errors
+                });
+            }
+        }
+        return true;
+    },
+    async deactivate() {
+        // Native resources are cleaned up by the Rust drop() mechanism
+        // when the process exits. No explicit cleanup needed.
+    }
+};
+export default corPlugin;
+//# sourceMappingURL=plugin.js.map