@soulcraft/cor 3.0.0-rc.1

package/dist/utils/NativeMetadataIndex.js

@@ -0,0 +1,2631 @@
+/**
+ * NativeMetadataIndex — TypeScript wrapper around the Rust NativeMetadataIndex.
+ *
+ * Implements the same public API as the old MetadataIndexManager (3,721 lines),
+ * delegating core operations (query, mutation, normalization) to Rust.
+ *
+ * Architecture:
+ * - Rust owns: posting-list algebra, filter evaluation, value normalization,
+ *   field extraction, text search, entity ID mapping, and the LSM engine
+ *   (the sole metadata storage engine — SSTables + manifest)
+ * - TS owns: async storage I/O, LSM SSTable/manifest persistence, lazy
+ *   value-enumeration field-index loading, rebuild orchestration, flush,
+ *   getSortedIdsForFilter (loads sort values from storage)
+ *
+ * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
+ * Rust operates in-memory, returns serialized state for TS to persist.
+ */
+import { existsSync, mkdirSync, rmSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { prodLog, getGlobalCache, FieldTypeInference, resolveEntityField } from '@soulcraft/brainy/internals';
+import { compareCodePoints } from './collation.js';
+/**
+ * Fields whose values are stored in the sparse index as BUCKETED values
+ * (rounded to a coarser granularity to keep the index compact). Sorting
+ * on these fields must bypass the index and read the actual value from
+ * entity storage to avoid precision loss.
+ *
+ * Mirrors BUCKETED_INDEX_FIELDS in brainy's src/utils/metadataIndex.ts —
+ * keep in lockstep when Brainy adds bucketed fields.
+ */
+const BUCKETED_INDEX_FIELDS = new Set([
+    'createdAt',
+    'updatedAt'
+]);
+import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '@soulcraft/brainy/types/graphTypes';
+import { loadNativeModule } from '../native/index.js';
+import { openOrCreateBinaryIdMapper } from './binaryIdMapperFactory.js';
+import { NativeColumnStore } from './NativeColumnStore.js';
+import { ResourceManager } from '../resource/ResourceManager.js';
+import { COR_EXPECTED_INDEX_EPOCH, IndexMigrationRequiredError, detectBrainSize, isEpochStale, readOnDiskBrainFormat, resolveAutoMigrate, resolveLargeBrainThreshold, } from '../migration/indexEpochGuard.js';
+/**
+ * Process-wide counter for unique `MetadataIndexManager` subscriber
+ * names. Each LSM-enabled instance reserves a fresh id so the
+ * `ResourceManager.registerMemoryBudgetSubscriber` slot doesn't collide
+ * across instances in the same process (e.g., multi-tenant deployments
+ * with one brain per workspace).
+ */
+let lsmSubscriberIdCounter = 0;
+function nextLsmSubscriberId() {
+    lsmSubscriberIdCounter += 1;
+    return `cor:metadata-lsm:${lsmSubscriberIdCounter}`;
+}
+/**
+ * Manifest format version this build writes and accepts. Mirrors the Rust
+ * `LSM_MANIFEST_VERSION` constant (native/src/metadata_index/lsm/format.rs).
+ * The wrapper stamps every manifest it writes with this value and refuses
+ * to cold-load a manifest carrying a different one — a forward-incompatible
+ * manifest must fail loud, never be half-read into a partial SSTable set.
+ */
+const LSM_MANIFEST_VERSION = 2;
+/**
+ * Storage key (flat `__metadata_*` system key, same family as
+ * `__metadata_field_registry__`) under which the LSM manifest JSON is
+ * persisted via `storage.saveMetadata` / `storage.getMetadata`.
+ *
+ * @remarks The original durability plan named this `_metadata/manifest.json`,
+ * but a slashed, non-UUID key routes through brainy's storage adapter as an
+ * "Unknown key format" system resource (noisy + fragile). The established
+ * clean pattern for index-level metadata is the flat `__metadata_*` key, so
+ * the manifest rides the exact same atomic `writeCanonicalObject` path the
+ * field registry already depends on.
+ */
+const LSM_MANIFEST_KEY = '__metadata_lsm_manifest__';
+/**
+ * **LSM durability (step 5).** Lower clamp (32 MiB) for the bounded-memtable
+ * size-flush trigger. Even when the dynamic 'metadata' budget is tiny, the
+ * trigger never fires below this — sub-32 MiB SSTables would amplify
+ * read-merge fan-out (every query unions across more on-disk files) without a
+ * matching memory win.
+ */
+const LSM_FLUSH_THRESHOLD_MIN_BYTES = 32 * 1024 * 1024;
+/**
+ * **LSM durability (step 5).** Upper clamp (256 MiB) for the bounded-memtable
+ * size-flush trigger. Caps a single flush's resident peak (and the resulting
+ * SSTable's build cost) even when the dynamic 'metadata' budget is many GiB,
+ * keeping flush latency bounded at billion-entity scale.
+ */
+const LSM_FLUSH_THRESHOLD_MAX_BYTES = 256 * 1024 * 1024;
+/**
+ * **LSM durability (step 5).** Resolve the memtable size-flush threshold from
+ * the engine's current admission ceiling (`allowedBytes`): ~50% of the
+ * ceiling, clamped to `[32 MiB, 256 MiB]`. Draining at half the ceiling
+ * leaves the upper half as headroom so a burst of writes between the trigger
+ * firing and the flush completing cannot reach the admission gate and wedge
+ * the writer — the gate stays a backstop, never the steady-state limiter.
+ *
+ * @param allowedBytes - The engine's current admission ceiling in bytes.
+ * @returns The clamped memtable-drain threshold in bytes.
+ */
+function computeLsmFlushThreshold(allowedBytes) {
+    const half = Math.floor(Math.max(0, allowedBytes) / 2);
+    return Math.min(LSM_FLUSH_THRESHOLD_MAX_BYTES, Math.max(LSM_FLUSH_THRESHOLD_MIN_BYTES, half));
+}
+/**
+ * Build a fresh, empty manifest ready to receive its first SSTable.
+ *
+ * @returns A version-stamped manifest with no SSTables and a zero durable epoch.
+ */
+function newEmptyLsmManifest() {
+    return { version: LSM_MANIFEST_VERSION, sstables: [], last_durable_epoch: 0 };
+}
+/**
+ * Allocate the next SSTable id: one past the highest id currently in the
+ * manifest. SSTables are append-only in this durability slice (compaction —
+ * which removes entries — is a later slice), so `max + 1` is strictly
+ * monotonic and can never collide with a live entry. It is also crash-safe:
+ * if a flush writes the `.sst` but crashes before persisting the manifest,
+ * the orphaned id is simply re-allocated on the next flush (the records
+ * replay from the still-uncommitted memtable log), overwriting a file no
+ * manifest ever referenced.
+ *
+ * @param manifest - The in-memory manifest whose ids bound the allocation.
+ * @returns The next monotonic SSTable id (≥ 1).
+ */
+function nextSstableId(manifest) {
+    let max = 0;
+    for (const e of manifest.sstables) {
+        if (e.id > max)
+            max = e.id;
+    }
+    return max + 1;
+}
+/**
+ * Build the binary-blob storage key for an SSTable. Both the write
+ * (`saveBinaryBlob`) and the mmap-open (`getBinaryBlobPath` →
+ * `registerSstablePath`) go through this single helper, so the logical key is
+ * internally consistent regardless of any extension the adapter appends.
+ *
+ * @param id - The SSTable's monotonic on-disk id.
+ * @returns Key like `_metadata/sstables/1.sst`.
+ */
+function lsmSstableKey(id) {
+    return `_metadata/sstables/${id}.sst`;
+}
+/**
+ * MetadataIndexManager — native Rust implementation with TS storage bridge.
+ *
+ * Drop-in replacement for the old pure-TS MetadataIndexManager.
+ * All bitmap AND/OR/NOT operations execute in Rust without crossing FFI.
+ *
+ * Implements {@link MetadataIndexProvider} so a compile error fires the moment
+ * this wrapper drops or drifts a member brainy calls via `this.metadataIndex.*`.
+ */
+/**
+ * **Piece 11 closeout — init-time gate.** Thrown by
+ * {@link MetadataIndexManager.init} when the brain carries cortex 2.x legacy
+ * JSON metadata state on disk. cor 3.0 uses the LSM engine exclusively (the
+ * legacy JSON chunk engine has been removed), so an un-migrated brain must be
+ * migrated rather than silently switched. Catchable for graceful fallback
+ * paths in callers that prefer a one-shot warning over a hard refusal; the
+ * default behaviour (uncaught) is the right one for production deployments —
+ * silent degradation to an empty LSM engine would mask the on-disk JSON state
+ * and surface as a mysteriously-empty brain at first query.
+ */
+export class MetadataIndexLegacyJsonStateDetected extends Error {
+    name = 'MetadataIndexLegacyJsonStateDetected';
+}
+export class MetadataIndexManager {
+    migrationCoordinatorGetter;
+    storage;
+    config;
+    native;
+    isRebuilding = false;
+    /**
+     * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
+     * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
+     * coordinated migration only AFTER the index is fully constructed. Flagging here
+     * (not kicking inline) avoids racing the migration's destructive rebuild
+     * (`reopenLsmEngineFresh` + `columnStore.reset`) against the remainder of the same
+     * init() (`columnStore.init` / `warmCache` / `lazyLoadCounts`) on shared state.
+     */
+    epochMigrationPending = false;
+    lastFlushTime = Date.now();
+    autoFlushThreshold = 10;
+    dirtyFields = new Set();
+    // Lazy field loading: tracks which fields have been loaded into Rust
+    loadedFields = new Set();
+    // Tracks which field indexes are known (from field registry)
+    knownFields = new Set();
+    /**
+     * **Piece 6 generation counter — reader-side cache.** Records the
+     * generation observed at the most recent `invalidateIfStale` call.
+     * Mutations on the native side bump `core.generation` via
+     * `bump_generation`; a subsequent reader's `invalidateIfStale` call
+     * sees a delta and clears `loadedFields`, forcing the next query to
+     * re-load from the current state instead of trusting the lazy-load
+     * snapshot.
+     *
+     * Default `-1n` means "never observed"; the first call will see any
+     * non-zero current generation and trigger an initial invalidate.
+     * (We don't invalidate on the very first call regardless because
+     * `loadedFields` is empty anyway.)
+     */
+    lastSeenGeneration = -1n;
+    /**
+     * **Piece 15 — per-field generation cache.** Keyed by field name,
+     * value is the per-field generation last observed via
+     * `native.fieldGeneration(field)`. Updated on every
+     * {@link invalidateFieldIfStale} call.
+     *
+     * Selective invalidation contract: when a probe of field X's
+     * counter advances, ONLY X is removed from `loadedFields` — every
+     * other loaded field stays valid. This is the load-bearing
+     * difference vs the index-wide {@link lastSeenGeneration} +
+     * {@link invalidateIfStale} path, which clears the entire
+     * `loadedFields` Set on any mutation.
+     *
+     * Cold-boot: an empty Map. The first probe of any field records
+     * its current native counter without invalidating (matches the
+     * `lastSeenGeneration: -1n` semantic — never-observed implies
+     * loadedFields is empty anyway).
+     */
+    lastSeenFieldGenerations = new Map();
+    // TS-only analytics (not in Rust)
+    fieldStats = new Map();
+    typeFieldAffinity = new Map();
+    totalEntitiesByType = new Map();
+    entityCountsByTypeFixed = new Uint32Array(NOUN_TYPE_COUNT);
+    verbCountsByTypeFixed = new Uint32Array(VERB_TYPE_COUNT);
+    // Unified cache for coordinated memory management
+    unifiedCache;
+    // Field Type Inference
+    fieldTypeInference;
+    /**
+     * **The native LSM engine (cor 3.0 — the sole metadata engine).** Opened
+     * in {@link init} (persistent + mmap-backed on filesystem storage, or a
+     * non-persistent in-RAM engine on non-filesystem adapters) and attached to
+     * `this.native` via `setLsmEngine` so the metadata-index dispatch routes
+     * through the engine's memtable + admission control. `null` only between
+     * construction and {@link openLsmEngine} during `init()`.
+     */
+    lsmEngine = null;
+    /**
+     * **LSM durability (steps 3-4).** In-memory mirror of the on-disk LSM
+     * manifest. `null` in JSON mode and on the non-persistent fallback engine
+     * (where there is no SSTable surface). Set by {@link openLsmEngine} at cold
+     * open (parsed from {@link LSM_MANIFEST_KEY}, or a fresh empty manifest) and
+     * grown by {@link flushLsm} as each memtable drain produces an SSTable.
+     */
+    lsmManifest = null;
+    /**
+     * **LSM durability (steps 3-4).** The resolved on-disk metadata directory
+     * (`getBinaryBlobPath('_metadata')`) backing the persistent engine's
+     * memtable logs. `null` in JSON mode and on the non-persistent fallback.
+     * Its presence is the load-bearing signal that {@link flushLsm} should drain
+     * the memtable to a durable SSTable.
+     */
+    lsmMetaDir = null;
+    /**
+     * **LSM durability (step 5) — bounded-memtable flush trigger.** The memtable
+     * resident-byte ceiling at which {@link addToIndex} proactively drains the
+     * memtable to a durable SSTable via {@link flushLsm}, so the memtable never
+     * climbs to the admission ceiling in steady state (which would wedge every
+     * writer — the failure mode the persistent-open switch would otherwise
+     * introduce). Computed by {@link computeLsmFlushThreshold} as ~50% of the
+     * engine's dynamic `allowedBytes`, clamped to `[32 MiB, 256 MiB]`, and
+     * RECOMPUTED inside the ResourceManager subscriber callback
+     * ({@link attachLsmBudgetSubscriber}) so it tracks the live per-category
+     * budget. Seeded to the floor until the first budget callback fires.
+     */
+    lsmFlushThresholdBytes = LSM_FLUSH_THRESHOLD_MIN_BYTES;
+    /**
+     * Deregistration hook returned by
+     * {@link ResourceManager.registerMemoryBudgetSubscriber}. Called from
+     * {@link close} so the rebalance loop stops firing the engine's
+     * `setAllowedBytes` after this index has shut down. `null` in JSON
+     * mode (no subscriber registered).
+     */
+    unsubscribeFromBudget = null;
+    /**
+     * Unique subscriber name used for both the
+     * `ResourceManager.registerMemoryBudgetSubscriber` call and the
+     * corresponding `reportNativeMemory` callback that drives the
+     * dynamic per-category rebalance. Each MetadataIndexManager instance
+     * gets a fresh name from {@link nextLsmSubscriberId} so multiple
+     * brains in one process don't collide on the same subscriber slot.
+     */
+    lsmSubscriberName = null;
+    /**
+     * Native column store — per-field sorted columns backing brainy's unfiltered
+     * `find({ orderBy })` path. Brainy calls `metadataIndex.columnStore.sortTopK(...)`
+     * directly, so this MUST be present (and is why it is public). Cor's metadata
+     * index does its own native filtering; the column store's jobs here are populate
+     * (on `addToIndex`), persist (on `flush`), and sort (`sortTopK`).
+     */
+    columnStore;
+    /**
+     * Lazily built id-mapper façade over the native UUID ↔ int methods, returned by
+     * {@link getIdMapper}. Cached so brainy's orderBy path reuses one object.
+     */
+    idMapperFacade = null;
+    /**
+     * **#72 Phase C.** The native mmap `BinaryIdMapper` injected into the
+     * Rust metadata core as its canonical UUID ↔ entity-int allocator,
+     * replacing the resident JSON id-maps (~56 GB @1B + the ~13M-UUID JSON
+     * cliff). Built in {@link init} from the brain's `_id_mapper/*` paths and
+     * handed to the core via `native.setIdMapper`. Kept here so {@link flush}
+     * can msync it and a full {@link rebuild} can truncate + re-inject it.
+     * `null` until init wires it (or on non-filesystem storage, where the
+     * mapper cannot engage — the metadata index then has no allocator and
+     * the brain must run unaccelerated).
+     */
+    idMapperNative = null;
+    /**
+     * Set by `getIdMapper().rebuild()` so the immediately-following
+     * `rebuild()` (brainy `restore()` runs metadataIndex.rebuild() in a
+     * `Promise.all` right after the mapper hook) is a no-op: re-clearing the
+     * entity-id mapper there would race the concurrent `graphIndex.rebuild()`
+     * that resolves verb endpoints through this same mapper. The hook already
+     * did the full, identical rebuild, so the second one is skipped and the
+     * mapper stays stable for the graph pass. One-shot: consumed by the next
+     * `rebuild()`. (#68)
+     */
+    skipRebuildOnce = false;
+    constructor(storage, config = {}, 
+    /**
+     * Lazy resolver to the shared {@link MigrationCoordinator} (the plugin creates
+     * one per brain and hands the same getter to all three derived-index providers).
+     * Undefined for a raw-napi construction (tests) or a tier that never migrates —
+     * then {@link isMigrating} is always `false` and the epoch guard's large-brain
+     * outcome falls through to Brainy's own path.
+     */
+    migrationCoordinatorGetter) {
+        this.migrationCoordinatorGetter = migrationCoordinatorGetter;
+        this.storage = storage;
+        this.config = {
+            maxIndexSize: config.maxIndexSize ?? 10000,
+            rebuildThreshold: config.rebuildThreshold ?? 0.1,
+            autoOptimize: config.autoOptimize ?? true,
+            indexedFields: config.indexedFields ?? [],
+            excludeFields: config.excludeFields ?? [
+                'embedding', 'vector', 'embeddings', 'vectors',
+                'content', 'data', 'originalData', '_data',
+                'id'
+            ],
+        };
+        // Build Rust config JSON
+        const nativeConfig = {};
+        if (this.config.excludeFields.length > 0) {
+            nativeConfig.excludeFields = this.config.excludeFields;
+        }
+        if (this.config.indexedFields.length > 0) {
+            nativeConfig.indexedFields = this.config.indexedFields;
+        }
+        const bindings = loadNativeModule();
+        // Cor 3.0 zero-config: pass 'u64' explicitly so the
+        // wrapper's LSM engine (also constructed with idSpace='u64'
+        // below) and the metadata index core agree. The raw napi
+        // constructor still defaults to u32 for back-compat with
+        // direct napi callers + internal tests.
+        this.native = new bindings.NativeMetadataIndex(Object.keys(nativeConfig).length > 0 ? JSON.stringify(nativeConfig) : null, 'u64');
+        this.unifiedCache = getGlobalCache();
+        this.fieldTypeInference = new FieldTypeInference(storage);
+        this.columnStore = new NativeColumnStore();
+        // LSM durability (steps 3-4): engine construction MOVED to init().
+        // The persistent engine is opened via `NativeLsmEngine.openWithDir`,
+        // which must be seeded from the on-disk manifest (an async storage
+        // read) so its epoch counter outranks every already-flushed SSTable.
+        // The constructor therefore does no LSM work — `init()` opens the
+        // engine (cold-loading any persisted SSTables) and registers the
+        // ResourceManager subscriber once the engine exists.
+    }
+    /**
+     * **LSM durability (steps 3-4) — the async open.** Construct the native LSM
+     * engine, attach it to the metadata index, register the ResourceManager
+     * budget subscriber, and cold-load every persisted SSTable into the
+     * engine's live read set. Called once from {@link init} (after
+     * {@link wireIdMapper}) — the LSM engine is the sole metadata engine.
+     *
+     * ## Persistent vs fallback
+     *
+     * When the storage is filesystem-backed (exposes `getBinaryBlobPath`), the
+     * engine is opened via `NativeLsmEngine.openWithDir(metaDir, manifestJson,
+     * …)`: its memtable is backed by mmap'd per-shard append logs and its epoch
+     * counter is seeded from the persisted manifest so new writes outrank every
+     * already-flushed SSTable. Non-filesystem storage falls back to a
+     * non-persistent in-RAM engine (`new NativeLsmEngine`) — reads/writes work
+     * for the session but there is no SSTable durability (and {@link flushLsm}
+     * becomes a no-op).
+     *
+     * ## Cold-load = no rebuild
+     *
+     * Cold start reads the manifest, seeds the epoch via `openWithDir`, then
+     * calls `registerSstablePath` once per SSTable — the entire restore. The
+     * O(N) {@link rebuild} path is NOT taken: the SSTables ARE the durable
+     * posting lists, served from mmap.
+     *
+     * @throws If a persisted SSTable's path cannot be resolved.
+     */
+    async openLsmEngine() {
+        if (this.lsmEngine !== null)
+            return; // already attached
+        const bindings = loadNativeModule();
+        const blobStorage = this.storage;
+        // Resolve the metadata directory for the persistent engine, mirroring
+        // wireIdMapper's filesystem guard. `getBinaryBlobPath` returns null on
+        // non-local adapters → fall back to the in-RAM engine.
+        const metaDir = typeof blobStorage.getBinaryBlobPath === 'function'
+            ? blobStorage.getBinaryBlobPath('_metadata')
+            : null;
+        if (metaDir === null) {
+            // Fallback: non-persistent in-RAM engine (the legacy `new` path).
+            // U64 IdSpace matches the cor 3.0 production default + the metadata
+            // index core constructed above.
+            this.lsmEngine = new bindings.NativeLsmEngine({ idSpace: 'u64' });
+            this.lsmManifest = null;
+            this.lsmMetaDir = null;
+        }
+        else {
+            // Persistent cold-load: parse the manifest (null on a fresh brain),
+            // seed the engine's epoch from it via openWithDir, then register every
+            // persisted SSTable below. NO rebuild.
+            const persisted = await this.loadLsmManifest();
+            const manifestJson = persisted !== null ? JSON.stringify(persisted) : null;
+            this.lsmEngine = bindings.NativeLsmEngine.openWithDir(metaDir, manifestJson, {
+                idSpace: 'u64',
+            });
+            this.lsmManifest = persisted ?? newEmptyLsmManifest();
+            this.lsmMetaDir = metaDir;
+        }
+        this.native.setLsmEngine(this.lsmEngine);
+        // Register the budget subscriber NOW that the engine exists, so its
+        // synchronous initial callback gives the engine a budgeted allowedBytes
+        // before the first write (closing the u64::MAX unwired-admission window).
+        this.attachLsmBudgetSubscriber();
+        // Cold-load SSTable registration (persistent engine only). The ordering
+        // invariant in flushLsm guarantees every entry the manifest references
+        // was made durable BEFORE the manifest — so each path exists here. Call
+        // getBinaryBlobPath as a METHOD on the storage object (not via an extracted
+        // reference) so its `this` binding survives — the adapter resolves the path
+        // through `this.blobPath(...)`.
+        if (this.lsmManifest !== null && metaDir !== null) {
+            const fsStorage = this.storage;
+            const engine = this.lsmEngine;
+            for (const entry of this.lsmManifest.sstables) {
+                const key = lsmSstableKey(entry.id);
+                const path = fsStorage.getBinaryBlobPath(key);
+                if (path === null) {
+                    throw new Error(`openLsmEngine: getBinaryBlobPath returned null for persisted SSTable '${key}'`);
+                }
+                engine.registerSstablePath(path);
+            }
+        }
+    }
+    /**
+     * **LSM durability (steps 3-4) — the sync part.** Register the
+     * ResourceManager budget subscriber that keeps the engine's admission
+     * ceiling in sync with the dynamic 'metadata' budget. Split out of the
+     * async {@link openLsmEngine} so the registration (and its synchronous
+     * initial callback) is an isolated, side-effect-free-on-failure step that
+     * runs immediately after the engine is attached.
+     *
+     * Idempotent: a second call (e.g. a re-init) is a no-op while a subscriber
+     * is live. The unique subscriber name keeps multiple brains in one process
+     * from colliding on the same ResourceManager slot.
+     *
+     * The callback path is load-bearing for the dynamic rebalance: every
+     * `notifySubscribers` cycle (rebalance + pressure tick) hits
+     * `this.lsmEngine.setAllowedBytes(BigInt(allowed))`, applied by the native
+     * engine on its very next admission check.
+     */
+    attachLsmBudgetSubscriber() {
+        if (this.unsubscribeFromBudget !== null)
+            return; // already subscribed
+        const rm = ResourceManager.getInstance();
+        this.lsmSubscriberName = nextLsmSubscriberId();
+        this.unsubscribeFromBudget = rm.registerMemoryBudgetSubscriber(this.lsmSubscriberName, 'metadata', (allowedBytes) => {
+            // Native side wants u64 BigInt. Negative or NaN values from a
+            // pathological budget calculation would round to 0n via the
+            // Math.max guard — admission would block every write, which
+            // is the SAFE failure (no over-budget writes) rather than the
+            // unsafe alternative (unwired u64::MAX → no admission gate).
+            const clamped = Math.max(0, Math.floor(allowedBytes));
+            this.lsmEngine?.setAllowedBytes(BigInt(clamped));
+            // **Step 5.** Track the dynamic budget: the memtable size-flush
+            // trigger ({@link addToIndex}) drains at ~50% of this admission
+            // ceiling (clamped [32 MiB, 256 MiB]), so the memtable empties
+            // well before admission would block a writer. Recomputed in the
+            // SAME callback the engine's ceiling is updated from so the two
+            // never drift apart across a rebalance.
+            this.lsmFlushThresholdBytes = computeLsmFlushThreshold(clamped);
+        });
+    }
+    /**
+     * **LSM durability (steps 3-4) — cold-load read.** Read and validate the
+     * persisted LSM manifest from storage. Returns `null` when no manifest
+     * exists yet (a fresh brain). Validates the format version + shape and
+     * throws on a mismatch rather than registering a partial SSTable set (which
+     * would silently drop rows) — a forward-incompatible or torn manifest must
+     * fail loud.
+     *
+     * @returns The parsed manifest, or `null` if absent.
+     * @throws If the manifest is present but malformed or carries an
+     *   unsupported {@link LsmManifest.version}.
+     */
+    async loadLsmManifest() {
+        let raw;
+        try {
+            raw = await this.storage.getMetadata(LSM_MANIFEST_KEY);
+        }
+        catch {
+            // Storage read error during the probe is treated as "no manifest" — a
+            // transient hiccup should not block engine open; a genuinely-present
+            // manifest is re-read on the next open.
+            return null;
+        }
+        if (raw === null || raw === undefined)
+            return null;
+        const m = raw;
+        if (typeof m.version !== 'number' || !Array.isArray(m.sstables)) {
+            throw new Error(`loadLsmManifest: malformed manifest at '${LSM_MANIFEST_KEY}' ` +
+                `(expected { version, sstables[], last_durable_epoch })`);
+        }
+        if (m.version !== LSM_MANIFEST_VERSION) {
+            throw new Error(`loadLsmManifest: manifest version ${m.version}, this build supports ` +
+                `${LSM_MANIFEST_VERSION}`);
+        }
+        return {
+            version: m.version,
+            sstables: m.sstables,
+            last_durable_epoch: typeof m.last_durable_epoch === 'number' ? m.last_durable_epoch : 0,
+        };
+    }
+    /**
+     * **LSM durability (step 6).** Whether the LSM engine recovered durable
+     * on-disk state at cold open — used by {@link init} to demote the
+     * field-registry-missing rebuild to disaster-recovery-only. Durable state
+     * is either ≥1 SSTable in the cold-loaded manifest OR a non-empty memtable
+     * replayed from the per-shard logs (records inserted but not yet flushed
+     * before the last shutdown/crash). When present, the posting lists are
+     * served from the engine independent of the JSON field registry, so a
+     * rebuild would be a redundant O(N) re-scan.
+     *
+     * @returns `true` when the engine holds durable posting-list state.
+     */
+    lsmHasDurableState() {
+        const engine = this.lsmEngine;
+        if (engine === null)
+            return false;
+        if (this.lsmManifest !== null && this.lsmManifest.sstables.length >= 1) {
+            return true;
+        }
+        if (Number(engine.memtableBytes()) > 0)
+            return true;
+        if (engine.memtableEntryCount() > 0n)
+            return true;
+        return false;
+    }
+    /**
+     * **LSM durability (step 6) — restore/rebuild re-open.** Re-open the LSM
+     * engine fresh against the (wiped + recreated) storage tree and re-attach
+     * it to the native core. Called from {@link rebuild} in LSM mode.
+     *
+     * ## Why a full re-open, not just a `mkdir`
+     *
+     * A rebuild commonly follows brainy's `restore()`, which WIPES and replaces
+     * the storage tree under the still-live engine. But `native.clear()` (the
+     * in-memory reset rebuild runs) deliberately does NOT touch the attached
+     * engine — the engine owns its own memtable + copy-on-write live-SSTable
+     * set, neither of which `clear()` can see. So WITHOUT this re-open the
+     * engine would keep:
+     *   - **stale live-SSTable readers** — mmap handles on the pre-restore
+     *     `.sst` files. On Linux an mmap survives the file's `unlink`, so those
+     *     readers keep serving PRE-restore rows; a query after rebuild would
+     *     merge the restored data WITH ghosts of the wiped entities.
+     *   - **a stale in-memory manifest** ({@link lsmManifest}) still listing the
+     *     deleted SSTables — the next cold open would try to `registerSstablePath`
+     *     files that no longer exist and fail loud.
+     *   - **a stale memtable** holding pre-restore writes not yet flushed.
+     *
+     * This method makes the engine adopt the new tree: it deletes the stale
+     * on-disk SSTables + memtable logs, drops the stale manifest, constructs a
+     * FRESH `openWithDir` engine (empty live set, empty memtable, epoch reset
+     * from the clean logs), and re-attaches it via `setLsmEngine` (which
+     * overwrites the core's engine Arc — verified in the Rust binding). The
+     * ResourceManager subscriber callback dereferences `this.lsmEngine`
+     * dynamically, so the fresh engine inherits budget tracking on the next
+     * tick; meanwhile the re-applied admission ceiling below keeps it bounded,
+     * and the step-5 size-flush trigger drains it during the rebuild's
+     * re-index walk.
+     *
+     * A no-op on the non-persistent fallback engine (no `lsmMetaDir`), where
+     * there is no SSTable surface to go stale.
+     */
+    async reopenLsmEngineFresh() {
+        if (this.lsmMetaDir === null)
+            return;
+        const metaDir = this.lsmMetaDir;
+        const bindings = loadNativeModule();
+        const blobStorage = this.storage;
+        // (1) Delete the stale on-disk SSTables of the pre-restore tree. The live
+        // engine's readers mmap the `.sst` files (surviving unlink), so they must
+        // be dropped together with the engine below; deleting the files also stops
+        // the next cold open from registering a ghost SSTable. The SSTables live
+        // under the blob root (resolved via the SAME `getBinaryBlobPath` key
+        // mapping flushLsm uses) — which is a SIBLING of `metaDir`, NOT a child of
+        // it (e.g. FileSystemStorage maps `_metadata` → `…/_metadata.bin` for the
+        // memtable logs but `_metadata/sstables/<id>.sst` → `…/_metadata/sstables/
+        // <id>.sst.bin`). Wiping the whole sstables directory reaps the
+        // manifest-listed SSTables AND any orphan from an interrupted flush.
+        if (typeof blobStorage.getBinaryBlobPath === 'function') {
+            const probe = blobStorage.getBinaryBlobPath(lsmSstableKey(0));
+            if (probe !== null) {
+                rmSync(dirname(probe), { recursive: true, force: true });
+            }
+        }
+        rmSync(join(metaDir, 'memtable'), { recursive: true, force: true });
+        // (2) Drop the stale manifest (it still references the deleted SSTables).
+        try {
+            await this.storage.saveMetadata(LSM_MANIFEST_KEY, null);
+        }
+        catch {
+            // Best-effort: a fresh-restore tree may already lack the manifest.
+        }
+        // (3) Recreate the memtable directory the fresh engine's lazy per-shard
+        // logs append into; the native append path fails LOUD on a missing dir.
+        mkdirSync(join(metaDir, 'memtable'), { recursive: true });
+        // (4) Construct a FRESH persistent engine (null manifest → epoch seeded
+        // from the now-clean logs) and re-attach. This drops the old engine's
+        // stale live-SSTable readers + memtable; `setLsmEngine` replaces the
+        // core's engine reference.
+        const engine = bindings.NativeLsmEngine.openWithDir(metaDir, null, {
+            idSpace: 'u64',
+        });
+        this.lsmEngine = engine;
+        this.lsmManifest = newEmptyLsmManifest();
+        this.native.setLsmEngine(engine);
+        // Re-apply the current admission ceiling immediately (the subscriber
+        // callback only refreshes it on the next rebalance/pressure tick). The
+        // fresh engine opens with `allowedBytes = u64::MAX`; seeding it now keeps
+        // the rebuild's writes inside the same budget the old engine had.
+        if (this.lsmSubscriberName !== null) {
+            const allowed = ResourceManager.getInstance().getCategoryAllowedBytes('metadata');
+            const clamped = Math.max(0, Math.floor(allowed));
+            engine.setAllowedBytes(BigInt(clamped));
+            this.lsmFlushThresholdBytes = computeLsmFlushThreshold(clamped);
+        }
+    }
+    /**
+     * **LSM durability (steps 3-4) — the flush/close drain.** Drain the LSM
+     * engine's memtable to a durable SSTable and splice it into the manifest.
+     * Called from {@link flush} (and therefore {@link close}, which flushes).
+     * No-op when there is no memtable content to flush, and on the
+     * non-persistent fallback engine (no manifest / no metaDir).
+     *
+     * ## Ordering invariant (load-bearing — a reviewer MUST preserve it)
+     *
+     * The four steps run in this exact order, and `commitFlushThrough` is NEVER
+     * called before the `.sst` bytes AND the manifest referencing them are
+     * durable:
+     *
+     *   1. write the `.sst` bytes (`saveBinaryBlob`, atomic tmp+rename)
+     *   2. write the manifest that references the new SSTable (`saveMetadata`)
+     *   3. register the SSTable into the engine's live read set
+     *      (`registerSstablePath` — mmap + SHA-256 verify)
+     *   4. ONLY NOW retire the memtable log below the flushed epoch
+     *      (`commitFlushThrough`)
+     *
+     * A crash between any two steps is safe: the just-flushed records still live
+     * in the uncommitted memtable log, so the next `openWithDir` replays them
+     * and merge-by-epoch idempotently collapses the transient double presence. A
+     * crash before step 2 leaves an orphaned `.sst` no manifest references — its
+     * id is harmlessly re-used on the next flush. Calling `commitFlushThrough`
+     * early would drop the log copy while the only on-disk copy might not yet be
+     * durable — unrecoverable data loss. Hence step 4 is strictly last.
+     */
+    async flushLsm() {
+        const engine = this.lsmEngine;
+        const manifest = this.lsmManifest;
+        // Only the persistent (filesystem-backed) engine has an SSTable surface.
+        if (engine === null || manifest === null || this.lsmMetaDir === null)
+            return;
+        // Defensive: skip the drain if the persistent memtable directory has been
+        // removed out from under us. The realistic trigger is teardown, not
+        // production: brainy registers a global process-exit `flushOnShutdown`
+        // hook that calls `metadataIndex.flush()` on EVERY still-registered brain,
+        // and a test commonly deletes its temp storage before that hook fires. The
+        // native memtable-log rotation in `commitFlushThrough` fails LOUD on a
+        // missing directory ("persistence guarantee broken — terminate"), aborting
+        // the process. There is nothing to persist when the directory is gone (the
+        // memtable logs lived there too), so a no-op is the correct, safe outcome.
+        // In production the directory never vanishes, so this guard always passes.
+        if (!existsSync(join(this.lsmMetaDir, 'memtable')))
+            return;
+        const result = engine.flush();
+        if (result == null)
+            return; // memtable empty — nothing to persist
+        const blobStorage = this.storage;
+        // Monotonic, manifest-persisted id (see nextSstableId for the crash-safety
+        // argument).
+        const id = nextSstableId(manifest);
+        const key = lsmSstableKey(id);
+        // --- Ordering invariant (see JSDoc) ---
+        // (1) .sst bytes durable.
+        await blobStorage.saveBinaryBlob(key, result.sstable);
+        // (2) manifest (references the .sst) durable. Stamp the native flush's
+        // placeholder-id entry with the allocated id, keep the (level, id) sort
+        // invariant, and advance the durable-epoch high-water mark.
+        const entry = JSON.parse(result.manifestEntryJson);
+        entry.id = id;
+        manifest.sstables.push(entry);
+        manifest.sstables.sort((a, b) => a.level - b.level || a.id - b.id);
+        manifest.last_durable_epoch = Number(result.lastDurableEpoch);
+        await this.storage.saveMetadata(LSM_MANIFEST_KEY, manifest);
+        // (3) register into the engine's live read set (mmap + verify).
+        const path = blobStorage.getBinaryBlobPath(key);
+        if (path === null) {
+            throw new Error(`flushLsm: getBinaryBlobPath returned null for '${key}'`);
+        }
+        engine.registerSstablePath(path);
+        // (4) Retire the memtable log below the flushed epoch. The `.sst` bytes
+        // AND the manifest referencing them are already durable here, so a failure
+        // at THIS step is non-fatal per the ordering invariant above: the records
+        // simply stay in the memtable log and replay on the next `openWithDir`,
+        // where merge-by-epoch idempotently collapses them against the SSTable
+        // copy (no double counting, no loss — the data lives safely in two places).
+        // The one realistic trigger is a `rebuild()` following brainy's `restore()`,
+        // which wipes + recreates the storage tree under the still-live engine; the
+        // native shard-log rotation can then fail to rename a log even though the
+        // flushed data is safely in the new SSTable. Warn + continue rather than
+        // abort the rebuild — the next clean flush retires the log.
+        try {
+            engine.commitFlushThrough(result.lastDurableEpoch);
+        }
+        catch (error) {
+            prodLog.warn(`flushLsm: commitFlushThrough deferred for SSTable ${id} (it + the ` +
+                `manifest are already durable; the memtable log replays + collapses ` +
+                `by epoch on the next open):`, error);
+        }
+    }
+    /**
+     * **Piece 11 closeout.** Detect cortex 2.x legacy JSON metadata
+     * state on storage. The canonical marker is the
+     * `__metadata_field_registry__` key the JSON path writes on
+     * every flush. If present, the brain was built before the LSM
+     * engine became the default and refuses to silently switch.
+     *
+     * On detection: throw an init-time error directing the operator to run
+     * the migration script (the cor 3.0 LSM engine is the only engine — the
+     * legacy JSON chunk path has been removed). The error message is the
+     * migration runbook.
+     *
+     * No legacy state present → the brain is either fresh (no prior
+     * cor install) or already on the LSM path → proceed normally
+     * with the LSM engine.
+     *
+     * @throws {@link MetadataIndexLegacyJsonStateDetected} when cortex 2.x
+     *   JSON metadata state is detected on disk.
+     */
+    async detectLegacyJsonStateAndGate() {
+        let registry;
+        try {
+            registry = await this.storage.getMetadata('__metadata_field_registry__');
+        }
+        catch {
+            // Storage errors during the probe are treated as "no legacy
+            // state" — we can't read the file, so we can't be sure it's
+            // there, and the default behaviour (use LSM) is safer than
+            // erroring out on every transient storage hiccup.
+            return;
+        }
+        if (registry === null || registry === undefined) {
+            // Fresh brain — no field registry — proceed with the LSM engine.
+            return;
+        }
+        // A reopened cor 3.0 LSM brain ALSO carries `__metadata_field_registry__`
+        // (the value-enumeration registry is shared — it is NOT a cortex 2.x
+        // marker by itself). The 2.x-specific marker is a per-field sparse index
+        // (`__sparse_index__<field>`), which the LSM path never writes — the
+        // metadata-storage analog of {@link assertNoLegacyLayout}'s chunk-file
+        // check. Probe the registered fields for one; only genuine 2.x chunk
+        // state trips the gate, so a legitimate LSM reopen passes through.
+        const fields = Array.isArray(registry.fields)
+            ? registry.fields.filter((f) => typeof f === 'string')
+            : [];
+        let hasLegacySparseIndex = false;
+        for (const field of fields) {
+            let sparse;
+            try {
+                sparse = await this.storage.getMetadata(`__sparse_index__${field}`);
+            }
+            catch {
+                continue;
+            }
+            if (sparse !== null && sparse !== undefined) {
+                hasLegacySparseIndex = true;
+                break;
+            }
+        }
+        if (!hasLegacySparseIndex) {
+            // Registry present but no 2.x sparse indices → an LSM-native brain.
+            return;
+        }
+        // Genuine cortex 2.x JSON chunk state. Refuse to proceed silently: the
+        // cor 3.0 LSM engine would shadow this on-disk state and surface as a
+        // mysteriously-empty brain on first query. The legacy JSON chunk engine
+        // has been removed, so the only path forward is migration.
+        throw new MetadataIndexLegacyJsonStateDetected(`[NativeMetadataIndex] cortex 2.x JSON metadata state detected ` +
+            `(__metadata_field_registry__ + __sparse_index__ chunk markers). ` +
+            `cor 3.0 uses the LSM metadata engine exclusively (the legacy JSON ` +
+            `chunk engine has been removed), which would shadow this on-disk ` +
+            `state and surface as a mysteriously-empty brain on first query.\n\n` +
+            `Migrate the brain to the LSM engine before reopening it:\n` +
+            `  - Run the migration script: ` +
+            `scripts/migrate-cortex-2x-to-3x.mjs <brain-path>\n` +
+            `    (it snapshots the brain via Brainy 7.x, wipes the legacy JSON ` +
+            `state, and restores via Brainy 8.0 + cor 3.0, producing the ` +
+            `LSM-native on-disk layout).`);
+    }
+    /**
+     * **#18 read-side epoch guard.** Detect 7.x → 8.0 derived-index epoch drift
+     * on open and route it to the right outcome. Called from {@link init} AFTER
+     * {@link detectLegacyJsonStateAndGate} (the cortex-2.x JSON gate) and
+     * {@link openLsmEngine} (the LSM cold-open), and BEFORE the column-store init
+     * and any rebuild branch — i.e. while reads are still un-served.
+     *
+     * The on-disk version-handshake marker (`_system/brain-format.json`) carries a
+     * shared, lockstep-bumped `indexEpoch`. cor reads it directly off storage (it
+     * cannot reach `brain.formatInfo()`, and Brainy's marker module is not
+     * exported) and compares against {@link COR_EXPECTED_INDEX_EPOCH}.
+     *
+     * Four fully-wired outcomes — no dangling flag:
+     *
+     * 1. **Epoch CURRENT (the common case)** — a single tiny marker read + an
+     *    integer compare, then return. NO size probe, NO further work: the
+     *    warm-restart path stays a cheap no-op.
+     * 2. **Stale + SMALL** (`< {@link resolveLargeBrainThreshold}`) — NO-OP. Brainy's
+     *    `rebuildIndexesIfNeeded` already drives the rebuild from the canonical
+     *    records for sub-threshold brains (its `AUTO_REBUILD_THRESHOLD` matches
+     *    ours) and re-stamps the marker after the rebuild verifies. cor must NOT
+     *    throw here; it simply lets Brainy's path run.
+     * 3. **Stale + LARGE + auto-migrate disabled** (`COR_AUTO_MIGRATE=false`) —
+     *    THROW {@link IndexMigrationRequiredError}, loud and pre-serve. A service
+     *    that refuses to start is recoverable; serving on indexes that predate
+     *    this build would surface stale/empty results with no signal.
+     * 4. **Stale + LARGE + auto-migrate enabled (default)** — NO-OP / fall through.
+     *    Brainy's existing large-brain path defers to a lazy first-query rebuild
+     *    (force=true, BLOCKING inline). That blocking rebuild is correct but not
+     *    yet "optimized": the non-blocking background build-new → verify → swap
+     *    that removes the first-query freeze is the **rc.8 increment** (it needs
+     *    Brainy's deference hook + a cor-invokable stamp method, neither shipped on
+     *    rc.7). We deliberately do NOT set an `epochMigrationPending` flag here:
+     *    nothing on rc.7 reads it (the MigrationCoordinator is the rc.8 unit), and
+     *    an unread flag would be unwired dead state.
+     *
+     * @throws {@link IndexMigrationRequiredError} on outcome 3 only.
+     */
+    async guardIndexEpoch() {
+        const onDisk = await readOnDiskBrainFormat(this.storage);
+        if (!isEpochStale(onDisk)) {
+            // Outcome 1: epoch current — zero-overhead no-op (no size probe).
+            return;
+        }
+        // Epoch is stale (drifted or absent/corrupt marker). Size the brain to
+        // decide who drives the rebuild.
+        const size = await detectBrainSize(this.storage);
+        const isLargeBrain = size >= resolveLargeBrainThreshold();
+        if (!isLargeBrain) {
+            // Outcome 2: small brain — Brainy's rebuildIndexesIfNeeded drives the
+            // rebuild from canonical and re-stamps. Nothing for cor to do.
+            return;
+        }
+        if (!resolveAutoMigrate()) {
+            // Outcome 3: large brain, auto-migrate disabled — refuse to serve.
+            throw new IndexMigrationRequiredError(`[NativeMetadataIndex] derived-index epoch drift detected on a large ` +
+                `brain (${size.toLocaleString()} entities ≥ migration threshold) and ` +
+                `COR_AUTO_MIGRATE is disabled. The on-disk derived indexes (DiskANN ` +
+                `vectors, LSM metadata postings, graph adjacency) predate this build ` +
+                `(on-disk indexEpoch ${onDisk?.indexEpoch ?? 'absent'} ≠ expected ` +
+                `${COR_EXPECTED_INDEX_EPOCH}) and would serve stale/empty results.\n\n` +
+                `Re-enable automatic migration (rebuild from the canonical records on ` +
+                `open) by unsetting COR_AUTO_MIGRATE, or set COR_AUTO_MIGRATE=true.`);
+        }
+        // Outcome 4: large brain, auto-migrate enabled — the #18 coordinated blocking
+        // migration will run, but we do NOT kick it here. guardIndexEpoch() runs in the
+        // MIDDLE of init() (before columnStore.init / warmCache / lazyLoadCounts), and
+        // the migration's first step (metadata.rebuild → reopenLsmEngineFresh +
+        // columnStore.reset, both destructive) would run CONCURRENTLY with — and race,
+        // unsynchronized — the remainder of THIS init(). Flag it instead; init() kicks
+        // the coordinator at its very END, so the rebuild starts against a fully
+        // settled index. The flag is read (init()'s tail) — not dead state.
+        this.epochMigrationPending = true;
+    }
+    /**
+     * @description **#18 migration lock — feature-detected by Brainy.** `true` while
+     * this brain is undergoing (or has failed) the coordinated 7.x → 8.0 rebuild, so
+     * Brainy holds every public read and write behind the lock. Delegates to the
+     * shared {@link MigrationCoordinator}; `false` when no coordinator is wired.
+     * @returns `true` when the brain must not be read from or written to.
+     */
+    isMigrating() {
+        return this.migrationCoordinatorGetter?.()?.isMigrating() ?? false;
+    }
+    /**
+     * @description **#18 migration lock — structured progress** Brainy relays into
+     * `getIndexStatus().migration`. `null` unless a migration is in flight.
+     * @returns The current {@link MigrationStatus}, or `null`.
+     */
+    migrationStatus() {
+        return this.migrationCoordinatorGetter?.()?.migrationStatus() ?? null;
+    }
+    /**
+     * **#18 in-place migration runner (stub).** Reads the un-migrated metadata
+     * state from storage and replays it through the LSM-attached mutation path,
+     * leaving the brain on the cor 3.0 LSM-native on-disk layout.
+     *
+     * **NOT YET IMPLEMENTED — GA path is the rc.8 increment.** The read-side
+     * epoch guard (#18, {@link guardIndexEpoch}) ships now and makes the SMALL
+     * 7.x → 8.0 brain GA on rc.7 (Brainy drives the rebuild from canonical). The
+     * LARGE-brain non-blocking background build-new → verify → swap that this
+     * runner will drive needs Brainy's deference hook + a cor-invokable stamp
+     * method, which land together in the rc.8 increment; until then a large stale
+     * brain falls through to Brainy's blocking inline rebuild on first query.
+     *
+     * The wrapper for this method exists so callers have a stable reference point
+     * in the API surface; today it throws with a pointer to the supported
+     * back-compat path. Until the in-place runner lands, migrate via the
+     * standalone script `scripts/migrate-cortex-2x-to-3x.mjs`, which snapshots the
+     * brain via Brainy 7.x, wipes the legacy state, and restores via Brainy 8.0 +
+     * cor 3.0 (producing the LSM-native on-disk layout).
+     *
+     * @throws Always — the in-place runner is the rc.8 increment.
+     */
+    async migrateToLsm() {
+        throw new Error(`[NativeMetadataIndex.migrateToLsm] not yet implemented — the in-place ` +
+            `runner is the #18 rc.8 increment (needs Brainy's deference hook + ` +
+            `stamp method). Until then, migrate via the standalone script: ` +
+            `scripts/migrate-cortex-2x-to-3x.mjs <brain-path>.`);
+    }
+    /**
+     * Get the shared entity id-mapper façade. Brainy's unfiltered `orderBy` path
+     * calls `metadataIndex.getIdMapper().getUuid(intId)` to turn the column store's
+     * sorted int ids back into UUIDs for batchGet. The façade delegates to the
+     * native index's O(1) UUID ↔ int methods — the canonical mapping that
+     * `addToIndex` assigns and the column store is populated against.
+     *
+     * @returns An object exposing the brainy `EntityIdMapper` methods callers use.
+     */
+    getIdMapper() {
+        if (this.idMapperFacade)
+            return this.idMapperFacade;
+        const native = this.native;
+        this.idMapperFacade = {
+            getUuid: (intId) => native.intToUuid(intId) ?? undefined,
+            getInt: (uuid) => native.uuidToInt(uuid) ?? undefined,
+            getOrAssign: (uuid) => {
+                // The native index assigns ids inside addToIndex. If a caller resolves an
+                // id before the entity is indexed, surface the existing mapping; an
+                // unmapped UUID has no int id yet and is reported as such rather than
+                // fabricating one (which would desync the canonical native mapper).
+                const existing = native.uuidToInt(uuid);
+                if (existing !== null)
+                    return existing;
+                throw new Error(`EntityIdMapper.getOrAssign: '${uuid}' has no native id yet — ids are ` +
+                    `assigned by addToIndex. Index the entity before resolving its int id.`);
+            },
+            intsIterableToUuids: (ints) => native.intsToUuids([...ints]),
+            // Brainy 8.0 `restore()` calls this BEFORE the metadata/vector/graph
+            // `Promise.all` so the entity-id mapper reflects the restored snapshot's
+            // assignments before `graphIndex.rebuild()` resolves verb endpoints through
+            // it. The metadata rebuild IS cor's canonical UUID→int assignment pass, so
+            // we run the full rebuild here, then mark the immediately-following
+            // `rebuild()` (the one in brainy's Promise.all) to skip — otherwise it would
+            // re-clear the mapper and race the concurrent graph rebuild (#68).
+            rebuild: async () => {
+                this.skipRebuildOnce = false; // ensure the call below actually rebuilds
+                await this.rebuild();
+                this.skipRebuildOnce = true; // brainy's following Promise.all rebuild() is a no-op
+            }
+        };
+        return this.idMapperFacade;
+    }
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    async init() {
+        // **Piece 11 closeout.** Detect cortex 2.x legacy JSON metadata state on
+        // disk. The cor 3.0 LSM engine is the only metadata engine, so an
+        // un-migrated 2.x brain would have its on-disk JSON state shadowed and
+        // surface as a mysteriously-empty brain on first query. Throw a clear
+        // migration error instead so the operator runs the migration script.
+        await this.detectLegacyJsonStateAndGate();
+        // Load field registry to discover persisted indices
+        await this.loadFieldRegistry();
+        // #72 Phase C: build the shared mmap BinaryIdMapper from the brain's
+        // `_id_mapper/*` paths and inject it as the core's canonical
+        // UUID ↔ entity-int allocator. Must run before columnStore.init (which
+        // resolves ids through getIdMapper()) and before any mutation.
+        this.wireIdMapper();
+        // LSM durability (steps 3-4): open the native LSM engine here (not in the
+        // constructor) so it can be seeded from the on-disk manifest and cold-load
+        // every persisted SSTable into its live read set — the entire restore,
+        // with NO O(N) rebuild. Runs after wireIdMapper (the engine's posting
+        // lists reference the mapper's entity ints) and before any read/mutation.
+        await this.openLsmEngine();
+        // **#18 read-side epoch guard.** AFTER the cortex-2.x legacy gate and the
+        // LSM cold-open, classify the brain's derived-index epoch against this
+        // build's expectation. The common case (current epoch) is a single tiny
+        // marker read + an integer compare, then return — no size probe, no
+        // overhead on the warm-restart path. Only an epoch drift (a 7.x→8.0 upgrade,
+        // or an absent/corrupt marker) does any further work. See
+        // {@link guardIndexEpoch}.
+        await this.guardIndexEpoch();
+        // Initialize the column store with the shared storage + id mapper façade.
+        // Discovers any persisted `_column_index/` segments and re-applies tombstones.
+        // Done before the rebuild branch below so that a rebuild repopulates a
+        // freshly-initialized column store via addToIndex.
+        await this.columnStore.init(this.storage, this.getIdMapper());
+        const hasFields = this.knownFields.size > 0;
+        if (!hasFields) {
+            // Don't trust "empty" — field registry may be missing due to interrupted flush.
+            // Probe storage for actual entities before concluding the workspace is empty.
+            try {
+                const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } });
+                const hasEntities = (probe.totalCount ?? 0) > 0 || probe.items.length > 0;
+                if (hasEntities) {
+                    // **LSM durability (step 6) — rebuild demoted to disaster-recovery.**
+                    // A missing field registry no longer implies the index is lost: in
+                    // LSM mode the posting lists live in the registered SSTables + the
+                    // replayed memtable log, wholly independent of the JSON field
+                    // registry. If `openLsmEngine` recovered durable state (≥1 manifest
+                    // SSTable OR a non-empty replayed memtable), reads serve from there
+                    // and an O(N) rebuild would be a redundant re-scan of the entire
+                    // corpus. Skip it. `rebuild()` stays reachable only for GENUINE
+                    // corruption — no durable LSM state AND entities on storage — which
+                    // falls through to the rebuild below.
+                    if (this.lsmHasDurableState()) {
+                        prodLog.info('[NativeMetadataIndex] Field registry missing but the LSM engine ' +
+                            'recovered durable state (SSTables/memtable) — serving from it, ' +
+                            'NO rebuild.');
+                        return;
+                    }
+                    console.warn(`[NativeMetadataIndex] Field registry missing but ${probe.totalCount ?? 'unknown'} entities exist — rebuilding index`);
+                    await this.rebuild();
+                    return;
+                }
+            }
+            catch {
+                // Storage probe failed — genuinely empty
+            }
+            return;
+        }
+        await this.warmCache();
+        await this.lazyLoadCounts();
+        this.syncTypeCountsToFixed();
+        // **#18 migration kick — LAST, after the index is fully constructed.** If the
+        // epoch guard flagged a large stale brain (outcome 4), start the coordinated
+        // blocking migration NOW: `start()` flips the lock (`isMigrating()` → true)
+        // SYNCHRONOUSLY before this init() returns (so Brainy sees it the instant init
+        // completes and holds reads+writes behind it), then runs the in-place
+        // rebuild-from-canonical of all three derived indexes asynchronously — the LOCK,
+        // not this call, serializes access, so init() is not blocked. Kicking HERE (not
+        // from guardIndexEpoch mid-init) is what keeps the migration's destructive
+        // rebuild from racing the rest of init(). Fire-and-forget: brainy polls
+        // isMigrating() at 250 ms (rc.9 dropped the whenMigrationComplete hook), so the
+        // `.catch` only prevents an unhandled rejection on the start() promise when a
+        // failed migration surfaces (it is logged loudly and keeps the brain locked). No coordinator
+        // wired (raw-napi test / non-migrating tier) → no-op.
+        if (this.epochMigrationPending) {
+            this.epochMigrationPending = false;
+            this.migrationCoordinatorGetter?.()?.start().catch(() => { });
+        }
+    }
+    // ==========================================================================
+    // Storage I/O helpers
+    // ==========================================================================
+    /**
+     * **#72 Phase C.** Build the shared mmap `BinaryIdMapper` from the
+     * brain's `_id_mapper/*` storage paths and inject it into the Rust
+     * core as the canonical UUID ↔ entity-int allocator. This replaces the
+     * legacy `loadEntityIdMapper`/`saveEntityIdMapper` JSON path, whose two
+     * resident HashMaps cost ~56 GB at 1 B entities and whose serialize
+     * threw at the V8 ~512 MB / ~13 M-UUID cliff.
+     *
+     * @throws If storage is not filesystem-backed (no `getBinaryBlobPath`)
+     *   — cor's metadata index requires local paths; brainy runs
+     *   unaccelerated on cloud adapters.
+     */
+    wireIdMapper() {
+        // The native core is always constructed in u64 mode (the cor 3.0
+        // billion-scale default); setIdMapper rejects a width mismatch.
+        const handle = openOrCreateBinaryIdMapper(this.storage, {
+            idSpace: 'u64',
+        });
+        this.idMapperNative = handle.mapper;
+        this.native.setIdMapper(handle.mapper);
+    }
+    async loadFieldRegistry() {
+        try {
+            const registry = await this.storage.getMetadata('__metadata_field_registry__');
+            if (!registry?.fields || !Array.isArray(registry.fields)) {
+                prodLog.debug('No field registry found - will build on first flush');
+                return;
+            }
+            // Load into Rust
+            this.native.loadFieldRegistry(JSON.stringify(registry));
+            // Track known fields
+            for (const field of registry.fields) {
+                if (typeof field === 'string' && field.length > 0) {
+                    this.knownFields.add(field);
+                }
+            }
+            prodLog.info(`Loaded field registry: ${registry.fields.length} persisted fields discovered\n` +
+                `   Fields: ${registry.fields.slice(0, 5).join(', ')}${registry.fields.length > 5 ? '...' : ''}`);
+        }
+        catch (error) {
+            prodLog.debug('Could not load field registry:', error);
+        }
+    }
+    async saveFieldRegistry() {
+        if (this.knownFields.size === 0)
+            return;
+        try {
+            const json = this.native.saveFieldRegistry();
+            const data = JSON.parse(json);
+            await this.storage.saveMetadata('__metadata_field_registry__', data);
+        }
+        catch (error) {
+            prodLog.warn('Failed to save field registry:', error);
+        }
+    }
+    /**
+     * **Piece 6 generation counter — invalidate stale lazy-load state.**
+     * Checks the native generation counter against our last-seen value.
+     * If a mutation has bumped the counter since we last looked,
+     * `loadedFields` is cleared so every subsequent `ensureFieldLoaded`
+     * re-pulls the current value-enumeration field index from storage.
+     *
+     * Called at the start of every read entry point. The check itself
+     * is one atomic load + one BigInt compare — sub-microsecond.
+     *
+     * Without this hook, after a mutation on the native side the
+     * cached `loadedFields` set would prevent re-reading the updated
+     * value enumeration, surfacing as stale-result bugs of the same shape
+     * as R5 (but for the lazy-load path rather than the filter path).
+     */
+    invalidateIfStale() {
+        // **Piece 15 nuclear path.** Pre-Piece-15 this method gated the
+        // clear on `currentGeneration()` having changed since the last
+        // observation; that gate was load-bearing when EVERY read entry
+        // point called this method (the counter delta was the canary
+        // for "some mutation happened"). Post-Piece-15, the read entry
+        // points call {@link invalidateFieldIfStale} instead — which
+        // never updates `lastSeenGeneration`, so the old delta check
+        // would never trigger from the on-path callers. The method
+        // survives as an unconditional reset for cross-cutting state
+        // transitions (rebuilds, debug callers, future code paths
+        // that need a clean slate) — the gate is gone because the
+        // method's purpose is now "wipe everything", full stop.
+        this.loadedFields.clear();
+        this.lastSeenFieldGenerations.clear();
+        this.lastSeenGeneration = this.native.currentGeneration();
+    }
+    /**
+     * **Piece 15 — per-field selective invalidation.** Reads the
+     * field's per-field generation from the native side and compares
+     * against {@link lastSeenFieldGenerations}. If the field's counter
+     * advanced since we last observed it, removes ONLY that field
+     * from {@link loadedFields} — every other loaded field stays
+     * valid. Then records the current value so the next call sees a
+     * no-op until another mutation touches this field.
+     *
+     * Called at the start of every read entry point that knows which
+     * fields it needs. The check is one FFI hop (native atomic load)
+     * + one Map get/set — sub-microsecond per field. For multi-field
+     * filters (`ensureFieldsLoaded`) the wrapper iterates this method
+     * once per field instead of doing the old index-wide check.
+     *
+     * Cold-boot semantic: first probe records the current counter
+     * without invalidating (matches the old `lastSeenGeneration ===
+     * -1n` initial-call behavior). A counter advance from 0 → 1 on
+     * the FIRST observation is treated as "first sighting", NOT
+     * "stale" — `loadedFields` doesn't have this field yet anyway, so
+     * there's nothing to invalidate.
+     */
+    invalidateFieldIfStale(field) {
+        const current = this.native.fieldGeneration(field);
+        const lastSeen = this.lastSeenFieldGenerations.get(field);
+        if (lastSeen !== undefined && current !== lastSeen) {
+            this.loadedFields.delete(field);
+        }
+        this.lastSeenFieldGenerations.set(field, current);
+    }
+    /**
+     * Ensure a field's persisted value-enumeration index is loaded into the
+     * native core (the LSM read paths enumerate candidate values from it).
+     * Lazy + idempotent — guarded by {@link loadedFields}.
+     */
+    async ensureFieldLoaded(field) {
+        // Piece 15: selective per-field invalidation. A mutation on
+        // ANOTHER field no longer drops this field's cached load —
+        // brainy's lazy-load survives the cross-field churn.
+        this.invalidateFieldIfStale(field);
+        if (this.loadedFields.has(field))
+            return;
+        await this.loadPersistedFieldIndex(field);
+        this.loadedFields.add(field);
+    }
+    /**
+     * **LSM durability (steps 3-4) — cold-load value enumeration.** Restore a
+     * field's persisted value-enumeration index
+     * (`__metadata_field_index__field_<field>`) into the native core. In LSM
+     * mode the read paths for field-wide `exists` / `missing` / `ne` and range
+     * (`between` / `gt` / `lt`) queries enumerate a field's distinct values from
+     * `core.field_indexes`. That map is built incrementally by `addToIndex`
+     * within a session but is EMPTY after a cold reopen — the registered
+     * SSTables hold the posting lists, but without the value enumeration the
+     * read paths have no value keys to look those lists up by. Loading this JSON
+     * (persisted by `persistMutationResult` / `flush`) closes that gap so the
+     * cold-loaded SSTables serve every filter shape with no rebuild.
+     *
+     * No-op for a field with no persisted index (a brand-new field this
+     * session) — `addToIndex` then populates `core.field_indexes` incrementally.
+     * Only the first access loads it (the caller guards via `loadedFields`), so
+     * an in-session field index built by `addToIndex` is never overwritten.
+     *
+     * @param field - The field whose value-enumeration index to restore.
+     */
+    async loadPersistedFieldIndex(field) {
+        try {
+            const data = await this.storage.getMetadata(`__metadata_field_index__field_${field}`);
+            if (data) {
+                this.native.loadFieldIndex(field, JSON.stringify(data));
+            }
+        }
+        catch (error) {
+            prodLog.debug(`Failed to load field index '${field}':`, error);
+        }
+    }
+    /**
+     * Ensure multiple fields are loaded (parallel).
+     *
+     * **Piece 15.** Per-field stale checks instead of one
+     * index-wide check. The wrapper now iterates {@link invalidateFieldIfStale}
+     * once per requested field — selectively dropping ONLY the
+     * fields whose counters advanced since we last looked. Cross-
+     * field churn (a mutation on `noun` while we're reading `status`)
+     * no longer invalidates `status`'s lazy-load. Cost: N FFI hops
+     * (each a single atomic Acquire load) instead of 1 — typically
+     * 3-5 hops on a brainy filter query, negligible vs the storage
+     * I/O the wrapper does next.
+     */
+    async ensureFieldsLoaded(fields) {
+        for (const f of fields) {
+            this.invalidateFieldIfStale(f);
+        }
+        const unloaded = fields.filter(f => !this.loadedFields.has(f));
+        if (unloaded.length === 0)
+            return;
+        // Pass a flag to skip the redundant per-field check.
+        await Promise.all(unloaded.map(f => this.ensureFieldLoadedNoCheck(f)));
+    }
+    /**
+     * Internal variant of `ensureFieldLoaded` that skips the generation
+     * check (used by `ensureFieldsLoaded` which already did one batched
+     * check at the top).
+     */
+    async ensureFieldLoadedNoCheck(field) {
+        if (this.loadedFields.has(field))
+            return;
+        await this.loadPersistedFieldIndex(field);
+        this.loadedFields.add(field);
+    }
+    /**
+     * Extract filter field names for lazy loading.
+     */
+    extractFilterFields(filter) {
+        if (!filter || typeof filter !== 'object')
+            return [];
+        const fields = [];
+        if (filter.allOf && Array.isArray(filter.allOf)) {
+            for (const sub of filter.allOf) {
+                fields.push(...this.extractFilterFields(sub));
+            }
+        }
+        if (filter.anyOf && Array.isArray(filter.anyOf)) {
+            for (const sub of filter.anyOf) {
+                fields.push(...this.extractFilterFields(sub));
+            }
+        }
+        for (const key of Object.keys(filter)) {
+            if (key !== 'allOf' && key !== 'anyOf' && key !== 'not') {
+                fields.push(key);
+            }
+        }
+        return [...new Set(fields)];
+    }
+    // ==========================================================================
+    // Dirty data persistence
+    // ==========================================================================
+    /**
+     * Persist the dirty value-enumeration field indexes + field registry from a
+     * MutationResult. Posting-list durability is owned by the LSM engine
+     * ({@link flushLsm}), so this only writes the JSON value-enumeration surface
+     * the LSM read paths cold-load.
+     */
+    async persistMutationResult(result) {
+        const promises = [];
+        // Save dirty field indexes
+        for (const field of result.dirtyFieldIndexes) {
+            const json = this.native.saveFieldIndex(field);
+            if (json) {
+                const indexId = `__metadata_field_index__field_${field}`;
+                promises.push(this.storage.saveMetadata(indexId, JSON.parse(json)));
+            }
+        }
+        // Track new fields
+        for (const field of result.newFields) {
+            this.knownFields.add(field);
+            this.dirtyFields.add(field);
+        }
+        if (result.dirtyFieldRegistry) {
+            promises.push(this.saveFieldRegistry());
+        }
+        // #72 Phase C: the entity-id mapper is the mmap BinaryIdMapper now —
+        // its writes are durable in the mapped files (msync'd at flush), so
+        // there is no per-mutation JSON blob to persist here.
+        await Promise.all(promises);
+    }
+    // ==========================================================================
+    // Cache warming
+    // ==========================================================================
+    async warmCache() {
+        const commonFields = ['noun', 'type', 'service', 'createdAt'];
+        await Promise.all(commonFields.map(async (field) => {
+            try {
+                await this.ensureFieldLoaded(field);
+            }
+            catch {
+                prodLog.debug(`Cache warming: field '${field}' not yet indexed`);
+            }
+        }));
+        await this.warmCacheForTopTypes(3);
+    }
+    async warmCacheForTopTypes(topN = 3) {
+        const topTypes = this.getTopNounTypes(topN);
+        if (topTypes.length === 0)
+            return;
+        for (const type of topTypes) {
+            const typeFields = this.typeFieldAffinity.get(type);
+            if (!typeFields)
+                continue;
+            const topFields = Array.from(typeFields.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 5)
+                .map(([field]) => field);
+            if (topFields.length === 0)
+                continue;
+            await Promise.all(topFields.map(async (field) => {
+                try {
+                    await this.ensureFieldLoaded(field);
+                }
+                catch {
+                    // Silently ignore
+                }
+            }));
+        }
+    }
+    // ==========================================================================
+    // Lazy load counts
+    // ==========================================================================
+    async lazyLoadCounts() {
+        try {
+            this.totalEntitiesByType.clear();
+            this.entityCountsByTypeFixed.fill(0);
+            this.verbCountsByTypeFixed.fill(0);
+            // Ensure 'noun' field is loaded into Rust
+            await this.ensureFieldLoaded('noun');
+            // Use Rust to get type counts via the noun field. Count natively
+            // (#78 pt2) — getIdsCount returns the mapped-id count without
+            // materializing one UUID string per entity. At 10B this is the
+            // difference between O(1) per type and allocating billions of strings
+            // just to read `.length`.
+            const nounValues = this.native.getFilterValues('noun');
+            for (const typeName of nounValues) {
+                const count = this.native.getIdsCount('noun', JSON.stringify(typeName));
+                if (count > 0) {
+                    this.totalEntitiesByType.set(typeName, count);
+                }
+            }
+            prodLog.debug(`Loaded type counts: ${this.totalEntitiesByType.size} types`);
+        }
+        catch (error) {
+            prodLog.debug('Could not load type counts:', error);
+        }
+    }
+    syncTypeCountsToFixed() {
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const type = TypeUtils.getNounFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.entityCountsByTypeFixed[i] = count;
+        }
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.verbCountsByTypeFixed[i] = count;
+        }
+    }
+    syncTypeCountsFromFixed() {
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+    }
+    // ==========================================================================
+    // Core query methods
+    // ==========================================================================
+    /**
+     * Serialize a query value to JSON for the native String param. `JSON.stringify`
+     * returns the JS value `undefined` (NOT a string) for `undefined`/functions —
+     * which the napi String param rejects ("Failed to convert Null/Undefined into
+     * String"). Coalesce to the JSON literal `'null'` so the Rust side parses it to
+     * `serde_json::Value::Null`, normalizes, and returns the no-match result (0/[]) —
+     * bug-for-bug compatible with brainy's "no match → empty". (#79 cluster B.)
+     */
+    serializeValue(value) {
+        const s = JSON.stringify(value);
+        return s === undefined ? 'null' : s;
+    }
+    async getIds(field, value) {
+        await this.ensureFieldLoaded(field);
+        return this.native.getIds(field, this.serializeValue(value));
+    }
+    /**
+     * Resolve a `where` filter to its matching entity ids.
+     *
+     * @param filter - Brainy field-operator filter (eq / allOf / anyOf / ne /
+     *   range / exists / …).
+     * @param opts - OPTIONAL page bound (#75). When `{ limit }` is supplied the
+     *   native producer early-stops and returns only the `[offset, offset+limit)`
+     *   PREFIX of the sorted match set — it short-circuits the posting-list →
+     *   UUID conversion at `offset+limit` instead of materializing the full list
+     *   and slicing, which removes the O(N) FFI marshal for a broad-filter
+     *   single-page `find()` at billion scale. Brainy calls this with
+     *   `{ limit: pageEnd, offset: 0 }` and re-windows the prefix itself
+     *   (visibility filter + slice), so the producer honors the literal `offset`
+     *   (brainy's `0` ⇒ the `[0, limit)` prefix). Omitting `opts` returns the
+     *   FULL match set (back-compat).
+     */
+    async getIdsForFilter(filter, opts) {
+        if (!filter || Object.keys(filter).length === 0)
+            return [];
+        // Ensure the filter's fields are loaded into the native index.
+        const fields = this.extractFilterFields(filter);
+        await this.ensureFieldsLoaded(fields);
+        // Negation operators (ne / exists:false / missing:true) need the all-ids universe.
+        // The native index self-sources it from the canonical in-memory idMapper — no storage
+        // scan, and correct even when the field was never registered on any entity (R4).
+        // `?? null` (not `|| null`) so a `limit`/`offset` of 0 reaches native intact.
+        return this.native.getIdsForFilter(JSON.stringify(filter), null, opts?.limit ?? null, opts?.offset ?? null);
+    }
+    async getIdsForMultipleFields(fieldValuePairs) {
+        if (fieldValuePairs.length === 0)
+            return [];
+        const fields = fieldValuePairs.map(p => p.field);
+        await this.ensureFieldsLoaded(fields);
+        const pairsJson = JSON.stringify(fieldValuePairs.map(p => ({
+            field: p.field,
+            value: p.value
+        })));
+        return this.native.getIdsForMultipleFields(pairsJson);
+    }
+    async getIdsForTextQuery(query) {
+        await this.ensureFieldLoaded('__words__');
+        const resultJson = this.native.getIdsForTextQuery(query);
+        return JSON.parse(resultJson);
+    }
+    async getSortedIdsForFilter(filter, orderBy, order = 'asc', topK) {
+        const filteredIds = await this.getIdsForFilter(filter);
+        if (filteredIds.length === 0)
+            return [];
+        const idValuePairs = [];
+        for (const id of filteredIds) {
+            const value = await this.getFieldValueForEntity(id, orderBy);
+            idValuePairs.push({ id, value });
+        }
+        idValuePairs.sort((a, b) => {
+            if (a.value == null && b.value == null)
+                return 0;
+            if (a.value == null)
+                return order === 'asc' ? 1 : -1;
+            if (b.value == null)
+                return order === 'asc' ? -1 : 1;
+            if (a.value === b.value)
+                return 0;
+            // Numbers compare numerically; everything else by code-point (UTF-8 byte)
+            // order, matching the native column store and brainy's getSortedIdsForFilter
+            // exactly (and deterministic across environments, unlike `<` for strings).
+            let comparison;
+            if (typeof a.value === 'number' && typeof b.value === 'number') {
+                comparison = a.value < b.value ? -1 : 1;
+            }
+            else {
+                comparison = compareCodePoints(String(a.value), String(b.value));
+            }
+            return order === 'asc' ? comparison : -comparison;
+        });
+        // Honor brainy's page bound (#79): brainy passes topK = offset+limit+hidden
+        // so a broad filter + orderBy returning a small page doesn't ship every
+        // matching sorted id across the FFI boundary into brainy's heap. The full
+        // sort still sees all matches (required to pick the true top-K); slicing
+        // bounds the RETURNED set. (Internal O(matches) value-fetch → native
+        // filteredSortTopK primitive is a tracked perf follow-up, #79.)
+        const bounded = topK != null && topK >= 0 ? idValuePairs.slice(0, topK) : idValuePairs;
+        return bounded.map(p => p.id);
+    }
+    /**
+     * Get field value for a specific entity (helper for sorted queries).
+     *
+     * Three-path lookup:
+     *
+     * 1. **Bucketed fields** (timestamps) — the native index stores values
+     *    rounded to coarser buckets to keep the index compact for range
+     *    queries. That bucketing loses precision, so sorting must read the
+     *    actual value directly from entity storage via resolveEntityField.
+     *
+     * 2. **Custom fields with no indexed values** — VFS fields like
+     *    `modified`/`accessed`, plus any user custom field not currently
+     *    present in the native index. Resolved from entity storage via
+     *    resolveEntityField, which knows the top-level-vs-metadata
+     *    shape contract.
+     *
+     * 3. **Indexed fields** — strings, enums, and low-cardinality ints that
+     *    the native index knows about. O(values) lookup.
+     *
+     * Mirrors getFieldValueForEntity in brainy's src/utils/metadataIndex.ts —
+     * keep in lockstep when the entity shape contract evolves.
+     */
+    async getFieldValueForEntity(entityId, field) {
+        // Path 1: Bucketed fields need the actual value from storage.
+        if (BUCKETED_INDEX_FIELDS.has(field)) {
+            const noun = await this.storage.getNoun(entityId);
+            return noun ? resolveEntityField(noun, field) : undefined;
+        }
+        // Path 3: Scan the native index for this entity's value.
+        await this.ensureFieldLoaded(field);
+        const values = this.native.getFilterValues(field);
+        for (const val of values) {
+            const matchIds = this.native.getIds(field, this.serializeValue(val));
+            if (matchIds.includes(entityId)) {
+                return this.denormalizeValue(val, field);
+            }
+        }
+        // Path 2: Not in the native index — fall back to entity storage.
+        // Covers VFS custom fields (modified, accessed) and user fields
+        // that haven't been inserted into the native index yet.
+        const noun = await this.storage.getNoun(entityId);
+        return noun ? resolveEntityField(noun, field) : undefined;
+    }
+    denormalizeValue(normalized, _field) {
+        const asNumber = Number(normalized);
+        if (!isNaN(asNumber))
+            return asNumber;
+        return normalized;
+    }
+    // ==========================================================================
+    // Filter discovery
+    // ==========================================================================
+    async getFilterValues(field) {
+        await this.ensureFieldLoaded(field);
+        return this.native.getFilterValues(field);
+    }
+    async getFilterFields() {
+        return this.native.getFilterFields();
+    }
+    // ==========================================================================
+    // Mutation methods
+    // ==========================================================================
+    async addToIndex(id, entityOrMetadata, skipFlush = false, deferWrites = false) {
+        // Extract field names from entity to know which fields to load
+        const fieldNames = this.native.extractFieldNames(JSON.stringify(entityOrMetadata));
+        await this.ensureFieldsLoaded(fieldNames);
+        // Execute mutation in Rust (single FFI call)
+        const resultJson = this.native.addToIndex(id, JSON.stringify(entityOrMetadata));
+        const result = JSON.parse(resultJson);
+        // Track new fields in loadedFields
+        for (const field of result.newFields) {
+            this.loadedFields.add(field);
+            this.knownFields.add(field);
+        }
+        // Update TS-side type tracking
+        this.updateTypeTracking(id, entityOrMetadata, 'add');
+        // Populate the column store. The native addToIndex above assigned this
+        // entity's int id, so uuidToInt now resolves. Feed the SAME indexable
+        // field→value map brainy's own MetadataIndexManager feeds its column store,
+        // so cor sorts/filters over an identical column set.
+        const intId = this.native.uuidToInt(id);
+        if (intId !== null) {
+            const fieldsMap = this.extractColumnFields(entityOrMetadata);
+            this.columnStore.addEntity(intId, fieldsMap);
+        }
+        // Persist dirty data (unless deferred for batch operations)
+        if (!deferWrites) {
+            await this.persistMutationResult(result);
+        }
+        // Adaptive auto-flush
+        if (!skipFlush) {
+            for (const field of fieldNames) {
+                this.dirtyFields.add(field);
+            }
+            const timeSinceLastFlush = Date.now() - this.lastFlushTime;
+            const shouldAutoFlush = this.dirtyFields.size >= this.autoFlushThreshold ||
+                (this.dirtyFields.size > 10 && timeSinceLastFlush > 5000);
+            if (shouldAutoFlush) {
+                const startTime = Date.now();
+                await this.flush();
+                const flushTime = Date.now() - startTime;
+                if (flushTime < 50) {
+                    this.autoFlushThreshold = Math.min(200, this.autoFlushThreshold * 1.2);
+                }
+                else if (flushTime > 200) {
+                    this.autoFlushThreshold = Math.max(20, this.autoFlushThreshold * 0.8);
+                }
+            }
+        }
+        // **LSM durability (step 5) — bounded-memtable flush trigger.** Drain the
+        // memtable to a durable SSTable the moment its resident estimate crosses
+        // the budget-tracked threshold, so it never climbs to the admission
+        // ceiling and wedges writers (the whole point of the bounded memtable).
+        // Runs on EVERY mutation, INDEPENDENT of `skipFlush` — `skipFlush` governs
+        // only the field-index JSON flush cadence, whereas the memtable bound is a
+        // hard native-memory safety constraint that must also hold on the
+        // deferred-write rebuild path (`addToIndex(id, md, true, true)` per
+        // entity), where a billion-entity re-index would otherwise grow the
+        // memtable unbounded until admission deadlocked the rebuild. Gated on a
+        // persistent engine (`lsmMetaDir !== null`): the non-persistent fallback
+        // has no SSTable surface, so `flushLsm` cannot drain it and the trigger
+        // would spin no-ops. When the field-index auto-flush above already drained
+        // the memtable this session, `memtableBytes()` reads ~0 and this no-ops.
+        const engine = this.lsmEngine;
+        if (engine !== null &&
+            this.lsmMetaDir !== null &&
+            Number(engine.memtableBytes()) >= this.lsmFlushThresholdBytes) {
+            await this.flushLsm();
+        }
+        this.reportLsmMemoryToResourceManager();
+    }
+    async removeFromIndex(id, metadata) {
+        // Capture the int id BEFORE the native removal (which may drop the mapping)
+        // so the column store can tombstone the same entity.
+        const intId = this.native.uuidToInt(id);
+        if (metadata) {
+            const fieldNames = this.native.extractFieldNames(JSON.stringify(metadata));
+            await this.ensureFieldsLoaded(fieldNames);
+            const resultJson = this.native.removeFromIndex(id, JSON.stringify(metadata));
+            const result = JSON.parse(resultJson);
+            this.updateTypeTracking(id, metadata, 'remove');
+            await this.persistMutationResult(result);
+        }
+        else {
+            // Remove from all known fields (slower path). Ensure every registered
+            // field's value-enumeration index is loaded so the native slow-path
+            // remove (which walks `field_indexes` against the LSM snapshot) sees
+            // the complete candidate set.
+            prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`);
+            for (const field of this.knownFields) {
+                await this.ensureFieldLoaded(field);
+            }
+            const resultJson = this.native.removeFromIndex(id);
+            const result = JSON.parse(resultJson);
+            await this.persistMutationResult(result);
+        }
+        // Tombstone in the column store so the entity drops out of sort/filter/range.
+        if (intId !== null) {
+            this.columnStore.removeEntity(intId);
+        }
+        this.reportLsmMemoryToResourceManager();
+    }
+    /**
+     * Report the LSM engine's current native memory footprint
+     * (memtable + reader cache) to the ResourceManager so the dynamic
+     * per-category rebalance can detect pressure on the 'metadata'
+     * category and shift capacity from underutilized neighbors.
+     *
+     * No-op in JSON mode (no engine attached) and after `close()` has
+     * unregistered the subscriber. Called from `addToIndex` /
+     * `removeFromIndex` on the production hot path — the underlying
+     * `nativeMemoryBytes()` is a single atomic load + sum, sub-microsecond
+     * cost.
+     *
+     * The reportNativeMemory call short-circuits inside ResourceManager
+     * if the subscriber name was already cleared (deregistered between
+     * the mutation and this call), so the unregister race is benign.
+     */
+    reportLsmMemoryToResourceManager() {
+        if (this.lsmEngine === null || this.lsmSubscriberName === null)
+            return;
+        const bytes = Number(this.lsmEngine.nativeMemoryBytes());
+        ResourceManager.getInstance().reportNativeMemory(this.lsmSubscriberName, bytes);
+    }
+    updateTypeTracking(id, entityOrMetadata, operation) {
+        const nounValue = entityOrMetadata?.type || entityOrMetadata?.noun;
+        if (!nounValue)
+            return;
+        const entityType = String(nounValue).toLowerCase().trim();
+        if (operation === 'add') {
+            const current = this.totalEntitiesByType.get(entityType) || 0;
+            const newCount = current + 1;
+            this.totalEntitiesByType.set(entityType, newCount);
+            try {
+                const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+            }
+            catch {
+                // Not a recognized noun type
+            }
+            // Update type-field affinity
+            if (!this.typeFieldAffinity.has(entityType)) {
+                this.typeFieldAffinity.set(entityType, new Map());
+            }
+            const typeFields = this.typeFieldAffinity.get(entityType);
+            const fieldNames = this.native.extractFieldNames(JSON.stringify(entityOrMetadata));
+            for (const field of fieldNames) {
+                const count = typeFields.get(field) || 0;
+                typeFields.set(field, count + 1);
+            }
+        }
+        else if (operation === 'remove') {
+            const current = this.totalEntitiesByType.get(entityType) || 0;
+            if (current > 1) {
+                const newCount = current - 1;
+                this.totalEntitiesByType.set(entityType, newCount);
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                }
+                catch { }
+            }
+            else {
+                this.totalEntitiesByType.delete(entityType);
+                this.typeFieldAffinity.delete(entityType);
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = 0;
+                }
+                catch { }
+            }
+        }
+    }
+    // ==========================================================================
+    // Tokenization (delegates to Rust via extractFieldNames, but also exposed)
+    // ==========================================================================
+    tokenize(text) {
+        if (!text)
+            return [];
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, ' ')
+            .split(/\s+/)
+            .filter(w => w.length >= 2 && w.length <= 50)
+            .filter((w, i, arr) => arr.indexOf(w) === i);
+    }
+    hashWord(word) {
+        let hash = 2166136261;
+        for (let i = 0; i < word.length; i++) {
+            hash ^= word.charCodeAt(i);
+            hash = Math.imul(hash, 16777619);
+        }
+        return hash | 0;
+    }
+    extractTextContent(data) {
+        if (data === null || data === undefined)
+            return '';
+        if (typeof data === 'string')
+            return data;
+        if (typeof data === 'number' || typeof data === 'boolean')
+            return String(data);
+        if (Array.isArray(data)) {
+            if (data.length > 0 && typeof data[0] === 'number')
+                return '';
+            return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ');
+        }
+        if (typeof data === 'object') {
+            const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']);
+            const texts = [];
+            for (const [key, value] of Object.entries(data)) {
+                if (skipKeys.has(key) || /^\d+$/.test(key))
+                    continue;
+                const text = this.extractTextContent(value);
+                if (text)
+                    texts.push(text);
+            }
+            return texts.join(' ');
+        }
+        return '';
+    }
+    /**
+     * Whether a field should be indexed, honoring this index's configured
+     * `excludeFields` / `indexedFields`. Mirrors the Rust `should_index_field`
+     * (and brainy's TS `shouldIndexField`) so the column store indexes exactly the
+     * fields the metadata index does.
+     *
+     * @param field - Fully-qualified field name (dotted for nested).
+     * @returns True if the field is indexable under the current config.
+     */
+    shouldIndexField(field) {
+        if (this.config.excludeFields.includes(field))
+            return false;
+        if (this.config.indexedFields.length > 0) {
+            return this.config.indexedFields.includes(field);
+        }
+        return true;
+    }
+    /**
+     * Build the indexable field→value map fed to the column store, identical to
+     * brainy's `MetadataIndexManager.extractIndexableFields()` → `fieldsMap`:
+     *
+     * - Recurses objects; flattens a top-level `metadata` object to the root.
+     * - Maps a top-level `type` field to `noun` (backward-compat with indexes).
+     * - Skips the never-index set (vector/embedding/connections/level/id), purely
+     *   numeric keys (array indices), config-excluded fields, and arrays longer
+     *   than 10 elements (likely vectors/bulk data).
+     * - Small arrays (≤10) become multi-value fields: one array entry per element.
+     * - Adds `__words__` as a multi-value array of FNV-1a int32 word hashes from
+     *   the entity's text content (capped at 5000 words), matching brainy.
+     *
+     * Replicated in TS rather than reusing the Rust extractor because the native
+     * `addToIndex` does not return the extracted pairs and the engine is frozen.
+     *
+     * @param data - The entity or plain-metadata object being indexed.
+     * @returns Field → value (or array of values) map for `columnStore.addEntity`.
+     */
+    extractColumnFields(data) {
+        const fieldsMap = {};
+        const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']);
+        const pushValue = (field, value) => {
+            const existing = fieldsMap[field];
+            if (Array.isArray(existing)) {
+                existing.push(value);
+            }
+            else if (existing !== undefined) {
+                // A second value for a scalar field → promote to a multi-value array,
+                // preserving insertion order (matches brainy feeding repeated pairs).
+                fieldsMap[field] = [existing, value];
+            }
+            else {
+                fieldsMap[field] = value;
+            }
+        };
+        const extract = (obj, prefix = '') => {
+            if (!obj || typeof obj !== 'object')
+                return;
+            for (const [key, value] of Object.entries(obj)) {
+                const fullKey = prefix ? `${prefix}.${key}` : key;
+                if (!prefix && NEVER_INDEX.has(key))
+                    continue;
+                if (/^\d+$/.test(key))
+                    continue;
+                if (!this.shouldIndexField(fullKey))
+                    continue;
+                // Flatten a top-level metadata object to the root (no prefix).
+                if (key === 'metadata' && !prefix && value && typeof value === 'object' && !Array.isArray(value)) {
+                    extract(value, '');
+                    continue;
+                }
+                if (Array.isArray(value)) {
+                    if (value.length > 10)
+                        continue;
+                    for (const item of value) {
+                        if (item !== null && typeof item !== 'object') {
+                            pushValue(fullKey, item);
+                        }
+                    }
+                }
+                else if (value && typeof value === 'object') {
+                    extract(value, fullKey);
+                }
+                else if (value !== null && value !== undefined) {
+                    const indexField = !prefix && key === 'type' ? 'noun' : fullKey;
+                    pushValue(indexField, value);
+                }
+            }
+        };
+        extract(data);
+        // Hybrid text-search words: FNV-1a int32 hashes, multi-value, capped at 5000.
+        const textContent = this.extractTextContent(data);
+        if (textContent) {
+            const MAX_WORDS_PER_ENTITY = 5000;
+            const words = this.tokenize(textContent).slice(0, MAX_WORDS_PER_ENTITY);
+            if (words.length > 0) {
+                fieldsMap.__words__ = words.map(w => this.hashWord(w));
+            }
+        }
+        return fieldsMap;
+    }
+    // ==========================================================================
+    // Entity counting
+    // ==========================================================================
+    getEntityCountByType(type) {
+        return this.totalEntitiesByType.get(type) || 0;
+    }
+    getTotalEntityCount() {
+        let total = 0;
+        for (const count of this.totalEntitiesByType.values()) {
+            total += count;
+        }
+        return total;
+    }
+    getAllEntityCounts() {
+        return new Map(this.totalEntitiesByType);
+    }
+    getEntityCountByTypeEnum(type) {
+        const index = TypeUtils.getNounIndex(type);
+        return this.entityCountsByTypeFixed[index];
+    }
+    getVerbCountByTypeEnum(type) {
+        const index = TypeUtils.getVerbIndex(type);
+        return this.verbCountsByTypeFixed[index];
+    }
+    getTopNounTypes(n) {
+        const types = [];
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        return types.sort((a, b) => b.count - a.count).slice(0, n).map(t => t.type);
+    }
+    getTopVerbTypes(n) {
+        const types = [];
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        return types.sort((a, b) => b.count - a.count).slice(0, n).map(t => t.type);
+    }
+    getAllNounTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                counts.set(TypeUtils.getNounFromIndex(i), count);
+            }
+        }
+        return counts;
+    }
+    getAllVerbTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                counts.set(TypeUtils.getVerbFromIndex(i), count);
+            }
+        }
+        return counts;
+    }
+    // ==========================================================================
+    // VFS Statistics (roaring bitmap intersection via Rust)
+    // ==========================================================================
+    async getVFSEntityCountByType(type) {
+        await this.ensureFieldsLoaded(['isVFSEntity', 'noun']);
+        const vfsIds = this.native.getIds('isVFSEntity', JSON.stringify(true));
+        const typeIds = this.native.getIds('noun', JSON.stringify(type));
+        const vfsSet = new Set(vfsIds);
+        return typeIds.filter(id => vfsSet.has(id)).length;
+    }
+    async getAllVFSEntityCounts() {
+        await this.ensureFieldsLoaded(['isVFSEntity', 'noun']);
+        const vfsIds = this.native.getIds('isVFSEntity', JSON.stringify(true));
+        if (vfsIds.length === 0)
+            return new Map();
+        const vfsSet = new Set(vfsIds);
+        const result = new Map();
+        for (const type of this.totalEntitiesByType.keys()) {
+            const typeIds = this.native.getIds('noun', JSON.stringify(type));
+            const count = typeIds.filter(id => vfsSet.has(id)).length;
+            if (count > 0)
+                result.set(type, count);
+        }
+        return result;
+    }
+    async getTotalVFSEntityCount() {
+        await this.ensureFieldLoaded('isVFSEntity');
+        // Native count — no per-entity UUID materialization (#78 pt2).
+        return this.native.getIdsCount('isVFSEntity', JSON.stringify(true));
+    }
+    // ==========================================================================
+    // Criteria-based counting
+    // ==========================================================================
+    async getCountForCriteria(field, value) {
+        // Native count — mirrors getIds(field, value).length exactly but allocates
+        // no UUID strings (#78 pt2).
+        await this.ensureFieldLoaded(field);
+        return this.native.getIdsCount(field, this.serializeValue(value));
+    }
+    // ==========================================================================
+    // Flush
+    // ==========================================================================
+    async flush() {
+        // Always save field registry + msync the entity ID mapper — even with
+        // no dirty fields. The field registry is the critical file init() needs
+        // to discover persisted indices. Without it, the metadata index appears
+        // empty after restart.
+        if (this.knownFields.size > 0) {
+            await this.saveFieldRegistry();
+        }
+        // #72 Phase C: msync the mmap BinaryIdMapper FIRST, before the bitmaps /
+        // column segments that reference its ints — so a crash can never leave a
+        // persisted bitmap pointing at an int the mapper has not yet durably
+        // assigned (which would silently drop those rows on reopen).
+        this.idMapperNative?.flush();
+        // LSM durability (steps 3-4): drain the memtable to a durable SSTable +
+        // manifest. Runs AFTER the id-mapper msync above (the SSTable's posting
+        // lists reference entity ints that must be durably assigned first) and
+        // BEFORE the dirty-field early-return below (the memtable can hold
+        // un-persisted writes even when no field index is dirty — e.g. a
+        // deferred-write batch). No-op when the memtable is empty or on the
+        // non-persistent fallback engine.
+        await this.flushLsm();
+        // Flush the column store unconditionally — drains tail buffers to L0 `.cidx`
+        // segments and persists manifests + deleted bitmaps. Must run even when no
+        // field indexes are dirty (e.g. a deferred-write batch buffered column data
+        // without touching the dirty-field set), so orderBy survives a reopen.
+        await this.columnStore.flush();
+        if (this.dirtyFields.size === 0)
+            return;
+        // Save dirty field indexes
+        const promises = [];
+        for (const field of this.dirtyFields) {
+            const json = this.native.saveFieldIndex(field);
+            if (json) {
+                const indexId = `__metadata_field_index__field_${field}`;
+                promises.push(this.storage.saveMetadata(indexId, JSON.parse(json)));
+            }
+        }
+        await Promise.all(promises);
+        // Save field registry (the mmap mapper was already msync'd above).
+        await this.saveFieldRegistry();
+        this.dirtyFields.clear();
+        this.lastFlushTime = Date.now();
+    }
+    /**
+     * Persist all state and release the column store's resources. Brainy's
+     * shutdown path calls `metadataIndex.close()` when present; flushing here
+     * guarantees the column store's tail buffers reach disk before the process
+     * exits, so a reopened index can serve `orderBy` from persisted segments.
+     */
+    async close() {
+        await this.flush();
+        await this.columnStore.close();
+        // P3-5: deregister the ResourceManager subscriber so the rebalance
+        // loop stops calling our callback after shutdown. The engine
+        // itself stays alive as long as `this.native` (via setLsmEngine's
+        // Arc clone) or any held snapshot holds a reference — extra
+        // post-close setAllowedBytes calls would be harmless but we want
+        // the loop to be efficient. Unsubscribing is idempotent; the
+        // returned deregistration function silently no-ops on second call.
+        if (this.unsubscribeFromBudget !== null) {
+            this.unsubscribeFromBudget();
+            this.unsubscribeFromBudget = null;
+            this.lsmSubscriberName = null;
+        }
+    }
+    // ==========================================================================
+    // Stats
+    // ==========================================================================
+    async getStats() {
+        const statsJson = this.native.getStats();
+        const stats = JSON.parse(statsJson);
+        return {
+            totalEntries: stats.totalEntries || 0,
+            totalIds: 0,
+            fieldsIndexed: stats.fieldsIndexed || [],
+            lastRebuild: Date.now(),
+            indexSize: (stats.totalEntries || 0) * 100
+        };
+    }
+    /**
+     * @description Report which index path a `find({ where: { [field]: ... } })`
+     * clause will hit. Brainy's `brain.explain()` calls this per `where` field so an
+     * operator can see *before* running a query whether the field has any index
+     * entries at all — a `where` on an unindexed field returns `[]` silently, and
+     * `path: 'none'` gives that empty result an explanation.
+     *
+     * The native engine serves all `where` lookups from the column store (the
+     * single source of truth — there is no separate legacy sparse-index path at
+     * query time, unlike brainy's TypeScript baseline). So a field either has
+     * column-store entries (`'column-store'`) or none (`'none'`). The field's
+     * segments are loaded first so `hasField` reflects on-disk state, not just
+     * what has been touched this session.
+     *
+     * @param field - The `where`-clause field name to explain.
+     * @returns The serving path and an operator-facing note.
+     * @example
+     * const { path, notes } = await metadataIndex.explainField('status')
+     * if (path === 'none') console.warn(notes)
+     */
+    async explainField(field) {
+        await this.ensureFieldLoaded(field);
+        if (this.columnStore.hasField(field)) {
+            return {
+                path: 'column-store',
+                notes: 'O(log n) binary search + roaring bitmap (native Rust column store). Best path.'
+            };
+        }
+        return {
+            path: 'none',
+            notes: `No index entries for field "${field}". A find({ where: { ${field}: ... } }) ` +
+                `will return an empty result regardless of whether matching entities exist on disk. ` +
+                `Likely causes: (1) the writer registered the field in memory but has not flushed; ` +
+                `(2) the field name does not match what was written (typo or casing); ` +
+                `(3) the field is genuinely absent from all entities. Call requestFlush() on the ` +
+                `writer or call brain.flush() before relying on the result.`
+        };
+    }
+    // ==========================================================================
+    // Validation
+    // ==========================================================================
+    async validateConsistency() {
+        const entityCount = this.native.entityIdMapperSize();
+        if (entityCount === 0) {
+            return {
+                healthy: true,
+                avgEntriesPerEntity: 0,
+                entityCount: 0,
+                indexEntryCount: 0,
+                recommendation: null
+            };
+        }
+        // Load all fields to count entries
+        for (const field of this.knownFields) {
+            await this.ensureFieldLoaded(field);
+        }
+        const statsJson = this.native.getStats();
+        const stats = JSON.parse(statsJson);
+        const indexEntryCount = stats.totalEntries || 0;
+        const avgEntriesPerEntity = indexEntryCount / entityCount;
+        const CORRUPTION_THRESHOLD = 100;
+        const healthy = avgEntriesPerEntity <= CORRUPTION_THRESHOLD;
+        let recommendation = null;
+        if (!healthy) {
+            recommendation = `Index corruption detected (${avgEntriesPerEntity.toFixed(1)} avg entries/entity, expected ~30). ` +
+                `Run brain.index.clearAllIndexData() followed by brain.index.rebuild() to repair.`;
+        }
+        return { healthy, avgEntriesPerEntity, entityCount, indexEntryCount, recommendation };
+    }
+    /**
+     * O(1)-ish cross-bucket consistency probe (brainy rc.6 matrix 7.7).
+     *
+     * Brainy calls this ONCE on the first read; on `false` it runs
+     * {@link detectAndRepairCorruption}. The check is fully internal — it never
+     * re-reads entity data. It loads a bounded set of value-enumeration field
+     * indexes (so the native sampler can enumerate each field's value buckets),
+     * biased toward `noun` (the canonical single-valued type field where the
+     * staff→timeslot class of phantom lands), then defers to the native
+     * {@link NativeMetadataIndexInstance.probeConsistency} bounded sampler.
+     *
+     * Best-effort: any error (a sampling glitch, a storage hiccup loading a field
+     * index) degrades to `true` so the probe NEVER breaks brainy's read path.
+     *
+     * @returns `true` if the index looks consistent (or the check is
+     *   indeterminate), `false` if a cross-bucket phantom was sampled.
+     */
+    async probeConsistency() {
+        try {
+            // Load value-enumeration field indexes so the native sampler can
+            // enumerate each field's buckets. Bounded to keep the probe cheap: at
+            // most PROBE_MAX_FIELDS small field-index JSON blobs, `noun` first. The
+            // native check itself is O(1)-ish (a bounded sample of ints against a
+            // bounded set of buckets) — it never walks the full index.
+            const PROBE_MAX_FIELDS = 64;
+            let loaded = 0;
+            if (this.knownFields.has('noun')) {
+                await this.ensureFieldLoaded('noun');
+                loaded++;
+            }
+            for (const field of this.knownFields) {
+                if (loaded >= PROBE_MAX_FIELDS)
+                    break;
+                if (field === 'noun' || field === '__words__')
+                    continue;
+                await this.ensureFieldLoaded(field);
+                loaded++;
+            }
+            return this.native.probeConsistency();
+        }
+        catch (error) {
+            // A probe must never throw into the read path — assume healthy.
+            prodLog.debug(`[probeConsistency] benign sampling error — assuming healthy: ${error}`);
+            return true;
+        }
+    }
+    async detectAndRepairCorruption() {
+        const validation = await this.validateConsistency();
+        if (!validation.healthy) {
+            prodLog.warn(`Index corruption detected (${validation.avgEntriesPerEntity.toFixed(1)} avg entries/entity)`);
+            prodLog.warn('Auto-rebuilding index to repair...');
+            await this.clearAllIndexData();
+            await this.rebuild();
+            const postRebuild = await this.validateConsistency();
+            if (postRebuild.healthy) {
+                prodLog.info(`Index rebuilt successfully (${postRebuild.avgEntriesPerEntity.toFixed(1)} avg entries/entity)`);
+            }
+            else {
+                prodLog.error(`Index still appears corrupted after rebuild (${postRebuild.avgEntriesPerEntity.toFixed(1)} avg entries/entity).`);
+            }
+            // The full rebuild re-indexed every entity from storage (the source of
+            // truth) — each entity lands in exactly its current value bucket, so no
+            // cross-bucket phantom can survive. The targeted purge below is therefore
+            // redundant after a rebuild.
+            return;
+        }
+        // The avg-entries heuristic above catches GROSS corruption (a runaway
+        // entries/entity ratio) but NOT the cross-bucket phantom: one stale posting
+        // that survived a value change barely moves the ratio. Run the targeted
+        // state-level purge so brainy's cold-open self-heal (matrix 7.8b) repairs a
+        // pre-poisoned index at the STATE level, not just masking it at query time.
+        await this.purgeCrossBucketPhantoms();
+    }
+    /**
+     * State-level cross-bucket phantom purge (brainy matrix 7.8b).
+     *
+     * Scans the native index for entities whose int sits in more than one value
+     * bucket of a field, then tombstones the stale buckets against each entity's
+     * CURRENT stored value(s) — read straight from storage (brainy's source of
+     * truth), NOT from the index being repaired. A multi-valued field keeps all
+     * its stored values (a no-op); a single-valued phantom is purged down to the
+     * one current value; an orphaned int (entity gone) is evicted entirely.
+     *
+     * Only reached after {@link probeConsistency} flagged the index, so the
+     * full field-index load + scan is acceptable here (unlike the O(1) probe).
+     */
+    async purgeCrossBucketPhantoms() {
+        // The scan enumerates each field's value buckets from the in-memory
+        // value-enumeration surface, so every known field must be loaded first
+        // (mirrors validateConsistency's load-all).
+        for (const field of this.knownFields) {
+            await this.ensureFieldLoaded(field);
+        }
+        // Cap the scan so a massively-corrupt index produces a bounded list; the
+        // remainder is healed on a subsequent cold-open (probe → repair again).
+        const PHANTOM_SCAN_CAP = 100_000;
+        let candidates;
+        try {
+            candidates = JSON.parse(this.native.detectCrossBucketPhantoms(PHANTOM_SCAN_CAP));
+        }
+        catch (error) {
+            prodLog.warn(`[purgeCrossBucketPhantoms] scan failed (skipping): ${error}`);
+            return;
+        }
+        if (candidates.length === 0)
+            return;
+        prodLog.warn(`[purgeCrossBucketPhantoms] ${candidates.length} cross-bucket candidate(s) — repairing against stored values`);
+        const repairedFields = new Set();
+        let purged = 0;
+        for (const { uuid, field } of candidates) {
+            // The entity's CURRENT value(s) for this field — brainy's source of
+            // truth, read from storage (NOT the index we're repairing). For `noun`
+            // resolveEntityField maps to entity.type; other fields read entity
+            // metadata. An empty result (entity gone / field dropped) keeps nothing,
+            // so the orphan int is evicted from every bucket of the field.
+            let currentValues = [];
+            try {
+                const noun = await this.storage.getNoun(uuid);
+                if (noun) {
+                    const raw = resolveEntityField(noun, field);
+                    if (raw !== undefined && raw !== null) {
+                        currentValues = Array.isArray(raw) ? raw : [raw];
+                    }
+                }
+            }
+            catch (error) {
+                prodLog.debug(`[purgeCrossBucketPhantoms] could not read stored ${uuid}.${field}: ${error}`);
+            }
+            try {
+                const didPurge = this.native.repairEntityFieldPhantom(uuid, field, JSON.stringify(currentValues));
+                if (didPurge) {
+                    purged++;
+                    repairedFields.add(field);
+                }
+            }
+            catch (error) {
+                prodLog.warn(`[purgeCrossBucketPhantoms] repair of ${uuid}.${field} failed: ${error}`);
+            }
+        }
+        // Persist the value-enumeration field indexes the purge mutated so a cold
+        // reopen doesn't re-enumerate an emptied stale value.
+        for (const field of repairedFields) {
+            const json = this.native.saveFieldIndex(field);
+            if (json) {
+                await this.storage.saveMetadata(`__metadata_field_index__field_${field}`, JSON.parse(json));
+            }
+        }
+        if (purged > 0) {
+            prodLog.info(`[purgeCrossBucketPhantoms] purged ${purged} stale cross-bucket posting set(s) across ${repairedFields.size} field(s)`);
+            // Drain the eviction tombstones to a durable SSTable so the repair
+            // survives a restart even without further writes.
+            await this.flush();
+        }
+    }
+    // ==========================================================================
+    // Rebuild
+    // ==========================================================================
+    async rebuild() {
+        if (this.skipRebuildOnce) {
+            // getIdMapper().rebuild() (brainy restore() mapper hook) already ran the
+            // full rebuild this cycle; skipping here keeps the entity-id mapper stable
+            // for the concurrent graphIndex.rebuild() in brainy's Promise.all instead
+            // of re-clearing it mid-flight (which silently drops resolved edges). (#68)
+            this.skipRebuildOnce = false;
+            prodLog.debug('NativeMetadataIndex.rebuild(): skipped — getIdMapper().rebuild() already rebuilt this restore cycle');
+            return;
+        }
+        if (this.isRebuilding)
+            return;
+        this.isRebuilding = true;
+        try {
+            prodLog.info('Starting metadata index rebuild with batch processing...');
+            // **LSM durability (step 6) — LSM-aware rebuild teardown.** A rebuild
+            // commonly follows brainy's `restore()`, which WIPES and replaces the
+            // storage tree under the still-live engine. `native.clear()` below
+            // cannot reset the attached engine (it owns its memtable + live-SSTable
+            // set out of `clear()`'s reach), so the engine must be re-opened fresh
+            // against the new tree BEFORE the re-index walk — otherwise its stale
+            // mmap'd readers (which survive the files' unlink) keep serving
+            // pre-restore rows and the stale manifest dangles. {@link
+            // reopenLsmEngineFresh} wipes the stale `_metadata/sstables/*` + manifest
+            // + memtable logs, recreates the memtable dir, opens a fresh
+            // `openWithDir` engine, and re-attaches it. No-op on the non-persistent
+            // fallback. (Supersedes the steps-3+4 minimal memtable-dir recreate —
+            // the wipe+re-open subsumes the `mkdir`.)
+            await this.reopenLsmEngineFresh();
+            // Clear in-memory index state (field registry + value enumeration).
+            // #72 Phase C: native.clear() does NOT reset the shared mmap id-mapper
+            // (it has no in-place clear, and truncating its files while the old mmap
+            // is still mapped would risk the kernel flushing stale MAP_SHARED pages
+            // over fresh data). The mapper persists, so the rebuild's addToIndex
+            // calls re-resolve each surviving UUID to its SAME int via get_or_assign
+            // — ints are stable across rebuild. The column store is still reset +
+            // repopulated below so its segments are rebuilt cleanly from those
+            // (identical) ints. The only residue is orphaned int→uuid entries for
+            // entities dropped by a restore-to-different-snapshot — a bounded space
+            // leak, never a read-correctness issue.
+            this.native.clear();
+            await this.columnStore.reset();
+            this.loadedFields.clear();
+            this.knownFields.clear();
+            this.dirtyFields.clear();
+            this.totalEntitiesByType.clear();
+            this.entityCountsByTypeFixed.fill(0);
+            this.verbCountsByTypeFixed.fill(0);
+            this.typeFieldAffinity.clear();
+            this.unifiedCache.clear('metadata');
+            // Delete existing persisted field indexes from storage
+            prodLog.info('Clearing existing metadata field indexes from storage...');
+            const existingFields = await this.getPersistedFieldList();
+            if (existingFields.length > 0) {
+                for (const field of existingFields) {
+                    await this.deletePersistedFieldIndex(field);
+                }
+                try {
+                    await this.storage.saveMetadata('__metadata_field_registry__', null);
+                }
+                catch { }
+                prodLog.info(`Cleared ${existingFields.length} field indexes from storage`);
+            }
+            // #72 Phase C: there is no `brainy:entityIdMapper` JSON blob to clear —
+            // the mapper is the mmap BinaryIdMapper, which persists across rebuild.
+            // Rebuild noun + verb metadata indexes via ONE cursor-paginated path for
+            // EVERY storage type (see rebuildEntityMetadata). The previous
+            // local-vs-cloud fork OOM'd / silently TRUNCATED at scale: the local
+            // branch loaded the whole corpus in a single getNouns/getVerbs({limit:1e6})
+            // call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
+            // batch(25) ≈ 250K. (#74 — perf-at-all-scales mandate.)
+            const totalNounsProcessed = await this.rebuildEntityMetadata('noun');
+            const totalVerbsProcessed = await this.rebuildEntityMetadata('verb');
+            // Final flush
+            await this.flushRebuildDirty();
+            await this.flush();
+            prodLog.info(`Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
+        }
+        finally {
+            this.isRebuilding = false;
+        }
+    }
+    /**
+     * Cursor-paginate every entity of a kind ('noun'|'verb') through the metadata
+     * index during rebuild. ONE streaming path for ALL storage types — replaces
+     * the old local-vs-cloud fork that OOM'd / silently TRUNCATED at scale: the
+     * local branch loaded the whole corpus in a single `getNouns/getVerbs({limit:
+     * 1e6})` call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
+     * batch(25) ≈ 250K. Cursor pagination is O(page) RAM and complete at any
+     * scale; it terminates on `hasMore === false` (8.0's cursor is no-rescan), with
+     * an empty-page backstop. (#74 — perf-at-all-scales mandate.)
+     * @returns the number of entities indexed (those carrying metadata).
+     */
+    async rebuildEntityMetadata(kind) {
+        const PAGE = 1000;
+        let cursor = undefined;
+        let hasMore = true;
+        let processed = 0;
+        let sinceFlush = 0;
+        while (hasMore) {
+            const page = kind === 'noun'
+                ? await this.storage.getNouns({ pagination: { limit: PAGE, cursor } })
+                : await this.storage.getVerbs({ pagination: { limit: PAGE, cursor } });
+            const items = page.items;
+            if (items.length === 0)
+                break; // empty page = done (defensive backstop)
+            const ids = items.map((it) => it.id);
+            const md = await this.loadRebuildMetadataBatch(kind, ids);
+            for (const it of items) {
+                const metadata = md.get(it.id);
+                if (metadata) {
+                    await this.addToIndex(it.id, metadata, true, true);
+                    processed++;
+                    if (++sinceFlush >= 5000) {
+                        await this.flushRebuildDirty();
+                        sinceFlush = 0;
+                    }
+                }
+            }
+            hasMore = page.hasMore;
+            cursor = page.nextCursor;
+            await this.yieldToEventLoop();
+        }
+        return processed;
+    }
+    /**
+     * Batch-load metadata for a page of noun/verb ids — prefers the storage
+     * adapter's batch API, falls back to per-id reads.
+     */
+    async loadRebuildMetadataBatch(kind, ids) {
+        if (kind === 'noun') {
+            if (this.storage.getMetadataBatch)
+                return this.storage.getMetadataBatch(ids);
+        }
+        else if (this.storage.getVerbMetadataBatch) {
+            return this.storage.getVerbMetadataBatch(ids);
+        }
+        const m = new Map();
+        for (const id of ids) {
+            try {
+                const md = kind === 'noun'
+                    ? await this.storage.getNounMetadata(id)
+                    : await this.storage.getVerbMetadata(id);
+                if (md)
+                    m.set(id, md);
+            }
+            catch { }
+        }
+        return m;
+    }
+    /**
+     * During rebuild, msync the durable state accumulated so far.
+     *
+     * Posting lists land in the LSM engine's memtable (drained to durable
+     * SSTables by the step-5 size-flush trigger as the re-index walk proceeds)
+     * and the entity ints land in the mmap id mapper — so the only durable
+     * write to force here is the mapper msync. The value-enumeration field
+     * indexes + the LSM memtable tail are persisted by the final {@link flush}
+     * at the end of the rebuild.
+     */
+    async flushRebuildDirty() {
+        // #72 Phase C: msync the mmap mapper (no JSON blob to write).
+        this.idMapperNative?.flush();
+    }
+    // ==========================================================================
+    // Clear
+    // ==========================================================================
+    async clearAllIndexData() {
+        prodLog.warn('Clearing ALL metadata index data from storage...');
+        const fields = await this.getPersistedFieldList();
+        for (const field of fields) {
+            await this.deletePersistedFieldIndex(field);
+        }
+        try {
+            await this.storage.saveMetadata('__metadata_field_registry__', null);
+        }
+        catch { }
+        // Clear in-memory index state. #72 Phase C: the shared mmap id-mapper
+        // persists (native.clear() no longer resets it); the rebuild that
+        // follows this repair re-resolves each surviving UUID to its SAME int,
+        // so the reset column store repopulates consistently.
+        this.native.clear();
+        await this.columnStore.reset();
+        this.loadedFields.clear();
+        this.knownFields.clear();
+        this.dirtyFields.clear();
+        this.totalEntitiesByType.clear();
+        this.entityCountsByTypeFixed.fill(0);
+        this.verbCountsByTypeFixed.fill(0);
+        this.typeFieldAffinity.clear();
+        this.unifiedCache.clear('metadata');
+        prodLog.info(`Cleared ${fields.length} field indexes and all in-memory state`);
+    }
+    async getPersistedFieldList() {
+        try {
+            const registry = await this.storage.getMetadata('__metadata_field_registry__');
+            if (!registry?.fields || !Array.isArray(registry.fields))
+                return [];
+            return registry.fields.filter((f) => typeof f === 'string' && f.length > 0);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Delete a field's persisted value-enumeration index
+     * (`__metadata_field_index__field_<field>`) from storage. Called by
+     * {@link rebuild} and {@link clearAllIndexData} to wipe the JSON
+     * value-enumeration surface before re-indexing; the posting lists
+     * themselves are wiped from the LSM engine separately
+     * ({@link reopenLsmEngineFresh}).
+     */
+    async deletePersistedFieldIndex(field) {
+        try {
+            await this.storage.saveMetadata(`__metadata_field_index__field_${field}`, null);
+        }
+        catch {
+            prodLog.debug(`Could not clear field index for field '${field}'`);
+        }
+    }
+    // ==========================================================================
+    // Field statistics (TS-only analytics)
+    // ==========================================================================
+    async getFieldStatistics() {
+        for (const field of this.knownFields) {
+            if (!this.fieldStats.has(field)) {
+                this.fieldStats.set(field, {
+                    cardinality: {
+                        uniqueValues: 0,
+                        totalValues: 0,
+                        distribution: 'uniform',
+                        updateFrequency: 0,
+                        lastAnalyzed: Date.now()
+                    },
+                    queryCount: 0,
+                    rangeQueryCount: 0,
+                    exactQueryCount: 0,
+                    avgQueryTime: 0,
+                    indexType: 'hash'
+                });
+            }
+        }
+        return new Map(this.fieldStats);
+    }
+    async getFieldCardinality(field) {
+        const stats = this.fieldStats.get(field);
+        return stats ? stats.cardinality : null;
+    }
+    async getFieldsWithCardinality() {
+        const fields = [];
+        for (const [field, stats] of this.fieldStats) {
+            fields.push({
+                field,
+                cardinality: stats.cardinality.uniqueValues,
+                distribution: stats.cardinality.distribution
+            });
+        }
+        fields.sort((a, b) => a.cardinality - b.cardinality);
+        return fields;
+    }
+    async getOptimalQueryPlan(filters) {
+        const fieldOrder = [];
+        let hasRangeQueries = false;
+        let totalEstimatedCost = 0;
+        for (const [field, value] of Object.entries(filters)) {
+            const stats = this.fieldStats.get(field);
+            if (!stats)
+                continue;
+            if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                hasRangeQueries = true;
+            }
+            const cardinality = stats.cardinality.uniqueValues;
+            totalEstimatedCost += Math.log2(Math.max(1, cardinality));
+            fieldOrder.push(field);
+        }
+        fieldOrder.sort((a, b) => {
+            const statsA = this.fieldStats.get(a);
+            const statsB = this.fieldStats.get(b);
+            if (!statsA || !statsB)
+                return 0;
+            return statsA.cardinality.uniqueValues - statsB.cardinality.uniqueValues;
+        });
+        return {
+            strategy: hasRangeQueries ? 'hybrid' : 'exact',
+            fieldOrder,
+            estimatedCost: totalEstimatedCost
+        };
+    }
+    async getFieldsForType(nounType) {
+        const typeFields = this.typeFieldAffinity.get(nounType);
+        const totalEntities = this.totalEntitiesByType.get(nounType);
+        if (!typeFields || !totalEntities)
+            return [];
+        const result = [];
+        for (const [field, count] of typeFields.entries()) {
+            result.push({
+                field,
+                affinity: count / totalEntities,
+                occurrences: count,
+                totalEntities
+            });
+        }
+        result.sort((a, b) => b.affinity - a.affinity);
+        return result;
+    }
+    // ==========================================================================
+    // Private helpers
+    // ==========================================================================
+    async yieldToEventLoop() {
+        return new Promise(resolve => setImmediate(resolve));
+    }
+}
+//# sourceMappingURL=NativeMetadataIndex.js.map