@soulcraft/cor 3.0.0-rc.1

package/dist/utils/NativeMetadataIndex.d.ts

@@ -0,0 +1,882 @@
+/**
+ * NativeMetadataIndex — TypeScript wrapper around the Rust NativeMetadataIndex.
+ *
+ * Implements the same public API as the old MetadataIndexManager (3,721 lines),
+ * delegating core operations (query, mutation, normalization) to Rust.
+ *
+ * Architecture:
+ * - Rust owns: posting-list algebra, filter evaluation, value normalization,
+ *   field extraction, text search, entity ID mapping, and the LSM engine
+ *   (the sole metadata storage engine — SSTables + manifest)
+ * - TS owns: async storage I/O, LSM SSTable/manifest persistence, lazy
+ *   value-enumeration field-index loading, rebuild orchestration, flush,
+ *   getSortedIdsForFilter (loads sort values from storage)
+ *
+ * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
+ * Rust operates in-memory, returns serialized state for TS to persist.
+ */
+import type { StorageAdapter } from '@soulcraft/brainy';
+import { NounType, VerbType } from '@soulcraft/brainy/types/graphTypes';
+import { NativeColumnStore } from './NativeColumnStore.js';
+import type { EntityIdMapperLike } from './columnStoreTypes.js';
+import type { MetadataIndexProvider } from '../providerContracts.js';
+import type { MigrationCoordinator, MigrationStatus } from '../migration/MigrationCoordinator.js';
+export interface MetadataIndexEntry {
+    field: string;
+    value: string | number | boolean;
+    ids: Set<string>;
+    lastUpdated: number;
+}
+export interface FieldIndexData {
+    values: Record<string, number>;
+    lastUpdated: number;
+}
+export interface MetadataIndexStats {
+    totalEntries: number;
+    totalIds: number;
+    fieldsIndexed: string[];
+    lastRebuild: number;
+    indexSize: number;
+}
+export interface MetadataIndexConfig {
+    maxIndexSize?: number;
+    rebuildThreshold?: number;
+    autoOptimize?: boolean;
+    indexedFields?: string[];
+    excludeFields?: string[];
+}
+interface CardinalityInfo {
+    uniqueValues: number;
+    totalValues: number;
+    distribution: 'uniform' | 'skewed' | 'sparse';
+    updateFrequency: number;
+    lastAnalyzed: number;
+}
+interface FieldStats {
+    cardinality: CardinalityInfo;
+    queryCount: number;
+    rangeQueryCount: number;
+    exactQueryCount: number;
+    avgQueryTime: number;
+    indexType: 'hash';
+    normalizationStrategy?: 'none' | 'precision' | 'bucket';
+}
+/**
+ * MetadataIndexManager — native Rust implementation with TS storage bridge.
+ *
+ * Drop-in replacement for the old pure-TS MetadataIndexManager.
+ * All bitmap AND/OR/NOT operations execute in Rust without crossing FFI.
+ *
+ * Implements {@link MetadataIndexProvider} so a compile error fires the moment
+ * this wrapper drops or drifts a member brainy calls via `this.metadataIndex.*`.
+ */
+/**
+ * **Piece 11 closeout — init-time gate.** Thrown by
+ * {@link MetadataIndexManager.init} when the brain carries cortex 2.x legacy
+ * JSON metadata state on disk. cor 3.0 uses the LSM engine exclusively (the
+ * legacy JSON chunk engine has been removed), so an un-migrated brain must be
+ * migrated rather than silently switched. Catchable for graceful fallback
+ * paths in callers that prefer a one-shot warning over a hard refusal; the
+ * default behaviour (uncaught) is the right one for production deployments —
+ * silent degradation to an empty LSM engine would mask the on-disk JSON state
+ * and surface as a mysteriously-empty brain at first query.
+ */
+export declare class MetadataIndexLegacyJsonStateDetected extends Error {
+    readonly name = "MetadataIndexLegacyJsonStateDetected";
+}
+export declare class MetadataIndexManager implements MetadataIndexProvider {
+    /**
+     * Lazy resolver to the shared {@link MigrationCoordinator} (the plugin creates
+     * one per brain and hands the same getter to all three derived-index providers).
+     * Undefined for a raw-napi construction (tests) or a tier that never migrates —
+     * then {@link isMigrating} is always `false` and the epoch guard's large-brain
+     * outcome falls through to Brainy's own path.
+     */
+    private readonly migrationCoordinatorGetter?;
+    private storage;
+    private config;
+    private native;
+    private isRebuilding;
+    /**
+     * **#18** — set true by {@link guardIndexEpoch} outcome 4 (large stale brain,
+     * auto-migrate enabled) and read once at the END of {@link init}, which kicks the
+     * coordinated migration only AFTER the index is fully constructed. Flagging here
+     * (not kicking inline) avoids racing the migration's destructive rebuild
+     * (`reopenLsmEngineFresh` + `columnStore.reset`) against the remainder of the same
+     * init() (`columnStore.init` / `warmCache` / `lazyLoadCounts`) on shared state.
+     */
+    private epochMigrationPending;
+    private lastFlushTime;
+    private autoFlushThreshold;
+    private dirtyFields;
+    private loadedFields;
+    private knownFields;
+    /**
+     * **Piece 6 generation counter — reader-side cache.** Records the
+     * generation observed at the most recent `invalidateIfStale` call.
+     * Mutations on the native side bump `core.generation` via
+     * `bump_generation`; a subsequent reader's `invalidateIfStale` call
+     * sees a delta and clears `loadedFields`, forcing the next query to
+     * re-load from the current state instead of trusting the lazy-load
+     * snapshot.
+     *
+     * Default `-1n` means "never observed"; the first call will see any
+     * non-zero current generation and trigger an initial invalidate.
+     * (We don't invalidate on the very first call regardless because
+     * `loadedFields` is empty anyway.)
+     */
+    private lastSeenGeneration;
+    /**
+     * **Piece 15 — per-field generation cache.** Keyed by field name,
+     * value is the per-field generation last observed via
+     * `native.fieldGeneration(field)`. Updated on every
+     * {@link invalidateFieldIfStale} call.
+     *
+     * Selective invalidation contract: when a probe of field X's
+     * counter advances, ONLY X is removed from `loadedFields` — every
+     * other loaded field stays valid. This is the load-bearing
+     * difference vs the index-wide {@link lastSeenGeneration} +
+     * {@link invalidateIfStale} path, which clears the entire
+     * `loadedFields` Set on any mutation.
+     *
+     * Cold-boot: an empty Map. The first probe of any field records
+     * its current native counter without invalidating (matches the
+     * `lastSeenGeneration: -1n` semantic — never-observed implies
+     * loadedFields is empty anyway).
+     */
+    private lastSeenFieldGenerations;
+    private fieldStats;
+    private typeFieldAffinity;
+    private totalEntitiesByType;
+    private entityCountsByTypeFixed;
+    private verbCountsByTypeFixed;
+    private unifiedCache;
+    private fieldTypeInference;
+    /**
+     * **The native LSM engine (cor 3.0 — the sole metadata engine).** Opened
+     * in {@link init} (persistent + mmap-backed on filesystem storage, or a
+     * non-persistent in-RAM engine on non-filesystem adapters) and attached to
+     * `this.native` via `setLsmEngine` so the metadata-index dispatch routes
+     * through the engine's memtable + admission control. `null` only between
+     * construction and {@link openLsmEngine} during `init()`.
+     */
+    private lsmEngine;
+    /**
+     * **LSM durability (steps 3-4).** In-memory mirror of the on-disk LSM
+     * manifest. `null` in JSON mode and on the non-persistent fallback engine
+     * (where there is no SSTable surface). Set by {@link openLsmEngine} at cold
+     * open (parsed from {@link LSM_MANIFEST_KEY}, or a fresh empty manifest) and
+     * grown by {@link flushLsm} as each memtable drain produces an SSTable.
+     */
+    private lsmManifest;
+    /**
+     * **LSM durability (steps 3-4).** The resolved on-disk metadata directory
+     * (`getBinaryBlobPath('_metadata')`) backing the persistent engine's
+     * memtable logs. `null` in JSON mode and on the non-persistent fallback.
+     * Its presence is the load-bearing signal that {@link flushLsm} should drain
+     * the memtable to a durable SSTable.
+     */
+    private lsmMetaDir;
+    /**
+     * **LSM durability (step 5) — bounded-memtable flush trigger.** The memtable
+     * resident-byte ceiling at which {@link addToIndex} proactively drains the
+     * memtable to a durable SSTable via {@link flushLsm}, so the memtable never
+     * climbs to the admission ceiling in steady state (which would wedge every
+     * writer — the failure mode the persistent-open switch would otherwise
+     * introduce). Computed by {@link computeLsmFlushThreshold} as ~50% of the
+     * engine's dynamic `allowedBytes`, clamped to `[32 MiB, 256 MiB]`, and
+     * RECOMPUTED inside the ResourceManager subscriber callback
+     * ({@link attachLsmBudgetSubscriber}) so it tracks the live per-category
+     * budget. Seeded to the floor until the first budget callback fires.
+     */
+    private lsmFlushThresholdBytes;
+    /**
+     * Deregistration hook returned by
+     * {@link ResourceManager.registerMemoryBudgetSubscriber}. Called from
+     * {@link close} so the rebalance loop stops firing the engine's
+     * `setAllowedBytes` after this index has shut down. `null` in JSON
+     * mode (no subscriber registered).
+     */
+    private unsubscribeFromBudget;
+    /**
+     * Unique subscriber name used for both the
+     * `ResourceManager.registerMemoryBudgetSubscriber` call and the
+     * corresponding `reportNativeMemory` callback that drives the
+     * dynamic per-category rebalance. Each MetadataIndexManager instance
+     * gets a fresh name from {@link nextLsmSubscriberId} so multiple
+     * brains in one process don't collide on the same subscriber slot.
+     */
+    private lsmSubscriberName;
+    /**
+     * Native column store — per-field sorted columns backing brainy's unfiltered
+     * `find({ orderBy })` path. Brainy calls `metadataIndex.columnStore.sortTopK(...)`
+     * directly, so this MUST be present (and is why it is public). Cor's metadata
+     * index does its own native filtering; the column store's jobs here are populate
+     * (on `addToIndex`), persist (on `flush`), and sort (`sortTopK`).
+     */
+    readonly columnStore: NativeColumnStore;
+    /**
+     * Lazily built id-mapper façade over the native UUID ↔ int methods, returned by
+     * {@link getIdMapper}. Cached so brainy's orderBy path reuses one object.
+     */
+    private idMapperFacade;
+    /**
+     * **#72 Phase C.** The native mmap `BinaryIdMapper` injected into the
+     * Rust metadata core as its canonical UUID ↔ entity-int allocator,
+     * replacing the resident JSON id-maps (~56 GB @1B + the ~13M-UUID JSON
+     * cliff). Built in {@link init} from the brain's `_id_mapper/*` paths and
+     * handed to the core via `native.setIdMapper`. Kept here so {@link flush}
+     * can msync it and a full {@link rebuild} can truncate + re-inject it.
+     * `null` until init wires it (or on non-filesystem storage, where the
+     * mapper cannot engage — the metadata index then has no allocator and
+     * the brain must run unaccelerated).
+     */
+    private idMapperNative;
+    /**
+     * Set by `getIdMapper().rebuild()` so the immediately-following
+     * `rebuild()` (brainy `restore()` runs metadataIndex.rebuild() in a
+     * `Promise.all` right after the mapper hook) is a no-op: re-clearing the
+     * entity-id mapper there would race the concurrent `graphIndex.rebuild()`
+     * that resolves verb endpoints through this same mapper. The hook already
+     * did the full, identical rebuild, so the second one is skipped and the
+     * mapper stays stable for the graph pass. One-shot: consumed by the next
+     * `rebuild()`. (#68)
+     */
+    private skipRebuildOnce;
+    constructor(storage: StorageAdapter, config?: MetadataIndexConfig, 
+    /**
+     * Lazy resolver to the shared {@link MigrationCoordinator} (the plugin creates
+     * one per brain and hands the same getter to all three derived-index providers).
+     * Undefined for a raw-napi construction (tests) or a tier that never migrates —
+     * then {@link isMigrating} is always `false` and the epoch guard's large-brain
+     * outcome falls through to Brainy's own path.
+     */
+    migrationCoordinatorGetter?: (() => MigrationCoordinator | undefined) | undefined);
+    /**
+     * **LSM durability (steps 3-4) — the async open.** Construct the native LSM
+     * engine, attach it to the metadata index, register the ResourceManager
+     * budget subscriber, and cold-load every persisted SSTable into the
+     * engine's live read set. Called once from {@link init} (after
+     * {@link wireIdMapper}) — the LSM engine is the sole metadata engine.
+     *
+     * ## Persistent vs fallback
+     *
+     * When the storage is filesystem-backed (exposes `getBinaryBlobPath`), the
+     * engine is opened via `NativeLsmEngine.openWithDir(metaDir, manifestJson,
+     * …)`: its memtable is backed by mmap'd per-shard append logs and its epoch
+     * counter is seeded from the persisted manifest so new writes outrank every
+     * already-flushed SSTable. Non-filesystem storage falls back to a
+     * non-persistent in-RAM engine (`new NativeLsmEngine`) — reads/writes work
+     * for the session but there is no SSTable durability (and {@link flushLsm}
+     * becomes a no-op).
+     *
+     * ## Cold-load = no rebuild
+     *
+     * Cold start reads the manifest, seeds the epoch via `openWithDir`, then
+     * calls `registerSstablePath` once per SSTable — the entire restore. The
+     * O(N) {@link rebuild} path is NOT taken: the SSTables ARE the durable
+     * posting lists, served from mmap.
+     *
+     * @throws If a persisted SSTable's path cannot be resolved.
+     */
+    private openLsmEngine;
+    /**
+     * **LSM durability (steps 3-4) — the sync part.** Register the
+     * ResourceManager budget subscriber that keeps the engine's admission
+     * ceiling in sync with the dynamic 'metadata' budget. Split out of the
+     * async {@link openLsmEngine} so the registration (and its synchronous
+     * initial callback) is an isolated, side-effect-free-on-failure step that
+     * runs immediately after the engine is attached.
+     *
+     * Idempotent: a second call (e.g. a re-init) is a no-op while a subscriber
+     * is live. The unique subscriber name keeps multiple brains in one process
+     * from colliding on the same ResourceManager slot.
+     *
+     * The callback path is load-bearing for the dynamic rebalance: every
+     * `notifySubscribers` cycle (rebalance + pressure tick) hits
+     * `this.lsmEngine.setAllowedBytes(BigInt(allowed))`, applied by the native
+     * engine on its very next admission check.
+     */
+    private attachLsmBudgetSubscriber;
+    /**
+     * **LSM durability (steps 3-4) — cold-load read.** Read and validate the
+     * persisted LSM manifest from storage. Returns `null` when no manifest
+     * exists yet (a fresh brain). Validates the format version + shape and
+     * throws on a mismatch rather than registering a partial SSTable set (which
+     * would silently drop rows) — a forward-incompatible or torn manifest must
+     * fail loud.
+     *
+     * @returns The parsed manifest, or `null` if absent.
+     * @throws If the manifest is present but malformed or carries an
+     *   unsupported {@link LsmManifest.version}.
+     */
+    private loadLsmManifest;
+    /**
+     * **LSM durability (step 6).** Whether the LSM engine recovered durable
+     * on-disk state at cold open — used by {@link init} to demote the
+     * field-registry-missing rebuild to disaster-recovery-only. Durable state
+     * is either ≥1 SSTable in the cold-loaded manifest OR a non-empty memtable
+     * replayed from the per-shard logs (records inserted but not yet flushed
+     * before the last shutdown/crash). When present, the posting lists are
+     * served from the engine independent of the JSON field registry, so a
+     * rebuild would be a redundant O(N) re-scan.
+     *
+     * @returns `true` when the engine holds durable posting-list state.
+     */
+    private lsmHasDurableState;
+    /**
+     * **LSM durability (step 6) — restore/rebuild re-open.** Re-open the LSM
+     * engine fresh against the (wiped + recreated) storage tree and re-attach
+     * it to the native core. Called from {@link rebuild} in LSM mode.
+     *
+     * ## Why a full re-open, not just a `mkdir`
+     *
+     * A rebuild commonly follows brainy's `restore()`, which WIPES and replaces
+     * the storage tree under the still-live engine. But `native.clear()` (the
+     * in-memory reset rebuild runs) deliberately does NOT touch the attached
+     * engine — the engine owns its own memtable + copy-on-write live-SSTable
+     * set, neither of which `clear()` can see. So WITHOUT this re-open the
+     * engine would keep:
+     *   - **stale live-SSTable readers** — mmap handles on the pre-restore
+     *     `.sst` files. On Linux an mmap survives the file's `unlink`, so those
+     *     readers keep serving PRE-restore rows; a query after rebuild would
+     *     merge the restored data WITH ghosts of the wiped entities.
+     *   - **a stale in-memory manifest** ({@link lsmManifest}) still listing the
+     *     deleted SSTables — the next cold open would try to `registerSstablePath`
+     *     files that no longer exist and fail loud.
+     *   - **a stale memtable** holding pre-restore writes not yet flushed.
+     *
+     * This method makes the engine adopt the new tree: it deletes the stale
+     * on-disk SSTables + memtable logs, drops the stale manifest, constructs a
+     * FRESH `openWithDir` engine (empty live set, empty memtable, epoch reset
+     * from the clean logs), and re-attaches it via `setLsmEngine` (which
+     * overwrites the core's engine Arc — verified in the Rust binding). The
+     * ResourceManager subscriber callback dereferences `this.lsmEngine`
+     * dynamically, so the fresh engine inherits budget tracking on the next
+     * tick; meanwhile the re-applied admission ceiling below keeps it bounded,
+     * and the step-5 size-flush trigger drains it during the rebuild's
+     * re-index walk.
+     *
+     * A no-op on the non-persistent fallback engine (no `lsmMetaDir`), where
+     * there is no SSTable surface to go stale.
+     */
+    private reopenLsmEngineFresh;
+    /**
+     * **LSM durability (steps 3-4) — the flush/close drain.** Drain the LSM
+     * engine's memtable to a durable SSTable and splice it into the manifest.
+     * Called from {@link flush} (and therefore {@link close}, which flushes).
+     * No-op when there is no memtable content to flush, and on the
+     * non-persistent fallback engine (no manifest / no metaDir).
+     *
+     * ## Ordering invariant (load-bearing — a reviewer MUST preserve it)
+     *
+     * The four steps run in this exact order, and `commitFlushThrough` is NEVER
+     * called before the `.sst` bytes AND the manifest referencing them are
+     * durable:
+     *
+     *   1. write the `.sst` bytes (`saveBinaryBlob`, atomic tmp+rename)
+     *   2. write the manifest that references the new SSTable (`saveMetadata`)
+     *   3. register the SSTable into the engine's live read set
+     *      (`registerSstablePath` — mmap + SHA-256 verify)
+     *   4. ONLY NOW retire the memtable log below the flushed epoch
+     *      (`commitFlushThrough`)
+     *
+     * A crash between any two steps is safe: the just-flushed records still live
+     * in the uncommitted memtable log, so the next `openWithDir` replays them
+     * and merge-by-epoch idempotently collapses the transient double presence. A
+     * crash before step 2 leaves an orphaned `.sst` no manifest references — its
+     * id is harmlessly re-used on the next flush. Calling `commitFlushThrough`
+     * early would drop the log copy while the only on-disk copy might not yet be
+     * durable — unrecoverable data loss. Hence step 4 is strictly last.
+     */
+    private flushLsm;
+    /**
+     * **Piece 11 closeout.** Detect cortex 2.x legacy JSON metadata
+     * state on storage. The canonical marker is the
+     * `__metadata_field_registry__` key the JSON path writes on
+     * every flush. If present, the brain was built before the LSM
+     * engine became the default and refuses to silently switch.
+     *
+     * On detection: throw an init-time error directing the operator to run
+     * the migration script (the cor 3.0 LSM engine is the only engine — the
+     * legacy JSON chunk path has been removed). The error message is the
+     * migration runbook.
+     *
+     * No legacy state present → the brain is either fresh (no prior
+     * cor install) or already on the LSM path → proceed normally
+     * with the LSM engine.
+     *
+     * @throws {@link MetadataIndexLegacyJsonStateDetected} when cortex 2.x
+     *   JSON metadata state is detected on disk.
+     */
+    private detectLegacyJsonStateAndGate;
+    /**
+     * **#18 read-side epoch guard.** Detect 7.x → 8.0 derived-index epoch drift
+     * on open and route it to the right outcome. Called from {@link init} AFTER
+     * {@link detectLegacyJsonStateAndGate} (the cortex-2.x JSON gate) and
+     * {@link openLsmEngine} (the LSM cold-open), and BEFORE the column-store init
+     * and any rebuild branch — i.e. while reads are still un-served.
+     *
+     * The on-disk version-handshake marker (`_system/brain-format.json`) carries a
+     * shared, lockstep-bumped `indexEpoch`. cor reads it directly off storage (it
+     * cannot reach `brain.formatInfo()`, and Brainy's marker module is not
+     * exported) and compares against {@link COR_EXPECTED_INDEX_EPOCH}.
+     *
+     * Four fully-wired outcomes — no dangling flag:
+     *
+     * 1. **Epoch CURRENT (the common case)** — a single tiny marker read + an
+     *    integer compare, then return. NO size probe, NO further work: the
+     *    warm-restart path stays a cheap no-op.
+     * 2. **Stale + SMALL** (`< {@link resolveLargeBrainThreshold}`) — NO-OP. Brainy's
+     *    `rebuildIndexesIfNeeded` already drives the rebuild from the canonical
+     *    records for sub-threshold brains (its `AUTO_REBUILD_THRESHOLD` matches
+     *    ours) and re-stamps the marker after the rebuild verifies. cor must NOT
+     *    throw here; it simply lets Brainy's path run.
+     * 3. **Stale + LARGE + auto-migrate disabled** (`COR_AUTO_MIGRATE=false`) —
+     *    THROW {@link IndexMigrationRequiredError}, loud and pre-serve. A service
+     *    that refuses to start is recoverable; serving on indexes that predate
+     *    this build would surface stale/empty results with no signal.
+     * 4. **Stale + LARGE + auto-migrate enabled (default)** — NO-OP / fall through.
+     *    Brainy's existing large-brain path defers to a lazy first-query rebuild
+     *    (force=true, BLOCKING inline). That blocking rebuild is correct but not
+     *    yet "optimized": the non-blocking background build-new → verify → swap
+     *    that removes the first-query freeze is the **rc.8 increment** (it needs
+     *    Brainy's deference hook + a cor-invokable stamp method, neither shipped on
+     *    rc.7). We deliberately do NOT set an `epochMigrationPending` flag here:
+     *    nothing on rc.7 reads it (the MigrationCoordinator is the rc.8 unit), and
+     *    an unread flag would be unwired dead state.
+     *
+     * @throws {@link IndexMigrationRequiredError} on outcome 3 only.
+     */
+    private guardIndexEpoch;
+    /**
+     * @description **#18 migration lock — feature-detected by Brainy.** `true` while
+     * this brain is undergoing (or has failed) the coordinated 7.x → 8.0 rebuild, so
+     * Brainy holds every public read and write behind the lock. Delegates to the
+     * shared {@link MigrationCoordinator}; `false` when no coordinator is wired.
+     * @returns `true` when the brain must not be read from or written to.
+     */
+    isMigrating(): boolean;
+    /**
+     * @description **#18 migration lock — structured progress** Brainy relays into
+     * `getIndexStatus().migration`. `null` unless a migration is in flight.
+     * @returns The current {@link MigrationStatus}, or `null`.
+     */
+    migrationStatus(): MigrationStatus | null;
+    /**
+     * **#18 in-place migration runner (stub).** Reads the un-migrated metadata
+     * state from storage and replays it through the LSM-attached mutation path,
+     * leaving the brain on the cor 3.0 LSM-native on-disk layout.
+     *
+     * **NOT YET IMPLEMENTED — GA path is the rc.8 increment.** The read-side
+     * epoch guard (#18, {@link guardIndexEpoch}) ships now and makes the SMALL
+     * 7.x → 8.0 brain GA on rc.7 (Brainy drives the rebuild from canonical). The
+     * LARGE-brain non-blocking background build-new → verify → swap that this
+     * runner will drive needs Brainy's deference hook + a cor-invokable stamp
+     * method, which land together in the rc.8 increment; until then a large stale
+     * brain falls through to Brainy's blocking inline rebuild on first query.
+     *
+     * The wrapper for this method exists so callers have a stable reference point
+     * in the API surface; today it throws with a pointer to the supported
+     * back-compat path. Until the in-place runner lands, migrate via the
+     * standalone script `scripts/migrate-cortex-2x-to-3x.mjs`, which snapshots the
+     * brain via Brainy 7.x, wipes the legacy state, and restores via Brainy 8.0 +
+     * cor 3.0 (producing the LSM-native on-disk layout).
+     *
+     * @throws Always — the in-place runner is the rc.8 increment.
+     */
+    migrateToLsm(): Promise<void>;
+    /**
+     * Get the shared entity id-mapper façade. Brainy's unfiltered `orderBy` path
+     * calls `metadataIndex.getIdMapper().getUuid(intId)` to turn the column store's
+     * sorted int ids back into UUIDs for batchGet. The façade delegates to the
+     * native index's O(1) UUID ↔ int methods — the canonical mapping that
+     * `addToIndex` assigns and the column store is populated against.
+     *
+     * @returns An object exposing the brainy `EntityIdMapper` methods callers use.
+     */
+    getIdMapper(): EntityIdMapperLike;
+    init(): Promise<void>;
+    /**
+     * **#72 Phase C.** Build the shared mmap `BinaryIdMapper` from the
+     * brain's `_id_mapper/*` storage paths and inject it into the Rust
+     * core as the canonical UUID ↔ entity-int allocator. This replaces the
+     * legacy `loadEntityIdMapper`/`saveEntityIdMapper` JSON path, whose two
+     * resident HashMaps cost ~56 GB at 1 B entities and whose serialize
+     * threw at the V8 ~512 MB / ~13 M-UUID cliff.
+     *
+     * @throws If storage is not filesystem-backed (no `getBinaryBlobPath`)
+     *   — cor's metadata index requires local paths; brainy runs
+     *   unaccelerated on cloud adapters.
+     */
+    private wireIdMapper;
+    private loadFieldRegistry;
+    private saveFieldRegistry;
+    /**
+     * **Piece 6 generation counter — invalidate stale lazy-load state.**
+     * Checks the native generation counter against our last-seen value.
+     * If a mutation has bumped the counter since we last looked,
+     * `loadedFields` is cleared so every subsequent `ensureFieldLoaded`
+     * re-pulls the current value-enumeration field index from storage.
+     *
+     * Called at the start of every read entry point. The check itself
+     * is one atomic load + one BigInt compare — sub-microsecond.
+     *
+     * Without this hook, after a mutation on the native side the
+     * cached `loadedFields` set would prevent re-reading the updated
+     * value enumeration, surfacing as stale-result bugs of the same shape
+     * as R5 (but for the lazy-load path rather than the filter path).
+     */
+    private invalidateIfStale;
+    /**
+     * **Piece 15 — per-field selective invalidation.** Reads the
+     * field's per-field generation from the native side and compares
+     * against {@link lastSeenFieldGenerations}. If the field's counter
+     * advanced since we last observed it, removes ONLY that field
+     * from {@link loadedFields} — every other loaded field stays
+     * valid. Then records the current value so the next call sees a
+     * no-op until another mutation touches this field.
+     *
+     * Called at the start of every read entry point that knows which
+     * fields it needs. The check is one FFI hop (native atomic load)
+     * + one Map get/set — sub-microsecond per field. For multi-field
+     * filters (`ensureFieldsLoaded`) the wrapper iterates this method
+     * once per field instead of doing the old index-wide check.
+     *
+     * Cold-boot semantic: first probe records the current counter
+     * without invalidating (matches the old `lastSeenGeneration ===
+     * -1n` initial-call behavior). A counter advance from 0 → 1 on
+     * the FIRST observation is treated as "first sighting", NOT
+     * "stale" — `loadedFields` doesn't have this field yet anyway, so
+     * there's nothing to invalidate.
+     */
+    private invalidateFieldIfStale;
+    /**
+     * Ensure a field's persisted value-enumeration index is loaded into the
+     * native core (the LSM read paths enumerate candidate values from it).
+     * Lazy + idempotent — guarded by {@link loadedFields}.
+     */
+    private ensureFieldLoaded;
+    /**
+     * **LSM durability (steps 3-4) — cold-load value enumeration.** Restore a
+     * field's persisted value-enumeration index
+     * (`__metadata_field_index__field_<field>`) into the native core. In LSM
+     * mode the read paths for field-wide `exists` / `missing` / `ne` and range
+     * (`between` / `gt` / `lt`) queries enumerate a field's distinct values from
+     * `core.field_indexes`. That map is built incrementally by `addToIndex`
+     * within a session but is EMPTY after a cold reopen — the registered
+     * SSTables hold the posting lists, but without the value enumeration the
+     * read paths have no value keys to look those lists up by. Loading this JSON
+     * (persisted by `persistMutationResult` / `flush`) closes that gap so the
+     * cold-loaded SSTables serve every filter shape with no rebuild.
+     *
+     * No-op for a field with no persisted index (a brand-new field this
+     * session) — `addToIndex` then populates `core.field_indexes` incrementally.
+     * Only the first access loads it (the caller guards via `loadedFields`), so
+     * an in-session field index built by `addToIndex` is never overwritten.
+     *
+     * @param field - The field whose value-enumeration index to restore.
+     */
+    private loadPersistedFieldIndex;
+    /**
+     * Ensure multiple fields are loaded (parallel).
+     *
+     * **Piece 15.** Per-field stale checks instead of one
+     * index-wide check. The wrapper now iterates {@link invalidateFieldIfStale}
+     * once per requested field — selectively dropping ONLY the
+     * fields whose counters advanced since we last looked. Cross-
+     * field churn (a mutation on `noun` while we're reading `status`)
+     * no longer invalidates `status`'s lazy-load. Cost: N FFI hops
+     * (each a single atomic Acquire load) instead of 1 — typically
+     * 3-5 hops on a brainy filter query, negligible vs the storage
+     * I/O the wrapper does next.
+     */
+    private ensureFieldsLoaded;
+    /**
+     * Internal variant of `ensureFieldLoaded` that skips the generation
+     * check (used by `ensureFieldsLoaded` which already did one batched
+     * check at the top).
+     */
+    private ensureFieldLoadedNoCheck;
+    /**
+     * Extract filter field names for lazy loading.
+     */
+    private extractFilterFields;
+    /**
+     * Persist the dirty value-enumeration field indexes + field registry from a
+     * MutationResult. Posting-list durability is owned by the LSM engine
+     * ({@link flushLsm}), so this only writes the JSON value-enumeration surface
+     * the LSM read paths cold-load.
+     */
+    private persistMutationResult;
+    private warmCache;
+    private warmCacheForTopTypes;
+    private lazyLoadCounts;
+    private syncTypeCountsToFixed;
+    private syncTypeCountsFromFixed;
+    /**
+     * Serialize a query value to JSON for the native String param. `JSON.stringify`
+     * returns the JS value `undefined` (NOT a string) for `undefined`/functions —
+     * which the napi String param rejects ("Failed to convert Null/Undefined into
+     * String"). Coalesce to the JSON literal `'null'` so the Rust side parses it to
+     * `serde_json::Value::Null`, normalizes, and returns the no-match result (0/[]) —
+     * bug-for-bug compatible with brainy's "no match → empty". (#79 cluster B.)
+     */
+    private serializeValue;
+    getIds(field: string, value: any): Promise<string[]>;
+    /**
+     * Resolve a `where` filter to its matching entity ids.
+     *
+     * @param filter - Brainy field-operator filter (eq / allOf / anyOf / ne /
+     *   range / exists / …).
+     * @param opts - OPTIONAL page bound (#75). When `{ limit }` is supplied the
+     *   native producer early-stops and returns only the `[offset, offset+limit)`
+     *   PREFIX of the sorted match set — it short-circuits the posting-list →
+     *   UUID conversion at `offset+limit` instead of materializing the full list
+     *   and slicing, which removes the O(N) FFI marshal for a broad-filter
+     *   single-page `find()` at billion scale. Brainy calls this with
+     *   `{ limit: pageEnd, offset: 0 }` and re-windows the prefix itself
+     *   (visibility filter + slice), so the producer honors the literal `offset`
+     *   (brainy's `0` ⇒ the `[0, limit)` prefix). Omitting `opts` returns the
+     *   FULL match set (back-compat).
+     */
+    getIdsForFilter(filter: any, opts?: {
+        limit?: number;
+        offset?: number;
+    }): Promise<string[]>;
+    getIdsForMultipleFields(fieldValuePairs: Array<{
+        field: string;
+        value: any;
+    }>): Promise<string[]>;
+    getIdsForTextQuery(query: string): Promise<Array<{
+        id: string;
+        matchCount: number;
+    }>>;
+    getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>;
+    /**
+     * Get field value for a specific entity (helper for sorted queries).
+     *
+     * Three-path lookup:
+     *
+     * 1. **Bucketed fields** (timestamps) — the native index stores values
+     *    rounded to coarser buckets to keep the index compact for range
+     *    queries. That bucketing loses precision, so sorting must read the
+     *    actual value directly from entity storage via resolveEntityField.
+     *
+     * 2. **Custom fields with no indexed values** — VFS fields like
+     *    `modified`/`accessed`, plus any user custom field not currently
+     *    present in the native index. Resolved from entity storage via
+     *    resolveEntityField, which knows the top-level-vs-metadata
+     *    shape contract.
+     *
+     * 3. **Indexed fields** — strings, enums, and low-cardinality ints that
+     *    the native index knows about. O(values) lookup.
+     *
+     * Mirrors getFieldValueForEntity in brainy's src/utils/metadataIndex.ts —
+     * keep in lockstep when the entity shape contract evolves.
+     */
+    getFieldValueForEntity(entityId: string, field: string): Promise<any>;
+    private denormalizeValue;
+    getFilterValues(field: string): Promise<string[]>;
+    getFilterFields(): Promise<string[]>;
+    addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise<void>;
+    removeFromIndex(id: string, metadata?: any): Promise<void>;
+    /**
+     * Report the LSM engine's current native memory footprint
+     * (memtable + reader cache) to the ResourceManager so the dynamic
+     * per-category rebalance can detect pressure on the 'metadata'
+     * category and shift capacity from underutilized neighbors.
+     *
+     * No-op in JSON mode (no engine attached) and after `close()` has
+     * unregistered the subscriber. Called from `addToIndex` /
+     * `removeFromIndex` on the production hot path — the underlying
+     * `nativeMemoryBytes()` is a single atomic load + sum, sub-microsecond
+     * cost.
+     *
+     * The reportNativeMemory call short-circuits inside ResourceManager
+     * if the subscriber name was already cleared (deregistered between
+     * the mutation and this call), so the unregister race is benign.
+     */
+    private reportLsmMemoryToResourceManager;
+    private updateTypeTracking;
+    tokenize(text: string): string[];
+    hashWord(word: string): number;
+    extractTextContent(data: any): string;
+    /**
+     * Whether a field should be indexed, honoring this index's configured
+     * `excludeFields` / `indexedFields`. Mirrors the Rust `should_index_field`
+     * (and brainy's TS `shouldIndexField`) so the column store indexes exactly the
+     * fields the metadata index does.
+     *
+     * @param field - Fully-qualified field name (dotted for nested).
+     * @returns True if the field is indexable under the current config.
+     */
+    private shouldIndexField;
+    /**
+     * Build the indexable field→value map fed to the column store, identical to
+     * brainy's `MetadataIndexManager.extractIndexableFields()` → `fieldsMap`:
+     *
+     * - Recurses objects; flattens a top-level `metadata` object to the root.
+     * - Maps a top-level `type` field to `noun` (backward-compat with indexes).
+     * - Skips the never-index set (vector/embedding/connections/level/id), purely
+     *   numeric keys (array indices), config-excluded fields, and arrays longer
+     *   than 10 elements (likely vectors/bulk data).
+     * - Small arrays (≤10) become multi-value fields: one array entry per element.
+     * - Adds `__words__` as a multi-value array of FNV-1a int32 word hashes from
+     *   the entity's text content (capped at 5000 words), matching brainy.
+     *
+     * Replicated in TS rather than reusing the Rust extractor because the native
+     * `addToIndex` does not return the extracted pairs and the engine is frozen.
+     *
+     * @param data - The entity or plain-metadata object being indexed.
+     * @returns Field → value (or array of values) map for `columnStore.addEntity`.
+     */
+    private extractColumnFields;
+    getEntityCountByType(type: string): number;
+    getTotalEntityCount(): number;
+    getAllEntityCounts(): Map<string, number>;
+    getEntityCountByTypeEnum(type: NounType): number;
+    getVerbCountByTypeEnum(type: VerbType): number;
+    getTopNounTypes(n: number): NounType[];
+    getTopVerbTypes(n: number): VerbType[];
+    getAllNounTypeCounts(): Map<NounType, number>;
+    getAllVerbTypeCounts(): Map<VerbType, number>;
+    getVFSEntityCountByType(type: string): Promise<number>;
+    getAllVFSEntityCounts(): Promise<Map<string, number>>;
+    getTotalVFSEntityCount(): Promise<number>;
+    getCountForCriteria(field: string, value: any): Promise<number>;
+    flush(): Promise<void>;
+    /**
+     * Persist all state and release the column store's resources. Brainy's
+     * shutdown path calls `metadataIndex.close()` when present; flushing here
+     * guarantees the column store's tail buffers reach disk before the process
+     * exits, so a reopened index can serve `orderBy` from persisted segments.
+     */
+    close(): Promise<void>;
+    getStats(): Promise<MetadataIndexStats>;
+    /**
+     * @description Report which index path a `find({ where: { [field]: ... } })`
+     * clause will hit. Brainy's `brain.explain()` calls this per `where` field so an
+     * operator can see *before* running a query whether the field has any index
+     * entries at all — a `where` on an unindexed field returns `[]` silently, and
+     * `path: 'none'` gives that empty result an explanation.
+     *
+     * The native engine serves all `where` lookups from the column store (the
+     * single source of truth — there is no separate legacy sparse-index path at
+     * query time, unlike brainy's TypeScript baseline). So a field either has
+     * column-store entries (`'column-store'`) or none (`'none'`). The field's
+     * segments are loaded first so `hasField` reflects on-disk state, not just
+     * what has been touched this session.
+     *
+     * @param field - The `where`-clause field name to explain.
+     * @returns The serving path and an operator-facing note.
+     * @example
+     * const { path, notes } = await metadataIndex.explainField('status')
+     * if (path === 'none') console.warn(notes)
+     */
+    explainField(field: string): Promise<{
+        path: 'column-store' | 'sparse-chunked' | 'none';
+        notes?: string;
+    }>;
+    validateConsistency(): Promise<{
+        healthy: boolean;
+        avgEntriesPerEntity: number;
+        entityCount: number;
+        indexEntryCount: number;
+        recommendation: string | null;
+    }>;
+    /**
+     * O(1)-ish cross-bucket consistency probe (brainy rc.6 matrix 7.7).
+     *
+     * Brainy calls this ONCE on the first read; on `false` it runs
+     * {@link detectAndRepairCorruption}. The check is fully internal — it never
+     * re-reads entity data. It loads a bounded set of value-enumeration field
+     * indexes (so the native sampler can enumerate each field's value buckets),
+     * biased toward `noun` (the canonical single-valued type field where the
+     * staff→timeslot class of phantom lands), then defers to the native
+     * {@link NativeMetadataIndexInstance.probeConsistency} bounded sampler.
+     *
+     * Best-effort: any error (a sampling glitch, a storage hiccup loading a field
+     * index) degrades to `true` so the probe NEVER breaks brainy's read path.
+     *
+     * @returns `true` if the index looks consistent (or the check is
+     *   indeterminate), `false` if a cross-bucket phantom was sampled.
+     */
+    probeConsistency(): Promise<boolean>;
+    detectAndRepairCorruption(): Promise<void>;
+    /**
+     * State-level cross-bucket phantom purge (brainy matrix 7.8b).
+     *
+     * Scans the native index for entities whose int sits in more than one value
+     * bucket of a field, then tombstones the stale buckets against each entity's
+     * CURRENT stored value(s) — read straight from storage (brainy's source of
+     * truth), NOT from the index being repaired. A multi-valued field keeps all
+     * its stored values (a no-op); a single-valued phantom is purged down to the
+     * one current value; an orphaned int (entity gone) is evicted entirely.
+     *
+     * Only reached after {@link probeConsistency} flagged the index, so the
+     * full field-index load + scan is acceptable here (unlike the O(1) probe).
+     */
+    private purgeCrossBucketPhantoms;
+    rebuild(): Promise<void>;
+    /**
+     * Cursor-paginate every entity of a kind ('noun'|'verb') through the metadata
+     * index during rebuild. ONE streaming path for ALL storage types — replaces
+     * the old local-vs-cloud fork that OOM'd / silently TRUNCATED at scale: the
+     * local branch loaded the whole corpus in a single `getNouns/getVerbs({limit:
+     * 1e6})` call (cap 1M), and the cloud branch capped at MAX_ITERATIONS(10000)×
+     * batch(25) ≈ 250K. Cursor pagination is O(page) RAM and complete at any
+     * scale; it terminates on `hasMore === false` (8.0's cursor is no-rescan), with
+     * an empty-page backstop. (#74 — perf-at-all-scales mandate.)
+     * @returns the number of entities indexed (those carrying metadata).
+     */
+    private rebuildEntityMetadata;
+    /**
+     * Batch-load metadata for a page of noun/verb ids — prefers the storage
+     * adapter's batch API, falls back to per-id reads.
+     */
+    private loadRebuildMetadataBatch;
+    /**
+     * During rebuild, msync the durable state accumulated so far.
+     *
+     * Posting lists land in the LSM engine's memtable (drained to durable
+     * SSTables by the step-5 size-flush trigger as the re-index walk proceeds)
+     * and the entity ints land in the mmap id mapper — so the only durable
+     * write to force here is the mapper msync. The value-enumeration field
+     * indexes + the LSM memtable tail are persisted by the final {@link flush}
+     * at the end of the rebuild.
+     */
+    private flushRebuildDirty;
+    clearAllIndexData(): Promise<void>;
+    private getPersistedFieldList;
+    /**
+     * Delete a field's persisted value-enumeration index
+     * (`__metadata_field_index__field_<field>`) from storage. Called by
+     * {@link rebuild} and {@link clearAllIndexData} to wipe the JSON
+     * value-enumeration surface before re-indexing; the posting lists
+     * themselves are wiped from the LSM engine separately
+     * ({@link reopenLsmEngineFresh}).
+     */
+    private deletePersistedFieldIndex;
+    getFieldStatistics(): Promise<Map<string, FieldStats>>;
+    getFieldCardinality(field: string): Promise<CardinalityInfo | null>;
+    getFieldsWithCardinality(): Promise<Array<{
+        field: string;
+        cardinality: number;
+        distribution: string;
+    }>>;
+    getOptimalQueryPlan(filters: Record<string, any>): Promise<{
+        strategy: 'exact' | 'range' | 'hybrid';
+        fieldOrder: string[];
+        estimatedCost: number;
+    }>;
+    getFieldsForType(nounType: NounType): Promise<Array<{
+        field: string;
+        affinity: number;
+        occurrences: number;
+        totalEntities: number;
+    }>>;
+    private yieldToEventLoop;
+}
+export {};
+//# sourceMappingURL=NativeMetadataIndex.d.ts.map