@avantf/dsh-mem 0.6.0 → 0.6.1

package/lib/engine/mem/store/vector_repair.d.ts

@@ -0,0 +1,248 @@
+/**
+ * The ONE vector-space repair flow both stores run.
+ *
+ * Changing the embedding space (model, width, pooling, normalization, input window, weights) is a
+ * DATA MIGRATION: persisted vectors keep decoding but live in another model's coordinates, so the
+ * semantic leg must not serve them. `common.ts` owns the cheap DETECTION (`reportStaleVectors`); this
+ * module owns the REPAIR — a bounded, resumable, switchable loop that re-encodes the corpus in
+ * slices and yields the event loop between them.
+ *
+ * It is a template method implemented by COMPOSITION (AGENTS.md / the design note in
+ * `docs/vector-repair-shared-flow.md`): the skeleton lives here as free functions, each store
+ * supplies a {@link VectorRepairTarget} adapter. No common base class — the two stores are separate
+ * databases with separate DAOs, and a base class would be a structural rewrite of two hub files.
+ *
+ * What is shared, and therefore must never be copied back into `store/memory.ts` /
+ * `store/knowledge.ts`: the batch size, the `yieldToEventLoop` call, the stop / resume / no-progress
+ * rules, the `semantic.auto_migrate` decision, the `dry_run` split, the order invariant
+ * (unusable bytes leave first, then re-encode), the warmup policy and the four log kinds.
+ *
+ * What stays in the stores (correct differences, DESIGN §2.3): the DAO calls and row shapes, the
+ * `conflict_checked` re-queue on the memory side, the `content_hash` reuse and failure counting on
+ * the knowledge side, the HRR column, and which live index a row is added to.
+ *
+ * @module store/vector_repair
+ */
+import type { AvantfLogger, VectorMigrationOutcome, VectorMigrationProgress, VectorsFixReport, VectorsFixStoreReport } from '../../mem-contract/index.js';
+import type { VectorStore } from '../../mem-retrieval/index.js';
+import { type StaleVectorCounts } from './common.js';
+/** The two stores this flow can drive. */
+export type VectorRepairStore = 'memory' | 'knowledge';
+/**
+ * Rows one background vector-space migration slice may re-encode.
+ *
+ * Small on purpose: the point of batching is that a query arriving mid-migration waits for at most
+ * one ONNX forward pass batch, not for a whole corpus. At ~30 ms per bge-base-zh encode a batch of 16
+ * is well under a second of contested model time, and the event loop is yielded between batches
+ * (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorSlice` takes whatever
+ * the caller needs.
+ */
+export declare const DEFAULT_VECTOR_MIGRATION_BATCH = 16;
+/**
+ * What one store's persisted-vector scan found, before any write.
+ *
+ * `stale` and `space_stale` mirror the cheap detection counts (`StaleVectorCounts`); `missing` is
+ * "ACTIVE row with no persisted vector at all" — the third reason the semantic leg cannot use a row.
+ */
+export interface VectorsClassification {
+    /** ACTIVE rows whose persisted vector has the wrong width (it cannot even be decoded). */
+    stale: number;
+    /** ACTIVE rows whose usable-width vector was recorded in another space. */
+    space_stale: number;
+    /** ACTIVE rows with no persisted vector at all. */
+    missing: number;
+    /**
+     * Ids whose unusable bytes must leave the store before a re-encode picks them up (the `limit`
+     * passed to {@link VectorRepairTarget.classifyVectors} is applied). Knowledge overwrites vectors
+     * in place, so it reports none.
+     */
+    dropIds: readonly number[];
+    /**
+     * Usable persisted vectors absent from the LIVE index, decoded and ready to re-add. Only stores
+     * whose diagnose surface tracks the live index report this (memory); knowledge rebuilds its index
+     * at open and keeps it in sync on every write, so it omits the property entirely.
+     */
+    unindexed?: readonly {
+        id: number;
+        vec: Float32Array;
+    }[];
+}
+/**
+ * One store's half of the repair flow — the "key methods" of the template.
+ *
+ * Every method is a PRIMITIVE the store can answer from its own DAO; none of them contains the
+ * loop, the batch size or the switch logic (those are in this module, once).
+ */
+export interface VectorRepairTarget {
+    readonly store: VectorRepairStore;
+    /** The manual entry the stale-vector warning points at, as the operator would type it. */
+    readonly manualEntry: string;
+    /** The representation fingerprint this store writes into now. */
+    vectorSpace(): string;
+    /** Is the embedding model loaded RIGHT NOW? (Never warms — the flow owns the warmup policy.) */
+    semanticAvailable(): boolean;
+    /** The RAW `semantic.auto_migrate` value; the flow decides what it means (`!== false`). */
+    autoMigrate(): boolean | undefined;
+    /** Cheap `{ stale, space_stale, legacy? }` — blob LENGTHS only, never decodes. */
+    vectorSpaceHealth(): StaleVectorCounts;
+    /** Cheap count of ACTIVE rows with no persisted vector (part of `remaining`). */
+    missingVectorCount(): number;
+    /** One scan classifying what needs dropping and re-encoding; `limit` bounds `dropIds`. */
+    classifyVectors(limit: number): VectorsClassification;
+    /**
+     * Optional: put usable persisted vectors missing from the live index back into it, returning how
+     * many were added. Called with the slice of {@link VectorsClassification.unindexed} the current
+     * `limit` allows, so the flow does not re-scan the corpus for it.
+     */
+    reindexUsable?(rows: readonly {
+        id: number;
+        vec: Float32Array;
+    }[]): number;
+    /** Clear unusable bytes (table + live index); returns how many rows were actually cleared. */
+    dropVectors(ids: readonly number[]): number;
+    /** Encode + persist up to `limit` rows into the current space, one row isolated at a time. */
+    reencodeVectors(limit: number): Promise<{
+        encoded: number;
+        failed: number;
+    }>;
+    /** Optional: wait for the embedding model (explicit repair only; the background never waits). */
+    warmup?(): Promise<boolean>;
+}
+/** A persisted vector row as read back from a store, for the shared open-time load. */
+export interface PersistedVectorRow {
+    id: number;
+    vec: Uint8Array | null;
+    embedding_model?: string | null;
+}
+/**
+ * Rebuild one store's live vstore from its persisted rows, count what is unusable in the current
+ * space, and say so once — the shared "open for business" step both stores used to write twice.
+ *
+ * The owning store's ONLY half is the row read (`facts.activeVectorRows()` /
+ * `chunks.vectorRows()`); the reload, the count and the loud warning are one implementation, so a
+ * change to what counts as stale cannot land on one store only (the incident this exists for: the
+ * memory store checked the SPACE while knowledge checked only the WIDTH, so a model swap mixed two
+ * spaces in one index until someone happened to run `kb_reindex`).
+ */
+export declare function loadVectorIndex(opts: {
+    store: VectorRepairStore;
+    vstore: VectorStore;
+    rows: readonly PersistedVectorRow[];
+    space: string;
+    manualEntry: string;
+}): StaleVectorCounts;
+/** One row the shared encode flow should turn into a vector. */
+export interface VectorEncodeRow {
+    id: number;
+    text: string;
+}
+/** One encoded row ready to be persisted. */
+export interface EncodedVector {
+    id: number;
+    bytes: Buffer;
+}
+/**
+ * One store's half of {@link encodeAndPersist}: how to encode, where the live index is, and how a
+ * group of vectors is persisted.
+ */
+export interface EncodeWriteTarget {
+    readonly store: VectorRepairStore;
+    /** The space recorded beside every persisted vector. */
+    readonly space: string;
+    /** One text → vector. Throwing skips that row (never the group). */
+    encode(text: string): Promise<Float32Array>;
+    /** Add a freshly encoded vector to THIS store's live index. */
+    add(id: number, vec: Float32Array): void;
+    /**
+     * Persist one group of encoded rows. Throwing fails the WHOLE group, which is what lets the store
+     * decide how to reconcile its live index (see {@link onWriteError}).
+     */
+    write(rows: readonly EncodedVector[]): void;
+    /** Rows per `write` call (default 1). Knowledge uses its measured `WRITE_BATCH`. */
+    readonly writeBatch?: number;
+    /** Called once per row whose encode (or live-index add) failed. */
+    onRowError?(row: VectorEncodeRow, error: unknown): void;
+    /** Called once per group whose `write` failed, with the rows that never reached the database. */
+    onWriteError?(rows: readonly EncodedVector[], error: unknown): void;
+}
+/**
+ * Encode rows one at a time, add each to the live index, and persist them in groups.
+ *
+ * The shared skeleton: serial encode (a MEASURED decision for chunks — a padded batched forward pass
+ * is 1.79x slower on real 53–550 character chunks), `add` right after a successful encode, one
+ * `write` call per `writeBatch` rows, per-row isolation of encode failures, per-group isolation of
+ * write failures, and the `{ encoded, failed }` count.
+ *
+ * What stays with the store: the backend call, the live index, the write statement, the replay of a
+ * failed group, and the summary log — the memory store logs per row and keeps a half-written index
+ * (documented as the benign direction for it), the knowledge store evicts the group so both sides
+ * agree that the vectors are missing.
+ */
+export declare function encodeAndPersist(rows: readonly VectorEncodeRow[], target: EncodeWriteTarget): Promise<{
+    encoded: number;
+    failed: number;
+}>;
+/**
+ * ONE bounded slice of the repair: re-add unusable-but-readable vectors that the live index lost,
+ * drop the rows a re-encode has to replace, then encode up to `limit` missing/foreign rows.
+ *
+ * `limit` is what makes the SAME logic usable as a background migration: one call is a bounded unit
+ * of work the caller can yield between. The explicit fix passes no limit (the operator action drains
+ * everything); the background drive passes its batch size.
+ *
+ * The ORDER is an invariant: unusable bytes leave the table and the live index first, so a
+ * re-encode can pick them up and a process that dies mid-migration leaves at most one batch
+ * vector-less — exactly the `missing` rows the next run resumes on.
+ */
+export declare function repairVectorSlice(target: VectorRepairTarget, opts: {
+    dryRun: boolean;
+    limit?: number;
+}): Promise<{
+    report: VectorsFixStoreReport;
+    remaining: number;
+}>;
+/**
+ * ONE bounded slice of the background migration — the unit the drive (and the tests) step.
+ *
+ * Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
+ * stale from the persisted width/space, so a restart simply continues where the last slice stopped.
+ */
+export declare function migrateVectorSlice(target: VectorRepairTarget, batchSize?: number): Promise<VectorMigrationProgress>;
+/** Options for the background drive; the four log kinds go to `log` (the host logger by default). */
+export interface DriveVectorRepairOptions {
+    batchSize?: number;
+    /** Plugin unmount / runtime shutdown: stop at the next batch boundary. */
+    shouldStop?: () => boolean;
+    onProgress?: (progress: VectorMigrationOutcome) => void;
+    /** Log sink; `retrievalLogger()` (the host logger once the runtime installed it) when omitted. */
+    log?: AvantfLogger;
+}
+/**
+ * Drive the bounded background migration for one store until it is current, the model is
+ * unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE
+ * never rejects: it is logged and reported as `remaining > 0`, which is what makes the next
+ * heartbeat a retry. Never blocks the event loop for longer than one batch.
+ *
+ * Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
+ * on background re-encoding can leave the loud warning and the manual entry as the only path. The
+ * switch is read through the adapter so a new store cannot forget it.
+ */
+export declare function driveVectorRepair(target: VectorRepairTarget, opts?: DriveVectorRepairOptions): Promise<VectorMigrationOutcome>;
+/**
+ * Drive every store in order. A stop request keeps the drive from starting another store, but each
+ * store still answers with its snapshot (so the caller can report what is left everywhere).
+ */
+export declare function driveVectorRepairs(targets: readonly VectorRepairTarget[], opts?: DriveVectorRepairOptions): Promise<VectorMigrationOutcome[]>;
+/**
+ * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`,
+ * covering one or both stores.
+ *
+ * `store` selects a single store (omitted = both); `dry_run` previews every count without writing —
+ * and without loading the model (so a preview also reports `would_warm`). The top-level
+ * `semantic_available` is AND over the selected stores: `false` means at least one store's repair
+ * could not encode anything.
+ */
+export declare function vectorsFix(targets: Partial<Record<VectorRepairStore, VectorRepairTarget>>, opts: {
+    dryRun: boolean;
+    store?: VectorRepairStore;
+}): Promise<VectorsFixReport>;

package/lib/engine/mem-contract/tools.d.ts

@@ -161,6 +161,10 @@ export declare const AdminUnion: z.ZodDiscriminatedUnion<[z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     action: z.ZodLiteral<"vectors_fix">;
     dry_run: z.ZodOptional<z.ZodBoolean>;
+    store: z.ZodOptional<z.ZodEnum<{
+        memory: "memory";
+        knowledge: "knowledge";
+    }>>;
 }, z.core.$strip>, z.ZodObject<{
     action: z.ZodLiteral<"contradict_check">;
 }, z.core.$strip>, z.ZodObject<{

package/lib/engine/mem-contract/types.d.ts

@@ -507,22 +507,53 @@ export interface VectorMigrationOutcome extends VectorMigrationProgress {
     enabled: boolean;
 }
 /**
- * `mem_admin vectors_fix`: what a repair did, or — with `dry_run` — what it WOULD do.
+ * ONE store's part of a vector repair (`mem_admin vectors_fix`).
+ *
+ * The common subset (`stale` / `space_stale` / `dropped` / `encoded` / `failed` /
+ * `semantic_available`) is what both stores report; `missing` and `would_warm` are shared steps of
+ * the repair flow, and `unindexed` / `reindexed` are only meaningful to a store that tracks its live
+ * index (memory — knowledge rebuilds it at open and keeps it in sync on every write).
  *
  * `semantic_available` is "the model is loaded RIGHT NOW" (a dry run must not load it), so the
  * preview also carries `would_warm`: together they separate "unknown until you run it" from
  * "the warmup was attempted and failed".
  */
-export interface VectorsFixReport {
-    missing: number;
+export interface VectorsFixStoreReport {
+    /** ACTIVE rows whose persisted vector has the wrong width. */
     stale: number;
+    /** ACTIVE rows whose usable-width vector was recorded in another space. */
     space_stale: number;
-    unindexed: number;
-    reindexed: number;
+    /** ACTIVE rows with no persisted vector at all (what the repair has to encode). */
+    missing: number;
+    /** Rows whose unusable old-space bytes were cleared so a re-encode could replace them. */
     dropped: number;
-    fixed: number;
+    /** Rows re-encoded into the current space by this repair. */
+    encoded: number;
+    /** Rows whose encode or write failed; they stay `missing` / `stale` for the next pass. */
+    failed: number;
+    /** The model is loaded right now. */
     semantic_available: boolean;
+    /** The model was NOT loaded at entry, so a real run would attempt a warmup. */
     would_warm: boolean;
+    /** Usable persisted vectors absent from the live index (memory only). */
+    unindexed?: number;
+    /** Usable persisted vectors put back into the live index by this repair (memory only). */
+    reindexed?: number;
+}
+/**
+ * `mem_admin vectors_fix`: what a repair did, or — with `dry_run` — what it WOULD do, per store.
+ *
+ * The report is per-store because one switch and one call now cover BOTH libraries (a model swap
+ * leaves memory and knowledge in the same state, and the repair used to exist for memory only).
+ * `store` on the request narrows the call to one library; the top-level `semantic_available` is the
+ * AND over the stores actually repaired.
+ */
+export interface VectorsFixReport {
+    stores: {
+        memory?: VectorsFixStoreReport;
+        knowledge?: VectorsFixStoreReport;
+    };
+    semantic_available: boolean;
     dry_run: boolean;
 }
 /** `kb_manage ingest`: what one document produced. */

package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts

@@ -0,0 +1,52 @@
+import type { Reranker } from '../interfaces.js';
+import { type ModelEnv } from './local_bge.js';
+/** One text-classification result: transformers.js yields `{label, score}`; tolerate a raw `{data}` tensor too. */
+type ClassifyResult = {
+    label?: string;
+    score?: number;
+    data?: number[];
+};
+/** The injected pipeline surface: a batch of (query, document) pairs in, scores out. */
+export type ClassifyFn = (inputs: [string, string][], opts?: Record<string, unknown>) => Promise<ClassifyResult[]>;
+/** Injectable seam: tests supply a fake classifier instead of the transformers.js import. */
+export type RerankerPipeFactory = (model: string, env: ModelEnv) => Promise<ClassifyFn>;
+/** Never hand the classifier a degenerate document; below this the score is meaningless. */
+export declare const MIN_DOC_TOKENS = 16;
+/**
+ * Local cross-encoder reranker via `@huggingface/transformers` (`bge-reranker`).
+ * Lazily imported; `warmUp()` configures the mirror/cache and triggers a load.
+ * Degrades to identity order when the package/model is not available.
+ *
+ * Like {@link LocalBgeBackend}, a failed warmup is retryable through
+ * `ensureWarm()` rather than being final until the next restart.
+ */
+export declare class LocalReranker implements Reranker {
+    readonly name = "bge_reranker";
+    private _available;
+    private _pipe;
+    private _warming;
+    private _lengthWarned;
+    private _window;
+    private readonly configuredMax;
+    private readonly model;
+    private readonly env;
+    private readonly pipeFactory;
+    private readonly gate;
+    constructor(model: string, env?: Partial<ModelEnv>, pipeFactory?: RerankerPipeFactory);
+    isAvailable(): boolean;
+    /** Fire-and-forget retry for call sites that only observe availability (see `warm_gate.ts`). */
+    ensureWarm(): void;
+    warmUp(): Promise<void>;
+    private _doWarmup;
+    private window;
+    /**
+     * Bound one document for a pair, given how much of the window the query already spends.
+     * Returns the document text unchanged when it fits.
+     */
+    private boundDocument;
+    rerank(query: string, candidates: {
+        id: number;
+        text: string;
+    }[]): Promise<number[]>;
+}
+export {};

package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts

@@ -0,0 +1,10 @@
+import type { Reranker } from '../interfaces.js';
+/** `none` — reranking disabled (the default). Reports unavailable so callers skip the rerank pass entirely. */
+export declare class NoneReranker implements Reranker {
+    readonly name = "none";
+    isAvailable(): boolean;
+    rerank(_query: string, candidates: {
+        id: number;
+        text: string;
+    }[]): Promise<number[]>;
+}

package/lib/engine/mem-retrieval/rerank.d.ts

@@ -0,0 +1,13 @@
+import type { Reranker } from './interfaces.js';
+/**
+ * The one rerank-and-reorder pass shared by every retrieval leg (memory search,
+ * knowledge search — the blocks were byte-identical and had already drifted once
+ * on the miss-sort sentinel).
+ *
+ * Contract: `reranker.rerank` returns candidate ids best-first; ids it omits sort
+ * last with their fused order preserved (stable sort). A disabled/unavailable
+ * reranker is a pass-through.
+ */
+export declare function rerankHits<T extends {
+    id: number;
+}>(reranker: Reranker, query: string, hits: T[], textOf: (id: number) => string): Promise<T[]>;