akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/storage/repositories/units-repository.js

@@ -0,0 +1,510 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `index.db` embedding-unit repository (docs/plans/index-fragment-vectors.md).
+ *
+ * Owns the durable, content-addressed vector store: `units_vec` (the vec0
+ * table, the ONE copy of every embedding), `units` (a plain lookup table
+ * mirroring `units_vec`'s `(unit_hash, identity)` key — vec0 auxiliary
+ * columns are not indexable, so a point lookup or an `IN (...)` filter on
+ * `units_vec` directly would be a full scan), and `entry_units` (the cheap,
+ * derived entry → ordinal → unit_hash mapping that a reindex rebuilds).
+ *
+ * Lifecycle invariant (the whole design, docs/plans/index-fragment-vectors.md
+ * "Vectors are content-addressed and never rebuilt"): `units` and `units_vec`
+ * are created if missing and NEVER dropped by a generation rebuild, by
+ * `akm index --full`, or by any embeddings purge — only {@link dropOtherIdentities}
+ * removes rows, and only for an identity other than the one being kept. A row
+ * exists in `units`/`units_vec` if and only if a vector was actually written
+ * for it: {@link upsertUnitVectors} and {@link dropOtherIdentities} are both
+ * no-ops when sqlite-vec is unavailable, rather than recording a hash with no
+ * vector behind it (which would make {@link listMissingHashes} lie).
+ */
+import { ConfigError } from "../../core/errors.js";
+import { withImmediateTransaction } from "../../core/state-db.js";
+import { getMeta } from "./index-meta-repository.js";
+import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
+import { isVecAvailable } from "./index-vec-repository.js";
+// ── Schema ──────────────────────────────────────────────────────────────────
+function createUnitsVecTable(db, dim) {
+    db.exec(`
+    CREATE VIRTUAL TABLE units_vec USING vec0(
+      unit_id   INTEGER PRIMARY KEY,
+      embedding FLOAT[${dim}],
+      +unit_hash TEXT,
+      +identity  TEXT
+    );
+  `);
+}
+function tableExists(db, name) {
+    return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) != null;
+}
+/**
+ * The vector width `units_vec` was created at, read back from its own DDL
+ * (`sqlite_master.sql` carries the `CREATE VIRTUAL TABLE` text verbatim) so
+ * there is one source of truth for "what width is this table at right now" —
+ * no separate meta key to keep in sync. Returns `undefined` when the table
+ * does not exist.
+ */
+function unitsVecDimension(db) {
+    const row = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'units_vec'").get();
+    const match = row?.sql.match(/FLOAT\[(\d+)\]/);
+    return match?.[1] === undefined ? undefined : Number(match[1]);
+}
+/**
+ * Create `units`, `entry_units` and (when sqlite-vec is loaded) `units_vec`
+ * if they do not already exist. Idempotent and safe to call on every schema
+ * ensure — it never touches an existing table's rows or drops anything.
+ *
+ * `dim` only matters the first time `units_vec` is created; an existing table
+ * keeps its width until {@link dropOtherIdentities} recreates it for a real
+ * dimension change.
+ */
+export function ensureUnitTables(db, dim) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS units (
+      unit_id   INTEGER PRIMARY KEY,
+      unit_hash TEXT NOT NULL,
+      identity  TEXT NOT NULL,
+      UNIQUE (unit_hash, identity)
+    );
+    CREATE TABLE IF NOT EXISTS entry_units (
+      entry_id    INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE,
+      ordinal     INTEGER NOT NULL,
+      fragment_id TEXT,
+      unit_hash   TEXT NOT NULL,
+      PRIMARY KEY (entry_id, ordinal)
+    );
+    CREATE INDEX IF NOT EXISTS entry_units_hash ON entry_units(unit_hash);
+  `);
+    // Mirrors entries_vec: a vec0 virtual table needs the extension loaded to
+    // even be created. `units`/`entry_units` above are plain tables and exist
+    // regardless, so lookups and the entry→hash mapping keep working; only the
+    // KNN path is unavailable (see `searchUnits`).
+    if (!isVecAvailable(db))
+        return;
+    if (tableExists(db, "units_vec"))
+        return;
+    createUnitsVecTable(db, dim);
+}
+// ── Vector storage ──────────────────────────────────────────────────────────
+function float32Buffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+}
+/**
+ * Write new (hash, identity) → vector rows. Rows whose (hash, identity)
+ * already exists in `units` are left as-is in `units` (content-addressed: the
+ * same text under the same identity is the same vector) but their `units_vec`
+ * row is still rewritten, so a prior run that wrote the `units` row but failed
+ * before its vec0 mirror (or ran while sqlite-vec was unavailable) self-heals
+ * on the next call instead of leaving a permanent gap `searchUnits` can never
+ * see.
+ *
+ * A no-op returning `{ inserted: 0, failed: 0 }` when sqlite-vec is
+ * unavailable: there is no BLOB fallback table for units
+ * (docs/plans/index-fragment-vectors.md — one copy of every vector, in
+ * vec0), so writing a `units` row with no vector behind it would make it
+ * look present to {@link listMissingHashes} forever.
+ *
+ * Each row commits in its OWN transaction (not one transaction for the whole
+ * batch): a vec0 insert can throw — most concretely a vector-width mismatch,
+ * "Dimension mismatch for inserted vector" — and a single malformed or
+ * wrong-width row from an otherwise-good provider response must not roll
+ * back every other row this call already wrote. A row that fails is left out
+ * of `units` entirely (any `units` placeholder this call itself just created
+ * for it is deleted again), not left dangling with no vector behind it, so
+ * {@link listMissingHashes} still sees it as missing and retries it on the
+ * next drain — the same "no row without a vector" invariant the module
+ * doc above states, just enforced per-row instead of per-batch.
+ */
+export function upsertUnitVectors(db, rows) {
+    if (rows.length === 0 || !isVecAvailable(db))
+        return { inserted: 0, failed: 0 };
+    const insertUnit = db.prepare("INSERT INTO units (unit_hash, identity) VALUES (?, ?) ON CONFLICT(unit_hash, identity) DO NOTHING");
+    const selectUnitId = db.prepare("SELECT unit_id FROM units WHERE unit_hash = ? AND identity = ?");
+    const deleteVec = db.prepare("DELETE FROM units_vec WHERE unit_id = ?");
+    const insertVec = db.prepare("INSERT INTO units_vec (unit_id, embedding, unit_hash, identity) VALUES (?, ?, ?, ?)");
+    const deleteUnit = db.prepare("DELETE FROM units WHERE unit_id = ?");
+    // Each row gets its OWN `BEGIN IMMEDIATE` transaction (not a shared bare
+    // `db.transaction()`), for two reasons: a bare deferred transaction whose
+    // body reads (selectUnitId) before it writes fails instantly with
+    // SQLITE_BUSY under a competing writer instead of honouring
+    // `busy_timeout` (docs/plans/index-redesign.md rule 5), and this call
+    // must still commit PER ROW even once immediate — a malformed vector must
+    // not roll back the rest of a provider batch (see the docstring above).
+    const writeOne = (row) => withImmediateTransaction(db, () => {
+        const result = insertUnit.run(row.hash, row.identity);
+        const freshlyInserted = Number(result.changes) > 0;
+        const unitRow = selectUnitId.get(row.hash, row.identity);
+        if (!unitRow)
+            return false;
+        // DELETE-then-INSERT (not INSERT OR REPLACE) — the established pattern
+        // for writing a fixed-rowid row into a vec0 table on this driver.
+        deleteVec.run(unitRow.unit_id);
+        try {
+            insertVec.run(unitRow.unit_id, float32Buffer(row.vector), row.hash, row.identity);
+        }
+        catch (err) {
+            // Undo the vector-less `units` row this call would otherwise leave
+            // behind — whether it was just created above or already existed (its
+            // prior vector, if any, is already gone via deleteVec either way).
+            deleteUnit.run(unitRow.unit_id);
+            throw err;
+        }
+        return freshlyInserted;
+    }, "index");
+    let inserted = 0;
+    let failed = 0;
+    for (const row of rows) {
+        try {
+            if (writeOne(row))
+                inserted++;
+        }
+        catch {
+            failed++;
+        }
+    }
+    return { inserted, failed };
+}
+/**
+ * Which of `hashes` have no vector yet for `identity`. The set-difference
+ * that drives the whole indexing loop (docs/plans/index-fragment-vectors.md
+ * "Indexing is a set difference"): callers hash every unit's text, ask what is
+ * missing, and only send that difference to the provider.
+ *
+ * Deduplicated and returned in first-occurrence order — the same unit text
+ * commonly recurs across entries (content-addressed), so de-duping here is
+ * what keeps a shared fragment from being requested from the provider twice
+ * in one pass.
+ */
+export function listMissingHashes(db, hashes, identity) {
+    const unique = [...new Set(hashes)];
+    if (unique.length === 0)
+        return [];
+    const present = new Set();
+    for (let offset = 0; offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT unit_hash FROM units WHERE identity = ? AND unit_hash IN (${placeholders})`)
+            .all(identity, ...chunk);
+        for (const row of rows)
+            present.add(row.unit_hash);
+    }
+    return unique.filter((hash) => !present.has(hash));
+}
+/**
+ * Remove rows for every identity except `keep`, recreating `units_vec` at
+ * `dim` first when its current width differs from `dim`.
+ *
+ * vec0's embedding column width is fixed for the whole table, so a genuine
+ * dimension change (a real model swap, not a rename) cannot be expressed as a
+ * row-level DELETE — the table is dropped and recreated instead. Any row that
+ * survives a width mismatch could only belong to `keep`, and it could only
+ * exist if it had already been written at `dim` — which the OLD, differently
+ * sized table would have rejected — so nothing behind `keep` is lost by the
+ * drop.
+ *
+ * The width check runs UNCONDITIONALLY, independent of `staleCount` (E6): a
+ * fresh index (or one whose only prior identity was already dropped) has
+ * zero rows under any identity, so gating the width check behind
+ * `staleCount > 0` — the round-2 shape of this bug — meant a `units_vec`
+ * created at a stale/misconfigured width (e.g. `embedding.dimension` set to
+ * a value the provider does not actually return) could never be corrected:
+ * every write fails "Dimension mismatch for inserted vector", the row this
+ * call itself just tried to write is discarded, `units` stays empty, and
+ * `staleCount` stays 0 forever — not even `akm index --reembed` (which
+ * reaches this same function) can recover, only deleting index.db could.
+ * Checking the width regardless of `staleCount` costs nothing extra when
+ * nothing is stale (`unitsVecDimension` is one cheap `sqlite_master` read)
+ * and fixes exactly this: the very first call for a new width recreates the
+ * table before anything is ever written under it.
+ *
+ * The drop/create/delete (or delete-stale-rows/delete) sequence runs in ONE
+ * transaction (E5a): as two separate autocommit statements, a concurrent
+ * `upsertUnitVectors` landing between the `DROP TABLE` and the `CREATE
+ * TABLE` would silently lose an already-paid-for embedding (counted
+ * `failed` and discarded) rather than simply retrying against the
+ * recreated table.
+ *
+ * A no-op when sqlite-vec is unavailable: `units`/`units_vec` only ever gain
+ * rows when a vector was actually written (see {@link upsertUnitVectors}), so
+ * there is nothing to remove and no vec0 table this could safely touch.
+ */
+export function dropOtherIdentities(db, keep, dim) {
+    if (!isVecAvailable(db))
+        return { removed: 0 };
+    ensureUnitTables(db, dim);
+    const currentDim = unitsVecDimension(db);
+    const widthChanged = currentDim !== undefined && currentDim !== dim;
+    const staleCount = db.prepare("SELECT COUNT(*) AS n FROM units WHERE identity != ?").get(keep).n;
+    if (staleCount === 0 && !widthChanged)
+        return { removed: 0 };
+    withImmediateTransaction(db, () => {
+        if (widthChanged) {
+            db.exec("DROP TABLE IF EXISTS units_vec");
+            createUnitsVecTable(db, dim);
+        }
+        else {
+            const staleIds = db.prepare("SELECT unit_id FROM units WHERE identity != ?").all(keep).map((row) => row.unit_id);
+            for (let offset = 0; offset < staleIds.length; offset += SQLITE_CHUNK_SIZE) {
+                const chunk = staleIds.slice(offset, offset + SQLITE_CHUNK_SIZE);
+                const placeholders = chunk.map(() => "?").join(",");
+                db.prepare(`DELETE FROM units_vec WHERE unit_id IN (${placeholders})`).run(...chunk);
+            }
+        }
+        db.prepare("DELETE FROM units WHERE identity != ?").run(keep);
+    }, "index");
+    return { removed: staleCount };
+}
+// ── Entry → unit mapping ────────────────────────────────────────────────────
+/** Replace every `entry_units` row for `entryId` with `units` (delete-then-insert). */
+export function replaceEntryUnits(db, entryId, units) {
+    // `applyChange` (indexer/reconcile.ts) normally already holds an outer
+    // `withImmediateTransaction` when it calls this — the join-if-open guard
+    // means this runs inside that transaction rather than opening its own.
+    withImmediateTransaction(db, () => {
+        db.prepare("DELETE FROM entry_units WHERE entry_id = ?").run(entryId);
+        const insert = db.prepare("INSERT INTO entry_units (entry_id, ordinal, fragment_id, unit_hash) VALUES (?, ?, ?, ?)");
+        for (const unit of units) {
+            insert.run(entryId, unit.ordinal, unit.fragmentId, unit.hash);
+        }
+    }, "index");
+}
+/** Remove every `entry_units` row for the given entries (e.g. entries deleted outside a cascade). */
+export function deleteEntryUnits(db, entryIds) {
+    const unique = [...new Set(entryIds)];
+    if (unique.length === 0)
+        return;
+    for (let offset = 0; offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        db.prepare(`DELETE FROM entry_units WHERE entry_id IN (${placeholders})`).run(...chunk);
+    }
+}
+// ── Search ──────────────────────────────────────────────────────────────────
+/**
+ * vec0 0.1.9 rejects a KNN query that also filters on an auxiliary column —
+ * "An illegal WHERE constraint was provided on a vec0 auxiliary column in a
+ * KNN query" — so `identity` cannot be pushed into the `MATCH` query itself
+ * and has to be filtered afterward. Over-fetching by this factor keeps that
+ * post-filter from starving recall during the rare, transient window where
+ * more than one identity's rows coexist in `units_vec` (the steady state,
+ * enforced by {@link dropOtherIdentities}, is exactly one identity).
+ *
+ * index-redesign integration note (kept, not dropped): the notes proposed
+ * dropping this filter/overfetch entirely for B5 and asserting the
+ * one-identity invariant where the store is written instead, keeping the
+ * overfetch only if a test proves two identities can coexist. One does:
+ * `tests/storage/units-repository.test.ts`'s "searchUnits returns nearest
+ * units first, scoped by identity" writes two identities into `units_vec`
+ * via the real {@link upsertUnitVectors} (not a raw-SQL seed) and asserts
+ * `searchUnits` still returns only the requested identity's nearest rows —
+ * without this post-filter that test's own tied-vector fixture would drop
+ * a real hit for the requested identity, not merely tolerate a stray one.
+ * `identity` is also a required part of this function's stage-1 contract
+ * signature, used by every real caller (B3's semantic search branch), so an
+ * unenforced parameter here would be a silent footgun. Left in place; no
+ * hard invariant assertion added elsewhere, since the transient
+ * two-identity window this docblock already describes is a real,
+ * non-error state a hard assertion would wrongly reject.
+ */
+const UNIT_SEARCH_OVERFETCH = 4;
+/**
+ * Which of a KNN candidate batch's unit hashes have an owning entry (via
+ * `entry_units` → `entries`) satisfying both a `typeFilter` and an
+ * `excludeTypes` predicate at once (index-redesign search-fix item 3).
+ *
+ * vec0 rejects a KNN query that also filters on an auxiliary column ("An
+ * illegal WHERE constraint was provided on a vec0 auxiliary column in a KNN
+ * query"), so the type predicate cannot be pushed INTO the `embedding MATCH`
+ * query itself. Solved instead by joining AROUND the KNN: run the KNN
+ * untyped (already over-fetching via {@link UNIT_SEARCH_OVERFETCH}), then
+ * filter the raw candidate hashes through `entry_units`/`entries` BEFORE the
+ * `k` cap below — the same "filter before the cap" fix as the lexical side
+ * (`runUnitsFtsQuery`, db-search.ts), so an eligible neighbor is never pushed
+ * out of the window by ineligible ones that rank closer.
+ */
+function eligibleUnitHashesByType(db, hashes, typeOpts) {
+    const eligible = new Set();
+    const clauses = [];
+    const baseParams = [];
+    if (typeOpts.typeFilter?.length) {
+        clauses.push(`e.type IN (${typeOpts.typeFilter.map(() => "?").join(",")})`);
+        baseParams.push(...typeOpts.typeFilter);
+    }
+    if (typeOpts.excludeTypes?.length) {
+        clauses.push(`e.type NOT IN (${typeOpts.excludeTypes.map(() => "?").join(",")})`);
+        baseParams.push(...typeOpts.excludeTypes);
+    }
+    for (let offset = 0; offset < hashes.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = hashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT DISTINCT eu.unit_hash AS hash FROM entry_units eu JOIN entries e ON e.id = eu.entry_id
+         WHERE eu.unit_hash IN (${placeholders}) AND ${clauses.join(" AND ")}`)
+            .all(...chunk, ...baseParams);
+        for (const row of rows)
+            eligible.add(row.hash);
+    }
+    return eligible;
+}
+/**
+ * KNN search over `units_vec` for the given `identity`, nearest first.
+ * Throws a {@link ConfigError} when sqlite-vec is not loaded — there is no JS
+ * fallback for units (docs/plans/index-fragment-vectors.md), so the caller is
+ * expected to catch this and fall back to lexical search.
+ *
+ * `typeOpts`, when given, is applied to the raw KNN candidates BEFORE the `k`
+ * cap (see {@link eligibleUnitHashesByType} for how — vec0 cannot filter this
+ * inside the KNN query itself), the same fix `runUnitsFtsQuery` applies on
+ * the lexical side.
+ */
+export function searchUnits(db, query, k, identity, typeOpts) {
+    if (!isVecAvailable(db)) {
+        throw new ConfigError("sqlite-vec is not loaded, so unit search cannot run its vec0 KNN query. Fall back to lexical search.", "EMBEDDING_VEC_UNAVAILABLE");
+    }
+    if (k <= 0)
+        return [];
+    const rows = db
+        .prepare("SELECT unit_id AS unitId, unit_hash AS hash, identity AS identity, distance AS distance " +
+        "FROM units_vec WHERE embedding MATCH ? AND k = ?")
+        .all(float32Buffer(query), k * UNIT_SEARCH_OVERFETCH);
+    const identityFiltered = rows.filter((row) => row.identity === identity);
+    const hasTypeFilter = Boolean(typeOpts?.typeFilter?.length || typeOpts?.excludeTypes?.length);
+    const typed = hasTypeFilter
+        ? (() => {
+            const eligible = eligibleUnitHashesByType(db, [...new Set(identityFiltered.map((row) => row.hash))], typeOpts);
+            return identityFiltered.filter((row) => eligible.has(row.hash));
+        })()
+        : identityFiltered;
+    return typed.slice(0, k).map(({ unitId, hash, distance }) => ({ unitId, hash, distance }));
+}
+/**
+ * Group unit hits to their owning entries, keeping the best (lowest distance)
+ * unit per entry. A unit hash can back more than one `entry_units` row (the
+ * same fragment text shared across entries, or multiple hits sharing a hash
+ * under different identities in the rare transient window `searchUnits`
+ * already tolerates), so both the hit list and the hash→entries fan-out are
+ * reduced to a minimum by distance.
+ */
+export function groupUnitHitsByEntry(db, hits) {
+    const result = new Map();
+    if (hits.length === 0)
+        return result;
+    const distanceByHash = new Map();
+    for (const hit of hits) {
+        const existing = distanceByHash.get(hit.hash);
+        if (existing === undefined || hit.distance < existing)
+            distanceByHash.set(hit.hash, hit.distance);
+    }
+    const hashes = [...distanceByHash.keys()];
+    for (let offset = 0; offset < hashes.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = hashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT entry_id AS entryId, fragment_id AS fragmentId, unit_hash AS hash FROM entry_units WHERE unit_hash IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows) {
+            const distance = distanceByHash.get(row.hash);
+            if (distance === undefined)
+                continue;
+            const existing = result.get(row.entryId);
+            if (!existing || distance < existing.distance) {
+                result.set(row.entryId, { distance, fragmentId: row.fragmentId, hash: row.hash });
+            }
+        }
+    }
+    return result;
+}
+/**
+ * The number of raw unit hits `getNeighborsByEntryId` over-fetches relative
+ * to the requested `k` other entries, one-sentence reason: an entry can own
+ * several units (its card plus every fragment), so the nearest raw unit hits
+ * collapse into fewer distinct OTHER entries once grouped by entry and the
+ * querying entry's own units are excluded — over-fetching keeps that
+ * collapse from starving the requested `k`.
+ */
+const NEIGHBOR_CANDIDATE_OVERFETCH = 4;
+/**
+ * Decode a `units_vec` embedding BLOB (the same `Buffer.from(new
+ * Float32Array(vector).buffer)` layout {@link upsertUnitVectors} writes) back
+ * into a plain vector, dimension inferred from byte length.
+ */
+function float32BufferToVector(buf) {
+    return Array.from(new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4)));
+}
+/**
+ * The `k` nearest OTHER entries to `id`, by distance between their card
+ * units (`entry_units` ordinal 0) under the active embedding identity —
+ * index-redesign-contract.md B5f item 4's replacement for the legacy
+ * `embeddings`/`entries_vec`-backed `getNeighborsByEntryId`
+ * (`index-vec-repository.ts`), which nothing wrote to since the units path
+ * became the only index. Those tables themselves — and everything that only
+ * ever read or wrote them — were deleted in B5h.
+ *
+ * No re-embedding, no network: reads the entry's own already-indexed card
+ * vector via a `units` rowid lookup (not a `units_vec` aux-column scan) and
+ * reuses {@link searchUnits}'s KNN, then groups the raw unit hits back to
+ * entries the same way search does ({@link groupUnitHitsByEntry}).
+ * The querying entry is excluded from its own result — its card is its own
+ * nearest neighbour at distance 0, and a caller asking for `k` neighbours
+ * wants `k` genuinely OTHER entries, not one slot spent confirming an entry
+ * is close to itself.
+ *
+ * Returns `[]` when there is no active identity yet, `id` has no card unit,
+ * or that unit has no vector for the active identity (drain has not reached
+ * it) — the caller (`consolidate.ts`'s `narrowToIncrementalCandidates`) fails
+ * open to the full pool on an empty/unusable result, same as it did for the
+ * legacy table being absent.
+ */
+export function getNeighborsByEntryId(db, id, k) {
+    if (k <= 0)
+        return [];
+    const identity = getMeta(db, "embeddingIdentity");
+    if (!identity)
+        return [];
+    const cardRow = db
+        .prepare("SELECT unit_hash AS unitHash FROM entry_units WHERE entry_id = ? AND ordinal = 0")
+        .get(id);
+    if (!cardRow)
+        return [];
+    const unitRow = db
+        .prepare("SELECT unit_id AS unitId FROM units WHERE unit_hash = ? AND identity = ?")
+        .get(cardRow.unitHash, identity);
+    if (!unitRow)
+        return [];
+    const vecRow = db.prepare("SELECT embedding FROM units_vec WHERE unit_id = ?").get(unitRow.unitId);
+    if (!vecRow)
+        return [];
+    const queryVector = float32BufferToVector(vecRow.embedding);
+    if (queryVector.length === 0)
+        return [];
+    const hits = searchUnits(db, queryVector, k * NEIGHBOR_CANDIDATE_OVERFETCH, identity);
+    const byEntry = groupUnitHitsByEntry(db, hits);
+    byEntry.delete(id);
+    return [...byEntry.entries()]
+        .sort((a, b) => a[1].distance - b[1].distance)
+        .slice(0, k)
+        .map(([entryId, match]) => ({ id: entryId, distance: match.distance }));
+}
+// ── Coverage ────────────────────────────────────────────────────────────────
+/** How much of the current `entry_units` mapping has a vector for `identity`, for progress reporting. */
+export function unitCoverage(db, identity) {
+    const entries = db.prepare("SELECT COUNT(DISTINCT entry_id) AS n FROM entry_units").get().n;
+    const unitsTotal = db.prepare("SELECT COUNT(*) AS n FROM entry_units").get().n;
+    const unitsPresent = db
+        .prepare("SELECT COUNT(*) AS n FROM entry_units eu JOIN units u ON u.unit_hash = eu.unit_hash AND u.identity = ?")
+        .get(identity).n;
+    const entriesFullyCovered = db
+        .prepare(`SELECT COUNT(*) AS n FROM (
+           SELECT eu.entry_id
+           FROM entry_units eu
+           LEFT JOIN units u ON u.unit_hash = eu.unit_hash AND u.identity = ?
+           GROUP BY eu.entry_id
+           HAVING COUNT(*) = COUNT(u.unit_id)
+         )`)
+        .get(identity).n;
+    return { entries, entriesFullyCovered, unitsTotal, unitsPresent };
+}

package/docs/migration/release-notes/0.9.15.md

@@ -85,27 +85,33 @@ scheduled or opportunistic run step aside instead of contending with a rebuild
 already in progress; the shipped `index-refresh` scheduled task already passes
 it.
 
-`embedding.maxTokens`'s default (the per-request token budget) is now 6000,
-down from 8000: a field report on an 8192-token llama.cpp embedder showed the
-4-chars-per-token estimator undercounts dense technical text by 7-55%, so the
-old default regularly overshot the endpoint's real context window. If you
-already set `embedding.maxTokens` explicitly, this default change does not
-affect you — your configured value is unchanged. `akm index` also now
-recovers automatically within a run: on the first request rejected for
-exceeding the endpoint's context window, it lowers its effective budget for
-the rest of that run (reported with one line) rather than continuing to hit
-the same wall on every following batch.
+`akm index` now packs embedding requests against the provider's own probed
+context window and slot count (llama.cpp's `GET /props`, Ollama's
+`POST /api/show`) instead of a flat configured token budget: an 8192-token
+llama.cpp embedder, for example, is packed against its real 8192 rather
+than a generic guess. An endpoint that answers neither probe (an
+OpenAI-compatible server, a gateway) gets a conservative built-in default,
+with the same run-scoped recovery as before — on the first request rejected
+for exceeding the endpoint's context window, `akm index` lowers its
+effective budget for the rest of that run (reported with one line) rather
+than continuing to hit the same wall on every following batch; a window akm
+actually probed from the endpoint is treated as authoritative and is never
+second-guessed this way. `embedding.maxTokens` and `embedding.batchSize`
+(the previous per-request token-budget and document-count config keys) are
+retired — see [Retired Configuration](../../reference/configuration.md#retired-configuration)
+— since the provider's own limits are now the source of truth.
 
 `embedding.concurrency` (positive integer, 1-16) overrides the number of
 embedding requests kept in flight at once, which otherwise defaults to 1 for
-a loopback endpoint and 2 for a remote one. Set it only for an endpoint that
-genuinely serves parallel requests — a local model server started with a
-multi-slot flag (llama.cpp's `--parallel N`, vLLM) — since the default
-already protects an ordinary single-slot server from reload-thrash.
-Embedding throughput is still tuned first by `embedding.batchSize`
-(documents per request) and `embedding.maxTokens` (token
-budget per request); the concurrency override is a second lever for a
-server that can actually use it.
+a loopback endpoint and 2 for a remote one, or the provider's own probed
+slot count (llama.cpp's `total_slots`) when the probe reports one. Set an
+explicit override only for an endpoint that genuinely serves parallel
+requests — a local model server started with a multi-slot flag (llama.cpp's
+`--parallel N`, vLLM) — since the default already protects an ordinary
+single-slot server from reload-thrash. Embedding throughput is still tuned
+first by request SIZE — the probed token budget and document-count cap
+above; the concurrency override is a second lever for a server that can
+actually use it.
 
 `embedding.timeoutMs` bounds each embedding request (default 120s, up from a
 prior fixed 30s that cut off a slow local model server mid-response). It is
@@ -142,24 +148,16 @@ this is a drop-in fix for anyone running `akm` under a scheduler,
 supervisor, or hook that can time out or kill the launcher process
 directly.
 
-`embedding.maxInputTokens` (default 512) now caps how much of a single
-document's text is sent to the embedding provider, truncating to the head
-instead of ever failing a whole batch over one oversized document.
-
-- An existing install's already-stored vectors are untouched and stay
-  valid — this only changes what happens for entries embedded *after*
-  upgrading.
-- New embeddings (any entry indexed for the first time, or re-indexed after
-  a content change) go through the new 512-token cap by default. If you
-  were relying on documents longer than ~2000 characters being embedded in
-  full, set `embedding.maxInputTokens` higher in `config.json`.
-- `akm index --reembed` re-embeds every entry under the new cap — run it if
-  you want your entire existing index rebuilt against the new default (or a
-  custom `embedding.maxInputTokens` you've set).
-- `embedding.contextLength` is Ollama's `num_ctx` only now; it no longer
-  also sets the per-request token budget (`embedding.maxTokens`). If you had
-  set `contextLength` specifically to control request batching (not your
-  Ollama server's context window), set `embedding.maxTokens` instead.
+A large document is no longer truncated for embedding — `akm index` splits
+it into multiple content-addressed units instead (one vector per section,
+sized against the embedding provider's own probed context window), so a
+long document's later sections are searchable too, not silently dropped
+past a head cap. `embedding.maxInputTokens`, the earlier per-document
+truncation cap this superseded within the same 0.9.15 cycle, is retired —
+see [Retired Configuration](../../reference/configuration.md#retired-configuration).
+`embedding.contextLength` is retired too: Ollama's `num_ctx` is now sent
+automatically from the same probe, or set explicitly via
+`embedding.ollamaOptions.num_ctx`.
 
 **Which token knob fixed the original 8k-context overflow.** A 0.9.15-beta
 field report described documents estimated under the request budget that