akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/indexer/passes/dir-staleness.js

@@ -1,161 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * Incremental dir-staleness engine.
- *
- * Decides, per stash directory, whether the directory's indexed rows are still
- * fresh relative to what is on disk — so an incremental `akm index` run can
- * skip unchanged directories instead of regenerating their metadata.
- *
- * Two persisted signals back the decision:
- *   1. The `entries` rows already indexed for the directory (`getEntriesByDir`).
- *   2. The `index_dir_state` row (`getIndexDirState`): the fingerprint of the
- *      directory's walked file set (basename set + max mtime, `computeDirFingerprint`)
- *      as of its last drain, plus the row count that drain persisted.
- *
- * `getCachedDirState` is the pre-drain gate (#900): a directory whose walked
- * fingerprint still matches its row is skipped before any file is read.
- */
-import { createHash } from "node:crypto";
-import fs from "node:fs";
-import path from "node:path";
-import { compareCodePoints } from "../../core/common.js";
-import { getEntriesByDir } from "../../storage/repositories/index-entries-repository.js";
-import { getIndexDirState } from "../../storage/repositories/index-meta-repository.js";
-/**
- * Post-drain freshness verdict. `files` is the recognized file set the drain
- * produced (compared against the persisted entries); `fingerprint` is the
- * walked-set fingerprint compared against the persisted row and defaults to
- * one computed over `files`.
- */
-export function getDirIndexState(db, dirPath, files, builtAtMs, indexVariant = "", fingerprint = computeDirFingerprint(dirPath, files, indexVariant)) {
-    const prevEntries = getEntriesByDir(db, dirPath);
-    if (prevEntries.length > 0) {
-        const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
-        if (staleReason)
-            return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
-        const cachedState = getIndexDirState(db, dirPath);
-        if (!cachedState || cachedState.fileSetHash !== fingerprint.fileSetHash) {
-            return {
-                stale: true,
-                reason: { kind: "index-context-changed", detail: indexVariant },
-                persistedRowCount: prevEntries.length,
-            };
-        }
-        return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
-    }
-    const cachedState = getIndexDirState(db, dirPath);
-    if (cachedState && cachedState.fileSetHash === fingerprint.fileSetHash) {
-        return {
-            stale: false,
-            reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
-            persistedRowCount: 0,
-        };
-    }
-    return {
-        stale: true,
-        reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
-        persistedRowCount: 0,
-    };
-}
-/**
- * Pre-drain gate (#900). A directory whose walked-set fingerprint matches its
- * persisted row cannot recognize differently than last time, so it is skipped
- * before `drainDirDocuments` reads a single file. A row that recorded a real
- * generation (`rowCount > 0`) is skipped outright; a zero-row or pre-#900 row
- * goes through the entries-aware check so the dedup-order guard still applies.
- */
-export function getCachedDirState(db, dirPath, files, builtAtMs, priorDirsChanged, indexVariant, fingerprint) {
-    const cached = getIndexDirState(db, dirPath);
-    if (!cached || cached.fileSetHash !== fingerprint.fileSetHash)
-        return undefined;
-    if (cached.rowCount !== undefined && cached.rowCount > 0) {
-        return { stale: false, reason: { kind: "unchanged-precheck" }, persistedRowCount: cached.rowCount };
-    }
-    const state = getDirIndexState(db, dirPath, files, builtAtMs, indexVariant, fingerprint);
-    if (state.stale || state.reason.kind !== "cached-zero-row-state")
-        return undefined;
-    if (!canUseIncrementalSkip(state, priorDirsChanged))
-        return undefined;
-    return state;
-}
-export function canUseIncrementalSkip(state, priorDirsChanged) {
-    return !(priorDirsChanged &&
-        state.reason.kind === "cached-zero-row-state" &&
-        state.reason.detail === "deduped-zero-row");
-}
-export function computeDirFingerprint(_dirPath, files, indexVariant = "") {
-    // One `statSync` per file — the same call this function has always made — but
-    // every field it returns that can witness a change is kept, per file, instead
-    // of being collapsed into a single max.
-    //
-    // `Math.max` over mtimes discarded everything except the newest file, so an
-    // edit to any other file landed below the max and was invisible; and mtime
-    // alone is writable by ordinary tooling (`touch -r`, `rsync --times`,
-    // `cp -p`, archive extraction), so a restored timestamp hid an edit outright.
-    // Size catches any length-changing edit; ctime catches the rest, because
-    // utimes(2) cannot hold the inode's change time back.
-    //
-    // This is still a heuristic: ctime also moves on metadata-only changes
-    // (chmod/chown) and after copying a tree, which costs an unnecessary rescan.
-    // That direction is safe — extra work, never stale content.
-    const entries = [];
-    let fileMtimeMaxMs = 0;
-    for (const file of [...new Set(files)].sort(compareCodePoints)) {
-        const name = path.basename(file);
-        try {
-            // `bigint: true` is the same syscall but reports nanoseconds. Millisecond
-            // floats would let an edit made inside the same millisecond as the last
-            // run's stat land on an identical digest.
-            const stat = fs.statSync(file, { bigint: true });
-            fileMtimeMaxMs = Math.max(fileMtimeMaxMs, Number(stat.mtimeMs));
-            entries.push(`${name}\0${stat.size}\0${stat.mtimeNs}\0${stat.ctimeNs}`);
-        }
-        catch {
-            // Unreadable or vanished: record it as such so the digest differs from
-            // any run where the file could be read, forcing a rescan.
-            entries.push(`${name}\0unreadable`);
-        }
-    }
-    const digest = createHash("sha256")
-        .update([indexVariant, ...entries].join("\n"), "utf8")
-        .digest("hex");
-    return { fileSetHash: digest, fileMtimeMaxMs };
-}
-function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
-    const prevFileNames = new Set(previousEntries
-        .map((ie) => {
-        const fromPath = path.basename(ie.filePath);
-        return fromPath || ie.entry.filename;
-    })
-        .filter((e) => !!e));
-    const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
-    if (prevFileNames.size !== currFileNames.size) {
-        return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
-    }
-    for (const name of currFileNames) {
-        if (!prevFileNames.has(name))
-            return { kind: "file-set-changed", detail: name };
-    }
-    for (const file of currentFiles) {
-        try {
-            if (fs.statSync(file).mtimeMs > builtAtMs)
-                return { kind: "mtime-changed", detail: path.basename(file) };
-        }
-        catch {
-            return { kind: "missing-file", detail: path.basename(file) };
-        }
-    }
-    return undefined;
-}
-export function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
-    if (dedupedRows > 0)
-        return "deduped-zero-row";
-    const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
-    if (workflowNoise)
-        return "workflow-noise";
-    if (!stash || stash.entries.length === 0)
-        return "empty-generated-set";
-    return `zero-row:${priorReason?.kind ?? "unknown"}`;
-}

package/dist/storage/repositories/embedding-salvage-repository.js

@@ -1,184 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * `index.db` embedding salvage (#955) — a transient, self-emptying table
- * that lets a full rebuild or an index-generation bump reuse vectors instead
- * of re-embedding a corpus whose content did not change.
- *
- * Zero steady-state cost by design: this is NOT a second embedding cache.
- * Rows are copied aside only at the moment they would otherwise be discarded
- * wholesale — a full-index wipe (`persistDirRecords`) or a generation bump
- * (`rebuildIncompatibleIndexGeneration`) — and are consumed by the very next
- * embedding pass (`generateEmbeddingsForDb`). A pass that completes without
- * abort or circuit-break purges whatever is left; an interrupted pass leaves
- * the table for the next attempt to pick up.
- *
- * Reuse is keyed on `sha256(search_text)` plus the fingerprint the vector was
- * generated under — a fingerprint mismatch or a single-byte content change
- * both correctly fall through to a real provider call. `content_hash` is the
- * PRIMARY KEY (not `(content_hash, fingerprint)`) so relabeling a whole
- * generation's fingerprint after a canary "keep" verdict is one UPDATE, and a
- * hash colliding across two discards simply keeps the most recent copy —
- * salvage is a best-effort optimization, not a durable multi-generation
- * archive.
- */
-import { createHash } from "node:crypto";
-import { blobToEmbedding } from "./embeddings-repository.js";
-import { getMeta } from "./index-meta-repository.js";
-import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
-/**
- * Create the salvage table. Additive-only DDL: it carries no bearing on the
- * `entries` generation fingerprint (`hasCanonicalEntrySchema`), so adding it
- * does not require an index-generation bump.
- */
-export function ensureEmbeddingSalvageTable(db) {
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS embedding_salvage (
-      content_hash TEXT PRIMARY KEY,
-      fingerprint  TEXT NOT NULL,
-      embedding    BLOB NOT NULL,
-      salvaged_at  TEXT NOT NULL
-    );
-  `);
-}
-/** The one hash function salvage writes and reuse lookups must agree on. */
-export function hashEmbeddableText(searchText) {
-    return createHash("sha256").update(searchText, "utf8").digest("hex");
-}
-function tableExists(db, name) {
-    return db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name) != null;
-}
-function tableHasColumn(db, table, column) {
-    const columns = db.prepare(`PRAGMA table_info(${table})`).all();
-    return columns.some((c) => c.name === column);
-}
-/**
- * Copy every (hash of search_text, embedding) pair about to be discarded
- * wholesale into `embedding_salvage`, tagged with the `embeddingFingerprint`
- * the discarded vectors were generated under. The caller MUST run this
- * inside the same transaction as the discard that follows it, so the copy
- * and the delete commit or roll back together.
- *
- * Streams `entries JOIN embeddings` in id-ordered pages of
- * {@link SQLITE_CHUNK_SIZE} instead of loading every row into memory before
- * hashing anything — a full rebuild of a large stash otherwise held the
- * entire corpus's search text and vectors in memory at once just to copy
- * them aside (#955, field-report follow-up).
- *
- * A no-op (returns 0) when there is no stored `embeddingFingerprint` to tag
- * rows with (nothing was ever verified against a provider, so there is
- * nothing worth reusing later) or the generation being discarded predates
- * the `entries.search_text` column or has no `embeddings` table at all — an
- * older generation than that has nothing this can safely read.
- */
-export function salvageEmbeddingsBeforeDiscard(db) {
-    const fingerprint = getMeta(db, "embeddingFingerprint");
-    if (!fingerprint)
-        return 0;
-    if (!tableExists(db, "entries") || !tableExists(db, "embeddings"))
-        return 0;
-    if (!tableHasColumn(db, "entries", "search_text"))
-        return 0;
-    const page = db.prepare("SELECT e.id AS id, e.search_text AS searchText, em.embedding AS embedding " +
-        "FROM entries e JOIN embeddings em ON em.id = e.id WHERE e.id > ? ORDER BY e.id LIMIT ?");
-    const insert = db.prepare("INSERT OR REPLACE INTO embedding_salvage (content_hash, fingerprint, embedding, salvaged_at) VALUES (?, ?, ?, ?)");
-    const salvagedAt = new Date().toISOString();
-    let lastId = 0;
-    let total = 0;
-    for (;;) {
-        const rows = page.all(lastId, SQLITE_CHUNK_SIZE);
-        if (rows.length === 0)
-            break;
-        for (const row of rows) {
-            insert.run(hashEmbeddableText(row.searchText), fingerprint, row.embedding, salvagedAt);
-        }
-        total += rows.length;
-        lastId = rows[rows.length - 1]?.id ?? lastId;
-        if (rows.length < SQLITE_CHUNK_SIZE)
-            break;
-    }
-    return total;
-}
-/**
- * Remove every salvage row. Called after an embedding pass completes without
- * abort or circuit-break (the salvaged generation has now either been reused
- * or superseded), and by `--reembed` / a canary "rebuild" verdict (the
- * salvaged vectors belong to a different model and are never reusable).
- */
-export function purgeEmbeddingSalvage(db) {
-    db.exec("DELETE FROM embedding_salvage");
-}
-/**
- * A canary "keep" verdict means the model did not actually change — only its
- * fingerprint STRING did (e.g. a gateway rename). Salvage rows tagged with
- * the old string are still valid vectors; rewrite them to the new string so
- * they remain reusable instead of silently going stale.
- */
-export function relabelEmbeddingSalvageFingerprint(db, fromFingerprint, toFingerprint) {
-    db.prepare("UPDATE embedding_salvage SET fingerprint = ? WHERE fingerprint = ?").run(toFingerprint, fromFingerprint);
-}
-/**
- * Reuse salvaged vectors for `entries` whose `searchText` hash matches a
- * salvage row tagged with the CURRENT `fingerprint` — never across
- * fingerprints, and never when `search_text` differs by even one byte (the
- * hash is exact-match only, by design). Matches are written via
- * `writeReused` in chunks of {@link SQLITE_CHUNK_SIZE}, each its own
- * transaction, mirroring the main pass's per-batch commit (#955) so an
- * interruption partway through the reuse step keeps whatever already wrote.
- *
- * The steady state of every ordinary run is an EMPTY salvage table (nothing
- * was just discarded), so this checks that first with one indexed lookup —
- * `SELECT 1 ... LIMIT 1` — before hashing a single pending entry. Hashing
- * every entry up front to look up a table that is empty 100% of the time
- * outside a rebuild was pure wasted work on the common path (#955,
- * field-report follow-up).
- */
-export function reuseSalvagedEmbeddings(db, entries, fingerprint, writeReused) {
-    if (entries.length === 0)
-        return { reusedCount: 0, remaining: [] };
-    const anySalvageForFingerprint = db
-        .prepare("SELECT 1 FROM embedding_salvage WHERE fingerprint = ? LIMIT 1")
-        .get(fingerprint);
-    if (!anySalvageForFingerprint)
-        return { reusedCount: 0, remaining: [...entries] };
-    const hashes = entries.map((entry) => hashEmbeddableText(entry.searchText));
-    const salvageByHash = new Map();
-    const uniqueHashes = [...new Set(hashes)];
-    for (let offset = 0; offset < uniqueHashes.length; offset += SQLITE_CHUNK_SIZE) {
-        const chunk = uniqueHashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
-        const placeholders = chunk.map(() => "?").join(",");
-        const rows = db
-            .prepare(`SELECT content_hash AS contentHash, embedding FROM embedding_salvage WHERE fingerprint = ? AND content_hash IN (${placeholders})`)
-            .all(fingerprint, ...chunk);
-        for (const row of rows)
-            salvageByHash.set(row.contentHash, row.embedding);
-    }
-    if (salvageByHash.size === 0)
-        return { reusedCount: 0, remaining: [...entries] };
-    let reusedCount = 0;
-    const remaining = [];
-    for (let offset = 0; offset < entries.length; offset += SQLITE_CHUNK_SIZE) {
-        const end = Math.min(offset + SQLITE_CHUNK_SIZE, entries.length);
-        const chunkMatches = [];
-        for (let i = offset; i < end; i++) {
-            const entry = entries[i];
-            const blob = salvageByHash.get(hashes[i]);
-            if (blob)
-                chunkMatches.push({ entry, blob });
-            else
-                remaining.push(entry);
-        }
-        if (chunkMatches.length === 0)
-            continue;
-        db.transaction(() => {
-            for (const { entry, blob } of chunkMatches) {
-                if (writeReused(entry, blobToEmbedding(blob)))
-                    reusedCount++;
-                else
-                    remaining.push(entry);
-            }
-        })();
-    }
-    return { reusedCount, remaining };
-}