akm-cli 0.9.0-beta.2 → 0.9.0-beta.26

package/dist/indexer/graph/graph-boost.js

@@ -241,13 +241,18 @@ export function collectGraphRelatedHit(context, filePath) {
  * Find graph files that share entities with the given file.
  *
  * Implementation: SQL self-join on graph_file_entities, scoped by stash_root,
- * grouped by entry_id, ordered by shared-entity count desc. Touches ~50-200
+ * grouped by file_path, ordered by shared-entity count desc. Touches ~50-200
  * rows instead of loading the entire snapshot into memory. Cold-call latency
  * drops from ~30-60ms (full snapshot parse) to ~2-5ms on typical stashes.
  *
+ * #624-P1: the graph tables are keyed on (stash_root, file_path, body_hash) —
+ * NOT entries.id — so candidates are identified by file_path (the unique index
+ * idx_graph_files_path guarantees one graph_files row per path).
+ *
  * The returned `ref` field carries the canonical asset ref (`type:name`)
- * resolved from entries.entry_key when the entry is indexed. Callers should
- * fall back to formatting `path` when `ref` is undefined (orphan graph row).
+ * resolved from entries.entry_key when the file is indexed. Callers should
+ * fall back to formatting `path` when `ref` is undefined (graph row with no
+ * matching entries row).
  */
 export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
     if (!db) {
@@ -255,113 +260,118 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
         // callers pass a handle), so degrade to empty rather than reopening.
         return [];
     }
-    // Resolve target's entry_id from the stash_root + file_path. The graph rows
-    // are keyed on entry_id; without it we can't run the join.
-    let targetEntryId;
+    // Confirm the target file has a graph row; without it there is nothing to
+    // relate. (Identity is file_path within the stash — one row per path.)
     try {
         const row = db
-            .prepare("SELECT entry_id FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
+            .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
             .get(stashRoot, filePath);
-        targetEntryId = row?.entry_id;
+        if (row === undefined)
+            return [];
     }
     catch {
         return [];
     }
-    if (targetEntryId == null)
-        return [];
     const effectiveLimit = Math.max(1, limit);
-    // Shared-entity count per candidate entry_id.
+    // Shared-entity count per candidate file_path. The target's entities are the
+    // rows for `filePath`; candidates are any OTHER file_path in the stash that
+    // shares a normalized entity.
     let candidateRows;
     try {
         candidateRows = db
-            .prepare(`SELECT gf.entry_id    AS entry_id,
-                gf.file_path   AS file_path,
+            .prepare(`SELECT gf.file_path   AS file_path,
                 gf.file_type   AS file_type,
                 COUNT(*)       AS shared
             FROM graph_file_entities target
             JOIN graph_file_entities e
               ON e.stash_root = target.stash_root
              AND e.entity_norm = target.entity_norm
-             AND e.entry_id  != target.entry_id
+             AND e.file_path  != target.file_path
             JOIN graph_files gf
-              ON gf.entry_id = e.entry_id
-          WHERE target.entry_id  = ?
+              ON gf.stash_root = e.stash_root
+             AND gf.file_path = e.file_path
+             AND gf.body_hash = e.body_hash
+          WHERE target.file_path  = ?
             AND target.stash_root = ?
-          GROUP BY gf.entry_id
+          GROUP BY gf.file_path
           ORDER BY shared DESC, gf.file_path ASC
           LIMIT ?`)
-            .all(targetEntryId, stashRoot, effectiveLimit);
+            .all(filePath, stashRoot, effectiveLimit);
     }
     catch {
         return [];
     }
     if (candidateRows.length === 0)
         return [];
-    const candidateIds = candidateRows.map((r) => r.entry_id);
-    const placeholders = candidateIds.map(() => "?").join(",");
+    const candidatePaths = candidateRows.map((r) => r.file_path);
+    const placeholders = candidatePaths.map(() => "?").join(",");
     // Pull the shared entity names (joined by normalized casing) for display.
     const sharedRows = db
-        .prepare(`SELECT e.entry_id AS entry_id, e.entity AS entity
+        .prepare(`SELECT e.file_path AS file_path, e.entity AS entity
          FROM graph_file_entities e
          JOIN graph_file_entities target
            ON target.stash_root = e.stash_root
            AND target.entity_norm = e.entity_norm
-         WHERE e.entry_id IN (${placeholders})
-           AND target.entry_id = ?
+         WHERE e.file_path IN (${placeholders})
+           AND e.stash_root = ?
+           AND target.file_path = ?
            AND target.stash_root = ?`)
-        .all(...candidateIds, targetEntryId, stashRoot);
-    const sharedByEntry = new Map();
+        .all(...candidatePaths, stashRoot, filePath, stashRoot);
+    const sharedByPath = new Map();
     for (const row of sharedRows) {
-        let bucket = sharedByEntry.get(row.entry_id);
+        let bucket = sharedByPath.get(row.file_path);
         if (!bucket) {
             bucket = new Set();
-            sharedByEntry.set(row.entry_id, bucket);
+            sharedByPath.set(row.file_path, bucket);
         }
         bucket.add(row.entity);
     }
     // Relation count for each candidate (relations where either endpoint
     // matches one of the shared entities).
-    const relationCountByEntry = new Map();
+    const relationCountByPath = new Map();
     const relationRows = db
-        .prepare(`SELECT entry_id, from_entity, to_entity
+        .prepare(`SELECT file_path, from_entity, to_entity
          FROM graph_file_relations
-        WHERE entry_id IN (${placeholders})`)
-        .all(...candidateIds);
+        WHERE file_path IN (${placeholders})
+          AND stash_root = ?`)
+        .all(...candidatePaths, stashRoot);
     for (const row of relationRows) {
-        const shared = sharedByEntry.get(row.entry_id);
+        const shared = sharedByPath.get(row.file_path);
         if (!shared)
             continue;
         if (shared.has(row.from_entity) || shared.has(row.to_entity)) {
-            relationCountByEntry.set(row.entry_id, (relationCountByEntry.get(row.entry_id) ?? 0) + 1);
+            relationCountByPath.set(row.file_path, (relationCountByPath.get(row.file_path) ?? 0) + 1);
         }
     }
     // Optional: ref lookup via entries.entry_key. entry_key is stored as
     // `${stash_dir}:${type}:${name}` — strip the stash-dir prefix to get the
-    // user-facing `type:name`.
-    const refByEntryId = new Map();
+    // user-facing `type:name`. Resolve by (stash_dir, file_path) now that the
+    // graph rows are no longer keyed on entries.id.
+    const refByPath = new Map();
     try {
         const entryRows = db
-            .prepare(`SELECT id, entry_key, stash_dir FROM entries WHERE id IN (${placeholders})`)
-            .all(...candidateIds);
+            .prepare(`SELECT entry_key, stash_dir, file_path FROM entries
+          WHERE file_path IN (${placeholders}) AND stash_dir = ?`)
+            .all(...candidatePaths, stashRoot);
         for (const row of entryRows) {
             const ref = stripStashPrefix(row.entry_key, row.stash_dir);
             if (ref)
-                refByEntryId.set(row.id, ref);
+                refByPath.set(row.file_path, ref);
         }
     }
     catch {
         /* ignore — refs are best-effort */
     }
     return candidateRows.map((row) => {
-        const sharedSet = sharedByEntry.get(row.entry_id) ?? new Set();
+        const sharedSet = sharedByPath.get(row.file_path) ?? new Set();
         const sharedEntities = [...sharedSet].sort((a, b) => a.localeCompare(b));
-        const ref = refByEntryId.get(row.entry_id);
+        const ref = refByPath.get(row.file_path);
         return {
             ...(ref ? { ref } : {}),
             path: row.file_path,
             type: row.file_type,
             sharedEntities,
-            relationCount: relationCountByEntry.get(row.entry_id) ?? 0,
+            relationCount: relationCountByPath.get(row.file_path) ?? 0,
         };
     });
 }

package/dist/indexer/index-writer-lock.js

@@ -0,0 +1,99 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import fs from "node:fs";
+import path from "node:path";
+import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
+import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
+const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
+const INDEX_WRITER_WAIT_MS = 100;
+const heldLocks = new Map();
+function buildPayload(purpose, pid = process.pid) {
+    return JSON.stringify({
+        pid,
+        purpose,
+        dbPath: getDbPath(),
+        startedAt: new Date().toISOString(),
+    });
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function throwIfAborted(signal) {
+    if (!signal?.aborted)
+        return;
+    throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
+}
+function releaseHeldLock(lockPath) {
+    const held = heldLocks.get(lockPath);
+    if (!held)
+        return;
+    held.depth -= 1;
+    if (held.depth > 0)
+        return;
+    heldLocks.delete(lockPath);
+    process.off("exit", held.exitHandler);
+    releaseLockIfOwned(lockPath, process.pid);
+}
+function retainHeldLock(lockPath) {
+    const existing = heldLocks.get(lockPath);
+    if (existing) {
+        existing.depth += 1;
+        return { lockPath, release: () => releaseHeldLock(lockPath) };
+    }
+    const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
+    process.on("exit", exitHandler);
+    heldLocks.set(lockPath, { depth: 1, exitHandler });
+    return { lockPath, release: () => releaseHeldLock(lockPath) };
+}
+function detachHeldLock(lockPath) {
+    const held = heldLocks.get(lockPath);
+    if (!held)
+        return;
+    heldLocks.delete(lockPath);
+    process.off("exit", held.exitHandler);
+}
+export async function acquireIndexWriterLease(options) {
+    const mode = options.mode ?? "wait";
+    const lockPath = getIndexWriterLockPath();
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    if (heldLocks.has(lockPath)) {
+        return retainHeldLock(lockPath);
+    }
+    while (true) {
+        throwIfAborted(options.signal);
+        if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
+            return retainHeldLock(lockPath);
+        }
+        const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
+        if (probe.state === "held" && probe.holderPid === process.pid) {
+            return retainHeldLock(lockPath);
+        }
+        if (probe.state === "stale") {
+            releaseLock(lockPath);
+            continue;
+        }
+        if (mode === "try")
+            return undefined;
+        await delay(INDEX_WRITER_WAIT_MS);
+    }
+}
+export async function withIndexWriterLease(options, run) {
+    const lease = await acquireIndexWriterLease(options);
+    if (!lease) {
+        throw new Error(`index writer lease unavailable for ${options.purpose}`);
+    }
+    try {
+        return await run();
+    }
+    finally {
+        lease.release();
+    }
+}
+export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
+    fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
+    detachHeldLock(lease.lockPath);
+}
+export function probeIndexWriterLease() {
+    return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
+}

package/dist/indexer/indexer.js

@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
 import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
 import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
 import { deleteStoredGraph } from "./db/graph-db.js";
+import { withIndexWriterLease } from "./index-writer-lock.js";
 import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
 import { buildSearchText } from "./search/search-fields.js";
 import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
 }
 // ── Indexer ──────────────────────────────────────────────────────────────────
 export async function akmIndex(options) {
-    const stashDir = options?.stashDir || resolveStashDir();
-    const onProgress = options?.onProgress ?? (() => { });
-    const signal = options?.signal;
-    const reEnrich = options?.reEnrich === true;
-    const full = options?.full === true;
-    const clean = options?.clean === true;
-    const dryRun = options?.dryRun === true;
-    // Load config and resolve all stash sources
-    const { loadConfig } = await import("../core/config/config.js");
-    const config = loadConfig();
-    // One-time, read-only guard: warn if the writable stash still holds an
-    // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
-    // entirely, so an unmigrated vault's `.env` data would silently never be
-    // indexed. Non-destructive — only stats, never reads/writes/deletes.
-    const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
-    warnOnUnmigratedVaults(stashDir);
-    // Ensure git stash caches are extracted before resolving stash dirs,
-    // so their content directories exist on disk for the walker to discover.
-    const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
-    await ensureSourceCaches(config, { force: full });
-    const allSourceEntries = resolveSourceEntries(stashDir, config);
-    const allSourceDirs = allSourceEntries.map((s) => s.path);
-    const t0 = Date.now();
-    // Open database — pass embedding dimension from config if available
-    const dbPath = getDbPath();
-    const embeddingDim = config.embedding?.dimension;
-    const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
-    try {
-        // Determine incremental vs full mode
-        const prevStashDir = getMeta(db, "stashDir");
-        const prevBuiltAt = getMeta(db, "builtAt");
-        const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
-        const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
-        // Assemble the run context
-        const ctx = {
-            db,
-            config,
-            sources: allSourceEntries,
-            sourceDirs: allSourceDirs,
-            full,
-            reEnrich,
-            stashDir,
-            onProgress,
-            signal,
-            timing: {
-                t0,
-                tWalkStart: t0,
-                tWalkEnd: t0,
-                tLlmEnd: t0,
-                tFtsEnd: t0,
-                tEmbedEnd: t0,
-            },
-            isIncremental,
-            builtAtMs,
-            hadRemovedSources: false,
-            scannedDirs: 0,
-            skippedDirs: 0,
-            generatedCount: 0,
-            walkWarnings: [],
-            dirsNeedingLlm: [],
-            embeddingResult: null,
-            graphExtractionResult: null,
-        };
-        onProgress({
-            phase: "summary",
-            message: buildIndexSummaryMessage({
+    return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
+        const stashDir = options?.stashDir || resolveStashDir();
+        const onProgress = options?.onProgress ?? (() => { });
+        const signal = options?.signal;
+        const reEnrich = options?.reEnrich === true;
+        const full = options?.full === true;
+        const clean = options?.clean === true;
+        const dryRun = options?.dryRun === true;
+        // Load config and resolve all stash sources
+        const { loadConfig } = await import("../core/config/config.js");
+        const config = loadConfig();
+        // One-time, read-only guard: warn if the writable stash still holds an
+        // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
+        // entirely, so an unmigrated vault's `.env` data would silently never be
+        // indexed. Non-destructive — only stats, never reads/writes/deletes.
+        const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
+        warnOnUnmigratedVaults(stashDir);
+        // Ensure git stash caches are extracted before resolving stash dirs,
+        // so their content directories exist on disk for the walker to discover.
+        const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
+        await ensureSourceCaches(config, { force: full });
+        const allSourceEntries = resolveSourceEntries(stashDir, config);
+        const allSourceDirs = allSourceEntries.map((s) => s.path);
+        const t0 = Date.now();
+        // Open database — pass embedding dimension from config if available
+        const dbPath = getDbPath();
+        const embeddingDim = config.embedding?.dimension;
+        const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
+        try {
+            // Determine incremental vs full mode
+            const prevStashDir = getMeta(db, "stashDir");
+            const prevBuiltAt = getMeta(db, "builtAt");
+            const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
+            const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
+            // Assemble the run context
+            const ctx = {
+                db,
+                config,
+                sources: allSourceEntries,
+                sourceDirs: allSourceDirs,
+                full,
+                reEnrich,
+                stashDir,
+                onProgress,
+                signal,
+                timing: {
+                    t0,
+                    tWalkStart: t0,
+                    tWalkEnd: t0,
+                    tLlmEnd: t0,
+                    tFtsEnd: t0,
+                    tEmbedEnd: t0,
+                },
+                isIncremental,
+                builtAtMs,
+                hadRemovedSources: false,
+                scannedDirs: 0,
+                skippedDirs: 0,
+                generatedCount: 0,
+                walkWarnings: [],
+                dirsNeedingLlm: [],
+                embeddingResult: null,
+                graphExtractionResult: null,
+            };
+            onProgress({
+                phase: "summary",
+                message: buildIndexSummaryMessage({
+                    mode: isIncremental ? "incremental" : "full",
+                    sourcesCount: allSourceDirs.length,
+                    semanticSearchMode: config.semanticSearchMode,
+                    embeddingProvider: getEmbeddingProvider(config.embedding),
+                    llmEnabled: !!resolveIndexPassLLM("enrichment", config),
+                    vecAvailable: isVecAvailable(db),
+                }),
+            });
+            // ── Phase sequence ───────────────────────────────────────────────────────
+            await runSourceCachePhase(ctx);
+            await runWalkPhase(ctx);
+            await runEmbeddingPhase(ctx);
+            await runFinalizePhase(ctx);
+            // ────────────────────────────────────────────────────────────────────────
+            const { _verification: verification, _totalEntries: totalEntries } = ctx;
+            const { timing } = ctx;
+            // ── Clean pass ───────────────────────────────────────────────────────────
+            // After the normal index completes, remove entries whose source files no
+            // longer exist on disk. Remote entries (empty file_path) are skipped.
+            let cleanResult;
+            if (clean) {
+                cleanResult = runCleanPass(db, dryRun);
+            }
+            // ────────────────────────────────────────────────────────────────────────
+            return {
+                stashDir,
+                totalEntries,
+                generatedMetadata: ctx.generatedCount,
+                indexPath: dbPath,
                 mode: isIncremental ? "incremental" : "full",
-                sourcesCount: allSourceDirs.length,
-                semanticSearchMode: config.semanticSearchMode,
-                embeddingProvider: getEmbeddingProvider(config.embedding),
-                llmEnabled: !!resolveIndexPassLLM("enrichment", config),
-                vecAvailable: isVecAvailable(db),
-            }),
-        });
-        // ── Phase sequence ───────────────────────────────────────────────────────
-        await runSourceCachePhase(ctx);
-        await runWalkPhase(ctx);
-        await runEmbeddingPhase(ctx);
-        await runFinalizePhase(ctx);
-        // ────────────────────────────────────────────────────────────────────────
-        const { _verification: verification, _totalEntries: totalEntries } = ctx;
-        const { timing } = ctx;
-        // ── Clean pass ───────────────────────────────────────────────────────────
-        // After the normal index completes, remove entries whose source files no
-        // longer exist on disk. Remote entries (empty file_path) are skipped.
-        let cleanResult;
-        if (clean) {
-            cleanResult = runCleanPass(db, dryRun);
+                directoriesScanned: ctx.scannedDirs,
+                directoriesSkipped: ctx.skippedDirs,
+                ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
+                verification,
+                timing: {
+                    totalMs: Date.now() - timing.t0,
+                    walkMs: timing.tWalkEnd - timing.tWalkStart,
+                    llmMs: timing.tLlmEnd - timing.tWalkEnd,
+                    embedMs: timing.tEmbedEnd - timing.tLlmEnd,
+                    ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
+                },
+                ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
+            };
         }
-        // ────────────────────────────────────────────────────────────────────────
-        return {
-            stashDir,
-            totalEntries,
-            generatedMetadata: ctx.generatedCount,
-            indexPath: dbPath,
-            mode: isIncremental ? "incremental" : "full",
-            directoriesScanned: ctx.scannedDirs,
-            directoriesSkipped: ctx.skippedDirs,
-            ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
-            verification,
-            timing: {
-                totalMs: Date.now() - timing.t0,
-                walkMs: timing.tWalkEnd - timing.tWalkStart,
-                llmMs: timing.tLlmEnd - timing.tWalkEnd,
-                embedMs: timing.tEmbedEnd - timing.tLlmEnd,
-                ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
-            },
-            ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
-        };
-    }
-    finally {
-        closeDatabase(db);
-    }
+        finally {
+            closeDatabase(db);
+        }
+    });
 }
 // ── Extracted helpers for indexing ────────────────────────────────────────────
 async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {