akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/indexer/index-writer-lock.js

@@ -1,17 +1,18 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { AsyncLocalStorage } from "node:async_hooks";
 import fs from "node:fs";
 import path from "node:path";
-import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
+import { createLockPayload, probeLock, reclaimStaleLock, releaseLock, tryAcquireLockSync, } from "../core/file-lock.js";
+import { tryAcquireMaintenanceBarrier } from "../core/maintenance-barrier.js";
 import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
 const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
 const INDEX_WRITER_WAIT_MS = 100;
 const DEFAULT_INDEX_WRITER_MAX_WAIT_MS = 10 * 60 * 1000;
-const heldLocks = new Map();
-function buildPayload(purpose, pid = process.pid) {
-    return JSON.stringify({
-        pid,
+const leaseContext = new AsyncLocalStorage();
+function buildPayload(purpose) {
+    return createLockPayload({
         purpose,
         dbPath: getDbPath(),
         startedAt: new Date().toISOString(),
@@ -25,27 +26,38 @@ function throwIfAborted(signal) {
         return;
     throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
 }
-function releaseHeldLock(lockPath) {
-    const held = heldLocks.get(lockPath);
-    if (!held)
-        return;
-    held.depth -= 1;
-    if (held.depth > 0)
-        return;
-    heldLocks.delete(lockPath);
-    process.off("exit", held.exitHandler);
-    releaseLockIfOwned(lockPath, process.pid);
+function createLease(lockPath, ownership) {
+    const exitHandler = () => releaseLock(ownership);
+    process.on("exit", exitHandler);
+    let released = false;
+    return {
+        lockPath,
+        release: () => {
+            if (released)
+                return;
+            released = true;
+            process.off("exit", exitHandler);
+            releaseLock(ownership);
+        },
+    };
 }
-function retainHeldLock(lockPath) {
-    const existing = heldLocks.get(lockPath);
-    if (existing) {
-        existing.depth += 1;
-        return { lockPath, release: () => releaseHeldLock(lockPath) };
+function tryAcquireIndexWriterLease(lockPath, purpose) {
+    while (true) {
+        const releaseBarrier = tryAcquireMaintenanceBarrier();
+        if (!releaseBarrier)
+            return undefined;
+        try {
+            const ownership = tryAcquireLockSync(lockPath, buildPayload(purpose));
+            if (ownership)
+                return createLease(lockPath, ownership);
+            const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
+            if (probe.state !== "stale" || !reclaimStaleLock(lockPath, probe))
+                return undefined;
+        }
+        finally {
+            releaseBarrier();
+        }
     }
-    const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
-    process.on("exit", exitHandler);
-    heldLocks.set(lockPath, { depth: 1, exitHandler });
-    return { lockPath, release: () => releaseHeldLock(lockPath) };
 }
 export async function acquireIndexWriterLease(options) {
     const mode = options.mode ?? "wait";
@@ -53,25 +65,13 @@ export async function acquireIndexWriterLease(options) {
     const startedAt = Date.now();
     const maxWaitMs = options.maxWaitMs ?? DEFAULT_INDEX_WRITER_MAX_WAIT_MS;
     fs.mkdirSync(path.dirname(lockPath), { recursive: true });
-    if (heldLocks.has(lockPath)) {
-        options.onAcquired?.({ waitedMs: 0 });
-        return retainHeldLock(lockPath);
-    }
     let lastWaitNoticeMs = 0;
     while (true) {
         throwIfAborted(options.signal);
-        if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
+        const lease = tryAcquireIndexWriterLease(lockPath, options.purpose);
+        if (lease) {
             options.onAcquired?.({ waitedMs: Date.now() - startedAt });
-            return retainHeldLock(lockPath);
-        }
-        const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
-        if (probe.state === "held" && probe.holderPid === process.pid) {
-            options.onAcquired?.({ waitedMs: Date.now() - startedAt });
-            return retainHeldLock(lockPath);
-        }
-        if (probe.state === "stale") {
-            releaseLock(lockPath);
-            continue;
+            return lease;
         }
         if (mode === "try")
             return undefined;
@@ -90,16 +90,60 @@ export async function acquireIndexWriterLease(options) {
     }
 }
 export async function withIndexWriterLease(options, run) {
-    const lease = await acquireIndexWriterLease(options);
-    if (!lease) {
-        throw new Error(`index writer lease unavailable for ${options.purpose}`);
-    }
-    try {
-        return await run();
-    }
-    finally {
-        lease.release();
-    }
+    const lockPath = getIndexWriterLockPath();
+    const inherited = leaseContext.getStore();
+    if (inherited?.has(lockPath))
+        return run();
+    const context = inherited ?? new Set();
+    const execute = async () => {
+        const lease = await acquireIndexWriterLease(options);
+        if (!lease)
+            throw new Error(`index writer lease unavailable for ${options.purpose}`);
+        context.add(lockPath);
+        try {
+            return await run();
+        }
+        finally {
+            context.delete(lockPath);
+            lease.release();
+        }
+    };
+    return inherited ? execute() : leaseContext.run(context, execute);
+}
+/** Asset writes and index rebuilds share one lease so scans cannot publish stale snapshots. */
+export function withAssetMutationLease(purpose, run) {
+    return withIndexWriterLease({ purpose }, run);
+}
+/** Synchronous asset-write boundary over the same interprocess lease. */
+export function withAssetMutationLeaseSync(purpose, run) {
+    const lockPath = getIndexWriterLockPath();
+    const inherited = leaseContext.getStore();
+    if (inherited?.has(lockPath))
+        return run();
+    const context = inherited ?? new Set();
+    const execute = () => {
+        const startedAt = Date.now();
+        fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+        let lease;
+        while (!lease) {
+            lease = tryAcquireIndexWriterLease(lockPath, purpose);
+            if (!lease) {
+                if (Date.now() - startedAt >= DEFAULT_INDEX_WRITER_MAX_WAIT_MS) {
+                    throw new Error(`timed out waiting for index writer lease for ${purpose}`);
+                }
+                Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, INDEX_WRITER_WAIT_MS);
+            }
+        }
+        context.add(lockPath);
+        try {
+            return run();
+        }
+        finally {
+            context.delete(lockPath);
+            lease.release();
+        }
+    };
+    return inherited ? execute() : leaseContext.run(context, execute);
 }
 export function probeIndexWriterLease() {
     return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });

package/dist/indexer/index-written-assets.js

@@ -9,23 +9,30 @@
  * writing). The memory write paths — `akm remember` / `writeMarkdownAsset`
  * and extract's session assets — historically did not, which is why reads
  * used to compensate with stale-triggered background reindexes (the
- * lock-contention footgun removed alongside this module's introduction; see
- * docs/design/read-path-reindex-contention-findings.md §7).
+ * lock-contention footgun removed alongside this module's introduction, per
+ * the 2026-07 read-path reindex-contention findings §7).
  *
  * This is NOT a general reindex. It upserts exactly the files the caller just
  * wrote: frontmatter/metadata via the shared matcher pipeline, the `entries`
  * row, and an incremental FTS refresh. Embeddings, index-time LLM passes,
  * graph extraction, `builtAt`, and the per-dir walk cache are all deliberately
  * untouched — the next full run heals them (the opportunistic-recovery
- * strategy of docs/technical/index-consistency-adr.md).
+ * strategy of the index-consistency ADR).
  */
 import fs from "node:fs";
 import path from "node:path";
+import { akmAdapter } from "../core/adapter/adapters/akm-adapter.js";
 import { getDbPath } from "../core/paths.js";
 import { warnVerbose } from "../core/warn.js";
-import { closeDatabase, getEntryCount, openExistingDatabase, rebuildFts, upsertEntry } from "./db/db.js";
-import { generateMetadataFlat } from "./passes/metadata.js";
+import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
+import { deleteEntriesByIds, getEntryCount, upsertEntry, upsertWorkflowDocument, } from "../storage/repositories/index-entries-repository.js";
+import { rebuildFts } from "../storage/repositories/index-fts-repository.js";
+import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
+import { withIndexWriterLease } from "./index-writer-lock.js";
+import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
+import { drainDirDocuments } from "./scan/drain-dir.js";
 import { buildSearchText } from "./search/search-fields.js";
+import { buildFileContext } from "./walk/file-context.js";
 /**
  * Busy-timeout (ms) for write-path index upserts. A real write — unlike the
  * 250ms telemetry inserts — but it must not hang `akm remember` for the full
@@ -46,60 +53,100 @@ export const WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5_000;
  * first read (`ensureIndex`) or an explicit `akm index`, which also cover
  * embeddings and the other passes this fast path skips.
  */
-export async function indexWrittenAssets(stashDir, filePaths) {
+export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
     try {
-        const dbPath = getDbPath();
-        if (!fs.existsSync(dbPath))
-            return;
-        // The full walk never descends into dot-directories (they hold state like
-        // `.meta/`, `.stash.json`), and `shouldIndexStashFile` relies on the walker
-        // for that — mirror it here so this fast path indexes exactly what a full
-        // run would.
-        const files = filePaths.filter((f) => {
-            if (!fs.existsSync(f))
-                return false;
-            const rel = path.relative(stashDir, f);
-            return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
-        });
-        if (files.length === 0)
-            return;
-        // Generate metadata BEFORE opening the DB so the write window stays
-        // short. One call per file keeps the entry↔path pairing exact.
-        const pairs = [];
-        for (const file of files) {
-            const generated = await generateMetadataFlat(stashDir, [file]);
-            const entry = generated.entries[0];
-            // Workflows carry a side-table document upsert this fast path doesn't
-            // do; no current caller writes them, but guard so one never lands
-            // half-indexed.
-            if (entry && entry.type !== "workflow")
-                pairs.push({ file, entry });
-        }
-        if (pairs.length === 0)
-            return;
-        const db = openExistingDatabase(dbPath);
-        try {
-            db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
-            if (getEntryCount(db) === 0)
-                return;
-            for (const { file, entry } of pairs) {
-                const entryKey = `${stashDir}:${entry.type}:${entry.name}`;
-                let entryWithSize = entry;
-                try {
-                    entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
+        return await withIndexWriterLease({ purpose: "index-written-assets" }, async () => {
+            const dbPath = getDbPath();
+            if (!fs.existsSync(dbPath))
+                return true;
+            // The full walk never descends into dot-directories (for example `.meta/`)
+            // — mirror that dot-segment skip here so this fast path indexes exactly
+            // what a full run would. Sensitive/
+            // infra abstention is the adapter's job now (see the `akmAdapter` note
+            // below), not a path pre-filter.
+            const files = filePaths.filter((f) => {
+                const rel = path.relative(stashDir, f);
+                return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
+            });
+            if (files.length === 0)
+                return true;
+            // Generate metadata BEFORE opening the DB so the write window stays
+            // short. One drain call per file keeps the entry↔path pairing exact and
+            // reuses the full-index recognize engine (F4a M-core-2 item 5): broken
+            // workflows drop, valid workflow docs are cached for the side-table upsert.
+            const component = deriveInstallations([
+                { path: stashDir, writable: true, ...(options.bundleId ? { registryId: options.bundleId } : {}) },
+            ])[0]?.components[0];
+            if (!component)
+                throw new Error(`Could not derive bundle provenance for ${stashDir}`);
+            const pairs = [];
+            const unindexable = new Set();
+            for (const file of files) {
+                if (!fs.existsSync(file)) {
+                    unindexable.add(file);
+                    continue;
+                }
+                const ctx = buildFileContext(stashDir, file);
+                // Hardcoded `akmAdapter` on purpose (owner ruling 2026-07-21): this
+                // write-path fast path only ever runs for assets a first-class akm
+                // mutation command just wrote into a managed akm source, so the akm
+                // adapter is always the right recognizer here — no per-component
+                // dispatch needed.
+                const drained = drainDirDocuments(akmAdapter, component, [ctx]);
+                const entry = drained.entries[0];
+                // Workflows also carry a workflow_documents side-table upsert — handled
+                // below, mirroring the full walk — since `akm mv` rewrites citer files
+                // that can be workflows. A broken workflow drains to zero entries (like
+                // the old skip-with-warning) and is treated as unindexable.
+                const conceptId = drained.conceptIdByFile.get(ctx.absPath);
+                if (entry && conceptId)
+                    pairs.push({ file, entry, conceptId, contentHash: drained.hashByFile.get(ctx.absPath) });
+                else
+                    unindexable.add(file);
+            }
+            const db = openExistingDatabase(dbPath);
+            try {
+                db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
+                if (getEntryCount(db) === 0)
+                    return true;
+                for (const file of unindexable) {
+                    const rows = db.prepare("SELECT id FROM entries WHERE file_path = ?").all(file);
+                    deleteEntriesByIds(db, rows.map((row) => row.id));
                 }
-                catch {
-                    // stat raced a delete — index without the size, like the full walk does.
+                for (const { file, entry, conceptId, contentHash } of pairs) {
+                    const entryKey = `${stashDir}:${entry.type}:${entry.name}`;
+                    let entryWithSize = entry;
+                    try {
+                        entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
+                    }
+                    catch {
+                        // stat raced a delete — index without the size, like the full walk does.
+                    }
+                    // Real provenance (F4a M-core-2 item 5): populate item_ref/content_hash
+                    // via the SAME derivation the full-index writer uses, so a write-path
+                    // row is never a NULL-item_ref straggler.
+                    const provenance = deriveEntryProvenance({ bundleId: component.id, componentId: component.id, adapterId: component.adapter }, entry.type, entry.name, conceptId);
+                    const entryId = upsertEntry(db, entryKey, path.dirname(file), file, stashDir, entryWithSize, buildSearchText(entry), provenance, contentHash);
+                    if (entry.type === "workflow") {
+                        // Same contract as the full walk (indexer.ts): the renderer cached
+                        // the parsed document during metadata generation; persist it so the
+                        // workflow runtime never sees an entry without its document.
+                        const doc = takeWorkflowDocument(entry);
+                        if (doc)
+                            upsertWorkflowDocument(db, entryId, doc, fs.readFileSync(file));
+                    }
                 }
-                upsertEntry(db, entryKey, path.dirname(file), file, stashDir, entryWithSize, buildSearchText(entry));
+                if (pairs.length > 0 || unindexable.size > 0)
+                    rebuildFts(db, { incremental: true });
             }
-            rebuildFts(db, { incremental: true });
-        }
-        finally {
-            closeDatabase(db);
-        }
+            finally {
+                closeDatabase(db);
+            }
+            return true;
+        });
     }
     catch (error) {
         warnVerbose("Write-path index update skipped (asset appears after the next full index):", error instanceof Error ? error.message : String(error));
+        return false;
     }
 }