akm-cli 0.9.16-alpha.1 → 0.9.16-alpha.2

package/dist/indexer/index-written-assets.js

@@ -2,130 +2,90 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Write-path indexing: index exactly what a write just produced, inline, in
- * the same call as the write. `remember` / `writeMarkdownAsset`, extract's
- * session assets, `source clone`, and proposal acceptance all call this
- * right after committing their file write so the asset is searchable
- * immediately, without a background reindex.
+ * Write-path indexing: targeted single-file index updates for asset writers.
  *
- * docs/plans/index-redesign-contract.md, module B2: a thin call to B1's
- * `reconcilePaths` (the per-file version of the stat-walk reconcile — parses,
- * upserts `entries`, derives units) followed by B4's `drainEmbeddingQueue`
- * scoped to exactly the units that reconcile just produced. No lock probe, no
- * rebuild detection, no background spawn — the redesign has no full-rebuild
- * pipeline for a write path to defer to; two writers reconciling the same
- * path converge on the same idempotent rows instead of contending for a lock.
+ * The index is maintained eagerly by every first-class mutation command
+ * (`source add`, `wiki`, `workflow`, `setup` all run `akmIndex()` after
+ * writing). The memory write paths — `akm remember` / `writeMarkdownAsset`
+ * and extract's session assets — historically did not, which is why reads
+ * used to compensate with stale-triggered background reindexes (the
+ * lock-contention footgun removed alongside this module's introduction, per
+ * the 2026-07 read-path reindex-contention findings §7).
  *
- * FAIL-OPEN at every step: any error (index.db absent, empty, unreadable,
- * unparseable file) is reduced to a verbose-only warning and the write
- * command succeeds untouched — the degraded outcome is the asset appearing
- * after the next reconcile (an explicit `akm index`, or the schedule) rather
- * than immediately.
+ * This is NOT a general reindex. It upserts exactly the files the caller just
+ * wrote: frontmatter/metadata via the shared matcher pipeline, the canonical
+ * row with its transactionally owned FTS projection, and vectors for changed
+ * entry IDs when semantic search is enabled. Index-time LLM passes, graph
+ * extraction, `builtAt`, and the per-dir walk cache remain full-index
+ * responsibilities.
  */
+import fs from "node:fs";
 import path from "node:path";
+import { akmAdapter } from "../core/adapter/adapters/akm-adapter.js";
 import { loadConfig } from "../core/config/config.js";
 import { isDataDirUnreadableError } from "../core/errors.js";
+import { probeLock } from "../core/file-lock.js";
 import { isPathAbsent } from "../core/path-access.js";
-import { getDbPath } from "../core/paths.js";
+import { getDbPath, getIndexRebuildLockPath } from "../core/paths.js";
+import { formatLockHolderPid } from "../core/run-lock.js";
 import { warn, warnVerbose } from "../core/warn.js";
 import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
-import { getEntryCount } from "../storage/repositories/index-entries-repository.js";
-import { SQLITE_CHUNK_SIZE } from "../storage/repositories/index-sql.js";
-import { drainEmbeddingQueue } from "./drain.js";
-import { deriveInstallations } from "./installations.js";
-import { reconcilePaths } from "./reconcile.js";
-import { resolveSourceEntries } from "./search/search-source.js";
+import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
+import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
+import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
+import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
+import { drainDirDocuments } from "./scan/drain-dir.js";
+import { buildSearchText } from "./search/search-fields.js";
+import { buildFileContext } from "./walk/file-context.js";
 /**
- * Busy-timeout (ms) for write-path index upserts. Every index write in the
- * redesign is a short immediate transaction relying on SQLite's own busy
- * timeout to serialize concurrent writers (rule 5, docs/plans/index-redesign.md)
- * rather than an application-level lock — this bounds how long one interactive
- * write waits behind another process's short reconcile/drain transaction.
+ * Busy-timeout (ms) for write-path index upserts. A real write — unlike the
+ * 250ms telemetry inserts — but it must not hang `akm remember` for the full
+ * default 30s behind a running full reindex. When it times out, the upsert is
+ * skipped and the asset becomes searchable after that reindex instead.
  */
 export const WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5_000;
-/**
- * The distinct unit hashes reachable from an entry at one of `files`, via the
- * `entry_units` mapping `reconcilePaths` just wrote — i.e. exactly the units
- * this write's reconcile pass added or touched, the scope B4's drain should
- * be asked to embed (module B2, docs/plans/index-redesign-contract.md:
- * "Compute 'the units this write added' from the mapping:
- * entries.file_path IN (paths) → entry_units → unit_hash"). A gone/unindexable
- * path has no `entries` row any more and so contributes nothing here, which
- * is correct — there is nothing left to embed for it.
- */
-function unitHashesForFiles(db, files) {
-    const hashes = new Set();
-    for (let offset = 0; offset < files.length; offset += SQLITE_CHUNK_SIZE) {
-        const chunk = files.slice(offset, offset + SQLITE_CHUNK_SIZE);
-        const placeholders = chunk.map(() => "?").join(",");
-        const rows = db
-            .prepare(`SELECT DISTINCT eu.unit_hash AS unitHash
-           FROM entries e
-           JOIN entry_units eu ON eu.entry_id = e.id
-          WHERE e.file_path IN (${placeholders})`)
-            .all(...chunk);
-        for (const row of rows)
-            hashes.add(row.unitHash);
-    }
-    return [...hashes];
-}
-/**
- * The bundle id a fresh write into `stashDir` should be recorded under, when
- * the caller did not already know it (`options.bundleId`).
- *
- * `stashDir` is matched against the CONFIGURED sources first, the way every
- * other write-path caller resolves its bundle id (`knowledge.ts`'s demoted-file
- * loop: `resolveSourceEntries` + `deriveInstallations`, matched by resolved
- * root). A stash that is ALSO a configured bundle (`akm bundle add <dir>
- * --name x`) must produce that bundle's id here, not a fresh slug of the raw
- * path — a mismatch leaves an undeletable duplicate row once a full `akm
- * index` reconciles the same path under its real, configured id (issue found
- * integrating write-path W1). Only a stash that is genuinely unconfigured (an
- * ad hoc write target) falls back to deriving an id from the raw path alone,
- * the way a fresh install would.
- */
-function resolveWrittenAssetBundleId(stashDir) {
-    const resolvedStashDir = path.resolve(stashDir);
-    const sourceEntries = resolveSourceEntries();
-    const index = sourceEntries.findIndex((entry) => path.resolve(entry.path) === resolvedStashDir);
-    if (index !== -1)
-        return deriveInstallations(sourceEntries)[index]?.id;
-    return deriveInstallations([{ path: stashDir, writable: true }])[0]?.id;
-}
 /**
  * Index the given just-written asset files into the existing local index.
  *
- * Returns `true` when the index is left in the state the caller expects —
- * either genuinely updated, or one of the deliberate skips below — and
- * `false` only for a real failure, which callers that gate their own success
- * on this boolean (`acceptProposal`, `source clone`) must treat as "the
- * write stands, but tell the operator the index needs a manual `akm index`".
+ * FAIL-OPEN at every step: any error (index.db absent, empty, locked past the
+ * busy timeout, unparseable file) is reduced to a verbose-only warning and the
+ * write command succeeds untouched. The degraded outcome is exactly the
+ * pre-write-path-indexing behavior: the asset appears after the next full
+ * `akm index` / improve-cron run.
  *
- * Deliberate `true` skips:
- *  - An absent index: bootstrap belongs to the first read (`ensureIndex`) or
- *    an explicit `akm index`, not this fast path.
- *  - An index with zero entries: same reasoning — nothing has bootstrapped
- *    it yet.
- *  - Every given path living under a dot-segment directory (e.g. `.meta/`):
- *    the full reconcile never descends into those either.
+ * An absent or empty index is skipped on purpose — bootstrap belongs to the
+ * first read (`ensureIndex`) or an explicit `akm index`, which also cover
+ * embeddings and the other passes this fast path skips.
  *
- * `false` on failure:
- *  - An index directory/file that exists but cannot be READ (not merely
- *    absent) is the one failure the next reconcile will not heal on its own
- *    (#791) — this warns audibly, not verbose-only.
- *  - Any other error during `reconcilePaths` (unparseable file, a locked
- *    database past the busy timeout, …) warns verbose-only; the write itself
- *    already succeeded and the asset appears after the next reconcile.
- *
- * The embedding drain that follows a successful reconcile is best-effort: a
- * failure there (a provider outage, a tripped circuit breaker) does NOT turn
- * this call into a `false` — the asset is already lexically searchable, and
- * the embedding queue is durable (module B4): any later drain, including the
- * next write, computes the same "no vector yet" query and picks it up.
+ * A live rebuild holding the rebuild lock is the SAME kind of skip, not a
+ * failure (#956 fix): a live `akm index` run owns bringing the index to the
+ * expected state on its own, so this call returns `true` on that skip
+ * exactly like the absent-index and empty-index cases above. Callers that
+ * gate their own success on this boolean (`acceptProposal`, `source clone`)
+ * must never fail or warn just because a concurrent rebuild is in progress.
  */
 export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
     try {
         return await (async () => {
+            // #956: a live `akm index` rebuild holds index.db under one long
+            // transaction (persistDirRecords), which this fast path's own 5s
+            // busy_timeout would just contend with pointlessly. Interactive
+            // commands (remember, import, extract session assets) must never wait
+            // on it — skip the inline upsert/embedding entirely; the write itself
+            // (file + commit) has already succeeded by the time this runs, and the
+            // rebuild in progress will pick up the change on its own.
+            const rebuildProbe = probeLock(getIndexRebuildLockPath());
+            if (rebuildProbe.state === "held") {
+                // #956: name the launcher pid alongside the holder pid when known —
+                // every process listing and task log shows the launcher's pid, not
+                // the bun/node child's.
+                const holderLabel = formatLockHolderPid({
+                    pid: rebuildProbe.holderPid,
+                    launcherPid: rebuildProbe.launcherPid ?? null,
+                });
+                warn(`index rebuild in progress (pid ${holderLabel}); the next index pass will index ${filePaths.join(", ")}`);
+                return true;
+            }
             const dbPath = getDbPath();
             // `true` here means "the index is in the state the caller expects" — and
             // `acceptProposal` advances its journal to `index-finalized` on the
@@ -134,54 +94,132 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
             // `openExistingDatabase` and surfaces as the honest `false` (#791).
             if (isPathAbsent(dbPath))
                 return true;
-            // The full reconcile never descends into dot-directories (for example
-            // `.meta/`) — mirror that dot-segment skip here so this fast path
-            // indexes exactly what a full reconcile would.
+            // The full walk never descends into dot-directories (for example `.meta/`)
+            // — mirror that dot-segment skip here so this fast path indexes exactly
+            // what a full run would. Sensitive/
+            // infra abstention is the adapter's job now (see the `akmAdapter` note
+            // below), not a path pre-filter.
             const files = filePaths.filter((f) => {
                 const rel = path.relative(stashDir, f);
                 return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
             });
             if (files.length === 0)
                 return true;
-            // An explicit bundleId wins outright (it IS the resulting installation
-            // id); otherwise resolve it config-aware, matching a configured source
-            // before ever falling back to deriving one from the raw path.
-            const bundleId = options.bundleId ?? resolveWrittenAssetBundleId(stashDir);
-            if (!bundleId)
+            // Generate metadata BEFORE opening the DB so the write window stays
+            // short. One drain call per file keeps the entry↔path pairing exact and
+            // reuses the full-index recognize engine (F4a M-core-2 item 5): broken
+            // workflows drop, valid workflow docs are cached for the side-table upsert.
+            const component = deriveInstallations([
+                { path: stashDir, writable: true, ...(options.bundleId ? { registryId: options.bundleId } : {}) },
+            ])[0]?.components[0];
+            if (!component)
                 throw new Error(`Could not derive bundle provenance for ${stashDir}`);
+            const pairs = [];
+            const unindexable = new Set();
+            const rejectedConceptIds = new Set();
+            for (const file of files) {
+                if (!fs.existsSync(file)) {
+                    let authoredDanglingSymlink = false;
+                    try {
+                        authoredDanglingSymlink = fs.lstatSync(file).isSymbolicLink();
+                    }
+                    catch {
+                        // A genuinely absent path has no source identity to preflight.
+                    }
+                    if (!authoredDanglingSymlink) {
+                        unindexable.add(file);
+                        continue;
+                    }
+                }
+                const ctx = buildFileContext(stashDir, file);
+                // Hardcoded `akmAdapter` on purpose (owner ruling 2026-07-21): this
+                // write-path fast path only ever runs for assets a first-class akm
+                // mutation command just wrote into a managed akm source, so the akm
+                // adapter is always the right recognizer here — no per-component
+                // dispatch needed.
+                const drained = drainDirDocuments(akmAdapter, component, [ctx]);
+                for (const rejectedPath of drained.rejectedPaths)
+                    unindexable.add(rejectedPath);
+                for (const conceptId of drained.rejectedConceptIds)
+                    rejectedConceptIds.add(conceptId);
+                const entry = drained.entries[0];
+                // A broken workflow drains to zero entries and is treated as
+                // unindexable; valid peer sources have already compiled to source IR.
+                const conceptId = drained.conceptIdByFile.get(ctx.absPath);
+                if (entry && conceptId)
+                    pairs.push({ file, entry, conceptId, contentHash: drained.hashByFile.get(ctx.absPath) });
+                else
+                    unindexable.add(file);
+            }
             const db = openExistingDatabase(dbPath);
             try {
                 db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
-                // An absent or empty index is skipped on purpose — bootstrap belongs
-                // to the first read (`ensureIndex`) or an explicit `akm index`.
                 if (getEntryCount(db) === 0)
                     return true;
-                // Pass `stashDir` through as `reconcilePaths`'s `opts.root` rather
-                // than relying on its config-based bundleId->root lookup: this
-                // caller already knows the exact root its write happened under, and
-                // a bundleId that is not (yet, or ever) a `bundles.<key>` config
-                // entry — an ad hoc write target, e.g. a proposal's named target —
-                // can resolve to the WRONG root through that lookup, producing a
-                // corrupt conceptId and a duplicate `entries` row instead of a clean
-                // no-op (caught integrating this against tests/integration/proposal/proposals.test.ts).
-                await reconcilePaths(db, files, bundleId, { root: stashDir });
-                try {
+                const targetEntryIds = new Set();
+                let mutated = false;
+                db.transaction(() => {
+                    const unindexableEntryIds = new Set();
+                    for (const file of unindexable) {
+                        const rows = db
+                            .prepare(`SELECT id FROM entries
+                  WHERE file_path = ?
+                    AND bundle_id = ?
+                    AND adapter_id = ?`)
+                            .all(file, component.id, component.adapter);
+                        for (const row of rows)
+                            unindexableEntryIds.add(row.id);
+                    }
+                    for (const conceptId of rejectedConceptIds) {
+                        const itemRef = `${component.id}//${conceptId}`;
+                        const rows = db
+                            .prepare(`SELECT id FROM entries
+                  WHERE bundle_id = ? AND adapter_id = ?
+                    AND type = 'workflow'
+                    AND (concept_id = ? OR item_ref = ?)`)
+                            .all(component.id, component.adapter, conceptId, itemRef);
+                        for (const row of rows)
+                            unindexableEntryIds.add(row.id);
+                    }
+                    deleteEntriesByIds(db, [...unindexableEntryIds]);
+                    mutated ||= unindexableEntryIds.size > 0;
+                    for (const { file, entry, conceptId, contentHash } of pairs) {
+                        let entryWithSize = entry;
+                        try {
+                            entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
+                            if (hasMarkdownFragmentContent(entry)) {
+                                setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
+                            }
+                        }
+                        catch {
+                            // stat raced a delete — index without the size, like the full walk does.
+                        }
+                        // Real provenance (F4a M-core-2 item 5): populate item_ref/content_hash
+                        // via the SAME derivation the full-index writer uses, so a write-path
+                        // row is never a NULL-item_ref straggler.
+                        const provenance = deriveEntryProvenance({ bundleId: component.id, componentId: component.id, adapterId: component.adapter }, entry.type, entry.name, conceptId);
+                        // A materialized file has one current owner. If a targeted write is
+                        // the ownership handoff (for example an explicitly named bundle
+                        // replacing an earlier path-derived bootstrap identity), remove the
+                        // superseded physical-path row before publishing the canonical ref.
+                        const supersededIds = db
+                            .prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?")
+                            .all(file, provenance.itemRef);
+                        deleteEntriesByIds(db, supersededIds.map((row) => row.id));
+                        targetEntryIds.add(upsertEntry(db, file, entryWithSize, buildSearchText(entry), provenance, contentHash));
+                        mutated = true;
+                    }
+                })();
+                if (mutated) {
                     const config = loadConfig();
-                    const onlyHashes = unitHashesForFiles(db, files);
-                    await drainEmbeddingQueue(db, config, { onlyHashes });
-                }
-                catch (drainError) {
-                    // Best-effort: the write is already lexically searchable via
-                    // reconcilePaths above. The embedding queue is durable, so a
-                    // failed drain here is not a failure of this call — see the
-                    // return-contract note above.
-                    warnVerbose("Write-path embedding drain skipped (vectors appear after the next drain):", drainError instanceof Error ? drainError.message : String(drainError));
+                    await generateEmbeddingsForDb(db, config, () => { }, undefined, [...targetEntryIds]);
+                    publishTargetedEmbeddingMeta(db, config);
                 }
-                return true;
             }
             finally {
                 closeDatabase(db);
             }
+            return true;
         })();
     }
     catch (error) {