akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/indexer/index-written-assets.js

@@ -2,90 +2,130 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Write-path indexing: targeted single-file index updates for asset writers.
+ * Write-path indexing: index exactly what a write just produced, inline, in
+ * the same call as the write. `remember` / `writeMarkdownAsset`, extract's
+ * session assets, `source clone`, and proposal acceptance all call this
+ * right after committing their file write so the asset is searchable
+ * immediately, without a background reindex.
  *
- * The index is maintained eagerly by every first-class mutation command
- * (`source add`, `wiki`, `workflow`, `setup` all run `akmIndex()` after
- * writing). The memory write paths — `akm remember` / `writeMarkdownAsset`
- * and extract's session assets — historically did not, which is why reads
- * used to compensate with stale-triggered background reindexes (the
- * lock-contention footgun removed alongside this module's introduction, per
- * the 2026-07 read-path reindex-contention findings §7).
+ * docs/plans/index-redesign-contract.md, module B2: a thin call to B1's
+ * `reconcilePaths` (the per-file version of the stat-walk reconcile — parses,
+ * upserts `entries`, derives units) followed by B4's `drainEmbeddingQueue`
+ * scoped to exactly the units that reconcile just produced. No lock probe, no
+ * rebuild detection, no background spawn — the redesign has no full-rebuild
+ * pipeline for a write path to defer to; two writers reconciling the same
+ * path converge on the same idempotent rows instead of contending for a lock.
  *
- * This is NOT a general reindex. It upserts exactly the files the caller just
- * wrote: frontmatter/metadata via the shared matcher pipeline, the canonical
- * row with its transactionally owned FTS projection, and vectors for changed
- * entry IDs when semantic search is enabled. Index-time LLM passes, graph
- * extraction, `builtAt`, and the per-dir walk cache remain full-index
- * responsibilities.
+ * FAIL-OPEN at every step: any error (index.db absent, empty, unreadable,
+ * unparseable file) is reduced to a verbose-only warning and the write
+ * command succeeds untouched — the degraded outcome is the asset appearing
+ * after the next reconcile (an explicit `akm index`, or the schedule) rather
+ * than immediately.
  */
-import fs from "node:fs";
 import path from "node:path";
-import { akmAdapter } from "../core/adapter/adapters/akm-adapter.js";
 import { loadConfig } from "../core/config/config.js";
 import { isDataDirUnreadableError } from "../core/errors.js";
-import { probeLock } from "../core/file-lock.js";
 import { isPathAbsent } from "../core/path-access.js";
-import { getDbPath, getIndexRebuildLockPath } from "../core/paths.js";
-import { formatLockHolderPid } from "../core/run-lock.js";
+import { getDbPath } from "../core/paths.js";
 import { warn, warnVerbose } from "../core/warn.js";
 import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
-import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
-import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
-import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
-import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
-import { drainDirDocuments } from "./scan/drain-dir.js";
-import { buildSearchText } from "./search/search-fields.js";
-import { buildFileContext } from "./walk/file-context.js";
+import { getEntryCount } from "../storage/repositories/index-entries-repository.js";
+import { SQLITE_CHUNK_SIZE } from "../storage/repositories/index-sql.js";
+import { drainEmbeddingQueue } from "./drain.js";
+import { deriveInstallations } from "./installations.js";
+import { reconcilePaths } from "./reconcile.js";
+import { resolveSourceEntries } from "./search/search-source.js";
 /**
- * Busy-timeout (ms) for write-path index upserts. A real write — unlike the
- * 250ms telemetry inserts — but it must not hang `akm remember` for the full
- * default 30s behind a running full reindex. When it times out, the upsert is
- * skipped and the asset becomes searchable after that reindex instead.
+ * Busy-timeout (ms) for write-path index upserts. Every index write in the
+ * redesign is a short immediate transaction relying on SQLite's own busy
+ * timeout to serialize concurrent writers (rule 5, docs/plans/index-redesign.md)
+ * rather than an application-level lock — this bounds how long one interactive
+ * write waits behind another process's short reconcile/drain transaction.
  */
 export const WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5_000;
+/**
+ * The distinct unit hashes reachable from an entry at one of `files`, via the
+ * `entry_units` mapping `reconcilePaths` just wrote — i.e. exactly the units
+ * this write's reconcile pass added or touched, the scope B4's drain should
+ * be asked to embed (module B2, docs/plans/index-redesign-contract.md:
+ * "Compute 'the units this write added' from the mapping:
+ * entries.file_path IN (paths) → entry_units → unit_hash"). A gone/unindexable
+ * path has no `entries` row any more and so contributes nothing here, which
+ * is correct — there is nothing left to embed for it.
+ */
+function unitHashesForFiles(db, files) {
+    const hashes = new Set();
+    for (let offset = 0; offset < files.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = files.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT DISTINCT eu.unit_hash AS unitHash
+           FROM entries e
+           JOIN entry_units eu ON eu.entry_id = e.id
+          WHERE e.file_path IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows)
+            hashes.add(row.unitHash);
+    }
+    return [...hashes];
+}
+/**
+ * The bundle id a fresh write into `stashDir` should be recorded under, when
+ * the caller did not already know it (`options.bundleId`).
+ *
+ * `stashDir` is matched against the CONFIGURED sources first, the way every
+ * other write-path caller resolves its bundle id (`knowledge.ts`'s demoted-file
+ * loop: `resolveSourceEntries` + `deriveInstallations`, matched by resolved
+ * root). A stash that is ALSO a configured bundle (`akm bundle add <dir>
+ * --name x`) must produce that bundle's id here, not a fresh slug of the raw
+ * path — a mismatch leaves an undeletable duplicate row once a full `akm
+ * index` reconciles the same path under its real, configured id (issue found
+ * integrating write-path W1). Only a stash that is genuinely unconfigured (an
+ * ad hoc write target) falls back to deriving an id from the raw path alone,
+ * the way a fresh install would.
+ */
+function resolveWrittenAssetBundleId(stashDir) {
+    const resolvedStashDir = path.resolve(stashDir);
+    const sourceEntries = resolveSourceEntries();
+    const index = sourceEntries.findIndex((entry) => path.resolve(entry.path) === resolvedStashDir);
+    if (index !== -1)
+        return deriveInstallations(sourceEntries)[index]?.id;
+    return deriveInstallations([{ path: stashDir, writable: true }])[0]?.id;
+}
 /**
  * Index the given just-written asset files into the existing local index.
  *
- * FAIL-OPEN at every step: any error (index.db absent, empty, locked past the
- * busy timeout, unparseable file) is reduced to a verbose-only warning and the
- * write command succeeds untouched. The degraded outcome is exactly the
- * pre-write-path-indexing behavior: the asset appears after the next full
- * `akm index` / improve-cron run.
+ * Returns `true` when the index is left in the state the caller expects —
+ * either genuinely updated, or one of the deliberate skips below — and
+ * `false` only for a real failure, which callers that gate their own success
+ * on this boolean (`acceptProposal`, `source clone`) must treat as "the
+ * write stands, but tell the operator the index needs a manual `akm index`".
  *
- * An absent or empty index is skipped on purpose — bootstrap belongs to the
- * first read (`ensureIndex`) or an explicit `akm index`, which also cover
- * embeddings and the other passes this fast path skips.
+ * Deliberate `true` skips:
+ *  - An absent index: bootstrap belongs to the first read (`ensureIndex`) or
+ *    an explicit `akm index`, not this fast path.
+ *  - An index with zero entries: same reasoning — nothing has bootstrapped
+ *    it yet.
+ *  - Every given path living under a dot-segment directory (e.g. `.meta/`):
+ *    the full reconcile never descends into those either.
  *
- * A live rebuild holding the rebuild lock is the SAME kind of skip, not a
- * failure (#956 fix): a live `akm index` run owns bringing the index to the
- * expected state on its own, so this call returns `true` on that skip
- * exactly like the absent-index and empty-index cases above. Callers that
- * gate their own success on this boolean (`acceptProposal`, `source clone`)
- * must never fail or warn just because a concurrent rebuild is in progress.
+ * `false` on failure:
+ *  - An index directory/file that exists but cannot be READ (not merely
+ *    absent) is the one failure the next reconcile will not heal on its own
+ *    (#791) — this warns audibly, not verbose-only.
+ *  - Any other error during `reconcilePaths` (unparseable file, a locked
+ *    database past the busy timeout, …) warns verbose-only; the write itself
+ *    already succeeded and the asset appears after the next reconcile.
+ *
+ * The embedding drain that follows a successful reconcile is best-effort: a
+ * failure there (a provider outage, a tripped circuit breaker) does NOT turn
+ * this call into a `false` — the asset is already lexically searchable, and
+ * the embedding queue is durable (module B4): any later drain, including the
+ * next write, computes the same "no vector yet" query and picks it up.
  */
 export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
     try {
         return await (async () => {
-            // #956: a live `akm index` rebuild holds index.db under one long
-            // transaction (persistDirRecords), which this fast path's own 5s
-            // busy_timeout would just contend with pointlessly. Interactive
-            // commands (remember, import, extract session assets) must never wait
-            // on it — skip the inline upsert/embedding entirely; the write itself
-            // (file + commit) has already succeeded by the time this runs, and the
-            // rebuild in progress will pick up the change on its own.
-            const rebuildProbe = probeLock(getIndexRebuildLockPath());
-            if (rebuildProbe.state === "held") {
-                // #956: name the launcher pid alongside the holder pid when known —
-                // every process listing and task log shows the launcher's pid, not
-                // the bun/node child's.
-                const holderLabel = formatLockHolderPid({
-                    pid: rebuildProbe.holderPid,
-                    launcherPid: rebuildProbe.launcherPid ?? null,
-                });
-                warn(`index rebuild in progress (pid ${holderLabel}); the next index pass will index ${filePaths.join(", ")}`);
-                return true;
-            }
             const dbPath = getDbPath();
             // `true` here means "the index is in the state the caller expects" — and
             // `acceptProposal` advances its journal to `index-finalized` on the
@@ -94,132 +134,54 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
             // `openExistingDatabase` and surfaces as the honest `false` (#791).
             if (isPathAbsent(dbPath))
                 return true;
-            // The full walk never descends into dot-directories (for example `.meta/`)
-            // — mirror that dot-segment skip here so this fast path indexes exactly
-            // what a full run would. Sensitive/
-            // infra abstention is the adapter's job now (see the `akmAdapter` note
-            // below), not a path pre-filter.
+            // The full reconcile never descends into dot-directories (for example
+            // `.meta/`) — mirror that dot-segment skip here so this fast path
+            // indexes exactly what a full reconcile would.
             const files = filePaths.filter((f) => {
                 const rel = path.relative(stashDir, f);
                 return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
             });
             if (files.length === 0)
                 return true;
-            // Generate metadata BEFORE opening the DB so the write window stays
-            // short. One drain call per file keeps the entry↔path pairing exact and
-            // reuses the full-index recognize engine (F4a M-core-2 item 5): broken
-            // workflows drop, valid workflow docs are cached for the side-table upsert.
-            const component = deriveInstallations([
-                { path: stashDir, writable: true, ...(options.bundleId ? { registryId: options.bundleId } : {}) },
-            ])[0]?.components[0];
-            if (!component)
+            // An explicit bundleId wins outright (it IS the resulting installation
+            // id); otherwise resolve it config-aware, matching a configured source
+            // before ever falling back to deriving one from the raw path.
+            const bundleId = options.bundleId ?? resolveWrittenAssetBundleId(stashDir);
+            if (!bundleId)
                 throw new Error(`Could not derive bundle provenance for ${stashDir}`);
-            const pairs = [];
-            const unindexable = new Set();
-            const rejectedConceptIds = new Set();
-            for (const file of files) {
-                if (!fs.existsSync(file)) {
-                    let authoredDanglingSymlink = false;
-                    try {
-                        authoredDanglingSymlink = fs.lstatSync(file).isSymbolicLink();
-                    }
-                    catch {
-                        // A genuinely absent path has no source identity to preflight.
-                    }
-                    if (!authoredDanglingSymlink) {
-                        unindexable.add(file);
-                        continue;
-                    }
-                }
-                const ctx = buildFileContext(stashDir, file);
-                // Hardcoded `akmAdapter` on purpose (owner ruling 2026-07-21): this
-                // write-path fast path only ever runs for assets a first-class akm
-                // mutation command just wrote into a managed akm source, so the akm
-                // adapter is always the right recognizer here — no per-component
-                // dispatch needed.
-                const drained = drainDirDocuments(akmAdapter, component, [ctx]);
-                for (const rejectedPath of drained.rejectedPaths)
-                    unindexable.add(rejectedPath);
-                for (const conceptId of drained.rejectedConceptIds)
-                    rejectedConceptIds.add(conceptId);
-                const entry = drained.entries[0];
-                // A broken workflow drains to zero entries and is treated as
-                // unindexable; valid peer sources have already compiled to source IR.
-                const conceptId = drained.conceptIdByFile.get(ctx.absPath);
-                if (entry && conceptId)
-                    pairs.push({ file, entry, conceptId, contentHash: drained.hashByFile.get(ctx.absPath) });
-                else
-                    unindexable.add(file);
-            }
             const db = openExistingDatabase(dbPath);
             try {
                 db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
+                // An absent or empty index is skipped on purpose — bootstrap belongs
+                // to the first read (`ensureIndex`) or an explicit `akm index`.
                 if (getEntryCount(db) === 0)
                     return true;
-                const targetEntryIds = new Set();
-                let mutated = false;
-                db.transaction(() => {
-                    const unindexableEntryIds = new Set();
-                    for (const file of unindexable) {
-                        const rows = db
-                            .prepare(`SELECT id FROM entries
-                  WHERE file_path = ?
-                    AND bundle_id = ?
-                    AND adapter_id = ?`)
-                            .all(file, component.id, component.adapter);
-                        for (const row of rows)
-                            unindexableEntryIds.add(row.id);
-                    }
-                    for (const conceptId of rejectedConceptIds) {
-                        const itemRef = `${component.id}//${conceptId}`;
-                        const rows = db
-                            .prepare(`SELECT id FROM entries
-                  WHERE bundle_id = ? AND adapter_id = ?
-                    AND type = 'workflow'
-                    AND (concept_id = ? OR item_ref = ?)`)
-                            .all(component.id, component.adapter, conceptId, itemRef);
-                        for (const row of rows)
-                            unindexableEntryIds.add(row.id);
-                    }
-                    deleteEntriesByIds(db, [...unindexableEntryIds]);
-                    mutated ||= unindexableEntryIds.size > 0;
-                    for (const { file, entry, conceptId, contentHash } of pairs) {
-                        let entryWithSize = entry;
-                        try {
-                            entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
-                            if (hasMarkdownFragmentContent(entry)) {
-                                setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
-                            }
-                        }
-                        catch {
-                            // stat raced a delete — index without the size, like the full walk does.
-                        }
-                        // Real provenance (F4a M-core-2 item 5): populate item_ref/content_hash
-                        // via the SAME derivation the full-index writer uses, so a write-path
-                        // row is never a NULL-item_ref straggler.
-                        const provenance = deriveEntryProvenance({ bundleId: component.id, componentId: component.id, adapterId: component.adapter }, entry.type, entry.name, conceptId);
-                        // A materialized file has one current owner. If a targeted write is
-                        // the ownership handoff (for example an explicitly named bundle
-                        // replacing an earlier path-derived bootstrap identity), remove the
-                        // superseded physical-path row before publishing the canonical ref.
-                        const supersededIds = db
-                            .prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?")
-                            .all(file, provenance.itemRef);
-                        deleteEntriesByIds(db, supersededIds.map((row) => row.id));
-                        targetEntryIds.add(upsertEntry(db, file, entryWithSize, buildSearchText(entry), provenance, contentHash));
-                        mutated = true;
-                    }
-                })();
-                if (mutated) {
+                // Pass `stashDir` through as `reconcilePaths`'s `opts.root` rather
+                // than relying on its config-based bundleId->root lookup: this
+                // caller already knows the exact root its write happened under, and
+                // a bundleId that is not (yet, or ever) a `bundles.<key>` config
+                // entry — an ad hoc write target, e.g. a proposal's named target —
+                // can resolve to the WRONG root through that lookup, producing a
+                // corrupt conceptId and a duplicate `entries` row instead of a clean
+                // no-op (caught integrating this against tests/integration/proposal/proposals.test.ts).
+                await reconcilePaths(db, files, bundleId, { root: stashDir });
+                try {
                     const config = loadConfig();
-                    await generateEmbeddingsForDb(db, config, () => { }, undefined, [...targetEntryIds]);
-                    publishTargetedEmbeddingMeta(db, config);
+                    const onlyHashes = unitHashesForFiles(db, files);
+                    await drainEmbeddingQueue(db, config, { onlyHashes });
+                }
+                catch (drainError) {
+                    // Best-effort: the write is already lexically searchable via
+                    // reconcilePaths above. The embedding queue is durable, so a
+                    // failed drain here is not a failure of this call — see the
+                    // return-contract note above.
+                    warnVerbose("Write-path embedding drain skipped (vectors appear after the next drain):", drainError instanceof Error ? drainError.message : String(drainError));
                 }
+                return true;
             }
             finally {
                 closeDatabase(db);
             }
-            return true;
         })();
     }
     catch (error) {