akm-cli 0.9.1 → 0.9.2-alpha.2

package/dist/indexer/installations.js

@@ -113,6 +113,12 @@ export function deriveInstallations(sources) {
     }
     return installations;
 }
+/** Canonical bundle ids whose configured component policy permits mutation. */
+export function deriveWritableBundleIds(sources) {
+    return new Set(deriveInstallations(sources)
+        .filter((installation) => installation.components.some((component) => component.writable === true))
+        .map((installation) => installation.id));
+}
 /**
  * Derive the durable `EntryProvenance` for an indexed entry (Chunk-5 flip
  * §14.4): `conceptId` is the D-R2 qualified `<stash-subdir>/<name>` spelling

package/dist/indexer/lookup/adapter-concept-owner.js

@@ -0,0 +1,283 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import fs from "node:fs";
+import path from "node:path";
+import { adapterForId } from "../../core/adapter/registry.js";
+import { isWithin } from "../../core/common.js";
+import { UsageError } from "../../core/errors.js";
+import { canonicalizeWorkflowName } from "../../core/recognition-util.js";
+import { resolveUniqueWorkflowSource, workflowNameForConceptId, } from "../../workflows/source-files.js";
+import { buildFileContext } from "../walk/file-context.js";
+const CONTENT_READ_REQUIRED = Symbol("adapter ownership probe requires content");
+const OWNER_SCAN_MAX_DIRECTORIES = 4_096;
+const OWNER_SCAN_MAX_FILES = 16_384;
+const OWNER_SCAN_SKIP_DIRECTORIES = new Set([".git", "node_modules", "bin", ".cache"]);
+const OWNER_SCAN_SKIP_FILES = new Set([".stash.json", ".gitignore", ".gitattributes"]);
+export class AdapterConceptOwnershipError extends UsageError {
+}
+export class AdapterConceptCollisionError extends AdapterConceptOwnershipError {
+    constructor(adapterId, conceptId, paths) {
+        const sorted = [...paths].sort(comparePaths);
+        super(`Adapter "${adapterId}" has multiple physical owners for "${conceptId}": ${sorted.join(", ")}.`, "RESOURCE_ALREADY_EXISTS");
+        this.name = "AdapterConceptCollisionError";
+        Object.setPrototypeOf(this, new.target.prototype);
+    }
+}
+export class AdapterConceptScanError extends AdapterConceptOwnershipError {
+    constructor(adapterId, sourcePath, detail) {
+        super(`Adapter "${adapterId}" could not safely enumerate ${sourcePath}: ${detail}.`, "INVALID_FLAG_VALUE");
+        this.name = "AdapterConceptScanError";
+        Object.setPrototypeOf(this, new.target.prototype);
+    }
+}
+function normalizedConceptId(conceptId) {
+    const normalized = conceptId.replaceAll("\\", "/");
+    if (!normalized ||
+        path.posix.isAbsolute(normalized) ||
+        normalized === ".." ||
+        normalized.startsWith("../") ||
+        path.posix.normalize(normalized) !== normalized) {
+        return undefined;
+    }
+    return normalized;
+}
+function componentFor(sourcePath, adapterId) {
+    return { id: adapterId, adapter: adapterId, root: sourcePath, writable: false };
+}
+function lexicallyWithin(candidate, root) {
+    const relative = path.relative(path.resolve(root), path.resolve(candidate));
+    return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
+}
+/** Add case-only suffix spellings beside an adapter-authored read candidate. */
+function candidateSpellings(candidate, sourcePath, realRoot) {
+    const authored = path.resolve(candidate.path);
+    if (!lexicallyWithin(authored, sourcePath))
+        return [{ ...candidate, path: authored }];
+    const parent = path.dirname(authored);
+    let realParent;
+    let entries;
+    try {
+        realParent = fs.realpathSync(parent);
+        entries = fs.readdirSync(parent, { withFileTypes: true });
+    }
+    catch {
+        return [{ ...candidate, path: authored }];
+    }
+    if (!isWithin(realParent, realRoot))
+        return [{ ...candidate, path: authored }];
+    const extension = path.extname(authored);
+    if (!extension)
+        return [{ ...candidate, path: authored }];
+    const stem = path.basename(authored, extension);
+    return entries
+        .filter((entry) => {
+        const entryExtension = path.extname(entry.name);
+        return (path.basename(entry.name, entryExtension) === stem && entryExtension.toLowerCase() === extension.toLowerCase());
+    })
+        .map((entry) => ({ ...candidate, path: path.join(parent, entry.name) }));
+}
+function inspectCandidate(sourcePath, realRoot, adapterId, candidate) {
+    const authoredPath = path.resolve(candidate.path);
+    if (!lexicallyWithin(authoredPath, sourcePath))
+        return undefined;
+    let authoredStat;
+    try {
+        authoredStat = fs.lstatSync(authoredPath);
+    }
+    catch {
+        return undefined;
+    }
+    if (!authoredStat.isFile() && !authoredStat.isSymbolicLink())
+        return undefined;
+    let realPath;
+    try {
+        realPath = fs.realpathSync(authoredPath);
+    }
+    catch {
+        return undefined;
+    }
+    if (!isWithin(realPath, realRoot))
+        return undefined;
+    try {
+        if (!fs.statSync(realPath).isFile())
+            return undefined;
+    }
+    catch {
+        return undefined;
+    }
+    return { path: authoredPath, realPath, conceptId: candidate.conceptId, adapterId };
+}
+function claimsWithoutContent(adapter, component, owner) {
+    const file = buildFileContext(component.root, owner.path);
+    let requestedBytes = false;
+    const denyBytes = () => {
+        requestedBytes = true;
+        throw CONTENT_READ_REQUIRED;
+    };
+    const noContentFile = { ...file, content: denyBytes, frontmatter: denyBytes };
+    try {
+        const document = adapter.recognize(component, noContentFile);
+        if (requestedBytes)
+            return true;
+        return document?.conceptId === owner.conceptId;
+    }
+    catch (error) {
+        if (error === CONTENT_READ_REQUIRED)
+            return true;
+        throw error;
+    }
+}
+function adapterPathContext(root, authoredPath) {
+    const file = buildFileContext(root, authoredPath);
+    return {
+        absPath: file.absPath,
+        relPath: file.relPath,
+        ext: file.ext,
+        fileName: file.fileName,
+        parentDir: file.parentDir,
+        parentDirAbs: file.parentDirAbs,
+        ancestorDirs: file.ancestorDirs,
+        stashRoot: file.stashRoot,
+    };
+}
+function scanRegularAuthoredPaths(sourcePath, realRoot, adapterId) {
+    const paths = [];
+    const stack = [path.resolve(sourcePath)];
+    let directories = 0;
+    let files = 0;
+    while (stack.length > 0) {
+        const current = stack.pop();
+        if (!current)
+            continue;
+        directories++;
+        if (directories > OWNER_SCAN_MAX_DIRECTORIES) {
+            throw new AdapterConceptScanError(adapterId, sourcePath, `directory limit ${OWNER_SCAN_MAX_DIRECTORIES} exceeded`);
+        }
+        let entries;
+        try {
+            entries = fs
+                .readdirSync(current, { withFileTypes: true })
+                .sort((left, right) => comparePaths(left.name, right.name));
+        }
+        catch (error) {
+            throw new AdapterConceptScanError(adapterId, sourcePath, `cannot read ${current}: ${String(error)}`);
+        }
+        for (let index = entries.length - 1; index >= 0; index--) {
+            const entry = entries[index];
+            if (!entry)
+                continue;
+            const authoredPath = path.join(current, entry.name);
+            if (entry.isSymbolicLink())
+                continue;
+            if (entry.isDirectory()) {
+                if (OWNER_SCAN_SKIP_DIRECTORIES.has(entry.name) || entry.name.startsWith("."))
+                    continue;
+                stack.push(authoredPath);
+                continue;
+            }
+            if (!entry.isFile() || OWNER_SCAN_SKIP_FILES.has(entry.name))
+                continue;
+            files++;
+            if (files > OWNER_SCAN_MAX_FILES) {
+                throw new AdapterConceptScanError(adapterId, sourcePath, `file limit ${OWNER_SCAN_MAX_FILES} exceeded`);
+            }
+            let realPath;
+            try {
+                realPath = fs.realpathSync(authoredPath);
+            }
+            catch {
+                continue;
+            }
+            if (isWithin(realPath, realRoot))
+                paths.push(authoredPath);
+        }
+    }
+    return paths.sort(comparePaths);
+}
+function scannedReadCandidates(adapter, component, sourcePath, realRoot, conceptId) {
+    const recognizePathCandidates = adapter.recognizePathCandidates;
+    if (!recognizePathCandidates)
+        return [];
+    return scanRegularAuthoredPaths(sourcePath, realRoot, adapter.id).flatMap((authoredPath) => recognizePathCandidates(component, adapterPathContext(sourcePath, authoredPath)).flatMap((candidateConceptId) => candidateConceptId === conceptId ? [{ path: authoredPath, conceptId: candidateConceptId }] : []));
+}
+/**
+ * Resolve one adapter/component's exact physical concept owner without reading
+ * authored bytes. Native workflow arbitration is reused verbatim; every other
+ * adapter supplies its own read placements and is probed with a byte-denying
+ * FileContext so path-level abstention remains authoritative.
+ */
+export function resolveAdapterConceptOwner(sourcePath, adapterId, conceptId) {
+    const adapter = adapterForId(adapterId);
+    const normalized = normalizedConceptId(conceptId);
+    if (!adapter || !normalized)
+        return undefined;
+    let realRoot;
+    try {
+        realRoot = fs.realpathSync(sourcePath);
+    }
+    catch {
+        return undefined;
+    }
+    const component = componentFor(sourcePath, adapterId);
+    const ownersByIdentity = new Map();
+    const workflowName = workflowNameForConceptId(adapterId, normalized);
+    const resolutionConceptId = workflowName === undefined
+        ? normalized
+        : adapterId === "akm"
+            ? `workflows/${canonicalizeWorkflowName(workflowName)}`
+            : canonicalizeWorkflowName(workflowName);
+    if (workflowName !== undefined) {
+        const workflowSource = resolveUniqueWorkflowSource(sourcePath, adapterId, workflowName);
+        if (workflowSource) {
+            const workflowOwner = {
+                path: workflowSource.path,
+                realPath: workflowSource.realPath,
+                conceptId: resolutionConceptId,
+                adapterId,
+                workflowSource,
+            };
+            ownersByIdentity.set(`${path.resolve(workflowOwner.path)}\0${resolutionConceptId}`, workflowOwner);
+        }
+    }
+    const directSpellings = (adapter.readCandidates?.(component, resolutionConceptId) ?? []).flatMap((candidate) => candidateSpellings(candidate, sourcePath, realRoot));
+    const spellings = [
+        ...directSpellings,
+        ...scannedReadCandidates(adapter, component, sourcePath, realRoot, resolutionConceptId),
+    ];
+    const inspected = [
+        ...new Map(spellings.map((candidate) => [`${path.resolve(candidate.path)}\0${candidate.conceptId}`, candidate])).values(),
+    ]
+        .sort((left, right) => comparePaths(left.path, right.path))
+        .flatMap((candidate) => {
+        const owner = inspectCandidate(sourcePath, realRoot, adapterId, candidate);
+        return owner ? [owner] : [];
+    });
+    for (const candidate of inspected) {
+        if (candidate.conceptId !== resolutionConceptId)
+            continue;
+        const identity = `${path.resolve(candidate.path)}\0${candidate.conceptId}`;
+        if (ownersByIdentity.has(identity))
+            continue;
+        if (claimsWithoutContent(adapter, component, candidate))
+            ownersByIdentity.set(identity, candidate);
+    }
+    const owners = [...ownersByIdentity.values()].filter((owner) => owner.conceptId === resolutionConceptId);
+    if (owners.length > 1) {
+        throw new AdapterConceptCollisionError(adapterId, resolutionConceptId, owners.map((owner) => path.relative(sourcePath, owner.path).replaceAll("\\", "/")));
+    }
+    return owners[0];
+}
+export function indexedPathMatchesOwner(indexedPath, owner) {
+    if (path.resolve(indexedPath) !== path.resolve(owner.path))
+        return false;
+    try {
+        return path.resolve(fs.realpathSync(indexedPath)) === path.resolve(owner.realPath);
+    }
+    catch {
+        return false;
+    }
+}
+function comparePaths(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}

package/dist/indexer/materialize-embeddings.js

@@ -0,0 +1,155 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { isVerbose, warn, warnVerbose } from "../core/warn.js";
+import { embedBatch } from "../llm/embedder.js";
+import { estimateTokenCount } from "../llm/embedders/remote.js";
+import { getEmbeddableEntryCount } from "../storage/repositories/index-entries-repository.js";
+import { deleteMeta, getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
+import { getAllEntriesForEmbedding, getEmbeddingCount, isVecAvailable, isVecFastPathComplete, isVecFastPathReady, purgeEmbeddings, setVecFastPathReady, upsertEmbedding, } from "../storage/repositories/index-vec-repository.js";
+import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw signal.reason instanceof Error ? signal.reason : new Error("index interrupted");
+    }
+}
+export async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds) {
+    throwIfAborted(signal);
+    if (config.semanticSearchMode === "off") {
+        onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
+        return { success: false, reason: "index-missing", message: "Semantic search is disabled." };
+    }
+    // A targeted call starts from an already-published generation. Preserve its
+    // trust decision in O(1): successful writes for the changed IDs keep a
+    // healthy fast path healthy, but can never promote a generation already
+    // marked degraded. Global runs can afford to verify the entire derived set.
+    const vecFastPathWasReady = isVecFastPathReady(db);
+    const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
+    const storedFingerprint = getMeta(db, "embeddingFingerprint");
+    let targetEntryIds = entryIds;
+    if (storedFingerprint && storedFingerprint !== currentFingerprint) {
+        purgeEmbeddings(db, { dropVecTable: true });
+        deleteMeta(db, "embeddingDim");
+        // A provider/model change invalidates the entire vector generation, even
+        // when a targeted write happened to discover it first.
+        targetEntryIds = undefined;
+    }
+    try {
+        throwIfAborted(signal);
+        const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
+        if (allEntries.length === 0) {
+            onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
+            setMeta(db, "embeddingFingerprint", currentFingerprint);
+            return { success: true };
+        }
+        onProgress({
+            phase: "embeddings",
+            message: `Generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}.`,
+        });
+        const texts = allEntries.map((entry) => entry.searchText);
+        if (isVerbose()) {
+            const EMBED_BATCH_SIZE = 100;
+            const totalBatches = Math.ceil(texts.length / EMBED_BATCH_SIZE);
+            for (const [i, entry] of allEntries.entries()) {
+                const batchNum = Math.floor(i / EMBED_BATCH_SIZE) + 1;
+                const chars = entry.searchText.length;
+                const tokens = estimateTokenCount(entry.searchText);
+                const ref = entry.itemRef;
+                warnVerbose(`[embed] ${ref} (${chars} chars, est. ${tokens} tokens) → batch ${batchNum}/${totalBatches}`);
+            }
+        }
+        let heartbeatTimer;
+        try {
+            heartbeatTimer = setInterval(() => {
+                onProgress({
+                    phase: "embeddings",
+                    message: `Still generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}; waiting on embedding provider.`,
+                });
+            }, 15000);
+            const embeddings = await embedBatch(texts, config.embedding, signal);
+            throwIfAborted(signal);
+            let storedCount = 0;
+            let skippedCount = 0;
+            let vecFailedCount = 0;
+            let vecUnavailableCount = 0;
+            db.transaction(() => {
+                for (const [i, entry] of allEntries.entries()) {
+                    const embedding = embeddings[i];
+                    if (!embedding)
+                        throw new Error(`Embedding provider returned no vector for ${entry.itemRef}.`);
+                    const result = upsertEmbedding(db, entry.id, embedding);
+                    if (result.stored)
+                        storedCount++;
+                    else
+                        skippedCount++;
+                    if (result.vec === "failed")
+                        vecFailedCount++;
+                    if (result.vec === "unavailable")
+                        vecUnavailableCount++;
+                }
+            })();
+            if (skippedCount > 0) {
+                warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
+            }
+            const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
+            setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
+            if (vecFailedCount > 0) {
+                warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed — ` +
+                    "semantic search will use the slower JS-cosine fallback over stored embeddings. " +
+                    "Rebuild with 'akm index --full' after resolving the vec table (often a vector-dimension mismatch).");
+            }
+            onProgress({
+                phase: "embeddings",
+                message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"}.`,
+            });
+            setMeta(db, "embeddingFingerprint", currentFingerprint);
+            return { success: true, vecInsertFailures: vecFailedCount };
+        }
+        finally {
+            if (heartbeatTimer)
+                clearInterval(heartbeatTimer);
+        }
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        warn("Embedding generation failed, continuing without:", message);
+        onProgress({ phase: "embeddings", message: `Embedding generation failed: ${message}` });
+        return {
+            success: false,
+            reason: classifySemanticFailure(message),
+            message: `Semantic search verification failed: ${message}`,
+        };
+    }
+}
+/** Publish the canonical semantic health snapshot after a targeted mutation. */
+export function publishTargetedSemanticStatus(db, config, result) {
+    if (config.semanticSearchMode === "off") {
+        clearSemanticStatus();
+        setMeta(db, "hasEmbeddings", "0");
+        return;
+    }
+    const entryCount = getEmbeddableEntryCount(db);
+    const embeddingCount = getEmbeddingCount(db);
+    let status;
+    if (entryCount === 0)
+        status = "pending";
+    else if (embeddingCount >= entryCount)
+        status = isVecAvailable(db) && isVecFastPathReady(db) ? "ready-vec" : "ready-js";
+    else
+        status = "blocked";
+    setMeta(db, "hasEmbeddings", status === "ready-js" || status === "ready-vec" ? "1" : "0");
+    writeSemanticStatus({
+        status,
+        ...(status === "blocked" ? { reason: result.reason ?? "index-failed" } : {}),
+        ...(status === "blocked"
+            ? {
+                message: result.message ??
+                    `Semantic search verification failed (${embeddingCount}/${entryCount} embeddings available).`,
+            }
+            : {}),
+        providerFingerprint: deriveSemanticProviderFingerprint(config.embedding),
+        lastCheckedAt: new Date().toISOString(),
+        entryCount,
+        embeddingCount,
+    });
+}