@cairnkeep/cli 2.15.4 → 2.17.0

package/mcp-memory-server/dist/context-pack.js

@@ -7,6 +7,7 @@ import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "nod
 import { promisify } from "node:util";
 import { z } from "zod";
 import { EmbeddingCache, cosineSimilarity, embedTexts, getEmbeddingConfig, hashText } from "./embeddings.js";
+import { doctorProgressiveContextCache, loadProgressiveContext, publicTree, selectHierarchicalFiles, } from "./context-pack-retrieval.js";
 import { indexOkfBundle, validateOkfBundle } from "./okf.js";
 import { atomicReplace, hardenPrivatePath, privatePathIsSafe } from "./platform-security.js";
 const execFileAsync = promisify(execFile);
@@ -547,7 +548,15 @@ async function acquirePointerLock(options) {
         catch (error) {
             if (error.code !== "EEXIST")
                 throw error;
-            const info = lstatSync(lock);
+            let info;
+            try {
+                info = lstatSync(lock);
+            }
+            catch (inspectionError) {
+                if (inspectionError.code === "ENOENT")
+                    continue;
+                throw inspectionError;
+            }
             if (!info.isDirectory() || info.isSymbolicLink())
                 throw new Error("Context pack project pointer lock is unsafe.");
             await delay(10);
@@ -782,7 +791,7 @@ export async function listVisibleContext(options) {
     }
     return { schema_version: 1, packs: [...grouped.values()] };
 }
-export async function searchVisibleContext(query, options) {
+async function flatSearchVisibleContext(query, options) {
     const normalized = query.trim().toLocaleLowerCase("en");
     if (!normalized)
         throw new Error("Context pack search query must not be empty.");
@@ -829,6 +838,137 @@ export async function searchVisibleContext(query, options) {
     results.sort((a, b) => b.score - a.score || a.pack_digest.localeCompare(b.pack_digest, "en") || a.path.localeCompare(b.path, "en") || a.offset - b.offset);
     return { schema_version: 1, query, search_mode: "substring", results: results.slice(0, options.limit ?? 20) };
 }
+function attachContextResultRefs(payload) {
+    const results = payload.results.map((result) => {
+        if (typeof result.file_digest !== "string")
+            throw new Error("Context search result is missing file provenance.");
+        return {
+            ...result,
+            chunk_digest: sha256("cairn-context-chunk-v1\0", result.pack_digest, "\0", result.path, "\0", result.file_digest, "\0", String(result.offset), "\0", result.text),
+        };
+    });
+    const resultDigest = sha256("cairn-context-results-v1\n", results.map(({ chunk_digest }) => chunk_digest).join("\n"));
+    return { ...payload, result_digest: resultDigest, results };
+}
+async function progressiveVisibleContext(options) {
+    const visible = await visiblePackFiles(options);
+    const inputs = await Promise.all(visible.map(async ({ pack, file }) => ({
+        pack_id: pack.manifest.id,
+        version: pack.manifest.version,
+        pack_digest: pack.digest,
+        pack_title: pack.manifest.title,
+        pack_description: pack.manifest.description,
+        path: file.path,
+        kind: file.kind,
+        title: file.title,
+        description: file.description,
+        keywords: file.keywords,
+        file_digest: file.sha256,
+        text: await readFile(join(pack.root, file.path), "utf8"),
+    })));
+    return { packs: await loadProgressiveContext(inputs, join(packBaseDir(), "cache", "context")), visible };
+}
+function packMatches(selector, pack) {
+    return pack.pack_digest === selector || pack.pack_digest.startsWith(selector) || pack.pack_id === selector || `${pack.pack_id}@${pack.version}` === selector;
+}
+export async function treeVisibleContext(options) {
+    const detail = options.detail ?? "abstract";
+    const { packs } = await progressiveVisibleContext(options);
+    const selected = options.pack ? packs.filter((pack) => packMatches(options.pack, pack)) : packs;
+    if (options.pack && selected.length !== 1)
+        throw new Error(selected.length ? "Context pack selector is ambiguous." : "Context pack is not enabled.");
+    return { schema_version: 1, detail, packs: selected.map((pack) => publicTree(pack, detail)) };
+}
+function progressiveFileKey(file) {
+    return `${file.pack_digest}:${file.path}`;
+}
+export async function searchVisibleContext(query, options) {
+    const strategy = options.strategy ?? "flat";
+    const detail = options.detail ?? "content";
+    if (strategy === "flat" && detail === "content" && !options.explain) {
+        const payload = await flatSearchVisibleContext(query, options);
+        return options.includeRefs ? attachContextResultRefs(payload) : payload;
+    }
+    const normalized = query.trim().toLocaleLowerCase("en");
+    if (!normalized)
+        throw new Error("Context pack search query must not be empty.");
+    const { packs, visible } = await progressiveVisibleContext(options);
+    const progressiveFiles = packs.flatMap((pack) => pack.files);
+    if (strategy === "flat") {
+        // Summary detail is one result per file. Retrieve all matching chunks
+        // before deduplication so a chatty first file cannot consume the public
+        // result limit and hide later matching files.
+        const flat = await flatSearchVisibleContext(query, { ...options, limit: Number.MAX_SAFE_INTEGER });
+        const summaries = new Map(progressiveFiles.map((file) => [progressiveFileKey(file), file]));
+        const seen = new Set();
+        const allResults = flat.results.flatMap((result) => {
+            const key = progressiveFileKey(result);
+            if (detail !== "content" && seen.has(key))
+                return [];
+            seen.add(key);
+            const summary = summaries.get(key);
+            return [{ ...result, offset: detail === "content" ? result.offset : 0, text: detail === "content" ? result.text : summary?.[detail] ?? result.text }];
+        });
+        const results = allResults.slice(0, options.limit ?? 20);
+        const trace = options.explain ? {
+            exact_leaf_bypass: false,
+            considered_files: progressiveFiles.length,
+            selected_files: new Set(results.map(progressiveFileKey)).size,
+            events: results.slice(0, 64).map((result) => ({ phase: "file", pack_digest: result.pack_digest, path: result.path, score: result.score, reason: "flat-result" })),
+            truncated: allResults.length > results.length || results.length > 64,
+        } : undefined;
+        const payload = { ...flat, strategy, detail, results, ...(trace ? { trace } : {}) };
+        return options.includeRefs ? attachContextResultRefs(payload) : payload;
+    }
+    const selection = selectHierarchicalFiles(query, packs);
+    const visibleByKey = new Map(visible.map((entry) => [`${entry.pack.digest}:${entry.file.path}`, entry]));
+    const selectedScores = new Map(selection.events.filter((event) => event.phase === "file" && event.path && event.pack_digest)
+        .map((event) => [`${event.pack_digest}:${event.path}`, event.score ?? 0]));
+    const results = [];
+    for (const file of selection.files) {
+        const source = visibleByKey.get(progressiveFileKey(file));
+        if (!source)
+            continue;
+        const base = { ...provenance(source.pack, source.file), ...await okfProvenance(source.pack, source.file) };
+        const summaryScore = selectedScores.get(progressiveFileKey(file)) ?? 0;
+        if (detail !== "content") {
+            results.push({ ...base, offset: 0, text: file[detail], score: summaryScore });
+            continue;
+        }
+        const fileChunks = chunks(file.text);
+        let matched = false;
+        for (const chunk of fileChunks) {
+            const searchable = `${file.title}\n${file.description}\n${file.keywords.join(" ")}\n${chunk.text}`.toLocaleLowerCase("en");
+            const occurrences = searchable.split(normalized).length - 1;
+            if (!occurrences && !selection.exactLeafBypass)
+                continue;
+            matched = true;
+            results.push({ ...base, offset: chunk.offset, text: chunk.text, score: selection.exactLeafBypass ? 100 : occurrences });
+        }
+        if (!matched && fileChunks[0])
+            results.push({ ...base, offset: fileChunks[0].offset, text: fileChunks[0].text, score: summaryScore });
+    }
+    results.sort((left, right) => right.score - left.score || left.pack_digest.localeCompare(right.pack_digest, "en") || left.path.localeCompare(right.path, "en") || left.offset - right.offset);
+    const limited = results.slice(0, options.limit ?? 20);
+    const trace = options.explain ? {
+        exact_leaf_bypass: selection.exactLeafBypass,
+        considered_packs: packs.length,
+        considered_files: progressiveFiles.length,
+        selected_files: selection.files.length,
+        events: selection.events,
+        truncated: selection.events.length >= 64 || results.length > limited.length,
+    } : undefined;
+    const payload = {
+        schema_version: 1,
+        query,
+        search_mode: "substring",
+        strategy,
+        detail,
+        results: limited,
+        ...(trace ? { trace } : {}),
+    };
+    return options.includeRefs ? attachContextResultRefs(payload) : payload;
+}
 export async function readVisibleContext(packSelector, path, options) {
     const normalized = normalizePackPath(path);
     const visible = await visiblePackFiles(options);
@@ -873,7 +1013,8 @@ export async function relatedVisibleContext(packSelector, path, options) {
     }
     return { schema_version: 1, ...provenance(pack, file), ...await okfProvenance(pack, file), direction, results, diagnostics: index.diagnostics.filter(({ path: diagnosticPath }) => diagnosticPath === file.path) };
 }
-export async function doctorContextPacks() {
+export async function doctorContextPacks(options = {}) {
+    const progressiveCachePacks = [];
     const issues = [];
     const remnants = [];
     let objects = 0;
@@ -883,6 +1024,23 @@ export async function doctorContextPacks() {
         objects = installed.length;
         for (const item of installed) {
             const pack = await validateContextPack(objectRoot(item.digest));
+            progressiveCachePacks.push({
+                pack_digest: pack.digest,
+                files: await Promise.all(pack.manifest.files.map(async (file) => ({
+                    pack_id: pack.manifest.id,
+                    version: pack.manifest.version,
+                    pack_digest: pack.digest,
+                    pack_title: pack.manifest.title,
+                    pack_description: pack.manifest.description,
+                    path: file.path,
+                    kind: file.kind,
+                    file_digest: file.sha256,
+                    title: file.title,
+                    description: file.description,
+                    keywords: file.keywords,
+                    text: await readFile(join(pack.root, file.path), "utf8"),
+                }))),
+            });
             if (pack.manifest.source_format?.name === "okf") {
                 const graph = await indexOkfBundle(pack.root, pack.manifest.files.map(({ path }) => path), pack.manifest.source_format.version);
                 expectedGraphs.set(pack.digest, JSON.stringify(graph));
@@ -984,6 +1142,16 @@ export async function doctorContextPacks() {
                 issues.push(`Stale context pack skill approval: ${path} -> ${approval.path}`);
         }
     }
+    const progressiveCache = await doctorProgressiveContextCache(join(packBaseDir(), "cache", "context"), progressiveCachePacks, { repair: options.repair });
+    issues.push(...progressiveCache.issues);
+    remnants.push(...progressiveCache.temporary_remnants);
+    if (options.repair) {
+        const contextCacheDirectory = join(packBaseDir(), "cache", "context");
+        for (let index = remnants.length - 1; index >= 0; index -= 1) {
+            if (remnants[index] === contextCacheDirectory || remnants[index].startsWith(`${contextCacheDirectory}${sep}`))
+                remnants.splice(index, 1);
+        }
+    }
     return { ok: issues.length === 0 && remnants.length === 0, objects, projects: pointers.length, issues, temporary_remnants: remnants };
 }
 export async function inspectContextPackUpdate(selector, options) {

package/mcp-memory-server/dist/domain-retrieval-provider.js

@@ -0,0 +1,257 @@
+import { isIP } from "node:net";
+const OPENVIKING_RESPONSE_LIMIT = 1024 * 1024;
+const OPENVIKING_DEFAULT_TIMEOUT_MS = 30_000;
+const OPENVIKING_MAX_TIMEOUT_MS = 120_000;
+export function resolveDomainRetrievalProvider(env = process.env) {
+    const configured = env.CAIRN_DOMAIN_RETRIEVAL_PROVIDER?.trim() || "anythingllm";
+    if (configured === "anythingllm" || configured === "openviking")
+        return configured;
+    throw new Error("CAIRN_DOMAIN_RETRIEVAL_PROVIDER must be either 'anythingllm' or 'openviking'.");
+}
+function isLoopbackHostname(hostname) {
+    const normalized = hostname.startsWith("[") && hostname.endsWith("]")
+        ? hostname.slice(1, -1)
+        : hostname;
+    if (normalized === "localhost" || normalized === "::1")
+        return true;
+    if (isIP(normalized) !== 4)
+        return false;
+    return normalized.split(".")[0] === "127";
+}
+function openVikingEndpoint(env) {
+    const configured = env.CAIRN_OPENVIKING_BASE_URL?.trim();
+    if (!configured) {
+        throw new Error("OpenViking base URL is not set; configure CAIRN_OPENVIKING_BASE_URL.");
+    }
+    let url;
+    try {
+        url = new URL(configured);
+    }
+    catch {
+        throw new Error("OpenViking base URL must be a valid HTTP(S) URL.");
+    }
+    if (url.username || url.password) {
+        throw new Error("OpenViking base URL must not contain embedded credentials.");
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new Error("OpenViking base URL must use HTTP or HTTPS.");
+    }
+    if (url.protocol === "http:" && !isLoopbackHostname(url.hostname)) {
+        throw new Error("OpenViking base URL must use HTTPS unless it targets a loopback host.");
+    }
+    if (url.search || url.hash) {
+        throw new Error("OpenViking base URL must not contain a query string or fragment.");
+    }
+    url.pathname = `${url.pathname.replace(/\/+$/, "")}/api/v1/search/find`;
+    return url;
+}
+function openVikingTimeout(env) {
+    const raw = env.CAIRN_OPENVIKING_TIMEOUT_MS?.trim();
+    if (!raw)
+        return OPENVIKING_DEFAULT_TIMEOUT_MS;
+    const timeout = Number(raw);
+    if (!Number.isInteger(timeout) || timeout < 100 || timeout > OPENVIKING_MAX_TIMEOUT_MS) {
+        throw new Error(`CAIRN_OPENVIKING_TIMEOUT_MS must be an integer from 100 to ${OPENVIKING_MAX_TIMEOUT_MS}.`);
+    }
+    return timeout;
+}
+function openVikingTargetUri(workspace) {
+    if (!workspace.startsWith("viking://")) {
+        return `viking://resources/${encodeURIComponent(workspace)}`;
+    }
+    if (workspace !== "viking://resources"
+        && !/^viking:\/\/resources\/[A-Za-z0-9._~!$&'()*+,;=:@%/-]+$/.test(workspace)) {
+        throw new Error("OpenViking workspace URI must stay below viking://resources.");
+    }
+    const path = workspace.slice("viking://resources".length);
+    let decoded;
+    try {
+        decoded = decodeURIComponent(path);
+    }
+    catch {
+        throw new Error("OpenViking workspace URI contains invalid percent encoding.");
+    }
+    if (decoded.split("/").some((segment) => segment === "." || segment === "..")) {
+        throw new Error("OpenViking workspace URI must not contain traversal segments.");
+    }
+    return workspace;
+}
+async function boundedResponseText(response) {
+    const advertisedLength = Number(response.headers.get("content-length"));
+    if (Number.isFinite(advertisedLength) && advertisedLength > OPENVIKING_RESPONSE_LIMIT) {
+        throw new Error("OpenViking response is too large.");
+    }
+    if (!response.body)
+        return "";
+    const reader = response.body.getReader();
+    const chunks = [];
+    let length = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        length += value.byteLength;
+        if (length > OPENVIKING_RESPONSE_LIMIT) {
+            await reader.cancel();
+            throw new Error("OpenViking response is too large.");
+        }
+        chunks.push(value);
+    }
+    const joined = new Uint8Array(length);
+    let offset = 0;
+    for (const chunk of chunks) {
+        joined.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return new TextDecoder("utf-8", { fatal: true }).decode(joined);
+}
+async function callAnythingLLM(workspace, query, env) {
+    const apiKey = env.ANYTHINGLLM_API_KEY;
+    if (!apiKey) {
+        throw new Error("ANYTHINGLLM_API_KEY is not set.");
+    }
+    const baseUrl = env.ANYTHINGLLM_BASE_URL ?? "http://localhost:3001";
+    const response = await fetch(`${baseUrl}/api/v1/workspace/${encodeURIComponent(workspace)}/chat`, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+            message: query,
+            mode: "query",
+        }),
+        signal: AbortSignal.timeout(120000),
+    });
+    if (!response.ok) {
+        const text = await response.text();
+        throw new Error(`AnythingLLM request failed with ${response.status}: ${text}`);
+    }
+    const payload = (await response.json());
+    const directText = [
+        payload.textResponse,
+        payload.response,
+        payload.message,
+        payload.text,
+    ].find((value) => typeof value === "string");
+    if (typeof directText === "string") {
+        return directText;
+    }
+    return JSON.stringify(payload, null, 2);
+}
+function parseOpenVikingResult(text) {
+    let payload;
+    try {
+        payload = JSON.parse(text);
+    }
+    catch {
+        throw new Error("OpenViking response is not valid JSON.");
+    }
+    if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+        throw new Error("OpenViking response does not contain a useful retrieval payload.");
+    }
+    const envelope = payload;
+    const result = envelope.result;
+    if (envelope.status !== "ok"
+        || !result
+        || typeof result !== "object"
+        || Array.isArray(result)) {
+        throw new Error("OpenViking response does not contain a useful retrieval payload.");
+    }
+    const candidate = result;
+    if (!Array.isArray(candidate.resources) || !Number.isInteger(candidate.total) || Number(candidate.total) < 0) {
+        throw new Error("OpenViking response does not contain a useful retrieval payload.");
+    }
+    const resources = [];
+    for (const resource of candidate.resources) {
+        if (!resource || typeof resource !== "object" || Array.isArray(resource)) {
+            throw new Error("OpenViking response does not contain a useful retrieval payload.");
+        }
+        const item = resource;
+        if (typeof item.uri !== "string"
+            || (item.uri !== "viking://resources" && !item.uri.startsWith("viking://resources/"))
+            || typeof item.abstract !== "string"
+            || (item.context_type !== undefined && item.context_type !== "resource")
+            || (item.level !== undefined && (!Number.isInteger(item.level) || Number(item.level) < 0 || Number(item.level) > 2))
+            || (item.score !== undefined && (typeof item.score !== "number" || !Number.isFinite(item.score)))
+            || (item.category !== undefined && typeof item.category !== "string")
+            || (item.match_reason !== undefined && typeof item.match_reason !== "string")
+            || (item.overview !== undefined && item.overview !== null && typeof item.overview !== "string")) {
+            throw new Error("OpenViking response does not contain a useful retrieval payload.");
+        }
+        resources.push({
+            ...(item.context_type === "resource" ? { context_type: item.context_type } : {}),
+            uri: item.uri,
+            ...(item.level !== undefined ? { level: Number(item.level) } : {}),
+            ...(item.score !== undefined ? { score: Number(item.score) } : {}),
+            ...(typeof item.category === "string" ? { category: item.category } : {}),
+            ...(typeof item.match_reason === "string" ? { match_reason: item.match_reason } : {}),
+            abstract: item.abstract,
+            ...(item.overview === null || typeof item.overview === "string" ? { overview: item.overview } : {}),
+        });
+    }
+    if (Number(candidate.total) < resources.length) {
+        throw new Error("OpenViking response does not contain a useful retrieval payload.");
+    }
+    return { resources, total: Number(candidate.total) };
+}
+async function callOpenViking(workspace, query, remote, env) {
+    if (env.CAIRN_OPENVIKING !== "1") {
+        throw new Error("OpenViking retrieval requires CAIRN_OPENVIKING=1.");
+    }
+    if (remote && env.CAIRN_OPENVIKING_MCP_HTTP !== "1") {
+        throw new Error("Remote OpenViking retrieval requires CAIRN_OPENVIKING_MCP_HTTP=1.");
+    }
+    const endpoint = openVikingEndpoint(env);
+    const timeout = openVikingTimeout(env);
+    const apiKey = (env.CAIRN_OPENVIKING_API_KEY ?? env.OPENVIKING_API_KEY)?.trim();
+    let response;
+    try {
+        response = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+                ...(apiKey ? { "X-API-Key": apiKey } : {}),
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                query,
+                target_uri: openVikingTargetUri(workspace),
+                context_type: ["resource"],
+                limit: 10,
+            }),
+            redirect: "manual",
+            signal: AbortSignal.timeout(timeout),
+        });
+    }
+    catch (error) {
+        if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) {
+            throw new Error("OpenViking request timed out.");
+        }
+        throw new Error(`OpenViking request failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (response.status >= 300 && response.status < 400) {
+        throw new Error(`OpenViking request refused redirect status ${response.status}.`);
+    }
+    if (!response.ok) {
+        throw new Error(`OpenViking request failed with ${response.status}.`);
+    }
+    if (!(response.headers.get("content-type") ?? "").toLowerCase().includes("application/json")) {
+        throw new Error("OpenViking response is not JSON.");
+    }
+    let text;
+    try {
+        text = await boundedResponseText(response);
+    }
+    catch (error) {
+        if (error instanceof TypeError)
+            throw new Error("OpenViking response is not valid UTF-8.");
+        throw error;
+    }
+    return JSON.stringify(parseOpenVikingResult(text), null, 2);
+}
+export async function queryDomainKnowledge({ workspace, query, remote = false, env = process.env, }) {
+    const provider = resolveDomainRetrievalProvider(env);
+    if (provider === "anythingllm")
+        return callAnythingLLM(workspace, query, env);
+    return callOpenViking(workspace, query, remote, env);
+}