attention-manifest-protocol 0.6.1 → 0.6.2

package/lib/scan/texScanner.js

@@ -0,0 +1,581 @@
+/**
+ * Bounded, literal LaTeX structure and project adapter. Never expands macros,
+ * executes TeX or resolves files outside the caller-provided inventory.
+ */
+import { posix } from "node:path";
+import { encodeDocumentDetail } from "../context/document.js";
+import { workspaceTerms } from "../context/workspace.js";
+const Levels = { part: 0, chapter: 1, section: 2, subsection: 3, subsubsection: 4, paragraph: 5, subparagraph: 6 };
+const Literals = new Set(["verbatim", "verbatim*", "Verbatim", "lstlisting", "minted", "comment", "filecontents", "filecontents*"]);
+const Dependencies = new Set([
+    "input",
+    "include",
+    "subfile",
+    "bibliography",
+    "addbibresource",
+    "includegraphics",
+    "documentclass",
+    "usepackage",
+    "RequirePackage",
+    "LoadClass",
+]);
+const References = new Set(["ref", "eqref", "pageref", "autoref", "cref", "Cref", "vref", "nameref"]);
+const Definitions = new Set(["newcommand", "renewcommand", "providecommand", "DeclareRobustCommand", "newenvironment", "renewenvironment"]);
+const short = (value) => value.replace(/\s+/g, " ").trim().slice(0, 180);
+const skipSpace = (text, start) => {
+    let offset = start;
+    while (offset < text.length) {
+        if (/\s/.test(text[offset])) {
+            offset += 1;
+        }
+        else if (text[offset] === "%") {
+            offset = text.indexOf("\n", offset);
+            if (offset < 0) {
+                return text.length;
+            }
+        }
+        else {
+            break;
+        }
+    }
+    return offset;
+};
+const group = (text, start, open = "{") => {
+    const begin = skipSpace(text, start);
+    if (text[begin] !== open) {
+        return (void 0);
+    }
+    const close = open === "{" ? "}" : open === "[" ? "]" : ")";
+    let depth = 1;
+    let braces = 0;
+    let value = "";
+    for (let offset = begin + 1; offset < text.length; offset += 1) {
+        const char = text[offset];
+        if (char === "\\") {
+            value += text.slice(offset, offset + 2);
+            offset += 1;
+            continue;
+        }
+        if (char === "%") {
+            const newline = text.indexOf("\n", offset);
+            offset = newline < 0 ? text.length : newline;
+            value += "\n";
+            continue;
+        }
+        if (open === "[" && char === "{") {
+            braces += 1;
+        }
+        else if (open === "[" && char === "}") {
+            braces -= 1;
+        }
+        if (braces === 0 && char === open) {
+            depth += 1;
+        }
+        else if (braces === 0 && char === close) {
+            depth -= 1;
+            if (depth === 0) {
+                return { value, end: offset + 1 };
+            }
+        }
+        if (depth + braces > 128 || braces < 0) {
+            throw new Error("TeX group nesting exceeds bounds");
+        }
+        value += char;
+    }
+    throw new Error("Unclosed TeX argument group");
+};
+const tokens = (text) => {
+    const result = [];
+    let offset = 0;
+    while (offset < text.length) {
+        if (text[offset] === "%") {
+            offset = text.indexOf("\n", offset);
+            if (offset < 0) {
+                break;
+            }
+        }
+        if (text[offset] !== "\\") {
+            offset += 1;
+            continue;
+        }
+        const start = offset;
+        const match = /^\\([A-Za-z@]+|[^\r\n])/.exec(text.slice(offset));
+        if (!match) {
+            offset += 1;
+            continue;
+        }
+        const name = match[1];
+        offset += match[0].length;
+        if (text[offset] === "*") {
+            offset += 1;
+        }
+        if (name === "verb") {
+            const delimiter = text[offset];
+            const end = delimiter ? text.indexOf(delimiter, offset + 1) : -1;
+            const newline = text.indexOf("\n", offset);
+            offset = end >= 0 && (newline < 0 || end < newline) ? end + 1 : newline < 0 ? text.length : newline;
+            continue;
+        }
+        if (["def", "gdef", "edef", "xdef"].includes(name)) {
+            const body = text.indexOf("{", offset);
+            offset = body < 0 ? text.length : group(text, body).end;
+            continue;
+        }
+        if (Definitions.has(name)) {
+            const macro = group(text, offset);
+            offset = macro?.end ?? skipSpace(text, offset);
+            if (!macro) {
+                offset += /^\\[A-Za-z@]+/.exec(text.slice(offset))?.[0].length ?? 0;
+            }
+            let option = group(text, offset, "[");
+            while (option) {
+                offset = option.end;
+                option = group(text, offset, "[");
+            }
+            const body = group(text, offset);
+            offset = body?.end ?? offset;
+            if (name.endsWith("environment")) {
+                offset = group(text, offset)?.end ?? offset;
+            }
+            continue;
+        }
+        if (!Object.hasOwn(Levels, name) &&
+            !["begin", "end", "label", "title"].includes(name) &&
+            !Dependencies.has(name) &&
+            !References.has(name) &&
+            !/^(?:[A-Za-z]*cite[A-Za-z]*|nocite)$/.test(name)) {
+            continue;
+        }
+        let option = group(text, offset, "[");
+        while (option) {
+            offset = option.end;
+            option = group(text, offset, "[");
+        }
+        let argument = group(text, offset);
+        if (!argument && name === "input") {
+            const begin = skipSpace(text, offset);
+            const filename = /^[^\s%{}\\]+/.exec(text.slice(begin))?.[0] ?? "";
+            if (filename) {
+                argument = { value: filename, end: begin + filename.length };
+            }
+        }
+        if (!argument) {
+            result.push({ name, value: "", start, end: offset });
+            continue;
+        }
+        offset = argument.end;
+        result.push({ name, value: argument.value.trim(), start, end: offset });
+        if (name === "begin" && Literals.has(argument.value.trim())) {
+            const closing = `\\end{${argument.value.trim()}}`;
+            const end = text.indexOf(closing, offset);
+            if (end < 0) {
+                throw new Error(`Unclosed literal environment ${argument.value}`);
+            }
+            result.push({ name: "end", value: argument.value.trim(), start: end, end: end + closing.length });
+            offset = end + closing.length;
+        }
+    }
+    return result;
+};
+const bibBlocks = (text) => {
+    const blocks = [];
+    let offset = 0;
+    while (offset < text.length) {
+        if (text[offset] === "%") {
+            const end = text.indexOf("\n", offset);
+            offset = end < 0 ? text.length : end + 1;
+            continue;
+        }
+        const match = /^@([A-Za-z]+)\s*([{(])/.exec(text.slice(offset));
+        if (!match) {
+            offset += 1;
+            continue;
+        }
+        const start = offset;
+        const open = match[2];
+        offset += match[0].length;
+        const bodyStart = offset;
+        let braces = open === "{" ? 1 : 0;
+        let quoted = false;
+        while (offset < text.length) {
+            const char = text[offset];
+            if (char === "\\") {
+                offset += 2;
+                continue;
+            }
+            if (char === '"' && (braces === 0 || (open === "{" && braces === 1))) {
+                quoted = !quoted;
+            }
+            if (!quoted && char === "{") {
+                braces += 1;
+            }
+            else if (!quoted && char === "}") {
+                braces -= 1;
+                if (open === "{" && braces === 0) {
+                    break;
+                }
+            }
+            else if (!quoted && open === "(" && char === ")" && braces === 0) {
+                break;
+            }
+            if (braces > 128) {
+                throw new Error("BibTeX nesting exceeds 128");
+            }
+            offset += 1;
+        }
+        if (offset >= text.length) {
+            throw new Error("Unclosed BibTeX entry");
+        }
+        const type = match[1].toLowerCase();
+        const key = text.slice(bodyStart, offset).split(",")[0].trim();
+        offset += 1;
+        if (!["comment", "preamble", "string"].includes(type) && key && !/[\s{}]/.test(key)) {
+            blocks.push({ start, end: offset, label: key, type: "paragraph", tag: `bibtex:${type}`, key });
+        }
+    }
+    return blocks;
+};
+/**
+ * Adds format facts to the shared snapshot. LaTeX build projects are knowledge
+ * views under their root files, not replacements for workspace ownership.
+ */
+export const addTexStructure = (snapshot, inputs, store) => {
+    const entities = new Map(snapshot.entities.map((entity) => [entity.id, entity]));
+    const parsed = new Map();
+    const roots = new Set();
+    const requests = new Map();
+    const keys = new Set();
+    const covered = new Set();
+    const edge = (from, to, kind, source) => {
+        const key = JSON.stringify([from, to, kind, source.uri, source.startLine]);
+        if (!keys.has(key)) {
+            keys.add(key);
+            snapshot.relations.push({ from, to, kind, source, method: "explicit" });
+        }
+    };
+    const diagnostic = (path, code, message) => {
+        snapshot.diagnostics.push({ uri: path, code, message });
+    };
+    const line = (file, offset) => {
+        let low = 0;
+        let high = file.lines.length;
+        while (low < high) {
+            const middle = (low + high) >>> 1;
+            if (file.lines[middle] <= offset) {
+                low = middle + 1;
+            }
+            else {
+                high = middle;
+            }
+        }
+        return low;
+    };
+    const source = (file, start, end) => ({
+        ...file.file.source,
+        startLine: line(file, start),
+        endLine: line(file, Math.max(start, end - 1)),
+        locator: { kind: "part", part: file.path, element: `chars:${start}:${end}` },
+    });
+    inputs
+        .filter((input) => /\.(?:tex|ltx|sty|cls|bib)$/i.test(input.path))
+        .forEach((input) => {
+        const file = {
+            path: input.path,
+            text: input.content,
+            tokens: [],
+            blocks: [],
+            lines: [0, ...[...input.content.matchAll(/\n/g)].map((match) => match.index + 1)],
+            file: entities.get(`file:${input.path}`),
+        };
+        try {
+            if (Buffer.byteLength(file.text) > 16 * 1024 * 1024) {
+                throw new Error("TeX source exceeds 16 MiB");
+            }
+            if (/\.bib$/i.test(file.path)) {
+                file.blocks = bibBlocks(file.text);
+            }
+            else {
+                file.tokens = tokens(file.text);
+                const sections = [];
+                const environments = [];
+                file.tokens.forEach((token) => {
+                    if (Object.hasOwn(Levels, token.name) && token.value) {
+                        const level = Levels[token.name];
+                        while (sections.length > 0 && sections.at(-1).level >= level) {
+                            sections.pop().end = token.start;
+                        }
+                        const block = { start: token.start, end: file.text.length, label: token.value, type: "section", tag: token.name, level };
+                        sections.push(block);
+                        file.blocks.push(block);
+                    }
+                    else if (token.name === "begin" && token.value && token.value !== "document") {
+                        const block = { start: token.start, end: file.text.length, label: token.value, type: "environment", tag: token.value };
+                        environments.push(block);
+                        file.blocks.push(block);
+                    }
+                    else if (token.name === "end" && token.value !== "document") {
+                        if (environments.at(-1)?.tag === token.value) {
+                            environments.pop().end = token.end;
+                        }
+                        else {
+                            diagnostic(file.path, "tex_environment_mismatch", `Unmatched \\end{${token.value}} at line ${line(file, token.start)}`);
+                        }
+                    }
+                });
+                environments.forEach((block) => diagnostic(file.path, "tex_environment_mismatch", `Unclosed ${block.tag} at line ${line(file, block.start)}`));
+                const list = [];
+                file.tokens
+                    .filter((token) => Dependencies.has(token.name))
+                    .forEach((token) => {
+                    const values = ["bibliography", "usepackage", "RequirePackage"].includes(token.name)
+                        ? token.value.split(",")
+                        : [token.value];
+                    values.forEach((value) => {
+                        list.push({ path: file.path, token, value: value.trim() });
+                    });
+                });
+                requests.set(file.path, list);
+            }
+        }
+        catch (error) {
+            diagnostic(file.path, "invalid_tex", error instanceof Error ? error.message : "TeX parsing failed");
+            return;
+        }
+        file.blocks.sort((left, right) => left.start - right.start || right.end - left.end);
+        const active = [];
+        const children = new Map();
+        file.blocks.forEach((block) => {
+            while (active.length > 0 && (active.at(-1).end < block.end || active.at(-1).end <= block.start)) {
+                active.pop();
+            }
+            const parentId = active.at(-1)?.node?.id ?? file.file.id;
+            block.parent = active.at(-1);
+            const body = file.text.slice(block.start, block.end);
+            const node = {
+                id: `knowledge:${file.path}#tex:${block.start}`,
+                kind: "knowledge",
+                projectId: file.file.projectId,
+                parentId,
+                label: block.label,
+                summary: short(`${block.tag} ${block.label}: ${body}`),
+                source: source(file, block.start, block.end),
+                structureKind: block.type,
+                tags: ["latex", block.tag],
+                searchTerms: workspaceTerms(body),
+                detailRef: store.put(encodeDocumentDetail({ kind: "text", text: body })),
+                summaryProvenance: { method: "extractive-v1", sources: [{ uri: file.path, revision: file.file.source.revision }], children: [] },
+            };
+            block.node = node;
+            entities.set(node.id, node);
+            const siblings = children.get(parentId) ?? [];
+            siblings.push(node);
+            children.set(parentId, siblings);
+            active.push(block);
+        });
+        file.blocks.forEach((block) => {
+            const node = block.node;
+            const nested = children.get(node.id) ?? [];
+            node.summaryProvenance.children = nested.map((child) => child.id);
+            node.summary = short(`${block.tag} ${block.label}; ${nested.map((child) => child.label).join("; ")} ${file.text.slice(block.start, block.end)}`);
+        });
+        file.file.structureKind = "document";
+        file.file.tags = [...(file.file.tags ?? []), /\.bib$/i.test(file.path) ? "bibtex" : "latex"];
+        file.file.summary = short(`${file.path}: ${file.blocks.map((block) => block.label).join("; ")} ${file.text}`);
+        file.file.searchTerms = workspaceTerms(file.text);
+        file.file.summaryProvenance = {
+            method: "extractive-v1",
+            sources: [{ uri: file.path, revision: file.file.source.revision }],
+            children: (children.get(file.file.id) ?? []).map((child) => child.id),
+        };
+        parsed.set(file.path, file);
+        if (file.tokens.some((token) => token.name === "documentclass" || (token.name === "begin" && token.value === "document"))) {
+            roots.add(file.path);
+        }
+        diagnostic(file.path, "tex_static_partial", "Literal structure only: no macro expansion, catcode/conditional evaluation, compilation, TeX search paths or BibTeX string expansion");
+    });
+    const owner = (file, offset) => {
+        let low = 0;
+        let high = file.blocks.length;
+        while (low < high) {
+            const middle = (low + high) >>> 1;
+            if (file.blocks[middle].start <= offset) {
+                low = middle + 1;
+            }
+            else {
+                high = middle;
+            }
+        }
+        let block = low > 0 ? file.blocks[low - 1] : (void 0);
+        while (block && block.end <= offset) {
+            block = block.parent;
+        }
+        return block?.node?.id ?? file.file.id;
+    };
+    const resolve = (request, root) => {
+        const value = request.value;
+        if (!value || /[\\{}#$\x00-\x1f]/.test(value) || posix.isAbsolute(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
+            return (void 0);
+        }
+        const command = request.token.name;
+        const extensions = ["bibliography", "addbibresource"].includes(command)
+            ? [".bib"]
+            : command === "includegraphics"
+                ? [".pdf", ".png", ".jpg", ".jpeg", ".eps", ".svg"]
+                : ["usepackage", "RequirePackage"].includes(command)
+                    ? [".sty"]
+                    : ["documentclass", "LoadClass"].includes(command)
+                        ? [".cls"]
+                        : [".tex", ".ltx"];
+        const names = posix.extname(value) ? [value] : extensions.map((extension) => `${value}${extension}`);
+        const bases = [...new Set([posix.dirname(root), posix.dirname(request.path)])];
+        for (let baseIndex = 0; baseIndex < bases.length; baseIndex += 1) {
+            const base = bases[baseIndex];
+            for (let index = 0; index < names.length; index += 1) {
+                const path = posix.normalize(posix.join(base, names[index]));
+                if (!path.startsWith("../") && entities.has(`file:${path}`)) {
+                    return path;
+                }
+            }
+        }
+        return (void 0);
+    };
+    parsed.forEach((file) => {
+        const hint = /^\s*%\s*!\s*TeX\s+root\s*=\s*(.+?)\s*$/im.exec(file.text)?.[1];
+        if (hint) {
+            const path = posix.normalize(posix.join(posix.dirname(file.path), hint));
+            if (parsed.has(path)) {
+                roots.add(path);
+            }
+            else {
+                diagnostic(file.path, "tex_unresolved_root", hint);
+            }
+        }
+    });
+    const visit = (root, project) => {
+        const pending = [root];
+        const seen = new Set();
+        const dependencies = [];
+        const projectId = `knowledge:${root}#latex-project`;
+        while (pending.length > 0) {
+            const path = pending.pop();
+            if (seen.has(path)) {
+                continue;
+            }
+            seen.add(path);
+            covered.add(path);
+            const file = parsed.get(path);
+            if (!file) {
+                continue;
+            }
+            (requests.get(path) ?? []).forEach((request) => {
+                const target = resolve(request, root);
+                const origin = source(file, request.token.start, request.token.end);
+                if (target) {
+                    dependencies.push({ from: path, to: target, command: request.token.name });
+                    edge(file.file.id, `file:${target}`, "imports", origin);
+                    edge(owner(file, request.token.start), `file:${target}`, "imports", origin);
+                    if (!seen.has(target)) {
+                        pending.push(target);
+                    }
+                }
+                else if (["documentclass", "usepackage", "RequirePackage", "LoadClass"].includes(request.token.name) && /^[\w.-]+$/.test(request.value)) {
+                    const id = `package:${JSON.stringify(["latex", file.file.projectId, request.token.name, request.value])}`;
+                    if (!entities.has(id)) {
+                        entities.set(id, {
+                            id,
+                            kind: "package",
+                            projectId: file.file.projectId,
+                            parentId: file.file.projectId,
+                            label: request.value,
+                            summary: `Declared TeX ${request.token.name}: ${request.value}; installation/version unverified`,
+                            source: origin,
+                            tags: ["latex", request.token.name],
+                        });
+                    }
+                    edge(file.file.id, id, "imports", origin);
+                }
+                else {
+                    diagnostic(path, "tex_unresolved_dependency", `${request.token.name}{${request.value}} from root ${root}`);
+                }
+            });
+        }
+        const labels = new Map();
+        const citations = new Map();
+        const addKey = (map, key, id) => {
+            const ids = map.get(key) ?? [];
+            ids.push(id);
+            map.set(key, ids);
+        };
+        seen.forEach((path) => {
+            const file = parsed.get(path);
+            file?.tokens
+                .filter((token) => token.name === "label")
+                .forEach((token) => addKey(labels, token.value, owner(file, token.start)));
+            file?.blocks.filter((block) => Boolean(block.key)).forEach((block) => addKey(citations, block.key, block.node.id));
+        });
+        seen.forEach((path) => {
+            const file = parsed.get(path);
+            file?.tokens.forEach((token) => {
+                const cite = /^(?:[A-Za-z]*cite[A-Za-z]*|nocite)$/.test(token.name);
+                if (!cite && !References.has(token.name)) {
+                    return;
+                }
+                token.value
+                    .split(",")
+                    .map((value) => value.trim())
+                    .forEach((key) => {
+                    if (token.name === "nocite" && key === "*") {
+                        citations.forEach((ids) => ids.forEach((id) => edge(owner(file, token.start), id, "references", source(file, token.start, token.end))));
+                        return;
+                    }
+                    const targets = (cite ? citations : labels).get(key) ?? [];
+                    if (targets.length === 1) {
+                        edge(owner(file, token.start), targets[0], "references", source(file, token.start, token.end));
+                    }
+                    else {
+                        diagnostic(path, targets.length > 1 ? "tex_ambiguous_reference" : "tex_unresolved_reference", `${token.name}{${key}} in ${root}`);
+                    }
+                });
+            });
+        });
+        if (!project) {
+            return;
+        }
+        const file = parsed.get(root);
+        const paths = [...seen].sort();
+        const names = paths
+            .flatMap((path) => parsed
+            .get(path)
+            ?.blocks.filter((block) => block.type === "section")
+            .map((block) => block.label) ?? [])
+            .join("; ");
+        entities.set(projectId, {
+            id: projectId,
+            kind: "knowledge",
+            projectId: file.file.projectId,
+            parentId: file.file.id,
+            label: `LaTeX project: ${root}`,
+            summary: short(`LaTeX project ${root}, ${paths.length} supplied files: ${names}`),
+            structureKind: "document",
+            tags: ["latex", "latex-project"],
+            source: { ...file.file.source },
+            searchTerms: workspaceTerms(`${root} ${paths.join(" ")} ${names}`),
+            detailRef: store.put(encodeDocumentDetail({ kind: "tex-project", root, files: paths, dependencies })),
+            summaryProvenance: {
+                method: "extractive-v1",
+                sources: paths.map((path) => ({ uri: path, revision: entities.get(`file:${path}`).source.revision })),
+                children: [],
+            },
+        });
+        paths.forEach((path) => edge(projectId, `file:${path}`, "documents", file.file.source));
+    };
+    [...roots]
+        .sort()
+        .filter((path) => parsed.has(path))
+        .forEach((path) => visit(path, true));
+    [...parsed.keys()]
+        .sort()
+        .filter((path) => /\.(?:tex|ltx)$/i.test(path) && !covered.has(path))
+        .forEach((path) => visit(path, false));
+    snapshot.entities = [...entities.values()];
+};

package/lib/scan/workspaceScanner.d.ts

@@ -0,0 +1,9 @@
+import type { ContentStore } from "../context/contentStore.js";
+import type { WorkspaceSnapshot } from "../context/workspace.js";
+export interface WorkspaceFile {
+    path: string;
+    content: string;
+}
+type WorkspaceScanner = (inputs: readonly WorkspaceFile[], store: ContentStore) => WorkspaceSnapshot;
+export declare const scanWorkspace: WorkspaceScanner;
+export {};