attention-manifest-protocol 0.6.1 → 0.6.2

package/CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
 project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.2] - 2026-11-03
+
+### Added
+
+- Publish the existing offline workspace, Office/document and TeX scanners through `attention-manifest-protocol/scan`. Inputs remain explicit file contents; the scanners never crawl the filesystem or execute project code.
+- Keep `tools/support` entrypoints as compatibility re-exports. TypeScript, fflate and fast-xml-parser are runtime dependencies of the published scanners.
+
+### Validation
+
+- 69 workspace, document, TeX, agent and read/write boundary regression tests pass against the relocated implementations.
+
 ## [0.6.1] - 2026-11-03
 
 ### Added

package/CHANGELOG.zh-CN.md

@@ -1,5 +1,11 @@
 # 更新日志
 
+## [0.6.2] - 2026-11-03
+
+- 将既有工作区、Office 文档与 TeX 离线扫描器发布为 `attention-manifest-protocol/scan`；输入仍为显式文件内容，不扫描宿主目录、不执行项目代码。
+- 保留 `tools/support` 兼容导出；扫描器使用的 TypeScript、fflate、fast-xml-parser 随 npm 安装。
+- 工作区、文档、TeX、agent 与读写边界的 69 项回归检查通过。
+
 本文件记录本项目的所有重要变更。
 
 格式遵循 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/)，版本号遵循

package/lib/scan/documentScanner.d.ts

@@ -0,0 +1,13 @@
+import type { ContentStore } from "../context/contentStore.js";
+import type { WorkspaceSnapshot } from "../context/workspace.js";
+export interface StructuredWorkspaceFile {
+    path: string;
+    content: string | Uint8Array;
+}
+type StructuredScanner = (inputs: readonly StructuredWorkspaceFile[], store: ContentStore) => WorkspaceSnapshot;
+/**
+ * Sources are canonical relative paths. Code/Markdown retain their existing
+ * adapters; modern Office documents add native structural knowledge nodes.
+ */
+export declare const scanStructuredWorkspace: StructuredScanner;
+export {};

package/lib/scan/documentScanner.js

@@ -0,0 +1,452 @@
+/**
+ * Offline structural adapters. OOXML archives are read in memory, never
+ * extracted to disk. Original packages remain available as scoped details.
+ */
+import { createHash } from "node:crypto";
+import { posix } from "node:path";
+import { strFromU8, unzipSync } from "fflate";
+import { XMLBuilder, XMLParser, XMLValidator } from "fast-xml-parser";
+import { encodeDocumentDetail } from "../context/document.js";
+import { workspaceTerms } from "../context/workspace.js";
+import { scanWorkspace } from "./workspaceScanner.js";
+import { addTexStructure } from "./texScanner.js";
+const MaxInputBytes = 16 * 1024 * 1024;
+const MaxExpandedBytes = 32 * 1024 * 1024;
+const Parser = new XMLParser({ preserveOrder: true, ignoreAttributes: false, parseTagValue: false, parseAttributeValue: false, trimValues: false });
+const Builder = new XMLBuilder({ preserveOrder: true, ignoreAttributes: false });
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+const short = (text) => text.replace(/\s+/g, " ").trim().slice(0, 180);
+const cellRange = (cells) => {
+    if (cells.length === 0) {
+        return "A1";
+    }
+    const addresses = cells.map((cell) => {
+        const match = /^([A-Z]+)(\d+)$/.exec(cell.address);
+        return {
+            column: match[1],
+            rank: [...match[1]].reduce((sum, char) => sum * 26 + char.charCodeAt(0) - 64, 0),
+            row: Number(match[2]),
+        };
+    });
+    addresses.sort((left, right) => left.rank - right.rank);
+    const rows = addresses.map((address) => address.row);
+    const first = `${addresses[0].column}${rows.reduce((minimum, row) => Math.min(minimum, row), Infinity)}`;
+    const last = `${addresses.at(-1).column}${rows.reduce((maximum, row) => Math.max(maximum, row), 1)}`;
+    return first === last ? first : `${first}:${last}`;
+};
+const local = (name) => name.split(":").at(-1);
+const attr = (node, name) => node === (void 0) ? "" : (node.attrs[name] ?? Object.entries(node.attrs).find(([key]) => local(key) === name)?.[1] ?? "");
+const relationshipId = (node) => node.attrs["r:id"] ?? Object.entries(node.attrs).find(([key]) => key.includes(":") && local(key) === "id")?.[1] ?? "";
+const children = (node, name) => node.children.filter((child) => local(child.name) === name);
+const descendants = (node, name) => node.children.flatMap((child) => [...(local(child.name) === name ? [child] : []), ...descendants(child, name)]);
+const text = (node) => node.text + node.children.map(text).join("");
+const paragraphs = (node) => descendants(node, "p")
+    .map((paragraph) => descendants(paragraph, "t").map(text).join(""))
+    .join("\n");
+const xml = (node) => Builder.build([node.raw]);
+const elements = (items) => items.flatMap((item) => {
+    const name = Object.keys(item).find((key) => key !== ":@" && key !== "#text" && !key.startsWith("?"));
+    if (name === (void 0)) {
+        return [];
+    }
+    const content = item[name];
+    return [
+        {
+            name,
+            attrs: Object.fromEntries(Object.entries((item[":@"] ?? {})).map(([key, value]) => [
+                key.slice(2),
+                value,
+            ])),
+            children: elements(content),
+            text: content
+                .filter((child) => "#text" in child)
+                .map((child) => `${child["#text"]}`)
+                .join(""),
+            raw: item,
+        },
+    ];
+});
+const parseXml = (content) => {
+    if (/<!DOCTYPE|<!ENTITY/i.test(content)) {
+        throw new Error("DTD/entity declarations are unsupported");
+    }
+    let depth = 0;
+    const tags = [...content.matchAll(/<[^>]+>/g)];
+    for (let tagIndex = 0; tagIndex < tags.length; tagIndex += 1) {
+        const match = tags[tagIndex];
+        if (/^<\//.test(match[0])) {
+            depth -= 1;
+        }
+        else if (!/^<[!?]/.test(match[0]) && !/\/>$/.test(match[0])) {
+            depth += 1;
+        }
+        if (depth > 128) {
+            throw new Error("XML nesting exceeds 128");
+        }
+    }
+    if (XMLValidator.validate(content) !== true) {
+        throw new Error("malformed XML");
+    }
+    const roots = elements(Parser.parse(content));
+    if (roots.length !== 1) {
+        throw new Error("XML requires one root");
+    }
+    return roots[0];
+};
+class Package {
+    parts;
+    parsed = new Map();
+    constructor(bytes) {
+        let total = 0;
+        const names = new Set();
+        this.parts = unzipSync(bytes, {
+            filter: (entry) => {
+                if (names.has(entry.name) ||
+                    names.size >= 4096 ||
+                    entry.name.startsWith("/") ||
+                    entry.name.includes("\\") ||
+                    posix.normalize(entry.name) !== entry.name ||
+                    entry.name.startsWith("../")) {
+                    throw new Error("invalid, duplicate or excessive ZIP entries");
+                }
+                names.add(entry.name);
+                total += entry.originalSize;
+                if (entry.originalSize > 8 * 1024 * 1024 || total > MaxExpandedBytes) {
+                    throw new Error("OOXML expansion limit exceeded");
+                }
+                return /\.(?:xml|rels)$/.test(entry.name);
+            },
+        });
+    }
+    has(part) {
+        return Object.hasOwn(this.parts, part);
+    }
+    read(part) {
+        const cached = this.parsed.get(part);
+        if (cached !== (void 0)) {
+            return cached;
+        }
+        if (!this.has(part)) {
+            throw new Error(`missing OOXML part ${part}`);
+        }
+        const root = parseXml(strFromU8(this.parts[part]));
+        this.parsed.set(part, root);
+        return root;
+    }
+    relationships(part) {
+        const rels = posix.join(posix.dirname(part), "_rels", `${posix.basename(part)}.rels`);
+        if (!this.has(rels)) {
+            return [];
+        }
+        return children(this.read(rels), "Relationship").map((node) => ({
+            id: attr(node, "Id"),
+            type: attr(node, "Type").split("/").at(-1),
+            target: attr(node, "TargetMode") === "External"
+                ? attr(node, "Target")
+                : posix.normalize(attr(node, "Target").startsWith("/") ? attr(node, "Target").slice(1) : posix.join(posix.dirname(part), attr(node, "Target"))),
+            external: attr(node, "TargetMode") === "External",
+        }));
+    }
+    target(part, id) {
+        const relation = this.relationships(part).find((item) => item.id === id);
+        if (!relation || relation.external) {
+            throw new Error(`missing or external OOXML relationship ${id}`);
+        }
+        return relation.target;
+    }
+}
+/**
+ * Sources are canonical relative paths. Code/Markdown retain their existing
+ * adapters; modern Office documents add native structural knowledge nodes.
+ */
+export const scanStructuredWorkspace = (inputs, store) => {
+    const normalized = inputs.map((file) => ({
+        ...file,
+        path: file.path.replace(/\\/g, "/"),
+    }));
+    const isText = (path) => /\.(?:md|txt|[cm]?[jt]sx?|json)$/i.test(path);
+    const snapshot = scanWorkspace(normalized.map((file) => ({
+        path: file.path,
+        content: isText(file.path) && !/\.txt$/i.test(file.path)
+            ? typeof (file.content) === "string"
+                ? file.content
+                : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(file.content)
+            : "",
+    })), store);
+    const revisions = new Map(normalized.map((file) => [file.path, hash(file.content)]));
+    snapshot.revision = hash(JSON.stringify([...revisions].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))));
+    snapshot.entities
+        .filter((entity) => entity.kind === "directory" || (entity.kind === "project" && entity.source.uri === "."))
+        .forEach((entity) => {
+        entity.source.revision = snapshot.revision;
+    });
+    const entities = new Map(snapshot.entities.map((entity) => [entity.id, entity]));
+    const texInputs = [];
+    for (let inputIndex = 0; inputIndex < normalized.length; inputIndex += 1) {
+        const input = normalized[inputIndex];
+        const path = input.path;
+        if (isText(path) && !/\.txt$/i.test(path)) {
+            continue;
+        }
+        const file = entities.get(`file:${path}`);
+        const revision = revisions.get(path);
+        file.source.revision = revision;
+        const bytes = typeof (input.content) === "string" ? new TextEncoder().encode(input.content) : input.content;
+        if (/\.(?:tex|ltx|sty|cls|bib)$/i.test(path)) {
+            try {
+                if (bytes.byteLength > MaxInputBytes) {
+                    throw new Error("TeX source exceeds 16 MiB");
+                }
+                const content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+                file.detailRef = store.put(content);
+                texInputs.push({ path, content });
+            }
+            catch (error) {
+                delete file.detailRef;
+                snapshot.diagnostics.push({ uri: path, code: "invalid_tex", message: error instanceof Error ? error.message : "TeX decoding failed" });
+            }
+            continue;
+        }
+        if (/\.(?:png|jpe?g|eps|svg|webp)$/i.test(path)) {
+            delete file.detailRef;
+            file.tags = ["document-asset"];
+            continue;
+        }
+        const added = [];
+        const source = (locator) => ({ uri: path, revision, locator });
+        const add = (key, parentId, label, locator, detail, body, kind) => {
+            const entity = {
+                id: `knowledge:${path}#${key}`,
+                kind: "knowledge",
+                projectId: file.projectId,
+                parentId,
+                label,
+                summary: short(`${label}: ${body}`),
+                source: source(locator),
+                structureKind: kind,
+                detailRef: store.put(encodeDocumentDetail(detail)),
+                searchTerms: workspaceTerms(body),
+                summaryProvenance: { method: "extractive-v1", sources: [{ uri: path, revision }], children: [] },
+            };
+            added.push(entity);
+            return entity;
+        };
+        try {
+            if (bytes.byteLength > MaxInputBytes) {
+                throw new Error("document exceeds 16 MiB input limit");
+            }
+            if (/\.txt$/i.test(path)) {
+                const body = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+                add("text", file.id, path, { kind: "part", part: path }, { kind: "text", text: body }, body, "document");
+                file.detailRef = store.put(body);
+            }
+            else if (/\.(?:docx|xlsx|pptx)$/i.test(path)) {
+                const pkg = new Package(bytes);
+                if (/\.docx$/i.test(path)) {
+                    const part = "word/document.xml";
+                    const root = pkg.read(part);
+                    const body = children(root, "body")[0];
+                    if (!body) {
+                        throw new Error("Word document body missing");
+                    }
+                    const sections = [];
+                    let ordinal = 0;
+                    for (let blockIndex = 0; blockIndex < body.children.length; blockIndex += 1) {
+                        const block = body.children[blockIndex];
+                        ordinal += 1;
+                        const name = local(block.name);
+                        if (!["p", "tbl"].includes(name)) {
+                            continue;
+                        }
+                        const locator = { kind: "part", part, element: `body/*[${ordinal}]` };
+                        if (name === "tbl") {
+                            const rows = children(block, "tr").map((row) => children(row, "tc").map(paragraphs));
+                            add(`table${ordinal}`, sections.at(-1)?.id ?? file.id, `Table ${ordinal}`, locator, { kind: "table", rows, xml: xml(block) }, rows.flat().join(" "), "table");
+                        }
+                        else {
+                            const style = attr(descendants(block, "pStyle")[0], "val");
+                            const outline = attr(descendants(block, "outlineLvl")[0], "val");
+                            const level = outline && Number(outline) < 9 ? Number(outline) + 1 : Number(/^Heading([1-9])$/i.exec(style)?.[1] ?? 0);
+                            const bodyText = descendants(block, "t").map(text).join("");
+                            if (level > 0) {
+                                while (sections.length > 0 && sections.at(-1).level >= level) {
+                                    sections.pop();
+                                }
+                            }
+                            const node = add(`p${ordinal}`, sections.at(-1)?.id ?? file.id, level ? bodyText : `Paragraph ${ordinal}`, locator, { kind: "text", text: bodyText, xml: xml(block) }, bodyText, level ? "section" : "paragraph");
+                            if (level > 0) {
+                                sections.push({ id: node.id, level });
+                            }
+                        }
+                    }
+                    snapshot.diagnostics.push({
+                        uri: path,
+                        code: "docx_structure_partial",
+                        message: "Main body, standard Heading styles and direct outline levels indexed; custom styles, headers, notes, revisions and images remain in original package",
+                    });
+                }
+                else if (/\.xlsx$/i.test(path)) {
+                    const part = "xl/workbook.xml";
+                    const root = pkg.read(part);
+                    const shared = pkg.has("xl/sharedStrings.xml")
+                        ? children(pkg.read("xl/sharedStrings.xml"), "si").map((item) => descendants(item, "t").map(text).join(""))
+                        : [];
+                    const sheets = [];
+                    const sheetElements = descendants(root, "sheet");
+                    for (let sheetIndex = 0; sheetIndex < sheetElements.length; sheetIndex += 1) {
+                        const sheet = sheetElements[sheetIndex];
+                        const name = attr(sheet, "name");
+                        const sheetPart = pkg.target(part, relationshipId(sheet));
+                        const sheetXml = pkg.read(sheetPart);
+                        const cells = descendants(sheetXml, "c").map((cell) => {
+                            const formula = children(cell, "f")[0];
+                            const value = children(cell, "v")[0];
+                            const type = attr(cell, "t") || "n";
+                            const address = attr(cell, "r");
+                            if (!/^[A-Z]+[1-9]\d*$/.test(address)) {
+                                throw new Error("sheet cell requires an A1 address");
+                            }
+                            return {
+                                address,
+                                type,
+                                ...(value ? { value: type === "s" ? (shared[Number(text(value))] ?? text(value)) : text(value) } : {}),
+                                ...(type === "inlineStr" ? { value: descendants(cell, "t").map(text).join("") } : {}),
+                                ...(formula ? { formula: text(formula), formulaAttributes: { ...formula.attrs } } : {}),
+                                ...(attr(cell, "s") ? { style: attr(cell, "s") } : {}),
+                            };
+                        });
+                        const range = cellRange(cells);
+                        const node = add(`sheet:${attr(sheet, "sheetId")}`, file.id, name, { kind: "cells", part: sheetPart, sheet: name, range }, {
+                            kind: "sheet",
+                            name,
+                            cells,
+                            merges: descendants(sheetXml, "mergeCell").map((item) => attr(item, "ref")),
+                            xml: xml(sheetXml),
+                        }, cells.map((cell) => `${cell.address} ${cell.value ?? ""} ${cell.formula ?? ""}`).join("\n"), "sheet");
+                        sheets.push({ name, node, cells, part: sheetPart });
+                    }
+                    for (let sheetIndex = 0; sheetIndex < sheets.length; sheetIndex += 1) {
+                        const sheet = sheets[sheetIndex];
+                        for (let cellIndex = 0; cellIndex < sheet.cells.length; cellIndex += 1) {
+                            const cell = sheet.cells[cellIndex];
+                            const references = [
+                                ...(cell.formula ?? "")
+                                    .replace(/"(?:[^"]|"")*"/g, "")
+                                    .matchAll(/(?<![\p{L}\p{N}_.\]])(?:'((?:[^']|'')+)'|([\p{L}_][\p{L}\p{N}_.]*))!\$?[A-Z]+\$?[1-9]\d*/gu),
+                            ];
+                            for (let referenceIndex = 0; referenceIndex < references.length; referenceIndex += 1) {
+                                const match = references[referenceIndex];
+                                const targetName = (match[1] ?? match[2]).replace(/''/g, "'");
+                                const target = sheets.find((item) => item.name.toLowerCase() === targetName.toLowerCase())?.node;
+                                if (target) {
+                                    snapshot.relations.push({
+                                        from: sheet.node.id,
+                                        to: target.id,
+                                        kind: "references",
+                                        method: "explicit",
+                                        source: source({ kind: "cells", part: sheet.part, sheet: sheet.name, range: cell.address }),
+                                    });
+                                }
+                            }
+                        }
+                    }
+                    snapshot.diagnostics.push({
+                        uri: path,
+                        code: "formula_graph_partial",
+                        message: "No recalculation: values are stored caches; graph covers literal cross-sheet A1 references only; named, dynamic, external and shared-formula expansion remain unanalysed. Styles/charts/table definitions remain in original package",
+                    });
+                }
+                else {
+                    const part = "ppt/presentation.xml";
+                    let ordinal = 0;
+                    const slideElements = descendants(pkg.read(part), "sldId");
+                    for (let slideIndex = 0; slideIndex < slideElements.length; slideIndex += 1) {
+                        const slide = slideElements[slideIndex];
+                        ordinal += 1;
+                        const slidePart = pkg.target(part, relationshipId(slide));
+                        const root = pkg.read(slidePart);
+                        const objects = [];
+                        const collect = (element) => {
+                            for (let childIndex = 0; childIndex < element.children.length; childIndex += 1) {
+                                const child = element.children[childIndex];
+                                if (["sp", "pic", "graphicFrame", "cxnSp"].includes(local(child.name))) {
+                                    const properties = descendants(child, "cNvPr")[0];
+                                    objects.push({
+                                        id: attr(properties, "id"),
+                                        kind: local(child.name),
+                                        text: paragraphs(child),
+                                        xml: xml(child),
+                                        locator: { kind: "slide", part: slidePart, slide: ordinal, objectId: attr(properties, "id") },
+                                    });
+                                }
+                                else {
+                                    collect(child);
+                                }
+                            }
+                        };
+                        collect(root);
+                        const note = pkg
+                            .relationships(slidePart)
+                            .find((item) => item.type === "notesSlide" && !item.external);
+                        const notes = note ? paragraphs(pkg.read(note.target)) : "";
+                        add(`slide${ordinal}`, file.id, `Slide ${ordinal}`, { kind: "slide", part: slidePart, slide: ordinal }, { kind: "slide", objects, notes, xml: xml(root) }, `${objects.map((object) => object.text).join("\n")}\n${notes}`, "slide");
+                    }
+                    snapshot.diagnostics.push({
+                        uri: path,
+                        code: "pptx_structure_partial",
+                        message: "Slide order, objects, native layout XML and speaker notes retained; charts/images/themes remain in original package; no OCR or layout inference",
+                    });
+                }
+                add("original", file.id, "Original package", { kind: "part", part: "/" }, {
+                    kind: "original",
+                    mediaType: `application/vnd.openxmlformats-officedocument.${/\.docx$/i.test(path) ? "wordprocessingml.document" : /\.xlsx$/i.test(path) ? "spreadsheetml.sheet" : "presentationml.presentation"}`,
+                    encoding: "base64",
+                    data: Buffer.from(bytes).toString("base64"),
+                }, "Original Office package; allocate this node to retrieve complete binary contents", "document");
+                delete file.detailRef;
+            }
+            else {
+                snapshot.diagnostics.push({
+                    uri: path,
+                    code: "unsupported_document",
+                    message: "Supported document formats: Markdown, UTF-8 TXT, TeX/LaTeX, BibTeX, DOCX, XLSX, PPTX. Convert PDF/EPUB/legacy Office formats before ingestion",
+                });
+                delete file.detailRef;
+                continue;
+            }
+            file.structureKind = "document";
+            const childrenByParent = new Map();
+            added.forEach((entity) => {
+                const siblings = childrenByParent.get(entity.parentId) ?? [];
+                siblings.push(entity);
+                childrenByParent.set(entity.parentId, siblings);
+            });
+            for (let entityIndex = added.length - 1; entityIndex >= 0; entityIndex -= 1) {
+                const entity = added[entityIndex];
+                const children = childrenByParent.get(entity.id) ?? [];
+                if (children.length > 0) {
+                    entity.summary = short(`${entity.summary} ${children.map((child) => child.summary).join("; ")}`);
+                }
+            }
+            for (let entityIndex = 0; entityIndex < added.length; entityIndex += 1) {
+                const entity = added[entityIndex];
+                entity.summaryProvenance.children = (childrenByParent.get(entity.id) ?? []).map((child) => child.id);
+                entities.set(entity.id, entity);
+            }
+            file.summary = short(`${path}: ${(childrenByParent.get(file.id) ?? []).map((entity) => entity.summary).join("; ")}`);
+            file.summaryProvenance = {
+                method: "extractive-v1",
+                sources: [{ uri: path, revision }],
+                children: (childrenByParent.get(file.id) ?? []).map((entity) => entity.id),
+            };
+        }
+        catch (error) {
+            delete file.detailRef;
+            snapshot.relations = snapshot.relations.filter((edge) => !added.some((entity) => entity.id === edge.from || entity.id === edge.to));
+            snapshot.diagnostics.push({ uri: path, code: "invalid_document", message: error instanceof Error ? error.message : "document parsing failed" });
+        }
+    }
+    snapshot.entities = [...entities.values()];
+    addTexStructure(snapshot, texInputs, store);
+    return snapshot;
+};

package/lib/scan/index.d.ts

@@ -0,0 +1,4 @@
+/** Offline scanners over explicit inputs; no filesystem traversal or execution. */
+export * from "./workspaceScanner.js";
+export * from "./documentScanner.js";
+export * from "./texScanner.js";

package/lib/scan/index.js

@@ -0,0 +1,4 @@
+/** Offline scanners over explicit inputs; no filesystem traversal or execution. */
+export * from "./workspaceScanner.js";
+export * from "./documentScanner.js";
+export * from "./texScanner.js";

package/lib/scan/texScanner.d.ts

@@ -0,0 +1,10 @@
+import type { ContentStore } from "../context/contentStore.js";
+import type { WorkspaceSnapshot } from "../context/workspace.js";
+import type { WorkspaceFile } from "./workspaceScanner.js";
+type TexAdapter = (snapshot: WorkspaceSnapshot, inputs: readonly WorkspaceFile[], store: ContentStore) => void;
+/**
+ * Adds format facts to the shared snapshot. LaTeX build projects are knowledge
+ * views under their root files, not replacements for workspace ownership.
+ */
+export declare const addTexStructure: TexAdapter;
+export {};