attention-manifest-protocol 0.4.2 → 0.4.3

package/lib/syntax/syntaxRegistry.js

@@ -0,0 +1,314 @@
+const IdPattern = /^[a-z][A-Za-z0-9]*$/;
+/**
+ * A mutable registry with duplicate and shape checks. The default instance is
+ * populated once at module load; `createSyntaxRegistry()` returns an empty one
+ * for a host that wants a different catalog.
+ */
+export class SyntaxRegistry {
+    languageById = new Map();
+    languageByExtension = new Map();
+    tokenKinds = new Map();
+    nodeKinds = new Map();
+    /**
+     * Register one language. An empty id, a duplicate id, a duplicate
+     * extension, an invalid extension or an unknown family is rejected.
+     */
+    registerLanguage(definition) {
+        if (definition === null || typeof (definition) !== "object") {
+            throw new TypeError("language definition must be an object");
+        }
+        if (!IdPattern.test(definition.id) || definition.id.length > 32) {
+            throw new RangeError(`invalid syntax language id ${JSON.stringify(definition.id)}`);
+        }
+        if (this.languageById.has(definition.id)) {
+            throw new Error(`duplicate syntax language ${definition.id}`);
+        }
+        if (!["code", "python", "markup", "text", "json", "style"].includes(definition.family)) {
+            throw new RangeError(`unknown syntax family ${definition.family}`);
+        }
+        const extensions = [];
+        for (let index = 0; index < definition.extensions.length; index += 1) {
+            const extension = definition.extensions[index].toLowerCase().replace(/^\./, "");
+            if (!/^[a-z0-9]+$/.test(extension)) {
+                throw new RangeError(`invalid syntax extension ${definition.extensions[index]}`);
+            }
+            if (this.languageByExtension.has(extension) && this.languageByExtension.get(extension) !== definition.id) {
+                throw new Error(`duplicate syntax extension ${extension}`);
+            }
+            extensions.push(extension);
+        }
+        const stored = {
+            ...definition,
+            extensions: Object.freeze(extensions),
+            ...(definition.keywords === (void 0) ? {} : { keywords: Object.freeze([...definition.keywords]) }),
+        };
+        this.languageById.set(definition.id, Object.freeze(stored));
+        extensions.forEach((extension) => {
+            this.languageByExtension.set(extension, definition.id);
+        });
+        return this;
+    }
+    /**
+     * Register a token category.
+     */
+    registerTokenKind(definition) {
+        if (!IdPattern.test(definition.id) || definition.description.length === 0) {
+            throw new RangeError(`invalid syntax token kind ${JSON.stringify(definition.id)}`);
+        }
+        if (this.tokenKinds.has(definition.id)) {
+            throw new Error(`duplicate syntax token kind ${definition.id}`);
+        }
+        this.tokenKinds.set(definition.id, Object.freeze({ ...definition }));
+        return this;
+    }
+    /**
+     * Register a node category.
+     */
+    registerNodeKind(definition) {
+        if (!/^[A-Z][A-Za-z0-9]*$/.test(definition.id) || definition.id.length > 32) {
+            throw new RangeError(`invalid syntax node kind ${JSON.stringify(definition.id)}`);
+        }
+        if (!["container", "declaration", "token", "markup", "text"].includes(definition.category)) {
+            throw new RangeError(`invalid syntax node category ${definition.category}`);
+        }
+        if (this.nodeKinds.has(definition.id)) {
+            throw new Error(`duplicate syntax node kind ${definition.id}`);
+        }
+        this.nodeKinds.set(definition.id, Object.freeze({ ...definition }));
+        return this;
+    }
+    /**
+     * Look up a language by id.
+     */
+    language(id) {
+        return this.languageById.get(id);
+    }
+    /**
+     * Resolve a path's extension. Unknown extensions return `unknown`; a path
+     * without an extension returns `plainText`.
+     */
+    languageOf(path) {
+        const name = path.split("/").pop() ?? path;
+        const dot = name.lastIndexOf(".");
+        if (dot <= 0 || dot === name.length - 1) {
+            return "plainText";
+        }
+        return this.languageByExtension.get(name.slice(dot + 1).toLowerCase()) ?? "unknown";
+    }
+    /**
+     * Rule object consumed by the tokenizer.
+     */
+    ruleOf(language) {
+        const definition = this.languageById.get(language);
+        if (definition === (void 0)) {
+            return { family: "code", lineComment: "//", blockStart: "/*", blockEnd: "*/" };
+        }
+        return {
+            family: definition.family,
+            ...(definition.lineComment === (void 0) ? {} : { lineComment: definition.lineComment }),
+            ...(definition.blockStart === (void 0) ? {} : { blockStart: definition.blockStart }),
+            ...(definition.blockEnd === (void 0) ? {} : { blockEnd: definition.blockEnd }),
+        };
+    }
+    /**
+     * Keyword set for one language.
+     */
+    keywordsOf(language) {
+        return new Set(this.languageById.get(language)?.keywords ?? []);
+    }
+    /**
+     * All registered languages, in registration order.
+     */
+    languages() {
+        return [...this.languageById.values()];
+    }
+    /**
+     * All registered token kinds.
+     */
+    tokenKindsList() {
+        return [...this.tokenKinds.values()];
+    }
+    /**
+     * All registered node kinds.
+     */
+    nodeKindsList() {
+        return [...this.nodeKinds.values()];
+    }
+    /**
+     * Whether every id is registered.
+     */
+    hasTokenKind(id) {
+        return this.tokenKinds.has(id);
+    }
+    /**
+     * Whether every id is registered.
+     */
+    hasNodeKind(id) {
+        return this.nodeKinds.has(id);
+    }
+}
+const Keywords = [
+    "abstract",
+    "as",
+    "async",
+    "await",
+    "break",
+    "case",
+    "catch",
+    "class",
+    "const",
+    "continue",
+    "default",
+    "delete",
+    "do",
+    "else",
+    "enum",
+    "export",
+    "extends",
+    "final",
+    "finally",
+    "for",
+    "from",
+    "function",
+    "if",
+    "implements",
+    "import",
+    "in",
+    "instanceof",
+    "interface",
+    "let",
+    "module",
+    "namespace",
+    "new",
+    "package",
+    "private",
+    "protected",
+    "public",
+    "record",
+    "return",
+    "static",
+    "struct",
+    "switch",
+    "throw",
+    "trait",
+    "try",
+    "type",
+    "typeof",
+    "use",
+    "var",
+    "void",
+    "while",
+    "with",
+    "yield",
+    "def",
+    "fn",
+    "func",
+    "impl",
+    "mod",
+    "pub",
+    "match",
+    "mut",
+    "ref",
+    "where",
+    "loop",
+    "unsafe",
+    "extern",
+    "crate",
+    "self",
+    "super",
+    "defer",
+    "go",
+    "select",
+    "chan",
+    "map",
+    "range",
+    "fallthrough",
+    "goto",
+    "typedef",
+    "union",
+    "using",
+    "template",
+    "virtual",
+    "override",
+    "operator",
+    "friend",
+    "inline",
+    "volatile",
+    "signed",
+    "unsigned",
+    "short",
+    "long",
+    "float",
+    "double",
+    "char",
+    "bool",
+    "boolean",
+    "byte",
+    "string",
+    "String",
+];
+const registerBuiltins = (registry) => {
+    registry.registerTokenKind({ id: "keyword", description: "Language keyword" });
+    registry.registerTokenKind({ id: "identifier", description: "Identifier or name" });
+    registry.registerTokenKind({ id: "number", description: "Numeric literal" });
+    registry.registerTokenKind({ id: "string", description: "String literal, including markup text" });
+    registry.registerTokenKind({ id: "comment", description: "Line or block comment" });
+    registry.registerTokenKind({ id: "punctuation", description: "Delimiter or operator" });
+    registry.registerTokenKind({ id: "text", description: "Unstructured text token" });
+    registry.registerNodeKind({ id: "SourceFile", category: "container" });
+    registry.registerNodeKind({ id: "Block", category: "container" });
+    registry.registerNodeKind({ id: "Declaration", category: "declaration" });
+    registry.registerNodeKind({ id: "Class", category: "declaration" });
+    registry.registerNodeKind({ id: "Interface", category: "declaration" });
+    registry.registerNodeKind({ id: "Struct", category: "declaration" });
+    registry.registerNodeKind({ id: "Trait", category: "declaration" });
+    registry.registerNodeKind({ id: "Enum", category: "declaration" });
+    registry.registerNodeKind({ id: "Namespace", category: "declaration" });
+    registry.registerNodeKind({ id: "Function", category: "declaration" });
+    registry.registerNodeKind({ id: "Method", category: "declaration" });
+    registry.registerNodeKind({ id: "Property", category: "declaration" });
+    registry.registerNodeKind({ id: "Import", category: "declaration" });
+    registry.registerNodeKind({ id: "Export", category: "declaration" });
+    registry.registerNodeKind({ id: "Parameter", category: "token" });
+    registry.registerNodeKind({ id: "Tag", category: "markup" });
+    registry.registerNodeKind({ id: "Attribute", category: "markup" });
+    registry.registerNodeKind({ id: "Text", category: "text" });
+    registry.registerNodeKind({ id: "Heading", category: "text" });
+    registry.registerNodeKind({ id: "Rule", category: "token" });
+    registry.registerNodeKind({ id: "Comment", category: "token" });
+    const language = (id, family, extensions, rule = {}) => ({ id, family, extensions, keywords: Keywords, ...rule });
+    registry.registerLanguage(language("typescript", "code", ["ts", "tsx", "mts", "cts"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("javascript", "code", ["js", "jsx", "mjs", "cjs"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("python", "python", ["py", "pyi"], { lineComment: "#", blockStart: '"""', blockEnd: '"""' }));
+    registry.registerLanguage(language("java", "code", ["java"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("c", "code", ["c", "h"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("cpp", "code", ["cc", "cpp", "cxx", "hpp", "hh"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("csharp", "code", ["cs"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("go", "code", ["go"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("rust", "code", ["rs"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("vue", "markup", ["vue"], { lineComment: "//", blockStart: "<!--", blockEnd: "-->" }));
+    registry.registerLanguage(language("html", "markup", ["html", "htm"], { blockStart: "<!--", blockEnd: "-->" }));
+    registry.registerLanguage(language("xml", "markup", ["xml", "xsd", "xsl", "xslt", "svg", "csproj", "fsproj", "vbproj", "props", "targets"], { blockStart: "<!--", blockEnd: "-->" }));
+    registry.registerLanguage(language("css", "style", ["css"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("scss", "style", ["scss", "sass", "less"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("json", "json", ["json", "jsonc"], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+    registry.registerLanguage(language("markdown", "text", ["md", "mdx", "markdown"], {}));
+    registry.registerLanguage(language("tex", "text", ["tex", "ltx", "sty", "cls"], { lineComment: "%" }));
+    registry.registerLanguage(language("plainText", "text", ["txt", "text"], {}));
+    registry.registerLanguage(language("unknown", "code", [], { lineComment: "//", blockStart: "/*", blockEnd: "*/" }));
+};
+/**
+ * The process-wide catalog used by the parser. It is frozen after module load;
+ * a host that needs a different catalog builds one with
+ * `createSyntaxRegistry()` and registers it explicitly.
+ */
+export const DefaultSyntaxRegistry = new SyntaxRegistry();
+registerBuiltins(DefaultSyntaxRegistry);
+/**
+ * Create an empty registry for a host that wants to register a custom catalog.
+ */
+export const createSyntaxRegistry = () => new SyntaxRegistry();
+/**
+ * Convenience wrapper over the default catalog.
+ */
+export const syntaxLanguageOf = (path) => DefaultSyntaxRegistry.languageOf(path);

package/lib/syntax/syntaxTree.d.ts

@@ -0,0 +1,106 @@
+import type { Charset } from "./types.js";
+import type { SourceRange as OutlineRange } from "./types.js";
+import type { SyntaxLanguage, SyntaxNodeKind, SyntaxRegistry, SyntaxTokenKind } from "./syntaxRegistry.js";
+/**
+ * Re-export the registry types so a consumer can import everything syntax
+ * related from one module.
+ */
+export type { SyntaxLanguage, SyntaxNodeKind, SyntaxTokenKind } from "./syntaxRegistry.js";
+export { DefaultSyntaxRegistry, SyntaxRegistry, createSyntaxRegistry, syntaxLanguageOf } from "./syntaxRegistry.js";
+export type { SyntaxLanguageDefinition, SyntaxNodeKindDefinition, SyntaxTokenKindDefinition } from "./syntaxRegistry.js";
+export type { SyntaxLanguage as SyntaxLanguageId, SyntaxNodeKind as SyntaxNodeKindId, SyntaxTokenKind as SyntaxTokenKindId } from "./syntaxRegistry.js";
+/**
+ * One lexical token with an exact UTF-8 byte / 1-based line range.
+ */
+export interface SyntaxToken {
+    readonly kind: SyntaxTokenKind;
+    readonly text: string;
+    readonly range: OutlineRange;
+}
+/**
+ * One syntax node. `tokens` are the tokens owned directly by this node; the
+ * tree-level `tokens` array is the complete stream, so no token is lost when a
+ * declaration body is nested.
+ */
+export interface SyntaxNode {
+    readonly kind: SyntaxNodeKind;
+    readonly name?: string;
+    readonly detail?: string;
+    readonly range: OutlineRange;
+    readonly children: SyntaxNode[];
+    readonly tokens: SyntaxToken[];
+}
+/**
+ * Complete syntax tree for one file.
+ */
+export interface SyntaxTree {
+    readonly path: string;
+    readonly language: SyntaxLanguage;
+    readonly fingerprint: string;
+    readonly charset: Charset;
+    readonly root: SyntaxNode;
+    readonly tokens: SyntaxToken[];
+    readonly tokenCount: number;
+    readonly nodeCount: number;
+    readonly truncated: boolean;
+}
+/**
+ * Wire projection: bigint ranges become decimal strings, matching the outline
+ * and file tools.
+ */
+export interface WireSyntaxToken extends Omit<SyntaxToken, "range"> {
+    readonly range: {
+        startByte: string;
+        endByte: string;
+        startLine: string;
+        endLine: string;
+    };
+}
+/**
+ * Wire node.
+ */
+export interface WireSyntaxNode extends Omit<SyntaxNode, "range" | "children" | "tokens"> {
+    readonly range: {
+        startByte: string;
+        endByte: string;
+        startLine: string;
+        endLine: string;
+    };
+    readonly children: readonly WireSyntaxNode[];
+    readonly tokens: readonly WireSyntaxToken[];
+}
+/**
+ * Wire tree.
+ */
+export interface WireSyntaxTree extends Omit<SyntaxTree, "root" | "tokens"> {
+    readonly root: WireSyntaxNode;
+    readonly tokens: readonly WireSyntaxToken[];
+}
+/**
+ * Bounds. A file beyond `maxBytes` is refused; token/node ceilings set
+ * `truncated` and stop the build instead of exhausting memory.
+ */
+export interface SyntaxTreeOptions {
+    readonly maxBytes?: number;
+    readonly maxTokens?: number;
+    readonly maxNodes?: number;
+    /**
+     * Catalog used to resolve language, keywords and lexical rules. Omitted
+     * means `DefaultSyntaxRegistry`; a host that added a language passes its own
+     * registry here and through `buildSyntaxTree`.
+     */
+    readonly registry?: SyntaxRegistry;
+}
+/**
+ * Build the tree for an already-decoded text.
+ */
+export declare const buildSyntaxTreeFromText: (path: string, text: string, language: SyntaxLanguage, options?: SyntaxTreeOptions) => SyntaxTree;
+/**
+ * Project a tree onto JSON-safe decimal strings.
+ */
+export declare const wireSyntaxTree: (tree: SyntaxTree) => WireSyntaxTree;
+/**
+ * Recursively free a node and its children. Kept for callers that want to drop
+ * a large tree from memory explicitly.
+ */
+export declare const disposeSyntaxNode: (node: SyntaxNode) => void;