attention-manifest-protocol 0.4.2 → 0.4.4

package/lib/syntax/byteIndex.js

@@ -0,0 +1,148 @@
+/**
+ * UTF-8 byte offsets for 1-based line ranges.
+ *
+ * Outline ranges are persisted and compared across tools, so they use UTF-8
+ * byte offsets, matching tree-sitter and SCIP, rather than JavaScript string
+ * indices (UTF-16 code units). The segment tree stores the byte length of every
+ * line and answers "where does line N start?" as a prefix sum in O(log n),
+ * which keeps large-book outlines from rebuilding a byte table per node.
+ *
+ * Line extents are half-open on bytes (`endByte` is the first byte after the
+ * range) and inclusive on lines (`endLine` is the last covered line). A line's
+ * byte span includes its trailing newline so consecutive ranges tile the file
+ * without gaps.
+ */
+import { SegmentTree } from "../structure/segmentTree.js";
+const startsOf = (text) => {
+    if (text.length === 0) {
+        return [];
+    }
+    const starts = [0];
+    for (let index = 0; index < text.length; index += 1) {
+        if (text.charCodeAt(index) === 10 && index + 1 < text.length) {
+            starts.push(index + 1);
+        }
+    }
+    return starts;
+};
+export class ByteLineIndex {
+    tree;
+    starts;
+    text;
+    total;
+    /**
+     * Bytes before the decoded text (a UTF-8/UTF-16 BOM). Symbol ranges are
+     * offsets in the original file, while the decoded text starts after the
+     * marker; adding the base here keeps both coordinate systems consistent.
+     */
+    base;
+    constructor(text, bomBytes = 0) {
+        this.text = text;
+        this.base = Number.isSafeInteger(bomBytes) && bomBytes > 0 ? bomBytes : 0;
+        this.starts = startsOf(text);
+        const lengths = [];
+        for (let index = 0; index < this.starts.length; index += 1) {
+            const from = this.starts[index];
+            const to = index + 1 < this.starts.length ? this.starts[index + 1] : text.length;
+            lengths.push(Buffer.byteLength(text.slice(from, to), "utf8"));
+        }
+        this.total = BigInt(Buffer.byteLength(text, "utf8") + this.base);
+        this.tree = new SegmentTree({
+            size: lengths.length,
+            operation: (left, right) => left + right,
+            identity: 0,
+            values: lengths,
+        });
+    }
+    /**
+     * Number of logical lines. The empty file has zero lines.
+     */
+    get lineCount() {
+        return this.starts.length;
+    }
+    /**
+     * Start byte offset of a 1-based line. Lines past EOF clamp to the end.
+     */
+    startByte(line) {
+        if (line <= 1) {
+            return BigInt(this.base);
+        }
+        const last = this.starts.length;
+        if (line > last) {
+            return this.total;
+        }
+        return BigInt(this.base + this.tree.query(0, line - 1));
+    }
+    /**
+     * Half-open end byte offset for an inclusive line.
+     */
+    endByte(line) {
+        return this.startByte(line + 1);
+    }
+    /**
+     * Byte length of the whole text.
+     */
+    byteLength() {
+        return this.total;
+    }
+    /**
+     * Build a half-open byte / inclusive line range. Inputs are clamped so a
+     * parser can be approximate without producing an invalid range.
+     */
+    range(startLine, endLine) {
+        const start = Math.max(1, Math.min(startLine, Math.max(1, this.starts.length)));
+        const end = Math.max(start, Math.min(endLine, Math.max(1, this.starts.length)));
+        return {
+            startByte: this.startByte(start),
+            endByte: this.endByte(end),
+            startLine: BigInt(start),
+            endLine: BigInt(end),
+        };
+    }
+    /**
+     * Range covering the entire text; used for a file anchor.
+     */
+    fullRange() {
+        return {
+            startByte: 0n,
+            endByte: this.total,
+            startLine: 1n,
+            endLine: BigInt(Math.max(1, this.starts.length)),
+        };
+    }
+    /**
+     * Source slice for a range, useful for a `detail`/signature preview.
+     */
+    slice(range) {
+        const from = this.offsetOf(range.startByte);
+        const to = this.offsetOf(range.endByte);
+        return this.text.slice(from, to);
+    }
+    offsetOf(byteOffset) {
+        if (byteOffset <= 0n) {
+            return 0;
+        }
+        const target = Number(byteOffset);
+        if (!Number.isSafeInteger(target) || target >= Number(this.total)) {
+            return this.text.length;
+        }
+        /**
+         * Line starts are already sorted, so a binary search over the byte
+         * prefix positions is cheaper than another tree walk for one offset.
+         */
+        let low = 0;
+        let high = this.starts.length - 1;
+        while (low < high) {
+            const mid = Math.ceil((low + high) / 2);
+            const midByte = Number(this.startByte(mid + 1));
+            if (midByte <= target) {
+                low = mid;
+            }
+            else {
+                high = mid - 1;
+            }
+        }
+        const lineStartByte = Number(this.startByte(low + 1));
+        return this.starts[low] + (target - lineStartByte);
+    }
+}

package/lib/syntax/charset.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Charset detection, decoding and line indexing.
+ *
+ * Detection is deliberately conservative. A file that does not decode as
+ * UTF-8/UTF-16 is only treated as Latin-1 when it looks like text; anything
+ * with NUL bytes or a high control-character ratio is `binary`, because
+ * guessing an encoding is how a tool silently corrupts a file on write.
+ */
+import type { Charset } from "./types.js";
+/**
+ * Result of sniffing a byte sequence. `bomLength` is the number of leading
+ * bytes that are part of the encoding marker and must not appear in the text.
+ */
+export interface DetectedCharset {
+    readonly charset: Charset;
+    readonly bomLength: number;
+    readonly binary: boolean;
+}
+/**
+ * Sniff a byte sequence.
+ *
+ * @param bytes - Raw file bytes.
+ * @returns Charset, BOM length and whether the content is binary.
+ */
+export declare const detectCharset: (bytes: Uint8Array) => DetectedCharset;
+/**
+ * Decode bytes with a declared charset. `binary` and `none` are refused: a
+ * caller must decide whether base64 is the right transport.
+ *
+ * @param bytes - Raw bytes.
+ * @param charset - Charset from detection or node metadata.
+ * @returns Decoded text without a BOM.
+ */
+export declare const decodeText: (bytes: Uint8Array, charset: Charset) => string;
+/**
+ * Encode text for a write. The returned bytes include the BOM for
+ * `utf-8-bom`/UTF-16 encodings, so a caller can write them verbatim.
+ *
+ * @param text - Text to encode.
+ * @param encoding - Target encoding; defaults to strict UTF-8.
+ * @returns Encoded bytes.
+ */
+export declare const encodeText: (text: string, encoding?: "utf-8" | "utf-8-bom" | "utf-16le" | "utf-16be" | "latin1") => Uint8Array;
+/**
+ * Count logical lines. The empty file has zero lines; a trailing newline does
+ * not add a phantom empty line, matching how editors number a file.
+ *
+ * @param text - Decoded text.
+ * @returns Line count as a bigint.
+ */
+export declare const countLines: (text: string) => bigint;
+/**
+ * Lazily-built newline index over one decoded text.
+ *
+ * Offsets are JavaScript string indices (UTF-16 code units), which is exactly
+ * what `String.prototype.slice` consumes. Byte offsets are intentionally not
+ * stored: a caller that needs bytes uses `Buffer.byteLength` on the slice.
+ */
+export declare class LineIndex {
+    private readonly text;
+    private offsets;
+    constructor(text: string);
+    /**
+     * Number of lines in the text.
+     */
+    get count(): number;
+    private starts;
+    /**
+     * Start offset of a 1-based line; lines past the end clamp to EOF.
+     */
+    start(line: number): number;
+    /**
+     * End offset (exclusive) of a 1-based line, including its newline.
+     */
+    end(line: number): number;
+    /**
+     * Slice a bounded run of lines.
+     *
+     * @param startLine - 1-based inclusive start.
+     * @param maxLines - Maximum number of lines.
+     * @returns Text without a trailing newline plus the inclusive range.
+     */
+    slice(startLine: number, maxLines: number): {
+        text: string;
+        startLine: number;
+        endLine: number;
+    };
+}

package/lib/syntax/charset.js

@@ -0,0 +1,276 @@
+import { fail } from "../file/errors.js";
+const Utf8Bom = [0xef, 0xbb, 0xbf];
+const Utf16LeBom = [0xff, 0xfe];
+const Utf16BeBom = [0xfe, 0xff];
+const startsWith = (bytes, prefix) => {
+    if (bytes.length < prefix.length) {
+        return false;
+    }
+    for (let index = 0; index < prefix.length; index += 1) {
+        if (bytes[index] !== prefix[index]) {
+            return false;
+        }
+    }
+    return true;
+};
+const isUtf8 = (bytes) => {
+    try {
+        new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * Heuristic for BOM-less UTF-16: sample ASCII code units and count NULs on
+ * each byte lane. A binary file with many NULs usually fails both lanes.
+ */
+const detectUtf16WithoutBom = (bytes) => {
+    if (bytes.length < 8 || bytes.length % 2 !== 0) {
+        return undefined;
+    }
+    const sample = Math.min(bytes.length, 512);
+    let evenNul = 0;
+    let oddNul = 0;
+    let evenTotal = 0;
+    let oddTotal = 0;
+    for (let index = 0; index + 1 < sample; index += 2) {
+        evenTotal += 1;
+        oddTotal += 1;
+        if (bytes[index] === 0) {
+            evenNul += 1;
+        }
+        if (bytes[index + 1] === 0) {
+            oddNul += 1;
+        }
+    }
+    if (oddNul / oddTotal > 0.6 && evenNul / evenTotal < 0.1) {
+        return "utf-16le";
+    }
+    if (evenNul / evenTotal > 0.6 && oddNul / oddTotal < 0.1) {
+        return "utf-16be";
+    }
+    return undefined;
+};
+/**
+ * Control-character ratio over the first 8 KiB. Tab/newline/carriage return are
+ * text; other C0 controls usually mean a container format.
+ */
+const controlRatio = (bytes) => {
+    const sample = Math.min(bytes.length, 8192);
+    let controls = 0;
+    for (let index = 0; index < sample; index += 1) {
+        const byte = bytes[index];
+        if (byte < 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0d) {
+            controls += 1;
+        }
+    }
+    return sample === 0 ? 0 : controls / sample;
+};
+/**
+ * Sniff a byte sequence.
+ *
+ * @param bytes - Raw file bytes.
+ * @returns Charset, BOM length and whether the content is binary.
+ */
+export const detectCharset = (bytes) => {
+    if (bytes.length === 0) {
+        return { charset: "utf-8", bomLength: 0, binary: false };
+    }
+    if (startsWith(bytes, Utf8Bom)) {
+        return { charset: "utf-8-bom", bomLength: Utf8Bom.length, binary: false };
+    }
+    if (startsWith(bytes, Utf16LeBom)) {
+        return { charset: "utf-16le", bomLength: Utf16LeBom.length, binary: false };
+    }
+    if (startsWith(bytes, Utf16BeBom)) {
+        return { charset: "utf-16be", bomLength: Utf16BeBom.length, binary: false };
+    }
+    const utf16 = detectUtf16WithoutBom(bytes);
+    if (utf16 !== undefined) {
+        return { charset: utf16, bomLength: 0, binary: false };
+    }
+    if (isUtf8(bytes)) {
+        return { charset: "utf-8", bomLength: 0, binary: controlRatio(bytes) > 0.05 };
+    }
+    return { charset: "latin1", bomLength: 0, binary: controlRatio(bytes) > 0.05 };
+};
+/**
+ * Decode bytes with a declared charset. `binary` and `none` are refused: a
+ * caller must decide whether base64 is the right transport.
+ *
+ * @param bytes - Raw bytes.
+ * @param charset - Charset from detection or node metadata.
+ * @returns Decoded text without a BOM.
+ */
+export const decodeText = (bytes, charset) => {
+    switch (charset) {
+        case "utf-8": {
+            return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        }
+        case "utf-8-bom": {
+            const bom = detectCharset(bytes);
+            const body = bom.bomLength === 0 ? bytes : bytes.subarray(bom.bomLength);
+            return new TextDecoder("utf-8", { fatal: true }).decode(body);
+        }
+        case "utf-16le": {
+            const body = startsWith(bytes, Utf16LeBom) ? bytes.subarray(2) : bytes;
+            return new TextDecoder("utf-16le", { fatal: true }).decode(body);
+        }
+        case "utf-16be": {
+            const body = startsWith(bytes, Utf16BeBom) ? bytes.subarray(2) : bytes;
+            return new TextDecoder("utf-16be", { fatal: true }).decode(body);
+        }
+        case "latin1": {
+            return Buffer.from(bytes).toString("latin1");
+        }
+        case "binary":
+        case "none": {
+            return fail("binary_file");
+        }
+        default: {
+            return fail("charset_unsupported");
+        }
+    }
+};
+/**
+ * Encode text for a write. The returned bytes include the BOM for
+ * `utf-8-bom`/UTF-16 encodings, so a caller can write them verbatim.
+ *
+ * @param text - Text to encode.
+ * @param encoding - Target encoding; defaults to strict UTF-8.
+ * @returns Encoded bytes.
+ */
+export const encodeText = (text, encoding = "utf-8") => {
+    switch (encoding) {
+        case "utf-8": {
+            return new TextEncoder().encode(text);
+        }
+        case "utf-8-bom": {
+            const body = new TextEncoder().encode(text);
+            const result = new Uint8Array(body.length + 3);
+            result.set(Utf8Bom);
+            result.set(body, 3);
+            return result;
+        }
+        case "utf-16le": {
+            const body = Buffer.from(text, "utf16le");
+            return new Uint8Array(Buffer.concat([Buffer.from(Utf16LeBom), body]));
+        }
+        case "utf-16be": {
+            const body = Buffer.from(text, "utf16le");
+            body.swap16();
+            return new Uint8Array(Buffer.concat([Buffer.from(Utf16BeBom), body]));
+        }
+        case "latin1": {
+            if ([...text].some((character) => character.codePointAt(0) > 0xff)) {
+                return fail("charset_unsupported");
+            }
+            return new Uint8Array(Buffer.from(text, "latin1"));
+        }
+        default: {
+            return fail("charset_unsupported");
+        }
+    }
+};
+/**
+ * Count logical lines. The empty file has zero lines; a trailing newline does
+ * not add a phantom empty line, matching how editors number a file.
+ *
+ * @param text - Decoded text.
+ * @returns Line count as a bigint.
+ */
+export const countLines = (text) => {
+    if (text.length === 0) {
+        return 0n;
+    }
+    let count = 0n;
+    for (let index = 0; index < text.length; index += 1) {
+        if (text.charCodeAt(index) === 10) {
+            count += 1n;
+        }
+    }
+    if (text.charCodeAt(text.length - 1) !== 10) {
+        count += 1n;
+    }
+    return count;
+};
+/**
+ * Lazily-built newline index over one decoded text.
+ *
+ * Offsets are JavaScript string indices (UTF-16 code units), which is exactly
+ * what `String.prototype.slice` consumes. Byte offsets are intentionally not
+ * stored: a caller that needs bytes uses `Buffer.byteLength` on the slice.
+ */
+export class LineIndex {
+    text;
+    offsets;
+    constructor(text) {
+        this.text = text;
+    }
+    /**
+     * Number of lines in the text.
+     */
+    get count() {
+        return this.starts().length;
+    }
+    starts() {
+        if (this.offsets !== undefined) {
+            return this.offsets;
+        }
+        const starts = [];
+        if (this.text.length > 0) {
+            starts.push(0);
+            for (let index = 0; index < this.text.length; index += 1) {
+                if (this.text.charCodeAt(index) === 10 && index + 1 < this.text.length) {
+                    starts.push(index + 1);
+                }
+            }
+        }
+        this.offsets = starts;
+        return starts;
+    }
+    /**
+     * Start offset of a 1-based line; lines past the end clamp to EOF.
+     */
+    start(line) {
+        const starts = this.starts();
+        if (line <= 1) {
+            return 0;
+        }
+        return starts[Math.min(line - 1, starts.length)] ?? this.text.length;
+    }
+    /**
+     * End offset (exclusive) of a 1-based line, including its newline.
+     */
+    end(line) {
+        return this.start(line + 1);
+    }
+    /**
+     * Slice a bounded run of lines.
+     *
+     * @param startLine - 1-based inclusive start.
+     * @param maxLines - Maximum number of lines.
+     * @returns Text without a trailing newline plus the inclusive range.
+     */
+    slice(startLine, maxLines) {
+        const starts = this.starts();
+        const total = starts.length;
+        if (total === 0 || startLine > total) {
+            return { text: "", startLine, endLine: startLine - 1 };
+        }
+        const first = Math.max(1, startLine);
+        const last = Math.min(total, first + Math.max(1, maxLines) - 1);
+        const from = starts[first - 1];
+        const to = last < total ? starts[last] : this.text.length;
+        let text = this.text.slice(from, to);
+        if (text.endsWith("\n")) {
+            text = text.slice(0, -1);
+        }
+        if (text.endsWith("\r")) {
+            text = text.slice(0, -1);
+        }
+        return { text, startLine: first, endLine: last };
+    }
+}

package/lib/syntax/index.d.ts

@@ -0,0 +1,6 @@
+/** Source encodings, byte/line indexing and bounded concrete syntax trees. */
+export * from "./types.js";
+export * from "./charset.js";
+export * from "./byteIndex.js";
+export * from "./syntaxTree.js";
+export type { SyntaxFamily, SyntaxRule } from "./syntaxRegistry.js";

package/lib/syntax/index.js

@@ -0,0 +1,5 @@
+/** Source encodings, byte/line indexing and bounded concrete syntax trees. */
+export * from "./types.js";
+export * from "./charset.js";
+export * from "./byteIndex.js";
+export * from "./syntaxTree.js";

package/lib/syntax/syntaxRegistry.d.ts

@@ -0,0 +1,150 @@
+/**
+ * Syntax registry.
+ *
+ * Languages, token categories and node categories are data, not three frozen
+ * unions spread across the parser. A host can add a language (or override an
+ * extension) without editing `syntaxTree.ts`; the parser only asks the registry
+ * for the language definition and the allowed token/node ids it already knows
+ * how to construct.
+ *
+ * The three public type aliases stay plain strings on purpose: the registry is
+ * the authority, so a value that did not pass `register*` is rejected at the
+ * boundary instead of being accepted by the type system and failing later.
+ */
+import type { SourceRange as OutlineRange } from "./types.js";
+/**
+ * Registered language id. Built-ins include `typescript`, `python`, `html`,
+ * `vue`, `json`, `markdown`, `tex` and `plainText`.
+ */
+export type SyntaxLanguage = string;
+/**
+ * Registered token id. Built-ins: `keyword`, `identifier`, `number`, `string`,
+ * `comment`, `punctuation`, `text`.
+ */
+export type SyntaxTokenKind = string;
+/**
+ * Registered node id. Built-ins: `SourceFile`, `Block`, `Declaration`, `Class`,
+ * `Interface`, `Struct`, `Trait`, `Enum`, `Namespace`, `Function`, `Method`,
+ * `Property`, `Import`, `Export`, `Parameter`, `Tag`, `Attribute`, `Text`,
+ * `Heading`, `Rule`, `Comment`.
+ */
+export type SyntaxNodeKind = string;
+/**
+ * Coarse parser family. It selects the tree builder, not the language id.
+ */
+export type SyntaxFamily = "code" | "python" | "markup" | "text" | "json" | "style";
+/**
+ * One language definition.
+ */
+export interface SyntaxLanguageDefinition {
+    readonly id: SyntaxLanguage;
+    readonly family: SyntaxFamily;
+    readonly extensions: readonly string[];
+    readonly lineComment?: string;
+    readonly blockStart?: string;
+    readonly blockEnd?: string;
+    readonly keywords?: readonly string[];
+}
+/**
+ * One token-category definition.
+ */
+export interface SyntaxTokenKindDefinition {
+    readonly id: SyntaxTokenKind;
+    readonly description: string;
+}
+/**
+ * One node-category definition.
+ */
+export interface SyntaxNodeKindDefinition {
+    readonly id: SyntaxNodeKind;
+    readonly category: "container" | "declaration" | "token" | "markup" | "text";
+}
+/**
+ * Lexical rules derived from one language definition.
+ */
+export interface SyntaxRule {
+    readonly lineComment?: string;
+    readonly blockStart?: string;
+    readonly blockEnd?: string;
+    readonly family: SyntaxFamily;
+}
+/**
+ * A mutable registry with duplicate and shape checks. The default instance is
+ * populated once at module load; `createSyntaxRegistry()` returns an empty one
+ * for a host that wants a different catalog.
+ */
+export declare class SyntaxRegistry {
+    private readonly languageById;
+    private readonly languageByExtension;
+    private readonly tokenKinds;
+    private readonly nodeKinds;
+    /**
+     * Register one language. An empty id, a duplicate id, a duplicate
+     * extension, an invalid extension or an unknown family is rejected.
+     */
+    registerLanguage(definition: SyntaxLanguageDefinition): this;
+    /**
+     * Register a token category.
+     */
+    registerTokenKind(definition: SyntaxTokenKindDefinition): this;
+    /**
+     * Register a node category.
+     */
+    registerNodeKind(definition: SyntaxNodeKindDefinition): this;
+    /**
+     * Look up a language by id.
+     */
+    language(id: SyntaxLanguage): SyntaxLanguageDefinition | undefined;
+    /**
+     * Resolve a path's extension. Unknown extensions return `unknown`; a path
+     * without an extension returns `plainText`.
+     */
+    languageOf(path: string): SyntaxLanguage;
+    /**
+     * Rule object consumed by the tokenizer.
+     */
+    ruleOf(language: SyntaxLanguage): SyntaxRule;
+    /**
+     * Keyword set for one language.
+     */
+    keywordsOf(language: SyntaxLanguage): ReadonlySet<string>;
+    /**
+     * All registered languages, in registration order.
+     */
+    languages(): SyntaxLanguageDefinition[];
+    /**
+     * All registered token kinds.
+     */
+    tokenKindsList(): SyntaxTokenKindDefinition[];
+    /**
+     * All registered node kinds.
+     */
+    nodeKindsList(): SyntaxNodeKindDefinition[];
+    /**
+     * Whether every id is registered.
+     */
+    hasTokenKind(id: SyntaxTokenKind): boolean;
+    /**
+     * Whether every id is registered.
+     */
+    hasNodeKind(id: SyntaxNodeKind): boolean;
+}
+/**
+ * The process-wide catalog used by the parser. It is frozen after module load;
+ * a host that needs a different catalog builds one with
+ * `createSyntaxRegistry()` and registers it explicitly.
+ */
+export declare const DefaultSyntaxRegistry: SyntaxRegistry;
+/**
+ * Create an empty registry for a host that wants to register a custom catalog.
+ */
+export declare const createSyntaxRegistry: () => SyntaxRegistry;
+/**
+ * Convenience wrapper over the default catalog.
+ */
+export declare const syntaxLanguageOf: (path: string) => SyntaxLanguage;
+/**
+ * A range used by syntax tokens/nodes; re-exported for callers that only import
+ * the registry module.
+ */
+export type SyntaxRange = OutlineRange;