@nette/neon 0.0.0-stage → 1.0.0

package/dist/scanner.js

@@ -0,0 +1,193 @@
+/** Reads tokens and trivia one by one. Returns null on an error, with errorAt set. */
+export class Scanner {
+    s;
+    #pos = 0;
+    errorAt = -1;
+    constructor(s) {
+        this.s = s;
+    }
+    get position() {
+        return this.#pos;
+    }
+    next() {
+        const s = this.s;
+        const start = this.#pos;
+        const c = s[start];
+        if (c === undefined) {
+            return null;
+        }
+        let kind;
+        let triviaKind;
+        let end;
+        if (start === 0 && s.charCodeAt(0) === 0xFEFF) {
+            kind = 'Trivia';
+            triviaKind = 'ByteOrderMark';
+            end = 1;
+        }
+        else if (c === '\'' || c === '"') {
+            kind = 'String';
+            end = s.startsWith(c + c + c, start) && this.isLineEnding(start + 3)
+                ? this.scanMultiline(start, c + c + c)
+                : -1;
+            if (end < 0) { // a triple without a closing line is read as single-line quotes
+                end = c === '\'' ? this.scanSingleQuoted(start) : this.scanDoubleQuoted(start);
+            }
+        }
+        else if (c === '#') {
+            kind = 'Trivia';
+            triviaKind = 'Comment';
+            end = start + 1;
+            while (end < s.length && !this.isLineEnding(end)) {
+                end++;
+            }
+        }
+        else if (this.isLineEnding(start)) {
+            kind = 'Trivia';
+            triviaKind = 'LineEnding';
+            end = start + (c === '\r' ? 2 : 1);
+        }
+        else if (this.isSpace(start)) {
+            kind = 'Trivia';
+            triviaKind = 'Whitespace';
+            end = start + 1;
+            while (this.isSpace(end)) {
+                end++;
+            }
+        }
+        else if (c in Punctuation) {
+            kind = Punctuation[c];
+            end = start + 1;
+        }
+        else if (c === ':' || c === '-') {
+            const next = s[start + 1];
+            const prev = s[start - 1];
+            if (next !== undefined && !this.isAny(start + 1, '"\',=[]{}()') && prev !== '"' && prev !== '\'') {
+                kind = 'Literal';
+                end = this.scanLiteral(start + 2); // the next character belongs to the start of the literal
+            }
+            else {
+                kind = c === ':' ? 'Colon' : 'Dash';
+                end = start + 1;
+            }
+        }
+        else if (!this.isAny(start, '#"\',:=[]{}()`')) {
+            kind = 'Literal';
+            end = this.scanLiteral(start + 1);
+        }
+        else {
+            end = -1;
+            kind = 'Trivia';
+        }
+        if (end < 0) {
+            this.errorAt = start;
+            return null;
+        }
+        this.#pos = end;
+        return { kind, triviaKind, start, end };
+    }
+    /** Is the character one of the given ones, a space, a tab, or a line ending? */
+    isAny(i, chars) {
+        const c = this.s[i];
+        return c === undefined || c === ' ' || c === '\t' || c === '\n' || c === '\r' || chars.includes(c);
+    }
+    isSpace(i) {
+        const c = this.s[i];
+        return c === ' ' || c === '\t' || (c === '\r' && this.s[i + 1] !== '\n');
+    }
+    isLineEnding(i) {
+        const c = this.s[i];
+        return c === '\n' || (c === '\r' && this.s[i + 1] === '\n');
+    }
+    scanLiteral(i) {
+        const s = this.s;
+        while (true) {
+            if (i < s.length && !this.isAny(i, ',:=]})(')) {
+                i++;
+                while (i < s.length && !this.isAny(i, ',:=]})(')) {
+                    i++;
+                }
+            }
+            else if (s[i] === ':' && i + 1 < s.length && !this.isAny(i + 1, ',]})')) {
+                i++;
+            }
+            else if (this.isSpace(i)) {
+                let j = i + 1;
+                while (this.isSpace(j)) {
+                    j++;
+                }
+                if (j < s.length && !this.isAny(j, '#,:=]})(')) {
+                    i = j;
+                }
+                else {
+                    return i;
+                }
+            }
+            else {
+                return i;
+            }
+        }
+    }
+    scanSingleQuoted(start) {
+        const s = this.s;
+        let i = start + 1;
+        while (i < s.length) {
+            if (s[i] === '\'' && s[i + 1] === '\'') {
+                i += 2;
+            }
+            else if (s[i] === '\'' || s[i] === '\n') {
+                break;
+            }
+            else {
+                i++;
+            }
+        }
+        return s[i] === '\'' ? i + 1 : -1;
+    }
+    scanDoubleQuoted(start) {
+        const s = this.s;
+        let i = start + 1;
+        while (i < s.length) {
+            if (s[i] === '\\' && i + 1 < s.length && !this.isLineEnding(i + 1)) {
+                i += 2;
+            }
+            else if (s[i] === '"' || s[i] === '\\' || s[i] === '\n') {
+                break;
+            }
+            else {
+                i++;
+            }
+        }
+        return s[i] === '"' ? i + 1 : -1;
+    }
+    /**
+     * Content runs to the first line after the first one that starts with optional spaces and the
+     * closing triple; only without such a line may the first line itself close an empty string.
+     */
+    scanMultiline(start, triple) {
+        const s = this.s;
+        const first = s.indexOf('\n', start) + 1;
+        for (let i = s.indexOf('\n', first); i >= 0; i = s.indexOf('\n', i + 1)) {
+            const end = this.closingAt(i + 1, triple);
+            if (end >= 0) {
+                return end;
+            }
+        }
+        return this.closingAt(first, triple);
+    }
+    closingAt(i, triple) {
+        while (this.isSpace(i)) {
+            i++;
+        }
+        return this.s.startsWith(triple, i) ? i + 3 : -1;
+    }
+}
+const Punctuation = {
+    ',': 'Comma',
+    '=': 'Equals',
+    '[': 'OpenBracket',
+    ']': 'CloseBracket',
+    '{': 'OpenBrace',
+    '}': 'CloseBrace',
+    '(': 'OpenParen',
+    ')': 'CloseParen',
+};

package/dist/token.d.ts

@@ -0,0 +1,67 @@
+import { forgetOrigin, setLeading, setParent, setTrailing } from './internal.ts';
+import { type Node } from './nodes.ts';
+export type TokenKind = 'String' | 'Literal' | 'Colon' | 'Equals' | 'Comma' | 'Dash' | 'OpenBracket' | 'CloseBracket' | 'OpenBrace' | 'CloseBrace' | 'OpenParen' | 'CloseParen' | 'EndOfFile';
+export type TriviaKind = 'Whitespace' | 'LineEnding' | 'Comment' | 'ByteOrderMark';
+/** Line and column are 1-based, offset 0-based; all in code points. utf16Offset indexes the JavaScript string. */
+export interface Position {
+    readonly line: number;
+    readonly column: number;
+    readonly offset: number;
+    readonly utf16Offset: number;
+}
+/** Whitespace, a line ending, a comment or a byte order mark around a token. */
+export declare class Trivia {
+    readonly kind: TriviaKind;
+    readonly text: string;
+    constructor(kind: TriviaKind, text: string);
+    static fromText(text: string): Trivia;
+    withText(text: string): Trivia;
+    is(...kinds: TriviaKind[]): boolean;
+    /** Text of a comment without # and trimmed, or empty string for other trivia. */
+    getCommentText(): string;
+}
+/** A token of the source with the trivia around it. */
+export declare class Token {
+    #private;
+    /** Prefer Token.fromText(); the text must be one token of the kind and the trivia canonical. */
+    constructor(kind: TokenKind, text: string, leadingTrivia?: readonly Trivia[], trailingTrivia?: readonly Trivia[], origin?: Position | null);
+    /** Makes a token of the kind the scanner reads the text as. */
+    static fromText(text: string): Token;
+    /** A token changes its kind only by being replaced. */
+    get kind(): TokenKind;
+    /** Where the token stood in the source it was read from; null for a token made otherwise. A clone keeps it. */
+    get origin(): Position | null;
+    get text(): string;
+    get leadingTrivia(): readonly Trivia[];
+    get trailingTrivia(): readonly Trivia[];
+    get parent(): Node | null;
+    /** Replaces the text by another one of the same kind. */
+    setText(text: string): this;
+    /** Sets the leading trivia; in a document also checks them against the token before. */
+    setLeadingTrivia(trivia: readonly Trivia[]): this;
+    /** Sets the trailing trivia; in a document also checks them against the token after. */
+    setTrailingTrivia(trivia: readonly Trivia[]): this;
+    /** Replaces a trivia, found by identity, in the leading or trailing trivia. */
+    replaceTrivia(old: Trivia, replacement: Trivia): this;
+    is(...kinds: TokenKind[]): boolean;
+    /** A detached copy with the same text, trivia and origin. */
+    clone(): Token;
+    /** The current position, computed by a walk from the root; null in a detached subtree. */
+    getPosition(): Position | null;
+    /** The source text of the token with its trivia. */
+    toString(): string;
+    /** @internal Sets the leading trivia without the checks of the place, for an edit that moves trivia in steps. */
+    [setLeading](trivia: readonly Trivia[]): this;
+    /** @internal */
+    [setTrailing](trivia: readonly Trivia[]): this;
+    /** @internal For a token made from a text that is not the source of the document. */
+    [forgetOrigin](): void;
+    /** @internal */
+    [setParent](parent: Node | null): void;
+}
+/** @internal A token whose text and trivia are known to be valid: one the lexer reads or a clone. */
+export declare function trustedToken(kind: TokenKind, text: string, leading: readonly Trivia[], trailing: readonly Trivia[], origin: Position | null): Token;
+/** The token of the tree right before or after one. */
+export declare function findNeighbour(token: Token, backward: boolean): Token | null;
+/** The tokens of the tree before or after one, the nearest first, read lazily from the nodes around it. */
+export declare function iterateNeighbours(token: Token, backward: boolean): Generator<Token>;

package/dist/token.js

@@ -0,0 +1,309 @@
+import { forgetOrigin, record, setLeading, setParent, setTrailing } from "./internal.js";
+import { Scanner } from "./scanner.js";
+import { DocumentNode } from "./nodes.js";
+import { Cursor } from "./position.js";
+/** Whitespace, a line ending, a comment or a byte order mark around a token. */
+export class Trivia {
+    kind;
+    text;
+    constructor(kind, text) {
+        this.kind = kind;
+        this.text = text;
+        if (classifyTrivia(text) !== kind) {
+            throw new TypeError(`${JSON.stringify(text)} is not ${kind}.`);
+        }
+        Object.freeze(this);
+    }
+    static fromText(text) {
+        const kind = classifyTrivia(text);
+        if (!kind) {
+            throw new TypeError(`${JSON.stringify(text)} is not one trivia.`);
+        }
+        return new Trivia(kind, text);
+    }
+    withText(text) {
+        const trivia = Trivia.fromText(text);
+        if (trivia.kind !== this.kind) {
+            throw new TypeError(`${JSON.stringify(text)} is not ${this.kind}.`);
+        }
+        return trivia;
+    }
+    is(...kinds) {
+        return kinds.includes(this.kind);
+    }
+    /** Text of a comment without # and trimmed, or empty string for other trivia. */
+    getCommentText() {
+        return this.kind === 'Comment' ? this.text.slice(1).trim() : '';
+    }
+}
+function classifyTrivia(text) {
+    if (/^(?:[ \t]|\r(?!\n))+$/.test(text)) {
+        return 'Whitespace';
+    }
+    else if (text === '\n' || text === '\r\n') {
+        return 'LineEnding';
+    }
+    else if (/^#[^\n]*$/.test(text) && text.isWellFormed()) {
+        return 'Comment';
+    }
+    else if (text === String.fromCharCode(0xFEFF)) {
+        return 'ByteOrderMark';
+    }
+    return null;
+}
+/** A token of the source with the trivia around it. */
+export class Token {
+    #kind;
+    #origin;
+    #text;
+    #leadingTrivia;
+    #trailingTrivia;
+    #parent = null;
+    /** Prefer Token.fromText(); the text must be one token of the kind and the trivia canonical. */
+    constructor(kind, text, leadingTrivia = [], trailingTrivia = [], origin = null) {
+        if (!trusted) {
+            checkText(kind, text);
+            checkTrivia(leadingTrivia, 'leading', kind === 'EndOfFile');
+            checkTrivia(trailingTrivia, 'trailing', false);
+            if (kind === 'EndOfFile' && trailingTrivia.length) {
+                throw new TypeError('EndOfFile has no trailing trivia.');
+            }
+        }
+        this.#kind = kind;
+        this.#text = text;
+        this.#leadingTrivia = Object.freeze([...leadingTrivia]);
+        this.#trailingTrivia = Object.freeze([...trailingTrivia]);
+        this.#origin = origin;
+    }
+    /** Makes a token of the kind the scanner reads the text as. */
+    static fromText(text) {
+        const kind = text.isWellFormed() ? scanSingleToken(text) : null;
+        if (!kind) {
+            throw new TypeError(`${JSON.stringify(text)} is not one token.`);
+        }
+        return new Token(kind, text);
+    }
+    /** A token changes its kind only by being replaced. */
+    get kind() {
+        return this.#kind;
+    }
+    /** Where the token stood in the source it was read from; null for a token made otherwise. A clone keeps it. */
+    get origin() {
+        return this.#origin;
+    }
+    get text() {
+        return this.#text;
+    }
+    get leadingTrivia() {
+        return this.#leadingTrivia;
+    }
+    get trailingTrivia() {
+        return this.#trailingTrivia;
+    }
+    get parent() {
+        return this.#parent;
+    }
+    /** Replaces the text by another one of the same kind. */
+    setText(text) {
+        checkText(this.#kind, text);
+        const old = this.#text;
+        record(() => this.#text = old);
+        this.#text = text;
+        return this;
+    }
+    /** Sets the leading trivia; in a document also checks them against the token before. */
+    setLeadingTrivia(trivia) {
+        checkTrivia(trivia, 'leading', this.#kind === 'EndOfFile');
+        const [before] = neighbours(this);
+        if (before && trivia.length && !before.trailingTrivia.some((t) => t.kind === 'LineEnding')) {
+            throw new TypeError('Leading trivia must follow a line ending; on the same line they are the trailing trivia of the token before.');
+        }
+        else if (before && trivia[0]?.kind === 'ByteOrderMark') {
+            throw new TypeError('A byte order mark may only stand at the start of the input.');
+        }
+        return this[setLeading](trivia);
+    }
+    /** Sets the trailing trivia; in a document also checks them against the token after. */
+    setTrailingTrivia(trivia) {
+        checkTrivia(trivia, 'trailing', this.#kind === 'EndOfFile');
+        const [, after] = neighbours(this);
+        if (after && after.leadingTrivia.length && !trivia.some((t) => t.kind === 'LineEnding')) {
+            throw new TypeError('Trailing trivia without a line ending must not be followed by leading trivia.');
+        }
+        else if (after && after.kind !== 'EndOfFile' && trivia.at(-1)?.kind === 'Comment') {
+            throw new TypeError('A comment must be followed by a line ending.');
+        }
+        return this[setTrailing](trivia);
+    }
+    /** Replaces a trivia, found by identity, in the leading or trailing trivia. */
+    replaceTrivia(old, replacement) {
+        const replace = (list) => list.map((trivia) => trivia === old ? replacement : trivia);
+        if (this.#leadingTrivia.includes(old)) {
+            this.setLeadingTrivia(replace(this.#leadingTrivia));
+        }
+        else if (this.#trailingTrivia.includes(old)) {
+            this.setTrailingTrivia(replace(this.#trailingTrivia));
+        }
+        else {
+            throw new TypeError('The trivia does not belong to the token.');
+        }
+        return this;
+    }
+    is(...kinds) {
+        return kinds.includes(this.#kind);
+    }
+    /** A detached copy with the same text, trivia and origin. */
+    clone() {
+        return trustedToken(this.#kind, this.#text, this.#leadingTrivia, this.#trailingTrivia, this.#origin);
+    }
+    /** The current position, computed by a walk from the root; null in a detached subtree. */
+    getPosition() {
+        let root = this.#parent;
+        while (root?.parent) {
+            root = root.parent;
+        }
+        if (!(root instanceof DocumentNode)) {
+            return null;
+        }
+        let text = '';
+        for (const token of root.getTokens()) {
+            if (token === this) {
+                for (const trivia of this.#leadingTrivia) {
+                    text += trivia.text;
+                }
+                return new Cursor(text).at(text.length);
+            }
+            text += token.toString();
+        }
+        return null;
+    }
+    /** The source text of the token with its trivia. */
+    toString() {
+        let res = '';
+        for (const trivia of this.#leadingTrivia) {
+            res += trivia.text;
+        }
+        res += this.#text;
+        for (const trivia of this.#trailingTrivia) {
+            res += trivia.text;
+        }
+        return res;
+    }
+    /** @internal Sets the leading trivia without the checks of the place, for an edit that moves trivia in steps. */
+    [setLeading](trivia) {
+        checkTrivia(trivia, 'leading', this.#kind === 'EndOfFile');
+        const old = this.#leadingTrivia;
+        record(() => this.#leadingTrivia = old);
+        this.#leadingTrivia = Object.freeze(uniqueTrivia(trivia));
+        return this;
+    }
+    /** @internal */
+    [setTrailing](trivia) {
+        checkTrivia(trivia, 'trailing', this.#kind === 'EndOfFile');
+        const old = this.#trailingTrivia;
+        record(() => this.#trailingTrivia = old);
+        this.#trailingTrivia = Object.freeze(uniqueTrivia(trivia));
+        return this;
+    }
+    /** @internal For a token made from a text that is not the source of the document. */
+    [forgetOrigin]() {
+        this.#origin = null;
+    }
+    /** @internal */
+    [setParent](parent) {
+        const old = this.#parent;
+        record(() => this.#parent = old);
+        this.#parent = parent;
+    }
+}
+let trusted = false;
+/** @internal A token whose text and trivia are known to be valid: one the lexer reads or a clone. */
+export function trustedToken(kind, text, leading, trailing, origin) {
+    trusted = true;
+    try {
+        return new Token(kind, text, leading, trailing, origin);
+    }
+    finally {
+        trusted = false;
+    }
+}
+function checkText(kind, text) {
+    if (!text.isWellFormed() || (kind === 'EndOfFile' ? text !== '' : scanSingleToken(text) !== kind)) {
+        throw new TypeError(`${JSON.stringify(text)} is not one token of kind ${kind}.`);
+    }
+}
+/** The tokens before and after one in a document; none when it is not in a document. */
+function neighbours(token) {
+    let root = token.parent;
+    while (root?.parent) {
+        root = root.parent;
+    }
+    return root instanceof DocumentNode ? [findNeighbour(token, true), findNeighbour(token, false)] : [null, null];
+}
+/** The token of the tree right before or after one. */
+export function findNeighbour(token, backward) {
+    for (const neighbour of iterateNeighbours(token, backward)) {
+        return neighbour;
+    }
+    return null;
+}
+/** The tokens of the tree before or after one, the nearest first, read lazily from the nodes around it. */
+export function* iterateNeighbours(token, backward) {
+    let child = token;
+    for (let node = token.parent; node; child = node, node = node.parent) {
+        const children = node.getChildren();
+        const i = children.indexOf(child);
+        for (const sibling of backward ? children.slice(0, i).reverse() : children.slice(i + 1)) {
+            yield* iterateTokens(sibling, backward);
+        }
+    }
+}
+function* iterateTokens(node, backward) {
+    if (node instanceof Token) {
+        yield node;
+        return;
+    }
+    const children = node.getChildren();
+    for (const child of backward ? children.toReversed() : children) {
+        yield* iterateTokens(child, backward);
+    }
+}
+function checkTrivia(list, side, endOfFile) {
+    if (side === 'trailing' && endOfFile && list.length) {
+        throw new TypeError('EndOfFile has no trailing trivia.');
+    }
+    list.forEach((trivia, i) => {
+        const next = list[i + 1];
+        if (!(trivia instanceof Trivia)) {
+            throw new TypeError('Trivia expected.');
+        }
+        else if (trivia.kind === 'Whitespace' && next?.kind === 'Whitespace') {
+            throw new TypeError('Two whitespace trivia must not stand next to each other.');
+        }
+        else if (trivia.kind === 'ByteOrderMark' && (side !== 'leading' || i > 0)) {
+            throw new TypeError('A byte order mark may only be the first leading trivia.');
+        }
+        else if (trivia.kind === 'LineEnding' && side === 'trailing' && next) {
+            throw new TypeError('A line ending must be the last of the trailing trivia.');
+        }
+        else if (trivia.kind === 'Comment' && next?.kind !== 'LineEnding' && (next || (side === 'leading' && !endOfFile))) {
+            throw new TypeError('A comment must be followed by a line ending.');
+        }
+        else if (trivia.text.endsWith('\r') && next?.text.startsWith('\n')) {
+            throw new TypeError('A carriage return before a line feed would make one line ending of the two.');
+        }
+    });
+}
+/** A copy of the list where an instance stands once, so that replaceTrivia() finds the one meant. */
+function uniqueTrivia(list) {
+    const seen = (new Set);
+    return list.map((trivia) => seen.has(trivia) ? new Trivia(trivia.kind, trivia.text) : (seen.add(trivia), trivia));
+}
+/** Kind of the text if the scanner reads it as exactly one token without trivia. */
+function scanSingleToken(text) {
+    const scanner = new Scanner(text);
+    const token = scanner.next();
+    return token && token.end === text.length && token.kind !== 'Trivia'
+        ? token.kind
+        : null;
+}

package/dist/traverser.d.ts

@@ -0,0 +1,31 @@
+import { type Node, ValueNode } from './nodes.ts';
+/** What a visitor returns to steer the walk; symbols, so that no other value means an action by accident. */
+export declare const TraverseAction: Readonly<{
+    readonly SkipChildren: symbol;
+    readonly Stop: symbol;
+    readonly Remove: symbol;
+}>;
+export type TraverseAction = typeof TraverseAction[keyof typeof TraverseAction];
+/** What a visitor returns: nothing to go on, a node to take the place of the visited one, or an instruction. */
+export type VisitResult = void | ValueNode | TraverseAction;
+export type Visitor = (node: Node) => VisitResult;
+/**
+ * Walks the tree depth-first, calling enter() before the children of a node and leave() after them.
+ * The children of a node are taken when it is entered, so nodes inserted around the visited one later
+ * are not visited, and a child taken out of the node meanwhile is skipped; a node the visitor takes out
+ * of its place itself is neither walked into nor left. A replacement is visited from its children on.
+ * Stop ends the walk at once: no node is entered or left after it. The walk is not one atomic edit:
+ * the edits before a failure stay.
+ */
+export declare class Traverser {
+    #private;
+    private readonly enter?;
+    private readonly leave?;
+    private constructor();
+    static traverse(node: Node, enter?: Visitor, leave?: Visitor): Node | null;
+    /** Returns the node that stands in the place of the visited one afterwards, null if it was removed or taken out. */
+    private visit;
+    /** Whether the node at the level and the nodes above it still stand where they were entered. */
+    private isInPlace;
+    private apply;
+}