@nette/neon 0.0.0-stage → 1.0.0

package/dist/parser.js

@@ -0,0 +1,404 @@
+import { NeonError } from "./errors.js";
+import { setSlots } from "./internal.js";
+import { quote, tokenize } from "./lexer.js";
+import { ArrayItemNode, BlockArrayNode, DocumentNode, EntityChainNode, EntityNode, InlineArrayNode, LiteralNode, StringNode, } from "./nodes.js";
+import { Cursor } from "./position.js";
+import { Chain, EscapeError, evaluateString, Int64Max, isIntegerKey, literalKey, MaxDepth } from "./values.js";
+/** Parses NEON into the lossless syntax tree. */
+export function parse(input) {
+    return new Parser(tokenize(input), input).parse();
+}
+const TooDeep = 'Too deep nesting';
+const Pairs = {
+    OpenBracket: 'CloseBracket',
+    OpenBrace: 'CloseBrace',
+    OpenParen: 'CloseParen',
+};
+/**
+ * The parser of nette/neon 3.x ported onto tokens with trivia. Line endings are trivia, but the
+ * parser sees them as it saw the Newline tokens of 3.x: before token i stands a line break if the
+ * trivia between token i-1 and token i contain a line ending.
+ */
+class Parser {
+    tokens;
+    #i = 0;
+    #newlineTaken = false;
+    #depth = 0;
+    #cursor;
+    constructor(tokens, input) {
+        this.tokens = tokens;
+        this.#cursor = new Cursor(input);
+    }
+    parse() {
+        while (this.consume('Newline')) {
+            ;
+        }
+        const value = this.parseBlock(this.getIndentation());
+        while (this.consume('Newline')) {
+            ;
+        }
+        if (this.isNext()) {
+            this.error();
+        }
+        const document = new DocumentNode(value, this.tokens.at(-1));
+        this.verify(document);
+        return document;
+    }
+    parseBlock(indent, onlyBullets = false) {
+        const depth = this.#depth;
+        try {
+            return this.parseBlockItems(indent, onlyBullets);
+        }
+        finally {
+            this.#depth = depth;
+        }
+    }
+    /**
+     * A block opened on the line of a dash is indented by a tab if the line where its indentation is first
+     * compared starts with one, by two spaces otherwise. 3.x tries the tab and then the spaces, which costs time
+     * exponential in nested dashes; the spaces fail on that line whenever the tab was chosen, so a block
+     * that fails with the tab fails as the spaces would.
+     */
+    parseDashBlock(base) {
+        const indent = { base, resolved: null, line: null };
+        try {
+            return this.parseBlock(indent);
+        }
+        catch (e) {
+            if (e instanceof NeonError && e.message !== TooDeep && indent.resolved?.endsWith('\t') && indent.line) {
+                this.#i = indent.line.i;
+                this.#newlineTaken = indent.line.newlineTaken;
+                this.error('Invalid combination of tabs and spaces');
+            }
+            throw e;
+        }
+    }
+    /** The indentation of a block, decided for a block opened on the line of a dash by the indentation of the next line. */
+    resolve(indent, next) {
+        if (typeof indent === 'string') {
+            return indent;
+        }
+        else if (indent.resolved === null) {
+            const base = this.resolve(indent.base, next);
+            indent.resolved = base + (next.startsWith(base + '\t') ? '\t' : '  ');
+            indent.line = this.mark();
+        }
+        return indent.resolved;
+    }
+    parseBlockItems(pending, onlyBullets) {
+        const items = [];
+        const keys = new KeyCheck(this);
+        while (true) {
+            const bullet = this.consumeToken('Dash');
+            let key = null;
+            let colon = null;
+            let value = null;
+            if (bullet) {
+                if (!items.length) {
+                    this.enter(bullet);
+                }
+            }
+            else if (!this.isNext() || onlyBullets) {
+                return items.length ? new BlockArrayNode(items) : null;
+            }
+            else {
+                const head = this.parseValue();
+                if ((colon = this.consumeToken('Colon', 'Equals'))) {
+                    key = keys.add(head);
+                    if (!items.length) {
+                        this.enter(head.getFirstToken());
+                    }
+                }
+                else {
+                    if (items.length) {
+                        this.error();
+                    }
+                    return head;
+                }
+            }
+            if (this.consume('Newline')) {
+                while (this.consume('Newline')) {
+                    ;
+                }
+                const nextIndent = this.getIndentation();
+                const indent = this.resolve(pending, nextIndent);
+                if (!comparable(nextIndent, indent)) {
+                    this.error('Invalid combination of tabs and spaces');
+                }
+                else if (nextIndent.length > indent.length) { // open new block
+                    value = this.parseBlock(nextIndent);
+                }
+                else if (nextIndent.length < indent.length) { // close block
+                    items.push(keys.item(bullet, key, colon, null));
+                    return new BlockArrayNode(items);
+                }
+                else if (key !== null && this.isNext('Dash')) { // special dash subblock
+                    value = this.parseBlock(indent, true);
+                }
+            }
+            else if (key === null) { // open new block after dash
+                value = this.parseDashBlock(pending);
+            }
+            else if (this.isNext()) {
+                value = this.parseValue();
+                if (this.isNext() && !this.isNext('Newline')) {
+                    this.error();
+                }
+            }
+            items.push(keys.item(bullet, key, colon, value));
+            while (this.consume('Newline')) {
+                ;
+            }
+            if (!this.isNext()) {
+                return new BlockArrayNode(items);
+            }
+            const nextIndent = this.getIndentation();
+            const indent = this.resolve(pending, nextIndent);
+            if (!comparable(nextIndent, indent)) {
+                this.error('Invalid combination of tabs and spaces');
+            }
+            else if (nextIndent.length > indent.length) {
+                this.error('Bad indentation');
+            }
+            else if (nextIndent.length < indent.length) { // close block
+                return new BlockArrayNode(items);
+            }
+        }
+    }
+    parseValue() {
+        let node;
+        let token;
+        if ((token = this.consumeToken('String'))) {
+            try {
+                evaluateString(token.text);
+            }
+            catch (e) {
+                if (e instanceof EscapeError) {
+                    this.error(e.message, token);
+                }
+                throw e;
+            }
+            node = new StringNode(token);
+        }
+        else if ((token = this.consumeToken('Literal'))) {
+            node = new LiteralNode(token);
+        }
+        else if (this.isNext('OpenBracket', 'OpenParen', 'OpenBrace')) {
+            node = this.parseBraces();
+        }
+        else {
+            this.error();
+        }
+        return this.parseEntity(node);
+    }
+    parseEntity(node) {
+        if (!this.isNext('OpenParen')) {
+            return node;
+        }
+        this.checkEntityHead(node);
+        const entities = [new EntityNode(node, this.parseBraces())];
+        let token;
+        while ((token = this.consumeToken('Literal'))) {
+            const head = new LiteralNode(token);
+            this.checkEntityHead(head);
+            if (this.isNext('OpenParen')) {
+                entities.push(new EntityNode(head, this.parseBraces()));
+            }
+            else {
+                entities.push(new EntityNode(head, null));
+                break;
+            }
+        }
+        return entities.length === 1
+            ? entities[0]
+            : new EntityChainNode(entities);
+    }
+    parseBraces() {
+        const depth = this.#depth;
+        try {
+            return this.parseBracesItems();
+        }
+        finally {
+            this.#depth = depth;
+        }
+    }
+    parseBracesItems() {
+        const open = this.consumeToken();
+        this.enter(open);
+        const end = Pairs[open.kind];
+        const items = [];
+        const keys = new KeyCheck(this);
+        while (true) {
+            while (this.consume('Newline')) {
+                ;
+            }
+            const close = this.consumeToken(end);
+            if (close) {
+                return new InlineArrayNode(open, items, close);
+            }
+            const head = this.parseValue();
+            let item;
+            const colon = this.consumeToken('Colon', 'Equals');
+            if (colon) {
+                const key = keys.add(head);
+                const value = this.isNext('Newline', 'Comma', end) ? null : this.parseValue();
+                item = keys.item(null, key, colon, value);
+            }
+            else {
+                item = keys.item(null, null, null, head);
+            }
+            items.push(item);
+            const before = this.mark();
+            while (this.consume('Newline')) {
+                ;
+            }
+            const comma = this.consumeToken('Comma');
+            if (comma) {
+                item[setSlots]({ separator: comma });
+            }
+            if (before.i !== this.#i || before.newlineTaken !== this.#newlineTaken) {
+                continue;
+            }
+            else if (!this.isNext(end)) {
+                this.error();
+            }
+        }
+    }
+    /** One level deeper into arrays. */
+    enter(token) {
+        if (++this.#depth > MaxDepth) {
+            this.error(TooDeep, token);
+        }
+    }
+    checkEntityHead(node) {
+        const value = node instanceof StringNode
+            ? node.toValue()
+            : node instanceof LiteralNode
+                ? node.token.text
+                : null;
+        if (value === Chain) {
+            this.error(`Entity name '${Chain}' is reserved`, node.getFirstToken());
+        }
+    }
+    /** Every token of the stream is in the tree exactly once, in the order of the stream. */
+    verify(document) {
+        const tokens = document.getTokens();
+        if (tokens.length !== this.tokens.length || tokens.some((token, i) => token !== this.tokens[i])) {
+            throw new Error('Internal error: the tree does not hold the tokens of the input in order.');
+        }
+    }
+    // stream of tokens with line breaks
+    newlineBefore(i) {
+        const trivia = i === 0 ? this.tokens[0].leadingTrivia : this.tokens[i - 1].trailingTrivia;
+        return trivia.some((t) => t.kind === 'LineEnding');
+    }
+    peek() {
+        return !this.#newlineTaken && this.newlineBefore(this.#i)
+            ? 'Newline'
+            : this.tokens[this.#i].kind;
+    }
+    /** Whether the next item is of one of the kinds; without kinds, whether anything but the end follows. */
+    isNext(...kinds) {
+        const next = this.peek();
+        return kinds.length ? kinds.includes(next) : next !== 'EndOfFile';
+    }
+    consume(kind) {
+        if (this.peek() === kind) {
+            this.#newlineTaken = true;
+            return true;
+        }
+        return false;
+    }
+    consumeToken(...kinds) {
+        const next = this.peek();
+        if (next === 'Newline' || next === 'EndOfFile' || (kinds.length && !kinds.includes(next))) {
+            return null;
+        }
+        this.#newlineTaken = false;
+        return this.tokens[this.#i++];
+    }
+    mark() {
+        return { i: this.#i, newlineTaken: this.#newlineTaken };
+    }
+    /** Indentation of the next token if it starts a line: the whitespace right after a line ending or the start. */
+    getIndentation() {
+        const i = this.#i;
+        const trivia = i === 0
+            ? this.tokens[0].leadingTrivia.filter((t) => t.kind !== 'ByteOrderMark')
+            : [...this.tokens[i - 1].trailingTrivia, ...this.tokens[i].leadingTrivia];
+        const last = trivia.at(-1);
+        const before = trivia.at(-2);
+        return last?.kind === 'Whitespace' && (trivia.length === 1 ? i === 0 : before?.kind === 'LineEnding')
+            ? last.text
+            : '';
+    }
+    error(message, token) {
+        if (token) {
+            throw new NeonError(message, token.origin);
+        }
+        const next = this.peek();
+        if (next === 'Newline') {
+            const [offset, count] = this.locateNewline();
+            throw new NeonError(message ?? `Unexpected '${'<new line>'.repeat(Math.min(count, 40))}'`, this.#cursor.at(offset));
+        }
+        const current = this.tokens[this.#i];
+        throw new NeonError(message ?? (next === 'EndOfFile' ? 'Unexpected end' : `Unexpected '${quote(current.text)}'`), current.origin);
+    }
+    /** Offset of the line break before the current token, and how many line endings follow one another there. */
+    locateNewline() {
+        const i = this.#i;
+        const prev = this.tokens[i - 1];
+        let offset = prev ? prev.origin.utf16Offset + prev.text.length : 0;
+        const trivia = [...prev?.trailingTrivia ?? [], ...this.tokens[i].leadingTrivia];
+        let k = 0;
+        while (trivia[k].kind !== 'LineEnding') {
+            offset += trivia[k++].text.length;
+        }
+        let count = 0;
+        while (trivia[k + count]?.kind === 'LineEnding') {
+            count++;
+        }
+        return [offset, count];
+    }
+}
+/** Checks explicit keys of one array for duplicates and implicit keys for overflow. */
+class KeyCheck {
+    parser;
+    #seen = (new Set);
+    #max = null;
+    constructor(parser) {
+        this.parser = parser;
+    }
+    add(node) {
+        if (!(node instanceof StringNode || node instanceof LiteralNode)) {
+            this.parser.error('Unacceptable key', node.getFirstToken());
+        }
+        const key = node instanceof StringNode ? node.toValue() : literalKey(node.token.text);
+        if (this.#seen.has(key)) {
+            this.parser.error(`Duplicated key '${key}'`, node.token);
+        }
+        this.#seen.add(key);
+        return node;
+    }
+    /** Makes the item and accounts its key, explicit or implicit. */
+    item(bullet, key, colon, value) {
+        const item = new ArrayItemNode(bullet, key, colon, value);
+        let canonical = item.getKeyValue();
+        if (canonical === null) {
+            const next = this.#max === null ? 0n : this.#max + 1n;
+            if (next > Int64Max) {
+                this.parser.error('Implicit key overflow', (value ?? item).getFirstToken());
+            }
+            canonical = String(next);
+        }
+        if (isIntegerKey(canonical)) {
+            const n = BigInt(canonical);
+            this.#max = this.#max === null || n > this.#max ? n : this.#max;
+        }
+        return item;
+    }
+}
+function comparable(a, b) {
+    const n = Math.min(a.length, b.length);
+    return a.slice(0, n) === b.slice(0, n);
+}

package/dist/position.d.ts

@@ -0,0 +1,8 @@
+import type { Position } from './token.ts';
+/** Computes positions of increasing UTF-16 offsets in one pass. */
+export declare class Cursor {
+    #private;
+    private readonly s;
+    constructor(s: string);
+    at(index: number): Position;
+}

package/dist/position.js

@@ -0,0 +1,40 @@
+/** Computes positions of increasing UTF-16 offsets in one pass. */
+export class Cursor {
+    s;
+    #index = 0;
+    #line = 1;
+    #column = 1;
+    #offset = 0;
+    constructor(s) {
+        this.s = s;
+    }
+    at(index) {
+        if (index < this.#index) {
+            this.#index = 0;
+            this.#line = 1;
+            this.#column = 1;
+            this.#offset = 0;
+        }
+        const s = this.s;
+        for (let i = this.#index; i < index; i++) {
+            const code = s.charCodeAt(i);
+            if (code === 10) {
+                this.#line++;
+                this.#column = 1;
+                this.#offset++;
+            }
+            else if (code >= 0xDC00 && code <= 0xDFFF && i > 0 && isHighSurrogate(s.charCodeAt(i - 1))) {
+                // second half of a code point
+            }
+            else {
+                this.#column++;
+                this.#offset++;
+            }
+        }
+        this.#index = index;
+        return Object.freeze({ line: this.#line, column: this.#column, offset: this.#offset, utf16Offset: index });
+    }
+}
+function isHighSurrogate(code) {
+    return code >= 0xD800 && code <= 0xDBFF;
+}

package/dist/printer.d.ts

@@ -0,0 +1,8 @@
+import type { Node } from './nodes.ts';
+import type { Position, Token } from './token.ts';
+/** The source text of the node, with the trivia of its tokens. */
+export declare function print(node: Node): string;
+/** The text of the node without the leading trivia of its first token and the trailing trivia of its last. */
+export declare function printText(node: Node): string;
+/** Current positions of all tokens of the tree in one walk, for an editor asking about many of them. */
+export declare function locateAll(root: Node): Map<Token, Position>;

package/dist/printer.js

@@ -0,0 +1,40 @@
+import { Cursor } from "./position.js";
+/** The source text of the node, with the trivia of its tokens. */
+export function print(node) {
+    return node.print();
+}
+/** The text of the node without the leading trivia of its first token and the trailing trivia of its last. */
+export function printText(node) {
+    const tokens = node.getTokens();
+    let res = '';
+    tokens.forEach((token, i) => {
+        if (i > 0) {
+            res += triviaText(token.leadingTrivia);
+        }
+        res += token.text;
+        if (i < tokens.length - 1) {
+            res += triviaText(token.trailingTrivia);
+        }
+    });
+    return res;
+}
+/** Current positions of all tokens of the tree in one walk, for an editor asking about many of them. */
+export function locateAll(root) {
+    const tokens = root.getTokens();
+    const offsets = [];
+    let text = '';
+    for (const token of tokens) {
+        text += triviaText(token.leadingTrivia);
+        offsets.push(text.length);
+        text += token.text + triviaText(token.trailingTrivia);
+    }
+    const cursor = new Cursor(text);
+    return new Map(tokens.map((token, i) => [token, cursor.at(offsets[i])]));
+}
+function triviaText(trivia) {
+    let res = '';
+    for (const t of trivia) {
+        res += t.text;
+    }
+    return res;
+}

package/dist/scanner.d.ts

@@ -0,0 +1,30 @@
+import type { TokenKind, TriviaKind } from './token.ts';
+export type ScannedKind = TokenKind | 'Trivia';
+export interface Scanned {
+    kind: ScannedKind;
+    triviaKind?: TriviaKind;
+    start: number;
+    end: number;
+}
+/** Reads tokens and trivia one by one. Returns null on an error, with errorAt set. */
+export declare class Scanner {
+    #private;
+    private readonly s;
+    errorAt: number;
+    constructor(s: string);
+    get position(): number;
+    next(): Scanned | null;
+    /** Is the character one of the given ones, a space, a tab, or a line ending? */
+    private isAny;
+    private isSpace;
+    private isLineEnding;
+    private scanLiteral;
+    private scanSingleQuoted;
+    private scanDoubleQuoted;
+    /**
+     * Content runs to the first line after the first one that starts with optional spaces and the
+     * closing triple; only without such a line may the first line itself close an empty string.
+     */
+    private scanMultiline;
+    private closingAt;
+}