@liquidmetal-ai/drizzle 0.0.1

package/dist/appify/parse.js

@@ -0,0 +1,579 @@
+const TOKEN_PATTERNS = [
+    { type: 'number', pattern: /^\d+(\.\d+)?/ },
+    { type: 'comment', pattern: /^\/\/[^\n]*/ },
+    { type: 'operator', pattern: /^[+\-*/%=]/ },
+    { type: 'whitespace', pattern: /^[ \t]+/ },
+    { type: 'boolean', pattern: /^(true|false)/ },
+    { type: 'string', pattern: /^'(?:[^'\\\n]|\\.)*'/ },
+    { type: 'string', pattern: /^"(?:[^"\\\n]|\\.)*"/ },
+    { type: 'identifier', pattern: /^[a-zA-Z_]\w*/ },
+    { type: 'newline', pattern: /^\n/ },
+    { type: 'curly', pattern: /^[{}]/ },
+    { type: 'comma', pattern: /^,/ },
+    // eslint-disable-next-line no-useless-escape
+    { type: 'bracket', pattern: /^[\[\]]/ },
+    { type: 'parenthesis', pattern: /^[()]/ },
+    { type: 'unknown', pattern: /^./ },
+];
+export class Tokenizer {
+    input;
+    cursor = 0;
+    line = 1;
+    column = 1;
+    constructor(input) {
+        this.input = input;
+    }
+    // Get the next token from the input. The optional ignore parameter
+    // can be used to skip certain token types. This could possibly be
+    // removed, but whitespace might be useful at some point.
+    next(ignore = ['whitespace']) {
+        if (this.eof()) {
+            return {
+                type: 'end',
+                value: '',
+                start: this.cursor,
+                end: this.cursor,
+                line: this.line,
+                column: this.column,
+            };
+        }
+        // Try to match a token in order of precedence.
+        for (const { type, pattern } of TOKEN_PATTERNS) {
+            const result = this.input.slice(this.cursor).match(pattern);
+            if (result) {
+                const token = {
+                    type,
+                    value: result[0],
+                    start: this.cursor,
+                    end: this.cursor + result[0].length,
+                    line: this.line,
+                    column: this.column,
+                };
+                this.cursor += result[0].length;
+                if (type === 'newline') {
+                    this.line++;
+                    this.column = 1;
+                }
+                else {
+                    this.column += result[0].length;
+                }
+                // Skip any ignored tokens.
+                if (ignore.includes(token.type)) {
+                    return this.next(ignore);
+                }
+                return token;
+            }
+        }
+        // This should be unreachable? Maybe?
+        throw new Error(`Unknown token at ${this.line}:${this.column}`);
+    }
+    peek(ignore = ['whitespace']) {
+        const cursor = this.cursor;
+        const line = this.line;
+        const column = this.column;
+        const token = this.next(ignore);
+        this.cursor = cursor;
+        this.line = line;
+        this.column = column;
+        return token;
+    }
+    eof() {
+        return this.cursor >= this.input.length;
+    }
+}
+// Parser class that takes a tokenizer and produces an AST. This
+// completely consumes the tokenizer.
+export class Parser {
+    tokenizer;
+    errors = [];
+    constructor(tokenizer) {
+        this.tokenizer = tokenizer;
+    }
+    // Parse the input and return the AST. The optional node parameter
+    // is used when this is recursed.
+    parse(root) {
+        if (root === undefined) {
+            root = {
+                type: 'block',
+                children: [],
+                line: this.tokenizer.line,
+                column: this.tokenizer.column,
+                start: this.tokenizer.cursor,
+                end: 0,
+            };
+        }
+        while (!this.tokenizer.eof()) {
+            const token = this.tokenizer.peek();
+            switch (token.type) {
+                case 'identifier':
+                    root.children.push(this.parseIdentifier());
+                    break;
+                case 'newline':
+                    root.children.push(this.tokenizer.next());
+                    break;
+                case 'comment':
+                    root.children.push(this.tokenizer.next());
+                    break;
+                case 'curly':
+                    if (token.value === '}') {
+                        root.end = this.tokenizer.cursor;
+                        return root;
+                    }
+                    else {
+                        // wtf?
+                        this.errors.push({
+                            expected: ['identifier'],
+                            received: 'curly',
+                            message: `Unexpected ${token.value} at ${token.line}:${token.column}`,
+                            line: token.line,
+                            column: token.column,
+                            start: token.start,
+                            end: token.end,
+                        });
+                        this.tokenizer.next();
+                    }
+                    break;
+                case 'end':
+                    root.end = this.tokenizer.cursor;
+                    return root;
+                default:
+                    this.errors.push({
+                        expected: ['identifier', 'newline', 'comment'],
+                        received: token.type,
+                        message: `Unexpected ${token.type} at ${token.line}:${token.column}`,
+                        line: token.line,
+                        column: token.column,
+                        start: token.start,
+                        end: token.end,
+                    });
+                    root.end = this.tokenizer.cursor;
+                    return root;
+            }
+        }
+        // We should never reach this point, but if we do, we're at the
+        // end.
+        root.end = this.tokenizer.cursor;
+        return root;
+    }
+    // Parse an identifier, which could be a stanza, assignment.
+    parseIdentifier() {
+        const token = this.tokenizer.next();
+        // Assert token is an identifier.
+        if (token.type !== 'identifier') {
+            // should be unreachable
+            throw new Error(`Expected identifier but got ${token.type} at ${token.line}:${token.column}`);
+        }
+        const next = this.tokenizer.peek();
+        if (next.type === 'curly' && next.value === '{') {
+            return this.parseStanza(token);
+        }
+        if (['string', 'number', 'boolean'].includes(next.type)) {
+            return this.parseStanza(token);
+        }
+        if (next.type === 'operator' && next.value === '=') {
+            return this.parseAssignment(token);
+        }
+        this.errors.push({
+            expected: ['curly', 'string', 'operator'],
+            received: next.type,
+            message: `Expected {, argument or operator at ${next.line}:${next.column}`,
+            line: next.line,
+            column: next.column,
+            start: next.start,
+            end: next.end,
+        });
+        return { type: 'empty', line: next.line, column: next.column, start: next.start, end: next.end };
+    }
+    // Parse a stanza, which is a block of assignments and other
+    // stanzas.
+    parseStanza(token) {
+        const stanza = {
+            type: 'stanza',
+            name: token.value,
+            args: [],
+            block: null,
+            line: token.line,
+            column: token.column,
+            start: token.start,
+            end: token.end,
+        };
+        let next = this.tokenizer.peek();
+        while (next.type !== 'curly') {
+            if (['string', 'number', 'boolean'].includes(next.type)) {
+                stanza.args.push(this.tokenizer.next());
+                next = this.tokenizer.peek();
+                continue;
+            }
+            this.errors.push({
+                expected: ['string', 'number', 'boolean'],
+                received: next.type,
+                message: `Expected argument at ${next.line}:${next.column}`,
+                line: next.line,
+                column: next.column,
+                start: next.start,
+                end: next.end,
+            });
+            this.tokenizer.next();
+            next = this.tokenizer.peek();
+        }
+        this.expect('curly', '{');
+        stanza.block = this.parse();
+        this.expect('curly', '}');
+        stanza.end = this.tokenizer.cursor;
+        return stanza;
+    }
+    // Parse an assignment, which is an assignment to an expression.
+    parseAssignment(token) {
+        if (token.type !== 'identifier') {
+            throw new Error(`Expected identifier but got ${token.type} at ${token.line}:${token.column}`);
+        }
+        const key = token;
+        const operator = this.expect('operator', '=');
+        const value = this.parseExpression();
+        return {
+            type: 'assignment',
+            operator,
+            key,
+            value,
+            line: key.line,
+            column: key.column,
+            start: key.start,
+            end: value.end,
+        };
+    }
+    // Parse an expression. We're not building a calculator (yet), so we
+    // just accept a value on the right for now.
+    parseExpression(ignore) {
+        const token = this.tokenizer.next(ignore || ['whitespace']);
+        switch (token.type) {
+            case 'string':
+            case 'number':
+            case 'boolean':
+                return token;
+            case 'bracket':
+                if (token.value === '[') {
+                    return this.parseArray();
+                }
+                break;
+            case 'curly':
+                if (token.value === '{') {
+                    return this.parseObject();
+                }
+                else {
+                    break;
+                }
+            default:
+                break;
+        }
+        // Recover by calling it a string.
+        this.errors.push({
+            expected: ['expression'],
+            received: token.type,
+            message: `Expected expression but got ${token.type} at ${token.line}:${token.column}`,
+            line: token.line,
+            column: token.column,
+            start: token.start,
+            end: token.end,
+        });
+        return {
+            type: 'string',
+            value: '"<error>"',
+            line: token.line,
+            column: token.column,
+            start: token.start,
+            end: token.end,
+        };
+    }
+    parseArray() {
+        // the first thing we expect are expression.
+        const elements = [];
+        // This whole ignoring newlines thing is really just context.
+        let next = this.tokenizer.peek(['whitespace', 'newline']);
+        while (true) {
+            elements.push(this.parseExpression(['whitespace', 'newline']));
+            next = this.tokenizer.peek(['whitespace', 'newline']);
+            if (next.type === 'bracket' && next.value === ']') {
+                this.tokenizer.next(['whitespace', 'newline']);
+                return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+            }
+            if (next.type === 'comma') {
+                this.tokenizer.next();
+                next = this.tokenizer.peek(['whitespace', 'newline']);
+                // Special case: allow an ending comma because it's nice.
+                if (next.type === 'bracket' && next.value === ']') {
+                    this.tokenizer.next(['whitespace', 'newline']);
+                    return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+                }
+                continue;
+            }
+            // Should have been a closing bracket or a comma, so we "close" and return for recovery.
+            this.errors.push({
+                expected: ['bracket', 'comma'],
+                received: next.type,
+                message: `Expected ] or , but got ${next.type} at ${next.line}:${next.column}`,
+                line: next.line,
+                column: next.column,
+                start: next.start,
+                end: next.end,
+            });
+            return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+        }
+    }
+    parseObject() {
+        const properties = [];
+        let next = this.tokenizer.peek(['whitespace', 'newline']);
+        while (true) {
+            if (next.type === 'end') {
+                this.errors.push({
+                    expected: ['identifier'],
+                    received: 'end',
+                    message: `Unexpected end of input at ${next.line}:${next.column}`,
+                    line: next.line,
+                    column: next.column,
+                    start: next.start,
+                    end: next.end,
+                });
+                break;
+            }
+            if (next.type === 'curly' && next.value === '}') {
+                this.tokenizer.next(['whitespace', 'newline']);
+                break;
+            }
+            if (next.type === 'identifier') {
+                properties.push(this.parseAssignment(this.tokenizer.next(['whitespace', 'newline'])));
+                next = this.tokenizer.peek(['whitespace', 'newline']);
+                continue;
+            }
+            const token = this.tokenizer.next(['whitespace', 'newline']);
+            next = this.tokenizer.peek(['whitespace', 'newline']);
+            this.errors.push({
+                expected: ['identifier'],
+                received: next.type,
+                message: `Expected identifier but got ${token.type} at ${next.line}:${next.column}`,
+                line: next.line,
+                column: next.column,
+                start: next.start,
+                end: next.end,
+            });
+        }
+        return { type: 'object', properties, line: next.line, column: next.column, start: next.start, end: next.end };
+    }
+    // Expect a token of a certain type, optionally with a certain
+    // value.
+    expect(type, value) {
+        const token = this.tokenizer.peek();
+        if (token.type === 'end' && type !== 'end') {
+            this.errors.push({
+                expected: [type],
+                received: 'end',
+                message: `Unexpected end of input at ${token.line}:${token.column}`,
+                line: token.line,
+                column: token.column,
+                start: token.start,
+                end: token.end,
+            });
+            return this.tokenizer.next();
+        }
+        if (token.type !== type) {
+            this.errors.push({
+                expected: [type],
+                received: token.type,
+                message: `Expected ${type} but got ${token.type} at ${token.line}:${token.column}`,
+                line: token.line,
+                column: token.column,
+                start: token.start,
+                end: token.end,
+            });
+            // Naively try to recover by returning a working token.
+            return {
+                type: type,
+                value: token.value,
+                start: token.start,
+                end: token.end,
+                line: token.line,
+                column: token.column,
+            };
+        }
+        if (value && token.value !== value) {
+            this.errors.push({
+                message: `Expected ${value} but got ${token.value} at ${token.line}:${token.column}`,
+                line: token.line,
+                column: token.column,
+                start: token.start,
+                end: token.end,
+            });
+            // Again, naively try to recover with a working token.
+            return {
+                type,
+                value,
+                start: token.start,
+                end: token.end,
+                line: token.line,
+                column: token.column,
+            };
+        }
+        // We match, so consume the token.
+        return this.tokenizer.next();
+    }
+}
+// JSONTranslator takes an AST and translates it into a JSON object
+// (any).
+export class JSONTranslator {
+    errors;
+    constructor() {
+        this.errors = [];
+    }
+    // Translate the AST into a JSON serializable object.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    translate(ast) {
+        return this.translateRoot(ast);
+    }
+    // Start at the root and translate the children.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    translateRoot(node) {
+        const obj = {}; // eslint-disable-line @typescript-eslint/no-explicit-any
+        for (const child of node.children) {
+            if (child.type === 'stanza') {
+                if (!obj[child.name]) {
+                    obj[child.name] = [];
+                }
+                const stanza = this.translateStanza(child);
+                obj[child.name].push(stanza);
+            }
+            else {
+                this.errors.push({
+                    message: `unexpected ${child.type} at ${child.line}:${child.column}`,
+                    line: 1,
+                    column: 1,
+                });
+            }
+        }
+        return obj;
+    }
+    // translateStanza takes a stanza node and translates it into a JSON
+    // object. This looks remarkably like translateRoot, and maybe
+    // there's some kind of generalized recursion we could use, but it's
+    // late.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    translateStanza(node) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const obj = {};
+        if (node.args && node.args[0]) {
+            obj['name'] = node.args[0].value.slice(1, -1);
+        }
+        const children = node.block ? node.block.children : [];
+        for (const child of children) {
+            if (child.type === 'assignment') {
+                // Remember we skipped expressions, so...
+                if (child.value.type === 'string') {
+                    obj[child.key.value] = child.value.value.slice(1, -1);
+                }
+                else if (child.value.type === 'number') {
+                    obj[child.key.value] = parseFloat(child.value.value);
+                }
+                else if (child.value.type === 'boolean') {
+                    obj[child.key.value] = child.value.value === 'true';
+                }
+                else if (child.value.type === 'array') {
+                    obj[child.key.value] = child.value.elements.map((el) => {
+                        if (el.type === 'string') {
+                            return el.value.slice(1, -1);
+                        }
+                        else if (el.type === 'number') {
+                            return parseFloat(el.value);
+                        }
+                        else if (el.type === 'boolean') {
+                            return el.value === 'true';
+                        }
+                        return null;
+                    });
+                }
+            }
+            if (child.type === 'stanza') {
+                if (!obj[child.name]) {
+                    obj[child.name] = [];
+                }
+                obj[child.name].push(this.translateStanza(child));
+            }
+        }
+        return obj;
+    }
+}
+// fmt is an ugly mess of string concatenation.
+export function fmt(ast, indent = 0, sibling) {
+    if (ast === null) {
+        return '';
+    }
+    const INDENT = ' '.repeat(indent);
+    let str = '';
+    // Just spit out token values as they are encountered.
+    switch (ast.type) {
+        case 'stanza':
+            str += `${INDENT}${ast.name} `;
+            for (const arg of ast.args) {
+                str += `${arg.value} `;
+            }
+            str += '{';
+            str += fmt(ast.block, indent + 2);
+            if (!ast.block?.children.every((child) => child.type === 'newline')) {
+                str += `${INDENT}`;
+            }
+            str += '}';
+            break;
+        case 'block':
+            if (ast.children.every((child) => child.type === 'newline')) {
+                // Do nothing.
+            }
+            else {
+                for (const child of ast.children) {
+                    str += fmt(child, indent, sibling);
+                    sibling = child;
+                }
+            }
+            break;
+        case 'assignment':
+            str += `${INDENT}${fmt(ast.key)} = ${fmt(ast.value)}`;
+            break;
+        case 'expression':
+            str += `${fmt(ast.left)} ${ast.operator.value} ${fmt(ast.right)}`;
+            break;
+        case 'comment':
+            if (sibling?.type !== 'newline') {
+                str += `  `;
+            }
+            else {
+                str += `${INDENT}`;
+            }
+            str += `${ast.value}`;
+            break;
+        case 'empty':
+            break;
+        case 'array':
+            // eslint-disable-next-line no-case-declarations
+            const oneLine = `[${ast.elements.map((el) => fmt(el)).join(', ')}]`;
+            if (oneLine.length > 80) {
+                str += `[`;
+                for (const el of ast.elements) {
+                    str += `\n${fmt(el, indent + 2)}`;
+                }
+                str += `\n${INDENT}]`;
+            }
+            else {
+                return oneLine;
+            }
+            break;
+        case 'object':
+            str += '{';
+            if (ast.properties.length > 0) {
+                str += '\n';
+                for (const assign of ast.properties) {
+                    str += `${fmt(assign, indent + 2)}\n`;
+                }
+                str += `\n${INDENT}}`;
+            }
+            break;
+        default:
+            str += `${ast?.value}`;
+    }
+    return str;
+}

package/dist/appify/parse.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=parse.test.d.ts.map

package/dist/appify/parse.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse.test.d.ts","sourceRoot":"","sources":["../../src/appify/parse.test.ts"],"names":[],"mappings":""}