@liquidmetal-ai/drizzle 0.0.1

package/src/appify/parse.ts

@@ -0,0 +1,744 @@
+type TokenBase = {
+  value: string;
+  start: number;
+  end: number;
+  line: number;
+  column: number;
+};
+
+export type TokenNumber = TokenBase & {
+  type: 'number';
+};
+
+export type TokenOperator = TokenBase & {
+  type: 'operator';
+};
+
+export type TokenParenthesis = TokenBase & {
+  type: 'parenthesis';
+};
+
+export type TokenWhitespace = TokenBase & {
+  type: 'whitespace';
+};
+
+export type TokenBoolean = TokenBase & {
+  type: 'boolean';
+};
+
+export type TokenString = TokenBase & {
+  type: 'string';
+};
+
+export type TokenIdentifier = TokenBase & {
+  type: 'identifier';
+};
+
+export type TokenComment = TokenBase & {
+  type: 'comment';
+};
+
+export type TokenNewline = TokenBase & {
+  type: 'newline';
+};
+
+export type TokenUnknown = TokenBase & {
+  type: 'unknown';
+};
+
+export type TokenCurly = TokenBase & {
+  type: 'curly';
+};
+
+export type TokenBracket = TokenBase & {
+  type: 'bracket';
+};
+
+export type TokenComma = TokenBase & {
+  type: 'comma';
+};
+
+export type TokenEnd = TokenBase & {
+  type: 'end';
+};
+
+type Token =
+  | TokenNumber
+  | TokenOperator
+  | TokenParenthesis
+  | TokenWhitespace
+  | TokenBoolean
+  | TokenString
+  | TokenIdentifier
+  | TokenComment
+  | TokenNewline
+  | TokenUnknown
+  | TokenCurly
+  | TokenBracket
+  | TokenEnd
+  | TokenComma;
+
+type TokenType = Token['type'];
+
+const TOKEN_PATTERNS: { type: TokenType; pattern: RegExp }[] = [
+  { type: 'number', pattern: /^\d+(\.\d+)?/ },
+  { type: 'comment', pattern: /^\/\/[^\n]*/ },
+  { type: 'operator', pattern: /^[+\-*/%=]/ },
+  { type: 'whitespace', pattern: /^[ \t]+/ },
+  { type: 'boolean', pattern: /^(true|false)/ },
+  { type: 'string', pattern: /^'(?:[^'\\\n]|\\.)*'/ },
+  { type: 'string', pattern: /^"(?:[^"\\\n]|\\.)*"/ },
+  { type: 'identifier', pattern: /^[a-zA-Z_]\w*/ },
+  { type: 'newline', pattern: /^\n/ },
+  { type: 'curly', pattern: /^[{}]/ },
+  { type: 'comma', pattern: /^,/ },
+  // eslint-disable-next-line no-useless-escape
+  { type: 'bracket', pattern: /^[\[\]]/ },
+  { type: 'parenthesis', pattern: /^[()]/ },
+  { type: 'unknown', pattern: /^./ },
+];
+
+export class Tokenizer {
+  cursor = 0;
+  line = 1;
+  column = 1;
+
+  constructor(private input: string) {}
+
+  // Get the next token from the input. The optional ignore parameter
+  // can be used to skip certain token types. This could possibly be
+  // removed, but whitespace might be useful at some point.
+  next(ignore: TokenType[] = ['whitespace']): Token {
+    if (this.eof()) {
+      return {
+        type: 'end',
+        value: '',
+        start: this.cursor,
+        end: this.cursor,
+        line: this.line,
+        column: this.column,
+      };
+    }
+
+    // Try to match a token in order of precedence.
+    for (const { type, pattern } of TOKEN_PATTERNS) {
+      const result = this.input.slice(this.cursor).match(pattern);
+      if (result) {
+        const token = {
+          type,
+          value: result[0],
+          start: this.cursor,
+          end: this.cursor + result[0].length,
+          line: this.line,
+          column: this.column,
+        };
+        this.cursor += result[0].length;
+        if (type === 'newline') {
+          this.line++;
+          this.column = 1;
+        } else {
+          this.column += result[0].length;
+        }
+        // Skip any ignored tokens.
+        if (ignore.includes(token.type)) {
+          return this.next(ignore);
+        }
+        return token;
+      }
+    }
+
+    // This should be unreachable? Maybe?
+    throw new Error(`Unknown token at ${this.line}:${this.column}`);
+  }
+
+  peek(ignore: TokenType[] = ['whitespace']): Token {
+    const cursor = this.cursor;
+    const line = this.line;
+    const column = this.column;
+    const token = this.next(ignore);
+    this.cursor = cursor;
+    this.line = line;
+    this.column = column;
+    return token;
+  }
+
+  eof(): boolean {
+    return this.cursor >= this.input.length;
+  }
+}
+
+export type NodeType = Node['type'] | TokenType;
+
+export type Node =
+  | StanzaNode
+  | AssignmentNode
+  | BlockNode
+  | ExpressionNode
+  | EmptyNode
+  | ArrayNode
+  | ObjectNode
+  | Token;
+
+type BaseNode = {
+  line: number;
+  column: number;
+  start: number;
+  end: number;
+};
+
+export type StanzaNode = BaseNode & {
+  type: 'stanza';
+  name: string;
+  args: (TokenString | TokenNumber | TokenBoolean)[];
+  block: BlockNode | null;
+};
+
+export type BlockNode = BaseNode & {
+  type: 'block';
+  children: (Node | TokenNewline | TokenComment)[];
+};
+
+export type AssignmentNode = BaseNode & {
+  type: 'assignment';
+  operator: TokenOperator;
+  key: Extract<Token, { type: 'identifier' }>;
+  value: ExpressionNode | Extract<Token, { type: 'string' | 'number' | 'boolean' }> | ArrayNode | ObjectNode;
+};
+
+export type ArrayNode = BaseNode & {
+  type: 'array';
+  elements: (ExpressionNode | Extract<Token, { type: 'string' | 'number' | 'boolean' }> | ArrayNode | ObjectNode)[];
+};
+
+export type ObjectNode = BaseNode & {
+  type: 'object';
+  properties: AssignmentNode[];
+};
+
+export type ExpressionNode = BaseNode & {
+  type: 'expression';
+  operator: TokenOperator;
+  left: Extract<Token, { type: 'string' | 'number' | 'boolean' }>;
+  right: ExpressionNode | Extract<Token, { type: 'string' | 'number' | 'boolean' }>;
+};
+
+export type EmptyNode = BaseNode & {
+  type: 'empty';
+};
+
+type ParserError = {
+  expected?: NodeType[];
+  received?: NodeType;
+  message: string;
+  line: number;
+  column: number;
+  start: number;
+  end: number;
+};
+
+// Parser class that takes a tokenizer and produces an AST. This
+// completely consumes the tokenizer.
+export class Parser {
+  errors: ParserError[] = [];
+
+  constructor(private tokenizer: Tokenizer) {}
+
+  // Parse the input and return the AST. The optional node parameter
+  // is used when this is recursed.
+  parse(root?: BlockNode): BlockNode {
+    if (root === undefined) {
+      root = {
+        type: 'block',
+        children: [],
+        line: this.tokenizer.line,
+        column: this.tokenizer.column,
+        start: this.tokenizer.cursor,
+        end: 0,
+      };
+    }
+    while (!this.tokenizer.eof()) {
+      const token = this.tokenizer.peek();
+      switch (token.type) {
+        case 'identifier':
+          root.children.push(this.parseIdentifier());
+          break;
+        case 'newline':
+          root.children.push(this.tokenizer.next() as TokenNewline);
+          break;
+        case 'comment':
+          root.children.push(this.tokenizer.next() as TokenComment);
+          break;
+        case 'curly':
+          if (token.value === '}') {
+            root.end = this.tokenizer.cursor;
+            return root;
+          } else {
+            // wtf?
+            this.errors.push({
+              expected: ['identifier'],
+              received: 'curly',
+              message: `Unexpected ${token.value} at ${token.line}:${token.column}`,
+              line: token.line,
+              column: token.column,
+              start: token.start,
+              end: token.end,
+            });
+            this.tokenizer.next();
+          }
+          break;
+        case 'end':
+          root.end = this.tokenizer.cursor;
+          return root;
+        default:
+          this.errors.push({
+            expected: ['identifier', 'newline', 'comment'],
+            received: token.type,
+            message: `Unexpected ${token.type} at ${token.line}:${token.column}`,
+            line: token.line,
+            column: token.column,
+            start: token.start,
+            end: token.end,
+          });
+          root.end = this.tokenizer.cursor;
+          return root;
+      }
+    }
+    // We should never reach this point, but if we do, we're at the
+    // end.
+    root.end = this.tokenizer.cursor;
+    return root;
+  }
+
+  // Parse an identifier, which could be a stanza, assignment.
+  parseIdentifier(): Node {
+    const token = this.tokenizer.next();
+    // Assert token is an identifier.
+    if (token.type !== 'identifier') {
+      // should be unreachable
+      throw new Error(`Expected identifier but got ${token.type} at ${token.line}:${token.column}`);
+    }
+
+    const next = this.tokenizer.peek();
+    if (next.type === 'curly' && next.value === '{') {
+      return this.parseStanza(token);
+    }
+    if (['string', 'number', 'boolean'].includes(next.type)) {
+      return this.parseStanza(token);
+    }
+    if (next.type === 'operator' && next.value === '=') {
+      return this.parseAssignment(token);
+    }
+    this.errors.push({
+      expected: ['curly', 'string', 'operator'],
+      received: next.type,
+      message: `Expected {, argument or operator at ${next.line}:${next.column}`,
+      line: next.line,
+      column: next.column,
+      start: next.start,
+      end: next.end,
+    });
+    return { type: 'empty', line: next.line, column: next.column, start: next.start, end: next.end };
+  }
+
+  // Parse a stanza, which is a block of assignments and other
+  // stanzas.
+  parseStanza(token: TokenIdentifier): Node {
+    const stanza: StanzaNode = {
+      type: 'stanza',
+      name: token.value,
+      args: [],
+      block: null,
+      line: token.line,
+      column: token.column,
+      start: token.start,
+      end: token.end,
+    };
+    let next = this.tokenizer.peek();
+    while (next.type !== 'curly') {
+      if (['string', 'number', 'boolean'].includes(next.type)) {
+        stanza.args.push(this.tokenizer.next() as TokenString | TokenNumber | TokenBoolean);
+        next = this.tokenizer.peek();
+        continue;
+      }
+      this.errors.push({
+        expected: ['string', 'number', 'boolean'],
+        received: next.type,
+        message: `Expected argument at ${next.line}:${next.column}`,
+        line: next.line,
+        column: next.column,
+        start: next.start,
+        end: next.end,
+      });
+      this.tokenizer.next();
+      next = this.tokenizer.peek();
+    }
+    this.expect('curly', '{');
+    stanza.block = this.parse();
+    this.expect('curly', '}');
+    stanza.end = this.tokenizer.cursor;
+    return stanza;
+  }
+
+  // Parse an assignment, which is an assignment to an expression.
+  parseAssignment(token: Token): AssignmentNode {
+    if (token.type !== 'identifier') {
+      throw new Error(`Expected identifier but got ${token.type} at ${token.line}:${token.column}`);
+    }
+    const key = token;
+    const operator = this.expect('operator', '=');
+    const value = this.parseExpression();
+    return {
+      type: 'assignment',
+      operator,
+      key,
+      value,
+      line: key.line,
+      column: key.column,
+      start: key.start,
+      end: value.end,
+    };
+  }
+
+  // Parse an expression. We're not building a calculator (yet), so we
+  // just accept a value on the right for now.
+  parseExpression(
+    ignore?: TokenType[],
+  ): TokenNumber | TokenString | TokenBoolean | ArrayNode | ObjectNode | ExpressionNode {
+    const token = this.tokenizer.next(ignore || ['whitespace']);
+    switch (token.type) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+        return token;
+      case 'bracket':
+        if (token.value === '[') {
+          return this.parseArray();
+        }
+        break;
+      case 'curly':
+        if (token.value === '{') {
+          return this.parseObject();
+        } else {
+          break;
+        }
+      default:
+        break;
+    }
+    // Recover by calling it a string.
+    this.errors.push({
+      expected: ['expression'],
+      received: token.type,
+      message: `Expected expression but got ${token.type} at ${token.line}:${token.column}`,
+      line: token.line,
+      column: token.column,
+      start: token.start,
+      end: token.end,
+    });
+    return {
+      type: 'string',
+      value: '"<error>"',
+      line: token.line,
+      column: token.column,
+      start: token.start,
+      end: token.end,
+    };
+  }
+
+  parseArray(): ArrayNode {
+    // the first thing we expect are expression.
+    const elements = [];
+    // This whole ignoring newlines thing is really just context.
+    let next = this.tokenizer.peek(['whitespace', 'newline']);
+    while (true) {
+      elements.push(this.parseExpression(['whitespace', 'newline']));
+      next = this.tokenizer.peek(['whitespace', 'newline']);
+      if (next.type === 'bracket' && next.value === ']') {
+        this.tokenizer.next(['whitespace', 'newline']);
+        return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+      }
+      if (next.type === 'comma') {
+        this.tokenizer.next();
+        next = this.tokenizer.peek(['whitespace', 'newline']);
+        // Special case: allow an ending comma because it's nice.
+        if (next.type === 'bracket' && next.value === ']') {
+          this.tokenizer.next(['whitespace', 'newline']);
+          return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+        }
+        continue;
+      }
+      // Should have been a closing bracket or a comma, so we "close" and return for recovery.
+      this.errors.push({
+        expected: ['bracket', 'comma'],
+        received: next.type,
+        message: `Expected ] or , but got ${next.type} at ${next.line}:${next.column}`,
+        line: next.line,
+        column: next.column,
+        start: next.start,
+        end: next.end,
+      });
+      return { type: 'array', elements, line: next.line, column: next.column, start: next.start, end: next.end };
+    }
+  }
+
+  parseObject(): ObjectNode {
+    const properties = [];
+    let next = this.tokenizer.peek(['whitespace', 'newline']);
+    while (true) {
+      if (next.type === 'end') {
+        this.errors.push({
+          expected: ['identifier'],
+          received: 'end',
+          message: `Unexpected end of input at ${next.line}:${next.column}`,
+          line: next.line,
+          column: next.column,
+          start: next.start,
+          end: next.end,
+        });
+        break;
+      }
+      if (next.type === 'curly' && next.value === '}') {
+        this.tokenizer.next(['whitespace', 'newline']);
+        break;
+      }
+      if (next.type === 'identifier') {
+        properties.push(this.parseAssignment(this.tokenizer.next(['whitespace', 'newline'])));
+        next = this.tokenizer.peek(['whitespace', 'newline']);
+        continue;
+      }
+      const token = this.tokenizer.next(['whitespace', 'newline']);
+      next = this.tokenizer.peek(['whitespace', 'newline']);
+      this.errors.push({
+        expected: ['identifier'],
+        received: next.type,
+        message: `Expected identifier but got ${token.type} at ${next.line}:${next.column}`,
+        line: next.line,
+        column: next.column,
+        start: next.start,
+        end: next.end,
+      });
+    }
+    return { type: 'object', properties, line: next.line, column: next.column, start: next.start, end: next.end };
+  }
+
+  // Expect a token of a certain type, optionally with a certain
+  // value.
+  expect<T extends TokenType>(type: T, value?: string): Extract<Token, { type: T }> {
+    const token = this.tokenizer.peek();
+    if (token.type === 'end' && type !== 'end') {
+      this.errors.push({
+        expected: [type],
+        received: 'end',
+        message: `Unexpected end of input at ${token.line}:${token.column}`,
+        line: token.line,
+        column: token.column,
+        start: token.start,
+        end: token.end,
+      });
+      return this.tokenizer.next() as Extract<Token, { type: T }>;
+    }
+    if (token.type !== type) {
+      this.errors.push({
+        expected: [type],
+        received: token.type,
+        message: `Expected ${type} but got ${token.type} at ${token.line}:${token.column}`,
+        line: token.line,
+        column: token.column,
+        start: token.start,
+        end: token.end,
+      });
+      // Naively try to recover by returning a working token.
+      return {
+        type: type,
+        value: token.value,
+        start: token.start,
+        end: token.end,
+        line: token.line,
+        column: token.column,
+      } as Token as Extract<Token, { type: T }>;
+    }
+    if (value && token.value !== value) {
+      this.errors.push({
+        message: `Expected ${value} but got ${token.value} at ${token.line}:${token.column}`,
+        line: token.line,
+        column: token.column,
+        start: token.start,
+        end: token.end,
+      });
+      // Again, naively try to recover with a working token.
+      return {
+        type,
+        value,
+        start: token.start,
+        end: token.end,
+        line: token.line,
+        column: token.column,
+      } as Token as Extract<Token, { type: T }>;
+    }
+    // We match, so consume the token.
+    return this.tokenizer.next() as Extract<Token, { type: T }>;
+  }
+}
+
+type TranslationError = {
+  message: string;
+  line: number;
+  column: number;
+};
+
+// JSONTranslator takes an AST and translates it into a JSON object
+// (any).
+export class JSONTranslator {
+  errors: TranslationError[];
+
+  constructor() {
+    this.errors = [];
+  }
+
+  // Translate the AST into a JSON serializable object.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  translate(ast: BlockNode): any {
+    return this.translateRoot(ast);
+  }
+
+  // Start at the root and translate the children.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  translateRoot(node: BlockNode): any {
+    const obj: any = {}; // eslint-disable-line @typescript-eslint/no-explicit-any
+    for (const child of node.children) {
+      if (child.type === 'stanza') {
+        if (!obj[child.name]) {
+          obj[child.name] = [];
+        }
+        const stanza = this.translateStanza(child);
+        obj[child.name].push(stanza);
+      } else {
+        this.errors.push({
+          message: `unexpected ${child.type} at ${child.line}:${child.column}`,
+          line: 1,
+          column: 1,
+        });
+      }
+    }
+    return obj;
+  }
+
+  // translateStanza takes a stanza node and translates it into a JSON
+  // object. This looks remarkably like translateRoot, and maybe
+  // there's some kind of generalized recursion we could use, but it's
+  // late.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  translateStanza(node: StanzaNode): any {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj: { [key: string]: any } = {};
+    if (node.args && node.args[0]) {
+      obj['name'] = node.args[0].value.slice(1, -1);
+    }
+    const children = node.block ? node.block.children : [];
+    for (const child of children) {
+      if (child.type === 'assignment') {
+        // Remember we skipped expressions, so...
+        if (child.value.type === 'string') {
+          obj[child.key.value] = child.value.value.slice(1, -1);
+        } else if (child.value.type === 'number') {
+          obj[child.key.value] = parseFloat(child.value.value);
+        } else if (child.value.type === 'boolean') {
+          obj[child.key.value] = child.value.value === 'true';
+        } else if (child.value.type === 'array') {
+          obj[child.key.value] = child.value.elements.map((el) => {
+            if (el.type === 'string') {
+              return el.value.slice(1, -1);
+            } else if (el.type === 'number') {
+              return parseFloat(el.value);
+            } else if (el.type === 'boolean') {
+              return el.value === 'true';
+            }
+            return null;
+          });
+        }
+      }
+      if (child.type === 'stanza') {
+        if (!obj[child.name]) {
+          obj[child.name] = [];
+        }
+        obj[child.name].push(this.translateStanza(child));
+      }
+    }
+    return obj;
+  }
+}
+
+// fmt is an ugly mess of string concatenation.
+export function fmt(ast: Node | Token | null, indent: number = 0, sibling?: Node | Token): string {
+  if (ast === null) {
+    return '';
+  }
+  const INDENT = ' '.repeat(indent);
+  let str = '';
+  // Just spit out token values as they are encountered.
+  switch (ast.type) {
+    case 'stanza':
+      str += `${INDENT}${ast.name} `;
+      for (const arg of ast.args) {
+        str += `${arg.value} `;
+      }
+      str += '{';
+      str += fmt(ast.block, indent + 2);
+      if (!ast.block?.children.every((child) => child.type === 'newline')) {
+        str += `${INDENT}`;
+      }
+      str += '}';
+      break;
+    case 'block':
+      if (ast.children.every((child) => child.type === 'newline')) {
+        // Do nothing.
+      } else {
+        for (const child of ast.children) {
+          str += fmt(child, indent, sibling);
+          sibling = child;
+        }
+      }
+      break;
+    case 'assignment':
+      str += `${INDENT}${fmt(ast.key)} = ${fmt(ast.value)}`;
+      break;
+    case 'expression':
+      str += `${fmt(ast.left)} ${ast.operator.value} ${fmt(ast.right)}`;
+      break;
+    case 'comment':
+      if (sibling?.type !== 'newline') {
+        str += `  `;
+      } else {
+        str += `${INDENT}`;
+      }
+      str += `${ast.value}`;
+      break;
+    case 'empty':
+      break;
+    case 'array':
+      // eslint-disable-next-line no-case-declarations
+      const oneLine = `[${ast.elements.map((el) => fmt(el)).join(', ')}]`;
+      if (oneLine.length > 80) {
+        str += `[`;
+        for (const el of ast.elements) {
+          str += `\n${fmt(el, indent + 2)}`;
+        }
+        str += `\n${INDENT}]`;
+      } else {
+        return oneLine;
+      }
+      break;
+    case 'object':
+      str += '{';
+      if (ast.properties.length > 0) {
+        str += '\n';
+        for (const assign of ast.properties) {
+          str += `${fmt(assign, indent + 2)}\n`;
+        }
+        str += `\n${INDENT}}`;
+      }
+      break;
+    default:
+      str += `${ast?.value}`;
+  }
+  return str;
+}