drift-parser 0.0.11

package/src/ast/expr.ts

@@ -0,0 +1,122 @@
+// import { ASTNode } from "./ast";
+// import { Token } from "../lexer/tokens";
+// import { Parser } from "../parser/parser";
+// import { parseExpr } from "../parser/expr";
+// import { BP } from "../parser/lookup";
+// import { makeExpressionClass } from "../utils/genexpr";
+
+// export abstract class Expression extends ASTNode {
+//   toJSON() {
+//     return {
+//       type: this.constructor.name,  // captures class name dynamically
+//       loc: this.loc,
+//       range: this.range,
+//     };
+//   }
+// }
+
+// export class SymbolExpression extends Expression {
+//   /** The name of the identifier. */
+//   public value: string;
+
+//   constructor(token: Token) {
+//     super(token.loc, token.range);
+//     this.value = token.value;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       value: this.value,
+//     };
+//   }
+// }
+
+// export class StringExpression extends Expression {
+//   /** The value of the string with escaped quotes. */
+//   public value: string;
+
+//   constructor(token: Token) {
+//     super(token.loc, token.range);
+//     this.value = token.value;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       value: this.value,
+//     };
+//   }
+// }
+
+// export class NumberExpression extends Expression {
+//   /** The value of the number as a string to not deal with bit issues. */
+//   public value: string;
+
+//   constructor(token: Token) {
+//     super(token.loc, token.range);
+//     this.value = token.value;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       value: this.value,
+//     };
+//   }
+// }
+
+// // export class PrefixExpression extends Expression {
+// //   public prefix: string;
+// //   public expression: Expression;
+
+// //   constructor(prefixToken: Token, rightExpression: Expression) {
+// //     super(prefixToken.loc, prefixToken.range);
+// //     this.prefix = prefixToken.value;
+// //     this.expression = rightExpression;
+// //   }
+
+// //   toJSON() {
+// //     return {
+// //       ...super.toJSON(),
+// //       prefix: this.prefix,
+// //       expression: this.expression.toJSON()
+// //     }
+// //   }
+// // }
+
+// export const PrefixExpression = makeExpressionClass({
+//   className: "PrefixExpression",
+//   fields: ["prefix", "expression"],
+//   constructor: (prefixToken: Token, rightExpression: Expression) => ({
+//     loc: prefixToken.loc,
+//     range: prefixToken.range,
+//     prefix: prefixToken.value,
+//     expression: rightExpression,
+//   }),
+//   toJSON: ({ prefix, expression }) => ({
+//     prefix,
+//     expression: expression.toJSON(),
+//   }),
+// });
+
+
+
+// export const BinaryExpression = makeExpressionClass({
+//   className: "PrefixExpression",
+//   fields: ["left", "op", "right"],
+//   constructor: (prefixToken: Token, rightExpression: Expression) => ({
+//     loc: prefixToken.loc,
+//     range: prefixToken.range,
+//     prefix: prefixToken.value,
+//     expression: rightExpression,
+//   }),
+//   toJSON: ({ prefix, expression }) => ({
+//     prefix,
+//     expression: expression.toJSON(),
+//   }),
+// });
+
+
+
+

package/src/ast/stmt.ts

@@ -0,0 +1,126 @@
+// // ./ast/stmt.ts
+// import { LocationInterface, Token, TokenType } from "../lexer/tokens";
+// import { ASTNode } from "./ast";
+// import { Expression } from "./expr";
+// import { Type } from "./type";
+
+// export abstract class Statement extends ASTNode {
+//   toJSON() {
+//     return {
+//       type: this.constructor.name,
+//       loc: this.loc,
+//       range: this.range,
+//     };
+//   }
+// }
+
+// export class BlockStatement extends Statement {
+//   public body: Statement[];
+
+//   constructor(body: Statement[], loc: LocationInterface, range: [number, number]) {
+//     super(loc, range);
+//     this.body = body;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       body: this.body.map((s) => s.toJSON()),
+//     };
+//   }
+// }
+
+// export class ExpressionStatement extends Statement {
+//   public expression: Expression;
+
+//   constructor(expression: Expression, loc: LocationInterface, range: [number, number]) {
+//     super(loc, range);
+//     this.expression = expression;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       expression: this.expression.toJSON(),
+//     };
+//   }
+// }
+
+
+// export class TerminatorStatement extends Statement {
+//   public kind: "semicolon" | "newline" | "eof";
+
+//   constructor(token: Token) {
+//     super(token.loc, token.range);
+
+//     if (token.type === TokenType.EOF) {
+//       this.kind = "eof";
+//     } else {
+//       this.kind = token.value === ";" ? "semicolon" : "newline";
+//     }
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       kind: this.kind,
+//     };
+//   }
+// }
+
+// export class UsingStatement extends Statement {
+//   public value: string;
+//   public as?: Expression;
+
+//   constructor(
+//     valueToken: Token,
+//     asExpression: Expression | undefined,
+//     _terminator: Token, // still required to build loc/range externally
+//     loc: LocationInterface,
+//     range: [number, number]
+//   ) {
+//     super(loc, range);
+//     this.value = valueToken.value;
+//     this.as = asExpression;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       value: this.value,
+//       as: this.as ? this.as.toJSON() : null,
+//     };
+//   }
+// }
+
+// export class VariableDeclarationStatement extends Statement {
+//   public identifier: string;
+//   public varType?: Type;
+//   public initializer?: Expression;
+//   public isConstant: boolean;
+
+//   constructor(
+//     nameToken: Token,
+//     varType: Type | undefined,
+//     initializer: Expression | undefined,
+//     _terminator: Token,
+//     isConstant: boolean,
+//     loc: LocationInterface,
+//     range: [number, number]
+//   ) {
+//     super(loc, range);
+//     this.identifier = nameToken.value;
+//     this.varType = varType;
+//     this.initializer = initializer;
+//     this.isConstant = isConstant;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       identifier: this.identifier,
+//       varType: this.varType ? this.varType.toJSON() : null,
+//       initializer: this.initializer ? this.initializer.toJSON() : null,
+//     };
+//   }
+// }

package/src/ast/type.ts

@@ -0,0 +1,46 @@
+import { LocationInterface } from "../lexer/tokens"
+
+export enum NodeKind {
+  Statement,
+  Expression,
+}
+
+export interface Node {
+  kind: NodeKind;
+  type: string;
+  loc: LocationInterface;
+  range: [number, number];
+  body: Node[];
+  [key: string]: any;
+}
+
+export interface Stmt extends Node {
+  kind: NodeKind.Statement;
+}
+
+export interface Expr extends Node {
+  kind: NodeKind.Expression;
+}
+
+type NodeFieldsRequired = {
+  type: string;
+  loc: LocationInterface;
+  range: [number, number];
+} & Record<string, any>;
+
+// Expr factory
+export function mkexpr(fields: NodeFieldsRequired & { body?: Node[] }): Expr {
+  return {
+    kind: NodeKind.Expression,
+    body: fields.body ?? [],
+    ...fields,
+  };
+}
+
+export function mkstmt(fields: NodeFieldsRequired & { body?: Node[] }): Stmt {
+  return {
+    kind: NodeKind.Statement,
+    body: fields.body ?? [],
+    ...fields,
+  };
+}

package/src/lexer/exports.ts

@@ -0,0 +1,2 @@
+export * from './tokenizer';
+export * from './tokens';

package/src/lexer/tokenizer.ts

@@ -0,0 +1,395 @@
+import { LocationInterface, Token, TokenType, Position, ReservedTokens } from "./tokens";
+
+type RegexHandler = (lex: Lexer, regex: RegExp) => void;
+
+interface RegexPattern {
+  regex: RegExp;
+  handler: RegexHandler;
+}
+
+interface LexerInterface {
+  patterns: RegexPattern[];
+  Tokens: Token[];
+  source: string;
+  pos: number;
+  line: number;
+  col: number;
+  remainingSource(): string;
+  end_of_file(): boolean;
+  push(token: Token): void;
+}
+
+class Lexer implements LexerInterface {
+  patterns: RegexPattern[];
+  Tokens: Token[];
+  source: string;
+  pos: number;
+  line: number;
+  col: number;
+
+  constructor(source: string, patterns: RegexPattern[]) {
+    this.source = source;
+    this.patterns = patterns;
+    this.Tokens = [];
+    this.pos = 0;
+    this.line = 1;
+    this.col = 0;
+  }
+
+  remainingSource(): string {
+    return this.source.slice(this.pos);
+  }
+
+  end_of_file(): boolean {
+    return this.pos >= this.source.length;
+  }
+
+  push(token: Token): void {
+    this.Tokens.push(token);
+  }
+}
+
+function stringHandler(tokenType: TokenType): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const match = lex.remainingSource().match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+function numberHandler(tokenType: TokenType): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const match = lex.remainingSource().match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+function multiCommentHandler(tokenType: TokenType): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const match = lex.remainingSource().match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+function singleCommentHandler(tokenType: TokenType): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const remaining = lex.remainingSource();
+    // Match comment line without the newline character itself
+    const match = remaining.match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+      const start: Position = { line: lex.line, col: lex.col };
+
+      // Count columns only, no newlines expected in single-line comments
+      let col = lex.col + text.length;
+      const end: Position = { line: lex.line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.col = col;
+
+      // **Do NOT consume the newline here. Leave it for the main lexer loop**
+    }
+  };
+}
+
+
+function whitespaceHandler(tokenType: TokenType): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const match = lex.remainingSource().match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+function defaultHandler(tokenType: TokenType, tokenValue: string): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const remaining = lex.remainingSource();
+    const match = remaining.match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      lex.push(new Token(tokenType, tokenValue, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+type ReservedMap = { [key: string]: TokenType };
+function symbolHandler(reservedKeywords: ReservedMap): RegexHandler {
+  return (lex: Lexer, regex: RegExp) => {
+    const remaining = lex.remainingSource();
+    const match = remaining.match(regex);
+    if (match && match.index === 0) {
+      const text = match[0];
+      const start: Position = { line: lex.line, col: lex.col };
+
+      let line = lex.line;
+      let col = lex.col;
+      for (const ch of text) {
+        if (ch === '\n') {
+          line++;
+          col = 0;
+        } else {
+          col++;
+        }
+      }
+      const end: Position = { line, col };
+      const loc: LocationInterface = { start, end };
+      const range: [number, number] = [lex.pos, lex.pos + text.length];
+
+      // Use reserved token type if it exists, else default to IDENTIFIER
+      const tokenType = reservedKeywords[text] ?? TokenType.IDENTIFIER;
+
+      lex.push(new Token(tokenType, text, loc, range));
+
+      lex.pos += text.length;
+      lex.line = line;
+      lex.col = col;
+    }
+  };
+}
+
+
+
+function CreateLexer(source: string): Lexer {
+  const patterns: RegexPattern[] = [
+    // Whitespace and newline — emit, don't skip (for formatter support)
+    { regex: /^[ \t]+/, handler: whitespaceHandler(TokenType.WHITESPACE) },
+    { regex: /^(?:\r\n|\r|\n)/, handler: defaultHandler(TokenType.NEWLINE, "\n") },
+
+    // Comments — single-line first, then multi-line
+    { regex: /^\/\/[^\r\n]*/, handler: singleCommentHandler(TokenType.SINGLE_COMMENT) },
+    { regex: /^\/\*[\s\S]*?\*\//, handler: multiCommentHandler(TokenType.MULTI_COMMENT) },
+
+    // Strings — assuming stringHandler handles quotes and escapes
+    { regex: /^"(?:[^"\\]|\\.)*"/, handler: stringHandler(TokenType.STRING) },
+    { regex: /^'(?:[^'\\]|\\.)*'/, handler: stringHandler(TokenType.STRING) },
+
+    // Numbers — integers and decimals
+    { regex: /^\d+\.\d+/, handler: numberHandler(TokenType.NUMBER) }, // decimal numbers
+    { regex: /^\d+/, handler: numberHandler(TokenType.NUMBER) },       // integers
+
+    // Identifiers — variable/function names, keywords, etc.
+    { regex: /^[A-Za-z_][A-Za-z0-9_]*!?$/, handler: symbolHandler(ReservedTokens) },
+
+    // Grouping & brackets
+    { regex: /^\[/, handler: defaultHandler(TokenType.LEFT_BRACKET, "[") },
+    { regex: /^\]/, handler: defaultHandler(TokenType.RIGHT_BRACKET, "]") },
+    { regex: /^\{/, handler: defaultHandler(TokenType.LEFT_BRACE, "{") },
+    { regex: /^\}/, handler: defaultHandler(TokenType.RIGHT_BRACE, "}") },
+    { regex: /^\(/, handler: defaultHandler(TokenType.LEFT_PAREN, "(") },
+    { regex: /^\)/, handler: defaultHandler(TokenType.RIGHT_PAREN, ")") },
+
+    // Comparison operators — longer first
+    { regex: /^==/, handler: defaultHandler(TokenType.EQUALS, "==") },
+    { regex: /^!=/, handler: defaultHandler(TokenType.NOT_EQUALS, "!=") },
+    { regex: /^<=/, handler: defaultHandler(TokenType.LESS_EQUALS, "<=") },
+    { regex: /^</, handler: defaultHandler(TokenType.LESS, "<") },
+    { regex: /^>=/, handler: defaultHandler(TokenType.GREATER_EQUALS, ">=") },
+    { regex: /^>/, handler: defaultHandler(TokenType.GREATER, ">") },
+
+    // Assignment and logical operators
+    { regex: /^&&/, handler: defaultHandler(TokenType.AND, "&&") },
+    { regex: /^\|\|/, handler: defaultHandler(TokenType.OR, "||") },
+    { regex: /^=/, handler: defaultHandler(TokenType.ASSINGMENT_EQUALS, "=") },
+    { regex: /^!/, handler: defaultHandler(TokenType.NOT, "!") },
+
+    // Range and dot — precedence important
+    { regex: /^\.\./, handler: defaultHandler(TokenType.RANGE_OPERATOR, "..") },
+    { regex: /^\./, handler: defaultHandler(TokenType.DOT, ".") },
+
+    // Punctuation and symbols
+    { regex: /^;/, handler: defaultHandler(TokenType.SEMI_COLON, ";") },
+    { regex: /^:/, handler: defaultHandler(TokenType.COLON, ":") },
+    { regex: /^\?/, handler: defaultHandler(TokenType.QUESTION_OPERATOR, "?") },
+    { regex: /^,/, handler: defaultHandler(TokenType.COMMA, ",") },
+
+    // Increment/decrement & compound assign — longer first
+    { regex: /^\+\+/, handler: defaultHandler(TokenType.PLUS_PLUS, "++") },
+    { regex: /^--/, handler: defaultHandler(TokenType.MINUS_MINUS, "--") },
+    { regex: /^\+=/, handler: defaultHandler(TokenType.PLUS_EQUALS, "+=") },
+    { regex: /^-=/, handler: defaultHandler(TokenType.MINUS_EQUALS, "-=") },
+
+    // Arithmetic operators
+    { regex: /^\+/, handler: defaultHandler(TokenType.PLUS, "+") },
+    { regex: /^-/, handler: defaultHandler(TokenType.MINUS, "-") },
+    { regex: /^\//, handler: defaultHandler(TokenType.DIVIDE, "/") },
+    { regex: /^\*/, handler: defaultHandler(TokenType.MUL, "*") },
+    { regex: /^%/, handler: defaultHandler(TokenType.MODULO, "%") },
+  ];
+
+  return new Lexer(source, patterns);
+}
+
+
+
+function Tokenize(source: string): Token[] {
+  const lex = CreateLexer(source);
+
+  while (!lex.end_of_file()) {
+    let matched = false;
+
+    for (const pattern of lex.patterns) {
+      const match = lex.remainingSource().match(pattern.regex);
+
+      if (match && match.index === 0) {
+        pattern.handler(lex, pattern.regex);
+        matched = true;
+        break;
+      }
+    }
+
+    if (!matched) {
+      const context = lex.remainingSource().slice(0, 20);
+      throw new Error(
+        `Unrecognized token at line ${lex.line}, column ${lex.col}: '${context}'`
+      );
+    }
+  }
+
+  // Push EOF token at end of source
+  const eofLoc: LocationInterface = {
+    start: { line: lex.line, col: lex.col },
+    end: { line: lex.line, col: lex.col },
+  };
+
+  lex.push(new Token(TokenType.EOF, "EOF", eofLoc, [lex.pos, lex.pos]));
+
+  return lex.Tokens;
+}
+
+
+interface PrintOptions {
+    shortenWhitespace: boolean;
+}
+
+function Print(tokens: Token[], options: PrintOptions | undefined) {
+    const shorten = options?.shortenWhitespace ?? false;
+    if (shorten) {
+        for (let i = 0; i < tokens.length; i++) {
+            if (tokens[i].type == TokenType.WHITESPACE) {
+                tokens[i].value = ' ';
+            }
+        }
+    }
+
+    return tokens.slice(0,-1).map(t => t.value).join('');
+} 
+
+
+export { Lexer, CreateLexer, Tokenize, Print };