drift-parser 0.0.11

package/src/lexer/tokens.ts

@@ -0,0 +1,241 @@
+export enum TokenType {
+    EOF,
+    /** SPACES ONLY */
+    WHITESPACE,
+    /** \n character */
+    NEWLINE,
+    LIFETIME_EXTEND, // x!
+    NULL,
+    TRUE,
+    FALSE,
+    NUMBER,
+    STRING,
+    IDENTIFIER,
+    ARRAYTYPE_IDENTIFIER,
+
+    // Grouping
+    LEFT_BRACKET,
+    RIGHT_BRACKET,
+    LEFT_BRACE,
+    RIGHT_BRACE,
+    LEFT_PAREN,
+    RIGHT_PAREN,
+
+    // EQUALITY
+    ASSINGMENT_EQUALS,
+    /** This is the equality check operator == */
+    EQUALS,
+    /** 
+     * b== 
+     * performs a byte by byte comparison for things like strings like "a" b== "a"
+     */
+    BYTE_EQUALS,
+    NOT_EQUALS,
+
+    // EQUIVALENCE
+    LESS,
+    LESS_EQUALS,
+    GREATER,
+    GREATER_EQUALS,
+
+    // LOGICAL
+
+    AND,
+    OR,
+    NOT,
+
+    DOT,
+    /** range as in : [0..10] */
+    RANGE_OPERATOR,
+    SPREAD_OPERATOR,
+    SEMI_COLON,
+    COLON, // for ternary + a lot more 
+    QUESTION_OPERATOR, // for ternary, eg condition ? result : elseresult
+    COMMA,
+
+    // shorthand operators
+    PLUS_PLUS,
+    MINUS_MINUS,
+    PLUS_EQUALS,
+    MINUS_EQUALS,
+    MUL_EQUALS, // *= 
+    DIV_EQUALS, // /= 
+    MOD_EQUALS, // %= 
+    EXPONENTIATION_EQUALS, // **= 
+
+    // Math operators
+    PLUS,
+    MINUS,
+    DIVIDE,
+    MUL,
+    MODULO,
+    EXPONENTATION,
+
+    //Bitwise operators
+    BW_NOT,
+    BW_AND,
+    BW_OR,
+    BW_XOR,
+    BW_LEFTSHIFT,
+    BW_RIGHTSHIFT,
+    BW_UNSIGNED_RIGHTSHIFT,
+    // maybe 
+    BW_ROL, // r<<
+    BW_ROR, // r >> - rotate right and left, macros of other bitwise ops that shift but rotate 
+
+    BW_AND_EQUALS, // &=
+    BW_OR_EQUALS, // |= 
+    BW_XOR_EQUALS, // ^= 
+    BW_LEFTSHIFT_EQUALS, // <<=
+    BW_RIGHTSHIFT_EQUALS, // >>=
+    // maybe
+    BW_ROL_EQUALS, // ridiculous but r<<=
+    BW_ROR_EQUALS, // r>>= 
+
+    //SEMANTIC
+    SINGLE_COMMENT, //
+    MULTI_COMMENT, // ### ... ###
+
+    // Reserved keywords + primitives
+
+    LET, // maybe change to set?
+    CONST,
+    PRIMITIVE_TYPE, //u8/byte, string, array(slice), u16 u32 u64 u128 + signed, float<-->f32, double<-->f64, quad<-->quadruple<-->f128
+    MYRESP, //my or myresp, can be a prefix to an expr,
+    TOFREE, //tf or tofree
+
+
+    /**
+     * import std = "std";
+     * import { add, mul } = "math.dr";
+     * potentially import module = c "module.h";
+     * 
+     */
+    IMPORT,
+    AS, // import { a as b } -- also single type casts if not in an import like 5::<i32> as const,
+    TURBOFISH, // :: - sorry original goals, its just nice to do 5::i32 or 5::my instead of (5) as <i32>, you can also do 5::<i32,const> for multiple
+
+    CLASS,
+    NEW,
+    FN,   // func def
+    RETURN,
+    IF,
+    ELSE,
+    ELSEIF,
+    FOR,
+    FOR_EACH, //idk yet
+    WHILE,
+    EXPORT,
+    /**
+     * The operator for an `@` decorator above a function.
+    */
+    TYPENAME, // typename "hi" ==
+    STRUCT,
+
+    //COMPILE OPTIONS
+    /**
+     Sets compile options, eg, USE unsafe_features;
+     And parser options like USE c_logicals; makes it || and && instead of or / and and ! instead of not.
+     */
+    USE,
+
+    //Memory management specific 
+
+    //MISC
+    /**
+     * Example function `a` demonstrating an unwrapped return from function b.
+     * 
+     * ```rust
+     * fn a() {
+     *      const b = unwrap {
+     *          let abc = 5; //local abc
+     *          return "top level return" + abc; //only returns the block
+     *      };
+     *      return b;
+     * }
+     * ```
+     */
+    UNRWAP, 
+    AT_COMPILE, // ATCOMPILE { } -> the same as UNWRAP but comptime 
+};
+
+export const ReservedTokens = {
+    "let": TokenType.LET,
+    "const": TokenType.CONST,
+    "import": TokenType.IMPORT,
+    "as": TokenType.AS,
+    "class": TokenType.CLASS,
+    "new": TokenType.NEW,
+    "fn": TokenType.FN,
+    "return": TokenType.RETURN,
+    "if": TokenType.IF,
+    "else": TokenType.ELSE,
+    "elseif": TokenType.ELSEIF,
+    "for": TokenType.FOR,
+    "foreach": TokenType.FOR_EACH,
+    "while": TokenType.WHILE,
+    "export": TokenType.EXPORT,
+    "typename": TokenType.TYPENAME,
+    "struct": TokenType.STRUCT,
+    "use": TokenType.USE,
+    "unwrap": TokenType.UNRWAP,
+};
+
+export function TypeName(type: TokenType): string {
+    return TokenType[type];
+}
+
+export interface Position {
+  line: number;
+  col: number;
+}
+
+export interface LocationInterface {
+  start: Position;
+  end: Position;
+}
+
+export class Token {
+    public type: TokenType;
+    public name: string;
+    public value: string;
+    public loc: LocationInterface;
+    public range: [number, number];
+
+    constructor(type: TokenType, value: string, location: LocationInterface, range: [number, number]) {
+        this.type = type;
+        this.name = TypeName(type);
+        this.value = value;
+        this.loc = location;
+        this.range = range;
+    }
+
+    /**
+    * Checks if the token's type matches any of the provided token types.
+    * 
+    * @param {...TokenType[]} checkTypes The token types to compare to.
+    * @returns {boolean} True if this token's type is in the list, otherwise false.
+    */
+    IsA(...checkTypes: TokenType[]): boolean {
+        for (let i = 0; i < checkTypes.length; i++) {
+            if (this.type == checkTypes[i]) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    Print(): void {
+        const { IDENTIFIER, NUMBER, STRING, SINGLE_COMMENT, MULTI_COMMENT } = TokenType;
+        if (this.IsA(IDENTIFIER, NUMBER, STRING, SINGLE_COMMENT, MULTI_COMMENT )) {
+            console.log(`${TypeName(this.type)}(${this.value})`)
+        } else {
+            console.log(`${TypeName(this.type)}()`)
+        }
+    }
+
+    toString(): string {
+        return this.value;
+    }
+}

package/src/parser/exports.ts

@@ -0,0 +1 @@
+export { Parser } from './parser'

package/src/parser/expr.ts

@@ -0,0 +1,82 @@
+import { TokenType } from "../lexer/tokens";
+import { Parser } from "./parser";
+// import { Expression, NumberExpression, PrefixExpression, StringExpression, SymbolExpression } from "../ast/exports";
+import { bp_lu, nud_lu, led_lu, BP } from "./lookup"; // wherever you put those
+import { Expr, mkexpr, Node, NodeKind } from "../ast/type";
+
+function tokenLocationInfo(token: { loc?: any; range?: [number, number] }): string {
+  const locStr = token.loc ? ` at ${JSON.stringify(token.loc)}` : "";
+  const rangeStr = token.range ? ` [${token.range[0]}, ${token.range[1]}]` : "";
+  return locStr + rangeStr;
+}
+
+export function parseExpr(p: Parser, bp: BP = BP.Default): Expr {
+  const tokenKind = p.currentTokenKind();
+  const nud = nud_lu[tokenKind];
+  if (!nud) {
+    const token = p.currentToken();
+    throw new Error(`nud handler expected for token ${TokenType[tokenKind]}${tokenLocationInfo(token)}`);
+  }
+
+  let left = nud(p);
+
+  while ((bp_lu[p.currentTokenKind()] ?? BP.Default) > bp) {
+    const nextTokenKind = p.currentTokenKind();
+    const led = led_lu[nextTokenKind];
+    if (!led) {
+      const token = p.currentToken();
+      throw new Error(`led handler expected for token ${TokenType[nextTokenKind]}${tokenLocationInfo(token)}`);
+    }
+    left = led(p, left, bp);
+  }
+  
+  return left;
+}
+
+
+export function parsePrimaryExpr(p: Parser): Expr {
+  switch (p.currentTokenKind()) {
+    case TokenType.NUMBER: {
+      const token = p.advance();
+      return mkexpr({
+        value: token.value,
+        type: 'NumberExpression',
+        loc: token.loc,
+        range: token.range,
+      })
+    }
+    case TokenType.STRING: {
+      const token = p.advance();
+      return mkexpr({
+        value: token.value,
+        type: 'StringExpression',
+        loc: token.loc,
+        range: token.range,
+      })
+    }
+    case TokenType.IDENTIFIER: {
+      const token = p.advance();
+      return mkexpr({
+        value: token.value,
+        type: 'SymbolExpression',
+        loc: token.loc,
+        range: token.range,
+      })
+    }
+    default: {
+      const token = p.currentToken();
+      throw new Error(`Can't create primary expression from token type ${TokenType[token.type]}${tokenLocationInfo(token)}`);
+    }
+  }
+}
+
+// type PrefixExpr = InstanceType<typeof PrefixExpression>;
+// export function parsePrefixExpression(p: Parser): PrefixExpr {
+//   const opToken = p.advance()
+//   const rhs = parseExpr(p, BP.Unary)
+//   return new PrefixExpression(opToken, rhs);
+// }
+
+// export function parseBinaryExpression(p: Parser): BinaryExpr {
+
+// }

package/src/parser/lookup.ts

@@ -0,0 +1,69 @@
+import { Token, TokenType } from "../lexer/tokens";
+import { Expr, Stmt } from "../ast/exports";
+import { Parser } from "./parser";
+import {  parsePrimaryExpr } from "./expr";
+// import { parseUsingStatement, parseVariableDeclaration } from "./stmt";
+
+export enum BP {
+  Default,
+  Comma,
+  Assignment,
+  Logical,
+  Relational,
+  Additive,
+  Multiplicative,
+  Unary,
+  Call,
+  Member,
+  Primary,
+}
+
+// Change StmtHandler to return Statement[]
+export type StmtHandler = (p: Parser) => Stmt[];
+export type NudHandler = (p: Parser) => Expr;
+export type LedHandler = (p: Parser, left: Expr, bp: BP) => Expr;
+
+const _bp_lu: Partial<Record<TokenType, BP>> = {};
+const _nud_lu: Partial<Record<TokenType, NudHandler>> = {};
+const _led_lu: Partial<Record<TokenType, LedHandler>> = {}
+const _stmt_lu: Partial<Record<TokenType, StmtHandler>> = {}
+export function led(kind: TokenType, bp: BP, ledFn: LedHandler): void {
+  _bp_lu[kind] = bp;
+  _led_lu[kind] = ledFn;
+}
+
+export function nud(kind: TokenType, nudFn: NudHandler): void {
+  _nud_lu[kind] = nudFn;
+}
+
+export function stmt(kind: TokenType, stmtFn: StmtHandler): void {
+  _bp_lu[kind] = BP.Default;
+  _stmt_lu[kind] = stmtFn;
+}
+
+export function createTokenLookups(): void {
+  // prefix expression nud handlers
+
+  nud(TokenType.NUMBER, parsePrimaryExpr);
+  nud(TokenType.IDENTIFIER, parsePrimaryExpr);
+  nud(TokenType.STRING, parsePrimaryExpr);
+  // nud(TokenType.MINUS, parsePrefixExpression);
+
+
+  led(TokenType.MINUS, BP.Additive, parsePrimaryExpr);
+
+  // statement handlers
+
+  // stmt(TokenType.USE, parseUsingStatement);
+  // stmt(TokenType.LET, parseVariableDeclaration);
+  // stmt(TokenType.CONST, parseVariableDeclaration)
+}
+
+createTokenLookups();
+
+export {
+  _bp_lu as bp_lu,
+  _nud_lu as nud_lu,
+  _led_lu as led_lu,
+  _stmt_lu as stmt_lu,
+};

package/src/parser/parser.ts

@@ -0,0 +1,166 @@
+import { mkstmt, Stmt } from "../ast/exports";
+import { Token, TokenType } from "../lexer/tokens";
+// import { _toJSON } from "../utils/mapAll";
+import { stmt } from "./lookup";
+import { parseStmt } from "./stmt"; 
+
+export class Parser {
+  private tokens: Token[];
+  private pos: number = 0;
+  public allowNewline = false;
+
+  constructor(tokens: Token[]) {
+    this.tokens = tokens;
+  }
+
+  static parse(tokens: Token[]): Stmt {
+    const parser = new Parser(tokens);
+    const body: Stmt[] = [];
+
+    while (parser.hasTokens()) {
+      body.push(...parseStmt(parser));
+    }
+
+    // Filter out the EOF token (if present) for accurate location data
+    const actualTokens = tokens.filter((t) => t.type !== TokenType.EOF);
+
+    if (actualTokens.length === 0) {
+      throw new Error("Cannot parse an empty token stream.");
+    }
+
+    const start = actualTokens[0];
+    const end = actualTokens[actualTokens.length - 1];
+
+    const loc = {
+      start: start.loc.start,
+      end: end.loc.end,
+    };
+
+    const range: [number, number] = [start.range[0], end.range[1]];
+
+    
+// export class BlockStatement extends Statement {
+//   public body: Statement[];
+
+//   constructor(body: Statement[], loc: LocationInterface, range: [number, number]) {
+//     super(loc, range);
+//     this.body = body;
+//   }
+
+//   toJSON() {
+//     return {
+//       ...super.toJSON(),
+//       body: this.body.map((s) => s.toJSON()),
+//     };
+//   }
+// }
+    return mkstmt({
+      type: 'BlockStatement',
+      body: body,
+      loc,
+      range,
+    })
+    // return new BlockStatement(body, loc, range);
+  }
+
+  // static parseJSON(tokens: Token[]): object {
+  //   return _toJSON(this.parse(tokens))
+  // }
+
+  mark(): number {
+    return this.pos;
+  }
+
+  reset(pos: number): void {
+    if (pos < 0 || pos > this.tokens.length) {
+      throw new Error(`Invalid reset position: ${pos}`);
+    }
+    this.pos = pos;
+  }
+  
+  currentToken(): Token {
+    return this.tokens[this.pos];
+  }
+
+  nextToken(): Token {
+    return this.tokens[this.pos + 1];
+  }
+
+  previousToken(): Token {
+    return this.tokens[this.pos - 1];
+  }
+
+  nthToken(n: number): Token {
+    return this.tokens[this.pos + n];
+  }
+
+  advance(): Token {
+    const token = this.currentToken();
+    do {
+      this.pos++;
+    } while (this.currentToken()?.type === TokenType.WHITESPACE);
+    return token;
+  }
+
+  hasTokens(): boolean {
+    return (
+      this.pos < this.tokens.length && this.currentTokenKind() !== TokenType.EOF
+    );
+  }
+
+  currentTokenKind(): TokenType {
+    return this.currentToken().type;
+  }
+
+  expect(expectedKind: TokenType): Token {
+    return this.expectError(expectedKind);
+  }
+
+  expectOne(...expectedKind: TokenType[]): Token {
+    return this.expectOneError(undefined, ...expectedKind);
+  }
+
+  parseTerminator(): Token {
+    const expectedKinds: TokenType[] = this.allowNewline
+      ? [TokenType.SEMI_COLON, TokenType.EOF]
+      : [TokenType.SEMI_COLON, TokenType.NEWLINE, TokenType.EOF];
+
+    const current = this.currentToken();
+
+    if (current.type === TokenType.EOF) {
+      return current;
+    }
+
+    return this.expectOne(...expectedKinds);
+  }
+
+
+
+  expectError(expectedKind: TokenType, err?: string): Token {
+    const token = this.currentToken();
+    const kind = token.type;
+
+    if (kind !== expectedKind) {
+      if (!err) {
+        err = `Expected token: ${TokenType[expectedKind]} but got ${TokenType[kind]}`;
+      }
+      throw new Error(err);
+    }
+
+    return this.advance();
+  }
+
+  expectOneError(err?: string, ...expectedKind: TokenType[]): Token {
+    const token = this.currentToken();
+    const kind = token.type;
+
+    if (!expectedKind.includes(kind)) {
+      if (!err) {
+        err = `Expected token: ${expectedKind.map(k => TokenType[k]).join(' OR ')} but got ${TokenType[kind]}`;
+      }
+      throw new Error(err);
+    }
+
+    return this.advance();
+  }
+}

package/src/parser/stmt.ts

@@ -0,0 +1,151 @@
+// ./parser/parse-stmt.ts
+import { Parser } from "./parser";
+import { parseExpr } from "./expr";
+import { stmt_lu } from "./lookup";
+// import {
+//   Statement,
+//   ExpressionStatement,
+//   TerminatorStatement,
+//   UsingStatement,
+//   VariableDeclarationStatement,
+// } from "../ast/stmt";
+import { TokenType } from "../lexer/tokens";
+import { BP } from "./lookup";
+// import { parse_type } from "./type";
+import { mkexpr, mkstmt, Stmt } from "../ast/type";
+
+export function parseStmt(p: Parser): Stmt[] {
+  const kind = p.currentTokenKind();
+
+  // if (kind === TokenType.SEMI_COLON || kind === TokenType.NEWLINE) {
+  if (kind === TokenType.SEMI_COLON) {
+    const tok = p.advance();
+    // export class TerminatorStatement extends Statement {
+    //   public kind: "semicolon" | "newline" | "eof";
+
+    //   constructor(token: Token) {
+    //     super(token.loc, token.range);
+
+    //     if (token.type === TokenType.EOF) {
+    //       this.kind = "eof";
+    //     } else {
+    //       this.kind = token.value === ";" ? "semicolon" : "newline";
+    //     }
+    //   }
+
+    //   toJSON() {
+    //     return {
+    //       ...super.toJSON(),
+    //       kind: this.kind,
+    //     };
+    //   }
+    // }
+    return [mkstmt({
+      type: "TerminatorStatement",
+      loc: tok.loc,
+      range: tok.range,
+      kind: tok.type == TokenType.EOF ? "EOF" : tok.value == ";" ? "semicolon" : "newline",
+    })];
+  }
+
+  const fn = stmt_lu[kind];
+  if (fn) {
+    const result = fn(p);
+    return Array.isArray(result) ? result : [result];
+  }
+
+  const expr = parseExpr(p, BP.Default);
+  const terminator = p.parseTerminator();
+
+  const loc = {
+    start: expr.loc.start,
+    end: terminator.loc.end,
+  };
+  const range: [number, number] = [expr.range[0], terminator.range[1]];
+
+
+  // export class ExpressionStatement extends Statement {
+  //   public expression: Expression;
+
+  //   constructor(expression: Expression, loc: LocationInterface, range: [number, number]) {
+  //     super(loc, range);
+  //     this.expression = expression;
+  //   }
+
+  //   toJSON() {
+  //     return {
+  //       ...super.toJSON(),
+  //       expression: this.expression.toJSON(),
+  //     };
+  //   }
+  // }
+
+
+  return [
+    mkstmt({
+      type: "ExpressionStatement",
+      expression: expr,
+      loc: loc,
+      range: range,
+    }),
+    mkstmt({
+      type: "TerminatorStatement",
+      loc: terminator.loc,
+      range: terminator.range,
+      kind: terminator.type == TokenType.EOF ? "EOF" : terminator.value == ";" ? "semicolon" : "newline",
+    })
+  ];
+}
+
+
+// export function parseUsingStatement(p: Parser): Stmt[] {
+//   const useTok = p.expect(TokenType.USE);
+//   const idTok = p.expect(TokenType.IDENTIFIER);
+
+//   let asExpr: Expr | undefined;
+//   if (p.currentTokenKind() === TokenType.AS) {
+//     p.advance();
+//     asExpr = parseExpr(p, BP.Default);
+//   }
+
+//   const terminator = p.parseTerminator();
+//   const loc = {
+//     start: useTok.loc.start,
+//     end: terminator.loc.end,
+//   };
+//   const range: [number, number] = [useTok.range[0], terminator.range[1]];
+
+//   return [
+//     new UsingStatement(idTok, asExpr, terminator, loc, range),
+//     new TerminatorStatement(terminator),
+//   ];
+// }
+
+// export function parseVariableDeclaration(p: Parser): Statement[] {
+//   const letTok = p.expectOne(TokenType.LET, TokenType.CONST);
+//   const nameTok = p.expect(TokenType.IDENTIFIER);
+
+//   let varType: Type | undefined;
+//   if (p.currentTokenKind() === TokenType.COLON) {
+//     p.advance();
+//     varType = parse_type(p, 0);
+//   }
+
+//   let initExpr: Expression | undefined;
+//   if (p.currentTokenKind() === TokenType.ASSINGMENT_EQUALS) {
+//     p.advance();
+//     initExpr = parseExpr(p, BP.Default);
+//   }
+
+//   const terminator = p.parseTerminator();
+//   const loc = {
+//     start: letTok.loc.start,
+//     end: terminator.loc.end,
+//   };
+//   const range: [number, number] = [letTok.range[0], terminator.range[1]];
+
+//   return [
+//     new VariableDeclarationStatement(nameTok, varType, initExpr, terminator, letTok.IsA(TokenType.CONST), loc, range),
+//     new TerminatorStatement(terminator),
+//   ];
+// }