mimo-lang 1.1.1 → 2.0.6

package/lexer/Lexer.js

@@ -0,0 +1,245 @@
+import { TokenType } from "./TokenTypes.js";
+import { createToken as _createTokenRaw } from "./createToken.js"; // <--- Add this import
+import { skipWhitespace } from "./tokenizers/whitespaceTokenizer.js";
+import {
+  isLiteralStart,
+  readIdentifier,
+  readNumber,
+  readString,
+  isAlpha,
+  isDigit,
+} from "./tokenizers/literalTokenizer.js";
+import { readSymbol } from "./tokenizers/symbolTokenizer.js";
+import { MimoError } from "../interpreter/MimoError.js";
+
+export class Lexer {
+  constructor(source, filePath = "unknown") {
+    // Add filePath
+    this.source = source;
+    this.filePath = filePath; // Store filePath
+    this.position = 0;
+    this.line = 1;
+    this.column = 1;
+    this.previousToken = null;
+
+    // State for template literals
+    // 0: Normal
+    // 1: Inside template, expecting StringFragment or ${ or `
+    // 2: Inside template, just saw ${, expecting expression (normal tokenizing)
+    // 3: Inside template, just saw }, expecting StringFragment or ${ or `
+    this.templateLiteralState = 0;
+    this.templateLiteralDepth = 0; // To handle nested templates if ever supported
+  }
+
+  isAtEnd() {
+    return this.position >= this.source.length;
+  }
+
+  peek(offset = 0) {
+    return this.position + offset < this.source.length
+      ? this.source[this.position + offset]
+      : null;
+  }
+
+  advance() {
+    if (!this.isAtEnd()) {
+      const char = this.source[this.position];
+      if (char === "\n") {
+        // <--- FIXED: Now checks for actual newline character
+        this.line++;
+        this.column = 1;
+      } else {
+        this.column++;
+      }
+      this.position++;
+    }
+  }
+
+  // Internal helper for Lexer's own token creation (e.g., for template literals)
+  // and for external tokenizers to use.
+  _createToken(type, value, startLine, startColumn, startPosition, length) {
+    // Calls the raw createToken from lexer/createToken.js
+    return _createTokenRaw(
+      type,
+      value,
+      startLine,
+      startColumn,
+      startPosition,
+      length,
+      this.filePath
+    );
+  }
+
+  // Helper for consistent error reporting (as discussed before)
+  error(message, code = "LEX000", suggestion = "") {
+    const errorToken = {
+      // Create a token-like object for error location
+      value:
+        this.peek() ||
+        this.source.substring(this.position - 1, this.position) ||
+        "",
+      line: this.line,
+      column: this.column,
+      start: this.position, // Point to the current position
+      length: 1,
+      file: this.filePath,
+    };
+    // CORRECTED: Pass arguments in the order expected by MimoError.lexerError(code, message, token, suggestion)
+    throw MimoError.lexerError(code, message, errorToken, suggestion);
+  }
+
+  nextToken() {
+    const startLine = this.line;
+    const startColumn = this.column;
+    const startPosition = this.position; // Capture start position for the token
+
+    skipWhitespace(this);
+
+    if (this.isAtEnd()) return null;
+
+    const char = this.peek(); // char needs to be defined here to be used later
+
+    // Handle template literal states (State 1 or 3)
+    if (this.templateLiteralState === 1 || this.templateLiteralState === 3) {
+      if (this.peek() === "`") {
+        this.advance();
+        this.templateLiteralState = 0;
+        this.templateLiteralDepth--;
+        return this._createToken(
+          TokenType.Backtick,
+          "`",
+          startLine,
+          startColumn,
+          startPosition,
+          1
+        );
+      }
+
+      if (this.peek() === "$" && this.peek(1) === "{") {
+        this.advance();
+        this.advance();
+        this.templateLiteralState = 2;
+        return this._createToken(
+          TokenType.InterpolationStart,
+          "${",
+          startLine,
+          startColumn,
+          startPosition,
+          2
+        );
+      }
+
+      let fragment = "";
+      // Loop to read string fragment, handling escapes
+      while (
+        !this.isAtEnd() &&
+        this.peek() !== "`" &&
+        !(this.peek() === "$" && this.peek(1) === "{")
+      ) {
+        const currentFragmentChar = this.peek(); // Renamed to avoid conflict with outer 'char'
+        if (currentFragmentChar === "\\") {
+          // <--- Correctly checking for backslash
+          this.advance(); // consume backslash
+          if (this.isAtEnd()) {
+            this.error(
+              "Unterminated escape sequence in template fragment.",
+              "LEX004",
+              "Complete the escape sequence or close the template literal."
+            );
+          }
+          const escapedChar = this.peek();
+          switch (escapedChar) {
+            case "n":
+              fragment += "\n";
+              break; // <--- FIXED: store actual newline
+            case "t":
+              fragment += "\t";
+              break; // <--- FIXED: store actual tab
+            case "r":
+              fragment += "\r";
+              break; // <--- FIXED: store actual carriage return
+            case "\\":
+              fragment += "\\";
+              break; // <--- FIXED: store actual backslash
+            case "`":
+              fragment += "`";
+              break;
+            case "$":
+              fragment += "$";
+              break;
+            case "{":
+              fragment += "{";
+              break;
+            default:
+              this.error(
+                `Unrecognized escape sequence in template: '\\${escapedChar}'.`,
+                "LEX005",
+                "Use valid escape sequences like '\\n', '\\t', '\\\\', '\\`', '\\$'."
+              );
+          }
+          this.advance(); // consume escaped character
+        } else if (currentFragmentChar === "\n") {
+          // <--- Handle literal newlines inside template fragments
+          fragment += currentFragmentChar;
+          this.advance();
+        } else {
+          fragment += currentFragmentChar;
+          this.advance();
+        }
+      }
+      if (fragment.length > 0) {
+        // Calculate length of the *original* source consumed for this fragment
+        return this._createToken(
+          TokenType.StringFragment,
+          fragment,
+          startLine,
+          startColumn,
+          startPosition,
+          this.position - startPosition
+        );
+      }
+    }
+
+    // Handle template literal start (` `)
+    if (char === "`") {
+      this.advance();
+      this.templateLiteralState = 1;
+      this.templateLiteralDepth++;
+      return this._createToken(
+        TokenType.Backtick,
+        "`",
+        startLine,
+        startColumn,
+        startPosition,
+        1
+      );
+    }
+    // Handle template interpolation end (})
+    if (this.templateLiteralState === 2 && char === "}") {
+      this.advance();
+      this.templateLiteralState = 3;
+      return this._createToken(
+        TokenType.InterpolationEnd,
+        "}",
+        startLine,
+        startColumn,
+        startPosition,
+        1
+      );
+    }
+
+    // Normal tokenizing mode
+    let token;
+    if (isAlpha(char)) {
+      token = readIdentifier(this);
+    } else if (isDigit(char)) {
+      token = readNumber(this);
+    } else if (char === '"') {
+      // Assuming only double quotes for strings based on current `readString`
+      token = readString(this);
+    } else {
+      token = readSymbol(this);
+    }
+    return token; // Tokenizers should return the token
+  }
+}

package/lexer/TokenTypes.js

@@ -0,0 +1,131 @@
+export const TokenType = {
+  Keyword: "keyword",
+  Identifier: "identifier",
+  Number: "number",
+  String: "string",
+  Boolean: "boolean",
+  Null: "null",
+  Operator: "operator",
+  LParen: "lparen",
+  RParen: "rparen",
+  LBracket: "lbracket",
+  RBracket: "rbracket",
+  LBrace: "lbrace",
+  RBrace: "rbrace",
+  Comma: "comma",
+  Range: "range",
+  Slice: "slice",
+  Spread: "spread",
+  Backtick: "backtick",
+  StringFragment: "string_fragment",
+  InterpolationStart: "interpolation_start",
+  InterpolationEnd: "interpolation_end",
+  Colon: "colon",
+  At: "at",
+};
+
+export const END_KEYWORDS = [
+  'end', 'else'
+];
+
+export const KEYWORDS = [
+  "set",
+  "let",
+  "const",
+  "global",
+  "destructure",
+  "from",
+  "if",
+  "guard",
+  "then",
+  "when",
+  "while",
+  "function",
+  "fn",
+  "call",
+  "show",
+  "return",
+  "try",
+  "catch",
+  "throw",
+  "for",
+  "in",
+  "match",
+  "case",
+  "default",
+  "break",
+  "continue",
+  "loop",
+  "true",
+  "false",
+  "null",
+  "import",
+  "as",
+  "export",
+  "not",
+  "and",
+  "or",
+  ...END_KEYWORDS
+];
+
+export const Punctuators = {
+  Semicolon: ";",
+  Comma: ",",
+  Dot: ".",
+  Question: "?",
+  Spread: "...",
+  Colon: ":",
+  LParen: "(",
+  RParen: ")",
+  LBracket: "[",
+  RBracket: "]",
+  LBrace: "{",
+  RBrace: "}",
+  Pipe: "|",
+  Ampersand: "&",
+};
+
+export const Keywords = {
+  set: TokenType.Keyword,
+  let: TokenType.Keyword,
+  const: TokenType.Keyword,
+  global: TokenType.Keyword,
+  if: TokenType.If,
+  while: TokenType.Keyword,
+  function: TokenType.Keyword,
+  call: TokenType.Keyword,
+  show: TokenType.Keyword,
+  return: TokenType.Keyword,
+  try: TokenType.Keyword,
+  catch: TokenType.Keyword,
+  throw: TokenType.Keyword,
+  for: TokenType.Keyword,
+  in: TokenType.Keyword,
+  match: TokenType.Keyword,
+  case: TokenType.Keyword,
+  default: TokenType.Keyword,
+  break: TokenType.Keyword,
+  continue: TokenType.Keyword,
+  loop: TokenType.Keyword,
+  true: TokenType.True,
+  false: TokenType.False,
+  null: TokenType.Null,
+  import: TokenType.Import,
+  export: TokenType.Export,
+};
+
+export const Operators = {
+  "+": TokenType.Plus,
+  "-": TokenType.Minus,
+  "=": TokenType.Equal,
+  "==": TokenType.EqualEqual,
+  "===": TokenType.EqualEqualEqual,
+  "!=": TokenType.BangEqual,
+  "!==": TokenType.BangEqualEqual,
+  ">": TokenType.Greater,
+  "<": TokenType.Less,
+  ">=": TokenType.GreaterEqual,
+  "<=": TokenType.LessEqual,
+  "&&": TokenType.And,
+  "||": TokenType.Or,
+};

package/lexer/createToken.js

@@ -0,0 +1,11 @@
+export function createToken(type, value, line, column, start, length, file = 'unknown') {
+    return {
+        type,
+        value,
+        line,
+        column,
+        start,  // Starting character index in the source string
+        length, // Number of characters this token spans in the source
+        file,   // The file path
+    };
+}

package/lexer/tokenizers/commentTokenizer.js

@@ -0,0 +1,45 @@
+export function isCommentStart(lexer) {
+  const char = lexer.peek();
+  return char === "/" && (lexer.peek(1) === "/" || lexer.peek(1) === "*");
+}
+
+export function skipSingleLineComment(lexer) {
+  // Skip the '//' characters
+  lexer.advance();
+  lexer.advance();
+
+  // Skip until end of line or end of file
+  while (!lexer.isAtEnd() && lexer.peek() !== "\n") {
+    lexer.advance();
+  }
+}
+
+export function skipMultiLineComment(lexer) {
+  // Skip the '/*' characters
+  lexer.advance();
+  lexer.advance();
+
+  // Skip until we find '*/' or reach end of file
+  while (!lexer.isAtEnd()) {
+    if (lexer.peek() === "*" && lexer.peek(1) === "/") {
+      // Found end of comment, skip the '*/' characters
+      lexer.advance();
+      lexer.advance();
+      break;
+    }
+    lexer.advance();
+  }
+}
+
+export function skipComments(lexer) {
+  while (!lexer.isAtEnd()) {
+    const char = lexer.peek();
+    if (char === "/" && lexer.peek(1) === "/") {
+      skipSingleLineComment(lexer);
+    } else if (char === "/" && lexer.peek(1) === "*") {
+      skipMultiLineComment(lexer);
+    } else {
+      break;
+    }
+  }
+}

package/lexer/tokenizers/literalTokenizer.js

@@ -0,0 +1,163 @@
+import { KEYWORDS, TokenType } from "../TokenTypes.js";
+import { createToken } from "../createToken.js";
+
+export function isLiteralStart(char) {
+  return isAlpha(char) || isDigit(char) || char === '"';
+}
+
+export function readIdentifier(lexer) {
+  const startLine = lexer.line;
+  const startColumn = lexer.column;
+  const startPosition = lexer.position; // Capture original start position
+  let value = "";
+
+  while (!lexer.isAtEnd() && isAlphaNumeric(lexer.peek())) {
+    value += lexer.peek();
+    lexer.advance();
+  }
+
+  const length = value.length; // Calculate length
+
+  // Handle boolean and null literals first, as they have specific TokenTypes
+  if (value === "true") {
+    return lexer._createToken(TokenType.Boolean, true, startLine, startColumn, startPosition, length);
+  }
+  if (value === "false") {
+    return lexer._createToken(TokenType.Boolean, false, startLine, startColumn, startPosition, length);
+  }
+  if (value === "null") {
+    return lexer._createToken(TokenType.Null, null, startLine, startColumn, startPosition, length);
+  }
+
+  // Check if the identifier is a keyword (using includes for arrays)
+  if (KEYWORDS.includes(value)) { // <--- FIXED: Use .includes() for array check
+    return lexer._createToken(TokenType.Keyword, value, startLine, startColumn, startPosition, length);
+  }
+
+  // Otherwise, it's a regular identifier
+  return lexer._createToken(TokenType.Identifier, value, startLine, startColumn, startPosition, length);
+}
+
+export function readNumber(lexer) {
+  const startLine = lexer.line;
+  const startCol = lexer.column; // Corrected to startColumn for consistency, though guide uses startCol
+  const startPosition = lexer.position; // Capture original start position
+  let value = "";
+  let hasDecimal = false;
+  let hasExponent = false;
+
+  while (!lexer.isAtEnd()) {
+    const char = lexer.peek();
+    if (isDigit(char)) {
+      value += char;
+    } else if (char === "." && !hasDecimal && !hasExponent && isDigit(lexer.peek(1))) {
+      value += char;
+      hasDecimal = true;
+    } else if ((char === "e" || char === "E") && !hasExponent) {
+      // Check if the next char is a sign or a digit for a valid exponent
+      const nextChar = lexer.peek(1);
+      if (nextChar === '+' || nextChar === '-' || isDigit(nextChar)) {
+        value += char; // Add 'e' or 'E'
+        lexer.advance(); // Consume 'e' or 'E'
+        if (nextChar === '+' || nextChar === '-') {
+          value += nextChar; // Add sign
+          lexer.advance(); // Consume sign
+        }
+        // Now read the exponent digits
+        while (!lexer.isAtEnd() && isDigit(lexer.peek())) {
+          value += lexer.peek();
+          lexer.advance();
+        }
+        hasExponent = true;
+        // Since we manually advanced for exponent parts, we break here
+        // The main loop's advance() would skip characters otherwise.
+        break;
+      } else {
+        // Not a valid exponent (e.g., "123e" followed by non-digit/non-sign)
+        break;
+      }
+    } else {
+      break;
+    }
+    lexer.advance();
+  }
+  // The guide.md snippet for readNumber is incomplete, so I'm using the logic from the `readIdentifier` and the `before` part of `readNumber`
+  // to ensure startPosition and length are correctly passed.
+  return lexer._createToken(TokenType.Number, parseFloat(value), startLine, startCol, startPosition, value.length);
+}
+
+export function readString(lexer) {
+  const startLine = lexer.line;
+  const startCol = lexer.column;
+  const startPos = lexer.position;
+  const quote = lexer.peek(); // Get the opening quote character
+  lexer.advance(); // Consume opening quote
+
+  let value = "";
+
+  while (!lexer.isAtEnd()) {
+    const char = lexer.peek();
+
+    if (char === quote) {
+      // End of string found
+      lexer.advance(); // Consume closing quote
+      const tokenLength = lexer.position - startPos;
+      return lexer._createToken(TokenType.String, value, startLine, startCol, startPos, tokenLength);
+    }
+
+    if (char === '\n') {
+      // Unterminated string error
+      lexer.error(
+        "Unterminated string literal. Newline encountered.",
+        'LEX004',
+        "String literals must be closed on the same line or use '\\n' for a newline character."
+      );
+    }
+
+    // --- CRITICAL FIX FOR ESCAPE CHARACTERS ---
+    if (char === '\\') {
+      lexer.advance(); // Consume the backslash
+
+      if (lexer.isAtEnd()) {
+        lexer.error("Unterminated escape sequence at end of file.", 'LEX005');
+      }
+
+      const escapedChar = lexer.peek();
+      switch (escapedChar) {
+        case 'n': value += '\n'; break;
+        case 't': value += '\t'; break;
+        case 'r': value += '\r'; break;
+        case '\\': value += '\\'; break;
+        case '"': value += '"'; break; // This handles \"
+        // You can add support for other quotes like \' if needed
+        default:
+          lexer.error(`Invalid escape sequence: \\${escapedChar}`, 'LEX003');
+      }
+    } else {
+      // Regular character
+      value += char;
+    }
+
+    lexer.advance(); // Move to the next character in the source
+  }
+
+  // If the loop finishes, it means we hit the end of the file without a closing quote.
+  lexer.error(
+    "Unterminated string literal.",
+    'LEX005',
+    `A string starting with ${quote} was not properly closed.`
+  );
+}
+
+// Helper functions
+export function isAlpha(char) {
+  return /^[a-zA-Z_]$/.test(char);
+}
+
+export function isAlphaNumeric(char) {
+  return /^[a-zA-Z0-9_]$/.test(char);
+}
+
+export function isDigit(char) {
+  return /^[0-9]$/.test(char);
+}

package/lexer/tokenizers/symbolTokenizer.js

@@ -0,0 +1,69 @@
+import { TokenType } from "../TokenTypes.js";
+import { createToken } from "../createToken.js";
+
+export const SYMBOLS = {
+  "(": TokenType.LParen,
+  ")": TokenType.RParen,
+  "[": TokenType.LBracket,
+  "]": TokenType.RBracket,
+  "{": TokenType.LBrace,
+  "}": TokenType.RBrace,
+  ",": TokenType.Comma,
+  "?.": TokenType.Operator,
+  ".": TokenType.Operator,
+  "+": TokenType.Operator,
+  "-": TokenType.Operator,
+  "*": TokenType.Operator,
+  "/": TokenType.Operator,
+  "%": TokenType.Operator,
+  ">": TokenType.Operator,
+  "<": TokenType.Operator,
+  "=": TokenType.Operator,
+  "==": TokenType.Operator,
+  "===": TokenType.Operator,
+  "!": TokenType.Operator,
+  "!=": TokenType.Operator,
+  "!==": TokenType.Operator,
+  ">=": TokenType.Operator,
+  "<=": TokenType.Operator,
+  "&&": TokenType.Operator,
+  "||": TokenType.Operator,
+  "|>": TokenType.Operator, // Pipe operator
+  "->": TokenType.Operator, // Arrow operator
+  "??": TokenType.Operator, // Null coalescing
+
+  "...": TokenType.Spread, // Spread operator
+  "..": TokenType.Range, // Range operator
+  ":": TokenType.Colon,
+  "@": TokenType.At,
+};
+
+
+
+export function readSymbol(lexer) {
+  const startLine = lexer.line;
+  const startColumn = lexer.column;
+  const startPosition = lexer.position; // Capture original start position
+
+  // Try to match longer symbols first (e.g., "==" before "=")
+  const sortedSymbols = Object.keys(SYMBOLS).sort((a, b) => b.length - a.length);
+
+  for (const symbolString of sortedSymbols) {
+    if (lexer.source.startsWith(symbolString, lexer.position)) {
+      // Found a match
+      for (let i = 0; i < symbolString.length; i++) {
+        lexer.advance(); // Consume the characters of the symbol
+      }
+      const tokenLength = symbolString.length;
+      return lexer._createToken(SYMBOLS[symbolString], symbolString, startLine, startColumn, startPosition, tokenLength); // <--- Use lexer._createToken
+    }
+  }
+
+  // If no recognized symbol prefix is found, it's an error
+  const char = lexer.peek(); // Get the current character that caused the issue
+  lexer.error( // <--- Use lexer.error
+    `Unrecognized symbol or character: '${char}'.`,
+    'LEX007',
+    `The symbol or character '${char}' is not recognized. Check for typos or unsupported operators.`
+  );
+}

package/lexer/tokenizers/whitespaceTokenizer.js

@@ -0,0 +1,36 @@
+import { skipComments } from "./commentTokenizer.js";
+
+export function isWhitespace(char) {
+  return char === " " || char === "\t" || char === "\n" || char === "\r";
+}
+
+export function skipWhitespace(lexer) {
+  while (!lexer.isAtEnd()) {
+    const char = lexer.peek();
+    
+    if (char === " " || char === "\t") {
+      lexer.advance();
+    } else if (char === "\n") {
+      lexer.advance();
+    } else if (char === "/" && (lexer.peek(1) === "/" || lexer.peek(1) === "*")) {
+      // Handle comments as part of whitespace skipping
+      skipComments(lexer);
+    } else {
+      break;
+    }
+  }
+}
+
+export function skipSimpleWhitespace(lexer) {
+  while (!lexer.isAtEnd()) {
+    const char = lexer.peek();
+    
+    if (char === " " || char === "\t") {
+      lexer.advance();
+    } else if (char === "\n") {
+      lexer.advance();
+    } else {
+      break;
+    }
+  }
+}