@fc-components/monaco-editor 0.2.1 → 0.3.1

package/src/expr/parser/lexer.ts

@@ -0,0 +1,377 @@
+/**
+ * Expr-lang lexer/tokenizer
+ * Handles: identifiers, numbers, strings, bytes literals, operators, comments
+ */
+
+import { Token, TokenKind, EOF_TOKEN } from './types';
+
+// Tokens that should be treated as Operators even if they look like identifiers
+const OPERATOR_TOKENS = new Set([
+  'let', 'if', 'else',
+  'not', 'and', 'or',
+  'in', 'matches', 'contains',
+  'startsWith', 'endsWith',
+  'hasPrefix', 'hasSuffix',
+]);
+
+export class Lexer {
+  private input = '';
+  private pos = 0;
+  private line = 1;
+  private column = 1;
+  private startLine = 1;
+  private startColumn = 1;
+
+  reset(input: string): void {
+    this.input = input;
+    this.pos = 0;
+    this.line = 1;
+    this.column = 1;
+    this.startLine = 1;
+    this.startColumn = 1;
+  }
+
+  next(): Token {
+    this.skipWhitespace();
+    this.skipComments();
+
+    if (this.pos >= this.input.length) {
+      return { ...EOF_TOKEN, start: this.pos, end: this.pos, line: this.line, column: this.column };
+    }
+
+    this.startLine = this.line;
+    this.startColumn = this.column;
+
+    const ch = this.input[this.pos];
+
+    // String literals
+    if (ch === '"' || ch === "'") {
+      return this.readString(ch);
+    }
+    if (ch === '`') {
+      return this.readBacktickString();
+    }
+
+    // Byte literals: b"..." or b'...' or B"..." or B'...'
+    if ((ch === 'b' || ch === 'B') && this.pos + 1 < this.input.length) {
+      const next = this.input[this.pos + 1];
+      if (next === '"' || next === "'") {
+        return this.readByteLiteral();
+      }
+    }
+
+    // Numbers
+    if (this.isDigit(ch) || (ch === '.' && this.pos + 1 < this.input.length && this.isDigit(this.input[this.pos + 1]))) {
+      return this.readNumber();
+    }
+
+    // Identifiers and keywords
+    if (this.isIdentStart(ch)) {
+      const token = this.readIdentOrKeyword();
+      return token;
+    }
+
+    // Operators and brackets
+    return this.readOperatorOrBracket();
+  }
+
+  private skipWhitespace(): void {
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (ch === ' ' || ch === '\t' || ch === '\r') {
+        this.pos++;
+        this.column++;
+      } else if (ch === '\n') {
+        this.pos++;
+        this.line++;
+        this.column = 1;
+      } else {
+        break;
+      }
+    }
+  }
+
+  private skipComments(): void {
+    while (this.pos < this.input.length) {
+      // Line comment //
+      if (this.input[this.pos] === '/' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '/') {
+        this.pos += 2;
+        this.column += 2;
+        while (this.pos < this.input.length && this.input[this.pos] !== '\n') {
+          this.pos++;
+          this.column++;
+        }
+        this.skipWhitespace();
+        continue;
+      }
+      // Block comment /* */
+      if (this.input[this.pos] === '/' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '*') {
+        this.pos += 2;
+        this.column += 2;
+        while (this.pos < this.input.length) {
+          if (this.input[this.pos] === '*' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '/') {
+            this.pos += 2;
+            this.column += 2;
+            break;
+          }
+          if (this.input[this.pos] === '\n') {
+            this.line++;
+            this.column = 1;
+          } else {
+            this.column++;
+          }
+          this.pos++;
+        }
+        this.skipWhitespace();
+        continue;
+      }
+      break;
+    }
+  }
+
+  private readString(quote: string): Token {
+    const start = this.pos;
+    this.pos++; // skip opening quote
+    this.column++;
+
+    let value = '';
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (ch === '\\') {
+        if (this.pos + 1 >= this.input.length) break;
+        const next = this.input[this.pos + 1];
+        switch (next) {
+          case 'n': value += '\n'; break;
+          case 't': value += '\t'; break;
+          case 'r': value += '\r'; break;
+          case '\\': value += '\\'; break;
+          case quote: value += quote; break;
+          case 'x': {
+            const hex = this.input.substr(this.pos + 2, 2);
+            value += String.fromCharCode(parseInt(hex, 16));
+            this.pos += 2;
+            this.column += 2;
+            break;
+          }
+          case 'u': {
+            const unicode = this.input.substr(this.pos + 2, 4);
+            value += String.fromCharCode(parseInt(unicode, 16));
+            this.pos += 2;
+            this.column += 2;
+            break;
+          }
+          default:
+            value += next;
+        }
+        this.pos += 2;
+        this.column += 2;
+      } else if (ch === quote) {
+        this.pos++; // skip closing quote
+        this.column++;
+        return this.makeToken(TokenKind.String, value, start);
+      } else {
+        value += ch;
+        this.pos++;
+        this.column++;
+        if (ch === '\n') {
+          this.line++;
+          this.column = 1;
+        }
+      }
+    }
+
+    return this.makeToken(TokenKind.String, value, start);
+  }
+
+  private readBacktickString(): Token {
+    const start = this.pos;
+    this.pos++; // skip opening backtick
+    this.column++;
+
+    let value = '';
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (ch === '`') {
+        this.pos++;
+        this.column++;
+        return this.makeToken(TokenKind.String, value, start);
+      }
+      value += ch;
+      this.pos++;
+      if (ch === '\n') {
+        this.line++;
+        this.column = 1;
+      } else {
+        this.column++;
+      }
+    }
+
+    return this.makeToken(TokenKind.String, value, start);
+  }
+
+  private readByteLiteral(): Token {
+    // b"..." or b'...' or B"..." or B'...'
+    const start = this.pos;
+    this.pos++; // skip 'b' or 'B'
+    this.column++;
+
+    const quote = this.input[this.pos];
+    this.pos++; // skip quote
+    this.column++;
+
+    let value = '';
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (ch === '\\') {
+        if (this.pos + 1 >= this.input.length) break;
+        const next = this.input[this.pos + 1];
+        switch (next) {
+          case 'n': value += '\n'; break;
+          case 't': value += '\t'; break;
+          case 'r': value += '\r'; break;
+          case '\\': value += '\\'; break;
+          case quote: value += quote; break;
+          case 'x': {
+            const hex = this.input.substr(this.pos + 2, 2);
+            value += String.fromCharCode(parseInt(hex, 16));
+            this.pos += 2;
+            this.column += 2;
+            break;
+          }
+          default:
+            // octal escape \NNN
+            if (this.isDigit(next)) {
+              const octal = this.input.substr(this.pos + 1, 3);
+              value += String.fromCharCode(parseInt(octal, 8));
+              this.pos += 2;
+              this.column += 2;
+              break;
+            }
+            value += next;
+        }
+        this.pos += 2;
+        this.column += 2;
+      } else if (ch === quote) {
+        this.pos++;
+        this.column++;
+        return this.makeToken(TokenKind.String, value, start);
+      } else {
+        value += ch;
+        this.pos++;
+        this.column++;
+      }
+    }
+
+    return this.makeToken(TokenKind.String, value, start);
+  }
+
+  private readNumber(): Token {
+    const start = this.pos;
+    let value = '';
+
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (this.isDigit(ch) || ch === '.' || ch === 'e' || ch === 'E' || ch === '+' || ch === '-' || ch === '_' || ch === 'x' || ch === 'X' || ch === 'o' || ch === 'O' || ch === 'b' || ch === 'B' || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')) {
+        value += ch;
+        this.pos++;
+        this.column++;
+      } else {
+        break;
+      }
+    }
+
+    return this.makeToken(TokenKind.Number, value, start);
+  }
+
+  private readIdentOrKeyword(): Token {
+    const start = this.pos;
+    let value = '';
+
+    while (this.pos < this.input.length) {
+      const ch = this.input[this.pos];
+      if (this.isIdentPart(ch)) {
+        value += ch;
+        this.pos++;
+        this.column++;
+      } else {
+        break;
+      }
+    }
+
+    // Keywords that are operators
+    if (OPERATOR_TOKENS.has(value)) {
+      return this.makeToken(TokenKind.Operator, value, start);
+    }
+
+    return this.makeToken(TokenKind.Identifier, value, start);
+  }
+
+  private readOperatorOrBracket(): Token {
+    const start = this.pos;
+    const ch = this.input[this.pos];
+
+    // Brackets
+    if ('()[]{}'.includes(ch)) {
+      this.pos++;
+      this.column++;
+      return this.makeToken(TokenKind.Bracket, ch, start);
+    }
+
+    // Multi-character operators
+    const twoChar = this.input.substr(this.pos, 2);
+    const threeChar = this.input.substr(this.pos, 3);
+
+    // 3-character operators: **=
+    // 2-character operators
+    const twoCharOps = ['==', '!=', '<=', '>=', '&&', '||', '??', '?.', '..', '**', '//', '::', '->'];
+    const threeCharOps = ['...', '<<=', '>>='];
+
+    if (threeCharOps.includes(threeChar)) {
+      this.pos += 3;
+      this.column += 3;
+      return this.makeToken(TokenKind.Operator, threeChar, start);
+    }
+
+    if (twoCharOps.includes(twoChar)) {
+      this.pos += 2;
+      this.column += 2;
+      return this.makeToken(TokenKind.Operator, twoChar, start);
+    }
+
+    // Single-character operators and separators
+    const singleCharOps = '+-*/%^=<>!&|?.,;:#@$~';
+    if (singleCharOps.includes(ch)) {
+      this.pos++;
+      this.column++;
+      return this.makeToken(TokenKind.Operator, ch, start);
+    }
+
+    // Unrecognized character
+    this.pos++;
+    this.column++;
+    return this.makeToken(TokenKind.Operator, ch, start);
+  }
+
+  private isDigit(ch: string): boolean {
+    return ch >= '0' && ch <= '9';
+  }
+
+  private isIdentStart(ch: string): boolean {
+    return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_' || ch === '$';
+  }
+
+  private isIdentPart(ch: string): boolean {
+    return this.isIdentStart(ch) || this.isDigit(ch);
+  }
+
+  private makeToken(kind: TokenKind, value: string, start: number): Token {
+    return {
+      kind,
+      value,
+      start,
+      end: this.pos,
+      line: this.startLine,
+      column: this.startColumn,
+    };
+  }
+}