ether-code 0.1.0

package/lexer/ether-lexer.js

@@ -0,0 +1,728 @@
+const {
+  Token,
+  TokenType,
+  LANG_BLOCKS,
+  normalizeAccents,
+  isCompoundKeyword,
+  couldBeCompoundStart
+} = require('./tokens')
+
+class EtherLexer {
+  constructor(source, options = {}) {
+    this.source = source
+    this.pos = 0
+    this.line = 1
+    this.column = 1
+    this.tokens = []
+    this.indentStack = [0]
+    this.currentIndent = 0
+    this.atLineStart = true
+    
+    this.options = {
+      trackComments: options.trackComments || false,
+      allowUnicode: options.allowUnicode !== false,
+      tabSize: options.tabSize || 2
+    }
+  }
+
+  peek(offset = 0) {
+    const idx = this.pos + offset
+    return idx < this.source.length ? this.source[idx] : null
+  }
+
+  advance() {
+    const char = this.source[this.pos]
+    this.pos++
+    if (char === '\n') {
+      this.line++
+      this.column = 1
+      this.atLineStart = true
+    } else {
+      this.column++
+    }
+    return char
+  }
+
+  match(expected) {
+    if (this.peek() === expected) {
+      this.advance()
+      return true
+    }
+    return false
+  }
+
+  isAtEnd() {
+    return this.pos >= this.source.length
+  }
+
+  isDigit(char) {
+    if (!char) return false
+    return char >= '0' && char <= '9'
+  }
+
+  isHexDigit(char) {
+    if (!char) return false
+    return this.isDigit(char) || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')
+  }
+
+  isAlpha(char) {
+    if (!char) return false
+    const code = char.charCodeAt(0)
+    if ((code >= 65 && code <= 90) || (code >= 97 && code <= 122)) return true
+    if (char === '_') return true
+    if (this.options.allowUnicode) {
+      if (code >= 0x00C0 && code <= 0x024F) return true
+      if (code >= 0x0400 && code <= 0x04FF) return true
+      if (code >= 0x4E00 && code <= 0x9FFF) return true
+      if (code >= 0x3040 && code <= 0x30FF) return true
+      if (code >= 0x0600 && code <= 0x06FF) return true
+      if (code >= 0x0370 && code <= 0x03FF) return true
+    }
+    return false
+  }
+
+  isAlphaNumeric(char) {
+    return this.isAlpha(char) || this.isDigit(char)
+  }
+
+  isWhitespace(char) {
+    if (!char) return false
+    return char === ' ' || char === '\t' || char === '\r'
+  }
+
+  addToken(type, value, startColumn = null) {
+    const col = startColumn !== null ? startColumn : this.column
+    this.tokens.push(new Token(type, value, this.line, col, this.currentIndent))
+  }
+
+  tokenize() {
+    while (!this.isAtEnd()) {
+      this.scanToken()
+    }
+
+    while (this.indentStack.length > 1) {
+      this.indentStack.pop()
+      this.tokens.push(new Token(TokenType.DEDENT, '', this.line, this.column, 0))
+    }
+
+    this.tokens.push(new Token(TokenType.EOF, '', this.line, this.column, 0))
+    return this.tokens
+  }
+
+  scanToken() {
+    if (this.atLineStart) {
+      this.handleIndentation()
+      if (this.isAtEnd()) return
+    }
+
+    const char = this.peek()
+
+    if (char === '\n') {
+      this.advance()
+      this.addToken(TokenType.NEWLINE, '\n')
+      return
+    }
+
+    if (this.isWhitespace(char)) {
+      this.advance()
+      return
+    }
+
+    if (char === '#') {
+      this.scanComment()
+      return
+    }
+
+    if (char === '/' && this.peek(1) === '/') {
+      this.scanComment()
+      return
+    }
+
+    if (char === '/' && this.peek(1) === '*') {
+      this.scanBlockComment()
+      return
+    }
+
+    if (char === '"' || char === "'") {
+      this.scanString(char)
+      return
+    }
+
+    if (char === '`') {
+      this.scanTemplateString()
+      return
+    }
+
+    if (this.isDigit(char) || (char === '-' && this.isDigit(this.peek(1)))) {
+      this.scanNumber()
+      return
+    }
+
+    if (this.isAlpha(char)) {
+      this.scanIdentifier()
+      return
+    }
+
+    this.scanOperator()
+  }
+
+  handleIndentation() {
+    let indent = 0
+    while (!this.isAtEnd() && (this.peek() === ' ' || this.peek() === '\t')) {
+      if (this.peek() === '\t') {
+        indent += this.options.tabSize
+      } else {
+        indent++
+      }
+      this.advance()
+    }
+
+    if (this.isAtEnd() || this.peek() === '\n' || this.peek() === '#') {
+      this.atLineStart = false
+      return
+    }
+
+    if (this.peek() === '/' && this.peek(1) === '/') {
+      this.atLineStart = false
+      return
+    }
+
+    this.currentIndent = indent
+    this.atLineStart = false
+
+    const currentLevel = this.indentStack[this.indentStack.length - 1]
+
+    if (indent > currentLevel) {
+      this.indentStack.push(indent)
+      this.tokens.push(new Token(TokenType.INDENT, indent, this.line, 1, indent))
+    } else if (indent < currentLevel) {
+      while (this.indentStack.length > 1 && this.indentStack[this.indentStack.length - 1] > indent) {
+        this.indentStack.pop()
+        this.tokens.push(new Token(TokenType.DEDENT, '', this.line, 1, indent))
+      }
+      if (this.indentStack[this.indentStack.length - 1] !== indent) {
+        this.tokens.push(new Token(TokenType.ERROR, 'Indentation incohérente', this.line, 1, indent))
+      }
+    }
+  }
+
+  scanComment() {
+    const startLine = this.line
+    const startCol = this.column
+    let comment = ''
+
+    if (this.peek() === '#') {
+      this.advance()
+    } else {
+      this.advance()
+      this.advance()
+    }
+
+    while (!this.isAtEnd() && this.peek() !== '\n') {
+      comment += this.advance()
+    }
+
+    if (this.options.trackComments) {
+      this.tokens.push(new Token(TokenType.COMMENT, comment.trim(), startLine, startCol, this.currentIndent))
+    }
+  }
+
+  scanBlockComment() {
+    const startLine = this.line
+    const startCol = this.column
+    let comment = ''
+
+    this.advance()
+    this.advance()
+
+    while (!this.isAtEnd()) {
+      if (this.peek() === '*' && this.peek(1) === '/') {
+        this.advance()
+        this.advance()
+        break
+      }
+      comment += this.advance()
+    }
+
+    if (this.options.trackComments) {
+      this.tokens.push(new Token(TokenType.BLOCK_COMMENT, comment.trim(), startLine, startCol, this.currentIndent))
+    }
+  }
+
+  scanString(quote) {
+    const startLine = this.line
+    const startCol = this.column
+    let value = ''
+    let isBlock = false
+
+    this.advance()
+
+    if (this.peek() === quote && this.peek(1) === quote) {
+      this.advance()
+      this.advance()
+      isBlock = true
+    }
+
+    while (!this.isAtEnd()) {
+      if (isBlock) {
+        if (this.peek() === quote && this.peek(1) === quote && this.peek(2) === quote) {
+          this.advance()
+          this.advance()
+          this.advance()
+          break
+        }
+      } else {
+        if (this.peek() === quote) {
+          this.advance()
+          break
+        }
+        if (this.peek() === '\n') {
+          this.tokens.push(new Token(TokenType.ERROR, 'Chaîne non terminée', startLine, startCol, this.currentIndent))
+          return
+        }
+      }
+
+      if (this.peek() === '\\' && !isBlock) {
+        this.advance()
+        const escaped = this.advance()
+        switch (escaped) {
+          case 'n': value += '\n'; break
+          case 't': value += '\t'; break
+          case 'r': value += '\r'; break
+          case '\\': value += '\\'; break
+          case "'": value += "'"; break
+          case '"': value += '"'; break
+          case '0': value += '\0'; break
+          case 'x':
+            let hex = ''
+            for (let i = 0; i < 2 && this.isHexDigit(this.peek()); i++) {
+              hex += this.advance()
+            }
+            value += String.fromCharCode(parseInt(hex, 16))
+            break
+          case 'u':
+            let unicode = ''
+            if (this.peek() === '{') {
+              this.advance()
+              while (this.isHexDigit(this.peek())) {
+                unicode += this.advance()
+              }
+              if (this.peek() === '}') this.advance()
+            } else {
+              for (let i = 0; i < 4 && this.isHexDigit(this.peek()); i++) {
+                unicode += this.advance()
+              }
+            }
+            value += String.fromCodePoint(parseInt(unicode, 16))
+            break
+          default:
+            value += escaped
+        }
+      } else {
+        value += this.advance()
+      }
+    }
+
+    this.tokens.push(new Token(
+      isBlock ? TokenType.BLOCK_STRING : TokenType.STRING,
+      value,
+      startLine,
+      startCol,
+      this.currentIndent
+    ))
+  }
+
+  scanTemplateString() {
+    const startLine = this.line
+    const startCol = this.column
+    let value = ''
+
+    this.advance()
+
+    while (!this.isAtEnd() && this.peek() !== '`') {
+      if (this.peek() === '\\') {
+        this.advance()
+        value += this.advance()
+      } else {
+        value += this.advance()
+      }
+    }
+
+    if (this.peek() === '`') {
+      this.advance()
+    }
+
+    this.tokens.push(new Token(TokenType.BLOCK_STRING, value, startLine, startCol, this.currentIndent))
+  }
+
+  scanNumber() {
+    const startLine = this.line
+    const startCol = this.column
+    let value = ''
+    let type = TokenType.INTEGER
+
+    if (this.peek() === '-') {
+      value += this.advance()
+    }
+
+    if (this.peek() === '0' && (this.peek(1) === 'x' || this.peek(1) === 'X')) {
+      value += this.advance()
+      value += this.advance()
+      while (this.isHexDigit(this.peek())) {
+        value += this.advance()
+      }
+      this.tokens.push(new Token(TokenType.HEX, value, startLine, startCol, this.currentIndent))
+      return
+    }
+
+    if (this.peek() === '0' && (this.peek(1) === 'b' || this.peek(1) === 'B')) {
+      this.advance()
+      this.advance()
+      let binValue = ''
+      while (this.peek() === '0' || this.peek() === '1') {
+        binValue += this.advance()
+      }
+      this.tokens.push(new Token(TokenType.INTEGER, parseInt(binValue, 2), startLine, startCol, this.currentIndent))
+      return
+    }
+
+    if (this.peek() === '0' && (this.peek(1) === 'o' || this.peek(1) === 'O')) {
+      this.advance()
+      this.advance()
+      let octValue = ''
+      while (this.peek() >= '0' && this.peek() <= '7') {
+        octValue += this.advance()
+      }
+      this.tokens.push(new Token(TokenType.INTEGER, parseInt(octValue, 8), startLine, startCol, this.currentIndent))
+      return
+    }
+
+    while (this.isDigit(this.peek()) || this.peek() === '_') {
+      if (this.peek() !== '_') {
+        value += this.advance()
+      } else {
+        this.advance()
+      }
+    }
+
+    if (this.peek() === '.' && this.isDigit(this.peek(1))) {
+      type = TokenType.FLOAT
+      value += this.advance()
+      while (this.isDigit(this.peek()) || this.peek() === '_') {
+        if (this.peek() !== '_') {
+          value += this.advance()
+        } else {
+          this.advance()
+        }
+      }
+    }
+
+    if (this.peek() === 'e' || this.peek() === 'E') {
+      type = TokenType.FLOAT
+      value += this.advance()
+      if (this.peek() === '+' || this.peek() === '-') {
+        value += this.advance()
+      }
+      while (this.isDigit(this.peek())) {
+        value += this.advance()
+      }
+    }
+
+    if (type === TokenType.FLOAT) {
+      this.tokens.push(new Token(type, parseFloat(value), startLine, startCol, this.currentIndent))
+    } else {
+      this.tokens.push(new Token(type, parseInt(value, 10), startLine, startCol, this.currentIndent))
+    }
+  }
+
+  scanIdentifier() {
+    const startLine = this.line
+    const startCol = this.column
+    let value = ''
+
+    while (this.isAlphaNumeric(this.peek()) || this.peek() === '-' || this.peek() === '_') {
+      value += this.advance()
+    }
+
+    let keepLooking = true
+    while (keepLooking && this.peek() === ' ') {
+      const savedPos = this.pos
+      const savedLine = this.line
+      const savedCol = this.column
+      
+      this.advance()
+      
+      let nextWord = ''
+      while (this.isAlphaNumeric(this.peek()) || this.peek() === '-' || this.peek() === '_') {
+        nextWord += this.advance()
+      }
+      
+      if (nextWord) {
+        const combined = value + ' ' + nextWord
+        if (isCompoundKeyword(combined) || couldBeCompoundStart(combined)) {
+          value = combined
+        } else {
+          this.pos = savedPos
+          this.line = savedLine
+          this.column = savedCol
+          keepLooking = false
+        }
+      } else {
+        this.pos = savedPos
+        this.line = savedLine
+        this.column = savedCol
+        keepLooking = false
+      }
+    }
+
+    const lowerValue = value.toLowerCase()
+    if (this.peek() === ':' && LANG_BLOCKS.includes(lowerValue)) {
+      this.advance()
+      this.tokens.push(new Token(TokenType.LANG_BLOCK, lowerValue, startLine, startCol, this.currentIndent))
+      return
+    }
+
+    this.tokens.push(new Token(TokenType.IDENTIFIER, value, startLine, startCol, this.currentIndent))
+  }
+
+  scanOperator() {
+    const startLine = this.line
+    const startCol = this.column
+    const char = this.advance()
+
+    switch (char) {
+      case ':':
+        if (this.match(':')) {
+          this.tokens.push(new Token(TokenType.DOUBLE_COLON, '::', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.COLON, ':', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '=':
+        if (this.match('=')) {
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.TRIPLE_EQUALS, '===', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.DOUBLE_EQUALS, '==', startLine, startCol, this.currentIndent))
+          }
+        } else if (this.match('>')) {
+          this.tokens.push(new Token(TokenType.FAT_ARROW, '=>', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.EQUALS, '=', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '!':
+        if (this.match('=')) {
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.NOT_DOUBLE_EQUALS, '!==', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.NOT_EQUALS, '!=', startLine, startCol, this.currentIndent))
+          }
+        } else {
+          this.tokens.push(new Token(TokenType.BANG, '!', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case ',':
+        this.tokens.push(new Token(TokenType.COMMA, ',', startLine, startCol, this.currentIndent))
+        break
+
+      case '.':
+        if (this.match('.')) {
+          if (this.match('.')) {
+            this.tokens.push(new Token(TokenType.SPREAD, '...', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.DOT, '.', startLine, startCol, this.currentIndent))
+            this.tokens.push(new Token(TokenType.DOT, '.', startLine, startCol + 1, this.currentIndent))
+          }
+        } else {
+          this.tokens.push(new Token(TokenType.DOT, '.', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '+':
+        if (this.match('+')) {
+          this.tokens.push(new Token(TokenType.PLUS_PLUS, '++', startLine, startCol, this.currentIndent))
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.PLUS_EQUALS, '+=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.PLUS, '+', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '-':
+        if (this.match('-')) {
+          this.tokens.push(new Token(TokenType.MINUS_MINUS, '--', startLine, startCol, this.currentIndent))
+        } else if (this.match('>')) {
+          this.tokens.push(new Token(TokenType.ARROW, '->', startLine, startCol, this.currentIndent))
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.MINUS_EQUALS, '-=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.MINUS, '-', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '*':
+        if (this.match('*')) {
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.DOUBLE_STAR_EQUALS, '**=', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.DOUBLE_STAR, '**', startLine, startCol, this.currentIndent))
+          }
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.STAR_EQUALS, '*=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.STAR, '*', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '/':
+        if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.SLASH_EQUALS, '/=', startLine, startCol, this.currentIndent))
+        } else if (this.match('/')) {
+          this.tokens.push(new Token(TokenType.DOUBLE_SLASH, '//', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.SLASH, '/', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '%':
+        if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.PERCENT_EQUALS, '%=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.PERCENT, '%', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '<':
+        if (this.match('=')) {
+          if (this.match('>')) {
+            this.tokens.push(new Token(TokenType.SPACESHIP, '<=>', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.LTE, '<=', startLine, startCol, this.currentIndent))
+          }
+        } else if (this.match('<')) {
+          this.tokens.push(new Token(TokenType.LT_LT, '<<', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.LT, '<', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '>':
+        if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.GTE, '>=', startLine, startCol, this.currentIndent))
+        } else if (this.match('>')) {
+          if (this.match('>')) {
+            this.tokens.push(new Token(TokenType.GT_GT_GT, '>>>', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.GT_GT, '>>', startLine, startCol, this.currentIndent))
+          }
+        } else {
+          this.tokens.push(new Token(TokenType.GT, '>', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '&':
+        if (this.match('&')) {
+          this.tokens.push(new Token(TokenType.DOUBLE_AMPERSAND, '&&', startLine, startCol, this.currentIndent))
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.AMPERSAND_EQUALS, '&=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.AMPERSAND, '&', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '|':
+        if (this.match('|')) {
+          this.tokens.push(new Token(TokenType.DOUBLE_PIPE, '||', startLine, startCol, this.currentIndent))
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.PIPE_EQUALS, '|=', startLine, startCol, this.currentIndent))
+        } else if (this.match('>')) {
+          this.tokens.push(new Token(TokenType.PIPE_GT, '|>', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.PIPE, '|', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '^':
+        if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.CARET_EQUALS, '^=', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.CARET, '^', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '~':
+        this.tokens.push(new Token(TokenType.TILDE, '~', startLine, startCol, this.currentIndent))
+        break
+
+      case '?':
+        if (this.match('?')) {
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.QUESTION_QUESTION_EQUALS, '??=', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.DOUBLE_QUESTION, '??', startLine, startCol, this.currentIndent))
+          }
+        } else if (this.match('.')) {
+          this.tokens.push(new Token(TokenType.QUESTION_DOT, '?.', startLine, startCol, this.currentIndent))
+        } else {
+          this.tokens.push(new Token(TokenType.QUESTION, '?', startLine, startCol, this.currentIndent))
+        }
+        break
+
+      case '@':
+        this.tokens.push(new Token(TokenType.AT, '@', startLine, startCol, this.currentIndent))
+        break
+
+      case '#':
+        this.tokens.push(new Token(TokenType.HASH, '#', startLine, startCol, this.currentIndent))
+        break
+
+      case '\\':
+        this.tokens.push(new Token(TokenType.BACKSLASH, '\\', startLine, startCol, this.currentIndent))
+        break
+
+      case '(':
+        this.tokens.push(new Token(TokenType.LPAREN, '(', startLine, startCol, this.currentIndent))
+        break
+
+      case ')':
+        this.tokens.push(new Token(TokenType.RPAREN, ')', startLine, startCol, this.currentIndent))
+        break
+
+      case '[':
+        this.tokens.push(new Token(TokenType.LBRACKET, '[', startLine, startCol, this.currentIndent))
+        break
+
+      case ']':
+        this.tokens.push(new Token(TokenType.RBRACKET, ']', startLine, startCol, this.currentIndent))
+        break
+
+      case '{':
+        this.tokens.push(new Token(TokenType.LBRACE, '{', startLine, startCol, this.currentIndent))
+        break
+
+      case '}':
+        this.tokens.push(new Token(TokenType.RBRACE, '}', startLine, startCol, this.currentIndent))
+        break
+
+      case ';':
+        this.tokens.push(new Token(TokenType.SEMICOLON, ';', startLine, startCol, this.currentIndent))
+        break
+
+      default:
+        this.tokens.push(new Token(TokenType.ERROR, `Caractère inattendu: ${char}`, startLine, startCol, this.currentIndent))
+    }
+  }
+
+  static tokenize(source, options = {}) {
+    const lexer = new EtherLexer(source, options)
+    return lexer.tokenize()
+  }
+}
+
+module.exports = {
+  EtherLexer
+}