ether-code 0.8.6 → 0.8.8

package/lexer/ether-lexer.js

@@ -31,6 +31,7 @@ const TokenType = {
   SPREAD: 'SPREAD',
   ARROW: 'ARROW',
   FAT_ARROW: 'FAT_ARROW',
+  SEMICOLON: 'SEMICOLON',
   
   PLUS: 'PLUS',
   MINUS: 'MINUS',
@@ -75,7 +76,6 @@ const TokenType = {
   
   COMMENT: 'COMMENT',
   BLOCK_COMMENT: 'BLOCK_COMMENT',
-  SEMICOLON: 'SEMICOLON',
   
   EOF: 'EOF',
   ERROR: 'ERROR'
@@ -100,17 +100,20 @@ class EtherLexer {
     this.indentStack = [0]
     this.currentIndent = 0
     this.atLineStart = true
+    this.options = options
+    this.lastTokenType = null
     
-    this.options = {
-      trackComments: options.trackComments || false,
-      allowUnicode: options.allowUnicode !== false,
-      tabSize: options.tabSize || 2
-    }
+    this.compoundKeywordsCache = null
+  }
+
+  isAtEnd() {
+    return this.pos >= this.source.length
   }
 
   peek(offset = 0) {
-    const idx = this.pos + offset
-    return idx < this.source.length ? this.source[idx] : null
+    const pos = this.pos + offset
+    if (pos >= this.source.length) return '\0'
+    return this.source[pos]
   }
 
   advance() {
@@ -119,7 +122,6 @@ class EtherLexer {
     if (char === '\n') {
       this.line++
       this.column = 1
-      this.atLineStart = true
     } else {
       this.column++
     }
@@ -127,65 +129,21 @@ class EtherLexer {
   }
 
   match(expected) {
-    if (this.peek() === expected) {
-      this.advance()
-      return true
-    }
-    return false
-  }
-
-  matchSequence(seq) {
-    for (let i = 0; i < seq.length; i++) {
-      if (this.peek(i) !== seq[i]) return false
-    }
-    for (let i = 0; i < seq.length; i++) {
-      this.advance()
-    }
+    if (this.isAtEnd()) return false
+    if (this.source[this.pos] !== expected) return false
+    this.advance()
     return true
   }
 
-  isAtEnd() {
-    return this.pos >= this.source.length
-  }
-
-  isDigit(char) {
-    if (!char) return false
-    return char >= '0' && char <= '9'
-  }
-
-  isHexDigit(char) {
-    if (!char) return false
-    return this.isDigit(char) || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')
-  }
-
-  isAlpha(char) {
-    if (!char) return false
-    const code = char.charCodeAt(0)
-    if ((code >= 65 && code <= 90) || (code >= 97 && code <= 122)) return true
-    if (char === '_') return true
-    if (this.options.allowUnicode) {
-      if (code >= 0x00C0 && code <= 0x024F) return true
-      if (code >= 0x0400 && code <= 0x04FF) return true
-      if (code >= 0x4E00 && code <= 0x9FFF) return true
-      if (code >= 0x3040 && code <= 0x30FF) return true
-      if (code >= 0x0600 && code <= 0x06FF) return true
-      if (code >= 0x0370 && code <= 0x03FF) return true
-    }
-    return false
-  }
-
-  isAlphaNumeric(char) {
-    return this.isAlpha(char) || this.isDigit(char)
-  }
-
-  isWhitespace(char) {
-    if (!char) return false
-    return char === ' ' || char === '\t' || char === '\r'
-  }
-
-  addToken(type, value, startColumn = null) {
-    const col = startColumn !== null ? startColumn : this.column
-    this.tokens.push(new Token(type, value, this.line, col, this.currentIndent))
+  skipWhitespace() {
+    while (!this.isAtEnd()) {
+      const char = this.peek()
+      if (char === ' ' || char === '\t' || char === '\r') {
+        this.advance()
+      } else {
+        break
+      }
+    }
   }
 
   tokenize() {
@@ -195,7 +153,7 @@ class EtherLexer {
 
     while (this.indentStack.length > 1) {
       this.indentStack.pop()
-      this.tokens.push(new Token(TokenType.DEDENT, '', this.line, this.column, 0))
+      this.tokens.push(new Token(TokenType.DEDENT, '', this.line, this.column, this.currentIndent))
     }
 
     this.tokens.push(new Token(TokenType.EOF, '', this.line, this.column, 0))
@@ -205,34 +163,22 @@ class EtherLexer {
   scanToken() {
     if (this.atLineStart) {
       this.handleIndentation()
-      if (this.isAtEnd()) return
+      this.atLineStart = false
     }
 
-    const char = this.peek()
+    this.skipWhitespace()
 
-    if (char === '\n') {
-      this.advance()
-      this.addToken(TokenType.NEWLINE, '\n')
-      return
-    }
+    if (this.isAtEnd()) return
 
-    if (this.isWhitespace(char)) {
-      this.advance()
-      return
-    }
+    const char = this.peek()
 
-    if (char === '#') {
-      const next = this.peek(1)
-      if (next && (this.isHexDigit(next) || this.isAlpha(next))) {
-        this.scanHashToken()
-        return
-      }
-      this.scanComment()
+    if (char === '\n') {
+      this.handleNewline()
       return
     }
 
     if (char === '/' && this.peek(1) === '/') {
-      this.scanComment()
+      this.scanLineComment()
       return
     }
 
@@ -241,8 +187,13 @@ class EtherLexer {
       return
     }
 
+    if (char === '#') {
+      this.scanLineComment()
+      return
+    }
+
     if (char === '"' || char === "'") {
-      this.scanString(char)
+      this.scanString()
       return
     }
 
@@ -251,24 +202,43 @@ class EtherLexer {
       return
     }
 
-    if (this.isDigit(char) || (char === '-' && this.isDigit(this.peek(1)))) {
+    if (this.isDigit(char) || (char === '.' && this.isDigit(this.peek(1)))) {
       this.scanNumber()
       return
     }
 
-    if (this.isAlpha(char)) {
+    if (this.isAlpha(char) || char === '_' || char === '$') {
       this.scanIdentifier()
       return
     }
 
-    this.scanOperator()
+    if (this.isOperator(char)) {
+      this.scanOperator()
+      return
+    }
+
+    this.advance()
+    this.tokens.push(new Token(TokenType.ERROR, `Caractère inattendu: ${char}`, this.line, this.column - 1, this.currentIndent))
+  }
+
+  handleNewline() {
+    const startLine = this.line
+    const startCol = this.column
+    this.advance()
+    
+    while (!this.isAtEnd() && this.peek() === '\n') {
+      this.advance()
+    }
+    
+    this.tokens.push(new Token(TokenType.NEWLINE, '\n', startLine, startCol, this.currentIndent))
+    this.atLineStart = true
   }
 
   handleIndentation() {
     let indent = 0
     while (!this.isAtEnd() && (this.peek() === ' ' || this.peek() === '\t')) {
       if (this.peek() === '\t') {
-        indent += this.options.tabSize
+        indent += 4
       } else {
         indent++
       }
@@ -276,368 +246,238 @@ class EtherLexer {
     }
 
     if (this.isAtEnd() || this.peek() === '\n') {
-      this.atLineStart = false
       return
     }
-    
-    if (this.peek() === '#') {
-      const next = this.peek(1)
-      if (!next || (!this.isAlpha(next) && !this.isHexDigit(next))) {
-        this.atLineStart = false
-        return
-      }
-    }
 
     if (this.peek() === '/' && this.peek(1) === '/') {
-      this.atLineStart = false
       return
     }
 
-    this.currentIndent = indent
-    this.atLineStart = false
+    if (this.peek() === '#') {
+      return
+    }
 
     const currentLevel = this.indentStack[this.indentStack.length - 1]
 
     if (indent > currentLevel) {
       this.indentStack.push(indent)
+      this.currentIndent = indent
       this.tokens.push(new Token(TokenType.INDENT, indent, this.line, 1, indent))
     } else if (indent < currentLevel) {
-      while (this.indentStack.length > 1 && this.indentStack[this.indentStack.length - 1] > indent) {
+      while (this.indentStack.length > 1 && indent < this.indentStack[this.indentStack.length - 1]) {
         this.indentStack.pop()
-        this.tokens.push(new Token(TokenType.DEDENT, '', this.line, 1, indent))
-      }
-      if (this.indentStack[this.indentStack.length - 1] !== indent && indent > 0) {
-        this.indentStack.push(indent)
+        this.tokens.push(new Token(TokenType.DEDENT, '', this.line, 1, this.indentStack[this.indentStack.length - 1]))
       }
+      this.currentIndent = this.indentStack[this.indentStack.length - 1]
     }
   }
 
-  scanComment() {
+  scanLineComment() {
     const startLine = this.line
     const startCol = this.column
-    let comment = ''
-
+    
     if (this.peek() === '#') {
       this.advance()
     } else {
       this.advance()
       this.advance()
     }
-
-    while (!this.isAtEnd() && this.peek() !== '\n') {
-      comment += this.advance()
-    }
-
-    if (this.options.trackComments) {
-      this.tokens.push(new Token(TokenType.COMMENT, comment.trim(), startLine, startCol, this.currentIndent))
-    }
-  }
-
-  scanHashToken() {
-    const startLine = this.line
-    const startCol = this.column
-    
-    this.advance()
     
     let value = ''
-    let isHexColor = true
-    let charCount = 0
-    
-    while (!this.isAtEnd()) {
-      const char = this.peek()
-      if (this.isHexDigit(char)) {
-        value += this.advance()
-        charCount++
-      } else if (this.isAlpha(char) && !this.isHexDigit(char)) {
-        isHexColor = false
-        value += this.advance()
-      } else if (char === '-' || char === '_') {
-        isHexColor = false
-        value += this.advance()
-      } else {
-        break
-      }
+    while (!this.isAtEnd() && this.peek() !== '\n') {
+      value += this.advance()
     }
     
-    if (isHexColor && (charCount === 3 || charCount === 4 || charCount === 6 || charCount === 8)) {
-      this.tokens.push(new Token(TokenType.HEX, '#' + value, startLine, startCol, this.currentIndent))
-    } else {
-      this.tokens.push(new Token(TokenType.HASH, '#', startLine, startCol, this.currentIndent))
-      if (value) {
-        this.tokens.push(new Token(TokenType.IDENTIFIER, value, startLine, startCol + 1, this.currentIndent))
-      }
-    }
+    this.tokens.push(new Token(TokenType.COMMENT, value.trim(), startLine, startCol, this.currentIndent))
   }
 
   scanBlockComment() {
     const startLine = this.line
     const startCol = this.column
-    let comment = ''
-
+    
     this.advance()
     this.advance()
-
+    
+    let value = ''
     while (!this.isAtEnd()) {
       if (this.peek() === '*' && this.peek(1) === '/') {
         this.advance()
         this.advance()
         break
       }
-      comment += this.advance()
-    }
-
-    if (this.options.trackComments) {
-      this.tokens.push(new Token(TokenType.BLOCK_COMMENT, comment.trim(), startLine, startCol, this.currentIndent))
+      value += this.advance()
     }
+    
+    this.tokens.push(new Token(TokenType.BLOCK_COMMENT, value.trim(), startLine, startCol, this.currentIndent))
   }
 
-  scanString(quote) {
+  scanString() {
     const startLine = this.line
     const startCol = this.column
-    let value = ''
-    let isBlock = false
-
-    this.advance()
-
+    const quote = this.advance()
+    
     if (this.peek() === quote && this.peek(1) === quote) {
       this.advance()
       this.advance()
-      isBlock = true
+      return this.scanBlockString(quote, startLine, startCol)
     }
-
+    
+    let value = ''
+    let escaped = false
+    
     while (!this.isAtEnd()) {
-      if (isBlock) {
-        if (this.peek() === quote && this.peek(1) === quote && this.peek(2) === quote) {
-          this.advance()
-          this.advance()
-          this.advance()
-          break
-        }
-      } else {
-        if (this.peek() === quote) {
-          this.advance()
-          break
-        }
-        if (this.peek() === '\n') {
-          this.tokens.push(new Token(TokenType.ERROR, 'Chaîne non terminée', startLine, startCol, this.currentIndent))
-          return
-        }
-      }
-
-      if (this.peek() === '\\' && !isBlock) {
-        this.advance()
-        const escaped = this.advance()
-        switch (escaped) {
+      const char = this.peek()
+      
+      if (escaped) {
+        switch (char) {
           case 'n': value += '\n'; break
           case 't': value += '\t'; break
           case 'r': value += '\r'; break
           case '\\': value += '\\'; break
           case "'": value += "'"; break
           case '"': value += '"'; break
-          case '0': value += '\0'; break
-          case 'x':
-            let hex = ''
-            for (let i = 0; i < 2 && this.isHexDigit(this.peek()); i++) {
-              hex += this.advance()
-            }
-            value += String.fromCharCode(parseInt(hex, 16))
-            break
-          case 'u':
-            let unicode = ''
-            if (this.peek() === '{') {
-              this.advance()
-              while (this.isHexDigit(this.peek())) {
-                unicode += this.advance()
-              }
-              if (this.peek() === '}') this.advance()
-            } else {
-              for (let i = 0; i < 4 && this.isHexDigit(this.peek()); i++) {
-                unicode += this.advance()
-              }
-            }
-            value += String.fromCodePoint(parseInt(unicode, 16))
-            break
-          default:
-            value += escaped
+          default: value += char
         }
+        escaped = false
+        this.advance()
+      } else if (char === '\\') {
+        escaped = true
+        this.advance()
+      } else if (char === quote) {
+        this.advance()
+        break
+      } else if (char === '\n') {
+        break
       } else {
         value += this.advance()
       }
     }
+    
+    this.tokens.push(new Token(TokenType.STRING, value, startLine, startCol, this.currentIndent))
+  }
 
-    this.tokens.push(new Token(
-      isBlock ? TokenType.BLOCK_STRING : TokenType.STRING,
-      value,
-      startLine,
-      startCol,
-      this.currentIndent
-    ))
+  scanBlockString(quote, startLine, startCol) {
+    let value = ''
+    
+    while (!this.isAtEnd()) {
+      if (this.peek() === quote && this.peek(1) === quote && this.peek(2) === quote) {
+        this.advance()
+        this.advance()
+        this.advance()
+        break
+      }
+      value += this.advance()
+    }
+    
+    this.tokens.push(new Token(TokenType.BLOCK_STRING, value, startLine, startCol, this.currentIndent))
   }
 
   scanTemplateString() {
     const startLine = this.line
     const startCol = this.column
-    let value = ''
-
     this.advance()
-
+    
+    let value = ''
+    
     while (!this.isAtEnd() && this.peek() !== '`') {
-      if (this.peek() === '\\') {
+      if (this.peek() === '\\' && this.peek(1) === '`') {
         this.advance()
         value += this.advance()
       } else {
         value += this.advance()
       }
     }
-
-    if (this.peek() === '`') {
+    
+    if (!this.isAtEnd()) {
       this.advance()
     }
-
-    this.tokens.push(new Token(TokenType.BLOCK_STRING, value, startLine, startCol, this.currentIndent))
+    
+    this.tokens.push(new Token(TokenType.STRING, value, startLine, startCol, this.currentIndent))
   }
 
   scanNumber() {
     const startLine = this.line
     const startCol = this.column
     let value = ''
-    let type = TokenType.INTEGER
-
-    if (this.peek() === '-') {
-      value += this.advance()
-    }
-
+    let tokenType = TokenType.INTEGER
+    
     if (this.peek() === '0' && (this.peek(1) === 'x' || this.peek(1) === 'X')) {
       value += this.advance()
       value += this.advance()
-      while (this.isHexDigit(this.peek())) {
+      while (!this.isAtEnd() && this.isHexDigit(this.peek())) {
         value += this.advance()
       }
       this.tokens.push(new Token(TokenType.HEX, value, startLine, startCol, this.currentIndent))
       return
     }
-
-    if (this.peek() === '0' && (this.peek(1) === 'b' || this.peek(1) === 'B')) {
-      this.advance()
-      this.advance()
-      let binValue = ''
-      while (this.peek() === '0' || this.peek() === '1') {
-        binValue += this.advance()
-      }
-      this.tokens.push(new Token(TokenType.INTEGER, parseInt(binValue, 2), startLine, startCol, this.currentIndent))
-      return
-    }
-
-    if (this.peek() === '0' && (this.peek(1) === 'o' || this.peek(1) === 'O')) {
-      this.advance()
-      this.advance()
-      let octValue = ''
-      while (this.peek() >= '0' && this.peek() <= '7') {
-        octValue += this.advance()
-      }
-      this.tokens.push(new Token(TokenType.INTEGER, parseInt(octValue, 8), startLine, startCol, this.currentIndent))
-      return
-    }
-
-    while (this.isDigit(this.peek()) || this.peek() === '_') {
-      if (this.peek() !== '_') {
-        value += this.advance()
-      } else {
-        this.advance()
-      }
+    
+    while (!this.isAtEnd() && this.isDigit(this.peek())) {
+      value += this.advance()
     }
-
+    
     if (this.peek() === '.' && this.isDigit(this.peek(1))) {
-      type = TokenType.FLOAT
+      tokenType = TokenType.FLOAT
       value += this.advance()
-      while (this.isDigit(this.peek()) || this.peek() === '_') {
-        if (this.peek() !== '_') {
-          value += this.advance()
-        } else {
-          this.advance()
-        }
-      }
-    }
-
-    if (this.peek() === 'e' || this.peek() === 'E') {
-      const nextChar = this.peek(1)
-      if (this.isDigit(nextChar) || nextChar === '+' || nextChar === '-') {
-        type = TokenType.FLOAT
+      while (!this.isAtEnd() && this.isDigit(this.peek())) {
         value += this.advance()
-        if (this.peek() === '+' || this.peek() === '-') {
-          value += this.advance()
-        }
-        while (this.isDigit(this.peek())) {
-          value += this.advance()
-        }
       }
     }
-
-    if (type === TokenType.FLOAT) {
-      let numValue = String(parseFloat(value))
-      const unit = this.scanCSSUnit()
-      if (unit) numValue += unit
-      this.tokens.push(new Token(type, numValue, startLine, startCol, this.currentIndent))
-    } else {
-      let numValue = String(parseInt(value, 10))
-      const unit = this.scanCSSUnit()
-      if (unit) numValue += unit
-      this.tokens.push(new Token(type, numValue, startLine, startCol, this.currentIndent))
-    }
-  }
-
-  scanCSSUnit() {
-    const units = ['px', 'em', 'rem', 'vh', 'vw', 'vmin', 'vmax', '%', 'fr', 's', 'ms', 'deg', 'rad', 'turn', 'ch', 'ex', 'cm', 'mm', 'in', 'pt', 'pc', 'cqw', 'cqh']
     
-    for (const unit of units) {
-      let matches = true
-      for (let i = 0; i < unit.length; i++) {
-        const char = this.peek(i)
-        if (!char || char.toLowerCase() !== unit[i].toLowerCase()) {
-          matches = false
-          break
-        }
+    if (this.peek() === 'e' || this.peek() === 'E') {
+      tokenType = TokenType.FLOAT
+      value += this.advance()
+      if (this.peek() === '+' || this.peek() === '-') {
+        value += this.advance()
       }
-      if (matches) {
-        const afterUnit = this.peek(unit.length)
-        if (!afterUnit || !this.isAlpha(afterUnit)) {
-          let result = ''
-          for (let i = 0; i < unit.length; i++) {
-            result += this.advance()
-          }
-          return result
-        }
+      while (!this.isAtEnd() && this.isDigit(this.peek())) {
+        value += this.advance()
       }
     }
-    return ''
+    
+    const numValue = tokenType === TokenType.INTEGER ? parseInt(value, 10) : parseFloat(value)
+    this.tokens.push(new Token(tokenType, numValue, startLine, startCol, this.currentIndent))
   }
 
   scanIdentifier() {
     const startLine = this.line
     const startCol = this.column
     let value = ''
-
-    while (this.isAlphaNumeric(this.peek()) || this.peek() === '-' || this.peek() === '_') {
+    
+    if (this.peek() === '$') {
+      value += this.advance()
+    }
+    
+    while (!this.isAtEnd() && (this.isAlphaNumeric(this.peek()) || this.peek() === '_' || this.peek() === '-' && this.isAlpha(this.peek(1)))) {
       value += this.advance()
     }
 
     let keepLooking = true
-    while (keepLooking && this.peek() === ' ') {
+    while (keepLooking && !this.isAtEnd()) {
       const savedPos = this.pos
       const savedLine = this.line
       const savedCol = this.column
       
-      this.advance()
-      
-      let nextWord = ''
-      while (this.isAlphaNumeric(this.peek()) || this.peek() === '-' || this.peek() === '_') {
-        nextWord += this.advance()
-      }
-      
-      if (nextWord) {
-        const combined = value + ' ' + nextWord
-        if (this.isCompoundKeyword(combined) || this.couldBeCompoundStart(combined)) {
-          value = combined
+      if (this.peek() === ' ') {
+        let spaces = ''
+        while (this.peek() === ' ') {
+          spaces += this.advance()
+        }
+        
+        if (this.isAlpha(this.peek()) || this.peek() === "'" || this.peek() === '-') {
+          let nextWord = ''
+          while (!this.isAtEnd() && (this.isAlphaNumeric(this.peek()) || this.peek() === '_' || this.peek() === "'" || this.peek() === '-' && this.isAlpha(this.peek(1)))) {
+            nextWord += this.advance()
+          }
+          
+          const compound = value + ' ' + nextWord
+          if (this.isCompoundKeyword(compound) || this.couldBeCompoundStart(compound)) {
+            value = compound
+          } else {
+            this.pos = savedPos
+            this.line = savedLine
+            this.column = savedCol
+            keepLooking = false
+          }
         } else {
           this.pos = savedPos
           this.line = savedLine
@@ -664,12 +504,7 @@ class EtherLexer {
 
   couldBeCompoundStart(partial) {
     const normalized = this.normalizeAccents(partial)
-    const compounds = [
-      'retour a la ligne', 'a la soumission', 'a la reinitialisation',
-      'inferieur ou egal', 'superieur ou egal',
-      'gauche a droite', 'droite a gauche',
-      'mettre a jour', 'case a cocher'
-    ]
+    const compounds = this.getCompoundKeywords()
     return compounds.some(c => c.startsWith(normalized + ' '))
   }
 
@@ -685,9 +520,12 @@ class EtherLexer {
     return str.toLowerCase().split('').map(c => accentsMap[c] || c).join('')
   }
 
-  isCompoundKeyword(phrase) {
-    const normalized = this.normalizeAccents(phrase)
-    const compounds = [
+  getCompoundKeywords() {
+    if (this.compoundKeywordsCache) {
+      return this.compoundKeywordsCache
+    }
+    
+    this.compoundKeywordsCache = [
       'en ligne', 'hors ligne', 'au survol', 'au clic', 'au focus',
       'au double clic', 'au changement', 'a la soumission', 
       'retour a la ligne', 'lecture seule', 'zone texte',
@@ -727,9 +565,173 @@ class EtherLexer {
       'couleur bordure', 'style bordure', 'largeur bordure',
       'espacement lettres', 'espacement mots',
       'espace blanc', 'debordement texte', 
-      'flou fond', 'filtre fond'
+      'flou fond', 'filtre fond',
+      
+      'obtenir type', 'definir type',
+      'est tableau', 'est booleen', 'est entier', 'est chaine',
+      'est nul', 'est flottant', 'est numerique', 'est objet',
+      'est ressource', 'est scalaire', 'est iterable', 'est comptable',
+      'est appelable', 'est vide',
+      'valeur entier', 'valeur flottant', 'valeur chaine', 'valeur booleen',
+      'afficher formate', 'formater chaine',
+      'afficher variable', 'exporter variable',
+      'imprimer lisible', 'deboguer zval',
+      'somme tableau', 'filtrer tableau', 'mapper tableau',
+      'fusionner tableaux', 'reduire tableau',
+      'trier tableau', 'inverser tableau', 'melanger tableau',
+      'premier tableau', 'dernier tableau',
+      'cles tableau', 'valeurs tableau',
+      'rechercher tableau', 'existe cle', 'existe tableau',
+      'couper espaces', 'couper gauche', 'couper droite',
+      'en majuscules', 'en minuscules',
+      'premiere majuscule', 'majuscules mots',
+      'longueur chaine', 'sous chaine',
+      'position chaine', 'remplacer chaine',
+      'exploser chaine', 'imploser tableau',
+      'lire fichier', 'ecrire fichier',
+      'fichier existe', 'supprimer fichier',
+      'creer dossier', 'lire dossier', 'supprimer dossier',
+      'encoder json', 'decoder json',
+      'encoder base64', 'decoder base64',
+      'encoder url', 'decoder url',
+      'hacher md5', 'hacher sha1', 'hacher sha256',
+      'hacher mot de passe', 'verifier mot de passe',
+      'mot de passe',
+      'demarrer session', 'detruire session',
+      'sinon si',
+      'nouvelle instance', 'nouveau datetime',
+      'classe abstraite', 'classe finale',
+      'methode abstraite', 'methode finale',
+      'propriete privee', 'propriete protegee', 'propriete publique',
+      'lecture seule',
+      
+      'creer base de donnees', 'creer table', 'creer vue',
+      'creer index', 'creer index unique',
+      'creer schema', 'creer sequence', 'creer type', 'creer domaine',
+      'creer fonction', 'creer procedure', 'creer declencheur',
+      'creer ou remplacer', 'creer role',
+      'si n\'existe pas', 'si nexiste pas',
+      'modifier table', 'modifier colonne', 'modifier base de donnees',
+      'modifier index', 'modifier sequence', 'modifier vue',
+      'ajouter colonne', 'supprimer colonne',
+      'renommer en', 'renommer colonne', 'modifier type',
+      'definir defaut', 'supprimer defaut',
+      'supprimer base de donnees', 'supprimer table', 'supprimer vue',
+      'supprimer index', 'supprimer schema', 'supprimer sequence',
+      'supprimer fonction', 'supprimer procedure', 'supprimer declencheur',
+      'supprimer role', 'supprimer contrainte',
+      'cle primaire', 'cle etrangere', 'non nul', 'par defaut',
+      'auto increment', 'unique contrainte',
+      'verifier contrainte', 'references table',
+      'sur suppression', 'sur mise a jour',
+      'cascade suppression', 'cascade mise a jour',
+      'definir null', 'sans action',
+      'inserer dans', 'inserer valeurs',
+      'mettre a jour', 'definir valeur',
+      'supprimer de', 'supprimer ou',
+      'selectionner depuis', 'selectionner tout',
+      'selectionner distinct', 'selectionner comme',
+      'depuis table', 'ou condition', 'et condition',
+      'grouper par', 'ordonner par', 'limiter a',
+      'decaler de', 'avoir condition',
+      'jointure gauche', 'jointure droite',
+      'jointure interne', 'jointure externe',
+      'jointure croisee', 'jointure naturelle',
+      'jointure complete',
+      'union tout', 'sauf tout', 'intersection tout',
+      'compter tout', 'somme de', 'moyenne de',
+      'maximum de', 'minimum de',
+      'comme motif', 'similaire a', 'correspond a regex',
+      'correspond regex insensible',
+      'ne correspond pas regex', 'ne correspond pas regex insensible',
+      'est null', 'n\'est pas null', 'nest pas null',
+      'est distinct de', 'n\'est pas distinct de', 'nest pas distinct de',
+      'entre et', 'pas entre',
+      'tout de', 'quelconque de', 'aucun de',
+      'dans liste', 'pas dans liste',
+      'existe sous requete', 'n\'existe pas sous requete',
+      'debut transaction', 'valider transaction', 'annuler transaction',
+      'point de sauvegarde', 'revenir a', 'liberer sauvegarde',
+      'definir transaction', 'lecture seule', 'lecture ecriture',
+      'niveau isolation', 'lecture non validee',
+      'lecture validee', 'lecture repetable', 'serialisable',
+      'verrouiller table', 'deverrouiller tables',
+      'verrouiller ligne', 'pour mise a jour',
+      'pour partage', 'sauter verrouille', 'sans attendre',
+      'accorder privileges', 'revoquer privileges',
+      'accorder tout', 'revoquer tout',
+      'avec option grant', 'option admin',
+      'creer utilisateur', 'supprimer utilisateur',
+      'modifier utilisateur', 'definir mot de passe',
+      'expliquer analyser', 'expliquer verbose',
+      'commentaire sur', 'commentaire ligne', 'commentaire bloc',
+      'tableau contient', 'contenu dans tableau',
+      'chevauchement tableau', 'longueur tableau',
+      'ajouter tableau', 'prefixer tableau',
+      'concatener tableaux', 'supprimer tableau',
+      'remplacer tableau', 'position tableau', 'positions tableau',
+      'vers chaine tableau', 'depuis chaine tableau',
+      'denombrer tableau',
+      'remplacer regex', 'correspondance regex', 'correspondances regex',
+      'diviser regex', 'tableau regex',
+      'extraire json', 'extraire texte json',
+      'contient json', 'existe json',
+      'type json', 'tableau json', 'objet json',
+      'cle json', 'valeur json',
+      'ensemble json', 'supprimer cle json',
+      'definir valeur json', 'inserer json',
+      'concatener json', 'chaque json',
+      'elements json', 'cles json',
+      'vers json', 'depuis json',
+      'maintenant timestamp', 'date courante', 'heure courante',
+      'timestamp courant', 'extraire date',
+      'partie date', 'tronquer date',
+      'age entre', 'intervalle temps',
+      'debut mois', 'fin mois',
+      'ajouter intervalle', 'soustraire intervalle',
+      'fuseau horaire', 'convertir fuseau',
+      'generer serie', 'generer dates',
+      'valeur suivante', 'valeur courante',
+      'definir valeur sequence', 'redemarrer sequence',
+      'fenetre sur', 'partitionner par',
+      'lignes precedentes', 'lignes suivantes',
+      'premiere valeur', 'derniere valeur',
+      'nieme valeur', 'rang dense', 'numero ligne',
+      'pourcentage rang', 'distribution cumulee',
+      'ntile groupe', 'valeur precedente', 'valeur suivante fenetre',
+      'somme cumulative', 'moyenne mobile',
+      'table temporaire', 'table non journalisee',
+      'sur conflit', 'ne rien faire', 'mettre a jour conflit',
+      'retourner tout', 'retourner insere',
+      'avec requete', 'requete recursive',
+      'table virtuelle', 'table materialisee',
+      'rafraichir vue', 'rafraichir concurrent',
+      'activer declencheur', 'desactiver declencheur',
+      'avant insertion', 'apres insertion',
+      'avant mise a jour', 'apres mise a jour',
+      'avant suppression', 'apres suppression',
+      'pour chaque ligne', 'pour chaque instruction',
+      'quand condition', 'executer procedure',
+      'declarer variable', 'definir variable',
+      'debut bloc', 'fin bloc',
+      'si condition', 'sinon si condition', 'sinon condition',
+      'boucle tant que', 'boucle pour', 'sortir boucle',
+      'continuer boucle', 'retourner valeur',
+      'lever exception', 'capturer exception',
+      'bloc exception', 'quand autres',
+      'afficher message', 'afficher avertissement', 'afficher erreur',
+      'curseur pour', 'ouvrir curseur', 'fermer curseur',
+      'recuperer suivant', 'recuperer precedent',
+      'recuperer premier', 'recuperer dernier',
+      'ligne trouvee', 'ligne non trouvee'
     ]
-    return compounds.includes(normalized)
+    
+    return this.compoundKeywordsCache
+  }
+
+  isCompoundKeyword(phrase) {
+    const normalized = this.normalizeAccents(phrase)
+    return this.getCompoundKeywords().includes(normalized)
   }
 
   scanOperator() {
@@ -801,19 +803,11 @@ class EtherLexer {
 
       case '-':
         if (this.match('-')) {
-          if (this.isAlpha(this.peek()) || this.peek() === '-') {
-            let varName = '--'
-            while (this.isAlphaNumeric(this.peek()) || this.peek() === '-' || this.peek() === '_') {
-              varName += this.advance()
-            }
-            this.tokens.push(new Token(TokenType.IDENTIFIER, varName, startLine, startCol, this.currentIndent))
-          } else {
-            this.tokens.push(new Token(TokenType.MINUS, '--', startLine, startCol, this.currentIndent))
-          }
-        } else if (this.match('>')) {
-          this.tokens.push(new Token(TokenType.ARROW, '->', startLine, startCol, this.currentIndent))
+          this.tokens.push(new Token(TokenType.MINUS, '--', startLine, startCol, this.currentIndent))
         } else if (this.match('=')) {
           this.tokens.push(new Token(TokenType.MINUS, '-=', startLine, startCol, this.currentIndent))
+        } else if (this.match('>')) {
+          this.tokens.push(new Token(TokenType.ARROW, '->', startLine, startCol, this.currentIndent))
         } else {
           this.tokens.push(new Token(TokenType.MINUS, '-', startLine, startCol, this.currentIndent))
         }
@@ -834,10 +828,10 @@ class EtherLexer {
         break
 
       case '/':
-        if (this.match('=')) {
-          this.tokens.push(new Token(TokenType.SLASH, '/=', startLine, startCol, this.currentIndent))
-        } else if (this.match('/')) {
+        if (this.match('/')) {
           this.tokens.push(new Token(TokenType.DOUBLE_SLASH, '//', startLine, startCol, this.currentIndent))
+        } else if (this.match('=')) {
+          this.tokens.push(new Token(TokenType.SLASH, '/=', startLine, startCol, this.currentIndent))
         } else {
           this.tokens.push(new Token(TokenType.SLASH, '/', startLine, startCol, this.currentIndent))
         }
@@ -859,7 +853,13 @@ class EtherLexer {
             this.tokens.push(new Token(TokenType.LTE, '<=', startLine, startCol, this.currentIndent))
           }
         } else if (this.match('<')) {
-          this.tokens.push(new Token(TokenType.LT, '<<', startLine, startCol, this.currentIndent))
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.LT, '<<=', startLine, startCol, this.currentIndent))
+          } else {
+            this.tokens.push(new Token(TokenType.LT, '<<', startLine, startCol, this.currentIndent))
+          }
+        } else if (this.match('>')) {
+          this.tokens.push(new Token(TokenType.NOT_EQUALS, '<>', startLine, startCol, this.currentIndent))
         } else {
           this.tokens.push(new Token(TokenType.LT, '<', startLine, startCol, this.currentIndent))
         }
@@ -869,8 +869,8 @@ class EtherLexer {
         if (this.match('=')) {
           this.tokens.push(new Token(TokenType.GTE, '>=', startLine, startCol, this.currentIndent))
         } else if (this.match('>')) {
-          if (this.match('>')) {
-            this.tokens.push(new Token(TokenType.GT, '>>>', startLine, startCol, this.currentIndent))
+          if (this.match('=')) {
+            this.tokens.push(new Token(TokenType.GT, '>>=', startLine, startCol, this.currentIndent))
           } else {
             this.tokens.push(new Token(TokenType.GT, '>>', startLine, startCol, this.currentIndent))
           }
@@ -894,8 +894,6 @@ class EtherLexer {
           this.tokens.push(new Token(TokenType.DOUBLE_PIPE, '||', startLine, startCol, this.currentIndent))
         } else if (this.match('=')) {
           this.tokens.push(new Token(TokenType.PIPE, '|=', startLine, startCol, this.currentIndent))
-        } else if (this.match('>')) {
-          this.tokens.push(new Token(TokenType.PIPE, '|>', startLine, startCol, this.currentIndent))
         } else {
           this.tokens.push(new Token(TokenType.PIPE, '|', startLine, startCol, this.currentIndent))
         }
@@ -922,6 +920,8 @@ class EtherLexer {
           }
         } else if (this.match('.')) {
           this.tokens.push(new Token(TokenType.QUESTION, '?.', startLine, startCol, this.currentIndent))
+        } else if (this.match('-') && this.match('>')) {
+          this.tokens.push(new Token(TokenType.ARROW, '?->', startLine, startCol, this.currentIndent))
         } else {
           this.tokens.push(new Token(TokenType.QUESTION, '?', startLine, startCol, this.currentIndent))
         }
@@ -972,6 +972,30 @@ class EtherLexer {
     }
   }
 
+  isDigit(char) {
+    return char >= '0' && char <= '9'
+  }
+
+  isHexDigit(char) {
+    return this.isDigit(char) || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')
+  }
+
+  isAlpha(char) {
+    return (char >= 'a' && char <= 'z') || 
+           (char >= 'A' && char <= 'Z') ||
+           char === '_' ||
+           char === '$' ||
+           char.charCodeAt(0) > 127
+  }
+
+  isAlphaNumeric(char) {
+    return this.isAlpha(char) || this.isDigit(char)
+  }
+
+  isOperator(char) {
+    return ':=!,.<>+-*/%&|^~?@#\\()[]{};\n\r\t '.indexOf(char) !== -1
+  }
+
   static tokenize(source, options = {}) {
     const lexer = new EtherLexer(source, options)
     return lexer.tokenize()