freelang-v4 4.3.0

package/src/self-host/lexer.fl

@@ -0,0 +1,305 @@
+// ============================================================
+// FreeLang v4 Self-Hosting Lexer (lexer.fl)
+// Tree-walking interpreter - Tokenization phase
+// ============================================================
+
+// ============================================================
+// Token Types (Enum)
+// ============================================================
+
+struct Token {
+  kind: str,
+  value: str,
+  line: i32,
+}
+
+// ============================================================
+// Lexer (Struct)
+// ============================================================
+
+struct Lexer {
+  source: str,
+  pos: i32,
+  line: i32,
+}
+
+// ============================================================
+// Helper Functions
+// ============================================================
+
+fn is_digit(c: str) -> bool {
+  var code = char_code(c)
+  return code >= 48 && code <= 57  // '0' = 48, '9' = 57
+}
+
+fn is_alpha(c: str) -> bool {
+  var code = char_code(c)
+  return (code >= 65 && code <= 90) || (code >= 97 && code <= 122) || code == 95  // A-Z, a-z, _
+}
+
+fn is_alphanumeric(c: str) -> bool {
+  return is_alpha(c) || is_digit(c)
+}
+
+fn is_whitespace(c: str) -> bool {
+  return c == " " || c == "\t" || c == "\n" || c == "\r"
+}
+
+fn char_code(c: str) -> i32 {
+  // ASCII code extraction - simplified for common chars
+  if c == " " { return 32 }
+  if c == "\t" { return 9 }
+  if c == "\n" { return 10 }
+  if c == "0" { return 48 }
+  if c == "1" { return 49 }
+  if c == "2" { return 50 }
+  if c == "3" { return 51 }
+  if c == "4" { return 52 }
+  if c == "5" { return 53 }
+  if c == "6" { return 54 }
+  if c == "7" { return 55 }
+  if c == "8" { return 56 }
+  if c == "9" { return 57 }
+  if c == "A" { return 65 }
+  if c == "Z" { return 90 }
+  if c == "a" { return 97 }
+  if c == "z" { return 122 }
+  if c == "_" { return 95 }
+  return 0
+}
+
+// ============================================================
+// Lexer Methods
+// ============================================================
+
+fn lexer_new(source: str) -> Lexer {
+  return Lexer { source: source, pos: 0, line: 1 }
+}
+
+fn lexer_current_char(lexer: Lexer) -> str {
+  if lexer.pos >= length(lexer.source) {
+    return ""
+  }
+  return char_at(lexer.source, lexer.pos)
+}
+
+fn lexer_peek_char(lexer: Lexer, offset: i32) -> str {
+  var p = lexer.pos + offset
+  if p >= length(lexer.source) {
+    return ""
+  }
+  return char_at(lexer.source, p)
+}
+
+fn lexer_advance(lexer: Lexer) -> Lexer {
+  var c = lexer_current_char(lexer)
+  if c == "\n" {
+    return Lexer { source: lexer.source, pos: lexer.pos + 1, line: lexer.line + 1 }
+  }
+  return Lexer { source: lexer.source, pos: lexer.pos + 1, line: lexer.line }
+}
+
+fn lexer_skip_whitespace(lexer: Lexer) -> Lexer {
+  var c = lexer_current_char(lexer)
+  if is_whitespace(c) && c != "" {
+    return lexer_skip_whitespace(lexer_advance(lexer))
+  }
+  return lexer
+}
+
+fn lexer_skip_comment(lexer: Lexer) -> Lexer {
+  // Skip // comment to end of line
+  var c = lexer_current_char(lexer)
+  if c == "/" && lexer_peek_char(lexer, 1) == "/" {
+    var l2 = lexer_advance(lexer)
+    var l3 = lexer_advance(l2)
+    var l4 = l3
+    while lexer_current_char(l4) != "\n" && lexer_current_char(l4) != "" {
+      l4 = lexer_advance(l4)
+    }
+    return l4
+  }
+  return lexer
+}
+
+// ============================================================
+// Number Lexing
+// ============================================================
+
+fn lex_number(lexer: Lexer) -> Token {
+  var start = lexer.pos
+  var l = lexer
+  var has_dot = false
+
+  while is_digit(lexer_current_char(l)) || (lexer_current_char(l) == "." && !has_dot) {
+    if lexer_current_char(l) == "." {
+      has_dot = true
+    }
+    l = lexer_advance(l)
+  }
+
+  var value = slice(lexer.source, start, l.pos)
+  var kind = if has_dot { "FLOAT" } else { "INT" }
+
+  return Token { kind: kind, value: value, line: lexer.line }
+}
+
+// ============================================================
+// String Lexing
+// ============================================================
+
+fn lex_string(lexer: Lexer) -> Token {
+  var l = lexer_advance(lexer)  // Skip opening "
+  var start = l.pos
+
+  while lexer_current_char(l) != "\"" && lexer_current_char(l) != "" {
+    if lexer_current_char(l) == "\\" {
+      l = lexer_advance(l)
+    }
+    l = lexer_advance(l)
+  }
+
+  var value = slice(lexer.source, start, l.pos)
+  var l2 = lexer_advance(l)  // Skip closing "
+
+  return Token { kind: "STRING", value: value, line: lexer.line }
+}
+
+// ============================================================
+// Identifier/Keyword Lexing
+// ============================================================
+
+fn lex_ident(lexer: Lexer) -> Token {
+  var start = lexer.pos
+  var l = lexer
+
+  while is_alphanumeric(lexer_current_char(l)) {
+    l = lexer_advance(l)
+  }
+
+  var value = slice(lexer.source, start, l.pos)
+  var kind = if is_keyword(value) { "KEYWORD" } else { "IDENT" }
+
+  return Token { kind: kind, value: value, line: lexer.line }
+}
+
+fn is_keyword(word: str) -> bool {
+  var keywords = ["var", "let", "const", "fn", "return", "if", "else", "while", "for", "match", "struct", "enum", "impl", "trait", "import", "export", "print", "println", "true", "false", "null"]
+  var i = 0
+  while i < length(keywords) {
+    if keywords[i] == word {
+      return true
+    }
+    i = i + 1
+  }
+  return false
+}
+
+// ============================================================
+// Main Lexer
+// ============================================================
+
+fn lex(source: str) -> [Token] {
+  var tokens: [Token] = []
+  var lexer = lexer_new(source)
+
+  while lexer.pos < length(source) {
+    lexer = lexer_skip_whitespace(lexer)
+    lexer = lexer_skip_comment(lexer)
+
+    if lexer.pos >= length(source) {
+      break
+    }
+
+    var c = lexer_current_char(lexer)
+
+    if is_digit(c) {
+      var token = lex_number(lexer)
+      tokens = push(tokens, token)
+      // Update lexer position - need to advance to token end
+      lexer.pos = lexer.pos + length(token.value)
+      lexer = Lexer { source: lexer.source, pos: lexer.pos, line: lexer.line }
+    } else if c == "\"" {
+      var token = lex_string(lexer)
+      tokens = push(tokens, token)
+      lexer.pos = lexer.pos + length(token.value) + 2  // +2 for quotes
+      lexer = Lexer { source: lexer.source, pos: lexer.pos, line: lexer.line }
+    } else if is_alpha(c) {
+      var token = lex_ident(lexer)
+      tokens = push(tokens, token)
+      lexer.pos = lexer.pos + length(token.value)
+      lexer = Lexer { source: lexer.source, pos: lexer.pos, line: lexer.line }
+    } else if c == "+" {
+      var token = Token { kind: "PLUS", value: "+", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "-" {
+      var token = Token { kind: "MINUS", value: "-", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "*" {
+      var token = Token { kind: "STAR", value: "*", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "/" {
+      var token = Token { kind: "SLASH", value: "/", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "=" {
+      if lexer_peek_char(lexer, 1) == "=" {
+        var token = Token { kind: "EQ", value: "==", line: lexer.line }
+        tokens = push(tokens, token)
+        lexer = lexer_advance(lexer)
+        lexer = lexer_advance(lexer)
+      } else {
+        var token = Token { kind: "ASSIGN", value: "=", line: lexer.line }
+        tokens = push(tokens, token)
+        lexer = lexer_advance(lexer)
+      }
+    } else if c == "(" {
+      var token = Token { kind: "LPAREN", value: "(", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == ")" {
+      var token = Token { kind: "RPAREN", value: ")", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "{" {
+      var token = Token { kind: "LBRACE", value: "{", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else if c == "}" {
+      var token = Token { kind: "RBRACE", value: "}", line: lexer.line }
+      tokens = push(tokens, token)
+      lexer = lexer_advance(lexer)
+    } else {
+      // Unknown character - skip
+      lexer = lexer_advance(lexer)
+    }
+  }
+
+  // Add EOF token
+  var eof = Token { kind: "EOF", value: "", line: lexer.line }
+  tokens = push(tokens, eof)
+
+  return tokens
+}
+
+// ============================================================
+// Test Entry Point
+// ============================================================
+
+fn test_lexer() -> void {
+  var source = "var x = 42"
+  var tokens = lex(source)
+
+  println("Tokens:")
+  var i = 0
+  while i < length(tokens) {
+    var t = tokens[i]
+    println(t.kind + ": " + t.value)
+    i = i + 1
+  }
+}
+
+test_lexer()