@devaloop/devalang 0.0.1-alpha.8 → 0.0.1-beta.1

package/rust/core/lexer/handler/identifier.rs

@@ -1,40 +1,47 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_identifier_lexer(
-    char: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize
-) {
-    let mut ident = char.to_string();
-
-    while let Some(&c) = chars.peek() {
-        if c.is_ascii_alphanumeric() || c == '_' {
-            ident.push(c);
-            chars.next();
-            *column += 1;
-        } else {
-            break;
-        }
-    }
-
-    let kind = match ident.as_str() {
-        "if" => TokenKind::If,
-        "else" => TokenKind::Else,
-        "bank" => TokenKind::Bank,
-        "bpm" => TokenKind::Tempo,
-        "loop" => TokenKind::Loop,
-        _ => TokenKind::Identifier,
-    };
-
-    tokens.push(Token {
-        kind: kind.clone(),
-        lexeme: ident,
-        line: *line,
-        column: *column,
-        indent: *current_indent,
-    });
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_identifier_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let mut ident = ch.to_string();
+
+    while let Some(&c) = chars.peek() {
+        if c.is_ascii_alphanumeric() || c == '_' {
+            ident.push(c);
+            chars.next();
+            *column += 1;
+        } else {
+            break;
+        }
+    }
+
+    let kind = match ident.as_str() {
+        "if" => TokenKind::If,
+        "else" => TokenKind::Else,
+        "bank" => TokenKind::Bank,
+        "pattern" => TokenKind::Pattern,
+        "bpm" => TokenKind::Tempo,
+        "loop" => TokenKind::Loop,
+        "for" => TokenKind::Loop,
+        "synth" => TokenKind::Synth,
+        "fn" => TokenKind::Function,
+        "as" => TokenKind::As,
+        "on" => TokenKind::On,
+        "emit" => TokenKind::Emit,
+        _ => TokenKind::Identifier,
+    };
+
+    tokens.push(Token {
+        kind: kind.clone(),
+        lexeme: ident,
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+}

package/rust/core/lexer/handler/indent.rs

@@ -1,52 +1,66 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_indent_lexer(
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize
-) {
-    *current_indent = 0;
-    let mut col = *column;
-
-    while let Some(&c) = chars.peek() {
-        if c == ' ' {
-            *current_indent += 1;
-            chars.next();
-            col += 1;
-        } else if c == '\t' {
-            *current_indent += 4;
-            chars.next();
-            col += 4;
-        } else {
-            break;
-        }
-    }
-
-    *column = col;
-
-    let last_indent = *indent_stack.last().unwrap();
-    if *current_indent > last_indent {
-        indent_stack.push(*current_indent);
-        tokens.push(Token {
-            kind: TokenKind::Indent,
-            lexeme: String::new(),
-            line: *line,
-            column: *column,
-            indent: *current_indent,
-        });
-    } else {
-        while *current_indent < *indent_stack.last().unwrap() {
-            indent_stack.pop();
-            tokens.push(Token {
-                kind: TokenKind::Dedent,
-                lexeme: String::new(),
-                line: *line,
-                column: *column,
-                indent: *current_indent,
-            });
-        }
-    }
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_indent_lexer(
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    indent_stack: &mut Vec<usize>,
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    *current_indent = 0;
+    let mut col = *column;
+
+    while let Some(&c) = chars.peek() {
+        if c == ' ' {
+            *current_indent += 1;
+            chars.next();
+            col += 1;
+            tokens.push(Token {
+                kind: TokenKind::Whitespace,
+                lexeme: " ".to_string(),
+                line: *line,
+                column: col,
+                indent: *current_indent,
+            });
+        } else if c == '\t' {
+            *current_indent += 4;
+            chars.next();
+            col += 4;
+            tokens.push(Token {
+                kind: TokenKind::Whitespace,
+                lexeme: "\t".to_string(),
+                line: *line,
+                column: col,
+                indent: *current_indent,
+            });
+        } else {
+            break;
+        }
+    }
+
+    *column = col;
+
+    let last_indent = *indent_stack.last().unwrap();
+    if *current_indent > last_indent {
+        indent_stack.push(*current_indent);
+        tokens.push(Token {
+            kind: TokenKind::Indent,
+            lexeme: String::from("<INDENT>"),
+            line: *line,
+            column: *column,
+            indent: *current_indent,
+        });
+    } else {
+        while *current_indent < *indent_stack.last().unwrap() {
+            indent_stack.pop();
+            tokens.push(Token {
+                kind: TokenKind::Dedent,
+                lexeme: String::from("<DEDENT>"),
+                line: *line,
+                column: *column,
+                indent: *current_indent,
+            });
+        }
+    }
+}

package/rust/core/lexer/handler/mod.rs

@@ -1,13 +1,15 @@
-pub mod driver;
-
-pub mod at;
-pub mod brace;
-pub mod colon;
-pub mod comment;
-pub mod dot;
-pub mod operator;
-pub mod identifier;
-pub mod newline;
-pub mod number;
-pub mod indent;
-pub mod string;
+pub mod arrow;
+pub mod at;
+pub mod brace;
+pub mod colon;
+pub mod comment;
+pub mod dot;
+pub mod driver;
+pub mod identifier;
+pub mod indent;
+pub mod newline;
+pub mod number;
+pub mod operator;
+pub mod parenthesis;
+pub mod slash;
+pub mod string;

package/rust/core/lexer/handler/newline.rs

@@ -1,23 +1,23 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_newline_lexer(
-    ch: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize,
-    at_line_start: &mut bool,
-    current_indent: &mut usize
-) {
-    tokens.push(Token {
-        kind: TokenKind::Newline,
-        lexeme: ch.to_string(),
-        line: *line,
-        column: 0,
-        indent: *current_indent,
-    });
-
-    *line += 1;
-    *column = 1;
-    *at_line_start = true;
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_newline_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+    at_line_start: &mut bool,
+    current_indent: &mut usize,
+) {
+    tokens.push(Token {
+        kind: TokenKind::Newline,
+        lexeme: ch.to_string(),
+        line: *line,
+        column: 0,
+        indent: *current_indent,
+    });
+
+    *line += 1;
+    *column = 1;
+    *at_line_start = true;
+}

package/rust/core/lexer/handler/number.rs

@@ -1,31 +1,31 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_number_lexer(
-    char: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize
-) {
-    let mut number = char.to_string();
-
-    while let Some(&c) = chars.peek() {
-        if c.is_ascii_digit() {
-            number.push(c);
-            chars.next();
-            *column += 1;
-        } else {
-            break;
-        }
-    }
-
-    tokens.push(Token {
-        kind: TokenKind::Number,
-        lexeme: number,
-        line: *line,
-        column: *column,
-        indent: *current_indent,
-    });
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_number_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let mut number = ch.to_string();
+
+    while let Some(&c) = chars.peek() {
+        if c.is_ascii_digit() {
+            number.push(c);
+            chars.next();
+            *column += 1;
+        } else {
+            break;
+        }
+    }
+
+    tokens.push(Token {
+        kind: TokenKind::Number,
+        lexeme: number,
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+}

package/rust/core/lexer/handler/operator.rs

@@ -1,44 +1,46 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_operator_lexer(
-    ch: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize
-) {
-    let next = chars.peek().copied();
-
-    let (kind, len) = match (ch, next) {
-        ('=', Some('=')) => (TokenKind::DoubleEquals, 2),
-        ('!', Some('=')) => (TokenKind::NotEquals, 2),
-        ('>', Some('=')) => (TokenKind::GreaterEqual, 2),
-        ('<', Some('=')) => (TokenKind::LessEqual, 2),
-        ('=', _) => (TokenKind::Equals, 1),
-        ('>', _) => (TokenKind::Greater, 1),
-        ('<', _) => (TokenKind::Less, 1),
-        _ => {
-            return;
-        }
-    };
-
-    if len == 2 {
-        chars.next(); // consume second char
-    }
-
-    tokens.push(Token {
-        kind,
-        lexeme: if len == 2 {
-            format!("{}{}", ch, next.unwrap())
-        } else {
-            ch.to_string()
-        },
-        line: *line,
-        column: *column,
-        indent: *current_indent,
-    });
-
-    *column += len;
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_operator_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let next = chars.peek().copied();
+
+    let (kind, len) = match (ch, next) {
+        ('=', Some('=')) => (TokenKind::DoubleEquals, 2),
+        ('!', Some('=')) => (TokenKind::NotEquals, 2),
+        ('>', Some('=')) => (TokenKind::GreaterEqual, 2),
+        ('<', Some('=')) => (TokenKind::LessEqual, 2),
+        ('+', _) => (TokenKind::Plus, 1),
+        ('*', _) => (TokenKind::Asterisk, 1),
+        ('=', _) => (TokenKind::Equals, 1),
+        ('>', _) => (TokenKind::Greater, 1),
+        ('<', _) => (TokenKind::Less, 1),
+        _ => {
+            return;
+        }
+    };
+
+    if len == 2 {
+        chars.next(); // consume second char
+    }
+
+    tokens.push(Token {
+        kind,
+        lexeme: if len == 2 {
+            format!("{}{}", ch, next.unwrap())
+        } else {
+            ch.to_string()
+        },
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+
+    *column += len;
+}

package/rust/core/lexer/handler/parenthesis.rs

@@ -0,0 +1,41 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_rparen_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    tokens.push(Token {
+        kind: TokenKind::RParen,
+        lexeme: ch.to_string(),
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+
+    *column += 1;
+}
+
+pub fn handle_lparen_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    tokens.push(Token {
+        kind: TokenKind::LParen,
+        lexeme: ch.to_string(),
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+
+    *column += 1;
+}

package/rust/core/lexer/handler/slash.rs

@@ -0,0 +1,21 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_slash_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let slash = ch.to_string();
+
+    tokens.push(Token {
+        kind: TokenKind::Slash,
+        lexeme: slash,
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+}

package/rust/core/lexer/handler/string.rs

@@ -1,63 +1,63 @@
-use crate::core::lexer::token::{ Token, TokenKind };
-
-pub fn handle_string_lexer(
-    ch: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize
-) {
-    let quote_char = ch;
-
-    let start_column = *column;
-    let start_line = *line;
-
-    *column += 1;
-
-    let mut string_content = String::new();
-
-    while let Some(&next_ch) = chars.peek() {
-        if next_ch == quote_char {
-            chars.next();
-            *column += 1;
-            break;
-        } else if next_ch == '\\' {
-            chars.next();
-            *column += 1;
-
-            if let Some(escaped) = chars.next() {
-                match escaped {
-                    'n' => string_content.push('\n'),
-                    't' => string_content.push('\t'),
-                    '\\' => string_content.push('\\'),
-                    '"' => string_content.push('"'),
-                    '\'' => string_content.push('\''),
-                    other => {
-                        string_content.push('\\');
-                        string_content.push(other);
-                    }
-                }
-                *column += 1;
-            }
-        } else {
-            chars.next();
-            if next_ch == '\n' {
-                *line += 1;
-                *column = 1;
-            } else {
-                *column += 1;
-            }
-            string_content.push(next_ch);
-        }
-    }
-
-    tokens.push(Token {
-        kind: TokenKind::String,
-        lexeme: string_content,
-        indent: *current_indent,
-        line: start_line,
-        column: start_column,
-    });
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_string_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let quote_char = ch;
+
+    let start_column = *column;
+    let start_line = *line;
+
+    *column += 1;
+
+    let mut string_content = String::new();
+
+    while let Some(&next_ch) = chars.peek() {
+        if next_ch == quote_char {
+            chars.next();
+            *column += 1;
+            break;
+        } else if next_ch == '\\' {
+            chars.next();
+            *column += 1;
+
+            if let Some(escaped) = chars.next() {
+                match escaped {
+                    'n' => string_content.push('\n'),
+                    't' => string_content.push('\t'),
+                    '\\' => string_content.push('\\'),
+                    '"' => string_content.push('"'),
+                    '\'' => string_content.push('\''),
+                    other => {
+                        string_content.push('\\');
+                        string_content.push(other);
+                    }
+                }
+                *column += 1;
+            }
+        } else {
+            chars.next();
+            if next_ch == '\n' {
+                *line += 1;
+                *column = 1;
+            } else {
+                *column += 1;
+            }
+            string_content.push(next_ch);
+        }
+    }
+
+    tokens.push(Token {
+        kind: TokenKind::String,
+        lexeme: string_content,
+        indent: *current_indent,
+        line: start_line,
+        column: start_column,
+    });
+}

package/rust/core/lexer/mod.rs

@@ -1,30 +1,3 @@
-pub mod handler;
-pub mod token;
-
-use std::fs;
-use crate::core::{
-    lexer::{ handler::driver::handle_content_lexing, token::Token },
-    utils::path::normalize_path,
-};
-
-pub struct Lexer {}
-
-impl Lexer {
-    pub fn new() -> Self {
-        Lexer {}
-    }
-
-    pub fn lex_from_source(&self, source: &str) -> Result<Vec<Token>, String> {
-        handle_content_lexing(source.to_string())
-    }
-
-    pub fn lex_tokens(&self, entrypoint: &str) -> Vec<Token> {
-        let path = normalize_path(entrypoint);
-
-        let file_content = fs::read_to_string(&path).expect("Failed to read the entrypoint file");
-
-        let tokens = handle_content_lexing(file_content).expect("Failed to lex the content");
-
-        tokens
-    }
-}
+pub mod driver;
+pub mod handler;
+pub mod token;