@devaloop/devalang 0.0.1-alpha.1 → 0.0.1-alpha.3

package/rust/core/lexer/handler/number.rs

@@ -0,0 +1,31 @@
+use crate::core::lexer::token::{ Token, TokenKind };
+
+pub fn handle_number_lexer(
+    char: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    indent_stack: &mut Vec<usize>,
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize
+) {
+    let mut number = char.to_string();
+
+    while let Some(&c) = chars.peek() {
+        if c.is_ascii_digit() {
+            number.push(c);
+            chars.next();
+            *column += 1;
+        } else {
+            break;
+        }
+    }
+
+    tokens.push(Token {
+        kind: TokenKind::Number,
+        lexeme: number,
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+}

package/rust/core/lexer/handler/string.rs

@@ -0,0 +1,66 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_string_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    indent_stack: &mut Vec<usize>,
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize
+) {
+    let quote_char = ch;
+
+    // Position de départ du token
+    let start_column = *column;
+    let start_line = *line;
+
+    // Consommer le guillemet ouvrant
+    chars.next();
+    *column += 1;
+
+    let mut string_content = String::new();
+
+    while let Some(&next_ch) = chars.peek() {
+        if next_ch == quote_char {
+            chars.next(); // Consomme le guillemet fermant
+            *column += 1;
+            break;
+        } else if next_ch == '\\' {
+            chars.next(); // Consomme '\'
+            *column += 1;
+
+            if let Some(escaped) = chars.next() {
+                match escaped {
+                    'n' => string_content.push('\n'),
+                    't' => string_content.push('\t'),
+                    '\\' => string_content.push('\\'),
+                    '"' => string_content.push('"'),
+                    '\'' => string_content.push('\''),
+                    other => {
+                        string_content.push('\\');
+                        string_content.push(other);
+                    }
+                }
+                *column += 1;
+            }
+        } else {
+            chars.next();
+            if next_ch == '\n' {
+                *line += 1;
+                *column = 1;
+            } else {
+                *column += 1;
+            }
+            string_content.push(next_ch);
+        }
+    }
+
+    tokens.push(Token {
+        kind: TokenKind::String,
+        lexeme: string_content,
+        indent: *current_indent,
+        line: start_line,
+        column: start_column,
+    });
+}

package/rust/core/lexer/mod.rs

@@ -1,333 +1,25 @@
-use crate::core::types::token::{ Token, TokenKind };
+pub mod handler;
+pub mod token;
 
-pub fn lex(input: String) -> Vec<Token> {
-    let mut tokens = Vec::new();
+use std::fs;
+use crate::core::{ lexer::{ handler::handle_content_lexing, token::Token }, utils::path::normalize_path };
 
-    let mut line = 1;
-    let mut column = 1;
+pub struct Lexer {}
 
-    let mut indent_stack: Vec<usize> = vec![0];
-    let mut current_indent = 0;
-    let mut at_line_start = true;
-
-    let mut chars = input.chars().peekable();
-
-    while let Some(_) = chars.peek() {
-        if at_line_start {
-            current_indent = 0;
-
-            while let Some(&c) = chars.peek() {
-                if c == ' ' {
-                    current_indent += 1;
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
-
-            let last_indent = *indent_stack.last().unwrap();
-            if current_indent > last_indent {
-                indent_stack.push(current_indent);
-                tokens.push(Token {
-                    kind: TokenKind::Indent,
-                    lexeme: String::new(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-            } else {
-                while current_indent < *indent_stack.last().unwrap() {
-                    indent_stack.pop();
-                    tokens.push(Token {
-                        kind: TokenKind::Dedent,
-                        lexeme: String::new(),
-                        line,
-                        column,
-                        indent: current_indent,
-                    });
-                }
-            }
-
-            at_line_start = false;
-        }
-
-        let Some(ch) = chars.next() else {
-            break;
-        };
-
-        if ch == '\n' {
-            tokens.push(Token {
-                kind: TokenKind::Newline,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-
-            line += 1;
-            column = 1;
-            at_line_start = true;
-
-            continue;
-        }
-
-        if ch == ' ' || ch == '\t' {
-            column += if ch == '\t' { 4 } else { 1 };
-            continue;
-        }
-
-        if ch == '#' {
-            let mut comment = String::new();
-            while let Some(&c) = chars.peek() {
-                if c == '\n' {
-                    break;
-                }
-                comment.push(c);
-                chars.next();
-                column += 1;
-            }
-            tokens.push(Token {
-                kind: TokenKind::Comment(comment.trim().to_string()),
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
-
-        if ch == ':' {
-            tokens.push(Token {
-                kind: TokenKind::Colon,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '=' {
-            if let Some('=') = chars.peek() {
-                chars.next();
-                tokens.push(Token {
-                    kind: TokenKind::DoubleEquals,
-                    lexeme: ch.to_string(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-                column += 2;
-            } else {
-                tokens.push(Token {
-                    kind: TokenKind::Equals,
-                    lexeme: ch.to_string(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-                column += 1;
-            }
-            continue;
-        }
-
-        if ch == '[' {
-            tokens.push(Token {
-                kind: TokenKind::LBracket,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == ']' {
-            tokens.push(Token {
-                kind: TokenKind::RBracket,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '{' {
-            tokens.push(Token {
-                kind: TokenKind::LBrace,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '}' {
-            tokens.push(Token {
-                kind: TokenKind::RBrace,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '"' {
-            let mut string_content = String::new();
-            column += 1; // skip the opening quote
-
-            while let Some(next_ch) = chars.next() {
-                column += 1;
-                if next_ch == '"' {
-                    break; // closing quote reached
-                } else {
-                    string_content.push(next_ch);
-                }
-            }
-
-            tokens.push(Token {
-                kind: TokenKind::String,
-                lexeme: string_content,
-                line,
-                column,
-                indent: current_indent,
-            });
-
-            continue;
-        }
-
-        if ch == '\'' {
-            let mut string_content = String::new();
-            column += 1;
-
-            while let Some(next_ch) = chars.next() {
-                column += 1;
-                if next_ch == '\'' {
-                    break;
-                } else {
-                    string_content.push(next_ch);
-                }
-            }
-
-            tokens.push(Token {
-                kind: TokenKind::String,
-                lexeme: string_content,
-                line,
-                column,
-                indent: current_indent,
-            });
-
-            continue;
-        }
-
-        if ch == '.' {
-            tokens.push(Token {
-                kind: TokenKind::Dot,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '@' {
-            tokens.push(Token {
-                kind: TokenKind::At,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch.is_ascii_digit() {
-            let mut number = ch.to_string();
-            while let Some(&c) = chars.peek() {
-                if c.is_ascii_digit() {
-                    number.push(c);
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
-
-            // let value = number.parse::<f32>().unwrap();
-            tokens.push(Token {
-                kind: TokenKind::Number,
-                lexeme: number,
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
-
-        if ch.is_ascii_alphabetic() {
-            let mut ident = ch.to_string();
-            while let Some(&c) = chars.peek() {
-                if c.is_ascii_alphanumeric() || c == '_' {
-                    ident.push(c);
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
+impl Lexer {
+    pub fn new() -> Self {
+        Lexer {}
+    }
 
-            let kind = match ident.as_str() {
-                "bank" => TokenKind::Bank,
-                "bpm" => TokenKind::Tempo,
-                "loop" => TokenKind::Loop,
-                _ => TokenKind::Identifier,
-            };
+    pub fn lex_tokens(&self, entrypoint: &str) -> Vec<Token> {
+        let path = normalize_path(entrypoint);
 
-            tokens.push(Token {
-                kind,
-                lexeme: ident,
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
+        let file_content = fs
+            ::read_to_string(&path)
+            .expect("Failed to read the entrypoint file");
 
-        // Skip unknown char
-        column += 1;
-    }
+        let tokens = handle_content_lexing(file_content);
 
-    while indent_stack.len() > 1 {
-        indent_stack.pop();
-        current_indent = *indent_stack.last().unwrap();
-        tokens.push(Token {
-            kind: TokenKind::Dedent,
-            lexeme: String::new(),
-            line,
-            column,
-            indent: current_indent,
-        });
+        tokens
     }
-
-    tokens.push(Token {
-        kind: TokenKind::EOF,
-        lexeme: String::new(),
-        line: line + 1, // EOF is considered to be on the next line
-        column: 0, // EOF has no column
-        indent: 0, // EOF has no indent
-    });
-
-    tokens
 }

package/rust/core/lexer/token.rs

@@ -0,0 +1,55 @@
+use serde::{ Deserialize, Serialize };
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Token {
+    pub kind: TokenKind,
+    pub lexeme: String,
+    pub indent: usize,
+    pub line: usize,
+    pub column: usize,
+}
+
+impl Token {
+    pub fn is_error(&self) -> bool {
+        match &self.kind {
+            TokenKind::Error(_) => {
+                return true;
+            }
+            _ => {
+                return false;
+            }
+        };
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum TokenKind {
+    At,
+    Tempo,
+    Bank,
+    Loop,
+    Identifier,
+    Map,
+    Array,
+    Number,
+    String,
+    Boolean,
+    Colon,
+    Comma,
+    Equals,
+    DoubleEquals,
+    Dot,
+    LBrace,
+    RBrace,
+    DbQuote,
+    Quote,
+    LBracket,
+    RBracket,
+    Newline,
+    Indent,
+    Dedent,
+    Comment,
+    Unknown,
+    Error(String),
+    EOF,
+}

package/rust/core/mod.rs

@@ -1,6 +1,9 @@
 pub mod lexer;
-pub mod parser;
 pub mod preprocessor;
+pub mod store;
+pub mod parser;
+pub mod shared;
 pub mod debugger;
+pub mod utils;
 pub mod builder;
-pub mod types;
+pub mod error;

package/rust/core/parser/handler/at.rs

@@ -0,0 +1,166 @@
+use crate::core::{
+    lexer::token::{ Token, TokenKind },
+    parser::{
+        handler::identifier::parse_identifier_token,
+        statement::{ Statement, StatementKind },
+        Parser,
+    },
+    shared::value::Value,
+    store::global::GlobalStore,
+};
+pub fn parse_at_token(parser: &mut Parser, _global_store: &mut GlobalStore) -> Statement {
+    parser.advance(); // consume '@'
+
+    let Some(token) = parser.peek_clone() else {
+        return Statement::unknown();
+    };
+
+    let keyword = token.lexeme.as_str();
+
+    match keyword {
+        "import" => {
+            parser.advance(); // consume 'import'
+
+            if !parser.match_token(TokenKind::LBrace) {
+                return Statement::error(token, "Expected '{{' after 'import'".to_string());
+            }
+
+            let mut names = Vec::new();
+            while let Some(token) = parser.peek() {
+                match &token.kind {
+                    TokenKind::Identifier => {
+                        names.push(token.lexeme.clone());
+                        parser.advance();
+                    }
+                    TokenKind::Comma => {
+                        parser.advance();
+                    }
+                    TokenKind::RBrace => {
+                        parser.advance();
+                        break;
+                    }
+                    _ => {
+                        let message = format!(
+                            "Unexpected token in import list: {:?}",
+                            token.kind.clone()
+                        );
+                        return Statement::error(token.clone(), message);
+                    }
+                }
+            }
+
+            let Some(from_token) = parser.peek_clone() else {
+                return Statement::error(token, "Expected 'from' after import list".to_string());
+            };
+
+            if from_token.lexeme != "from" {
+                return Statement::error(token, "Expected keyword 'from'".to_string());
+            }
+
+            parser.advance(); // consume 'from'
+
+            let Some(source_token) = parser.peek() else {
+                return Statement::error(token, "Expected string after 'from'".to_string());
+            };
+
+            if source_token.kind != TokenKind::String {
+                return Statement::error(token, "Expected string after 'from'".to_string());
+            }
+
+            let source = source_token.lexeme.clone();
+            parser.advance(); // consume string
+
+            Statement {
+                kind: StatementKind::Import { names, source },
+                value: Value::Null,
+                indent: token.indent,
+                line: token.line,
+                column: token.column,
+            }
+        }
+
+        "export" => {
+            parser.advance(); // consume 'export'
+
+            parser.advance(); // consume '{'
+
+            let names_tokens = parser.collect_until(|t| TokenKind::RBrace == t.kind);
+            let mut names: Vec<String> = Vec::new();
+
+            for token in names_tokens {
+                if token.kind == TokenKind::Identifier {
+                    names.push(token.lexeme.clone());
+                } else if token.kind == TokenKind::Comma {
+                    continue; // Ignore commas
+                } else if token.kind == TokenKind::RBrace {
+                    break; // Stop at the closing brace
+                } else {
+                    return Statement::error(token, "Unexpected token in export list".to_string());
+                }
+            }
+
+            Statement {
+                kind: StatementKind::Export {
+                    names: names.clone(),
+                    source: parser.current_module.clone(),
+                },
+                value: Value::Null,
+                indent: token.indent,
+                line: token.line,
+                column: token.column,
+            }
+        }
+
+        "load" => {
+            parser.advance(); // consume 'load'
+
+            // Exemple : @load "preset.mydeva"
+            let Some(path_token) = parser.peek() else {
+                return Statement::error(token, "Expected string after 'load'".to_string());
+            };
+
+            if path_token.kind != TokenKind::String {
+                return Statement::error(token, "Expected string after 'load'".to_string());
+            }
+
+            let path = path_token.lexeme.clone();
+
+            parser.advance(); // consume string
+            parser.advance(); // consume 'as'
+
+            let Some(as_token) = parser.peek_clone() else {
+                return Statement::error(
+                    token,
+                    "Expected 'as' after path in load statement".to_string()
+                );
+            };
+
+            if as_token.kind != TokenKind::Identifier {
+                return Statement::error(
+                    token,
+                    "Expected identifier after 'as' in load statement".to_string()
+                );
+            }
+
+            let alias = as_token.lexeme.clone();
+
+            parser.advance(); // consume identifier
+
+            Statement {
+                kind: StatementKind::Load {
+                    source: path,
+                    alias,
+                },
+                value: Value::Null,
+                indent: token.indent,
+                line: token.line,
+                column: token.column,
+            }
+        }
+
+        _ => {
+            let message = format!("Unknown keyword after '@' : {}", keyword);
+            Statement::error(token, message)
+        }
+    }
+}