@devaloop/devalang 0.0.1-alpha.2 → 0.0.1-alpha.3

package/rust/core/parser/mod.rs

@@ -1,201 +1,239 @@
-pub mod identifer;
-pub mod variable;
-pub mod at;
-pub mod dot;
-pub mod bank;
-pub mod loop_;
-pub mod tempo;
-
-use crate::{
-    core::{
-        parser::{
-            at::parse_at,
-            bank::parse_bank,
-            dot::parse_dot,
-            identifer::parse_identifier,
-            loop_::parse_loop,
-            tempo::parse_tempo,
-        },
-        preprocessor::resolver::resolve_statement,
-        types::{
-            module::Module,
-            parser::Parser,
-            statement::{ Statement, StatementKind, StatementResolved },
-            store::{ GlobalStore },
-            token::{ Token, TokenKind },
-            variable::VariableValue,
+pub mod statement;
+pub mod handler;
+
+use crate::core::{
+    lexer::token::{ Token, TokenKind },
+    parser::{
+        handler::{
+            at::parse_at_token,
+            bank::parse_bank_token,
+            dot::parse_dot_token,
+            identifier::parse_identifier_token,
+            loop_::parse_loop_token,
+            tempo::parse_tempo_token,
         },
+        statement::{ Statement, StatementKind },
     },
-    utils::logger::log_message,
+    shared::value::Value,
+    store::global::GlobalStore,
 };
 
-pub fn parse_without_resolving(
-    tokens: Vec<Token>,
-    mut parser: &mut Parser,
-    global_store: &mut GlobalStore
-) -> Vec<Statement> {
-    let mut statements = Vec::new();
-
-    while !parser.is_eof() {
-        let mut error_statement = Statement {
-            kind: StatementKind::Error,
-            value: VariableValue::Null,
-            line: parser.peek().map_or(0, |t| t.line),
-            column: parser.peek().map_or(0, |t| t.column),
-            indent: parser.peek().map_or(0, |t| t.indent),
-        };
-
-        match parser.peek().map(|t| t.kind.clone()) {
-            Some(TokenKind::Identifier) => {
-                match parse_identifier(&mut parser, global_store) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+#[derive(Debug, Clone, PartialEq)]
+pub struct Parser {
+    pub resolve_modules: bool,
+    pub tokens: Vec<Token>,
+    pub token_index: usize,
+    pub current_module: String,
+    pub previous: Option<Token>,
+}
 
-            Some(TokenKind::Bank) => {
-                match parse_bank(&mut parser, global_store) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+impl Parser {
+    pub fn new() -> Self {
+        Parser {
+            resolve_modules: false,
+            tokens: Vec::new(),
+            token_index: 0,
+            current_module: String::new(),
+            previous: None,
+        }
+    }
 
-            Some(TokenKind::At) => {
-                match parse_at(&mut parser) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+    pub fn set_current_module(&mut self, module_path: String) {
+        self.current_module = module_path;
+    }
 
-            Some(TokenKind::Dot) => {
-                match parse_dot(&mut parser, global_store) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+    pub fn advance(&mut self) -> Option<&Token> {
+        if self.is_eof() {
+            return None;
+        }
 
-            Some(TokenKind::Loop) => {
-                match parse_loop(&mut parser, global_store) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+        self.previous = self.tokens.get(self.token_index).cloned(); // mémorise avant de bouger
+        self.token_index += 1;
 
-            Some(TokenKind::Tempo) => {
-                match parse_tempo(&mut parser, global_store) {
-                    Ok(statement) => statements.push(statement),
-                    Err(e) => {
-                        error_statement.value = VariableValue::Text(e.to_string());
-                        statements.push(error_statement);
-                    }
-                }
-            }
+        self.tokens.get(self.token_index - 1)
+    }
 
-            | Some(TokenKind::LBrace)
-            | Some(TokenKind::RBrace)
-            | Some(TokenKind::LBracket)
-            | Some(TokenKind::RBracket)
-            | Some(TokenKind::DbQuote)
-            | Some(TokenKind::Quote)
-            | Some(TokenKind::Number)
-            | Some(TokenKind::String)
-            | Some(TokenKind::Newline)
-            | Some(TokenKind::Indent)
-            | Some(TokenKind::Dedent) => {
-                parser.next();
-            }
-            Some(_) => {
-                parser.next();
-            }
-            None => {
-                break;
+    pub fn match_token(&mut self, kind: TokenKind) -> bool {
+        if let Some(tok) = self.peek() {
+            if tok.kind == kind {
+                self.advance();
+                return true;
             }
         }
+        false
     }
 
-    statements
-}
+    pub fn previous_clone(&self) -> Option<Token> {
+        self.previous.clone()
+    }
+
+    pub fn parse_block(
+        &self,
+        tokens: Vec<Token>,
+        global_store: &mut GlobalStore
+    ) -> Vec<Statement> {
+        let mut inner_parser = Parser {
+            resolve_modules: self.resolve_modules,
+            tokens,
+            token_index: 0,
+            current_module: self.current_module.clone(),
+            previous: None,
+        };
+
+        inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
+    }
 
-pub fn parse_without_resolving_with_module(tokens: Vec<Token>, module: &Module) -> Vec<Statement> {
-    let mut parser = Parser::new(tokens.clone());
+    pub fn parse_tokens(
+        &mut self,
+        tokens: Vec<Token>,
+        global_store: &mut GlobalStore
+    ) -> Vec<Statement> {
+        self.tokens = tokens;
+        self.token_index = 0;
+
+        let mut statements = Vec::new();
+
+        while !self.is_eof() {
+            let token = match self.peek() {
+                Some(t) => t.clone(),
+                None => {
+                    break;
+                }
+            };
+
+            let mut statement = match &token.kind {
+                TokenKind::At => parse_at_token(self, global_store),
+                TokenKind::Identifier => parse_identifier_token(self, global_store),
+                TokenKind::Dot => parse_dot_token(self, global_store),
+                TokenKind::Tempo => parse_tempo_token(self, global_store),
+                TokenKind::Bank => parse_bank_token(self, global_store),
+                TokenKind::Loop => parse_loop_token(self, global_store),
+
+                | TokenKind::Comment
+                | TokenKind::Equals
+                | TokenKind::Colon
+                | TokenKind::Number
+                | TokenKind::String
+                | TokenKind::LBrace
+                | TokenKind::RBrace
+                | TokenKind::Comma
+                | TokenKind::Newline
+                | TokenKind::Dedent
+                | TokenKind::Indent => {
+                    self.advance();
+                    continue;
+                }
 
-    parser.current_module = module.path.clone();
+                TokenKind::EOF => {
+                    break;
+                }
 
-    let mut global_store = GlobalStore::new();
-    global_store.insert_module(module.path.clone(), module.clone());
+                _ => {
+                    println!("Unhandled token: {:?}", token);
+                    self.advance();
+                    Statement::unknown()
+                }
+            };
 
-    let statements = parse_without_resolving(tokens, &mut parser, &mut global_store);
+            statements.push(statement);
+        }
 
-    let mut updated_module = module.clone();
-    updated_module.statements = statements.clone();
+        statements
+    }
+
+    pub fn check_token(&self, kind: TokenKind) -> bool {
+        self.peek().map_or(false, |t| t.kind == kind)
+    }
 
-    let mut errors: Vec<String> = Vec::new();
+    pub fn parse_map_value(&mut self) -> Option<Value> {
+        if !self.match_token(TokenKind::LBrace) {
+            return None;
+        }
 
-    statements.iter().for_each(|statement| {
-        match &statement.kind {
-            StatementKind::Error => {
-                let error_message = match &statement.value {
-                    VariableValue::Text(text) => text.clone(),
-                    _ => "Unknown error".to_string(),
-                };
+        let mut map = std::collections::HashMap::new();
 
-                errors.push(format!(
-                    "Error in module '{}': {} at line {}, column {}",
-                    updated_module.path,
-                    error_message,
-                    statement.line,
-                    statement.column
-                ));
+        while !self.check_token(TokenKind::RBrace) && !self.is_eof() {
+            let key = if let Some(token) = self.advance() {
+                token.lexeme.clone()
+            } else {
+                break;
+            };
 
-                log_message(&format!("Error: {}", error_message), "ERROR");
+            if !self.match_token(TokenKind::Colon) {
+                println!("Expected ':' after map key '{}'", key);
+                break;
             }
-            _ => {}
-        }
-    });
 
-    if errors.len() > 0 {
-        log_message(&format!("{} error(s) found. Parsing stopped.", errors.len()), "INFO");
+            let value = if let Some(token) = self.peek_clone() {
+                match token.kind {
+                    TokenKind::String => {
+                        self.advance();
+                        Value::String(token.lexeme.clone())
+                    }
+                    TokenKind::Number => {
+                        self.advance();
+                        Value::Number(token.lexeme.parse().unwrap_or(0.0))
+                    }
+                    TokenKind::Identifier => {
+                        self.advance();
+                        Value::Identifier(token.lexeme.clone())
+                    }
+                    _ => {
+                        println!("Unexpected token in map value: {:?}", token);
+                        Value::Null
+                    }
+                }
+            } else {
+                Value::Null
+            };
 
-        statements
-    } else {
-        statements
-    }
-}
+            map.insert(key, value);
+        }
 
-pub fn parse_with_resolving_with_module(
-    tokens: Vec<Token>,
-    module: &Module
-) -> Vec<StatementResolved> {
-    let mut parser = Parser::new(tokens.clone());
+        if !self.match_token(TokenKind::RBrace) {
+            println!("Expected '}}' at end of map");
+        }
 
-    parser.current_module = module.path.clone();
+        Some(Value::Map(map))
+    }
 
-    let mut global_store = GlobalStore::new();
-    global_store.insert_module(module.path.clone(), module.clone());
+    pub fn peek(&self) -> Option<&Token> {
+        self.tokens.get(self.token_index)
+    }
 
-    let statements = parse_without_resolving(tokens, &mut parser, &mut global_store);
+    pub fn peek_clone(&self) -> Option<Token> {
+        self.tokens.get(self.token_index).cloned()
+    }
 
-    let mut resolved_statements = Vec::new();
+    pub fn expect(&mut self, kind: TokenKind) -> Result<&Token, String> {
+        let tok = self.advance().ok_or("Unexpected end of input")?;
+        if tok.kind == kind {
+            Ok(tok)
+        } else {
+            Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
+        }
+    }
 
-    for statement in statements {
-        let resolved_statement = resolve_statement(&statement, &mut module.clone());
-        resolved_statements.push(resolved_statement);
+    pub fn collect_until<F>(&mut self, condition: F) -> Vec<Token> where F: Fn(&Token) -> bool {
+        let mut collected = Vec::new();
+        while let Some(token) = self.peek() {
+            if token.kind == TokenKind::Newline || token.kind == TokenKind::Indent {
+                self.advance(); // Skip newlines and indents
+                continue;
+            }
+            if token.kind == TokenKind::EOF {
+                break;
+            }
+            if condition(token) {
+                self.advance(); // Consume the token that matches the condition
+                break;
+            }
+            collected.push(self.advance().unwrap().clone());
+        }
+        collected
     }
 
-    return resolved_statements;
+    pub fn is_eof(&self) -> bool {
+        self.token_index >= self.tokens.len()
+    }
 }

package/rust/core/parser/statement.rs

@@ -0,0 +1,91 @@
+use serde::{ Deserialize, Serialize };
+
+use crate::core::{ lexer::token::Token, shared::{ duration::Duration, value::Value } };
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct Statement {
+    pub kind: StatementKind,
+    pub value: Value,
+    pub indent: usize,
+    pub line: usize,
+    pub column: usize,
+}
+
+impl Statement {
+    pub fn unknown() -> Self {
+        Statement {
+            kind: StatementKind::Unknown,
+            value: Value::Null,
+            indent: 0,
+            line: 0,
+            column: 0,
+        }
+    }
+
+    pub fn error(token: Token, message: String) -> Self {
+        Statement {
+            kind: StatementKind::Error { message },
+            value: Value::Null,
+            indent: token.indent,
+            line: token.line,
+            column: token.column,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Clone, Deserialize, PartialEq)]
+/// Represents the kind of a statement
+pub enum StatementKind {
+    // Trigger statements
+    Trigger {
+        entity: String,
+        duration: Duration,
+        // params: Vec<TokenParam>,
+    },
+
+    // Variable statements
+    Let {
+        name: String,
+    },
+
+    // Loop statements
+    Loop,
+
+    // Conditional statements
+    // If {
+    //     // condition: ConditionParts,
+    //     condition_state: bool,
+    //     body: Vec<Statement>,
+    // },
+
+    // Keyword statements
+    Tempo,
+    Bank,
+
+    // At (@) statements
+    Include(String),
+    Export {
+        names: Vec<String>,
+        source: String,
+    },
+    Import {
+        names: Vec<String>,
+        source: String,
+    },
+    Load {
+        source: String,
+        alias: String,
+    },
+
+    // Error & Unknown statements
+    Unknown,
+    Error {
+        message: String,
+    },
+
+    // Empty or ignored statements
+    Comment,
+    Indent,
+    Dedent,
+    NewLine,
+}

package/rust/core/preprocessor/loader.rs

@@ -0,0 +1,105 @@
+use std::collections::HashMap;
+
+use crate::{
+    core::{
+        debugger::{ lexer::write_lexer_log_file, preprocessor::write_preprocessor_log_file },
+        error::ErrorHandler,
+        lexer::Lexer,
+        parser::{ statement::{ Statement, StatementKind }, Parser },
+        preprocessor::{
+            module::Module,
+            processor::process_modules,
+            resolver::{ resolve_all_modules, resolve_and_flatten_all_modules },
+        },
+        store::global::GlobalStore,
+    },
+    utils::logger::{ LogLevel, Logger },
+};
+
+pub struct ModuleLoader {
+    pub entry: String,
+    pub output: String,
+}
+
+impl ModuleLoader {
+    pub fn new(entry: &str, output: &str) -> Self {
+        Self {
+            entry: entry.to_string(),
+            output: output.to_string(),
+        }
+    }
+
+    pub fn load_all(&self, global_store: &mut GlobalStore) -> HashMap<String, Vec<Statement>> {
+        // SECTION Load the entry module and its dependencies
+        self.load_module_recursively(&self.entry, global_store);
+
+        // SECTION Process and resolve modules
+        process_modules(self, global_store);
+        resolve_all_modules(self, global_store);
+
+        let resolved = resolve_and_flatten_all_modules(global_store);
+
+        // SECTION Write resolved statements to log file
+        write_preprocessor_log_file(&self.output, "resolved_statements.log", resolved.clone());
+
+        // Return the resolved statements
+        resolved
+    }
+
+    fn load_module_recursively(&self, path: &str, global_store: &mut GlobalStore) {
+        if global_store.modules.contains_key(path) {
+            return;
+        }
+
+        let lexer = Lexer::new();
+        let tokens = lexer.lex_tokens(path);
+
+        let mut parser = Parser::new();
+        parser.set_current_module(path.to_string());
+
+        let statements = parser.parse_tokens(tokens.clone(), global_store);
+
+        // SECTION Error handling
+        let mut error_handler = ErrorHandler::new();
+        error_handler.detect_from_statements(&mut parser, &statements);
+
+        error_handler.has_errors().then(|| {
+            let logger = Logger::new();
+            let errors = error_handler.get_errors();
+
+            for error in errors {
+                let stacktrace = format!("{}:{}:{}", path, error.line, error.column);
+                logger.log_error_with_stacktrace(&error.message, &stacktrace);
+            }
+        });
+
+        // SECTION Module creation
+        let mut module = Module::new(path);
+        module.tokens = tokens;
+        module.statements = statements;
+
+        global_store.insert_module(path.to_string(), module);
+
+        // Then load the imports recursively
+        self.load_module_imports(&path.to_string(), global_store);
+    }
+
+    fn load_module_imports(&self, path: &String, global_store: &mut GlobalStore) {
+        let imports = global_store.modules
+            .get(path)
+            .unwrap()
+            .statements.iter()
+            .filter_map(|stmt| {
+                if let StatementKind::Import { source, .. } = &stmt.kind {
+                    Some(source.clone())
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        for import_path in imports {
+            self.load_module_recursively(&import_path, global_store);
+        }
+    }
+}

package/rust/core/preprocessor/mod.rs

@@ -1,26 +1,4 @@
 pub mod module;
-pub mod dependencies;
+pub mod loader;
 pub mod resolver;
-
-use crate::core::{
-    preprocessor::{
-        dependencies::collect_dependencies_recursively,
-        module::load_module_into_global_store,
-    },
-    types::{ store::GlobalStore },
-};
-
-pub fn preprocess(entry_file: &str) -> GlobalStore {
-    println!("📦 Collecting dependencies for: {}", entry_file);
-
-    let files = collect_dependencies_recursively(entry_file);
-    let mut store = GlobalStore::default();
-
-    for file in &files {
-        if let Err(e) = load_module_into_global_store(file, &mut store) {
-            eprintln!("❌ Error while loading {}: {}", file, e);
-        }
-    }
-
-    store
-}
+pub mod processor;