@devaloop/devalang 0.0.1-beta.1 → 0.0.1-beta.3

package/rust/core/preprocessor/loader/mod.rs

@@ -0,0 +1,235 @@
+#[cfg(feature = "cli")]
+use crate::core::preprocessor::resolver::driver::{
+    resolve_all_modules, resolve_and_flatten_all_modules,
+};
+// resolve_relative_path moved to loader_helpers
+use crate::core::{
+    error::ErrorHandler,
+    lexer::{driver::Lexer, token::Token},
+    parser::{driver::parser::Parser, statement::Statement},
+    preprocessor::{module::Module, processor::handlers::process_modules},
+    store::global::GlobalStore,
+};
+use devalang_utils::path::normalize_path;
+use std::{collections::HashMap, path::Path};
+
+mod inject;
+mod loader_helpers;
+
+pub struct ModuleLoader {
+    pub entry: String,
+    pub output: String,
+    pub base_dir: String,
+}
+
+impl ModuleLoader {
+    pub fn new(entry: &str, output: &str) -> Self {
+        let base_dir = Path::new(entry)
+            .parent()
+            .unwrap_or(Path::new(""))
+            .to_string_lossy()
+            .replace('\\', "/");
+
+        Self {
+            entry: entry.to_string(),
+            output: output.to_string(),
+            base_dir,
+        }
+    }
+
+    pub fn from_raw_source(
+        entry_path: &str,
+        output_path: &str,
+        content: &str,
+        global_store: &mut GlobalStore,
+    ) -> Self {
+        let normalized_entry_path = normalize_path(entry_path);
+
+        let mut module = Module::new(entry_path);
+        module.content = content.to_string();
+
+        // Insert a module stub containing the provided content into the
+        // global store. This is used by the WASM APIs and tests which
+        // operate on in-memory sources instead of files on disk.
+        global_store.insert_module(normalized_entry_path.to_string(), module);
+
+        Self {
+            entry: normalized_entry_path.to_string(),
+            output: output_path.to_string(),
+            base_dir: "".to_string(),
+        }
+    }
+
+    pub fn extract_statements_map(
+        &self,
+        global_store: &GlobalStore,
+    ) -> HashMap<String, Vec<Statement>> {
+        global_store
+            .modules
+            .iter()
+            .map(|(path, module)| (path.clone(), module.statements.clone()))
+            .collect()
+    }
+
+    pub fn load_single_module(&self, global_store: &mut GlobalStore) -> Result<Module, String> {
+        let mut module = global_store
+            .modules
+            .remove(&self.entry)
+            .ok_or_else(|| format!("Module not found in store for path: {}", self.entry))?;
+
+        // SECTION Lexing the module content
+        let lexer = Lexer::new();
+        let tokens = lexer
+            .lex_from_source(&module.content)
+            .map_err(|e| format!("Lexer failed: {}", e))?;
+
+        module.tokens = tokens.clone();
+
+        // SECTION Parsing tokens into statements
+        let mut parser = Parser::new();
+        parser.set_current_module(self.entry.clone());
+        let statements = parser.parse_tokens(tokens, global_store);
+        module.statements = statements;
+
+        // SECTION Injecting bank triggers if any (legacy default for single-module run)
+        if let Err(e) = inject::inject_bank_triggers(&mut module, "808", None) {
+            return Err(format!("Failed to inject bank triggers: {}", e));
+        }
+
+        for (plugin_name, alias) in inject::extract_plugin_uses(&module.statements) {
+            inject::load_plugin_and_register(&mut module, &plugin_name, &alias, global_store);
+        }
+
+        global_store
+            .modules
+            .insert(self.entry.clone(), module.clone());
+
+        // SECTION Error handling
+        let mut error_handler = ErrorHandler::new();
+        error_handler.detect_from_statements(&mut parser, &module.statements);
+
+        Ok(module)
+    }
+
+    pub fn load_wasm_module(&self, global_store: &mut GlobalStore) -> Result<(), String> {
+        // Step one : Load the module from the global store
+        let module = {
+            let module_ref = global_store
+                .modules
+                .get(&self.entry)
+                .ok_or_else(|| format!("❌ Module not found for path: {}", self.entry))?;
+
+            Module::from_existing(&self.entry, module_ref.content.clone())
+        };
+
+        // Step two : lexing
+        let lexer = Lexer::new();
+        let tokens = lexer
+            .lex_from_source(&module.content)
+            .map_err(|e| format!("Lexer failed: {}", e))?;
+
+        // Step three : parsing
+        let mut parser = Parser::new();
+        parser.set_current_module(self.entry.clone());
+
+        let statements = parser.parse_tokens(tokens.clone(), global_store);
+
+        let mut updated_module = module;
+        updated_module.tokens = tokens;
+        updated_module.statements = statements;
+
+        // Step four : Injecting bank triggers if any
+        if let Err(e) = inject::inject_bank_triggers(&mut updated_module, "808", None) {
+            return Err(format!("Failed to inject bank triggers: {}", e));
+        }
+
+        // Insert the updated module into the global store before processing so
+        // process_modules can operate on it and populate variable_table, imports,
+        // and other derived structures.
+        global_store
+            .modules
+            .insert(self.entry.clone(), updated_module.clone());
+
+        // Process modules to populate module.variable_table, import/export tables,
+        // and other derived structures so runtime execution can resolve groups/synths.
+        process_modules(self, global_store);
+
+        for (plugin_name, alias) in inject::extract_plugin_uses(&updated_module.statements) {
+            inject::load_plugin_and_register(
+                &mut updated_module,
+                &plugin_name,
+                &alias,
+                global_store,
+            );
+        }
+
+        // Step four : error handling
+        let mut error_handler = ErrorHandler::new();
+        error_handler.detect_from_statements(&mut parser, &updated_module.statements);
+
+        // Final step : also expose module-level variables and functions into the global store
+        // so runtime evaluation (render_audio) can find group/synth definitions.
+        // Use the module instance that was actually processed by `process_modules`
+        // (it lives in `global_store.modules`) because `updated_module` is a local
+        // clone and won't contain the mutations applied by `process_modules`.
+        if let Some(stored_module) = global_store.modules.get(&self.entry) {
+            global_store
+                .variables
+                .variables
+                .extend(stored_module.variable_table.variables.clone());
+            global_store
+                .functions
+                .functions
+                .extend(stored_module.function_table.functions.clone());
+        } else {
+            // Fallback to the local updated_module if for any reason the module
+            // wasn't inserted into the store (defensive programming).
+            global_store
+                .variables
+                .variables
+                .extend(updated_module.variable_table.variables.clone());
+            global_store
+                .functions
+                .functions
+                .extend(updated_module.function_table.functions.clone());
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "cli")]
+    pub fn load_all_modules(
+        &self,
+        global_store: &mut GlobalStore,
+    ) -> (HashMap<String, Vec<Token>>, HashMap<String, Vec<Statement>>) {
+        // SECTION Load the entry module and its dependencies
+        let tokens_by_module = self.load_module_recursively(&self.entry, global_store);
+
+        // SECTION Process and resolve modules
+        process_modules(self, global_store);
+        resolve_all_modules(self, global_store);
+
+        // SECTION Flatten all modules to get statements (+ injects)
+        let statements_by_module = resolve_and_flatten_all_modules(global_store);
+
+        (tokens_by_module, statements_by_module)
+    }
+
+    #[cfg(feature = "cli")]
+    fn load_module_recursively(
+        &self,
+        raw_path: &str,
+        global_store: &mut GlobalStore,
+    ) -> HashMap<String, Vec<Token>> {
+        crate::core::preprocessor::loader::loader_helpers::load_module_recursively(
+            raw_path,
+            global_store,
+        )
+    }
+
+    #[cfg(feature = "cli")]
+    #[allow(dead_code)]
+    fn load_module_imports(&self, path: &String, global_store: &mut GlobalStore) {
+        crate::core::preprocessor::loader::loader_helpers::load_module_imports(path, global_store)
+    }
+}

package/rust/core/preprocessor/module.rs

@@ -1,60 +1,55 @@
-use crate::core::{
-    lexer::token::Token,
-    parser::statement::Statement,
-    store::{
-        export::ExportTable, function::FunctionTable, import::ImportTable, variable::VariableTable,
-    },
-};
-
-#[derive(Debug, Clone)]
-pub struct Module {
-    pub path: String,
-    pub resolved: bool,
-    pub tokens: Vec<Token>,
-    pub statements: Vec<Statement>,
-    pub variable_table: VariableTable,
-    pub function_table: FunctionTable,
-    pub export_table: ExportTable,
-    pub import_table: ImportTable,
-    pub content: String,
-    pub current_dir: String,
-}
-
-impl Module {
-    pub fn new(path: &str) -> Self {
-        Module {
-            path: path.to_string(),
-            tokens: Vec::new(),
-            statements: Vec::new(),
-            variable_table: VariableTable::new(),
-            function_table: FunctionTable::new(),
-            export_table: ExportTable::new(),
-            import_table: ImportTable::new(),
-            resolved: false,
-            content: String::new(),
-            current_dir: String::new(),
-        }
-    }
-
-    pub fn is_resolved(&self) -> bool {
-        self.resolved
-    }
-
-    pub fn set_resolved(&mut self, resolved: bool) {
-        self.resolved = resolved;
-    }
-
-    pub fn add_token(&mut self, token: Token) {
-        self.tokens.push(token);
-    }
-
-    pub fn add_statement(&mut self, statement: Statement) {
-        self.statements.push(statement);
-    }
-
-    pub fn from_existing(path: &str, content: String) -> Self {
-        let mut module = Module::new(path);
-        module.content = content;
-        module
-    }
-}
+use crate::core::{lexer::token::Token, parser::statement::Statement};
+use devalang_types::{ExportTable, FunctionTable, ImportTable, VariableTable};
+
+#[derive(Debug, Clone)]
+pub struct Module {
+    pub path: String,
+    pub resolved: bool,
+    pub tokens: Vec<Token>,
+    pub statements: Vec<Statement>,
+    pub variable_table: VariableTable,
+    pub function_table: FunctionTable,
+    pub export_table: ExportTable,
+    pub import_table: ImportTable,
+    pub content: String,
+    pub current_dir: String,
+}
+
+impl Module {
+    pub fn new(path: &str) -> Self {
+        Module {
+            path: path.to_string(),
+            tokens: Vec::new(),
+            statements: Vec::new(),
+            variable_table: VariableTable::new(),
+            function_table: FunctionTable::new(),
+            export_table: ExportTable::new(),
+            import_table: ImportTable::new(),
+            resolved: false,
+            content: String::new(),
+            current_dir: String::new(),
+        }
+    }
+
+    pub fn is_resolved(&self) -> bool {
+        self.resolved
+    }
+
+    pub fn set_resolved(&mut self, resolved: bool) {
+        self.resolved = resolved;
+    }
+
+    pub fn add_token(&mut self, token: Token) {
+        self.tokens.push(token);
+    }
+
+    pub fn add_statement(&mut self, statement: Statement) {
+        self.statements.push(statement);
+    }
+
+    pub fn from_existing(path: &str, content: String) -> Self {
+        let mut module = Module::new(path);
+        module.content = content;
+        module
+    }
+}

package/rust/core/preprocessor/{processor.rs → processor/handlers.rs}

@@ -1,114 +1,107 @@
-use std::{collections::HashMap, path::Path};
-
-use crate::core::{
-    parser::statement::StatementKind,
-    preprocessor::loader::ModuleLoader,
-    store::global::GlobalStore,
-    utils::path::{normalize_path, resolve_relative_path},
-};
-use devalang_types::Value;
-
-pub fn process_modules(_module_loader: &ModuleLoader, global_store: &mut GlobalStore) {
-    for module in global_store.modules.values_mut() {
-        let logger = devalang_utils::logger::Logger::new();
-        use devalang_utils::logger::LogLevel;
-
-        for stmt in &module.statements {
-            match &stmt.kind {
-                StatementKind::Let { name } => {
-                    if let Value::Null = stmt.value {
-                        logger.log_message(
-                            LogLevel::Error,
-                            &format!("Variable '{}' is declared but not initialized.", name),
-                        );
-
-                        module.variable_table.variables.insert(
-                            name.clone(),
-                            Value::StatementKind(Box::new(stmt.kind.clone())),
-                        );
-
-                        continue;
-                    }
-
-                    if module.variable_table.get(name).is_some() {
-                        logger.log_message(
-                            LogLevel::Error,
-                            &format!("Variable '{}' is already defined in this scope.", name),
-                        );
-                        continue;
-                    }
-
-                    if let Some(module_variable) = module.variable_table.variables.get(name) {
-                        logger.log_message(
-                            LogLevel::Error,
-                            &format!(
-                                "Variable '{}' is already defined globally with value: {:?}",
-                                name, module_variable
-                            ),
-                        );
-                        continue;
-                    }
-
-                    module
-                        .variable_table
-                        .variables
-                        .insert(name.clone(), stmt.value.clone());
-                }
-
-                StatementKind::Load { source, alias } => {
-                    let module_dir = Path::new(&module.path).parent().unwrap_or(Path::new(""));
-
-                    let resolved_path = normalize_path(module_dir.join(source));
-
-                    module
-                        .variable_table
-                        .variables
-                        .insert(alias.clone(), Value::Sample(resolved_path));
-                }
-
-                StatementKind::Export { names, source: _ } => {
-                    for name in names {
-                        if let Some(val) = module.variable_table.get(name) {
-                            module.export_table.add_export(name.clone(), val.clone());
-                        }
-                    }
-                }
-
-                StatementKind::Import { names, source } => {
-                    let resolved = resolve_relative_path(&module.path, source);
-                    for name in names {
-                        module
-                            .import_table
-                            .add_import(name.clone(), Value::String(resolved.clone()));
-                    }
-                }
-
-                StatementKind::Group => {
-                    if let Value::Map(map) = &stmt.value {
-                        if let (Some(Value::String(name)), Some(Value::Block(body))) =
-                            (map.get("identifier"), map.get("body"))
-                        {
-                            let mut stored_map = HashMap::new();
-
-                            stored_map
-                                .insert("identifier".to_string(), Value::String(name.clone()));
-
-                            stored_map.insert("body".to_string(), Value::Block(body.clone()));
-
-                            module
-                                .variable_table
-                                .set(name.to_string(), Value::Map(stored_map));
-                        } else {
-                            logger.log_message(
-                                LogLevel::Error,
-                                &format!("Invalid group definition: {:?}", stmt.value),
-                            );
-                        }
-                    }
-                }
-
-                _ => {}
-            }
-        }
-    }
-}
+use crate::core::{
+    parser::statement::StatementKind, preprocessor::loader::ModuleLoader,
+    store::global::GlobalStore,
+};
+use devalang_types::Value;
+use devalang_utils::path::{normalize_path, resolve_relative_path};
+use std::collections::HashMap;
+
+pub fn process_modules(_module_loader: &ModuleLoader, global_store: &mut GlobalStore) {
+    for module in global_store.modules.values_mut() {
+        let logger = devalang_utils::logger::Logger::new();
+        use devalang_utils::logger::LogLevel;
+
+        for stmt in &module.statements {
+            match &stmt.kind {
+                StatementKind::Let { name } => {
+                    if let Value::Null = stmt.value {
+                        logger.log_message(
+                            LogLevel::Error,
+                            &format!("Variable '{}' is declared but not initialized.", name),
+                        );
+                        module.variable_table.variables.insert(
+                            name.clone(),
+                            Value::StatementKind(Box::new(stmt.kind.clone())),
+                        );
+                        continue;
+                    }
+
+                    if module.variable_table.get(name).is_some() {
+                        logger.log_message(
+                            LogLevel::Error,
+                            &format!("Variable '{}' is already defined in this scope.", name),
+                        );
+                        continue;
+                    }
+
+                    if let Some(module_variable) = module.variable_table.variables.get(name) {
+                        logger.log_message(
+                            LogLevel::Error,
+                            &format!(
+                                "Variable '{}' is already defined globally with value: {:?}",
+                                name, module_variable
+                            ),
+                        );
+                        continue;
+                    }
+
+                    module
+                        .variable_table
+                        .variables
+                        .insert(name.clone(), stmt.value.clone());
+                }
+
+                StatementKind::Load { source, alias } => {
+                    let module_dir = std::path::Path::new(&module.path)
+                        .parent()
+                        .unwrap_or(std::path::Path::new(""));
+                    let resolved_path = normalize_path(module_dir.join(source));
+                    module
+                        .variable_table
+                        .variables
+                        .insert(alias.clone(), Value::Sample(resolved_path));
+                }
+
+                StatementKind::Export { names, source: _ } => {
+                    for name in names {
+                        if let Some(val) = module.variable_table.get(name) {
+                            module.export_table.add_export(name.clone(), val.clone());
+                        }
+                    }
+                }
+
+                StatementKind::Import { names, source } => {
+                    let resolved = resolve_relative_path(&module.path, source);
+                    for name in names {
+                        module
+                            .import_table
+                            .add_import(name.clone(), Value::String(resolved.clone()));
+                    }
+                }
+
+                StatementKind::Group => {
+                    if let Value::Map(map) = &stmt.value {
+                        if let (Some(Value::String(name)), Some(Value::Block(body))) =
+                            (map.get("identifier"), map.get("body"))
+                        {
+                            let mut stored_map = HashMap::new();
+                            stored_map
+                                .insert("identifier".to_string(), Value::String(name.clone()));
+                            stored_map.insert("body".to_string(), Value::Block(body.clone()));
+                            module
+                                .variable_table
+                                .set(name.to_string(), Value::Map(stored_map));
+                        } else {
+                            logger.log_message(
+                                LogLevel::Error,
+                                &format!("Invalid group definition: {:?}", stmt.value),
+                            );
+                        }
+                    }
+                }
+
+                _ => {}
+            }
+        }
+    }
+}

package/rust/core/preprocessor/processor/mod.rs

@@ -0,0 +1 @@
+pub mod handlers;