@devaloop/devalang 0.0.1-beta.1 → 0.0.1-beta.2

package/rust/core/lexer/handler/string.rs

@@ -1,63 +1,63 @@
-use crate::core::lexer::token::{Token, TokenKind};
-
-pub fn handle_string_lexer(
-    ch: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
-    current_indent: &mut usize,
-    _indent_stack: &mut [usize],
-    tokens: &mut Vec<Token>,
-    line: &mut usize,
-    column: &mut usize,
-) {
-    let quote_char = ch;
-
-    let start_column = *column;
-    let start_line = *line;
-
-    *column += 1;
-
-    let mut string_content = String::new();
-
-    while let Some(&next_ch) = chars.peek() {
-        if next_ch == quote_char {
-            chars.next();
-            *column += 1;
-            break;
-        } else if next_ch == '\\' {
-            chars.next();
-            *column += 1;
-
-            if let Some(escaped) = chars.next() {
-                match escaped {
-                    'n' => string_content.push('\n'),
-                    't' => string_content.push('\t'),
-                    '\\' => string_content.push('\\'),
-                    '"' => string_content.push('"'),
-                    '\'' => string_content.push('\''),
-                    other => {
-                        string_content.push('\\');
-                        string_content.push(other);
-                    }
-                }
-                *column += 1;
-            }
-        } else {
-            chars.next();
-            if next_ch == '\n' {
-                *line += 1;
-                *column = 1;
-            } else {
-                *column += 1;
-            }
-            string_content.push(next_ch);
-        }
-    }
-
-    tokens.push(Token {
-        kind: TokenKind::String,
-        lexeme: string_content,
-        indent: *current_indent,
-        line: start_line,
-        column: start_column,
-    });
-}
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_string_lexer(
+    ch: char,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let quote_char = ch;
+
+    let start_column = *column;
+    let start_line = *line;
+
+    *column += 1;
+
+    let mut string_content = String::new();
+
+    while let Some(&next_ch) = chars.peek() {
+        if next_ch == quote_char {
+            chars.next();
+            *column += 1;
+            break;
+        } else if next_ch == '\\' {
+            chars.next();
+            *column += 1;
+
+            if let Some(escaped) = chars.next() {
+                match escaped {
+                    'n' => string_content.push('\n'),
+                    't' => string_content.push('\t'),
+                    '\\' => string_content.push('\\'),
+                    '"' => string_content.push('"'),
+                    '\'' => string_content.push('\''),
+                    other => {
+                        string_content.push('\\');
+                        string_content.push(other);
+                    }
+                }
+                *column += 1;
+            }
+        } else {
+            chars.next();
+            if next_ch == '\n' {
+                *line += 1;
+                *column = 1;
+            } else {
+                *column += 1;
+            }
+            string_content.push(next_ch);
+        }
+    }
+
+    tokens.push(Token {
+        kind: TokenKind::String,
+        lexeme: string_content,
+        indent: *current_indent,
+        line: start_line,
+        column: start_column,
+    });
+}

package/rust/core/lexer/mod.rs

@@ -1,3 +1,3 @@
-pub mod driver;
-pub mod handler;
-pub mod token;
+pub mod driver;
+pub mod handler;
+pub mod token;

package/rust/core/mod.rs

@@ -7,4 +7,3 @@ pub mod parser;
 pub mod plugin;
 pub mod preprocessor;
 pub mod store;
-pub mod utils;

package/rust/core/parser/driver/block.rs

@@ -0,0 +1,111 @@
+use crate::core::lexer::token::Token;
+use crate::core::lexer::token::TokenKind;
+use crate::core::store::global::GlobalStore;
+
+pub fn collect_block_tokens(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+) -> Vec<Token> {
+    let mut tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            tokens.push(t.clone());
+        } else {
+            // Unexpected EOF while collecting block tokens: stop collecting
+            break;
+        }
+    }
+
+    tokens
+}
+
+pub fn collect_until<F>(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    condition: F,
+) -> Vec<Token>
+where
+    F: Fn(&Token) -> bool,
+{
+    let mut collected = Vec::new();
+    while let Some(token) = parser.peek() {
+        if condition(token) {
+            break;
+        }
+        if token.kind == TokenKind::EOF {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            collected.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    collected
+}
+
+pub fn parse_block(
+    parser: &crate::core::parser::driver::parser::Parser,
+    tokens: Vec<Token>,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut inner_parser = crate::core::parser::driver::parser::Parser {
+        resolve_modules: parser.resolve_modules,
+        tokens,
+        token_index: 0,
+        current_module: parser.current_module.clone(),
+        previous: None,
+    };
+
+    inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
+}
+
+pub fn parse_block_until_next_else(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut block_tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        // Stop if we encounter an 'else' at same indent level
+        if tok.lexeme == "else" && tok.indent == base_indent {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            block_tokens.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    parse_block(parser, block_tokens, global_store)
+}
+
+pub fn parse_block_until_else_or_dedent(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        if tok.lexeme == "else" && tok.indent == base_indent {
+            break;
+        }
+        if tok.indent < base_indent && tok.kind != TokenKind::Newline {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            tokens.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    parse_block(parser, tokens, global_store)
+}

package/rust/core/parser/driver/cursor.rs

@@ -0,0 +1,82 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn advance_impl(parser: &mut crate::core::parser::driver::parser::Parser) -> Option<&Token> {
+    if is_eof_impl(parser) {
+        return None;
+    }
+
+    parser.previous = parser.tokens.get(parser.token_index).cloned();
+    parser.token_index += 1;
+
+    parser.tokens.get(parser.token_index - 1)
+}
+
+pub fn peek_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<&Token> {
+    parser.tokens.get(parser.token_index)
+}
+
+pub fn peek_clone_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<Token> {
+    parser.tokens.get(parser.token_index).cloned()
+}
+
+pub fn peek_nth_impl(
+    parser: &crate::core::parser::driver::parser::Parser,
+    n: usize,
+) -> Option<&Token> {
+    if parser.token_index + n < parser.tokens.len() {
+        parser.tokens.get(parser.token_index + n)
+    } else {
+        None
+    }
+}
+
+pub fn peek_nth_kind_impl(
+    parser: &crate::core::parser::driver::parser::Parser,
+    n: usize,
+) -> Option<TokenKind> {
+    peek_nth_impl(parser, n).map(|t| t.kind.clone())
+}
+
+pub fn is_eof_impl(parser: &crate::core::parser::driver::parser::Parser) -> bool {
+    parser.token_index >= parser.tokens.len()
+}
+
+pub fn previous_clone_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<Token> {
+    parser.previous.clone()
+}
+
+pub fn match_token_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> bool {
+    if let Some(tok) = peek_impl(parser) {
+        if tok.kind == kind {
+            advance_impl(parser);
+            return true;
+        }
+    }
+    false
+}
+
+pub fn advance_if_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> bool {
+    match_token_impl(parser, kind)
+}
+
+pub fn peek_is_impl(parser: &crate::core::parser::driver::parser::Parser, expected: &str) -> bool {
+    peek_impl(parser).is_some_and(|t| t.lexeme == expected)
+}
+
+pub fn expect_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> Result<&Token, String> {
+    let tok = advance_impl(parser).ok_or("Unexpected end of input")?;
+    if tok.kind == kind {
+        Ok(tok)
+    } else {
+        Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
+    }
+}

package/rust/core/parser/driver/driver_impl.rs

@@ -0,0 +1,139 @@
+use crate::core::lexer::token::TokenKind;
+use crate::core::store::global::GlobalStore;
+use devalang_types::Value;
+
+pub fn parse_tokens_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    tokens: Vec<crate::core::lexer::token::Token>,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
+    // rely on end-of-line semantics.
+    parser.tokens = tokens
+        .into_iter()
+        .filter(|t| t.kind != TokenKind::Whitespace)
+        .collect();
+    parser.token_index = 0;
+
+    let mut statements = Vec::new();
+
+    while !crate::core::parser::driver::cursor::is_eof_impl(parser) {
+        let token = match crate::core::parser::driver::cursor::peek_impl(parser) {
+            Some(t) => t.clone(),
+            None => {
+                break;
+            }
+        };
+
+        if token.kind == TokenKind::Newline {
+            crate::core::parser::driver::cursor::advance_impl(parser);
+            continue;
+        }
+
+        let statement = match &token.kind {
+            TokenKind::At => crate::core::parser::handler::at::parse_at_token(parser, global_store),
+            TokenKind::Identifier => {
+                if
+                    let Some(next) = crate::core::parser::driver::cursor
+                        ::peek_nth_impl(parser, 1)
+                        .cloned()
+                {
+                    if next.kind == TokenKind::Arrow {
+                        crate::core::parser::handler::arrow_call::parse_arrow_call(
+                            parser,
+                            global_store
+                        )
+                    } else {
+                        crate::core::parser::handler::identifier::parse_identifier_token(
+                            parser,
+                            global_store
+                        )
+                    }
+                } else {
+                    crate::core::parser::handler::identifier::parse_identifier_token(
+                        parser,
+                        global_store
+                    )
+                }
+            }
+            TokenKind::Dot =>
+                crate::core::parser::handler::dot::parse_dot_token(parser, global_store),
+            TokenKind::Tempo =>
+                crate::core::parser::handler::tempo::parse_tempo_token(parser, global_store),
+            TokenKind::Bank =>
+                crate::core::parser::handler::bank::parse_bank_token(parser, global_store),
+            TokenKind::Pattern =>
+                crate::core::parser::handler::pattern::parse_pattern_token(parser, global_store),
+            TokenKind::Loop =>
+                crate::core::parser::handler::loop_::parse_loop_token(parser, global_store),
+            TokenKind::If =>
+                crate::core::parser::handler::condition::parse_condition_token(
+                    parser,
+                    global_store
+                ),
+            TokenKind::Function =>
+                crate::core::parser::handler::identifier::function::parse_function_token(
+                    parser,
+                    global_store
+                ),
+            TokenKind::On =>
+                crate::core::parser::handler::identifier::on::parse_on_token(parser, global_store),
+            TokenKind::Emit =>
+                crate::core::parser::handler::identifier::emit::parse_emit_token(
+                    parser,
+                    token.clone(),
+                    global_store
+                ),
+
+            | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
+            | TokenKind::Comment
+            | TokenKind::Equals
+            | TokenKind::Colon
+            | TokenKind::Number
+            | TokenKind::String
+            | TokenKind::LBrace
+            | TokenKind::RBrace
+            | TokenKind::Comma
+            | TokenKind::Dedent
+            | TokenKind::Indent => {
+                crate::core::parser::driver::cursor::advance_impl(parser);
+                continue;
+            }
+
+            TokenKind::EOF => {
+                break;
+            }
+
+            _ => {
+                crate::core::parser::driver::cursor::advance_impl(parser);
+                crate::core::parser::statement::Statement::unknown_with_pos(
+                    token.indent,
+                    token.line,
+                    token.column
+                )
+            }
+        };
+
+        statements.push(statement);
+    }
+
+    statements
+}
+
+pub fn parse_condition_until_colon_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+) -> Option<Value> {
+    let tokens =
+        crate::core::parser::driver::block::collect_until(parser, |t| t.kind == TokenKind::Colon);
+    if tokens.is_empty() {
+        return None;
+    }
+
+    let condition = tokens
+        .iter()
+        .map(|t| t.lexeme.clone())
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    Some(Value::String(condition))
+}

package/rust/core/parser/driver/mod.rs

@@ -0,0 +1,6 @@
+pub mod block;
+pub mod cursor;
+pub mod driver_impl;
+pub mod parse_array;
+pub mod parse_map;
+pub mod parser;

package/rust/core/parser/driver/parse_array.rs

@@ -0,0 +1,120 @@
+use crate::core::lexer::token::TokenKind;
+use devalang_types::Value;
+
+pub fn parse_array_value(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+) -> Option<Value> {
+    let logger = devalang_utils::logger::Logger::new();
+    use devalang_utils::logger::LogLevel;
+    if !parser.match_token(TokenKind::LBracket) {
+        return None;
+    }
+
+    let mut arr: Vec<Value> = Vec::new();
+
+    while !parser.check_token(TokenKind::RBracket) && !parser.is_eof() {
+        // Skip formatting tokens
+        while parser.check_token(TokenKind::Newline)
+            || parser.check_token(TokenKind::Whitespace)
+            || parser.check_token(TokenKind::Indent)
+            || parser.check_token(TokenKind::Dedent)
+            || parser.check_token(TokenKind::Comma)
+        {
+            parser.advance();
+        }
+
+        if parser.check_token(TokenKind::RBracket) {
+            break;
+        }
+
+        if let Some(token) = parser.peek_clone() {
+            let value = match token.kind {
+                TokenKind::String => {
+                    parser.advance();
+                    Value::String(token.lexeme.clone())
+                }
+                TokenKind::Number => {
+                    // Support decimals and fraction literals (e.g., 1/4 -> Duration::Beat("1/4"))
+                    let mut number_str = token.lexeme.clone();
+                    parser.advance();
+                    if let Some(dot) = parser.peek_clone() {
+                        if dot.kind == TokenKind::Dot {
+                            if let Some(next) = parser.peek_nth(1).cloned() {
+                                if next.kind == TokenKind::Number {
+                                    parser.advance(); // consume dot
+                                    parser.advance(); // consume next number
+                                    number_str.push('.');
+                                    number_str.push_str(&next.lexeme);
+                                }
+                            }
+                        }
+                    }
+
+                    // Fraction form: number '/' number or identifier
+                    if let Some(slash_tok) = parser.peek_clone() {
+                        if slash_tok.kind == TokenKind::Slash {
+                            // consume '/'
+                            parser.advance();
+                            if let Some(den_tok) = parser.peek_clone() {
+                                match den_tok.kind {
+                                    TokenKind::Number | TokenKind::Identifier => {
+                                        let frac = format!("{}/{}", number_str, den_tok.lexeme);
+                                        parser.advance();
+                                        Value::Duration(devalang_types::Duration::Beat(frac))
+                                    }
+                                    _ => Value::Number(number_str.parse::<f32>().unwrap_or(0.0)),
+                                }
+                            } else {
+                                Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                            }
+                        } else {
+                            Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                        }
+                    } else {
+                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                    }
+                }
+                TokenKind::Identifier => {
+                    parser.advance();
+                    Value::Identifier(token.lexeme.clone())
+                }
+                TokenKind::LBrace => {
+                    // Allow inline maps inside arrays
+                    if let Some(v) = crate::core::parser::driver::parse_map::parse_map_value(parser)
+                    {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                TokenKind::LBracket => {
+                    // Nested arrays
+                    if let Some(v) = parse_array_value(parser) {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                _ => {
+                    parser.advance();
+                    Value::Null
+                }
+            };
+
+            // Only push non-null (retain alignment with permissive parsing)
+            if value != Value::Null {
+                arr.push(value);
+            }
+
+            // Optional trailing comma handled by the skipper at loop start
+        } else {
+            break;
+        }
+    }
+
+    if !parser.match_token(TokenKind::RBracket) {
+        logger.log_message(LogLevel::Error, "Expected ']' at end of array");
+    }
+
+    Some(Value::Array(arr))
+}