@devaloop/devalang 0.0.1-alpha.9 → 0.0.1-beta.2

package/rust/core/parser/driver/block.rs

@@ -0,0 +1,111 @@
+use crate::core::lexer::token::Token;
+use crate::core::lexer::token::TokenKind;
+use crate::core::store::global::GlobalStore;
+
+pub fn collect_block_tokens(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+) -> Vec<Token> {
+    let mut tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            tokens.push(t.clone());
+        } else {
+            // Unexpected EOF while collecting block tokens: stop collecting
+            break;
+        }
+    }
+
+    tokens
+}
+
+pub fn collect_until<F>(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    condition: F,
+) -> Vec<Token>
+where
+    F: Fn(&Token) -> bool,
+{
+    let mut collected = Vec::new();
+    while let Some(token) = parser.peek() {
+        if condition(token) {
+            break;
+        }
+        if token.kind == TokenKind::EOF {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            collected.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    collected
+}
+
+pub fn parse_block(
+    parser: &crate::core::parser::driver::parser::Parser,
+    tokens: Vec<Token>,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut inner_parser = crate::core::parser::driver::parser::Parser {
+        resolve_modules: parser.resolve_modules,
+        tokens,
+        token_index: 0,
+        current_module: parser.current_module.clone(),
+        previous: None,
+    };
+
+    inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
+}
+
+pub fn parse_block_until_next_else(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut block_tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        // Stop if we encounter an 'else' at same indent level
+        if tok.lexeme == "else" && tok.indent == base_indent {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            block_tokens.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    parse_block(parser, block_tokens, global_store)
+}
+
+pub fn parse_block_until_else_or_dedent(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    base_indent: usize,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    let mut tokens = Vec::new();
+
+    while let Some(tok) = parser.peek() {
+        if tok.lexeme == "else" && tok.indent == base_indent {
+            break;
+        }
+        if tok.indent < base_indent && tok.kind != TokenKind::Newline {
+            break;
+        }
+        if let Some(t) = parser.advance() {
+            tokens.push(t.clone());
+        } else {
+            break;
+        }
+    }
+
+    parse_block(parser, tokens, global_store)
+}

package/rust/core/parser/driver/cursor.rs

@@ -0,0 +1,82 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn advance_impl(parser: &mut crate::core::parser::driver::parser::Parser) -> Option<&Token> {
+    if is_eof_impl(parser) {
+        return None;
+    }
+
+    parser.previous = parser.tokens.get(parser.token_index).cloned();
+    parser.token_index += 1;
+
+    parser.tokens.get(parser.token_index - 1)
+}
+
+pub fn peek_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<&Token> {
+    parser.tokens.get(parser.token_index)
+}
+
+pub fn peek_clone_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<Token> {
+    parser.tokens.get(parser.token_index).cloned()
+}
+
+pub fn peek_nth_impl(
+    parser: &crate::core::parser::driver::parser::Parser,
+    n: usize,
+) -> Option<&Token> {
+    if parser.token_index + n < parser.tokens.len() {
+        parser.tokens.get(parser.token_index + n)
+    } else {
+        None
+    }
+}
+
+pub fn peek_nth_kind_impl(
+    parser: &crate::core::parser::driver::parser::Parser,
+    n: usize,
+) -> Option<TokenKind> {
+    peek_nth_impl(parser, n).map(|t| t.kind.clone())
+}
+
+pub fn is_eof_impl(parser: &crate::core::parser::driver::parser::Parser) -> bool {
+    parser.token_index >= parser.tokens.len()
+}
+
+pub fn previous_clone_impl(parser: &crate::core::parser::driver::parser::Parser) -> Option<Token> {
+    parser.previous.clone()
+}
+
+pub fn match_token_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> bool {
+    if let Some(tok) = peek_impl(parser) {
+        if tok.kind == kind {
+            advance_impl(parser);
+            return true;
+        }
+    }
+    false
+}
+
+pub fn advance_if_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> bool {
+    match_token_impl(parser, kind)
+}
+
+pub fn peek_is_impl(parser: &crate::core::parser::driver::parser::Parser, expected: &str) -> bool {
+    peek_impl(parser).is_some_and(|t| t.lexeme == expected)
+}
+
+pub fn expect_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    kind: TokenKind,
+) -> Result<&Token, String> {
+    let tok = advance_impl(parser).ok_or("Unexpected end of input")?;
+    if tok.kind == kind {
+        Ok(tok)
+    } else {
+        Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
+    }
+}

package/rust/core/parser/driver/driver_impl.rs

@@ -0,0 +1,139 @@
+use crate::core::lexer::token::TokenKind;
+use crate::core::store::global::GlobalStore;
+use devalang_types::Value;
+
+pub fn parse_tokens_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+    tokens: Vec<crate::core::lexer::token::Token>,
+    global_store: &mut GlobalStore,
+) -> Vec<crate::core::parser::statement::Statement> {
+    // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
+    // rely on end-of-line semantics.
+    parser.tokens = tokens
+        .into_iter()
+        .filter(|t| t.kind != TokenKind::Whitespace)
+        .collect();
+    parser.token_index = 0;
+
+    let mut statements = Vec::new();
+
+    while !crate::core::parser::driver::cursor::is_eof_impl(parser) {
+        let token = match crate::core::parser::driver::cursor::peek_impl(parser) {
+            Some(t) => t.clone(),
+            None => {
+                break;
+            }
+        };
+
+        if token.kind == TokenKind::Newline {
+            crate::core::parser::driver::cursor::advance_impl(parser);
+            continue;
+        }
+
+        let statement = match &token.kind {
+            TokenKind::At => crate::core::parser::handler::at::parse_at_token(parser, global_store),
+            TokenKind::Identifier => {
+                if
+                    let Some(next) = crate::core::parser::driver::cursor
+                        ::peek_nth_impl(parser, 1)
+                        .cloned()
+                {
+                    if next.kind == TokenKind::Arrow {
+                        crate::core::parser::handler::arrow_call::parse_arrow_call(
+                            parser,
+                            global_store
+                        )
+                    } else {
+                        crate::core::parser::handler::identifier::parse_identifier_token(
+                            parser,
+                            global_store
+                        )
+                    }
+                } else {
+                    crate::core::parser::handler::identifier::parse_identifier_token(
+                        parser,
+                        global_store
+                    )
+                }
+            }
+            TokenKind::Dot =>
+                crate::core::parser::handler::dot::parse_dot_token(parser, global_store),
+            TokenKind::Tempo =>
+                crate::core::parser::handler::tempo::parse_tempo_token(parser, global_store),
+            TokenKind::Bank =>
+                crate::core::parser::handler::bank::parse_bank_token(parser, global_store),
+            TokenKind::Pattern =>
+                crate::core::parser::handler::pattern::parse_pattern_token(parser, global_store),
+            TokenKind::Loop =>
+                crate::core::parser::handler::loop_::parse_loop_token(parser, global_store),
+            TokenKind::If =>
+                crate::core::parser::handler::condition::parse_condition_token(
+                    parser,
+                    global_store
+                ),
+            TokenKind::Function =>
+                crate::core::parser::handler::identifier::function::parse_function_token(
+                    parser,
+                    global_store
+                ),
+            TokenKind::On =>
+                crate::core::parser::handler::identifier::on::parse_on_token(parser, global_store),
+            TokenKind::Emit =>
+                crate::core::parser::handler::identifier::emit::parse_emit_token(
+                    parser,
+                    token.clone(),
+                    global_store
+                ),
+
+            | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
+            | TokenKind::Comment
+            | TokenKind::Equals
+            | TokenKind::Colon
+            | TokenKind::Number
+            | TokenKind::String
+            | TokenKind::LBrace
+            | TokenKind::RBrace
+            | TokenKind::Comma
+            | TokenKind::Dedent
+            | TokenKind::Indent => {
+                crate::core::parser::driver::cursor::advance_impl(parser);
+                continue;
+            }
+
+            TokenKind::EOF => {
+                break;
+            }
+
+            _ => {
+                crate::core::parser::driver::cursor::advance_impl(parser);
+                crate::core::parser::statement::Statement::unknown_with_pos(
+                    token.indent,
+                    token.line,
+                    token.column
+                )
+            }
+        };
+
+        statements.push(statement);
+    }
+
+    statements
+}
+
+pub fn parse_condition_until_colon_impl(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+) -> Option<Value> {
+    let tokens =
+        crate::core::parser::driver::block::collect_until(parser, |t| t.kind == TokenKind::Colon);
+    if tokens.is_empty() {
+        return None;
+    }
+
+    let condition = tokens
+        .iter()
+        .map(|t| t.lexeme.clone())
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    Some(Value::String(condition))
+}

package/rust/core/parser/driver/mod.rs

@@ -0,0 +1,6 @@
+pub mod block;
+pub mod cursor;
+pub mod driver_impl;
+pub mod parse_array;
+pub mod parse_map;
+pub mod parser;

package/rust/core/parser/driver/parse_array.rs

@@ -0,0 +1,120 @@
+use crate::core::lexer::token::TokenKind;
+use devalang_types::Value;
+
+pub fn parse_array_value(
+    parser: &mut crate::core::parser::driver::parser::Parser,
+) -> Option<Value> {
+    let logger = devalang_utils::logger::Logger::new();
+    use devalang_utils::logger::LogLevel;
+    if !parser.match_token(TokenKind::LBracket) {
+        return None;
+    }
+
+    let mut arr: Vec<Value> = Vec::new();
+
+    while !parser.check_token(TokenKind::RBracket) && !parser.is_eof() {
+        // Skip formatting tokens
+        while parser.check_token(TokenKind::Newline)
+            || parser.check_token(TokenKind::Whitespace)
+            || parser.check_token(TokenKind::Indent)
+            || parser.check_token(TokenKind::Dedent)
+            || parser.check_token(TokenKind::Comma)
+        {
+            parser.advance();
+        }
+
+        if parser.check_token(TokenKind::RBracket) {
+            break;
+        }
+
+        if let Some(token) = parser.peek_clone() {
+            let value = match token.kind {
+                TokenKind::String => {
+                    parser.advance();
+                    Value::String(token.lexeme.clone())
+                }
+                TokenKind::Number => {
+                    // Support decimals and fraction literals (e.g., 1/4 -> Duration::Beat("1/4"))
+                    let mut number_str = token.lexeme.clone();
+                    parser.advance();
+                    if let Some(dot) = parser.peek_clone() {
+                        if dot.kind == TokenKind::Dot {
+                            if let Some(next) = parser.peek_nth(1).cloned() {
+                                if next.kind == TokenKind::Number {
+                                    parser.advance(); // consume dot
+                                    parser.advance(); // consume next number
+                                    number_str.push('.');
+                                    number_str.push_str(&next.lexeme);
+                                }
+                            }
+                        }
+                    }
+
+                    // Fraction form: number '/' number or identifier
+                    if let Some(slash_tok) = parser.peek_clone() {
+                        if slash_tok.kind == TokenKind::Slash {
+                            // consume '/'
+                            parser.advance();
+                            if let Some(den_tok) = parser.peek_clone() {
+                                match den_tok.kind {
+                                    TokenKind::Number | TokenKind::Identifier => {
+                                        let frac = format!("{}/{}", number_str, den_tok.lexeme);
+                                        parser.advance();
+                                        Value::Duration(devalang_types::Duration::Beat(frac))
+                                    }
+                                    _ => Value::Number(number_str.parse::<f32>().unwrap_or(0.0)),
+                                }
+                            } else {
+                                Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                            }
+                        } else {
+                            Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                        }
+                    } else {
+                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                    }
+                }
+                TokenKind::Identifier => {
+                    parser.advance();
+                    Value::Identifier(token.lexeme.clone())
+                }
+                TokenKind::LBrace => {
+                    // Allow inline maps inside arrays
+                    if let Some(v) = crate::core::parser::driver::parse_map::parse_map_value(parser)
+                    {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                TokenKind::LBracket => {
+                    // Nested arrays
+                    if let Some(v) = parse_array_value(parser) {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                _ => {
+                    parser.advance();
+                    Value::Null
+                }
+            };
+
+            // Only push non-null (retain alignment with permissive parsing)
+            if value != Value::Null {
+                arr.push(value);
+            }
+
+            // Optional trailing comma handled by the skipper at loop start
+        } else {
+            break;
+        }
+    }
+
+    if !parser.match_token(TokenKind::RBracket) {
+        logger.log_message(LogLevel::Error, "Expected ']' at end of array");
+    }
+
+    Some(Value::Array(arr))
+}

package/rust/core/parser/driver/parse_map.rs

@@ -0,0 +1,223 @@
+use crate::core::lexer::token::TokenKind;
+use devalang_types::Value;
+
+pub fn parse_map_value(parser: &mut crate::core::parser::driver::parser::Parser) -> Option<Value> {
+    let logger = devalang_utils::logger::Logger::new();
+    use devalang_utils::logger::LogLevel;
+    if !parser.match_token(TokenKind::LBrace) {
+        return None;
+    }
+
+    let mut map = std::collections::HashMap::new();
+
+    while !parser.check_token(TokenKind::RBrace) && !parser.is_eof() {
+        // Skip separators and formatting before the key
+        while parser.check_token(TokenKind::Newline)
+            || parser.check_token(TokenKind::Whitespace)
+            || parser.check_token(TokenKind::Indent)
+            || parser.check_token(TokenKind::Dedent)
+            || parser.check_token(TokenKind::Comma)
+        {
+            parser.advance();
+        }
+
+        // Check if we are at the closing brace of the map
+        if parser.check_token(TokenKind::RBrace) {
+            break;
+        }
+
+        let key = if let Some(token) = parser.advance() {
+            match token.kind {
+                TokenKind::Whitespace
+                | TokenKind::Indent
+                | TokenKind::Dedent
+                | TokenKind::Newline => {
+                    continue;
+                }
+                _ => token.lexeme.clone(),
+            }
+        } else {
+            break;
+        };
+
+        // Skip newlines and whitespace before colon
+        while parser.check_token(TokenKind::Newline) || parser.check_token(TokenKind::Whitespace) {
+            parser.advance();
+        }
+
+        if !parser.match_token(TokenKind::Colon) {
+            logger.log_message(
+                LogLevel::Error,
+                &format!("Expected ':' after map key '{}'", key),
+            );
+            break;
+        }
+
+        // Skip separators and formatting before value
+        while parser.check_token(TokenKind::Newline)
+            || parser.check_token(TokenKind::Whitespace)
+            || parser.check_token(TokenKind::Indent)
+            || parser.check_token(TokenKind::Dedent)
+            || parser.check_token(TokenKind::Comma)
+        {
+            parser.advance();
+        }
+
+        let value = if let Some(token) = parser.peek_clone() {
+            match token.kind {
+                TokenKind::String => {
+                    parser.advance();
+                    Value::String(token.lexeme.clone())
+                }
+                TokenKind::Number => {
+                    // Handle number, decimal number and optional fraction form (e.g., 1/4)
+                    let mut number_str = token.lexeme.clone();
+                    parser.advance(); // consume the first number
+
+                    // decimal support: number '.' number
+                    if let Some(dot_token) = parser.peek_clone() {
+                        if dot_token.kind == TokenKind::Dot {
+                            parser.advance(); // consume the dot
+
+                            if let Some(decimal_token) = parser.peek_clone() {
+                                if decimal_token.kind == TokenKind::Number {
+                                    parser.advance(); // consume the number after the dot
+                                    number_str.push('.');
+                                    number_str.push_str(&decimal_token.lexeme);
+                                } else {
+                                    logger.log_message(
+                                        LogLevel::Error,
+                                        &format!(
+                                            "Expected number after dot, got {:?}",
+                                            decimal_token
+                                        ),
+                                    );
+                                    return Some(Value::Null);
+                                }
+                            } else {
+                                logger.log_message(
+                                    LogLevel::Error,
+                                    "Expected number after dot, but reached EOF",
+                                );
+                                return Some(Value::Null);
+                            }
+                        }
+                    }
+
+                    // Fraction support: number '/' number  -> Duration::Beat("num/den")
+                    if let Some(slash_tok) = parser.peek_clone() {
+                        if slash_tok.kind == TokenKind::Slash {
+                            // consume '/'
+                            parser.advance();
+                            if let Some(den_tok) = parser.peek_clone() {
+                                match den_tok.kind {
+                                    TokenKind::Number | TokenKind::Identifier => {
+                                        let frac = format!("{}/{}", number_str, den_tok.lexeme);
+                                        parser.advance();
+                                        return Some(Value::Duration(
+                                            devalang_types::Duration::Beat(frac),
+                                        ));
+                                    }
+                                    _ => {
+                                        logger.log_message(
+                                            LogLevel::Error,
+                                            &format!(
+                                                "Expected number or identifier after '/', got {:?}",
+                                                den_tok
+                                            ),
+                                        );
+                                        return Some(Value::Null);
+                                    }
+                                }
+                            } else {
+                                logger.log_message(
+                                    LogLevel::Error,
+                                    "Expected denominator after '/', but reached EOF",
+                                );
+                                return Some(Value::Null);
+                            }
+                        }
+                    }
+
+                    Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
+                }
+
+                TokenKind::Identifier => {
+                    // Support dotted identifiers in map values: alias.param or nested
+                    let current_line = token.line;
+                    let mut parts: Vec<String> = vec![token.lexeme.clone()];
+                    parser.advance();
+                    loop {
+                        let Some(next) = parser.peek_clone() else {
+                            break;
+                        };
+                        if next.line != current_line {
+                            break;
+                        }
+                        if next.kind == TokenKind::Dot {
+                            // Consume '.' and the following identifier/number on same line
+                            parser.advance(); // dot
+                            if let Some(id2) = parser.peek_clone() {
+                                if id2.line == current_line
+                                    && (id2.kind == TokenKind::Identifier
+                                        || id2.kind == TokenKind::Number)
+                                {
+                                    parts.push(id2.lexeme.clone());
+                                    parser.advance(); // consume part
+                                    continue;
+                                }
+                            }
+                            break;
+                        } else {
+                            break;
+                        }
+                    }
+                    Value::Identifier(parts.join("."))
+                }
+                TokenKind::LBracket => {
+                    // Allow arrays as map values
+                    if let Some(v) =
+                        crate::core::parser::driver::parse_array::parse_array_value(parser)
+                    {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                TokenKind::LBrace => {
+                    // Allow inline nested maps as map values
+                    if let Some(v) = parse_map_value(parser) {
+                        v
+                    } else {
+                        Value::Null
+                    }
+                }
+                _ => {
+                    logger.log_message(
+                        LogLevel::Error,
+                        &format!("Unexpected token in map value: {:?}", token),
+                    );
+                    Value::Null
+                }
+            }
+        } else {
+            Value::Null
+        };
+
+        map.insert(key, value);
+
+        // Optionally skip a trailing comma after the value
+        while parser.check_token(TokenKind::Comma)
+            || parser.check_token(TokenKind::Whitespace)
+            || parser.check_token(TokenKind::Newline)
+        {
+            parser.advance();
+        }
+    }
+
+    if !parser.match_token(TokenKind::RBrace) {
+        logger.log_message(LogLevel::Error, "Expected '}' at end of map");
+    }
+
+    Some(Value::Map(map))
+}