@devaloop/devalang 0.0.1-alpha.9 → 0.0.1-beta.2

package/rust/core/lexer/handler/driver.rs

@@ -1,22 +1,39 @@
 use crate::core::lexer::{
     handler::{
-        arrow::handle_arrow_lexer, at::handle_at_lexer, brace::{ handle_lbrace_lexer, handle_rbrace_lexer }, colon::handle_colon_lexer, comment::handle_comment_lexer, dot::handle_dot_lexer, identifier::handle_identifier_lexer, indent::handle_indent_lexer, newline::handle_newline_lexer, number::handle_number_lexer, operator::handle_operator_lexer, string::handle_string_lexer
+        arrow::handle_arrow_lexer,
+        at::handle_at_lexer,
+        brace::{handle_lbrace_lexer, handle_rbrace_lexer},
+        colon::handle_colon_lexer,
+        comment::handle_comment_lexer,
+        dot::handle_dot_lexer,
+        identifier::handle_identifier_lexer,
+        indent::handle_indent_lexer,
+        newline::handle_newline_lexer,
+        number::handle_number_lexer,
+        operator::handle_operator_lexer,
+        parenthesis::{handle_lparen_lexer, handle_rparen_lexer},
+        slash::handle_slash_lexer,
+        string::handle_string_lexer,
     },
-    token::{ Token, TokenKind },
+    token::{Token, TokenKind},
 };
 
 fn advance_char<I: Iterator<Item = char>>(
     chars: &mut std::iter::Peekable<I>,
-    line: &mut usize,
-    column: &mut usize
+    _line: &mut usize,
+    column: &mut usize,
 ) -> Option<char> {
-    let c = chars.next()?;
-    if c == '\n' {
-        // Do not increment column on newline here
-    } else {
-        *column += 1;
+    for c in chars.by_ref() {
+        if c == '\r' {
+            continue;
+        } else if c == '\n' {
+            // newline: don't increment column
+        } else {
+            *column += 1;
+        }
+        return Some(c);
     }
-    Some(c)
+    None
 }
 
 pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
@@ -39,7 +56,7 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                 &mut indent_stack,
                 &mut tokens,
                 &mut line,
-                &mut column
+                &mut column,
             );
 
             at_line_start = false;
@@ -58,7 +75,7 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut line,
                     &mut column,
                     &mut at_line_start,
-                    &mut current_indent
+                    &mut current_indent,
                 );
             }
             ' ' | '\t' => {
@@ -72,7 +89,7 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
             ':' => {
@@ -83,10 +100,10 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
-            '=' | '!' | '<' | '>' => {
+            '=' | '!' | '<' | '>' | '+' | '*' => {
                 handle_operator_lexer(
                     ch,
                     &mut chars,
@@ -94,7 +111,18 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
+                );
+            }
+            '/' => {
+                handle_slash_lexer(
+                    ch,
+                    &mut chars,
+                    &mut current_indent,
+                    &mut indent_stack,
+                    &mut tokens,
+                    &mut line,
+                    &mut column,
                 );
             }
             '-' => {
@@ -105,8 +133,22 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
+                // If not parsed as arrow or number, fallback as Minus token
+                if let Some(last) = tokens.last() {
+                    if last.kind == TokenKind::Unknown && last.lexeme == "-" {
+                        // replace last with Minus
+                        let _ = tokens.pop();
+                        tokens.push(Token {
+                            kind: TokenKind::Minus,
+                            lexeme: "-".to_string(),
+                            line,
+                            column,
+                            indent: current_indent,
+                        });
+                    }
+                }
             }
             '{' => {
                 handle_lbrace_lexer(
@@ -116,7 +158,7 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
             '}' => {
@@ -127,7 +169,56 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
+                );
+            }
+            '[' => {
+                tokens.push(Token {
+                    kind: TokenKind::LBracket,
+                    lexeme: "[".to_string(),
+                    line,
+                    column,
+                    indent: current_indent,
+                });
+            }
+            ']' => {
+                tokens.push(Token {
+                    kind: TokenKind::RBracket,
+                    lexeme: "]".to_string(),
+                    line,
+                    column,
+                    indent: current_indent,
+                });
+            }
+            ',' => {
+                tokens.push(Token {
+                    kind: TokenKind::Comma,
+                    lexeme: ",".to_string(),
+                    line,
+                    column,
+                    indent: current_indent,
+                });
+            }
+            '(' => {
+                handle_lparen_lexer(
+                    ch,
+                    &mut chars,
+                    &mut current_indent,
+                    &mut indent_stack,
+                    &mut tokens,
+                    &mut line,
+                    &mut column,
+                );
+            }
+            ')' => {
+                handle_rparen_lexer(
+                    ch,
+                    &mut chars,
+                    &mut current_indent,
+                    &mut indent_stack,
+                    &mut tokens,
+                    &mut line,
+                    &mut column,
                 );
             }
             '.' => {
@@ -138,7 +229,7 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
             '@' => {
@@ -149,40 +240,60 @@ pub fn handle_content_lexing(content: String) -> Result<Vec<Token>, String> {
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
-            '\"' | '\'' => {
-                handle_string_lexer(
+            '$' => {
+                // Treat `$` as start of a special identifier like `$env` or `$math`
+                let mut ident = String::from("$");
+                while let Some(&c) = chars.peek() {
+                    if c.is_ascii_alphanumeric() || c == '_' {
+                        ident.push(c);
+                        chars.next();
+                        column += 1;
+                    } else {
+                        break;
+                    }
+                }
+                tokens.push(Token {
+                    kind: TokenKind::Identifier,
+                    lexeme: ident,
+                    line,
+                    column,
+                    indent: current_indent,
+                });
+            }
+            '0'..='9' => {
+                handle_number_lexer(
                     ch,
                     &mut chars,
                     &mut current_indent,
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
-            c if c.is_ascii_digit() => {
-                handle_number_lexer(
-                    c,
+            'a'..='z' | 'A'..='Z' | '_' => {
+                handle_identifier_lexer(
+                    ch,
                     &mut chars,
                     &mut current_indent,
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
-            c if c.is_ascii_alphabetic() => {
-                handle_identifier_lexer(
-                    c,
+            '"' | '\'' => {
+                handle_string_lexer(
+                    ch,
                     &mut chars,
                     &mut current_indent,
                     &mut indent_stack,
                     &mut tokens,
                     &mut line,
-                    &mut column
+                    &mut column,
                 );
             }
             _ => {

package/rust/core/lexer/handler/identifier.rs

@@ -1,15 +1,15 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_identifier_lexer(
-    char: char,
+    ch: char,
     chars: &mut std::iter::Peekable<std::str::Chars>,
     current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
+    _indent_stack: &mut [usize],
     tokens: &mut Vec<Token>,
     line: &mut usize,
-    column: &mut usize
+    column: &mut usize,
 ) {
-    let mut ident = char.to_string();
+    let mut ident = ch.to_string();
 
     while let Some(&c) = chars.peek() {
         if c.is_ascii_alphanumeric() || c == '_' {
@@ -25,9 +25,15 @@ pub fn handle_identifier_lexer(
         "if" => TokenKind::If,
         "else" => TokenKind::Else,
         "bank" => TokenKind::Bank,
+        "pattern" => TokenKind::Pattern,
         "bpm" => TokenKind::Tempo,
         "loop" => TokenKind::Loop,
+        "for" => TokenKind::Loop,
         "synth" => TokenKind::Synth,
+        "fn" => TokenKind::Function,
+        "as" => TokenKind::As,
+        "on" => TokenKind::On,
+        "emit" => TokenKind::Emit,
         _ => TokenKind::Identifier,
     };
 

package/rust/core/lexer/handler/indent.rs

@@ -1,4 +1,4 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_indent_lexer(
     chars: &mut std::iter::Peekable<std::str::Chars>,
@@ -6,7 +6,7 @@ pub fn handle_indent_lexer(
     indent_stack: &mut Vec<usize>,
     tokens: &mut Vec<Token>,
     line: &mut usize,
-    column: &mut usize
+    column: &mut usize,
 ) {
     *current_indent = 0;
     let mut col = *column;
@@ -16,10 +16,24 @@ pub fn handle_indent_lexer(
             *current_indent += 1;
             chars.next();
             col += 1;
+            tokens.push(Token {
+                kind: TokenKind::Whitespace,
+                lexeme: " ".to_string(),
+                line: *line,
+                column: col,
+                indent: *current_indent,
+            });
         } else if c == '\t' {
             *current_indent += 4;
             chars.next();
             col += 4;
+            tokens.push(Token {
+                kind: TokenKind::Whitespace,
+                lexeme: "\t".to_string(),
+                line: *line,
+                column: col,
+                indent: *current_indent,
+            });
         } else {
             break;
         }
@@ -32,7 +46,7 @@ pub fn handle_indent_lexer(
         indent_stack.push(*current_indent);
         tokens.push(Token {
             kind: TokenKind::Indent,
-            lexeme: String::new(),
+            lexeme: String::from("<INDENT>"),
             line: *line,
             column: *column,
             indent: *current_indent,
@@ -42,7 +56,7 @@ pub fn handle_indent_lexer(
             indent_stack.pop();
             tokens.push(Token {
                 kind: TokenKind::Dedent,
-                lexeme: String::new(),
+                lexeme: String::from("<DEDENT>"),
                 line: *line,
                 column: *column,
                 indent: *current_indent,

package/rust/core/lexer/handler/mod.rs

@@ -1,14 +1,15 @@
-pub mod driver;
-
+pub mod arrow;
 pub mod at;
 pub mod brace;
 pub mod colon;
 pub mod comment;
 pub mod dot;
-pub mod operator;
+pub mod driver;
 pub mod identifier;
+pub mod indent;
 pub mod newline;
 pub mod number;
-pub mod indent;
+pub mod operator;
+pub mod parenthesis;
+pub mod slash;
 pub mod string;
-pub mod arrow;

package/rust/core/lexer/handler/newline.rs

@@ -1,13 +1,13 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_newline_lexer(
     ch: char,
-    chars: &mut std::iter::Peekable<std::str::Chars>,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
     tokens: &mut Vec<Token>,
     line: &mut usize,
     column: &mut usize,
     at_line_start: &mut bool,
-    current_indent: &mut usize
+    current_indent: &mut usize,
 ) {
     tokens.push(Token {
         kind: TokenKind::Newline,

package/rust/core/lexer/handler/number.rs

@@ -1,15 +1,15 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_number_lexer(
-    char: char,
+    ch: char,
     chars: &mut std::iter::Peekable<std::str::Chars>,
     current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
+    _indent_stack: &mut [usize],
     tokens: &mut Vec<Token>,
     line: &mut usize,
-    column: &mut usize
+    column: &mut usize,
 ) {
-    let mut number = char.to_string();
+    let mut number = ch.to_string();
 
     while let Some(&c) = chars.peek() {
         if c.is_ascii_digit() {

package/rust/core/lexer/handler/operator.rs

@@ -1,13 +1,13 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_operator_lexer(
     ch: char,
     chars: &mut std::iter::Peekable<std::str::Chars>,
     current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
+    _indent_stack: &mut [usize],
     tokens: &mut Vec<Token>,
     line: &mut usize,
-    column: &mut usize
+    column: &mut usize,
 ) {
     let next = chars.peek().copied();
 
@@ -16,6 +16,8 @@ pub fn handle_operator_lexer(
         ('!', Some('=')) => (TokenKind::NotEquals, 2),
         ('>', Some('=')) => (TokenKind::GreaterEqual, 2),
         ('<', Some('=')) => (TokenKind::LessEqual, 2),
+        ('+', _) => (TokenKind::Plus, 1),
+        ('*', _) => (TokenKind::Asterisk, 1),
         ('=', _) => (TokenKind::Equals, 1),
         ('>', _) => (TokenKind::Greater, 1),
         ('<', _) => (TokenKind::Less, 1),

package/rust/core/lexer/handler/parenthesis.rs

@@ -0,0 +1,41 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_rparen_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    tokens.push(Token {
+        kind: TokenKind::RParen,
+        lexeme: ch.to_string(),
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+
+    *column += 1;
+}
+
+pub fn handle_lparen_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    tokens.push(Token {
+        kind: TokenKind::LParen,
+        lexeme: ch.to_string(),
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+
+    *column += 1;
+}

package/rust/core/lexer/handler/slash.rs

@@ -0,0 +1,21 @@
+use crate::core::lexer::token::{Token, TokenKind};
+
+pub fn handle_slash_lexer(
+    ch: char,
+    _chars: &mut std::iter::Peekable<std::str::Chars>,
+    current_indent: &mut usize,
+    _indent_stack: &mut [usize],
+    tokens: &mut Vec<Token>,
+    line: &mut usize,
+    column: &mut usize,
+) {
+    let slash = ch.to_string();
+
+    tokens.push(Token {
+        kind: TokenKind::Slash,
+        lexeme: slash,
+        line: *line,
+        column: *column,
+        indent: *current_indent,
+    });
+}

package/rust/core/lexer/handler/string.rs

@@ -1,13 +1,13 @@
-use crate::core::lexer::token::{ Token, TokenKind };
+use crate::core::lexer::token::{Token, TokenKind};
 
 pub fn handle_string_lexer(
     ch: char,
     chars: &mut std::iter::Peekable<std::str::Chars>,
     current_indent: &mut usize,
-    indent_stack: &mut Vec<usize>,
+    _indent_stack: &mut [usize],
     tokens: &mut Vec<Token>,
     line: &mut usize,
-    column: &mut usize
+    column: &mut usize,
 ) {
     let quote_char = ch;
 

package/rust/core/lexer/mod.rs

@@ -1,51 +1,3 @@
+pub mod driver;
 pub mod handler;
 pub mod token;
-
-use std::fs;
-use std::path::{Path, PathBuf};
-use crate::core::{
-    lexer::{ handler::driver::handle_content_lexing, token::Token },
-    utils::path::normalize_path,
-};
-
-pub struct Lexer {}
-
-impl Lexer {
-    pub fn new() -> Self {
-        Lexer {}
-    }
-
-    pub fn lex_from_source(&self, source: &str) -> Result<Vec<Token>, String> {
-        handle_content_lexing(source.to_string())
-    }
-
-    pub fn lex_tokens(&self, entrypoint: &str) -> Vec<Token> {
-        let path = normalize_path(entrypoint);
-        let resolved_path = Self::resolve_entry_path(&path);
-
-        let file_content =
-            fs::read_to_string(&resolved_path).expect("Failed to read the entrypoint file");
-
-        handle_content_lexing(file_content).expect("Failed to lex the content")
-    }
-
-    fn resolve_entry_path(path: &str) -> String {
-        let candidate = Path::new(path);
-
-        if candidate.is_dir() {
-            let index_path = candidate.join("index.deva");
-            if index_path.exists() {
-                return index_path.to_string_lossy().replace("\\", "/");
-            } else {
-                panic!(
-                    "Expected 'index.deva' in directory '{}', but it was not found",
-                    path
-                );
-            }
-        } else if candidate.is_file() {
-            return path.to_string();
-        } else {
-            panic!("Provided entrypoint '{}' is not a valid file or directory", path);
-        }
-    }
-}

package/rust/core/lexer/token.rs

@@ -1,4 +1,4 @@
-use serde::{ Deserialize, Serialize };
+use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Token {
@@ -11,14 +11,7 @@ pub struct Token {
 
 impl Token {
     pub fn is_error(&self) -> bool {
-        match &self.kind {
-            TokenKind::Error(_) => {
-                return true;
-            }
-            _ => {
-                return false;
-            }
-        };
+        matches!(&self.kind, TokenKind::Error(_))
     }
 }
 
@@ -28,7 +21,12 @@ pub enum TokenKind {
     At,
     Tempo,
     Bank,
+    Pattern,
     Loop,
+    Function,
+    As,
+    On,
+    Emit,
 
     // ───── Instruments ─────
     Synth,
@@ -49,6 +47,10 @@ pub enum TokenKind {
     Comma,
     Equals,
     Dot,
+    Slash,
+    Plus,
+    Asterisk,
+    Minus,
 
     // ───── Operators ─────
     DoubleEquals,
@@ -59,13 +61,15 @@ pub enum TokenKind {
     Less,
 
     // ───── Brackets ─────
-    LBrace, // {
-    RBrace, // }
+    LBrace,   // {
+    RBrace,   // }
     LBracket, // [
     RBracket, // ]
+    LParen,   // (
+    RParen,   // )
 
     // ───── Quotes ─────
-    Quote, // '
+    Quote,   // '
     DbQuote, // "
 
     // ───── Formatting ─────
@@ -80,6 +84,7 @@ pub enum TokenKind {
     ElseIf,
 
     // ───── Special / Internal ─────
+    Whitespace,
     Unknown,
     Error(String),
     EOF,

package/rust/core/mod.rs

@@ -1,10 +1,9 @@
-pub mod lexer;
-pub mod preprocessor;
-pub mod store;
-pub mod parser;
-pub mod shared;
-pub mod debugger;
-pub mod utils;
-pub mod builder;
-pub mod error;
-pub mod audio;
+pub mod audio;
+pub mod builder;
+pub mod debugger;
+pub mod error;
+pub mod lexer;
+pub mod parser;
+pub mod plugin;
+pub mod preprocessor;
+pub mod store;