@devaloop/devalang 0.0.1-alpha.1 → 0.0.1-alpha.11

package/rust/core/lexer/mod.rs

@@ -1,333 +1,51 @@
-use crate::core::types::token::{ Token, TokenKind };
+pub mod handler;
+pub mod token;
 
-pub fn lex(input: String) -> Vec<Token> {
-    let mut tokens = Vec::new();
+use std::fs;
+use std::path::{Path, PathBuf};
+use crate::core::{
+    lexer::{ handler::driver::handle_content_lexing, token::Token },
+    utils::path::normalize_path,
+};
 
-    let mut line = 1;
-    let mut column = 1;
+pub struct Lexer {}
 
-    let mut indent_stack: Vec<usize> = vec![0];
-    let mut current_indent = 0;
-    let mut at_line_start = true;
-
-    let mut chars = input.chars().peekable();
-
-    while let Some(_) = chars.peek() {
-        if at_line_start {
-            current_indent = 0;
-
-            while let Some(&c) = chars.peek() {
-                if c == ' ' {
-                    current_indent += 1;
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
-
-            let last_indent = *indent_stack.last().unwrap();
-            if current_indent > last_indent {
-                indent_stack.push(current_indent);
-                tokens.push(Token {
-                    kind: TokenKind::Indent,
-                    lexeme: String::new(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-            } else {
-                while current_indent < *indent_stack.last().unwrap() {
-                    indent_stack.pop();
-                    tokens.push(Token {
-                        kind: TokenKind::Dedent,
-                        lexeme: String::new(),
-                        line,
-                        column,
-                        indent: current_indent,
-                    });
-                }
-            }
-
-            at_line_start = false;
-        }
-
-        let Some(ch) = chars.next() else {
-            break;
-        };
-
-        if ch == '\n' {
-            tokens.push(Token {
-                kind: TokenKind::Newline,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
+impl Lexer {
+    pub fn new() -> Self {
+        Lexer {}
+    }
 
-            line += 1;
-            column = 1;
-            at_line_start = true;
+    pub fn lex_from_source(&self, source: &str) -> Result<Vec<Token>, String> {
+        handle_content_lexing(source.to_string())
+    }
 
-            continue;
-        }
+    pub fn lex_tokens(&self, entrypoint: &str) -> Vec<Token> {
+        let path = normalize_path(entrypoint);
+        let resolved_path = Self::resolve_entry_path(&path);
 
-        if ch == ' ' || ch == '\t' {
-            column += if ch == '\t' { 4 } else { 1 };
-            continue;
-        }
+        let file_content =
+            fs::read_to_string(&resolved_path).expect("Failed to read the entrypoint file");
 
-        if ch == '#' {
-            let mut comment = String::new();
-            while let Some(&c) = chars.peek() {
-                if c == '\n' {
-                    break;
-                }
-                comment.push(c);
-                chars.next();
-                column += 1;
-            }
-            tokens.push(Token {
-                kind: TokenKind::Comment(comment.trim().to_string()),
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
+        handle_content_lexing(file_content).expect("Failed to lex the content")
+    }
 
-        if ch == ':' {
-            tokens.push(Token {
-                kind: TokenKind::Colon,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
+    fn resolve_entry_path(path: &str) -> String {
+        let candidate = Path::new(path);
 
-        if ch == '=' {
-            if let Some('=') = chars.peek() {
-                chars.next();
-                tokens.push(Token {
-                    kind: TokenKind::DoubleEquals,
-                    lexeme: ch.to_string(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-                column += 2;
+        if candidate.is_dir() {
+            let index_path = candidate.join("index.deva");
+            if index_path.exists() {
+                return index_path.to_string_lossy().replace("\\", "/");
             } else {
-                tokens.push(Token {
-                    kind: TokenKind::Equals,
-                    lexeme: ch.to_string(),
-                    line,
-                    column,
-                    indent: current_indent,
-                });
-                column += 1;
+                panic!(
+                    "Expected 'index.deva' in directory '{}', but it was not found",
+                    path
+                );
             }
-            continue;
-        }
-
-        if ch == '[' {
-            tokens.push(Token {
-                kind: TokenKind::LBracket,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == ']' {
-            tokens.push(Token {
-                kind: TokenKind::RBracket,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '{' {
-            tokens.push(Token {
-                kind: TokenKind::LBrace,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
+        } else if candidate.is_file() {
+            return path.to_string();
+        } else {
+            panic!("Provided entrypoint '{}' is not a valid file or directory", path);
         }
-
-        if ch == '}' {
-            tokens.push(Token {
-                kind: TokenKind::RBrace,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '"' {
-            let mut string_content = String::new();
-            column += 1; // skip the opening quote
-
-            while let Some(next_ch) = chars.next() {
-                column += 1;
-                if next_ch == '"' {
-                    break; // closing quote reached
-                } else {
-                    string_content.push(next_ch);
-                }
-            }
-
-            tokens.push(Token {
-                kind: TokenKind::String,
-                lexeme: string_content,
-                line,
-                column,
-                indent: current_indent,
-            });
-
-            continue;
-        }
-
-        if ch == '\'' {
-            let mut string_content = String::new();
-            column += 1;
-
-            while let Some(next_ch) = chars.next() {
-                column += 1;
-                if next_ch == '\'' {
-                    break;
-                } else {
-                    string_content.push(next_ch);
-                }
-            }
-
-            tokens.push(Token {
-                kind: TokenKind::String,
-                lexeme: string_content,
-                line,
-                column,
-                indent: current_indent,
-            });
-
-            continue;
-        }
-
-        if ch == '.' {
-            tokens.push(Token {
-                kind: TokenKind::Dot,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch == '@' {
-            tokens.push(Token {
-                kind: TokenKind::At,
-                lexeme: ch.to_string(),
-                line,
-                column,
-                indent: current_indent,
-            });
-            column += 1;
-            continue;
-        }
-
-        if ch.is_ascii_digit() {
-            let mut number = ch.to_string();
-            while let Some(&c) = chars.peek() {
-                if c.is_ascii_digit() {
-                    number.push(c);
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
-
-            // let value = number.parse::<f32>().unwrap();
-            tokens.push(Token {
-                kind: TokenKind::Number,
-                lexeme: number,
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
-
-        if ch.is_ascii_alphabetic() {
-            let mut ident = ch.to_string();
-            while let Some(&c) = chars.peek() {
-                if c.is_ascii_alphanumeric() || c == '_' {
-                    ident.push(c);
-                    chars.next();
-                    column += 1;
-                } else {
-                    break;
-                }
-            }
-
-            let kind = match ident.as_str() {
-                "bank" => TokenKind::Bank,
-                "bpm" => TokenKind::Tempo,
-                "loop" => TokenKind::Loop,
-                _ => TokenKind::Identifier,
-            };
-
-            tokens.push(Token {
-                kind,
-                lexeme: ident,
-                line,
-                column,
-                indent: current_indent,
-            });
-            continue;
-        }
-
-        // Skip unknown char
-        column += 1;
     }
-
-    while indent_stack.len() > 1 {
-        indent_stack.pop();
-        current_indent = *indent_stack.last().unwrap();
-        tokens.push(Token {
-            kind: TokenKind::Dedent,
-            lexeme: String::new(),
-            line,
-            column,
-            indent: current_indent,
-        });
-    }
-
-    tokens.push(Token {
-        kind: TokenKind::EOF,
-        lexeme: String::new(),
-        line: line + 1, // EOF is considered to be on the next line
-        column: 0, // EOF has no column
-        indent: 0, // EOF has no indent
-    });
-
-    tokens
 }

package/rust/core/lexer/token.rs

@@ -0,0 +1,87 @@
+use serde::{ Deserialize, Serialize };
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Token {
+    pub kind: TokenKind,
+    pub lexeme: String,
+    pub indent: usize,
+    pub line: usize,
+    pub column: usize,
+}
+
+impl Token {
+    pub fn is_error(&self) -> bool {
+        match &self.kind {
+            TokenKind::Error(_) => {
+                return true;
+            }
+            _ => {
+                return false;
+            }
+        };
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum TokenKind {
+    // ───── Keywords ─────
+    At,
+    Tempo,
+    Bank,
+    Loop,
+
+    // ───── Instruments ─────
+    Synth,
+
+    // ───── Literals ─────
+    Identifier,
+    Number,
+    String,
+    Boolean,
+    Arrow,
+
+    // ───── Structures ─────
+    Map,
+    Array,
+
+    // ───── Symbols ─────
+    Colon,
+    Comma,
+    Equals,
+    Dot,
+    Slash,
+
+    // ───── Operators ─────
+    DoubleEquals,
+    NotEquals,
+    GreaterEqual,
+    LessEqual,
+    Greater,
+    Less,
+
+    // ───── Brackets ─────
+    LBrace, // {
+    RBrace, // }
+    LBracket, // [
+    RBracket, // ]
+
+    // ───── Quotes ─────
+    Quote, // '
+    DbQuote, // "
+
+    // ───── Formatting ─────
+    Newline,
+    Indent,
+    Dedent,
+    Comment,
+
+    // ───── Conditions ─────
+    If,
+    Else,
+    ElseIf,
+
+    // ───── Special / Internal ─────
+    Unknown,
+    Error(String),
+    EOF,
+}

package/rust/core/mod.rs

@@ -1,6 +1,10 @@
 pub mod lexer;
-pub mod parser;
 pub mod preprocessor;
+pub mod store;
+pub mod parser;
+pub mod shared;
 pub mod debugger;
+pub mod utils;
 pub mod builder;
-pub mod types;
+pub mod error;
+pub mod audio;