@nova-lang/cli 0.1.0

package/src/interpreter.zig

@@ -0,0 +1,288 @@
+const std = @import("std");
+const types = @import("types.zig");
+const Node = types.Node;
+
+pub const Env = struct {
+    allocator: std.mem.Allocator,
+    vars: std.StringHashMap([]const u8),
+    macros: std.StringHashMap(types.MacroDef),
+    parent: ?*Env,
+
+    pub fn init(allocator: std.mem.Allocator) Env {
+        return .{
+            .allocator = allocator,
+            .vars = std.StringHashMap([]const u8).init(allocator),
+            .macros = std.StringHashMap(types.MacroDef).init(allocator),
+            .parent = null,
+        };
+    }
+
+    pub fn deinit(self: *Env) void {
+        self.vars.deinit();
+        self.macros.deinit();
+    }
+
+    pub fn get(self: *Env, name: []const u8) ?[]const u8 {
+        if (self.vars.get(name)) |v| return v;
+        if (self.parent) |p| return p.get(name);
+        return null;
+    }
+
+    pub fn set(self: *Env, name: []const u8, value: []const u8) void {
+        self.vars.put(name, value) catch {};
+    }
+
+    pub fn defMacro(self: *Env, name: []const u8, macro: types.MacroDef) void {
+        if (self.parent) |p| {
+            p.defMacro(name, macro);
+        } else {
+            self.macros.put(name, macro) catch {};
+        }
+    }
+
+    pub fn lookupMacro(self: *Env, name: []const u8) ?types.MacroDef {
+        if (self.macros.get(name)) |m| return m;
+        if (self.parent) |p| return p.lookupMacro(name);
+        return null;
+    }
+
+    pub fn clone(self: *Env) !Env {
+        var e = Env.init(self.allocator);
+        e.parent = self;
+        return e;
+    }
+};
+
+pub const Interpreter = struct {
+    allocator: std.mem.Allocator,
+    global_env: Env,
+
+    pub fn init(allocator: std.mem.Allocator) Interpreter {
+        return .{
+            .allocator = allocator,
+            .global_env = Env.init(allocator),
+        };
+    }
+
+    pub fn deinit(self: *Interpreter) void {
+        self.global_env.deinit();
+    }
+
+    pub fn evalDocument(self: *Interpreter, doc: *Node) !Node {
+        const doc_data = &doc.data.document;
+        var new_children = std.ArrayList(Node).init(self.allocator);
+        for (doc_data.children.items) |child| {
+            const result = try self.evalNode(child, &self.global_env);
+            if (result) |n| try new_children.append(n);
+        }
+        doc_data.children = new_children;
+        return doc.*;
+    }
+
+    fn evalNode(self: *Interpreter, node: Node, env: *Env) !?Node {
+        switch (node.tag) {
+            .document => return self.evalDocument(@constCast(&node)),
+            .block => return self.evalBlock(&node.data.block, env),
+            .inline_block => return self.evalInline(&node.data.inline_block, env),
+            .text => return self.evalText(&node.data.text, env),
+            .interpolation => return self.evalInterpolation(&node.data.interpolation, env),
+            .math_inline, .math_display => return node,
+            .list_block => return self.evalList(&node.data.list_block, env),
+            .table_block => return node,
+            .macro_def => {
+                env.defMacro(node.data.macro_def.name, node.data.macro_def);
+                return null;
+            },
+            .schema_def, .service_def => return node,
+            .if_block => return self.evalIf(&node.data.if_block, env),
+            .for_block => return self.evalFor(&node.data.for_block, env),
+            .use_block => {
+                std.debug.print("Warning: @use \"{s}\" is not yet supported (module system not implemented)\n", .{node.data.use_block.path});
+                return null;
+            },
+            .meta_block => return node,
+            else => return node,
+        }
+    }
+
+    fn evalBlock(self: *Interpreter, block: *types.Block, env: *Env) !?Node {
+        if (block.name.len == 0) {
+            block.children = try self.evalChildren(block.children, env);
+            return Node.init(.block, .{ .block = block.* });
+        }
+        const macro = env.lookupMacro(block.name);
+        if (macro) |m| {
+            return self.expandMacro(&m, block, env);
+        }
+        block.children = try self.evalChildren(block.children, env);
+        block.inline_content = try self.evalInlineList(block.inline_content, env);
+        return Node.init(.block, .{ .block = block.* });
+    }
+
+    fn evalInline(self: *Interpreter, inl: *types.InlineBlock, env: *Env) !Node {
+        inl.content = try self.evalInlineList(inl.content, env);
+        return Node.init(.inline_block, .{ .inline_block = inl.* });
+    }
+
+    fn evalText(_: *Interpreter, _text: *types.Text, _: *Env) !Node {
+        return Node.init(.text, .{ .text = _text.* });
+    }
+
+    fn evalInterpolation(_: *Interpreter, interp: *types.Interpolation, _: *Env) !Node {
+        return Node.init(.text, .{ .text = .{ .value = interp.expression } });
+    }
+
+    fn evalChildren(self: *Interpreter, children: std.ArrayList(Node), env: *Env) !std.ArrayList(Node) {
+        var result = std.ArrayList(Node).init(self.allocator);
+        for (children.items) |child| {
+            const evaled = try self.evalNode(child, env);
+            if (evaled) |n| try result.append(n);
+        }
+        return result;
+    }
+
+    fn evalInlineList(self: *Interpreter, nodes: std.ArrayList(Node), env: *Env) !std.ArrayList(Node) {
+        var result = std.ArrayList(Node).init(self.allocator);
+        for (nodes.items) |node| {
+            const evaled = try self.evalNode(node, env);
+            if (evaled) |n| try result.append(n);
+        }
+        return result;
+    }
+
+    fn evalList(self: *Interpreter, list: *types.ListBlock, env: *Env) !Node {
+        var new_items = std.ArrayList(Node).init(self.allocator);
+        for (list.items.items) |item| {
+            if (item.tag == .block) {
+                var blk = item.data.block;
+                blk.children = try self.evalChildren(blk.children, env);
+                try new_items.append(Node.init(.block, .{ .block = blk }));
+            } else {
+                try new_items.append(item);
+            }
+        }
+        list.items = new_items;
+        return Node.init(.list_block, .{ .list_block = list.* });
+    }
+
+    fn evalIf(self: *Interpreter, ifblk: *types.IfBlock, env: *Env) !?Node {
+        const cond = try self.evaluateCondition(ifblk.condition, env);
+        const result = if (cond)
+            try self.evalChildren(ifblk.then_body, env)
+        else
+            try self.evalChildren(ifblk.else_body, env);
+        if (result.items.len == 1) return result.items[0];
+        return Node.init(.document, .{
+            .document = .{
+                .children = result,
+                .meta = std.StringHashMap([]const u8).init(self.allocator),
+            },
+        });
+    }
+
+    fn evalFor(self: *Interpreter, forblk: *types.ForBlock, env: *Env) !?Node {
+        var child_env = try env.clone();
+        var result_nodes = std.ArrayList(Node).init(self.allocator);
+
+        const iterable_val = env.get(forblk.iterable) orelse forblk.iterable;
+        var items = std.ArrayList([]const u8).init(self.allocator);
+        defer items.deinit();
+
+        if (std.mem.eql(u8, forblk.iterable, "true") or std.mem.eql(u8, forblk.iterable, "false") or std.mem.eql(u8, forblk.iterable, "null")) {
+        } else if (std.ascii.isDigit(forblk.iterable[0]) or forblk.iterable[0] == '-') {
+            const count = std.fmt.parseInt(i32, forblk.iterable, 10) catch 1;
+            var i: i32 = 0;
+            while (i < count) : (i += 1) {
+                var buf: [16]u8 = undefined;
+                const val = try std.fmt.bufPrint(&buf, "{}", .{i});
+                try items.append(try self.allocator.dupe(u8, val));
+            }
+        } else {
+            var it = std.mem.splitScalar(u8, iterable_val, ',');
+            while (it.next()) |part| {
+                const trimmed = std.mem.trim(u8, part, " ");
+                if (trimmed.len > 0) try items.append(trimmed);
+            }
+        }
+
+        for (items.items) |item_val| {
+            child_env.set(forblk.var_name, item_val);
+            const evaled = try self.evalChildren(forblk.body, &child_env);
+            for (evaled.items) |n| try result_nodes.append(n);
+        }
+
+        return Node.init(.document, .{
+            .document = .{
+                .children = result_nodes,
+                .meta = std.StringHashMap([]const u8).init(self.allocator),
+            },
+        });
+    }
+
+    fn evaluateCondition(self: *Interpreter, condition: []const u8, env: *Env) !bool {
+        if (condition.len == 0) return true;
+        const trimmed = std.mem.trim(u8, condition, " ");
+        if (std.mem.eql(u8, trimmed, "true")) return true;
+        if (std.mem.eql(u8, trimmed, "false")) return false;
+        if (std.mem.eql(u8, trimmed, "null")) return false;
+        if (std.mem.startsWith(u8, trimmed, "@defined(")) {
+            const inner = std.mem.trim(u8, trimmed["@defined(".len..trimmed.len - 1], " ");
+            return env.get(inner) != null;
+        }
+        if (std.mem.startsWith(u8, trimmed, "!")) {
+            const rest = std.mem.trim(u8, trimmed[1..], " ");
+            return !try self.evaluateCondition(rest, env);
+        }
+        if (env.get(trimmed)) |val| {
+            return !(val.len == 0 or std.mem.eql(u8, val, "false") or std.mem.eql(u8, val, "null"));
+        }
+        const num = std.fmt.parseInt(i32, trimmed, 10) catch return condition.len > 0;
+        return num != 0;
+    }
+
+    fn expandMacro(self: *Interpreter, macro: *types.MacroDef, call: *types.Block, env: *Env) !Node {
+        var child_env = try env.clone();
+
+        var arg_idx: usize = 0;
+        for (macro.params.items) |param_name| {
+            var param_val: []const u8 = "";
+            if (arg_idx < call.inline_content.items.len) {
+                const arg_node = call.inline_content.items[arg_idx];
+                if (arg_node.tag == .text) {
+                    param_val = arg_node.data.text.value;
+                }
+            }
+            child_env.set(param_name, param_val);
+            arg_idx += 1;
+        }
+
+        var kw_iter = macro.kwargs.iterator();
+        while (kw_iter.next()) |entry| {
+            const kw_name = entry.key_ptr.*;
+            const call_val = call.attrs.get(kw_name) orelse entry.value_ptr.*;
+            child_env.set(kw_name, call_val);
+        }
+
+        var evaled_body = std.ArrayList(Node).init(self.allocator);
+        for (macro.body.items) |body_node| {
+            if (body_node.tag == .block and std.mem.eql(u8, body_node.data.block.name, macro.block_param_name) and macro.has_block_param) {
+                for (call.children.items) |call_child| {
+                    const evaled = try self.evalNode(call_child, &child_env);
+                    if (evaled) |n| try evaled_body.append(n);
+                }
+            } else {
+                const evaled = try self.evalNode(body_node, &child_env);
+                if (evaled) |n| try evaled_body.append(n);
+            }
+        }
+
+        return Node.init(.block, .{
+            .block = .{
+                .name = macro.name,
+                .attrs = std.StringHashMap([]const u8).init(self.allocator),
+                .inline_content = std.ArrayList(Node).init(self.allocator),
+                .children = evaled_body,
+            },
+        });
+    }
+};

package/src/lexer.zig

@@ -0,0 +1,400 @@
+const std = @import("std");
+const types = @import("types.zig");
+const TokenType = types.TokenType;
+const Token = types.Token;
+
+const ESCAPE_MAP = std.ComptimeStringMap(u8, .{
+    .{ "n", '\n' }, .{ "t", '\t' }, .{ "r", '\r' },
+    .{ "\"", '"' }, .{ "'", '\'' }, .{ "\\", '\\' },
+    .{ "#", '#' }, .{ "$", '$' },
+});
+
+const INDENT_WIDTH: u32 = 2;
+
+pub const Lexer = struct {
+    allocator: std.mem.Allocator,
+    source: []const u8,
+    pos: u32,
+    line: u32,
+    col: u32,
+    tokens: std.ArrayList(Token),
+    indent_stack: std.ArrayList(u32),
+    line_start: bool,
+
+    pub fn init(allocator: std.mem.Allocator, source: []const u8) Lexer {
+        var stack = std.ArrayList(u32).init(allocator);
+        stack.append(0) catch {};
+        return .{
+            .allocator = allocator,
+            .source = source,
+            .pos = 0,
+            .line = 1,
+            .col = 1,
+            .tokens = std.ArrayList(Token).init(allocator),
+            .indent_stack = stack,
+            .line_start = true,
+        };
+    }
+
+    pub fn deinit(self: *Lexer) void {
+        self.tokens.deinit();
+        self.indent_stack.deinit();
+    }
+
+    pub fn getTokens(self: *Lexer) !std.ArrayList(Token) {
+        try self.tokenize();
+        return self.tokens;
+    }
+
+    fn peek(self: *Lexer, offset: u32) u8 {
+        const idx = self.pos + offset;
+        return if (idx < self.source.len) self.source[idx] else 0;
+    }
+
+    fn advance(self: *Lexer) u8 {
+        const ch = self.source[self.pos];
+        self.pos += 1;
+        if (ch == '\n') {
+            self.line += 1;
+            self.col = 1;
+        } else {
+            self.col += 1;
+        }
+        return ch;
+    }
+
+    fn syntaxError(self: *Lexer, msg: []const u8) !void {
+        std.debug.print("SyntaxError: {s} at line {}, col {}\n", .{ msg, self.line, self.col });
+        return error.SyntaxError;
+    }
+
+    fn emit(self: *Lexer, token_type: TokenType, value: []const u8) !void {
+        try self.tokens.append(.{
+            .type = token_type,
+            .value = value,
+            .line = self.line,
+            .col = self.col,
+        });
+    }
+
+    fn skipComment(self: *Lexer) !void {
+        if (self.peek(0) != '/') return;
+        self.pos += 1;
+        if (self.peek(0) == '/') {
+            self.pos += 1;
+            while (self.pos < self.source.len and self.source[self.pos] != '\n') {
+                self.pos += 1;
+            }
+        } else if (self.peek(0) == '*') {
+            self.pos += 1;
+            var depth: u32 = 1;
+            while (depth > 0 and self.pos < self.source.len) {
+                if (self.peek(0) == '/' and self.peek(1) == '*') {
+                    self.pos += 2;
+                    depth += 1;
+                } else if (self.peek(0) == '*' and self.peek(1) == '/') {
+                    self.pos += 2;
+                    depth -= 1;
+                } else {
+                    self.pos += 1;
+                }
+            }
+        } else {
+            self.pos -= 1;
+        }
+    }
+
+    fn readString(self: *Lexer, quote: u8) ![]const u8 {
+        var buf = std.ArrayList(u8).init(self.allocator);
+        defer buf.deinit();
+        while (self.pos < self.source.len) {
+            const ch = self.advance();
+            if (ch == '\\') {
+                if (self.pos < self.source.len) {
+                    const next = self.advance();
+                    const escaped = ESCAPE_MAP.get(&.{next});
+                    try buf.append('\\');
+                    if (escaped) |e| try buf.append(e) else try buf.append(next);
+                }
+            } else if (ch == '#' and self.peek(0) == '{') {
+                return buf.toOwnedSlice();
+            } else if (ch == quote) {
+                return buf.toOwnedSlice();
+            } else {
+                try buf.append(ch);
+            }
+        }
+        return error.UnterminatedString;
+    }
+
+    fn readStringInterp(self: *Lexer, quote: u8) !void {
+        while (true) {
+            var buf = std.ArrayList(u8).init(self.allocator);
+            defer buf.deinit();
+            var has_content = false;
+            while (self.pos < self.source.len) {
+                const ch = self.advance();
+                if (ch == '\\') {
+                    if (self.pos < self.source.len) {
+                        const next = self.advance();
+                        const escaped = ESCAPE_MAP.get(&.{next});
+                        try buf.append('\\');
+                        if (escaped) |e| try buf.append(e) else try buf.append(next);
+                        has_content = true;
+                    }
+                } else if (ch == '#' and self.peek(0) == '{') {
+                    self.pos += 1;
+                    if (has_content or buf.items.len > 0) {
+                        try self.emit(.string, try buf.toOwnedSlice());
+                    }
+                    var depth: u32 = 1;
+                    const start = self.pos;
+                    while (self.pos < self.source.len and depth > 0) {
+                        const c = self.source[self.pos];
+                        self.pos += 1;
+                        if (c == '{') {
+                            depth += 1;
+                        } else if (c == '}') {
+                            depth -= 1;
+                        }
+                    }
+                    try self.emit(.interp_start, self.source[start .. self.pos - 1]);
+                    break;
+                } else if (ch == quote) {
+                    if (has_content or buf.items.len > 0) {
+                        try self.emit(.string, buf.items);
+                    }
+                    return;
+                } else {
+                    try buf.append(ch);
+                    has_content = true;
+                }
+            } else {
+                return;
+            }
+        }
+    }
+
+    fn readNumber(self: *Lexer) ![]const u8 {
+        const start = self.pos;
+        if (self.peek(0) == '+' or self.peek(0) == '-') {
+            self.pos += 1;
+        }
+        if (self.peek(0) == '0' and (self.peek(1) == 'x' or self.peek(1) == 'X')) {
+            self.pos += 2;
+            while (self.pos < self.source.len and std.ascii.isHexDigit(self.source[self.pos])) {
+                self.pos += 1;
+            }
+            return self.source[start..self.pos];
+        }
+        while (self.pos < self.source.len and (std.ascii.isDigit(self.source[self.pos]) or self.source[self.pos] == '.')) {
+            self.pos += 1;
+        }
+        if (self.pos < self.source.len and (self.source[self.pos] == 'e' or self.source[self.pos] == 'E')) {
+            self.pos += 1;
+            if (self.pos < self.source.len and (self.source[self.pos] == '+' or self.source[self.pos] == '-')) {
+                self.pos += 1;
+            }
+            while (self.pos < self.source.len and std.ascii.isDigit(self.source[self.pos])) {
+                self.pos += 1;
+            }
+        }
+        return self.source[start..self.pos];
+    }
+
+    fn readIdent(self: *Lexer) []const u8 {
+        const start = self.pos;
+        while (self.pos < self.source.len) {
+            const ch = self.source[self.pos];
+            if (std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '-') {
+                self.pos += 1;
+            } else {
+                break;
+            }
+        }
+        return self.source[start..self.pos];
+    }
+
+    fn readText(self: *Lexer) []const u8 {
+        const start = self.pos;
+        while (self.pos < self.source.len) {
+            const ch = self.source[self.pos];
+            const next = if (self.pos + 1 < self.source.len) self.source[self.pos + 1] else 0;
+            if (ch == '@' or ch == '{' or ch == '}' or ch == '$' or ch == '\n' or ch == '"' or ch == '\'') break;
+            if (ch == '/' and (next == '/' or next == '*')) break;
+            if (ch == '#' and next == '{') break;
+            self.pos += 1;
+        }
+        return self.source[start..self.pos];
+    }
+
+    fn handleIndent(self: *Lexer) !void {
+        var count: u32 = 0;
+        while (self.pos < self.source.len and self.source[self.pos] == ' ') {
+            self.pos += 1;
+            count += 1;
+        }
+        while (self.pos < self.source.len and self.source[self.pos] == '\t') {
+            self.pos += 1;
+            count += INDENT_WIDTH;
+        }
+        if (self.pos >= self.source.len) return;
+        if (self.source[self.pos] == '\n') return;
+        if (self.source[self.pos] == '#') return;
+
+        const top = self.indent_stack.getLast();
+        if (count > top) {
+            try self.indent_stack.append(count);
+            try self.emit(.indent, "");
+        } else if (count < top) {
+            while (self.indent_stack.items.len > 0 and count < self.indent_stack.getLast()) {
+                _ = self.indent_stack.pop();
+                try self.emit(.dedent, "");
+            }
+            if (self.indent_stack.items.len > 0 and count != self.indent_stack.getLast()) {
+                return self.syntaxError("Inconsistent indentation");
+            }
+        }
+    }
+
+    fn tokenize(self: *Lexer) !void {
+        while (self.pos < self.source.len) {
+            if (self.line_start) {
+                self.line_start = false;
+                try self.handleIndent();
+                if (self.pos >= self.source.len) break;
+            }
+
+            const ch = self.peek(0);
+
+            if (ch == '\n') {
+                self.pos += 1;
+                self.line += 1;
+                self.col = 1;
+                self.line_start = true;
+                try self.emit(.newline, "");
+                continue;
+            }
+
+            if (ch == '/' and (self.peek(1) == '/' or self.peek(1) == '*')) {
+                try self.skipComment();
+                continue;
+            }
+
+            if (ch == ' ' or ch == '\t') {
+                self.pos += 1;
+                self.col += 1;
+                continue;
+            }
+
+            if (ch == '$') {
+                if (self.peek(1) == '$') {
+                    self.pos += 2;
+                    const start = self.pos;
+                    while (self.pos < self.source.len) {
+                        if (self.peek(0) == '$' and self.peek(1) == '$') {
+                            try self.emit(.math_display, self.source[start..self.pos]);
+                            self.pos += 2;
+                            break;
+                        }
+                        self.pos += 1;
+                    } else {
+                        return self.syntaxError("Unterminated display math");
+                    }
+                } else {
+                    self.pos += 1;
+                    const start = self.pos;
+                    while (self.pos < self.source.len and self.source[self.pos] != '$') {
+                        self.pos += 1;
+                    }
+                    if (self.pos >= self.source.len) return self.syntaxError("Unterminated inline math");
+                    try self.emit(.math_inline, self.source[start..self.pos]);
+                    self.pos += 1;
+                }
+                continue;
+            }
+
+            if (ch == '@') {
+                self.pos += 1;
+                try self.emit(.at, "");
+                continue;
+            }
+
+            if (ch == '#') {
+                if (self.peek(1) == '{') {
+                    self.pos += 2;
+                    var depth: u32 = 1;
+                    const start = self.pos;
+                    while (self.pos < self.source.len and depth > 0) {
+                        const c = self.source[self.pos];
+                        self.pos += 1;
+                        if (c == '{') {
+                            depth += 1;
+                        } else if (c == '}') {
+                            depth -= 1;
+                        }
+                    }
+                    try self.emit(.interp_start, self.source[start .. self.pos - 1]);
+                    continue;
+                }
+            }
+
+            if (ch == '"' or ch == '\'') {
+                self.pos += 1;
+                try self.readStringInterp(ch);
+                continue;
+            }
+
+            if (ch == '(') { self.pos += 1; try self.emit(.lparen, ""); continue; }
+            if (ch == ')') { self.pos += 1; try self.emit(.rparen, ""); continue; }
+            if (ch == '{') { self.pos += 1; try self.emit(.lbrace, ""); continue; }
+            if (ch == '}') { self.pos += 1; try self.emit(.rbrace, ""); continue; }
+            if (ch == '[') { self.pos += 1; try self.emit(.lbracket, ""); continue; }
+            if (ch == ']') { self.pos += 1; try self.emit(.rbracket, ""); continue; }
+            if (ch == ':') { self.pos += 1; try self.emit(.colon, ":"); continue; }
+            if (ch == ',') { self.pos += 1; try self.emit(.comma, ""); continue; }
+            if (ch == '-') { self.pos += 1; try self.emit(.dash, ""); continue; }
+            if (ch == '*') { self.pos += 1; try self.emit(.star, ""); continue; }
+            if (ch == '+') { self.pos += 1; try self.emit(.plus, ""); continue; }
+            if (ch == '|') { self.pos += 1; try self.emit(.pipe, ""); continue; }
+            if (ch == '=') { self.pos += 1; try self.emit(.equals, ""); continue; }
+            if (ch == '.') { self.pos += 1; try self.emit(.dot, ""); continue; }
+            if (ch == '?') { self.pos += 1; try self.emit(.question, ""); continue; }
+            if (ch == '<') { self.pos += 1; try self.emit(.lt, ""); continue; }
+            if (ch == '>') { self.pos += 1; try self.emit(.gt, ""); continue; }
+
+            if (std.ascii.isDigit(ch) or ((ch == '+' or ch == '-') and self.peek(1) != 0 and std.ascii.isDigit(self.peek(1)))) {
+                const num = try self.readNumber();
+                try self.emit(.number, num);
+                continue;
+            }
+
+            if (std.ascii.isAlpha(ch) or ch == '_') {
+                const ident = self.readIdent();
+                if (std.mem.eql(u8, ident, "true") or std.mem.eql(u8, ident, "false")) {
+                    try self.emit(.bool, ident);
+                } else if (std.mem.eql(u8, ident, "null")) {
+                    try self.emit(.null, ident);
+                } else {
+                    try self.emit(.ident, ident);
+                }
+                continue;
+            }
+
+            const text = self.readText();
+            if (text.len > 0) {
+                try self.emit(.text, text);
+                continue;
+            }
+
+            if (ch == '\r') { self.pos += 1; continue; }
+            self.pos += 1;
+        }
+
+        while (self.indent_stack.items.len > 1) {
+            _ = self.indent_stack.pop();
+            try self.emit(.dedent, "");
+        }
+        try self.emit(.eof, "");
+    }
+};