mt-lang 0.3.23 → 0.3.25

data/lib/milk_tea/core/c_backend/runtime_helpers.rb

@@ -340,7 +340,7 @@ module MilkTea
         lines
       end
 
-      def emit_text_buffer_helpers
+      def emit_utf8_validation_helpers
         [
           "static bool mt_is_utf8_continuation_byte(unsigned char byte) {",
           "#{INDENT}return (byte & 0xC0u) == 0x80u;",

data/lib/milk_tea/core/c_backend/statements.rb

@@ -19,7 +19,7 @@ module MilkTea
 
       def emit_statement(statement, level, function:, used_labels:, loop_continue_label: nil, loop_break_label: nil, remaining_statements: [])
         indent = INDENT * level
-        aliases = checked_index_aliases_for_statement(statement)
+        aliases = collect_checked_index_aliases_for_statement(statement)
         alias_lines = emit_checked_index_alias_declarations(aliases, indent)
         if statement.respond_to?(:line) && statement.line
           @current_line = statement.line
@@ -56,7 +56,7 @@ module MilkTea
               ["#{indent}#{emit_expression(statement.target)} #{statement.operator} #{emit_expression(statement.value)};"]
             end
           when IR::BlockStmt
-            if block_requires_scope?(statement.body)
+            if statements_require_scope?(statement.body)
               lines = ["#{indent}{"]
               lines.concat(emit_statement_sequence(statement.body, level + 1, function:, used_labels:, loop_continue_label:, loop_break_label:))
               lines << "#{indent}}"
@@ -89,7 +89,7 @@ module MilkTea
             body_continue_label = loop_continue_label_name(statement.body)
             body = body_continue_label ? statement.body[0...-1] : statement.body
             body_break_label = loop_break_label_name(body, remaining_statements)
-            @suppressed_labels << body_break_label if body_break_label && !statements_need_explicit_break_label_after_emission?(body, body_break_label, loop_break_label_active: true)
+            @suppressed_labels << body_break_label if body_break_label && !block_requires_break_label?(body, body_break_label, loop_break_label_active: true)
             if @debug_guards
               @loop_guard_id += 1
               guard_name = "__mt_loop_#{@loop_guard_id}"
@@ -111,12 +111,12 @@ module MilkTea
             body_continue_label = loop_continue_label_name(statement.body)
             body = body_continue_label ? statement.body[0...-1] : statement.body
             body_break_label = loop_break_label_name(body, remaining_statements)
-            @suppressed_labels << body_break_label if body_break_label && !statements_need_explicit_break_label_after_emission?(body, body_break_label, loop_break_label_active: true)
+            @suppressed_labels << body_break_label if body_break_label && !block_requires_break_label?(body, body_break_label, loop_break_label_active: true)
             if @debug_guards
               @loop_guard_id += 1
               guard_name = "__mt_loop_#{@loop_guard_id}"
               lines = ["#{indent}{ uintptr_t #{guard_name} = 0;"]
-              lines << "#{indent}for (#{emit_for_clause_statement(statement.init)}; #{emit_expression(statement.condition)}; #{emit_for_clause_statement(statement.post)}) {"
+              lines << "#{indent}for (#{emit_for_loop_clause(statement.init)}; #{emit_expression(statement.condition)}; #{emit_for_loop_clause(statement.post)}) {"
               guard = "#{INDENT * (level + 1)}if (++#{guard_name} > #{LOOP_GUARD_MAX_ITERATIONS}) mt_fatal(\"loop iteration limit exceeded in #{function.linkage_name}\");"
               lines << guard
               lines.concat(emit_statement_sequence(body, level + 1, function:, used_labels:, loop_continue_label: body_continue_label, loop_break_label: body_break_label))
@@ -124,7 +124,7 @@ module MilkTea
               lines << "#{indent}}"
               lines
             else
-              lines = ["#{indent}for (#{emit_for_clause_statement(statement.init)}; #{emit_expression(statement.condition)}; #{emit_for_clause_statement(statement.post)}) {"]
+              lines = ["#{indent}for (#{emit_for_loop_clause(statement.init)}; #{emit_expression(statement.condition)}; #{emit_for_loop_clause(statement.post)}) {"]
               lines.concat(emit_statement_sequence(body, level + 1, function:, used_labels:, loop_continue_label: body_continue_label, loop_break_label: body_break_label))
               lines << "#{indent}}"
               lines
@@ -149,7 +149,7 @@ module MilkTea
           when IR::IfStmt
             emit_if_statement(statement, level, function:, used_labels:, loop_continue_label:, loop_break_label:)
           when IR::SwitchStmt
-            if switch_emittable_as_if?(statement, loop_break_label:)
+            if can_emit_switch_as_if?(statement, loop_break_label:)
               emit_switch_as_if_statement(statement, level, function:, used_labels:, loop_continue_label:, loop_break_label:)
             else
             lines = ["#{indent}switch (#{emit_expression(statement.expression)}) {"]
@@ -183,8 +183,8 @@ module MilkTea
         alias_lines + line_directive + statement_lines
       end
 
-      def compact_generated_statement_sequence(statements)
-        transformed = statements.map { |statement| transform_compactable_nested_bodies(statement) }
+      def fold_redundant_statement_temps(statements)
+        transformed = statements.map { |statement| fold_nested_redundant_statement_temps(statement) }
         compacted = []
         index = 0
         reachable = true
@@ -226,26 +226,26 @@ module MilkTea
         compacted
       end
 
-      def transform_compactable_nested_bodies(statement)
+      def fold_nested_redundant_statement_temps(statement)
         case statement
         when IR::BlockStmt
-          IR::BlockStmt.new(body: compact_generated_statement_sequence(statement.body))
+          IR::BlockStmt.new(body: fold_redundant_statement_temps(statement.body))
         when IR::WhileStmt
           canonicalize_top_guarded_while(
-            IR::WhileStmt.new(condition: statement.condition, body: compact_generated_statement_sequence(statement.body))
+            IR::WhileStmt.new(condition: statement.condition, body: fold_redundant_statement_temps(statement.body))
           )
         when IR::ForStmt
           IR::ForStmt.new(
             init: statement.init,
             condition: statement.condition,
             post: statement.post,
-            body: compact_generated_statement_sequence(statement.body),
+            body: fold_redundant_statement_temps(statement.body),
           )
         when IR::IfStmt
           IR::IfStmt.new(
             condition: statement.condition,
-            then_body: compact_generated_statement_sequence(statement.then_body),
-            else_body: statement.else_body ? compact_generated_statement_sequence(statement.else_body) : nil,
+            then_body: fold_redundant_statement_temps(statement.then_body),
+            else_body: statement.else_body ? fold_redundant_statement_temps(statement.else_body) : nil,
           )
         when IR::SwitchStmt
           IR::SwitchStmt.new(
@@ -253,9 +253,9 @@ module MilkTea
             exhaustive: statement.exhaustive,
             cases: statement.cases.map do |switch_case|
               if switch_case.is_a?(IR::SwitchDefaultCase)
-                IR::SwitchDefaultCase.new(body: compact_generated_statement_sequence(switch_case.body))
+                IR::SwitchDefaultCase.new(body: fold_redundant_statement_temps(switch_case.body))
               else
-                IR::SwitchCase.new(value: switch_case.value, body: compact_generated_statement_sequence(switch_case.body))
+                IR::SwitchCase.new(value: switch_case.value, body: fold_redundant_statement_temps(switch_case.body))
               end
             end,
           )
@@ -384,7 +384,7 @@ module MilkTea
         type.is_a?(Types::Primitive) && type.name == "bool"
       end
 
-      def checked_index_aliases_for_statement(statement)
+      def collect_checked_index_aliases_for_statement(statement)
         expressions = case statement
         when IR::LocalDecl
           [statement.value]
@@ -524,11 +524,11 @@ module MilkTea
         nil
       end
 
-      def block_requires_scope?(statements)
+      def statements_require_scope?(statements)
         statements.any? { |statement| statement.is_a?(IR::LocalDecl) }
       end
 
-      def emit_for_clause_statement(statement)
+      def emit_for_loop_clause(statement)
         case statement
         when IR::LocalDecl
           raise CBackendError.new("array for-loop init declarations are unsupported", line: 0, column: 0, path: @path) if array_type?(statement.type)

data/lib/milk_tea/core/c_backend/type_declaration.rb

@@ -137,7 +137,7 @@ module MilkTea
       def function_params(function)
         params = []
         params << array_out_param_declaration(function.return_type, ARRAY_OUT_PARAM_NAME) if array_type?(function.return_type)
-        params.concat(emitted_function_params(function).map { |param| c_declaration(param.pointer ? pointer_to(param.type) : param.type, param.linkage_name) })
+        params.concat(call_emission_params(function).map { |param| c_declaration(param.pointer ? pointer_to(param.type) : param.type, param.linkage_name) })
 
         if params.empty?
           "void"
@@ -150,7 +150,7 @@ module MilkTea
         @checked_index_alias_stack.clear
         @checked_index_alias_id = 0
         @suppressed_labels = []
-        body = compact_generated_statement_sequence(function.body)
+        body = fold_redundant_statement_temps(function.body)
         lines = ["#{function_signature(function)} {"]
         used_labels = collect_used_labels(body)
         if body.empty?
@@ -194,7 +194,7 @@ module MilkTea
       def emit_simd_type(type)
         name = simd_type_name(type)
         elem_c = primitive_c_type(type.element_type.name)
-        width = type.lane_count * Layout.primitive_layout(type.element_type).first
+        width = type.lane_count * Types::Layout.primitive_layout(type.element_type).first
         ["typedef #{elem_c} #{name} __attribute__((__vector_size__(#{width})));"]
       end
 
@@ -207,7 +207,7 @@ module MilkTea
         " __attribute__((#{attributes.join(', ')}))"
       end
 
-      def emit_vector_math_types
+      def emit_vector_math_typedefs
         lines = []
         lines << "typedef struct mt_vec2 { float x; float y; } mt_vec2;"
         lines << "typedef struct mt_vec3 { float x; float y; float z; } mt_vec3;"

data/lib/milk_tea/core/c_backend/type_system.rb

@@ -33,7 +33,7 @@ module MilkTea
       end
 
       def c_field_declaration(type, name)
-        return "uint8_t #{name}" if void_storage_field?(type)
+        return "uint8_t #{name}" if void_typed_field?(type)
 
         c_declaration(type, name)
       end
@@ -224,6 +224,7 @@ module MilkTea
       end
 
       def named_type_c_name(type)
+        return nullable_opt_type_name(type) if type.is_a?(Types::Nullable)
         return task_type_name(type) if type.is_a?(Types::Task)
         if type.is_a?(Types::VariantArmPayload)
           return "#{named_type_c_name(type.variant_type)}_#{type.arm_name}"

data/lib/milk_tea/core/c_backend.rb

@@ -127,7 +127,7 @@ module MilkTea
         lines << ""
       end
       if uses_vector_math_types?
-        lines.concat(emit_vector_math_types)
+        lines.concat(emit_vector_math_typedefs)
         lines << ""
       end
       if uses_fatal_helper?
@@ -146,8 +146,8 @@ module MilkTea
         lines.concat(emit_str_equality_helper)
         lines << ""
       end
-      if uses_text_buffer_helpers?
-        lines.concat(emit_text_buffer_helpers)
+      if uses_str_buffer_helpers?
+        lines.concat(emit_utf8_validation_helpers)
         lines << ""
       end
       if uses_async_memory_helpers?

data/lib/milk_tea/core/compile_time.rb

@@ -3,7 +3,19 @@
 require_relative "types/layout"
 
 module MilkTea
-  module ConstEval
+  module CompileTime
+    Layout = ::MilkTea::Types::Layout
+
+    class ReturnValue < StandardError
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+        super("return #{value.inspect}")
+      end
+    end
+
+    class Error < StandardError; end
 
     def self.evaluate(expression, resolve_identifier:, resolve_member_access:, resolve_type_ref: nil, resolve_call: nil)
       Evaluator.new(
@@ -78,7 +90,7 @@ module MilkTea
           evaluate_binary(expression)
         when AST::IfExpr
           condition = evaluate(expression.condition)
-          return unless ConstEval.boolean_value?(condition)
+          return unless CompileTime.boolean_value?(condition)
 
           evaluate(condition ? expression.then_expression : expression.else_expression)
         else
@@ -87,9 +99,24 @@ module MilkTea
       end
 
       def resolve_layout_type(type_ref)
-        return unless @resolve_type_ref
+        result = begin
+          @resolve_type_ref&.call(type_ref)
+        rescue SemanticError
+          nil
+        end
+        return result if result
+
+        return unless type_ref.respond_to?(:name) && type_ref.name.parts.length >= 1
 
-        @resolve_type_ref.call(type_ref)
+        expression = ::MilkTea::AST.build_chain_from_parts(type_ref.name.parts)
+        return unless expression
+
+        value = evaluate(expression)
+        return value if value.is_a?(Types::Struct) || value.is_a?(Types::Primitive) ||
+          value.is_a?(Types::Union) || value.is_a?(Types::Nullable) ||
+          value.is_a?(Types::StructInstance)
+
+        nil
       end
 
       def evaluate_unary(expression)
@@ -103,7 +130,7 @@ module MilkTea
         when "~"
           operand.is_a?(Integer) ? ~operand : nil
         when "not"
-          ConstEval.boolean_value?(operand) ? !operand : nil
+          CompileTime.boolean_value?(operand) ? !operand : nil
         end
       end
 
@@ -112,19 +139,19 @@ module MilkTea
 
         case expression.operator
         when "and"
-          return unless ConstEval.boolean_value?(left)
+          return unless CompileTime.boolean_value?(left)
           return false if left == false
 
           right = evaluate(expression.right)
-          return right if ConstEval.boolean_value?(right)
+          return right if CompileTime.boolean_value?(right)
 
           return nil
         when "or"
-          return unless ConstEval.boolean_value?(left)
+          return unless CompileTime.boolean_value?(left)
           return true if left == true
 
           right = evaluate(expression.right)
-          return right if ConstEval.boolean_value?(right)
+          return right if CompileTime.boolean_value?(right)
 
           return nil
         end
@@ -133,9 +160,9 @@ module MilkTea
 
         case expression.operator
         when "=="
-          ConstEval.equality_result(left, right)
+          CompileTime.equality_result(left, right)
         when "!="
-          result = ConstEval.equality_result(left, right)
+          result = CompileTime.equality_result(left, right)
           result.nil? ? nil : !result
         when "+"
           left.is_a?(Numeric) && right.is_a?(Numeric) ? left + right : nil
@@ -145,12 +172,12 @@ module MilkTea
           left.is_a?(Numeric) && right.is_a?(Numeric) ? left * right : nil
         when "/"
           return unless left.is_a?(Numeric) && right.is_a?(Numeric)
-          raise CompileTime::Error, "division by zero" if zero_numeric?(right)
+          raise Error, "division by zero" if zero_numeric?(right)
 
           left / right
         when "%"
           return unless left.is_a?(Integer) && right.is_a?(Integer)
-          raise CompileTime::Error, "modulo by zero" if right.zero?
+          raise Error, "modulo by zero" if right.zero?
 
           left % right
         when "<<"
@@ -178,23 +205,6 @@ module MilkTea
         (value.is_a?(Integer) && value.zero?) || (value.is_a?(Float) && value.zero?)
       end
     end
-  end
-end
-
-module MilkTea
-  module CompileTime
-    Layout = ::MilkTea::Layout
-
-    class ReturnValue < StandardError
-      attr_reader :value
-
-      def initialize(value)
-        @value = value
-        super("return #{value.inspect}")
-      end
-    end
-
-    class Error < StandardError; end
 
     class BlockContext
       attr_reader :checker
@@ -458,45 +468,6 @@ module MilkTea
       end
     end
 
-    class Evaluator < ConstEval::Evaluator
-
-      def resolve_layout_type(type_ref)
-        result = begin
-          super
-        rescue SemanticError
-          nil
-        end
-        return result if result
-        return result if result
-
-        return unless type_ref.respond_to?(:name) && type_ref.name.parts.length >= 1
-
-        expression = ::MilkTea::AST.build_chain_from_parts(type_ref.name.parts)
-        return unless expression
-
-        value = evaluate(expression)
-        return value if value.is_a?(Types::Struct) || value.is_a?(Types::Primitive) ||
-          value.is_a?(Types::Union) || value.is_a?(Types::Nullable) ||
-          value.is_a?(Types::StructInstance)
-
-        nil
-      end
-
-    end
-
-    def self.evaluate(expression, resolve_identifier:, resolve_member_access:, resolve_type_ref: nil, resolve_call: nil)
-      Evaluator.new(
-        resolve_identifier:,
-        resolve_member_access:,
-        resolve_type_ref:,
-        resolve_call:,
-      ).evaluate(expression)
-    end
-
-    def self.boolean_value?(value)
-      ConstEval.boolean_value?(value)
-    end
-
     module Reflection
       def self.core_field_handle(struct_handle, field_name)
         field_decl = struct_handle.declaration.fields.find { |f| f.name == field_name }
@@ -559,4 +530,4 @@ module MilkTea
       end
     end
   end
-end
+end

data/lib/milk_tea/core/control_flow/builder.rb

@@ -118,7 +118,7 @@ module MilkTea
             )
             @graph.add_edge(cond_id, then_entry, label: :true_branch)
             @graph.add_edge(cond_id, fallback_entry, label: :false_branch)
-            emit_null_test_refinements(cond_id, then_entry, fallback_entry, branch.condition)
+            add_null_test_refinements(cond_id, then_entry, fallback_entry, branch.condition)
             fallback_entry = cond_id
           end
           fallback_entry
@@ -177,7 +177,7 @@ module MilkTea
           body_entry = build_block(stmt.body, cond_id, break_target: next_id, continue_target: cond_id)
           @graph.add_edge(cond_id, body_entry, label: :true_branch)
           @graph.add_edge(cond_id, next_id, label: :false_branch)
-          emit_null_test_refinements(cond_id, body_entry, next_id, stmt.condition)
+          add_null_test_refinements(cond_id, body_entry, next_id, stmt.condition)
           cond_id
         when AST::ForStmt
           writes = Set.new
@@ -582,7 +582,7 @@ module MilkTea
         name
       end
 
-      def emit_null_test_refinements(cond_id, true_succ, false_succ, condition)
+      def add_null_test_refinements(cond_id, true_succ, false_succ, condition)
         pairs = null_check_pairs(condition, positive: true)
         return if pairs.empty?
 

data/lib/milk_tea/core/control_flow/constant_propagation.rb

@@ -105,7 +105,7 @@ module MilkTea
       def self.eval_expr_const(expr, state, binding_resolution:, strict_binding_ids:)
         return NAC if expr.nil?
 
-        value = ConstEval.evaluate(
+        value = CompileTime.evaluate(
           expr,
           resolve_identifier: lambda do |identifier_expression|
             key = identifier_key(identifier_expression, binding_resolution:, strict_binding_ids:)

data/lib/milk_tea/core/control_flow/graph.rb

@@ -80,7 +80,7 @@ module MilkTea
         order
       end
 
-      def read_bindings
+      def all_read_binding_keys
         keys = Set.new
         each_node { |node| keys.merge(node.reads) }
         keys

data/lib/milk_tea/core/control_flow/termination.rb

@@ -11,7 +11,7 @@ module MilkTea
         !reachability.reachable_ids.include?(graph.exit_id)
       end
 
-      def self.loop_body_always_exits?(statements, **builder_options)
+      def self.loop_body_always_terminates?(statements, **builder_options)
         return false if statements.nil? || statements.empty?
 
         graph = Builder.new(**builder_options).build_loop_body(statements)

data/lib/milk_tea/core/cst.rb

@@ -1,6 +1,22 @@
 # frozen_string_literal: true
 
 module MilkTea
+  class CSTBuilder
+    def self.build(source, path: nil)
+      new(source, path:).build
+    end
+
+    def initialize(source, path: nil)
+      @source = source
+      @path = path
+    end
+
+    def build
+      lexed = Lexer.lex_with_trivia(@source, path: @path)
+      CST::SourceFile.new(source: @source, tokens: lexed.tokens, trivia: lexed.trivia)
+    end
+  end
+
   module CST
     SourceFile = Data.define(:source, :tokens, :trivia) do
       def reconstruct

data/lib/milk_tea/core/flow_scope.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class FlowScope
+    def initialize = (@bindings = {})
+    def [](key) = @bindings[key]
+    def []=(key, val); @bindings[key] = val; end
+    def key?(key) = @bindings.key?(key)
+    def empty? = @bindings.empty?
+    def each(&block) = @bindings.each(&block)
+    def each_with_object(init, &block) = @bindings.each_with_object(init, &block)
+  end
+end

data/lib/milk_tea/core/lexer/character_classes.rb

@@ -21,7 +21,7 @@ module MilkTea
           else nil
         end
 
-        @tokens << token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
+        @tokens << build_token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
         index
       end
 
@@ -41,7 +41,7 @@ module MilkTea
         byte ? IDENT_PART_BYTE[byte] : false
       end
 
-      def identifier_start_token(line, start_index)
+      def extract_identifier_at(line, start_index)
         return "" unless start_index < line.length && identifier_start?(line[start_index])
 
         finish = start_index + 1

data/lib/milk_tea/core/lexer/format_strings.rb

@@ -5,7 +5,7 @@ module MilkTea
     # Format string lexing (`f"...#{expr}..."`) and the interpolation
     # scanning / format-spec splitting shared with format heredocs.
     module FormatStrings
-      def parse_format_heredoc_parts(content, start_line:, start_column:)
+      def split_format_heredoc_parts(content, start_line:, start_column:)
         parts = []
         text = +""
         index = 0
@@ -66,7 +66,7 @@ module MilkTea
           if char == '"'
             parts << { kind: :text, value: text } unless text.empty?
             lexeme = line[start..index]
-            @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
+            @tokens << build_token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
             return index + 1
           end
 
@@ -103,7 +103,7 @@ module MilkTea
           @recovery_errors << LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path)
           parts << { kind: :text, value: text } unless text.empty?
           lexeme = line[start..]
-          @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + line.length)
+          @tokens << build_token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + line.length)
           return line.length
         end
 
@@ -117,7 +117,7 @@ module MilkTea
           char = line[index]
 
           if char == '"'
-            index = skip_string_contents(line, index, line_number)
+            index = skip_nested_string_literal(line, index, line_number)
             next
           end
 
@@ -146,7 +146,7 @@ module MilkTea
         raise LexError.new("unterminated format interpolation", line: line_number, column:, path: @path)
       end
 
-      def skip_string_contents(line, index, line_number)
+      def skip_nested_string_literal(line, index, line_number)
         index += 1
 
         while index < line.length

data/lib/milk_tea/core/lexer/heredocs.rb

@@ -89,7 +89,7 @@ module MilkTea
             content_margin = heredoc_content_margin(content_lines)
 
             literal = if format
-              parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
+              split_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
             else
               value
             end
@@ -102,8 +102,8 @@ module MilkTea
               :string
             end
 
-            @tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
-            emit_line_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
+            @tokens << build_token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
+            emit_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
             return resync_line_number ? (resync_line_number - line_number) : (scan_line_index - line_index)
           end
 
@@ -117,7 +117,7 @@ module MilkTea
         content_margin = heredoc_content_margin(content_lines)
 
         literal = if format
-          parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
+          split_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
         else
           value
         end
@@ -130,8 +130,8 @@ module MilkTea
           :string
         end
 
-        @tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
-        emit_line_newline(terminator_line, terminator_line_number, terminator_line_offset, terminator_has_newline)
+        @tokens << build_token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
+        emit_newline(terminator_line, terminator_line_number, terminator_line_offset, terminator_has_newline)
 
         (scan_line_index - line_index) + 1
       end

data/lib/milk_tea/core/lexer/indentation.rb

@@ -20,13 +20,13 @@ module MilkTea
           end
 
           @indent_stack << indent
-          @tokens << token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          @tokens << build_token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
           return
         end
 
         while @indent_stack.last > indent
           @indent_stack.pop
-          @tokens << token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          @tokens << build_token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
         end
 
         return if @indent_stack.last == indent
@@ -44,13 +44,13 @@ module MilkTea
 
         if recovered_indent > current_indent
           @indent_stack << recovered_indent
-          @tokens << token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          @tokens << build_token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
           return
         end
 
         while @indent_stack.last > recovered_indent
           @indent_stack.pop
-          @tokens << token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          @tokens << build_token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
         end
 
         return if @indent_stack.last == recovered_indent

data/lib/milk_tea/core/lexer/numbers.rb

@@ -58,7 +58,7 @@ module MilkTea
           normalized = lexeme.delete("_").delete_suffix("f").delete_suffix("d")
           literal = normalized.to_f
         end
-        @tokens << token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
+        @tokens << build_token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
         index
       end