mt-lang 0.2.0

data/lib/milk_tea/core/lexer/indentation.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Indentation-to-INDENT/DEDENT token conversion and error recovery.
+    module Indentation
+      private
+
+      def emit_indentation(indent, line_number, line_offset)
+        if (indent % 4) != 0
+          raise LexError.new("indentation must use multiples of 4 spaces", line: line_number, column: indent + 1, path: @path)
+        end
+
+        current_indent = @indent_stack.last
+        if indent == current_indent
+          return
+        end
+
+        if indent > current_indent
+          if indent != current_indent + 4
+            raise LexError.new("indentation may only increase by 4 spaces at a time", line: line_number, column: 1, path: @path)
+          end
+
+          @indent_stack << indent
+          @tokens << token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          return
+        end
+
+        while @indent_stack.last > indent
+          @indent_stack.pop
+          @tokens << token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+        end
+
+        return if @indent_stack.last == indent
+
+        raise LexError.new("indentation does not match any open block", line: line_number, column: 1, path: @path)
+      end
+
+      def recover_indentation(indent, line_number, line_offset)
+        recovered_indent = indent - (indent % 4)
+        current_indent = @indent_stack.last
+
+        if recovered_indent > current_indent + 4
+          recovered_indent = current_indent + 4
+        end
+
+        if recovered_indent > current_indent
+          @indent_stack << recovered_indent
+          @tokens << token(:indent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+          return
+        end
+
+        while @indent_stack.last > recovered_indent
+          @indent_stack.pop
+          @tokens << token(:dedent, "", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
+        end
+
+        return if @indent_stack.last == recovered_indent
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/numbers.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Integer and float literal lexing, including type suffixes and exponents.
+    module Numbers
+      private
+
+      def lex_number(line, index, line_number, line_offset:)
+        start = index
+        type = :integer
+
+        if line[index] == "0" && %w[x X b B].include?(line[index + 1])
+          base_char = line[index + 1]
+          index += 2
+          allowed = base_char.downcase == "x" ? HEX_DIGIT_BYTE : BIN_DIGIT_BYTE
+          while index < line.length && (byte = line.getbyte(index)) && allowed[byte]
+            index += 1
+          end
+        else
+          while index < line.length && (byte = line.getbyte(index)) && NUMERIC_PART_BYTE[byte]
+            index += 1
+          end
+
+          if line[index] == "." && digit?(line[index + 1])
+            type = :float
+            index += 1
+            while index < line.length && (byte = line.getbyte(index)) && NUMERIC_PART_BYTE[byte]
+              index += 1
+            end
+          end
+
+          if exponent_part?(line, index)
+            type = :float
+            index += 1
+            index += 1 if %w[+ -].include?(line[index])
+            while index < line.length && (byte = line.getbyte(index)) && NUMERIC_PART_BYTE[byte]
+              index += 1
+            end
+          end
+        end
+
+        int_suffix = nil
+        if type == :integer
+          int_suffix = scan_integer_suffix_at(line, index)
+          index += int_suffix.length if int_suffix
+        end
+
+        if type == :float && line[index] == "f" && !identifier_part?((line[index + 1] || " ").to_s)
+          index += 1
+        elsif type == :float && line[index] == "d" && !identifier_part?((line[index + 1] || " ").to_s)
+          index += 1
+        end
+
+        lexeme = line[start...index]
+        if type == :integer
+          cleaned = int_suffix ? lexeme.delete_suffix(int_suffix).delete("_") : lexeme.delete("_")
+          literal = parse_integer(cleaned)
+        else
+          normalized = lexeme.delete("_").delete_suffix("f").delete_suffix("d")
+          literal = normalized.to_f
+        end
+        @tokens << token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
+        index
+      end
+
+      def scan_integer_suffix_at(line, index)
+        return nil if index >= line.length
+
+        INTEGER_SUFFIX_STRINGS.find do |suffix|
+          line[index, suffix.length] == suffix && !identifier_part?((line[index + suffix.length] || " ").to_s)
+        end
+      end
+
+      def exponent_part?(line, index)
+        return false unless %w[e E].include?(line[index])
+
+        exponent_index = index + 1
+        exponent_index += 1 if %w[+ -].include?(line[exponent_index])
+        digit?(line[exponent_index])
+      end
+
+      def parse_integer(lexeme)
+        cleaned = lexeme.delete("_")
+        case cleaned
+        when /\A0[xX]/
+          cleaned[2..].to_i(16)
+        when /\A0[bB]/
+          cleaned[2..].to_i(2)
+        else
+          cleaned.to_i(10)
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/recovery.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Error-recovery heuristics: detecting a top-level declaration line to
+    # resynchronize on after an unterminated grouping or heredoc.
+    module Recovery
+      private
+
+      def top_level_resync_line?(line)
+        return false if line.strip.empty?
+        return false if leading_space_count(line).positive?
+
+        first_word = line.strip.split(/\s+/, 3)[0]
+        second_word = line.strip.split(/\s+/, 3)[1]
+
+        case first_word
+        when *TOP_LEVEL_RESYNC_PREFIXES
+          true
+        when "async"
+          second_word == "function"
+        when "public"
+          %w[function struct union enum flags variant type const var opaque interface extending attribute event].include?(second_word)
+        when "foreign", "external"
+          second_word == "function"
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/strings.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # String literal lexing (including adjacent-literal continuation),
+    # character literal lexing, and escape-sequence decoding.
+    module Strings
+      private
+
+      def lex_string(lines, line_index, line, index, line_number, line_offset:, cstring: false)
+        segment = scan_string_segment(line, index, line_number, cstring:, recover: @recovery_errors)
+        consumed_lines = 1
+        value = +segment.value
+        last_line = line
+        last_line_number = line_number
+        last_line_offset = line_offset
+        last_segment_end = segment.next_index
+        last_line_has_newline = lines.fetch(line_index).end_with?("\n")
+        remainder = line[segment.next_index..] || ""
+        line_indent = leading_space_count(line)
+        recovered = segment.recovered
+
+        if remainder.strip.empty? && !recovered
+          scan_line_index = line_index + 1
+          scan_line_number = line_number + 1
+          scan_line_offset = line_offset + lines.fetch(line_index).bytesize
+
+          while scan_line_index < lines.length
+            raw_line = lines.fetch(scan_line_index)
+            scan_line = raw_line.delete_suffix("\n").b
+            segment_start = leading_space_count(scan_line)
+            prefix = cstring ? 'c"' : '"'
+
+            break unless segment_start > line_indent
+            break if scan_line[segment_start].nil?
+            break if scan_line[segment_start] == "#"
+            break unless scan_line[segment_start, prefix.length] == prefix
+
+            continued_segment = scan_string_segment(scan_line, segment_start, scan_line_number, cstring:, recover: @recovery_errors)
+            continued_remainder = scan_line[continued_segment.next_index..] || ""
+            break unless continued_remainder.strip.empty?
+
+            value << continued_segment.value
+            consumed_lines += 1
+            last_line = scan_line
+            last_line_number = scan_line_number
+            last_line_offset = scan_line_offset
+            last_segment_end = continued_segment.next_index
+            last_line_has_newline = raw_line.end_with?("\n")
+            recovered ||= continued_segment.recovered
+
+            scan_line_offset += raw_line.bytesize
+            scan_line_number += 1
+            scan_line_index += 1
+
+            break if continued_segment.recovered
+          end
+        end
+
+        start_offset = line_offset + index
+        if consumed_lines == 1
+          lexeme = line[index...segment.next_index]
+          @tokens << token(cstring ? :cstring : :string, lexeme, value, line_number, index + 1, start_offset:, end_offset: line_offset + segment.next_index)
+          return StringLexResult.new(consumed_lines:, next_index: segment.next_index)
+        end
+
+        end_offset = last_line_offset + last_segment_end
+        lexeme = @source.byteslice(start_offset, end_offset - start_offset)
+        @tokens << token(cstring ? :cstring : :string, lexeme, value, line_number, index + 1, start_offset:, end_offset:)
+        emit_line_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
+        StringLexResult.new(consumed_lines:, next_index: last_segment_end)
+      end
+
+      def scan_string_segment(line, index, line_number, cstring: false, recover: false)
+        start = index
+        index += cstring ? 2 : 1
+        value = +""
+
+        while index < line.length
+          char = line[index]
+          if char == '"'
+            return StringSegment.new(next_index: index + 1, value:)
+          end
+
+          if char == "\\"
+            escape = line[index + 1]
+            if escape.nil?
+              if recover
+                @recovery_errors << LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path) if @recovery_errors
+                return StringSegment.new(next_index: line.length, value:, recovered: true)
+              end
+
+              raise LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path)
+            end
+
+            value << decode_escape(escape)
+            index += 2
+            next
+          end
+
+          value << char
+          index += 1
+        end
+
+        if recover
+          @recovery_errors << LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path) if @recovery_errors
+          return StringSegment.new(next_index: line.length, value:, recovered: true)
+        end
+
+        raise LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path)
+      end
+
+      def lex_char_literal(line, index, line_number, line_offset:)
+        start = index
+        index += 1
+        if index >= line.length
+          raise LexError.new("unterminated character literal", line: line_number, column: start + 1, path: @path)
+        end
+
+        char = line[index]
+        if char == "\\"
+          index += 1
+          if index >= line.length
+            raise LexError.new("unterminated escape in character literal", line: line_number, column: start + 1, path: @path)
+          end
+          escape_char = line[index]
+          if escape_char == "x"
+            hex = line[index + 1, 2]
+            unless hex&.match?(/\A[0-9a-fA-F]{2}\z/)
+              raise LexError.new("invalid hex escape in character literal", line: line_number, column: index + 1, path: @path)
+            end
+            value = hex.to_i(16)
+            index += 3
+          else
+            value = decode_escape(escape_char).ord
+            index += 1
+          end
+        else
+          value = char.ord
+          index += 1
+        end
+
+        if index >= line.length || line[index] != "'"
+          raise LexError.new("expected closing ' in character literal", line: line_number, column: index + 1, path: @path)
+        end
+        index += 1
+
+        lexeme = line[start...index]
+        @tokens << token(:char_literal, lexeme, value, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
+        index
+      end
+
+      def decode_escape(char)
+        case char
+        when "n" then "\n"
+        when "r" then "\r"
+        when "t" then "\t"
+        when "0" then "\0"
+        when '"' then '"'
+        when "'" then "'"
+        when "\\" then "\\"
+        else char
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/symbols.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Operator / punctuation lexing and grouping-delimiter depth tracking.
+    module Symbols
+      private
+
+      def lex_symbol(line, index, line_number, line_offset:)
+        start = index
+        lexeme = line[index, 3]
+        if lexeme && THREE_CHAR_TOKENS.key?(lexeme)
+          type = THREE_CHAR_TOKENS.fetch(lexeme)
+          @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 3)
+          return index + 3
+        end
+
+        lexeme = line[index, 2]
+        if lexeme && TWO_CHAR_TOKENS.key?(lexeme)
+          type = TWO_CHAR_TOKENS.fetch(lexeme)
+          @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 2)
+          adjust_grouping_depth(type, line_number, start + 1)
+          return index + 2
+        end
+
+        lexeme = line[index]
+        type = ONE_CHAR_TOKENS[lexeme]
+        unless type
+          if @recovery_errors && lexeme == "}"
+            @recovery_errors << LexError.new("unexpected closing delimiter", line: line_number, column: start + 1, path: @path)
+            return index + 1
+          end
+
+          if @recovery_errors
+            @recovery_errors << LexError.new("unexpected character #{lexeme.inspect}", line: line_number, column: start + 1, path: @path)
+            return index + 1
+          end
+
+          raise LexError.new("unexpected character #{lexeme.inspect}", line: line_number, column: start + 1, path: @path)
+        end
+
+        @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
+        adjust_grouping_depth(type, line_number, start + 1)
+        index + 1
+      end
+
+      def adjust_grouping_depth(type, line_number, column)
+        case type
+        when :lparen, :lbracket
+          if @grouping_depth.zero?
+            @grouping_start_line = line_number
+            @grouping_start_column = column
+          end
+          @grouping_depth += 1
+        when :rparen, :rbracket
+          @grouping_depth -= 1
+          if @grouping_depth.negative?
+            error = LexError.new("unexpected closing delimiter", line: line_number, column: column, path: @path)
+            if @recovery_errors
+              @recovery_errors << error
+              @grouping_depth = 0
+              return
+            end
+
+            raise error
+          end
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/trivia.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Trivia collection (comments, blank lines, whitespace) used only when
+    # lexing in :with_trivia mode for the concrete syntax tree / formatter.
+    module Trivia
+      private
+
+      def with_trivia?
+        @mode == :with_trivia
+      end
+
+      def register_detached_line_trivia(kind, line, line_number, line_offset, has_newline:)
+        return unless with_trivia?
+
+        text = has_newline ? (line + "\n") : line
+        trivia = TriviaToken.new(
+          kind:,
+          text:,
+          line: line_number,
+          column: 1,
+          start_offset: line_offset,
+          end_offset: line_offset + text.bytesize,
+        )
+        push_pending_leading_trivia(trivia)
+      end
+
+      def push_pending_leading_trivia(trivia)
+        return unless with_trivia?
+
+        @trivia << trivia
+        @pending_leading_trivia << trivia
+      end
+
+      def append_trailing_or_pending(trivia)
+        return unless with_trivia?
+
+        @trivia << trivia
+        if @tokens.empty?
+          @pending_leading_trivia << trivia
+        else
+          trailing = @pending_leading_trivia
+          @pending_leading_trivia = []
+          token = @tokens[-1]
+          trailing.each { |entry| token = token.with_appended_trailing_trivia(entry) }
+          token = token.with_appended_trailing_trivia(trivia)
+          @tokens[-1] = token
+        end
+      end
+    end
+  end
+end