mt-lang 0.2.0

data/lib/milk_tea/core/ir.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module MilkTea
+  module IR
+    Program = Data.define(:module_name, :includes, :constants, :globals, :opaques, :structs, :unions, :enums, :variants, :static_asserts, :functions, :source_path) do
+      def initialize(module_name:, includes:, constants:, globals:, opaques:, structs:, unions:, enums:, variants:, static_asserts:, functions:, source_path: nil) = super
+    end
+    Include = Data.define(:header)
+    Constant = Data.define(:name, :linkage_name, :type, :value)
+    Global = Data.define(:name, :linkage_name, :type, :value)
+    OpaqueDecl = Data.define(:name, :linkage_name, :forward_declarable, :source_module) do
+      def initialize(name:, linkage_name:, forward_declarable:, source_module: nil) = super
+    end
+    StructDecl = Data.define(:name, :linkage_name, :fields, :packed, :alignment, :source_module) do
+      def initialize(name:, linkage_name:, fields:, packed:, alignment:, source_module: nil) = super
+    end
+    UnionDecl = Data.define(:name, :linkage_name, :fields, :source_module) do
+      def initialize(name:, linkage_name:, fields:, source_module: nil) = super
+    end
+    EnumDecl = Data.define(:name, :linkage_name, :backing_type, :members, :flags)
+    EnumMember = Data.define(:name, :linkage_name, :value)
+    Field = Data.define(:name, :type)
+    Function = Data.define(:name, :linkage_name, :params, :return_type, :body, :entry_point, :method_receiver_param) do
+      def initialize(name:, linkage_name:, params:, return_type:, body:, entry_point:, method_receiver_param: false) = super
+    end
+    Param = Data.define(:name, :linkage_name, :type, :pointer)
+
+    LocalDecl = Data.define(:name, :linkage_name, :type, :value, :line, :source_path) do
+      def initialize(name:, linkage_name:, type:, value:, line: nil, source_path: nil) = super
+    end
+    Assignment = Data.define(:target, :operator, :value)
+    BlockStmt = Data.define(:body)
+    IfStmt = Data.define(:condition, :then_body, :else_body)
+    SwitchCase = Data.define(:value, :body)
+    SwitchDefaultCase = Data.define(:body)
+    SwitchStmt = Data.define(:expression, :cases, :exhaustive) do
+      def initialize(expression:, cases:, exhaustive: false) = super
+    end
+    WhileStmt = Data.define(:condition, :body)
+    ForStmt = Data.define(:init, :condition, :post, :body)
+    BreakStmt = Data.define()
+    ContinueStmt = Data.define()
+    GotoStmt = Data.define(:label)
+    LabelStmt = Data.define(:name)
+    StaticAssert = Data.define(:condition, :message)
+    ReturnStmt = Data.define(:value, :line, :source_path) do
+      def initialize(value:, line: nil, source_path: nil) = super
+    end
+    ExpressionStmt = Data.define(:expression, :line, :source_path) do
+      def initialize(expression:, line: nil, source_path: nil) = super
+    end
+
+    Name = Data.define(:name, :type, :pointer)
+    Member = Data.define(:receiver, :member, :type)
+    Index = Data.define(:receiver, :index, :type)
+    CheckedIndex = Data.define(:receiver, :index, :receiver_type, :type)
+    CheckedSpanIndex = Data.define(:receiver, :index, :receiver_type, :type)
+    NullableIndex = Data.define(:receiver, :index, :receiver_type, :type)
+    NullableSpanIndex = Data.define(:receiver, :index, :receiver_type, :type)
+    Call = Data.define(:callee, :arguments, :type)
+    Unary = Data.define(:operator, :operand, :type)
+    Binary = Data.define(:operator, :left, :right, :type)
+    Conditional = Data.define(:condition, :then_expression, :else_expression, :type)
+    ReinterpretExpr = Data.define(:target_type, :source_type, :expression, :type)
+    SizeofExpr = Data.define(:target_type, :type)
+    AlignofExpr = Data.define(:target_type, :type)
+    OffsetofExpr = Data.define(:target_type, :field, :type)
+    IntegerLiteral = Data.define(:value, :type)
+    FloatLiteral = Data.define(:value, :type)
+    StringLiteral = Data.define(:value, :type, :cstring)
+    BooleanLiteral = Data.define(:value, :type)
+    NullLiteral = Data.define(:type)
+    ZeroInit = Data.define(:type)
+    AddressOf = Data.define(:expression, :type)
+    Cast = Data.define(:target_type, :expression, :type)
+    AggregateLiteral = Data.define(:type, :fields)
+    AggregateField = Data.define(:name, :value)
+    ArrayLiteral = Data.define(:type, :elements)
+    VariantDecl = Data.define(:name, :linkage_name, :arms, :source_module) do
+      def initialize(name:, linkage_name:, arms:, source_module: nil) = super
+    end
+    VariantArm = Data.define(:name, :linkage_name, :fields)
+    VariantLiteral = Data.define(:type, :arm_name, :fields)
+  end
+end

data/lib/milk_tea/core/keywords.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module MilkTea
+  KEYWORDS = {
+    "align_of" => :align_of,
+    "and" => :and,
+    "as" => :as,
+    "async" => :async,
+    "attribute" => :attribute,
+    "attribute_arg" => :attribute_arg,
+    "attribute_of" => :attribute_of,
+    "attributes_of" => :attributes_of,
+    "await" => :await,
+    "break" => :break,
+    "const" => :const,
+    "compiler_flag" => :compiler_flag,
+    "gather" => :gather,
+    "continue" => :continue,
+    "function" => :function,
+    "has_attribute" => :has_attribute,
+    "defer" => :defer,
+    "detach" => :detach,
+    "dyn" => :dyn,
+    "editable" => :editable,
+    "enum" => :enum,
+    "else" => :else,
+    "emit" => :emit,
+    "event" => :event,
+    "external" => :external,
+    "false" => :false,
+    "callable_of" => :callable_of,
+    "fields_of" => :fields_of,
+    "field_of" => :field_of,
+    "flags" => :flags,
+    "fn" => :fn,
+    "for" => :for,
+    "foreign" => :foreign,
+    "if" => :if,
+    "implements" => :implements,
+    "include" => :include,
+    "in" => :in,
+    "inline" => :inline,
+    "inout" => :inout,
+    "import" => :import,
+    "interface" => :interface,
+    "is" => :is,
+    "let" => :let,
+    "link" => :link,
+    "match" => :match,
+    "members_of" => :members_of,
+    "extending" => :extending,
+    "module" => :module,
+    "not" => :not,
+    "null" => :null,
+    "offset_of" => :offset_of,
+    "opaque" => :opaque,
+    "consuming" => :consuming,
+    "or" => :or,
+    "out" => :out,
+    "parallel" => :parallel,
+    "pass" => :pass,
+    "proc" => :proc,
+    "public" => :public,
+    "return" => :return,
+    "size_of" => :size_of,
+    "static" => :static,
+    "static_assert" => :static_assert,
+    "struct" => :struct,
+    "type" => :type,
+    "unsafe" => :unsafe,
+    "true" => :true,
+    "union" => :union,
+    "var" => :var,
+    "variant" => :variant,
+    "when" => :when,
+    "while" => :while,
+  }.freeze
+
+  BUILTIN_PRIMITIVE_NAMES = %w[
+    bool byte ubyte char short ushort int uint long ulong ptr_int ptr_uint float double void str cstr
+    vec2 vec3 vec4 ivec2 ivec3 ivec4 mat3 mat4 quat
+  ].freeze
+
+  RESERVED_VALUE_TYPE_NAMES = (BUILTIN_PRIMITIVE_NAMES + %w[
+    Option Result
+  ]).freeze
+
+  RESERVED_IMPORT_ALIAS_NAMES = %w[
+    Option Result
+  ].freeze
+
+  BUILTIN_TYPE_NAMES = (BUILTIN_PRIMITIVE_NAMES + %w[
+    ptr const_ptr own ref span array str_buffer atomic Task Option Result SoA
+    struct_handle field_handle callable_handle attribute_handle member_handle type
+    EventError Subscription
+  ]).freeze
+end

data/lib/milk_tea/core/lexer/character_classes.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Identifier lexing plus the raw-byte character-classification helpers.
+    module CharacterClasses
+      private
+
+      def lex_identifier(line, index, line_number, line_offset:)
+        start = index
+        length = line.length
+        index += 1
+        while index < length && (byte = line.getbyte(index)) && IDENT_PART_BYTE[byte]
+          index += 1
+        end
+
+        lexeme = line[start...index]
+        type = Token::KEYWORDS.fetch(lexeme, :identifier)
+        literal = case type
+                  when :true then true
+                  when :false then false
+                  when :null then nil
+                  else nil
+                  end
+
+        @tokens << token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
+        index
+      end
+
+      def leading_space_count(line)
+        index = 0
+        index += 1 while line.getbyte(index) == SPACE_BYTE
+        index
+      end
+
+      def identifier_start?(char)
+        byte = char && char.getbyte(0)
+        byte ? IDENT_START_BYTE[byte] : false
+      end
+
+      def identifier_part?(char)
+        byte = char && char.getbyte(0)
+        byte ? IDENT_PART_BYTE[byte] : false
+      end
+
+      def identifier_start_token(line, start_index)
+        return "" unless start_index < line.length && identifier_start?(line[start_index])
+
+        finish = start_index + 1
+        finish += 1 while finish < line.length && identifier_part?(line[finish])
+        line[start_index...finish]
+      end
+
+      def digit?(char)
+        byte = char && char.getbyte(0)
+        byte ? DIGIT_BYTE[byte] : false
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/format_strings.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Format string lexing (`f"...#{expr}..."`) and the interpolation
+    # scanning / format-spec splitting shared with format heredocs.
+    module FormatStrings
+      private
+
+      def parse_format_heredoc_parts(content, start_line:, start_column:)
+        parts = []
+        text = +""
+        index = 0
+        line = start_line
+        column = start_column
+        base_column = start_column
+
+        while index < content.length
+          char = content[index]
+          if char == "#" && content[index + 1] == "{"
+            parts << { kind: :text, value: text } unless text.empty?
+            text = +""
+
+            expr_start = index + 2
+            expr_line = line
+            expr_column = column + 2
+            expr_end = scan_format_interpolation_end(content, expr_start, expr_line, expr_column, recover: @recovery_errors)
+            raw_source = content[expr_start...expr_end]
+            if raw_source.strip.empty?
+              raise LexError.new("empty format interpolation", line: line, column:, path: @path)
+            end
+
+            source, format_spec = split_format_interpolation_source(raw_source)
+            parts << { kind: :expr, source:, format_spec:, line: expr_line, column: expr_column }
+
+            while index <= expr_end
+              line, column = advance_heredoc_position(char: content[index], line:, column:, base_column:)
+              index += 1
+            end
+            next
+          end
+
+          text << char
+          line, column = advance_heredoc_position(char:, line:, column:, base_column:)
+          index += 1
+        end
+
+        parts << { kind: :text, value: text } unless text.empty?
+        parts
+      end
+
+      def advance_heredoc_position(char:, line:, column:, base_column:)
+        if char == "\n"
+          [line + 1, base_column]
+        else
+          [line, column + 1]
+        end
+      end
+
+      def lex_format_string(line, index, line_number, line_offset:)
+        start = index
+        index += 2
+        text = +""
+        parts = []
+
+        while index < line.length
+          char = line[index]
+          if char == '"'
+            parts << { kind: :text, value: text } unless text.empty?
+            lexeme = line[start..index]
+            @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
+            return index + 1
+          end
+
+          if char == "#" && line[index + 1] == "{"
+            parts << { kind: :text, value: text } unless text.empty?
+            text = +""
+            expr_start = index + 2
+            expr_end = scan_format_interpolation_end(line, expr_start, line_number, start + 1, recover: @recovery_errors)
+            raw_source = line[expr_start...expr_end]
+            if raw_source.strip.empty?
+              raise LexError.new("empty format interpolation", line: line_number, column: index + 1, path: @path)
+            end
+
+            source, format_spec = split_format_interpolation_source(raw_source)
+            parts << { kind: :expr, source:, format_spec:, line: line_number, column: expr_start + 1 }
+            index = expr_end + 1
+            next
+          end
+
+          if char == "\\"
+            next_char = line[index + 1]
+            raise LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path) unless next_char
+
+            text << decode_escape(next_char)
+            index += 2
+            next
+          end
+
+          text << char
+          index += 1
+        end
+
+        if @recovery_errors
+          @recovery_errors << LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path)
+          parts << { kind: :text, value: text } unless text.empty?
+          lexeme = line[start..]
+          @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + line.length)
+          return line.length
+        end
+
+        raise LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path)
+      end
+
+      def scan_format_interpolation_end(line, index, line_number, column, recover: false)
+        depth = 1
+
+        while index < line.length
+          char = line[index]
+
+          if char == '"'
+            index = skip_string_contents(line, index, line_number)
+            next
+          end
+
+          if char == "{"
+            depth += 1
+            index += 1
+            next
+          end
+
+          if char == "}"
+            depth -= 1
+            return index if depth == 0
+
+            index += 1
+            next
+          end
+
+          index += 1
+        end
+
+        if recover
+          @recovery_errors << LexError.new("unterminated format interpolation", line: line_number, column:, path: @path) if @recovery_errors
+          return line.length
+        end
+
+        raise LexError.new("unterminated format interpolation", line: line_number, column:, path: @path)
+      end
+
+      def skip_string_contents(line, index, line_number)
+        index += 1
+
+        while index < line.length
+          char = line[index]
+          return index + 1 if char == '"'
+
+          if char == "\\"
+            raise LexError.new("unterminated string literal", line: line_number, column: index + 1, path: @path) unless line[index + 1]
+
+            index += 2
+            next
+          end
+
+          index += 1
+        end
+
+        raise LexError.new("unterminated string literal", line: line_number, column: index + 1, path: @path)
+      end
+
+      def split_format_interpolation_source(source)
+        depth = 0
+        format_spec_index = nil
+        index = 0
+
+        while index < source.length
+          char = source[index]
+
+          if char == '"'
+            index = skip_interpolation_string_contents(source, index)
+            next
+          end
+
+          case char
+          when "(", "[", "{"
+            depth += 1
+          when ")", "]", "}"
+            depth -= 1 if depth.positive?
+          when ":"
+            suffix = source[(index + 1)..]
+            format_spec_index = index if depth.zero? && format_spec_suffix?(suffix)
+          end
+
+          index += 1
+        end
+
+        return [source, nil] unless format_spec_index
+
+        [source[0...format_spec_index], source[(format_spec_index + 1)..]]
+      end
+
+      def skip_interpolation_string_contents(source, index)
+        index += 1
+
+        while index < source.length
+          char = source[index]
+          return index + 1 if char == '"'
+
+          if char == "\\"
+            index += 2
+            next
+          end
+
+          index += 1
+        end
+
+        source.length
+      end
+
+      def format_spec_suffix?(source)
+        source && source.strip.match?(/\A(?:\.\d+|[xXoObB])\z/)
+      end
+    end
+  end
+end

data/lib/milk_tea/core/lexer/heredocs.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class Lexer
+    # Heredoc lexing (`<<-TAG`, `c<<-TAG`, `f<<-TAG`) including content
+    # dedenting, terminator detection, and recovery.
+    module Heredocs
+      private
+
+      def lex_heredoc(lines, line_index, index, line_number, line_offset, cstring:, format: false)
+        line = lines.fetch(line_index).delete_suffix("\n").b
+        prefix_length = heredoc_prefix(cstring:, format:).length
+        tag_start = index + prefix_length
+        tag_end = tag_start
+        tag_end += 1 while tag_end < line.length && identifier_part?(line[tag_end])
+        tag = line[tag_start...tag_end]
+        remainder = line[tag_end..] || ""
+
+        unless heredoc_context_allowed?
+          return lex_symbol(line, index, line_number, line_offset:)
+        end
+
+        unless remainder.strip.empty?
+          raise LexError.new("unexpected characters after heredoc tag", line: line_number, column: tag_end + 1, path: @path)
+        end
+
+        content_lines = []
+        terminator_line = nil
+        terminator_line_number = nil
+        terminator_line_offset = nil
+        terminator_has_newline = false
+        last_line = line
+        last_line_number = line_number
+        last_line_offset = line_offset
+        last_line_has_newline = lines.fetch(line_index).end_with?("\n")
+        resync_line_offset = nil
+        resync_line_number = nil
+        scan_line_number = line_number + 1
+        scan_line_offset = line_offset + lines.fetch(line_index).bytesize
+        scan_line_index = line_index + 1
+
+        content_min_indent = nil
+
+        while scan_line_index < lines.length
+          raw_line = lines.fetch(scan_line_index)
+          raw_text = raw_line.delete_suffix("\n")
+          if heredoc_terminator?(raw_text, tag)
+            terminator_line = raw_text
+            terminator_line_number = scan_line_number
+            terminator_line_offset = scan_line_offset
+            terminator_has_newline = raw_line.end_with?("\n")
+            break
+          end
+
+          if @recovery_errors && content_min_indent&.positive? && top_level_resync_line?(raw_text)
+            resync_line_offset = scan_line_offset
+            resync_line_number = scan_line_number
+            break
+          end
+
+          content_lines << raw_line
+          last_line = raw_text
+          last_line_number = scan_line_number
+          last_line_offset = scan_line_offset
+          last_line_has_newline = raw_line.end_with?("\n")
+          scan_line_offset += raw_line.bytesize
+          scan_line_number += 1
+          scan_line_index += 1
+
+          unless raw_text.strip.empty?
+            line_indent = leading_space_count(raw_text)
+            content_min_indent = line_indent if content_min_indent.nil? || line_indent < content_min_indent
+          end
+        end
+
+        if terminator_line.nil? && @recovery_errors && resync_line_number.nil? && scan_line_index < lines.length
+          trailing_text = lines.fetch(scan_line_index).delete_suffix("\n")
+          if top_level_resync_line?(trailing_text)
+            resync_line_offset = scan_line_offset
+            resync_line_number = scan_line_number
+          end
+        end
+
+        if terminator_line.nil?
+          if @recovery_errors
+            @recovery_errors << LexError.new("unterminated heredoc literal", line: line_number, column: index + 1, path: @path)
+            start_offset = line_offset + index
+            end_offset = resync_line_offset || scan_line_offset
+            lexeme = @source.byteslice(start_offset, end_offset - start_offset)
+            value = dedent_heredoc_content(content_lines)
+            content_margin = heredoc_content_margin(content_lines)
+
+            literal = if format
+                         parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
+                       else
+                         value
+                       end
+            token_type = if format
+                           :fstring
+                         elsif cstring
+                           :cstring
+                         else
+                           :string
+                         end
+
+            @tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
+            emit_line_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
+            return resync_line_number ? (resync_line_number - line_number) : (scan_line_index - line_index)
+          end
+
+          raise LexError.new("unterminated heredoc literal", line: line_number, column: index + 1, path: @path)
+        end
+
+        start_offset = line_offset + index
+        end_offset = terminator_line_offset + terminator_line.bytesize
+        lexeme = @source.byteslice(start_offset, end_offset - start_offset)
+        value = dedent_heredoc_content(content_lines)
+        content_margin = heredoc_content_margin(content_lines)
+
+        literal = if format
+                     parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
+                   else
+                     value
+                   end
+        token_type = if format
+                       :fstring
+                     elsif cstring
+                       :cstring
+                     else
+                       :string
+                     end
+
+        @tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
+        emit_line_newline(terminator_line, terminator_line_number, terminator_line_offset, terminator_has_newline)
+
+        (scan_line_index - line_index) + 1
+      end
+
+      def heredoc_start?(line, index, cstring: false, format: false)
+        prefix = heredoc_prefix(cstring:, format:)
+        line[index, prefix.length] == prefix && identifier_start?(line[index + prefix.length])
+      end
+
+      def heredoc_prefix(cstring: false, format: false)
+        operator = "<<-"
+        if cstring
+          "c#{operator}"
+        elsif format
+          "f#{operator}"
+        else
+          operator
+        end
+      end
+
+      def heredoc_context_allowed?
+        allowed = %i[
+          newline indent dedent
+          equal plus_equal minus_equal star_equal slash_equal percent_equal
+          amp_equal pipe_equal caret_equal shift_left_equal shift_right_equal
+          lparen lbracket comma colon
+          return defer if else while match in
+          or and not out inout
+          plus minus star slash percent amp pipe caret shift_left shift_right
+          less less_equal greater greater_equal equal_equal bang_equal
+        ]
+        previous_type = @tokens.last&.type
+        previous_type.nil? || allowed.include?(previous_type)
+      end
+
+      def heredoc_terminator?(line, tag)
+        line.match?(Regexp.new("\\A *#{Regexp.escape(tag)} *\\z"))
+      end
+
+      def heredoc_content_margin(raw_lines)
+        raw_lines
+          .map { |raw_line| raw_line.delete_suffix("\n") }
+          .reject { |text| text.strip.empty? }
+          .map { |text| leading_space_count(text) }
+          .min || 0
+      end
+
+      def dedent_heredoc_content(raw_lines)
+        dedent_heredoc_lines(raw_lines).each_with_index.map do |text, index|
+          text + (raw_lines[index].end_with?("\n") ? "\n" : "")
+        end.join
+      end
+
+      def dedent_heredoc_lines(raw_lines)
+        margin = heredoc_content_margin(raw_lines)
+
+        raw_lines.map do |raw_line|
+          text = raw_line.delete_suffix("\n")
+          text = text.strip.empty? ? "" : text[margin..]
+          text || ""
+        end
+      end
+    end
+  end
+end