mt-lang 0.2.0

data/lib/milk_tea/core/parser/types.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+module MilkTea
+  module Parse
+    module Types
+      private
+
+      def parse_params(allow_variadic: false)
+        parse_parameter_list(allow_variadic:) { parse_param }
+      end
+
+      def parse_signature_params
+        parse_parameter_list { parse_param }
+      end
+
+      def parse_foreign_params(allow_variadic: false)
+        parse_parameter_list(allow_variadic:) { parse_foreign_param }
+      end
+
+      def parse_parameter_list(allow_variadic: false)
+        consume(:lparen, "expected '('")
+        params = []
+        variadic = false
+
+        unless check(:rparen)
+          loop do
+            if allow_variadic && match(:ellipsis)
+              variadic = true
+              break
+            end
+
+            params << yield
+            break unless match(:comma)
+            break if check(:rparen)
+          end
+        end
+
+        consume(:rparen, "expected ')' after parameters")
+        return [params, variadic] if allow_variadic
+
+        params
+      end
+
+      def parse_param
+        name_token = consume_name("expected parameter name")
+        raise error(name_token, "expected ':' and parameter type") unless match(:colon)
+
+        param_type = parse_type_ref
+
+        AST::Param.new(name: name_token.lexeme, type: param_type, line: name_token.line, column: name_token.column)
+      rescue ParseError => e
+        raise unless @recovery_errors
+
+        @recovery_errors << e
+        param_name = name_token&.lexeme || "error"
+        error_type = recovery_error_expr(e)
+        AST::Param.new(name: param_name, type: error_type, line: name_token&.line || 1, column: name_token&.column || 1)
+      end
+
+      def parse_foreign_param
+        mode = if foreign_param_qualifier_mode?
+                 advance.type
+               else
+                 :plain
+               end
+        name_token = consume_name("expected parameter name")
+        raise error(name_token, "expected ':' and parameter type") unless match(:colon)
+
+        param_type = parse_type_ref
+        boundary_type = match(:as) ? parse_type_ref : nil
+
+        AST::ForeignParam.new(name: name_token.lexeme, type: param_type, mode:, boundary_type:)
+      rescue ParseError => e
+        raise unless @recovery_errors
+
+        @recovery_errors << e
+        param_name = name_token&.lexeme || "error"
+        error_type = recovery_error_expr(e)
+        AST::ForeignParam.new(name: param_name, type: error_type, mode: mode || :plain, boundary_type: nil)
+      end
+
+      def parse_type_ref
+        return parse_function_type_ref if match(:fn)
+        return parse_proc_type_ref if match(:proc)
+        return parse_dyn_type_ref if match(:dyn)
+        return parse_tuple_type_ref if check(:lparen)
+
+        first_token = peek
+        if match(:at)
+          lt_token = consume_name("expected lifetime name after @")
+          return AST::TypeRef.new(name: AST::QualifiedName.new(parts: ["@#{lt_token.lexeme}"]), arguments: [], nullable: false, lifetime: nil, line: first_token.line, column: first_token.column, length: lt_token.lexeme.length + 1)
+        end
+        name = parse_qualified_name
+        arguments = []
+        lifetime = nil
+        if match(:lbracket)
+          if check(:at) && name.to_s == "ref"
+            match(:at)
+            lt_token = consume_name("expected lifetime name after @")
+            lifetime = "@#{lt_token.lexeme}"
+            consume(:comma, "expected ',' after lifetime in type arguments")
+          end
+          arguments = parse_comma_separated_until(:rbracket) do
+            AST::TypeArgument.new(value: parse_type_argument)
+          end
+          consume(:rbracket, "expected ']' after type arguments")
+        end
+
+        nullable = match(:question)
+        type_name = name.to_s
+        length = type_name.length + (nullable ? 1 : 0)
+        AST::TypeRef.new(name:, arguments:, nullable:, lifetime:, line: first_token.line, column: first_token.column, length:)
+      end
+
+      def parse_function_type_ref
+        parse_callable_type_ref(keyword: "fn", param_context: "function type parameters") do |params, return_type|
+          AST::FunctionType.new(params:, return_type:)
+        end
+      end
+
+      def parse_proc_type_ref
+        parse_callable_type_ref(keyword: "proc", param_context: "proc type parameters") do |params, return_type|
+          AST::ProcType.new(params:, return_type:)
+        end
+      end
+
+      def parse_dyn_type_ref
+        first_token = previous
+        consume(:lbracket, "expected '[' after dyn")
+        interface = parse_qualified_name_with_type_arguments
+        consume(:rbracket, "expected ']' after interface name")
+        nullable = match(:question)
+        AST::DynType.new(interface:, nullable:, line: first_token.line, column: first_token.column, length: 3)
+      end
+
+      def parse_tuple_type_ref
+        match(:lparen)
+        first_type = parse_type_ref
+        if match(:comma)
+          element_types = [first_type]
+          loop do
+            element_types << parse_type_ref
+            break unless match(:comma)
+          end
+          consume(:rparen, "expected ')' after tuple type elements")
+          nullable = match(:question)
+          AST::TupleType.new(element_types:, nullable:)
+        else
+          consume(:rparen, "expected ')' after type")
+          first_type
+        end
+      end
+
+      def parse_callable_type_ref(keyword:, param_context:)
+        consume(:lparen, "expected '(' after #{keyword}")
+        params = parse_comma_separated_until(:rparen) { parse_function_type_param }
+
+        consume(:rparen, "expected ')' after #{param_context}")
+        consume(:arrow, "expected '->' after #{param_context}")
+        return_type = parse_type_ref
+        yield params, return_type
+      end
+
+      def parse_function_type_param
+        name_token = consume_name("expected function type parameter name")
+        name = name_token.lexeme
+        consume(:colon, "expected ':' after function type parameter name")
+        type = parse_type_ref
+        AST::Param.new(name:, type:, line: name_token.line, column: name_token.column)
+      end
+
+      def parse_type_argument
+        if match(:integer)
+          token = previous
+          AST::IntegerLiteral.new(lexeme: token.lexeme, value: token.literal)
+        elsif match(:float)
+          token = previous
+          AST::FloatLiteral.new(lexeme: token.lexeme, value: token.literal)
+        else
+          parse_type_ref
+        end
+      end
+
+      def parse_declaration_type_params
+        return [] unless match(:lbracket)
+
+        params = parse_comma_separated_until(:rbracket) do
+          name_token = consume_name("expected type parameter name")
+          name = name_token.lexeme
+          if match(:colon)
+            value_type = parse_type_ref
+            AST::ValueTypeParam.new(
+              name:,
+              type: value_type,
+              line: name_token.line,
+              column: name_token.column,
+              length: name.length,
+            )
+          else
+            constraints = parse_type_param_constraints
+            AST::TypeParam.new(
+              name:,
+              constraints:,
+              line: name_token.line,
+              column: name_token.column,
+              length: name.length,
+            )
+          end
+        end
+
+        consume(:rbracket, "expected ']' after type parameters")
+        params
+      end
+
+      def parse_type_param_constraints
+        constraints = []
+        interface_constraint_mode = false
+
+        constraint, interface_constraint_mode = parse_type_param_constraint(interface_constraint_mode)
+        return constraints unless constraint
+
+        constraints << constraint
+
+        while match(:and)
+          constraint, interface_constraint_mode = parse_type_param_constraint(interface_constraint_mode)
+          raise error(peek, "expected constraint after 'and'") unless constraint
+
+          constraints << constraint
+        end
+
+        constraints
+      end
+
+      def parse_type_param_constraint(interface_constraint_mode)
+        if match(:implements)
+          return [AST::TypeParamConstraint.new(kind: :interface, interface_ref: parse_qualified_name_with_type_arguments), true]
+        end
+
+        return [AST::TypeParamConstraint.new(kind: :interface, interface_ref: parse_qualified_name_with_type_arguments), true] if interface_constraint_mode
+
+        [nil, interface_constraint_mode]
+      end
+
+      def parse_implements_clause
+        return [] unless match(:implements)
+
+        implements = [parse_qualified_name_with_type_arguments]
+        implements << parse_qualified_name_with_type_arguments while match(:comma)
+        implements
+      end
+
+      def parse_qualified_name_with_type_arguments
+        first_token = consume_path_component("expected identifier")
+        parts = [first_token.lexeme]
+        while match(:dot)
+          parts << consume_path_component("expected identifier after '.'").lexeme
+        end
+        type_arguments = if match(:lbracket)
+                           args = parse_comma_separated_until(:rbracket) do
+                             parse_type_ref
+                           end
+                           consume(:rbracket, "expected ']' after type arguments")
+                           args
+                         else
+                           []
+                         end
+        AST::QualifiedName.new(parts:, type_arguments:, line: first_token.line, column: first_token.column)
+      end
+    end
+  end
+end

data/lib/milk_tea/core/parser.rb

@@ -0,0 +1,400 @@
+# frozen_string_literal: true
+
+require_relative "parser/blocks"
+require_relative "parser/recovery"
+require_relative "parser/types"
+require_relative "parser/attributes"
+require_relative "parser/expressions"
+require_relative "parser/declarations"
+require_relative "parser/statements"
+
+module MilkTea
+  class ParseError < StandardError
+    attr_reader :token, :path
+
+    def initialize(message, token:, path: nil)
+      @token = token
+      @path = path
+
+      location = [path, "#{token.line}:#{token.column}"].compact.join(":")
+      super(location.empty? ? message : "#{message} at #{location}")
+    end
+
+    def line
+      @token.line
+    end
+
+    def column
+      @token.column
+    end
+
+    def code
+      "parse/error"
+    end
+  end
+
+  class Parser
+    ParseRecoveryResult = Data.define(:ast, :errors) do
+      def initialize(ast:, errors: []) = super
+    end
+
+    include Parse::Blocks
+    include Parse::Recovery
+    include Parse::Types
+    include Parse::Attributes
+    include Parse::Expressions
+    include Parse::Declarations
+    include Parse::Statements
+
+    def self.parse(source = nil, path: nil, tokens: nil)
+      token_stream = tokens || Lexer.lex(source, path: path)
+      new(token_stream, path: path).parse
+    end
+
+    def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
+      if tokens
+        return new(tokens, path: path).parse_collecting_errors
+      end
+
+      lex_errors = []
+      token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
+      parse_result = new(token_stream, path: path).parse_collecting_errors
+      ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
+    end
+
+    def initialize(tokens, path: nil)
+      @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
+      @path = path
+      @current = 0
+      @known_type_names = {}
+      @known_import_aliases = {}
+      @known_generic_callable_names = {}
+      @current_type_param_names = []
+      @in_inline_block_body = false
+      seed_known_names
+    end
+
+    def parse
+      parse_source_file
+    end
+
+    def parse_collecting_errors
+      errors = @recovery_errors ? @recovery_errors.dup : []
+      previous_recovery_errors = @recovery_errors
+      @recovery_errors = errors
+      ast = parse_source_file(errors:)
+      ParseRecoveryResult.new(ast:, errors:)
+    rescue ParseError => e
+      errors ||= []
+      errors << e
+      ParseRecoveryResult.new(ast: nil, errors:)
+    ensure
+      @recovery_errors = previous_recovery_errors
+    end
+
+    private
+
+    BUILTIN_ATTRIBUTE_NAME_LEXEMES = %w[packed align].freeze
+
+    def parse_source_file(errors: nil)
+      skip_newlines
+
+      module_name = nil
+      module_kind = :module
+      module_line = nil
+      imports = []
+      directives = []
+      declarations = []
+
+      if external_file_header?
+        advance
+        module_line = previous.line
+        module_kind = :raw_module
+        consume(:newline, "expected newline after external") unless eof?
+        skip_newlines
+        imports, directives, declarations = parse_raw_module_body(errors:)
+        skip_newlines
+        raise error(peek, "expected end of file after external declarations") unless eof?
+
+        return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
+      end
+
+      while match(:import)
+        if errors
+          begin
+            imports << parse_import
+          rescue ParseError => e
+            errors << e
+            synchronize_to_top_level_boundary
+          end
+        else
+          imports << parse_import
+        end
+        skip_newlines
+      end
+
+      until eof?
+        if errors
+          begin
+            declarations << parse_declaration
+          rescue ParseError => e
+            errors << e
+            synchronize_to_top_level_boundary
+          end
+        else
+          declarations << parse_declaration
+        end
+        skip_newlines
+      end
+
+      AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
+    end
+
+    def external_file_header?
+      return false unless check(:external)
+
+      next_token = @tokens[@current + 1]
+      next_token.nil? || %i[newline eof].include?(next_token.type)
+    end
+
+    def match(*types)
+      return false unless types.any? { |type| check(type) }
+
+      advance
+      true
+    end
+
+    def consume(type, message)
+      return advance if check(type)
+
+      raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
+    end
+
+    def consume_name(message)
+      if keyword_token?(peek)
+        name_role = message.sub(/\A(?:expected|required)\s+/, "")
+        clearer_message = if name_role == message
+                            message
+                          else
+                            "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
+                          end
+        raise error(peek, clearer_message)
+      end
+
+      consume(:identifier, message)
+    end
+
+    def consume_name_allowing_keywords(message)
+      if check(:identifier)
+        advance
+      elsif keyword_token?(peek)
+        advance
+      else
+        raise error(peek, message)
+      end
+    end
+
+    def keyword_token?(token)
+      token && Token::KEYWORDS.key?(token.lexeme)
+    end
+
+    def check(type)
+      return false if eof?
+
+      peek.type == type
+    end
+
+    def check_name
+      !eof? && peek.type == :identifier
+    end
+
+    def check_next(type)
+      return false if (@current + 1) >= @tokens.length
+
+      @tokens[@current + 1].type == type
+    end
+
+    def match_name
+      return false unless check_name
+
+      advance
+      true
+    end
+
+    def advance
+      @current += 1 unless eof?
+      previous
+    end
+
+    def eof?
+      peek.type == :eof
+    end
+
+    def peek
+      @tokens[@current]
+    end
+
+    def previous
+      @tokens[@current - 1]
+    end
+
+    def error(token, message)
+      ParseError.new(message, token:, path: @path)
+    end
+
+    def skip_newlines
+      advance while check(:newline)
+    end
+
+    def consume_end_of_statement
+      return if @in_inline_block_body
+
+      consume(:newline, "expected end of statement")
+    end
+
+    def block_expression?(expression)
+      expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr)
+    end
+
+    def parse_qualified_name
+      parts = [consume_path_component("expected identifier").lexeme]
+      while match(:dot)
+        parts << consume_path_component("expected identifier after '.'").lexeme
+      end
+      AST::QualifiedName.new(parts:)
+    end
+
+    def consume_path_component(message)
+      return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))
+
+      raise error(peek, message)
+    end
+
+    def parse_comma_separated_until(closing_type)
+      items = []
+
+      unless check(closing_type)
+        loop do
+          items << yield
+          break unless match(:comma)
+          break if check(closing_type)
+        end
+      end
+
+      items
+    end
+
+    def foreign_param_qualifier_mode?
+      return false unless %i[out in inout consuming].include?(peek.type)
+
+      @tokens[@current + 1]&.type == :identifier
+    end
+
+    def legacy_layout_modifier_start?(token)
+      token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
+    end
+
+    def known_type_like_name?(name)
+      @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
+    end
+
+    def with_type_param_names(names)
+      saved_names = @current_type_param_names
+      @current_type_param_names = @current_type_param_names + names
+      yield
+    ensure
+      @current_type_param_names = saved_names
+    end
+
+    def seed_known_names
+      MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }
+
+      depth = 0
+      index = 0
+      while index < @tokens.length
+        token = @tokens[index]
+        case token.type
+        when :indent
+          depth += 1
+        when :dedent
+          depth -= 1 if depth.positive?
+        when :import
+          index = seed_import_alias(index + 1) if depth.zero?
+        when :function
+          if depth.zero?
+            name_token = @tokens[index + 1]
+            type_param_token = @tokens[index + 2]
+            if type_name_token?(name_token) && type_param_token&.type == :lbracket
+              @known_generic_callable_names[name_token.lexeme] = true
+            end
+          end
+        when :async
+          if depth.zero? && @tokens[index + 1]&.type == :function
+            name_token = @tokens[index + 2]
+            type_param_token = @tokens[index + 3]
+            if type_name_token?(name_token) && type_param_token&.type == :lbracket
+              @known_generic_callable_names[name_token.lexeme] = true
+            end
+          end
+        when :foreign
+          if depth.zero? && @tokens[index + 1]&.type == :function
+            name_token = @tokens[index + 2]
+            type_param_token = @tokens[index + 3]
+            if type_name_token?(name_token) && type_param_token&.type == :lbracket
+              @known_generic_callable_names[name_token.lexeme] = true
+            end
+          end
+        when :struct, :union, :enum, :flags, :opaque, :type, :variant
+          if depth.zero?
+            name_token = @tokens[index + 1]
+            @known_type_names[name_token.lexeme] = true if type_name_token?(name_token)
+          end
+        end
+
+        index += 1
+      end
+    end
+
+    def seed_import_alias(start_index)
+      cursor = start_index
+      last_part = nil
+
+      while cursor < @tokens.length && @tokens[cursor].type != :newline
+        token = @tokens[cursor]
+        if token.type == :as
+          alias_token = @tokens[cursor + 1]
+          @known_import_aliases[alias_token.lexeme] = true if type_name_token?(alias_token)
+          return cursor
+        end
+
+        last_part = token.lexeme if type_name_token?(token)
+        cursor += 1
+      end
+
+      @known_import_aliases[last_part] = true if last_part
+      cursor
+    end
+
+    def type_name_token?(token)
+      token&.type == :identifier
+    end
+
+    def matching_rbracket_index(start_index)
+      depth = 0
+      index = start_index
+
+      while index < @tokens.length
+        case @tokens[index].type
+        when :lbracket
+          depth += 1
+        when :rbracket
+          depth -= 1
+          return index if depth.zero?
+        end
+        index += 1
+      end
+
+      nil
+    end
+  end
+end

data/lib/milk_tea/core/prelude_installer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class PreludeInstaller
+    PRELUDE_MODULE_PATHS = %w[std.option std.result].freeze
+
+    def initialize(resolve_module_path:, check_block:, bind_block:)
+      @resolve_module_path = resolve_module_path
+      @check_block = check_block
+      @bind_block = bind_block
+    end
+
+    def install_prelude_modules!(ast, modules, importer_path: nil, collecting_errors: false)
+      current_module_name = ast.module_name&.to_s
+      return if PRELUDE_MODULE_PATHS.include?(current_module_name)
+
+      PRELUDE_MODULE_PATHS.each do |module_path|
+        next if modules.key?(module_path)
+
+        begin
+          import_path = @resolve_module_path.call(module_path, importer_path:)
+          import_analysis = @check_block.call(import_path, collecting_errors)
+          modules[module_path] = @bind_block.call(import_analysis)
+        rescue ModuleLoadError
+        end
+      end
+    end
+  end
+end