mt-lang 0.2.0

data/lib/milk_tea/core/semantic_analyzer.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+require_relative "semantic_analyzer/type_declaration"
+require_relative "semantic_analyzer/attributes"
+require_relative "semantic_analyzer/function_binding"
+require_relative "semantic_analyzer/top_level"
+require_relative "semantic_analyzer/interface_conformance"
+require_relative "semantic_analyzer/nullability"
+require_relative "semantic_analyzer/statements"
+require_relative "semantic_analyzer/expressions"
+require_relative "semantic_analyzer/calls"
+require_relative "semantic_analyzer/analysis_context"
+require_relative "semantic_analyzer/type_compatibility"
+require_relative "semantic_analyzer/flow_refinement"
+require_relative "semantic_analyzer/name_resolution"
+require_relative "semantic_analyzer/generics"
+require_relative "semantic_analyzer/foreign_functions"
+require_relative "semantic_analyzer/module_context"
+
+module MilkTea
+  class SemanticError < StandardError
+    attr_reader :line, :column, :length, :path, :suggestion
+
+    def initialize(msg = nil, line: nil, column: nil, length: nil, path: nil, suggestion: nil)
+      super(msg)
+      @line = line
+      @column = column
+      @length = length
+      @path = path
+      @suggestion = suggestion
+    end
+
+    def code
+      "sema/error"
+    end
+
+    def to_diagnostic(path: nil)
+      Diagnostic.new(
+        path:,
+        line: @line,
+        column: @column,
+        length: @length,
+        code: "sema/error",
+        message: message,
+        severity: :error,
+      )
+    end
+  end
+
+  class SemanticAnalyzer
+    Analysis = Data.define(:ast, :module_name, :module_kind, :directives, :imports, :types, :interfaces, :attributes, :attribute_applications, :values, :functions, :methods, :implemented_interfaces, :local_completion_frames, :binding_resolution, :callable_value_identifier_sites, :callable_value_member_access_sites, :required_unsafe_lines, :uses_parallel_for, :resolved_expr_types, :resolved_call_kinds, :const_values) do
+      def initialize(ast:, module_name:, module_kind:, directives:, imports:, types:, interfaces:, attributes:, attribute_applications:, values:, functions:, methods:, implemented_interfaces:, local_completion_frames:, binding_resolution:, callable_value_identifier_sites:, callable_value_member_access_sites:, required_unsafe_lines:, uses_parallel_for:, resolved_expr_types: {}, resolved_call_kinds: {}, const_values: {}) = super
+    end
+    ToolingSnapshot = Data.define(:facts, :diagnostics)
+    LocalCompletionFrame = Data.define(:start_line, :end_line, :function_name, :receiver_type, :snapshots)
+    LocalCompletionSnapshot = Data.define(:line, :column, :bindings)
+    BindingResolution = Data.define(
+      :identifier_binding_ids,
+      :declaration_binding_ids,
+      :mutating_argument_identifier_ids,
+      :editable_receiver_expression_ids,
+      :mutable_lvalue_argument_identifier_ids,
+      :binding_types,
+    )
+    InterfaceMethodBinding = Data.define(:name, :params, :return_type, :kind, :async, :ast)
+    InterfaceBinding = Data.define(:name, :methods, :ast, :module_name, :type_arguments) do
+      def initialize(name:, methods:, ast:, module_name:, type_arguments: nil) = super
+    end
+
+    GenericInterfaceBinding = Data.define(:name, :type_params, :type_param_constraints, :methods, :ast, :module_name) do
+      def instantiate(arguments)
+        raise ArgumentError, "#{name} expects #{type_params.length} type arguments, got #{arguments.length}" unless arguments.length == type_params.length
+
+        substitutions = type_params.zip(arguments).to_h
+        substituted_methods = methods.transform_values do |method|
+          InterfaceMethodBinding.new(
+            name: method.name,
+            params: method.params.map { |p| Types::Registry.parameter(p.name, Types.substitute_type_variables(p.type, substitutions)) },
+            return_type: Types.substitute_type_variables(method.return_type, substitutions),
+            kind: method.kind,
+            async: method.async,
+            ast: method.ast,
+          )
+        end
+
+        InterfaceBinding.new(
+          name:,
+          methods: substituted_methods.freeze,
+          ast:,
+          module_name:,
+          type_arguments: arguments.freeze,
+        )
+      end
+    end
+    ResolvedAttributeApplication = Data.define(:binding, :argument_values)
+    AttributePresenceKey = Data.define(:target, :attribute_module_name, :attribute_name)
+    TypeParamConstraintBinding = Data.define(:interfaces) do
+      def initialize(interfaces: []) = super
+    end
+    DefaultResolution = Data.define(:target_type, :binding)
+    HashResolution = Data.define(:target_type, :binding)
+    EqualResolution = Data.define(:target_type, :binding)
+    OrderResolution = Data.define(:target_type, :binding)
+
+    INSTALLABLE_BUILTIN_TYPE_NAMES = (MilkTea::BUILTIN_PRIMITIVE_NAMES + %w[
+      Subscription EventError
+      struct_handle field_handle callable_handle attribute_handle member_handle type
+    ]).freeze
+
+    def self.check(ast, imported_modules: {}, allow_missing_imports: false, path: nil, global_import_index: {})
+      Checker.new(ast, imported_modules:, allow_missing_imports:, path:, global_import_index:).check
+    end
+
+    # LSP-oriented entry point: runs all sema phases and collects every error
+    # instead of stopping at the first one.  Structural phases collect per-
+    # declaration, and function-body phases collect per function/method.
+    # Returns { analysis: Analysis|nil, errors: [SemanticError] }.
+    def self.check_collecting_errors(ast, imported_modules: {}, allow_missing_imports: false, path: nil)
+      Checker.new(ast, imported_modules:, allow_missing_imports:, path:).check_collecting_errors
+    rescue SemanticError => e
+      { analysis: nil, errors: [e] }
+    end
+
+    def self.tooling_snapshot(ast, imported_modules: {}, allow_missing_imports: false, path: nil)
+      result = check_collecting_errors(ast, imported_modules:, allow_missing_imports:, path:)
+      diagnostics = Array(result[:errors]).map { |error| error.to_diagnostic(path:) }.freeze
+
+      ToolingSnapshot.new(facts: result[:analysis], diagnostics:)
+    end
+
+    class Checker
+      include CompatibilityHelpers
+
+      attr_reader :ctx
+
+      def module_name
+        @ctx.module_name
+      end
+
+      def initialize(ast, imported_modules: {}, allow_missing_imports: false, path: nil, global_import_index: {})
+        @path = path
+        @allow_missing_imports = allow_missing_imports
+        @ctx = ModuleContext.new(
+          ast:,
+          module_name: ast.module_name&.to_s,
+          module_kind: ast.module_kind,
+          imported_modules:,
+          global_import_index:,
+          const_declarations: ast.declarations.grep(AST::ConstDecl).each_with_object({}) { |decl, result| result[decl.name] = decl },
+        )
+        @null_type = Types::Null.new
+        @error_type = Types::Error.new
+        @loop_depth = 0
+        @unsafe_depth = 0
+        @compile_time_depth = 0
+        @foreign_mapping_depth = 0
+        @async_function_depth = 0
+        @checked_function_bindings = {}
+        @checking_function_bindings = {}
+        @evaluating_const_values = []
+        @evaluated_const_values = {}
+        @error_node_stack = []
+        @local_completion_frames = []
+        @active_local_completion_stack = []
+        @resolved_expr_types = {}
+        @resolved_call_kinds = {}
+        @const_values = {}
+        @next_binding_id = 1
+        @binding_name_by_id = {}
+        @binding_type_by_id = {}
+        @identifier_binding_ids = {}
+        @declaration_binding_ids = {}
+        @mutating_argument_identifier_ids = {}
+        @mutable_lvalue_argument_identifier_ids = {}
+        @editable_receiver_expression_ids = {}
+        @preassigned_local_binding_ids = {}
+        @nullability_flow_result = nil
+        @unsafe_statement_lines = []
+        @callable_value_identifier_sites = {}
+        @callable_value_member_access_sites = {}
+        @required_unsafe_lines = []
+        @uses_parallel_for = false
+        @current_specialization_owner = nil
+        @return_context_stack = []
+      end
+
+      STRUCTURAL_PHASES = %i[
+        install_builtin_types
+        install_builtin_attributes
+        install_imports
+        install_prelude_types
+        declare_named_types
+        resolve_generic_type_param_constraints
+        resolve_type_aliases
+        declare_attributes
+        resolve_aggregate_fields
+        resolve_enum_members
+        resolve_variant_arms
+        collect_emit_declarations
+        declare_top_level_values
+        check_attribute_applications
+        declare_functions
+        check_interface_conformances
+      ].freeze
+
+      def check
+        STRUCTURAL_PHASES.each { |phase| send(phase) }
+        check_top_level_values
+        finalize_top_level_const_values
+        check_top_level_static_asserts
+        check_functions
+
+        build_analysis
+      end
+
+      def collect_emit_declarations
+        collect_emit_from_declarations(expanded_declarations)
+        @ctx.ast.declarations.grep(AST::ConstDecl).each { |decl| @ctx.const_declarations[decl.name] ||= decl }
+      end
+
+      def collect_emit_from_declarations(declarations)
+        declarations.each do |decl|
+          case decl
+          when AST::FunctionDef
+            next unless decl.const && decl.body
+
+            collect_emit_from_statements(decl.body)
+          when AST::WhenStmt
+            body = when_chosen_body(decl)
+            collect_emit_from_declarations(body) if body
+          when AST::StructDecl, AST::ExtendingBlock
+            # no emit in structs/extending blocks
+          end
+        end
+      end
+
+      def collect_emit_from_statements(statements)
+        statements.each do |stmt|
+          case stmt
+          when AST::EmitStmt
+            emit_decl = stmt.declaration
+            next if emit_decl.is_a?(AST::ErrorExpr)
+
+            @ctx.ast.declarations << emit_decl
+            collect_emit_from_node(emit_decl)
+          when AST::WhenStmt
+            body = when_chosen_body(stmt) || []
+            body.each { |nested| collect_emit_from_statements([nested]) }
+          when AST::ForStmt, AST::WhileStmt, AST::IfStmt, AST::MatchStmt
+            next unless stmt.inline
+            stmt.body&.each { |s| collect_emit_from_statements([s]) }
+            if stmt.is_a?(AST::IfStmt)
+              stmt.else_body&.each { |s| collect_emit_from_statements([s]) }
+            end
+          end
+        end
+      end
+
+      def collect_emit_from_node(node)
+        case node
+        when AST::FunctionDef
+          node.body&.each do |stmt|
+            if stmt.is_a?(AST::EmitStmt)
+              nested = stmt.declaration
+              next if nested.is_a?(AST::ErrorExpr)
+
+              @ctx.ast.declarations << nested
+              collect_emit_from_node(nested)
+            end
+          end
+        end
+      end
+
+      # Like check, but collects per-function errors instead of raising at first.
+      # Structural phases (imports, type resolution, declaration) collect errors per
+      # declaration so that the maximum number of diagnostics are surfaced.
+      # Returns { analysis: Analysis, errors: [SemanticError] }.
+      def check_collecting_errors
+        @collecting_errors = true
+        @structural_errors = []
+
+        STRUCTURAL_PHASES.each { |phase| catch_structural { send(phase) } }
+
+        errors = @structural_errors.dup
+
+        begin
+          check_top_level_values
+        rescue SemanticError => e
+          errors << e
+        end
+        errors.concat(@structural_errors.drop(errors.length))
+
+        begin
+          finalize_top_level_const_values
+        rescue SemanticError => e
+          errors << e
+        end
+
+        begin
+          check_top_level_static_asserts
+        rescue SemanticError => e
+          errors << e
+        end
+
+        check_functions_collecting(errors)
+
+        analysis = build_analysis
+
+        { analysis: analysis, errors: errors.uniq { |e| [e.message, e.line, e.column, e.length] } }
+      end
+
+      def catch_structural
+        yield
+      rescue SemanticError => e
+        @structural_errors << e
+      end
+
+      def collect_structural_error(error)
+        raise error unless @collecting_errors
+
+        @structural_errors << error
+      end
+
+
+    end
+  end
+end

data/lib/milk_tea/core/token.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module MilkTea
+  TriviaToken = Data.define(:kind, :text, :line, :column, :start_offset, :end_offset)
+
+  class Token < Data.define(:type, :lexeme, :literal, :line, :column, :start_offset, :end_offset, :leading_trivia, :trailing_trivia)
+    KEYWORDS = MilkTea::KEYWORDS
+
+    ASSIGNMENT_TYPES = %i[
+      equal plus_equal minus_equal star_equal slash_equal percent_equal
+      amp_equal pipe_equal caret_equal shift_left_equal shift_right_equal
+    ].freeze
+
+    def assignment?
+      ASSIGNMENT_TYPES.include?(type)
+    end
+
+    def eof?
+      type == :eof
+    end
+
+    def with_appended_trailing_trivia(trivia)
+      with(trailing_trivia: trailing_trivia + [trivia])
+    end
+  end
+end

data/lib/milk_tea/core/token_stream.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class SyntaxTokenStream
+    def initialize(tokens)
+      @tokens = tokens
+    end
+
+    def [](index)
+      @tokens[index]
+    end
+
+    def length
+      @tokens.length
+    end
+
+    def to_a
+      @tokens
+    end
+  end
+
+  class TriviaTokenStream
+    attr_reader :tokens, :trivia
+
+    def initialize(tokens, trivia)
+      @tokens = tokens
+      @trivia = trivia
+    end
+  end
+end

data/lib/milk_tea/core/types/layout.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "fiddle"
+require_relative "types"
+
+module MilkTea
+  module Layout
+    module_function
+
+    POINTER_SIZE = Fiddle::SIZEOF_VOIDP
+
+    def size_of(type)
+      size_and_alignment(type, {})&.first
+    end
+
+    def alignment_of(type)
+      size_and_alignment(type, {})&.last
+    end
+
+    def offset_of(type, field_name)
+      case type
+      when Types::Struct, Types::StructInstance, Types::Span, Types::StringView, Types::Task
+        fields = ordered_fields(type)
+        return unless fields.key?(field_name)
+
+        struct_layout(fields, packed: packed_layout?(type), alignment: explicit_alignment(type), stack: {})[:offsets][field_name]
+      when Types::Union
+        type.field(field_name) ? 0 : nil
+      when Types::GenericInstance
+        return unless str_buffer_type?(type)
+
+        fields = str_buffer_fields(type)
+        return unless fields.key?(field_name)
+
+        struct_layout(fields, packed: false, alignment: nil, stack: {})[:offsets][field_name]
+      else
+        nil
+      end
+    end
+
+    def size_and_alignment(type, stack)
+      case type
+      when Types::Primitive
+        primitive_layout(type)
+      when Types::EnumBase
+        size_and_alignment(type.backing_type, stack)
+      when Types::Nullable, Types::Function
+        if type.is_a?(Types::Function) || layout_pointer_like_nullable_base?(type.base)
+          [POINTER_SIZE, POINTER_SIZE]
+        else
+          with_stack(type, stack) do
+            layout = struct_layout(nullable_opt_layout_fields(type.base), packed: false, alignment: nil, stack:)
+            [layout[:size], layout[:alignment]]
+          end
+        end
+      when Types::StringView, Types::Span, Types::Task, Types::Struct, Types::StructInstance
+        with_stack(type, stack) do
+          layout = struct_layout(ordered_fields(type), packed: packed_layout?(type), alignment: explicit_alignment(type), stack:)
+          [layout[:size], layout[:alignment]]
+        end
+      when Types::Union
+        with_stack(type, stack) do
+          layout = union_layout(ordered_fields(type), packed: packed_layout?(type), alignment: explicit_alignment(type), stack:)
+          [layout[:size], layout[:alignment]]
+        end
+      when Types::Variant, Types::VariantInstance
+        with_stack(type, stack) do
+          layout = variant_layout(type, stack:)
+          [layout[:size], layout[:alignment]]
+        end
+      when Types::GenericInstance
+        generic_layout(type, stack)
+      else
+        nil
+      end
+    end
+
+    def primitive_layout(type)
+      case type.name
+      when "bool", "byte", "ubyte", "char"
+        [1, 1]
+      when "short", "ushort"
+        [2, 2]
+      when "int", "uint", "float"
+        [4, 4]
+      when "long", "ulong", "double"
+        [8, 8]
+      when "ptr_int", "ptr_uint", "cstr"
+        [POINTER_SIZE, POINTER_SIZE]
+      else
+        nil
+      end
+    end
+
+    def generic_layout(type, stack)
+      case type.name
+      when "ptr", "const_ptr", "own", "ref"
+        [POINTER_SIZE, POINTER_SIZE]
+      when "array"
+        return unless array_type?(type)
+
+        element_layout = size_and_alignment(type.arguments.first, stack)
+        return unless element_layout
+
+        [element_layout.first * type.arguments[1].value, element_layout.last]
+      when "str_buffer"
+        return unless str_buffer_type?(type)
+
+        with_stack(type, stack) do
+          layout = struct_layout(str_buffer_fields(type), packed: false, alignment: nil, stack:)
+          [layout[:size], layout[:alignment]]
+        end
+      end
+    end
+
+    def variant_layout(type, stack:)
+      payload_layouts = type.arm_names.filter_map do |arm_name|
+        fields = type.arm(arm_name)
+        next if fields.nil? || fields.empty?
+
+        struct_layout(fields, packed: false, alignment: nil, stack:)
+      end
+
+      data_layout = union_layout_from_layouts(payload_layouts, packed: false, alignment: nil)
+      field_infos = [["kind", 4, 4]]
+      field_infos << ["data", data_layout[:size], data_layout[:alignment]] if data_layout[:size].positive?
+      struct_layout_from_infos(field_infos, packed: false, alignment: nil)
+    end
+
+    def struct_layout(fields, packed:, alignment:, stack:)
+      field_infos = fields.map do |field_name, field_type|
+        field_layout = size_and_alignment(field_type, stack)
+        return nil unless field_layout
+
+        [field_name, field_layout.first, field_layout.last]
+      end
+
+      struct_layout_from_infos(field_infos, packed:, alignment:)
+    end
+
+    def union_layout(fields, packed:, alignment:, stack:)
+      field_infos = fields.map do |_field_name, field_type|
+        field_layout = size_and_alignment(field_type, stack)
+        return nil unless field_layout
+
+        field_layout
+      end
+
+      union_layout_from_layouts(field_infos.map { |size, field_alignment| { size:, alignment: field_alignment } }, packed:, alignment:)
+    end
+
+    def struct_layout_from_infos(field_infos, packed:, alignment:)
+      offsets = {}
+      offset = 0
+      natural_alignment = 1
+
+      field_infos.each do |field_name, field_size, field_alignment|
+        effective_alignment = packed ? 1 : field_alignment
+        natural_alignment = [natural_alignment, effective_alignment].max
+        offset = align_up(offset, effective_alignment) unless packed
+        offsets[field_name] = offset
+        offset += field_size
+      end
+
+      overall_alignment = [natural_alignment, alignment || 1].max
+      {
+        size: align_up(offset, overall_alignment),
+        alignment: overall_alignment,
+        offsets:,
+      }
+    end
+
+    def union_layout_from_layouts(layouts, packed:, alignment:)
+      natural_alignment = packed ? 1 : (layouts.map { |layout| layout[:alignment] }.max || 1)
+      overall_alignment = [natural_alignment, alignment || 1].max
+      size = layouts.map { |layout| layout[:size] }.max || 0
+
+      {
+        size: align_up(size, overall_alignment),
+        alignment: overall_alignment,
+      }
+    end
+
+    def ordered_fields(type)
+      type.fields
+    end
+
+    def packed_layout?(type)
+      type.respond_to?(:packed) && type.packed
+    end
+
+    def explicit_alignment(type)
+      type.respond_to?(:alignment) ? type.alignment : nil
+    end
+
+    def array_type?(type)
+      type.arguments.length == 2 && type.arguments[1].is_a?(Types::LiteralTypeArg) && type.arguments[1].value.is_a?(Integer)
+    end
+
+    def str_buffer_type?(type)
+      type.arguments.length == 1 && type.arguments.first.is_a?(Types::LiteralTypeArg) && type.arguments.first.value.is_a?(Integer)
+    end
+
+    def str_buffer_fields(type)
+      storage_capacity = type.arguments.first.value + 1
+      {
+        "data" => Types::Registry.generic_instance("array", [Types::Registry.primitive("char"), Types::LiteralTypeArg.new(storage_capacity)]),
+        "len" => Types::Registry.primitive("ptr_uint"),
+        "dirty" => Types::Registry.primitive("bool"),
+      }
+    end
+
+    def layout_pointer_like_nullable_base?(base)
+      return true if base.is_a?(Types::Function) || base.is_a?(Types::Proc) || base.is_a?(Types::Opaque)
+      return true if base.is_a?(Types::Primitive) && base.name == "cstr"
+
+      base.is_a?(Types::GenericInstance) && %w[ptr const_ptr own ref].include?(base.name)
+    end
+
+    def nullable_opt_layout_fields(base)
+      {
+        "has_value" => Types::Registry.primitive("bool"),
+        "value" => base,
+      }
+    end
+
+    def align_up(value, alignment)
+      return value if alignment <= 1
+
+      remainder = value % alignment
+      remainder.zero? ? value : value + alignment - remainder
+    end
+
+    def with_stack(type, stack)
+      return if stack[type]
+
+      stack[type] = true
+      yield
+    ensure
+      stack.delete(type)
+    end
+  end
+end