mt-lang 0.2.0

data/lib/milk_tea/bindings/imported_bindings/method_source.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module MilkTea
+  module ImportedBindings
+    class Generator
+      module GeneratorMethodSource
+        private
+
+        def load_method_sources(method_specs)
+          method_specs.each_with_object({}) do |spec, sources|
+            next unless spec[:module_name]
+
+            key = method_source_key(spec)
+            next if sources.key?(key)
+
+            sources[key] = load_method_source(spec)
+          end
+        end
+
+        def load_method_source(spec)
+          module_name = spec.fetch(:module_name)
+          module_path = resolve_module_path(module_name)
+          source_ast = ModuleLoader.new(module_roots: @module_roots).load_file(module_path)
+          unless source_ast.module_name&.to_s == module_name
+            raise Error, "expected #{module_path} to define module #{module_name}"
+          end
+
+          declarations = index_method_source_declarations(source_ast)
+          MethodSource.new(
+            module_name:,
+            module_path:,
+            module_kind: source_ast.module_kind,
+            import_alias: spec.fetch(:module_import_alias),
+            imports_by_alias: source_ast.imports.to_h { |import| [import.alias_name, import.path.parts.join(".")] },
+            public_type_names: declarations[:types].keys,
+            functions: declarations[:functions],
+            function_order: declarations[:function_order],
+            import_specs: [{ module_name:, alias: spec.fetch(:module_import_alias) }],
+          )
+        end
+
+        def method_source_import_specs(method_sources)
+          method_sources.values.flat_map(&:import_specs)
+        end
+
+        def method_source_key(spec)
+          [spec.fetch(:module_name), spec.fetch(:module_import_alias)]
+        end
+
+        def index_method_source_declarations(source_ast)
+          types = {}
+          type_order = []
+          functions = {}
+          function_order = []
+
+          source_ast.declarations.each do |declaration|
+            case declaration
+            when AST::TypeAliasDecl, AST::StructDecl, AST::UnionDecl, AST::EnumDecl, AST::FlagsDecl, AST::OpaqueDecl
+              next unless visible_from_method_source?(declaration, module_kind: source_ast.module_kind)
+
+              types[declaration.name] = declaration
+              type_order << declaration.name
+            when AST::ExternFunctionDecl, AST::ForeignFunctionDecl
+              next unless visible_from_method_source?(declaration, module_kind: source_ast.module_kind)
+
+              functions[declaration.name] = declaration
+              function_order << declaration.name
+            end
+          end
+
+          {
+            types:,
+            type_order:,
+            functions:,
+            function_order:,
+          }
+        end
+
+        def visible_from_method_source?(declaration, module_kind:)
+          return true if module_kind == :raw_module
+
+          declaration.respond_to?(:visibility) && declaration.visibility == :public
+        end
+
+        def plan_method_source_function(declaration, source:)
+          if declaration.respond_to?(:variadic) && declaration.variadic
+            raise Error, "method generation for #{declaration.name} in #{source.module_path} cannot use variadic functions"
+          end
+
+          {
+            raw_name: declaration.name,
+            public_name: declaration.name,
+            call_name: "#{source.import_alias}.#{declaration.name}",
+            type_params: raw_type_param_names(declaration),
+            params: declaration.params.map { |param| project_method_source_param(param, source:) },
+            return_type: project_method_source_type(declaration.return_type, source:),
+          }
+        end
+
+        def project_method_source_param(param, source:)
+          case param
+          when AST::ForeignParam
+            {
+              "name" => param.name,
+              "type" => project_method_source_type(param.type, source:),
+              "mode" => normalize_method_source_param_mode(param.mode),
+            }
+          when AST::Param
+            {
+              "name" => param.name,
+              "type" => project_method_source_type(param.type, source:),
+            }
+          else
+            raise Error, "unsupported method source parameter #{param.class.name} in #{source.module_path}"
+          end
+        end
+
+        def normalize_method_source_param_mode(mode)
+          return nil if mode == :plain || mode.nil?
+
+          mode.to_s
+        end
+
+        def project_method_source_type(type, source:)
+          case type
+          when AST::TypeRef
+            text = project_method_source_type_name(type.name.to_s, source:)
+            unless type.arguments.empty?
+              rendered_arguments = type.arguments.map { |argument| project_method_source_type_argument(argument.value, source:) }
+              text << "[#{rendered_arguments.join(', ')}]"
+            end
+            text << "?" if type.nullable
+            text
+          when AST::FunctionType
+            params = type.params.map { |param| project_method_source_function_type_param(param, source:) }.join(', ')
+            "fn(#{params}) -> #{project_method_source_type(type.return_type, source:)}"
+          else
+            raise Error, "unsupported method source type node #{type.class.name} in #{source.module_path}"
+          end
+        end
+
+        def project_method_source_function_type_param(param, source:)
+          case param
+          when AST::Param
+            "#{param.name}: #{project_method_source_type(param.type, source:)}"
+          when AST::ForeignParam
+            text = +""
+            text << "#{param.mode} " if param.mode
+            text << "#{param.name}: #{project_method_source_type(param.type, source:)}"
+            if param.boundary_type
+              text << " as #{project_method_source_type(param.boundary_type, source:)}"
+            end
+            text
+          else
+            project_method_source_type(param, source:)
+          end
+        end
+
+        def project_method_source_type_argument(argument, source:)
+          case argument
+          when AST::TypeRef, AST::FunctionType
+            project_method_source_type(argument, source:)
+          when AST::IntegerLiteral
+            argument.lexeme
+          when AST::Identifier
+            argument.name
+          else
+            raise Error, "unsupported method source type argument #{argument.class.name} in #{source.module_path}"
+          end
+        end
+
+        def project_method_source_type_name(raw_name, source:)
+          parts = raw_name.split(".")
+          if parts.length > 1
+            import_alias = parts.first
+            imported_name = parts[1..].join(".")
+            imported_module_name = source.imports_by_alias[import_alias]
+            return rendered_type_name(imported_name) if imported_module_name == @raw_module_name
+            return imported_name if imported_module_name == @module_name
+            return raw_name if imported_module_name
+          end
+
+          return "#{source.import_alias}.#{raw_name}" if source.public_type_names.include?(raw_name)
+
+          rendered_type_name(raw_name)
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/bindings/imported_bindings/naming.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+module MilkTea
+  module ImportedBindings
+    class Generator
+      module GeneratorNaming
+        OPENGL_TYPED_SUFFIX_TOKENS = [
+          ["ui64v", %w[uint64 values]],
+          ["i64v", %w[int64 values]],
+          ["uiv", %w[uint values]],
+          ["iv", %w[int values]],
+          ["fi", %w[float int]],
+          ["fv", %w[float values]],
+          ["dv", %w[double values]],
+          ["ubv", %w[ubyte values]],
+          ["usv", %w[ushort values]],
+          ["bv", %w[byte values]],
+          ["sv", %w[short values]],
+          ["ui64", %w[uint64]],
+          ["i64", %w[int64]],
+          ["ui", %w[uint]],
+          ["i", %w[int]],
+          ["f", %w[float]],
+          ["d", %w[double]],
+          ["ub", %w[ubyte]],
+          ["us", %w[ushort]],
+          ["b", %w[byte]],
+          ["s", %w[short]],
+          ["v", %w[values]],
+        ].freeze
+        OPENGL_DOMAIN_MARKERS = {
+          "i" => "integer",
+          "l" => "long",
+          "p" => "packed",
+        }.freeze
+        OPENGL_INDEXED_PARAM_NAMES = %w[buf index mask_number].freeze
+        OPENGL_TYPED_ALPHA_STEMS = %w[
+          array
+          attrib
+          block
+          boolean
+          buffer
+          double
+          feedback
+          float
+          framebuffer
+          indexed
+          integer
+          internalformat
+          interface
+          multisample
+          object
+          parameter
+          pipeline
+          pointer
+          program
+          query
+          resource
+          shader
+          stage
+          subroutine
+          subroutines
+          sync
+          uniform
+          uniforms
+        ].freeze
+        OPENGL_TYPED_QUERY_BASES = %w[boolean double float integer].freeze
+        OPENGL_NUMERIC_SUFFIX_CONTEXTS = %w[
+          attrib
+          boolean
+          double
+          feedback
+          float
+          indexed
+          int
+          integer
+          integer64
+          long
+          matrix
+          normalized
+          object
+          packed
+          parameter
+          uint
+          uniform
+        ].freeze
+
+        private
+
+        def normalize_opengl_snake_case(name)
+          tokens = name.to_s.split("_").reject(&:empty?)
+          normalized = []
+          index = 0
+
+          while index < tokens.length
+            token = tokens[index]
+            next_token = tokens[index + 1]
+
+            case token
+            when "getn"
+              normalized.concat(%w[get n])
+              index += 1
+              next
+            when "64i"
+              if next_token == "v"
+                if normalized.last == "integer"
+                  normalized[-1] = "integer64"
+                  normalized.concat(%w[indexed values])
+                else
+                  normalized.concat(%w[int64 indexed values])
+                end
+                index += 2
+                next
+              end
+            when "i"
+              if next_token == "v"
+                if OPENGL_TYPED_QUERY_BASES.include?(normalized.last)
+                  normalized.concat(%w[indexed values])
+                else
+                  normalized.concat(%w[int indexed values])
+                end
+                index += 2
+                next
+              end
+            when "i64"
+              if next_token == "v"
+                normalized.concat(%w[int64 indexed values])
+                index += 2
+                next
+              end
+            when "ui64"
+              if next_token == "v"
+                normalized.concat(%w[uint64 indexed values])
+                index += 2
+                next
+              end
+            when "64v"
+              case normalized.last
+              when "integer"
+                normalized[-1] = "integer64"
+                normalized << "values"
+                index += 1
+                next
+              when "int"
+                normalized[-1] = "int64"
+                normalized << "values"
+                index += 1
+                next
+              when "uint"
+                normalized[-1] = "uint64"
+                normalized << "values"
+                index += 1
+                next
+              end
+            end
+
+            if (domain_marker = opengl_domain_marker(token, next_token))
+              normalized << domain_marker
+              index += 1
+              next
+            end
+
+            if (expanded = expand_opengl_special_token(token, numeric_context: opengl_numeric_suffix_context?(normalized)))
+              normalized.concat(expanded)
+              index += 1
+              next
+            end
+
+            normalized << token
+            index += 1
+          end
+
+          normalized.join("_")
+        end
+
+        def normalize_opengl_terminal_suffix(name, raw_name)
+          normalized = if name.end_with?("indexedfv")
+                         "#{name.delete_suffix('indexedfv')}indexed_float_values"
+                       elsif name.end_with?("indexedf")
+                         "#{name.delete_suffix('indexedf')}indexed_float"
+                       else
+                         name
+                       end
+
+          raw_text = raw_name.to_s
+          if raw_text.end_with?("i") && normalized.end_with?("i")
+            suffix = opengl_indexed_terminal_variant?(raw_name) ? "indexed" : "int"
+            return "#{normalized.delete_suffix('i')}_#{suffix}"
+          end
+
+          return "#{normalized.delete_suffix('f')}_float" if raw_text.end_with?("f") && normalized.end_with?("f")
+
+          normalized
+        end
+
+        def opengl_indexed_terminal_variant?(raw_name)
+          raw_declaration = @raw_function_declarations[raw_name]
+          return false unless raw_declaration
+
+          raw_declaration.params
+            .map { |param| snake_case(param.name) }
+            .any? { |name| OPENGL_INDEXED_PARAM_NAMES.include?(name) }
+        end
+
+        def opengl_domain_marker(token, next_token)
+          return unless OPENGL_DOMAIN_MARKERS.key?(token)
+          return unless next_token
+          return unless next_token == "format" || next_token == "pointer" || next_token.match?(/\A\d/)
+
+          OPENGL_DOMAIN_MARKERS.fetch(token)
+        end
+
+        def opengl_numeric_suffix_context?(normalized_tokens)
+          last_token = normalized_tokens.last
+          last_token && OPENGL_NUMERIC_SUFFIX_CONTEXTS.include?(last_token)
+        end
+
+        def expand_opengl_special_token(token, numeric_context:)
+          return %w[integer int values] if token == "iiv"
+          return %w[integer uint values] if token == "iuiv"
+
+          if token.start_with?("n") && token.length > 1
+            expanded = expand_opengl_typed_token(token[1..], numeric_context: true)
+            return ["normalized", *expanded] if expanded
+          end
+
+          expand_opengl_typed_token(token, numeric_context:)
+        end
+
+        def expand_opengl_typed_token(token, numeric_context:)
+          OPENGL_TYPED_SUFFIX_TOKENS.each do |suffix, replacement|
+            next unless token.end_with?(suffix)
+
+            prefix = token.delete_suffix(suffix)
+            if prefix.empty?
+              return nil if suffix == "v"
+              return replacement if numeric_context
+
+              next
+            end
+
+            if prefix.match?(/\A\d+\z|\A\d+x\d+\z/)
+              return nil unless numeric_context
+
+              return [prefix, *replacement]
+            end
+
+            if prefix.match?(/\A[a-z]+\z/) && OPENGL_TYPED_ALPHA_STEMS.include?(prefix)
+              return [prefix, *replacement]
+            end
+          end
+
+          nil
+        end
+
+        def snake_case(name)
+          name.to_s
+            .gsub(/([A-Z]+)([A-Z][a-z])/, '\\1_\\2')
+            .gsub(/([a-z0-9])([A-Z])/, '\\1_\\2')
+            .downcase
+            .gsub(/([a-z])(\d+)_d\b/, '\1_\2d')
+        end
+
+        def camelize_binding_name(name)
+          parts = name.to_s.split("_").reject(&:empty?)
+          return "" if parts.empty?
+
+          parts.map do |part|
+            if part.match?(/\A\d+\z/)
+              part
+            else
+              part[0].upcase + part[1..].to_s.downcase
+            end
+          end.join
+        end
+
+        def openglize_binding_name(name)
+          name.to_s
+            .gsub(/([A-Za-z])((?:i64|i|ui)_v)(?=\z|_)/, '\\1_\\2')
+            .gsub(/(?<!_)([A-Za-z])(\d[A-Za-z0-9]*)(?=[A-Z_]|\z)/, '\1_\2')
+            .gsub(/_(\d)D(?=[A-Z_]|\z)/, '_\1d')
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/bindings/imported_bindings.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../tooling/formatter"
+require_relative "../core/token"
+require_relative "../core/types/types"
+
+require_relative "imported_bindings/generator"
+require_relative "imported_bindings/method_source"
+require_relative "imported_bindings/naming"
+require_relative "imported_bindings/defaults"
+
+module MilkTea
+  module ImportedBindings
+    class Error < StandardError; end
+
+    class Binding
+      attr_reader :name, :module_name, :binding_path, :raw_module_name, :policy_path, :import_alias
+
+      def initialize(name:, module_name:, binding_path:, raw_module_name:, policy_path:, import_alias: "c")
+        @name = name.to_s
+        @module_name = module_name
+        @binding_path = File.expand_path(binding_path.to_s)
+        @raw_module_name = raw_module_name
+        @policy_path = File.expand_path(policy_path.to_s)
+        @import_alias = import_alias
+      end
+
+      def task_name
+        "imported_bindings:#{name}"
+      end
+
+      def check_task_name
+        "imported_bindings:check:#{name}"
+      end
+
+      def generate(module_roots: [MilkTea.root])
+        Generator.new(
+          module_name:,
+          raw_module_name:,
+          raw_module_path: resolve_module_path(raw_module_name, module_roots),
+          policy_path:,
+          import_alias:,
+          module_roots:,
+        ).generate
+      end
+
+      def write!(module_roots: [MilkTea.root])
+        raw_module_path = resolve_module_path(raw_module_name, module_roots)
+        source = Generator.new(
+          module_name:,
+          raw_module_name:,
+          raw_module_path:,
+          policy_path:,
+          import_alias:,
+          module_roots:,
+        ).generate
+        File.write(binding_path, source)
+        raw_module_path
+      end
+
+      def check!(module_roots: [MilkTea.root])
+        raw_module_path = resolve_module_path(raw_module_name, module_roots)
+        actual = Generator.new(
+          module_name:,
+          raw_module_name:,
+          raw_module_path:,
+          policy_path:,
+          import_alias:,
+          module_roots:,
+        ).generate
+        expected = File.read(binding_path)
+
+        if expected != actual
+          raise Error, <<~MESSAGE
+            #{binding_path} is out of date for #{raw_module_path} and #{policy_path}
+            Run `rake #{task_name}` to regenerate it.
+          MESSAGE
+        end
+
+        MilkTea::ModuleLoader.new(module_roots:).check_file(binding_path)
+        raw_module_path
+      end
+
+      private
+
+      def resolve_module_path(module_name, module_roots)
+        relative_path = File.join(*module_name.split(".")) + ".mt"
+        candidate = module_roots.lazy.map { |root| File.join(File.expand_path(root.to_s), relative_path) }.find { |path| File.file?(path) }
+        raise Error, "module not found: #{module_name}" unless candidate
+
+        File.expand_path(candidate)
+      end
+    end
+
+    class Registry
+      include Enumerable
+
+      def initialize(bindings = [])
+        @bindings = {}
+        @bindings_by_module_name = {}
+        bindings.each { |binding| register(binding) }
+      end
+
+      def register(binding)
+        raise Error, "duplicate imported binding #{binding.name}" if @bindings.key?(binding.name)
+        raise Error, "duplicate imported binding module #{binding.module_name}" if @bindings_by_module_name.key?(binding.module_name)
+
+        @bindings[binding.name] = binding
+        @bindings_by_module_name[binding.module_name] = binding
+      end
+
+      def fetch(name)
+        @bindings.fetch(name.to_s)
+      rescue KeyError
+        raise Error, "unknown imported binding #{name}"
+      end
+
+      def each(&block)
+        @bindings.each_value(&block)
+      end
+
+      def find_by_module_name(module_name)
+        @bindings_by_module_name[module_name]
+      end
+
+      def task_names
+        map(&:task_name)
+      end
+
+      def check_task_names
+        map(&:check_task_name)
+      end
+    end
+
+    class Generator
+      MethodSource = Data.define(:module_name, :module_path, :module_kind, :import_alias, :imports_by_alias, :public_type_names, :functions, :function_order, :import_specs)
+
+      def initialize(module_name:, raw_module_name:, raw_module_path:, policy_path:, import_alias:, module_roots:)
+        @module_name = module_name
+        @raw_module_name = raw_module_name
+        @raw_module_path = File.expand_path(raw_module_path)
+        @policy_path = File.expand_path(policy_path)
+        @import_alias = import_alias
+        @module_roots = module_roots.map { |root| File.expand_path(root.to_s) }
+        @public_type_names_by_raw_name = {}
+        @public_type_kinds_by_raw_name = {}
+        @raw_function_declarations = {}
+        @raw_type_declarations = {}
+      end
+
+      def generate
+        policy = load_policy
+        validate_policy!(policy)
+
+        raw_ast = ModuleLoader.new(module_roots: @module_roots).load_file(@raw_module_path)
+        validate_raw_module!(raw_ast)
+        declarations = index_raw_declarations(raw_ast)
+        import_specs = raw_import_specs(raw_ast)
+        validate_import_specs!(import_specs)
+        extra_import_specs = policy_import_specs(policy)
+        type_spec = normalize_alias_spec(policy["types"], context: "type")
+        @native_type_mapping = type_spec[:native_types]
+        const_spec = normalize_alias_spec(policy["constants"], context: "constant")
+        function_spec = normalize_function_spec(policy["functions"])
+        method_specs = normalize_method_specs(policy["methods"])
+        method_sources = load_method_sources(method_specs)
+        @raw_function_declarations = declarations[:functions]
+        @raw_type_declarations = declarations[:types]
+        @public_type_names_by_raw_name = build_public_type_names(type_spec, declarations)
+        function_entries = plan_foreign_functions(function_spec, declarations)
+        generated_import_specs = merge_import_specs(raw_import_specs(raw_ast), extra_import_specs, method_source_import_specs(method_sources))
+        validate_import_specs!(generated_import_specs)
+        const_lines = emit_const_aliases(const_spec, declarations)
+        function_lines = emit_foreign_functions(function_entries)
+        method_lines = emit_methods(method_specs, function_entries, declarations, method_sources:)
+        type_lines = emit_type_aliases(
+          type_spec,
+          declarations,
+          referenced_lines: const_lines + function_lines + method_lines,
+          method_sources:,
+        )
+
+        lines = []
+        lines << "# generated by mtc imported-bindings from #{@raw_module_name} using #{policy_label}"
+        lines << ""
+        lines << "import #{@raw_module_name} as #{@import_alias}"
+        generated_import_specs.each do |spec|
+          lines << "import #{spec[:module_name]} as #{spec[:alias]}"
+        end
+
+        sections = [
+          type_lines,
+          const_lines,
+          function_lines,
+          method_lines,
+        ].reject(&:empty?)
+
+        sections.each do |section_lines|
+          lines << ""
+          lines.concat(section_lines)
+        end
+
+        source = lines.join("\n") + "\n"
+        Formatter.format_source(source, path: generated_module_path, mode: :tidy)
+      end
+
+      private
+
+      include GeneratorPolicy
+      include GeneratorMethodSource
+      include GeneratorNaming
+    end
+  end
+end