rbs 4.1.0.pre.2-java

data/lib/rbs/wasm/deserializer.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require_relative "serialization_schema"
+
+module RBS
+  module WASM
+    # Rebuilds RBS::AST objects from the binary buffer produced by
+    # `rbs_serialize_node` (src/serialize.c), driven by the generated
+    # SerializationSchema. This is the pure-Ruby counterpart of the C extension's
+    # ast_translation.c, used when the parser runs inside WebAssembly (JRuby).
+    #
+    # All locations are reconstructed through the public RBS::Location API, so the
+    # same code works whether RBS::Location is backed by the C extension (CRuby)
+    # or by a pure-Ruby implementation (JRuby).
+    class Deserializer
+      # Deserialize a buffer produced for a whole signature, returning
+      # `[directives, declarations]` to match RBS::Parser._parse_signature.
+      def self.deserialize(bytes, buffer)
+        new(bytes, buffer).read_node
+      end
+
+      # Deserialize a bare node list (rbs_serialize_node_list), e.g. the result
+      # of RBS::Parser._parse_type_params.
+      def self.deserialize_node_list(bytes, buffer)
+        new(bytes, buffer).read_node_list
+      end
+
+      # Deserialize the token stream produced by rbs_wasm_lex into the
+      # [type, location] pairs RBS::Parser._lex returns.
+      def self.deserialize_tokens(bytes, buffer)
+        new(bytes, buffer).read_tokens
+      end
+
+      def initialize(bytes, buffer)
+        @bytes = bytes
+        @buffer = buffer
+        # Symbols and rbs_string fields (comments, annotations) inherit the
+        # source encoding, matching ast_translation.c. String/Integer literal
+        # nodes are always UTF-8 (see read_node).
+        @encoding = buffer.content.encoding
+        @pos = 0
+        @class_cache = {} #: Hash[String, untyped]
+      end
+
+      def read_node
+        tag = read_u8
+        return nil if tag == 0
+        return read_string(@encoding).to_sym if tag == SerializationSchema::SYMBOL_TAG
+
+        entry = SerializationSchema::SCHEMA[tag] or raise "Unknown node tag: #{tag}"
+
+        case entry[0]
+        when :node then read_struct(entry)
+        when :bool then read_u8 != 0
+        when :integer then read_string(Encoding::UTF_8).to_i
+        when :string_value then read_string(Encoding::UTF_8)
+        when :record_field then [read_node, read_u8 != 0]
+        when :signature then [read_node_list, read_node_list]
+        when :namespace then RBS::Namespace[read_node_list, read_u8 != 0]
+        when :type_name then RBS::TypeName[read_node, read_node]
+        else raise "Unknown schema entry kind: #{entry[0].inspect}"
+        end
+      end
+
+      def read_node_list
+        Array.new(read_count) { read_node }
+      end
+
+      # The lex stream has no leading count: read records until the buffer is
+      # exhausted. Each is a token type name followed by its character range.
+      def read_tokens
+        tokens = [] #: Array[[ Symbol, Location ]]
+        until @pos >= @bytes.bytesize
+          type = read_string(Encoding::UTF_8).to_sym
+          start_char = read_i32
+          end_char = read_i32
+          tokens << [type, RBS::Location.new(@buffer, start_char, end_char)]
+        end
+        tokens
+      end
+
+      private
+
+      def read_struct(entry)
+        _, class_name, expose_location, loc_children, fields, resolve_type_params = entry
+
+        location = read_location(loc_children) if expose_location
+
+        kwargs = {} #: Hash[Symbol, untyped]
+        (fields || []).each do |name, reader|
+          kwargs[name] = read_field(reader)
+        end
+
+        RBS::AST::TypeParam.resolve_variables(kwargs[:type_params]) if resolve_type_params
+
+        klass = class_for(class_name)
+        if expose_location
+          klass.new(location: location, **kwargs)
+        else
+          klass.new(**kwargs)
+        end
+      end
+
+      def read_field(reader)
+        case reader
+        when :node then read_node
+        when :node_list then read_node_list
+        when :hash then read_hash
+        when :string then read_string(@encoding)
+        when :bool then read_u8 != 0
+        when :location_range then read_location_value
+        when :location_range_list then read_location_value_list
+        when :attr_ivar_name then read_attr_ivar_name
+        else # [:enum, [value_or_nil, ...]]
+          reader[1][read_u8]
+        end
+      end
+
+      def read_hash
+        hash = {} #: Hash[untyped, untyped]
+        read_count.times do
+          key = read_node
+          hash[key] = read_node
+        end
+        hash
+      end
+
+      # A count of nested items. Each item is at least one byte, so a count that
+      # exceeds the bytes remaining signals the cursor has drifted out of sync.
+      def read_count
+        count = read_u32
+        if count > @bytes.bytesize - @pos
+          raise "Corrupt buffer: count #{count} exceeds #{@bytes.bytesize - @pos} remaining bytes at offset #{@pos}"
+        end
+        count
+      end
+
+      # The base location of a node, followed by its named child ranges.
+      def read_location(loc_children)
+        base = read_range
+        children = (loc_children || []).map { |name, required| [name, required, read_range] }
+
+        return nil unless base
+
+        location = RBS::Location.new(@buffer, base[0], base[1])
+        children.each do |name, required, range|
+          if required
+            location.add_required_child(name, range[0]...range[1]) if range
+          else
+            location.add_optional_child(name, range ? (range[0]...range[1]) : nil)
+          end
+        end
+        location
+      end
+
+      # A standalone location range field: nil or an RBS::Location without children.
+      def read_location_value
+        range = read_range
+        range && RBS::Location.new(@buffer, range[0], range[1])
+      end
+
+      def read_location_value_list
+        Array.new(read_count) { read_location_value }
+      end
+
+      def read_attr_ivar_name
+        case read_u8
+        when 0 then nil   # inferred instance variable
+        when 1 then false # no instance variable
+        else read_string(@encoding).to_sym
+        end
+      end
+
+      # Reads a presence byte and, when present, the start/end character positions.
+      def read_range
+        return nil if read_u8 == 0
+
+        start_char = read_i32
+        end_char = read_i32
+        [start_char, end_char]
+      end
+
+      def read_u8
+        byte = @bytes.getbyte(@pos) or raise "Unexpected end of buffer"
+        @pos += 1
+        byte
+      end
+
+      def read_u32
+        value = @bytes.unpack1("L<", offset: @pos) #: Integer
+        @pos += 4
+        value
+      end
+
+      def read_i32
+        value = @bytes.unpack1("l<", offset: @pos) #: Integer
+        @pos += 4
+        value
+      end
+
+      def read_string(encoding)
+        length = read_u32
+        string = @bytes.byteslice(@pos, length) or raise "Unexpected end of buffer"
+        @pos += length
+        string.force_encoding(encoding)
+      end
+
+      def class_for(name)
+        @class_cache[name] ||= Object.const_get(name)
+      end
+    end
+  end
+end

data/lib/rbs/wasm/location.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RBS
+  # Pure-Ruby implementation of the primitives that back RBS::Location.
+  #
+  # On CRuby these come from the C extension (ext/rbs_extension/legacy_location.c).
+  # JRuby loads this instead, before rbs/location_aux.rb layers the public API on
+  # top, so RBS::Location behaves identically without the native extension.
+  class Location
+    attr_reader :buffer
+
+    def initialize(buffer, start_pos, end_pos)
+      @buffer = buffer
+      @start_pos = start_pos
+      @end_pos = end_pos
+      @required_children = {} #: Hash[Symbol, [ Integer, Integer ]]
+      @optional_children = {} #: Hash[Symbol, [ Integer, Integer ]?]
+    end
+
+    def _start_pos
+      @start_pos
+    end
+
+    def _end_pos
+      @end_pos
+    end
+
+    def _add_required_child(name, start_pos, end_pos)
+      @required_children[name] = [start_pos, end_pos]
+    end
+
+    def _add_optional_child(name, start_pos, end_pos)
+      @optional_children[name] = [start_pos, end_pos]
+    end
+
+    def _add_optional_no_child(name)
+      @optional_children[name] = nil
+    end
+
+    def _required_keys
+      @required_children.keys
+    end
+
+    def _optional_keys
+      @optional_children.keys
+    end
+
+    def [](name)
+      if (range = @required_children[name])
+        return Location.new(@buffer, range[0], range[1])
+      end
+
+      if @optional_children.key?(name)
+        range = @optional_children[name]
+        return range && Location.new(@buffer, range[0], range[1])
+      end
+
+      raise "Unknown child name given: #{name}"
+    end
+  end
+end

data/lib/rbs/wasm/parser.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require_relative "runtime"
+require_relative "deserializer"
+
+module RBS
+  # WebAssembly-backed implementation of the parser primitives.
+  #
+  # On CRuby these come from the C extension (ext/rbs_extension/main.c). JRuby
+  # loads this instead: it runs the parser inside WebAssembly, then rebuilds the
+  # AST with RBS::WASM::Deserializer. rbs/parser_aux.rb layers the public
+  # RBS::Parser API on top, exactly as it does for the C extension.
+  class Parser
+    class << self
+      def _parse_signature(buffer, start_pos, end_pos)
+        validate_position_range(start_pos, end_pos)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_signature(buffer.content, encoding, start_pos, end_pos)
+        raise_parsing_error(buffer, bytes) unless success
+
+        WASM::Deserializer.deserialize(bytes, buffer)
+      end
+
+      def _parse_type(buffer, start_pos, end_pos, variables, require_eof, void_allowed, self_allowed, classish_allowed)
+        validate_position_range(start_pos, end_pos)
+        validate_variables(variables)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_type(buffer.content, encoding, start_pos, end_pos, variables, require_eof, void_allowed, self_allowed, classish_allowed)
+        raise_parsing_error(buffer, bytes) unless success
+
+        deserialize_or_nil(bytes, buffer)
+      end
+
+      def _parse_method_type(buffer, start_pos, end_pos, variables, require_eof)
+        validate_position_range(start_pos, end_pos)
+        validate_variables(variables)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_method_type(buffer.content, encoding, start_pos, end_pos, variables, require_eof)
+        raise_parsing_error(buffer, bytes) unless success
+
+        deserialize_or_nil(bytes, buffer)
+      end
+
+      def _parse_type_params(buffer, start_pos, end_pos, module_type_params)
+        validate_position_range(start_pos, end_pos)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_type_params(buffer.content, encoding, start_pos, end_pos, module_type_params)
+        raise_parsing_error(buffer, bytes) unless success
+
+        bytes.empty? ? nil : WASM::Deserializer.deserialize_node_list(bytes, buffer)
+      end
+
+      def _lex(buffer, end_pos)
+        encoding = buffer.content.encoding.name
+        _success, bytes = WASM::Runtime.instance.lex(buffer.content, encoding, end_pos)
+
+        WASM::Deserializer.deserialize_tokens(bytes, buffer)
+      end
+
+      def _parse_inline_leading_annotation(buffer, start_pos, end_pos, variables)
+        validate_position_range(start_pos, end_pos)
+        validate_variables(variables)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_inline_leading_annotation(buffer.content, encoding, start_pos, end_pos, variables)
+        raise_parsing_error(buffer, bytes) unless success
+
+        deserialize_or_nil(bytes, buffer)
+      end
+
+      def _parse_inline_trailing_annotation(buffer, start_pos, end_pos, variables)
+        validate_position_range(start_pos, end_pos)
+        validate_variables(variables)
+        encoding = buffer.content.encoding.name
+        success, bytes = WASM::Runtime.instance.parse_inline_trailing_annotation(buffer.content, encoding, start_pos, end_pos, variables)
+        raise_parsing_error(buffer, bytes) unless success
+
+        deserialize_or_nil(bytes, buffer)
+      end
+
+      private
+
+      # Reject negative or reversed ranges before handing them to the parser,
+      # matching validate_position_range in the C extension (main.c). A reversed
+      # range would otherwise make the lexer loop forever inside WebAssembly.
+      def validate_position_range(start_pos, end_pos)
+        if start_pos < 0 || end_pos < 0
+          raise ArgumentError, "negative position range: #{start_pos}...#{end_pos}"
+        end
+        if start_pos > end_pos
+          raise ArgumentError, "invalid position range: #{start_pos}...#{end_pos}"
+        end
+      end
+
+      # Reject anything that is not nil or an Array of Symbols, matching
+      # declare_type_variables in the C extension (main.c).
+      def validate_variables(variables)
+        return if variables.nil?
+
+        unless variables.is_a?(Array)
+          raise TypeError, "wrong argument type #{variables.class} (must be an Array of Symbols or nil)"
+        end
+
+        variables.each do |variable|
+          unless variable.is_a?(Symbol)
+            raise TypeError, "Type variables Array contains invalid value #{variable.inspect} of type #{variable.class} (must be an Array of Symbols or nil)"
+          end
+        end
+      end
+
+      # An empty result means the parser reached EOF immediately (`nil`).
+      def deserialize_or_nil(bytes, buffer)
+        bytes.empty? ? nil : WASM::Deserializer.deserialize(bytes, buffer)
+      end
+
+      # Decodes the error blob written by set_error_result (rbs_wasm.c) and raises
+      # the same error the C extension would (see raise_error in main.c).
+      def raise_parsing_error(buffer, blob)
+        start_char, end_char, syntax_error = blob.unpack("l<l<C")
+
+        raise "Unexpected error" if syntax_error.zero?
+
+        offset = 9
+        token_type_length = blob.unpack1("L<", offset: offset)
+        offset += 4
+        token_type = blob.byteslice(offset, token_type_length).to_s.force_encoding(Encoding::UTF_8)
+        offset += token_type_length
+
+        message_length = blob.unpack1("L<", offset: offset)
+        offset += 4
+        message = blob.byteslice(offset, message_length).to_s.force_encoding(Encoding::UTF_8)
+
+        location = Location.new(buffer, start_char, end_char)
+        raise ParsingError.new(location, message, token_type)
+      end
+    end
+  end
+end

data/lib/rbs/wasm/runtime.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "java"
+require "monitor"
+
+module RBS
+  module WASM
+    # Loads rbs_parser.wasm into a JVM WebAssembly runtime (Chicory) and drives
+    # it. This is the JRuby counterpart of the C extension's main.c: it copies a
+    # source string into the module's linear memory, runs the parser, and returns
+    # the serialized result for RBS::WASM::Deserializer to rebuild.
+    #
+    # Chicory is a pure-Java runtime, so there is no native dependency: only the
+    # `.wasm` and the Chicory jars need to ship with the gem.
+    class Runtime
+      include MonitorMixin
+
+      # The Chicory jars the runtime needs at load time.
+      # Jars Chicory needs to load and run the module.
+      JARS = %w[wasm runtime log wasi].freeze
+
+      # Jars for Chicory's ahead-of-time compiler (wasm -> JVM bytecode), which
+      # runs the parser ~8x faster than the interpreter. Optional: the runtime
+      # falls back to the interpreter when they are absent. asm* are the ow2 ASM
+      # libraries the compiler depends on.
+      OPTIONAL_JARS = %w[compiler asm asm-tree asm-util asm-commons asm-analysis].freeze
+
+      class << self
+        def instance
+          @instance ||= new
+        end
+
+        def wasm_path
+          ENV["RBS_WASM_PARSER"] || File.expand_path("rbs_parser.wasm", __dir__)
+        end
+
+        def jars_dir
+          ENV["RBS_WASM_JARS"] || File.expand_path("jars", __dir__)
+        end
+      end
+
+      def initialize
+        super()
+        load_jars
+        @wasm = build_instance
+        @memory = @wasm.memory
+        @alloc = @wasm.export("rbs_wasm_alloc")
+        @free = @wasm.export("rbs_wasm_free")
+        @result_ptr = @wasm.export("rbs_wasm_result_ptr")
+        @result_len = @wasm.export("rbs_wasm_result_len")
+        @parse_signature = @wasm.export("rbs_wasm_parse_signature")
+        @parse_type = @wasm.export("rbs_wasm_parse_type")
+        @parse_method_type = @wasm.export("rbs_wasm_parse_method_type")
+        @parse_type_params = @wasm.export("rbs_wasm_parse_type_params")
+        @parse_inline_leading_annotation = @wasm.export("rbs_wasm_parse_inline_leading_annotation")
+        @parse_inline_trailing_annotation = @wasm.export("rbs_wasm_parse_inline_trailing_annotation")
+        @lex = @wasm.export("rbs_wasm_lex")
+      end
+
+      # `content` is the whole buffer; `start_pos`/`end_pos` are the character
+      # range within it to parse. Each method returns [success, bytes]: on success
+      # `bytes` is the serialized AST, otherwise it is the error blob (see
+      # set_error_result in rbs_wasm.c).
+
+      def parse_signature(content, encoding, start_pos, end_pos)
+        run(content, encoding) { |ptr, len, enc_ptr, enc_len| @parse_signature.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos)[0] }
+      end
+
+      def parse_type(content, encoding, start_pos, end_pos, variables, require_eof, void_allowed, self_allowed, classish_allowed)
+        with_variables(variables) do |vars_ptr, vars_len|
+          run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+            @parse_type.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos, vars_ptr, vars_len, bool(require_eof), bool(void_allowed), bool(self_allowed), bool(classish_allowed))[0]
+          end
+        end
+      end
+
+      def parse_method_type(content, encoding, start_pos, end_pos, variables, require_eof)
+        with_variables(variables) do |vars_ptr, vars_len|
+          run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+            @parse_method_type.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos, vars_ptr, vars_len, bool(require_eof))[0]
+          end
+        end
+      end
+
+      def parse_type_params(content, encoding, start_pos, end_pos, module_type_params)
+        run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+          @parse_type_params.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos, bool(module_type_params))[0]
+        end
+      end
+
+      def parse_inline_leading_annotation(content, encoding, start_pos, end_pos, variables)
+        with_variables(variables) do |vars_ptr, vars_len|
+          run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+            @parse_inline_leading_annotation.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos, vars_ptr, vars_len)[0]
+          end
+        end
+      end
+
+      def parse_inline_trailing_annotation(content, encoding, start_pos, end_pos, variables)
+        with_variables(variables) do |vars_ptr, vars_len|
+          run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+            @parse_inline_trailing_annotation.apply(ptr, len, enc_ptr, enc_len, start_pos, end_pos, vars_ptr, vars_len)[0]
+          end
+        end
+      end
+
+      def lex(content, encoding, end_pos)
+        run(content, encoding) do |ptr, len, enc_ptr, enc_len|
+          @lex.apply(ptr, len, enc_ptr, enc_len, end_pos)[0]
+        end
+      end
+
+      private
+
+      # Copies `source` and its encoding name into linear memory, yields their
+      # pointers/lengths to the block (which invokes the parser and returns its
+      # status), then reads the result back out. Serialized through the monitor
+      # because the module keeps its result in a single shared location.
+      def run(source, encoding)
+        synchronize do
+          bytes = source.b
+          length = bytes.bytesize
+          name = encoding.to_s.b
+          name_length = name.bytesize
+          source_ptr = @alloc.apply(length)[0]
+          name_ptr = @alloc.apply(name_length)[0]
+          begin
+            @memory.write(source_ptr, bytes.to_java_bytes)
+            @memory.write(name_ptr, name.to_java_bytes) unless name_length.zero?
+            status = yield(source_ptr, length, name_ptr, name_length)
+            [status == 1, read_result]
+          ensure
+            @free.apply(source_ptr)
+            @free.apply(name_ptr)
+          end
+        end
+      end
+
+      def read_result
+        pointer = @result_ptr.apply[0]
+        length = @result_len.apply[0]
+        return "".b if length.zero?
+
+        String.from_java_bytes(@memory.read_bytes(pointer, length)).b
+      end
+
+      # Allocates a buffer of newline-separated variable names and yields its
+      # pointer/length. A nil `variables` is passed as length -1 ("no variables").
+      def with_variables(variables)
+        names = variables&.map(&:to_s)&.join("\n")
+
+        if names.nil? || names.empty?
+          return yield(0, variables.nil? ? -1 : 0)
+        end
+
+        bytes = names.b
+        length = bytes.bytesize
+        synchronize do
+          pointer = @alloc.apply(length)[0]
+          begin
+            @memory.write(pointer, bytes.to_java_bytes)
+            yield(pointer, length)
+          ensure
+            @free.apply(pointer)
+          end
+        end
+      end
+
+      def bool(value)
+        value ? 1 : 0
+      end
+
+      def build_instance
+        parser = Java::ComDylibsoChicoryWasm::Parser
+        instance_class = Java::ComDylibsoChicoryRuntime::Instance
+        import_values = Java::ComDylibsoChicoryRuntime::ImportValues
+        wasi_preview1 = Java::ComDylibsoChicoryWasi::WasiPreview1
+        wasi_options = Java::ComDylibsoChicoryWasi::WasiOptions
+
+        wasm_module = parser.parse(java.io.File.new(self.class.wasm_path))
+        wasi = wasi_preview1.builder.with_options(wasi_options.builder.build).build
+        imports = import_values.builder.add_function(wasi.to_host_functions).build
+
+        builder = instance_class.builder(wasm_module).with_import_values(imports)
+        if (factory = machine_factory(wasm_module))
+          builder = builder.with_machine_factory(factory)
+        end
+
+        wasm = builder.build
+        wasm.export("_initialize").apply
+        wasm
+      end
+
+      # Chicory's AOT compiler when its jars are present and usable, otherwise nil
+      # (the builder then uses the interpreter). NameError covers a missing
+      # compiler class; LinkageError covers an incompatible/missing ASM (so a bad
+      # jar set degrades to the interpreter instead of crashing).
+      def machine_factory(wasm_module)
+        Java::ComDylibsoChicoryCompiler::MachineFactoryCompiler.compile(wasm_module)
+      rescue NameError, Java::JavaLang::LinkageError
+        nil
+      end
+
+      def load_jars
+        JARS.each { |name| require jar_path(name) }
+        OPTIONAL_JARS.each do |name|
+          path = jar_path(name)
+          require path if File.exist?(path)
+        end
+      end
+
+      def jar_path(name)
+        File.join(self.class.jars_dir, "#{name}.jar")
+      end
+    end
+  end
+end