rbs 4.0.3 → 4.1.0.pre.2

data/include/rbs/defines.h

@@ -83,4 +83,17 @@
 #define RBS_ATTRIBUTE_UNUSED
 #endif
 
+/**
+ * Nullability annotations for pointer types.
+ * Clang supports _Nullable and _Nonnull to indicate whether a pointer may be NULL.
+ * On other compilers, these expand to nothing.
+ */
+#ifdef __clang__
+#define RBS_NULLABLE _Nullable
+#define RBS_NONNULL _Nonnull
+#else
+#define RBS_NULLABLE
+#define RBS_NONNULL
+#endif
+
 #endif

data/include/rbs/lexer.h

@@ -70,6 +70,7 @@ enum RBSTokenType {
     kATRBS,        /* @rbs */
     kSKIP,         /* skip */
     kRETURN,       /* return */
+    kMODULESELF,   /* module-self */
 
     tLIDENT,    /* Identifiers starting with lower case */
     tUIDENT,    /* Identifiers starting with upper case */

data/include/rbs/serialize.h

@@ -0,0 +1,39 @@
+/*----------------------------------------------------------------------------*/
+/* This file is generated by the templates/template.rb script and should not  */
+/* be modified manually.                                                      */
+/* To change the template see                                                 */
+/* templates/include/rbs/serialize.h.erb                                      */
+/*----------------------------------------------------------------------------*/
+
+#ifndef RBS__SERIALIZE_H
+#define RBS__SERIALIZE_H
+
+#include "rbs/ast.h"
+#include "rbs/string.h"
+#include "rbs/util/rbs_allocator.h"
+#include "rbs/util/rbs_constant_pool.h"
+
+/**
+ * Serialize a parsed AST node into a compact, portable binary buffer.
+ *
+ * The format is consumed by RBS::WASM::Deserializer on the Ruby side, which
+ * rebuilds the same `RBS::AST` objects that the C extension would have built
+ * directly. This is what lets RBS run on Ruby implementations that cannot load
+ * the C extension (notably JRuby): the parser runs inside WebAssembly, produces
+ * this buffer, and the host reconstructs the tree in pure Ruby.
+ *
+ * The buffer is allocated from `allocator`, so its lifetime is tied to that
+ * allocator. `constant_pool` must be the pool the node was parsed with; it is
+ * used to resolve interned symbol/identifier ids back into their bytes.
+ *
+ * See `docs/wasm_serialization.md` for the wire format.
+ */
+rbs_string_t rbs_serialize_node(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_t *node);
+
+/**
+ * Like rbs_serialize_node, but for a bare node list (e.g. the result of
+ * rbs_parse_type_params). Decoded by RBS::WASM::Deserializer.deserialize_node_list.
+ */
+rbs_string_t rbs_serialize_node_list(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_list_t *list);
+
+#endif

data/lib/rbs/annotate/rdoc_annotator.rb

@@ -13,25 +13,22 @@ module RBS
         @include_filename = true
       end
 
-      def annotate_file(path, preserve:)
-        content = path.read()
+      def annotate_file(path, preserve: true)
+        buffer, _, decls = Parser.parse_signature(path.read())
 
-        _, _, decls = Parser.parse_signature(content)
+        rewriter = Rewriter.new(buffer)
+        annotate_decls(decls, rewriter)
 
-        annotate_decls(decls)
-
-        path.open("w") do |io|
-          Writer.new(out: io).preserve!(preserve: preserve).write(decls)
-        end
+        path.write(rewriter.string)
       end
 
-      def annotate_decls(decls, outer: [])
+      def annotate_decls(decls, rewriter, outer: [])
         decls.each do |decl|
           case decl
           when AST::Declarations::Class, AST::Declarations::Module
-            annotate_class(decl, outer: outer)
+            annotate_class(decl, rewriter, outer: outer)
           when AST::Declarations::Constant
-            annotate_constant(decl, outer: outer)
+            annotate_constant(decl, rewriter, outer: outer)
           end
         end
       end
@@ -244,7 +241,7 @@ module RBS
         end
       end
 
-      def annotate_class(decl, outer:)
+      def annotate_class(decl, rewriter, outer:)
         annots = annotations(decl)
 
         full_name = resolve_name(decl.name, outer: outer)
@@ -252,7 +249,7 @@ module RBS
           text = resolve_doc_source(annots.copy_annotation, tester: annots) { doc_for_class(full_name, tester: annots) }
         end
 
-        replace_comment(decl, text)
+        replace_comment(decl, text, rewriter)
 
         unless annots.skip_all?
           outer_ = outer + [decl.name.to_namespace]
@@ -260,28 +257,28 @@ module RBS
           decl.each_member do |member|
             case member
             when AST::Members::MethodDefinition
-              annotate_method(full_name, member)
+              annotate_method(full_name, member, rewriter)
             when AST::Members::Alias
-              annotate_alias(full_name, member)
+              annotate_alias(full_name, member, rewriter)
             when AST::Members::AttrReader, AST::Members::AttrAccessor, AST::Members::AttrWriter
-              annotate_attribute(full_name, member)
+              annotate_attribute(full_name, member, rewriter)
             end
           end
 
-          annotate_decls(decl.each_decl.to_a, outer: outer_)
+          annotate_decls(decl.each_decl.to_a, rewriter, outer: outer_)
         end
       end
 
-      def annotate_constant(const, outer:)
+      def annotate_constant(const, rewriter, outer:)
         annots = Annotations.new([])
 
         full_name = resolve_name(const.name, outer: outer)
         text = doc_for_constant(full_name, tester: annots)
 
-        replace_comment(const, text)
+        replace_comment(const, text, rewriter)
       end
 
-      def annotate_alias(typename, als)
+      def annotate_alias(typename, als, rewriter)
         annots = annotations(als)
 
         unless annots.skip?
@@ -295,7 +292,7 @@ module RBS
           end
         end
 
-        replace_comment(als, text)
+        replace_comment(als, text, rewriter)
       end
 
       def join_docs(docs, separator: "----")
@@ -311,7 +308,7 @@ module RBS
         end
       end
 
-      def annotate_method(typename, method)
+      def annotate_method(typename, method, rewriter)
         annots = annotations(method)
 
         unless annots.skip?
@@ -337,10 +334,10 @@ module RBS
           }
         end
 
-        replace_comment(method, text)
+        replace_comment(method, text, rewriter)
       end
 
-      def annotate_attribute(typename, attr)
+      def annotate_attribute(typename, attr, rewriter)
         annots = annotations(attr)
 
         unless annots.skip?
@@ -368,18 +365,17 @@ module RBS
           end
         end
 
-        replace_comment(attr, text)
+        replace_comment(attr, text, rewriter)
       end
 
-      def replace_comment(commented, string)
+      def replace_comment(commented, string, rewriter)
         if string
           if string.empty?
-            commented.instance_variable_set(:@comment, nil)
+            rewriter.delete_comment(commented.comment) if commented.comment
+          elsif commented.comment
+            rewriter.replace_comment(commented.comment, content: string)
           else
-            commented.instance_variable_set(
-              :@comment,
-              AST::Comment.new(location: nil, string: string)
-            )
+            rewriter.add_comment(commented.location || raise, *commented.annotations.filter_map(&:location), content: string)
           end
         end
       end

data/lib/rbs/ast/ruby/annotations.rb

@@ -353,6 +353,48 @@ module RBS
           end
         end
 
+        class ModuleSelfAnnotation < Base
+          attr_reader :keyword_location, :colon_location, :name, :args, :open_bracket_location, :close_bracket_location, :args_comma_locations, :comment_location
+
+          def initialize(location:, prefix_location:, keyword_location:, colon_location:, name:, args:, open_bracket_location:, close_bracket_location:, args_comma_locations:, comment_location:)
+            super(location, prefix_location)
+            @keyword_location = keyword_location
+            @colon_location = colon_location
+            @name = name
+            @args = args
+            @open_bracket_location = open_bracket_location
+            @close_bracket_location = close_bracket_location
+            @args_comma_locations = args_comma_locations
+            @comment_location = comment_location
+          end
+
+          def map_type_name
+            mapped_args = args.map { |type| type.map_type_name { yield _1 } }
+
+            self.class.new(
+              location:,
+              prefix_location:,
+              keyword_location:,
+              colon_location:,
+              name: yield(name),
+              args: mapped_args,
+              open_bracket_location:,
+              close_bracket_location:,
+              args_comma_locations:,
+              comment_location:
+            ) #: self
+          end
+
+          def type_fingerprint
+            [
+              "annots/module_self",
+              name.to_s,
+              args.map(&:to_s),
+              comment_location&.source
+            ]
+          end
+        end
+
         class BlockParamTypeAnnotation < Base
           attr_reader :ampersand_location, :name_location, :colon_location, :question_location, :type_location, :type, :comment_location
 

data/lib/rbs/ast/ruby/comment_block.rb

@@ -15,6 +15,7 @@ module RBS
           prefix_str = "# "
 
           ranges = [] #: Array[Range[Integer]]
+          byte_ranges = [] #: Array[Range[Integer]]
 
           comments.each do |comment|
             tuple = [comment, 2] #: [Prism::Comment, Integer]
@@ -25,12 +26,13 @@ module RBS
 
             offsets << tuple
 
-            start_char = comment.location.start_character_offset + tuple[1]
-            end_char = comment.location.end_character_offset
+            start_char = source_buffer.character_offset(comment.location.start_offset) + tuple[1]
+            end_char = source_buffer.character_offset(comment.location.end_offset)
             ranges << (start_char ... end_char)
+            byte_ranges << ((comment.location.start_offset + tuple[1]) ... comment.location.end_offset)
           end
 
-          @comment_buffer = source_buffer.sub_buffer(lines: ranges)
+          @comment_buffer = source_buffer.sub_buffer(lines: ranges, byte_lines_hint: byte_ranges)
         end
 
         def leading?
@@ -53,7 +55,7 @@ module RBS
 
         def line_starts
           offsets.map do |comment, prefix_size|
-            comment.location.start_character_offset + prefix_size
+            comment_buffer.character_offset(comment.location.start_offset) + prefix_size
           end
         end
 

data/lib/rbs/ast/ruby/declarations.rb

@@ -137,7 +137,17 @@ module RBS
 
           def type_params = []
 
-          def self_types = []
+          def self_types
+            members.filter_map do |member|
+              if member.is_a?(Members::ModuleSelfMember)
+                AST::Declarations::Module::Self.new(
+                  name: member.name,
+                  args: member.args,
+                  location: member.location
+                )
+              end
+            end
+          end
 
           def location
             rbs_location(node.location)

data/lib/rbs/ast/ruby/helpers/location_helper.rb

@@ -6,7 +6,7 @@ module RBS
       module Helpers
         module LocationHelper
           def rbs_location(location)
-            Location.new(buffer, location.start_character_offset, location.end_character_offset)
+            Location.new(buffer, buffer.character_offset(location.start_offset), buffer.character_offset(location.end_offset))
           end
         end
       end

data/lib/rbs/ast/ruby/members.rb

@@ -549,17 +549,27 @@ module RBS
 
           attr_reader :name
           attr_reader :node
+          attr_reader :kind
           attr_reader :method_type
           attr_reader :leading_comment
 
-          def initialize(buffer, name, node, method_type, leading_comment)
+          def initialize(buffer, name, node, method_type, leading_comment, kind: :instance)
             super(buffer)
             @name = name
             @node = node
+            @kind = kind
             @method_type = method_type
             @leading_comment = leading_comment
           end
 
+          def singleton?
+            kind == :singleton
+          end
+
+          def instance?
+            kind == :instance
+          end
+
           def location
             rbs_location(node.location)
           end
@@ -583,6 +593,7 @@ module RBS
           def type_fingerprint
             [
               "members/def",
+              kind.to_s,
               name.to_s,
               method_type.type_fingerprint,
               leading_comment&.as_comment&.string
@@ -717,6 +728,34 @@ module RBS
             ]
           end
         end
+
+        class ModuleSelfMember < Base
+          attr_reader :annotation
+
+          def initialize(buffer, annotation)
+            super(buffer)
+            @annotation = annotation
+          end
+
+          def name
+            annotation.name
+          end
+
+          def args
+            annotation.args
+          end
+
+          def location
+            annotation.location
+          end
+
+          def type_fingerprint
+            [
+              "members/module_self",
+              annotation.type_fingerprint
+            ]
+          end
+        end
       end
     end
   end

data/lib/rbs/buffer.rb

@@ -87,25 +87,62 @@ module RBS
       "#<RBS::Buffer:#{__id__} @name=#{name}, @content=#{content.bytesize} bytes, @lines=#{ranges.size} lines,>"
     end
 
+    def character_offset(byte_offset)
+      top = top_buffer
+      return top.character_offset(byte_offset) unless top.equal?(self)
+
+      keys, vals = (@character_offset_cache ||= [[0], [0]])
+
+      idx = keys.bsearch_index { |k| k > byte_offset }
+      lo = idx ? idx - 1 : keys.size - 1
+
+      base_byte = keys[lo]
+      base_char = vals[lo]
+      delta = byte_offset - base_byte
+      return base_char if delta == 0
+
+      result = base_char + (content.byteslice(base_byte, delta) or raise).length
+
+      if base_byte == keys[-1]
+        keys << byte_offset
+        vals << result
+      end
+
+      result
+    end
+
     def rbs_location(location, loc2=nil)
+      top = top_buffer
       if loc2
-        Location.new(self.top_buffer, location.start_character_offset, loc2.end_character_offset)
+        Location.new(top, character_offset(location.start_offset), character_offset(loc2.end_offset))
       else
-        Location.new(self.top_buffer, location.start_character_offset, location.end_character_offset)
+        Location.new(top, character_offset(location.start_offset), character_offset(location.end_offset))
       end
     end
 
-    def sub_buffer(lines:)
+    def sub_buffer(lines:, byte_lines_hint: nil)
       buf = +""
-      lines.each_with_index do |range, index|
-        start_pos = range.begin
-        end_pos = range.end
-        slice = content[start_pos...end_pos] or raise
-        if slice.include?("\n")
-          raise "Line #{index + 1} cannot contain newline character."
+
+      if byte_lines_hint
+        byte_lines_hint.each_with_index do |range, index|
+          slice = content.byteslice(range.begin, range.end - range.begin) or raise
+          if slice.include?("\n")
+            raise "Line #{index + 1} cannot contain newline character."
+          end
+          buf << slice
+          buf << "\n"
+        end
+      else
+        lines.each_with_index do |range, index|
+          start_pos = range.begin
+          end_pos = range.end
+          slice = content[start_pos...end_pos] or raise
+          if slice.include?("\n")
+            raise "Line #{index + 1} cannot contain newline character."
+          end
+          buf << slice
+          buf << "\n"
         end
-        buf << slice
-        buf << "\n"
       end
 
       buf.chomp!

data/lib/rbs/cli.rb

@@ -962,8 +962,6 @@ Options:
       source = RBS::Annotate::RDocSource.new()
       annotator = RBS::Annotate::RDocAnnotator.new(source: source)
 
-      preserve = true
-
       OptionParser.new do |opts|
         opts.banner = <<-EOB
 Usage: rbs annotate [options...] [files...]
@@ -984,7 +982,7 @@ Options:
         opts.on("-d", "--dir DIRNAME", "Load RDoc from DIRNAME") {|d| source.extra_dirs << Pathname(d) }
         opts.on("--[no-]arglists", "Generate arglists section (defaults to true)") {|b| annotator.include_arg_lists = b }
         opts.on("--[no-]filename", "Include source file name in the documentation (defaults to true)") {|b| annotator.include_filename = b }
-        opts.on("--[no-]preserve", "Try preserve the format of the original file (defaults to true)") {|b| preserve = b }
+        opts.on("--[no-]preserve", "[Deprecated] It always preserves the format") { stdout.puts "The `--preserve` option is deprecated. The tool always preserves the format of RBS files." }
       end.parse!(args)
 
       source.load()
@@ -994,11 +992,11 @@ Options:
         if path.directory?
           Pathname.glob((path + "**/*.rbs").to_s).each do |path|
             stdout.puts "Processing #{path}..."
-            annotator.annotate_file(path, preserve: preserve)
+            annotator.annotate_file(path)
           end
         else
           stdout.puts "Processing #{path}..."
-          annotator.annotate_file(path, preserve: preserve)
+          annotator.annotate_file(path)
         end
       end
 

data/lib/rbs/collection/config/lockfile_generator.rb

@@ -21,6 +21,17 @@ module RBS
           "pstore" => nil,
         }
 
+        NONGEM_STDLIBS = Set[
+          "cgi-escape",
+          "coverage",
+          "monitor",
+          "objspace",
+          "pathname",
+          "pty",
+          "ripper",
+          "socket",
+        ]
+
         class GemfileLockMismatchError < StandardError
           def initialize(expected:, actual:)
             @expected = expected
@@ -168,7 +179,9 @@ module RBS
               end
             end
           else
-            RBS.logger.warn "Cannot find `#{name}` gem. Using incorrect Bundler context? (#{definition.lockfile})"
+            unless NONGEM_STDLIBS.include?(name)
+              RBS.logger.warn "Cannot find `#{name}` gem. Using incorrect Bundler context? (#{definition.lockfile})"
+            end
           end
         end
 

data/lib/rbs/collection/sources/git.rb

@@ -140,6 +140,12 @@ module RBS
                 # git v2.27.0 or greater
                 git 'clone', '--filter=blob:none', remote, git_dir.to_s, chdir: nil
               rescue CommandError
+                # The failed `--filter=blob:none` clone may leave behind a
+                # partial repository (e.g. when the server returned an error
+                # mid-clone), which would cause the fallback `git clone` to
+                # fail with "destination path ... already exists and is not
+                # an empty directory". Clean it up before retrying.
+                FileUtils.rm_rf(git_dir)
                 git 'clone', remote, git_dir.to_s, chdir: nil
               end
             end

data/lib/rbs/definition_builder/method_builder.rb

@@ -143,12 +143,14 @@ module RBS
                   decl.members.each do |member|
                     case member
                     when AST::Ruby::Members::DefMember
-                      build_method(
-                        methods,
-                        type,
-                        member: member,
-                        accessibility: :public
-                      )
+                      if member.instance?
+                        build_method(
+                          methods,
+                          type,
+                          member: member,
+                          accessibility: :public
+                        )
+                      end
                     when AST::Ruby::Members::AttrReaderMember, AST::Ruby::Members::AttrWriterMember, AST::Ruby::Members::AttrAccessorMember
                       build_ruby_attribute(methods, type, member: member, accessibility: :public)
                     end
@@ -181,6 +183,10 @@ module RBS
                     if member.kind == :singleton
                       build_alias(methods, type, member: member)
                     end
+                  when AST::Ruby::Members::DefMember
+                    if member.singleton?
+                      build_method(methods, type, member: member, accessibility: :public)
+                    end
                   end
                 end
               end

data/lib/rbs/environment.rb

@@ -173,7 +173,7 @@ module RBS
           unless type_name.namespace.empty?
             parent = type_name.namespace.to_type_name
             normalized_parent = normalize_module_name?(parent) or return
-            constant_name = TypeName.new(name: type_name.name, namespace: normalized_parent.to_namespace)
+            constant_name = TypeName[normalized_parent.to_namespace, type_name.name]
             constant_decls.fetch(constant_name, nil)
           end
         end
@@ -193,7 +193,7 @@ module RBS
           parent = normalize_module_name?(parent)
           return parent unless parent
 
-          TypeName.new(namespace: parent.to_namespace, name: name.name)
+          TypeName[parent.to_namespace, name.name]
         else
           name
         end
@@ -795,7 +795,8 @@ module RBS
           member.name,
           member.node,
           member.method_type.map_type_name {|name, _, _| absolute_type_name(resolver, nil, name, context: context) },
-          member.leading_comment
+          member.leading_comment,
+          kind: member.kind
         )
       when AST::Ruby::Members::IncludeMember
         resolved_annotation = member.annotation&.map_type_name {|name, _, _| absolute_type_name(resolver, nil, name, context: context) }
@@ -854,6 +855,12 @@ module RBS
           member.buffer,
           resolved_annotation
         )
+      when AST::Ruby::Members::ModuleSelfMember
+        resolved_annotation = member.annotation.map_type_name {|name| absolute_type_name(resolver, nil, name, context: context) }
+        AST::Ruby::Members::ModuleSelfMember.new(
+          member.buffer,
+          resolved_annotation
+        )
       else
         raise "Unknown member type: #{member.class}"
       end