rbs 3.9.2 → 4.0.0.dev.1

data/include/rbs/util/rbs_encoding.h

@@ -0,0 +1,280 @@
+/**
+ * @file encoding.h
+ *
+ * The encoding interface and implementations used by the parser.
+ */
+#ifndef RBS_RBS_ENCODING_H
+#define RBS_RBS_ENCODING_H
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * This struct defines the functions necessary to implement the encoding
+ * interface so we can determine how many bytes the subsequent character takes.
+ * Each callback should return the number of bytes, or 0 if the next bytes are
+ * invalid for the encoding and type.
+ */
+typedef struct {
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding. Does not read more than n bytes. It is assumed that n is
+     * at least 1.
+     */
+    size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphabetical. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphanumeric. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return true if the next character is valid in the encoding and is an
+     * uppercase character. Does not read more than n bytes. It is assumed that
+     * n is at least 1.
+     */
+    bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * The name of the encoding. This should correspond to a value that can be
+     * passed to Encoding.find in Ruby.
+     */
+    const char *name;
+
+    /**
+     * Return true if the encoding is a multibyte encoding.
+     */
+    bool multibyte;
+} rbs_encoding_t;
+
+/**
+ * All of the lookup tables use the first bit of each embedded byte to indicate
+ * whether the codepoint is alphabetical.
+ */
+#define RBS_ENCODING_ALPHABETIC_BIT 1 << 0
+
+/**
+ * All of the lookup tables use the second bit of each embedded byte to indicate
+ * whether the codepoint is alphanumeric.
+ */
+#define RBS_ENCODING_ALPHANUMERIC_BIT 1 << 1
+
+/**
+ * All of the lookup tables use the third bit of each embedded byte to indicate
+ * whether the codepoint is uppercase.
+ */
+#define RBS_ENCODING_UPPERCASE_BIT 1 << 2
+
+/**
+ * Return the size of the next character in the UTF-8 encoding.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphabetical character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphanumeric character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return true if the next character in the UTF-8 encoding if it is an uppercase
+ * character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns True if the next character is valid in the encoding and is an
+ *     uppercase character, or false if it is not.
+ */
+bool rbs_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * This lookup table is referenced in both the UTF-8 encoding file and the
+ * parser directly in order to speed up the default encoding processing. It is
+ * used to indicate whether a character is alphabetical, alphanumeric, or
+ * uppercase in unicode mappings.
+ */
+extern const uint8_t rbs_encoding_unicode_table[256];
+
+/**
+ * These are all of the encodings that prism supports.
+ */
+typedef enum {
+    RBS_ENCODING_UTF_8 = 0,
+    RBS_ENCODING_US_ASCII,
+    RBS_ENCODING_ASCII_8BIT,
+    RBS_ENCODING_EUC_JP,
+    RBS_ENCODING_WINDOWS_31J,
+
+// We optionally support excluding the full set of encodings to only support the
+// minimum necessary to process Ruby code without encoding comments.
+#ifndef RBS_ENCODING_EXCLUDE_FULL
+    RBS_ENCODING_BIG5,
+    RBS_ENCODING_BIG5_HKSCS,
+    RBS_ENCODING_BIG5_UAO,
+    RBS_ENCODING_CESU_8,
+    RBS_ENCODING_CP51932,
+    RBS_ENCODING_CP850,
+    RBS_ENCODING_CP852,
+    RBS_ENCODING_CP855,
+    RBS_ENCODING_CP949,
+    RBS_ENCODING_CP950,
+    RBS_ENCODING_CP951,
+    RBS_ENCODING_EMACS_MULE,
+    RBS_ENCODING_EUC_JP_MS,
+    RBS_ENCODING_EUC_JIS_2004,
+    RBS_ENCODING_EUC_KR,
+    RBS_ENCODING_EUC_TW,
+    RBS_ENCODING_GB12345,
+    RBS_ENCODING_GB18030,
+    RBS_ENCODING_GB1988,
+    RBS_ENCODING_GB2312,
+    RBS_ENCODING_GBK,
+    RBS_ENCODING_IBM437,
+    RBS_ENCODING_IBM720,
+    RBS_ENCODING_IBM737,
+    RBS_ENCODING_IBM775,
+    RBS_ENCODING_IBM852,
+    RBS_ENCODING_IBM855,
+    RBS_ENCODING_IBM857,
+    RBS_ENCODING_IBM860,
+    RBS_ENCODING_IBM861,
+    RBS_ENCODING_IBM862,
+    RBS_ENCODING_IBM863,
+    RBS_ENCODING_IBM864,
+    RBS_ENCODING_IBM865,
+    RBS_ENCODING_IBM866,
+    RBS_ENCODING_IBM869,
+    RBS_ENCODING_ISO_8859_1,
+    RBS_ENCODING_ISO_8859_2,
+    RBS_ENCODING_ISO_8859_3,
+    RBS_ENCODING_ISO_8859_4,
+    RBS_ENCODING_ISO_8859_5,
+    RBS_ENCODING_ISO_8859_6,
+    RBS_ENCODING_ISO_8859_7,
+    RBS_ENCODING_ISO_8859_8,
+    RBS_ENCODING_ISO_8859_9,
+    RBS_ENCODING_ISO_8859_10,
+    RBS_ENCODING_ISO_8859_11,
+    RBS_ENCODING_ISO_8859_13,
+    RBS_ENCODING_ISO_8859_14,
+    RBS_ENCODING_ISO_8859_15,
+    RBS_ENCODING_ISO_8859_16,
+    RBS_ENCODING_KOI8_R,
+    RBS_ENCODING_KOI8_U,
+    RBS_ENCODING_MAC_CENT_EURO,
+    RBS_ENCODING_MAC_CROATIAN,
+    RBS_ENCODING_MAC_CYRILLIC,
+    RBS_ENCODING_MAC_GREEK,
+    RBS_ENCODING_MAC_ICELAND,
+    RBS_ENCODING_MAC_JAPANESE,
+    RBS_ENCODING_MAC_ROMAN,
+    RBS_ENCODING_MAC_ROMANIA,
+    RBS_ENCODING_MAC_THAI,
+    RBS_ENCODING_MAC_TURKISH,
+    RBS_ENCODING_MAC_UKRAINE,
+    RBS_ENCODING_SHIFT_JIS,
+    RBS_ENCODING_SJIS_DOCOMO,
+    RBS_ENCODING_SJIS_KDDI,
+    RBS_ENCODING_SJIS_SOFTBANK,
+    RBS_ENCODING_STATELESS_ISO_2022_JP,
+    RBS_ENCODING_STATELESS_ISO_2022_JP_KDDI,
+    RBS_ENCODING_TIS_620,
+    RBS_ENCODING_UTF8_MAC,
+    RBS_ENCODING_UTF8_DOCOMO,
+    RBS_ENCODING_UTF8_KDDI,
+    RBS_ENCODING_UTF8_SOFTBANK,
+    RBS_ENCODING_WINDOWS_1250,
+    RBS_ENCODING_WINDOWS_1251,
+    RBS_ENCODING_WINDOWS_1252,
+    RBS_ENCODING_WINDOWS_1253,
+    RBS_ENCODING_WINDOWS_1254,
+    RBS_ENCODING_WINDOWS_1255,
+    RBS_ENCODING_WINDOWS_1256,
+    RBS_ENCODING_WINDOWS_1257,
+    RBS_ENCODING_WINDOWS_1258,
+    RBS_ENCODING_WINDOWS_874,
+#endif
+
+    RBS_ENCODING_MAXIMUM
+} rbs_encoding_type_t;
+
+/**
+ * This is the table of all of the encodings that prism supports.
+ */
+extern const rbs_encoding_t rbs_encodings[RBS_ENCODING_MAXIMUM];
+
+/**
+ * This is the default UTF-8 encoding. We need a reference to it to quickly
+ * create parsers.
+ */
+#define RBS_ENCODING_UTF_8_ENTRY (&rbs_encodings[RBS_ENCODING_UTF_8])
+
+/**
+ * This is the US-ASCII encoding. We need a reference to it to be able to
+ * compare against it when a string is being created because it could possibly
+ * need to fall back to ASCII-8BIT.
+ */
+#define RBS_ENCODING_US_ASCII_ENTRY (&rbs_encodings[RBS_ENCODING_US_ASCII])
+
+/**
+ * This is the ASCII-8BIT encoding. We need a reference to it so that rbs_strpbrk
+ * can compare against it because invalid multibyte characters are not a thing
+ * in this encoding. It is also needed for handling Regexp encoding flags.
+ */
+#define RBS_ENCODING_ASCII_8BIT_ENTRY (&rbs_encodings[RBS_ENCODING_ASCII_8BIT])
+
+/**
+ * This is the EUC-JP encoding. We need a reference to it to quickly process
+ * regular expression modifiers.
+ */
+#define RBS_ENCODING_EUC_JP_ENTRY (&rbs_encodings[RBS_ENCODING_EUC_JP])
+
+/**
+ * This is the Windows-31J encoding. We need a reference to it to quickly
+ * process regular expression modifiers.
+ */
+#define RBS_ENCODING_WINDOWS_31J_ENTRY (&rbs_encodings[RBS_ENCODING_WINDOWS_31J])
+
+/**
+ * Parse the given name of an encoding and return a pointer to the corresponding
+ * encoding struct if one can be found, otherwise return NULL.
+ *
+ * @param start A pointer to the first byte of the name.
+ * @param end A pointer to the last byte of the name.
+ * @returns A pointer to the encoding struct if one is found, otherwise NULL.
+ */
+const rbs_encoding_t * rbs_encoding_find(const uint8_t *start, const uint8_t *end);
+
+#endif

data/include/rbs/util/rbs_unescape.h

@@ -0,0 +1,23 @@
+#ifndef RBS_RBS_UNESCAPE_H
+#define RBS_RBS_UNESCAPE_H
+
+#include <stddef.h>
+#include "rbs/util/rbs_allocator.h"
+#include "rbs/string.h"
+
+/**
+ * Receives `rbs_parser_t` and `range`, which represents a string token or symbol token, and returns a string VALUE.
+ *
+ *    Input token | Output string
+ *    ------------+-------------
+ *    "foo\\n"    | foo\n
+ *    'foo'       | foo
+ *    `bar`       | bar
+ *    :"baz\\t"   | baz\t
+ *    :'baz'      | baz
+ *
+ * @returns A new owned string that will be freed when the allocator is freed.
+ * */
+rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input);
+
+#endif // RBS_RBS_UNESCAPE_H

data/include/rbs.h

@@ -1,7 +1,6 @@
 #ifndef RBS_H
 #define RBS_H
 
-#include "rbs/constants.h"
-#include "rbs/ruby_objs.h"
+#include "rbs/parser.h"
 
 #endif

data/lib/rbs/annotate/formatter.rb

@@ -59,20 +59,10 @@ module RBS
 
       def self.each_part(doc, &block)
         if block
-          document =
-            case doc
-            when String
-              raise
-            when RDoc::Comment
-              document = doc.parse
-            when RDoc::Markup::Document
-              document = doc
-            end
-
-          if document.file
-            yield document
+          if doc.file
+            yield doc
           else
-            document.each do |d|
+            doc.each do |d|
               each_part(d, &block)
             end
           end

data/lib/rbs/annotate/rdoc_annotator.rb

@@ -39,7 +39,9 @@ module RBS
       def each_part(subjects, tester:)
         if block_given?
           subjects.each do |subject, docs|
-            Formatter.each_part(subject.comment) do |doc|
+            comment = subject.comment
+            raise if comment.is_a?(String)
+            Formatter.each_part(comment.parse) do |doc|
               if tester.test_path(doc.file || raise)
                 yield [doc, subject]
               end

data/lib/rbs/annotate/rdoc_source.rb

@@ -23,7 +23,7 @@ module RBS
         @stores.clear()
 
         RDoc::RI::Paths.each(with_system_dir, with_site_dir, with_home_dir, with_gems_dir ? :latest : false, *extra_dirs.map(&:to_s)) do |path, type|
-          store = RDoc::Store.new(path, type)
+          store = RDoc::Store.new(RDoc::Options.new, path:, type:)
           store.load_all
 
           @stores << store

data/lib/rbs/ast/ruby/annotations.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module RBS
+  module AST
+    module Ruby
+      module Annotations
+        class Base
+          attr_reader :location, :prefix_location
+
+          def initialize(location, prefix_location)
+            @location = location
+            @prefix_location = prefix_location
+          end
+
+          def buffer
+            location.buffer
+          end
+        end
+
+        class NodeTypeAssertion < Base
+          attr_reader :type
+
+          def initialize(location:, prefix_location:, type:)
+            super(location, prefix_location)
+            @type = type
+          end
+
+          def map_type_name
+            self.class.new(
+              location:, prefix_location:,
+              type: type.map_type_name { yield _1 }
+            ) #: self
+          end
+        end
+
+        class ColonMethodTypeAnnotation < Base
+          attr_reader :annotations, :method_type
+
+          def initialize(location:, prefix_location:, annotations:, method_type:)
+            super(location, prefix_location)
+            @annotations = annotations
+            @method_type = method_type
+          end
+
+          def map_type_name
+            self.class.new(
+              location:,
+              prefix_location:,
+              annotations: annotations,
+              method_type: method_type.map_type {|type| type.map_type_name { yield _1 }}
+            ) #: self
+          end
+        end
+
+        class MethodTypesAnnotation < Base
+          Overload = AST::Members::MethodDefinition::Overload
+
+          attr_reader :overloads, :vertical_bar_locations
+
+          def initialize(location:, prefix_location:, overloads:, vertical_bar_locations:)
+            super(location, prefix_location)
+            @overloads = overloads
+            @vertical_bar_locations = vertical_bar_locations
+          end
+
+          def map_type_name(&block)
+            ovs = overloads.map do |overload|
+              Overload.new(
+                method_type: overload.method_type.map_type {|type| type.map_type_name { yield _1 } },
+                annotations: overload.annotations
+              )
+            end
+
+            self.class.new(location:, prefix_location:, overloads: ovs, vertical_bar_locations:) #: self
+          end
+        end
+
+        class SkipAnnotation < Base
+          attr_reader :skip_location, :comment_location
+
+          def initialize(location:, prefix_location:, skip_location:, comment_location:)
+            super(location, prefix_location)
+            @skip_location = skip_location
+            @comment_location = comment_location
+          end
+        end
+
+        class ReturnTypeAnnotation < Base
+          attr_reader :return_location
+
+          attr_reader :colon_location
+
+          attr_reader :return_type
+
+          attr_reader :comment_location
+
+          def initialize(location:, prefix_location:, return_location:, colon_location:, return_type:, comment_location:)
+            super(location, prefix_location)
+            @return_location = return_location
+            @colon_location = colon_location
+            @return_type = return_type
+            @comment_location = comment_location
+          end
+
+          def map_type_name(&block)
+            self.class.new(
+              location:,
+              prefix_location:,
+              return_location: return_location,
+              colon_location: colon_location,
+              return_type: return_type.map_type_name { yield _1 },
+              comment_location: comment_location
+            ) #: self
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rbs/ast/ruby/comment_block.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module RBS
+  module AST
+    module Ruby
+      class CommentBlock
+        attr_reader :name, :offsets, :comment_buffer
+
+        def initialize(source_buffer, comments)
+          @name = source_buffer.name
+
+          @offsets = []
+
+          # Assume the comment starts with a prefix whitespace
+          prefix_str = "# "
+
+          ranges = [] #: Array[Range[Integer]]
+
+          comments.each do |comment|
+            tuple = [comment, 2] #: [Prism::Comment, Integer]
+
+            unless comment.location.slice.start_with?(prefix_str)
+              tuple[1] = 1
+            end
+
+            offsets << tuple
+
+            start_char = comment.location.start_character_offset + tuple[1]
+            end_char = comment.location.end_character_offset
+            ranges << (start_char ... end_char)
+          end
+
+          @comment_buffer = source_buffer.sub_buffer(lines: ranges)
+        end
+
+        def leading?
+          comment = offsets[0][0] or raise
+          comment.location.start_line_slice.index(/\S/) ? false : true
+        end
+
+        def trailing?
+          comment = offsets[0][0] or raise
+          comment.location.start_line_slice.index(/\S/) ? true : false
+        end
+
+        def start_line
+          comments[0].location.start_line
+        end
+
+        def end_line
+          comments[-1].location.end_line
+        end
+
+        def line_starts
+          offsets.map do |comment, prefix_size|
+            comment.location.start_character_offset + prefix_size
+          end
+        end
+
+        def self.build(buffer, comments)
+          blocks = [] #: Array[CommentBlock]
+
+          comments = comments.filter {|comment| comment.is_a?(Prism::InlineComment) }
+
+          until comments.empty?
+            block_comments = [] #: Array[Prism::Comment]
+
+            until comments.empty?
+              comment = comments.first or raise
+              last_comment = block_comments.last
+
+              if last_comment
+                if last_comment.location.end_line + 1 == comment.location.start_line
+                  if last_comment.location.start_column == comment.location.start_column
+                    unless comment.location.start_line_slice.index(/\S/)
+                      block_comments << comments.shift
+                      next
+                    end
+                  end
+                end
+
+                break
+              else
+                block_comments << comments.shift
+              end
+            end
+
+            unless block_comments.empty?
+              blocks << CommentBlock.new(buffer, block_comments.dup)
+            end
+          end
+
+          blocks
+        end
+
+        AnnotationSyntaxError = _ = Struct.new(:location, :error)
+
+        def each_paragraph(variables, &block)
+          if block
+            if leading_annotation?(0)
+              yield_annotation(0, 0, 0, variables, &block)
+            else
+              yield_paragraph(0, 0, variables, &block)
+            end
+          else
+            enum_for :each_paragraph, variables
+          end
+        end
+
+        def yield_paragraph(start_line, current_line, variables, &block)
+          # We already know at start_line..current_line are paragraph.
+
+          while true
+            next_line = current_line + 1
+
+            if next_line >= comment_buffer.line_count
+              yield line_location(start_line, current_line)
+              return
+            end
+
+            if leading_annotation?(next_line)
+              yield line_location(start_line, current_line)
+              return yield_annotation(next_line, next_line, next_line, variables, &block)
+            else
+              current_line = next_line
+            end
+          end
+        end
+
+        def yield_annotation(start_line, end_line, current_line, variables, &block)
+          # We already know at start_line..end_line are annotation.
+          while true
+            next_line = current_line + 1
+
+            if next_line >= comment_buffer.line_count
+              annotation = parse_annotation_lines(start_line, end_line, variables)
+              yield annotation
+
+              if end_line > current_line
+                yield_paragraph(end_line + 1, end_line + 1, variables, &block)
+              end
+
+              return
+            end
+
+            line_text = text(next_line)
+            if leading_spaces = line_text.index(/\S/)
+              if leading_spaces == 0
+                # End of annotation
+                yield parse_annotation_lines(start_line, end_line, variables)
+
+                if leading_annotation?(end_line + 1)
+                  yield_annotation(end_line + 1, end_line + 1, end_line + 1, variables, &block)
+                else
+                  yield_paragraph(end_line + 1, end_line + 1, variables, &block)
+                end
+
+                return
+              else
+                current_line = next_line
+                end_line = next_line
+              end
+            else
+              current_line = next_line
+            end
+          end
+        end
+
+        def text(comment_index)
+          range = comment_buffer.ranges[comment_index]
+          comment_buffer.content[range] or raise
+        end
+
+        def line_location(start_line, end_line)
+          start_offset = comment_buffer.ranges[start_line].begin
+          end_offset = comment_buffer.ranges[end_line].end
+          Location.new(comment_buffer, start_offset, end_offset)
+        end
+
+        def parse_annotation_lines(start_line, end_line, variables)
+          start_pos = comment_buffer.ranges[start_line].begin
+          end_pos = comment_buffer.ranges[end_line].end
+          begin
+            Parser.parse_inline_leading_annotation(comment_buffer, start_pos...end_pos, variables: variables)
+          rescue ParsingError => error
+            AnnotationSyntaxError.new(line_location(start_line, end_line), error)
+          end
+        end
+
+        def trailing_annotation(variables)
+          if trailing?
+            comment = comments[0] or raise
+            if comment.location.slice.start_with?(/#[:\[]/)
+              begin
+                Parser.parse_inline_trailing_annotation(comment_buffer, 0...comment_buffer.last_position, variables: variables)
+              rescue ParsingError => error
+                location = line_location(0, offsets.size - 1)
+                AnnotationSyntaxError.new(location, error)
+              end
+            end
+          end
+        end
+
+        def comments
+          offsets.map { _1[0]}
+        end
+
+        def leading_annotation?(index)
+          if index < comment_buffer.line_count
+            text(index).start_with?(/@rbs\b/) and return true
+
+            comment = offsets[index][0]
+            comment.location.slice.start_with?(/\#:/) and return true
+          end
+
+          false
+        end
+      end
+    end
+  end
+end