rbs 3.9.5 → 3.10.0.pre.1

data/include/rbs/util/rbs_constant_pool.h

@@ -10,6 +10,8 @@
 #ifndef RBS_CONSTANT_POOL_H
 #define RBS_CONSTANT_POOL_H
 
+#include "rbs/util/rbs_encoding.h"
+
 #include <assert.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -28,70 +30,6 @@
  */
 typedef uint32_t rbs_constant_id_t;
 
-/**
- * A list of constant IDs. Usually used to represent a set of locals.
- */
-typedef struct {
-    /** The number of constant ids in the list. */
-    size_t size;
-
-    /** The number of constant ids that have been allocated in the list. */
-    size_t capacity;
-
-    /** The constant ids in the list. */
-    rbs_constant_id_t *ids;
-} rbs_constant_id_list_t;
-
-/**
- * Initialize a list of constant ids.
- *
- * @param list The list to initialize.
- */
-void rbs_constant_id_list_init(rbs_constant_id_list_t *list);
-
-/**
- * Initialize a list of constant ids with a given capacity.
- *
- * @param list The list to initialize.
- * @param capacity The initial capacity of the list.
- */
-void rbs_constant_id_list_init_capacity(rbs_constant_id_list_t *list, size_t capacity);
-
-/**
- * Append a constant id to a list of constant ids. Returns false if any
- * potential reallocations fail.
- *
- * @param list The list to append to.
- * @param id The id to append.
- * @return Whether the append succeeded.
- */
-bool rbs_constant_id_list_append(rbs_constant_id_list_t *list, rbs_constant_id_t id);
-
-/**
- * Insert a constant id into a list of constant ids at the specified index.
- *
- * @param list The list to insert into.
- * @param index The index at which to insert.
- * @param id The id to insert.
- */
-void rbs_constant_id_list_insert(rbs_constant_id_list_t *list, size_t index, rbs_constant_id_t id);
-
-/**
- * Checks if the current constant id list includes the given constant id.
- *
- * @param list The list to check.
- * @param id The id to check for.
- * @return Whether the list includes the given id.
- */
-bool rbs_constant_id_list_includes(rbs_constant_id_list_t *list, rbs_constant_id_t id);
-
-/**
- * Free the memory associated with a list of constant ids.
- *
- * @param list The list to free.
- */
-void rbs_constant_id_list_free(rbs_constant_id_list_t *list);
-
 /**
  * The type of bucket in the constant pool hash map. This determines how the
  * bucket should be freed.
@@ -110,10 +48,10 @@ static const rbs_constant_pool_bucket_type_t RBS_CONSTANT_POOL_BUCKET_CONSTANT =
 /** A bucket in the hash map. */
 typedef struct {
     /** The incremental ID used for indexing back into the pool. */
-    unsigned int id: 30;
+    unsigned int id : 30;
 
     /** The type of the bucket, which determines how to free it. */
-    rbs_constant_pool_bucket_type_t type: 2;
+    rbs_constant_pool_bucket_type_t type : 2;
 
     /** The hash of the bucket. */
     uint32_t hash;
@@ -162,7 +100,7 @@ bool rbs_constant_pool_init(rbs_constant_pool_t *pool, uint32_t capacity);
  * @param constant_id The id of the constant to get.
  * @return A pointer to the constant.
  */
-rbs_constant_t * rbs_constant_pool_id_to_constant(const rbs_constant_pool_t *pool, rbs_constant_id_t constant_id);
+rbs_constant_t *rbs_constant_pool_id_to_constant(const rbs_constant_pool_t *pool, rbs_constant_id_t constant_id);
 
 /**
  * Find a constant in a constant pool. Returns the id of the constant, or 0 if
@@ -185,6 +123,7 @@ rbs_constant_id_t rbs_constant_pool_find(const rbs_constant_pool_t *pool, const
  * @return The id of the constant.
  */
 rbs_constant_id_t rbs_constant_pool_insert_shared(rbs_constant_pool_t *pool, const uint8_t *start, size_t length);
+rbs_constant_id_t rbs_constant_pool_insert_shared_with_encoding(rbs_constant_pool_t *pool, const uint8_t *start, size_t length, const rbs_encoding_t *encoding);
 
 /**
  * Insert a constant into a constant pool from memory that is now owned by the

data/include/rbs/util/rbs_encoding.h

@@ -0,0 +1,282 @@
+/**
+ * @file encoding.h
+ *
+ * The encoding interface and implementations used by the parser.
+ */
+#ifndef RBS_RBS_ENCODING_H
+#define RBS_RBS_ENCODING_H
+
+#include "rbs/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * This struct defines the functions necessary to implement the encoding
+ * interface so we can determine how many bytes the subsequent character takes.
+ * Each callback should return the number of bytes, or 0 if the next bytes are
+ * invalid for the encoding and type.
+ */
+typedef struct {
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding. Does not read more than n bytes. It is assumed that n is
+     * at least 1.
+     */
+    size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphabetical. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphanumeric. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return true if the next character is valid in the encoding and is an
+     * uppercase character. Does not read more than n bytes. It is assumed that
+     * n is at least 1.
+     */
+    bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * The name of the encoding. This should correspond to a value that can be
+     * passed to Encoding.find in Ruby.
+     */
+    const char *name;
+
+    /**
+     * Return true if the encoding is a multibyte encoding.
+     */
+    bool multibyte;
+} rbs_encoding_t;
+
+/**
+ * All of the lookup tables use the first bit of each embedded byte to indicate
+ * whether the codepoint is alphabetical.
+ */
+#define RBS_ENCODING_ALPHABETIC_BIT 1 << 0
+
+/**
+ * All of the lookup tables use the second bit of each embedded byte to indicate
+ * whether the codepoint is alphanumeric.
+ */
+#define RBS_ENCODING_ALPHANUMERIC_BIT 1 << 1
+
+/**
+ * All of the lookup tables use the third bit of each embedded byte to indicate
+ * whether the codepoint is uppercase.
+ */
+#define RBS_ENCODING_UPPERCASE_BIT 1 << 2
+
+/**
+ * Return the size of the next character in the UTF-8 encoding.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphabetical character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphanumeric character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t rbs_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return true if the next character in the UTF-8 encoding if it is an uppercase
+ * character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns True if the next character is valid in the encoding and is an
+ *     uppercase character, or false if it is not.
+ */
+bool rbs_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * This lookup table is referenced in both the UTF-8 encoding file and the
+ * parser directly in order to speed up the default encoding processing. It is
+ * used to indicate whether a character is alphabetical, alphanumeric, or
+ * uppercase in unicode mappings.
+ */
+extern const uint8_t rbs_encoding_unicode_table[256];
+
+/**
+ * These are all of the encodings that prism supports.
+ */
+typedef enum {
+    RBS_ENCODING_UTF_8 = 0,
+    RBS_ENCODING_US_ASCII,
+    RBS_ENCODING_ASCII_8BIT,
+    RBS_ENCODING_EUC_JP,
+    RBS_ENCODING_WINDOWS_31J,
+
+// We optionally support excluding the full set of encodings to only support the
+// minimum necessary to process Ruby code without encoding comments.
+#ifndef RBS_ENCODING_EXCLUDE_FULL
+    RBS_ENCODING_BIG5,
+    RBS_ENCODING_BIG5_HKSCS,
+    RBS_ENCODING_BIG5_UAO,
+    RBS_ENCODING_CESU_8,
+    RBS_ENCODING_CP51932,
+    RBS_ENCODING_CP850,
+    RBS_ENCODING_CP852,
+    RBS_ENCODING_CP855,
+    RBS_ENCODING_CP949,
+    RBS_ENCODING_CP950,
+    RBS_ENCODING_CP951,
+    RBS_ENCODING_EMACS_MULE,
+    RBS_ENCODING_EUC_JP_MS,
+    RBS_ENCODING_EUC_JIS_2004,
+    RBS_ENCODING_EUC_KR,
+    RBS_ENCODING_EUC_TW,
+    RBS_ENCODING_GB12345,
+    RBS_ENCODING_GB18030,
+    RBS_ENCODING_GB1988,
+    RBS_ENCODING_GB2312,
+    RBS_ENCODING_GBK,
+    RBS_ENCODING_IBM437,
+    RBS_ENCODING_IBM720,
+    RBS_ENCODING_IBM737,
+    RBS_ENCODING_IBM775,
+    RBS_ENCODING_IBM852,
+    RBS_ENCODING_IBM855,
+    RBS_ENCODING_IBM857,
+    RBS_ENCODING_IBM860,
+    RBS_ENCODING_IBM861,
+    RBS_ENCODING_IBM862,
+    RBS_ENCODING_IBM863,
+    RBS_ENCODING_IBM864,
+    RBS_ENCODING_IBM865,
+    RBS_ENCODING_IBM866,
+    RBS_ENCODING_IBM869,
+    RBS_ENCODING_ISO_8859_1,
+    RBS_ENCODING_ISO_8859_2,
+    RBS_ENCODING_ISO_8859_3,
+    RBS_ENCODING_ISO_8859_4,
+    RBS_ENCODING_ISO_8859_5,
+    RBS_ENCODING_ISO_8859_6,
+    RBS_ENCODING_ISO_8859_7,
+    RBS_ENCODING_ISO_8859_8,
+    RBS_ENCODING_ISO_8859_9,
+    RBS_ENCODING_ISO_8859_10,
+    RBS_ENCODING_ISO_8859_11,
+    RBS_ENCODING_ISO_8859_13,
+    RBS_ENCODING_ISO_8859_14,
+    RBS_ENCODING_ISO_8859_15,
+    RBS_ENCODING_ISO_8859_16,
+    RBS_ENCODING_KOI8_R,
+    RBS_ENCODING_KOI8_U,
+    RBS_ENCODING_MAC_CENT_EURO,
+    RBS_ENCODING_MAC_CROATIAN,
+    RBS_ENCODING_MAC_CYRILLIC,
+    RBS_ENCODING_MAC_GREEK,
+    RBS_ENCODING_MAC_ICELAND,
+    RBS_ENCODING_MAC_JAPANESE,
+    RBS_ENCODING_MAC_ROMAN,
+    RBS_ENCODING_MAC_ROMANIA,
+    RBS_ENCODING_MAC_THAI,
+    RBS_ENCODING_MAC_TURKISH,
+    RBS_ENCODING_MAC_UKRAINE,
+    RBS_ENCODING_SHIFT_JIS,
+    RBS_ENCODING_SJIS_DOCOMO,
+    RBS_ENCODING_SJIS_KDDI,
+    RBS_ENCODING_SJIS_SOFTBANK,
+    RBS_ENCODING_STATELESS_ISO_2022_JP,
+    RBS_ENCODING_STATELESS_ISO_2022_JP_KDDI,
+    RBS_ENCODING_TIS_620,
+    RBS_ENCODING_UTF8_MAC,
+    RBS_ENCODING_UTF8_DOCOMO,
+    RBS_ENCODING_UTF8_KDDI,
+    RBS_ENCODING_UTF8_SOFTBANK,
+    RBS_ENCODING_WINDOWS_1250,
+    RBS_ENCODING_WINDOWS_1251,
+    RBS_ENCODING_WINDOWS_1252,
+    RBS_ENCODING_WINDOWS_1253,
+    RBS_ENCODING_WINDOWS_1254,
+    RBS_ENCODING_WINDOWS_1255,
+    RBS_ENCODING_WINDOWS_1256,
+    RBS_ENCODING_WINDOWS_1257,
+    RBS_ENCODING_WINDOWS_1258,
+    RBS_ENCODING_WINDOWS_874,
+#endif
+
+    RBS_ENCODING_MAXIMUM
+} rbs_encoding_type_t;
+
+/**
+ * This is the table of all of the encodings that prism supports.
+ */
+extern const rbs_encoding_t rbs_encodings[RBS_ENCODING_MAXIMUM];
+
+/**
+ * This is the default UTF-8 encoding. We need a reference to it to quickly
+ * create parsers.
+ */
+#define RBS_ENCODING_UTF_8_ENTRY (&rbs_encodings[RBS_ENCODING_UTF_8])
+
+/**
+ * This is the US-ASCII encoding. We need a reference to it to be able to
+ * compare against it when a string is being created because it could possibly
+ * need to fall back to ASCII-8BIT.
+ */
+#define RBS_ENCODING_US_ASCII_ENTRY (&rbs_encodings[RBS_ENCODING_US_ASCII])
+
+/**
+ * This is the ASCII-8BIT encoding. We need a reference to it so that rbs_strpbrk
+ * can compare against it because invalid multibyte characters are not a thing
+ * in this encoding. It is also needed for handling Regexp encoding flags.
+ */
+#define RBS_ENCODING_ASCII_8BIT_ENTRY (&rbs_encodings[RBS_ENCODING_ASCII_8BIT])
+
+/**
+ * This is the EUC-JP encoding. We need a reference to it to quickly process
+ * regular expression modifiers.
+ */
+#define RBS_ENCODING_EUC_JP_ENTRY (&rbs_encodings[RBS_ENCODING_EUC_JP])
+
+/**
+ * This is the Windows-31J encoding. We need a reference to it to quickly
+ * process regular expression modifiers.
+ */
+#define RBS_ENCODING_WINDOWS_31J_ENTRY (&rbs_encodings[RBS_ENCODING_WINDOWS_31J])
+
+/**
+ * Parse the given name of an encoding and return a pointer to the corresponding
+ * encoding struct if one can be found, otherwise return NULL.
+ *
+ * @param start A pointer to the first byte of the name.
+ * @param end A pointer to the last byte of the name.
+ * @returns A pointer to the encoding struct if one is found, otherwise NULL.
+ */
+const rbs_encoding_t *rbs_encoding_find(const uint8_t *start, const uint8_t *end);
+
+#endif

data/include/rbs/util/rbs_unescape.h

@@ -0,0 +1,23 @@
+#ifndef RBS_RBS_UNESCAPE_H
+#define RBS_RBS_UNESCAPE_H
+
+#include <stddef.h>
+#include "rbs/util/rbs_allocator.h"
+#include "rbs/string.h"
+
+/**
+ * Receives `rbs_parser_t` and `range`, which represents a string token or symbol token, and returns a string VALUE.
+ *
+ *    Input token | Output string
+ *    ------------+-------------
+ *    "foo\\n"    | foo\n
+ *    'foo'       | foo
+ *    `bar`       | bar
+ *    :"baz\\t"   | baz\t
+ *    :'baz'      | baz
+ *
+ * @returns A new owned string that will be freed when the allocator is freed.
+ * */
+rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input);
+
+#endif // RBS_RBS_UNESCAPE_H

data/include/rbs.h

@@ -1,7 +1,6 @@
 #ifndef RBS_H
 #define RBS_H
 
-#include "rbs/constants.h"
-#include "rbs/ruby_objs.h"
+#include "rbs/parser.h"
 
 #endif

data/lib/rbs/annotate/formatter.rb

@@ -59,20 +59,10 @@ module RBS
 
       def self.each_part(doc, &block)
         if block
-          document =
-            case doc
-            when String
-              raise
-            when RDoc::Comment
-              document = doc.parse
-            when RDoc::Markup::Document
-              document = doc
-            end
-
-          if document.file
-            yield document
+          if doc.file
+            yield doc
           else
-            document.each do |d|
+            doc.each do |d|
               each_part(d, &block)
             end
           end

data/lib/rbs/annotate/rdoc_annotator.rb

@@ -39,7 +39,9 @@ module RBS
       def each_part(subjects, tester:)
         if block_given?
           subjects.each do |subject, docs|
-            Formatter.each_part(subject.comment) do |doc|
+            comment = subject.comment
+            raise if comment.is_a?(String)
+            Formatter.each_part(comment.parse) do |doc|
               if tester.test_path(doc.file || raise)
                 yield [doc, subject]
               end

data/lib/rbs/annotate/rdoc_source.rb

@@ -23,7 +23,7 @@ module RBS
         @stores.clear()
 
         RDoc::RI::Paths.each(with_system_dir, with_site_dir, with_home_dir, with_gems_dir ? :latest : false, *extra_dirs.map(&:to_s)) do |path, type|
-          store = RDoc::Store.new(path, type)
+          store = RDoc::Store.new(RDoc::Options.new, path:, type:)
           store.load_all
 
           @stores << store

data/lib/rbs/cli/validate.rb

@@ -131,7 +131,7 @@ EOU
 
                 self_params =
                   if self_type.name.class?
-                    @env.normalized_module_entry(self_type.name)&.type_params
+                    @env.module_entry(self_type.name, normalized: true)&.type_params
                   else
                     @env.interface_decls[self_type.name]&.decl&.type_params
                   end
@@ -188,7 +188,7 @@ EOU
                 end
                 params =
                   if member.name.class?
-                    module_decl = @env.normalized_module_entry(member.name) or raise
+                    module_decl = @env.module_entry(member.name, normalized: true) or raise
                     module_decl.type_params
                   else
                     interface_decl = @env.interface_decls.fetch(member.name)

data/lib/rbs/cli.rb

@@ -947,7 +947,7 @@ Import documents from RDoc and update RBS files.
 
 Examples:
 
-  $ rbs annotate stdlib/pathname/**/*.rbs
+  $ rbs annotate stdlib/logger/**/*.rbs
 
 Options:
         EOB

data/lib/rbs/collection/config/lockfile_generator.rb

@@ -14,6 +14,7 @@ module RBS
           "net-smtp" => nil,
           "nkf" => nil,
           "observer" => nil,
+          "cgi" => nil,
         }
 
         class GemfileLockMismatchError < StandardError

data/lib/rbs/definition_builder/ancestor_builder.rb

@@ -220,7 +220,7 @@ module RBS
               InvalidTypeApplicationError.check2!(type_name: super_class.name, args: super_class.args, env: env, location: super_class.location)
             end
 
-            super_entry = env.normalized_class_entry(super_name) or raise
+            super_entry = env.class_entry(super_name, normalized: true) or raise
             super_args = AST::TypeParam.normalize_args(super_entry.type_params, super_args)
 
             ancestors = OneAncestors.class_instance(
@@ -248,7 +248,7 @@ module RBS
 
               module_name = module_self.name
               if module_name.class?
-                module_entry = env.normalized_module_class_entry(module_name) or raise
+                module_entry = env.module_class_entry(module_name, normalized: true) or raise
                 module_name = module_entry.name
                 self_args = AST::TypeParam.normalize_args(module_entry.type_params, module_self.args)
               end
@@ -359,7 +359,7 @@ module RBS
               MixinClassError.check!(type_name: type_name, env: env, member: member)
               NoMixinFoundError.check!(member.name, env: env, member: member)
 
-              module_decl = env.normalized_module_entry(module_name) or raise
+              module_decl = env.module_entry(module_name, normalized: true) or raise
               module_args = AST::TypeParam.normalize_args(module_decl.type_params, module_args)
 
               module_name = env.normalize_module_name(module_name)
@@ -378,7 +378,7 @@ module RBS
               MixinClassError.check!(type_name: type_name, env: env, member: member)
               NoMixinFoundError.check!(member.name, env: env, member: member)
 
-              module_decl = env.normalized_module_entry(member.name) or raise
+              module_decl = env.module_entry(member.name, normalized: true) or raise
               module_name = module_decl.name
 
               module_args = member.args.map {|type| align_params ? type.sub(align_params) : type }
@@ -396,7 +396,7 @@ module RBS
               MixinClassError.check!(type_name: type_name, env: env, member: member)
               NoMixinFoundError.check!(member.name, env: env, member: member)
 
-              module_decl = env.normalized_module_entry(module_name) or raise
+              module_decl = env.module_entry(module_name, normalized: true) or raise
               module_args = AST::TypeParam.normalize_args(module_decl.type_params, module_args)
 
               module_name = env.normalize_module_name(module_name)