rbs 3.8.0 → 3.9.0.dev.1

data/ext/rbs_extension/parserstate.c

@@ -1,4 +1,5 @@
 #include "rbs_extension.h"
+#include "rbs/util/rbs_constant_pool.h"
 
 #define RESET_TABLE_P(table) (table->size == 0)
 
@@ -8,7 +9,7 @@ id_table *alloc_empty_table(void) {
   *table = (id_table) {
     .size = 10,
     .count = 0,
-    .ids = calloc(10, sizeof(ID)),
+    .ids = calloc(10, sizeof(rbs_constant_id_t)),
     .next = NULL,
   };
 
@@ -61,7 +62,7 @@ void parser_pop_typevar_table(parserstate *state) {
   }
 }
 
-void parser_insert_typevar(parserstate *state, ID id) {
+void parser_insert_typevar(parserstate *state, rbs_constant_id_t id) {
   id_table *table = state->vars;
 
   if (RESET_TABLE_P(table)) {
@@ -70,17 +71,17 @@ void parser_insert_typevar(parserstate *state, ID id) {
 
   if (table->size == table->count) {
     // expand
-    ID *ptr = table->ids;
+    rbs_constant_id_t *ptr = table->ids;
     table->size += 10;
-    table->ids = calloc(table->size, sizeof(ID));
-    memcpy(table->ids, ptr, sizeof(ID) * table->count);
+    table->ids = calloc(table->size, sizeof(rbs_constant_id_t));
+    memcpy(table->ids, ptr, sizeof(rbs_constant_id_t) * table->count);
     free(ptr);
   }
 
   table->ids[table->count++] = id;
 }
 
-bool parser_typevar_member(parserstate *state, ID id) {
+bool parser_typevar_member(parserstate *state, rbs_constant_id_t id) {
   id_table *table = state->vars;
 
   while (table && !RESET_TABLE_P(table)) {
@@ -332,8 +333,30 @@ parserstate *alloc_parser(VALUE buffer, lexstate *lexer, int start_pos, int end_
 
     .vars = NULL,
     .last_comment = NULL,
+
+    .constant_pool = {},
   };
 
+  // The parser's constant pool is mainly used for storing the names of type variables, which usually aren't many.
+  // Below are some statistics gathered from the current test suite. We can see that 56% of parsers never add to their
+  // constant pool at all. The initial capacity needs to be a power of 2. Picking 2 means that we won't need to realloc
+  // in 85% of cases.
+  //
+  // TODO: recalculate these statistics based on a real world codebase, rather than the test suite.
+  //
+  // | Size | Count | Cumulative | % Coverage |
+  // |------|-------|------------|------------|
+  // |   0  | 7,862 |      7,862 |     56%    |
+  // |   1  | 3,196 |     11,058 |     79%    |
+  // |   2  |   778 |     12,719 |     85%    |
+  // |   3  |   883 |     11,941 |     91%    |
+  // |   4  |   478 |     13,197 |     95%    |
+  // |   5  |   316 |     13,513 |     97%    |
+  // |   6  |   288 |     13,801 |     99%    |
+  // |   7  |   144 |     13,945 |    100%    |
+  const size_t initial_pool_capacity = 2;
+  rbs_constant_pool_init(&parser->constant_pool, initial_pool_capacity);
+
   parser_advance(parser);
   parser_advance(parser);
   parser_advance(parser);
@@ -348,9 +371,16 @@ parserstate *alloc_parser(VALUE buffer, lexstate *lexer, int start_pos, int end_
     parser_push_typevar_table(parser, true);
 
     for (long i = 0; i < rb_array_len(variables); i++) {
-      VALUE index = INT2FIX(i);
-      VALUE symbol = rb_ary_aref(1, &index, variables);
-      parser_insert_typevar(parser, SYM2ID(symbol));
+      VALUE symbol = rb_ary_entry(variables, i);
+      VALUE name = rb_sym2str(symbol);
+
+      rbs_constant_id_t id = rbs_constant_pool_insert_shared(
+        &parser->constant_pool,
+        (const uint8_t *) RSTRING_PTR(name),
+        RSTRING_LEN(name)
+      );
+
+      parser_insert_typevar(parser, id);
     }
   }
 
@@ -362,5 +392,6 @@ void free_parser(parserstate *parser) {
   if (parser->last_comment) {
     free_comment(parser->last_comment);
   }
+  rbs_constant_pool_free(&parser->constant_pool);
   free(parser);
 }

data/ext/rbs_extension/parserstate.h

@@ -7,13 +7,13 @@
 #include "location.h"
 
 /**
- * id_table represents a set of IDs.
+ * id_table represents a set of RBS constant IDs.
  * This is used to manage the set of bound variables.
  * */
 typedef struct id_table {
   size_t size;
   size_t count;
-  ID *ids;
+  rbs_constant_id_t *ids;
   struct id_table *next;
 } id_table;
 
@@ -55,6 +55,8 @@ typedef struct {
 
   id_table *vars;         /* Known type variables */
   comment *last_comment;  /* Last read comment */
+
+  rbs_constant_pool_t constant_pool;
 } parserstate;
 
 comment *alloc_comment(token comment_token, comment *last_comment);
@@ -84,14 +86,14 @@ void parser_pop_typevar_table(parserstate *state);
 /**
  * Insert new type variable into the latest table.
  * */
-void parser_insert_typevar(parserstate *state, ID id);
+void parser_insert_typevar(parserstate *state, rbs_constant_id_t id);
 
 /**
  * Returns true if given type variable is recorded in the table.
  * If not found, it goes one table up, if it's not a reset table.
  * Or returns false, if it's a reset table.
  * */
-bool parser_typevar_member(parserstate *state, ID id);
+bool parser_typevar_member(parserstate *state, rbs_constant_id_t id);
 
 /**
  * Allocate new lexstate object.

data/include/rbs/ruby_objs.h

@@ -14,13 +14,13 @@ VALUE rbs_ast_annotation(VALUE string, VALUE location);
 VALUE rbs_ast_comment(VALUE string, VALUE location);
 VALUE rbs_ast_decl_class(VALUE name, VALUE type_params, VALUE super_class, VALUE members, VALUE annotations, VALUE location, VALUE comment);
 VALUE rbs_ast_decl_class_super(VALUE name, VALUE args, VALUE location);
-VALUE rbs_ast_decl_class_alias(VALUE new_name, VALUE old_name, VALUE location, VALUE comment);
-VALUE rbs_ast_decl_constant(VALUE name, VALUE type, VALUE location, VALUE comment);
-VALUE rbs_ast_decl_global(VALUE name, VALUE type, VALUE location, VALUE comment);
+VALUE rbs_ast_decl_class_alias(VALUE new_name, VALUE old_name, VALUE location, VALUE comment, VALUE annotations);
+VALUE rbs_ast_decl_constant(VALUE name, VALUE type, VALUE location, VALUE comment, VALUE annotations);
+VALUE rbs_ast_decl_global(VALUE name, VALUE type, VALUE location, VALUE comment, VALUE annotations);
 VALUE rbs_ast_decl_interface(VALUE name, VALUE type_params, VALUE members, VALUE annotations, VALUE location, VALUE comment);
 VALUE rbs_ast_decl_module(VALUE name, VALUE type_params, VALUE self_types, VALUE members, VALUE annotations, VALUE location, VALUE comment);
 VALUE rbs_ast_decl_module_self(VALUE name, VALUE args, VALUE location);
-VALUE rbs_ast_decl_module_alias(VALUE new_name, VALUE old_name, VALUE location, VALUE comment);
+VALUE rbs_ast_decl_module_alias(VALUE new_name, VALUE old_name, VALUE location, VALUE comment, VALUE annotations);
 VALUE rbs_ast_decl_type_alias(VALUE name, VALUE type_params, VALUE type, VALUE annotations, VALUE location, VALUE comment);
 VALUE rbs_ast_directives_use(VALUE clauses, VALUE location);
 VALUE rbs_ast_directives_use_single_clause(VALUE type_name, VALUE new_name, VALUE location);
@@ -39,12 +39,12 @@ VALUE rbs_ast_members_method_definition_overload(VALUE annotations, VALUE method
 VALUE rbs_ast_members_prepend(VALUE name, VALUE args, VALUE annotations, VALUE location, VALUE comment);
 VALUE rbs_ast_members_private(VALUE location);
 VALUE rbs_ast_members_public(VALUE location);
-VALUE rbs_ast_type_param(VALUE name, VALUE variance, VALUE upper_bound, VALUE default_type, VALUE location);
+VALUE rbs_ast_type_param(VALUE name, VALUE variance, VALUE upper_bound, VALUE default_type, VALUE unchecked, VALUE location);
 VALUE rbs_method_type(VALUE type_params, VALUE type, VALUE block, VALUE location);
 VALUE rbs_namespace(VALUE path, VALUE absolute);
 VALUE rbs_type_name(VALUE namespace, VALUE name);
 VALUE rbs_alias(VALUE name, VALUE args, VALUE location);
-VALUE rbs_bases_any(VALUE location);
+VALUE rbs_bases_any(VALUE todo, VALUE location);
 VALUE rbs_bases_bool(VALUE location);
 VALUE rbs_bases_bottom(VALUE location);
 VALUE rbs_bases_class(VALUE location);

data/include/rbs/util/rbs_constant_pool.h

@@ -0,0 +1,219 @@
+/**
+ * @file rbs_constant_pool.h
+ *
+ * A data structure that stores a set of strings.
+ *
+ * Each string is assigned a unique id, which can be used to compare strings for
+ * equality. This comparison ends up being much faster than strcmp, since it
+ * only requires a single integer comparison.
+ */
+#ifndef RBS_CONSTANT_POOL_H
+#define RBS_CONSTANT_POOL_H
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * When we allocate constants into the pool, we reserve 0 to mean that the slot
+ * is not yet filled. This constant is reused in other places to indicate the
+ * lack of a constant id.
+ */
+#define RBS_CONSTANT_ID_UNSET 0
+
+/**
+ * A constant id is a unique identifier for a constant in the constant pool.
+ */
+typedef uint32_t rbs_constant_id_t;
+
+/**
+ * A list of constant IDs. Usually used to represent a set of locals.
+ */
+typedef struct {
+    /** The number of constant ids in the list. */
+    size_t size;
+
+    /** The number of constant ids that have been allocated in the list. */
+    size_t capacity;
+
+    /** The constant ids in the list. */
+    rbs_constant_id_t *ids;
+} rbs_constant_id_list_t;
+
+/**
+ * Initialize a list of constant ids.
+ *
+ * @param list The list to initialize.
+ */
+void rbs_constant_id_list_init(rbs_constant_id_list_t *list);
+
+/**
+ * Initialize a list of constant ids with a given capacity.
+ *
+ * @param list The list to initialize.
+ * @param capacity The initial capacity of the list.
+ */
+void rbs_constant_id_list_init_capacity(rbs_constant_id_list_t *list, size_t capacity);
+
+/**
+ * Append a constant id to a list of constant ids. Returns false if any
+ * potential reallocations fail.
+ *
+ * @param list The list to append to.
+ * @param id The id to append.
+ * @return Whether the append succeeded.
+ */
+bool rbs_constant_id_list_append(rbs_constant_id_list_t *list, rbs_constant_id_t id);
+
+/**
+ * Insert a constant id into a list of constant ids at the specified index.
+ *
+ * @param list The list to insert into.
+ * @param index The index at which to insert.
+ * @param id The id to insert.
+ */
+void rbs_constant_id_list_insert(rbs_constant_id_list_t *list, size_t index, rbs_constant_id_t id);
+
+/**
+ * Checks if the current constant id list includes the given constant id.
+ *
+ * @param list The list to check.
+ * @param id The id to check for.
+ * @return Whether the list includes the given id.
+ */
+bool rbs_constant_id_list_includes(rbs_constant_id_list_t *list, rbs_constant_id_t id);
+
+/**
+ * Free the memory associated with a list of constant ids.
+ *
+ * @param list The list to free.
+ */
+void rbs_constant_id_list_free(rbs_constant_id_list_t *list);
+
+/**
+ * The type of bucket in the constant pool hash map. This determines how the
+ * bucket should be freed.
+ */
+typedef unsigned int rbs_constant_pool_bucket_type_t;
+
+/** By default, each constant is a slice of the source. */
+static const rbs_constant_pool_bucket_type_t RBS_CONSTANT_POOL_BUCKET_DEFAULT = 0;
+
+/** An owned constant is one for which memory has been allocated. */
+static const rbs_constant_pool_bucket_type_t RBS_CONSTANT_POOL_BUCKET_OWNED = 1;
+
+/** A constant constant is known at compile time. */
+static const rbs_constant_pool_bucket_type_t RBS_CONSTANT_POOL_BUCKET_CONSTANT = 2;
+
+/** A bucket in the hash map. */
+typedef struct {
+    /** The incremental ID used for indexing back into the pool. */
+    unsigned int id: 30;
+
+    /** The type of the bucket, which determines how to free it. */
+    rbs_constant_pool_bucket_type_t type: 2;
+
+    /** The hash of the bucket. */
+    uint32_t hash;
+} rbs_constant_pool_bucket_t;
+
+/** A constant in the pool which effectively stores a string. */
+typedef struct {
+    /** A pointer to the start of the string. */
+    const uint8_t *start;
+
+    /** The length of the string. */
+    size_t length;
+} rbs_constant_t;
+
+/** The overall constant pool, which stores constants found while parsing. */
+typedef struct {
+    /** The buckets in the hash map. */
+    rbs_constant_pool_bucket_t *buckets;
+
+    /** The constants that are stored in the buckets. */
+    rbs_constant_t *constants;
+
+    /** The number of buckets in the hash map. */
+    uint32_t size;
+
+    /** The number of buckets that have been allocated in the hash map. */
+    uint32_t capacity;
+} rbs_constant_pool_t;
+
+// A global constant pool for storing permenant keywords, such as the names of location children in `parser.c`.
+extern rbs_constant_pool_t *RBS_GLOBAL_CONSTANT_POOL;
+
+/**
+ * Initialize a new constant pool with a given capacity.
+ *
+ * @param pool The pool to initialize.
+ * @param capacity The initial capacity of the pool.
+ * @return Whether the initialization succeeded.
+ */
+bool rbs_constant_pool_init(rbs_constant_pool_t *pool, uint32_t capacity);
+
+/**
+ * Return a pointer to the constant indicated by the given constant id.
+ *
+ * @param pool The pool to get the constant from.
+ * @param constant_id The id of the constant to get.
+ * @return A pointer to the constant.
+ */
+rbs_constant_t * rbs_constant_pool_id_to_constant(const rbs_constant_pool_t *pool, rbs_constant_id_t constant_id);
+
+/**
+ * Find a constant in a constant pool. Returns the id of the constant, or 0 if
+ * the constant is not found.
+ *
+ * @param pool The pool to find the constant in.
+ * @param start A pointer to the start of the constant.
+ * @param length The length of the constant.
+ * @return The id of the constant.
+ */
+rbs_constant_id_t rbs_constant_pool_find(const rbs_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+/**
+ * Insert a constant into a constant pool that is a slice of a source string.
+ * Returns the id of the constant, or 0 if any potential calls to resize fail.
+ *
+ * @param pool The pool to insert the constant into.
+ * @param start A pointer to the start of the constant.
+ * @param length The length of the constant.
+ * @return The id of the constant.
+ */
+rbs_constant_id_t rbs_constant_pool_insert_shared(rbs_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+/**
+ * Insert a constant into a constant pool from memory that is now owned by the
+ * constant pool. Returns the id of the constant, or 0 if any potential calls to
+ * resize fail.
+ *
+ * @param pool The pool to insert the constant into.
+ * @param start A pointer to the start of the constant.
+ * @param length The length of the constant.
+ * @return The id of the constant.
+ */
+rbs_constant_id_t rbs_constant_pool_insert_owned(rbs_constant_pool_t *pool, uint8_t *start, size_t length);
+
+/**
+ * Insert a constant into a constant pool from memory that is constant. Returns
+ * the id of the constant, or 0 if any potential calls to resize fail.
+ *
+ * @param pool The pool to insert the constant into.
+ * @param start A pointer to the start of the constant.
+ * @param length The length of the constant.
+ * @return The id of the constant.
+ */
+rbs_constant_id_t rbs_constant_pool_insert_constant(rbs_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+/**
+ * Free the memory associated with a constant pool.
+ *
+ * @param pool The pool to free.
+ */
+void rbs_constant_pool_free(rbs_constant_pool_t *pool);
+
+#endif

data/lib/rbs/ast/declarations.rb

@@ -349,12 +349,14 @@ module RBS
         attr_reader :type
         attr_reader :location
         attr_reader :comment
+        attr_reader :annotations
 
-        def initialize(name:, type:, location:, comment:)
+        def initialize(name:, type:, location:, comment:, annotations: [])
           @name = name
           @type = type
           @location = location
           @comment = comment
+          @annotations = annotations || []
         end
 
         def ==(other)
@@ -385,12 +387,14 @@ module RBS
         attr_reader :type
         attr_reader :location
         attr_reader :comment
+        attr_reader :annotations
 
-        def initialize(name:, type:, location:, comment:)
+        def initialize(name:, type:, location:, comment:, annotations: [])
           @name = name
           @type = type
           @location = location
           @comment = comment
+          @annotations = annotations
         end
 
         def ==(other)
@@ -417,13 +421,14 @@ module RBS
       end
 
       class AliasDecl < Base
-        attr_reader :new_name, :old_name, :location, :comment
+        attr_reader :new_name, :old_name, :location, :comment, :annotations
 
-        def initialize(new_name:, old_name:, location:, comment:)
+        def initialize(new_name:, old_name:, location:, comment:, annotations: [])
           @new_name = new_name
           @old_name = old_name
           @location = location
           @comment = comment
+          @annotations = annotations
         end
 
         def ==(other)

data/lib/rbs/ast/directives.rb

@@ -34,6 +34,16 @@ module RBS
         end
       end
 
+      class ResolveTypeNames < Base
+        attr_reader :location
+
+        attr_reader :value
+
+        def initialize(value:, location:)
+          @value = value
+          @location = location
+        end
+      end
     end
   end
 end

data/lib/rbs/ast/members.rb

@@ -30,6 +30,8 @@ module RBS
           end
 
           def sub(subst)
+            return self if subst.empty?
+
             update(method_type: self.method_type.sub(subst))
           end
 

data/lib/rbs/ast/type_param.rb

@@ -5,13 +5,13 @@ module RBS
     class TypeParam
       attr_reader :name, :variance, :location, :upper_bound_type, :default_type
 
-      def initialize(name:, variance:, upper_bound:, location:, default_type: nil)
+      def initialize(name:, variance:, upper_bound:, location:, default_type: nil, unchecked: false)
         @name = name
         @variance = variance
         @upper_bound_type = upper_bound
         @location = location
-        @unchecked = false
         @default_type = default_type
+        @unchecked = unchecked
       end
 
       def upper_bound

data/lib/rbs/cli/validate.rb

@@ -196,6 +196,7 @@ EOU
                   end
                 InvalidTypeApplicationError.check!(type_name: member.name, params: params, args: member.args, location: member.location)
               when AST::Members::Var
+                @validator.validate_variable(member)
                 void_type_context_validator(member.type)
                 if member.is_a?(AST::Members::ClassVariable)
                   no_self_type_validator(member.type)

data/lib/rbs/cli.rb

@@ -587,7 +587,7 @@ EOU
             merge = true
           end
           opts.on("--todo", "Generates only undefined methods compared to objects") do
-            Warning.warn("Geneating prototypes with `--todo` option is experimental\n", category: :experimental)
+            Warning.warn("Generating prototypes with `--todo` option is experimental\n", category: :experimental)
             todo = true
           end
           opts.on("--method-owner CLASS", "Generate method prototypes if the owner of the method is [CLASS]") do |klass|
@@ -1155,7 +1155,7 @@ EOB
             # Generate RBS files from the codebase.
             $ rbs prototype rb lib/ > generated.rbs
 
-            # Write more descrictive types by hand.
+            # Write more descriptive types by hand.
             $ $EDITOR handwritten.rbs
 
             # Remove hand-written method definitions from generated.rbs.
@@ -1163,7 +1163,7 @@ EOB
 
           Options:
         HELP
-        opts.on('-w', '--write', 'Overwrite files directry') { write_to_file = true }
+        opts.on('-w', '--write', 'Overwrite files directory') { write_to_file = true }
         opts.on('--subtrahend=PATH', '') { |path| subtrahend_paths << path }
         opts.parse!(args)
       end

data/lib/rbs/collection/config/lockfile_generator.rb

@@ -4,7 +4,17 @@ module RBS
   module Collection
     class Config
       class LockfileGenerator
-        ALUMNI_STDLIBS = { "mutex_m" => ">= 0.3.0" }
+        ALUMNI_STDLIBS = {
+          "mutex_m" => ">= 0.3.0",
+          "abbrev" => nil,
+          "base64" => nil,
+          "bigdecimal" => nil,
+          "csv" => nil,
+          "minitest" => nil,
+          "net-smtp" => nil,
+          "nkf" => nil,
+          "observer" => nil,
+        }
 
         class GemfileLockMismatchError < StandardError
           def initialize(expected:, actual:)
@@ -161,17 +171,24 @@ module RBS
           return if lockfile.gems.key?(name)
 
           case name
-          when 'rubygems', 'set'
-            msg = "`#{name}` has been moved to core library, so it is always loaded. Remove explicit loading `#{name}`"
-            msg << " from `#{from_gem}`" if from_gem
-            msg << "."
+          when 'bigdecimal-math'
+            # The `bigdecimal-math` is never released as a gem.
+            # Therefore, `assign_gem` should not be called.
+            RBS.logger.info {
+              from = from_gem || "rbs_collection.yaml"
+              "`#{name}` is included in the RBS dependencies of `#{from}`, but the type definition as a stdlib in rbs-gem is deprecated. Delete `#{name}` from the RBS dependencies of `#{from}`."
+            }
+            source = find_source(name: name)
+            if source&.is_a?(Sources::Stdlib)
+              lockfile.gems[name] = { name: name, version: "0", source: source }
+            end
             return
           when *ALUMNI_STDLIBS.keys
             version = ALUMNI_STDLIBS.fetch(name)
             if from_gem
               # From `dependencies:` of a `manifest.yaml` of a gem
               source = find_source(name: name) or raise
-              if source.is_a?(Sources::Stdlib)
+              if source.is_a?(Sources::Stdlib) && version
                 RBS.logger.warn {
                   "`#{name}` is included in the RBS dependencies of `#{from_gem}`, but the type definition as a stdlib in rbs-gem is deprecated. Add `#{name}` (#{version}) to the dependency of your Ruby program to use the gem-bundled type definition."
                 }
@@ -185,7 +202,11 @@ module RBS
             else
               # From `gems:` of a `rbs_collection.yaml`
               RBS.logger.warn {
-                "`#{name}` as a stdlib in rbs-gem is deprecated. Add `#{name}` (#{version}) to the dependency of your Ruby program to use the gem-bundled type definition."
+                if version
+                  "`#{name}` as a stdlib in rbs-gem is deprecated. Add `#{name}` (#{version}) to the dependency of your Ruby program to use the gem-bundled type definition."
+                else
+                  "`#{name}` as a stdlib in rbs-gem is deprecated. Delete `#{name}` from the RBS dependencies in your rbs_collection.yaml."
+                end
               }
             end
           end

data/lib/rbs/collection/sources/rubygems.rb

@@ -5,7 +5,7 @@ require 'singleton'
 module RBS
   module Collection
     module Sources
-      # Signatures that are inclduded in gem package as sig/ directory.
+      # Signatures that are included in gem package as sig/ directory.
       class Rubygems
         include Base
         include Singleton