rbs 3.5.0.pre.2 → 3.5.1.pre.1

data/ext/rbs_extension/lexer.h

@@ -78,6 +78,8 @@ enum TokenType {
   tCOMMENT,         /* Comment */
   tLINECOMMENT,     /* Comment of all line */
 
+  tTRIVIA,          /* Trivia tokens -- space and new line */
+
   tDQSTRING,        /* Double quoted string */
   tSQSTRING,        /* Single quoted string */
   tINTEGER,         /* Integer */

data/ext/rbs_extension/lexer.re

@@ -3,7 +3,6 @@
 token rbsparser_next_token(lexstate *state) {
   lexstate backup;
 
-start:
   backup = *state;
 
   /*!re2c
@@ -139,9 +138,9 @@ start:
 
       "$" global_ident      { return next_token(state, tGIDENT); }
 
-      skip = [ \t\n\r]+;
+      skip = ([ \t]+|[\r\n]);
 
-      skip     { state->start = state->current; goto start; }
+      skip     { return next_token(state, tTRIVIA); }
       "\x00"   { return next_token(state, pEOF); }
       *        { return next_token(state, ErrorToken); }
   */

data/ext/rbs_extension/lexstate.c

@@ -77,6 +77,8 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
   "tCOMMENT",
   "tLINECOMMENT",
 
+  "tTRIVIA",
+
   "tDQSTRING",        /* Double quoted string */
   "tSQSTRING",        /* Single quoted string */
   "tINTEGER",         /* Integer */
@@ -120,7 +122,9 @@ token next_token(lexstate *state, enum TokenType type) {
   t.range.start = state->start;
   t.range.end = state->current;
   state->start = state->current;
-  state->first_token_of_line = false;
+  if (type != tTRIVIA) {
+    state->first_token_of_line = false;
+  }
 
   return t;
 }

data/ext/rbs_extension/location.c

@@ -2,6 +2,7 @@
 
 #define RBS_LOC_REQUIRED_P(loc, i) ((loc)->children->required_p & (1 << (i)))
 #define RBS_LOC_OPTIONAL_P(loc, i) (!RBS_LOC_REQUIRED_P((loc), (i)))
+#define RBS_LOC_CHILDREN_SIZE(cap) (sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * ((cap) - 1))
 
 VALUE RBS_Location;
 
@@ -25,7 +26,7 @@ static void check_children_max(unsigned short n) {
 void rbs_loc_alloc_children(rbs_loc *loc, unsigned short cap) {
   check_children_max(cap);
 
-  size_t s = sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * cap;
+  size_t s = RBS_LOC_CHILDREN_SIZE(cap);
   loc->children = malloc(s);
 
   loc->children->len = 0;
@@ -39,7 +40,7 @@ static void check_children_cap(rbs_loc *loc) {
   } else {
     if (loc->children->len == loc->children->cap) {
       check_children_max(loc->children->cap + 1);
-      size_t s = sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * (++loc->children->cap);
+      size_t s = RBS_LOC_CHILDREN_SIZE(++loc->children->cap);
       loc->children = realloc(loc->children, s);
     }
   }
@@ -85,7 +86,7 @@ static size_t rbs_loc_memsize(const void *ptr) {
   if (loc->children == NULL) {
     return sizeof(rbs_loc);
   } else {
-    return sizeof(rbs_loc) + sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * loc->children->cap;
+    return sizeof(rbs_loc) + RBS_LOC_CHILDREN_SIZE(loc->children->cap);
   }
 }
 
@@ -129,7 +130,7 @@ static VALUE location_initialize_copy(VALUE self, VALUE other) {
   self_loc->rg = other_loc->rg;
   if (other_loc->children != NULL) {
     rbs_loc_alloc_children(self_loc, other_loc->children->cap);
-    memcpy(self_loc->children, other_loc->children, sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * other_loc->children->cap);
+    memcpy(self_loc->children, other_loc->children, RBS_LOC_CHILDREN_SIZE(other_loc->children->cap));
   }
 
   return Qnil;

data/ext/rbs_extension/location.h

@@ -16,17 +16,19 @@ typedef struct {
 
 typedef unsigned int rbs_loc_entry_bitmap;
 
+// The flexible array always allocates, but it's okay.
+// This struct is not allocated when the `rbs_loc` doesn't have children.
 typedef struct {
   unsigned short len;
   unsigned short cap;
   rbs_loc_entry_bitmap required_p;
-  rbs_loc_entry entries[0];
+  rbs_loc_entry entries[1];
 } rbs_loc_children;
 
 typedef struct {
   VALUE buffer;
   range rg;
-  rbs_loc_children *children;
+  rbs_loc_children *children; // NULL when no children is allocated
 } rbs_loc;
 
 /**

data/ext/rbs_extension/parser.c

@@ -2805,56 +2805,103 @@ VALUE parse_signature(parserstate *state) {
   return ret;
 }
 
+struct parse_type_arg {
+  parserstate *parser;
+  VALUE require_eof;
+};
+
 static VALUE
-rbsparser_parse_type(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos, VALUE variables, VALUE require_eof)
-{
-  parserstate *parser = alloc_parser(buffer, FIX2INT(start_pos), FIX2INT(end_pos), variables);
+ensure_free_parser(VALUE parser) {
+  free_parser((parserstate *)parser);
+  return Qnil;
+}
+
+static VALUE
+parse_type_try(VALUE a) {
+  struct parse_type_arg *arg = (struct parse_type_arg *)a;
 
-  if (parser->next_token.type == pEOF) {
-    free_parser(parser);
+  if (arg->parser->next_token.type == pEOF) {
     return Qnil;
   }
 
-  VALUE type = parse_type(parser);
+  VALUE type = parse_type(arg->parser);
 
-  if (RB_TEST(require_eof)) {
-    parser_advance_assert(parser, pEOF);
+  if (RB_TEST(arg->require_eof)) {
+    parser_advance_assert(arg->parser, pEOF);
   }
 
-  free_parser(parser);
-
   return type;
 }
 
 static VALUE
-rbsparser_parse_method_type(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos, VALUE variables, VALUE require_eof)
+rbsparser_parse_type(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos, VALUE variables, VALUE require_eof)
 {
   parserstate *parser = alloc_parser(buffer, FIX2INT(start_pos), FIX2INT(end_pos), variables);
+  struct parse_type_arg arg = {
+    parser,
+    require_eof
+  };
+  return rb_ensure(parse_type_try, (VALUE)&arg, ensure_free_parser, (VALUE)parser);
+}
 
-  if (parser->next_token.type == pEOF) {
-    free_parser(parser);
+static VALUE
+parse_method_type_try(VALUE a) {
+  struct parse_type_arg *arg = (struct parse_type_arg *)a;
+
+  if (arg->parser->next_token.type == pEOF) {
     return Qnil;
   }
 
-  VALUE method_type = parse_method_type(parser);
+  VALUE method_type = parse_method_type(arg->parser);
 
-  if (RB_TEST(require_eof)) {
-    parser_advance_assert(parser, pEOF);
+  if (RB_TEST(arg->require_eof)) {
+    parser_advance_assert(arg->parser, pEOF);
   }
 
-  free_parser(parser);
-
   return method_type;
 }
 
+static VALUE
+rbsparser_parse_method_type(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos, VALUE variables, VALUE require_eof)
+{
+  parserstate *parser = alloc_parser(buffer, FIX2INT(start_pos), FIX2INT(end_pos), variables);
+  struct parse_type_arg arg = {
+    parser,
+    require_eof
+  };
+  return rb_ensure(parse_method_type_try, (VALUE)&arg, ensure_free_parser, (VALUE)parser);
+}
+
+static VALUE
+parse_signature_try(VALUE a) {
+  parserstate *parser = (parserstate *)a;
+  return parse_signature(parser);
+}
+
 static VALUE
 rbsparser_parse_signature(VALUE self, VALUE buffer, VALUE end_pos)
 {
   parserstate *parser = alloc_parser(buffer, 0, FIX2INT(end_pos), Qnil);
-  VALUE pair = parse_signature(parser);
-  free_parser(parser);
+  return rb_ensure(parse_signature_try, (VALUE)parser, ensure_free_parser, (VALUE)parser);
+}
+
+static VALUE
+rbsparser_lex(VALUE self, VALUE buffer, VALUE end_pos) {
+  lexstate *lexer = alloc_lexer(buffer, 0, FIX2INT(end_pos));
+  VALUE results = rb_ary_new();
+
+  token token = NullToken;
+  while (token.type != pEOF) {
+    token = rbsparser_next_token(lexer);
+    VALUE type = ID2SYM(rb_intern(token_type_str(token.type)));
+    VALUE location = rbs_new_location(buffer, token.range);
+    VALUE pair = rb_ary_new3(2, type, location);
+    rb_ary_push(results, pair);
+  }
+
+  free(lexer);
 
-  return pair;
+  return results;
 }
 
 void rbs__init_parser(void) {
@@ -2862,4 +2909,5 @@ void rbs__init_parser(void) {
   rb_define_singleton_method(RBS_Parser, "_parse_type", rbsparser_parse_type, 5);
   rb_define_singleton_method(RBS_Parser, "_parse_method_type", rbsparser_parse_method_type, 5);
   rb_define_singleton_method(RBS_Parser, "_parse_signature", rbsparser_parse_signature, 2);
+  rb_define_singleton_method(RBS_Parser, "_lex", rbsparser_lex, 2);
 }

data/ext/rbs_extension/parserstate.c

@@ -109,6 +109,8 @@ void parser_advance(parserstate *state) {
       // skip
     } else if (state->next_token3.type == tLINECOMMENT) {
       insert_comment_line(state, state->next_token3);
+    } else if (state->next_token3.type == tTRIVIA) {
+      //skip
     } else {
       break;
     }
@@ -272,7 +274,7 @@ VALUE comment_to_ruby(comment *com, VALUE buffer) {
   );
 }
 
-parserstate *alloc_parser(VALUE buffer, int start_pos, int end_pos, VALUE variables) {
+lexstate *alloc_lexer(VALUE buffer, int start_pos, int end_pos) {
   VALUE string = rb_funcall(buffer, rb_intern("content"), 0);
 
   StringValue(string);
@@ -290,6 +292,11 @@ parserstate *alloc_parser(VALUE buffer, int start_pos, int end_pos, VALUE variab
   lexer->start = lexer->current;
   lexer->first_token_of_line = lexer->current.column == 0;
 
+  return lexer;
+}
+
+parserstate *alloc_parser(VALUE buffer, int start_pos, int end_pos, VALUE variables) {
+  lexstate *lexer = alloc_lexer(buffer, start_pos, end_pos);
   parserstate *parser = calloc(1, sizeof(parserstate));
   parser->lexstate = lexer;
   parser->buffer = buffer;

data/ext/rbs_extension/parserstate.h

@@ -93,6 +93,15 @@ void parser_insert_typevar(parserstate *state, ID id);
  * */
 bool parser_typevar_member(parserstate *state, ID id);
 
+/**
+ * Allocate new lexstate object.
+ *
+ * ```
+ * alloc_lexer(buffer, 0, 31)    // New lexstate with buffer
+ * ```
+ * */
+lexstate *alloc_lexer(VALUE buffer, int start_pos, int end_pos);
+
 /**
  * Allocate new parserstate object.
  *

data/lib/rbs/cli/validate.rb

@@ -66,8 +66,8 @@ Examples:
   $ rbs validate
 EOU
 
-          opts.on("--silent") do
-            RBS.print_warning { "`--silent` option is deprecated." }
+          opts.on("--silent", "This option has been deprecated and does nothing.") do
+            RBS.print_warning { "`--silent` option is deprecated because it's silent by default. You can use --log-level option of rbs command to display more information." }
           end
           opts.on("--[no-]exit-error-on-syntax-error", "exit(1) if syntax error is detected") {|bool|
             exit_error = bool

data/lib/rbs/cli.rb

@@ -3,7 +3,6 @@
 require "open3"
 require "optparse"
 require "shellwords"
-require "abbrev"
 require "stringio"
 
 module RBS
@@ -1062,14 +1061,13 @@ EOB
       config_path = options.config_path or raise
       lock_path = Collection::Config.to_lockfile_path(config_path)
 
-      subcommand = Abbrev.abbrev(['install', 'update', 'help'])[args[0]] || args[0]
-      case subcommand
-      when 'install'
+      case args[0]
+      when 'install', 'instal', 'insta', 'inst', 'ins', 'in', 'i'
         unless params[:frozen]
           Collection::Config.generate_lockfile(config_path: config_path, definition: Bundler.definition)
         end
         Collection::Installer.new(lockfile_path: lock_path, stdout: stdout).install_from_lockfile
-      when 'update'
+      when 'update', 'updat', 'upda', 'upd', 'up', 'u'
         # TODO: Be aware of argv to update only specified gem
         Collection::Config.generate_lockfile(config_path: config_path, definition: Bundler.definition, with_lockfile: false)
         Collection::Installer.new(lockfile_path: lock_path, stdout: stdout).install_from_lockfile
@@ -1107,7 +1105,7 @@ EOB
           exit 1
         end
         Collection::Cleaner.new(lockfile_path: lock_path)
-      when 'help'
+      when 'help', 'hel', 'he', 'h'
         puts opts.help
       else
         puts opts.help

data/lib/rbs/location_aux.rb

@@ -5,7 +5,12 @@ module RBS
     def inspect
       rks = each_required_key.to_a
       ops = each_optional_key.to_a.map {|x| "?#{x}" }
-      "#<#{self.class}:#{self.__id__} buffer=#{buffer.name}, start=#{start_line}:#{start_column}, pos=#{start_pos}...#{end_pos}, children=#{(rks + ops).join(",")} source='#{source.lines.first&.chomp}'>"
+      src = if source.length <= 1
+        source.inspect
+      else
+        source.each_line.first&.chomp&.inspect
+      end
+      "#<#{self.class}:#{self.__id__} buffer=#{buffer.name}, start=#{start_line}:#{start_column}, pos=#{start_pos}...#{end_pos}, children=#{(rks + ops).join(",")} source=#{src}>"
     end
 
     def self.new(buffer_ = nil, start_pos_ = nil, end_pos_ = nil, buffer: nil, start_pos: nil, end_pos: nil)

data/lib/rbs/parser/lex_result.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RBS
+  class Parser
+    class LexResult
+      attr_reader :buffer
+      attr_reader :value
+
+      def initialize(buffer:, value:)
+        @buffer = buffer
+        @value = value
+      end
+    end
+  end
+end

data/lib/rbs/parser/token.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RBS
+  class Parser
+    class Token
+      attr_reader :type
+      attr_reader :location
+
+      def initialize(type:, location:)
+        @type = type
+        @location = location
+      end
+
+      def value
+        @location.source
+      end
+
+      def comment?
+        @type == :tCOMMENT || @type == :tLINECOMMENT
+      end
+    end
+  end
+end

data/lib/rbs/parser_aux.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "parser/lex_result"
+require_relative "parser/token"
+
 module RBS
   class Parser
     def self.parse_type(source, range: 0..., variables: [], require_eof: false)
@@ -19,6 +22,15 @@ module RBS
       [buf, dirs, decls]
     end
 
+    def self.lex(source)
+      buf = buffer(source)
+      list = _lex(buf, buf.last_position)
+      value = list.map do |type, location|
+        Token.new(type: type, location: location)
+      end
+      LexResult.new(buffer: buf, value: value)
+    end
+
     def self.buffer(source)
       case source
       when String

data/lib/rbs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RBS
-  VERSION = "3.5.0.pre.2"
+  VERSION = "3.5.1.pre.1"
 end

data/rbs.gemspec

@@ -28,7 +28,12 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|bin|steep|benchmark)/}) }
+    `git ls-files -z`.split("\x0").reject do |f|
+      [
+        %r{^(test|spec|features|bin|steep|benchmark)/},
+        /Gemfile/
+      ].any? {|r| f.match(r) }
+    end
   end
   spec.extensions = %w{ext/rbs_extension/extconf.rb}
 
@@ -36,5 +41,4 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.required_ruby_version = ">= 3.0"
-  spec.add_dependency "abbrev"
 end

data/sig/manifest.yaml

@@ -5,4 +5,3 @@ dependencies:
   - name: optparse
   - name: tsort
   - name: rdoc
-  - name: abbrev

data/sig/parser.rbs

@@ -1,5 +1,23 @@
 module RBS
   class Parser
+    # Result of `Parser.lex`
+    class LexResult
+      attr_reader buffer: Buffer
+      attr_reader value: Array[Token]
+
+      def initialize: (buffer: Buffer, value: Array[Token]) -> void
+    end
+
+    # Represents a token per result of `Parser.lex`.
+    class Token
+      attr_reader type: Symbol
+      attr_reader location: Location[untyped, untyped]
+
+      def initialize: (type: Symbol, location: Location[untyped, untyped]) -> void
+      def value: () -> String
+      def comment?: () -> bool
+    end
+
     # Parse a method type and return it
     #
     # When `range` keyword is specified, it starts parsing from the `begin` to the `end` of the range.
@@ -50,6 +68,14 @@ module RBS
     #
     def self.parse_signature: (Buffer | String) -> [Buffer, Array[AST::Directives::t], Array[AST::Declarations::t]]
 
+    # Parse whole RBS file and return result.
+    #
+    # ```ruby
+    # RBS::Parser.lex("# Comment\nmodule A\nend\n").value.map(&:type)
+    # # => [:tLINECOMMENT, :kMODULE, :tUIDENT, :kEND, :pEOF]
+    # ```
+    def self.lex: (Buffer | String) -> LexResult
+
     KEYWORDS: Hash[String, bot]
 
     private
@@ -62,6 +88,8 @@ module RBS
 
     def self._parse_signature: (Buffer, Integer end_pos) -> [Array[AST::Directives::t], Array[AST::Declarations::t]]
 
+    def self._lex: (Buffer, Integer end_pos) -> Array[[Symbol, Location[untyped, untyped]]]
+
     class LocatedValue
     end
   end

data/stdlib/uri/0/mailto.rbs

@@ -4,5 +4,89 @@ module URI
   # RFC6068, the mailto URL scheme.
   #
   class MailTo < Generic
+    EMAIL_REGEXP: Regexp
+
+    # <!--
+    #   rdoc-file=lib/uri/mailto.rb
+    #   - build(args)
+    # -->
+    # ## Description
+    #
+    # Creates a new URI::MailTo object from components, with syntax checking.
+    #
+    # Components can be provided as an Array or Hash. If an Array is used, the
+    # components must be supplied as `[to, headers]`.
+    #
+    # If a Hash is used, the keys are the component names preceded by colons.
+    #
+    # The headers can be supplied as a pre-encoded string, such as
+    # `"subject=subscribe&cc=address"`, or as an Array of Arrays like `[['subject',
+    # 'subscribe'], ['cc', 'address']]`.
+    #
+    # Examples:
+    #
+    #     require 'uri'
+    #
+    #     m1 = URI::MailTo.build(['joe@example.com', 'subject=Ruby'])
+    #     m1.to_s  # => "mailto:joe@example.com?subject=Ruby"
+    #
+    #     m2 = URI::MailTo.build(['john@example.com', [['Subject', 'Ruby'], ['Cc', 'jack@example.com']]])
+    #     m2.to_s  # => "mailto:john@example.com?Subject=Ruby&Cc=jack@example.com"
+    #
+    #     m3 = URI::MailTo.build({:to => 'listman@example.com', :headers => [['subject', 'subscribe']]})
+    #     m3.to_s  # => "mailto:listman@example.com?subject=subscribe"
+    #
+    def self.build: (Array[String]) -> instance
+                  | ([String, Array[Array[String]]]) -> instance
+                  | (Hash[Symbol, String | Array[Array[String]]]) -> instance
+
+    # <!-- rdoc-file=lib/uri/mailto.rb -->
+    # E-mail headers set by the URL, as an Array of Arrays.
+    #
+    def headers: () -> Array[[String, String]]
+
+    # <!--
+    #   rdoc-file=lib/uri/mailto.rb
+    #   - headers=(v)
+    # -->
+    # Setter for headers `v`.
+    #
+    def headers=: (String) -> String
+
+    # <!-- rdoc-file=lib/uri/mailto.rb -->
+    # The primary e-mail address of the URL, as a String.
+    #
+    def to: () -> String
+
+    # <!--
+    #   rdoc-file=lib/uri/mailto.rb
+    #   - to=(v)
+    # -->
+    # Setter for to `v`.
+    #
+    def to=: (String) -> String
+
+    # <!--
+    #   rdoc-file=lib/uri/mailto.rb
+    #   - to_mailtext()
+    # -->
+    # Returns the RFC822 e-mail text equivalent of the URL, as a String.
+    #
+    # Example:
+    #
+    #     require 'uri'
+    #
+    #     uri = URI.parse("mailto:ruby-list@ruby-lang.org?Subject=subscribe&cc=myaddr")
+    #     uri.to_mailtext
+    #     # => "To: ruby-list@ruby-lang.org\nSubject: subscribe\nCc: myaddr\n\n\n"
+    #
+    def to_mailtext: () -> String
+
+    # <!--
+    #   rdoc-file=lib/uri/mailto.rb
+    #   - to_rfc822text()
+    # -->
+    #
+    def to_rfc822text: () -> String
   end
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rbs
 version: !ruby/object:Gem::Version
-  version: 3.5.0.pre.2
+  version: 3.5.1.pre.1
 platform: ruby
 authors:
 - Soutaro Matsumoto
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-23 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: abbrev
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+date: 2024-06-07 00:00:00.000000000 Z
+dependencies: []
 description: RBS is the language for type signatures for Ruby and standard library
   definitions.
 email:
@@ -36,6 +22,7 @@ extra_rdoc_files: []
 files:
 - ".github/dependabot.yml"
 - ".github/workflows/comments.yml"
+- ".github/workflows/dependabot.yml"
 - ".github/workflows/ruby.yml"
 - ".github/workflows/typecheck.yml"
 - ".gitignore"
@@ -43,8 +30,6 @@ files:
 - BSDL
 - CHANGELOG.md
 - COPYING
-- Gemfile
-- Gemfile.lock
 - README.md
 - Rakefile
 - Steepfile
@@ -210,6 +195,8 @@ files:
 - lib/rbs/locator.rb
 - lib/rbs/method_type.rb
 - lib/rbs/namespace.rb
+- lib/rbs/parser/lex_result.rb
+- lib/rbs/parser/token.rb
 - lib/rbs/parser_aux.rb
 - lib/rbs/prototype/helpers.rb
 - lib/rbs/prototype/node_usage.rb