prism 1.9.0 → 2.0.0.rc2

data/lib/prism.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 # :markup: markdown
+#--
+# rbs_inline: enabled
 
 # The Prism Ruby parser.
 #
@@ -21,7 +23,7 @@ module Prism
   autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
   autoload :MutationCompiler, "prism/mutation_compiler"
-  autoload :Pack, "prism/pack"
+  autoload :NodeFind, "prism/node_find"
   autoload :Pattern, "prism/pattern"
   autoload :Reflection, "prism/reflection"
   autoload :Relocation, "prism/relocation"
@@ -33,19 +35,24 @@ module Prism
   # Some of these constants are not meant to be exposed, so marking them as
   # private here.
 
-  private_constant :LexCompat
+  if RUBY_ENGINE != "jruby"
+    private_constant :LexCompat
+    private_constant :NodeFind
+  end
 
   # Raised when requested to parse as the currently running Ruby version but Prism has no support for it.
   class CurrentVersionError < ArgumentError
     # Initialize a new exception for the given ruby version string.
+    #
+    #: (String version) -> void
     def initialize(version)
       message = +"invalid version: Requested to parse as `version: 'current'`; "
-      segments =
+      major, minor, =
         if version.match?(/\A\d+\.\d+.\d+\z/)
           version.split(".").map(&:to_i)
         end
 
-      if segments && ((segments[0] < 3) || (segments[0] == 3 && segments[1] < 3))
+      if major && minor && ((major < 3) || (major == 3 && minor < 3))
         message << " #{version} is below the minimum supported syntax."
       else
         message << " #{version} is unknown. Please update the `prism` gem."
@@ -56,23 +63,63 @@ module Prism
   end
 
   # :call-seq:
-  #   Prism::lex_compat(source, **options) -> LexCompat::Result
+  #   lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
-  # resembles the return value of Ripper::lex.
+  # resembles the return value of Ripper.lex.
+  #
+  # For supported options, see Prism.parse.
   #
-  # For supported options, see Prism::parse.
+  #: (String source, **untyped options) -> LexCompat::Result
   def self.lex_compat(source, **options)
     LexCompat.new(source, **options).result # steep:ignore
   end
 
   # :call-seq:
-  #   Prism::load(source, serialized, freeze) -> ParseResult
+  #   load(source, serialized, freeze) -> ParseResult
   #
   # Load the serialized AST using the source as a reference into a tree.
+  #
+  #: (String source, String serialized, ?bool freeze) -> ParseResult
   def self.load(source, serialized, freeze = false)
     Serialize.load_parse(source, serialized, freeze)
   end
+
+  # Given a Method, UnboundMethod, Proc, or Thread::Backtrace::Location,
+  # returns the Prism node representing it. On CRuby, this uses node_id for
+  # an exact match. On other implementations, it falls back to best-effort
+  # matching by source location line number.
+  #
+  #: (Method | UnboundMethod | Proc | Thread::Backtrace::Location callable) -> Node?
+  def self.find(callable)
+    NodeFind.find(callable)
+  end
+
+  # @rbs!
+  #    VERSION: String
+  #    BACKEND: :CEXT | :FFI
+  #
+  #    interface _Stream
+  #      def gets: (?Integer integer) -> (String | nil)
+  #    end
+  #
+  #    def self.parse:               (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.profile:             (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+  #    def self.lex:                 (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+  #    def self.parse_lex:           (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+  #    def self.dump:                (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+  #    def self.parse_comments:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+  #    def self.parse_success?:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_failure?:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_stream:        (_Stream stream, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.parse_file:          (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.profile_file:        (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+  #    def self.lex_file:            (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+  #    def self.parse_lex_file:      (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+  #    def self.dump_file:           (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+  #    def self.parse_file_comments: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+  #    def self.parse_file_success?: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_file_failure?: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
 end
 
 require_relative "prism/polyfill/byteindex"
@@ -89,7 +136,30 @@ if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
   # The C extension is the default backend on CRuby.
   Prism::BACKEND = :CEXT
 
-  require "prism/prism"
+  begin
+    # The precompiled native libraries are in <gem_dir>/lib/prism/<ruby_version>
+    require_relative "prism/#{RUBY_VERSION[/\d+\.\d+/]}/prism"
+  rescue LoadError => e
+    if e.message.include?("GLIBC")
+      warn(<<~EOM)
+
+        ERROR: It looks like you're trying to use Prism as a precompiled native gem on a system
+               with an unsupported version of glibc.
+
+          #{e.message}
+
+          If that's the case, then please install Prism via the `ruby` platform gem:
+              gem install prism --platform=ruby
+          or, in your Gemfile:
+              gem "prism", force_ruby_platform: true
+
+      EOM
+      raise e
+    end
+
+    # Precompiled library isn't available, fall back to the library compiled at installation time.
+    require "prism/prism"
+  end
 else
   # The FFI backend is used on other Ruby implementations.
   Prism::BACKEND = :FFI

data/prism.gemspec

@@ -2,9 +2,9 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.9.0"
-  spec.authors = ["Shopify"]
-  spec.email = ["ruby@shopify.com"]
+  spec.version = "2.0.0.rc2"
+  spec.authors = ["Ruby developers"]
+  spec.email = ["ruby-core@ruby-lang.org"]
 
   spec.summary = "Prism Ruby parser"
   spec.homepage = "https://github.com/ruby/prism"
@@ -42,31 +42,74 @@ Gem::Specification.new do |spec|
     "docs/serialization.md",
     "docs/testing.md",
     "ext/prism/api_node.c",
-    "ext/prism/api_pack.c",
+    "ext/prism/extconf.rb",
     "ext/prism/extension.c",
     "ext/prism/extension.h",
     "include/prism.h",
+    "include/prism/compiler/accel.h",
+    "include/prism/compiler/align.h",
+    "include/prism/compiler/assume.h",
+    "include/prism/compiler/bswap.h",
+    "include/prism/compiler/endianness.h",
+    "include/prism/compiler/exported.h",
+    "include/prism/compiler/fallthrough.h",
+    "include/prism/compiler/filesystem.h",
+    "include/prism/compiler/flex_array.h",
+    "include/prism/compiler/force_inline.h",
+    "include/prism/compiler/format.h",
+    "include/prism/compiler/inline.h",
+    "include/prism/compiler/nodiscard.h",
+    "include/prism/compiler/nonnull.h",
+    "include/prism/compiler/unused.h",
+    "include/prism/internal/allocator.h",
+    "include/prism/internal/allocator_debug.h",
+    "include/prism/internal/arena.h",
+    "include/prism/internal/bit.h",
+    "include/prism/internal/buffer.h",
+    "include/prism/internal/char.h",
+    "include/prism/internal/comments.h",
+    "include/prism/internal/constant_pool.h",
+    "include/prism/internal/diagnostic.h",
+    "include/prism/internal/encoding.h",
+    "include/prism/internal/integer.h",
+    "include/prism/internal/isinf.h",
+    "include/prism/internal/line_offset_list.h",
+    "include/prism/internal/list.h",
+    "include/prism/internal/magic_comments.h",
+    "include/prism/internal/memchr.h",
+    "include/prism/internal/node.h",
+    "include/prism/internal/options.h",
+    "include/prism/internal/parser.h",
+    "include/prism/internal/regexp.h",
+    "include/prism/internal/serialize.h",
+    "include/prism/internal/source.h",
+    "include/prism/internal/static_literals.h",
+    "include/prism/internal/strncasecmp.h",
+    "include/prism/internal/stringy.h",
+    "include/prism/internal/strpbrk.h",
+    "include/prism/internal/tokens.h",
+    "include/prism/internal/xxh3.h",
+    "include/prism/arena.h",
     "include/prism/ast.h",
-    "include/prism/defines.h",
+    "include/prism/buffer.h",
+    "include/prism/comments.h",
+    "include/prism/constant_pool.h",
     "include/prism/diagnostic.h",
-    "include/prism/encoding.h",
+    "include/prism/errors_format.h",
+    "include/prism/excludes.h",
+    "include/prism/integer.h",
+    "include/prism/json.h",
+    "include/prism/line_offset_list.h",
+    "include/prism/magic_comments.h",
     "include/prism/node.h",
     "include/prism/options.h",
-    "include/prism/pack.h",
     "include/prism/parser.h",
     "include/prism/prettyprint.h",
-    "include/prism/regexp.h",
-    "include/prism/static_literals.h",
-    "include/prism/util/pm_buffer.h",
-    "include/prism/util/pm_char.h",
-    "include/prism/util/pm_constant_pool.h",
-    "include/prism/util/pm_integer.h",
-    "include/prism/util/pm_list.h",
-    "include/prism/util/pm_memchr.h",
-    "include/prism/util/pm_newline_list.h",
-    "include/prism/util/pm_strncasecmp.h",
-    "include/prism/util/pm_string.h",
-    "include/prism/util/pm_strpbrk.h",
+    "include/prism/serialize.h",
+    "include/prism/source.h",
+    "include/prism/stream.h",
+    "include/prism/string_query.h",
+    "include/prism/stringy.h",
     "include/prism/version.h",
     "lib/prism.rb",
     "lib/prism/compiler.rb",
@@ -79,10 +122,9 @@ Gem::Specification.new do |spec|
     "lib/prism/lex_compat.rb",
     "lib/prism/mutation_compiler.rb",
     "lib/prism/node_ext.rb",
+    "lib/prism/node_find.rb",
     "lib/prism/node.rb",
-    "lib/prism/pack.rb",
     "lib/prism/parse_result.rb",
-    "lib/prism/parse_result/comments.rb",
     "lib/prism/parse_result/errors.rb",
     "lib/prism/parse_result/newlines.rb",
     "lib/prism/pattern.rb",
@@ -110,59 +152,83 @@ Gem::Specification.new do |spec|
     "lib/prism/translation/ruby_parser.rb",
     "lib/prism/visitor.rb",
     "prism.gemspec",
-    "rbi/prism.rbi",
-    "rbi/prism/compiler.rbi",
-    "rbi/prism/dsl.rbi",
-    "rbi/prism/inspect_visitor.rbi",
-    "rbi/prism/node_ext.rbi",
-    "rbi/prism/node.rbi",
-    "rbi/prism/parse_result.rbi",
-    "rbi/prism/reflection.rbi",
-    "rbi/prism/string_query.rbi",
+    "rbi/generated/prism.rbi",
+    "rbi/generated/prism/compiler.rbi",
+    "rbi/generated/prism/desugar_compiler.rbi",
+    "rbi/generated/prism/dispatcher.rbi",
+    "rbi/generated/prism/dot_visitor.rbi",
+    "rbi/generated/prism/dsl.rbi",
+    "rbi/generated/prism/inspect_visitor.rbi",
+    "rbi/generated/prism/lex_compat.rbi",
+    "rbi/generated/prism/mutation_compiler.rbi",
+    "rbi/generated/prism/node.rbi",
+    "rbi/generated/prism/node_ext.rbi",
+    "rbi/generated/prism/node_find.rbi",
+    "rbi/generated/prism/parse_result.rbi",
+    "rbi/generated/prism/pattern.rbi",
+    "rbi/generated/prism/reflection.rbi",
+    "rbi/generated/prism/relocation.rbi",
+    "rbi/generated/prism/serialize.rbi",
+    "rbi/generated/prism/string_query.rbi",
+    "rbi/generated/prism/translation.rbi",
+    "rbi/generated/prism/visitor.rbi",
+    "rbi/generated/prism/parse_result/errors.rbi",
+    "rbi/generated/prism/parse_result/newlines.rbi",
     "rbi/prism/translation/parser.rbi",
     "rbi/prism/translation/parser_versions.rbi",
     "rbi/prism/translation/ripper.rbi",
-    "rbi/prism/visitor.rbi",
-    "sig/prism.rbs",
-    "sig/prism/compiler.rbs",
-    "sig/prism/dispatcher.rbs",
-    "sig/prism/dot_visitor.rbs",
-    "sig/prism/dsl.rbs",
-    "sig/prism/inspect_visitor.rbs",
-    "sig/prism/lex_compat.rbs",
-    "sig/prism/mutation_compiler.rbs",
-    "sig/prism/node_ext.rbs",
-    "sig/prism/node.rbs",
-    "sig/prism/pack.rbs",
-    "sig/prism/parse_result.rbs",
-    "sig/prism/parse_result/comments.rbs",
-    "sig/prism/pattern.rbs",
-    "sig/prism/reflection.rbs",
-    "sig/prism/relocation.rbs",
-    "sig/prism/serialize.rbs",
-    "sig/prism/string_query.rbs",
-    "sig/prism/visitor.rbs",
+    "rbi/rubyvm/node_find.rbi",
+    "sig/generated/prism.rbs",
+    "sig/generated/prism/compiler.rbs",
+    "sig/generated/prism/desugar_compiler.rbs",
+    "sig/generated/prism/dispatcher.rbs",
+    "sig/generated/prism/dot_visitor.rbs",
+    "sig/generated/prism/dsl.rbs",
+    "sig/generated/prism/inspect_visitor.rbs",
+    "sig/generated/prism/lex_compat.rbs",
+    "sig/generated/prism/mutation_compiler.rbs",
+    "sig/generated/prism/node.rbs",
+    "sig/generated/prism/node_ext.rbs",
+    "sig/generated/prism/node_find.rbs",
+    "sig/generated/prism/parse_result.rbs",
+    "sig/generated/prism/pattern.rbs",
+    "sig/generated/prism/reflection.rbs",
+    "sig/generated/prism/relocation.rbs",
+    "sig/generated/prism/serialize.rbs",
+    "sig/generated/prism/string_query.rbs",
+    "sig/generated/prism/translation.rbs",
+    "sig/generated/prism/visitor.rbs",
+    "sig/generated/prism/parse_result/errors.rbs",
+    "sig/generated/prism/parse_result/newlines.rbs",
+    "sig/manifest.yaml",
+    "src/arena.c",
+    "src/buffer.c",
+    "src/char.c",
+    "src/comments.c",
+    "src/constant_pool.c",
     "src/diagnostic.c",
     "src/encoding.c",
+    "src/errors_format.c",
+    "src/integer.c",
+    "src/json.c",
+    "src/line_offset_list.c",
+    "src/list.c",
+    "src/memchr.c",
     "src/node.c",
     "src/options.c",
-    "src/pack.c",
+    "src/parser.c",
     "src/prettyprint.c",
     "src/prism.c",
     "src/regexp.c",
     "src/serialize.c",
+    "src/source.c",
     "src/static_literals.c",
-    "src/token_type.c",
-    "src/util/pm_buffer.c",
-    "src/util/pm_char.c",
-    "src/util/pm_constant_pool.c",
-    "src/util/pm_integer.c",
-    "src/util/pm_list.c",
-    "src/util/pm_memchr.c",
-    "src/util/pm_newline_list.c",
-    "src/util/pm_string.c",
-    "src/util/pm_strncasecmp.c",
-    "src/util/pm_strpbrk.c"
+    "src/string_query.c",
+    "src/stringy.c",
+    "src/strncasecmp.c",
+    "src/strpbrk.c",
+    "src/tokens.c",
+    "src/xxh3.c"
   ]
 
   spec.extensions = ["ext/prism/extconf.rb"]