prism 1.8.1 → 2.0.0.rc1

data/lib/prism.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 # :markup: markdown
+#--
+# rbs_inline: enabled
 
 # The Prism Ruby parser.
 #
@@ -20,9 +22,8 @@ module Prism
   autoload :DSL, "prism/dsl"
   autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
-  autoload :LexRipper, "prism/lex_ripper"
   autoload :MutationCompiler, "prism/mutation_compiler"
-  autoload :Pack, "prism/pack"
+  autoload :NodeFind, "prism/node_find"
   autoload :Pattern, "prism/pattern"
   autoload :Reflection, "prism/reflection"
   autoload :Relocation, "prism/relocation"
@@ -34,20 +35,24 @@ module Prism
   # Some of these constants are not meant to be exposed, so marking them as
   # private here.
 
-  private_constant :LexCompat
-  private_constant :LexRipper
+  if RUBY_ENGINE != "jruby"
+    private_constant :LexCompat
+    private_constant :NodeFind
+  end
 
   # Raised when requested to parse as the currently running Ruby version but Prism has no support for it.
   class CurrentVersionError < ArgumentError
     # Initialize a new exception for the given ruby version string.
+    #
+    #: (String version) -> void
     def initialize(version)
       message = +"invalid version: Requested to parse as `version: 'current'`; "
-      segments =
+      major, minor, =
         if version.match?(/\A\d+\.\d+.\d+\z/)
           version.split(".").map(&:to_i)
         end
 
-      if segments && ((segments[0] < 3) || (segments[0] == 3 && segments[1] < 3))
+      if major && minor && ((major < 3) || (major == 3 && minor < 3))
         message << " #{version} is below the minimum supported syntax."
       else
         message << " #{version} is unknown. Please update the `prism` gem."
@@ -58,34 +63,63 @@ module Prism
   end
 
   # :call-seq:
-  #   Prism::lex_compat(source, **options) -> LexCompat::Result
+  #   lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
-  # resembles the return value of Ripper::lex. The main difference is that the
-  # `:on_sp` token is not emitted.
+  # resembles the return value of Ripper.lex.
+  #
+  # For supported options, see Prism.parse.
   #
-  # For supported options, see Prism::parse.
+  #: (String source, **untyped options) -> LexCompat::Result
   def self.lex_compat(source, **options)
     LexCompat.new(source, **options).result # steep:ignore
   end
 
   # :call-seq:
-  #   Prism::lex_ripper(source) -> Array
-  #
-  # This lexes with the Ripper lex. It drops any space events but otherwise
-  # returns the same tokens. Raises SyntaxError if the syntax in source is
-  # invalid.
-  def self.lex_ripper(source)
-    LexRipper.new(source).result # steep:ignore
-  end
-
-  # :call-seq:
-  #   Prism::load(source, serialized, freeze) -> ParseResult
+  #   load(source, serialized, freeze) -> ParseResult
   #
   # Load the serialized AST using the source as a reference into a tree.
+  #
+  #: (String source, String serialized, ?bool freeze) -> ParseResult
   def self.load(source, serialized, freeze = false)
     Serialize.load_parse(source, serialized, freeze)
   end
+
+  # Given a Method, UnboundMethod, Proc, or Thread::Backtrace::Location,
+  # returns the Prism node representing it. On CRuby, this uses node_id for
+  # an exact match. On other implementations, it falls back to best-effort
+  # matching by source location line number.
+  #
+  #: (Method | UnboundMethod | Proc | Thread::Backtrace::Location callable) -> Node?
+  def self.find(callable)
+    NodeFind.find(callable)
+  end
+
+  # @rbs!
+  #    VERSION: String
+  #    BACKEND: :CEXT | :FFI
+  #
+  #    interface _Stream
+  #      def gets: (?Integer integer) -> (String | nil)
+  #    end
+  #
+  #    def self.parse:               (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.profile:             (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+  #    def self.lex:                 (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+  #    def self.parse_lex:           (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+  #    def self.dump:                (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+  #    def self.parse_comments:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+  #    def self.parse_success?:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_failure?:      (String source,  ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_stream:        (_Stream stream, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.parse_file:          (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.profile_file:        (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+  #    def self.lex_file:            (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+  #    def self.parse_lex_file:      (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+  #    def self.dump_file:           (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+  #    def self.parse_file_comments: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+  #    def self.parse_file_success?: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_file_failure?: (path filepath,                     ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
 end
 
 require_relative "prism/polyfill/byteindex"
@@ -102,7 +136,30 @@ if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
   # The C extension is the default backend on CRuby.
   Prism::BACKEND = :CEXT
 
-  require "prism/prism"
+  begin
+    # The precompiled native libraries are in <gem_dir>/lib/prism/<ruby_version>
+    require_relative "prism/#{RUBY_VERSION[/\d+\.\d+/]}/prism"
+  rescue LoadError => e
+    if e.message.include?("GLIBC")
+      warn(<<~EOM)
+
+        ERROR: It looks like you're trying to use Prism as a precompiled native gem on a system
+               with an unsupported version of glibc.
+
+          #{e.message}
+
+          If that's the case, then please install Prism via the `ruby` platform gem:
+              gem install prism --platform=ruby
+          or, in your Gemfile:
+              gem "prism", force_ruby_platform: true
+
+      EOM
+      raise e
+    end
+
+    # Precompiled library isn't available, fall back to the library compiled at installation time.
+    require "prism/prism"
+  end
 else
   # The FFI backend is used on other Ruby implementations.
   Prism::BACKEND = :FFI

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.8.1"
+  spec.version = "2.0.0.rc1"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -42,31 +42,71 @@ Gem::Specification.new do |spec|
     "docs/serialization.md",
     "docs/testing.md",
     "ext/prism/api_node.c",
-    "ext/prism/api_pack.c",
+    "ext/prism/extconf.rb",
     "ext/prism/extension.c",
     "ext/prism/extension.h",
     "include/prism.h",
+    "include/prism/compiler/accel.h",
+    "include/prism/compiler/align.h",
+    "include/prism/compiler/assume.h",
+    "include/prism/compiler/exported.h",
+    "include/prism/compiler/fallthrough.h",
+    "include/prism/compiler/filesystem.h",
+    "include/prism/compiler/flex_array.h",
+    "include/prism/compiler/force_inline.h",
+    "include/prism/compiler/format.h",
+    "include/prism/compiler/inline.h",
+    "include/prism/compiler/nodiscard.h",
+    "include/prism/compiler/nonnull.h",
+    "include/prism/compiler/unused.h",
+    "include/prism/internal/allocator.h",
+    "include/prism/internal/allocator_debug.h",
+    "include/prism/internal/arena.h",
+    "include/prism/internal/bit.h",
+    "include/prism/internal/buffer.h",
+    "include/prism/internal/char.h",
+    "include/prism/internal/comments.h",
+    "include/prism/internal/constant_pool.h",
+    "include/prism/internal/diagnostic.h",
+    "include/prism/internal/encoding.h",
+    "include/prism/internal/integer.h",
+    "include/prism/internal/isinf.h",
+    "include/prism/internal/line_offset_list.h",
+    "include/prism/internal/list.h",
+    "include/prism/internal/magic_comments.h",
+    "include/prism/internal/memchr.h",
+    "include/prism/internal/node.h",
+    "include/prism/internal/options.h",
+    "include/prism/internal/parser.h",
+    "include/prism/internal/regexp.h",
+    "include/prism/internal/serialize.h",
+    "include/prism/internal/source.h",
+    "include/prism/internal/static_literals.h",
+    "include/prism/internal/strncasecmp.h",
+    "include/prism/internal/stringy.h",
+    "include/prism/internal/strpbrk.h",
+    "include/prism/internal/tokens.h",
+    "include/prism/arena.h",
     "include/prism/ast.h",
-    "include/prism/defines.h",
+    "include/prism/buffer.h",
+    "include/prism/comments.h",
+    "include/prism/constant_pool.h",
     "include/prism/diagnostic.h",
-    "include/prism/encoding.h",
+    "include/prism/errors_format.h",
+    "include/prism/excludes.h",
+    "include/prism/integer.h",
+    "include/prism/json.h",
+    "include/prism/line_offset_list.h",
+    "include/prism/magic_comments.h",
     "include/prism/node.h",
     "include/prism/options.h",
-    "include/prism/pack.h",
     "include/prism/parser.h",
     "include/prism/prettyprint.h",
-    "include/prism/regexp.h",
-    "include/prism/static_literals.h",
-    "include/prism/util/pm_buffer.h",
-    "include/prism/util/pm_char.h",
-    "include/prism/util/pm_constant_pool.h",
-    "include/prism/util/pm_integer.h",
-    "include/prism/util/pm_list.h",
-    "include/prism/util/pm_memchr.h",
-    "include/prism/util/pm_newline_list.h",
-    "include/prism/util/pm_strncasecmp.h",
-    "include/prism/util/pm_string.h",
-    "include/prism/util/pm_strpbrk.h",
+    "include/prism/serialize.h",
+    "include/prism/source.h",
+    "include/prism/stream.h",
+    "include/prism/string_query.h",
+    "include/prism/stringy.h",
     "include/prism/version.h",
     "lib/prism.rb",
     "lib/prism/compiler.rb",
@@ -77,13 +117,11 @@ Gem::Specification.new do |spec|
     "lib/prism/ffi.rb",
     "lib/prism/inspect_visitor.rb",
     "lib/prism/lex_compat.rb",
-    "lib/prism/lex_ripper.rb",
     "lib/prism/mutation_compiler.rb",
     "lib/prism/node_ext.rb",
+    "lib/prism/node_find.rb",
     "lib/prism/node.rb",
-    "lib/prism/pack.rb",
     "lib/prism/parse_result.rb",
-    "lib/prism/parse_result/comments.rb",
     "lib/prism/parse_result/errors.rb",
     "lib/prism/parse_result/newlines.rb",
     "lib/prism/pattern.rb",
@@ -104,65 +142,89 @@ Gem::Specification.new do |spec|
     "lib/prism/translation/parser/compiler.rb",
     "lib/prism/translation/parser/lexer.rb",
     "lib/prism/translation/ripper.rb",
+    "lib/prism/translation/ripper/filter.rb",
     "lib/prism/translation/ripper/lexer.rb",
     "lib/prism/translation/ripper/sexp.rb",
     "lib/prism/translation/ripper/shim.rb",
     "lib/prism/translation/ruby_parser.rb",
     "lib/prism/visitor.rb",
     "prism.gemspec",
-    "rbi/prism.rbi",
-    "rbi/prism/compiler.rbi",
-    "rbi/prism/dsl.rbi",
-    "rbi/prism/inspect_visitor.rbi",
-    "rbi/prism/node_ext.rbi",
-    "rbi/prism/node.rbi",
-    "rbi/prism/parse_result.rbi",
-    "rbi/prism/reflection.rbi",
-    "rbi/prism/string_query.rbi",
+    "rbi/generated/prism.rbi",
+    "rbi/generated/prism/compiler.rbi",
+    "rbi/generated/prism/desugar_compiler.rbi",
+    "rbi/generated/prism/dispatcher.rbi",
+    "rbi/generated/prism/dot_visitor.rbi",
+    "rbi/generated/prism/dsl.rbi",
+    "rbi/generated/prism/inspect_visitor.rbi",
+    "rbi/generated/prism/lex_compat.rbi",
+    "rbi/generated/prism/mutation_compiler.rbi",
+    "rbi/generated/prism/node.rbi",
+    "rbi/generated/prism/node_ext.rbi",
+    "rbi/generated/prism/node_find.rbi",
+    "rbi/generated/prism/parse_result.rbi",
+    "rbi/generated/prism/pattern.rbi",
+    "rbi/generated/prism/reflection.rbi",
+    "rbi/generated/prism/relocation.rbi",
+    "rbi/generated/prism/serialize.rbi",
+    "rbi/generated/prism/string_query.rbi",
+    "rbi/generated/prism/translation.rbi",
+    "rbi/generated/prism/visitor.rbi",
+    "rbi/generated/prism/parse_result/errors.rbi",
+    "rbi/generated/prism/parse_result/newlines.rbi",
     "rbi/prism/translation/parser.rbi",
     "rbi/prism/translation/parser_versions.rbi",
     "rbi/prism/translation/ripper.rbi",
-    "rbi/prism/visitor.rbi",
-    "sig/prism.rbs",
-    "sig/prism/compiler.rbs",
-    "sig/prism/dispatcher.rbs",
-    "sig/prism/dot_visitor.rbs",
-    "sig/prism/dsl.rbs",
-    "sig/prism/inspect_visitor.rbs",
-    "sig/prism/lex_compat.rbs",
-    "sig/prism/mutation_compiler.rbs",
-    "sig/prism/node_ext.rbs",
-    "sig/prism/node.rbs",
-    "sig/prism/pack.rbs",
-    "sig/prism/parse_result.rbs",
-    "sig/prism/parse_result/comments.rbs",
-    "sig/prism/pattern.rbs",
-    "sig/prism/reflection.rbs",
-    "sig/prism/relocation.rbs",
-    "sig/prism/serialize.rbs",
-    "sig/prism/string_query.rbs",
-    "sig/prism/visitor.rbs",
+    "rbi/rubyvm/node_find.rbi",
+    "sig/generated/prism.rbs",
+    "sig/generated/prism/compiler.rbs",
+    "sig/generated/prism/desugar_compiler.rbs",
+    "sig/generated/prism/dispatcher.rbs",
+    "sig/generated/prism/dot_visitor.rbs",
+    "sig/generated/prism/dsl.rbs",
+    "sig/generated/prism/inspect_visitor.rbs",
+    "sig/generated/prism/lex_compat.rbs",
+    "sig/generated/prism/mutation_compiler.rbs",
+    "sig/generated/prism/node.rbs",
+    "sig/generated/prism/node_ext.rbs",
+    "sig/generated/prism/node_find.rbs",
+    "sig/generated/prism/parse_result.rbs",
+    "sig/generated/prism/pattern.rbs",
+    "sig/generated/prism/reflection.rbs",
+    "sig/generated/prism/relocation.rbs",
+    "sig/generated/prism/serialize.rbs",
+    "sig/generated/prism/string_query.rbs",
+    "sig/generated/prism/translation.rbs",
+    "sig/generated/prism/visitor.rbs",
+    "sig/generated/prism/parse_result/errors.rbs",
+    "sig/generated/prism/parse_result/newlines.rbs",
+    "sig/manifest.yaml",
+    "src/arena.c",
+    "src/buffer.c",
+    "src/char.c",
+    "src/comments.c",
+    "src/constant_pool.c",
     "src/diagnostic.c",
     "src/encoding.c",
+    "src/errors_format.c",
+    "src/integer.c",
+    "src/json.c",
+    "src/line_offset_list.c",
+    "src/list.c",
+    "src/memchr.c",
     "src/node.c",
     "src/options.c",
-    "src/pack.c",
+    "src/parser.c",
     "src/prettyprint.c",
     "src/prism.c",
     "src/regexp.c",
     "src/serialize.c",
+    "src/source.c",
     "src/static_literals.c",
-    "src/token_type.c",
-    "src/util/pm_buffer.c",
-    "src/util/pm_char.c",
-    "src/util/pm_constant_pool.c",
-    "src/util/pm_integer.c",
-    "src/util/pm_list.c",
-    "src/util/pm_memchr.c",
-    "src/util/pm_newline_list.c",
-    "src/util/pm_string.c",
-    "src/util/pm_strncasecmp.c",
-    "src/util/pm_strpbrk.c"
+    "src/string_query.c",
+    "src/stringy.c",
+    "src/strncasecmp.c",
+    "src/strpbrk.c",
+    "src/tokens.c"
   ]
 
   spec.extensions = ["ext/prism/extconf.rb"]