prism 0.20.0 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 742f60637c4fd77f62b7713f484e87d70249e6e9dfeefc2c2ba0bfe667eed034
-  data.tar.gz: 67bd239271c1d848536389668722a0617419b19b6bb250ed39cf17b7a414da4c
+  metadata.gz: 407353660e25fcada7fe3b7e7c79d58584bf3bc1c4d9ffad8f74d178a0fbe2b7
+  data.tar.gz: a23b28dada65ba503b2d2c6cb74e9389386bb9c59a504820ea46bdeddcc16169
 SHA512:
-  metadata.gz: d0a90337f2635d35b08b0932ad6d928610406bb3f908c1b7b601f5fcb08b404604745f93bffd9a4bb84fc13cde0b6b4a71015390546a077daa4e05d7d8cf965e
-  data.tar.gz: 231693786022302c486d3c4ea2c8841636e3c94cb37f6b5e410f1ab6ac4ce7fc12e53cd4d67f43d6a38f3292867fda808e655391528619d626823a5163cbf722
+  metadata.gz: aa848ed878d8fbc2b83b04789748d7d228226a5ca0963601c3dadfa901bc5b6c528667791d220e187399a91f6b1d8c2edcaee2c65721473683c349a7c8e4cebc
+  data.tar.gz: 6c7c4671fc9f53ca898fdce09de14eb4a79bf2ace45f88efff8302f99d6ae871c0336730d143c5bd87a8a8698a558a3d008fa68e9797c4502c7515fc67cc3099

data/CHANGELOG.md

@@ -6,7 +6,40 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
-## [0.20.0] - 2024-01-01
+## [0.22.0] - 2024-02-07
+
+### Added
+
+- More support for `Prism::RipperCompat` is added.
+- Support for Ruby 2.7 has been added, and the minimum Ruby requirement has been lowered to 2.7.
+
+### Changed
+
+- The error for an invalid source encoding has a new `:argument` level to indicate it raises an argument error.
+- `BeginNode` nodes that are used when a class, singleton class, module, method definition, or block have an inline `rescue`/`ensure`/`else` now have their opening locations set to the beginning of the respective keyword.
+- Improved error messages for invalid characters.
+- `Prism.parse_file` and similar APIs will raise more appropriate errors when the file does not exist or cannot be mapped.
+- Correctly handle the `recover` parameter for `Prism::Translation::Parser`.
+
+## [0.21.0] - 2024-02-05
+
+### Added
+
+- Add the `pm_constant_pool_find` API for finding a constant.
+
+### Changed
+
+- Fixes for `Prism::Translation::Parser`.
+  - Ensure all errors flow through `parser.diagnostics.process`.
+  - Fix the find pattern node.
+  - Fix block forwarding with `NumberedParametersNode`.
+  - Ensure we can parse strings with invalid bytes for the encoding.
+  - Fix hash pairs in pattern matching.
+- Properly reject operator writes on operator calls, e.g., `a.+ -= b`.
+- Fix multi-byte escapes.
+- Handle missing body in `begin` within the receiver of a method call.
+
+## [0.20.0] - 2024-02-01
 
 ### Added
 
@@ -323,7 +356,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v0.20.0...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v0.22.0...HEAD
+[0.22.0]: https://github.com/ruby/prism/compare/v0.21.0...v0.22.0
+[0.21.0]: https://github.com/ruby/prism/compare/v0.20.0...v0.21.0
 [0.20.0]: https://github.com/ruby/prism/compare/v0.19.0...v0.20.0
 [0.19.0]: https://github.com/ruby/prism/compare/v0.18.0...v0.19.0
 [0.18.0]: https://github.com/ruby/prism/compare/v0.17.1...v0.18.0

data/docs/parser_translation.md

@@ -9,7 +9,7 @@ The `parser` gem provides multiple parsers to support different versions of the
 You can use the `prism` parser like you would any other. After requiring the parser, you should be able to call any of the regular `Parser::Base` APIs that you would normally use.
 
 ```ruby
-require "prism/translation/parser"
+require "prism"
 
 Prism::Translation::Parser.parse_file("path/to/file.rb")
 ```

data/docs/releasing.md

@@ -18,6 +18,24 @@ To release a new version of Prism, perform the following steps:
   * `templates/javascript/src/deserialize.js.erb` in the version constants
   * `templates/lib/prism/serialize.rb.erb` in the version constants
 * Run `bundle install` to update the `Gemfile.lock` file.
+
+```sh
+bundle install
+```
+
+* Run `bundle install` with the various other gemfiles:
+
+```sh
+chruby ruby-2.7.8 && BUNDLE_GEMFILE=gemfiles/2.7/Gemfile bundle install
+chruby ruby-3.0.6 && BUNDLE_GEMFILE=gemfiles/3.0/Gemfile bundle install
+chruby ruby-3.1.4 && BUNDLE_GEMFILE=gemfiles/3.1/Gemfile bundle install
+chruby ruby-3.2.3 && BUNDLE_GEMFILE=gemfiles/3.2/Gemfile bundle install
+chruby ruby-3.3.0 && BUNDLE_GEMFILE=gemfiles/3.3/Gemfile bundle install
+chruby ruby-3.4.0-dev && BUNDLE_GEMFILE=gemfiles/3.4/Gemfile bundle install
+chruby jruby-9.4.5.0 && BUNDLE_GEMFILE=gemfiles/jruby/Gemfile bundle install
+chruby truffleruby-23.1.2 && BUNDLE_GEMFILE=gemfiles/truffleruby/Gemfile bundle install
+```
+
 * Update `rust/ruby-prism-sys/Cargo.toml` to match the new version and run `cargo build`
 * Update `rust/ruby-prism/Cargo.toml` to match the new version and run `cargo build`
 * Commit all of the updated files.

data/ext/prism/extension.c

@@ -1,5 +1,9 @@
 #include "prism/extension.h"
 
+#ifdef _WIN32
+#include <ruby/win32.h>
+#endif
+
 // NOTE: this file should contain only bindings. All non-trivial logic should be
 // in libprism so it can be shared its the various callers.
 
@@ -212,20 +216,29 @@ string_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options)
 /**
  * Read options for methods that look like (filepath, **options).
  */
-static bool
+static void
 file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
     VALUE filepath;
     VALUE keywords;
     rb_scan_args(argc, argv, "1:", &filepath, &keywords);
 
+    Check_Type(filepath, T_STRING);
+
     extract_options(options, filepath, keywords);
 
-    if (!pm_string_mapped_init(input, (const char *) pm_string_source(&options->filepath))) {
+    const char * string_source = (const char *) pm_string_source(&options->filepath);
+
+    if (!pm_string_mapped_init(input, string_source)) {
         pm_options_free(options);
-        return false;
-    }
 
-    return true;
+#ifdef _WIN32
+        int e = rb_w32_map_errno(GetLastError());
+#else
+        int e = errno;
+#endif
+
+        rb_syserr_fail(e, string_source);
+    }
 }
 
 /******************************************************************************/
@@ -299,7 +312,8 @@ static VALUE
 dump_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = dump_input(&input, &options);
     pm_string_free(&input);
@@ -404,6 +418,9 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
             case PM_ERROR_LEVEL_FATAL:
                 level = ID2SYM(rb_intern("fatal"));
                 break;
+            case PM_ERROR_LEVEL_ARGUMENT:
+                level = ID2SYM(rb_intern("argument"));
+                break;
             default:
                 rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level);
         }
@@ -606,7 +623,8 @@ static VALUE
 lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_lex_input(&input, &options, false);
     pm_string_free(&input);
@@ -707,7 +725,8 @@ static VALUE
 parse_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_input(&input, &options);
     pm_string_free(&input);
@@ -767,7 +786,8 @@ static VALUE
 parse_file_comments(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_input_comments(&input, &options);
     pm_string_free(&input);
@@ -821,7 +841,8 @@ static VALUE
 parse_lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_lex_input(&input, &options, true);
     pm_string_free(&input);
@@ -878,7 +899,8 @@ static VALUE
 parse_file_success_p(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE result = parse_input_success_p(&input, &options);
     pm_string_free(&input);
@@ -956,7 +978,17 @@ profile_file(VALUE self, VALUE filepath) {
     pm_string_t input;
 
     const char *checked = check_string(filepath);
-    if (!pm_string_mapped_init(&input, checked)) return Qnil;
+    Check_Type(filepath, T_STRING);
+
+    if (!pm_string_mapped_init(&input, checked)) {
+#ifdef _WIN32
+        int e = rb_w32_map_errno(GetLastError());
+#else
+        int e = errno;
+#endif
+
+        rb_syserr_fail(e, checked);
+    }
 
     pm_options_t options = { 0 };
     pm_options_filepath_set(&options, checked);

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "0.20.0"
+#define EXPECTED_PRISM_VERSION "0.22.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>

data/include/prism/ast.h

@@ -1042,7 +1042,7 @@ static const pm_node_flags_t PM_NODE_FLAG_COMMON_MASK = (1 << (PM_NODE_FLAG_BITS
  * Cast the type to an enum to allow the compiler to provide exhaustiveness
  * checking.
  */
-#define PM_NODE_TYPE(node) ((enum pm_node_type) node->type)
+#define PM_NODE_TYPE(node) ((enum pm_node_type) (node)->type)
 
 /**
  * Return true if the type of the given node matches the given type.

data/include/prism/diagnostic.h

@@ -19,7 +19,10 @@
  */
 typedef enum {
     /** For errors that cannot be recovered from. */
-    PM_ERROR_LEVEL_FATAL = 0
+    PM_ERROR_LEVEL_FATAL = 0,
+
+    /** For errors that should raise an argument error. */
+    PM_ERROR_LEVEL_ARGUMENT = 1
 } pm_error_level_t;
 
 /**
@@ -28,6 +31,7 @@ typedef enum {
 typedef enum {
     /** For warnings which should be emitted if $VERBOSE != nil. */
     PM_WARNING_LEVEL_DEFAULT = 0,
+
     /** For warnings which should be emitted if $VERBOSE == true. */
     PM_WARNING_LEVEL_VERBOSE = 1
 } pm_warning_level_t;
@@ -192,8 +196,10 @@ typedef enum {
     PM_ERR_INVALID_NUMBER_HEXADECIMAL,
     PM_ERR_INVALID_NUMBER_OCTAL,
     PM_ERR_INVALID_NUMBER_UNDERSCORE,
+    PM_ERR_INVALID_CHARACTER,
+    PM_ERR_INVALID_MULTIBYTE_CHARACTER,
+    PM_ERR_INVALID_PRINTABLE_CHARACTER,
     PM_ERR_INVALID_PERCENT,
-    PM_ERR_INVALID_TOKEN,
     PM_ERR_INVALID_VARIABLE_GLOBAL,
     PM_ERR_IT_NOT_ALLOWED,
     PM_ERR_LAMBDA_OPEN,

data/include/prism/parser.h

@@ -626,7 +626,7 @@ struct pm_parser {
      * This is the path of the file being parsed. We use the filepath when
      * constructing SourceFileNodes.
      */
-    pm_string_t filepath_string;
+    pm_string_t filepath;
 
     /**
      * This constant pool keeps all of the constants defined throughout the file

data/include/prism/util/pm_constant_pool.h

@@ -154,6 +154,17 @@ bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
  */
 pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
 
+/**
+ * Find a constant in a constant pool. Returns the id of the constant, or 0 if
+ * the constant is not found.
+ *
+ * @param pool The pool to find the constant in.
+ * @param start A pointer to the start of the constant.
+ * @param length The length of the constant.
+ * @return The id of the constant.
+ */
+pm_constant_id_t pm_constant_pool_find(pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
 /**
  * Insert a constant into a constant pool that is a slice of a source string.
  * Returns the id of the constant, or 0 if any potential calls to resize fail.

data/include/prism/version.h

@@ -14,7 +14,7 @@
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 20
+#define PRISM_VERSION_MINOR 22
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "0.20.0"
+#define PRISM_VERSION "0.22.0"
 
 #endif

data/lib/prism/ffi.rb

@@ -160,8 +160,13 @@ module Prism
         pointer = FFI::MemoryPointer.new(SIZEOF)
 
         begin
-          raise unless LibRubyParser.pm_string_mapped_init(pointer, filepath)
-          yield new(pointer)
+          raise TypeError unless filepath.is_a?(String)
+
+          if LibRubyParser.pm_string_mapped_init(pointer, filepath)
+            yield new(pointer)
+          else
+            raise SystemCallError.new(filepath, FFI.errno)
+          end
         ensure
           LibRubyParser.pm_string_free(pointer)
           pointer.free

data/lib/prism/lex_compat.rb

@@ -860,7 +860,7 @@ module Prism
       previous = []
       results = []
 
-      Ripper.lex(source, raise_errors: true).each do |token|
+      lex(source).each do |token|
         case token[1]
         when :on_sp
           # skip
@@ -886,6 +886,21 @@ module Prism
 
       results
     end
+
+    private
+
+    if Ripper.method(:lex).parameters.assoc(:keyrest)
+      def lex(source)
+        Ripper.lex(source, raise_errors: true)
+      end
+    else
+      def lex(source)
+        ripper = Ripper::Lexer.new(source)
+        ripper.lex.tap do |result|
+          raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
+        end
+      end
+    end
   end
 
   private_constant :LexRipper