ripper_ruby_parser 1.8.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 799011b06831426c0ebecad641082d58bfaa8da0aea9359a3a2d152ea01bd196
-  data.tar.gz: eec3804031490d2382f59ddc450af14f73b696b9e14f0d2919b64a3df3557853
+  metadata.gz: 4e4ee66b4ce6abf5b0eeacd55fd137dd7217bf728bc59f1ad6bdf0b197011f86
+  data.tar.gz: a35a48bfa5178c5a9cec64017842246a4acb8b10a0c09fba2749de9aa650772c
 SHA512:
-  metadata.gz: eab6dc676f7c5806bcd09f9adf87e5b67dd5320e043c07370014fdfc8e5ca1dffbaa9583754ac74bc354a30c6b27392c3310a7172b20699d475d9122e9dae748
-  data.tar.gz: b776c9d833e7808c7b2ac17ec6bb5a358a93cb1bb925ee94199aeddf85c0cca2e102cdcdfff17ca8d17ac406e0f76c4b981b640085fbd02d2868bca36fdd840d
+  metadata.gz: 0574addabb87b0f843c2a72a3e5c03b6ffff2461a9f1fa551412ee706841a2baeb354c1decd3ca7c902e1e78c2c2b7c4a346784ebb6c5b1064211b4ca08b8039
+  data.tar.gz: cffd4a9f013b443e82b3b4d6149719b02119f182aafe86fc2984d81b131bc047bd1143ca5b06cd28dc32e71876cbb67452c9db47979a50d766e28ea7b7f590a2

data/CHANGELOG.md

@@ -6,6 +6,15 @@ This project adheres to [Semantic Versioning 2.0.0][1].
 
 This document is formatted based on [Keep A CHANGELOG][2].
 
+## 1.9.0 / 2021-08-10
+
+* Fix escape sequence handling in non-interpolating strings and word lists
+  ([#152])
+* Handle strings with mixed valid and invalid utf8 ([#153])
+* Handle argument forwarding, updating the RubyParser compatibility target to
+  3.17.0 ([#154])
+* Fix encoding for non-interpolating strings as well ([#155])
+
 ## 1.8.0 / 2021-05-18
 
 ### Changes
@@ -217,6 +226,10 @@ This document is formatted based on [Keep A CHANGELOG][2].
 * Initial release
 
 <!-- Pull request links -->
+[#155]: https://github.com/mvz/ripper_ruby_parser/pull/155
+[#154]: https://github.com/mvz/ripper_ruby_parser/pull/154
+[#153]: https://github.com/mvz/ripper_ruby_parser/pull/153
+[#152]: https://github.com/mvz/ripper_ruby_parser/pull/152
 [#145]: https://github.com/mvz/ripper_ruby_parser/pull/145
 [#123]: https://github.com/mvz/ripper_ruby_parser/pull/123
 [#106]: https://github.com/mvz/ripper_ruby_parser/pull/106

data/README.md

@@ -11,7 +11,7 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 * Drop-in replacement for RubyParser
 * Should handle 1.9 and later syntax gracefully
 * Requires Ruby 2.5 or higher
-* Compatible with RubyParser 3.15.0
+* Compatible with RubyParser 3.17.0
 
 ## Known incompatibilities
 

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -118,8 +118,8 @@ module RipperRubyParser
       end
 
       OPERATOR_ASSIGNMENT_MAP = {
-        '||': :op_asgn_or,
-        '&&': :op_asgn_and
+        "||": :op_asgn_or,
+        "&&": :op_asgn_and
       }.freeze
 
       def create_operator_assignment_sub_type(lvalue, value, operator)

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -7,7 +7,7 @@ module RipperRubyParser
       # character literals
       def process_at_CHAR(exp)
         _, val, pos = exp.shift 3
-        with_position(pos, s(:str, unescape(val[1..-1])))
+        with_position(pos, s(:str, fix_encoding(unescape(val[1..-1]))))
       end
 
       def process_array(exp)

data/lib/ripper_ruby_parser/sexp_handlers/method_calls.rb

@@ -26,7 +26,11 @@ module RipperRubyParser
         _, call, parens = exp.shift 3
         call = process(call)
         parens = process(parens)
-        call.push(*parens.sexp_body)
+        if parens.sexp_type == :forward_args
+          call.push(parens)
+        else
+          call.push(*parens.sexp_body)
+        end
       end
 
       # Handle implied hashes, such as at the end of argument lists.
@@ -36,9 +40,9 @@ module RipperRubyParser
       end
 
       CALL_OP_MAP = {
-        '.': :call,
-        '::': :call,
-        '&.': :safe_call
+        ".": :call,
+        "::": :call,
+        "&.": :safe_call
       }.freeze
 
       def process_call(exp)

data/lib/ripper_ruby_parser/sexp_handlers/methods.rb

@@ -72,6 +72,11 @@ module RipperRubyParser
         s(:alias, process(left), process(right))
       end
 
+      def process_args_forward(exp)
+        _ = exp.shift
+        s(:forward_args)
+      end
+
       private
 
       def in_method

data/lib/ripper_ruby_parser/sexp_handlers/operators.rb

@@ -5,8 +5,8 @@ module RipperRubyParser
     # Sexp handlers for operators
     module Operators
       BINARY_OPERATOR_MAP = {
-        '&&': :and,
-        '||': :or,
+        "&&": :and,
+        "||": :or,
         and: :and,
         or: :or
       }.freeze
@@ -16,7 +16,7 @@ module RipperRubyParser
       }.freeze
 
       NEGATED_BINARY_OPERATOR_MAP = {
-        '!~': :=~
+        "!~": :=~
       }.freeze
 
       SHIFT_OPERATORS = [:<<, :>>].freeze

data/lib/ripper_ruby_parser/sexp_handlers/string_literals.rb

@@ -123,8 +123,7 @@ module RipperRubyParser
 
       def process_at_tstring_content(exp)
         _, content, pos, delim = exp.shift 4
-        string = handle_string_unescaping(content, delim)
-        string = handle_string_encoding(string, delim)
+        string = fix_encoding handle_string_unescaping(content, delim)
         with_position(pos, s(:str, string))
       end
 
@@ -243,24 +242,15 @@ module RipperRubyParser
         when INTERPOLATING_WORD_LIST
           unescape_wordlist_word(content)
         when *NON_INTERPOLATING_STRINGS
-          simple_unescape(content)
+          simple_unescape(content, delim)
         when *REGEXP_LITERALS
           unescape_regexp(content)
         when NON_INTERPOLATING_WORD_LIST
-          simple_unescape_wordlist_word(content)
+          simple_unescape_wordlist_word(content, delim)
         else
           content
         end
       end
-
-      def handle_string_encoding(string, delim)
-        case delim
-        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
-          fix_encoding string
-        else
-          string
-        end
-      end
     end
   end
 end

data/lib/ripper_ruby_parser/unescape.rb

@@ -37,25 +37,34 @@ module RipperRubyParser
     SINGLE_LETTER_ESCAPES_REGEXP =
       Regexp.new("^[#{SINGLE_LETTER_ESCAPES.keys.join}]$")
 
-    def simple_unescape(string)
+    DELIMITER_PAIRS = {
+      "(" => "()",
+      "<" => "<>",
+      "[" => "[]",
+      "{" => "{}"
+    }.freeze
+
+    def simple_unescape(string, delimiter)
+      delimiters = delimiter_regexp_pattern(delimiter)
       string.gsub(/
                   \\ # a backslash
                   (  # followed by a
-                   '   | # single quote or
-                   \\    # backslash
+                    #{delimiters} | # delimiter or
+                    \\              # backslash
                   )/x) do
                     Regexp.last_match[1]
                   end
     end
 
-    def simple_unescape_wordlist_word(string)
+    def simple_unescape_wordlist_word(string, delimiter)
+      delimiters = delimiter_regexp_pattern(delimiter)
       string.gsub(/
                   \\ # a backslash
                   (  # followed by a
-                    '   | # single quote or
-                    \\  | # backslash or
-                    [ ] | # space or
-                    \n    # newline
+                    #{delimiters} | # delimiter or
+                    \\            | # backslash or
+                    [ ]           | # space or
+                    \n              # newline
                   )
                   /x) do
                     Regexp.last_match[1]
@@ -63,12 +72,14 @@ module RipperRubyParser
     end
 
     def unescape(string)
+      string = string.dup if string.frozen?
+      string.force_encoding("ASCII-8BIT")
       string.gsub(ESCAPE_SEQUENCE_REGEXP) do
         bare = Regexp.last_match[1]
         if bare == "\n"
           ""
         else
-          unescaped_value(bare)
+          unescaped_value(bare).force_encoding("ASCII-8BIT")
         end
       end
     end
@@ -105,7 +116,7 @@ module RipperRubyParser
     def unescaped_value(bare)
       case bare
       when SINGLE_LETTER_ESCAPES_REGEXP
-        SINGLE_LETTER_ESCAPES[bare]
+        SINGLE_LETTER_ESCAPES[bare].dup
       when /^x/
         unescape_hex_char bare
       when /^u/
@@ -164,5 +175,11 @@ module RipperRubyParser
     def meta(val)
       val | 0b1000_0000
     end
+
+    def delimiter_regexp_pattern(delimiter)
+      delimiter = delimiter[-1]
+      delimiters = DELIMITER_PAIRS.fetch(delimiter, delimiter)
+      delimiters.each_char.map { |it| Regexp.escape it }.join(" | ")
+    end
   end
 end

data/lib/ripper_ruby_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RipperRubyParser
-  VERSION = "1.8.0"
+  VERSION = "1.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ripper_ruby_parser
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.0
 platform: ruby
 authors:
 - Matijs van Zuijlen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-18 00:00:00.000000000 Z
+date: 2021-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.6'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.18.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.18.0
 - !ruby/object:Gem::Dependency
   name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.1
+        version: 0.15.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.1
+        version: 0.15.0
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.16.0
+        version: 3.17.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.16.0
+        version: 3.17.0
 - !ruby/object:Gem::Dependency
   name: sexp_processor
   requirement: !ruby/object:Gem::Requirement
@@ -224,7 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Parse with Ripper, produce sexps that are compatible with RubyParser.