prism 1.8.2 → 1.9.0

data/sig/prism.rbs

@@ -14,119 +14,128 @@ module Prism
 
   def self.parse: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> ParseResult
 
   def self.profile: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> nil
 
   def self.lex: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> LexResult
 
   def self.lex_compat: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> LexCompat::Result
 
   def self.parse_lex: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> ParseLexResult
 
   def self.dump: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> String
 
   def self.parse_comments: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> Array[comment]
 
   def self.parse_success?: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> bool
 
   def self.parse_failure?: (
     String source,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> bool
 
   def self.load: (
@@ -135,106 +144,110 @@ module Prism
     ?bool freeze
   ) -> ParseResult
 
-  def self.lex_ripper: (
-    String source
-  ) -> Array[[[Integer, Integer], Symbol, String, untyped]]
-
   # Methods taking a path to a Ruby file:
 
   def self.parse_file: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> ParseResult
 
   def self.profile_file: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> nil
 
   def self.lex_file: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> LexResult
 
   def self.parse_lex_file: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> ParseLexResult
 
   def self.dump_file: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> String
 
   def self.parse_file_comments: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> Array[comment]
 
   def self.parse_file_success?: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> bool
 
   def self.parse_file_failure?: (
     String filepath,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> bool
 
   interface _Stream
@@ -243,15 +256,16 @@ module Prism
 
   def self.parse_stream: (
     _Stream stream,
+    ?command_line: String,
     ?encoding: Encoding | false,
     ?filepath: String,
     ?freeze: bool,
     ?frozen_string_literal: bool,
     ?line: Integer,
     ?main_script: bool,
-    ?offset: Integer,
+    ?partial_script: bool,
     ?scopes: Array[Array[Symbol]],
-    ?verbose: bool
+    ?version: String
   ) -> ParseResult
 
   def self.scope: (?locals: Array[Symbol], ?forwarding: Array[Symbol]) -> Scope

data/src/diagnostic.c

@@ -10,7 +10,7 @@
 
 #include "prism/diagnostic.h"
 
-#define PM_DIAGNOSTIC_ID_MAX 326
+#define PM_DIAGNOSTIC_ID_MAX 324
 
 /** This struct holds the data for each diagnostic. */
 typedef struct {
@@ -112,8 +112,6 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = {
     [PM_ERR_ARGUMENT_FORWARDING_UNBOUND]        = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND]   = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES]    = { "unexpected ... when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX },
-    [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_LAMBDA] = { "unexpected ... in lambda argument", PM_ERROR_LEVEL_SYNTAX },
-    [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_BLOCK]  = { "unexpected ... in block argument", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_ARGUMENT_NO_FORWARDING_STAR]        = { "unexpected `*`; no anonymous rest parameter", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR]   = { "unexpected `**`; no anonymous keyword rest parameter", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT]   = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_SYNTAX },
@@ -444,8 +442,6 @@ pm_diagnostic_id_human(pm_diagnostic_id_t diag_id) {
         case PM_ERR_ARGUMENT_FORWARDING_UNBOUND: return "argument_forwarding_unbound";
         case PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND: return "argument_no_forwarding_ampersand";
         case PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES: return "argument_no_forwarding_ellipses";
-        case PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_LAMBDA: return "argument_no_forwarding_ellipses_lambda";
-        case PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_BLOCK: return "argument_no_forwarding_ellipses_block";
         case PM_ERR_ARGUMENT_NO_FORWARDING_STAR: return "argument_no_forwarding_star";
         case PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR: return "argument_no_forwarding_star_star";
         case PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT: return "argument_splat_after_assoc_splat";
@@ -761,7 +757,6 @@ pm_diagnostic_id_human(pm_diagnostic_id_t diag_id) {
 static inline const char *
 pm_diagnostic_message(pm_diagnostic_id_t diag_id) {
     assert(diag_id < PM_DIAGNOSTIC_ID_MAX);
-    PRISM_ASSUME(diag_id < PM_DIAGNOSTIC_ID_MAX);
 
     const char *message = diagnostic_messages[diag_id].message;
     assert(message);
@@ -772,7 +767,6 @@ pm_diagnostic_message(pm_diagnostic_id_t diag_id) {
 static inline uint8_t
 pm_diagnostic_level(pm_diagnostic_id_t diag_id) {
     assert(diag_id < PM_DIAGNOSTIC_ID_MAX);
-    PRISM_ASSUME(diag_id < PM_DIAGNOSTIC_ID_MAX);
 
     return (uint8_t) diagnostic_messages[diag_id].level;
 }

data/src/prism.c

@@ -9979,21 +9979,8 @@ parser_lex(pm_parser_t *parser) {
                                 following && (
                                     (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '&') ||
                                     (peek_at(parser, following) == '|' && peek_at(parser, following + 1) == '|') ||
-                                    (
-                                        peek_at(parser, following) == 'a' &&
-                                        peek_at(parser, following + 1) == 'n' &&
-                                        peek_at(parser, following + 2) == 'd' &&
-                                        peek_at(parser, next_content + 3) != '!' &&
-                                        peek_at(parser, next_content + 3) != '?' &&
-                                        !char_is_identifier(parser, following + 3, parser->end - (following + 3))
-                                    ) ||
-                                    (
-                                        peek_at(parser, following) == 'o' &&
-                                        peek_at(parser, following + 1) == 'r' &&
-                                        peek_at(parser, next_content + 2) != '!' &&
-                                        peek_at(parser, next_content + 2) != '?' &&
-                                        !char_is_identifier(parser, following + 2, parser->end - (following + 2))
-                                    )
+                                    (peek_at(parser, following) == 'a' && peek_at(parser, following + 1) == 'n' && peek_at(parser, following + 2) == 'd' && !char_is_identifier(parser, following + 3, parser->end - (following + 3))) ||
+                                    (peek_at(parser, following) == 'o' && peek_at(parser, following + 1) == 'r' && !char_is_identifier(parser, following + 2, parser->end - (following + 2)))
                                 )
                             ) {
                                 if (!lexed_comment) parser_lex_ignored_newline(parser);
@@ -10064,8 +10051,6 @@ parser_lex(pm_parser_t *parser) {
                                 peek_at(parser, next_content) == 'a' &&
                                 peek_at(parser, next_content + 1) == 'n' &&
                                 peek_at(parser, next_content + 2) == 'd' &&
-                                peek_at(parser, next_content + 3) != '!' &&
-                                peek_at(parser, next_content + 3) != '?' &&
                                 !char_is_identifier(parser, next_content + 3, parser->end - (next_content + 3))
                             ) {
                                 if (!lexed_comment) parser_lex_ignored_newline(parser);
@@ -10082,8 +10067,6 @@ parser_lex(pm_parser_t *parser) {
                             if (
                                 peek_at(parser, next_content) == 'o' &&
                                 peek_at(parser, next_content + 1) == 'r' &&
-                                peek_at(parser, next_content + 2) != '!' &&
-                                peek_at(parser, next_content + 2) != '?' &&
                                 !char_is_identifier(parser, next_content + 2, parser->end - (next_content + 2))
                             ) {
                                 if (!lexed_comment) parser_lex_ignored_newline(parser);
@@ -10774,10 +10757,7 @@ parser_lex(pm_parser_t *parser) {
                     }
 
                     if (lex_state_spcarg_p(parser, space_seen)) {
-                        // https://bugs.ruby-lang.org/issues/21994
-                        if (parser->version <= PM_OPTIONS_VERSION_CRUBY_4_0) {
-                            pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_SLASH);
-                        }
+                        pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_SLASH);
                         lex_mode_push_regexp(parser, '\0', '/');
                         LEX(PM_TOKEN_REGEXP_BEGIN);
                     }
@@ -12458,7 +12438,7 @@ expect1_opening(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t di
 
     pm_parser_err(parser, opening->start, opening->end, diag_id);
 
-    parser->previous.start = opening->end;
+    parser->previous.start = parser->previous.end;
     parser->previous.type = PM_TOKEN_MISSING;
 }
 
@@ -13879,7 +13859,6 @@ parse_parameters(
     bool allows_forwarding_parameters,
     bool accepts_blocks_in_defaults,
     bool in_block,
-    pm_diagnostic_id_t diag_id_forwarding,
     uint16_t depth
 ) {
     pm_do_loop_stack_push(parser, false);
@@ -13935,7 +13914,7 @@ parse_parameters(
             }
             case PM_TOKEN_UDOT_DOT_DOT: {
                 if (!allows_forwarding_parameters) {
-                    pm_parser_err_current(parser, diag_id_forwarding);
+                    pm_parser_err_current(parser, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES);
                 }
 
                 bool succeeded = update_parameter_state(parser, &parser->current, &order);
@@ -14615,7 +14594,6 @@ parse_block_parameters(
             false,
             accepts_blocks_in_defaults,
             true,
-            is_lambda_literal ? PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_LAMBDA : PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_BLOCK,
             (uint16_t) (depth + 1)
         );
         if (!is_lambda_literal) {
@@ -18858,17 +18836,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
                     if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
                         params = NULL;
                     } else {
-                        params = parse_parameters(
-                            parser,
-                            PM_BINDING_POWER_DEFINED,
-                            true,
-                            false,
-                            true,
-                            true,
-                            false,
-                            PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
-                            (uint16_t) (depth + 1)
-                        );
+                        params = parse_parameters(parser, PM_BINDING_POWER_DEFINED, true, false, true, true, false, (uint16_t) (depth + 1));
                     }
 
                     lex_state_set(parser, PM_LEX_STATE_BEG);
@@ -18893,17 +18861,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
                     lparen = not_provided(parser);
                     rparen = not_provided(parser);
-                    params = parse_parameters(
-                        parser,
-                        PM_BINDING_POWER_DEFINED,
-                        false,
-                        false,
-                        true,
-                        true,
-                        false,
-                        PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
-                        (uint16_t) (depth + 1)
-                    );
+                    params = parse_parameters(parser, PM_BINDING_POWER_DEFINED, false, false, true, true, false, (uint16_t) (depth + 1));
 
                     // Reject `def * = 1` and similar. We have to specifically check
                     // for them because they create ambiguity with optional arguments.
@@ -18959,7 +18917,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
                     context_push(parser, PM_CONTEXT_RESCUE_MODIFIER);
 
                     pm_token_t rescue_keyword = parser->previous;
-                    pm_node_t *value = parse_expression(parser, PM_BINDING_POWER_MATCH + 1, false, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
+                    pm_node_t *value = parse_expression(parser, pm_binding_powers[PM_TOKEN_KEYWORD_RESCUE_MODIFIER].right, false, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
                     context_pop(parser);
 
                     statement = UP(pm_rescue_modifier_node_create(parser, statement, &rescue_keyword, value));
@@ -20256,7 +20214,7 @@ parse_assignment_value(pm_parser_t *parser, pm_binding_power_t previous_binding_
         pm_token_t rescue = parser->current;
         parser_lex(parser);
 
-        pm_node_t *right = parse_expression(parser, PM_BINDING_POWER_MATCH + 1, false, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
+        pm_node_t *right = parse_expression(parser, pm_binding_powers[PM_TOKEN_KEYWORD_RESCUE_MODIFIER].right, false, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
         context_pop(parser);
 
         return UP(pm_rescue_modifier_node_create(parser, value, &rescue, right));
@@ -20362,7 +20320,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding
             }
         }
 
-        pm_node_t *right = parse_expression(parser, PM_BINDING_POWER_MATCH + 1, accepts_command_call_inner, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
+        pm_node_t *right = parse_expression(parser, pm_binding_powers[PM_TOKEN_KEYWORD_RESCUE_MODIFIER].right, accepts_command_call_inner, false, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
         context_pop(parser);
 
         return UP(pm_rescue_modifier_node_create(parser, value, &rescue, right));
@@ -21681,6 +21639,12 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc
      ) {
         node = parse_expression_infix(parser, node, binding_power, current_binding_powers.right, accepts_command_call, (uint16_t) (depth + 1));
 
+        if (context_terminator(parser->current_context->context, &parser->current)) {
+            // If this token terminates the current context, then we need to
+            // stop parsing the expression, as it has become a statement.
+            return node;
+        }
+
         switch (PM_NODE_TYPE(node)) {
             case PM_MULTI_WRITE_NODE:
                 // Multi-write nodes are statements, and cannot be followed by
@@ -21793,17 +21757,6 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc
                     break;
             }
         }
-
-        if (context_terminator(parser->current_context->context, &parser->current)) {
-            pm_binding_powers_t next_binding_powers = pm_binding_powers[parser->current.type];
-            if (
-                !next_binding_powers.binary ||
-                binding_power > next_binding_powers.left ||
-                (PM_NODE_TYPE_P(node, PM_CALL_NODE) && pm_call_node_command_p((pm_call_node_t *) node))
-            ) {
-                return node;
-            }
-        }
     }
 
     return node;

data/src/util/pm_memchr.c

@@ -7,7 +7,7 @@
  * we need to search for a character in a buffer that could be the trailing byte
  * of a multibyte character.
  */
-const void *
+void *
 pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, const pm_encoding_t *encoding) {
     if (encoding_changed && encoding->multibyte && character >= PRISM_MEMCHR_TRAILING_BYTE_MINIMUM) {
         const uint8_t *source = (const uint8_t *) memory;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 1.9.0
 platform: ruby
 authors:
 - Shopify
@@ -80,7 +80,6 @@ files:
 - lib/prism/ffi.rb
 - lib/prism/inspect_visitor.rb
 - lib/prism/lex_compat.rb
-- lib/prism/lex_ripper.rb
 - lib/prism/mutation_compiler.rb
 - lib/prism/node.rb
 - lib/prism/node_ext.rb
@@ -107,6 +106,7 @@ files:
 - lib/prism/translation/parser_current.rb
 - lib/prism/translation/parser_versions.rb
 - lib/prism/translation/ripper.rb
+- lib/prism/translation/ripper/filter.rb
 - lib/prism/translation/ripper/lexer.rb
 - lib/prism/translation/ripper/sexp.rb
 - lib/prism/translation/ripper/shim.rb

data/lib/prism/lex_ripper.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-# :markup: markdown
-
-require "ripper"
-
-module Prism
-  # This is a class that wraps the Ripper lexer to produce almost exactly the
-  # same tokens.
-  class LexRipper # :nodoc:
-    attr_reader :source
-
-    def initialize(source)
-      @source = source
-    end
-
-    def result
-      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
-      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
-
-      lex(source).each do |token|
-        case token[1]
-        when :on_sp
-          # skip
-        when :on_tstring_content
-          if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
-        when :on_words_sep
-          if previous[1] == :on_words_sep
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
-        else
-          results << token
-          previous = token
-        end
-      end
-
-      results
-    end
-
-    private
-
-    if Ripper.method(:lex).parameters.assoc(:keyrest)
-      def lex(source)
-        Ripper.lex(source, raise_errors: true)
-      end
-    else
-      def lex(source)
-        ripper = Ripper::Lexer.new(source)
-        ripper.lex.tap do |result|
-          raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
-        end
-      end
-    end
-  end
-
-  private_constant :LexRipper
-end