ruby_parser 3.8.2 → 3.8.3

data/lib/ruby23_parser.y

@@ -19,12 +19,8 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END
       tSTRING tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA
       tLAMBEG tDSTAR tCHAR tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DEND tUBANG
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
       tRATIONAL tIMAGINARY
-#endif
-#if defined(RUBY22 || defined(RUBY23))
       tLABEL_END
-#endif
        tLONELY
 
 prechigh
@@ -695,9 +691,7 @@ rule
                     {
                       result = new_call val[0], :**, argl(val[2])
                     }
-#if   defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
                 | tUMINUS_NUM simple_numeric tPOW arg
-#endif
                     {
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
                     }
@@ -1729,12 +1723,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
        word_list: none
                     {
-                      result = s(:array)
+                      result = new_word_list
                     }
                 | word_list word tSPACE
                     {
-                      word = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1]
-                      result = val[0].dup << word
+                      result = val[0].dup << new_word_list_entry(val)
                     }
 
             word: string_content
@@ -1754,23 +1747,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
      symbol_list: none
                     {
-                      result = s(:array)
+                      result = new_symbol_list
                     }
                 | symbol_list word tSPACE
                     {
-                      list, sym, _ = val
-
-                      case sym[0]
-                      when :dstr then
-                        sym[0] = :dsym
-                      when :str then
-                        sym = s(:lit, sym.last.to_sym)
-                      else
-                        debug20 24
-                        sym = s(:dsym, "", result)
-                      end
-
-                      result = list.dup << sym
+                      result = val[0].dup << new_symbol_list_entry(val)
                     }
 
           qwords: tQWORDS_BEG tSPACE tSTRING_END
@@ -1793,20 +1774,20 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       qword_list: none
                     {
-                      result = s(:array)
+                      result = new_qword_list
                     }
                 | qword_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0].dup << s(:str, val[1])
+                      result = val[0].dup << new_qword_list_entry(val)
                     }
 
        qsym_list: none
                     {
-                      result = s(:array)
+                      result = new_qsym_list
                     }
                 | qsym_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0].dup << s(:lit, val[1].to_sym)
+                      result = val[0].dup << new_qsym_list_entry(val)
                     }
 
  string_contents: none
@@ -1854,10 +1835,10 @@ regexp_contents: none
                     }
                 | tSTRING_DBEG
                     {
-                      result = [lexer.lex_strterm, 
-                                lexer.brace_nest, 
+                      result = [lexer.lex_strterm,
+                                lexer.brace_nest,
                                 lexer.string_nest, # TODO: remove
-                                lexer.cond.store, 
+                                lexer.cond.store,
                                 lexer.cmdarg.store,
                                 lexer.lex_state,
                                ]
@@ -1938,21 +1919,17 @@ regexp_contents: none
                       end
                     }
 
-#if   defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
          numeric: simple_numeric
                 | tUMINUS_NUM simple_numeric
-#endif
                     {
                       result = -val[1] # TODO: pt_testcase
                     }
 
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
   simple_numeric: tINTEGER
                 | tFLOAT
                 | tRATIONAL
                 | tIMAGINARY
 
-#endif
    user_variable: tIDENTIFIER
                 | tIVAR
                 | tGVAR
@@ -2039,15 +2016,15 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = args val
                     }
-		| f_kwarg opt_f_block_arg
+                | f_kwarg opt_f_block_arg
                     {
                       result = args val
                     }
-		| f_kwrest opt_f_block_arg
+                | f_kwrest opt_f_block_arg
                     {
                       result = args val
                     }
-		| f_block_arg
+                | f_block_arg
 
    opt_args_tail: tCOMMA args_tail
                     {
@@ -2145,7 +2122,6 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = identifier
                     }
 
-#if defined(RUBY22) || defined(RUBY23))
       f_arg_asgn: f_norm_arg
 
       f_arg_item: f_arg_asgn
@@ -2153,13 +2129,6 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = val[1]
                     }
-#else
-      f_arg_item: f_norm_arg
-                | tLPAREN f_margs rparen
-                    {
-                      result = val[1]
-                    }
-#endif
 
            f_arg: f_arg_item
                     {

data/lib/ruby_lexer.rb

@@ -258,7 +258,18 @@ class RubyLexer
 
   def int_with_base base
     rb_compile_error "Invalid numeric format" if matched =~ /__/
-    return result(:expr_end, :tINTEGER, matched.to_i(base))
+
+    text = matched
+    case
+    when text.end_with?('ri')
+      return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop.to_i(base))))
+    when text.end_with?('r')
+      return result(:expr_end, :tRATIONAL, Rational(text.chop.to_i(base)))
+    when text.end_with?('i')
+      return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_i(base)))
+    else
+      return result(:expr_end, :tINTEGER, text.to_i(base))
+    end
   end
 
   def is_arg?
@@ -406,7 +417,17 @@ class RubyLexer
 
   def process_float text
     rb_compile_error "Invalid numeric format" if text =~ /__/
-    return result(:expr_end, :tFLOAT, text.to_f)
+
+    case
+    when text.end_with?('ri')
+      return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop)))
+    when text.end_with?('r')
+      return result(:expr_end, :tRATIONAL, Rational(text.chop))
+    when text.end_with?('i')
+      return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_f))
+    else
+      return result(:expr_end, :tFLOAT, text.to_f)
+    end
   end
 
   def process_gvar text
@@ -903,6 +924,17 @@ class RubyLexer
     ss.check re
   end
 
+  def eat_whitespace
+    r = scan(/\s+/)
+    self.extra_lineno += r.count("\n") if r
+    r
+  end
+
+  def fixup_lineno extra = 0
+    self.lineno += self.extra_lineno + extra
+    self.extra_lineno = 0
+  end
+
   def scanner_class # TODO: design this out of oedipus_lex. or something.
     RPStringScanner
   end
@@ -1043,7 +1075,8 @@ class RubyLexer
   def unescape s
     r = ESCAPES[s]
 
-    self.extra_lineno -= 1 if r && s == "n"
+    self.extra_lineno += 1 if s == "\n"     # eg backslash newline strings
+    self.extra_lineno -= 1 if r && s == "n" # literal \n, not newline
 
     return r if r
 
@@ -1131,10 +1164,10 @@ class RubyLexer
                               when 'q' then
                                 [:tSTRING_BEG,   STR_SQUOTE]
                               when 'W' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'w' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tQWORDS_BEG,   STR_SQUOTE | STR_FUNC_QWORDS]
                               when 'x' then
                                 [:tXSTRING_BEG,  STR_XQUOTE]
@@ -1144,10 +1177,10 @@ class RubyLexer
                                 self.lex_state  = :expr_fname
                                 [:tSYMBEG,       STR_SSYM]
                               when 'I' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'i' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
                               end
 
@@ -1177,7 +1210,7 @@ class RubyLexer
       return :tSTRING_END, nil
     end
 
-    space = true if qwords and scan(/\s+/)
+    space = true if qwords and eat_whitespace
 
     if self.string_nest == 0 && scan(/#{term_re}/) then
       if qwords then

data/lib/ruby_lexer.rex

@@ -12,12 +12,12 @@ macro
   SIMPLE_STRING /(#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*/o
   SSTRING       /(\\.|[^\'])*/
 
-  INT_DEC       /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0d[0-9_]+)/i
-  INT_HEX       /[+]?0x[a-f0-9_]+/i
-  INT_BIN       /[+]?0b[01_]+/i
-  INT_OCT       /[+]?0o?[0-7_]+|0o/i
-  FLOAT         /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?\b|[+]?[\d_]+e[+-]?[\d_]+\b/i
-  INT_DEC2      /[+]?\d[0-9_]*(?![e])/i
+  INT_DEC       /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)(ri|r|i)?\b|0d[0-9_]+)(ri|r|i)?/i
+  INT_HEX       /[+]?0x[a-f0-9_]+(ri|r|i)?/i
+  INT_BIN       /[+]?0b[01_]+(ri|r|i)?/i
+  INT_OCT       /[+]?0o?[0-7_]+(ri|r|i)?|0o(ri|r|i)?/i
+  FLOAT         /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?(?:(ri|r|i)\b)?|[+]?[\d_]+e[+-]?[\d_]+(?:(ri|r|i)\b)?/i
+  INT_DEC2      /[+]?\d[0-9_]*(?![e])((ri|r|i)\b)?/i
 
   NUM_BAD       /[+]?0[xbd]\b/i
   INT_OCT_BAD   /[+]?0o?[0-7_]*[89]/i

data/lib/ruby_lexer.rex.rb

@@ -15,17 +15,18 @@ class RubyLexer
   ESC           = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
   SIMPLE_STRING = /(#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*/o
   SSTRING       = /(\\.|[^\'])*/
-  INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0d[0-9_]+)/i
-  INT_HEX       = /[+]?0x[a-f0-9_]+/i
-  INT_BIN       = /[+]?0b[01_]+/i
-  INT_OCT       = /[+]?0o?[0-7_]+|0o/i
-  FLOAT         = /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?\b|[+]?[\d_]+e[+-]?[\d_]+\b/i
-  INT_DEC2      = /[+]?\d[0-9_]*(?![e])/i
+  INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)(ri|r|i)?\b|0d[0-9_]+)(ri|r|i)?/i
+  INT_HEX       = /[+]?0x[a-f0-9_]+(ri|r|i)?/i
+  INT_BIN       = /[+]?0b[01_]+(ri|r|i)?/i
+  INT_OCT       = /[+]?0o?[0-7_]+(ri|r|i)?|0o(ri|r|i)?/i
+  FLOAT         = /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?(?:(ri|r|i)\b)?|[+]?[\d_]+e[+-]?[\d_]+(?:(ri|r|i)\b)?/i
+  INT_DEC2      = /[+]?\d[0-9_]*(?![e])((ri|r|i)\b)?/i
   NUM_BAD       = /[+]?0[xbd]\b/i
   INT_OCT_BAD   = /[+]?0o?[0-7_]*[89]/i
   FLOAT_BAD     = /[+]?\d[\d_]*_(e|\.)/i
 
-  class ScanError < StandardError ; end
+  class LexerError < StandardError ; end
+  class ScanError < LexerError ; end
 
   attr_accessor :filename
   attr_accessor :ss
@@ -43,7 +44,6 @@ class RubyLexer
     yield
   end
 
-
   def scanner_class
     StringScanner
   end unless instance_methods(false).map(&:to_s).include?("scanner_class")
@@ -62,6 +62,12 @@ class RubyLexer
     end
   end
 
+  def location
+    [
+      (filename || "<input>"),
+    ].compact.join(":")
+  end
+
   def next_token
     return process_string if lex_strterm
     self.command_state = self.command_start
@@ -297,16 +303,16 @@ class RubyLexer
             action { rb_compile_error "Invalid char #{text.inspect} in expression" }
           else
             text = ss.string[ss.pos .. -1]
-            raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
           end
         else
-          raise ScanError, "undefined state: '#{state}'"
+          raise ScanError, "undefined state at #{location}: '#{state}'"
         end # token = case state
 
       next unless token # allow functions to trigger redo w/ nil
     end # while
 
-    raise "bad lexical result: #{token.inspect}" unless
+    raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless
       token.nil? || (Array === token && token.size >= 2)
 
     # auto-switch state

data/lib/ruby_parser.yy

@@ -27,10 +27,10 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END
       tSTRING tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA
       tLAMBEG tDSTAR tCHAR tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DEND tUBANG
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
       tRATIONAL tIMAGINARY
 #endif
-#if defined(RUBY22 || defined(RUBY23))
+#if defined(RUBY22) || defined(RUBY23)
       tLABEL_END
 #endif
 #if defined(RUBY23)
@@ -720,7 +720,7 @@ rule
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
                     }
                 | tUMINUS_NUM tFLOAT tPOW arg
-#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
+#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
                 | tUMINUS_NUM simple_numeric tPOW arg
 #endif
                     {
@@ -1758,12 +1758,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
        word_list: none
                     {
-                      result = s(:array)
+                      result = new_word_list
                     }
                 | word_list word tSPACE
                     {
-                      word = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1]
-                      result = val[0].dup << word
+                      result = val[0].dup << new_word_list_entry(val)
                     }
 
             word: string_content
@@ -1783,23 +1782,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
      symbol_list: none
                     {
-                      result = s(:array)
+                      result = new_symbol_list
                     }
                 | symbol_list word tSPACE
                     {
-                      list, sym, _ = val
-
-                      case sym[0]
-                      when :dstr then
-                        sym[0] = :dsym
-                      when :str then
-                        sym = s(:lit, sym.last.to_sym)
-                      else
-                        debug20 24
-                        sym = s(:dsym, "", result)
-                      end
-
-                      result = list.dup << sym
+                      result = val[0].dup << new_symbol_list_entry(val)
                     }
 
           qwords: tQWORDS_BEG tSPACE tSTRING_END
@@ -1822,20 +1809,20 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       qword_list: none
                     {
-                      result = s(:array)
+                      result = new_qword_list
                     }
                 | qword_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0].dup << s(:str, val[1])
+                      result = val[0].dup << new_qword_list_entry(val)
                     }
 
        qsym_list: none
                     {
-                      result = s(:array)
+                      result = new_qsym_list
                     }
                 | qsym_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0].dup << s(:lit, val[1].to_sym)
+                      result = val[0].dup << new_qsym_list_entry(val)
                     }
 
  string_contents: none
@@ -1883,10 +1870,10 @@ regexp_contents: none
                     }
                 | tSTRING_DBEG
                     {
-                      result = [lexer.lex_strterm, 
-                                lexer.brace_nest, 
+                      result = [lexer.lex_strterm,
+                                lexer.brace_nest,
                                 lexer.string_nest, # TODO: remove
-                                lexer.cond.store, 
+                                lexer.cond.store,
                                 lexer.cmdarg.store,
                                 lexer.lex_state,
                                ]
@@ -1973,7 +1960,7 @@ regexp_contents: none
          numeric: tINTEGER
                 | tFLOAT
                 | tUMINUS_NUM tINTEGER =tLOWEST
-#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
+#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
          numeric: simple_numeric
                 | tUMINUS_NUM simple_numeric
 #endif
@@ -1987,7 +1974,7 @@ regexp_contents: none
 #endif
                     }
 
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
   simple_numeric: tINTEGER
                 | tFLOAT
                 | tRATIONAL
@@ -2080,15 +2067,15 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = args val
                     }
-		| f_kwarg opt_f_block_arg
+                | f_kwarg opt_f_block_arg
                     {
                       result = args val
                     }
-		| f_kwrest opt_f_block_arg
+                | f_kwrest opt_f_block_arg
                     {
                       result = args val
                     }
-		| f_block_arg
+                | f_block_arg
 
    opt_args_tail: tCOMMA args_tail
                     {
@@ -2186,7 +2173,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = identifier
                     }
 
-#if defined(RUBY22) || defined(RUBY23))
+#if defined(RUBY22) || defined(RUBY23)
       f_arg_asgn: f_norm_arg
 
       f_arg_item: f_arg_asgn