ruby_parser 3.13.0 → 3.15.0

data/lib/ruby_lexer.rex

@@ -6,9 +6,9 @@ class RubyLexer
 
 macro
 
-  IDENT         /^#{IDENT_CHAR}+/o
+  IDENT_CHAR    /[a-zA-Z0-9_[:^ascii:]]/
 
-  ESC           /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
+  ESC           /\\((?>[0-7]{1,3}|x\h{1,2}|M-[^\\]|(C-|c)[^\\]|u\h{1,4}|u\{\h+(?:\s+\h+)*\}|[^0-7xMCc]))/
   SIMPLE_STRING /((#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*)/o
   SSTRING       /((\\.|[^\'])*)/
 
@@ -44,18 +44,17 @@ rule
                 /[\]\)\}]/              process_brace_close
 
 : /\!/
-| in_arg_state? /\!\@/                  { result :expr_arg, :tUBANG, "!@" }
+| is_after_operator? /\!\@/             { result EXPR_ARG, :tUBANG, "!@" }
 |               /\![=~]?/               { result :arg_state, TOKENS[text], text }
 
 : /\./
-|               /\.\.\.?/               { result :expr_beg, TOKENS[text], text }
+|               /\.\.\.?/               { result EXPR_BEG, TOKENS[text], text }
 |               /\.\d/                  { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
-|               /\./                    { self.lex_state = :expr_beg; result :expr_dot, :tDOT, "." }
+|               /\./                    { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
 
                 /\(/                    process_paren
 
-# TODO: :expr_beg|:expr_label
-                /\,/                    { result :expr_beg, TOKENS[text], text }
+                /\,/                    { result EXPR_PAR, TOKENS[text], text }
 
 : /=/
 |               /\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/ { result arg_state, TOKENS[text], text }
@@ -63,7 +62,7 @@ rule
 |               /\=(?=begin\b)/         { result arg_state, TOKENS[text], text }
 
 ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
-                /\"(#{SIMPLE_STRING})\"/o { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
+                /\"(#{SIMPLE_STRING})\"/o process_simple_string
                 /\"/                    { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
 
                 /\@\@?\d/               { rb_compile_error "`#{text}` is not allowed as a variable name" }
@@ -76,7 +75,7 @@ ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
 |               /\:\:/                      process_colon2
 |               /\:/                        process_colon1
 
-                /->/                    { result :expr_endfn, :tLAMBDA, nil }
+                /->/                    { result EXPR_ENDFN, :tLAMBDA, nil }
 
                 /[+-]/                  process_plus_minus
 
@@ -97,57 +96,57 @@ ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
 was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 
 : /\|/
-|               /\|\|\=/                { result :expr_beg, :tOP_ASGN, "||" }
-|               /\|\|/                  { result :expr_beg, :tOROP,    "||" }
-|               /\|\=/                  { result :expr_beg, :tOP_ASGN, "|" }
-|               /\|/                    { result :arg_state, :tPIPE,    "|" }
+|               /\|\|\=/                { result EXPR_BEG, :tOP_ASGN, "||" }
+|               /\|\|/                  { result EXPR_BEG, :tOROP,    "||" }
+|               /\|\=/                  { result EXPR_BEG, :tOP_ASGN, "|" }
+|               /\|/                    { state = is_after_operator? ? EXPR_ARG : EXPR_PAR; result state, :tPIPE, "|" }
 
                 /\{/                    process_brace_open
 
 : /\*/
-|               /\*\*=/                 { result :expr_beg, :tOP_ASGN, "**" }
+|               /\*\*=/                 { result EXPR_BEG, :tOP_ASGN, "**" }
 |               /\*\*/                  { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
-|               /\*\=/                  { result(:expr_beg, :tOP_ASGN, "*") }
+|               /\*\=/                  { result(EXPR_BEG, :tOP_ASGN, "*") }
 |               /\*/                    { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
 
 # TODO: fix result+process_lchevron to set command_start = true
 : /</
 |               /\<\=\>/                { result :arg_state, :tCMP, "<=>"    }
 |               /\<\=/                  { result :arg_state, :tLEQ, "<="     }
-|               /\<\<\=/                { result :expr_beg,  :tOP_ASGN, "<<" }
+|               /\<\<\=/                { result EXPR_BEG,  :tOP_ASGN, "<<" }
 |               /\<\</                  process_lchevron
 |               /\</                    { result :arg_state, :tLT, "<"       }
 
 : />/
 |               /\>\=/                  { result :arg_state, :tGEQ, ">="     }
-|               /\>\>=/                 { result :expr_beg,  :tOP_ASGN, ">>" }
+|               /\>\>=/                 { result EXPR_BEG,  :tOP_ASGN, ">>" }
 |               /\>\>/                  { result :arg_state, :tRSHFT, ">>"   }
 |               /\>/                    { result :arg_state, :tGT, ">"       }
 
 : /\`/
-| expr_fname?   /\`/                   { result(:expr_end, :tBACK_REF2, "`") }
-| expr_dot?     /\`/                   { result((cmd_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
+| expr_fname?   /\`/                   { result(EXPR_END, :tBACK_REF2, "`") }
+| expr_dot?     /\`/                   { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
 |               /\`/                   { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
 
                 /\?/                    process_questionmark
 
 : /&/
-|               /\&\&\=/                { result(:expr_beg, :tOP_ASGN, "&&") }
-|               /\&\&/                  { result(:expr_beg, :tANDOP,   "&&") }
-|               /\&\=/                  { result(:expr_beg, :tOP_ASGN, "&" ) }
-|               /\&\./                  { result(:expr_dot, :tLONELY,  "&.") }
+|               /\&\&\=/                { result(EXPR_BEG, :tOP_ASGN, "&&") }
+|               /\&\&/                  { result(EXPR_BEG, :tANDOP,   "&&") }
+|               /\&\=/                  { result(EXPR_BEG, :tOP_ASGN, "&" ) }
+|               /\&\./                  { result(EXPR_DOT, :tLONELY,  "&.") }
 |               /\&/                    process_amper
 
                 /\//                    process_slash
 
 : /\^/
-|               /\^=/                   { result(:expr_beg, :tOP_ASGN, "^") }
+|               /\^=/                   { result(EXPR_BEG, :tOP_ASGN, "^") }
 |               /\^/                    { result(:arg_state, :tCARET, "^") }
 
-                /\;/                    { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
+                /\;/                    { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
 
 : /~/
-| in_arg_state? /\~@/                   { result(:arg_state, :tTILDE, "~") }
+| is_after_operator? /\~@/              { result(:arg_state, :tTILDE, "~") }
 |               /\~/                    { result(:arg_state, :tTILDE, "~") }
 
 : /\\/
@@ -165,12 +164,12 @@ was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 | in_fname?     /\$([1-9]\d*)/                   process_gvar
 |               /\$([1-9]\d*)/                   process_nthref
 |               /\$0/                            process_gvar
+|               /\$#{IDENT_CHAR}+/               process_gvar
 |               /\$\W|\$\z/                      process_gvar_oddity
-|               /\$\w+/                          process_gvar
 
                 /\_/                    process_underscore
 
-                /#{IDENT}/o             process_token
+                /#{IDENT_CHAR}+/o       process_token
 
                 /\004|\032|\000|\Z/     { [RubyLexer::EOF, RubyLexer::EOF] }
 

data/lib/ruby_lexer.rex.rb

@@ -1,18 +1,23 @@
 # encoding: UTF-8
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.5.0.
+# Generated by: oedipus_lex version 2.5.2.
 # Source: lib/ruby_lexer.rex
 #++
 
 #
 # lexical scanner definition for ruby
 
+
+##
+# The generated lexer RubyLexer
+
 class RubyLexer
   require 'strscan'
 
-  IDENT         = /^#{IDENT_CHAR}+/o
-  ESC           = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
+  # :stopdoc:
+  IDENT_CHAR    = /[a-zA-Z0-9_[:^ascii:]]/
+  ESC           = /\\((?>[0-7]{1,3}|x\h{1,2}|M-[^\\]|(C-|c)[^\\]|u\h{1,4}|u\{\h+(?:\s+\h+)*\}|[^0-7xMCc]))/
   SIMPLE_STRING = /((#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*)/o
   SSTRING       = /((\\.|[^\'])*)/
   INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)(ri|r|i)?\b|0d[0-9_]+)(ri|r|i)?/i
@@ -24,30 +29,56 @@ class RubyLexer
   NUM_BAD       = /[+]?0[xbd]\b/i
   INT_OCT_BAD   = /[+]?0o?[0-7_]*[89]/i
   FLOAT_BAD     = /[+]?\d[\d_]*_(e|\.)/i
-
+  # :startdoc:
+  # :stopdoc:
   class LexerError < StandardError ; end
   class ScanError < LexerError ; end
+  # :startdoc:
+
+  ##
+  # The file name / path
 
   attr_accessor :filename
+
+  ##
+  # The StringScanner for this lexer.
+
   attr_accessor :ss
+
+  ##
+  # The current lexical state.
+
   attr_accessor :state
 
   alias :match :ss
 
+  ##
+  # The match groups for the current scan.
+
   def matches
     m = (1..9).map { |i| ss[i] }
     m.pop until m[-1] or m.empty?
     m
   end
 
+  ##
+  # Yields on the current action.
+
   def action
     yield
   end
 
+
+  ##
+  # The current scanner class. Must be overridden in subclasses.
+
   def scanner_class
     StringScanner
   end unless instance_methods(false).map(&:to_s).include?("scanner_class")
 
+  ##
+  # Parse the given string.
+
   def parse str
     self.ss     = scanner_class.new str
     self.state  ||= nil
@@ -55,6 +86,9 @@ class RubyLexer
     do_parse
   end
 
+  ##
+  # Read in and parse the file at +path+.
+
   def parse_file path
     self.filename = path
     open path do |f|
@@ -62,12 +96,18 @@ class RubyLexer
     end
   end
 
+  ##
+  # The current location in the parse.
+
   def location
     [
       (filename || "<input>"),
     ].compact.join(":")
   end
 
+  ##
+  # Lex the next token.
+
   def next_token
     return process_string if lex_strterm
     self.cmd_state = self.command_start
@@ -90,24 +130,24 @@ class RubyLexer
             process_brace_close text
           when ss.match?(/\!/) then
             case
-            when in_arg_state? && (ss.skip(/\!\@/)) then
-              action { result :expr_arg, :tUBANG, "!@" }
+            when is_after_operator? && (ss.skip(/\!\@/)) then
+              action { result EXPR_ARG, :tUBANG, "!@" }
             when text = ss.scan(/\![=~]?/) then
               action { result :arg_state, TOKENS[text], text }
             end # group /\!/
           when ss.match?(/\./) then
             case
             when text = ss.scan(/\.\.\.?/) then
-              action { result :expr_beg, TOKENS[text], text }
+              action { result EXPR_BEG, TOKENS[text], text }
             when ss.skip(/\.\d/) then
               action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
             when ss.skip(/\./) then
-              action { self.lex_state = :expr_beg; result :expr_dot, :tDOT, "." }
+              action { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
             end # group /\./
           when text = ss.scan(/\(/) then
             process_paren text
           when text = ss.scan(/\,/) then
-            action { result :expr_beg, TOKENS[text], text }
+            action { result EXPR_PAR, TOKENS[text], text }
           when ss.match?(/=/) then
             case
             when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
@@ -120,7 +160,7 @@ class RubyLexer
           when ruby22_label? && (text = ss.scan(/\"#{SIMPLE_STRING}\":/o)) then
             process_label text
           when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
-            action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
+            process_simple_string text
           when text = ss.scan(/\"/) then
             action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
           when text = ss.scan(/\@\@?\d/) then
@@ -141,7 +181,7 @@ class RubyLexer
               process_colon1 text
             end # group /:/
           when ss.skip(/->/) then
-            action { result :expr_endfn, :tLAMBDA, nil }
+            action { result EXPR_ENDFN, :tLAMBDA, nil }
           when text = ss.scan(/[+-]/) then
             process_plus_minus text
           when ss.match?(/[+\d]/) then
@@ -174,24 +214,24 @@ class RubyLexer
           when ss.match?(/\|/) then
             case
             when ss.skip(/\|\|\=/) then
-              action { result :expr_beg, :tOP_ASGN, "||" }
+              action { result EXPR_BEG, :tOP_ASGN, "||" }
             when ss.skip(/\|\|/) then
-              action { result :expr_beg, :tOROP,    "||" }
+              action { result EXPR_BEG, :tOROP,    "||" }
             when ss.skip(/\|\=/) then
-              action { result :expr_beg, :tOP_ASGN, "|" }
+              action { result EXPR_BEG, :tOP_ASGN, "|" }
             when ss.skip(/\|/) then
-              action { result :arg_state, :tPIPE,    "|" }
+              action { state = is_after_operator? ? EXPR_ARG : EXPR_PAR; result state, :tPIPE, "|" }
             end # group /\|/
           when text = ss.scan(/\{/) then
             process_brace_open text
           when ss.match?(/\*/) then
             case
             when ss.skip(/\*\*=/) then
-              action { result :expr_beg, :tOP_ASGN, "**" }
+              action { result EXPR_BEG, :tOP_ASGN, "**" }
             when ss.skip(/\*\*/) then
               action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
             when ss.skip(/\*\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "*") }
+              action { result(EXPR_BEG, :tOP_ASGN, "*") }
             when ss.skip(/\*/) then
               action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
             end # group /\*/
@@ -202,7 +242,7 @@ class RubyLexer
             when ss.skip(/\<\=/) then
               action { result :arg_state, :tLEQ, "<="     }
             when ss.skip(/\<\<\=/) then
-              action { result :expr_beg,  :tOP_ASGN, "<<" }
+              action { result EXPR_BEG,  :tOP_ASGN, "<<" }
             when text = ss.scan(/\<\</) then
               process_lchevron text
             when ss.skip(/\</) then
@@ -213,7 +253,7 @@ class RubyLexer
             when ss.skip(/\>\=/) then
               action { result :arg_state, :tGEQ, ">="     }
             when ss.skip(/\>\>=/) then
-              action { result :expr_beg,  :tOP_ASGN, ">>" }
+              action { result EXPR_BEG,  :tOP_ASGN, ">>" }
             when ss.skip(/\>\>/) then
               action { result :arg_state, :tRSHFT, ">>"   }
             when ss.skip(/\>/) then
@@ -222,9 +262,9 @@ class RubyLexer
           when ss.match?(/\`/) then
             case
             when expr_fname? && (ss.skip(/\`/)) then
-              action { result(:expr_end, :tBACK_REF2, "`") }
+              action { result(EXPR_END, :tBACK_REF2, "`") }
             when expr_dot? && (ss.skip(/\`/)) then
-              action { result((cmd_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
+              action { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
             when ss.skip(/\`/) then
               action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
             end # group /\`/
@@ -233,13 +273,13 @@ class RubyLexer
           when ss.match?(/&/) then
             case
             when ss.skip(/\&\&\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "&&") }
+              action { result(EXPR_BEG, :tOP_ASGN, "&&") }
             when ss.skip(/\&\&/) then
-              action { result(:expr_beg, :tANDOP,   "&&") }
+              action { result(EXPR_BEG, :tANDOP,   "&&") }
             when ss.skip(/\&\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "&" ) }
+              action { result(EXPR_BEG, :tOP_ASGN, "&" ) }
             when ss.skip(/\&\./) then
-              action { result(:expr_dot, :tLONELY,  "&.") }
+              action { result(EXPR_DOT, :tLONELY,  "&.") }
             when text = ss.scan(/\&/) then
               process_amper text
             end # group /&/
@@ -248,15 +288,15 @@ class RubyLexer
           when ss.match?(/\^/) then
             case
             when ss.skip(/\^=/) then
-              action { result(:expr_beg, :tOP_ASGN, "^") }
+              action { result(EXPR_BEG, :tOP_ASGN, "^") }
             when ss.skip(/\^/) then
               action { result(:arg_state, :tCARET, "^") }
             end # group /\^/
           when ss.skip(/\;/) then
-            action { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
+            action { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
           when ss.match?(/~/) then
             case
-            when in_arg_state? && (ss.skip(/\~@/)) then
+            when is_after_operator? && (ss.skip(/\~@/)) then
               action { result(:arg_state, :tTILDE, "~") }
             when ss.skip(/\~/) then
               action { result(:arg_state, :tTILDE, "~") }
@@ -288,14 +328,14 @@ class RubyLexer
               process_nthref text
             when text = ss.scan(/\$0/) then
               process_gvar text
+            when text = ss.scan(/\$#{IDENT_CHAR}+/) then
+              process_gvar text
             when text = ss.scan(/\$\W|\$\z/) then
               process_gvar_oddity text
-            when text = ss.scan(/\$\w+/) then
-              process_gvar text
             end # group /\$/
           when text = ss.scan(/\_/) then
             process_underscore text
-          when text = ss.scan(/#{IDENT}/o) then
+          when text = ss.scan(/#{IDENT_CHAR}+/o) then
             process_token text
           when ss.skip(/\004|\032|\000|\Z/) then
             action { [RubyLexer::EOF, RubyLexer::EOF] }

data/lib/ruby_parser.rb

@@ -9,28 +9,29 @@ class RubyParser
 
   VERSIONS = []
 
-  class Parser < Racc::Parser
-    include RubyParserStuff
+  attr_accessor :current
 
-    def self.inherited x
-      RubyParser::VERSIONS << x
-    end
-
-    def self.version= v
-       @version = v
-    end
+  def self.for_current_ruby
+    name  = "V#{RUBY_VERSION[/^\d+\.\d+/].delete "."}"
+    klass = if const_defined? name then
+              const_get name
+            else
+              latest = VERSIONS.first
+              warn "NOTE: RubyParser::#{name} undefined, using #{latest}."
+              latest
+            end
 
-    def self.version
-      @version ||= Parser > self && self.name[/(?:V|Ruby)(\d+)/, 1].to_i
-    end
+    klass.new
   end
 
-  class SyntaxError < RuntimeError; end
+  def self.latest
+    VERSIONS.first.new
+  end
 
   def process s, f = "(string)", t = 10
     e = nil
     VERSIONS.each do |klass|
-      parser = klass.new
+      self.current = parser = klass.new
       begin
         return parser.process s, f, t
       rescue Racc::ParseError, RubyParser::SyntaxError => exc
@@ -46,22 +47,23 @@ class RubyParser
     # do nothing
   end
 
-  def self.latest
-    VERSIONS.first.new
-  end
+  class Parser < Racc::Parser
+    include RubyParserStuff
 
-  def self.for_current_ruby
-    name  = "V#{RUBY_VERSION[/^\d+\.\d+/].delete "."}"
-    klass = if const_defined? name then
-              const_get name
-            else
-              latest = VERSIONS.first
-              warn "NOTE: RubyParser::#{name} undefined, using #{latest}."
-              latest
-            end
+    def self.inherited x
+      RubyParser::VERSIONS << x
+    end
 
-    klass.new
+    def self.version= v
+       @version = v
+    end
+
+    def self.version
+      @version ||= Parser > self && self.name[/(?:V|Ruby)(\d+)/, 1].to_i
+    end
   end
+
+  class SyntaxError < RuntimeError; end
 end
 
 ##
@@ -76,10 +78,12 @@ require "ruby23_parser"
 require "ruby24_parser"
 require "ruby25_parser"
 require "ruby26_parser"
+require "ruby27_parser"
 
 class RubyParser # HACK
   VERSIONS.clear # also a HACK caused by racc namespace issues
 
+  class V27 < ::Ruby27Parser; end
   class V26 < ::Ruby26Parser; end
   class V25 < ::Ruby25Parser; end
   class V24 < ::Ruby24Parser; end