ruby_parser 3.12.0 → 3.18.1

data/lib/ruby_lexer.rex

@@ -4,11 +4,16 @@
 
 class RubyLexer
 
+option
+
+  lineno
+  column
+
 macro
 
-  IDENT         /^#{IDENT_CHAR}+/o
+  IDENT_CHAR    /[a-zA-Z0-9_[:^ascii:]]/
 
-  ESC           /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
+  ESC           /\\((?>[0-7]{1,3}|x\h{1,2}|M-[^\\]|(C-|c)[^\\]|u\h{1,4}|u\{\h+(?:\s+\h+)*\}|[^0-7xMCc]))/
   SIMPLE_STRING /((#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*)/o
   SSTRING       /((\\.|[^\'])*)/
 
@@ -25,11 +30,12 @@ macro
 
 start
 
-  return process_string if lex_strterm
+  maybe_pop_stack
+  return process_string_or_heredoc if lex_strterm
 
-  self.command_state = self.command_start
+  self.cmd_state = self.command_start
   self.command_start = false
-  self.space_seen    = false
+  self.space_seen    = false # TODO: rename token_seen?
   self.last_state    = lex_state
 
 rule
@@ -37,24 +43,24 @@ rule
 # [:state]      pattern                 [actions]
 
                 # \s - \n + \v
-                /[\ \t\r\f\v]/          { self.space_seen = true; next }
+                /[\ \t\r\f\v]+/         { self.space_seen = true; next }
 
                 /\n|\#/                 process_newline_or_comment
 
-                /[\]\)\}]/              process_bracing
+                /[\]\)\}]/              process_brace_close
 
 : /\!/
-| in_arg_state? /\!\@/                  { result :expr_arg, :tUBANG, "!@" }
+| is_after_operator? /\!\@/             { result EXPR_ARG,   TOKENS[text], text }
 |               /\![=~]?/               { result :arg_state, TOKENS[text], text }
 
 : /\./
-|               /\.\.\.?/               { result :expr_beg, TOKENS[text], text }
+|               /\.\.\.?/               process_dots
 |               /\.\d/                  { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
-|               /\./                    { result :expr_dot, :tDOT, "." }
+|               /\./                    { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
 
                 /\(/                    process_paren
 
-                /\,/                    { result :expr_beg, TOKENS[text], text }
+                /\,/                    { result EXPR_PAR, TOKENS[text], text }
 
 : /=/
 |               /\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/ { result arg_state, TOKENS[text], text }
@@ -62,8 +68,8 @@ rule
 |               /\=(?=begin\b)/         { result arg_state, TOKENS[text], text }
 
 ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
-                /\"(#{SIMPLE_STRING})\"/o { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
-                /\"/                    { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
+                /\"(#{SIMPLE_STRING})\"/o process_simple_string
+                /\"/                    { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
 
                 /\@\@?\d/               { rb_compile_error "`#{text}` is not allowed as a variable name" }
                 /\@\@?#{IDENT_CHAR}+/o  process_ivar
@@ -75,7 +81,7 @@ ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
 |               /\:\:/                      process_colon2
 |               /\:/                        process_colon1
 
-                /->/                    { result :expr_endfn, :tLAMBDA, nil }
+                /->/                    { result EXPR_ENDFN, :tLAMBDA, text }
 
                 /[+-]/                  process_plus_minus
 
@@ -94,59 +100,61 @@ ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
                 /\[/                    process_square_bracket
 
 was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
+                  /\'/                  { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
 
 : /\|/
-|               /\|\|\=/                { result :expr_beg, :tOP_ASGN, "||" }
-|               /\|\|/                  { result :expr_beg, :tOROP,    "||" }
-|               /\|\=/                  { result :expr_beg, :tOP_ASGN, "|" }
-|               /\|/                    { result :arg_state, :tPIPE,    "|" }
+|               /\|\|\=/                { result EXPR_BEG, :tOP_ASGN, "||" }
+|               /\|\|/                  { result EXPR_BEG, :tOROP,    "||" }
+|               /\|\=/                  { result EXPR_BEG, :tOP_ASGN, "|" }
+|               /\|/                    { state = is_after_operator? ? EXPR_ARG : EXPR_PAR; result state, :tPIPE, "|" }
 
-                /\{/                    process_curly_brace
+                /\{/                    process_brace_open
 
 : /\*/
-|               /\*\*=/                 { result :expr_beg, :tOP_ASGN, "**" }
-|               /\*\*/                  { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
-|               /\*\=/                  { result(:expr_beg, :tOP_ASGN, "*") }
-|               /\*/                    { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
+|               /\*\*=/                 { result EXPR_BEG, :tOP_ASGN, "**" }
+|               /\*\*/                  { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
+|               /\*\=/                  { result EXPR_BEG, :tOP_ASGN, "*" }
+|               /\*/                    { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
 
+# TODO: fix result+process_lchevron to set command_start = true
 : /</
 |               /\<\=\>/                { result :arg_state, :tCMP, "<=>"    }
 |               /\<\=/                  { result :arg_state, :tLEQ, "<="     }
-|               /\<\<\=/                { result :arg_state, :tOP_ASGN, "<<" }
+|               /\<\<\=/                { result EXPR_BEG,  :tOP_ASGN, "<<" }
 |               /\<\</                  process_lchevron
 |               /\</                    { result :arg_state, :tLT, "<"       }
 
 : />/
 |               /\>\=/                  { result :arg_state, :tGEQ, ">="     }
-|               /\>\>=/                 { result :arg_state, :tOP_ASGN, ">>" }
+|               /\>\>=/                 { result EXPR_BEG,  :tOP_ASGN, ">>" }
 |               /\>\>/                  { result :arg_state, :tRSHFT, ">>"   }
 |               /\>/                    { result :arg_state, :tGT, ">"       }
 
 : /\`/
-| expr_fname?   /\`/                   { result(:expr_end, :tBACK_REF2, "`") }
-| expr_dot?     /\`/                   { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
-|               /\`/                   { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
+| expr_fname?   /\`/                   { result EXPR_END, :tBACK_REF2, "`" }
+| expr_dot?     /\`/                   { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
+|               /\`/                   { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
 
                 /\?/                    process_questionmark
 
 : /&/
-|               /\&\&\=/                { result(:expr_beg, :tOP_ASGN, "&&") }
-|               /\&\&/                  { result(:expr_beg, :tANDOP,   "&&") }
-|               /\&\=/                  { result(:expr_beg, :tOP_ASGN, "&" ) }
-|               /\&\./                  { result(:expr_dot, :tLONELY,  "&.") }
+|               /\&\&\=/                { result EXPR_BEG, :tOP_ASGN, "&&" }
+|               /\&\&/                  { result EXPR_BEG, :tANDOP,   "&&" }
+|               /\&\=/                  { result EXPR_BEG, :tOP_ASGN, "&"  }
+|               /\&\./                  { result EXPR_DOT, :tLONELY,  "&." }
 |               /\&/                    process_amper
 
                 /\//                    process_slash
 
 : /\^/
-|               /\^=/                   { result(:expr_beg, :tOP_ASGN, "^") }
-|               /\^/                    { result(:arg_state, :tCARET, "^") }
+|               /\^=/                   { result EXPR_BEG, :tOP_ASGN, "^" }
+|               /\^/                    { result :arg_state, :tCARET, "^" }
 
-                /\;/                    { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
+                /\;/                    { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
 
 : /~/
-| in_arg_state? /\~@/                   { result(:arg_state, :tTILDE, "~") }
-|               /\~/                    { result(:arg_state, :tTILDE, "~") }
+| is_after_operator? /\~@/              { result :arg_state, :tTILDE, "~" }
+|               /\~/                    { result :arg_state, :tTILDE, "~" }
 
 : /\\/
 |               /\\\r?\n/               { self.lineno += 1; self.space_seen = true; next }
@@ -163,12 +171,12 @@ was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 | in_fname?     /\$([1-9]\d*)/                   process_gvar
 |               /\$([1-9]\d*)/                   process_nthref
 |               /\$0/                            process_gvar
-|               /\$\W|\$\z/                      process_gvar_oddity
-|               /\$\w+/                          process_gvar
+|               /\$#{IDENT_CHAR}+/               process_gvar
+|               /\$\W/                           process_gvar_oddity
 
                 /\_/                    process_underscore
 
-                /#{IDENT}/o             process_token
+                /#{IDENT_CHAR}+/o       process_token
 
                 /\004|\032|\000|\Z/     { [RubyLexer::EOF, RubyLexer::EOF] }
 

data/lib/ruby_lexer.rex.rb

@@ -1,18 +1,24 @@
+# frozen_string_literal: true
 # encoding: UTF-8
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.5.0.
+# Generated by: oedipus_lex version 2.6.0.
 # Source: lib/ruby_lexer.rex
 #++
 
 #
 # lexical scanner definition for ruby
 
+
+##
+# The generated lexer RubyLexer
+
 class RubyLexer
   require 'strscan'
 
-  IDENT         = /^#{IDENT_CHAR}+/o
-  ESC           = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
+  # :stopdoc:
+  IDENT_CHAR    = /[a-zA-Z0-9_[:^ascii:]]/
+  ESC           = /\\((?>[0-7]{1,3}|x\h{1,2}|M-[^\\]|(C-|c)[^\\]|u\h{1,4}|u\{\h+(?:\s+\h+)*\}|[^0-7xMCc]))/
   SIMPLE_STRING = /((#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*)/o
   SSTRING       = /((\\.|[^\'])*)/
   INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)(ri|r|i)?\b|0d[0-9_]+)(ri|r|i)?/i
@@ -24,37 +30,90 @@ class RubyLexer
   NUM_BAD       = /[+]?0[xbd]\b/i
   INT_OCT_BAD   = /[+]?0o?[0-7_]*[89]/i
   FLOAT_BAD     = /[+]?\d[\d_]*_(e|\.)/i
-
+  # :startdoc:
+  # :stopdoc:
   class LexerError < StandardError ; end
   class ScanError < LexerError ; end
+  # :startdoc:
+
+  ##
+  # The current line number.
+
+  attr_accessor :lineno
+  ##
+  # The file name / path
 
   attr_accessor :filename
+
+  ##
+  # The StringScanner for this lexer.
+
   attr_accessor :ss
+
+  ##
+  # The current lexical state.
+
   attr_accessor :state
 
   alias :match :ss
 
+  ##
+  # The match groups for the current scan.
+
   def matches
     m = (1..9).map { |i| ss[i] }
     m.pop until m[-1] or m.empty?
     m
   end
 
+  ##
+  # Yields on the current action.
+
   def action
     yield
   end
 
+  ##
+  # The previous position. Only available if the :column option is on.
+
+  attr_accessor :old_pos
+
+  ##
+  # The position of the start of the current line. Only available if the
+  # :column option is on.
+
+  attr_accessor :start_of_current_line_pos
+
+  ##
+  # The current column, starting at 0. Only available if the
+  # :column option is on.
+  def column
+    old_pos - start_of_current_line_pos
+  end
+
+
+  ##
+  # The current scanner class. Must be overridden in subclasses.
+
   def scanner_class
     StringScanner
   end unless instance_methods(false).map(&:to_s).include?("scanner_class")
 
+  ##
+  # Parse the given string.
+
   def parse str
     self.ss     = scanner_class.new str
+    self.lineno = 1
+    self.start_of_current_line_pos = 0
     self.state  ||= nil
 
     do_parse
   end
 
+  ##
+  # Read in and parse the file at +path+.
+
   def parse_file path
     self.filename = path
     open path do |f|
@@ -62,52 +121,67 @@ class RubyLexer
     end
   end
 
+  ##
+  # The current location in the parse.
+
   def location
     [
       (filename || "<input>"),
+      lineno,
+      column,
     ].compact.join(":")
   end
 
+  ##
+  # Lex the next token.
+
   def next_token
-    return process_string if lex_strterm
-    self.command_state = self.command_start
+    maybe_pop_stack
+    return process_string_or_heredoc if lex_strterm
+    self.cmd_state = self.command_start
     self.command_start = false
-    self.space_seen    = false
+    self.space_seen    = false # TODO: rename token_seen?
     self.last_state    = lex_state
 
     token = nil
 
     until ss.eos? or token do
+      if ss.check(/\n/) then
+        self.lineno += 1
+        # line starts 1 position after the newline
+        self.start_of_current_line_pos = ss.pos + 1
+      end
+      self.old_pos = ss.pos
       token =
         case state
         when nil then
           case
-          when ss.skip(/[\ \t\r\f\v]/) then
+          when ss.skip(/[\ \t\r\f\v]+/) then
             action { self.space_seen = true; next }
           when text = ss.scan(/\n|\#/) then
             process_newline_or_comment text
           when text = ss.scan(/[\]\)\}]/) then
-            process_bracing text
+            process_brace_close text
           when ss.match?(/\!/) then
             case
-            when in_arg_state? && (ss.skip(/\!\@/)) then
-              action { result :expr_arg, :tUBANG, "!@" }
+            when is_after_operator? && (text = ss.scan(/\!\@/)) then
+              action { result EXPR_ARG,   TOKENS[text], text }
             when text = ss.scan(/\![=~]?/) then
               action { result :arg_state, TOKENS[text], text }
             end # group /\!/
           when ss.match?(/\./) then
             case
             when text = ss.scan(/\.\.\.?/) then
-              action { result :expr_beg, TOKENS[text], text }
+              process_dots text
             when ss.skip(/\.\d/) then
               action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
             when ss.skip(/\./) then
-              action { result :expr_dot, :tDOT, "." }
+              action { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
             end # group /\./
           when text = ss.scan(/\(/) then
             process_paren text
           when text = ss.scan(/\,/) then
-            action { result :expr_beg, TOKENS[text], text }
+            action { result EXPR_PAR, TOKENS[text], text }
           when ss.match?(/=/) then
             case
             when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
@@ -120,9 +194,9 @@ class RubyLexer
           when ruby22_label? && (text = ss.scan(/\"#{SIMPLE_STRING}\":/o)) then
             process_label text
           when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
-            action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
+            process_simple_string text
           when text = ss.scan(/\"/) then
-            action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
+            action { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
           when text = ss.scan(/\@\@?\d/) then
             action { rb_compile_error "`#{text}` is not allowed as a variable name" }
           when text = ss.scan(/\@\@?#{IDENT_CHAR}+/o) then
@@ -140,8 +214,8 @@ class RubyLexer
             when text = ss.scan(/\:/) then
               process_colon1 text
             end # group /:/
-          when ss.skip(/->/) then
-            action { result :expr_endfn, :tLAMBDA, nil }
+          when text = ss.scan(/->/) then
+            action { result EXPR_ENDFN, :tLAMBDA, text }
           when text = ss.scan(/[+-]/) then
             process_plus_minus text
           when ss.match?(/[+\d]/) then
@@ -171,29 +245,31 @@ class RubyLexer
             process_square_bracket text
           when was_label? && (text = ss.scan(/\'#{SSTRING}\':?/o)) then
             process_label_or_string text
+          when text = ss.scan(/\'/) then
+            action { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
           when ss.match?(/\|/) then
             case
             when ss.skip(/\|\|\=/) then
-              action { result :expr_beg, :tOP_ASGN, "||" }
+              action { result EXPR_BEG, :tOP_ASGN, "||" }
             when ss.skip(/\|\|/) then
-              action { result :expr_beg, :tOROP,    "||" }
+              action { result EXPR_BEG, :tOROP,    "||" }
             when ss.skip(/\|\=/) then
-              action { result :expr_beg, :tOP_ASGN, "|" }
+              action { result EXPR_BEG, :tOP_ASGN, "|" }
             when ss.skip(/\|/) then
-              action { result :arg_state, :tPIPE,    "|" }
+              action { state = is_after_operator? ? EXPR_ARG : EXPR_PAR; result state, :tPIPE, "|" }
             end # group /\|/
           when text = ss.scan(/\{/) then
-            process_curly_brace text
+            process_brace_open text
           when ss.match?(/\*/) then
             case
             when ss.skip(/\*\*=/) then
-              action { result :expr_beg, :tOP_ASGN, "**" }
+              action { result EXPR_BEG, :tOP_ASGN, "**" }
             when ss.skip(/\*\*/) then
-              action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
+              action { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
             when ss.skip(/\*\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "*") }
+              action { result EXPR_BEG, :tOP_ASGN, "*" }
             when ss.skip(/\*/) then
-              action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
+              action { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
             end # group /\*/
           when ss.match?(/</) then
             case
@@ -202,7 +278,7 @@ class RubyLexer
             when ss.skip(/\<\=/) then
               action { result :arg_state, :tLEQ, "<="     }
             when ss.skip(/\<\<\=/) then
-              action { result :arg_state, :tOP_ASGN, "<<" }
+              action { result EXPR_BEG,  :tOP_ASGN, "<<" }
             when text = ss.scan(/\<\</) then
               process_lchevron text
             when ss.skip(/\</) then
@@ -213,7 +289,7 @@ class RubyLexer
             when ss.skip(/\>\=/) then
               action { result :arg_state, :tGEQ, ">="     }
             when ss.skip(/\>\>=/) then
-              action { result :arg_state, :tOP_ASGN, ">>" }
+              action { result EXPR_BEG,  :tOP_ASGN, ">>" }
             when ss.skip(/\>\>/) then
               action { result :arg_state, :tRSHFT, ">>"   }
             when ss.skip(/\>/) then
@@ -222,24 +298,24 @@ class RubyLexer
           when ss.match?(/\`/) then
             case
             when expr_fname? && (ss.skip(/\`/)) then
-              action { result(:expr_end, :tBACK_REF2, "`") }
+              action { result EXPR_END, :tBACK_REF2, "`" }
             when expr_dot? && (ss.skip(/\`/)) then
-              action { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
+              action { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
             when ss.skip(/\`/) then
-              action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
+              action { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
             end # group /\`/
           when text = ss.scan(/\?/) then
             process_questionmark text
           when ss.match?(/&/) then
             case
             when ss.skip(/\&\&\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "&&") }
+              action { result EXPR_BEG, :tOP_ASGN, "&&" }
             when ss.skip(/\&\&/) then
-              action { result(:expr_beg, :tANDOP,   "&&") }
+              action { result EXPR_BEG, :tANDOP,   "&&" }
             when ss.skip(/\&\=/) then
-              action { result(:expr_beg, :tOP_ASGN, "&" ) }
+              action { result EXPR_BEG, :tOP_ASGN, "&"  }
             when ss.skip(/\&\./) then
-              action { result(:expr_dot, :tLONELY,  "&.") }
+              action { result EXPR_DOT, :tLONELY,  "&." }
             when text = ss.scan(/\&/) then
               process_amper text
             end # group /&/
@@ -248,18 +324,18 @@ class RubyLexer
           when ss.match?(/\^/) then
             case
             when ss.skip(/\^=/) then
-              action { result(:expr_beg, :tOP_ASGN, "^") }
+              action { result EXPR_BEG, :tOP_ASGN, "^" }
             when ss.skip(/\^/) then
-              action { result(:arg_state, :tCARET, "^") }
+              action { result :arg_state, :tCARET, "^" }
             end # group /\^/
           when ss.skip(/\;/) then
-            action { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
+            action { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
           when ss.match?(/~/) then
             case
-            when in_arg_state? && (ss.skip(/\~@/)) then
-              action { result(:arg_state, :tTILDE, "~") }
+            when is_after_operator? && (ss.skip(/\~@/)) then
+              action { result :arg_state, :tTILDE, "~" }
             when ss.skip(/\~/) then
-              action { result(:arg_state, :tTILDE, "~") }
+              action { result :arg_state, :tTILDE, "~" }
             end # group /~/
           when ss.match?(/\\/) then
             case
@@ -288,14 +364,14 @@ class RubyLexer
               process_nthref text
             when text = ss.scan(/\$0/) then
               process_gvar text
-            when text = ss.scan(/\$\W|\$\z/) then
-              process_gvar_oddity text
-            when text = ss.scan(/\$\w+/) then
+            when text = ss.scan(/\$#{IDENT_CHAR}+/) then
               process_gvar text
+            when text = ss.scan(/\$\W/) then
+              process_gvar_oddity text
             end # group /\$/
           when text = ss.scan(/\_/) then
             process_underscore text
-          when text = ss.scan(/#{IDENT}/o) then
+          when text = ss.scan(/#{IDENT_CHAR}+/o) then
             process_token text
           when ss.skip(/\004|\032|\000|\Z/) then
             action { [RubyLexer::EOF, RubyLexer::EOF] }