ruby_parser 3.15.1 → 3.18.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/History.rdoc +107 -0
  4. data/Manifest.txt +5 -0
  5. data/README.rdoc +1 -0
  6. data/Rakefile +137 -29
  7. data/bin/ruby_parse_extract_error +1 -1
  8. data/compare/normalize.rb +8 -3
  9. data/debugging.md +133 -0
  10. data/gauntlet.md +106 -0
  11. data/lib/rp_extensions.rb +15 -36
  12. data/lib/rp_stringscanner.rb +20 -51
  13. data/lib/ruby20_parser.rb +3568 -3502
  14. data/lib/ruby20_parser.y +342 -251
  15. data/lib/ruby21_parser.rb +3659 -3617
  16. data/lib/ruby21_parser.y +337 -248
  17. data/lib/ruby22_parser.rb +3699 -3631
  18. data/lib/ruby22_parser.y +341 -250
  19. data/lib/ruby23_parser.rb +3638 -3576
  20. data/lib/ruby23_parser.y +341 -250
  21. data/lib/ruby24_parser.rb +3721 -3657
  22. data/lib/ruby24_parser.y +341 -250
  23. data/lib/ruby25_parser.rb +3721 -3657
  24. data/lib/ruby25_parser.y +341 -250
  25. data/lib/ruby26_parser.rb +3724 -3661
  26. data/lib/ruby26_parser.y +341 -249
  27. data/lib/ruby27_parser.rb +5018 -3725
  28. data/lib/ruby27_parser.y +937 -248
  29. data/lib/ruby30_parser.rb +8751 -0
  30. data/lib/ruby30_parser.y +3472 -0
  31. data/lib/ruby3_parser.yy +3476 -0
  32. data/lib/ruby_lexer.rb +273 -602
  33. data/lib/ruby_lexer.rex +28 -21
  34. data/lib/ruby_lexer.rex.rb +60 -24
  35. data/lib/ruby_lexer_strings.rb +638 -0
  36. data/lib/ruby_parser.rb +2 -0
  37. data/lib/ruby_parser.yy +978 -255
  38. data/lib/ruby_parser_extras.rb +297 -116
  39. data/test/test_ruby_lexer.rb +213 -129
  40. data/test/test_ruby_parser.rb +1479 -281
  41. data/tools/munge.rb +36 -8
  42. data/tools/ripper.rb +15 -10
  43. data.tar.gz.sig +0 -0
  44. metadata +36 -23
  45. metadata.gz.sig +0 -0
data/lib/ruby_lexer.rex CHANGED
@@ -4,6 +4,11 @@
4
4
 
5
5
  class RubyLexer
6
6
 
7
+ option
8
+
9
+ lineno
10
+ column
11
+
7
12
  macro
8
13
 
9
14
  IDENT_CHAR /[a-zA-Z0-9_[:^ascii:]]/
@@ -25,7 +30,8 @@ macro
25
30
 
26
31
  start
27
32
 
28
- return process_string if lex_strterm
33
+ maybe_pop_stack
34
+ return process_string_or_heredoc if lex_strterm
29
35
 
30
36
  self.cmd_state = self.command_start
31
37
  self.command_start = false
@@ -37,18 +43,18 @@ rule
37
43
  # [:state] pattern [actions]
38
44
 
39
45
  # \s - \n + \v
40
- /[\ \t\r\f\v]/ { self.space_seen = true; next }
46
+ /[\ \t\r\f\v]+/ { self.space_seen = true; next }
41
47
 
42
48
  /\n|\#/ process_newline_or_comment
43
49
 
44
50
  /[\]\)\}]/ process_brace_close
45
51
 
46
52
  : /\!/
47
- | is_after_operator? /\!\@/ { result EXPR_ARG, :tUBANG, "!@" }
53
+ | is_after_operator? /\!\@/ { result EXPR_ARG, TOKENS[text], text }
48
54
  | /\![=~]?/ { result :arg_state, TOKENS[text], text }
49
55
 
50
56
  : /\./
51
- | /\.\.\.?/ { result EXPR_BEG, TOKENS[text], text }
57
+ | /\.\.\.?/ process_dots
52
58
  | /\.\d/ { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
53
59
  | /\./ { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
54
60
 
@@ -63,7 +69,7 @@ rule
63
69
 
64
70
  ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
65
71
  /\"(#{SIMPLE_STRING})\"/o process_simple_string
66
- /\"/ { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
72
+ /\"/ { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
67
73
 
68
74
  /\@\@?\d/ { rb_compile_error "`#{text}` is not allowed as a variable name" }
69
75
  /\@\@?#{IDENT_CHAR}+/o process_ivar
@@ -75,7 +81,7 @@ ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
75
81
  | /\:\:/ process_colon2
76
82
  | /\:/ process_colon1
77
83
 
78
- /->/ { result EXPR_ENDFN, :tLAMBDA, nil }
84
+ /->/ { result EXPR_ENDFN, :tLAMBDA, text }
79
85
 
80
86
  /[+-]/ process_plus_minus
81
87
 
@@ -94,6 +100,7 @@ ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
94
100
  /\[/ process_square_bracket
95
101
 
96
102
  was_label? /\'#{SSTRING}\':?/o process_label_or_string
103
+ /\'/ { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
97
104
 
98
105
  : /\|/
99
106
  | /\|\|\=/ { result EXPR_BEG, :tOP_ASGN, "||" }
@@ -105,9 +112,9 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
105
112
 
106
113
  : /\*/
107
114
  | /\*\*=/ { result EXPR_BEG, :tOP_ASGN, "**" }
108
- | /\*\*/ { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
109
- | /\*\=/ { result(EXPR_BEG, :tOP_ASGN, "*") }
110
- | /\*/ { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
115
+ | /\*\*/ { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
116
+ | /\*\=/ { result EXPR_BEG, :tOP_ASGN, "*" }
117
+ | /\*/ { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
111
118
 
112
119
  # TODO: fix result+process_lchevron to set command_start = true
113
120
  : /</
@@ -124,30 +131,30 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
124
131
  | /\>/ { result :arg_state, :tGT, ">" }
125
132
 
126
133
  : /\`/
127
- | expr_fname? /\`/ { result(EXPR_END, :tBACK_REF2, "`") }
134
+ | expr_fname? /\`/ { result EXPR_END, :tBACK_REF2, "`" }
128
135
  | expr_dot? /\`/ { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
129
- | /\`/ { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
136
+ | /\`/ { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
130
137
 
131
138
  /\?/ process_questionmark
132
139
 
133
140
  : /&/
134
- | /\&\&\=/ { result(EXPR_BEG, :tOP_ASGN, "&&") }
135
- | /\&\&/ { result(EXPR_BEG, :tANDOP, "&&") }
136
- | /\&\=/ { result(EXPR_BEG, :tOP_ASGN, "&" ) }
137
- | /\&\./ { result(EXPR_DOT, :tLONELY, "&.") }
141
+ | /\&\&\=/ { result EXPR_BEG, :tOP_ASGN, "&&" }
142
+ | /\&\&/ { result EXPR_BEG, :tANDOP, "&&" }
143
+ | /\&\=/ { result EXPR_BEG, :tOP_ASGN, "&" }
144
+ | /\&\./ { result EXPR_DOT, :tLONELY, "&." }
138
145
  | /\&/ process_amper
139
146
 
140
147
  /\// process_slash
141
148
 
142
149
  : /\^/
143
- | /\^=/ { result(EXPR_BEG, :tOP_ASGN, "^") }
144
- | /\^/ { result(:arg_state, :tCARET, "^") }
150
+ | /\^=/ { result EXPR_BEG, :tOP_ASGN, "^" }
151
+ | /\^/ { result :arg_state, :tCARET, "^" }
145
152
 
146
- /\;/ { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
153
+ /\;/ { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
147
154
 
148
155
  : /~/
149
- | is_after_operator? /\~@/ { result(:arg_state, :tTILDE, "~") }
150
- | /\~/ { result(:arg_state, :tTILDE, "~") }
156
+ | is_after_operator? /\~@/ { result :arg_state, :tTILDE, "~" }
157
+ | /\~/ { result :arg_state, :tTILDE, "~" }
151
158
 
152
159
  : /\\/
153
160
  | /\\\r?\n/ { self.lineno += 1; self.space_seen = true; next }
@@ -165,7 +172,7 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
165
172
  | /\$([1-9]\d*)/ process_nthref
166
173
  | /\$0/ process_gvar
167
174
  | /\$#{IDENT_CHAR}+/ process_gvar
168
- | /\$\W|\$\z/ process_gvar_oddity
175
+ | /\$\W/ process_gvar_oddity
169
176
 
170
177
  /\_/ process_underscore
171
178
 
data/lib/ruby_lexer.rex.rb CHANGED
@@ -1,7 +1,8 @@
1
+ # frozen_string_literal: true
1
2
  # encoding: UTF-8
2
3
  #--
3
4
  # This file is automatically generated. Do not modify it.
4
- # Generated by: oedipus_lex version 2.5.2.
5
+ # Generated by: oedipus_lex version 2.6.0.
5
6
  # Source: lib/ruby_lexer.rex
6
7
  #++
7
8
 
@@ -35,6 +36,10 @@ class RubyLexer
35
36
  class ScanError < LexerError ; end
36
37
  # :startdoc:
37
38
 
39
+ ##
40
+ # The current line number.
41
+
42
+ attr_accessor :lineno
38
43
  ##
39
44
  # The file name / path
40
45
 
@@ -68,6 +73,24 @@ class RubyLexer
68
73
  yield
69
74
  end
70
75
 
76
+ ##
77
+ # The previous position. Only available if the :column option is on.
78
+
79
+ attr_accessor :old_pos
80
+
81
+ ##
82
+ # The position of the start of the current line. Only available if the
83
+ # :column option is on.
84
+
85
+ attr_accessor :start_of_current_line_pos
86
+
87
+ ##
88
+ # The current column, starting at 0. Only available if the
89
+ # :column option is on.
90
+ def column
91
+ old_pos - start_of_current_line_pos
92
+ end
93
+
71
94
 
72
95
  ##
73
96
  # The current scanner class. Must be overridden in subclasses.
@@ -81,6 +104,8 @@ class RubyLexer
81
104
 
82
105
  def parse str
83
106
  self.ss = scanner_class.new str
107
+ self.lineno = 1
108
+ self.start_of_current_line_pos = 0
84
109
  self.state ||= nil
85
110
 
86
111
  do_parse
@@ -102,6 +127,8 @@ class RubyLexer
102
127
  def location
103
128
  [
104
129
  (filename || "<input>"),
130
+ lineno,
131
+ column,
105
132
  ].compact.join(":")
106
133
  end
107
134
 
@@ -109,7 +136,8 @@ class RubyLexer
109
136
  # Lex the next token.
110
137
 
111
138
  def next_token
112
- return process_string if lex_strterm
139
+ maybe_pop_stack
140
+ return process_string_or_heredoc if lex_strterm
113
141
  self.cmd_state = self.command_start
114
142
  self.command_start = false
115
143
  self.space_seen = false # TODO: rename token_seen?
@@ -118,11 +146,17 @@ class RubyLexer
118
146
  token = nil
119
147
 
120
148
  until ss.eos? or token do
149
+ if ss.check(/\n/) then
150
+ self.lineno += 1
151
+ # line starts 1 position after the newline
152
+ self.start_of_current_line_pos = ss.pos + 1
153
+ end
154
+ self.old_pos = ss.pos
121
155
  token =
122
156
  case state
123
157
  when nil then
124
158
  case
125
- when ss.skip(/[\ \t\r\f\v]/) then
159
+ when ss.skip(/[\ \t\r\f\v]+/) then
126
160
  action { self.space_seen = true; next }
127
161
  when text = ss.scan(/\n|\#/) then
128
162
  process_newline_or_comment text
@@ -130,15 +164,15 @@ class RubyLexer
130
164
  process_brace_close text
131
165
  when ss.match?(/\!/) then
132
166
  case
133
- when is_after_operator? && (ss.skip(/\!\@/)) then
134
- action { result EXPR_ARG, :tUBANG, "!@" }
167
+ when is_after_operator? && (text = ss.scan(/\!\@/)) then
168
+ action { result EXPR_ARG, TOKENS[text], text }
135
169
  when text = ss.scan(/\![=~]?/) then
136
170
  action { result :arg_state, TOKENS[text], text }
137
171
  end # group /\!/
138
172
  when ss.match?(/\./) then
139
173
  case
140
174
  when text = ss.scan(/\.\.\.?/) then
141
- action { result EXPR_BEG, TOKENS[text], text }
175
+ process_dots text
142
176
  when ss.skip(/\.\d/) then
143
177
  action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
144
178
  when ss.skip(/\./) then
@@ -162,7 +196,7 @@ class RubyLexer
162
196
  when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
163
197
  process_simple_string text
164
198
  when text = ss.scan(/\"/) then
165
- action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
199
+ action { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
166
200
  when text = ss.scan(/\@\@?\d/) then
167
201
  action { rb_compile_error "`#{text}` is not allowed as a variable name" }
168
202
  when text = ss.scan(/\@\@?#{IDENT_CHAR}+/o) then
@@ -180,8 +214,8 @@ class RubyLexer
180
214
  when text = ss.scan(/\:/) then
181
215
  process_colon1 text
182
216
  end # group /:/
183
- when ss.skip(/->/) then
184
- action { result EXPR_ENDFN, :tLAMBDA, nil }
217
+ when text = ss.scan(/->/) then
218
+ action { result EXPR_ENDFN, :tLAMBDA, text }
185
219
  when text = ss.scan(/[+-]/) then
186
220
  process_plus_minus text
187
221
  when ss.match?(/[+\d]/) then
@@ -211,6 +245,8 @@ class RubyLexer
211
245
  process_square_bracket text
212
246
  when was_label? && (text = ss.scan(/\'#{SSTRING}\':?/o)) then
213
247
  process_label_or_string text
248
+ when text = ss.scan(/\'/) then
249
+ action { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
214
250
  when ss.match?(/\|/) then
215
251
  case
216
252
  when ss.skip(/\|\|\=/) then
@@ -229,11 +265,11 @@ class RubyLexer
229
265
  when ss.skip(/\*\*=/) then
230
266
  action { result EXPR_BEG, :tOP_ASGN, "**" }
231
267
  when ss.skip(/\*\*/) then
232
- action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
268
+ action { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
233
269
  when ss.skip(/\*\=/) then
234
- action { result(EXPR_BEG, :tOP_ASGN, "*") }
270
+ action { result EXPR_BEG, :tOP_ASGN, "*" }
235
271
  when ss.skip(/\*/) then
236
- action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
272
+ action { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
237
273
  end # group /\*/
238
274
  when ss.match?(/</) then
239
275
  case
@@ -262,24 +298,24 @@ class RubyLexer
262
298
  when ss.match?(/\`/) then
263
299
  case
264
300
  when expr_fname? && (ss.skip(/\`/)) then
265
- action { result(EXPR_END, :tBACK_REF2, "`") }
301
+ action { result EXPR_END, :tBACK_REF2, "`" }
266
302
  when expr_dot? && (ss.skip(/\`/)) then
267
303
  action { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
268
304
  when ss.skip(/\`/) then
269
- action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
305
+ action { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
270
306
  end # group /\`/
271
307
  when text = ss.scan(/\?/) then
272
308
  process_questionmark text
273
309
  when ss.match?(/&/) then
274
310
  case
275
311
  when ss.skip(/\&\&\=/) then
276
- action { result(EXPR_BEG, :tOP_ASGN, "&&") }
312
+ action { result EXPR_BEG, :tOP_ASGN, "&&" }
277
313
  when ss.skip(/\&\&/) then
278
- action { result(EXPR_BEG, :tANDOP, "&&") }
314
+ action { result EXPR_BEG, :tANDOP, "&&" }
279
315
  when ss.skip(/\&\=/) then
280
- action { result(EXPR_BEG, :tOP_ASGN, "&" ) }
316
+ action { result EXPR_BEG, :tOP_ASGN, "&" }
281
317
  when ss.skip(/\&\./) then
282
- action { result(EXPR_DOT, :tLONELY, "&.") }
318
+ action { result EXPR_DOT, :tLONELY, "&." }
283
319
  when text = ss.scan(/\&/) then
284
320
  process_amper text
285
321
  end # group /&/
@@ -288,18 +324,18 @@ class RubyLexer
288
324
  when ss.match?(/\^/) then
289
325
  case
290
326
  when ss.skip(/\^=/) then
291
- action { result(EXPR_BEG, :tOP_ASGN, "^") }
327
+ action { result EXPR_BEG, :tOP_ASGN, "^" }
292
328
  when ss.skip(/\^/) then
293
- action { result(:arg_state, :tCARET, "^") }
329
+ action { result :arg_state, :tCARET, "^" }
294
330
  end # group /\^/
295
331
  when ss.skip(/\;/) then
296
- action { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
332
+ action { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
297
333
  when ss.match?(/~/) then
298
334
  case
299
335
  when is_after_operator? && (ss.skip(/\~@/)) then
300
- action { result(:arg_state, :tTILDE, "~") }
336
+ action { result :arg_state, :tTILDE, "~" }
301
337
  when ss.skip(/\~/) then
302
- action { result(:arg_state, :tTILDE, "~") }
338
+ action { result :arg_state, :tTILDE, "~" }
303
339
  end # group /~/
304
340
  when ss.match?(/\\/) then
305
341
  case
@@ -330,7 +366,7 @@ class RubyLexer
330
366
  process_gvar text
331
367
  when text = ss.scan(/\$#{IDENT_CHAR}+/) then
332
368
  process_gvar text
333
- when text = ss.scan(/\$\W|\$\z/) then
369
+ when text = ss.scan(/\$\W/) then
334
370
  process_gvar_oddity text
335
371
  end # group /\$/
336
372
  when text = ss.scan(/\_/) then