ruby_parser 3.15.0 → 3.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/History.rdoc +101 -0
  4. data/Manifest.txt +5 -0
  5. data/README.rdoc +1 -0
  6. data/Rakefile +128 -30
  7. data/bin/ruby_parse_extract_error +1 -1
  8. data/compare/normalize.rb +8 -3
  9. data/debugging.md +133 -0
  10. data/gauntlet.md +106 -0
  11. data/lib/rp_extensions.rb +15 -36
  12. data/lib/rp_stringscanner.rb +20 -51
  13. data/lib/ruby20_parser.rb +3559 -3499
  14. data/lib/ruby20_parser.y +333 -248
  15. data/lib/ruby21_parser.rb +3650 -3614
  16. data/lib/ruby21_parser.y +328 -245
  17. data/lib/ruby22_parser.rb +3690 -3628
  18. data/lib/ruby22_parser.y +332 -247
  19. data/lib/ruby23_parser.rb +3629 -3573
  20. data/lib/ruby23_parser.y +332 -247
  21. data/lib/ruby24_parser.rb +3712 -3654
  22. data/lib/ruby24_parser.y +332 -247
  23. data/lib/ruby25_parser.rb +3712 -3654
  24. data/lib/ruby25_parser.y +332 -247
  25. data/lib/ruby26_parser.rb +3715 -3658
  26. data/lib/ruby26_parser.y +332 -246
  27. data/lib/ruby27_parser.rb +5009 -3722
  28. data/lib/ruby27_parser.y +928 -245
  29. data/lib/ruby30_parser.rb +8741 -0
  30. data/lib/ruby30_parser.y +3463 -0
  31. data/lib/ruby3_parser.yy +3467 -0
  32. data/lib/ruby_lexer.rb +273 -602
  33. data/lib/ruby_lexer.rex +28 -21
  34. data/lib/ruby_lexer.rex.rb +60 -24
  35. data/lib/ruby_lexer_strings.rb +638 -0
  36. data/lib/ruby_parser.rb +2 -0
  37. data/lib/ruby_parser.yy +969 -252
  38. data/lib/ruby_parser_extras.rb +297 -116
  39. data/test/test_ruby_lexer.rb +213 -129
  40. data/test/test_ruby_parser.rb +1288 -110
  41. data/tools/munge.rb +36 -8
  42. data/tools/ripper.rb +15 -10
  43. data.tar.gz.sig +0 -0
  44. metadata +48 -35
  45. metadata.gz.sig +1 -4
data/lib/ruby_lexer.rex CHANGED
@@ -4,6 +4,11 @@
4
4
 
5
5
  class RubyLexer
6
6
 
7
+ option
8
+
9
+ lineno
10
+ column
11
+
7
12
  macro
8
13
 
9
14
  IDENT_CHAR /[a-zA-Z0-9_[:^ascii:]]/
@@ -25,7 +30,8 @@ macro
25
30
 
26
31
  start
27
32
 
28
- return process_string if lex_strterm
33
+ maybe_pop_stack
34
+ return process_string_or_heredoc if lex_strterm
29
35
 
30
36
  self.cmd_state = self.command_start
31
37
  self.command_start = false
@@ -37,18 +43,18 @@ rule
37
43
  # [:state] pattern [actions]
38
44
 
39
45
  # \s - \n + \v
40
- /[\ \t\r\f\v]/ { self.space_seen = true; next }
46
+ /[\ \t\r\f\v]+/ { self.space_seen = true; next }
41
47
 
42
48
  /\n|\#/ process_newline_or_comment
43
49
 
44
50
  /[\]\)\}]/ process_brace_close
45
51
 
46
52
  : /\!/
47
- | is_after_operator? /\!\@/ { result EXPR_ARG, :tUBANG, "!@" }
53
+ | is_after_operator? /\!\@/ { result EXPR_ARG, TOKENS[text], text }
48
54
  | /\![=~]?/ { result :arg_state, TOKENS[text], text }
49
55
 
50
56
  : /\./
51
- | /\.\.\.?/ { result EXPR_BEG, TOKENS[text], text }
57
+ | /\.\.\.?/ process_dots
52
58
  | /\.\d/ { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
53
59
  | /\./ { self.lex_state = EXPR_BEG; result EXPR_DOT, :tDOT, "." }
54
60
 
@@ -63,7 +69,7 @@ rule
63
69
 
64
70
  ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
65
71
  /\"(#{SIMPLE_STRING})\"/o process_simple_string
66
- /\"/ { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
72
+ /\"/ { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
67
73
 
68
74
  /\@\@?\d/ { rb_compile_error "`#{text}` is not allowed as a variable name" }
69
75
  /\@\@?#{IDENT_CHAR}+/o process_ivar
@@ -75,7 +81,7 @@ ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
75
81
  | /\:\:/ process_colon2
76
82
  | /\:/ process_colon1
77
83
 
78
- /->/ { result EXPR_ENDFN, :tLAMBDA, nil }
84
+ /->/ { result EXPR_ENDFN, :tLAMBDA, text }
79
85
 
80
86
  /[+-]/ process_plus_minus
81
87
 
@@ -94,6 +100,7 @@ ruby22_label? /\"#{SIMPLE_STRING}\":/o process_label
94
100
  /\[/ process_square_bracket
95
101
 
96
102
  was_label? /\'#{SSTRING}\':?/o process_label_or_string
103
+ /\'/ { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
97
104
 
98
105
  : /\|/
99
106
  | /\|\|\=/ { result EXPR_BEG, :tOP_ASGN, "||" }
@@ -105,9 +112,9 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
105
112
 
106
113
  : /\*/
107
114
  | /\*\*=/ { result EXPR_BEG, :tOP_ASGN, "**" }
108
- | /\*\*/ { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
109
- | /\*\=/ { result(EXPR_BEG, :tOP_ASGN, "*") }
110
- | /\*/ { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
115
+ | /\*\*/ { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
116
+ | /\*\=/ { result EXPR_BEG, :tOP_ASGN, "*" }
117
+ | /\*/ { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
111
118
 
112
119
  # TODO: fix result+process_lchevron to set command_start = true
113
120
  : /</
@@ -124,30 +131,30 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
124
131
  | /\>/ { result :arg_state, :tGT, ">" }
125
132
 
126
133
  : /\`/
127
- | expr_fname? /\`/ { result(EXPR_END, :tBACK_REF2, "`") }
134
+ | expr_fname? /\`/ { result EXPR_END, :tBACK_REF2, "`" }
128
135
  | expr_dot? /\`/ { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
129
- | /\`/ { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
136
+ | /\`/ { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
130
137
 
131
138
  /\?/ process_questionmark
132
139
 
133
140
  : /&/
134
- | /\&\&\=/ { result(EXPR_BEG, :tOP_ASGN, "&&") }
135
- | /\&\&/ { result(EXPR_BEG, :tANDOP, "&&") }
136
- | /\&\=/ { result(EXPR_BEG, :tOP_ASGN, "&" ) }
137
- | /\&\./ { result(EXPR_DOT, :tLONELY, "&.") }
141
+ | /\&\&\=/ { result EXPR_BEG, :tOP_ASGN, "&&" }
142
+ | /\&\&/ { result EXPR_BEG, :tANDOP, "&&" }
143
+ | /\&\=/ { result EXPR_BEG, :tOP_ASGN, "&" }
144
+ | /\&\./ { result EXPR_DOT, :tLONELY, "&." }
138
145
  | /\&/ process_amper
139
146
 
140
147
  /\// process_slash
141
148
 
142
149
  : /\^/
143
- | /\^=/ { result(EXPR_BEG, :tOP_ASGN, "^") }
144
- | /\^/ { result(:arg_state, :tCARET, "^") }
150
+ | /\^=/ { result EXPR_BEG, :tOP_ASGN, "^" }
151
+ | /\^/ { result :arg_state, :tCARET, "^" }
145
152
 
146
- /\;/ { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
153
+ /\;/ { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
147
154
 
148
155
  : /~/
149
- | is_after_operator? /\~@/ { result(:arg_state, :tTILDE, "~") }
150
- | /\~/ { result(:arg_state, :tTILDE, "~") }
156
+ | is_after_operator? /\~@/ { result :arg_state, :tTILDE, "~" }
157
+ | /\~/ { result :arg_state, :tTILDE, "~" }
151
158
 
152
159
  : /\\/
153
160
  | /\\\r?\n/ { self.lineno += 1; self.space_seen = true; next }
@@ -165,7 +172,7 @@ was_label? /\'#{SSTRING}\':?/o process_label_or_string
165
172
  | /\$([1-9]\d*)/ process_nthref
166
173
  | /\$0/ process_gvar
167
174
  | /\$#{IDENT_CHAR}+/ process_gvar
168
- | /\$\W|\$\z/ process_gvar_oddity
175
+ | /\$\W/ process_gvar_oddity
169
176
 
170
177
  /\_/ process_underscore
171
178
 
data/lib/ruby_lexer.rex.rb CHANGED
@@ -1,7 +1,8 @@
1
+ # frozen_string_literal: true
1
2
  # encoding: UTF-8
2
3
  #--
3
4
  # This file is automatically generated. Do not modify it.
4
- # Generated by: oedipus_lex version 2.5.2.
5
+ # Generated by: oedipus_lex version 2.6.0.
5
6
  # Source: lib/ruby_lexer.rex
6
7
  #++
7
8
 
@@ -35,6 +36,10 @@ class RubyLexer
35
36
  class ScanError < LexerError ; end
36
37
  # :startdoc:
37
38
 
39
+ ##
40
+ # The current line number.
41
+
42
+ attr_accessor :lineno
38
43
  ##
39
44
  # The file name / path
40
45
 
@@ -68,6 +73,24 @@ class RubyLexer
68
73
  yield
69
74
  end
70
75
 
76
+ ##
77
+ # The previous position. Only available if the :column option is on.
78
+
79
+ attr_accessor :old_pos
80
+
81
+ ##
82
+ # The position of the start of the current line. Only available if the
83
+ # :column option is on.
84
+
85
+ attr_accessor :start_of_current_line_pos
86
+
87
+ ##
88
+ # The current column, starting at 0. Only available if the
89
+ # :column option is on.
90
+ def column
91
+ old_pos - start_of_current_line_pos
92
+ end
93
+
71
94
 
72
95
  ##
73
96
  # The current scanner class. Must be overridden in subclasses.
@@ -81,6 +104,8 @@ class RubyLexer
81
104
 
82
105
  def parse str
83
106
  self.ss = scanner_class.new str
107
+ self.lineno = 1
108
+ self.start_of_current_line_pos = 0
84
109
  self.state ||= nil
85
110
 
86
111
  do_parse
@@ -102,6 +127,8 @@ class RubyLexer
102
127
  def location
103
128
  [
104
129
  (filename || "<input>"),
130
+ lineno,
131
+ column,
105
132
  ].compact.join(":")
106
133
  end
107
134
 
@@ -109,7 +136,8 @@ class RubyLexer
109
136
  # Lex the next token.
110
137
 
111
138
  def next_token
112
- return process_string if lex_strterm
139
+ maybe_pop_stack
140
+ return process_string_or_heredoc if lex_strterm
113
141
  self.cmd_state = self.command_start
114
142
  self.command_start = false
115
143
  self.space_seen = false # TODO: rename token_seen?
@@ -118,11 +146,17 @@ class RubyLexer
118
146
  token = nil
119
147
 
120
148
  until ss.eos? or token do
149
+ if ss.check(/\n/) then
150
+ self.lineno += 1
151
+ # line starts 1 position after the newline
152
+ self.start_of_current_line_pos = ss.pos + 1
153
+ end
154
+ self.old_pos = ss.pos
121
155
  token =
122
156
  case state
123
157
  when nil then
124
158
  case
125
- when ss.skip(/[\ \t\r\f\v]/) then
159
+ when ss.skip(/[\ \t\r\f\v]+/) then
126
160
  action { self.space_seen = true; next }
127
161
  when text = ss.scan(/\n|\#/) then
128
162
  process_newline_or_comment text
@@ -130,15 +164,15 @@ class RubyLexer
130
164
  process_brace_close text
131
165
  when ss.match?(/\!/) then
132
166
  case
133
- when is_after_operator? && (ss.skip(/\!\@/)) then
134
- action { result EXPR_ARG, :tUBANG, "!@" }
167
+ when is_after_operator? && (text = ss.scan(/\!\@/)) then
168
+ action { result EXPR_ARG, TOKENS[text], text }
135
169
  when text = ss.scan(/\![=~]?/) then
136
170
  action { result :arg_state, TOKENS[text], text }
137
171
  end # group /\!/
138
172
  when ss.match?(/\./) then
139
173
  case
140
174
  when text = ss.scan(/\.\.\.?/) then
141
- action { result EXPR_BEG, TOKENS[text], text }
175
+ process_dots text
142
176
  when ss.skip(/\.\d/) then
143
177
  action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
144
178
  when ss.skip(/\./) then
@@ -162,7 +196,7 @@ class RubyLexer
162
196
  when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
163
197
  process_simple_string text
164
198
  when text = ss.scan(/\"/) then
165
- action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
199
+ action { string STR_DQUOTE, '"'; result nil, :tSTRING_BEG, text }
166
200
  when text = ss.scan(/\@\@?\d/) then
167
201
  action { rb_compile_error "`#{text}` is not allowed as a variable name" }
168
202
  when text = ss.scan(/\@\@?#{IDENT_CHAR}+/o) then
@@ -180,8 +214,8 @@ class RubyLexer
180
214
  when text = ss.scan(/\:/) then
181
215
  process_colon1 text
182
216
  end # group /:/
183
- when ss.skip(/->/) then
184
- action { result EXPR_ENDFN, :tLAMBDA, nil }
217
+ when text = ss.scan(/->/) then
218
+ action { result EXPR_ENDFN, :tLAMBDA, text }
185
219
  when text = ss.scan(/[+-]/) then
186
220
  process_plus_minus text
187
221
  when ss.match?(/[+\d]/) then
@@ -211,6 +245,8 @@ class RubyLexer
211
245
  process_square_bracket text
212
246
  when was_label? && (text = ss.scan(/\'#{SSTRING}\':?/o)) then
213
247
  process_label_or_string text
248
+ when text = ss.scan(/\'/) then
249
+ action { string STR_SQUOTE, "'"; result nil, :tSTRING_BEG, text }
214
250
  when ss.match?(/\|/) then
215
251
  case
216
252
  when ss.skip(/\|\|\=/) then
@@ -229,11 +265,11 @@ class RubyLexer
229
265
  when ss.skip(/\*\*=/) then
230
266
  action { result EXPR_BEG, :tOP_ASGN, "**" }
231
267
  when ss.skip(/\*\*/) then
232
- action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
268
+ action { result :arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**" }
233
269
  when ss.skip(/\*\=/) then
234
- action { result(EXPR_BEG, :tOP_ASGN, "*") }
270
+ action { result EXPR_BEG, :tOP_ASGN, "*" }
235
271
  when ss.skip(/\*/) then
236
- action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
272
+ action { result :arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*" }
237
273
  end # group /\*/
238
274
  when ss.match?(/</) then
239
275
  case
@@ -262,24 +298,24 @@ class RubyLexer
262
298
  when ss.match?(/\`/) then
263
299
  case
264
300
  when expr_fname? && (ss.skip(/\`/)) then
265
- action { result(EXPR_END, :tBACK_REF2, "`") }
301
+ action { result EXPR_END, :tBACK_REF2, "`" }
266
302
  when expr_dot? && (ss.skip(/\`/)) then
267
303
  action { result((cmd_state ? EXPR_CMDARG : EXPR_ARG), :tBACK_REF2, "`") }
268
304
  when ss.skip(/\`/) then
269
- action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
305
+ action { string STR_XQUOTE, '`'; result nil, :tXSTRING_BEG, "`" }
270
306
  end # group /\`/
271
307
  when text = ss.scan(/\?/) then
272
308
  process_questionmark text
273
309
  when ss.match?(/&/) then
274
310
  case
275
311
  when ss.skip(/\&\&\=/) then
276
- action { result(EXPR_BEG, :tOP_ASGN, "&&") }
312
+ action { result EXPR_BEG, :tOP_ASGN, "&&" }
277
313
  when ss.skip(/\&\&/) then
278
- action { result(EXPR_BEG, :tANDOP, "&&") }
314
+ action { result EXPR_BEG, :tANDOP, "&&" }
279
315
  when ss.skip(/\&\=/) then
280
- action { result(EXPR_BEG, :tOP_ASGN, "&" ) }
316
+ action { result EXPR_BEG, :tOP_ASGN, "&" }
281
317
  when ss.skip(/\&\./) then
282
- action { result(EXPR_DOT, :tLONELY, "&.") }
318
+ action { result EXPR_DOT, :tLONELY, "&." }
283
319
  when text = ss.scan(/\&/) then
284
320
  process_amper text
285
321
  end # group /&/
@@ -288,18 +324,18 @@ class RubyLexer
288
324
  when ss.match?(/\^/) then
289
325
  case
290
326
  when ss.skip(/\^=/) then
291
- action { result(EXPR_BEG, :tOP_ASGN, "^") }
327
+ action { result EXPR_BEG, :tOP_ASGN, "^" }
292
328
  when ss.skip(/\^/) then
293
- action { result(:arg_state, :tCARET, "^") }
329
+ action { result :arg_state, :tCARET, "^" }
294
330
  end # group /\^/
295
331
  when ss.skip(/\;/) then
296
- action { self.command_start = true; result(EXPR_BEG, :tSEMI, ";") }
332
+ action { self.command_start = true; result EXPR_BEG, :tSEMI, ";" }
297
333
  when ss.match?(/~/) then
298
334
  case
299
335
  when is_after_operator? && (ss.skip(/\~@/)) then
300
- action { result(:arg_state, :tTILDE, "~") }
336
+ action { result :arg_state, :tTILDE, "~" }
301
337
  when ss.skip(/\~/) then
302
- action { result(:arg_state, :tTILDE, "~") }
338
+ action { result :arg_state, :tTILDE, "~" }
303
339
  end # group /~/
304
340
  when ss.match?(/\\/) then
305
341
  case
@@ -330,7 +366,7 @@ class RubyLexer
330
366
  process_gvar text
331
367
  when text = ss.scan(/\$#{IDENT_CHAR}+/) then
332
368
  process_gvar text
333
- when text = ss.scan(/\$\W|\$\z/) then
369
+ when text = ss.scan(/\$\W/) then
334
370
  process_gvar_oddity text
335
371
  end # group /\$/
336
372
  when text = ss.scan(/\_/) then