ruby_parser 3.12.0 → 3.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.autotest +18 -29
  4. data/History.rdoc +283 -0
  5. data/Manifest.txt +12 -4
  6. data/README.rdoc +4 -3
  7. data/Rakefile +189 -51
  8. data/bin/ruby_parse +3 -1
  9. data/bin/ruby_parse_extract_error +19 -36
  10. data/compare/normalize.rb +76 -4
  11. data/debugging.md +190 -0
  12. data/gauntlet.md +106 -0
  13. data/lib/rp_extensions.rb +14 -42
  14. data/lib/rp_stringscanner.rb +20 -51
  15. data/lib/ruby20_parser.rb +4659 -4218
  16. data/lib/ruby20_parser.y +953 -602
  17. data/lib/ruby21_parser.rb +4723 -4308
  18. data/lib/ruby21_parser.y +956 -605
  19. data/lib/ruby22_parser.rb +4762 -4337
  20. data/lib/ruby22_parser.y +960 -612
  21. data/lib/ruby23_parser.rb +4761 -4342
  22. data/lib/ruby23_parser.y +961 -613
  23. data/lib/ruby24_parser.rb +4791 -4341
  24. data/lib/ruby24_parser.y +968 -612
  25. data/lib/ruby25_parser.rb +4791 -4341
  26. data/lib/ruby25_parser.y +968 -612
  27. data/lib/ruby26_parser.rb +7287 -0
  28. data/lib/ruby26_parser.y +2749 -0
  29. data/lib/ruby27_parser.rb +8517 -0
  30. data/lib/ruby27_parser.y +3346 -0
  31. data/lib/ruby30_parser.rb +8751 -0
  32. data/lib/ruby30_parser.y +3472 -0
  33. data/lib/ruby3_parser.yy +3476 -0
  34. data/lib/ruby_lexer.rb +611 -826
  35. data/lib/ruby_lexer.rex +48 -40
  36. data/lib/ruby_lexer.rex.rb +122 -46
  37. data/lib/ruby_lexer_strings.rb +638 -0
  38. data/lib/ruby_parser.rb +38 -34
  39. data/lib/ruby_parser.yy +1710 -704
  40. data/lib/ruby_parser_extras.rb +987 -553
  41. data/test/test_ruby_lexer.rb +1718 -1539
  42. data/test/test_ruby_parser.rb +3957 -2164
  43. data/test/test_ruby_parser_extras.rb +39 -4
  44. data/tools/munge.rb +250 -0
  45. data/tools/ripper.rb +44 -0
  46. data.tar.gz.sig +0 -0
  47. metadata +68 -47
  48. metadata.gz.sig +0 -0
  49. data/lib/ruby18_parser.rb +0 -5793
  50. data/lib/ruby18_parser.y +0 -1908
  51. data/lib/ruby19_parser.rb +0 -6185
  52. data/lib/ruby19_parser.y +0 -2116
@@ -1,4 +1,6 @@
1
1
  # encoding: ASCII-8BIT
2
+ # frozen_string_literal: true
3
+ # TODO: remove encoding comment
2
4
 
3
5
  require "sexp"
4
6
  require "ruby_lexer"
@@ -6,12 +8,50 @@ require "timeout"
6
8
  require "rp_extensions"
7
9
  require "rp_stringscanner"
8
10
 
11
+ class Sexp
12
+ def check_line_numbers
13
+ raise "bad nil line for:\n%s" % [self.pretty_inspect] if nil_line?
14
+ raise "bad line number for:\n%s" % [self.pretty_inspect] unless
15
+ Integer === self.line &&
16
+ self.line >= 1 &&
17
+ self.line <= self.line_min
18
+ end
19
+
20
+ ##
21
+ # Returns the maximum line number of the children of self.
22
+
23
+ def line_min
24
+ @line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
25
+ end
26
+
27
+ def nil_line?
28
+ self.deep_each.map(&:line).any?(&:nil?)
29
+ end
30
+ end
31
+
9
32
  module RubyParserStuff
10
- VERSION = "3.12.0"
33
+ VERSION = "3.18.1"
11
34
 
12
35
  attr_accessor :lexer, :in_def, :in_single, :file
36
+ attr_accessor :in_kwarg
13
37
  attr_reader :env, :comments
14
38
 
39
+ ##
40
+ # Canonicalize conditionals. Eg:
41
+ #
42
+ # not x ? a : b
43
+ #
44
+ # becomes:
45
+ #
46
+ # x ? b : a
47
+
48
+ attr_accessor :canonicalize_conditions
49
+
50
+ ##
51
+ # The last token type returned from #next_token
52
+
53
+ attr_accessor :last_token_type
54
+
15
55
  $good20 = []
16
56
 
17
57
  %w[
@@ -30,6 +70,28 @@ module RubyParserStuff
30
70
  end
31
71
  end
32
72
 
73
+ ##
74
+ # for pure ruby systems only
75
+
76
+ def do_parse
77
+ _racc_do_parse_rb(_racc_setup, false)
78
+ end if ENV["PURE_RUBY"] || ENV["CHECK_LINE_NUMS"]
79
+
80
+ if ENV["CHECK_LINE_NUMS"] then
81
+ def _racc_do_reduce arg, act
82
+ x = super
83
+
84
+ @racc_vstack.grep(Sexp).each do |sexp|
85
+ sexp.check_line_numbers
86
+ end
87
+ x
88
+ end
89
+ end
90
+
91
+ ARG_TYPES = [:arglist, :call_args, :array, :args].map { |k|
92
+ [k, true]
93
+ }.to_h
94
+
33
95
  has_enc = "".respond_to? :encoding
34
96
 
35
97
  # This is in sorted order of occurrence according to
@@ -47,113 +109,85 @@ module RubyParserStuff
47
109
  Encoding::EUC_JP
48
110
  ] if has_enc
49
111
 
50
- def syntax_error msg
51
- raise RubyParser::SyntaxError, msg
52
- end
112
+ JUMP_TYPE = [:return, :next, :break, :yield].map { |k| [k, true] }.to_h
53
113
 
54
- def arg_blk_pass node1, node2 # TODO: nuke
55
- node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.sexp_type
56
- node1 << node2 if node2
57
- node1
58
- end
114
+ TAB_WIDTH = 8
59
115
 
60
- def arg_concat node1, node2 # TODO: nuke
61
- raise "huh" unless node2
62
- node1 << s(:splat, node2).compact
63
- node1
64
- end
65
-
66
- def clean_mlhs sexp
67
- case sexp.sexp_type
68
- when :masgn then
69
- if sexp.size == 2 and sexp[1].sexp_type == :array then
70
- s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
71
- else
72
- debug20 5
73
- sexp
74
- end
75
- when :gasgn, :iasgn, :lasgn, :cvasgn then
76
- if sexp.size == 2 then
77
- sexp.last
78
- else
79
- debug20 7
80
- sexp # optional value
81
- end
82
- else
83
- raise "unsupported type: #{sexp.inspect}"
84
- end
85
- end
116
+ def initialize(options = {})
117
+ super()
86
118
 
87
- def block_var *args
88
- result = self.args args
89
- result.sexp_type = :masgn
90
- result
91
- end
119
+ v = self.class.name[/[23]\d/]
120
+ raise "Bad Class name #{self.class}" unless v
92
121
 
93
- def block_var18 ary, splat, block
94
- ary ||= s(:array)
122
+ self.lexer = RubyLexer.new v && v.to_i
123
+ self.lexer.parser = self
124
+ self.in_kwarg = false
95
125
 
96
- if splat then
97
- splat = splat[1] unless Symbol === splat
98
- ary << "*#{splat}".to_sym
99
- end
126
+ @env = RubyParserStuff::Environment.new
127
+ @comments = []
100
128
 
101
- ary << "&#{block[1]}".to_sym if block
129
+ @canonicalize_conditions = true
102
130
 
103
- if ary.length > 2 or ary.splat then # HACK
104
- s(:masgn, *ary.sexp_body)
105
- else
106
- ary.last
107
- end
131
+ self.reset
108
132
  end
109
133
 
110
- def array_to_hash array
111
- case array.sexp_type
112
- when :kwsplat then
113
- array
114
- else
115
- s(:hash, *array.sexp_body)
116
- end
117
- end
134
+ def arg_concat node1, node2 # TODO: nuke
135
+ raise "huh" unless node2
118
136
 
119
- def call_args args
120
- result = s(:call_args)
137
+ splat = s(:splat, node2)
138
+ splat.line node2.line
121
139
 
122
- args.each do |arg|
123
- case arg
124
- when Sexp then
125
- case arg.sexp_type
126
- when :array, :args, :call_args then # HACK? remove array at some point
127
- result.concat arg.sexp_body
128
- else
129
- result << arg
130
- end
131
- when Symbol then
132
- result << arg
133
- when ",", nil then
134
- # ignore
135
- else
136
- raise "unhandled: #{arg.inspect} in #{args.inspect}"
137
- end
138
- end
140
+ node1 << splat
141
+ end
139
142
 
140
- result
143
+ def argl x
144
+ x = s(:arglist, x) if x and x.sexp_type == :array
145
+ x
141
146
  end
142
147
 
143
148
  def args args
144
149
  result = s(:args)
145
150
 
151
+ ss = args.grep Sexp
152
+ if ss.empty? then
153
+ result.line lexer.lineno
154
+ else
155
+ result.line ss.first.line
156
+ end
157
+
146
158
  args.each do |arg|
159
+ if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
160
+ arg = arg.first
161
+ end
162
+
147
163
  case arg
148
164
  when Sexp then
149
165
  case arg.sexp_type
150
166
  when :args, :block, :array, :call_args then # HACK call_args mismatch
151
- result.concat arg.sexp_body
167
+ rest = arg.sexp_body
168
+
169
+ rest.map! { |x|
170
+ if x.instance_of? Array and x.size == 2 and Numeric === x.last then
171
+ x.first
172
+ else
173
+ x
174
+ end
175
+ }
176
+
177
+ result.concat rest
178
+ when :forward_args then
179
+ self.env[:*] = :lvar # TODO: arg_var(p, idFWD_REST) ?
180
+ self.env[:**] = :lvar
181
+ self.env[:&] = :lvar
182
+
183
+ result << arg
152
184
  when :block_arg then
153
185
  result << :"&#{arg.last}"
154
186
  when :shadow then
187
+ name = arg.last
188
+ self.env[name] = :lvar
155
189
  if Sexp === result.last and result.last.sexp_type == :shadow then
156
- result.last << arg.last
190
+ result.last << name
157
191
  else
158
192
  result << arg
159
193
  end
@@ -166,6 +200,8 @@ module RubyParserStuff
166
200
  name = arg.to_s.delete("&*")
167
201
  self.env[name.to_sym] = :lvar unless name.empty?
168
202
  result << arg
203
+ when true, false then
204
+ self.in_kwarg = arg
169
205
  when ",", "|", ";", "(", ")", nil then
170
206
  # ignore
171
207
  else
@@ -176,21 +212,46 @@ module RubyParserStuff
176
212
  result
177
213
  end
178
214
 
215
+ def end_args args
216
+ lexer.lex_state = RubyLexer::State::Values::EXPR_BEG
217
+ lexer.command_start = true
218
+ self.args args
219
+ end
220
+
221
+ def endless_method_name defn_or_defs
222
+ name = defn_or_defs[1]
223
+ name = defn_or_defs[2] unless Symbol === name
224
+
225
+ if name.end_with? "=" then
226
+ yyerror "setter method cannot be defined in an endless method definition"
227
+ end
228
+
229
+ # TODO? token_info_drop(p, "def", loc->beg_pos);
230
+ end
231
+
232
+ def array_to_hash array
233
+ case array.sexp_type
234
+ when :kwsplat then
235
+ array
236
+ else
237
+ s(:hash, *array.sexp_body).line array.line
238
+ end
239
+ end
240
+
179
241
  def aryset receiver, index
180
242
  index ||= s()
181
- s(:attrasgn, receiver, :"[]=", *index.sexp_body).compact # [].sexp_body => nil
243
+ l = receiver.line
244
+ result = s(:attrasgn, receiver, :"[]=",
245
+ *index.sexp_body).compact # [].sexp_body => nil
246
+ result.line = l
247
+ result
182
248
  end
183
249
 
184
250
  def assignable(lhs, value = nil)
185
- id = lhs.to_sym unless Sexp === lhs
186
- id = id.to_sym if Sexp === id
187
-
188
- raise "write a test 1" if id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
251
+ id, line = lhs
252
+ id = id.to_sym
189
253
 
190
- raise SyntaxError, "Can't change the value of #{id}" if
191
- id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
192
-
193
- result = case id.to_s
254
+ result = case id
194
255
  when /^@@/ then
195
256
  asgn = in_def || in_single > 0
196
257
  s((asgn ? :cvasgn : :cvdecl), id)
@@ -212,8 +273,22 @@ module RubyParserStuff
212
273
  self.env[id] ||= :lvar if result.sexp_type == :lasgn
213
274
 
214
275
  result << value if value
276
+ result.line line
277
+ result
278
+ end
215
279
 
216
- return result
280
+ def backref_assign_error ref
281
+ # TODO: need a test for this... obviously
282
+ case ref.sexp_type
283
+ when :nth_ref then
284
+ raise "write a test 2"
285
+ raise SyntaxError, "Can't set variable %p" % ref.last
286
+ when :back_ref then
287
+ raise "write a test 3"
288
+ raise SyntaxError, "Can't set back reference %p" % ref.last
289
+ else
290
+ raise "Unknown backref type: #{ref.inspect}"
291
+ end
217
292
  end
218
293
 
219
294
  def block_append(head, tail)
@@ -223,12 +298,82 @@ module RubyParserStuff
223
298
  line = [head.line, tail.line].compact.min
224
299
 
225
300
  head = remove_begin(head)
226
- head = s(:block, head) unless head.node_type == :block
301
+ head = s(:block, head).line(line) unless head.sexp_type == :block
227
302
 
228
- head.line = line
303
+ # head.line = line
229
304
  head << tail
230
305
  end
231
306
 
307
+ def block_dup_check call_or_args, block
308
+ syntax_error "Both block arg and actual block given." if
309
+ block and call_or_args.block_pass?
310
+ end
311
+
312
+ def block_var *args
313
+ result = self.args args
314
+ result.sexp_type = :masgn
315
+ result
316
+ end
317
+
318
+ def call_args args
319
+ result = s(:call_args)
320
+
321
+ a = args.grep(Sexp).first
322
+ if a then
323
+ result.line a.line
324
+ else
325
+ result.line lexer.lineno
326
+ end
327
+
328
+ args.each do |arg|
329
+ if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
330
+ arg = arg.first
331
+ end
332
+
333
+ case arg
334
+ when Sexp then
335
+ case arg.sexp_type
336
+ when :array, :args, :call_args then # HACK? remove array at some point
337
+ result.concat arg.sexp_body
338
+ else
339
+ result << arg
340
+ end
341
+ when Symbol then
342
+ result << arg
343
+ when Array then
344
+ id, _line = arg
345
+ result << id
346
+ when ",", nil, "(" then
347
+ # ignore
348
+ else
349
+ raise "unhandled: #{arg.inspect} in #{args.inspect}"
350
+ end
351
+ end
352
+
353
+ result
354
+ end
355
+
356
+ def clean_mlhs sexp
357
+ case sexp.sexp_type
358
+ when :masgn then
359
+ if sexp.size == 2 and sexp[1].sexp_type == :array then
360
+ s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
361
+ else
362
+ debug20 5
363
+ sexp
364
+ end
365
+ when :gasgn, :iasgn, :lasgn, :cvasgn then
366
+ if sexp.size == 2 then
367
+ sexp.last
368
+ else
369
+ debug20 7
370
+ sexp # optional value
371
+ end
372
+ else
373
+ raise "unsupported type: #{sexp.inspect}"
374
+ end
375
+ end
376
+
232
377
  def cond node
233
378
  return nil if node.nil?
234
379
  node = value_expr node
@@ -236,63 +381,116 @@ module RubyParserStuff
236
381
  case node.sexp_type
237
382
  when :lit then
238
383
  if Regexp === node.last then
239
- return s(:match, node)
384
+ s(:match, node)
240
385
  else
241
- return node
386
+ node
242
387
  end
243
388
  when :and then
244
- return s(:and, cond(node[1]), cond(node[2]))
389
+ _, lhs, rhs = node
390
+ s(:and, cond(lhs), cond(rhs))
245
391
  when :or then
246
- return s(:or, cond(node[1]), cond(node[2]))
392
+ _, lhs, rhs = node
393
+ s(:or, cond(lhs), cond(rhs))
247
394
  when :dot2 then
248
395
  label = "flip#{node.hash}"
249
396
  env[label] = :lvar
250
397
  _, lhs, rhs = node
251
- return s(:flip2, lhs, rhs)
398
+ s(:flip2, lhs, rhs) # TODO: recurse?
252
399
  when :dot3 then
253
400
  label = "flip#{node.hash}"
254
401
  env[label] = :lvar
255
402
  _, lhs, rhs = node
256
- return s(:flip3, lhs, rhs)
403
+ s(:flip3, lhs, rhs)
257
404
  else
258
- return node
259
- end
405
+ node
406
+ end.line node.line
260
407
  end
261
408
 
262
- ##
263
- # for pure ruby systems only
409
+ def dedent sexp
410
+ dedent_count = dedent_size sexp
264
411
 
265
- def do_parse
266
- _racc_do_parse_rb(_racc_setup, false)
267
- end if ENV['PURE_RUBY']
412
+ skip_one = false
413
+ sexp.map { |obj|
414
+ case obj
415
+ when Symbol then
416
+ obj
417
+ when String then
418
+ obj.lines.map { |l| remove_whitespace_width l, dedent_count }.join
419
+ when Sexp then
420
+ case obj.sexp_type
421
+ when :evstr then
422
+ skip_one = true
423
+ obj
424
+ when :str then
425
+ _, str = obj
426
+ str = if skip_one then
427
+ skip_one = false
428
+ s1, *rest = str.lines
429
+ s1 + rest.map { |l| remove_whitespace_width l, dedent_count }.join
430
+ else
431
+ str.lines.map { |l| remove_whitespace_width l, dedent_count }.join
432
+ end
268
433
 
269
- def new_match lhs, rhs
270
- if lhs then
271
- case lhs.sexp_type
272
- when :dregx, :dregx_once then
273
- return s(:match2, lhs, rhs).line(lhs.line)
274
- when :lit then
275
- return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
434
+ s(:str, str).line obj.line
435
+ else
436
+ warn "unprocessed sexp %p" % [obj]
437
+ end
438
+ else
439
+ warn "unprocessed: %p" % [obj]
276
440
  end
277
- end
441
+ }
442
+ end
278
443
 
279
- if rhs then
280
- case rhs.sexp_type
281
- when :dregx, :dregx_once then
282
- return s(:match3, rhs, lhs).line(lhs.line)
283
- when :lit then
284
- return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
444
+ def dedent_size sexp
445
+ skip_one = false
446
+ sexp.flat_map { |s|
447
+ case s
448
+ when Symbol then
449
+ next
450
+ when String then
451
+ s.lines
452
+ when Sexp then
453
+ case s.sexp_type
454
+ when :evstr then
455
+ skip_one = true
456
+ next
457
+ when :str then
458
+ _, str = s
459
+ lines = str.lines
460
+ if skip_one then
461
+ skip_one = false
462
+ lines.shift
463
+ end
464
+ lines
465
+ else
466
+ warn "unprocessed sexp %p" % [s]
467
+ end
468
+ else
469
+ warn "unprocessed: %p" % [s]
470
+ end.map { |l| whitespace_width l.chomp }
471
+ }.compact.min
472
+ end
473
+
474
+ def dedent_string string, width
475
+ characters_skipped = 0
476
+ indentation_skipped = 0
477
+
478
+ string.chars.each do |char|
479
+ break if indentation_skipped >= width
480
+ if char == " "
481
+ characters_skipped += 1
482
+ indentation_skipped += 1
483
+ elsif char == "\t"
484
+ proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1)
485
+ break if proposed > width
486
+ characters_skipped += 1
487
+ indentation_skipped = proposed
285
488
  end
286
489
  end
287
-
288
- return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
490
+ string[characters_skipped..-1]
289
491
  end
290
492
 
291
- # TODO: remove in 4.0 or 2018-01, whichever is first
292
- deprecate :get_match_node, :new_match
293
-
294
493
  def gettable(id)
295
- lineno = id.lineno if id.respond_to? :lineno
296
494
  id = id.to_sym if String === id
297
495
 
298
496
  result = case id.to_s
@@ -313,38 +511,99 @@ module RubyParserStuff
313
511
  end
314
512
  end
315
513
 
316
- result.line lineno if lineno
317
-
318
514
  raise "identifier #{id.inspect} is not valid" unless result
319
515
 
320
516
  result
321
517
  end
322
518
 
519
+ def hack_encoding str, extra = nil
520
+ encodings = ENCODING_ORDER.dup
521
+ encodings.unshift(extra) unless extra.nil?
522
+
523
+ # terrible, horrible, no good, very bad, last ditch effort.
524
+ encodings.each do |enc|
525
+ begin
526
+ str.force_encoding enc
527
+ if str.valid_encoding? then
528
+ str.encode! Encoding::UTF_8
529
+ break
530
+ end
531
+ rescue ArgumentError # unknown encoding name
532
+ # do nothing
533
+ rescue Encoding::InvalidByteSequenceError
534
+ # do nothing
535
+ rescue Encoding::UndefinedConversionError
536
+ # do nothing
537
+ end
538
+ end
539
+
540
+ # no amount of pain is enough for you.
541
+ raise "Bad encoding. Need a magic encoding comment." unless
542
+ str.encoding.name == "UTF-8"
543
+ end
544
+
323
545
  ##
324
- # Canonicalize conditionals. Eg:
325
- #
326
- # not x ? a : b
546
+ # Returns a UTF-8 encoded string after processing BOMs and magic
547
+ # encoding comments.
327
548
  #
328
- # becomes:
549
+ # Holy crap... ok. Here goes:
329
550
  #
330
- # x ? b : a
551
+ # Ruby's file handling and encoding support is insane. We need to be
552
+ # able to lex a file. The lexer file is explicitly UTF-8 to make
553
+ # things cleaner. This allows us to deal with extended chars in
554
+ # class and method names. In order to do this, we need to encode all
555
+ # input source files as UTF-8. First, we look for a UTF-8 BOM by
556
+ # looking at the first line while forcing its encoding to
557
+ # ASCII-8BIT. If we find a BOM, we strip it and set the expected
558
+ # encoding to UTF-8. Then, we search for a magic encoding comment.
559
+ # If found, it overrides the BOM. Finally, we force the encoding of
560
+ # the input string to whatever was found, and then encode that to
561
+ # UTF-8 for compatibility with the lexer.
331
562
 
332
- attr_accessor :canonicalize_conditions
563
+ def handle_encoding str
564
+ str = str.dup
565
+ has_enc = str.respond_to? :encoding
566
+ encoding = nil
333
567
 
334
- def initialize(options = {})
335
- super()
568
+ header = str.each_line.first(2)
569
+ header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
336
570
 
337
- v = self.class.name[/1[89]|2[01]/]
571
+ first = header.first || ""
572
+ encoding, str = +"utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
338
573
 
339
- self.lexer = RubyLexer.new v && v.to_i
340
- self.lexer.parser = self
574
+ encoding = $1.strip if header.find { |s|
575
+ s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
576
+ s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
577
+ }
341
578
 
342
- @env = RubyParserStuff::Environment.new
343
- @comments = []
579
+ if encoding then
580
+ if has_enc then
581
+ encoding.sub!(/utf-8-.+$/, "utf-8") # HACK for stupid emacs formats
582
+ hack_encoding str, encoding
583
+ else
584
+ warn "Skipping magic encoding comment"
585
+ end
586
+ else
587
+ # nothing specified... ugh. try to encode as utf-8
588
+ hack_encoding str if has_enc
589
+ end
344
590
 
345
- @canonicalize_conditions = true
591
+ str
592
+ end
346
593
 
347
- self.reset
594
+ def invert_block_call val
595
+ ret, iter = val
596
+ type, call = ret
597
+
598
+ iter.insert 1, call
599
+
600
+ ret = s(type).line ret.line
601
+
602
+ [iter, ret]
603
+ end
604
+
605
+ def inverted? val
606
+ JUMP_TYPE[val[0].sexp_type]
348
607
  end
349
608
 
350
609
  def list_append list, item # TODO: nuke me *sigh*
@@ -365,12 +624,14 @@ module RubyParserStuff
365
624
 
366
625
  htype, ttype = head.sexp_type, tail.sexp_type
367
626
 
368
- head = s(:dstr, '', head) if htype == :evstr
627
+ head = s(:dstr, "", head).line head.line if htype == :evstr
369
628
 
370
629
  case ttype
371
630
  when :str then
372
631
  if htype == :str
373
- head.last << tail.last
632
+ a, b = head.last, tail.last
633
+ b = b.dup.force_encoding a.encoding unless Encoding.compatible?(a, b)
634
+ a << b
374
635
  elsif htype == :dstr and head.size == 2 then
375
636
  head.last << tail.last
376
637
  else
@@ -384,8 +645,8 @@ module RubyParserStuff
384
645
  head.line = lineno
385
646
  else
386
647
  tail.sexp_type = :array
387
- tail[1] = s(:str, tail[1])
388
- tail.delete_at 1 if tail[1] == s(:str, '')
648
+ tail[1] = s(:str, tail[1]).line tail.line
649
+ tail.delete_at 1 if tail[1] == s(:str, "")
389
650
 
390
651
  head.push(*tail.sexp_body)
391
652
  end
@@ -411,6 +672,13 @@ module RubyParserStuff
411
672
  return head
412
673
  end
413
674
 
675
+ def local_pop in_def
676
+ lexer.cond.pop # group = local_pop
677
+ lexer.cmdarg.pop
678
+ self.env.unextend
679
+ self.in_def = in_def
680
+ end
681
+
414
682
  def logical_op type, left, right
415
683
  left = value_expr left
416
684
 
@@ -423,23 +691,135 @@ module RubyParserStuff
423
691
  node = rhs
424
692
  end
425
693
 
426
- node[2] = s(type, rhs, right)
694
+ node.pop
695
+ node << s(type, rhs, right).line(rhs.line)
427
696
 
428
697
  return left
429
698
  end
430
699
 
431
- return s(type, left, right)
700
+ result = s(type, left, right)
701
+ result.line left.line if left.line
702
+ result
432
703
  end
433
704
 
434
- # TODO: remove in 4.0 or 2018-01, whichever is first
435
- deprecate :logop, :logical_op
436
-
437
705
  def new_aref val
438
706
  val[2] ||= s(:arglist)
439
707
  val[2].sexp_type = :arglist if val[2].sexp_type == :array # REFACTOR
440
708
  new_call val[0], :"[]", val[2]
441
709
  end
442
710
 
711
+ def new_arg val
712
+ arg, = val
713
+
714
+ case arg
715
+ when Symbol then
716
+ result = s(:args, arg).line line
717
+ when Sexp then
718
+ result = arg
719
+ when Array then
720
+ (arg, line), = val
721
+ result = s(:args, arg).line line
722
+ else
723
+ debug20 32
724
+ raise "Unknown f_arg type: #{val.inspect}"
725
+ end
726
+
727
+ result
728
+ end
729
+
730
+ def new_array_pattern const, pre_arg, arypat, loc
731
+ result = s(:array_pat, const).line loc
732
+ result << pre_arg if pre_arg
733
+
734
+ if arypat && arypat.sexp_type == :array_TAIL then
735
+ result.concat arypat.sexp_body
736
+ else
737
+ raise "NO?: %p" % [arypat]
738
+ end
739
+
740
+ result
741
+ end
742
+
743
+ def array_pat_concat lhs, rhs
744
+ case lhs.sexp_type
745
+ when :PATTERN then
746
+ lhs.sexp_type = :array_pat
747
+ end
748
+
749
+ if rhs then
750
+ case rhs.sexp_type
751
+ when :array_pat, :array_TAIL, :PATTERN then
752
+ lhs.concat rhs.sexp_body
753
+ else
754
+ lhs << rhs
755
+ end
756
+ end
757
+ end
758
+
759
+ def new_array_pattern_tail pre_args, has_rest, rest_arg, post_args
760
+ # TODO: remove has_rest once all tests pass !!!
761
+ rest_arg = if has_rest then
762
+ :"*#{rest_arg}"
763
+ else
764
+ nil
765
+ end
766
+
767
+ result = s(:array_TAIL).line 666
768
+
769
+ array_pat_concat result, pre_args
770
+
771
+ result << rest_arg if rest_arg
772
+
773
+ array_pat_concat result, post_args
774
+
775
+ result
776
+ end
777
+
778
+ def new_assign lhs, rhs
779
+ return nil unless lhs
780
+
781
+ rhs = value_expr rhs
782
+
783
+ case lhs.sexp_type
784
+ when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
785
+ lhs << rhs
786
+ when :const then
787
+ lhs.sexp_type = :cdecl
788
+ lhs << rhs
789
+ else
790
+ raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
791
+ end
792
+
793
+ lhs
794
+ end
795
+
796
+ def new_attrasgn recv, meth, call_op = :"."
797
+ call_op = call_op.first if Array === call_op
798
+
799
+ meth = :"#{meth}="
800
+
801
+ result = case call_op.to_sym
802
+ when :"."
803
+ s(:attrasgn, recv, meth)
804
+ when :"&."
805
+ s(:safe_attrasgn, recv, meth)
806
+ else
807
+ raise "unknown call operator: `#{type.inspect}`"
808
+ end
809
+
810
+ result.line = recv.line
811
+ result
812
+ end
813
+
814
+ def new_begin val
815
+ _, lineno, body, _ = val
816
+
817
+ result = body ? s(:begin, body) : s(:nil)
818
+ result.line lineno
819
+
820
+ result
821
+ end
822
+
443
823
  def new_body val
444
824
  body, resbody, elsebody, ensurebody = val
445
825
 
@@ -463,39 +843,29 @@ module RubyParserStuff
463
843
 
464
844
  if elsebody and not resbody then
465
845
  warning("else without rescue is useless")
466
- result = s(:begin, result) if result
846
+ result = s(:begin, result).line result.line if result
467
847
  result = block_append(result, elsebody)
468
848
  end
469
849
 
470
- result = s(:ensure, result, ensurebody).compact if ensurebody
850
+ if ensurebody
851
+ lineno = (result || ensurebody).line
852
+ result = s(:ensure, result, ensurebody).compact.line lineno
853
+ end
471
854
 
472
855
  result
473
856
  end
474
857
 
475
- def argl x
476
- x = s(:arglist, x) if x and x.sexp_type == :array
477
- x
858
+ def new_brace_body args, body, lineno
859
+ new_iter(nil, args, body).line lineno
478
860
  end
479
861
 
480
- def backref_assign_error ref
481
- # TODO: need a test for this... obviously
482
- case ref.sexp_type
483
- when :nth_ref then
484
- raise "write a test 2"
485
- raise SyntaxError, "Can't set variable %p" % ref.last
486
- when :back_ref then
487
- raise "write a test 3"
488
- raise SyntaxError, "Can't set back reference %p" % ref.last
489
- else
490
- raise "Unknown backref type: #{ref.inspect}"
491
- end
492
- end
862
+ def new_call recv, meth, args = nil, call_op = :"."
863
+ call_op = call_op.first if Array === call_op
493
864
 
494
- def new_call recv, meth, args = nil, call_op = :'.'
495
865
  result = case call_op.to_sym
496
- when :'.'
866
+ when :"."
497
867
  s(:call, recv, meth)
498
- when :'&.'
868
+ when :"&."
499
869
  s(:safe_call, recv, meth)
500
870
  else
501
871
  raise "unknown call operator: `#{type.inspect}`"
@@ -505,39 +875,28 @@ module RubyParserStuff
505
875
  # TODO: need a test with f(&b) { } to produce warning
506
876
 
507
877
  if args
508
- if [:arglist, :args, :array, :call_args].include? args.sexp_type
878
+ if ARG_TYPES[args.sexp_type] then
509
879
  result.concat args.sexp_body
510
880
  else
511
881
  result << args
512
882
  end
513
883
  end
514
884
 
515
- line = result.grep(Sexp).map(&:line).compact.min
516
- result.line = line if line
885
+ # line = result.grep(Sexp).map(&:line).compact.min
886
+ result.line = recv.line if recv
887
+ result.line ||= lexer.lineno
517
888
 
518
889
  result
519
890
  end
520
891
 
521
- def new_attrasgn recv, meth, call_op
522
- meth = :"#{meth}="
523
-
524
- result = case call_op.to_sym
525
- when :'.'
526
- s(:attrasgn, recv, meth)
527
- when :'&.'
528
- s(:safe_attrasgn, recv, meth)
529
- else
530
- raise "unknown call operator: `#{type.inspect}`"
531
- end
532
-
533
- result.line = recv.line
534
- result
892
+ def new_in pat, body, cases, line
893
+ s(:in, pat, body, cases).line line
535
894
  end
536
895
 
537
896
  def new_case expr, body, line
538
897
  result = s(:case, expr)
539
898
 
540
- while body and body.node_type == :when
899
+ while body and [:when, :in].include? body.sexp_type
541
900
  result << body
542
901
  body = body.delete_at 3
543
902
  end
@@ -556,8 +915,11 @@ module RubyParserStuff
556
915
  end
557
916
 
558
917
  def new_class val
918
+ # TODO: get line from class keyword
559
919
  line, path, superclass, body = val[1], val[2], val[3], val[5]
560
920
 
921
+ path = path.first if path.instance_of? Array
922
+
561
923
  result = s(:class, path, superclass)
562
924
 
563
925
  if body then
@@ -579,44 +941,87 @@ module RubyParserStuff
579
941
  result
580
942
  end
581
943
 
944
+ def new_const_op_asgn val
945
+ lhs, (asgn_op, _), rhs = val
946
+ asgn_op = asgn_op.to_sym
947
+
948
+ result = case asgn_op
949
+ when :"||" then
950
+ s(:op_asgn_or, lhs, rhs)
951
+ when :"&&" then
952
+ s(:op_asgn_and, lhs, rhs)
953
+ else
954
+ s(:op_asgn, lhs, asgn_op, rhs)
955
+ end
956
+
957
+ result.line = lhs.line
958
+ result
959
+ end
960
+
582
961
  def new_defn val
583
- (_, line), name, _, args, body, * = val
584
- body ||= s(:nil)
962
+ _, (name, line), in_def, args, body, _ = val
585
963
 
586
- result = s(:defn, name.to_sym, args)
964
+ body ||= s(:nil).line line
587
965
 
588
- if body then
589
- if body.sexp_type == :block then
590
- result.push(*body.sexp_body)
591
- else
592
- result.push body
593
- end
966
+ args.line line
967
+
968
+ result = s(:defn, name.to_sym, args).line line
969
+
970
+ if body.sexp_type == :block then
971
+ result.push(*body.sexp_body)
972
+ else
973
+ result.push body
594
974
  end
595
975
 
596
- args.line line
597
- result.line = line
598
976
  result.comments = self.comments.pop
599
977
 
600
- result
978
+ [result, in_def]
601
979
  end
602
980
 
603
981
  def new_defs val
604
- recv, name, args, body = val[1], val[4], val[6], val[7]
605
- body ||= s(:nil)
982
+ _, recv, (name, line), in_def, args, body, _ = val
606
983
 
607
- result = s(:defs, recv, name.to_sym, args)
984
+ body ||= s(:nil).line line
608
985
 
609
- if body then
610
- if body.sexp_type == :block then
611
- result.push(*body.sexp_body)
612
- else
613
- result.push body
614
- end
986
+ args.line line
987
+
988
+ result = s(:defs, recv, name.to_sym, args).line line
989
+
990
+ # TODO: remove_begin
991
+ # TODO: reduce_nodes
992
+
993
+ if body.sexp_type == :block then
994
+ result.push(*body.sexp_body)
995
+ else
996
+ result.push body
615
997
  end
616
998
 
617
- result.line = recv.line
618
999
  result.comments = self.comments.pop
619
- result
1000
+
1001
+ [result, in_def]
1002
+ end
1003
+
1004
+ def new_do_body args, body, lineno
1005
+ new_iter(nil, args, body).line(lineno)
1006
+ end
1007
+
1008
+ def new_find_pattern const, pat
1009
+ pat.sexp_type = :find_pat
1010
+ pat.insert 1, const
1011
+ end
1012
+
1013
+ def new_find_pattern_tail lhs, mid, rhs
1014
+ lhs_id, line = lhs
1015
+ rhs_id, _line = rhs
1016
+
1017
+ # TODO: fpinfo->pre_rest_arg = pre_rest_arg ? assignable(p, pre_rest_arg, 0, loc) : NODE_SPECIAL_NO_NAME_REST;
1018
+
1019
+ lhs_id = "*#{lhs_id}".to_sym
1020
+ rhs_id = "*#{rhs_id}".to_sym
1021
+
1022
+ mid.sexp_type = :array_pat # HACK?
1023
+
1024
+ s(:find_pat_TAIL, lhs_id, mid, rhs_id).line line
620
1025
  end
621
1026
 
622
1027
  def new_for expr, var, body
@@ -626,7 +1031,49 @@ module RubyParserStuff
626
1031
  end
627
1032
 
628
1033
  def new_hash val
629
- s(:hash, *val[2].values).line(val[1])
1034
+ _, line, assocs = val
1035
+
1036
+ s(:hash).line(line).concat assocs.sexp_body
1037
+ end
1038
+
1039
+ def new_hash_pattern const, hash_pat, loc
1040
+ _, pat, kw_args, kw_rest_arg = hash_pat
1041
+
1042
+ line = (const||hash_pat).line
1043
+
1044
+ result = s(:hash_pat, const).line line
1045
+ result.concat pat.sexp_body if pat
1046
+ result << kw_args if kw_args
1047
+ result << kw_rest_arg if kw_rest_arg
1048
+ result
1049
+ end
1050
+
1051
+ def new_hash_pattern_tail kw_args, kw_rest_arg, line # TODO: remove line arg
1052
+ # kw_rest_arg = assignable(kw_rest_arg, nil).line line if kw_rest_arg
1053
+
1054
+ result = s(:hash_pat).line line
1055
+ result << kw_args
1056
+
1057
+ if kw_rest_arg then
1058
+ name = kw_rest_arg.value
1059
+ # TODO: I _hate_ this:
1060
+ assignable [name, kw_rest_arg.line] if name != :**
1061
+ result << kw_rest_arg
1062
+ end
1063
+
1064
+ result
1065
+ end
1066
+
1067
+ def push_pktbl
1068
+ end
1069
+
1070
+ def pop_pktbl
1071
+ end
1072
+
1073
+ def push_pvtbl
1074
+ end
1075
+
1076
+ def pop_pvtbl
630
1077
  end
631
1078
 
632
1079
  def new_if c, t, f
@@ -647,33 +1094,68 @@ module RubyParserStuff
647
1094
  result << args
648
1095
  result << body if body
649
1096
 
650
- args.sexp_type = :args unless args == 0
1097
+ result.line call.line if call
651
1098
 
652
- result
653
- end
1099
+ unless args == 0 then
1100
+ args.line call.line if call
1101
+ args.sexp_type = :args
1102
+ end
654
1103
 
655
- def new_masgn_arg rhs, wrap = false
656
- rhs = value_expr(rhs)
657
- rhs = s(:to_ary, rhs) if wrap # HACK: could be array if lhs isn't right
658
- rhs
1104
+ result
659
1105
  end
660
1106
 
661
1107
  def new_masgn lhs, rhs, wrap = false
662
1108
  _, ary = lhs
663
1109
 
1110
+ line = rhs.line
664
1111
  rhs = value_expr(rhs)
665
1112
  rhs = ary ? s(:to_ary, rhs) : s(:array, rhs) if wrap
1113
+ rhs.line line if wrap
1114
+
1115
+ lhs.delete_at 1 if ary.nil?
1116
+ lhs << rhs
1117
+
1118
+ lhs
1119
+ end
1120
+
1121
+ def new_masgn_arg rhs, wrap = false
1122
+ rhs = value_expr(rhs)
1123
+ # HACK: could be array if lhs isn't right
1124
+ rhs = s(:to_ary, rhs).line rhs.line if wrap
1125
+ rhs
1126
+ end
1127
+
1128
+ def new_match lhs, rhs
1129
+ if lhs then
1130
+ case lhs.sexp_type
1131
+ when :dregx, :dregx_once then
1132
+ # TODO: no test coverage
1133
+ return s(:match2, lhs, rhs).line(lhs.line)
1134
+ when :lit then
1135
+ return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
1136
+ end
1137
+ end
666
1138
 
667
- lhs.delete_at 1 if ary.nil?
668
- lhs << rhs
1139
+ if rhs then
1140
+ case rhs.sexp_type
1141
+ when :dregx, :dregx_once then
1142
+ # TODO: no test coverage
1143
+ return s(:match3, rhs, lhs).line(lhs.line)
1144
+ when :lit then
1145
+ return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
1146
+ end
1147
+ end
669
1148
 
670
- lhs
1149
+ new_call(lhs, :"=~", argl(rhs)).line lhs.line
671
1150
  end
672
1151
 
673
1152
  def new_module val
1153
+ # TODO: get line from module keyword
674
1154
  line, path, body = val[1], val[2], val[4]
675
1155
 
676
- result = s(:module, path)
1156
+ path = path.first if path.instance_of? Array
1157
+
1158
+ result = s(:module, path).line line
677
1159
 
678
1160
  if body then # REFACTOR?
679
1161
  if body.sexp_type == :block then
@@ -683,39 +1165,49 @@ module RubyParserStuff
683
1165
  end
684
1166
  end
685
1167
 
686
- result.line = line
687
1168
  result.comments = self.comments.pop
688
1169
  result
689
1170
  end
690
1171
 
691
1172
  def new_op_asgn val
692
- lhs, asgn_op, arg = val[0], val[1].to_sym, val[2]
693
- name = lhs.value
694
- arg = remove_begin(arg)
695
- result = case asgn_op # REFACTOR
1173
+ lhs, (op, _line), rhs = val
1174
+ op = op.to_sym
1175
+
1176
+ name = gettable(lhs.last).line lhs.line
1177
+ arg = remove_begin rhs
1178
+ result = case op # REFACTOR
696
1179
  when :"||" then
697
1180
  lhs << arg
698
- s(:op_asgn_or, self.gettable(name), lhs)
1181
+ s(:op_asgn_or, name, lhs).line lhs.line
699
1182
  when :"&&" then
700
1183
  lhs << arg
701
- s(:op_asgn_and, self.gettable(name), lhs)
1184
+ s(:op_asgn_and, name, lhs).line lhs.line
702
1185
  else
703
- # TODO: why [2] ?
704
- lhs[2] = new_call(self.gettable(name), asgn_op, argl(arg))
1186
+ lhs << new_call(name, op, argl(arg))
705
1187
  lhs
706
1188
  end
707
- result.line = lhs.line
1189
+
1190
+ result
1191
+ end
1192
+
1193
+ def new_op_asgn1 val
1194
+ lhs, _, args, _, (op, _), rhs = val
1195
+
1196
+ args.sexp_type = :arglist if args
1197
+
1198
+ result = s(:op_asgn1, lhs, args, op.to_sym, rhs)
1199
+ result.line lhs.line
708
1200
  result
709
1201
  end
710
1202
 
711
1203
  def new_op_asgn2 val
712
- recv, call_op, meth, op, arg = val
1204
+ recv, (call_op, _), (meth, _), (op, _), arg = val
713
1205
  meth = :"#{meth}="
714
1206
 
715
1207
  result = case call_op.to_sym
716
- when :'.'
1208
+ when :"."
717
1209
  s(:op_asgn2, recv, meth, op.to_sym, arg)
718
- when :'&.'
1210
+ when :"&."
719
1211
  s(:safe_op_asgn2, recv, meth, op.to_sym, arg)
720
1212
  else
721
1213
  raise "unknown call operator: `#{type.inspect}`"
@@ -725,27 +1217,44 @@ module RubyParserStuff
725
1217
  result
726
1218
  end
727
1219
 
1220
+ def new_qsym_list
1221
+ s(:array).line lexer.lineno
1222
+ end
1223
+
1224
+ def new_qsym_list_entry val
1225
+ _, (str, line), _ = val
1226
+ s(:lit, str.to_sym).line line
1227
+ end
1228
+
1229
+ def new_qword_list
1230
+ s(:array).line lexer.lineno
1231
+ end
1232
+
1233
+ def new_qword_list_entry val
1234
+ _, (str, line), _ = val
1235
+ str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
1236
+ s(:str, str).line line
1237
+ end
1238
+
728
1239
  def new_regexp val
729
- node = val[1] || s(:str, '')
730
- options = val[2]
1240
+ (_, line), node, (options, _) = val
1241
+
1242
+ node ||= s(:str, "").line line
731
1243
 
732
1244
  o, k = 0, nil
733
1245
  options.split(//).uniq.each do |c| # FIX: this has a better home
734
1246
  v = {
735
- 'x' => Regexp::EXTENDED,
736
- 'i' => Regexp::IGNORECASE,
737
- 'm' => Regexp::MULTILINE,
738
- 'o' => Regexp::ONCE,
739
- 'n' => Regexp::ENC_NONE,
740
- 'e' => Regexp::ENC_EUC,
741
- 's' => Regexp::ENC_SJIS,
742
- 'u' => Regexp::ENC_UTF8,
1247
+ "x" => Regexp::EXTENDED,
1248
+ "i" => Regexp::IGNORECASE,
1249
+ "m" => Regexp::MULTILINE,
1250
+ "o" => Regexp::ONCE,
1251
+ "n" => Regexp::ENC_NONE,
1252
+ "e" => Regexp::ENC_EUC,
1253
+ "s" => Regexp::ENC_SJIS,
1254
+ "u" => Regexp::ENC_UTF8,
743
1255
  }[c]
744
1256
  raise "unknown regexp option: #{c}" unless v
745
1257
  o += v
746
-
747
- # encoding options are ignored on 1.9+
748
- k = c if c =~ /[esu]/ if RUBY_VERSION < "1.9"
749
1258
  end
750
1259
 
751
1260
  case node.sexp_type
@@ -757,12 +1266,12 @@ module RubyParserStuff
757
1266
  begin
758
1267
  Regexp.new(node[1], o)
759
1268
  rescue RegexpError => e
760
- warn "WA\RNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
1269
+ warn "WARNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
761
1270
  begin
762
- warn "WA\RNING: trying to recover with ENC_UTF8"
1271
+ warn "WARNING: trying to recover with ENC_UTF8"
763
1272
  Regexp.new(node[1], Regexp::ENC_UTF8)
764
1273
  rescue RegexpError => e
765
- warn "WA\RNING: trying to recover with ENC_NONE"
1274
+ warn "WARNING: trying to recover with ENC_NONE"
766
1275
  Regexp.new(node[1], Regexp::ENC_NONE)
767
1276
  end
768
1277
  end
@@ -775,7 +1284,7 @@ module RubyParserStuff
775
1284
  end
776
1285
  node << o if o and o != 0
777
1286
  else
778
- node = s(:dregx, '', node);
1287
+ node = s(:dregx, "", node).line line
779
1288
  node.sexp_type = :dregx_once if options =~ /o/
780
1289
  node << o if o and o != 0
781
1290
  end
@@ -789,9 +1298,14 @@ module RubyParserStuff
789
1298
  else
790
1299
  body = [body]
791
1300
  end
1301
+
792
1302
  s(:resbody, cond, *body).line cond.line
793
1303
  end
794
1304
 
1305
+ def new_rescue body, resbody
1306
+ s(:rescue, body, resbody).line body.line
1307
+ end
1308
+
795
1309
  def new_sclass val
796
1310
  recv, in_def, in_single, body = val[3], val[4], val[6], val[7]
797
1311
 
@@ -812,90 +1326,55 @@ module RubyParserStuff
812
1326
  end
813
1327
 
814
1328
  def new_string val
815
- str = val[0]
816
- str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
817
- result = s(:str, str)
818
- self.lexer.fixup_lineno str.count("\n")
819
- result
820
- end
821
-
822
- def new_qword_list_entry val
823
- str = val[1]
824
- str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
825
- result = s(:str, str)
826
- self.lexer.fixup_lineno
827
- result
828
- end
829
-
830
- def new_qword_list
831
- result = s(:array)
832
- self.lexer.fixup_lineno
833
- result
834
- end
835
-
836
- def new_word_list
837
- result = s(:array)
838
- self.lexer.fixup_lineno
839
- result
840
- end
1329
+ (str, line), = val
841
1330
 
842
- def new_word_list_entry val
843
- result = val[1].sexp_type == :evstr ? s(:dstr, "", val[1]) : val[1]
844
- self.lexer.fixup_lineno
845
- result
1331
+ str.force_encoding("UTF-8")
1332
+ # TODO: remove:
1333
+ str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
1334
+ s(:str, str).line line
846
1335
  end
847
1336
 
848
- def new_qsym_list
849
- result = s(:array)
850
- self.lexer.fixup_lineno
851
- result
1337
+ def new_super args
1338
+ if args && args.sexp_type == :block_pass then
1339
+ s(:super, args).line args.line
1340
+ else
1341
+ args ||= s(:arglist).line lexer.lineno
1342
+ s(:super, *args.sexp_body).line args.line
1343
+ end
852
1344
  end
853
1345
 
854
- def new_qsym_list_entry val
855
- result = s(:lit, val[1].to_sym)
856
- self.lexer.fixup_lineno
857
- result
1346
+ def new_symbol val
1347
+ name = val.last
1348
+ s(:lit, name.to_sym).line lexer.lineno
858
1349
  end
859
1350
 
860
1351
  def new_symbol_list
861
- result = s(:array)
862
- self.lexer.fixup_lineno
863
- result
1352
+ # TODO: hunt down and try to remove ALL lexer.lineno usage!
1353
+ s(:array).line lexer.lineno
864
1354
  end
865
1355
 
866
1356
  def new_symbol_list_entry val
867
- _list, sym, _nil = val # TODO: use _list
868
- result = val[1]
1357
+ _, sym, _ = val
869
1358
 
870
- result ||= s(:str, "")
1359
+ sym ||= s(:str, "").line lexer.lineno
871
1360
 
872
1361
  case sym.sexp_type
873
1362
  when :dstr then
874
1363
  sym.sexp_type = :dsym
875
1364
  when :str then
876
- sym = s(:lit, sym.last.to_sym)
1365
+ sym = s(:lit, sym.last.to_sym).line sym.line
877
1366
  else
878
- debug20 24
879
- sym = s(:dsym, "", sym || s(:str, ""))
1367
+ sym = s(:dsym, "", sym).line sym.line
880
1368
  end
881
- self.lexer.fixup_lineno
882
- sym
883
- end
884
1369
 
885
- def new_super args
886
- if args && args.node_type == :block_pass then
887
- s(:super, args)
888
- else
889
- args ||= s(:arglist)
890
- s(:super, *args.sexp_body)
891
- end
1370
+ sym
892
1371
  end
893
1372
 
894
1373
  def new_undef n, m = nil
895
1374
  if m then
896
- block_append(n, s(:undef, m))
1375
+ block_append(n, s(:undef, m).line(m.line))
897
1376
  else
898
- s(:undef, n)
1377
+ s(:undef, n).line n.line
899
1378
  end
900
1379
  end
901
1380
 
@@ -928,138 +1407,78 @@ module RubyParserStuff
928
1407
  new_until_or_while :while, block, expr, pre
929
1408
  end
930
1409
 
931
- def new_xstring str
932
- if str then
933
- case str.sexp_type
1410
+ def new_word_list
1411
+ s(:array).line lexer.lineno
1412
+ end
1413
+
1414
+ def new_word_list_entry val
1415
+ _, word, _ = val
1416
+ word.sexp_type == :evstr ? s(:dstr, "", word).line(word.line) : word
1417
+ end
1418
+
1419
+ def new_xstring val
1420
+ _, node = val
1421
+
1422
+ node ||= s(:str, "").line lexer.lineno
1423
+
1424
+ if node then
1425
+ case node.sexp_type
934
1426
  when :str
935
- str.sexp_type = :xstr
1427
+ node.sexp_type = :xstr
936
1428
  when :dstr
937
- str.sexp_type = :dxstr
1429
+ node.sexp_type = :dxstr
938
1430
  else
939
- str = s(:dxstr, '', str)
1431
+ node = s(:dxstr, "", node).line node.line
940
1432
  end
941
- str
942
- else
943
- s(:xstr, '')
944
1433
  end
1434
+
1435
+ node
945
1436
  end
946
1437
 
947
1438
  def new_yield args = nil
948
1439
  # TODO: raise args.inspect unless [:arglist].include? args.first # HACK
949
- raise "write a test 4" if args && args.node_type == :block_pass
1440
+ raise "write a test 4" if args && args.sexp_type == :block_pass
950
1441
  raise SyntaxError, "Block argument should not be given." if
951
- args && args.node_type == :block_pass
1442
+ args && args.sexp_type == :block_pass
952
1443
 
953
- args ||= s(:arglist)
1444
+ args ||= s(:arglist).line lexer.lineno
954
1445
 
955
1446
  args.sexp_type = :arglist if [:call_args, :array].include? args.sexp_type
956
- args = s(:arglist, args) unless args.sexp_type == :arglist
957
-
958
- return s(:yield, *args.sexp_body)
959
- end
1447
+ args = s(:arglist, args).line args.line unless args.sexp_type == :arglist
960
1448
 
961
- def next_token
962
- token = self.lexer.next_token
963
-
964
- if token and token.first != RubyLexer::EOF then
965
- return token
966
- else
967
- return [false, '$end']
968
- end
1449
+ s(:yield, *args.sexp_body).line args.line
969
1450
  end
970
1451
 
971
- def new_assign lhs, rhs
972
- return nil unless lhs
973
-
974
- rhs = value_expr rhs
975
-
976
- case lhs.sexp_type
977
- when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
978
- lhs << rhs
979
- when :const then
980
- lhs.sexp_type = :cdecl
981
- lhs << rhs
1452
+ def prev_value_to_lineno v
1453
+ s, n = v
1454
+ if String === s then
1455
+ n
982
1456
  else
983
- raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
1457
+ lexer.lineno
984
1458
  end
985
-
986
- lhs
987
1459
  end
988
1460
 
989
- # TODO: remove in 4.0 or 2018-01, whichever is first
990
- deprecate :node_assign, :new_assign
991
-
992
- ##
993
- # Returns a UTF-8 encoded string after processing BOMs and magic
994
- # encoding comments.
995
- #
996
- # Holy crap... ok. Here goes:
997
- #
998
- # Ruby's file handling and encoding support is insane. We need to be
999
- # able to lex a file. The lexer file is explicitly UTF-8 to make
1000
- # things cleaner. This allows us to deal with extended chars in
1001
- # class and method names. In order to do this, we need to encode all
1002
- # input source files as UTF-8. First, we look for a UTF-8 BOM by
1003
- # looking at the first line while forcing its encoding to
1004
- # ASCII-8BIT. If we find a BOM, we strip it and set the expected
1005
- # encoding to UTF-8. Then, we search for a magic encoding comment.
1006
- # If found, it overrides the BOM. Finally, we force the encoding of
1007
- # the input string to whatever was found, and then encode that to
1008
- # UTF-8 for compatibility with the lexer.
1009
-
1010
- def handle_encoding str
1011
- str = str.dup
1012
- has_enc = str.respond_to? :encoding
1013
- encoding = nil
1014
-
1015
- header = str.each_line.first(2)
1016
- header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
1017
-
1018
- first = header.first || ""
1019
- encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
1020
-
1021
- encoding = $1.strip if header.find { |s|
1022
- s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
1023
- s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
1024
- }
1461
+ def next_token
1462
+ token = self.lexer.next_token
1025
1463
 
1026
- if encoding then
1027
- if has_enc then
1028
- encoding.sub!(/utf-8-.+$/, 'utf-8') # HACK for stupid emacs formats
1029
- hack_encoding str, encoding
1030
- else
1031
- warn "Skipping magic encoding comment"
1032
- end
1464
+ if token and token.first != RubyLexer::EOF then
1465
+ self.last_token_type = token
1466
+ return token
1467
+ elsif !token
1468
+ return self.lexer.next_token
1033
1469
  else
1034
- # nothing specified... ugh. try to encode as utf-8
1035
- hack_encoding str if has_enc
1470
+ return [false, false]
1036
1471
  end
1037
-
1038
- str
1039
1472
  end
1040
1473
 
1041
- def hack_encoding str, extra = nil
1042
- encodings = ENCODING_ORDER.dup
1043
- encodings.unshift(extra) unless extra.nil?
1044
-
1045
- # terrible, horrible, no good, very bad, last ditch effort.
1046
- encodings.each do |enc|
1047
- begin
1048
- str.force_encoding enc
1049
- if str.valid_encoding? then
1050
- str.encode! Encoding::UTF_8
1051
- break
1052
- end
1053
- rescue Encoding::InvalidByteSequenceError
1054
- # do nothing
1055
- rescue Encoding::UndefinedConversionError
1056
- # do nothing
1057
- end
1058
- end
1059
-
1060
- # no amount of pain is enough for you.
1061
- raise "Bad encoding. Need a magic encoding comment." unless
1062
- str.encoding.name == "UTF-8"
1474
+ def on_error(et, ev, values)
1475
+ ev = ev.first if ev.instance_of?(Array) && ev.size == 2 && ev.last.is_a?(Integer)
1476
+ super
1477
+ rescue Racc::ParseError => e
1478
+ # I don't like how the exception obscures the error message
1479
+ e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
1480
+ warn e.message if $DEBUG
1481
+ raise
1063
1482
  end
1064
1483
 
1065
1484
  ##
@@ -1067,56 +1486,44 @@ module RubyParserStuff
1067
1486
  # Timeout::Error if it runs for more than +time+ seconds.
1068
1487
 
1069
1488
  def process(str, file = "(string)", time = 10)
1489
+ str.freeze
1490
+
1070
1491
  Timeout.timeout time do
1071
1492
  raise "bad val: #{str.inspect}" unless String === str
1072
1493
 
1073
- str = handle_encoding str
1494
+ self.lexer.string = handle_encoding str
1074
1495
 
1075
1496
  self.file = file.dup
1076
1497
 
1077
- @yydebug = ENV.has_key? 'DEBUG'
1078
-
1079
- # HACK -- need to get tests passing more than have graceful code
1080
- self.lexer.ss = RPStringScanner.new str
1498
+ @yydebug = ENV.has_key? "DEBUG"
1081
1499
 
1082
1500
  do_parse
1083
1501
  end
1084
1502
  end
1085
1503
 
1086
- alias :parse :process
1504
+ alias parse process
1087
1505
 
1088
1506
  def remove_begin node
1089
- oldnode = node
1090
- if node and node.sexp_type == :begin and node.size == 2 then
1091
- node = node.last
1092
- node.line = oldnode.line
1093
- end
1507
+ line = node.line
1508
+
1509
+ node = node.last while node and node.sexp_type == :begin and node.size == 2
1510
+
1511
+ node = s(:nil) if node == s(:begin)
1512
+
1513
+ node.line ||= line
1514
+
1094
1515
  node
1095
1516
  end
1096
1517
 
1518
+ alias value_expr remove_begin # TODO: for now..? could check the tree, but meh?
1519
+
1097
1520
  def reset
1098
1521
  lexer.reset
1099
1522
  self.in_def = false
1100
1523
  self.in_single = 0
1101
1524
  self.env.reset
1102
1525
  self.comments.clear
1103
- end
1104
-
1105
- def block_dup_check call_or_args, block
1106
- syntax_error "Both block arg and actual block given." if
1107
- block and call_or_args.block_pass?
1108
- end
1109
-
1110
- def inverted? val
1111
- [:return, :next, :break, :yield].include? val[0].sexp_type
1112
- end
1113
-
1114
- def invert_block_call val
1115
- (type, call), iter = val
1116
-
1117
- iter.insert 1, call
1118
-
1119
- [iter, s(type)]
1526
+ self.last_token_type = nil
1120
1527
  end
1121
1528
 
1122
1529
  def ret_args node
@@ -1131,7 +1538,7 @@ module RubyParserStuff
1131
1538
 
1132
1539
  # HACK matz wraps ONE of the FOUR splats in a newline to
1133
1540
  # distinguish. I use paren for now. ugh
1134
- node = s(:svalue, node) if node.sexp_type == :splat and not node.paren
1541
+ node = s(:svalue, node).line node.line if node.sexp_type == :splat and not node.paren
1135
1542
  node.sexp_type = :svalue if node.sexp_type == :arglist && node[1].sexp_type == :splat
1136
1543
  end
1137
1544
 
@@ -1140,18 +1547,25 @@ module RubyParserStuff
1140
1547
 
1141
1548
  def s(*args)
1142
1549
  result = Sexp.new(*args)
1143
- result.line ||= lexer.lineno if lexer.ss # otherwise...
1550
+ # result.line ||= lexer.lineno if lexer.ss unless ENV["CHECK_LINE_NUMS"] # otherwise...
1144
1551
  result.file = self.file
1145
1552
  result
1146
1553
  end
1147
1554
 
1148
- def value_expr oldnode # HACK
1149
- node = remove_begin oldnode
1150
- node.line = oldnode.line if oldnode
1151
- node[2] = value_expr node[2] if node and node.sexp_type == :if
1152
- node
1555
+ def debug n
1556
+ if ENV["PRY"] then
1557
+ require "pry"; binding.pry
1558
+ end
1559
+
1560
+ raise RubyParser::SyntaxError, "debug #{n}"
1561
+ end
1562
+
1563
+ def syntax_error msg
1564
+ raise RubyParser::SyntaxError, msg
1153
1565
  end
1154
1566
 
1567
+ alias yyerror syntax_error
1568
+
1155
1569
  def void_stmts node
1156
1570
  return nil unless node
1157
1571
  return node unless node.sexp_type == :block
@@ -1169,18 +1583,39 @@ module RubyParserStuff
1169
1583
  # do nothing for now
1170
1584
  end
1171
1585
 
1172
- alias yyerror syntax_error
1586
+ def whitespace_width line, remove_width = nil
1587
+ col = 0
1588
+ idx = 0
1173
1589
 
1174
- def on_error(et, ev, values)
1175
- super
1176
- rescue Racc::ParseError => e
1177
- # I don't like how the exception obscures the error message
1178
- e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
1179
- warn e.message if $DEBUG
1180
- raise
1590
+ line.chars.each do |c|
1591
+ break if remove_width && col >= remove_width
1592
+ case c
1593
+ when " " then
1594
+ col += 1
1595
+ when "\t" then
1596
+ n = TAB_WIDTH * (col / TAB_WIDTH + 1)
1597
+ break if remove_width && n > remove_width
1598
+ col = n
1599
+ else
1600
+ break
1601
+ end
1602
+ idx += 1
1603
+ end
1604
+
1605
+ if remove_width then
1606
+ line[idx..-1]
1607
+ elsif line[idx].nil?
1608
+ nil
1609
+ else
1610
+ col
1611
+ end
1181
1612
  end
1182
1613
 
1614
+ alias remove_whitespace_width whitespace_width
1615
+
1183
1616
  class Keyword
1617
+ include RubyLexer::State::Values
1618
+
1184
1619
  class KWtable
1185
1620
  attr_accessor :name, :state, :id0, :id1
1186
1621
  def initialize(name, id=[], state=nil)
@@ -1193,82 +1628,75 @@ module RubyParserStuff
1193
1628
  ##
1194
1629
  # :stopdoc:
1195
1630
  #
1196
- # :expr_beg = ignore newline, +/- is a sign.
1197
- # :expr_end = newline significant, +/- is a operator.
1198
- # :expr_arg = newline significant, +/- is a operator.
1199
- # :expr_cmdarg = newline significant, +/- is a operator.
1200
- # :expr_endarg = newline significant, +/- is a operator.
1201
- # :expr_mid = newline significant, +/- is a operator.
1202
- # :expr_fname = ignore newline, no reserved words.
1203
- # :expr_dot = right after . or ::, no reserved words.
1204
- # :expr_class = immediate after class, no here document.
1631
+ # :expr_beg = ignore newline, +/- is a sign.
1632
+ # :expr_end = newline significant, +/- is an operator.
1633
+ # :expr_endarg = ditto, and unbound braces.
1634
+ # :expr_endfn = ditto, and unbound braces.
1635
+ # :expr_arg = newline significant, +/- is an operator.
1636
+ # :expr_cmdarg = ditto
1637
+ # :expr_mid = ditto
1638
+ # :expr_fname = ignore newline, no reserved words.
1639
+ # :expr_dot = right after . or ::, no reserved words.
1640
+ # :expr_class = immediate after class, no here document.
1641
+ # :expr_label = flag bit, label is allowed.
1642
+ # :expr_labeled = flag bit, just after a label.
1643
+ # :expr_fitem = symbol literal as FNAME.
1644
+ # :expr_value = :expr_beg -- work to remove. Need multi-state support.
1645
+
1646
+ expr_woot = EXPR_FNAME|EXPR_FITEM
1205
1647
 
1206
1648
  wordlist = [
1207
- ["end", [:kEND, :kEND ], :expr_end ],
1208
- ["else", [:kELSE, :kELSE ], :expr_beg ],
1209
- ["case", [:kCASE, :kCASE ], :expr_beg ],
1210
- ["ensure", [:kENSURE, :kENSURE ], :expr_beg ],
1211
- ["module", [:kMODULE, :kMODULE ], :expr_beg ],
1212
- ["elsif", [:kELSIF, :kELSIF ], :expr_beg ],
1213
- ["def", [:kDEF, :kDEF ], :expr_fname ],
1214
- ["rescue", [:kRESCUE, :kRESCUE_MOD ], :expr_mid ],
1215
- ["not", [:kNOT, :kNOT ], :expr_beg ],
1216
- ["then", [:kTHEN, :kTHEN ], :expr_beg ],
1217
- ["yield", [:kYIELD, :kYIELD ], :expr_arg ],
1218
- ["for", [:kFOR, :kFOR ], :expr_beg ],
1219
- ["self", [:kSELF, :kSELF ], :expr_end ],
1220
- ["false", [:kFALSE, :kFALSE ], :expr_end ],
1221
- ["retry", [:kRETRY, :kRETRY ], :expr_end ],
1222
- ["return", [:kRETURN, :kRETURN ], :expr_mid ],
1223
- ["true", [:kTRUE, :kTRUE ], :expr_end ],
1224
- ["if", [:kIF, :kIF_MOD ], :expr_beg ],
1225
- ["defined?", [:kDEFINED, :kDEFINED ], :expr_arg ],
1226
- ["super", [:kSUPER, :kSUPER ], :expr_arg ],
1227
- ["undef", [:kUNDEF, :kUNDEF ], :expr_fname ],
1228
- ["break", [:kBREAK, :kBREAK ], :expr_mid ],
1229
- ["in", [:kIN, :kIN ], :expr_beg ],
1230
- ["do", [:kDO, :kDO ], :expr_beg ],
1231
- ["nil", [:kNIL, :kNIL ], :expr_end ],
1232
- ["until", [:kUNTIL, :kUNTIL_MOD ], :expr_beg ],
1233
- ["unless", [:kUNLESS, :kUNLESS_MOD ], :expr_beg ],
1234
- ["or", [:kOR, :kOR ], :expr_beg ],
1235
- ["next", [:kNEXT, :kNEXT ], :expr_mid ],
1236
- ["when", [:kWHEN, :kWHEN ], :expr_beg ],
1237
- ["redo", [:kREDO, :kREDO ], :expr_end ],
1238
- ["and", [:kAND, :kAND ], :expr_beg ],
1239
- ["begin", [:kBEGIN, :kBEGIN ], :expr_beg ],
1240
- ["__LINE__", [:k__LINE__, :k__LINE__ ], :expr_end ],
1241
- ["class", [:kCLASS, :kCLASS ], :expr_class ],
1242
- ["__FILE__", [:k__FILE__, :k__FILE__ ], :expr_end ],
1243
- ["END", [:klEND, :klEND ], :expr_end ],
1244
- ["BEGIN", [:klBEGIN, :klBEGIN ], :expr_end ],
1245
- ["while", [:kWHILE, :kWHILE_MOD ], :expr_beg ],
1246
- ["alias", [:kALIAS, :kALIAS ], :expr_fname ],
1247
- ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], :expr_end],
1248
- ].map { |args| KWtable.new(*args) }
1649
+ ["alias", [:kALIAS, :kALIAS ], expr_woot ],
1650
+ ["and", [:kAND, :kAND ], EXPR_BEG ],
1651
+ ["begin", [:kBEGIN, :kBEGIN ], EXPR_BEG ],
1652
+ ["break", [:kBREAK, :kBREAK ], EXPR_MID ],
1653
+ ["case", [:kCASE, :kCASE ], EXPR_BEG ],
1654
+ ["class", [:kCLASS, :kCLASS ], EXPR_CLASS ],
1655
+ ["def", [:kDEF, :kDEF ], EXPR_FNAME ],
1656
+ ["defined?", [:kDEFINED, :kDEFINED ], EXPR_ARG ],
1657
+ ["do", [:kDO, :kDO ], EXPR_BEG ],
1658
+ ["else", [:kELSE, :kELSE ], EXPR_BEG ],
1659
+ ["elsif", [:kELSIF, :kELSIF ], EXPR_BEG ],
1660
+ ["end", [:kEND, :kEND ], EXPR_END ],
1661
+ ["ensure", [:kENSURE, :kENSURE ], EXPR_BEG ],
1662
+ ["false", [:kFALSE, :kFALSE ], EXPR_END ],
1663
+ ["for", [:kFOR, :kFOR ], EXPR_BEG ],
1664
+ ["if", [:kIF, :kIF_MOD ], EXPR_BEG ],
1665
+ ["in", [:kIN, :kIN ], EXPR_BEG ],
1666
+ ["module", [:kMODULE, :kMODULE ], EXPR_BEG ],
1667
+ ["next", [:kNEXT, :kNEXT ], EXPR_MID ],
1668
+ ["nil", [:kNIL, :kNIL ], EXPR_END ],
1669
+ ["not", [:kNOT, :kNOT ], EXPR_ARG ],
1670
+ ["or", [:kOR, :kOR ], EXPR_BEG ],
1671
+ ["redo", [:kREDO, :kREDO ], EXPR_END ],
1672
+ ["rescue", [:kRESCUE, :kRESCUE_MOD ], EXPR_MID ],
1673
+ ["retry", [:kRETRY, :kRETRY ], EXPR_END ],
1674
+ ["return", [:kRETURN, :kRETURN ], EXPR_MID ],
1675
+ ["self", [:kSELF, :kSELF ], EXPR_END ],
1676
+ ["super", [:kSUPER, :kSUPER ], EXPR_ARG ],
1677
+ ["then", [:kTHEN, :kTHEN ], EXPR_BEG ],
1678
+ ["true", [:kTRUE, :kTRUE ], EXPR_END ],
1679
+ ["undef", [:kUNDEF, :kUNDEF ], expr_woot ],
1680
+ ["unless", [:kUNLESS, :kUNLESS_MOD ], EXPR_BEG ],
1681
+ ["until", [:kUNTIL, :kUNTIL_MOD ], EXPR_BEG ],
1682
+ ["when", [:kWHEN, :kWHEN ], EXPR_BEG ],
1683
+ ["while", [:kWHILE, :kWHILE_MOD ], EXPR_BEG ],
1684
+ ["yield", [:kYIELD, :kYIELD ], EXPR_ARG ],
1685
+ ["BEGIN", [:klBEGIN, :klBEGIN ], EXPR_END ],
1686
+ ["END", [:klEND, :klEND ], EXPR_END ],
1687
+ ["__FILE__", [:k__FILE__, :k__FILE__ ], EXPR_END ],
1688
+ ["__LINE__", [:k__LINE__, :k__LINE__ ], EXPR_END ],
1689
+ ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], EXPR_END],
1690
+ ].map { |args|
1691
+ KWtable.new(*args)
1692
+ }
1249
1693
 
1250
1694
  # :startdoc:
1251
1695
 
1252
- WORDLIST18 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
1253
- WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
1254
-
1255
- WORDLIST18.delete "__ENCODING__"
1256
-
1257
- %w[and case elsif for if in module or unless until when while].each do |k|
1258
- WORDLIST19[k] = WORDLIST19[k].dup
1259
- WORDLIST19[k].state = :expr_value
1260
- end
1261
- %w[not].each do |k|
1262
- WORDLIST19[k] = WORDLIST19[k].dup
1263
- WORDLIST19[k].state = :expr_arg
1264
- end
1265
-
1266
- def self.keyword18 str # REFACTOR
1267
- WORDLIST18[str]
1268
- end
1696
+ WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
1269
1697
 
1270
- def self.keyword19 str
1271
- WORDLIST19[str]
1698
+ def self.keyword str
1699
+ WORDLIST[str]
1272
1700
  end
1273
1701
  end
1274
1702
 
@@ -1328,51 +1756,57 @@ module RubyParserStuff
1328
1756
  @debug = debug
1329
1757
  end
1330
1758
 
1331
- def reset
1332
- @stack = [false]
1333
- warn "#{name}_stack(set): 0" if debug
1334
- end
1335
-
1336
1759
  def inspect
1337
1760
  "StackState(#{@name}, #{@stack.inspect})"
1338
1761
  end
1339
1762
 
1340
1763
  def is_in_state
1764
+ log :is_in_state if debug
1341
1765
  @stack.last
1342
1766
  end
1343
1767
 
1344
1768
  def lexpop
1345
- warn "#{name}_stack.lexpop" if debug
1346
1769
  raise if @stack.size == 0
1347
1770
  a = @stack.pop
1348
1771
  b = @stack.pop
1349
1772
  @stack.push(a || b)
1773
+ log :lexpop if debug
1774
+ end
1775
+
1776
+ def log action
1777
+ c = caller[1]
1778
+ c = caller[2] if c =~ /expr_result/
1779
+ warn "%s_stack.%s: %p at %s" % [name, action, @stack, c.clean_caller]
1780
+ nil
1350
1781
  end
1351
1782
 
1352
1783
  def pop
1353
1784
  r = @stack.pop
1354
- warn "#{name}_stack.pop" if debug
1355
- @stack.push false if @stack.size == 0
1785
+ @stack.push false if @stack.empty?
1786
+ log :pop if debug
1356
1787
  r
1357
1788
  end
1358
1789
 
1359
1790
  def push val
1360
1791
  @stack.push val
1361
- return unless debug
1362
- c = caller.first
1363
- c = caller[1] if c =~ /expr_result/
1364
- warn "#{name}_stack(push): #{val} at line #{c.clean_caller}"
1365
- nil
1792
+ log :push if debug
1366
1793
  end
1367
1794
 
1368
- def store
1369
- result = @stack.dup
1370
- @stack.replace [false]
1371
- result
1795
+ def reset
1796
+ @stack = [false]
1797
+ log :reset if debug
1372
1798
  end
1373
1799
 
1374
1800
  def restore oldstate
1375
1801
  @stack.replace oldstate
1802
+ log :restore if debug
1803
+ end
1804
+
1805
+ def store base = false
1806
+ result = @stack.dup
1807
+ @stack.replace [base]
1808
+ log :store if debug
1809
+ result
1376
1810
  end
1377
1811
  end
1378
1812
  end