ruby_parser 3.12.0 → 3.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.autotest +18 -29
- data/History.rdoc +283 -0
- data/Manifest.txt +12 -4
- data/README.rdoc +4 -3
- data/Rakefile +189 -51
- data/bin/ruby_parse +3 -1
- data/bin/ruby_parse_extract_error +19 -36
- data/compare/normalize.rb +76 -4
- data/debugging.md +190 -0
- data/gauntlet.md +106 -0
- data/lib/rp_extensions.rb +14 -42
- data/lib/rp_stringscanner.rb +20 -51
- data/lib/ruby20_parser.rb +4659 -4218
- data/lib/ruby20_parser.y +953 -602
- data/lib/ruby21_parser.rb +4723 -4308
- data/lib/ruby21_parser.y +956 -605
- data/lib/ruby22_parser.rb +4762 -4337
- data/lib/ruby22_parser.y +960 -612
- data/lib/ruby23_parser.rb +4761 -4342
- data/lib/ruby23_parser.y +961 -613
- data/lib/ruby24_parser.rb +4791 -4341
- data/lib/ruby24_parser.y +968 -612
- data/lib/ruby25_parser.rb +4791 -4341
- data/lib/ruby25_parser.y +968 -612
- data/lib/ruby26_parser.rb +7287 -0
- data/lib/ruby26_parser.y +2749 -0
- data/lib/ruby27_parser.rb +8517 -0
- data/lib/ruby27_parser.y +3346 -0
- data/lib/ruby30_parser.rb +8751 -0
- data/lib/ruby30_parser.y +3472 -0
- data/lib/ruby3_parser.yy +3476 -0
- data/lib/ruby_lexer.rb +611 -826
- data/lib/ruby_lexer.rex +48 -40
- data/lib/ruby_lexer.rex.rb +122 -46
- data/lib/ruby_lexer_strings.rb +638 -0
- data/lib/ruby_parser.rb +38 -34
- data/lib/ruby_parser.yy +1710 -704
- data/lib/ruby_parser_extras.rb +987 -553
- data/test/test_ruby_lexer.rb +1718 -1539
- data/test/test_ruby_parser.rb +3957 -2164
- data/test/test_ruby_parser_extras.rb +39 -4
- data/tools/munge.rb +250 -0
- data/tools/ripper.rb +44 -0
- data.tar.gz.sig +0 -0
- metadata +68 -47
- metadata.gz.sig +0 -0
- data/lib/ruby18_parser.rb +0 -5793
- data/lib/ruby18_parser.y +0 -1908
- data/lib/ruby19_parser.rb +0 -6185
- data/lib/ruby19_parser.y +0 -2116
data/lib/ruby_parser_extras.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# encoding: ASCII-8BIT
|
2
|
+
# frozen_string_literal: true
|
3
|
+
# TODO: remove encoding comment
|
2
4
|
|
3
5
|
require "sexp"
|
4
6
|
require "ruby_lexer"
|
@@ -6,12 +8,50 @@ require "timeout"
|
|
6
8
|
require "rp_extensions"
|
7
9
|
require "rp_stringscanner"
|
8
10
|
|
11
|
+
class Sexp
|
12
|
+
def check_line_numbers
|
13
|
+
raise "bad nil line for:\n%s" % [self.pretty_inspect] if nil_line?
|
14
|
+
raise "bad line number for:\n%s" % [self.pretty_inspect] unless
|
15
|
+
Integer === self.line &&
|
16
|
+
self.line >= 1 &&
|
17
|
+
self.line <= self.line_min
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# Returns the maximum line number of the children of self.
|
22
|
+
|
23
|
+
def line_min
|
24
|
+
@line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
|
25
|
+
end
|
26
|
+
|
27
|
+
def nil_line?
|
28
|
+
self.deep_each.map(&:line).any?(&:nil?)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
9
32
|
module RubyParserStuff
|
10
|
-
VERSION = "3.
|
33
|
+
VERSION = "3.18.1"
|
11
34
|
|
12
35
|
attr_accessor :lexer, :in_def, :in_single, :file
|
36
|
+
attr_accessor :in_kwarg
|
13
37
|
attr_reader :env, :comments
|
14
38
|
|
39
|
+
##
|
40
|
+
# Canonicalize conditionals. Eg:
|
41
|
+
#
|
42
|
+
# not x ? a : b
|
43
|
+
#
|
44
|
+
# becomes:
|
45
|
+
#
|
46
|
+
# x ? b : a
|
47
|
+
|
48
|
+
attr_accessor :canonicalize_conditions
|
49
|
+
|
50
|
+
##
|
51
|
+
# The last token type returned from #next_token
|
52
|
+
|
53
|
+
attr_accessor :last_token_type
|
54
|
+
|
15
55
|
$good20 = []
|
16
56
|
|
17
57
|
%w[
|
@@ -30,6 +70,28 @@ module RubyParserStuff
|
|
30
70
|
end
|
31
71
|
end
|
32
72
|
|
73
|
+
##
|
74
|
+
# for pure ruby systems only
|
75
|
+
|
76
|
+
def do_parse
|
77
|
+
_racc_do_parse_rb(_racc_setup, false)
|
78
|
+
end if ENV["PURE_RUBY"] || ENV["CHECK_LINE_NUMS"]
|
79
|
+
|
80
|
+
if ENV["CHECK_LINE_NUMS"] then
|
81
|
+
def _racc_do_reduce arg, act
|
82
|
+
x = super
|
83
|
+
|
84
|
+
@racc_vstack.grep(Sexp).each do |sexp|
|
85
|
+
sexp.check_line_numbers
|
86
|
+
end
|
87
|
+
x
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
ARG_TYPES = [:arglist, :call_args, :array, :args].map { |k|
|
92
|
+
[k, true]
|
93
|
+
}.to_h
|
94
|
+
|
33
95
|
has_enc = "".respond_to? :encoding
|
34
96
|
|
35
97
|
# This is in sorted order of occurrence according to
|
@@ -47,113 +109,85 @@ module RubyParserStuff
|
|
47
109
|
Encoding::EUC_JP
|
48
110
|
] if has_enc
|
49
111
|
|
50
|
-
|
51
|
-
raise RubyParser::SyntaxError, msg
|
52
|
-
end
|
112
|
+
JUMP_TYPE = [:return, :next, :break, :yield].map { |k| [k, true] }.to_h
|
53
113
|
|
54
|
-
|
55
|
-
node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.sexp_type
|
56
|
-
node1 << node2 if node2
|
57
|
-
node1
|
58
|
-
end
|
114
|
+
TAB_WIDTH = 8
|
59
115
|
|
60
|
-
def
|
61
|
-
|
62
|
-
node1 << s(:splat, node2).compact
|
63
|
-
node1
|
64
|
-
end
|
65
|
-
|
66
|
-
def clean_mlhs sexp
|
67
|
-
case sexp.sexp_type
|
68
|
-
when :masgn then
|
69
|
-
if sexp.size == 2 and sexp[1].sexp_type == :array then
|
70
|
-
s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
|
71
|
-
else
|
72
|
-
debug20 5
|
73
|
-
sexp
|
74
|
-
end
|
75
|
-
when :gasgn, :iasgn, :lasgn, :cvasgn then
|
76
|
-
if sexp.size == 2 then
|
77
|
-
sexp.last
|
78
|
-
else
|
79
|
-
debug20 7
|
80
|
-
sexp # optional value
|
81
|
-
end
|
82
|
-
else
|
83
|
-
raise "unsupported type: #{sexp.inspect}"
|
84
|
-
end
|
85
|
-
end
|
116
|
+
def initialize(options = {})
|
117
|
+
super()
|
86
118
|
|
87
|
-
|
88
|
-
|
89
|
-
result.sexp_type = :masgn
|
90
|
-
result
|
91
|
-
end
|
119
|
+
v = self.class.name[/[23]\d/]
|
120
|
+
raise "Bad Class name #{self.class}" unless v
|
92
121
|
|
93
|
-
|
94
|
-
|
122
|
+
self.lexer = RubyLexer.new v && v.to_i
|
123
|
+
self.lexer.parser = self
|
124
|
+
self.in_kwarg = false
|
95
125
|
|
96
|
-
|
97
|
-
|
98
|
-
ary << "*#{splat}".to_sym
|
99
|
-
end
|
126
|
+
@env = RubyParserStuff::Environment.new
|
127
|
+
@comments = []
|
100
128
|
|
101
|
-
|
129
|
+
@canonicalize_conditions = true
|
102
130
|
|
103
|
-
|
104
|
-
s(:masgn, *ary.sexp_body)
|
105
|
-
else
|
106
|
-
ary.last
|
107
|
-
end
|
131
|
+
self.reset
|
108
132
|
end
|
109
133
|
|
110
|
-
def
|
111
|
-
|
112
|
-
when :kwsplat then
|
113
|
-
array
|
114
|
-
else
|
115
|
-
s(:hash, *array.sexp_body)
|
116
|
-
end
|
117
|
-
end
|
134
|
+
def arg_concat node1, node2 # TODO: nuke
|
135
|
+
raise "huh" unless node2
|
118
136
|
|
119
|
-
|
120
|
-
|
137
|
+
splat = s(:splat, node2)
|
138
|
+
splat.line node2.line
|
121
139
|
|
122
|
-
|
123
|
-
|
124
|
-
when Sexp then
|
125
|
-
case arg.sexp_type
|
126
|
-
when :array, :args, :call_args then # HACK? remove array at some point
|
127
|
-
result.concat arg.sexp_body
|
128
|
-
else
|
129
|
-
result << arg
|
130
|
-
end
|
131
|
-
when Symbol then
|
132
|
-
result << arg
|
133
|
-
when ",", nil then
|
134
|
-
# ignore
|
135
|
-
else
|
136
|
-
raise "unhandled: #{arg.inspect} in #{args.inspect}"
|
137
|
-
end
|
138
|
-
end
|
140
|
+
node1 << splat
|
141
|
+
end
|
139
142
|
|
140
|
-
|
143
|
+
def argl x
|
144
|
+
x = s(:arglist, x) if x and x.sexp_type == :array
|
145
|
+
x
|
141
146
|
end
|
142
147
|
|
143
148
|
def args args
|
144
149
|
result = s(:args)
|
145
150
|
|
151
|
+
ss = args.grep Sexp
|
152
|
+
if ss.empty? then
|
153
|
+
result.line lexer.lineno
|
154
|
+
else
|
155
|
+
result.line ss.first.line
|
156
|
+
end
|
157
|
+
|
146
158
|
args.each do |arg|
|
159
|
+
if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
|
160
|
+
arg = arg.first
|
161
|
+
end
|
162
|
+
|
147
163
|
case arg
|
148
164
|
when Sexp then
|
149
165
|
case arg.sexp_type
|
150
166
|
when :args, :block, :array, :call_args then # HACK call_args mismatch
|
151
|
-
|
167
|
+
rest = arg.sexp_body
|
168
|
+
|
169
|
+
rest.map! { |x|
|
170
|
+
if x.instance_of? Array and x.size == 2 and Numeric === x.last then
|
171
|
+
x.first
|
172
|
+
else
|
173
|
+
x
|
174
|
+
end
|
175
|
+
}
|
176
|
+
|
177
|
+
result.concat rest
|
178
|
+
when :forward_args then
|
179
|
+
self.env[:*] = :lvar # TODO: arg_var(p, idFWD_REST) ?
|
180
|
+
self.env[:**] = :lvar
|
181
|
+
self.env[:&] = :lvar
|
182
|
+
|
183
|
+
result << arg
|
152
184
|
when :block_arg then
|
153
185
|
result << :"&#{arg.last}"
|
154
186
|
when :shadow then
|
187
|
+
name = arg.last
|
188
|
+
self.env[name] = :lvar
|
155
189
|
if Sexp === result.last and result.last.sexp_type == :shadow then
|
156
|
-
result.last <<
|
190
|
+
result.last << name
|
157
191
|
else
|
158
192
|
result << arg
|
159
193
|
end
|
@@ -166,6 +200,8 @@ module RubyParserStuff
|
|
166
200
|
name = arg.to_s.delete("&*")
|
167
201
|
self.env[name.to_sym] = :lvar unless name.empty?
|
168
202
|
result << arg
|
203
|
+
when true, false then
|
204
|
+
self.in_kwarg = arg
|
169
205
|
when ",", "|", ";", "(", ")", nil then
|
170
206
|
# ignore
|
171
207
|
else
|
@@ -176,21 +212,46 @@ module RubyParserStuff
|
|
176
212
|
result
|
177
213
|
end
|
178
214
|
|
215
|
+
def end_args args
|
216
|
+
lexer.lex_state = RubyLexer::State::Values::EXPR_BEG
|
217
|
+
lexer.command_start = true
|
218
|
+
self.args args
|
219
|
+
end
|
220
|
+
|
221
|
+
def endless_method_name defn_or_defs
|
222
|
+
name = defn_or_defs[1]
|
223
|
+
name = defn_or_defs[2] unless Symbol === name
|
224
|
+
|
225
|
+
if name.end_with? "=" then
|
226
|
+
yyerror "setter method cannot be defined in an endless method definition"
|
227
|
+
end
|
228
|
+
|
229
|
+
# TODO? token_info_drop(p, "def", loc->beg_pos);
|
230
|
+
end
|
231
|
+
|
232
|
+
def array_to_hash array
|
233
|
+
case array.sexp_type
|
234
|
+
when :kwsplat then
|
235
|
+
array
|
236
|
+
else
|
237
|
+
s(:hash, *array.sexp_body).line array.line
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
179
241
|
def aryset receiver, index
|
180
242
|
index ||= s()
|
181
|
-
|
243
|
+
l = receiver.line
|
244
|
+
result = s(:attrasgn, receiver, :"[]=",
|
245
|
+
*index.sexp_body).compact # [].sexp_body => nil
|
246
|
+
result.line = l
|
247
|
+
result
|
182
248
|
end
|
183
249
|
|
184
250
|
def assignable(lhs, value = nil)
|
185
|
-
id = lhs
|
186
|
-
id = id.to_sym
|
187
|
-
|
188
|
-
raise "write a test 1" if id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
|
251
|
+
id, line = lhs
|
252
|
+
id = id.to_sym
|
189
253
|
|
190
|
-
|
191
|
-
id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
|
192
|
-
|
193
|
-
result = case id.to_s
|
254
|
+
result = case id
|
194
255
|
when /^@@/ then
|
195
256
|
asgn = in_def || in_single > 0
|
196
257
|
s((asgn ? :cvasgn : :cvdecl), id)
|
@@ -212,8 +273,22 @@ module RubyParserStuff
|
|
212
273
|
self.env[id] ||= :lvar if result.sexp_type == :lasgn
|
213
274
|
|
214
275
|
result << value if value
|
276
|
+
result.line line
|
277
|
+
result
|
278
|
+
end
|
215
279
|
|
216
|
-
|
280
|
+
def backref_assign_error ref
|
281
|
+
# TODO: need a test for this... obviously
|
282
|
+
case ref.sexp_type
|
283
|
+
when :nth_ref then
|
284
|
+
raise "write a test 2"
|
285
|
+
raise SyntaxError, "Can't set variable %p" % ref.last
|
286
|
+
when :back_ref then
|
287
|
+
raise "write a test 3"
|
288
|
+
raise SyntaxError, "Can't set back reference %p" % ref.last
|
289
|
+
else
|
290
|
+
raise "Unknown backref type: #{ref.inspect}"
|
291
|
+
end
|
217
292
|
end
|
218
293
|
|
219
294
|
def block_append(head, tail)
|
@@ -223,12 +298,82 @@ module RubyParserStuff
|
|
223
298
|
line = [head.line, tail.line].compact.min
|
224
299
|
|
225
300
|
head = remove_begin(head)
|
226
|
-
head = s(:block, head) unless head.
|
301
|
+
head = s(:block, head).line(line) unless head.sexp_type == :block
|
227
302
|
|
228
|
-
head.line = line
|
303
|
+
# head.line = line
|
229
304
|
head << tail
|
230
305
|
end
|
231
306
|
|
307
|
+
def block_dup_check call_or_args, block
|
308
|
+
syntax_error "Both block arg and actual block given." if
|
309
|
+
block and call_or_args.block_pass?
|
310
|
+
end
|
311
|
+
|
312
|
+
def block_var *args
|
313
|
+
result = self.args args
|
314
|
+
result.sexp_type = :masgn
|
315
|
+
result
|
316
|
+
end
|
317
|
+
|
318
|
+
def call_args args
|
319
|
+
result = s(:call_args)
|
320
|
+
|
321
|
+
a = args.grep(Sexp).first
|
322
|
+
if a then
|
323
|
+
result.line a.line
|
324
|
+
else
|
325
|
+
result.line lexer.lineno
|
326
|
+
end
|
327
|
+
|
328
|
+
args.each do |arg|
|
329
|
+
if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
|
330
|
+
arg = arg.first
|
331
|
+
end
|
332
|
+
|
333
|
+
case arg
|
334
|
+
when Sexp then
|
335
|
+
case arg.sexp_type
|
336
|
+
when :array, :args, :call_args then # HACK? remove array at some point
|
337
|
+
result.concat arg.sexp_body
|
338
|
+
else
|
339
|
+
result << arg
|
340
|
+
end
|
341
|
+
when Symbol then
|
342
|
+
result << arg
|
343
|
+
when Array then
|
344
|
+
id, _line = arg
|
345
|
+
result << id
|
346
|
+
when ",", nil, "(" then
|
347
|
+
# ignore
|
348
|
+
else
|
349
|
+
raise "unhandled: #{arg.inspect} in #{args.inspect}"
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
result
|
354
|
+
end
|
355
|
+
|
356
|
+
def clean_mlhs sexp
|
357
|
+
case sexp.sexp_type
|
358
|
+
when :masgn then
|
359
|
+
if sexp.size == 2 and sexp[1].sexp_type == :array then
|
360
|
+
s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
|
361
|
+
else
|
362
|
+
debug20 5
|
363
|
+
sexp
|
364
|
+
end
|
365
|
+
when :gasgn, :iasgn, :lasgn, :cvasgn then
|
366
|
+
if sexp.size == 2 then
|
367
|
+
sexp.last
|
368
|
+
else
|
369
|
+
debug20 7
|
370
|
+
sexp # optional value
|
371
|
+
end
|
372
|
+
else
|
373
|
+
raise "unsupported type: #{sexp.inspect}"
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
232
377
|
def cond node
|
233
378
|
return nil if node.nil?
|
234
379
|
node = value_expr node
|
@@ -236,63 +381,116 @@ module RubyParserStuff
|
|
236
381
|
case node.sexp_type
|
237
382
|
when :lit then
|
238
383
|
if Regexp === node.last then
|
239
|
-
|
384
|
+
s(:match, node)
|
240
385
|
else
|
241
|
-
|
386
|
+
node
|
242
387
|
end
|
243
388
|
when :and then
|
244
|
-
|
389
|
+
_, lhs, rhs = node
|
390
|
+
s(:and, cond(lhs), cond(rhs))
|
245
391
|
when :or then
|
246
|
-
|
392
|
+
_, lhs, rhs = node
|
393
|
+
s(:or, cond(lhs), cond(rhs))
|
247
394
|
when :dot2 then
|
248
395
|
label = "flip#{node.hash}"
|
249
396
|
env[label] = :lvar
|
250
397
|
_, lhs, rhs = node
|
251
|
-
|
398
|
+
s(:flip2, lhs, rhs) # TODO: recurse?
|
252
399
|
when :dot3 then
|
253
400
|
label = "flip#{node.hash}"
|
254
401
|
env[label] = :lvar
|
255
402
|
_, lhs, rhs = node
|
256
|
-
|
403
|
+
s(:flip3, lhs, rhs)
|
257
404
|
else
|
258
|
-
|
259
|
-
end
|
405
|
+
node
|
406
|
+
end.line node.line
|
260
407
|
end
|
261
408
|
|
262
|
-
|
263
|
-
|
409
|
+
def dedent sexp
|
410
|
+
dedent_count = dedent_size sexp
|
264
411
|
|
265
|
-
|
266
|
-
|
267
|
-
|
412
|
+
skip_one = false
|
413
|
+
sexp.map { |obj|
|
414
|
+
case obj
|
415
|
+
when Symbol then
|
416
|
+
obj
|
417
|
+
when String then
|
418
|
+
obj.lines.map { |l| remove_whitespace_width l, dedent_count }.join
|
419
|
+
when Sexp then
|
420
|
+
case obj.sexp_type
|
421
|
+
when :evstr then
|
422
|
+
skip_one = true
|
423
|
+
obj
|
424
|
+
when :str then
|
425
|
+
_, str = obj
|
426
|
+
str = if skip_one then
|
427
|
+
skip_one = false
|
428
|
+
s1, *rest = str.lines
|
429
|
+
s1 + rest.map { |l| remove_whitespace_width l, dedent_count }.join
|
430
|
+
else
|
431
|
+
str.lines.map { |l| remove_whitespace_width l, dedent_count }.join
|
432
|
+
end
|
268
433
|
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
|
434
|
+
s(:str, str).line obj.line
|
435
|
+
else
|
436
|
+
warn "unprocessed sexp %p" % [obj]
|
437
|
+
end
|
438
|
+
else
|
439
|
+
warn "unprocessed: %p" % [obj]
|
276
440
|
end
|
277
|
-
|
441
|
+
}
|
442
|
+
end
|
278
443
|
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
when
|
284
|
-
|
444
|
+
def dedent_size sexp
|
445
|
+
skip_one = false
|
446
|
+
sexp.flat_map { |s|
|
447
|
+
case s
|
448
|
+
when Symbol then
|
449
|
+
next
|
450
|
+
when String then
|
451
|
+
s.lines
|
452
|
+
when Sexp then
|
453
|
+
case s.sexp_type
|
454
|
+
when :evstr then
|
455
|
+
skip_one = true
|
456
|
+
next
|
457
|
+
when :str then
|
458
|
+
_, str = s
|
459
|
+
lines = str.lines
|
460
|
+
if skip_one then
|
461
|
+
skip_one = false
|
462
|
+
lines.shift
|
463
|
+
end
|
464
|
+
lines
|
465
|
+
else
|
466
|
+
warn "unprocessed sexp %p" % [s]
|
467
|
+
end
|
468
|
+
else
|
469
|
+
warn "unprocessed: %p" % [s]
|
470
|
+
end.map { |l| whitespace_width l.chomp }
|
471
|
+
}.compact.min
|
472
|
+
end
|
473
|
+
|
474
|
+
def dedent_string string, width
|
475
|
+
characters_skipped = 0
|
476
|
+
indentation_skipped = 0
|
477
|
+
|
478
|
+
string.chars.each do |char|
|
479
|
+
break if indentation_skipped >= width
|
480
|
+
if char == " "
|
481
|
+
characters_skipped += 1
|
482
|
+
indentation_skipped += 1
|
483
|
+
elsif char == "\t"
|
484
|
+
proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1)
|
485
|
+
break if proposed > width
|
486
|
+
characters_skipped += 1
|
487
|
+
indentation_skipped = proposed
|
285
488
|
end
|
286
489
|
end
|
287
|
-
|
288
|
-
return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
|
490
|
+
string[characters_skipped..-1]
|
289
491
|
end
|
290
492
|
|
291
|
-
# TODO: remove in 4.0 or 2018-01, whichever is first
|
292
|
-
deprecate :get_match_node, :new_match
|
293
|
-
|
294
493
|
def gettable(id)
|
295
|
-
lineno = id.lineno if id.respond_to? :lineno
|
296
494
|
id = id.to_sym if String === id
|
297
495
|
|
298
496
|
result = case id.to_s
|
@@ -313,38 +511,99 @@ module RubyParserStuff
|
|
313
511
|
end
|
314
512
|
end
|
315
513
|
|
316
|
-
result.line lineno if lineno
|
317
|
-
|
318
514
|
raise "identifier #{id.inspect} is not valid" unless result
|
319
515
|
|
320
516
|
result
|
321
517
|
end
|
322
518
|
|
519
|
+
def hack_encoding str, extra = nil
|
520
|
+
encodings = ENCODING_ORDER.dup
|
521
|
+
encodings.unshift(extra) unless extra.nil?
|
522
|
+
|
523
|
+
# terrible, horrible, no good, very bad, last ditch effort.
|
524
|
+
encodings.each do |enc|
|
525
|
+
begin
|
526
|
+
str.force_encoding enc
|
527
|
+
if str.valid_encoding? then
|
528
|
+
str.encode! Encoding::UTF_8
|
529
|
+
break
|
530
|
+
end
|
531
|
+
rescue ArgumentError # unknown encoding name
|
532
|
+
# do nothing
|
533
|
+
rescue Encoding::InvalidByteSequenceError
|
534
|
+
# do nothing
|
535
|
+
rescue Encoding::UndefinedConversionError
|
536
|
+
# do nothing
|
537
|
+
end
|
538
|
+
end
|
539
|
+
|
540
|
+
# no amount of pain is enough for you.
|
541
|
+
raise "Bad encoding. Need a magic encoding comment." unless
|
542
|
+
str.encoding.name == "UTF-8"
|
543
|
+
end
|
544
|
+
|
323
545
|
##
|
324
|
-
#
|
325
|
-
#
|
326
|
-
# not x ? a : b
|
546
|
+
# Returns a UTF-8 encoded string after processing BOMs and magic
|
547
|
+
# encoding comments.
|
327
548
|
#
|
328
|
-
#
|
549
|
+
# Holy crap... ok. Here goes:
|
329
550
|
#
|
330
|
-
#
|
551
|
+
# Ruby's file handling and encoding support is insane. We need to be
|
552
|
+
# able to lex a file. The lexer file is explicitly UTF-8 to make
|
553
|
+
# things cleaner. This allows us to deal with extended chars in
|
554
|
+
# class and method names. In order to do this, we need to encode all
|
555
|
+
# input source files as UTF-8. First, we look for a UTF-8 BOM by
|
556
|
+
# looking at the first line while forcing its encoding to
|
557
|
+
# ASCII-8BIT. If we find a BOM, we strip it and set the expected
|
558
|
+
# encoding to UTF-8. Then, we search for a magic encoding comment.
|
559
|
+
# If found, it overrides the BOM. Finally, we force the encoding of
|
560
|
+
# the input string to whatever was found, and then encode that to
|
561
|
+
# UTF-8 for compatibility with the lexer.
|
331
562
|
|
332
|
-
|
563
|
+
def handle_encoding str
|
564
|
+
str = str.dup
|
565
|
+
has_enc = str.respond_to? :encoding
|
566
|
+
encoding = nil
|
333
567
|
|
334
|
-
|
335
|
-
|
568
|
+
header = str.each_line.first(2)
|
569
|
+
header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
|
336
570
|
|
337
|
-
|
571
|
+
first = header.first || ""
|
572
|
+
encoding, str = +"utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
|
338
573
|
|
339
|
-
|
340
|
-
|
574
|
+
encoding = $1.strip if header.find { |s|
|
575
|
+
s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
|
576
|
+
s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
|
577
|
+
}
|
341
578
|
|
342
|
-
|
343
|
-
|
579
|
+
if encoding then
|
580
|
+
if has_enc then
|
581
|
+
encoding.sub!(/utf-8-.+$/, "utf-8") # HACK for stupid emacs formats
|
582
|
+
hack_encoding str, encoding
|
583
|
+
else
|
584
|
+
warn "Skipping magic encoding comment"
|
585
|
+
end
|
586
|
+
else
|
587
|
+
# nothing specified... ugh. try to encode as utf-8
|
588
|
+
hack_encoding str if has_enc
|
589
|
+
end
|
344
590
|
|
345
|
-
|
591
|
+
str
|
592
|
+
end
|
346
593
|
|
347
|
-
|
594
|
+
def invert_block_call val
|
595
|
+
ret, iter = val
|
596
|
+
type, call = ret
|
597
|
+
|
598
|
+
iter.insert 1, call
|
599
|
+
|
600
|
+
ret = s(type).line ret.line
|
601
|
+
|
602
|
+
[iter, ret]
|
603
|
+
end
|
604
|
+
|
605
|
+
def inverted? val
|
606
|
+
JUMP_TYPE[val[0].sexp_type]
|
348
607
|
end
|
349
608
|
|
350
609
|
def list_append list, item # TODO: nuke me *sigh*
|
@@ -365,12 +624,14 @@ module RubyParserStuff
|
|
365
624
|
|
366
625
|
htype, ttype = head.sexp_type, tail.sexp_type
|
367
626
|
|
368
|
-
head = s(:dstr,
|
627
|
+
head = s(:dstr, "", head).line head.line if htype == :evstr
|
369
628
|
|
370
629
|
case ttype
|
371
630
|
when :str then
|
372
631
|
if htype == :str
|
373
|
-
head.last
|
632
|
+
a, b = head.last, tail.last
|
633
|
+
b = b.dup.force_encoding a.encoding unless Encoding.compatible?(a, b)
|
634
|
+
a << b
|
374
635
|
elsif htype == :dstr and head.size == 2 then
|
375
636
|
head.last << tail.last
|
376
637
|
else
|
@@ -384,8 +645,8 @@ module RubyParserStuff
|
|
384
645
|
head.line = lineno
|
385
646
|
else
|
386
647
|
tail.sexp_type = :array
|
387
|
-
tail[1] = s(:str, tail[1])
|
388
|
-
tail.delete_at 1 if tail[1] == s(:str,
|
648
|
+
tail[1] = s(:str, tail[1]).line tail.line
|
649
|
+
tail.delete_at 1 if tail[1] == s(:str, "")
|
389
650
|
|
390
651
|
head.push(*tail.sexp_body)
|
391
652
|
end
|
@@ -411,6 +672,13 @@ module RubyParserStuff
|
|
411
672
|
return head
|
412
673
|
end
|
413
674
|
|
675
|
+
def local_pop in_def
|
676
|
+
lexer.cond.pop # group = local_pop
|
677
|
+
lexer.cmdarg.pop
|
678
|
+
self.env.unextend
|
679
|
+
self.in_def = in_def
|
680
|
+
end
|
681
|
+
|
414
682
|
def logical_op type, left, right
|
415
683
|
left = value_expr left
|
416
684
|
|
@@ -423,23 +691,135 @@ module RubyParserStuff
|
|
423
691
|
node = rhs
|
424
692
|
end
|
425
693
|
|
426
|
-
node
|
694
|
+
node.pop
|
695
|
+
node << s(type, rhs, right).line(rhs.line)
|
427
696
|
|
428
697
|
return left
|
429
698
|
end
|
430
699
|
|
431
|
-
|
700
|
+
result = s(type, left, right)
|
701
|
+
result.line left.line if left.line
|
702
|
+
result
|
432
703
|
end
|
433
704
|
|
434
|
-
# TODO: remove in 4.0 or 2018-01, whichever is first
|
435
|
-
deprecate :logop, :logical_op
|
436
|
-
|
437
705
|
def new_aref val
|
438
706
|
val[2] ||= s(:arglist)
|
439
707
|
val[2].sexp_type = :arglist if val[2].sexp_type == :array # REFACTOR
|
440
708
|
new_call val[0], :"[]", val[2]
|
441
709
|
end
|
442
710
|
|
711
|
+
def new_arg val
|
712
|
+
arg, = val
|
713
|
+
|
714
|
+
case arg
|
715
|
+
when Symbol then
|
716
|
+
result = s(:args, arg).line line
|
717
|
+
when Sexp then
|
718
|
+
result = arg
|
719
|
+
when Array then
|
720
|
+
(arg, line), = val
|
721
|
+
result = s(:args, arg).line line
|
722
|
+
else
|
723
|
+
debug20 32
|
724
|
+
raise "Unknown f_arg type: #{val.inspect}"
|
725
|
+
end
|
726
|
+
|
727
|
+
result
|
728
|
+
end
|
729
|
+
|
730
|
+
def new_array_pattern const, pre_arg, arypat, loc
|
731
|
+
result = s(:array_pat, const).line loc
|
732
|
+
result << pre_arg if pre_arg
|
733
|
+
|
734
|
+
if arypat && arypat.sexp_type == :array_TAIL then
|
735
|
+
result.concat arypat.sexp_body
|
736
|
+
else
|
737
|
+
raise "NO?: %p" % [arypat]
|
738
|
+
end
|
739
|
+
|
740
|
+
result
|
741
|
+
end
|
742
|
+
|
743
|
+
def array_pat_concat lhs, rhs
|
744
|
+
case lhs.sexp_type
|
745
|
+
when :PATTERN then
|
746
|
+
lhs.sexp_type = :array_pat
|
747
|
+
end
|
748
|
+
|
749
|
+
if rhs then
|
750
|
+
case rhs.sexp_type
|
751
|
+
when :array_pat, :array_TAIL, :PATTERN then
|
752
|
+
lhs.concat rhs.sexp_body
|
753
|
+
else
|
754
|
+
lhs << rhs
|
755
|
+
end
|
756
|
+
end
|
757
|
+
end
|
758
|
+
|
759
|
+
def new_array_pattern_tail pre_args, has_rest, rest_arg, post_args
|
760
|
+
# TODO: remove has_rest once all tests pass !!!
|
761
|
+
rest_arg = if has_rest then
|
762
|
+
:"*#{rest_arg}"
|
763
|
+
else
|
764
|
+
nil
|
765
|
+
end
|
766
|
+
|
767
|
+
result = s(:array_TAIL).line 666
|
768
|
+
|
769
|
+
array_pat_concat result, pre_args
|
770
|
+
|
771
|
+
result << rest_arg if rest_arg
|
772
|
+
|
773
|
+
array_pat_concat result, post_args
|
774
|
+
|
775
|
+
result
|
776
|
+
end
|
777
|
+
|
778
|
+
def new_assign lhs, rhs
|
779
|
+
return nil unless lhs
|
780
|
+
|
781
|
+
rhs = value_expr rhs
|
782
|
+
|
783
|
+
case lhs.sexp_type
|
784
|
+
when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
|
785
|
+
lhs << rhs
|
786
|
+
when :const then
|
787
|
+
lhs.sexp_type = :cdecl
|
788
|
+
lhs << rhs
|
789
|
+
else
|
790
|
+
raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
|
791
|
+
end
|
792
|
+
|
793
|
+
lhs
|
794
|
+
end
|
795
|
+
|
796
|
+
def new_attrasgn recv, meth, call_op = :"."
|
797
|
+
call_op = call_op.first if Array === call_op
|
798
|
+
|
799
|
+
meth = :"#{meth}="
|
800
|
+
|
801
|
+
result = case call_op.to_sym
|
802
|
+
when :"."
|
803
|
+
s(:attrasgn, recv, meth)
|
804
|
+
when :"&."
|
805
|
+
s(:safe_attrasgn, recv, meth)
|
806
|
+
else
|
807
|
+
raise "unknown call operator: `#{type.inspect}`"
|
808
|
+
end
|
809
|
+
|
810
|
+
result.line = recv.line
|
811
|
+
result
|
812
|
+
end
|
813
|
+
|
814
|
+
def new_begin val
|
815
|
+
_, lineno, body, _ = val
|
816
|
+
|
817
|
+
result = body ? s(:begin, body) : s(:nil)
|
818
|
+
result.line lineno
|
819
|
+
|
820
|
+
result
|
821
|
+
end
|
822
|
+
|
443
823
|
def new_body val
|
444
824
|
body, resbody, elsebody, ensurebody = val
|
445
825
|
|
@@ -463,39 +843,29 @@ module RubyParserStuff
|
|
463
843
|
|
464
844
|
if elsebody and not resbody then
|
465
845
|
warning("else without rescue is useless")
|
466
|
-
result = s(:begin, result) if result
|
846
|
+
result = s(:begin, result).line result.line if result
|
467
847
|
result = block_append(result, elsebody)
|
468
848
|
end
|
469
849
|
|
470
|
-
|
850
|
+
if ensurebody
|
851
|
+
lineno = (result || ensurebody).line
|
852
|
+
result = s(:ensure, result, ensurebody).compact.line lineno
|
853
|
+
end
|
471
854
|
|
472
855
|
result
|
473
856
|
end
|
474
857
|
|
475
|
-
def
|
476
|
-
|
477
|
-
x
|
858
|
+
def new_brace_body args, body, lineno
|
859
|
+
new_iter(nil, args, body).line lineno
|
478
860
|
end
|
479
861
|
|
480
|
-
def
|
481
|
-
|
482
|
-
case ref.sexp_type
|
483
|
-
when :nth_ref then
|
484
|
-
raise "write a test 2"
|
485
|
-
raise SyntaxError, "Can't set variable %p" % ref.last
|
486
|
-
when :back_ref then
|
487
|
-
raise "write a test 3"
|
488
|
-
raise SyntaxError, "Can't set back reference %p" % ref.last
|
489
|
-
else
|
490
|
-
raise "Unknown backref type: #{ref.inspect}"
|
491
|
-
end
|
492
|
-
end
|
862
|
+
def new_call recv, meth, args = nil, call_op = :"."
|
863
|
+
call_op = call_op.first if Array === call_op
|
493
864
|
|
494
|
-
def new_call recv, meth, args = nil, call_op = :'.'
|
495
865
|
result = case call_op.to_sym
|
496
|
-
when :
|
866
|
+
when :"."
|
497
867
|
s(:call, recv, meth)
|
498
|
-
when :
|
868
|
+
when :"&."
|
499
869
|
s(:safe_call, recv, meth)
|
500
870
|
else
|
501
871
|
raise "unknown call operator: `#{type.inspect}`"
|
@@ -505,39 +875,28 @@ module RubyParserStuff
|
|
505
875
|
# TODO: need a test with f(&b) { } to produce warning
|
506
876
|
|
507
877
|
if args
|
508
|
-
if [
|
878
|
+
if ARG_TYPES[args.sexp_type] then
|
509
879
|
result.concat args.sexp_body
|
510
880
|
else
|
511
881
|
result << args
|
512
882
|
end
|
513
883
|
end
|
514
884
|
|
515
|
-
line = result.grep(Sexp).map(&:line).compact.min
|
516
|
-
result.line = line if
|
885
|
+
# line = result.grep(Sexp).map(&:line).compact.min
|
886
|
+
result.line = recv.line if recv
|
887
|
+
result.line ||= lexer.lineno
|
517
888
|
|
518
889
|
result
|
519
890
|
end
|
520
891
|
|
521
|
-
def
|
522
|
-
|
523
|
-
|
524
|
-
result = case call_op.to_sym
|
525
|
-
when :'.'
|
526
|
-
s(:attrasgn, recv, meth)
|
527
|
-
when :'&.'
|
528
|
-
s(:safe_attrasgn, recv, meth)
|
529
|
-
else
|
530
|
-
raise "unknown call operator: `#{type.inspect}`"
|
531
|
-
end
|
532
|
-
|
533
|
-
result.line = recv.line
|
534
|
-
result
|
892
|
+
def new_in pat, body, cases, line
|
893
|
+
s(:in, pat, body, cases).line line
|
535
894
|
end
|
536
895
|
|
537
896
|
def new_case expr, body, line
|
538
897
|
result = s(:case, expr)
|
539
898
|
|
540
|
-
while body and body.
|
899
|
+
while body and [:when, :in].include? body.sexp_type
|
541
900
|
result << body
|
542
901
|
body = body.delete_at 3
|
543
902
|
end
|
@@ -556,8 +915,11 @@ module RubyParserStuff
|
|
556
915
|
end
|
557
916
|
|
558
917
|
def new_class val
|
918
|
+
# TODO: get line from class keyword
|
559
919
|
line, path, superclass, body = val[1], val[2], val[3], val[5]
|
560
920
|
|
921
|
+
path = path.first if path.instance_of? Array
|
922
|
+
|
561
923
|
result = s(:class, path, superclass)
|
562
924
|
|
563
925
|
if body then
|
@@ -579,44 +941,87 @@ module RubyParserStuff
|
|
579
941
|
result
|
580
942
|
end
|
581
943
|
|
944
|
+
def new_const_op_asgn val
|
945
|
+
lhs, (asgn_op, _), rhs = val
|
946
|
+
asgn_op = asgn_op.to_sym
|
947
|
+
|
948
|
+
result = case asgn_op
|
949
|
+
when :"||" then
|
950
|
+
s(:op_asgn_or, lhs, rhs)
|
951
|
+
when :"&&" then
|
952
|
+
s(:op_asgn_and, lhs, rhs)
|
953
|
+
else
|
954
|
+
s(:op_asgn, lhs, asgn_op, rhs)
|
955
|
+
end
|
956
|
+
|
957
|
+
result.line = lhs.line
|
958
|
+
result
|
959
|
+
end
|
960
|
+
|
582
961
|
def new_defn val
|
583
|
-
|
584
|
-
body ||= s(:nil)
|
962
|
+
_, (name, line), in_def, args, body, _ = val
|
585
963
|
|
586
|
-
|
964
|
+
body ||= s(:nil).line line
|
587
965
|
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
966
|
+
args.line line
|
967
|
+
|
968
|
+
result = s(:defn, name.to_sym, args).line line
|
969
|
+
|
970
|
+
if body.sexp_type == :block then
|
971
|
+
result.push(*body.sexp_body)
|
972
|
+
else
|
973
|
+
result.push body
|
594
974
|
end
|
595
975
|
|
596
|
-
args.line line
|
597
|
-
result.line = line
|
598
976
|
result.comments = self.comments.pop
|
599
977
|
|
600
|
-
result
|
978
|
+
[result, in_def]
|
601
979
|
end
|
602
980
|
|
603
981
|
def new_defs val
|
604
|
-
recv, name,
|
605
|
-
body ||= s(:nil)
|
982
|
+
_, recv, (name, line), in_def, args, body, _ = val
|
606
983
|
|
607
|
-
|
984
|
+
body ||= s(:nil).line line
|
608
985
|
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
986
|
+
args.line line
|
987
|
+
|
988
|
+
result = s(:defs, recv, name.to_sym, args).line line
|
989
|
+
|
990
|
+
# TODO: remove_begin
|
991
|
+
# TODO: reduce_nodes
|
992
|
+
|
993
|
+
if body.sexp_type == :block then
|
994
|
+
result.push(*body.sexp_body)
|
995
|
+
else
|
996
|
+
result.push body
|
615
997
|
end
|
616
998
|
|
617
|
-
result.line = recv.line
|
618
999
|
result.comments = self.comments.pop
|
619
|
-
|
1000
|
+
|
1001
|
+
[result, in_def]
|
1002
|
+
end
|
1003
|
+
|
1004
|
+
def new_do_body args, body, lineno
|
1005
|
+
new_iter(nil, args, body).line(lineno)
|
1006
|
+
end
|
1007
|
+
|
1008
|
+
def new_find_pattern const, pat
|
1009
|
+
pat.sexp_type = :find_pat
|
1010
|
+
pat.insert 1, const
|
1011
|
+
end
|
1012
|
+
|
1013
|
+
def new_find_pattern_tail lhs, mid, rhs
|
1014
|
+
lhs_id, line = lhs
|
1015
|
+
rhs_id, _line = rhs
|
1016
|
+
|
1017
|
+
# TODO: fpinfo->pre_rest_arg = pre_rest_arg ? assignable(p, pre_rest_arg, 0, loc) : NODE_SPECIAL_NO_NAME_REST;
|
1018
|
+
|
1019
|
+
lhs_id = "*#{lhs_id}".to_sym
|
1020
|
+
rhs_id = "*#{rhs_id}".to_sym
|
1021
|
+
|
1022
|
+
mid.sexp_type = :array_pat # HACK?
|
1023
|
+
|
1024
|
+
s(:find_pat_TAIL, lhs_id, mid, rhs_id).line line
|
620
1025
|
end
|
621
1026
|
|
622
1027
|
def new_for expr, var, body
|
@@ -626,7 +1031,49 @@ module RubyParserStuff
|
|
626
1031
|
end
|
627
1032
|
|
628
1033
|
def new_hash val
|
629
|
-
|
1034
|
+
_, line, assocs = val
|
1035
|
+
|
1036
|
+
s(:hash).line(line).concat assocs.sexp_body
|
1037
|
+
end
|
1038
|
+
|
1039
|
+
def new_hash_pattern const, hash_pat, loc
|
1040
|
+
_, pat, kw_args, kw_rest_arg = hash_pat
|
1041
|
+
|
1042
|
+
line = (const||hash_pat).line
|
1043
|
+
|
1044
|
+
result = s(:hash_pat, const).line line
|
1045
|
+
result.concat pat.sexp_body if pat
|
1046
|
+
result << kw_args if kw_args
|
1047
|
+
result << kw_rest_arg if kw_rest_arg
|
1048
|
+
result
|
1049
|
+
end
|
1050
|
+
|
1051
|
+
def new_hash_pattern_tail kw_args, kw_rest_arg, line # TODO: remove line arg
|
1052
|
+
# kw_rest_arg = assignable(kw_rest_arg, nil).line line if kw_rest_arg
|
1053
|
+
|
1054
|
+
result = s(:hash_pat).line line
|
1055
|
+
result << kw_args
|
1056
|
+
|
1057
|
+
if kw_rest_arg then
|
1058
|
+
name = kw_rest_arg.value
|
1059
|
+
# TODO: I _hate_ this:
|
1060
|
+
assignable [name, kw_rest_arg.line] if name != :**
|
1061
|
+
result << kw_rest_arg
|
1062
|
+
end
|
1063
|
+
|
1064
|
+
result
|
1065
|
+
end
|
1066
|
+
|
1067
|
+
def push_pktbl
|
1068
|
+
end
|
1069
|
+
|
1070
|
+
def pop_pktbl
|
1071
|
+
end
|
1072
|
+
|
1073
|
+
def push_pvtbl
|
1074
|
+
end
|
1075
|
+
|
1076
|
+
def pop_pvtbl
|
630
1077
|
end
|
631
1078
|
|
632
1079
|
def new_if c, t, f
|
@@ -647,33 +1094,68 @@ module RubyParserStuff
|
|
647
1094
|
result << args
|
648
1095
|
result << body if body
|
649
1096
|
|
650
|
-
|
1097
|
+
result.line call.line if call
|
651
1098
|
|
652
|
-
|
653
|
-
|
1099
|
+
unless args == 0 then
|
1100
|
+
args.line call.line if call
|
1101
|
+
args.sexp_type = :args
|
1102
|
+
end
|
654
1103
|
|
655
|
-
|
656
|
-
rhs = value_expr(rhs)
|
657
|
-
rhs = s(:to_ary, rhs) if wrap # HACK: could be array if lhs isn't right
|
658
|
-
rhs
|
1104
|
+
result
|
659
1105
|
end
|
660
1106
|
|
661
1107
|
def new_masgn lhs, rhs, wrap = false
|
662
1108
|
_, ary = lhs
|
663
1109
|
|
1110
|
+
line = rhs.line
|
664
1111
|
rhs = value_expr(rhs)
|
665
1112
|
rhs = ary ? s(:to_ary, rhs) : s(:array, rhs) if wrap
|
1113
|
+
rhs.line line if wrap
|
1114
|
+
|
1115
|
+
lhs.delete_at 1 if ary.nil?
|
1116
|
+
lhs << rhs
|
1117
|
+
|
1118
|
+
lhs
|
1119
|
+
end
|
1120
|
+
|
1121
|
+
def new_masgn_arg rhs, wrap = false
|
1122
|
+
rhs = value_expr(rhs)
|
1123
|
+
# HACK: could be array if lhs isn't right
|
1124
|
+
rhs = s(:to_ary, rhs).line rhs.line if wrap
|
1125
|
+
rhs
|
1126
|
+
end
|
1127
|
+
|
1128
|
+
def new_match lhs, rhs
|
1129
|
+
if lhs then
|
1130
|
+
case lhs.sexp_type
|
1131
|
+
when :dregx, :dregx_once then
|
1132
|
+
# TODO: no test coverage
|
1133
|
+
return s(:match2, lhs, rhs).line(lhs.line)
|
1134
|
+
when :lit then
|
1135
|
+
return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
|
1136
|
+
end
|
1137
|
+
end
|
666
1138
|
|
667
|
-
|
668
|
-
|
1139
|
+
if rhs then
|
1140
|
+
case rhs.sexp_type
|
1141
|
+
when :dregx, :dregx_once then
|
1142
|
+
# TODO: no test coverage
|
1143
|
+
return s(:match3, rhs, lhs).line(lhs.line)
|
1144
|
+
when :lit then
|
1145
|
+
return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
|
1146
|
+
end
|
1147
|
+
end
|
669
1148
|
|
670
|
-
lhs
|
1149
|
+
new_call(lhs, :"=~", argl(rhs)).line lhs.line
|
671
1150
|
end
|
672
1151
|
|
673
1152
|
def new_module val
|
1153
|
+
# TODO: get line from module keyword
|
674
1154
|
line, path, body = val[1], val[2], val[4]
|
675
1155
|
|
676
|
-
|
1156
|
+
path = path.first if path.instance_of? Array
|
1157
|
+
|
1158
|
+
result = s(:module, path).line line
|
677
1159
|
|
678
1160
|
if body then # REFACTOR?
|
679
1161
|
if body.sexp_type == :block then
|
@@ -683,39 +1165,49 @@ module RubyParserStuff
|
|
683
1165
|
end
|
684
1166
|
end
|
685
1167
|
|
686
|
-
result.line = line
|
687
1168
|
result.comments = self.comments.pop
|
688
1169
|
result
|
689
1170
|
end
|
690
1171
|
|
691
1172
|
def new_op_asgn val
|
692
|
-
lhs,
|
693
|
-
|
694
|
-
|
695
|
-
|
1173
|
+
lhs, (op, _line), rhs = val
|
1174
|
+
op = op.to_sym
|
1175
|
+
|
1176
|
+
name = gettable(lhs.last).line lhs.line
|
1177
|
+
arg = remove_begin rhs
|
1178
|
+
result = case op # REFACTOR
|
696
1179
|
when :"||" then
|
697
1180
|
lhs << arg
|
698
|
-
s(:op_asgn_or,
|
1181
|
+
s(:op_asgn_or, name, lhs).line lhs.line
|
699
1182
|
when :"&&" then
|
700
1183
|
lhs << arg
|
701
|
-
s(:op_asgn_and,
|
1184
|
+
s(:op_asgn_and, name, lhs).line lhs.line
|
702
1185
|
else
|
703
|
-
|
704
|
-
lhs[2] = new_call(self.gettable(name), asgn_op, argl(arg))
|
1186
|
+
lhs << new_call(name, op, argl(arg))
|
705
1187
|
lhs
|
706
1188
|
end
|
707
|
-
|
1189
|
+
|
1190
|
+
result
|
1191
|
+
end
|
1192
|
+
|
1193
|
+
def new_op_asgn1 val
|
1194
|
+
lhs, _, args, _, (op, _), rhs = val
|
1195
|
+
|
1196
|
+
args.sexp_type = :arglist if args
|
1197
|
+
|
1198
|
+
result = s(:op_asgn1, lhs, args, op.to_sym, rhs)
|
1199
|
+
result.line lhs.line
|
708
1200
|
result
|
709
1201
|
end
|
710
1202
|
|
711
1203
|
def new_op_asgn2 val
|
712
|
-
recv, call_op, meth, op, arg = val
|
1204
|
+
recv, (call_op, _), (meth, _), (op, _), arg = val
|
713
1205
|
meth = :"#{meth}="
|
714
1206
|
|
715
1207
|
result = case call_op.to_sym
|
716
|
-
when :
|
1208
|
+
when :"."
|
717
1209
|
s(:op_asgn2, recv, meth, op.to_sym, arg)
|
718
|
-
when :
|
1210
|
+
when :"&."
|
719
1211
|
s(:safe_op_asgn2, recv, meth, op.to_sym, arg)
|
720
1212
|
else
|
721
1213
|
raise "unknown call operator: `#{type.inspect}`"
|
@@ -725,27 +1217,44 @@ module RubyParserStuff
|
|
725
1217
|
result
|
726
1218
|
end
|
727
1219
|
|
1220
|
+
def new_qsym_list
|
1221
|
+
s(:array).line lexer.lineno
|
1222
|
+
end
|
1223
|
+
|
1224
|
+
def new_qsym_list_entry val
|
1225
|
+
_, (str, line), _ = val
|
1226
|
+
s(:lit, str.to_sym).line line
|
1227
|
+
end
|
1228
|
+
|
1229
|
+
def new_qword_list
|
1230
|
+
s(:array).line lexer.lineno
|
1231
|
+
end
|
1232
|
+
|
1233
|
+
def new_qword_list_entry val
|
1234
|
+
_, (str, line), _ = val
|
1235
|
+
str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
|
1236
|
+
s(:str, str).line line
|
1237
|
+
end
|
1238
|
+
|
728
1239
|
def new_regexp val
|
729
|
-
|
730
|
-
|
1240
|
+
(_, line), node, (options, _) = val
|
1241
|
+
|
1242
|
+
node ||= s(:str, "").line line
|
731
1243
|
|
732
1244
|
o, k = 0, nil
|
733
1245
|
options.split(//).uniq.each do |c| # FIX: this has a better home
|
734
1246
|
v = {
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
1247
|
+
"x" => Regexp::EXTENDED,
|
1248
|
+
"i" => Regexp::IGNORECASE,
|
1249
|
+
"m" => Regexp::MULTILINE,
|
1250
|
+
"o" => Regexp::ONCE,
|
1251
|
+
"n" => Regexp::ENC_NONE,
|
1252
|
+
"e" => Regexp::ENC_EUC,
|
1253
|
+
"s" => Regexp::ENC_SJIS,
|
1254
|
+
"u" => Regexp::ENC_UTF8,
|
743
1255
|
}[c]
|
744
1256
|
raise "unknown regexp option: #{c}" unless v
|
745
1257
|
o += v
|
746
|
-
|
747
|
-
# encoding options are ignored on 1.9+
|
748
|
-
k = c if c =~ /[esu]/ if RUBY_VERSION < "1.9"
|
749
1258
|
end
|
750
1259
|
|
751
1260
|
case node.sexp_type
|
@@ -757,12 +1266,12 @@ module RubyParserStuff
|
|
757
1266
|
begin
|
758
1267
|
Regexp.new(node[1], o)
|
759
1268
|
rescue RegexpError => e
|
760
|
-
warn "
|
1269
|
+
warn "WARNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
|
761
1270
|
begin
|
762
|
-
warn "
|
1271
|
+
warn "WARNING: trying to recover with ENC_UTF8"
|
763
1272
|
Regexp.new(node[1], Regexp::ENC_UTF8)
|
764
1273
|
rescue RegexpError => e
|
765
|
-
warn "
|
1274
|
+
warn "WARNING: trying to recover with ENC_NONE"
|
766
1275
|
Regexp.new(node[1], Regexp::ENC_NONE)
|
767
1276
|
end
|
768
1277
|
end
|
@@ -775,7 +1284,7 @@ module RubyParserStuff
|
|
775
1284
|
end
|
776
1285
|
node << o if o and o != 0
|
777
1286
|
else
|
778
|
-
node = s(:dregx,
|
1287
|
+
node = s(:dregx, "", node).line line
|
779
1288
|
node.sexp_type = :dregx_once if options =~ /o/
|
780
1289
|
node << o if o and o != 0
|
781
1290
|
end
|
@@ -789,9 +1298,14 @@ module RubyParserStuff
|
|
789
1298
|
else
|
790
1299
|
body = [body]
|
791
1300
|
end
|
1301
|
+
|
792
1302
|
s(:resbody, cond, *body).line cond.line
|
793
1303
|
end
|
794
1304
|
|
1305
|
+
def new_rescue body, resbody
|
1306
|
+
s(:rescue, body, resbody).line body.line
|
1307
|
+
end
|
1308
|
+
|
795
1309
|
def new_sclass val
|
796
1310
|
recv, in_def, in_single, body = val[3], val[4], val[6], val[7]
|
797
1311
|
|
@@ -812,90 +1326,55 @@ module RubyParserStuff
|
|
812
1326
|
end
|
813
1327
|
|
814
1328
|
def new_string val
|
815
|
-
str = val
|
816
|
-
str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
|
817
|
-
result = s(:str, str)
|
818
|
-
self.lexer.fixup_lineno str.count("\n")
|
819
|
-
result
|
820
|
-
end
|
821
|
-
|
822
|
-
def new_qword_list_entry val
|
823
|
-
str = val[1]
|
824
|
-
str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
|
825
|
-
result = s(:str, str)
|
826
|
-
self.lexer.fixup_lineno
|
827
|
-
result
|
828
|
-
end
|
829
|
-
|
830
|
-
def new_qword_list
|
831
|
-
result = s(:array)
|
832
|
-
self.lexer.fixup_lineno
|
833
|
-
result
|
834
|
-
end
|
835
|
-
|
836
|
-
def new_word_list
|
837
|
-
result = s(:array)
|
838
|
-
self.lexer.fixup_lineno
|
839
|
-
result
|
840
|
-
end
|
1329
|
+
(str, line), = val
|
841
1330
|
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
1331
|
+
str.force_encoding("UTF-8")
|
1332
|
+
# TODO: remove:
|
1333
|
+
str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
|
1334
|
+
s(:str, str).line line
|
846
1335
|
end
|
847
1336
|
|
848
|
-
def
|
849
|
-
|
850
|
-
|
851
|
-
|
1337
|
+
def new_super args
|
1338
|
+
if args && args.sexp_type == :block_pass then
|
1339
|
+
s(:super, args).line args.line
|
1340
|
+
else
|
1341
|
+
args ||= s(:arglist).line lexer.lineno
|
1342
|
+
s(:super, *args.sexp_body).line args.line
|
1343
|
+
end
|
852
1344
|
end
|
853
1345
|
|
854
|
-
def
|
855
|
-
|
856
|
-
|
857
|
-
result
|
1346
|
+
def new_symbol val
|
1347
|
+
name = val.last
|
1348
|
+
s(:lit, name.to_sym).line lexer.lineno
|
858
1349
|
end
|
859
1350
|
|
860
1351
|
def new_symbol_list
|
861
|
-
|
862
|
-
|
863
|
-
result
|
1352
|
+
# TODO: hunt down and try to remove ALL lexer.lineno usage!
|
1353
|
+
s(:array).line lexer.lineno
|
864
1354
|
end
|
865
1355
|
|
866
1356
|
def new_symbol_list_entry val
|
867
|
-
|
868
|
-
result = val[1]
|
1357
|
+
_, sym, _ = val
|
869
1358
|
|
870
|
-
|
1359
|
+
sym ||= s(:str, "").line lexer.lineno
|
871
1360
|
|
872
1361
|
case sym.sexp_type
|
873
1362
|
when :dstr then
|
874
1363
|
sym.sexp_type = :dsym
|
875
1364
|
when :str then
|
876
|
-
sym = s(:lit, sym.last.to_sym)
|
1365
|
+
sym = s(:lit, sym.last.to_sym).line sym.line
|
877
1366
|
else
|
878
|
-
|
879
|
-
sym = s(:dsym, "", sym || s(:str, ""))
|
1367
|
+
sym = s(:dsym, "", sym).line sym.line
|
880
1368
|
end
|
881
|
-
self.lexer.fixup_lineno
|
882
|
-
sym
|
883
|
-
end
|
884
1369
|
|
885
|
-
|
886
|
-
if args && args.node_type == :block_pass then
|
887
|
-
s(:super, args)
|
888
|
-
else
|
889
|
-
args ||= s(:arglist)
|
890
|
-
s(:super, *args.sexp_body)
|
891
|
-
end
|
1370
|
+
sym
|
892
1371
|
end
|
893
1372
|
|
894
1373
|
def new_undef n, m = nil
|
895
1374
|
if m then
|
896
|
-
block_append(n, s(:undef, m))
|
1375
|
+
block_append(n, s(:undef, m).line(m.line))
|
897
1376
|
else
|
898
|
-
s(:undef, n)
|
1377
|
+
s(:undef, n).line n.line
|
899
1378
|
end
|
900
1379
|
end
|
901
1380
|
|
@@ -928,138 +1407,78 @@ module RubyParserStuff
|
|
928
1407
|
new_until_or_while :while, block, expr, pre
|
929
1408
|
end
|
930
1409
|
|
931
|
-
def
|
932
|
-
|
933
|
-
|
1410
|
+
def new_word_list
|
1411
|
+
s(:array).line lexer.lineno
|
1412
|
+
end
|
1413
|
+
|
1414
|
+
def new_word_list_entry val
|
1415
|
+
_, word, _ = val
|
1416
|
+
word.sexp_type == :evstr ? s(:dstr, "", word).line(word.line) : word
|
1417
|
+
end
|
1418
|
+
|
1419
|
+
def new_xstring val
|
1420
|
+
_, node = val
|
1421
|
+
|
1422
|
+
node ||= s(:str, "").line lexer.lineno
|
1423
|
+
|
1424
|
+
if node then
|
1425
|
+
case node.sexp_type
|
934
1426
|
when :str
|
935
|
-
|
1427
|
+
node.sexp_type = :xstr
|
936
1428
|
when :dstr
|
937
|
-
|
1429
|
+
node.sexp_type = :dxstr
|
938
1430
|
else
|
939
|
-
|
1431
|
+
node = s(:dxstr, "", node).line node.line
|
940
1432
|
end
|
941
|
-
str
|
942
|
-
else
|
943
|
-
s(:xstr, '')
|
944
1433
|
end
|
1434
|
+
|
1435
|
+
node
|
945
1436
|
end
|
946
1437
|
|
947
1438
|
def new_yield args = nil
|
948
1439
|
# TODO: raise args.inspect unless [:arglist].include? args.first # HACK
|
949
|
-
raise "write a test 4" if args && args.
|
1440
|
+
raise "write a test 4" if args && args.sexp_type == :block_pass
|
950
1441
|
raise SyntaxError, "Block argument should not be given." if
|
951
|
-
args && args.
|
1442
|
+
args && args.sexp_type == :block_pass
|
952
1443
|
|
953
|
-
args ||= s(:arglist)
|
1444
|
+
args ||= s(:arglist).line lexer.lineno
|
954
1445
|
|
955
1446
|
args.sexp_type = :arglist if [:call_args, :array].include? args.sexp_type
|
956
|
-
args = s(:arglist, args) unless args.sexp_type == :arglist
|
957
|
-
|
958
|
-
return s(:yield, *args.sexp_body)
|
959
|
-
end
|
1447
|
+
args = s(:arglist, args).line args.line unless args.sexp_type == :arglist
|
960
1448
|
|
961
|
-
|
962
|
-
token = self.lexer.next_token
|
963
|
-
|
964
|
-
if token and token.first != RubyLexer::EOF then
|
965
|
-
return token
|
966
|
-
else
|
967
|
-
return [false, '$end']
|
968
|
-
end
|
1449
|
+
s(:yield, *args.sexp_body).line args.line
|
969
1450
|
end
|
970
1451
|
|
971
|
-
def
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
case lhs.sexp_type
|
977
|
-
when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
|
978
|
-
lhs << rhs
|
979
|
-
when :const then
|
980
|
-
lhs.sexp_type = :cdecl
|
981
|
-
lhs << rhs
|
1452
|
+
def prev_value_to_lineno v
|
1453
|
+
s, n = v
|
1454
|
+
if String === s then
|
1455
|
+
n
|
982
1456
|
else
|
983
|
-
|
1457
|
+
lexer.lineno
|
984
1458
|
end
|
985
|
-
|
986
|
-
lhs
|
987
1459
|
end
|
988
1460
|
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
##
|
993
|
-
# Returns a UTF-8 encoded string after processing BOMs and magic
|
994
|
-
# encoding comments.
|
995
|
-
#
|
996
|
-
# Holy crap... ok. Here goes:
|
997
|
-
#
|
998
|
-
# Ruby's file handling and encoding support is insane. We need to be
|
999
|
-
# able to lex a file. The lexer file is explicitly UTF-8 to make
|
1000
|
-
# things cleaner. This allows us to deal with extended chars in
|
1001
|
-
# class and method names. In order to do this, we need to encode all
|
1002
|
-
# input source files as UTF-8. First, we look for a UTF-8 BOM by
|
1003
|
-
# looking at the first line while forcing its encoding to
|
1004
|
-
# ASCII-8BIT. If we find a BOM, we strip it and set the expected
|
1005
|
-
# encoding to UTF-8. Then, we search for a magic encoding comment.
|
1006
|
-
# If found, it overrides the BOM. Finally, we force the encoding of
|
1007
|
-
# the input string to whatever was found, and then encode that to
|
1008
|
-
# UTF-8 for compatibility with the lexer.
|
1009
|
-
|
1010
|
-
def handle_encoding str
|
1011
|
-
str = str.dup
|
1012
|
-
has_enc = str.respond_to? :encoding
|
1013
|
-
encoding = nil
|
1014
|
-
|
1015
|
-
header = str.each_line.first(2)
|
1016
|
-
header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
|
1017
|
-
|
1018
|
-
first = header.first || ""
|
1019
|
-
encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
|
1020
|
-
|
1021
|
-
encoding = $1.strip if header.find { |s|
|
1022
|
-
s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
|
1023
|
-
s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
|
1024
|
-
}
|
1461
|
+
def next_token
|
1462
|
+
token = self.lexer.next_token
|
1025
1463
|
|
1026
|
-
if
|
1027
|
-
|
1028
|
-
|
1029
|
-
|
1030
|
-
|
1031
|
-
warn "Skipping magic encoding comment"
|
1032
|
-
end
|
1464
|
+
if token and token.first != RubyLexer::EOF then
|
1465
|
+
self.last_token_type = token
|
1466
|
+
return token
|
1467
|
+
elsif !token
|
1468
|
+
return self.lexer.next_token
|
1033
1469
|
else
|
1034
|
-
|
1035
|
-
hack_encoding str if has_enc
|
1470
|
+
return [false, false]
|
1036
1471
|
end
|
1037
|
-
|
1038
|
-
str
|
1039
1472
|
end
|
1040
1473
|
|
1041
|
-
def
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
#
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1049
|
-
if str.valid_encoding? then
|
1050
|
-
str.encode! Encoding::UTF_8
|
1051
|
-
break
|
1052
|
-
end
|
1053
|
-
rescue Encoding::InvalidByteSequenceError
|
1054
|
-
# do nothing
|
1055
|
-
rescue Encoding::UndefinedConversionError
|
1056
|
-
# do nothing
|
1057
|
-
end
|
1058
|
-
end
|
1059
|
-
|
1060
|
-
# no amount of pain is enough for you.
|
1061
|
-
raise "Bad encoding. Need a magic encoding comment." unless
|
1062
|
-
str.encoding.name == "UTF-8"
|
1474
|
+
def on_error(et, ev, values)
|
1475
|
+
ev = ev.first if ev.instance_of?(Array) && ev.size == 2 && ev.last.is_a?(Integer)
|
1476
|
+
super
|
1477
|
+
rescue Racc::ParseError => e
|
1478
|
+
# I don't like how the exception obscures the error message
|
1479
|
+
e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
|
1480
|
+
warn e.message if $DEBUG
|
1481
|
+
raise
|
1063
1482
|
end
|
1064
1483
|
|
1065
1484
|
##
|
@@ -1067,56 +1486,44 @@ module RubyParserStuff
|
|
1067
1486
|
# Timeout::Error if it runs for more than +time+ seconds.
|
1068
1487
|
|
1069
1488
|
def process(str, file = "(string)", time = 10)
|
1489
|
+
str.freeze
|
1490
|
+
|
1070
1491
|
Timeout.timeout time do
|
1071
1492
|
raise "bad val: #{str.inspect}" unless String === str
|
1072
1493
|
|
1073
|
-
|
1494
|
+
self.lexer.string = handle_encoding str
|
1074
1495
|
|
1075
1496
|
self.file = file.dup
|
1076
1497
|
|
1077
|
-
@yydebug = ENV.has_key?
|
1078
|
-
|
1079
|
-
# HACK -- need to get tests passing more than have graceful code
|
1080
|
-
self.lexer.ss = RPStringScanner.new str
|
1498
|
+
@yydebug = ENV.has_key? "DEBUG"
|
1081
1499
|
|
1082
1500
|
do_parse
|
1083
1501
|
end
|
1084
1502
|
end
|
1085
1503
|
|
1086
|
-
alias
|
1504
|
+
alias parse process
|
1087
1505
|
|
1088
1506
|
def remove_begin node
|
1089
|
-
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1093
|
-
|
1507
|
+
line = node.line
|
1508
|
+
|
1509
|
+
node = node.last while node and node.sexp_type == :begin and node.size == 2
|
1510
|
+
|
1511
|
+
node = s(:nil) if node == s(:begin)
|
1512
|
+
|
1513
|
+
node.line ||= line
|
1514
|
+
|
1094
1515
|
node
|
1095
1516
|
end
|
1096
1517
|
|
1518
|
+
alias value_expr remove_begin # TODO: for now..? could check the tree, but meh?
|
1519
|
+
|
1097
1520
|
def reset
|
1098
1521
|
lexer.reset
|
1099
1522
|
self.in_def = false
|
1100
1523
|
self.in_single = 0
|
1101
1524
|
self.env.reset
|
1102
1525
|
self.comments.clear
|
1103
|
-
|
1104
|
-
|
1105
|
-
def block_dup_check call_or_args, block
|
1106
|
-
syntax_error "Both block arg and actual block given." if
|
1107
|
-
block and call_or_args.block_pass?
|
1108
|
-
end
|
1109
|
-
|
1110
|
-
def inverted? val
|
1111
|
-
[:return, :next, :break, :yield].include? val[0].sexp_type
|
1112
|
-
end
|
1113
|
-
|
1114
|
-
def invert_block_call val
|
1115
|
-
(type, call), iter = val
|
1116
|
-
|
1117
|
-
iter.insert 1, call
|
1118
|
-
|
1119
|
-
[iter, s(type)]
|
1526
|
+
self.last_token_type = nil
|
1120
1527
|
end
|
1121
1528
|
|
1122
1529
|
def ret_args node
|
@@ -1131,7 +1538,7 @@ module RubyParserStuff
|
|
1131
1538
|
|
1132
1539
|
# HACK matz wraps ONE of the FOUR splats in a newline to
|
1133
1540
|
# distinguish. I use paren for now. ugh
|
1134
|
-
node = s(:svalue, node) if node.sexp_type == :splat and not node.paren
|
1541
|
+
node = s(:svalue, node).line node.line if node.sexp_type == :splat and not node.paren
|
1135
1542
|
node.sexp_type = :svalue if node.sexp_type == :arglist && node[1].sexp_type == :splat
|
1136
1543
|
end
|
1137
1544
|
|
@@ -1140,18 +1547,25 @@ module RubyParserStuff
|
|
1140
1547
|
|
1141
1548
|
def s(*args)
|
1142
1549
|
result = Sexp.new(*args)
|
1143
|
-
result.line ||= lexer.lineno if lexer.ss
|
1550
|
+
# result.line ||= lexer.lineno if lexer.ss unless ENV["CHECK_LINE_NUMS"] # otherwise...
|
1144
1551
|
result.file = self.file
|
1145
1552
|
result
|
1146
1553
|
end
|
1147
1554
|
|
1148
|
-
def
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1555
|
+
def debug n
|
1556
|
+
if ENV["PRY"] then
|
1557
|
+
require "pry"; binding.pry
|
1558
|
+
end
|
1559
|
+
|
1560
|
+
raise RubyParser::SyntaxError, "debug #{n}"
|
1561
|
+
end
|
1562
|
+
|
1563
|
+
def syntax_error msg
|
1564
|
+
raise RubyParser::SyntaxError, msg
|
1153
1565
|
end
|
1154
1566
|
|
1567
|
+
alias yyerror syntax_error
|
1568
|
+
|
1155
1569
|
def void_stmts node
|
1156
1570
|
return nil unless node
|
1157
1571
|
return node unless node.sexp_type == :block
|
@@ -1169,18 +1583,39 @@ module RubyParserStuff
|
|
1169
1583
|
# do nothing for now
|
1170
1584
|
end
|
1171
1585
|
|
1172
|
-
|
1586
|
+
def whitespace_width line, remove_width = nil
|
1587
|
+
col = 0
|
1588
|
+
idx = 0
|
1173
1589
|
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1590
|
+
line.chars.each do |c|
|
1591
|
+
break if remove_width && col >= remove_width
|
1592
|
+
case c
|
1593
|
+
when " " then
|
1594
|
+
col += 1
|
1595
|
+
when "\t" then
|
1596
|
+
n = TAB_WIDTH * (col / TAB_WIDTH + 1)
|
1597
|
+
break if remove_width && n > remove_width
|
1598
|
+
col = n
|
1599
|
+
else
|
1600
|
+
break
|
1601
|
+
end
|
1602
|
+
idx += 1
|
1603
|
+
end
|
1604
|
+
|
1605
|
+
if remove_width then
|
1606
|
+
line[idx..-1]
|
1607
|
+
elsif line[idx].nil?
|
1608
|
+
nil
|
1609
|
+
else
|
1610
|
+
col
|
1611
|
+
end
|
1181
1612
|
end
|
1182
1613
|
|
1614
|
+
alias remove_whitespace_width whitespace_width
|
1615
|
+
|
1183
1616
|
class Keyword
|
1617
|
+
include RubyLexer::State::Values
|
1618
|
+
|
1184
1619
|
class KWtable
|
1185
1620
|
attr_accessor :name, :state, :id0, :id1
|
1186
1621
|
def initialize(name, id=[], state=nil)
|
@@ -1193,82 +1628,75 @@ module RubyParserStuff
|
|
1193
1628
|
##
|
1194
1629
|
# :stopdoc:
|
1195
1630
|
#
|
1196
|
-
# :expr_beg
|
1197
|
-
# :expr_end
|
1198
|
-
# :
|
1199
|
-
# :
|
1200
|
-
# :
|
1201
|
-
# :
|
1202
|
-
# :
|
1203
|
-
# :
|
1204
|
-
# :
|
1631
|
+
# :expr_beg = ignore newline, +/- is a sign.
|
1632
|
+
# :expr_end = newline significant, +/- is an operator.
|
1633
|
+
# :expr_endarg = ditto, and unbound braces.
|
1634
|
+
# :expr_endfn = ditto, and unbound braces.
|
1635
|
+
# :expr_arg = newline significant, +/- is an operator.
|
1636
|
+
# :expr_cmdarg = ditto
|
1637
|
+
# :expr_mid = ditto
|
1638
|
+
# :expr_fname = ignore newline, no reserved words.
|
1639
|
+
# :expr_dot = right after . or ::, no reserved words.
|
1640
|
+
# :expr_class = immediate after class, no here document.
|
1641
|
+
# :expr_label = flag bit, label is allowed.
|
1642
|
+
# :expr_labeled = flag bit, just after a label.
|
1643
|
+
# :expr_fitem = symbol literal as FNAME.
|
1644
|
+
# :expr_value = :expr_beg -- work to remove. Need multi-state support.
|
1645
|
+
|
1646
|
+
expr_woot = EXPR_FNAME|EXPR_FITEM
|
1205
1647
|
|
1206
1648
|
wordlist = [
|
1207
|
-
["
|
1208
|
-
["
|
1209
|
-
["
|
1210
|
-
["
|
1211
|
-
["
|
1212
|
-
["
|
1213
|
-
["def", [:kDEF, :kDEF ],
|
1214
|
-
["
|
1215
|
-
["
|
1216
|
-
["
|
1217
|
-
["
|
1218
|
-
["
|
1219
|
-
["
|
1220
|
-
["false", [:kFALSE, :kFALSE ],
|
1221
|
-
["
|
1222
|
-
["
|
1223
|
-
["
|
1224
|
-
["
|
1225
|
-
["
|
1226
|
-
["
|
1227
|
-
["
|
1228
|
-
["
|
1229
|
-
["
|
1230
|
-
["
|
1231
|
-
["
|
1232
|
-
["
|
1233
|
-
["
|
1234
|
-
["
|
1235
|
-
["
|
1236
|
-
["
|
1237
|
-
["
|
1238
|
-
["
|
1239
|
-
["
|
1240
|
-
["
|
1241
|
-
["
|
1242
|
-
["
|
1243
|
-
["
|
1244
|
-
["
|
1245
|
-
["
|
1246
|
-
["
|
1247
|
-
["__ENCODING__", [:k__ENCODING__, :k__ENCODING__],
|
1248
|
-
].map { |args|
|
1649
|
+
["alias", [:kALIAS, :kALIAS ], expr_woot ],
|
1650
|
+
["and", [:kAND, :kAND ], EXPR_BEG ],
|
1651
|
+
["begin", [:kBEGIN, :kBEGIN ], EXPR_BEG ],
|
1652
|
+
["break", [:kBREAK, :kBREAK ], EXPR_MID ],
|
1653
|
+
["case", [:kCASE, :kCASE ], EXPR_BEG ],
|
1654
|
+
["class", [:kCLASS, :kCLASS ], EXPR_CLASS ],
|
1655
|
+
["def", [:kDEF, :kDEF ], EXPR_FNAME ],
|
1656
|
+
["defined?", [:kDEFINED, :kDEFINED ], EXPR_ARG ],
|
1657
|
+
["do", [:kDO, :kDO ], EXPR_BEG ],
|
1658
|
+
["else", [:kELSE, :kELSE ], EXPR_BEG ],
|
1659
|
+
["elsif", [:kELSIF, :kELSIF ], EXPR_BEG ],
|
1660
|
+
["end", [:kEND, :kEND ], EXPR_END ],
|
1661
|
+
["ensure", [:kENSURE, :kENSURE ], EXPR_BEG ],
|
1662
|
+
["false", [:kFALSE, :kFALSE ], EXPR_END ],
|
1663
|
+
["for", [:kFOR, :kFOR ], EXPR_BEG ],
|
1664
|
+
["if", [:kIF, :kIF_MOD ], EXPR_BEG ],
|
1665
|
+
["in", [:kIN, :kIN ], EXPR_BEG ],
|
1666
|
+
["module", [:kMODULE, :kMODULE ], EXPR_BEG ],
|
1667
|
+
["next", [:kNEXT, :kNEXT ], EXPR_MID ],
|
1668
|
+
["nil", [:kNIL, :kNIL ], EXPR_END ],
|
1669
|
+
["not", [:kNOT, :kNOT ], EXPR_ARG ],
|
1670
|
+
["or", [:kOR, :kOR ], EXPR_BEG ],
|
1671
|
+
["redo", [:kREDO, :kREDO ], EXPR_END ],
|
1672
|
+
["rescue", [:kRESCUE, :kRESCUE_MOD ], EXPR_MID ],
|
1673
|
+
["retry", [:kRETRY, :kRETRY ], EXPR_END ],
|
1674
|
+
["return", [:kRETURN, :kRETURN ], EXPR_MID ],
|
1675
|
+
["self", [:kSELF, :kSELF ], EXPR_END ],
|
1676
|
+
["super", [:kSUPER, :kSUPER ], EXPR_ARG ],
|
1677
|
+
["then", [:kTHEN, :kTHEN ], EXPR_BEG ],
|
1678
|
+
["true", [:kTRUE, :kTRUE ], EXPR_END ],
|
1679
|
+
["undef", [:kUNDEF, :kUNDEF ], expr_woot ],
|
1680
|
+
["unless", [:kUNLESS, :kUNLESS_MOD ], EXPR_BEG ],
|
1681
|
+
["until", [:kUNTIL, :kUNTIL_MOD ], EXPR_BEG ],
|
1682
|
+
["when", [:kWHEN, :kWHEN ], EXPR_BEG ],
|
1683
|
+
["while", [:kWHILE, :kWHILE_MOD ], EXPR_BEG ],
|
1684
|
+
["yield", [:kYIELD, :kYIELD ], EXPR_ARG ],
|
1685
|
+
["BEGIN", [:klBEGIN, :klBEGIN ], EXPR_END ],
|
1686
|
+
["END", [:klEND, :klEND ], EXPR_END ],
|
1687
|
+
["__FILE__", [:k__FILE__, :k__FILE__ ], EXPR_END ],
|
1688
|
+
["__LINE__", [:k__LINE__, :k__LINE__ ], EXPR_END ],
|
1689
|
+
["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], EXPR_END],
|
1690
|
+
].map { |args|
|
1691
|
+
KWtable.new(*args)
|
1692
|
+
}
|
1249
1693
|
|
1250
1694
|
# :startdoc:
|
1251
1695
|
|
1252
|
-
|
1253
|
-
WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
|
1254
|
-
|
1255
|
-
WORDLIST18.delete "__ENCODING__"
|
1256
|
-
|
1257
|
-
%w[and case elsif for if in module or unless until when while].each do |k|
|
1258
|
-
WORDLIST19[k] = WORDLIST19[k].dup
|
1259
|
-
WORDLIST19[k].state = :expr_value
|
1260
|
-
end
|
1261
|
-
%w[not].each do |k|
|
1262
|
-
WORDLIST19[k] = WORDLIST19[k].dup
|
1263
|
-
WORDLIST19[k].state = :expr_arg
|
1264
|
-
end
|
1265
|
-
|
1266
|
-
def self.keyword18 str # REFACTOR
|
1267
|
-
WORDLIST18[str]
|
1268
|
-
end
|
1696
|
+
WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
|
1269
1697
|
|
1270
|
-
def self.
|
1271
|
-
|
1698
|
+
def self.keyword str
|
1699
|
+
WORDLIST[str]
|
1272
1700
|
end
|
1273
1701
|
end
|
1274
1702
|
|
@@ -1328,51 +1756,57 @@ module RubyParserStuff
|
|
1328
1756
|
@debug = debug
|
1329
1757
|
end
|
1330
1758
|
|
1331
|
-
def reset
|
1332
|
-
@stack = [false]
|
1333
|
-
warn "#{name}_stack(set): 0" if debug
|
1334
|
-
end
|
1335
|
-
|
1336
1759
|
def inspect
|
1337
1760
|
"StackState(#{@name}, #{@stack.inspect})"
|
1338
1761
|
end
|
1339
1762
|
|
1340
1763
|
def is_in_state
|
1764
|
+
log :is_in_state if debug
|
1341
1765
|
@stack.last
|
1342
1766
|
end
|
1343
1767
|
|
1344
1768
|
def lexpop
|
1345
|
-
warn "#{name}_stack.lexpop" if debug
|
1346
1769
|
raise if @stack.size == 0
|
1347
1770
|
a = @stack.pop
|
1348
1771
|
b = @stack.pop
|
1349
1772
|
@stack.push(a || b)
|
1773
|
+
log :lexpop if debug
|
1774
|
+
end
|
1775
|
+
|
1776
|
+
def log action
|
1777
|
+
c = caller[1]
|
1778
|
+
c = caller[2] if c =~ /expr_result/
|
1779
|
+
warn "%s_stack.%s: %p at %s" % [name, action, @stack, c.clean_caller]
|
1780
|
+
nil
|
1350
1781
|
end
|
1351
1782
|
|
1352
1783
|
def pop
|
1353
1784
|
r = @stack.pop
|
1354
|
-
|
1355
|
-
|
1785
|
+
@stack.push false if @stack.empty?
|
1786
|
+
log :pop if debug
|
1356
1787
|
r
|
1357
1788
|
end
|
1358
1789
|
|
1359
1790
|
def push val
|
1360
1791
|
@stack.push val
|
1361
|
-
|
1362
|
-
c = caller.first
|
1363
|
-
c = caller[1] if c =~ /expr_result/
|
1364
|
-
warn "#{name}_stack(push): #{val} at line #{c.clean_caller}"
|
1365
|
-
nil
|
1792
|
+
log :push if debug
|
1366
1793
|
end
|
1367
1794
|
|
1368
|
-
def
|
1369
|
-
|
1370
|
-
|
1371
|
-
result
|
1795
|
+
def reset
|
1796
|
+
@stack = [false]
|
1797
|
+
log :reset if debug
|
1372
1798
|
end
|
1373
1799
|
|
1374
1800
|
def restore oldstate
|
1375
1801
|
@stack.replace oldstate
|
1802
|
+
log :restore if debug
|
1803
|
+
end
|
1804
|
+
|
1805
|
+
def store base = false
|
1806
|
+
result = @stack.dup
|
1807
|
+
@stack.replace [base]
|
1808
|
+
log :store if debug
|
1809
|
+
result
|
1376
1810
|
end
|
1377
1811
|
end
|
1378
1812
|
end
|