ripper_ruby_parser 1.4.1 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f75852f6918c25ff33b1edfdd19ed7dc6e09662dd5786b37afe5cc900433db6b
4
- data.tar.gz: 1f8b9b066b7dff6170ba7a1c0487a474314a1c67e8d5b63c3720aae4ae498775
3
+ metadata.gz: ab036613dacd6865ca8103db78c67273d6fad340e3a6a7ed1fe5450c5511cc05
4
+ data.tar.gz: 182feec3370738d6d5516e5615022a045ed717edf489cc694ac02eda97e6e6bb
5
5
  SHA512:
6
- metadata.gz: 040b94d89f959fdab917a244cea6649d27dbf1f9fa390753f09b025f6feac6778425a154592d9f252b1f60d28473865e78f6e993d6ba75658cd8d67aeed95d53
7
- data.tar.gz: 0d947570348eb461628a027200ad456968261980925087915c9b7ec7f281ec9e789b3d9facf03090f7cecf3f879e4e7fcf1a1ff41a259fb036737d371d6a31e1
6
+ metadata.gz: 9c5f17c00265aaf9df0741c5afee3b90f000141d4285bce067953bb1feac645779c8e4b9db2319baa9a3f6822b46a9f6edbeef87af1177b22a03cd437fecad51
7
+ data.tar.gz: 7001633d0756ab5afb2b2813572550f3cea1df75c11e82fe8117e4243aaa6cf4386512b3a7754b76f0a5a8be7c4b49f8b1e5a0148b4465e7aa72658b7c951acc
@@ -1,5 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.4.2 / 2018-04-03
4
+
5
+ * Fix handling of strings delimited by %()
6
+ * Handle line continuations in stringlike literals
7
+ - Handle line continuations in string and regexp literals
8
+ - Handle escaped line continuations
9
+ - Hanlde line continuations in word and symbol list literals
10
+ * Force encoding of string literals to UTF-8 if the result is valid
11
+ * Fix handling of range operators with float literals
12
+
3
13
  ## 1.4.1 / 2018-03-31
4
14
 
5
15
  * Properly pop delimiter stack after parsing a symbol
@@ -7,7 +17,8 @@
7
17
  ## 1.4.0 / 2018-03-30
8
18
 
9
19
  * Handle begin..end blocks with postfix conditionals
10
- * Correctly handle string variants that do not allow escape sequences
20
+ * Match RubyParser's handling of string literals that do not allow escape
21
+ sequences
11
22
 
12
23
  ## 1.3.0 / 2018-02-17
13
24
 
@@ -179,10 +179,18 @@ module RipperRubyParser
179
179
 
180
180
  def on_tstring_content(content)
181
181
  content = case @delimiter_stack.last
182
- when '"', '`', ':"', /^<</, /^%I/, /^%Q/, /^%W/
182
+ when /^<</
183
183
  Unescape.unescape(content)
184
- when "'", ":'"
184
+ when '"', '`', ':"', /^%Q.$/, /^%.$/
185
+ Unescape.fix_encoding(Unescape.unescape(content))
186
+ when /^%[WI].$/
187
+ Unescape.fix_encoding(Unescape.unescape_wordlist_word(content))
188
+ when "'", ":'", /^%q.$/
185
189
  Unescape.simple_unescape(content)
190
+ when '/', /^%r.$/
191
+ Unescape.unescape_regexp(content)
192
+ when /^%[wi].$/
193
+ Unescape.simple_unescape_wordlist_word(content)
186
194
  else
187
195
  content
188
196
  end
@@ -36,8 +36,8 @@ module RipperRubyParser
36
36
  items.push(*map_process_list(rest))
37
37
  end
38
38
 
39
- def literal?(exp)
40
- exp.sexp_type == :lit
39
+ def integer_literal?(exp)
40
+ exp.sexp_type == :lit && exp[1].is_a?(Integer)
41
41
  end
42
42
 
43
43
  def reject_void_stmt(body)
@@ -46,7 +46,7 @@ module RipperRubyParser
46
46
  _, left, right = exp.shift 3
47
47
  left = process(left)
48
48
  right = process(right)
49
- if literal?(left) && literal?(right)
49
+ if integer_literal?(left) && integer_literal?(right)
50
50
  s(:lit, Range.new(left[1], right[1]))
51
51
  else
52
52
  s(:dot2, left, right)
@@ -57,7 +57,7 @@ module RipperRubyParser
57
57
  _, left, right = exp.shift 3
58
58
  left = process(left)
59
59
  right = process(right)
60
- if literal?(left) && literal?(right)
60
+ if integer_literal?(left) && integer_literal?(right)
61
61
  s(:lit, Range.new(left[1], right[1], true))
62
62
  else
63
63
  s(:dot3, left, right)
@@ -5,6 +5,22 @@ module RipperRubyParser
5
5
  module Unescape
6
6
  module_function
7
7
 
8
+ ESCAPE_SEQUENCE_REGEXP =
9
+ /\\(
10
+ [0-7]{1,3} | # octal character
11
+ x[0-9a-fA-F]{1,2} | # hex byte
12
+ u[0-9a-fA-F]{4} | # unicode character
13
+ M-\\C-. | # meta-ctrl
14
+ C-\\M-. | # ctrl-meta
15
+ M-\\c. | # meta-ctrl (shorthand)
16
+ c\\M-. | # ctrl-meta (shorthand)
17
+ C-. | # control (regular)
18
+ c. | # control (shorthand)
19
+ M-. | # meta
20
+ \n | # line continuation
21
+ . # single-character
22
+ )/x
23
+
8
24
  SINGLE_LETTER_ESCAPES = {
9
25
  'a' => "\a",
10
26
  'b' => "\b",
@@ -29,40 +45,75 @@ module RipperRubyParser
29
45
  end
30
46
  end
31
47
 
32
- def unescape(string)
48
+ def simple_unescape_wordlist_word(string)
33
49
  string.gsub(/\\(
34
- [0-7]{1,3} | # octal character
35
- x[0-9a-fA-F]{1,2} | # hex byte
36
- u[0-9a-fA-F]{4} | # unicode character
37
- M-\\C-. | # meta-ctrl
38
- C-\\M-. | # ctrl-meta
39
- M-\\c. | # meta-ctrl (shorthand)
40
- c\\M-. | # ctrl-meta (shorthand)
41
- C-. | # control (regular)
42
- c. | # control (shorthand)
43
- M-. | # meta
44
- . # single-character
50
+ ' | # single quote
51
+ \\ | # backslash
52
+ \n # newline
45
53
  )/x) do
54
+ Regexp.last_match[1]
55
+ end
56
+ end
57
+
58
+ def unescape(string)
59
+ string.gsub(ESCAPE_SEQUENCE_REGEXP) do
60
+ bare = Regexp.last_match[1]
61
+ if bare == "\n"
62
+ ''
63
+ else
64
+ unescaped_value(bare)
65
+ end
66
+ end
67
+ end
68
+
69
+ def unescape_wordlist_word(string)
70
+ string.gsub(ESCAPE_SEQUENCE_REGEXP) do
71
+ bare = Regexp.last_match[1]
72
+ unescaped_value(bare)
73
+ end
74
+ end
75
+
76
+ def fix_encoding(string)
77
+ unless string.encoding == Encoding::UTF_8
78
+ dup = string.dup.force_encoding Encoding::UTF_8
79
+ return dup if dup.valid_encoding?
80
+ end
81
+ string
82
+ end
83
+
84
+ def unescape_regexp(string)
85
+ string.gsub(/\\(\n|\\)/) do
46
86
  bare = Regexp.last_match[1]
47
87
  case bare
48
- when SINGLE_LETTER_ESCAPES_REGEXP
49
- SINGLE_LETTER_ESCAPES[bare]
50
- when /^x/
51
- bare[1..-1].to_i(16).chr
52
- when /^u/
53
- bare[1..-1].to_i(16).chr(Encoding::UTF_8)
54
- when /^(c|C-).$/
55
- (bare[-1].ord & 0b1001_1111).chr
56
- when /^M-.$/
57
- (bare[-1].ord | 0b1000_0000).chr
58
- when /^(M-\\C-|C-\\M-|M-\\c|c\\M-).$/
59
- (bare[-1].ord & 0b1001_1111 | 0b1000_0000).chr
60
- when /^[0-7]+/
61
- bare.to_i(8).chr
88
+ when "\n"
89
+ ''
62
90
  else
63
- bare
91
+ '\\\\'
64
92
  end
65
93
  end
66
94
  end
95
+
96
+ def unescaped_value(bare)
97
+ case bare
98
+ when SINGLE_LETTER_ESCAPES_REGEXP
99
+ SINGLE_LETTER_ESCAPES[bare]
100
+ when /^x/
101
+ bare[1..-1].to_i(16).chr
102
+ when /^u/
103
+ bare[1..-1].to_i(16).chr(Encoding::UTF_8)
104
+ when /^(c|C-).$/
105
+ (bare[-1].ord & 0b1001_1111).chr
106
+ when /^M-.$/
107
+ (bare[-1].ord | 0b1000_0000).chr
108
+ when /^(M-\\C-|C-\\M-|M-\\c|c\\M-).$/
109
+ (bare[-1].ord & 0b1001_1111 | 0b1000_0000).chr
110
+ when /^[0-7]+/
111
+ bare.to_i(8).chr
112
+ when "\n"
113
+ bare
114
+ else
115
+ bare
116
+ end
117
+ end
67
118
  end
68
119
  end
@@ -1,3 +1,3 @@
1
1
  module RipperRubyParser
2
- VERSION = '1.4.1'.freeze
2
+ VERSION = '1.4.2'.freeze
3
3
  end
@@ -99,78 +99,4 @@ describe 'Using RipperRubyParser and RubyParser' do
99
99
  program.must_be_parsed_as_before
100
100
  end
101
101
  end
102
-
103
- describe 'for an example with regular expressions with different encoding flags' do
104
- it 'gives the same result' do
105
- program = <<-END
106
- regular = /foo/
107
- noenc = /foo/n
108
- utf8 = /foo/u
109
- euc = /foo/e
110
- sjis = /foo/s
111
-
112
- regular = /foo\#{bar}/
113
- noenc = /foo\#{bar}/n
114
- utf8 = /foo\#{bar}/u
115
- euc = /foo\#{bar}/e
116
- sjis = /foo\#{bar}/s
117
- END
118
-
119
- program.must_be_parsed_as_before
120
- end
121
- end
122
-
123
- describe 'for an example with __ENCODING__' do
124
- it 'gives the same result' do
125
- program = 'foo = __ENCODING__'
126
- program.must_be_parsed_as_before
127
- end
128
- end
129
-
130
- describe 'for an example with self[]' do
131
- # https://github.com/seattlerb/ruby_parser/issues/250
132
- it 'gives the same result' do
133
- program = 'self[:foo]'
134
- program.must_be_parsed_as_before
135
- end
136
- end
137
-
138
- describe 'for an example with required keyword arguments and no parentheses' do
139
- # https://github.com/seattlerb/ruby_parser/pull/254
140
- let(:program) do
141
- <<-END
142
- def foo a:, b:
143
- # body
144
- end
145
- END
146
- end
147
-
148
- it 'gives the same result' do
149
- program.must_be_parsed_as_before
150
- end
151
- end
152
-
153
- describe 'for an example combining begin..end and diverse operators' do
154
- let(:program) do
155
- <<-END
156
- begin end
157
- begin; foo; end
158
- begin; foo; bar; end
159
- - begin; foo; end
160
- begin; bar; end + foo
161
- foo + begin; bar; end
162
- begin; foo; end ? bar : baz
163
- foo ? begin; bar; end : baz
164
- foo ? bar : begin; baz; end
165
- begin; bar; end and foo
166
- foo and begin; bar; end
167
- begin; foo; end if bar
168
- begin; foo; end unless bar
169
- END
170
- end
171
-
172
- it 'gives the same result' do
173
- program.must_be_parsed_as_before
174
- end
175
- end
176
102
  end
@@ -10,7 +10,7 @@ describe 'Using RipperRubyParser and RubyParser' do
10
10
  RubyParser.new
11
11
  end
12
12
 
13
- Dir.glob('test/**/*.rb').each do |file|
13
+ Dir.glob('test/ripper_ruby_parser/**/*.rb').each do |file|
14
14
  describe "for #{file}" do
15
15
  let :program do
16
16
  File.read file
@@ -845,54 +845,6 @@ describe RipperRubyParser::Parser do
845
845
  must_be_parsed_as s(:call, s(:lit, 1), :!)
846
846
  end
847
847
 
848
- it 'handles the range operator with positive number literals' do
849
- '1..2'.
850
- must_be_parsed_as s(:lit, 1..2)
851
- end
852
-
853
- it 'handles the range operator with negative number literals' do
854
- '-1..-2'.
855
- must_be_parsed_as s(:lit, -1..-2)
856
- end
857
-
858
- it 'handles the range operator with string literals' do
859
- "'a'..'z'".
860
- must_be_parsed_as s(:dot2,
861
- s(:str, 'a'),
862
- s(:str, 'z'))
863
- end
864
-
865
- it 'handles the range operator with non-literals' do
866
- 'foo..bar'.
867
- must_be_parsed_as s(:dot2,
868
- s(:call, nil, :foo),
869
- s(:call, nil, :bar))
870
- end
871
-
872
- it 'handles the exclusive range operator with positive number literals' do
873
- '1...2'.
874
- must_be_parsed_as s(:lit, 1...2)
875
- end
876
-
877
- it 'handles the exclusive range operator with negative number literals' do
878
- '-1...-2'.
879
- must_be_parsed_as s(:lit, -1...-2)
880
- end
881
-
882
- it 'handles the exclusive range operator with string literals' do
883
- "'a'...'z'".
884
- must_be_parsed_as s(:dot3,
885
- s(:str, 'a'),
886
- s(:str, 'z'))
887
- end
888
-
889
- it 'handles the exclusive range operator with non-literals' do
890
- 'foo...bar'.
891
- must_be_parsed_as s(:dot3,
892
- s(:call, nil, :foo),
893
- s(:call, nil, :bar))
894
- end
895
-
896
848
  it 'handles the ternary operator' do
897
849
  'foo ? bar : baz'.
898
850
  must_be_parsed_as s(:if,
@@ -46,6 +46,23 @@ describe RipperRubyParser::Parser do
46
46
  result.inspect.must_equal s(:lit, /foo/n).inspect
47
47
  end
48
48
 
49
+ it 'works with line continuation' do
50
+ "/foo\\\nbar/".
51
+ must_be_parsed_as s(:lit, /foobar/)
52
+ end
53
+
54
+ describe 'for a %r-delimited regex literal' do
55
+ it 'works for the simple case with escape sequences' do
56
+ '%r[foo\nbar]'.
57
+ must_be_parsed_as s(:lit, /foo\nbar/)
58
+ end
59
+
60
+ it 'works with odd delimiters and escape sequences' do
61
+ '%r_foo\nbar_'.
62
+ must_be_parsed_as s(:lit, /foo\nbar/)
63
+ end
64
+ end
65
+
49
66
  describe 'with interpolations' do
50
67
  it 'works for a simple interpolation' do
51
68
  '/foo#{bar}baz/'.
@@ -139,6 +156,16 @@ describe RipperRubyParser::Parser do
139
156
  result[1].encoding.to_s.must_equal 'UTF-8'
140
157
  end
141
158
 
159
+ it 'handles line continuation with double-quoted strings' do
160
+ "\"foo\\\nbar\"".
161
+ must_be_parsed_as s(:str, 'foobar')
162
+ end
163
+
164
+ it 'escapes line continuation with double-quoted strings' do
165
+ "\"foo\\\\\nbar\"".
166
+ must_be_parsed_as s(:str, "foo\\\nbar")
167
+ end
168
+
142
169
  describe 'with double-quoted strings with escape sequences' do
143
170
  it 'works for strings with escape sequences' do
144
171
  '"\\n"'.
@@ -213,6 +240,17 @@ describe RipperRubyParser::Parser do
213
240
  it 'works with unicode escapes (unlike RubyParser)' do
214
241
  '"foo\\u273bbar"'.must_be_parsed_as s(:str, 'foo✻bar')
215
242
  end
243
+
244
+ it 'converts to unicode if possible' do
245
+ '"2\302\275"'.must_be_parsed_as s(:str, '2½')
246
+ end
247
+
248
+ it 'does not convert to unicode if result is not valid' do
249
+ parser = RipperRubyParser::Parser.new
250
+ result = parser.parse '"2\x82\302\275"'
251
+ expected = s(:str, "2\x82\xC2\xBD".force_encoding(Encoding::US_ASCII))
252
+ result.inspect.must_equal expected.inspect
253
+ end
216
254
  end
217
255
 
218
256
  describe 'with interpolations' do
@@ -332,6 +370,11 @@ describe RipperRubyParser::Parser do
332
370
  "'foo\\\\\\abar'".
333
371
  must_be_parsed_as s(:str, 'foo\\\\abar')
334
372
  end
373
+
374
+ it 'does not process line continuation' do
375
+ "'foo\\\nbar'".
376
+ must_be_parsed_as s(:str, "foo\\\nbar")
377
+ end
335
378
  end
336
379
 
337
380
  describe 'with %Q-delimited strings' do
@@ -344,6 +387,28 @@ describe RipperRubyParser::Parser do
344
387
  '%Q[foo\\nbar]'.
345
388
  must_be_parsed_as s(:str, "foo\nbar")
346
389
  end
390
+
391
+ it 'handles line continuation' do
392
+ "%Q[foo\\\nbar]".
393
+ must_be_parsed_as s(:str, 'foobar')
394
+ end
395
+ end
396
+
397
+ describe 'with %-delimited strings' do
398
+ it 'works for the simple case' do
399
+ '%(bar)'.
400
+ must_be_parsed_as s(:str, 'bar')
401
+ end
402
+
403
+ it 'works for escape sequences' do
404
+ '%(foo\nbar)'.
405
+ must_be_parsed_as s(:str, "foo\nbar")
406
+ end
407
+
408
+ it 'works for odd delimiters' do
409
+ '%!foo\nbar!'.
410
+ must_be_parsed_as s(:str, "foo\nbar")
411
+ end
347
412
  end
348
413
 
349
414
  describe 'with string concatenation' do
@@ -406,16 +471,45 @@ describe RipperRubyParser::Parser do
406
471
  "<<FOO\nbar\\tbaz\nFOO".
407
472
  must_be_parsed_as s(:str, "bar\tbaz\n")
408
473
  end
474
+
475
+ it 'handles line continuation' do
476
+ "<<FOO\nbar\\\nbaz\nFOO".
477
+ must_be_parsed_as s(:str, "barbaz\n")
478
+ end
479
+
480
+ it 'escapes line continuation' do
481
+ "<<FOO\nbar\\\\\nbaz\nFOO".
482
+ must_be_parsed_as s(:str, "bar\\\nbaz\n")
483
+ end
484
+
485
+ it 'does not convert to unicode even if possible' do
486
+ parser = RipperRubyParser::Parser.new
487
+ result = parser.parse "<<FOO\n2\\302\\275\nFOO"
488
+ expected = s(:str, "2\xC2\xBD\n".force_encoding(Encoding::US_ASCII))
489
+ result.inspect.must_equal expected.inspect
490
+ end
409
491
  end
410
492
  end
411
493
 
412
- describe 'for word list literals' do
413
- it 'works for the simple case with %w' do
494
+ describe 'for word list literals with %w delimiter' do
495
+ it 'works for the simple case' do
414
496
  '%w(foo bar)'.
415
497
  must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar'))
416
498
  end
417
499
 
418
- it 'works for the simple case with %W' do
500
+ it 'does not perform interpolation' do
501
+ '%w(foo\\nbar baz)'.
502
+ must_be_parsed_as s(:array, s(:str, 'foo\\nbar'), s(:str, 'baz'))
503
+ end
504
+
505
+ it 'handles line continuation' do
506
+ "%w(foo\\\nbar baz)".
507
+ must_be_parsed_as s(:array, s(:str, "foo\nbar"), s(:str, 'baz'))
508
+ end
509
+ end
510
+
511
+ describe 'for word list literals with %W delimiter' do
512
+ it 'works for the simple case' do
419
513
  '%W(foo bar)'.
420
514
  must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar'))
421
515
  end
@@ -452,6 +546,13 @@ describe RipperRubyParser::Parser do
452
546
  s(:str, "foo\nbar"),
453
547
  s(:str, 'baz'))
454
548
  end
549
+
550
+ it 'correctly handles line continuation' do
551
+ "%W(foo\\\nbar baz)".
552
+ must_be_parsed_as s(:array,
553
+ s(:str, "foo\nbar"),
554
+ s(:str, 'baz'))
555
+ end
455
556
  end
456
557
 
457
558
  describe 'for symbol list literals with %i delimiter' do
@@ -459,6 +560,16 @@ describe RipperRubyParser::Parser do
459
560
  '%i(foo bar)'.
460
561
  must_be_parsed_as s(:array, s(:lit, :foo), s(:lit, :bar))
461
562
  end
563
+
564
+ it 'does not perform interpolation' do
565
+ '%i(foo\\nbar baz)'.
566
+ must_be_parsed_as s(:array, s(:lit, :"foo\\nbar"), s(:lit, :baz))
567
+ end
568
+
569
+ it 'handles line continuation' do
570
+ "%i(foo\\\nbar baz)".
571
+ must_be_parsed_as s(:array, s(:lit, :"foo\nbar"), s(:lit, :baz))
572
+ end
462
573
  end
463
574
 
464
575
  describe 'for symbol list literals with %I delimiter' do
@@ -491,6 +602,13 @@ describe RipperRubyParser::Parser do
491
602
  s(:evstr, s(:call, nil, :bar)),
492
603
  s(:str, 'baz')))
493
604
  end
605
+
606
+ it 'correctly handles line continuation' do
607
+ "%I(foo\\\nbar baz)".
608
+ must_be_parsed_as s(:array,
609
+ s(:lit, :"foo\nbar"),
610
+ s(:lit, :baz))
611
+ end
494
612
  end
495
613
 
496
614
  describe 'for character literals' do