ripper_ruby_parser 1.7.1 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 943fa1c037a2a2a257d13aed73d33194756f57d0d582207062331f60b5629cd3
4
- data.tar.gz: 94fffed38a9e3354a8f9a903add44e26d5bbc8ea599afb72141fbb877f4e2e22
3
+ metadata.gz: 5d4b6b4c9b802cf538662716b22a68c98d26af9dd0fba04aa72c53515d869738
4
+ data.tar.gz: e34339faf0080e288368618bd0688525e253300c25883fc74b8f86adfd572f9b
5
5
  SHA512:
6
- metadata.gz: b677389067d2759571fb431a8055a4909b58026c127b43fde51d158e6de48633f423cb88f448a996da0406e2e02ff60e9834a25aeff2c5eb49274e80d63d6558
7
- data.tar.gz: 47df66fb3e972978919e3224522957ef7ccc3a430ee09318775efd1191c1d87aaf54432aca181ea36b4abc729847eda1355809e5d785a6e68ea1c2ae15c96968
6
+ metadata.gz: 1262aa2a0cb8b14e99650b8e49f1f1cf4ba672c59ed662b46204c0fcbd1d83580681d3b16440047b29dbc542a8c3dc2bd794071d32e09290a51a4fa4c6180476
7
+ data.tar.gz: aedb6d0c231afd60183a0cf1152f34c5cc14ec088a22360d63ae4d3aacb50d2e8bc07548e8617611086ed1eddfa1ebb4c6d9aeb12f2093ad6d677e86e2ee363f
@@ -1,5 +1,19 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.7.2 / 2020-02-28
4
+
5
+ ### Bug fixes
6
+
7
+ * Support imaginary number literals ([#100])
8
+ * Handle anonymous kwrest arguments ([#95])
9
+
10
+ ### Internal changes
11
+
12
+ * Update tests ([#101])
13
+ * Prepare for testing on Ruby 2.7 ([#99])
14
+ * Various improvements ([#98])
15
+ * Split long module ([#97])
16
+
3
17
  ## 1.7.1 / 2019-11-03
4
18
 
5
19
  * Handle unicode escapes with five or six hex digits ([#94])
@@ -187,6 +201,12 @@
187
201
  * Initial release
188
202
 
189
203
  <!-- Pull request links -->
204
+ [#101]: https://github.com/mvz/ripper_ruby_parser/pull/101
205
+ [#100]: https://github.com/mvz/ripper_ruby_parser/pull/100
206
+ [#99]: https://github.com/mvz/ripper_ruby_parser/pull/99
207
+ [#98]: https://github.com/mvz/ripper_ruby_parser/pull/98
208
+ [#97]: https://github.com/mvz/ripper_ruby_parser/pull/97
209
+ [#95]: https://github.com/mvz/ripper_ruby_parser/pull/95
190
210
  [#94]: https://github.com/mvz/ripper_ruby_parser/pull/94
191
211
  [#92]: https://github.com/mvz/ripper_ruby_parser/pull/92
192
212
  [#90]: https://github.com/mvz/ripper_ruby_parser/pull/90
@@ -10,6 +10,7 @@ require "ripper_ruby_parser/sexp_handlers/loops"
10
10
  require "ripper_ruby_parser/sexp_handlers/method_calls"
11
11
  require "ripper_ruby_parser/sexp_handlers/methods"
12
12
  require "ripper_ruby_parser/sexp_handlers/operators"
13
+ require "ripper_ruby_parser/sexp_handlers/string_literals"
13
14
 
14
15
  module RipperRubyParser
15
16
  # Umbrella module for handlers of particular sexp types
@@ -28,6 +29,7 @@ module RipperRubyParser
28
29
  include MethodCalls
29
30
  include Methods
30
31
  include Operators
32
+ include StringLiterals
31
33
  end
32
34
  end
33
35
  end
@@ -35,7 +35,7 @@ module RipperRubyParser
35
35
 
36
36
  def process_kwrest_param(exp)
37
37
  _, sym, = exp.shift 3
38
- process(sym)
38
+ process(sym) || s(:lvar, :"")
39
39
  end
40
40
 
41
41
  def process_block_var(exp)
@@ -189,7 +189,12 @@ module RipperRubyParser
189
189
  def handle_double_splat(doublesplat)
190
190
  return [] unless doublesplat
191
191
 
192
- [s(:dsplat, process(doublesplat))]
192
+ # Anonymous kwrest arguments are parsed into an Integer in Ruby 2.4
193
+ if RUBY_VERSION < "2.5.0" && doublesplat.is_a?(Integer)
194
+ [s(:dsplat, s(:lvar, :""))]
195
+ else
196
+ [s(:dsplat, process(doublesplat))]
197
+ end
193
198
  end
194
199
 
195
200
  def handle_block_argument(block)
@@ -4,134 +4,10 @@ module RipperRubyParser
4
4
  module SexpHandlers
5
5
  # Sexp handlers for literals
6
6
  module Literals
7
- def process_string_literal(exp)
8
- _, content = exp.shift 2
9
- process(content)
10
- end
11
-
12
- def process_string_content(exp)
13
- _, *rest = shift_all exp
14
- line, string, rest = extract_string_parts(rest)
15
-
16
- if rest.empty?
17
- with_line_number(line, s(:str, string))
18
- else
19
- s(:dstr, string, *rest)
20
- end
21
- end
22
-
23
- alias process_word process_string_content
24
-
25
- def process_string_embexpr(exp)
26
- _, list = exp.shift 2
27
-
28
- val = process(list.sexp_body.first)
29
-
30
- case val.sexp_type
31
- when :str, :dstr
32
- val
33
- when :void_stmt
34
- s(:dstr, "", s(:evstr))
35
- else
36
- s(:dstr, "", s(:evstr, val))
37
- end
38
- end
39
-
40
- def process_string_dvar(exp)
41
- _, list = exp.shift 2
42
- val = process(list)
43
- s(:dstr, "", s(:evstr, val))
44
- end
45
-
46
- def process_string_concat(exp)
47
- _, left, right = exp.shift 3
48
-
49
- left = process(left)
50
- right = process(right)
51
-
52
- if left.sexp_type == :str
53
- merge_left_into_right(left, right)
54
- else
55
- merge_right_into_left(left, right)
56
- end
57
- end
58
-
59
- def process_xstring_literal(exp)
60
- _, content = exp.shift 2
61
- process(content)
62
- end
63
-
64
- def process_xstring(exp)
65
- _, *rest = shift_all exp
66
- line, string, rest = extract_string_parts(rest)
67
- if rest.empty?
68
- s(:xstr, string).line(line)
69
- else
70
- s(:dxstr, string, *rest)
71
- end
72
- end
73
-
74
- def process_regexp_literal(exp)
75
- _, content, (_, flags,) = exp.shift 3
76
-
77
- content = process(content)
78
- numflags = character_flags_to_numerical flags
79
-
80
- if content.length == 2
81
- return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
82
- end
83
-
84
- content.sexp_type = :dregx_once if /o/.match?(flags)
85
- content << numflags unless numflags == 0
86
- content
87
- end
88
-
89
- def process_regexp(exp)
90
- _, *rest = shift_all exp
91
- line, string, rest = extract_string_parts(rest)
92
- with_line_number(line, s(:dregx, string, *rest))
93
- end
94
-
95
- def process_symbol_literal(exp)
96
- _, symbol = exp.shift 2
97
- handle_symbol_content(symbol)
98
- end
99
-
100
- def process_symbol(exp)
101
- _, node = exp.shift 2
102
- handle_symbol_content(node)
103
- end
104
-
105
- def process_dyna_symbol(exp)
106
- _, node = exp.shift 2
107
- handle_dyna_symbol_content(node)
108
- end
109
-
110
- def process_qsymbols(exp)
111
- _, *items = shift_all(exp)
112
- items = items.map { |item| handle_symbol_content(item) }
113
- s(:qsymbols, *items)
114
- end
115
-
116
- def process_symbols(exp)
117
- _, *items = shift_all(exp)
118
- items = items.map { |item| handle_dyna_symbol_content(item) }
119
- s(:symbols, *items)
120
- end
121
-
122
- INTERPOLATING_HEREDOC = /^<<[-~]?[^-~']/.freeze
123
- NON_INTERPOLATING_HEREDOC = /^<<[-~]?'/.freeze
124
- INTERPOLATING_STRINGS = ['"', "`", ':"', /^%Q.$/, /^%.$/].freeze
125
- NON_INTERPOLATING_STRINGS = ["'", ":'", /^%q.$/].freeze
126
- INTERPOLATING_WORD_LIST = /^%[WI].$/.freeze
127
- NON_INTERPOLATING_WORD_LIST = /^%[wi].$/.freeze
128
- REGEXP_LITERALS = ["/", /^%r.$/].freeze
129
-
130
- def process_at_tstring_content(exp)
131
- _, content, pos, delim = exp.shift 4
132
- string = handle_string_unescaping(content, delim)
133
- string = handle_string_encoding(string, delim)
134
- with_position(pos, s(:str, string))
7
+ # character literals
8
+ def process_at_CHAR(exp)
9
+ _, val, pos = exp.shift 3
10
+ with_position(pos, s(:str, unescape(val[1..-1])))
135
11
  end
136
12
 
137
13
  def process_array(exp)
@@ -163,134 +39,31 @@ module RipperRubyParser
163
39
  s(:kwsplat, process(param))
164
40
  end
165
41
 
166
- private
167
-
168
- def extract_string_parts(list)
169
- return nil, "", [] if list.empty?
170
-
171
- list = merge_raw_string_literals list
172
- list = map_process_list list
173
-
174
- parts = list.flat_map do |item|
175
- type, val, *rest = item
176
- if type == :dstr
177
- if val.empty?
178
- rest
179
- else
180
- [s(:str, val), *rest]
181
- end
182
- else
183
- [item]
184
- end
185
- end
186
-
187
- string = ""
188
- while parts.first&.sexp_type == :str
189
- str = parts.shift
190
- line ||= str.line
191
- string += str.last
192
- end
193
-
194
- return line, string, parts
195
- end
196
-
197
- def merge_raw_string_literals(list)
198
- chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
199
- chunks.flat_map do |is_simple, items|
200
- if is_simple && items.count > 1
201
- head = items.first
202
- contents = items.map { |it| it[1] }.join
203
- [s(:@tstring_content, contents, head[2], head[3])]
204
- else
205
- items
206
- end
207
- end
208
- end
209
-
210
- def character_flags_to_numerical(flags)
211
- numflags = 0
212
-
213
- numflags = Regexp::MULTILINE if /m/.match?(flags)
214
- numflags |= Regexp::EXTENDED if /x/.match?(flags)
215
- numflags |= Regexp::IGNORECASE if /i/.match?(flags)
216
-
217
- numflags |= Regexp::NOENCODING if /n/.match?(flags)
218
- numflags |= Regexp::FIXEDENCODING if /[ues]/.match?(flags)
219
-
220
- numflags
221
- end
222
-
223
- def handle_dyna_symbol_content(node)
224
- type, *body = *process(node)
225
- case type
226
- when :str, :xstr
227
- s(:lit, body.first.to_sym)
228
- when :dstr, :dxstr
229
- s(:dsym, *body)
230
- end
42
+ # number literals
43
+ def process_at_int(exp)
44
+ make_literal(exp) { |val| Integer(val) }
231
45
  end
232
46
 
233
- def handle_symbol_content(node)
234
- if node.sexp_type == :'@kw'
235
- symbol, position = extract_node_symbol_with_position(node)
236
- with_position(position, s(:lit, symbol))
237
- else
238
- processed = process(node)
239
- symbol = processed.last.to_sym
240
- line = processed.line
241
- with_line_number(line, s(:lit, symbol))
242
- end
47
+ def process_at_float(exp)
48
+ make_literal(exp, &:to_f)
243
49
  end
244
50
 
245
- def merge_left_into_right(left, right)
246
- right[1] = left.last + right[1]
247
- right
51
+ def process_at_rational(exp)
52
+ make_literal(exp, &:to_r)
248
53
  end
249
54
 
250
- def merge_right_into_left(left, right)
251
- if right.sexp_type == :str
252
- left.push right
253
- else
254
- _, first, *rest = right
255
- left.push s(:str, first) unless first.empty?
256
- left.push(*rest)
257
- end
55
+ def process_at_imaginary(exp)
56
+ make_literal(exp, &:to_c)
258
57
  end
259
58
 
260
- def handle_string_unescaping(content, delim)
261
- case delim
262
- when INTERPOLATING_HEREDOC
263
- unescape(content)
264
- when *INTERPOLATING_STRINGS
265
- unescape(content)
266
- when INTERPOLATING_WORD_LIST
267
- unescape_wordlist_word(content)
268
- when *NON_INTERPOLATING_STRINGS
269
- simple_unescape(content)
270
- when *REGEXP_LITERALS
271
- unescape_regexp(content)
272
- when NON_INTERPOLATING_WORD_LIST
273
- simple_unescape_wordlist_word(content)
274
- else
275
- content
276
- end
277
- end
278
-
279
- def handle_string_encoding(string, delim)
280
- case delim
281
- when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
282
- fix_encoding string
283
- else
284
- string
285
- end
286
- end
59
+ private
287
60
 
288
61
  # Process list of items that can be either :assoc_new or :assoc_splat
289
62
  def make_hash_items(elems)
290
63
  result = s()
291
64
  elems.each do |sub_exp|
292
65
  if sub_exp.sexp_type == :assoc_new
293
- sub_exp.sexp_body.each { |elem| result << process(elem) }
66
+ result += process(sub_exp).sexp_body
294
67
  else # :assoc_splat
295
68
  result << process(sub_exp)
296
69
  end
@@ -99,20 +99,22 @@ module RipperRubyParser
99
99
 
100
100
  def convert_arguments(args)
101
101
  args.line ||= args.sexp_body.first&.line
102
- args.map! do |item|
103
- if item.is_a? Symbol
104
- item
102
+ args.map! { |item| convert_argument item }
103
+ end
104
+
105
+ def convert_argument(item)
106
+ if item.is_a? Symbol
107
+ item
108
+ else
109
+ case item.sexp_type
110
+ when :lvar
111
+ item.last
112
+ when *SPECIAL_ARG_MARKER.keys
113
+ convert_marked_argument(item)
114
+ when :masgn
115
+ convert_masgn_argument(item)
105
116
  else
106
- case item.sexp_type
107
- when :lvar
108
- item.last
109
- when *SPECIAL_ARG_MARKER.keys
110
- convert_marked_argument(item)
111
- when :masgn
112
- convert_masgn_argument(item)
113
- else
114
- item
115
- end
117
+ item
116
118
  end
117
119
  end
118
120
  end
@@ -0,0 +1,260 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RipperRubyParser
4
+ module SexpHandlers
5
+ # Sexp handlers for string and stringlike literals
6
+ module StringLiterals
7
+ def process_string_literal(exp)
8
+ _, content = exp.shift 2
9
+ process(content)
10
+ end
11
+
12
+ def process_string_content(exp)
13
+ _, *rest = shift_all exp
14
+ line, string, rest = extract_string_parts(rest)
15
+
16
+ if rest.empty?
17
+ with_line_number(line, s(:str, string))
18
+ else
19
+ s(:dstr, string, *rest)
20
+ end
21
+ end
22
+
23
+ alias process_word process_string_content
24
+
25
+ def process_string_embexpr(exp)
26
+ _, list = exp.shift 2
27
+
28
+ val = process(list.sexp_body.first)
29
+
30
+ case val.sexp_type
31
+ when :str, :dstr
32
+ val
33
+ when :void_stmt
34
+ s(:dstr, "", s(:evstr))
35
+ else
36
+ s(:dstr, "", s(:evstr, val))
37
+ end
38
+ end
39
+
40
+ def process_string_dvar(exp)
41
+ _, list = exp.shift 2
42
+ val = process(list)
43
+ s(:dstr, "", s(:evstr, val))
44
+ end
45
+
46
+ def process_string_concat(exp)
47
+ _, left, right = exp.shift 3
48
+
49
+ left = process(left)
50
+ right = process(right)
51
+
52
+ if left.sexp_type == :str
53
+ merge_left_into_right(left, right)
54
+ else
55
+ merge_right_into_left(left, right)
56
+ end
57
+ end
58
+
59
+ def process_xstring_literal(exp)
60
+ _, content = exp.shift 2
61
+ process(content)
62
+ end
63
+
64
+ def process_xstring(exp)
65
+ _, *rest = shift_all exp
66
+ line, string, rest = extract_string_parts(rest)
67
+ result = if rest.empty?
68
+ s(:xstr, string)
69
+ else
70
+ s(:dxstr, string, *rest)
71
+ end
72
+ result.line = line
73
+ result
74
+ end
75
+
76
+ def process_regexp_literal(exp)
77
+ _, content, (_, flags,) = exp.shift 3
78
+
79
+ content = process(content)
80
+ numflags = character_flags_to_numerical flags
81
+
82
+ if content.length == 2
83
+ return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
84
+ end
85
+
86
+ content.sexp_type = :dregx_once if /o/.match?(flags)
87
+ content << numflags unless numflags == 0
88
+ content
89
+ end
90
+
91
+ def process_regexp(exp)
92
+ _, *rest = shift_all exp
93
+ line, string, rest = extract_string_parts(rest)
94
+ with_line_number(line, s(:dregx, string, *rest))
95
+ end
96
+
97
+ def process_symbol_literal(exp)
98
+ _, symbol = exp.shift 2
99
+ handle_symbol_content(symbol)
100
+ end
101
+
102
+ def process_symbol(exp)
103
+ _, node = exp.shift 2
104
+ handle_symbol_content(node)
105
+ end
106
+
107
+ def process_dyna_symbol(exp)
108
+ _, node = exp.shift 2
109
+ handle_dyna_symbol_content(node)
110
+ end
111
+
112
+ def process_qsymbols(exp)
113
+ _, *items = shift_all(exp)
114
+ items = items.map { |item| handle_symbol_content(item) }
115
+ s(:qsymbols, *items)
116
+ end
117
+
118
+ def process_symbols(exp)
119
+ _, *items = shift_all(exp)
120
+ items = items.map { |item| handle_dyna_symbol_content(item) }
121
+ s(:symbols, *items)
122
+ end
123
+
124
+ def process_at_tstring_content(exp)
125
+ _, content, pos, delim = exp.shift 4
126
+ string = handle_string_unescaping(content, delim)
127
+ string = handle_string_encoding(string, delim)
128
+ with_position(pos, s(:str, string))
129
+ end
130
+
131
+ private
132
+
133
+ def extract_string_parts(list)
134
+ return nil, "", [] if list.empty?
135
+
136
+ list = merge_raw_string_literals list
137
+ list = map_process_list list
138
+
139
+ parts = list.flat_map do |item|
140
+ type, val, *rest = item
141
+ if type == :dstr
142
+ if val.empty?
143
+ rest
144
+ else
145
+ [s(:str, val), *rest]
146
+ end
147
+ else
148
+ [item]
149
+ end
150
+ end
151
+
152
+ string = ""
153
+ while parts.first&.sexp_type == :str
154
+ str = parts.shift
155
+ line ||= str.line
156
+ string += str.last
157
+ end
158
+
159
+ return line, string, parts
160
+ end
161
+
162
+ def merge_raw_string_literals(list)
163
+ chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
164
+ chunks.flat_map do |is_simple, items|
165
+ if is_simple && items.count > 1
166
+ head = items.first
167
+ contents = items.map { |it| it[1] }.join
168
+ [s(:@tstring_content, contents, head[2], head[3])]
169
+ else
170
+ items
171
+ end
172
+ end
173
+ end
174
+
175
+ def character_flags_to_numerical(flags)
176
+ numflags = 0
177
+
178
+ numflags = Regexp::MULTILINE if /m/.match?(flags)
179
+ numflags |= Regexp::EXTENDED if /x/.match?(flags)
180
+ numflags |= Regexp::IGNORECASE if /i/.match?(flags)
181
+
182
+ numflags |= Regexp::NOENCODING if /n/.match?(flags)
183
+ numflags |= Regexp::FIXEDENCODING if /[ues]/.match?(flags)
184
+
185
+ numflags
186
+ end
187
+
188
+ def handle_dyna_symbol_content(node)
189
+ type, *body = *process(node)
190
+ case type
191
+ when :str, :xstr
192
+ s(:lit, body.first.to_sym)
193
+ when :dstr, :dxstr
194
+ s(:dsym, *body)
195
+ end
196
+ end
197
+
198
+ def handle_symbol_content(node)
199
+ if node.sexp_type == :'@kw'
200
+ symbol, position = extract_node_symbol_with_position(node)
201
+ with_position(position, s(:lit, symbol))
202
+ else
203
+ processed = process(node)
204
+ symbol = processed.last.to_sym
205
+ line = processed.line
206
+ with_line_number(line, s(:lit, symbol))
207
+ end
208
+ end
209
+
210
+ def merge_left_into_right(left, right)
211
+ right[1] = left.last + right[1]
212
+ right
213
+ end
214
+
215
+ def merge_right_into_left(left, right)
216
+ if right.sexp_type == :str
217
+ left.push right
218
+ else
219
+ _, first, *rest = right
220
+ left.push s(:str, first) unless first.empty?
221
+ left.push(*rest)
222
+ end
223
+ end
224
+
225
+ INTERPOLATING_HEREDOC = /^<<[-~]?[^-~']/.freeze
226
+ NON_INTERPOLATING_HEREDOC = /^<<[-~]?'/.freeze
227
+ INTERPOLATING_STRINGS = ['"', "`", ':"', /^%Q.$/, /^%.$/].freeze
228
+ NON_INTERPOLATING_STRINGS = ["'", ":'", /^%q.$/].freeze
229
+ INTERPOLATING_WORD_LIST = /^%[WI].$/.freeze
230
+ NON_INTERPOLATING_WORD_LIST = /^%[wi].$/.freeze
231
+ REGEXP_LITERALS = ["/", /^%r.$/].freeze
232
+
233
+ def handle_string_unescaping(content, delim)
234
+ case delim
235
+ when INTERPOLATING_HEREDOC, *INTERPOLATING_STRINGS
236
+ unescape(content)
237
+ when INTERPOLATING_WORD_LIST
238
+ unescape_wordlist_word(content)
239
+ when *NON_INTERPOLATING_STRINGS
240
+ simple_unescape(content)
241
+ when *REGEXP_LITERALS
242
+ unescape_regexp(content)
243
+ when NON_INTERPOLATING_WORD_LIST
244
+ simple_unescape_wordlist_word(content)
245
+ else
246
+ content
247
+ end
248
+ end
249
+
250
+ def handle_string_encoding(string, delim)
251
+ case delim
252
+ when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
253
+ fix_encoding string
254
+ else
255
+ string
256
+ end
257
+ end
258
+ end
259
+ end
260
+ end