irb 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/irb/cmd/irb_info.rb +14 -18
- data/lib/irb/cmd/nop.rb +4 -8
- data/lib/irb/cmd/show_source.rb +3 -3
- data/lib/irb/completion.rb +2 -2
- data/lib/irb/context.rb +19 -16
- data/lib/irb/ext/loader.rb +2 -0
- data/lib/irb/extend-command.rb +1 -1
- data/lib/irb/nesting_parser.rb +227 -0
- data/lib/irb/ruby-lex.rb +220 -495
- data/lib/irb/version.rb +2 -2
- data/lib/irb.rb +23 -29
- metadata +3 -3
- data/lib/irb/cmd/fork.rb +0 -34
data/lib/irb/ruby-lex.rb
CHANGED
@@ -6,6 +6,7 @@
|
|
6
6
|
|
7
7
|
require "ripper"
|
8
8
|
require "jruby" if RUBY_ENGINE == "jruby"
|
9
|
+
require_relative "nesting_parser"
|
9
10
|
|
10
11
|
# :stopdoc:
|
11
12
|
class RubyLex
|
@@ -54,8 +55,7 @@ class RubyLex
|
|
54
55
|
if @io.respond_to?(:check_termination)
|
55
56
|
@io.check_termination do |code|
|
56
57
|
if Reline::IOGate.in_pasting?
|
57
|
-
|
58
|
-
rest = lex.check_termination_in_prev_line(code)
|
58
|
+
rest = check_termination_in_prev_line(code)
|
59
59
|
if rest
|
60
60
|
Reline.delete_text
|
61
61
|
rest.bytes.reverse_each do |c|
|
@@ -69,65 +69,36 @@ class RubyLex
|
|
69
69
|
# Accept any single-line input for symbol aliases or commands that transform args
|
70
70
|
next true if single_line_command?(code)
|
71
71
|
|
72
|
-
|
73
|
-
|
74
|
-
false
|
75
|
-
else
|
76
|
-
true
|
77
|
-
end
|
72
|
+
_tokens, _opens, terminated = check_code_state(code)
|
73
|
+
terminated
|
78
74
|
end
|
79
75
|
end
|
80
76
|
end
|
81
77
|
if @io.respond_to?(:dynamic_prompt)
|
82
78
|
@io.dynamic_prompt do |lines|
|
83
79
|
lines << '' if lines.empty?
|
84
|
-
result = []
|
85
80
|
tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
unprocessed_tokens << t
|
93
|
-
if t.tok.include?("\n")
|
94
|
-
t_str = t.tok
|
95
|
-
t_str.each_line("\n") do |s|
|
96
|
-
code << s
|
97
|
-
next unless s.include?("\n")
|
98
|
-
ltype, indent, continue, code_block_open = check_state(code, partial_tokens)
|
99
|
-
result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
|
100
|
-
line_num_offset += 1
|
101
|
-
end
|
102
|
-
unprocessed_tokens = []
|
103
|
-
else
|
104
|
-
code << t.tok
|
81
|
+
line_results = IRB::NestingParser.parse_by_line(tokens)
|
82
|
+
tokens_until_line = []
|
83
|
+
line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset|
|
84
|
+
line_tokens.each do |token, _s|
|
85
|
+
# Avoid appending duplicated token. Tokens that include "\n" like multiline tstring_content can exist in multiple lines.
|
86
|
+
tokens_until_line << token if token != tokens_until_line.last
|
105
87
|
end
|
88
|
+
continue = should_continue?(tokens_until_line)
|
89
|
+
prompt(next_opens, continue, line_num_offset)
|
106
90
|
end
|
107
|
-
|
108
|
-
unless unprocessed_tokens.empty?
|
109
|
-
ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens)
|
110
|
-
result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
|
111
|
-
end
|
112
|
-
result
|
113
91
|
end
|
114
92
|
end
|
115
93
|
|
116
94
|
if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
|
117
95
|
@io.auto_indent do |lines, line_index, byte_pointer, is_newline|
|
118
|
-
if
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
else
|
125
|
-
code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
|
126
|
-
last_line = lines[line_index]&.byteslice(0, byte_pointer)
|
127
|
-
code += last_line if last_line
|
128
|
-
@tokens = self.class.ripper_lex_without_warning(code, context: @context)
|
129
|
-
check_corresponding_token_depth(lines, line_index)
|
130
|
-
end
|
96
|
+
next nil if lines == [nil] # Workaround for exit IRB with CTRL+d
|
97
|
+
next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/)
|
98
|
+
|
99
|
+
code = lines[0..line_index].map { |l| "#{l}\n" }.join
|
100
|
+
tokens = self.class.ripper_lex_without_warning(code, context: @context)
|
101
|
+
process_indent_level(tokens, lines, line_index, is_newline)
|
131
102
|
end
|
132
103
|
end
|
133
104
|
end
|
@@ -149,9 +120,42 @@ class RubyLex
|
|
149
120
|
"#{local_variables.join('=')}=nil;" unless local_variables.empty?
|
150
121
|
end
|
151
122
|
|
123
|
+
# Some part of the code is not included in Ripper's token.
|
124
|
+
# Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr.
|
125
|
+
# With interpolated tokens, tokens.map(&:tok).join will be equal to code.
|
126
|
+
def self.interpolate_ripper_ignored_tokens(code, tokens)
|
127
|
+
line_positions = [0]
|
128
|
+
code.lines.each do |line|
|
129
|
+
line_positions << line_positions.last + line.bytesize
|
130
|
+
end
|
131
|
+
prev_byte_pos = 0
|
132
|
+
interpolated = []
|
133
|
+
prev_line = 1
|
134
|
+
tokens.each do |t|
|
135
|
+
line, col = t.pos
|
136
|
+
byte_pos = line_positions[line - 1] + col
|
137
|
+
if prev_byte_pos < byte_pos
|
138
|
+
tok = code.byteslice(prev_byte_pos...byte_pos)
|
139
|
+
pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
|
140
|
+
interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
|
141
|
+
prev_line += tok.count("\n")
|
142
|
+
end
|
143
|
+
interpolated << t
|
144
|
+
prev_byte_pos = byte_pos + t.tok.bytesize
|
145
|
+
prev_line += t.tok.count("\n")
|
146
|
+
end
|
147
|
+
if prev_byte_pos < code.bytesize
|
148
|
+
tok = code.byteslice(prev_byte_pos..)
|
149
|
+
pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
|
150
|
+
interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
|
151
|
+
end
|
152
|
+
interpolated
|
153
|
+
end
|
154
|
+
|
152
155
|
def self.ripper_lex_without_warning(code, context: nil)
|
153
156
|
verbose, $VERBOSE = $VERBOSE, nil
|
154
157
|
lvars_code = generate_local_variables_assign_code(context&.local_variables || [])
|
158
|
+
original_code = code
|
155
159
|
if lvars_code
|
156
160
|
code = "#{lvars_code}\n#{code}"
|
157
161
|
line_no = 0
|
@@ -161,7 +165,8 @@ class RubyLex
|
|
161
165
|
|
162
166
|
compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no|
|
163
167
|
lexer = Ripper::Lexer.new(inner_code, '-', line_no)
|
164
|
-
|
168
|
+
tokens = []
|
169
|
+
lexer.scan.each do |t|
|
165
170
|
next if t.pos.first == 0
|
166
171
|
prev_tk = tokens.last
|
167
172
|
position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize
|
@@ -171,55 +176,45 @@ class RubyLex
|
|
171
176
|
tokens << t
|
172
177
|
end
|
173
178
|
end
|
179
|
+
interpolate_ripper_ignored_tokens(original_code, tokens)
|
174
180
|
end
|
175
181
|
ensure
|
176
182
|
$VERBOSE = verbose
|
177
183
|
end
|
178
184
|
|
179
|
-
def
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
line_count = 0
|
184
|
-
@tokens.each_with_index do |t, i|
|
185
|
-
if t.tok.include?("\n")
|
186
|
-
line_count += t.tok.count("\n")
|
187
|
-
if line_count >= line_index
|
188
|
-
return prev_spaces
|
189
|
-
end
|
190
|
-
next if t.event == :on_tstring_content || t.event == :on_words_sep
|
191
|
-
if (@tokens.size - 1) > i
|
192
|
-
md = @tokens[i + 1].tok.match(/(\A +)/)
|
193
|
-
prev_spaces = md.nil? ? 0 : md[1].count(' ')
|
194
|
-
end
|
195
|
-
end
|
196
|
-
end
|
197
|
-
prev_spaces
|
198
|
-
end
|
199
|
-
|
200
|
-
def check_state(code, tokens)
|
201
|
-
ltype = process_literal_type(tokens)
|
202
|
-
indent = process_nesting_level(tokens)
|
203
|
-
continue = process_continue(tokens)
|
204
|
-
lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables)
|
205
|
-
code = "#{lvars_code}\n#{code}" if lvars_code
|
206
|
-
code_block_open = check_code_block(code, tokens)
|
207
|
-
[ltype, indent, continue, code_block_open]
|
185
|
+
def prompt(opens, continue, line_num_offset)
|
186
|
+
ltype = ltype_from_open_tokens(opens)
|
187
|
+
indent_level = calc_indent_level(opens)
|
188
|
+
@prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset)
|
208
189
|
end
|
209
190
|
|
210
191
|
def check_code_state(code)
|
211
192
|
check_target_code = code.gsub(/\s*\z/, '').concat("\n")
|
212
193
|
tokens = self.class.ripper_lex_without_warning(check_target_code, context: @context)
|
213
|
-
|
194
|
+
opens = IRB::NestingParser.open_tokens(tokens)
|
195
|
+
[tokens, opens, code_terminated?(code, tokens, opens)]
|
196
|
+
end
|
197
|
+
|
198
|
+
def code_terminated?(code, tokens, opens)
|
199
|
+
case check_code_syntax(code)
|
200
|
+
when :unrecoverable_error
|
201
|
+
true
|
202
|
+
when :recoverable_error
|
203
|
+
false
|
204
|
+
when :other_error
|
205
|
+
opens.empty? && !should_continue?(tokens)
|
206
|
+
when :valid
|
207
|
+
!should_continue?(tokens)
|
208
|
+
end
|
214
209
|
end
|
215
210
|
|
216
|
-
def save_prompt_to_context_io(
|
211
|
+
def save_prompt_to_context_io(opens, continue, line_num_offset)
|
217
212
|
# Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`.
|
218
|
-
|
213
|
+
prompt(opens, continue, line_num_offset)
|
219
214
|
end
|
220
215
|
|
221
216
|
def readmultiline
|
222
|
-
save_prompt_to_context_io(
|
217
|
+
save_prompt_to_context_io([], false, 0)
|
223
218
|
|
224
219
|
# multiline
|
225
220
|
return @input.call if @io.respond_to?(:check_termination)
|
@@ -237,11 +232,12 @@ class RubyLex
|
|
237
232
|
# Accept any single-line input for symbol aliases or commands that transform args
|
238
233
|
return code if single_line_command?(code)
|
239
234
|
|
240
|
-
|
241
|
-
return code
|
235
|
+
tokens, opens, terminated = check_code_state(code)
|
236
|
+
return code if terminated
|
242
237
|
|
243
238
|
line_offset += 1
|
244
|
-
|
239
|
+
continue = should_continue?(tokens)
|
240
|
+
save_prompt_to_context_io(opens, continue, line_offset)
|
245
241
|
end
|
246
242
|
end
|
247
243
|
|
@@ -259,32 +255,33 @@ class RubyLex
|
|
259
255
|
end
|
260
256
|
end
|
261
257
|
|
262
|
-
def
|
263
|
-
# last token
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
258
|
+
def should_continue?(tokens)
|
259
|
+
# Look at the last token and check if IRB need to continue reading next line.
|
260
|
+
# Example code that should continue: `a\` `a +` `a.`
|
261
|
+
# Trailing spaces, newline, comments are skipped
|
262
|
+
return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n"
|
263
|
+
|
264
|
+
tokens.reverse_each do |token|
|
265
|
+
case token.event
|
266
|
+
when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end
|
267
|
+
# Skip
|
268
|
+
when :on_regexp_end, :on_heredoc_end, :on_semicolon
|
269
|
+
# State is EXPR_BEG but should not continue
|
270
|
+
return false
|
271
|
+
else
|
272
|
+
# Endless range should not continue
|
273
|
+
return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/)
|
274
|
+
|
275
|
+
# EXPR_DOT and most of the EXPR_BEG should continue
|
276
|
+
return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT)
|
277
|
+
end
|
279
278
|
end
|
280
279
|
false
|
281
280
|
end
|
282
281
|
|
283
|
-
def
|
284
|
-
|
285
|
-
|
286
|
-
return true
|
287
|
-
end
|
282
|
+
def check_code_syntax(code)
|
283
|
+
lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables)
|
284
|
+
code = "#{lvars_code}\n#{code}"
|
288
285
|
|
289
286
|
begin # check if parser error are available
|
290
287
|
verbose, $VERBOSE = $VERBOSE, nil
|
@@ -303,6 +300,7 @@ class RubyLex
|
|
303
300
|
end
|
304
301
|
rescue EncodingError
|
305
302
|
# This is for a hash with invalid encoding symbol, {"\xAE": 1}
|
303
|
+
:unrecoverable_error
|
306
304
|
rescue SyntaxError => e
|
307
305
|
case e.message
|
308
306
|
when /unterminated (?:string|regexp) meets end of file/
|
@@ -315,7 +313,7 @@ class RubyLex
|
|
315
313
|
#
|
316
314
|
# example:
|
317
315
|
# '
|
318
|
-
return
|
316
|
+
return :recoverable_error
|
319
317
|
when /syntax error, unexpected end-of-input/
|
320
318
|
# "syntax error, unexpected end-of-input, expecting keyword_end"
|
321
319
|
#
|
@@ -325,7 +323,7 @@ class RubyLex
|
|
325
323
|
# if false
|
326
324
|
# fuga
|
327
325
|
# end
|
328
|
-
return
|
326
|
+
return :recoverable_error
|
329
327
|
when /syntax error, unexpected keyword_end/
|
330
328
|
# "syntax error, unexpected keyword_end"
|
331
329
|
#
|
@@ -335,402 +333,160 @@ class RubyLex
|
|
335
333
|
#
|
336
334
|
# example:
|
337
335
|
# end
|
338
|
-
return
|
336
|
+
return :unrecoverable_error
|
339
337
|
when /syntax error, unexpected '\.'/
|
340
338
|
# "syntax error, unexpected '.'"
|
341
339
|
#
|
342
340
|
# example:
|
343
341
|
# .
|
344
|
-
return
|
342
|
+
return :unrecoverable_error
|
345
343
|
when /unexpected tREGEXP_BEG/
|
346
344
|
# "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
|
347
345
|
#
|
348
346
|
# example:
|
349
347
|
# method / f /
|
350
|
-
return
|
348
|
+
return :unrecoverable_error
|
349
|
+
else
|
350
|
+
return :other_error
|
351
351
|
end
|
352
352
|
ensure
|
353
353
|
$VERBOSE = verbose
|
354
354
|
end
|
355
|
-
|
356
|
-
last_lex_state = tokens.last.state
|
357
|
-
|
358
|
-
if last_lex_state.allbits?(Ripper::EXPR_BEG)
|
359
|
-
return false
|
360
|
-
elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
|
361
|
-
return true
|
362
|
-
elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
|
363
|
-
return true
|
364
|
-
elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
|
365
|
-
return true
|
366
|
-
elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
|
367
|
-
return true
|
368
|
-
elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
|
369
|
-
return false
|
370
|
-
end
|
371
|
-
|
372
|
-
false
|
355
|
+
:valid
|
373
356
|
end
|
374
357
|
|
375
|
-
def
|
376
|
-
|
377
|
-
|
378
|
-
tokens.each_with_index { |t, index|
|
379
|
-
# detecting one-liner method definition
|
380
|
-
if in_oneliner_def.nil?
|
381
|
-
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
382
|
-
in_oneliner_def = :ENDFN
|
383
|
-
end
|
384
|
-
else
|
385
|
-
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
386
|
-
# continuing
|
387
|
-
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
388
|
-
if t.tok == '='
|
389
|
-
in_oneliner_def = :BODY
|
390
|
-
end
|
391
|
-
else
|
392
|
-
if in_oneliner_def == :BODY
|
393
|
-
# one-liner method definition
|
394
|
-
indent -= 1
|
395
|
-
end
|
396
|
-
in_oneliner_def = nil
|
397
|
-
end
|
398
|
-
end
|
399
|
-
|
358
|
+
def calc_indent_level(opens)
|
359
|
+
indent_level = 0
|
360
|
+
opens.each_with_index do |t, index|
|
400
361
|
case t.event
|
401
|
-
when :
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
when 'do'
|
409
|
-
syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index)
|
410
|
-
indent += 1 if syntax_of_do == :method_calling
|
411
|
-
when 'def', 'case', 'for', 'begin', 'class', 'module'
|
412
|
-
indent += 1
|
413
|
-
when 'if', 'unless', 'while', 'until'
|
414
|
-
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
|
415
|
-
indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL)
|
416
|
-
when 'end'
|
417
|
-
indent -= 1
|
362
|
+
when :on_heredoc_beg
|
363
|
+
if opens[index + 1]&.event != :on_heredoc_beg
|
364
|
+
if t.tok.match?(/^<<[~-]/)
|
365
|
+
indent_level += 1
|
366
|
+
else
|
367
|
+
indent_level = 0
|
368
|
+
end
|
418
369
|
end
|
370
|
+
when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick
|
371
|
+
# can be indented if t.tok starts with `%`
|
372
|
+
when :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_embexpr_beg
|
373
|
+
# can be indented but not indented in current implementation
|
374
|
+
when :on_embdoc_beg
|
375
|
+
indent_level = 0
|
376
|
+
else
|
377
|
+
indent_level += 1
|
419
378
|
end
|
420
|
-
|
421
|
-
|
422
|
-
indent
|
379
|
+
end
|
380
|
+
indent_level
|
423
381
|
end
|
424
382
|
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
return true
|
430
|
-
elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident
|
431
|
-
non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp }
|
432
|
-
if non_sp_index
|
433
|
-
prev_tk = tokens[non_sp_index]
|
434
|
-
if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period
|
435
|
-
# The target method call with receiver to pass the block with "do".
|
436
|
-
return true
|
437
|
-
end
|
438
|
-
end
|
439
|
-
end
|
440
|
-
false
|
383
|
+
FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg]
|
384
|
+
|
385
|
+
def free_indent_token?(token)
|
386
|
+
FREE_INDENT_TOKENS.include?(token&.event)
|
441
387
|
end
|
442
388
|
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
if is_method_calling?(tokens, i)
|
457
|
-
syntax_of_do = :method_calling
|
458
|
-
break if first_in_fomula
|
459
|
-
elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok)
|
460
|
-
# A loop syntax in front of "do" found.
|
461
|
-
#
|
462
|
-
# while cond do # also "until" or "for"
|
463
|
-
# end
|
464
|
-
#
|
465
|
-
# This "do" doesn't increment indent because the loop syntax already
|
466
|
-
# incremented.
|
467
|
-
syntax_of_do = :loop_syntax
|
468
|
-
break if first_in_fomula
|
389
|
+
# Calculates the difference of pasted code's indent and indent calculated from tokens
|
390
|
+
def indent_difference(lines, line_results, line_index)
|
391
|
+
loop do
|
392
|
+
_tokens, prev_opens, _next_opens, min_depth = line_results[line_index]
|
393
|
+
open_token = prev_opens.last
|
394
|
+
if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token))
|
395
|
+
# If the leading whitespace is an indent, return the difference
|
396
|
+
indent_level = calc_indent_level(prev_opens.take(min_depth))
|
397
|
+
calculated_indent = 2 * indent_level
|
398
|
+
actual_indent = lines[line_index][/^ */].size
|
399
|
+
return actual_indent - calculated_indent
|
400
|
+
elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/)
|
401
|
+
return 0
|
469
402
|
end
|
403
|
+
# If the leading whitespace is not an indent but part of a multiline token
|
404
|
+
# Calculate base_indent of the multiline token's beginning line
|
405
|
+
line_index = open_token.pos[0] - 1
|
470
406
|
end
|
471
|
-
syntax_of_do
|
472
407
|
end
|
473
408
|
|
474
|
-
def
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
first_in_fomula = true
|
484
|
-
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
|
485
|
-
first_in_fomula = true
|
486
|
-
end
|
487
|
-
if tk.event == :on_kw && tk.tok == 'for'
|
488
|
-
# A loop syntax in front of "do" found.
|
489
|
-
#
|
490
|
-
# while cond do # also "until" or "for"
|
491
|
-
# end
|
492
|
-
#
|
493
|
-
# This "do" doesn't increment indent because the loop syntax already
|
494
|
-
# incremented.
|
495
|
-
syntax_of_in = :for
|
496
|
-
end
|
497
|
-
break if first_in_fomula
|
409
|
+
def process_indent_level(tokens, lines, line_index, is_newline)
|
410
|
+
line_results = IRB::NestingParser.parse_by_line(tokens)
|
411
|
+
result = line_results[line_index]
|
412
|
+
if result
|
413
|
+
_tokens, prev_opens, next_opens, min_depth = result
|
414
|
+
else
|
415
|
+
# When last line is empty
|
416
|
+
prev_opens = next_opens = line_results.last[2]
|
417
|
+
min_depth = next_opens.size
|
498
418
|
end
|
499
|
-
syntax_of_in
|
500
|
-
end
|
501
419
|
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
in_oneliner_def = nil
|
506
|
-
@tokens.each_with_index do |t, index|
|
507
|
-
# detecting one-liner method definition
|
508
|
-
if in_oneliner_def.nil?
|
509
|
-
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
510
|
-
in_oneliner_def = :ENDFN
|
511
|
-
end
|
512
|
-
else
|
513
|
-
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
514
|
-
# continuing
|
515
|
-
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
516
|
-
if t.tok == '='
|
517
|
-
in_oneliner_def = :BODY
|
518
|
-
end
|
519
|
-
else
|
520
|
-
if in_oneliner_def == :BODY
|
521
|
-
# one-liner method definition
|
522
|
-
depth_difference -= 1
|
523
|
-
end
|
524
|
-
in_oneliner_def = nil
|
525
|
-
end
|
526
|
-
end
|
420
|
+
# To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation.
|
421
|
+
# Shortest open tokens can be calculated by `opens.take(min_depth)`
|
422
|
+
indent = 2 * calc_indent_level(prev_opens.take(min_depth))
|
527
423
|
|
528
|
-
|
529
|
-
when :on_ignored_nl, :on_nl, :on_comment
|
530
|
-
if index != (@tokens.size - 1) and in_oneliner_def != :BODY
|
531
|
-
depth_difference = 0
|
532
|
-
open_brace_on_line = 0
|
533
|
-
end
|
534
|
-
next
|
535
|
-
when :on_sp
|
536
|
-
next
|
537
|
-
end
|
424
|
+
preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size
|
538
425
|
|
539
|
-
|
540
|
-
|
541
|
-
depth_difference += 1
|
542
|
-
open_brace_on_line += 1
|
543
|
-
when :on_rbracket, :on_rbrace, :on_rparen
|
544
|
-
depth_difference -= 1 if open_brace_on_line > 0
|
545
|
-
when :on_kw
|
546
|
-
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
|
547
|
-
case t.tok
|
548
|
-
when 'do'
|
549
|
-
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
|
550
|
-
depth_difference += 1 if syntax_of_do == :method_calling
|
551
|
-
when 'def', 'case', 'for', 'begin', 'class', 'module'
|
552
|
-
depth_difference += 1
|
553
|
-
when 'if', 'unless', 'while', 'until', 'rescue'
|
554
|
-
# postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
|
555
|
-
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
556
|
-
depth_difference += 1
|
557
|
-
end
|
558
|
-
when 'else', 'elsif', 'ensure', 'when'
|
559
|
-
depth_difference += 1
|
560
|
-
when 'in'
|
561
|
-
unless is_the_in_correspond_to_a_for(@tokens, index)
|
562
|
-
depth_difference += 1
|
563
|
-
end
|
564
|
-
when 'end'
|
565
|
-
depth_difference -= 1
|
566
|
-
end
|
567
|
-
end
|
568
|
-
end
|
569
|
-
depth_difference
|
570
|
-
end
|
426
|
+
prev_open_token = prev_opens.last
|
427
|
+
next_open_token = next_opens.last
|
571
428
|
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
if heredoc_scope?
|
582
|
-
return lines[line_index][/^ */].length
|
429
|
+
# Calculates base indent for pasted code on the line where prev_open_token is located
|
430
|
+
# irb(main):001:1* if a # base_indent is 2, indent calculated from tokens is 0
|
431
|
+
# irb(main):002:1* if b # base_indent is 6, indent calculated from tokens is 2
|
432
|
+
# irb(main):003:0> c # base_indent is 6, indent calculated from tokens is 4
|
433
|
+
if prev_open_token
|
434
|
+
base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max
|
435
|
+
else
|
436
|
+
base_indent = 0
|
583
437
|
end
|
584
438
|
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
in_oneliner_def = :ENDFN
|
590
|
-
end
|
439
|
+
if free_indent_token?(prev_open_token)
|
440
|
+
if is_newline && prev_open_token.pos[0] == line_index
|
441
|
+
# First newline inside free-indent token
|
442
|
+
base_indent + indent
|
591
443
|
else
|
592
|
-
|
593
|
-
|
594
|
-
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
595
|
-
if t.tok == '='
|
596
|
-
in_oneliner_def = :BODY
|
597
|
-
end
|
598
|
-
else
|
599
|
-
if in_oneliner_def == :BODY
|
600
|
-
# one-liner method definition
|
601
|
-
if is_first_printable_of_line
|
602
|
-
corresponding_token_depth = spaces_of_nest.pop
|
603
|
-
else
|
604
|
-
spaces_of_nest.pop
|
605
|
-
corresponding_token_depth = nil
|
606
|
-
end
|
607
|
-
end
|
608
|
-
in_oneliner_def = nil
|
609
|
-
end
|
444
|
+
# Accept any number of indent inside free-indent token
|
445
|
+
preserve_indent
|
610
446
|
end
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
is_first_printable_of_line = true
|
619
|
-
open_brace_on_line = 0
|
620
|
-
end
|
621
|
-
next
|
622
|
-
when :on_sp
|
623
|
-
spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line
|
624
|
-
is_first_spaces_of_line = false
|
625
|
-
next
|
447
|
+
elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg
|
448
|
+
if prev_open_token&.event == next_open_token&.event
|
449
|
+
# Accept any number of indent inside embdoc content
|
450
|
+
preserve_indent
|
451
|
+
else
|
452
|
+
# =begin or =end
|
453
|
+
0
|
626
454
|
end
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
455
|
+
elsif prev_open_token&.event == :on_heredoc_beg
|
456
|
+
tok = prev_open_token.tok
|
457
|
+
if prev_opens.size <= next_opens.size
|
458
|
+
if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token
|
459
|
+
# First line in heredoc
|
460
|
+
tok.match?(/^<<[-~]/) ? base_indent + indent : indent
|
461
|
+
elsif tok.match?(/^<<~/)
|
462
|
+
# Accept extra indent spaces inside `<<~` heredoc
|
463
|
+
[base_indent + indent, preserve_indent].max
|
635
464
|
else
|
636
|
-
|
637
|
-
|
638
|
-
end
|
639
|
-
open_brace_on_line -= 1
|
640
|
-
when :on_kw
|
641
|
-
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
|
642
|
-
case t.tok
|
643
|
-
when 'do'
|
644
|
-
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
|
645
|
-
if syntax_of_do == :method_calling
|
646
|
-
spaces_of_nest.push(spaces_at_line_head)
|
647
|
-
end
|
648
|
-
when 'def', 'case', 'for', 'begin', 'class', 'module'
|
649
|
-
spaces_of_nest.push(spaces_at_line_head)
|
650
|
-
when 'rescue'
|
651
|
-
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
652
|
-
corresponding_token_depth = spaces_of_nest.last
|
653
|
-
end
|
654
|
-
when 'if', 'unless', 'while', 'until'
|
655
|
-
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
|
656
|
-
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
657
|
-
spaces_of_nest.push(spaces_at_line_head)
|
658
|
-
end
|
659
|
-
when 'else', 'elsif', 'ensure', 'when'
|
660
|
-
corresponding_token_depth = spaces_of_nest.last
|
661
|
-
when 'in'
|
662
|
-
if in_keyword_case_scope?
|
663
|
-
corresponding_token_depth = spaces_of_nest.last
|
664
|
-
end
|
665
|
-
when 'end'
|
666
|
-
if is_first_printable_of_line
|
667
|
-
corresponding_token_depth = spaces_of_nest.pop
|
668
|
-
else
|
669
|
-
spaces_of_nest.pop
|
670
|
-
corresponding_token_depth = nil
|
671
|
-
end
|
465
|
+
# Accept any number of indent inside other heredoc
|
466
|
+
preserve_indent
|
672
467
|
end
|
468
|
+
else
|
469
|
+
# Heredoc close
|
470
|
+
prev_line_indent_level = calc_indent_level(prev_opens)
|
471
|
+
tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0
|
673
472
|
end
|
674
|
-
|
675
|
-
|
473
|
+
else
|
474
|
+
base_indent + indent
|
676
475
|
end
|
677
|
-
corresponding_token_depth
|
678
476
|
end
|
679
477
|
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
t = tokens[i]
|
687
|
-
case t.event
|
688
|
-
when *end_type.last
|
689
|
-
start_token.pop
|
690
|
-
end_type.pop
|
691
|
-
when :on_tstring_beg
|
692
|
-
start_token << t
|
693
|
-
end_type << [:on_tstring_end, :on_label_end]
|
694
|
-
when :on_regexp_beg
|
695
|
-
start_token << t
|
696
|
-
end_type << :on_regexp_end
|
697
|
-
when :on_symbeg
|
698
|
-
acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick}
|
699
|
-
if (i + 1) < tokens.size
|
700
|
-
if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st }
|
701
|
-
start_token << t
|
702
|
-
end_type << :on_tstring_end
|
703
|
-
else
|
704
|
-
i += 1
|
705
|
-
end
|
706
|
-
end
|
707
|
-
when :on_backtick
|
708
|
-
if t.state.allbits?(Ripper::EXPR_BEG)
|
709
|
-
start_token << t
|
710
|
-
end_type << :on_tstring_end
|
711
|
-
end
|
712
|
-
when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
|
713
|
-
start_token << t
|
714
|
-
end_type << :on_tstring_end
|
715
|
-
when :on_heredoc_beg
|
716
|
-
pending_heredocs << t
|
717
|
-
end
|
478
|
+
LTYPE_TOKENS = %i[
|
479
|
+
on_heredoc_beg on_tstring_beg
|
480
|
+
on_regexp_beg on_symbeg on_backtick
|
481
|
+
on_symbols_beg on_qsymbols_beg
|
482
|
+
on_words_beg on_qwords_beg
|
483
|
+
]
|
718
484
|
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
end_type << :on_heredoc_end
|
723
|
-
end
|
724
|
-
pending_heredocs = []
|
725
|
-
end
|
726
|
-
i += 1
|
485
|
+
def ltype_from_open_tokens(opens)
|
486
|
+
start_token = opens.reverse_each.find do |tok|
|
487
|
+
LTYPE_TOKENS.include?(tok.event)
|
727
488
|
end
|
728
|
-
|
729
|
-
end
|
730
|
-
|
731
|
-
def process_literal_type(tokens)
|
732
|
-
start_token = check_string_literal(tokens)
|
733
|
-
return nil if start_token == ""
|
489
|
+
return nil unless start_token
|
734
490
|
|
735
491
|
case start_token&.event
|
736
492
|
when :on_tstring_beg
|
@@ -783,47 +539,16 @@ class RubyLex
|
|
783
539
|
end
|
784
540
|
end
|
785
541
|
|
786
|
-
if first_token.
|
787
|
-
return false
|
788
|
-
elsif first_token && first_token.state == Ripper::EXPR_DOT
|
789
|
-
return false
|
790
|
-
else
|
542
|
+
if first_token && first_token.state != Ripper::EXPR_DOT
|
791
543
|
tokens_without_last_line = tokens[0..index]
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
if ltype or indent > 0 or continue or code_block_open
|
797
|
-
return false
|
798
|
-
else
|
799
|
-
return last_line_tokens.map(&:tok).join('')
|
544
|
+
code_without_last_line = tokens_without_last_line.map(&:tok).join
|
545
|
+
opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line)
|
546
|
+
if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
|
547
|
+
return last_line_tokens.map(&:tok).join
|
800
548
|
end
|
801
549
|
end
|
802
550
|
end
|
803
551
|
false
|
804
552
|
end
|
805
|
-
|
806
|
-
private
|
807
|
-
|
808
|
-
def heredoc_scope?
|
809
|
-
heredoc_tokens = @tokens.select { |t| [:on_heredoc_beg, :on_heredoc_end].include?(t.event) }
|
810
|
-
heredoc_tokens[-1]&.event == :on_heredoc_beg
|
811
|
-
end
|
812
|
-
|
813
|
-
def in_keyword_case_scope?
|
814
|
-
kw_tokens = @tokens.select { |t| t.event == :on_kw && ['case', 'for', 'end'].include?(t.tok) }
|
815
|
-
counter = 0
|
816
|
-
kw_tokens.reverse.each do |t|
|
817
|
-
if t.tok == 'case'
|
818
|
-
return true if counter.zero?
|
819
|
-
counter += 1
|
820
|
-
elsif t.tok == 'for'
|
821
|
-
counter += 1
|
822
|
-
elsif t.tok == 'end'
|
823
|
-
counter -= 1
|
824
|
-
end
|
825
|
-
end
|
826
|
-
false
|
827
|
-
end
|
828
553
|
end
|
829
554
|
# :startdoc:
|