irb 1.7.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/irb/ruby-lex.rb CHANGED
@@ -6,6 +6,7 @@
6
6
 
7
7
  require "ripper"
8
8
  require "jruby" if RUBY_ENGINE == "jruby"
9
+ require_relative "nesting_parser"
9
10
 
10
11
  # :stopdoc:
11
12
  class RubyLex
@@ -54,8 +55,7 @@ class RubyLex
54
55
  if @io.respond_to?(:check_termination)
55
56
  @io.check_termination do |code|
56
57
  if Reline::IOGate.in_pasting?
57
- lex = RubyLex.new(@context)
58
- rest = lex.check_termination_in_prev_line(code)
58
+ rest = check_termination_in_prev_line(code)
59
59
  if rest
60
60
  Reline.delete_text
61
61
  rest.bytes.reverse_each do |c|
@@ -69,65 +69,36 @@ class RubyLex
69
69
  # Accept any single-line input for symbol aliases or commands that transform args
70
70
  next true if single_line_command?(code)
71
71
 
72
- ltype, indent, continue, code_block_open = check_code_state(code)
73
- if ltype or indent > 0 or continue or code_block_open
74
- false
75
- else
76
- true
77
- end
72
+ _tokens, _opens, terminated = check_code_state(code)
73
+ terminated
78
74
  end
79
75
  end
80
76
  end
81
77
  if @io.respond_to?(:dynamic_prompt)
82
78
  @io.dynamic_prompt do |lines|
83
79
  lines << '' if lines.empty?
84
- result = []
85
80
  tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
86
- code = String.new
87
- partial_tokens = []
88
- unprocessed_tokens = []
89
- line_num_offset = 0
90
- tokens.each do |t|
91
- partial_tokens << t
92
- unprocessed_tokens << t
93
- if t.tok.include?("\n")
94
- t_str = t.tok
95
- t_str.each_line("\n") do |s|
96
- code << s
97
- next unless s.include?("\n")
98
- ltype, indent, continue, code_block_open = check_state(code, partial_tokens)
99
- result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
100
- line_num_offset += 1
101
- end
102
- unprocessed_tokens = []
103
- else
104
- code << t.tok
81
+ line_results = IRB::NestingParser.parse_by_line(tokens)
82
+ tokens_until_line = []
83
+ line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset|
84
+ line_tokens.each do |token, _s|
85
+ # Avoid appending duplicated token. Tokens that include "\n" like multiline tstring_content can exist in multiple lines.
86
+ tokens_until_line << token if token != tokens_until_line.last
105
87
  end
88
+ continue = should_continue?(tokens_until_line)
89
+ prompt(next_opens, continue, line_num_offset)
106
90
  end
107
-
108
- unless unprocessed_tokens.empty?
109
- ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens)
110
- result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
111
- end
112
- result
113
91
  end
114
92
  end
115
93
 
116
94
  if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
117
95
  @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
118
- if is_newline
119
- @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: @context)
120
- prev_spaces = find_prev_spaces(line_index)
121
- depth_difference = check_newline_depth_difference
122
- depth_difference = 0 if depth_difference < 0
123
- prev_spaces + depth_difference * 2
124
- else
125
- code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
126
- last_line = lines[line_index]&.byteslice(0, byte_pointer)
127
- code += last_line if last_line
128
- @tokens = self.class.ripper_lex_without_warning(code, context: @context)
129
- check_corresponding_token_depth(lines, line_index)
130
- end
96
+ next nil if lines == [nil] # Workaround for exit IRB with CTRL+d
97
+ next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/)
98
+
99
+ code = lines[0..line_index].map { |l| "#{l}\n" }.join
100
+ tokens = self.class.ripper_lex_without_warning(code, context: @context)
101
+ process_indent_level(tokens, lines, line_index, is_newline)
131
102
  end
132
103
  end
133
104
  end
@@ -149,9 +120,42 @@ class RubyLex
149
120
  "#{local_variables.join('=')}=nil;" unless local_variables.empty?
150
121
  end
151
122
 
123
+ # Some part of the code is not included in Ripper's token.
124
+ # Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr.
125
+ # With interpolated tokens, tokens.map(&:tok).join will be equal to code.
126
+ def self.interpolate_ripper_ignored_tokens(code, tokens)
127
+ line_positions = [0]
128
+ code.lines.each do |line|
129
+ line_positions << line_positions.last + line.bytesize
130
+ end
131
+ prev_byte_pos = 0
132
+ interpolated = []
133
+ prev_line = 1
134
+ tokens.each do |t|
135
+ line, col = t.pos
136
+ byte_pos = line_positions[line - 1] + col
137
+ if prev_byte_pos < byte_pos
138
+ tok = code.byteslice(prev_byte_pos...byte_pos)
139
+ pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
140
+ interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
141
+ prev_line += tok.count("\n")
142
+ end
143
+ interpolated << t
144
+ prev_byte_pos = byte_pos + t.tok.bytesize
145
+ prev_line += t.tok.count("\n")
146
+ end
147
+ if prev_byte_pos < code.bytesize
148
+ tok = code.byteslice(prev_byte_pos..)
149
+ pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
150
+ interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
151
+ end
152
+ interpolated
153
+ end
154
+
152
155
  def self.ripper_lex_without_warning(code, context: nil)
153
156
  verbose, $VERBOSE = $VERBOSE, nil
154
157
  lvars_code = generate_local_variables_assign_code(context&.local_variables || [])
158
+ original_code = code
155
159
  if lvars_code
156
160
  code = "#{lvars_code}\n#{code}"
157
161
  line_no = 0
@@ -161,7 +165,8 @@ class RubyLex
161
165
 
162
166
  compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no|
163
167
  lexer = Ripper::Lexer.new(inner_code, '-', line_no)
164
- lexer.scan.each_with_object([]) do |t, tokens|
168
+ tokens = []
169
+ lexer.scan.each do |t|
165
170
  next if t.pos.first == 0
166
171
  prev_tk = tokens.last
167
172
  position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize
@@ -171,55 +176,45 @@ class RubyLex
171
176
  tokens << t
172
177
  end
173
178
  end
179
+ interpolate_ripper_ignored_tokens(original_code, tokens)
174
180
  end
175
181
  ensure
176
182
  $VERBOSE = verbose
177
183
  end
178
184
 
179
- def find_prev_spaces(line_index)
180
- return 0 if @tokens.size == 0
181
- md = @tokens[0].tok.match(/(\A +)/)
182
- prev_spaces = md.nil? ? 0 : md[1].count(' ')
183
- line_count = 0
184
- @tokens.each_with_index do |t, i|
185
- if t.tok.include?("\n")
186
- line_count += t.tok.count("\n")
187
- if line_count >= line_index
188
- return prev_spaces
189
- end
190
- next if t.event == :on_tstring_content || t.event == :on_words_sep
191
- if (@tokens.size - 1) > i
192
- md = @tokens[i + 1].tok.match(/(\A +)/)
193
- prev_spaces = md.nil? ? 0 : md[1].count(' ')
194
- end
195
- end
196
- end
197
- prev_spaces
198
- end
199
-
200
- def check_state(code, tokens)
201
- ltype = process_literal_type(tokens)
202
- indent = process_nesting_level(tokens)
203
- continue = process_continue(tokens)
204
- lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables)
205
- code = "#{lvars_code}\n#{code}" if lvars_code
206
- code_block_open = check_code_block(code, tokens)
207
- [ltype, indent, continue, code_block_open]
185
+ def prompt(opens, continue, line_num_offset)
186
+ ltype = ltype_from_open_tokens(opens)
187
+ indent_level = calc_indent_level(opens)
188
+ @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset)
208
189
  end
209
190
 
210
191
  def check_code_state(code)
211
192
  check_target_code = code.gsub(/\s*\z/, '').concat("\n")
212
193
  tokens = self.class.ripper_lex_without_warning(check_target_code, context: @context)
213
- check_state(check_target_code, tokens)
194
+ opens = IRB::NestingParser.open_tokens(tokens)
195
+ [tokens, opens, code_terminated?(code, tokens, opens)]
196
+ end
197
+
198
+ def code_terminated?(code, tokens, opens)
199
+ case check_code_syntax(code)
200
+ when :unrecoverable_error
201
+ true
202
+ when :recoverable_error
203
+ false
204
+ when :other_error
205
+ opens.empty? && !should_continue?(tokens)
206
+ when :valid
207
+ !should_continue?(tokens)
208
+ end
214
209
  end
215
210
 
216
- def save_prompt_to_context_io(ltype, indent, continue, line_num_offset)
211
+ def save_prompt_to_context_io(opens, continue, line_num_offset)
217
212
  # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`.
218
- @prompt.call(ltype, indent, continue, @line_no + line_num_offset)
213
+ prompt(opens, continue, line_num_offset)
219
214
  end
220
215
 
221
216
  def readmultiline
222
- save_prompt_to_context_io(nil, 0, false, 0)
217
+ save_prompt_to_context_io([], false, 0)
223
218
 
224
219
  # multiline
225
220
  return @input.call if @io.respond_to?(:check_termination)
@@ -237,11 +232,12 @@ class RubyLex
237
232
  # Accept any single-line input for symbol aliases or commands that transform args
238
233
  return code if single_line_command?(code)
239
234
 
240
- ltype, indent, continue, code_block_open = check_code_state(code)
241
- return code unless ltype or indent > 0 or continue or code_block_open
235
+ tokens, opens, terminated = check_code_state(code)
236
+ return code if terminated
242
237
 
243
238
  line_offset += 1
244
- save_prompt_to_context_io(ltype, indent, continue, line_offset)
239
+ continue = should_continue?(tokens)
240
+ save_prompt_to_context_io(opens, continue, line_offset)
245
241
  end
246
242
  end
247
243
 
@@ -259,32 +255,33 @@ class RubyLex
259
255
  end
260
256
  end
261
257
 
262
- def process_continue(tokens)
263
- # last token is always newline
264
- if tokens.size >= 2 and tokens[-2].event == :on_regexp_end
265
- # end of regexp literal
266
- return false
267
- elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon
268
- return false
269
- elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok)
270
- return false
271
- elsif !tokens.empty? and tokens.last.tok == "\\\n"
272
- return true
273
- elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n"
274
- return false
275
- elsif tokens.size >= 2 and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/
276
- # end of literal except for regexp
277
- # endless range at end of line is not a continue
278
- return true
258
+ def should_continue?(tokens)
259
+ # Look at the last token and check if IRB need to continue reading next line.
260
+ # Example code that should continue: `a\` `a +` `a.`
261
+ # Trailing spaces, newline, comments are skipped
262
+ return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n"
263
+
264
+ tokens.reverse_each do |token|
265
+ case token.event
266
+ when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end
267
+ # Skip
268
+ when :on_regexp_end, :on_heredoc_end, :on_semicolon
269
+ # State is EXPR_BEG but should not continue
270
+ return false
271
+ else
272
+ # Endless range should not continue
273
+ return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/)
274
+
275
+ # EXPR_DOT and most of the EXPR_BEG should continue
276
+ return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT)
277
+ end
279
278
  end
280
279
  false
281
280
  end
282
281
 
283
- def check_code_block(code, tokens)
284
- return true if tokens.empty?
285
- if tokens.last.event == :on_heredoc_beg
286
- return true
287
- end
282
+ def check_code_syntax(code)
283
+ lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables)
284
+ code = "#{lvars_code}\n#{code}"
288
285
 
289
286
  begin # check if parser error are available
290
287
  verbose, $VERBOSE = $VERBOSE, nil
@@ -303,6 +300,7 @@ class RubyLex
303
300
  end
304
301
  rescue EncodingError
305
302
  # This is for a hash with invalid encoding symbol, {"\xAE": 1}
303
+ :unrecoverable_error
306
304
  rescue SyntaxError => e
307
305
  case e.message
308
306
  when /unterminated (?:string|regexp) meets end of file/
@@ -315,7 +313,7 @@ class RubyLex
315
313
  #
316
314
  # example:
317
315
  # '
318
- return true
316
+ return :recoverable_error
319
317
  when /syntax error, unexpected end-of-input/
320
318
  # "syntax error, unexpected end-of-input, expecting keyword_end"
321
319
  #
@@ -325,7 +323,7 @@ class RubyLex
325
323
  # if false
326
324
  # fuga
327
325
  # end
328
- return true
326
+ return :recoverable_error
329
327
  when /syntax error, unexpected keyword_end/
330
328
  # "syntax error, unexpected keyword_end"
331
329
  #
@@ -335,402 +333,160 @@ class RubyLex
335
333
  #
336
334
  # example:
337
335
  # end
338
- return false
336
+ return :unrecoverable_error
339
337
  when /syntax error, unexpected '\.'/
340
338
  # "syntax error, unexpected '.'"
341
339
  #
342
340
  # example:
343
341
  # .
344
- return false
342
+ return :unrecoverable_error
345
343
  when /unexpected tREGEXP_BEG/
346
344
  # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
347
345
  #
348
346
  # example:
349
347
  # method / f /
350
- return false
348
+ return :unrecoverable_error
349
+ else
350
+ return :other_error
351
351
  end
352
352
  ensure
353
353
  $VERBOSE = verbose
354
354
  end
355
-
356
- last_lex_state = tokens.last.state
357
-
358
- if last_lex_state.allbits?(Ripper::EXPR_BEG)
359
- return false
360
- elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
361
- return true
362
- elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
363
- return true
364
- elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
365
- return true
366
- elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
367
- return true
368
- elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
369
- return false
370
- end
371
-
372
- false
355
+ :valid
373
356
  end
374
357
 
375
- def process_nesting_level(tokens)
376
- indent = 0
377
- in_oneliner_def = nil
378
- tokens.each_with_index { |t, index|
379
- # detecting one-liner method definition
380
- if in_oneliner_def.nil?
381
- if t.state.allbits?(Ripper::EXPR_ENDFN)
382
- in_oneliner_def = :ENDFN
383
- end
384
- else
385
- if t.state.allbits?(Ripper::EXPR_ENDFN)
386
- # continuing
387
- elsif t.state.allbits?(Ripper::EXPR_BEG)
388
- if t.tok == '='
389
- in_oneliner_def = :BODY
390
- end
391
- else
392
- if in_oneliner_def == :BODY
393
- # one-liner method definition
394
- indent -= 1
395
- end
396
- in_oneliner_def = nil
397
- end
398
- end
399
-
358
+ def calc_indent_level(opens)
359
+ indent_level = 0
360
+ opens.each_with_index do |t, index|
400
361
  case t.event
401
- when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
402
- indent += 1
403
- when :on_rbracket, :on_rbrace, :on_rparen
404
- indent -= 1
405
- when :on_kw
406
- next if index > 0 and tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
407
- case t.tok
408
- when 'do'
409
- syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index)
410
- indent += 1 if syntax_of_do == :method_calling
411
- when 'def', 'case', 'for', 'begin', 'class', 'module'
412
- indent += 1
413
- when 'if', 'unless', 'while', 'until'
414
- # postfix if/unless/while/until must be Ripper::EXPR_LABEL
415
- indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL)
416
- when 'end'
417
- indent -= 1
362
+ when :on_heredoc_beg
363
+ if opens[index + 1]&.event != :on_heredoc_beg
364
+ if t.tok.match?(/^<<[~-]/)
365
+ indent_level += 1
366
+ else
367
+ indent_level = 0
368
+ end
418
369
  end
370
+ when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick
371
+ # can be indented if t.tok starts with `%`
372
+ when :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_embexpr_beg
373
+ # can be indented but not indented in current implementation
374
+ when :on_embdoc_beg
375
+ indent_level = 0
376
+ else
377
+ indent_level += 1
419
378
  end
420
- # percent literals are not indented
421
- }
422
- indent
379
+ end
380
+ indent_level
423
381
  end
424
382
 
425
- def is_method_calling?(tokens, index)
426
- tk = tokens[index]
427
- if tk.state.anybits?(Ripper::EXPR_CMDARG) and tk.event == :on_ident
428
- # The target method call to pass the block with "do".
429
- return true
430
- elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident
431
- non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp }
432
- if non_sp_index
433
- prev_tk = tokens[non_sp_index]
434
- if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period
435
- # The target method call with receiver to pass the block with "do".
436
- return true
437
- end
438
- end
439
- end
440
- false
383
+ FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg]
384
+
385
+ def free_indent_token?(token)
386
+ FREE_INDENT_TOKENS.include?(token&.event)
441
387
  end
442
388
 
443
- def take_corresponding_syntax_to_kw_do(tokens, index)
444
- syntax_of_do = nil
445
- # Finding a syntax corresponding to "do".
446
- index.downto(0) do |i|
447
- tk = tokens[i]
448
- # In "continue", the token isn't the corresponding syntax to "do".
449
- non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
450
- first_in_fomula = false
451
- if non_sp_index.nil?
452
- first_in_fomula = true
453
- elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
454
- first_in_fomula = true
455
- end
456
- if is_method_calling?(tokens, i)
457
- syntax_of_do = :method_calling
458
- break if first_in_fomula
459
- elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok)
460
- # A loop syntax in front of "do" found.
461
- #
462
- # while cond do # also "until" or "for"
463
- # end
464
- #
465
- # This "do" doesn't increment indent because the loop syntax already
466
- # incremented.
467
- syntax_of_do = :loop_syntax
468
- break if first_in_fomula
389
+ # Calculates the difference of pasted code's indent and indent calculated from tokens
390
+ def indent_difference(lines, line_results, line_index)
391
+ loop do
392
+ _tokens, prev_opens, _next_opens, min_depth = line_results[line_index]
393
+ open_token = prev_opens.last
394
+ if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token))
395
+ # If the leading whitespace is an indent, return the difference
396
+ indent_level = calc_indent_level(prev_opens.take(min_depth))
397
+ calculated_indent = 2 * indent_level
398
+ actual_indent = lines[line_index][/^ */].size
399
+ return actual_indent - calculated_indent
400
+ elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/)
401
+ return 0
469
402
  end
403
+ # If the leading whitespace is not an indent but part of a multiline token
404
+ # Calculate base_indent of the multiline token's beginning line
405
+ line_index = open_token.pos[0] - 1
470
406
  end
471
- syntax_of_do
472
407
  end
473
408
 
474
- def is_the_in_correspond_to_a_for(tokens, index)
475
- syntax_of_in = nil
476
- # Finding a syntax corresponding to "do".
477
- index.downto(0) do |i|
478
- tk = tokens[i]
479
- # In "continue", the token isn't the corresponding syntax to "do".
480
- non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
481
- first_in_fomula = false
482
- if non_sp_index.nil?
483
- first_in_fomula = true
484
- elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
485
- first_in_fomula = true
486
- end
487
- if tk.event == :on_kw && tk.tok == 'for'
488
- # A loop syntax in front of "do" found.
489
- #
490
- # while cond do # also "until" or "for"
491
- # end
492
- #
493
- # This "do" doesn't increment indent because the loop syntax already
494
- # incremented.
495
- syntax_of_in = :for
496
- end
497
- break if first_in_fomula
409
+ def process_indent_level(tokens, lines, line_index, is_newline)
410
+ line_results = IRB::NestingParser.parse_by_line(tokens)
411
+ result = line_results[line_index]
412
+ if result
413
+ _tokens, prev_opens, next_opens, min_depth = result
414
+ else
415
+ # When last line is empty
416
+ prev_opens = next_opens = line_results.last[2]
417
+ min_depth = next_opens.size
498
418
  end
499
- syntax_of_in
500
- end
501
419
 
502
- def check_newline_depth_difference
503
- depth_difference = 0
504
- open_brace_on_line = 0
505
- in_oneliner_def = nil
506
- @tokens.each_with_index do |t, index|
507
- # detecting one-liner method definition
508
- if in_oneliner_def.nil?
509
- if t.state.allbits?(Ripper::EXPR_ENDFN)
510
- in_oneliner_def = :ENDFN
511
- end
512
- else
513
- if t.state.allbits?(Ripper::EXPR_ENDFN)
514
- # continuing
515
- elsif t.state.allbits?(Ripper::EXPR_BEG)
516
- if t.tok == '='
517
- in_oneliner_def = :BODY
518
- end
519
- else
520
- if in_oneliner_def == :BODY
521
- # one-liner method definition
522
- depth_difference -= 1
523
- end
524
- in_oneliner_def = nil
525
- end
526
- end
420
+ # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation.
421
+ # Shortest open tokens can be calculated by `opens.take(min_depth)`
422
+ indent = 2 * calc_indent_level(prev_opens.take(min_depth))
527
423
 
528
- case t.event
529
- when :on_ignored_nl, :on_nl, :on_comment
530
- if index != (@tokens.size - 1) and in_oneliner_def != :BODY
531
- depth_difference = 0
532
- open_brace_on_line = 0
533
- end
534
- next
535
- when :on_sp
536
- next
537
- end
424
+ preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size
538
425
 
539
- case t.event
540
- when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
541
- depth_difference += 1
542
- open_brace_on_line += 1
543
- when :on_rbracket, :on_rbrace, :on_rparen
544
- depth_difference -= 1 if open_brace_on_line > 0
545
- when :on_kw
546
- next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
547
- case t.tok
548
- when 'do'
549
- syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
550
- depth_difference += 1 if syntax_of_do == :method_calling
551
- when 'def', 'case', 'for', 'begin', 'class', 'module'
552
- depth_difference += 1
553
- when 'if', 'unless', 'while', 'until', 'rescue'
554
- # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
555
- unless t.state.allbits?(Ripper::EXPR_LABEL)
556
- depth_difference += 1
557
- end
558
- when 'else', 'elsif', 'ensure', 'when'
559
- depth_difference += 1
560
- when 'in'
561
- unless is_the_in_correspond_to_a_for(@tokens, index)
562
- depth_difference += 1
563
- end
564
- when 'end'
565
- depth_difference -= 1
566
- end
567
- end
568
- end
569
- depth_difference
570
- end
426
+ prev_open_token = prev_opens.last
427
+ next_open_token = next_opens.last
571
428
 
572
- def check_corresponding_token_depth(lines, line_index)
573
- corresponding_token_depth = nil
574
- is_first_spaces_of_line = true
575
- is_first_printable_of_line = true
576
- spaces_of_nest = []
577
- spaces_at_line_head = 0
578
- open_brace_on_line = 0
579
- in_oneliner_def = nil
580
-
581
- if heredoc_scope?
582
- return lines[line_index][/^ */].length
429
+ # Calculates base indent for pasted code on the line where prev_open_token is located
430
+ # irb(main):001:1* if a # base_indent is 2, indent calculated from tokens is 0
431
+ # irb(main):002:1* if b # base_indent is 6, indent calculated from tokens is 2
432
+ # irb(main):003:0> c # base_indent is 6, indent calculated from tokens is 4
433
+ if prev_open_token
434
+ base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max
435
+ else
436
+ base_indent = 0
583
437
  end
584
438
 
585
- @tokens.each_with_index do |t, index|
586
- # detecting one-liner method definition
587
- if in_oneliner_def.nil?
588
- if t.state.allbits?(Ripper::EXPR_ENDFN)
589
- in_oneliner_def = :ENDFN
590
- end
439
+ if free_indent_token?(prev_open_token)
440
+ if is_newline && prev_open_token.pos[0] == line_index
441
+ # First newline inside free-indent token
442
+ base_indent + indent
591
443
  else
592
- if t.state.allbits?(Ripper::EXPR_ENDFN)
593
- # continuing
594
- elsif t.state.allbits?(Ripper::EXPR_BEG)
595
- if t.tok == '='
596
- in_oneliner_def = :BODY
597
- end
598
- else
599
- if in_oneliner_def == :BODY
600
- # one-liner method definition
601
- if is_first_printable_of_line
602
- corresponding_token_depth = spaces_of_nest.pop
603
- else
604
- spaces_of_nest.pop
605
- corresponding_token_depth = nil
606
- end
607
- end
608
- in_oneliner_def = nil
609
- end
444
+ # Accept any number of indent inside free-indent token
445
+ preserve_indent
610
446
  end
611
-
612
- case t.event
613
- when :on_ignored_nl, :on_nl, :on_comment, :on_heredoc_end, :on_embdoc_end
614
- if in_oneliner_def != :BODY
615
- corresponding_token_depth = nil
616
- spaces_at_line_head = 0
617
- is_first_spaces_of_line = true
618
- is_first_printable_of_line = true
619
- open_brace_on_line = 0
620
- end
621
- next
622
- when :on_sp
623
- spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line
624
- is_first_spaces_of_line = false
625
- next
447
+ elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg
448
+ if prev_open_token&.event == next_open_token&.event
449
+ # Accept any number of indent inside embdoc content
450
+ preserve_indent
451
+ else
452
+ # =begin or =end
453
+ 0
626
454
  end
627
-
628
- case t.event
629
- when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
630
- spaces_of_nest.push(spaces_at_line_head + open_brace_on_line * 2)
631
- open_brace_on_line += 1
632
- when :on_rbracket, :on_rbrace, :on_rparen
633
- if is_first_printable_of_line
634
- corresponding_token_depth = spaces_of_nest.pop
455
+ elsif prev_open_token&.event == :on_heredoc_beg
456
+ tok = prev_open_token.tok
457
+ if prev_opens.size <= next_opens.size
458
+ if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token
459
+ # First line in heredoc
460
+ tok.match?(/^<<[-~]/) ? base_indent + indent : indent
461
+ elsif tok.match?(/^<<~/)
462
+ # Accept extra indent spaces inside `<<~` heredoc
463
+ [base_indent + indent, preserve_indent].max
635
464
  else
636
- spaces_of_nest.pop
637
- corresponding_token_depth = nil
638
- end
639
- open_brace_on_line -= 1
640
- when :on_kw
641
- next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
642
- case t.tok
643
- when 'do'
644
- syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
645
- if syntax_of_do == :method_calling
646
- spaces_of_nest.push(spaces_at_line_head)
647
- end
648
- when 'def', 'case', 'for', 'begin', 'class', 'module'
649
- spaces_of_nest.push(spaces_at_line_head)
650
- when 'rescue'
651
- unless t.state.allbits?(Ripper::EXPR_LABEL)
652
- corresponding_token_depth = spaces_of_nest.last
653
- end
654
- when 'if', 'unless', 'while', 'until'
655
- # postfix if/unless/while/until must be Ripper::EXPR_LABEL
656
- unless t.state.allbits?(Ripper::EXPR_LABEL)
657
- spaces_of_nest.push(spaces_at_line_head)
658
- end
659
- when 'else', 'elsif', 'ensure', 'when'
660
- corresponding_token_depth = spaces_of_nest.last
661
- when 'in'
662
- if in_keyword_case_scope?
663
- corresponding_token_depth = spaces_of_nest.last
664
- end
665
- when 'end'
666
- if is_first_printable_of_line
667
- corresponding_token_depth = spaces_of_nest.pop
668
- else
669
- spaces_of_nest.pop
670
- corresponding_token_depth = nil
671
- end
465
+ # Accept any number of indent inside other heredoc
466
+ preserve_indent
672
467
  end
468
+ else
469
+ # Heredoc close
470
+ prev_line_indent_level = calc_indent_level(prev_opens)
471
+ tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0
673
472
  end
674
- is_first_spaces_of_line = false
675
- is_first_printable_of_line = false
473
+ else
474
+ base_indent + indent
676
475
  end
677
- corresponding_token_depth
678
476
  end
679
477
 
680
- def check_string_literal(tokens)
681
- i = 0
682
- start_token = []
683
- end_type = []
684
- pending_heredocs = []
685
- while i < tokens.size
686
- t = tokens[i]
687
- case t.event
688
- when *end_type.last
689
- start_token.pop
690
- end_type.pop
691
- when :on_tstring_beg
692
- start_token << t
693
- end_type << [:on_tstring_end, :on_label_end]
694
- when :on_regexp_beg
695
- start_token << t
696
- end_type << :on_regexp_end
697
- when :on_symbeg
698
- acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick}
699
- if (i + 1) < tokens.size
700
- if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st }
701
- start_token << t
702
- end_type << :on_tstring_end
703
- else
704
- i += 1
705
- end
706
- end
707
- when :on_backtick
708
- if t.state.allbits?(Ripper::EXPR_BEG)
709
- start_token << t
710
- end_type << :on_tstring_end
711
- end
712
- when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
713
- start_token << t
714
- end_type << :on_tstring_end
715
- when :on_heredoc_beg
716
- pending_heredocs << t
717
- end
478
+ LTYPE_TOKENS = %i[
479
+ on_heredoc_beg on_tstring_beg
480
+ on_regexp_beg on_symbeg on_backtick
481
+ on_symbols_beg on_qsymbols_beg
482
+ on_words_beg on_qwords_beg
483
+ ]
718
484
 
719
- if pending_heredocs.any? && t.tok.include?("\n")
720
- pending_heredocs.reverse_each do |t|
721
- start_token << t
722
- end_type << :on_heredoc_end
723
- end
724
- pending_heredocs = []
725
- end
726
- i += 1
485
+ def ltype_from_open_tokens(opens)
486
+ start_token = opens.reverse_each.find do |tok|
487
+ LTYPE_TOKENS.include?(tok.event)
727
488
  end
728
- pending_heredocs.first || start_token.last
729
- end
730
-
731
- def process_literal_type(tokens)
732
- start_token = check_string_literal(tokens)
733
- return nil if start_token == ""
489
+ return nil unless start_token
734
490
 
735
491
  case start_token&.event
736
492
  when :on_tstring_beg
@@ -783,47 +539,16 @@ class RubyLex
783
539
  end
784
540
  end
785
541
 
786
- if first_token.nil?
787
- return false
788
- elsif first_token && first_token.state == Ripper::EXPR_DOT
789
- return false
790
- else
542
+ if first_token && first_token.state != Ripper::EXPR_DOT
791
543
  tokens_without_last_line = tokens[0..index]
792
- ltype = process_literal_type(tokens_without_last_line)
793
- indent = process_nesting_level(tokens_without_last_line)
794
- continue = process_continue(tokens_without_last_line)
795
- code_block_open = check_code_block(tokens_without_last_line.map(&:tok).join(''), tokens_without_last_line)
796
- if ltype or indent > 0 or continue or code_block_open
797
- return false
798
- else
799
- return last_line_tokens.map(&:tok).join('')
544
+ code_without_last_line = tokens_without_last_line.map(&:tok).join
545
+ opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line)
546
+ if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
547
+ return last_line_tokens.map(&:tok).join
800
548
  end
801
549
  end
802
550
  end
803
551
  false
804
552
  end
805
-
806
- private
807
-
808
- def heredoc_scope?
809
- heredoc_tokens = @tokens.select { |t| [:on_heredoc_beg, :on_heredoc_end].include?(t.event) }
810
- heredoc_tokens[-1]&.event == :on_heredoc_beg
811
- end
812
-
813
- def in_keyword_case_scope?
814
- kw_tokens = @tokens.select { |t| t.event == :on_kw && ['case', 'for', 'end'].include?(t.tok) }
815
- counter = 0
816
- kw_tokens.reverse.each do |t|
817
- if t.tok == 'case'
818
- return true if counter.zero?
819
- counter += 1
820
- elsif t.tok == 'for'
821
- counter += 1
822
- elsif t.tok == 'end'
823
- counter -= 1
824
- end
825
- end
826
- false
827
- end
828
553
  end
829
554
  # :startdoc: