irb 1.0.0 → 1.1.0.pre.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -58,8 +58,6 @@ module IRB
58
58
  end
59
59
 
60
60
  module HistorySavingAbility # :nodoc:
61
- include Readline
62
-
63
61
  def HistorySavingAbility.extended(obj)
64
62
  IRB.conf[:AT_EXIT].push proc{obj.save_history}
65
63
  obj.load_history
@@ -67,18 +65,22 @@ module IRB
67
65
  end
68
66
 
69
67
  def load_history
68
+ return unless self.class.const_defined?(:HISTORY)
69
+ history = self.class::HISTORY
70
70
  if history_file = IRB.conf[:HISTORY_FILE]
71
71
  history_file = File.expand_path(history_file)
72
72
  end
73
73
  history_file = IRB.rc_file("_history") unless history_file
74
74
  if File.exist?(history_file)
75
75
  open(history_file) do |f|
76
- f.each {|l| HISTORY << l.chomp}
76
+ f.each {|l| history << l.chomp}
77
77
  end
78
78
  end
79
79
  end
80
80
 
81
81
  def save_history
82
+ return unless self.class.const_defined?(:HISTORY)
83
+ history = self.class::HISTORY
82
84
  if num = IRB.conf[:SAVE_HISTORY] and (num = num.to_i) > 0
83
85
  if history_file = IRB.conf[:HISTORY_FILE]
84
86
  history_file = File.expand_path(history_file)
@@ -96,7 +98,7 @@ module IRB
96
98
  end
97
99
 
98
100
  open(history_file, 'w', 0600 ) do |f|
99
- hist = HISTORY.to_a
101
+ hist = history.to_a
100
102
  f.puts(hist[-num..-1] || hist)
101
103
  end
102
104
  end
data/lib/irb/init.rb CHANGED
@@ -44,6 +44,7 @@ module IRB # :nodoc:
44
44
  @CONF[:IRB_RC] = nil
45
45
 
46
46
  @CONF[:USE_READLINE] = false unless defined?(ReadlineInputMethod)
47
+ @CONF[:USE_COLORIZE] = true
47
48
  @CONF[:INSPECT_MODE] = true
48
49
  @CONF[:USE_TRACER] = false
49
50
  @CONF[:USE_LOADER] = false
@@ -112,8 +113,6 @@ module IRB # :nodoc:
112
113
  @CONF[:LC_MESSAGES] = Locale.new
113
114
 
114
115
  @CONF[:AT_EXIT] = []
115
-
116
- @CONF[:DEBUG_LEVEL] = 0
117
116
  end
118
117
 
119
118
  def IRB.init_error
@@ -165,6 +164,10 @@ module IRB # :nodoc:
165
164
  @CONF[:USE_READLINE] = true
166
165
  when "--noreadline"
167
166
  @CONF[:USE_READLINE] = false
167
+ when "--reidline"
168
+ @CONF[:USE_REIDLINE] = true
169
+ when "--noreidline"
170
+ @CONF[:USE_REIDLINE] = false
168
171
  when "--echo"
169
172
  @CONF[:ECHO] = true
170
173
  when "--noecho"
@@ -173,6 +176,10 @@ module IRB # :nodoc:
173
176
  @CONF[:VERBOSE] = true
174
177
  when "--noverbose"
175
178
  @CONF[:VERBOSE] = false
179
+ when "--colorize"
180
+ @CONF[:USE_COLORIZE] = true
181
+ when "--nocolorize"
182
+ @CONF[:USE_COLORIZE] = false
176
183
  when /^--prompt-mode(?:=(.+))?/, /^--prompt(?:=(.+))?/
177
184
  opt = $1 || argv.shift
178
185
  prompt_mode = opt.upcase.tr("-", "_").intern
@@ -191,8 +198,6 @@ module IRB # :nodoc:
191
198
  @CONF[:CONTEXT_MODE] = ($1 || argv.shift).to_i
192
199
  when "--single-irb"
193
200
  @CONF[:SINGLE_IRB] = true
194
- when /^--irb_debug(?:=(.+))?/
195
- @CONF[:DEBUG_LEVEL] = ($1 || argv.shift).to_i
196
201
  when "-v", "--version"
197
202
  print IRB.version, "\n"
198
203
  exit 0
@@ -11,6 +11,8 @@
11
11
  #
12
12
  require_relative 'src_encoding'
13
13
  require_relative 'magic-file'
14
+ require_relative 'completion'
15
+ require 'reline'
14
16
 
15
17
  module IRB
16
18
  STDIN_FILE_NAME = "(line)" # :nodoc:
@@ -140,6 +142,12 @@ module IRB
140
142
 
141
143
  @stdin = IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
142
144
  @stdout = IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
145
+
146
+ if Readline.respond_to?("basic_word_break_characters=")
147
+ Readline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
148
+ end
149
+ Readline.completion_append_character = nil
150
+ Readline.completion_proc = IRB::InputCompletor::CompletionProc
143
151
  end
144
152
 
145
153
  # Reads the next line from this input method.
@@ -186,7 +194,95 @@ module IRB
186
194
  def encoding
187
195
  @stdin.external_encoding
188
196
  end
197
+
198
+ if Readline.respond_to?("basic_word_break_characters=")
199
+ Readline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
200
+ end
201
+ Readline.completion_append_character = nil
202
+ Readline.completion_proc = IRB::InputCompletor::CompletionProc
189
203
  end
190
204
  rescue LoadError
191
205
  end
206
+
207
+ class ReidlineInputMethod < InputMethod
208
+ include Reline
209
+ # Creates a new input method object using Readline
210
+ def initialize
211
+ super
212
+
213
+ @line_no = 0
214
+ @line = []
215
+ @eof = false
216
+
217
+ @stdin = ::IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
218
+ @stdout = ::IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
219
+
220
+ if Reline.respond_to?("basic_word_break_characters=")
221
+ Reline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
222
+ end
223
+ Reline.completion_append_character = nil
224
+ Reline.completion_proc = IRB::InputCompletor::CompletionProc
225
+ Reline.output_modifier_proc =
226
+ if IRB.conf[:USE_COLORIZE]
227
+ proc do |output, complete:|
228
+ next unless IRB::Color.colorable?
229
+ IRB::Color.colorize_code(output, complete: complete)
230
+ end
231
+ else
232
+ proc do |output|
233
+ Reline::Unicode.escape_for_print(output)
234
+ end
235
+ end
236
+ Reline.dig_perfect_match_proc = IRB::InputCompletor::PerfectMatchedProc
237
+ end
238
+
239
+ def check_termination(&block)
240
+ @check_termination_proc = block
241
+ end
242
+
243
+ # Reads the next line from this input method.
244
+ #
245
+ # See IO#gets for more information.
246
+ def gets
247
+ Reline.input = @stdin
248
+ Reline.output = @stdout
249
+ if l = readmultiline(@prompt, false, &@check_termination_proc)
250
+ HISTORY.push(l) if !l.empty?
251
+ @line[@line_no += 1] = l + "\n"
252
+ else
253
+ @eof = true
254
+ l
255
+ end
256
+ end
257
+
258
+ # Whether the end of this input method has been reached, returns +true+
259
+ # if there is no more data to read.
260
+ #
261
+ # See IO#eof? for more information.
262
+ def eof?
263
+ super
264
+ end
265
+
266
+ # Whether this input method is still readable when there is no more data to
267
+ # read.
268
+ #
269
+ # See IO#eof for more information.
270
+ def readable_after_eof?
271
+ true
272
+ end
273
+
274
+ # Returns the current line number for #io.
275
+ #
276
+ # #line counts the number of times #gets is called.
277
+ #
278
+ # See IO#lineno for more information.
279
+ def line(line_no)
280
+ @line[line_no]
281
+ end
282
+
283
+ # The external encoding for standard input.
284
+ def encoding
285
+ @stdin.external_encoding
286
+ end
287
+ end
192
288
  end
data/lib/irb/inspector.rb CHANGED
@@ -106,12 +106,22 @@ module IRB # :nodoc:
106
106
  Inspector.def_inspector([false, :to_s, :raw]){|v| v.to_s}
107
107
  Inspector.def_inspector([true, :p, :inspect]){|v|
108
108
  begin
109
- v.inspect
109
+ result = v.inspect
110
+ if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
111
+ result = Color.colorize_code(result)
112
+ end
113
+ result
110
114
  rescue NoMethodError
111
115
  puts "(Object doesn't support #inspect)"
112
116
  end
113
117
  }
114
- Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v| v.pretty_inspect.chomp}
118
+ Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v|
119
+ result = v.pretty_inspect.chomp
120
+ if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
121
+ result = Color.colorize_code(result)
122
+ end
123
+ result
124
+ }
115
125
  Inspector.def_inspector([:yaml, :YAML], proc{require "yaml"}){|v|
116
126
  begin
117
127
  YAML.dump(v)
@@ -26,6 +26,8 @@ Usage: irb.rb [options] [programfile] [arguments]
26
26
  --noinspect Don't use inspect for output
27
27
  --readline Use Readline extension module
28
28
  --noreadline Don't use Readline extension module
29
+ --colorize Use colorization
30
+ --nocolorize Don't use colorization
29
31
  --prompt prompt-mode/--prompt-mode prompt-mode
30
32
  Switch prompt mode. Pre-defined prompt modes are
31
33
  `default', `simple', `xmp' and `inf-ruby'
@@ -39,7 +41,6 @@ Usage: irb.rb [options] [programfile] [arguments]
39
41
  --back-trace-limit n
40
42
  Display backtrace top n and tail n. The default
41
43
  value is 16.
42
- --irb_debug n Set internal debug level to n (not for popular use)
43
44
  --verbose Show details
44
45
  --noverbose Don't show details
45
46
  -v, --version Print the version of irb
@@ -25,6 +25,8 @@ Usage: irb.rb [options] [programfile] [arguments]
25
25
  --noinspect 結果出力にinspectを用いない.
26
26
  --readline readlineライブラリを利用する.
27
27
  --noreadline readlineライブラリを利用しない.
28
+ --colorize 色付けを利用する.
29
+ --nocolorize 色付けを利用しない.
28
30
  --prompt prompt-mode/--prompt-mode prompt-mode
29
31
  プロンプトモードを切替えます. 現在定義されているプ
30
32
  ロンプトモードは, default, simple, xmp, inf-rubyが
@@ -41,8 +43,6 @@ Usage: irb.rb [options] [programfile] [arguments]
41
43
  バックトレース表示をバックトレースの頭から n, 後ろ
42
44
  からnだけ行なう. デフォルトは16
43
45
 
44
- --irb_debug n irbのデバッグレベルをnに設定する(非推奨).
45
-
46
46
  --verbose 詳細なメッセージを出力する.
47
47
  --noverbose 詳細なメッセージを出力しない(デフォルト).
48
48
  -v, --version irbのバージョンを表示する.
data/lib/irb/ruby-lex.rb CHANGED
@@ -11,73 +11,40 @@
11
11
  #
12
12
 
13
13
  require "e2mmap"
14
- require_relative "slex"
15
- require_relative "ruby-token"
14
+ require "ripper"
16
15
 
17
16
  # :stopdoc:
18
17
  class RubyLex
19
18
 
20
19
  extend Exception2MessageMapper
21
- def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
22
- def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
23
- def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
24
- def_exception(:TkReading2TokenDuplicateError,
25
- "key duplicate(token_n='%s', key='%s')")
26
- def_exception(:SyntaxError, "%s")
27
-
28
20
  def_exception(:TerminateLineInput, "Terminate Line Input")
29
21
 
30
- include RubyToken
31
-
32
- class << self
33
- attr_accessor :debug_level
34
- def debug?
35
- @debug_level > 0
36
- end
37
- end
38
- @debug_level = 0
39
-
40
22
  def initialize
41
- lex_init
42
- set_input(STDIN)
43
-
44
- @seek = 0
45
23
  @exp_line_no = @line_no = 1
46
- @base_char_no = 0
47
- @char_no = 0
48
- @rests = []
49
- @readed = []
50
- @here_readed = []
51
-
52
24
  @indent = 0
53
- @indent_stack = []
54
- @lex_state = EXPR_BEG
55
- @space_seen = false
56
- @here_header = false
57
- @post_symbeg = false
58
-
59
25
  @continue = false
60
26
  @line = ""
61
-
62
- @skip_space = false
63
- @readed_auto_clean_up = false
64
- @exception_on_syntax_error = true
65
-
66
27
  @prompt = nil
67
28
  end
68
29
 
69
- attr_accessor :skip_space
70
- attr_accessor :readed_auto_clean_up
71
- attr_accessor :exception_on_syntax_error
72
-
73
- attr_reader :seek
74
- attr_reader :char_no
75
- attr_reader :line_no
76
- attr_reader :indent
77
-
78
30
  # io functions
79
31
  def set_input(io, p = nil, &block)
80
32
  @io = io
33
+ if @io.respond_to?(:check_termination)
34
+ @io.check_termination do |code|
35
+ code.gsub!(/\s*\z/, '').concat("\n")
36
+ @tokens = Ripper.lex(code)
37
+ continue = process_continue
38
+ code_block_open = check_code_block(code)
39
+ indent = process_nesting_level
40
+ ltype = process_literal_type
41
+ if code_block_open or ltype or continue or indent > 0
42
+ false
43
+ else
44
+ true
45
+ end
46
+ end
47
+ end
81
48
  if p.respond_to?(:call)
82
49
  @input = p
83
50
  elsif block_given?
@@ -87,112 +54,6 @@ class RubyLex
87
54
  end
88
55
  end
89
56
 
90
- def get_readed
91
- if idx = @readed.rindex("\n")
92
- @base_char_no = @readed.size - (idx + 1)
93
- else
94
- @base_char_no += @readed.size
95
- end
96
-
97
- readed = @readed.join("")
98
- @readed = []
99
- readed
100
- end
101
-
102
- def getc
103
- while @rests.empty?
104
- @rests.push nil unless buf_input
105
- end
106
- c = @rests.shift
107
- if @here_header
108
- @here_readed.push c
109
- else
110
- @readed.push c
111
- end
112
- @seek += 1
113
- if c == "\n"
114
- @line_no += 1
115
- @char_no = 0
116
- else
117
- @char_no += 1
118
- end
119
- c
120
- end
121
-
122
- def gets
123
- l = ""
124
- while c = getc
125
- l.concat(c)
126
- break if c == "\n"
127
- end
128
- return nil if l == "" and c.nil?
129
- l
130
- end
131
-
132
- def eof?
133
- @io.eof?
134
- end
135
-
136
- def getc_of_rests
137
- if @rests.empty?
138
- nil
139
- else
140
- getc
141
- end
142
- end
143
-
144
- def ungetc(c = nil)
145
- if @here_readed.empty?
146
- c2 = @readed.pop
147
- else
148
- c2 = @here_readed.pop
149
- end
150
- c = c2 unless c
151
- @rests.unshift c #c =
152
- @seek -= 1
153
- if c == "\n"
154
- @line_no -= 1
155
- if idx = @readed.rindex("\n")
156
- @char_no = idx + 1
157
- else
158
- @char_no = @base_char_no + @readed.size
159
- end
160
- else
161
- @char_no -= 1
162
- end
163
- end
164
-
165
- def peek_equal?(str)
166
- chrs = str.split(//)
167
- until @rests.size >= chrs.size
168
- return false unless buf_input
169
- end
170
- @rests[0, chrs.size] == chrs
171
- end
172
-
173
- def peek_match?(regexp)
174
- while @rests.empty?
175
- return false unless buf_input
176
- end
177
- regexp =~ @rests.join("")
178
- end
179
-
180
- def peek(i = 0)
181
- while @rests.size <= i
182
- return nil unless buf_input
183
- end
184
- @rests[i]
185
- end
186
-
187
- def buf_input
188
- prompt
189
- line = @input.call
190
- return nil unless line
191
- @rests.concat line.chars.to_a
192
- true
193
- end
194
- private :buf_input
195
-
196
57
  def set_prompt(p = nil, &block)
197
58
  p = block if block_given?
198
59
  if p.respond_to?(:call)
@@ -210,20 +71,11 @@ class RubyLex
210
71
 
211
72
  def initialize_input
212
73
  @ltype = nil
213
- @quoted = nil
214
74
  @indent = 0
215
- @indent_stack = []
216
- @lex_state = EXPR_BEG
217
- @space_seen = false
218
- @here_header = false
219
-
220
75
  @continue = false
221
- @post_symbeg = false
222
-
223
- prompt
224
-
225
76
  @line = ""
226
77
  @exp_line_no = @line_no
78
+ @code_block_open = false
227
79
  end
228
80
 
229
81
  def each_top_level_statement
@@ -231,13 +83,14 @@ class RubyLex
231
83
  catch(:TERM_INPUT) do
232
84
  loop do
233
85
  begin
234
- @continue = false
235
86
  prompt
236
87
  unless l = lex
237
88
  throw :TERM_INPUT if @line == ''
238
89
  else
90
+ @line_no += 1
91
+ next if l == "\n"
239
92
  @line.concat l
240
- if @ltype or @continue or @indent > 0
93
+ if @code_block_open or @ltype or @continue or @indent > 0
241
94
  next
242
95
  end
243
96
  end
@@ -245,935 +98,235 @@ class RubyLex
245
98
  @line.force_encoding(@io.encoding)
246
99
  yield @line, @exp_line_no
247
100
  end
248
- break unless l
101
+ break if @io.eof?
249
102
  @line = ''
250
103
  @exp_line_no = @line_no
251
104
 
252
105
  @indent = 0
253
- @indent_stack = []
254
- prompt
255
106
  rescue TerminateLineInput
256
107
  initialize_input
257
108
  prompt
258
- get_readed
259
109
  end
260
110
  end
261
111
  end
262
112
  end
263
113
 
264
114
  def lex
265
- continue = @continue
266
- while tk = token
267
- case tk
268
- when TkNL, TkEND_OF_SCRIPT
269
- @continue = continue unless continue.nil?
270
- break unless @continue
271
- when TkSPACE, TkCOMMENT
272
- when TkSEMICOLON, TkBEGIN, TkELSE
273
- @continue = continue = false
274
- else
275
- continue = nil
276
- end
277
- end
278
- line = get_readed
279
- if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
280
- nil
281
- else
282
- line
283
- end
284
- end
285
-
286
- def token
287
- @prev_seek = @seek
288
- @prev_line_no = @line_no
289
- @prev_char_no = @char_no
290
- begin
291
- begin
292
- tk = @OP.match(self)
293
- @space_seen = tk.kind_of?(TkSPACE)
294
- @lex_state = EXPR_END if @post_symbeg && tk.kind_of?(TkOp)
295
- @post_symbeg = tk.kind_of?(TkSYMBEG)
296
- rescue SyntaxError
297
- raise if @exception_on_syntax_error
298
- tk = TkError.new(@seek, @line_no, @char_no)
299
- end
300
- end while @skip_space and tk.kind_of?(TkSPACE)
301
- if @readed_auto_clean_up
302
- get_readed
303
- end
304
- tk
305
- end
306
-
307
- ENINDENT_CLAUSE = [
308
- "case", "class", "def", "do", "for", "if",
309
- "module", "unless", "until", "while", "begin"
310
- ]
311
- DEINDENT_CLAUSE = ["end"
312
- ]
313
-
314
- PERCENT_LTYPE = {
315
- "q" => "\'",
316
- "Q" => "\"",
317
- "x" => "\`",
318
- "r" => "/",
319
- "w" => "]",
320
- "W" => "]",
321
- "i" => "]",
322
- "I" => "]",
323
- "s" => ":"
324
- }
325
-
326
- PERCENT_PAREN = {
327
- "{" => "}",
328
- "[" => "]",
329
- "<" => ">",
330
- "(" => ")"
331
- }
332
-
333
- Ltype2Token = {
334
- "\'" => TkSTRING,
335
- "\"" => TkSTRING,
336
- "\`" => TkXSTRING,
337
- "/" => TkREGEXP,
338
- "]" => TkDSTRING,
339
- ":" => TkSYMBOL
340
- }
341
- DLtype2Token = {
342
- "\"" => TkDSTRING,
343
- "\`" => TkDXSTRING,
344
- "/" => TkDREGEXP,
345
- }
346
-
347
- def lex_init()
348
- @OP = IRB::SLex.new
349
- @OP.def_rules("\0", "\004", "\032") do |op, io|
350
- Token(TkEND_OF_SCRIPT)
351
- end
352
-
353
- @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
354
- @space_seen = true
355
- while getc =~ /[ \t\f\r\13]/; end
356
- ungetc
357
- Token(TkSPACE)
358
- end
359
-
360
- @OP.def_rule("#") do |op, io|
361
- identify_comment
362
- end
363
-
364
- @OP.def_rule("=begin",
365
- proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
366
- |op, io|
367
- @ltype = "="
368
- until getc == "\n"; end
369
- until peek_equal?("=end") && peek(4) =~ /\s/
370
- until getc == "\n"; end
371
- end
372
- gets
373
- @ltype = nil
374
- Token(TkRD_COMMENT)
375
- end
376
-
377
- @OP.def_rule("\n") do |op, io|
378
- print "\\n\n" if RubyLex.debug?
379
- case @lex_state
380
- when EXPR_BEG, EXPR_FNAME, EXPR_DOT
381
- @continue = true
382
- else
383
- @continue = false
384
- @lex_state = EXPR_BEG
385
- until (@indent_stack.empty? ||
386
- [TkLPAREN, TkLBRACK, TkLBRACE,
387
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
388
- @indent_stack.pop
389
- end
390
- end
391
- @here_header = false
392
- @here_readed = []
393
- Token(TkNL)
394
- end
395
-
396
- @OP.def_rules("*", "**",
397
- "=", "==", "===",
398
- "=~", "<=>",
399
- "<", "<=",
400
- ">", ">=", ">>",
401
- "!", "!=", "!~") do
402
- |op, io|
403
- case @lex_state
404
- when EXPR_FNAME, EXPR_DOT
405
- @lex_state = EXPR_ARG
406
- else
407
- @lex_state = EXPR_BEG
408
- end
409
- Token(op)
410
- end
411
-
412
- @OP.def_rules("<<") do
413
- |op, io|
414
- tk = nil
415
- if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
416
- (@lex_state != EXPR_ARG || @space_seen)
417
- c = peek(0)
418
- if /[-~"'`\w]/ =~ c
419
- tk = identify_here_document
420
- end
421
- end
422
- unless tk
423
- tk = Token(op)
424
- case @lex_state
425
- when EXPR_FNAME, EXPR_DOT
426
- @lex_state = EXPR_ARG
427
- else
428
- @lex_state = EXPR_BEG
429
- end
430
- end
431
- tk
432
- end
433
-
434
- @OP.def_rules("'", '"') do
435
- |op, io|
436
- identify_string(op)
437
- end
438
-
439
- @OP.def_rules("`") do
440
- |op, io|
441
- if @lex_state == EXPR_FNAME
442
- @lex_state = EXPR_END
443
- Token(op)
444
- else
445
- identify_string(op)
446
- end
447
- end
448
-
449
- @OP.def_rules('?') do
450
- |op, io|
451
- if @lex_state == EXPR_END
452
- @lex_state = EXPR_BEG
453
- Token(TkQUESTION)
454
- else
455
- ch = getc
456
- if @lex_state == EXPR_ARG && ch =~ /\s/
457
- ungetc
458
- @lex_state = EXPR_BEG;
459
- Token(TkQUESTION)
460
- else
461
- if (ch == '\\')
462
- read_escape
463
- end
464
- @lex_state = EXPR_END
465
- Token(TkINTEGER)
466
- end
467
- end
468
- end
469
-
470
- @OP.def_rules("&", "&&", "|", "||") do
471
- |op, io|
472
- @lex_state = EXPR_BEG
473
- Token(op)
474
- end
475
-
476
- @OP.def_rules("+=", "-=", "*=", "**=",
477
- "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
478
- |op, io|
479
- @lex_state = EXPR_BEG
480
- op =~ /^(.*)=$/
481
- Token(TkOPASGN, $1)
482
- end
483
-
484
- @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
485
- |op, io|
486
- @lex_state = EXPR_ARG
487
- Token(op)
488
- end
489
-
490
- @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
491
- |op, io|
492
- @lex_state = EXPR_ARG
493
- Token(op)
494
- end
495
-
496
- @OP.def_rules("+", "-") do
497
- |op, io|
498
- catch(:RET) do
499
- if @lex_state == EXPR_ARG
500
- if @space_seen and peek(0) =~ /[0-9]/
501
- throw :RET, identify_number
502
- else
503
- @lex_state = EXPR_BEG
504
- end
505
- elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
506
- throw :RET, identify_number
507
- else
508
- @lex_state = EXPR_BEG
509
- end
510
- Token(op)
511
- end
512
- end
513
-
514
- @OP.def_rule(".") do
515
- |op, io|
516
- @lex_state = EXPR_BEG
517
- if peek(0) =~ /[0-9]/
518
- ungetc
519
- identify_number
520
- else
521
- # for "obj.if" etc.
522
- @lex_state = EXPR_DOT
523
- Token(TkDOT)
524
- end
525
- end
526
-
527
- @OP.def_rules("..", "...") do
528
- |op, io|
529
- @lex_state = EXPR_BEG
530
- Token(op)
531
- end
532
-
533
- lex_int2
534
- end
535
-
536
- def lex_int2
537
- @OP.def_rules("]", "}", ")") do
538
- |op, io|
539
- @lex_state = EXPR_END
540
- @indent -= 1
541
- @indent_stack.pop
542
- Token(op)
543
- end
544
-
545
- @OP.def_rule(":") do
546
- |op, io|
547
- if @lex_state == EXPR_END || peek(0) =~ /\s/
548
- @lex_state = EXPR_BEG
549
- Token(TkCOLON)
550
- else
551
- @lex_state = EXPR_FNAME
552
- Token(TkSYMBEG)
553
- end
554
- end
555
-
556
- @OP.def_rule("::") do
557
- |op, io|
558
- if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
559
- @lex_state = EXPR_BEG
560
- Token(TkCOLON3)
561
- else
562
- @lex_state = EXPR_DOT
563
- Token(TkCOLON2)
564
- end
565
- end
566
-
567
- @OP.def_rule("/") do
568
- |op, io|
569
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
570
- identify_string(op)
571
- elsif peek(0) == '='
572
- getc
573
- @lex_state = EXPR_BEG
574
- Token(TkOPASGN, "/") #/)
575
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
576
- identify_string(op)
577
- else
578
- @lex_state = EXPR_BEG
579
- Token("/") #/)
580
- end
581
- end
582
-
583
- @OP.def_rules("^") do
584
- |op, io|
585
- @lex_state = EXPR_BEG
586
- Token("^")
587
- end
588
-
589
- @OP.def_rules(",") do
590
- |op, io|
591
- @lex_state = EXPR_BEG
592
- Token(op)
593
- end
594
-
595
- @OP.def_rules(";") do
596
- |op, io|
597
- @lex_state = EXPR_BEG
598
- until (@indent_stack.empty? ||
599
- [TkLPAREN, TkLBRACK, TkLBRACE,
600
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
601
- @indent_stack.pop
602
- end
603
- Token(op)
604
- end
605
-
606
- @OP.def_rule("~") do
607
- |op, io|
608
- @lex_state = EXPR_BEG
609
- Token("~")
610
- end
611
-
612
- @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
613
- |op, io|
614
- @lex_state = EXPR_BEG
615
- Token("~")
616
- end
617
-
618
- @OP.def_rule("(") do
619
- |op, io|
620
- @indent += 1
621
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
622
- @lex_state = EXPR_BEG
623
- tk_c = TkfLPAREN
624
- else
625
- @lex_state = EXPR_BEG
626
- tk_c = TkLPAREN
627
- end
628
- @indent_stack.push tk_c
629
- Token(tk_c)
630
- end
631
-
632
- @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
633
- |op, io|
634
- @lex_state = EXPR_ARG
635
- Token("[]")
636
- end
637
-
638
- @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
639
- |op, io|
640
- @lex_state = EXPR_ARG
641
- Token("[]=")
642
- end
643
-
644
- @OP.def_rule("[") do
645
- |op, io|
646
- @indent += 1
647
- if @lex_state == EXPR_FNAME
648
- tk_c = TkfLBRACK
649
- else
650
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
651
- tk_c = TkLBRACK
652
- elsif @lex_state == EXPR_ARG && @space_seen
653
- tk_c = TkLBRACK
654
- else
655
- tk_c = TkfLBRACK
656
- end
657
- @lex_state = EXPR_BEG
658
- end
659
- @indent_stack.push tk_c
660
- Token(tk_c)
661
- end
662
-
663
- @OP.def_rule("{") do
664
- |op, io|
665
- @indent += 1
666
- if @lex_state != EXPR_END && @lex_state != EXPR_ARG
667
- tk_c = TkLBRACE
668
- else
669
- tk_c = TkfLBRACE
670
- end
671
- @lex_state = EXPR_BEG
672
- @indent_stack.push tk_c
673
- Token(tk_c)
674
- end
675
-
676
- @OP.def_rule('\\') do
677
- |op, io|
678
- if getc == "\n"
679
- @space_seen = true
680
- @continue = true
681
- Token(TkSPACE)
682
- else
683
- read_escape
684
- Token("\\")
685
- end
686
- end
687
-
688
- @OP.def_rule('%') do
689
- |op, io|
690
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
691
- identify_quotation
692
- elsif peek(0) == '='
693
- getc
694
- Token(TkOPASGN, :%)
695
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
696
- identify_quotation
697
- else
698
- @lex_state = EXPR_BEG
699
- Token("%") #))
700
- end
701
- end
702
-
703
- @OP.def_rule('$') do
704
- |op, io|
705
- identify_gvar
706
- end
707
-
708
- @OP.def_rule('@') do
709
- |op, io|
710
- if peek(0) =~ /[\w@]/
711
- ungetc
712
- identify_identifier
713
- else
714
- Token("@")
715
- end
716
- end
717
-
718
- @OP.def_rule("") do
719
- |op, io|
720
- printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
721
- if peek(0) =~ /[0-9]/
722
- t = identify_number
723
- elsif peek(0) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
724
- t = identify_identifier
725
- end
726
- printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
727
- t
728
- end
729
-
730
- p @OP if RubyLex.debug?
731
- end
732
-
733
- def identify_gvar
734
- @lex_state = EXPR_END
735
-
736
- case ch = getc
737
- when /[~_*$?!@\/\\;,=:<>".]/ #"
738
- Token(TkGVAR, "$" + ch)
739
- when "-"
740
- Token(TkGVAR, "$-" + getc)
741
- when "&", "`", "'", "+"
742
- Token(TkBACK_REF, "$"+ch)
743
- when /[1-9]/
744
- while getc =~ /[0-9]/; end
745
- ungetc
746
- Token(TkNTH_REF)
747
- when /\w/
748
- ungetc
749
- ungetc
750
- identify_identifier
751
- else
752
- ungetc
753
- Token("$")
754
- end
115
+ line = @input.call
116
+ if @io.respond_to?(:check_termination)
117
+ return line # multiline
118
+ end
119
+ code = @line + (line.nil? ? '' : line)
120
+ code.gsub!(/\s*\z/, '').concat("\n")
121
+ @tokens = Ripper.lex(code)
122
+ @continue = process_continue
123
+ @code_block_open = check_code_block(code)
124
+ @indent = process_nesting_level
125
+ @ltype = process_literal_type
126
+ line
755
127
  end
756
128
 
757
- def identify_identifier
758
- token = ""
759
- if peek(0) =~ /[$@]/
760
- token.concat(c = getc)
761
- if c == "@" and peek(0) == "@"
762
- token.concat getc
763
- end
764
- end
765
-
766
- while (ch = getc) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
767
- print ":", ch, ":" if RubyLex.debug?
768
- token.concat ch
769
- end
770
- ungetc
771
-
772
- if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
773
- token.concat getc
774
- end
775
-
776
- # almost fix token
777
-
778
- case token
779
- when /^\$/
780
- return Token(TkGVAR, token)
781
- when /^\@\@/
782
- @lex_state = EXPR_END
783
- # p Token(TkCVAR, token)
784
- return Token(TkCVAR, token)
785
- when /^\@/
786
- @lex_state = EXPR_END
787
- return Token(TkIVAR, token)
788
- end
789
-
790
- if @lex_state != EXPR_DOT
791
- print token, "\n" if RubyLex.debug?
792
-
793
- token_c, *trans = TkReading2Token[token]
794
- if token_c
795
- # reserved word?
796
-
797
- if (@lex_state != EXPR_BEG &&
798
- @lex_state != EXPR_FNAME &&
799
- trans[1])
800
- # modifiers
801
- token_c = TkSymbol2Token[trans[1]]
802
- @lex_state = trans[0]
803
- else
804
- if @lex_state != EXPR_FNAME and peek(0) != ':'
805
- if ENINDENT_CLAUSE.include?(token)
806
- # check for ``class = val'' etc.
807
- valid = true
808
- case token
809
- when "class"
810
- valid = false unless peek_match?(/^\s*(<<|\w|::)/)
811
- when "def"
812
- valid = false if peek_match?(/^\s*(([+\-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
813
- when "do"
814
- valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&)/)
815
- when *ENINDENT_CLAUSE
816
- valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&|\|)/)
817
- else
818
- # no nothing
819
- end
820
- if valid
821
- if token == "do"
822
- if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
823
- @indent += 1
824
- @indent_stack.push token_c
825
- end
826
- else
827
- @indent += 1
828
- @indent_stack.push token_c
829
- end
830
- end
831
-
832
- elsif DEINDENT_CLAUSE.include?(token)
833
- @indent -= 1
834
- @indent_stack.pop
835
- end
836
- @lex_state = trans[0]
837
- else
838
- @lex_state = EXPR_END
839
- end
840
- end
841
- return Token(token_c, token)
842
- end
843
- end
844
-
845
- if @lex_state == EXPR_FNAME
846
- @lex_state = EXPR_END
847
- if peek(0) == '='
848
- token.concat getc
849
- end
850
- elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
851
- @lex_state = EXPR_ARG
852
- else
853
- @lex_state = EXPR_END
854
- end
855
-
856
- if token[0, 1] =~ /[A-Z]/
857
- return Token(TkCONSTANT, token)
858
- elsif token[token.size - 1, 1] =~ /[!?]/
859
- return Token(TkFID, token)
860
- else
861
- return Token(TkIDENTIFIER, token)
862
- end
129
+ def process_continue
130
+ continued_bits = Ripper::EXPR_BEG | Ripper::EXPR_FNAME
131
+ # last token is always newline
132
+ if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end
133
+ # end of regexp literal
134
+ return false
135
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon
136
+ return false
137
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and (@tokens[-2][2] == 'begin' or @tokens[-2][2] == 'else')
138
+ return false
139
+ elsif @tokens.size >= 3 and @tokens[-3][1] == :on_symbeg and @tokens[-2][1] == :on_ivar
140
+ # This is for :@a or :@1 because :@1 ends with EXPR_FNAME
141
+ return false
142
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_ivar and @tokens[-2][2] =~ /\A@\d+\z/
143
+ # This is for @1
144
+ return false
145
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_cvar and @tokens[-1][1] == :on_int
146
+ # This is for @@1 or :@@1 and ends with on_int because it's syntax error
147
+ return false
148
+ elsif !@tokens.empty? and @tokens.last[2] == "\\\n"
149
+ return true
150
+ elsif @tokens.size >= 1 and @tokens[-1][1] == :on_heredoc_end # "EOH\n"
151
+ return false
152
+ elsif @tokens.size >= 2 and @tokens[-2][3].anybits?(continued_bits)
153
+ # end of literal except for regexp
154
+ return true
155
+ end
156
+ false
863
157
  end
864
158
 
865
- def identify_here_document
866
- ch = getc
867
- if ch == "-" || ch == "~"
868
- ch = getc
869
- indent = true
870
- end
871
- if /['"`]/ =~ ch
872
- lt = ch
873
- quoted = ""
874
- while (c = getc) && c != lt
875
- quoted.concat c
876
- end
877
- else
878
- lt = '"'
879
- quoted = ch.dup
880
- while (c = getc) && c =~ /\w/
881
- quoted.concat c
882
- end
883
- ungetc
884
- end
885
-
886
- ltback, @ltype = @ltype, lt
887
- reserve = []
888
- while ch = getc
889
- reserve.push ch
890
- if ch == "\\"
891
- reserve.push ch = getc
892
- elsif ch == "\n"
893
- break
894
- end
895
- end
896
-
897
- @here_header = false
898
-
899
- line = ""
900
- while ch = getc
901
- if ch == "\n"
902
- if line == quoted
903
- break
904
- end
905
- line = ""
906
- else
907
- line.concat ch unless indent && line == "" && /\s/ =~ ch
908
- if @ltype != "'" && ch == "#" && peek(0) == "{"
909
- identify_string_dvar
910
- end
159
+ def check_code_block(code)
160
+ return true if @tokens.empty?
161
+ if @tokens.last[1] == :on_heredoc_beg
162
+ return true
163
+ end
164
+
165
+ begin # check if parser error are available
166
+ verbose, $VERBOSE = $VERBOSE, nil
167
+ RubyVM::InstructionSequence.compile(code)
168
+ rescue SyntaxError => e
169
+ case e.message
170
+ when /unterminated (?:string|regexp) meets end of file/
171
+ # "unterminated regexp meets end of file"
172
+ #
173
+ # example:
174
+ # /
175
+ #
176
+ # "unterminated string meets end of file"
177
+ #
178
+ # example:
179
+ # '
180
+ return true
181
+ when /syntax error, unexpected end-of-input/
182
+ # "syntax error, unexpected end-of-input, expecting keyword_end"
183
+ #
184
+ # example:
185
+ # if ture
186
+ # hoge
187
+ # if false
188
+ # fuga
189
+ # end
190
+ return true
191
+ when /syntax error, unexpected keyword_end/
192
+ # "syntax error, unexpected keyword_end"
193
+ #
194
+ # example:
195
+ # if (
196
+ # end
197
+ #
198
+ # example:
199
+ # end
200
+ return false
201
+ when /syntax error, unexpected '\.'/
202
+ # "syntax error, unexpected '.'"
203
+ #
204
+ # example:
205
+ # .
206
+ return false
207
+ when /unexpected tREGEXP_BEG/
208
+ # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
209
+ #
210
+ # example:
211
+ # method / f /
212
+ return false
213
+ when /numbered parameter outside block/
214
+ # "numbered parameter outside block"
215
+ #
216
+ # example:
217
+ # :@1
218
+ return false
911
219
  end
220
+ ensure
221
+ $VERBOSE = verbose
912
222
  end
913
223
 
914
- @here_header = true
915
- @here_readed.concat reserve
916
- while ch = reserve.pop
917
- ungetc ch
918
- end
919
-
920
- @ltype = ltback
921
- @lex_state = EXPR_END
922
- Token(Ltype2Token[lt])
923
- end
924
-
925
- def identify_quotation
926
- ch = getc
927
- if lt = PERCENT_LTYPE[ch]
928
- ch = getc
929
- elsif ch =~ /\W/
930
- lt = "\""
931
- else
932
- RubyLex.fail SyntaxError, "unknown type of %string"
933
- end
934
- @quoted = ch unless @quoted = PERCENT_PAREN[ch]
935
- identify_string(lt, @quoted)
936
- end
937
-
938
- def identify_number
939
- @lex_state = EXPR_END
940
-
941
- if peek(0) == "0" && peek(1) !~ /[.eE]/
942
- getc
943
- case peek(0)
944
- when /[xX]/
945
- ch = getc
946
- match = /[0-9a-fA-F_]/
947
- when /[bB]/
948
- ch = getc
949
- match = /[01_]/
950
- when /[oO]/
951
- ch = getc
952
- match = /[0-7_]/
953
- when /[dD]/
954
- ch = getc
955
- match = /[0-9_]/
956
- when /[0-7]/
957
- match = /[0-7_]/
958
- when /[89]/
959
- RubyLex.fail SyntaxError, "Invalid octal digit"
960
- else
961
- return Token(TkINTEGER)
962
- end
963
-
964
- len0 = true
965
- non_digit = false
966
- while ch = getc
967
- if match =~ ch
968
- if ch == "_"
969
- if non_digit
970
- RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
971
- else
972
- non_digit = ch
973
- end
974
- else
975
- non_digit = false
976
- len0 = false
977
- end
978
- else
979
- ungetc
980
- if len0
981
- RubyLex.fail SyntaxError, "numeric literal without digits"
982
- end
983
- if non_digit
984
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
985
- end
986
- break
987
- end
988
- end
989
- return Token(TkINTEGER)
224
+ last_lex_state = @tokens.last[3]
225
+ if last_lex_state.allbits?(Ripper::EXPR_BEG)
226
+ return false
227
+ elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
228
+ return true
229
+ elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
230
+ return true
231
+ elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
232
+ return true
233
+ elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
234
+ return true
235
+ elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
236
+ return false
990
237
  end
991
238
 
992
- type = TkINTEGER
993
- allow_point = true
994
- allow_e = true
995
- non_digit = false
996
- while ch = getc
997
- case ch
998
- when /[0-9]/
999
- non_digit = false
1000
- when "_"
1001
- non_digit = ch
1002
- when allow_point && "."
1003
- if non_digit
1004
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1005
- end
1006
- type = TkFLOAT
1007
- if peek(0) !~ /[0-9]/
1008
- type = TkINTEGER
1009
- ungetc
1010
- break
1011
- end
1012
- allow_point = false
1013
- when allow_e && "e", allow_e && "E"
1014
- if non_digit
1015
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016
- end
1017
- type = TkFLOAT
1018
- if peek(0) =~ /[+-]/
1019
- getc
1020
- end
1021
- allow_e = false
1022
- allow_point = false
1023
- non_digit = ch
1024
- else
1025
- if non_digit
1026
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027
- end
1028
- ungetc
1029
- break
1030
- end
1031
- end
1032
- Token(type)
239
+ false
1033
240
  end
1034
241
 
1035
- def identify_string(ltype, quoted = ltype)
1036
- @ltype = ltype
1037
- @quoted = quoted
1038
- subtype = nil
1039
- begin
1040
- nest = 0
1041
- while ch = getc
1042
- if @quoted == ch and nest == 0
1043
- break
1044
- elsif @ltype != "'" && ch == "#" && peek(0) == "{"
1045
- identify_string_dvar
1046
- elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1047
- subtype = true
1048
- elsif ch == '\\' and @ltype == "'" #'
1049
- case ch = getc
1050
- when "\\", "\n", "'"
1051
- else
1052
- ungetc
1053
- end
1054
- elsif ch == '\\' #'
1055
- read_escape
1056
- end
1057
- if PERCENT_PAREN.values.include?(@quoted)
1058
- if PERCENT_PAREN[ch] == @quoted
1059
- nest += 1
1060
- elsif ch == @quoted
1061
- nest -= 1
1062
- end
1063
- end
1064
- end
1065
- if @ltype == "/"
1066
- while /[imxoesun]/ =~ peek(0)
1067
- getc
242
+ def process_nesting_level
243
+ @tokens.inject(0) { |indent, t|
244
+ case t[1]
245
+ when :on_lbracket, :on_lbrace, :on_lparen
246
+ indent += 1
247
+ when :on_rbracket, :on_rbrace, :on_rparen
248
+ indent -= 1
249
+ when :on_kw
250
+ case t[2]
251
+ when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
252
+ indent += 1
253
+ when 'if', 'unless', 'while', 'until', 'rescue'
254
+ # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
255
+ indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
256
+ when 'end'
257
+ indent -= 1
1068
258
  end
1069
259
  end
1070
- if subtype
1071
- Token(DLtype2Token[ltype])
1072
- else
1073
- Token(Ltype2Token[ltype])
1074
- end
1075
- ensure
1076
- @ltype = nil
1077
- @quoted = nil
1078
- @lex_state = EXPR_END
1079
- end
260
+ # percent literals are not indented
261
+ indent
262
+ }
1080
263
  end
1081
264
 
1082
- def identify_string_dvar
1083
- begin
1084
- getc
1085
-
1086
- reserve_continue = @continue
1087
- reserve_ltype = @ltype
1088
- reserve_indent = @indent
1089
- reserve_indent_stack = @indent_stack
1090
- reserve_state = @lex_state
1091
- reserve_quoted = @quoted
1092
-
1093
- @ltype = nil
1094
- @quoted = nil
1095
- @indent = 0
1096
- @indent_stack = []
1097
- @lex_state = EXPR_BEG
1098
-
1099
- loop do
1100
- @continue = false
1101
- prompt
1102
- tk = token
1103
- if @ltype or @continue or @indent >= 0
1104
- next
265
+ def check_string_literal
266
+ i = 0
267
+ start_token = []
268
+ end_type = []
269
+ while i < @tokens.size
270
+ t = @tokens[i]
271
+ case t[1]
272
+ when :on_tstring_beg
273
+ start_token << t
274
+ end_type << [:on_tstring_end, :on_label_end]
275
+ when :on_regexp_beg
276
+ start_token << t
277
+ end_type << :on_regexp_end
278
+ when :on_symbeg
279
+ acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw}
280
+ if (i + 1) < @tokens.size and acceptable_single_tokens.all?{ |t| @tokens[i + 1][1] != t }
281
+ start_token << t
282
+ end_type << :on_tstring_end
1105
283
  end
1106
- break if tk.kind_of?(TkRBRACE)
1107
- end
1108
- ensure
1109
- @continue = reserve_continue
1110
- @ltype = reserve_ltype
1111
- @indent = reserve_indent
1112
- @indent_stack = reserve_indent_stack
1113
- @lex_state = reserve_state
1114
- @quoted = reserve_quoted
1115
- end
284
+ when :on_backtick
285
+ start_token << t
286
+ end_type << :on_tstring_end
287
+ when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
288
+ start_token << t
289
+ end_type << :on_tstring_end
290
+ when :on_heredoc_beg
291
+ start_token << t
292
+ end_type << :on_heredoc_end
293
+ when *end_type.last
294
+ start_token.pop
295
+ end_type.pop
296
+ end
297
+ i += 1
298
+ end
299
+ start_token.last.nil? ? '' : start_token.last
1116
300
  end
1117
301
 
1118
- def identify_comment
1119
- @ltype = "#"
1120
-
1121
- while ch = getc
1122
- if ch == "\n"
1123
- @ltype = nil
1124
- ungetc
1125
- break
1126
- end
1127
- end
1128
- return Token(TkCOMMENT)
1129
- end
1130
-
1131
- def read_escape
1132
- case ch = getc
1133
- when "\n", "\r", "\f"
1134
- when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1135
- when /[0-7]/
1136
- ungetc ch
1137
- 3.times do
1138
- case ch = getc
1139
- when /[0-7]/
1140
- when nil
1141
- break
1142
- else
1143
- ungetc
1144
- break
1145
- end
1146
- end
1147
-
1148
- when "x"
1149
- 2.times do
1150
- case ch = getc
1151
- when /[0-9a-fA-F]/
1152
- when nil
1153
- break
1154
- else
1155
- ungetc
1156
- break
1157
- end
1158
- end
1159
-
1160
- when "M"
1161
- if (ch = getc) != '-'
1162
- ungetc
1163
- else
1164
- if (ch = getc) == "\\" #"
1165
- read_escape
1166
- end
1167
- end
1168
-
1169
- when "C", "c" #, "^"
1170
- if ch == "C" and (ch = getc) != "-"
1171
- ungetc
1172
- elsif (ch = getc) == "\\" #"
1173
- read_escape
302
+ def process_literal_type
303
+ start_token = check_string_literal
304
+ case start_token[1]
305
+ when :on_tstring_beg
306
+ case start_token[2]
307
+ when ?" then ?"
308
+ when /^%.$/ then ?"
309
+ when /^%Q.$/ then ?"
310
+ when ?' then ?'
311
+ when /^%q.$/ then ?'
312
+ end
313
+ when :on_regexp_beg then ?/
314
+ when :on_symbeg then ?:
315
+ when :on_backtick then ?`
316
+ when :on_qwords_beg then ?]
317
+ when :on_words_beg then ?]
318
+ when :on_qsymbols_beg then ?]
319
+ when :on_symbols_beg then ?]
320
+ when :on_heredoc_beg
321
+ start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
322
+ case $1
323
+ when ?" then ?"
324
+ when ?' then ?'
325
+ when ?` then ?`
326
+ else ?"
1174
327
  end
1175
328
  else
1176
- # other characters
329
+ nil
1177
330
  end
1178
331
  end
1179
332
  end