irb 1.0.0 → 1.1.0.pre.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -58,8 +58,6 @@ module IRB
58
58
  end
59
59
 
60
60
  module HistorySavingAbility # :nodoc:
61
- include Readline
62
-
63
61
  def HistorySavingAbility.extended(obj)
64
62
  IRB.conf[:AT_EXIT].push proc{obj.save_history}
65
63
  obj.load_history
@@ -67,18 +65,22 @@ module IRB
67
65
  end
68
66
 
69
67
  def load_history
68
+ return unless self.class.const_defined?(:HISTORY)
69
+ history = self.class::HISTORY
70
70
  if history_file = IRB.conf[:HISTORY_FILE]
71
71
  history_file = File.expand_path(history_file)
72
72
  end
73
73
  history_file = IRB.rc_file("_history") unless history_file
74
74
  if File.exist?(history_file)
75
75
  open(history_file) do |f|
76
- f.each {|l| HISTORY << l.chomp}
76
+ f.each {|l| history << l.chomp}
77
77
  end
78
78
  end
79
79
  end
80
80
 
81
81
  def save_history
82
+ return unless self.class.const_defined?(:HISTORY)
83
+ history = self.class::HISTORY
82
84
  if num = IRB.conf[:SAVE_HISTORY] and (num = num.to_i) > 0
83
85
  if history_file = IRB.conf[:HISTORY_FILE]
84
86
  history_file = File.expand_path(history_file)
@@ -96,7 +98,7 @@ module IRB
96
98
  end
97
99
 
98
100
  open(history_file, 'w', 0600 ) do |f|
99
- hist = HISTORY.to_a
101
+ hist = history.to_a
100
102
  f.puts(hist[-num..-1] || hist)
101
103
  end
102
104
  end
data/lib/irb/init.rb CHANGED
@@ -44,6 +44,7 @@ module IRB # :nodoc:
44
44
  @CONF[:IRB_RC] = nil
45
45
 
46
46
  @CONF[:USE_READLINE] = false unless defined?(ReadlineInputMethod)
47
+ @CONF[:USE_COLORIZE] = true
47
48
  @CONF[:INSPECT_MODE] = true
48
49
  @CONF[:USE_TRACER] = false
49
50
  @CONF[:USE_LOADER] = false
@@ -112,8 +113,6 @@ module IRB # :nodoc:
112
113
  @CONF[:LC_MESSAGES] = Locale.new
113
114
 
114
115
  @CONF[:AT_EXIT] = []
115
-
116
- @CONF[:DEBUG_LEVEL] = 0
117
116
  end
118
117
 
119
118
  def IRB.init_error
@@ -165,6 +164,10 @@ module IRB # :nodoc:
165
164
  @CONF[:USE_READLINE] = true
166
165
  when "--noreadline"
167
166
  @CONF[:USE_READLINE] = false
167
+ when "--reidline"
168
+ @CONF[:USE_REIDLINE] = true
169
+ when "--noreidline"
170
+ @CONF[:USE_REIDLINE] = false
168
171
  when "--echo"
169
172
  @CONF[:ECHO] = true
170
173
  when "--noecho"
@@ -173,6 +176,10 @@ module IRB # :nodoc:
173
176
  @CONF[:VERBOSE] = true
174
177
  when "--noverbose"
175
178
  @CONF[:VERBOSE] = false
179
+ when "--colorize"
180
+ @CONF[:USE_COLORIZE] = true
181
+ when "--nocolorize"
182
+ @CONF[:USE_COLORIZE] = false
176
183
  when /^--prompt-mode(?:=(.+))?/, /^--prompt(?:=(.+))?/
177
184
  opt = $1 || argv.shift
178
185
  prompt_mode = opt.upcase.tr("-", "_").intern
@@ -191,8 +198,6 @@ module IRB # :nodoc:
191
198
  @CONF[:CONTEXT_MODE] = ($1 || argv.shift).to_i
192
199
  when "--single-irb"
193
200
  @CONF[:SINGLE_IRB] = true
194
- when /^--irb_debug(?:=(.+))?/
195
- @CONF[:DEBUG_LEVEL] = ($1 || argv.shift).to_i
196
201
  when "-v", "--version"
197
202
  print IRB.version, "\n"
198
203
  exit 0
@@ -11,6 +11,8 @@
11
11
  #
12
12
  require_relative 'src_encoding'
13
13
  require_relative 'magic-file'
14
+ require_relative 'completion'
15
+ require 'reline'
14
16
 
15
17
  module IRB
16
18
  STDIN_FILE_NAME = "(line)" # :nodoc:
@@ -140,6 +142,12 @@ module IRB
140
142
 
141
143
  @stdin = IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
142
144
  @stdout = IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
145
+
146
+ if Readline.respond_to?("basic_word_break_characters=")
147
+ Readline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
148
+ end
149
+ Readline.completion_append_character = nil
150
+ Readline.completion_proc = IRB::InputCompletor::CompletionProc
143
151
  end
144
152
 
145
153
  # Reads the next line from this input method.
@@ -186,7 +194,95 @@ module IRB
186
194
  def encoding
187
195
  @stdin.external_encoding
188
196
  end
197
+
198
+ if Readline.respond_to?("basic_word_break_characters=")
199
+ Readline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
200
+ end
201
+ Readline.completion_append_character = nil
202
+ Readline.completion_proc = IRB::InputCompletor::CompletionProc
189
203
  end
190
204
  rescue LoadError
191
205
  end
206
+
207
+ class ReidlineInputMethod < InputMethod
208
+ include Reline
209
+ # Creates a new input method object using Readline
210
+ def initialize
211
+ super
212
+
213
+ @line_no = 0
214
+ @line = []
215
+ @eof = false
216
+
217
+ @stdin = ::IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
218
+ @stdout = ::IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
219
+
220
+ if Reline.respond_to?("basic_word_break_characters=")
221
+ Reline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
222
+ end
223
+ Reline.completion_append_character = nil
224
+ Reline.completion_proc = IRB::InputCompletor::CompletionProc
225
+ Reline.output_modifier_proc =
226
+ if IRB.conf[:USE_COLORIZE]
227
+ proc do |output, complete:|
228
+ next unless IRB::Color.colorable?
229
+ IRB::Color.colorize_code(output, complete: complete)
230
+ end
231
+ else
232
+ proc do |output|
233
+ Reline::Unicode.escape_for_print(output)
234
+ end
235
+ end
236
+ Reline.dig_perfect_match_proc = IRB::InputCompletor::PerfectMatchedProc
237
+ end
238
+
239
+ def check_termination(&block)
240
+ @check_termination_proc = block
241
+ end
242
+
243
+ # Reads the next line from this input method.
244
+ #
245
+ # See IO#gets for more information.
246
+ def gets
247
+ Reline.input = @stdin
248
+ Reline.output = @stdout
249
+ if l = readmultiline(@prompt, false, &@check_termination_proc)
250
+ HISTORY.push(l) if !l.empty?
251
+ @line[@line_no += 1] = l + "\n"
252
+ else
253
+ @eof = true
254
+ l
255
+ end
256
+ end
257
+
258
+ # Whether the end of this input method has been reached, returns +true+
259
+ # if there is no more data to read.
260
+ #
261
+ # See IO#eof? for more information.
262
+ def eof?
263
+ super
264
+ end
265
+
266
+ # Whether this input method is still readable when there is no more data to
267
+ # read.
268
+ #
269
+ # See IO#eof for more information.
270
+ def readable_after_eof?
271
+ true
272
+ end
273
+
274
+ # Returns the current line number for #io.
275
+ #
276
+ # #line counts the number of times #gets is called.
277
+ #
278
+ # See IO#lineno for more information.
279
+ def line(line_no)
280
+ @line[line_no]
281
+ end
282
+
283
+ # The external encoding for standard input.
284
+ def encoding
285
+ @stdin.external_encoding
286
+ end
287
+ end
192
288
  end
data/lib/irb/inspector.rb CHANGED
@@ -106,12 +106,22 @@ module IRB # :nodoc:
106
106
  Inspector.def_inspector([false, :to_s, :raw]){|v| v.to_s}
107
107
  Inspector.def_inspector([true, :p, :inspect]){|v|
108
108
  begin
109
- v.inspect
109
+ result = v.inspect
110
+ if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
111
+ result = Color.colorize_code(result)
112
+ end
113
+ result
110
114
  rescue NoMethodError
111
115
  puts "(Object doesn't support #inspect)"
112
116
  end
113
117
  }
114
- Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v| v.pretty_inspect.chomp}
118
+ Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v|
119
+ result = v.pretty_inspect.chomp
120
+ if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
121
+ result = Color.colorize_code(result)
122
+ end
123
+ result
124
+ }
115
125
  Inspector.def_inspector([:yaml, :YAML], proc{require "yaml"}){|v|
116
126
  begin
117
127
  YAML.dump(v)
@@ -26,6 +26,8 @@ Usage: irb.rb [options] [programfile] [arguments]
26
26
  --noinspect Don't use inspect for output
27
27
  --readline Use Readline extension module
28
28
  --noreadline Don't use Readline extension module
29
+ --colorize Use colorization
30
+ --nocolorize Don't use colorization
29
31
  --prompt prompt-mode/--prompt-mode prompt-mode
30
32
  Switch prompt mode. Pre-defined prompt modes are
31
33
  `default', `simple', `xmp' and `inf-ruby'
@@ -39,7 +41,6 @@ Usage: irb.rb [options] [programfile] [arguments]
39
41
  --back-trace-limit n
40
42
  Display backtrace top n and tail n. The default
41
43
  value is 16.
42
- --irb_debug n Set internal debug level to n (not for popular use)
43
44
  --verbose Show details
44
45
  --noverbose Don't show details
45
46
  -v, --version Print the version of irb
@@ -25,6 +25,8 @@ Usage: irb.rb [options] [programfile] [arguments]
25
25
  --noinspect 結果出力にinspectを用いない.
26
26
  --readline readlineライブラリを利用する.
27
27
  --noreadline readlineライブラリを利用しない.
28
+ --colorize 色付けを利用する.
29
+ --nocolorize 色付けを利用しない.
28
30
  --prompt prompt-mode/--prompt-mode prompt-mode
29
31
  プロンプトモードを切替えます. 現在定義されているプ
30
32
  ロンプトモードは, default, simple, xmp, inf-rubyが
@@ -41,8 +43,6 @@ Usage: irb.rb [options] [programfile] [arguments]
41
43
  バックトレース表示をバックトレースの頭から n, 後ろ
42
44
  からnだけ行なう. デフォルトは16
43
45
 
44
- --irb_debug n irbのデバッグレベルをnに設定する(非推奨).
45
-
46
46
  --verbose 詳細なメッセージを出力する.
47
47
  --noverbose 詳細なメッセージを出力しない(デフォルト).
48
48
  -v, --version irbのバージョンを表示する.
data/lib/irb/ruby-lex.rb CHANGED
@@ -11,73 +11,40 @@
11
11
  #
12
12
 
13
13
  require "e2mmap"
14
- require_relative "slex"
15
- require_relative "ruby-token"
14
+ require "ripper"
16
15
 
17
16
  # :stopdoc:
18
17
  class RubyLex
19
18
 
20
19
  extend Exception2MessageMapper
21
- def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
22
- def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
23
- def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
24
- def_exception(:TkReading2TokenDuplicateError,
25
- "key duplicate(token_n='%s', key='%s')")
26
- def_exception(:SyntaxError, "%s")
27
-
28
20
  def_exception(:TerminateLineInput, "Terminate Line Input")
29
21
 
30
- include RubyToken
31
-
32
- class << self
33
- attr_accessor :debug_level
34
- def debug?
35
- @debug_level > 0
36
- end
37
- end
38
- @debug_level = 0
39
-
40
22
  def initialize
41
- lex_init
42
- set_input(STDIN)
43
-
44
- @seek = 0
45
23
  @exp_line_no = @line_no = 1
46
- @base_char_no = 0
47
- @char_no = 0
48
- @rests = []
49
- @readed = []
50
- @here_readed = []
51
-
52
24
  @indent = 0
53
- @indent_stack = []
54
- @lex_state = EXPR_BEG
55
- @space_seen = false
56
- @here_header = false
57
- @post_symbeg = false
58
-
59
25
  @continue = false
60
26
  @line = ""
61
-
62
- @skip_space = false
63
- @readed_auto_clean_up = false
64
- @exception_on_syntax_error = true
65
-
66
27
  @prompt = nil
67
28
  end
68
29
 
69
- attr_accessor :skip_space
70
- attr_accessor :readed_auto_clean_up
71
- attr_accessor :exception_on_syntax_error
72
-
73
- attr_reader :seek
74
- attr_reader :char_no
75
- attr_reader :line_no
76
- attr_reader :indent
77
-
78
30
  # io functions
79
31
  def set_input(io, p = nil, &block)
80
32
  @io = io
33
+ if @io.respond_to?(:check_termination)
34
+ @io.check_termination do |code|
35
+ code.gsub!(/\s*\z/, '').concat("\n")
36
+ @tokens = Ripper.lex(code)
37
+ continue = process_continue
38
+ code_block_open = check_code_block(code)
39
+ indent = process_nesting_level
40
+ ltype = process_literal_type
41
+ if code_block_open or ltype or continue or indent > 0
42
+ false
43
+ else
44
+ true
45
+ end
46
+ end
47
+ end
81
48
  if p.respond_to?(:call)
82
49
  @input = p
83
50
  elsif block_given?
@@ -87,112 +54,6 @@ class RubyLex
87
54
  end
88
55
  end
89
56
 
90
- def get_readed
91
- if idx = @readed.rindex("\n")
92
- @base_char_no = @readed.size - (idx + 1)
93
- else
94
- @base_char_no += @readed.size
95
- end
96
-
97
- readed = @readed.join("")
98
- @readed = []
99
- readed
100
- end
101
-
102
- def getc
103
- while @rests.empty?
104
- @rests.push nil unless buf_input
105
- end
106
- c = @rests.shift
107
- if @here_header
108
- @here_readed.push c
109
- else
110
- @readed.push c
111
- end
112
- @seek += 1
113
- if c == "\n"
114
- @line_no += 1
115
- @char_no = 0
116
- else
117
- @char_no += 1
118
- end
119
- c
120
- end
121
-
122
- def gets
123
- l = ""
124
- while c = getc
125
- l.concat(c)
126
- break if c == "\n"
127
- end
128
- return nil if l == "" and c.nil?
129
- l
130
- end
131
-
132
- def eof?
133
- @io.eof?
134
- end
135
-
136
- def getc_of_rests
137
- if @rests.empty?
138
- nil
139
- else
140
- getc
141
- end
142
- end
143
-
144
- def ungetc(c = nil)
145
- if @here_readed.empty?
146
- c2 = @readed.pop
147
- else
148
- c2 = @here_readed.pop
149
- end
150
- c = c2 unless c
151
- @rests.unshift c #c =
152
- @seek -= 1
153
- if c == "\n"
154
- @line_no -= 1
155
- if idx = @readed.rindex("\n")
156
- @char_no = idx + 1
157
- else
158
- @char_no = @base_char_no + @readed.size
159
- end
160
- else
161
- @char_no -= 1
162
- end
163
- end
164
-
165
- def peek_equal?(str)
166
- chrs = str.split(//)
167
- until @rests.size >= chrs.size
168
- return false unless buf_input
169
- end
170
- @rests[0, chrs.size] == chrs
171
- end
172
-
173
- def peek_match?(regexp)
174
- while @rests.empty?
175
- return false unless buf_input
176
- end
177
- regexp =~ @rests.join("")
178
- end
179
-
180
- def peek(i = 0)
181
- while @rests.size <= i
182
- return nil unless buf_input
183
- end
184
- @rests[i]
185
- end
186
-
187
- def buf_input
188
- prompt
189
- line = @input.call
190
- return nil unless line
191
- @rests.concat line.chars.to_a
192
- true
193
- end
194
- private :buf_input
195
-
196
57
  def set_prompt(p = nil, &block)
197
58
  p = block if block_given?
198
59
  if p.respond_to?(:call)
@@ -210,20 +71,11 @@ class RubyLex
210
71
 
211
72
  def initialize_input
212
73
  @ltype = nil
213
- @quoted = nil
214
74
  @indent = 0
215
- @indent_stack = []
216
- @lex_state = EXPR_BEG
217
- @space_seen = false
218
- @here_header = false
219
-
220
75
  @continue = false
221
- @post_symbeg = false
222
-
223
- prompt
224
-
225
76
  @line = ""
226
77
  @exp_line_no = @line_no
78
+ @code_block_open = false
227
79
  end
228
80
 
229
81
  def each_top_level_statement
@@ -231,13 +83,14 @@ class RubyLex
231
83
  catch(:TERM_INPUT) do
232
84
  loop do
233
85
  begin
234
- @continue = false
235
86
  prompt
236
87
  unless l = lex
237
88
  throw :TERM_INPUT if @line == ''
238
89
  else
90
+ @line_no += 1
91
+ next if l == "\n"
239
92
  @line.concat l
240
- if @ltype or @continue or @indent > 0
93
+ if @code_block_open or @ltype or @continue or @indent > 0
241
94
  next
242
95
  end
243
96
  end
@@ -245,935 +98,235 @@ class RubyLex
245
98
  @line.force_encoding(@io.encoding)
246
99
  yield @line, @exp_line_no
247
100
  end
248
- break unless l
101
+ break if @io.eof?
249
102
  @line = ''
250
103
  @exp_line_no = @line_no
251
104
 
252
105
  @indent = 0
253
- @indent_stack = []
254
- prompt
255
106
  rescue TerminateLineInput
256
107
  initialize_input
257
108
  prompt
258
- get_readed
259
109
  end
260
110
  end
261
111
  end
262
112
  end
263
113
 
264
114
  def lex
265
- continue = @continue
266
- while tk = token
267
- case tk
268
- when TkNL, TkEND_OF_SCRIPT
269
- @continue = continue unless continue.nil?
270
- break unless @continue
271
- when TkSPACE, TkCOMMENT
272
- when TkSEMICOLON, TkBEGIN, TkELSE
273
- @continue = continue = false
274
- else
275
- continue = nil
276
- end
277
- end
278
- line = get_readed
279
- if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
280
- nil
281
- else
282
- line
283
- end
284
- end
285
-
286
- def token
287
- @prev_seek = @seek
288
- @prev_line_no = @line_no
289
- @prev_char_no = @char_no
290
- begin
291
- begin
292
- tk = @OP.match(self)
293
- @space_seen = tk.kind_of?(TkSPACE)
294
- @lex_state = EXPR_END if @post_symbeg && tk.kind_of?(TkOp)
295
- @post_symbeg = tk.kind_of?(TkSYMBEG)
296
- rescue SyntaxError
297
- raise if @exception_on_syntax_error
298
- tk = TkError.new(@seek, @line_no, @char_no)
299
- end
300
- end while @skip_space and tk.kind_of?(TkSPACE)
301
- if @readed_auto_clean_up
302
- get_readed
303
- end
304
- tk
305
- end
306
-
307
- ENINDENT_CLAUSE = [
308
- "case", "class", "def", "do", "for", "if",
309
- "module", "unless", "until", "while", "begin"
310
- ]
311
- DEINDENT_CLAUSE = ["end"
312
- ]
313
-
314
- PERCENT_LTYPE = {
315
- "q" => "\'",
316
- "Q" => "\"",
317
- "x" => "\`",
318
- "r" => "/",
319
- "w" => "]",
320
- "W" => "]",
321
- "i" => "]",
322
- "I" => "]",
323
- "s" => ":"
324
- }
325
-
326
- PERCENT_PAREN = {
327
- "{" => "}",
328
- "[" => "]",
329
- "<" => ">",
330
- "(" => ")"
331
- }
332
-
333
- Ltype2Token = {
334
- "\'" => TkSTRING,
335
- "\"" => TkSTRING,
336
- "\`" => TkXSTRING,
337
- "/" => TkREGEXP,
338
- "]" => TkDSTRING,
339
- ":" => TkSYMBOL
340
- }
341
- DLtype2Token = {
342
- "\"" => TkDSTRING,
343
- "\`" => TkDXSTRING,
344
- "/" => TkDREGEXP,
345
- }
346
-
347
- def lex_init()
348
- @OP = IRB::SLex.new
349
- @OP.def_rules("\0", "\004", "\032") do |op, io|
350
- Token(TkEND_OF_SCRIPT)
351
- end
352
-
353
- @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
354
- @space_seen = true
355
- while getc =~ /[ \t\f\r\13]/; end
356
- ungetc
357
- Token(TkSPACE)
358
- end
359
-
360
- @OP.def_rule("#") do |op, io|
361
- identify_comment
362
- end
363
-
364
- @OP.def_rule("=begin",
365
- proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
366
- |op, io|
367
- @ltype = "="
368
- until getc == "\n"; end
369
- until peek_equal?("=end") && peek(4) =~ /\s/
370
- until getc == "\n"; end
371
- end
372
- gets
373
- @ltype = nil
374
- Token(TkRD_COMMENT)
375
- end
376
-
377
- @OP.def_rule("\n") do |op, io|
378
- print "\\n\n" if RubyLex.debug?
379
- case @lex_state
380
- when EXPR_BEG, EXPR_FNAME, EXPR_DOT
381
- @continue = true
382
- else
383
- @continue = false
384
- @lex_state = EXPR_BEG
385
- until (@indent_stack.empty? ||
386
- [TkLPAREN, TkLBRACK, TkLBRACE,
387
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
388
- @indent_stack.pop
389
- end
390
- end
391
- @here_header = false
392
- @here_readed = []
393
- Token(TkNL)
394
- end
395
-
396
- @OP.def_rules("*", "**",
397
- "=", "==", "===",
398
- "=~", "<=>",
399
- "<", "<=",
400
- ">", ">=", ">>",
401
- "!", "!=", "!~") do
402
- |op, io|
403
- case @lex_state
404
- when EXPR_FNAME, EXPR_DOT
405
- @lex_state = EXPR_ARG
406
- else
407
- @lex_state = EXPR_BEG
408
- end
409
- Token(op)
410
- end
411
-
412
- @OP.def_rules("<<") do
413
- |op, io|
414
- tk = nil
415
- if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
416
- (@lex_state != EXPR_ARG || @space_seen)
417
- c = peek(0)
418
- if /[-~"'`\w]/ =~ c
419
- tk = identify_here_document
420
- end
421
- end
422
- unless tk
423
- tk = Token(op)
424
- case @lex_state
425
- when EXPR_FNAME, EXPR_DOT
426
- @lex_state = EXPR_ARG
427
- else
428
- @lex_state = EXPR_BEG
429
- end
430
- end
431
- tk
432
- end
433
-
434
- @OP.def_rules("'", '"') do
435
- |op, io|
436
- identify_string(op)
437
- end
438
-
439
- @OP.def_rules("`") do
440
- |op, io|
441
- if @lex_state == EXPR_FNAME
442
- @lex_state = EXPR_END
443
- Token(op)
444
- else
445
- identify_string(op)
446
- end
447
- end
448
-
449
- @OP.def_rules('?') do
450
- |op, io|
451
- if @lex_state == EXPR_END
452
- @lex_state = EXPR_BEG
453
- Token(TkQUESTION)
454
- else
455
- ch = getc
456
- if @lex_state == EXPR_ARG && ch =~ /\s/
457
- ungetc
458
- @lex_state = EXPR_BEG;
459
- Token(TkQUESTION)
460
- else
461
- if (ch == '\\')
462
- read_escape
463
- end
464
- @lex_state = EXPR_END
465
- Token(TkINTEGER)
466
- end
467
- end
468
- end
469
-
470
- @OP.def_rules("&", "&&", "|", "||") do
471
- |op, io|
472
- @lex_state = EXPR_BEG
473
- Token(op)
474
- end
475
-
476
- @OP.def_rules("+=", "-=", "*=", "**=",
477
- "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
478
- |op, io|
479
- @lex_state = EXPR_BEG
480
- op =~ /^(.*)=$/
481
- Token(TkOPASGN, $1)
482
- end
483
-
484
- @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
485
- |op, io|
486
- @lex_state = EXPR_ARG
487
- Token(op)
488
- end
489
-
490
- @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
491
- |op, io|
492
- @lex_state = EXPR_ARG
493
- Token(op)
494
- end
495
-
496
- @OP.def_rules("+", "-") do
497
- |op, io|
498
- catch(:RET) do
499
- if @lex_state == EXPR_ARG
500
- if @space_seen and peek(0) =~ /[0-9]/
501
- throw :RET, identify_number
502
- else
503
- @lex_state = EXPR_BEG
504
- end
505
- elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
506
- throw :RET, identify_number
507
- else
508
- @lex_state = EXPR_BEG
509
- end
510
- Token(op)
511
- end
512
- end
513
-
514
- @OP.def_rule(".") do
515
- |op, io|
516
- @lex_state = EXPR_BEG
517
- if peek(0) =~ /[0-9]/
518
- ungetc
519
- identify_number
520
- else
521
- # for "obj.if" etc.
522
- @lex_state = EXPR_DOT
523
- Token(TkDOT)
524
- end
525
- end
526
-
527
- @OP.def_rules("..", "...") do
528
- |op, io|
529
- @lex_state = EXPR_BEG
530
- Token(op)
531
- end
532
-
533
- lex_int2
534
- end
535
-
536
- def lex_int2
537
- @OP.def_rules("]", "}", ")") do
538
- |op, io|
539
- @lex_state = EXPR_END
540
- @indent -= 1
541
- @indent_stack.pop
542
- Token(op)
543
- end
544
-
545
- @OP.def_rule(":") do
546
- |op, io|
547
- if @lex_state == EXPR_END || peek(0) =~ /\s/
548
- @lex_state = EXPR_BEG
549
- Token(TkCOLON)
550
- else
551
- @lex_state = EXPR_FNAME
552
- Token(TkSYMBEG)
553
- end
554
- end
555
-
556
- @OP.def_rule("::") do
557
- |op, io|
558
- if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
559
- @lex_state = EXPR_BEG
560
- Token(TkCOLON3)
561
- else
562
- @lex_state = EXPR_DOT
563
- Token(TkCOLON2)
564
- end
565
- end
566
-
567
- @OP.def_rule("/") do
568
- |op, io|
569
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
570
- identify_string(op)
571
- elsif peek(0) == '='
572
- getc
573
- @lex_state = EXPR_BEG
574
- Token(TkOPASGN, "/") #/)
575
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
576
- identify_string(op)
577
- else
578
- @lex_state = EXPR_BEG
579
- Token("/") #/)
580
- end
581
- end
582
-
583
- @OP.def_rules("^") do
584
- |op, io|
585
- @lex_state = EXPR_BEG
586
- Token("^")
587
- end
588
-
589
- @OP.def_rules(",") do
590
- |op, io|
591
- @lex_state = EXPR_BEG
592
- Token(op)
593
- end
594
-
595
- @OP.def_rules(";") do
596
- |op, io|
597
- @lex_state = EXPR_BEG
598
- until (@indent_stack.empty? ||
599
- [TkLPAREN, TkLBRACK, TkLBRACE,
600
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
601
- @indent_stack.pop
602
- end
603
- Token(op)
604
- end
605
-
606
- @OP.def_rule("~") do
607
- |op, io|
608
- @lex_state = EXPR_BEG
609
- Token("~")
610
- end
611
-
612
- @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
613
- |op, io|
614
- @lex_state = EXPR_BEG
615
- Token("~")
616
- end
617
-
618
- @OP.def_rule("(") do
619
- |op, io|
620
- @indent += 1
621
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
622
- @lex_state = EXPR_BEG
623
- tk_c = TkfLPAREN
624
- else
625
- @lex_state = EXPR_BEG
626
- tk_c = TkLPAREN
627
- end
628
- @indent_stack.push tk_c
629
- Token(tk_c)
630
- end
631
-
632
- @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
633
- |op, io|
634
- @lex_state = EXPR_ARG
635
- Token("[]")
636
- end
637
-
638
- @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
639
- |op, io|
640
- @lex_state = EXPR_ARG
641
- Token("[]=")
642
- end
643
-
644
- @OP.def_rule("[") do
645
- |op, io|
646
- @indent += 1
647
- if @lex_state == EXPR_FNAME
648
- tk_c = TkfLBRACK
649
- else
650
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
651
- tk_c = TkLBRACK
652
- elsif @lex_state == EXPR_ARG && @space_seen
653
- tk_c = TkLBRACK
654
- else
655
- tk_c = TkfLBRACK
656
- end
657
- @lex_state = EXPR_BEG
658
- end
659
- @indent_stack.push tk_c
660
- Token(tk_c)
661
- end
662
-
663
- @OP.def_rule("{") do
664
- |op, io|
665
- @indent += 1
666
- if @lex_state != EXPR_END && @lex_state != EXPR_ARG
667
- tk_c = TkLBRACE
668
- else
669
- tk_c = TkfLBRACE
670
- end
671
- @lex_state = EXPR_BEG
672
- @indent_stack.push tk_c
673
- Token(tk_c)
674
- end
675
-
676
- @OP.def_rule('\\') do
677
- |op, io|
678
- if getc == "\n"
679
- @space_seen = true
680
- @continue = true
681
- Token(TkSPACE)
682
- else
683
- read_escape
684
- Token("\\")
685
- end
686
- end
687
-
688
- @OP.def_rule('%') do
689
- |op, io|
690
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
691
- identify_quotation
692
- elsif peek(0) == '='
693
- getc
694
- Token(TkOPASGN, :%)
695
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
696
- identify_quotation
697
- else
698
- @lex_state = EXPR_BEG
699
- Token("%") #))
700
- end
701
- end
702
-
703
- @OP.def_rule('$') do
704
- |op, io|
705
- identify_gvar
706
- end
707
-
708
- @OP.def_rule('@') do
709
- |op, io|
710
- if peek(0) =~ /[\w@]/
711
- ungetc
712
- identify_identifier
713
- else
714
- Token("@")
715
- end
716
- end
717
-
718
- @OP.def_rule("") do
719
- |op, io|
720
- printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
721
- if peek(0) =~ /[0-9]/
722
- t = identify_number
723
- elsif peek(0) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
724
- t = identify_identifier
725
- end
726
- printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
727
- t
728
- end
729
-
730
- p @OP if RubyLex.debug?
731
- end
732
-
733
- def identify_gvar
734
- @lex_state = EXPR_END
735
-
736
- case ch = getc
737
- when /[~_*$?!@\/\\;,=:<>".]/ #"
738
- Token(TkGVAR, "$" + ch)
739
- when "-"
740
- Token(TkGVAR, "$-" + getc)
741
- when "&", "`", "'", "+"
742
- Token(TkBACK_REF, "$"+ch)
743
- when /[1-9]/
744
- while getc =~ /[0-9]/; end
745
- ungetc
746
- Token(TkNTH_REF)
747
- when /\w/
748
- ungetc
749
- ungetc
750
- identify_identifier
751
- else
752
- ungetc
753
- Token("$")
754
- end
115
+ line = @input.call
116
+ if @io.respond_to?(:check_termination)
117
+ return line # multiline
118
+ end
119
+ code = @line + (line.nil? ? '' : line)
120
+ code.gsub!(/\s*\z/, '').concat("\n")
121
+ @tokens = Ripper.lex(code)
122
+ @continue = process_continue
123
+ @code_block_open = check_code_block(code)
124
+ @indent = process_nesting_level
125
+ @ltype = process_literal_type
126
+ line
755
127
  end
756
128
 
757
- def identify_identifier
758
- token = ""
759
- if peek(0) =~ /[$@]/
760
- token.concat(c = getc)
761
- if c == "@" and peek(0) == "@"
762
- token.concat getc
763
- end
764
- end
765
-
766
- while (ch = getc) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
767
- print ":", ch, ":" if RubyLex.debug?
768
- token.concat ch
769
- end
770
- ungetc
771
-
772
- if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
773
- token.concat getc
774
- end
775
-
776
- # almost fix token
777
-
778
- case token
779
- when /^\$/
780
- return Token(TkGVAR, token)
781
- when /^\@\@/
782
- @lex_state = EXPR_END
783
- # p Token(TkCVAR, token)
784
- return Token(TkCVAR, token)
785
- when /^\@/
786
- @lex_state = EXPR_END
787
- return Token(TkIVAR, token)
788
- end
789
-
790
- if @lex_state != EXPR_DOT
791
- print token, "\n" if RubyLex.debug?
792
-
793
- token_c, *trans = TkReading2Token[token]
794
- if token_c
795
- # reserved word?
796
-
797
- if (@lex_state != EXPR_BEG &&
798
- @lex_state != EXPR_FNAME &&
799
- trans[1])
800
- # modifiers
801
- token_c = TkSymbol2Token[trans[1]]
802
- @lex_state = trans[0]
803
- else
804
- if @lex_state != EXPR_FNAME and peek(0) != ':'
805
- if ENINDENT_CLAUSE.include?(token)
806
- # check for ``class = val'' etc.
807
- valid = true
808
- case token
809
- when "class"
810
- valid = false unless peek_match?(/^\s*(<<|\w|::)/)
811
- when "def"
812
- valid = false if peek_match?(/^\s*(([+\-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
813
- when "do"
814
- valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&)/)
815
- when *ENINDENT_CLAUSE
816
- valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&|\|)/)
817
- else
818
- # no nothing
819
- end
820
- if valid
821
- if token == "do"
822
- if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
823
- @indent += 1
824
- @indent_stack.push token_c
825
- end
826
- else
827
- @indent += 1
828
- @indent_stack.push token_c
829
- end
830
- end
831
-
832
- elsif DEINDENT_CLAUSE.include?(token)
833
- @indent -= 1
834
- @indent_stack.pop
835
- end
836
- @lex_state = trans[0]
837
- else
838
- @lex_state = EXPR_END
839
- end
840
- end
841
- return Token(token_c, token)
842
- end
843
- end
844
-
845
- if @lex_state == EXPR_FNAME
846
- @lex_state = EXPR_END
847
- if peek(0) == '='
848
- token.concat getc
849
- end
850
- elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
851
- @lex_state = EXPR_ARG
852
- else
853
- @lex_state = EXPR_END
854
- end
855
-
856
- if token[0, 1] =~ /[A-Z]/
857
- return Token(TkCONSTANT, token)
858
- elsif token[token.size - 1, 1] =~ /[!?]/
859
- return Token(TkFID, token)
860
- else
861
- return Token(TkIDENTIFIER, token)
862
- end
129
+ def process_continue
130
+ continued_bits = Ripper::EXPR_BEG | Ripper::EXPR_FNAME
131
+ # last token is always newline
132
+ if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end
133
+ # end of regexp literal
134
+ return false
135
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon
136
+ return false
137
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and (@tokens[-2][2] == 'begin' or @tokens[-2][2] == 'else')
138
+ return false
139
+ elsif @tokens.size >= 3 and @tokens[-3][1] == :on_symbeg and @tokens[-2][1] == :on_ivar
140
+ # This is for :@a or :@1 because :@1 ends with EXPR_FNAME
141
+ return false
142
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_ivar and @tokens[-2][2] =~ /\A@\d+\z/
143
+ # This is for @1
144
+ return false
145
+ elsif @tokens.size >= 2 and @tokens[-2][1] == :on_cvar and @tokens[-1][1] == :on_int
146
+ # This is for @@1 or :@@1 and ends with on_int because it's syntax error
147
+ return false
148
+ elsif !@tokens.empty? and @tokens.last[2] == "\\\n"
149
+ return true
150
+ elsif @tokens.size >= 1 and @tokens[-1][1] == :on_heredoc_end # "EOH\n"
151
+ return false
152
+ elsif @tokens.size >= 2 and @tokens[-2][3].anybits?(continued_bits)
153
+ # end of literal except for regexp
154
+ return true
155
+ end
156
+ false
863
157
  end
864
158
 
865
- def identify_here_document
866
- ch = getc
867
- if ch == "-" || ch == "~"
868
- ch = getc
869
- indent = true
870
- end
871
- if /['"`]/ =~ ch
872
- lt = ch
873
- quoted = ""
874
- while (c = getc) && c != lt
875
- quoted.concat c
876
- end
877
- else
878
- lt = '"'
879
- quoted = ch.dup
880
- while (c = getc) && c =~ /\w/
881
- quoted.concat c
882
- end
883
- ungetc
884
- end
885
-
886
- ltback, @ltype = @ltype, lt
887
- reserve = []
888
- while ch = getc
889
- reserve.push ch
890
- if ch == "\\"
891
- reserve.push ch = getc
892
- elsif ch == "\n"
893
- break
894
- end
895
- end
896
-
897
- @here_header = false
898
-
899
- line = ""
900
- while ch = getc
901
- if ch == "\n"
902
- if line == quoted
903
- break
904
- end
905
- line = ""
906
- else
907
- line.concat ch unless indent && line == "" && /\s/ =~ ch
908
- if @ltype != "'" && ch == "#" && peek(0) == "{"
909
- identify_string_dvar
910
- end
159
+ def check_code_block(code)
160
+ return true if @tokens.empty?
161
+ if @tokens.last[1] == :on_heredoc_beg
162
+ return true
163
+ end
164
+
165
+ begin # check if parser error are available
166
+ verbose, $VERBOSE = $VERBOSE, nil
167
+ RubyVM::InstructionSequence.compile(code)
168
+ rescue SyntaxError => e
169
+ case e.message
170
+ when /unterminated (?:string|regexp) meets end of file/
171
+ # "unterminated regexp meets end of file"
172
+ #
173
+ # example:
174
+ # /
175
+ #
176
+ # "unterminated string meets end of file"
177
+ #
178
+ # example:
179
+ # '
180
+ return true
181
+ when /syntax error, unexpected end-of-input/
182
+ # "syntax error, unexpected end-of-input, expecting keyword_end"
183
+ #
184
+ # example:
185
+ # if ture
186
+ # hoge
187
+ # if false
188
+ # fuga
189
+ # end
190
+ return true
191
+ when /syntax error, unexpected keyword_end/
192
+ # "syntax error, unexpected keyword_end"
193
+ #
194
+ # example:
195
+ # if (
196
+ # end
197
+ #
198
+ # example:
199
+ # end
200
+ return false
201
+ when /syntax error, unexpected '\.'/
202
+ # "syntax error, unexpected '.'"
203
+ #
204
+ # example:
205
+ # .
206
+ return false
207
+ when /unexpected tREGEXP_BEG/
208
+ # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
209
+ #
210
+ # example:
211
+ # method / f /
212
+ return false
213
+ when /numbered parameter outside block/
214
+ # "numbered parameter outside block"
215
+ #
216
+ # example:
217
+ # :@1
218
+ return false
911
219
  end
220
+ ensure
221
+ $VERBOSE = verbose
912
222
  end
913
223
 
914
- @here_header = true
915
- @here_readed.concat reserve
916
- while ch = reserve.pop
917
- ungetc ch
918
- end
919
-
920
- @ltype = ltback
921
- @lex_state = EXPR_END
922
- Token(Ltype2Token[lt])
923
- end
924
-
925
- def identify_quotation
926
- ch = getc
927
- if lt = PERCENT_LTYPE[ch]
928
- ch = getc
929
- elsif ch =~ /\W/
930
- lt = "\""
931
- else
932
- RubyLex.fail SyntaxError, "unknown type of %string"
933
- end
934
- @quoted = ch unless @quoted = PERCENT_PAREN[ch]
935
- identify_string(lt, @quoted)
936
- end
937
-
938
- def identify_number
939
- @lex_state = EXPR_END
940
-
941
- if peek(0) == "0" && peek(1) !~ /[.eE]/
942
- getc
943
- case peek(0)
944
- when /[xX]/
945
- ch = getc
946
- match = /[0-9a-fA-F_]/
947
- when /[bB]/
948
- ch = getc
949
- match = /[01_]/
950
- when /[oO]/
951
- ch = getc
952
- match = /[0-7_]/
953
- when /[dD]/
954
- ch = getc
955
- match = /[0-9_]/
956
- when /[0-7]/
957
- match = /[0-7_]/
958
- when /[89]/
959
- RubyLex.fail SyntaxError, "Invalid octal digit"
960
- else
961
- return Token(TkINTEGER)
962
- end
963
-
964
- len0 = true
965
- non_digit = false
966
- while ch = getc
967
- if match =~ ch
968
- if ch == "_"
969
- if non_digit
970
- RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
971
- else
972
- non_digit = ch
973
- end
974
- else
975
- non_digit = false
976
- len0 = false
977
- end
978
- else
979
- ungetc
980
- if len0
981
- RubyLex.fail SyntaxError, "numeric literal without digits"
982
- end
983
- if non_digit
984
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
985
- end
986
- break
987
- end
988
- end
989
- return Token(TkINTEGER)
224
+ last_lex_state = @tokens.last[3]
225
+ if last_lex_state.allbits?(Ripper::EXPR_BEG)
226
+ return false
227
+ elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
228
+ return true
229
+ elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
230
+ return true
231
+ elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
232
+ return true
233
+ elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
234
+ return true
235
+ elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
236
+ return false
990
237
  end
991
238
 
992
- type = TkINTEGER
993
- allow_point = true
994
- allow_e = true
995
- non_digit = false
996
- while ch = getc
997
- case ch
998
- when /[0-9]/
999
- non_digit = false
1000
- when "_"
1001
- non_digit = ch
1002
- when allow_point && "."
1003
- if non_digit
1004
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1005
- end
1006
- type = TkFLOAT
1007
- if peek(0) !~ /[0-9]/
1008
- type = TkINTEGER
1009
- ungetc
1010
- break
1011
- end
1012
- allow_point = false
1013
- when allow_e && "e", allow_e && "E"
1014
- if non_digit
1015
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016
- end
1017
- type = TkFLOAT
1018
- if peek(0) =~ /[+-]/
1019
- getc
1020
- end
1021
- allow_e = false
1022
- allow_point = false
1023
- non_digit = ch
1024
- else
1025
- if non_digit
1026
- RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027
- end
1028
- ungetc
1029
- break
1030
- end
1031
- end
1032
- Token(type)
239
+ false
1033
240
  end
1034
241
 
1035
- def identify_string(ltype, quoted = ltype)
1036
- @ltype = ltype
1037
- @quoted = quoted
1038
- subtype = nil
1039
- begin
1040
- nest = 0
1041
- while ch = getc
1042
- if @quoted == ch and nest == 0
1043
- break
1044
- elsif @ltype != "'" && ch == "#" && peek(0) == "{"
1045
- identify_string_dvar
1046
- elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1047
- subtype = true
1048
- elsif ch == '\\' and @ltype == "'" #'
1049
- case ch = getc
1050
- when "\\", "\n", "'"
1051
- else
1052
- ungetc
1053
- end
1054
- elsif ch == '\\' #'
1055
- read_escape
1056
- end
1057
- if PERCENT_PAREN.values.include?(@quoted)
1058
- if PERCENT_PAREN[ch] == @quoted
1059
- nest += 1
1060
- elsif ch == @quoted
1061
- nest -= 1
1062
- end
1063
- end
1064
- end
1065
- if @ltype == "/"
1066
- while /[imxoesun]/ =~ peek(0)
1067
- getc
242
+ def process_nesting_level
243
+ @tokens.inject(0) { |indent, t|
244
+ case t[1]
245
+ when :on_lbracket, :on_lbrace, :on_lparen
246
+ indent += 1
247
+ when :on_rbracket, :on_rbrace, :on_rparen
248
+ indent -= 1
249
+ when :on_kw
250
+ case t[2]
251
+ when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
252
+ indent += 1
253
+ when 'if', 'unless', 'while', 'until', 'rescue'
254
+ # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
255
+ indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
256
+ when 'end'
257
+ indent -= 1
1068
258
  end
1069
259
  end
1070
- if subtype
1071
- Token(DLtype2Token[ltype])
1072
- else
1073
- Token(Ltype2Token[ltype])
1074
- end
1075
- ensure
1076
- @ltype = nil
1077
- @quoted = nil
1078
- @lex_state = EXPR_END
1079
- end
260
+ # percent literals are not indented
261
+ indent
262
+ }
1080
263
  end
1081
264
 
1082
- def identify_string_dvar
1083
- begin
1084
- getc
1085
-
1086
- reserve_continue = @continue
1087
- reserve_ltype = @ltype
1088
- reserve_indent = @indent
1089
- reserve_indent_stack = @indent_stack
1090
- reserve_state = @lex_state
1091
- reserve_quoted = @quoted
1092
-
1093
- @ltype = nil
1094
- @quoted = nil
1095
- @indent = 0
1096
- @indent_stack = []
1097
- @lex_state = EXPR_BEG
1098
-
1099
- loop do
1100
- @continue = false
1101
- prompt
1102
- tk = token
1103
- if @ltype or @continue or @indent >= 0
1104
- next
265
+ def check_string_literal
266
+ i = 0
267
+ start_token = []
268
+ end_type = []
269
+ while i < @tokens.size
270
+ t = @tokens[i]
271
+ case t[1]
272
+ when :on_tstring_beg
273
+ start_token << t
274
+ end_type << [:on_tstring_end, :on_label_end]
275
+ when :on_regexp_beg
276
+ start_token << t
277
+ end_type << :on_regexp_end
278
+ when :on_symbeg
279
+ acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw}
280
+ if (i + 1) < @tokens.size and acceptable_single_tokens.all?{ |t| @tokens[i + 1][1] != t }
281
+ start_token << t
282
+ end_type << :on_tstring_end
1105
283
  end
1106
- break if tk.kind_of?(TkRBRACE)
1107
- end
1108
- ensure
1109
- @continue = reserve_continue
1110
- @ltype = reserve_ltype
1111
- @indent = reserve_indent
1112
- @indent_stack = reserve_indent_stack
1113
- @lex_state = reserve_state
1114
- @quoted = reserve_quoted
1115
- end
284
+ when :on_backtick
285
+ start_token << t
286
+ end_type << :on_tstring_end
287
+ when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
288
+ start_token << t
289
+ end_type << :on_tstring_end
290
+ when :on_heredoc_beg
291
+ start_token << t
292
+ end_type << :on_heredoc_end
293
+ when *end_type.last
294
+ start_token.pop
295
+ end_type.pop
296
+ end
297
+ i += 1
298
+ end
299
+ start_token.last.nil? ? '' : start_token.last
1116
300
  end
1117
301
 
1118
- def identify_comment
1119
- @ltype = "#"
1120
-
1121
- while ch = getc
1122
- if ch == "\n"
1123
- @ltype = nil
1124
- ungetc
1125
- break
1126
- end
1127
- end
1128
- return Token(TkCOMMENT)
1129
- end
1130
-
1131
- def read_escape
1132
- case ch = getc
1133
- when "\n", "\r", "\f"
1134
- when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1135
- when /[0-7]/
1136
- ungetc ch
1137
- 3.times do
1138
- case ch = getc
1139
- when /[0-7]/
1140
- when nil
1141
- break
1142
- else
1143
- ungetc
1144
- break
1145
- end
1146
- end
1147
-
1148
- when "x"
1149
- 2.times do
1150
- case ch = getc
1151
- when /[0-9a-fA-F]/
1152
- when nil
1153
- break
1154
- else
1155
- ungetc
1156
- break
1157
- end
1158
- end
1159
-
1160
- when "M"
1161
- if (ch = getc) != '-'
1162
- ungetc
1163
- else
1164
- if (ch = getc) == "\\" #"
1165
- read_escape
1166
- end
1167
- end
1168
-
1169
- when "C", "c" #, "^"
1170
- if ch == "C" and (ch = getc) != "-"
1171
- ungetc
1172
- elsif (ch = getc) == "\\" #"
1173
- read_escape
302
+ def process_literal_type
303
+ start_token = check_string_literal
304
+ case start_token[1]
305
+ when :on_tstring_beg
306
+ case start_token[2]
307
+ when ?" then ?"
308
+ when /^%.$/ then ?"
309
+ when /^%Q.$/ then ?"
310
+ when ?' then ?'
311
+ when /^%q.$/ then ?'
312
+ end
313
+ when :on_regexp_beg then ?/
314
+ when :on_symbeg then ?:
315
+ when :on_backtick then ?`
316
+ when :on_qwords_beg then ?]
317
+ when :on_words_beg then ?]
318
+ when :on_qsymbols_beg then ?]
319
+ when :on_symbols_beg then ?]
320
+ when :on_heredoc_beg
321
+ start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
322
+ case $1
323
+ when ?" then ?"
324
+ when ?' then ?'
325
+ when ?` then ?`
326
+ else ?"
1174
327
  end
1175
328
  else
1176
- # other characters
329
+ nil
1177
330
  end
1178
331
  end
1179
332
  end