irb 1.1.0 → 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -106,22 +106,12 @@ module IRB # :nodoc:
106
106
  Inspector.def_inspector([false, :to_s, :raw]){|v| v.to_s}
107
107
  Inspector.def_inspector([true, :p, :inspect]){|v|
108
108
  begin
109
- result = v.inspect
110
- if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
111
- result = Color.colorize_code(result)
112
- end
113
- result
109
+ v.inspect
114
110
  rescue NoMethodError
115
111
  puts "(Object doesn't support #inspect)"
116
112
  end
117
113
  }
118
- Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v|
119
- result = v.pretty_inspect.chomp
120
- if IRB.conf[:MAIN_CONTEXT]&.use_colorize? && Color.inspect_colorable?(v)
121
- result = Color.colorize_code(result)
122
- end
123
- result
124
- }
114
+ Inspector.def_inspector([:pp, :pretty_inspect], proc{require "pp"}){|v| v.pretty_inspect.chomp}
125
115
  Inspector.def_inspector([:yaml, :YAML], proc{require "yaml"}){|v|
126
116
  begin
127
117
  YAML.dump(v)
@@ -22,19 +22,15 @@ Usage: irb.rb [options] [programfile] [arguments]
22
22
  when new workspace was created
23
23
  --echo Show result(default)
24
24
  --noecho Don't show result
25
- --inspect Use `inspect' for output
26
- --noinspect Don't use inspect for output
27
- --multiline Use multiline editor module
28
- --nomultiline Don't use multiline editor module
29
- --singleline Use singleline editor module
30
- --nosingleline Don't use singleline editor module
31
- --colorize Use colorization
32
- --nocolorize Don't use colorization
25
+ --inspect Use `inspect' for output (default except for bc mode)
26
+ --noinspect Don't use inspect for output
27
+ --readline Use Readline extension module
28
+ --noreadline Don't use Readline extension module
33
29
  --prompt prompt-mode/--prompt-mode prompt-mode
34
30
  Switch prompt mode. Pre-defined prompt modes are
35
31
  `default', `simple', `xmp' and `inf-ruby'
36
32
  --inf-ruby-mode Use prompt appropriate for inf-ruby-mode on emacs.
37
- Suppresses --multiline and --singleline.
33
+ Suppresses --readline.
38
34
  --sample-book-mode/--simple-prompt
39
35
  Simple prompt mode
40
36
  --noprompt No prompt mode
@@ -43,6 +39,7 @@ Usage: irb.rb [options] [programfile] [arguments]
43
39
  --back-trace-limit n
44
40
  Display backtrace top n and tail n. The default
45
41
  value is 16.
42
+ --irb_debug n Set internal debug level to n (not for popular use)
46
43
  --verbose Show details
47
44
  --noverbose Don't show details
48
45
  -v, --version Print the version of irb
@@ -21,21 +21,16 @@ Usage: irb.rb [options] [programfile] [arguments]
21
21
  オブジェクトの作成方法を 0 から 3 のいずれかに設定する.
22
22
  --echo 実行結果を表示する(デフォルト).
23
23
  --noecho 実行結果を表示しない.
24
- --inspect 結果出力にinspectを用いる.
24
+ --inspect 結果出力にinspectを用いる(bcモード以外はデフォルト).
25
25
  --noinspect 結果出力にinspectを用いない.
26
- --multiline マルチラインエディタを利用する.
27
- --nomultiline マルチラインエディタを利用しない.
28
- --singleline シングルラインエディタを利用する.
29
- --nosingleline シングルラインエディタを利用しない.
30
- --colorize 色付けを利用する.
31
- --nocolorize 色付けを利用しない.
26
+ --readline readlineライブラリを利用する.
27
+ --noreadline readlineライブラリを利用しない.
32
28
  --prompt prompt-mode/--prompt-mode prompt-mode
33
29
  プロンプトモードを切替えます. 現在定義されているプ
34
30
  ロンプトモードは, default, simple, xmp, inf-rubyが
35
31
  用意されています.
36
32
  --inf-ruby-mode emacsのinf-ruby-mode用のプロンプト表示を行なう. 特
37
- に指定がない限り, シングルラインエディタとマルチラ
38
- インエディタは使わなくなる.
33
+ に指定がない限り, readlineライブラリは使わなくなる.
39
34
  --sample-book-mode/--simple-prompt
40
35
  非常にシンプルなプロンプトを用いるモードです.
41
36
  --noprompt プロンプト表示を行なわない.
@@ -46,6 +41,8 @@ Usage: irb.rb [options] [programfile] [arguments]
46
41
  バックトレース表示をバックトレースの頭から n, 後ろ
47
42
  からnだけ行なう. デフォルトは16
48
43
 
44
+ --irb_debug n irbのデバッグレベルをnに設定する(非推奨).
45
+
49
46
  --verbose 詳細なメッセージを出力する.
50
47
  --noverbose 詳細なメッセージを出力しない(デフォルト).
51
48
  -v, --version irbのバージョンを表示する.
@@ -11,48 +11,73 @@
11
11
  #
12
12
 
13
13
  require "e2mmap"
14
- require "ripper"
14
+ require_relative "slex"
15
+ require_relative "ruby-token"
15
16
 
16
17
  # :stopdoc:
17
18
  class RubyLex
18
19
 
19
20
  extend Exception2MessageMapper
21
+ def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
22
+ def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
23
+ def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
24
+ def_exception(:TkReading2TokenDuplicateError,
25
+ "key duplicate(token_n='%s', key='%s')")
26
+ def_exception(:SyntaxError, "%s")
27
+
20
28
  def_exception(:TerminateLineInput, "Terminate Line Input")
21
29
 
30
+ include RubyToken
31
+
32
+ class << self
33
+ attr_accessor :debug_level
34
+ def debug?
35
+ @debug_level > 0
36
+ end
37
+ end
38
+ @debug_level = 0
39
+
22
40
  def initialize
41
+ lex_init
42
+ set_input(STDIN)
43
+
44
+ @seek = 0
23
45
  @exp_line_no = @line_no = 1
46
+ @base_char_no = 0
47
+ @char_no = 0
48
+ @rests = []
49
+ @readed = []
50
+ @here_readed = []
51
+
24
52
  @indent = 0
53
+ @indent_stack = []
54
+ @lex_state = EXPR_BEG
55
+ @space_seen = false
56
+ @here_header = false
57
+ @post_symbeg = false
58
+
25
59
  @continue = false
26
60
  @line = ""
61
+
62
+ @skip_space = false
63
+ @readed_auto_clean_up = false
64
+ @exception_on_syntax_error = true
65
+
27
66
  @prompt = nil
28
67
  end
29
68
 
69
+ attr_accessor :skip_space
70
+ attr_accessor :readed_auto_clean_up
71
+ attr_accessor :exception_on_syntax_error
72
+
73
+ attr_reader :seek
74
+ attr_reader :char_no
75
+ attr_reader :line_no
76
+ attr_reader :indent
77
+
30
78
  # io functions
31
79
  def set_input(io, p = nil, &block)
32
80
  @io = io
33
- if @io.respond_to?(:check_termination)
34
- @io.check_termination do |code|
35
- code.gsub!(/\s*\z/, '').concat("\n")
36
- ltype, indent, continue, code_block_open = check_state(code)
37
- if ltype or indent > 0 or continue or code_block_open
38
- false
39
- else
40
- true
41
- end
42
- end
43
- end
44
- if @io.respond_to?(:dynamic_prompt)
45
- @io.dynamic_prompt do |lines|
46
- lines << '' if lines.empty?
47
- result = []
48
- lines.each_index { |i|
49
- c = lines[0..i].map{ |l| l + "\n" }.join
50
- ltype, indent, continue, code_block_open = check_state(c)
51
- result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + i)
52
- }
53
- result
54
- end
55
- end
56
81
  if p.respond_to?(:call)
57
82
  @input = p
58
83
  elsif block_given?
@@ -62,54 +87,119 @@ class RubyLex
62
87
  end
63
88
  end
64
89
 
65
- def set_prompt(p = nil, &block)
66
- p = block if block_given?
67
- if p.respond_to?(:call)
68
- @prompt = p
90
+ def get_readed
91
+ if idx = @readed.rindex("\n")
92
+ @base_char_no = @readed.size - (idx + 1)
69
93
  else
70
- @prompt = Proc.new{print p}
94
+ @base_char_no += @readed.size
71
95
  end
96
+
97
+ readed = @readed.join("")
98
+ @readed = []
99
+ readed
72
100
  end
73
101
 
74
- def ripper_lex_without_warning(code)
75
- verbose, $VERBOSE = $VERBOSE, nil
76
- tokens = Ripper.lex(code)
77
- $VERBOSE = verbose
78
- tokens
102
+ def getc
103
+ while @rests.empty?
104
+ @rests.push nil unless buf_input
105
+ end
106
+ c = @rests.shift
107
+ if @here_header
108
+ @here_readed.push c
109
+ else
110
+ @readed.push c
111
+ end
112
+ @seek += 1
113
+ if c == "\n"
114
+ @line_no += 1
115
+ @char_no = 0
116
+ else
117
+ @char_no += 1
118
+ end
119
+ c
79
120
  end
80
121
 
81
- def set_auto_indent(context)
82
- if @io.respond_to?(:auto_indent) and context.auto_indent_mode
83
- @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
84
- if is_newline
85
- md = lines[line_index - 1].match(/(\A +)/)
86
- prev_spaces = md.nil? ? 0 : md[1].count(' ')
87
- @tokens = ripper_lex_without_warning(lines[0..line_index].join("\n"))
88
- depth_difference = check_newline_depth_difference
89
- prev_spaces + depth_difference * 2
90
- else
91
- code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
92
- last_line = lines[line_index]&.byteslice(0, byte_pointer)
93
- code += last_line if last_line
94
- @tokens = ripper_lex_without_warning(code)
95
- corresponding_token_depth = check_corresponding_token_depth
96
- if corresponding_token_depth
97
- corresponding_token_depth
98
- else
99
- nil
100
- end
101
- end
122
+ def gets
123
+ l = ""
124
+ while c = getc
125
+ l.concat(c)
126
+ break if c == "\n"
127
+ end
128
+ return nil if l == "" and c.nil?
129
+ l
130
+ end
131
+
132
+ def eof?
133
+ @io.eof?
134
+ end
135
+
136
+ def getc_of_rests
137
+ if @rests.empty?
138
+ nil
139
+ else
140
+ getc
141
+ end
142
+ end
143
+
144
+ def ungetc(c = nil)
145
+ if @here_readed.empty?
146
+ c2 = @readed.pop
147
+ else
148
+ c2 = @here_readed.pop
149
+ end
150
+ c = c2 unless c
151
+ @rests.unshift c #c =
152
+ @seek -= 1
153
+ if c == "\n"
154
+ @line_no -= 1
155
+ if idx = @readed.rindex("\n")
156
+ @char_no = idx + 1
157
+ else
158
+ @char_no = @base_char_no + @readed.size
102
159
  end
160
+ else
161
+ @char_no -= 1
103
162
  end
104
163
  end
105
164
 
106
- def check_state(code)
107
- @tokens = ripper_lex_without_warning(code)
108
- ltype = process_literal_type
109
- indent = process_nesting_level
110
- continue = process_continue
111
- code_block_open = check_code_block(code)
112
- [ltype, indent, continue, code_block_open]
165
+ def peek_equal?(str)
166
+ chrs = str.split(//)
167
+ until @rests.size >= chrs.size
168
+ return false unless buf_input
169
+ end
170
+ @rests[0, chrs.size] == chrs
171
+ end
172
+
173
+ def peek_match?(regexp)
174
+ while @rests.empty?
175
+ return false unless buf_input
176
+ end
177
+ regexp =~ @rests.join("")
178
+ end
179
+
180
+ def peek(i = 0)
181
+ while @rests.size <= i
182
+ return nil unless buf_input
183
+ end
184
+ @rests[i]
185
+ end
186
+
187
+ def buf_input
188
+ prompt
189
+ line = @input.call
190
+ return nil unless line
191
+ @rests.concat line.chars.to_a
192
+ true
193
+ end
194
+ private :buf_input
195
+
196
+ def set_prompt(p = nil, &block)
197
+ p = block if block_given?
198
+ if p.respond_to?(:call)
199
+ @prompt = p
200
+ else
201
+ @prompt = Proc.new{print p}
202
+ end
113
203
  end
114
204
 
115
205
  def prompt
@@ -120,11 +210,20 @@ class RubyLex
120
210
 
121
211
  def initialize_input
122
212
  @ltype = nil
213
+ @quoted = nil
123
214
  @indent = 0
215
+ @indent_stack = []
216
+ @lex_state = EXPR_BEG
217
+ @space_seen = false
218
+ @here_header = false
219
+
124
220
  @continue = false
221
+ @post_symbeg = false
222
+
223
+ prompt
224
+
125
225
  @line = ""
126
226
  @exp_line_no = @line_no
127
- @code_block_open = false
128
227
  end
129
228
 
130
229
  def each_top_level_statement
@@ -132,14 +231,13 @@ class RubyLex
132
231
  catch(:TERM_INPUT) do
133
232
  loop do
134
233
  begin
234
+ @continue = false
135
235
  prompt
136
236
  unless l = lex
137
237
  throw :TERM_INPUT if @line == ''
138
238
  else
139
- @line_no += l.count("\n")
140
- next if l == "\n"
141
239
  @line.concat l
142
- if @code_block_open or @ltype or @continue or @indent > 0
240
+ if @ltype or @continue or @indent > 0
143
241
  next
144
242
  end
145
243
  end
@@ -147,352 +245,935 @@ class RubyLex
147
245
  @line.force_encoding(@io.encoding)
148
246
  yield @line, @exp_line_no
149
247
  end
150
- break if @io.eof?
248
+ break unless l
151
249
  @line = ''
152
250
  @exp_line_no = @line_no
153
251
 
154
252
  @indent = 0
253
+ @indent_stack = []
254
+ prompt
155
255
  rescue TerminateLineInput
156
256
  initialize_input
157
257
  prompt
258
+ get_readed
158
259
  end
159
260
  end
160
261
  end
161
262
  end
162
263
 
163
264
  def lex
164
- line = @input.call
165
- if @io.respond_to?(:check_termination)
166
- return line # multiline
167
- end
168
- code = @line + (line.nil? ? '' : line)
169
- code.gsub!(/\s*\z/, '').concat("\n")
170
- @tokens = ripper_lex_without_warning(code)
171
- @continue = process_continue
172
- @code_block_open = check_code_block(code)
173
- @indent = process_nesting_level
174
- @ltype = process_literal_type
175
- line
265
+ continue = @continue
266
+ while tk = token
267
+ case tk
268
+ when TkNL, TkEND_OF_SCRIPT
269
+ @continue = continue unless continue.nil?
270
+ break unless @continue
271
+ when TkSPACE, TkCOMMENT
272
+ when TkSEMICOLON, TkBEGIN, TkELSE
273
+ @continue = continue = false
274
+ else
275
+ continue = nil
276
+ end
277
+ end
278
+ line = get_readed
279
+ if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
280
+ nil
281
+ else
282
+ line
283
+ end
176
284
  end
177
285
 
178
- def process_continue
179
- # last token is always newline
180
- if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end
181
- # end of regexp literal
182
- return false
183
- elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon
184
- return false
185
- elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(@tokens[-2][2])
186
- return false
187
- elsif @tokens.size >= 3 and @tokens[-3][1] == :on_symbeg and @tokens[-2][1] == :on_ivar
188
- # This is for :@a or :@1 because :@1 ends with EXPR_FNAME
189
- return false
190
- elsif @tokens.size >= 2 and @tokens[-2][1] == :on_ivar and @tokens[-2][2] =~ /\A@\d+\z/
191
- # This is for @1
192
- return false
193
- elsif @tokens.size >= 2 and @tokens[-2][1] == :on_cvar and @tokens[-1][1] == :on_int
194
- # This is for @@1 or :@@1 and ends with on_int because it's syntax error
195
- return false
196
- elsif !@tokens.empty? and @tokens.last[2] == "\\\n"
197
- return true
198
- elsif @tokens.size >= 1 and @tokens[-1][1] == :on_heredoc_end # "EOH\n"
199
- return false
200
- elsif @tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and @tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME)
201
- # end of literal except for regexp
202
- return true
203
- end
204
- false
286
+ def token
287
+ @prev_seek = @seek
288
+ @prev_line_no = @line_no
289
+ @prev_char_no = @char_no
290
+ begin
291
+ begin
292
+ tk = @OP.match(self)
293
+ @space_seen = tk.kind_of?(TkSPACE)
294
+ @lex_state = EXPR_END if @post_symbeg && tk.kind_of?(TkOp)
295
+ @post_symbeg = tk.kind_of?(TkSYMBEG)
296
+ rescue SyntaxError
297
+ raise if @exception_on_syntax_error
298
+ tk = TkError.new(@seek, @line_no, @char_no)
299
+ end
300
+ end while @skip_space and tk.kind_of?(TkSPACE)
301
+ if @readed_auto_clean_up
302
+ get_readed
303
+ end
304
+ tk
205
305
  end
206
306
 
207
- def check_code_block(code)
208
- return true if @tokens.empty?
209
- if @tokens.last[1] == :on_heredoc_beg
210
- return true
307
+ ENINDENT_CLAUSE = [
308
+ "case", "class", "def", "do", "for", "if",
309
+ "module", "unless", "until", "while", "begin"
310
+ ]
311
+ DEINDENT_CLAUSE = ["end"
312
+ ]
313
+
314
+ PERCENT_LTYPE = {
315
+ "q" => "\'",
316
+ "Q" => "\"",
317
+ "x" => "\`",
318
+ "r" => "/",
319
+ "w" => "]",
320
+ "W" => "]",
321
+ "i" => "]",
322
+ "I" => "]",
323
+ "s" => ":"
324
+ }
325
+
326
+ PERCENT_PAREN = {
327
+ "{" => "}",
328
+ "[" => "]",
329
+ "<" => ">",
330
+ "(" => ")"
331
+ }
332
+
333
+ Ltype2Token = {
334
+ "\'" => TkSTRING,
335
+ "\"" => TkSTRING,
336
+ "\`" => TkXSTRING,
337
+ "/" => TkREGEXP,
338
+ "]" => TkDSTRING,
339
+ ":" => TkSYMBOL
340
+ }
341
+ DLtype2Token = {
342
+ "\"" => TkDSTRING,
343
+ "\`" => TkDXSTRING,
344
+ "/" => TkDREGEXP,
345
+ }
346
+
347
+ def lex_init()
348
+ @OP = IRB::SLex.new
349
+ @OP.def_rules("\0", "\004", "\032") do |op, io|
350
+ Token(TkEND_OF_SCRIPT)
351
+ end
352
+
353
+ @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
354
+ @space_seen = true
355
+ while getc =~ /[ \t\f\r\13]/; end
356
+ ungetc
357
+ Token(TkSPACE)
211
358
  end
212
359
 
213
- begin # check if parser error are available
214
- verbose, $VERBOSE = $VERBOSE, nil
215
- case RUBY_ENGINE
216
- when 'jruby'
217
- JRuby.compile_ir(code)
360
+ @OP.def_rule("#") do |op, io|
361
+ identify_comment
362
+ end
363
+
364
+ @OP.def_rule("=begin",
365
+ proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
366
+ |op, io|
367
+ @ltype = "="
368
+ until getc == "\n"; end
369
+ until peek_equal?("=end") && peek(4) =~ /\s/
370
+ until getc == "\n"; end
371
+ end
372
+ gets
373
+ @ltype = nil
374
+ Token(TkRD_COMMENT)
375
+ end
376
+
377
+ @OP.def_rule("\n") do |op, io|
378
+ print "\\n\n" if RubyLex.debug?
379
+ case @lex_state
380
+ when EXPR_BEG, EXPR_FNAME, EXPR_DOT
381
+ @continue = true
218
382
  else
219
- RubyVM::InstructionSequence.compile(code)
220
- end
221
- rescue SyntaxError => e
222
- case e.message
223
- when /unterminated (?:string|regexp) meets end of file/
224
- # "unterminated regexp meets end of file"
225
- #
226
- # example:
227
- # /
228
- #
229
- # "unterminated string meets end of file"
230
- #
231
- # example:
232
- # '
233
- return true
234
- when /syntax error, unexpected end-of-input/
235
- # "syntax error, unexpected end-of-input, expecting keyword_end"
236
- #
237
- # example:
238
- # if ture
239
- # hoge
240
- # if false
241
- # fuga
242
- # end
243
- return true
244
- when /syntax error, unexpected keyword_end/
245
- # "syntax error, unexpected keyword_end"
246
- #
247
- # example:
248
- # if (
249
- # end
250
- #
251
- # example:
252
- # end
253
- return false
254
- when /syntax error, unexpected '\.'/
255
- # "syntax error, unexpected '.'"
256
- #
257
- # example:
258
- # .
259
- return false
260
- when /unexpected tREGEXP_BEG/
261
- # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
262
- #
263
- # example:
264
- # method / f /
265
- return false
266
- when /numbered parameter outside block/
267
- # "numbered parameter outside block"
268
- #
269
- # example:
270
- # :@1
271
- return false
383
+ @continue = false
384
+ @lex_state = EXPR_BEG
385
+ until (@indent_stack.empty? ||
386
+ [TkLPAREN, TkLBRACK, TkLBRACE,
387
+ TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
388
+ @indent_stack.pop
389
+ end
272
390
  end
273
- ensure
274
- $VERBOSE = verbose
391
+ @here_header = false
392
+ @here_readed = []
393
+ Token(TkNL)
275
394
  end
276
395
 
277
- if defined?(Ripper::EXPR_BEG)
278
- last_lex_state = @tokens.last[3]
279
- if last_lex_state.allbits?(Ripper::EXPR_BEG)
280
- return false
281
- elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
282
- return true
283
- elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
284
- return true
285
- elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
286
- return true
287
- elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
288
- return true
289
- elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
290
- return false
396
+ @OP.def_rules("*", "**",
397
+ "=", "==", "===",
398
+ "=~", "<=>",
399
+ "<", "<=",
400
+ ">", ">=", ">>",
401
+ "!", "!=", "!~") do
402
+ |op, io|
403
+ case @lex_state
404
+ when EXPR_FNAME, EXPR_DOT
405
+ @lex_state = EXPR_ARG
406
+ else
407
+ @lex_state = EXPR_BEG
291
408
  end
409
+ Token(op)
292
410
  end
293
411
 
294
- false
295
- end
412
+ @OP.def_rules("<<") do
413
+ |op, io|
414
+ tk = nil
415
+ if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
416
+ (@lex_state != EXPR_ARG || @space_seen)
417
+ c = peek(0)
418
+ if /[-~"'`\w]/ =~ c
419
+ tk = identify_here_document
420
+ end
421
+ end
422
+ unless tk
423
+ tk = Token(op)
424
+ case @lex_state
425
+ when EXPR_FNAME, EXPR_DOT
426
+ @lex_state = EXPR_ARG
427
+ else
428
+ @lex_state = EXPR_BEG
429
+ end
430
+ end
431
+ tk
432
+ end
433
+
434
+ @OP.def_rules("'", '"') do
435
+ |op, io|
436
+ identify_string(op)
437
+ end
438
+
439
+ @OP.def_rules("`") do
440
+ |op, io|
441
+ if @lex_state == EXPR_FNAME
442
+ @lex_state = EXPR_END
443
+ Token(op)
444
+ else
445
+ identify_string(op)
446
+ end
447
+ end
448
+
449
+ @OP.def_rules('?') do
450
+ |op, io|
451
+ if @lex_state == EXPR_END
452
+ @lex_state = EXPR_BEG
453
+ Token(TkQUESTION)
454
+ else
455
+ ch = getc
456
+ if @lex_state == EXPR_ARG && ch =~ /\s/
457
+ ungetc
458
+ @lex_state = EXPR_BEG;
459
+ Token(TkQUESTION)
460
+ else
461
+ if (ch == '\\')
462
+ read_escape
463
+ end
464
+ @lex_state = EXPR_END
465
+ Token(TkINTEGER)
466
+ end
467
+ end
468
+ end
469
+
470
+ @OP.def_rules("&", "&&", "|", "||") do
471
+ |op, io|
472
+ @lex_state = EXPR_BEG
473
+ Token(op)
474
+ end
475
+
476
+ @OP.def_rules("+=", "-=", "*=", "**=",
477
+ "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
478
+ |op, io|
479
+ @lex_state = EXPR_BEG
480
+ op =~ /^(.*)=$/
481
+ Token(TkOPASGN, $1)
482
+ end
296
483
 
297
- def process_nesting_level
298
- indent = 0
299
- @tokens.each_with_index { |t, index|
300
- case t[1]
301
- when :on_lbracket, :on_lbrace, :on_lparen
302
- indent += 1
303
- when :on_rbracket, :on_rbrace, :on_rparen
304
- indent -= 1
305
- when :on_kw
306
- next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
307
- case t[2]
308
- when 'do'
309
- if index > 0 and @tokens[index - 1][3].anybits?(Ripper::EXPR_CMDARG | Ripper::EXPR_ENDFN)
310
- # method_with_block do; end
311
- indent += 1
484
+ @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
485
+ |op, io|
486
+ @lex_state = EXPR_ARG
487
+ Token(op)
488
+ end
489
+
490
+ @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
491
+ |op, io|
492
+ @lex_state = EXPR_ARG
493
+ Token(op)
494
+ end
495
+
496
+ @OP.def_rules("+", "-") do
497
+ |op, io|
498
+ catch(:RET) do
499
+ if @lex_state == EXPR_ARG
500
+ if @space_seen and peek(0) =~ /[0-9]/
501
+ throw :RET, identify_number
312
502
  else
313
- # while cond do; end # also "until" or "for"
314
- # This "do" doesn't increment indent because "while" already
315
- # incremented.
503
+ @lex_state = EXPR_BEG
316
504
  end
317
- when 'def', 'case', 'for', 'begin', 'class', 'module'
318
- indent += 1
319
- when 'if', 'unless', 'while', 'until'
320
- # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
321
- indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
322
- when 'end'
323
- indent -= 1
505
+ elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
506
+ throw :RET, identify_number
507
+ else
508
+ @lex_state = EXPR_BEG
324
509
  end
510
+ Token(op)
325
511
  end
326
- # percent literals are not indented
327
- }
328
- indent
512
+ end
513
+
514
+ @OP.def_rule(".") do
515
+ |op, io|
516
+ @lex_state = EXPR_BEG
517
+ if peek(0) =~ /[0-9]/
518
+ ungetc
519
+ identify_number
520
+ else
521
+ # for "obj.if" etc.
522
+ @lex_state = EXPR_DOT
523
+ Token(TkDOT)
524
+ end
525
+ end
526
+
527
+ @OP.def_rules("..", "...") do
528
+ |op, io|
529
+ @lex_state = EXPR_BEG
530
+ Token(op)
531
+ end
532
+
533
+ lex_int2
329
534
  end
330
535
 
331
- def check_newline_depth_difference
332
- depth_difference = 0
333
- @tokens.each_with_index do |t, index|
334
- case t[1]
335
- when :on_ignored_nl, :on_nl, :on_comment
336
- if index != (@tokens.size - 1)
337
- depth_difference = 0
536
+ def lex_int2
537
+ @OP.def_rules("]", "}", ")") do
538
+ |op, io|
539
+ @lex_state = EXPR_END
540
+ @indent -= 1
541
+ @indent_stack.pop
542
+ Token(op)
543
+ end
544
+
545
+ @OP.def_rule(":") do
546
+ |op, io|
547
+ if @lex_state == EXPR_END || peek(0) =~ /\s/
548
+ @lex_state = EXPR_BEG
549
+ Token(TkCOLON)
550
+ else
551
+ @lex_state = EXPR_FNAME
552
+ Token(TkSYMBEG)
553
+ end
554
+ end
555
+
556
+ @OP.def_rule("::") do
557
+ |op, io|
558
+ if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
559
+ @lex_state = EXPR_BEG
560
+ Token(TkCOLON3)
561
+ else
562
+ @lex_state = EXPR_DOT
563
+ Token(TkCOLON2)
564
+ end
565
+ end
566
+
567
+ @OP.def_rule("/") do
568
+ |op, io|
569
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
570
+ identify_string(op)
571
+ elsif peek(0) == '='
572
+ getc
573
+ @lex_state = EXPR_BEG
574
+ Token(TkOPASGN, "/") #/)
575
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
576
+ identify_string(op)
577
+ else
578
+ @lex_state = EXPR_BEG
579
+ Token("/") #/)
580
+ end
581
+ end
582
+
583
+ @OP.def_rules("^") do
584
+ |op, io|
585
+ @lex_state = EXPR_BEG
586
+ Token("^")
587
+ end
588
+
589
+ @OP.def_rules(",") do
590
+ |op, io|
591
+ @lex_state = EXPR_BEG
592
+ Token(op)
593
+ end
594
+
595
+ @OP.def_rules(";") do
596
+ |op, io|
597
+ @lex_state = EXPR_BEG
598
+ until (@indent_stack.empty? ||
599
+ [TkLPAREN, TkLBRACK, TkLBRACE,
600
+ TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
601
+ @indent_stack.pop
602
+ end
603
+ Token(op)
604
+ end
605
+
606
+ @OP.def_rule("~") do
607
+ |op, io|
608
+ @lex_state = EXPR_BEG
609
+ Token("~")
610
+ end
611
+
612
+ @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
613
+ |op, io|
614
+ @lex_state = EXPR_BEG
615
+ Token("~")
616
+ end
617
+
618
+ @OP.def_rule("(") do
619
+ |op, io|
620
+ @indent += 1
621
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
622
+ @lex_state = EXPR_BEG
623
+ tk_c = TkfLPAREN
624
+ else
625
+ @lex_state = EXPR_BEG
626
+ tk_c = TkLPAREN
627
+ end
628
+ @indent_stack.push tk_c
629
+ Token(tk_c)
630
+ end
631
+
632
+ @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
633
+ |op, io|
634
+ @lex_state = EXPR_ARG
635
+ Token("[]")
636
+ end
637
+
638
+ @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
639
+ |op, io|
640
+ @lex_state = EXPR_ARG
641
+ Token("[]=")
642
+ end
643
+
644
+ @OP.def_rule("[") do
645
+ |op, io|
646
+ @indent += 1
647
+ if @lex_state == EXPR_FNAME
648
+ tk_c = TkfLBRACK
649
+ else
650
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
651
+ tk_c = TkLBRACK
652
+ elsif @lex_state == EXPR_ARG && @space_seen
653
+ tk_c = TkLBRACK
654
+ else
655
+ tk_c = TkfLBRACK
338
656
  end
339
- next
340
- when :on_sp
341
- next
342
- end
343
- case t[1]
344
- when :on_lbracket, :on_lbrace, :on_lparen
345
- depth_difference += 1
346
- when :on_rbracket, :on_rbrace, :on_rparen
347
- depth_difference -= 1
348
- when :on_kw
349
- next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
350
- case t[2]
351
- when 'do'
352
- if index > 0 and @tokens[index - 1][3].anybits?(Ripper::EXPR_CMDARG | Ripper::EXPR_ENDFN)
353
- # method_with_block do; end
354
- depth_difference += 1
657
+ @lex_state = EXPR_BEG
658
+ end
659
+ @indent_stack.push tk_c
660
+ Token(tk_c)
661
+ end
662
+
663
+ @OP.def_rule("{") do
664
+ |op, io|
665
+ @indent += 1
666
+ if @lex_state != EXPR_END && @lex_state != EXPR_ARG
667
+ tk_c = TkLBRACE
668
+ else
669
+ tk_c = TkfLBRACE
670
+ end
671
+ @lex_state = EXPR_BEG
672
+ @indent_stack.push tk_c
673
+ Token(tk_c)
674
+ end
675
+
676
+ @OP.def_rule('\\') do
677
+ |op, io|
678
+ if getc == "\n"
679
+ @space_seen = true
680
+ @continue = true
681
+ Token(TkSPACE)
682
+ else
683
+ read_escape
684
+ Token("\\")
685
+ end
686
+ end
687
+
688
+ @OP.def_rule('%') do
689
+ |op, io|
690
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
691
+ identify_quotation
692
+ elsif peek(0) == '='
693
+ getc
694
+ Token(TkOPASGN, :%)
695
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
696
+ identify_quotation
697
+ else
698
+ @lex_state = EXPR_BEG
699
+ Token("%") #))
700
+ end
701
+ end
702
+
703
+ @OP.def_rule('$') do
704
+ |op, io|
705
+ identify_gvar
706
+ end
707
+
708
+ @OP.def_rule('@') do
709
+ |op, io|
710
+ if peek(0) =~ /[\w@]/
711
+ ungetc
712
+ identify_identifier
713
+ else
714
+ Token("@")
715
+ end
716
+ end
717
+
718
+ @OP.def_rule("") do
719
+ |op, io|
720
+ printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
721
+ if peek(0) =~ /[0-9]/
722
+ t = identify_number
723
+ elsif peek(0) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
724
+ t = identify_identifier
725
+ end
726
+ printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
727
+ t
728
+ end
729
+
730
+ p @OP if RubyLex.debug?
731
+ end
732
+
733
+ def identify_gvar
734
+ @lex_state = EXPR_END
735
+
736
+ case ch = getc
737
+ when /[~_*$?!@\/\\;,=:<>".]/ #"
738
+ Token(TkGVAR, "$" + ch)
739
+ when "-"
740
+ Token(TkGVAR, "$-" + getc)
741
+ when "&", "`", "'", "+"
742
+ Token(TkBACK_REF, "$"+ch)
743
+ when /[1-9]/
744
+ while getc =~ /[0-9]/; end
745
+ ungetc
746
+ Token(TkNTH_REF)
747
+ when /\w/
748
+ ungetc
749
+ ungetc
750
+ identify_identifier
751
+ else
752
+ ungetc
753
+ Token("$")
754
+ end
755
+ end
756
+
757
+ def identify_identifier
758
+ token = ""
759
+ if peek(0) =~ /[$@]/
760
+ token.concat(c = getc)
761
+ if c == "@" and peek(0) == "@"
762
+ token.concat getc
763
+ end
764
+ end
765
+
766
+ while (ch = getc) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
767
+ print ":", ch, ":" if RubyLex.debug?
768
+ token.concat ch
769
+ end
770
+ ungetc
771
+
772
+ if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
773
+ token.concat getc
774
+ end
775
+
776
+ # almost fix token
777
+
778
+ case token
779
+ when /^\$/
780
+ return Token(TkGVAR, token)
781
+ when /^\@\@/
782
+ @lex_state = EXPR_END
783
+ # p Token(TkCVAR, token)
784
+ return Token(TkCVAR, token)
785
+ when /^\@/
786
+ @lex_state = EXPR_END
787
+ return Token(TkIVAR, token)
788
+ end
789
+
790
+ if @lex_state != EXPR_DOT
791
+ print token, "\n" if RubyLex.debug?
792
+
793
+ token_c, *trans = TkReading2Token[token]
794
+ if token_c
795
+ # reserved word?
796
+
797
+ if (@lex_state != EXPR_BEG &&
798
+ @lex_state != EXPR_FNAME &&
799
+ trans[1])
800
+ # modifiers
801
+ token_c = TkSymbol2Token[trans[1]]
802
+ @lex_state = trans[0]
803
+ else
804
+ if @lex_state != EXPR_FNAME and peek(0) != ':'
805
+ if ENINDENT_CLAUSE.include?(token)
806
+ # check for ``class = val'' etc.
807
+ valid = true
808
+ case token
809
+ when "class"
810
+ valid = false unless peek_match?(/^\s*(<<|\w|::)/)
811
+ when "def"
812
+ valid = false if peek_match?(/^\s*(([+\-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
813
+ when "do"
814
+ valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&)/)
815
+ when *ENINDENT_CLAUSE
816
+ valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&|\|)/)
817
+ else
818
+ # no nothing
819
+ end
820
+ if valid
821
+ if token == "do"
822
+ if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
823
+ @indent += 1
824
+ @indent_stack.push token_c
825
+ end
826
+ else
827
+ @indent += 1
828
+ @indent_stack.push token_c
829
+ end
830
+ end
831
+
832
+ elsif DEINDENT_CLAUSE.include?(token)
833
+ @indent -= 1
834
+ @indent_stack.pop
835
+ end
836
+ @lex_state = trans[0]
355
837
  else
356
- # while cond do; end # also "until" or "for"
357
- # This "do" doesn't increment indent because "while" already
358
- # incremented.
838
+ @lex_state = EXPR_END
359
839
  end
360
- when 'def', 'case', 'for', 'begin', 'class', 'module'
361
- depth_difference += 1
362
- when 'if', 'unless', 'while', 'until'
363
- # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
364
- unless t[3].allbits?(Ripper::EXPR_LABEL)
365
- depth_difference += 1
366
- end
367
- when 'else', 'elsif', 'rescue', 'ensure', 'when', 'in'
368
- depth_difference += 1
369
840
  end
841
+ return Token(token_c, token)
842
+ end
843
+ end
844
+
845
+ if @lex_state == EXPR_FNAME
846
+ @lex_state = EXPR_END
847
+ if peek(0) == '='
848
+ token.concat getc
370
849
  end
850
+ elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
851
+ @lex_state = EXPR_ARG
852
+ else
853
+ @lex_state = EXPR_END
854
+ end
855
+
856
+ if token[0, 1] =~ /[A-Z]/
857
+ return Token(TkCONSTANT, token)
858
+ elsif token[token.size - 1, 1] =~ /[!?]/
859
+ return Token(TkFID, token)
860
+ else
861
+ return Token(TkIDENTIFIER, token)
371
862
  end
372
- depth_difference
373
863
  end
374
864
 
375
- def check_corresponding_token_depth
376
- corresponding_token_depth = nil
377
- is_first_spaces_of_line = true
378
- is_first_printable_of_line = true
379
- spaces_of_nest = []
380
- spaces_at_line_head = 0
381
- @tokens.each_with_index do |t, index|
382
- corresponding_token_depth = nil
383
- case t[1]
384
- when :on_ignored_nl, :on_nl, :on_comment
385
- spaces_at_line_head = 0
386
- is_first_spaces_of_line = true
387
- is_first_printable_of_line = true
388
- next
389
- when :on_sp
390
- spaces_at_line_head = t[2].count(' ') if is_first_spaces_of_line
391
- is_first_spaces_of_line = false
392
- next
393
- end
394
- case t[1]
395
- when :on_lbracket, :on_lbrace, :on_lparen
396
- spaces_of_nest.push(spaces_at_line_head)
397
- when :on_rbracket, :on_rbrace, :on_rparen
398
- if is_first_printable_of_line
399
- corresponding_token_depth = spaces_of_nest.pop
400
- else
401
- spaces_of_nest.pop
402
- corresponding_token_depth = nil
865
+ def identify_here_document
866
+ ch = getc
867
+ if ch == "-" || ch == "~"
868
+ ch = getc
869
+ indent = true
870
+ end
871
+ if /['"`]/ =~ ch
872
+ lt = ch
873
+ quoted = ""
874
+ while (c = getc) && c != lt
875
+ quoted.concat c
876
+ end
877
+ else
878
+ lt = '"'
879
+ quoted = ch.dup
880
+ while (c = getc) && c =~ /\w/
881
+ quoted.concat c
882
+ end
883
+ ungetc
884
+ end
885
+
886
+ ltback, @ltype = @ltype, lt
887
+ reserve = []
888
+ while ch = getc
889
+ reserve.push ch
890
+ if ch == "\\"
891
+ reserve.push ch = getc
892
+ elsif ch == "\n"
893
+ break
894
+ end
895
+ end
896
+
897
+ @here_header = false
898
+
899
+ line = ""
900
+ while ch = getc
901
+ if ch == "\n"
902
+ if line == quoted
903
+ break
904
+ end
905
+ line = ""
906
+ else
907
+ line.concat ch unless indent && line == "" && /\s/ =~ ch
908
+ if @ltype != "'" && ch == "#" && peek(0) == "{"
909
+ identify_string_dvar
403
910
  end
404
- when :on_kw
405
- next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
406
- case t[2]
407
- when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
408
- spaces_of_nest.push(spaces_at_line_head)
409
- when 'if', 'unless', 'while', 'until'
410
- # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
411
- unless t[3].allbits?(Ripper::EXPR_LABEL)
412
- spaces_of_nest.push(spaces_at_line_head)
911
+ end
912
+ end
913
+
914
+ @here_header = true
915
+ @here_readed.concat reserve
916
+ while ch = reserve.pop
917
+ ungetc ch
918
+ end
919
+
920
+ @ltype = ltback
921
+ @lex_state = EXPR_END
922
+ Token(Ltype2Token[lt])
923
+ end
924
+
925
+ def identify_quotation
926
+ ch = getc
927
+ if lt = PERCENT_LTYPE[ch]
928
+ ch = getc
929
+ elsif ch =~ /\W/
930
+ lt = "\""
931
+ else
932
+ RubyLex.fail SyntaxError, "unknown type of %string"
933
+ end
934
+ @quoted = ch unless @quoted = PERCENT_PAREN[ch]
935
+ identify_string(lt, @quoted)
936
+ end
937
+
938
+ def identify_number
939
+ @lex_state = EXPR_END
940
+
941
+ if peek(0) == "0" && peek(1) !~ /[.eE]/
942
+ getc
943
+ case peek(0)
944
+ when /[xX]/
945
+ ch = getc
946
+ match = /[0-9a-fA-F_]/
947
+ when /[bB]/
948
+ ch = getc
949
+ match = /[01_]/
950
+ when /[oO]/
951
+ ch = getc
952
+ match = /[0-7_]/
953
+ when /[dD]/
954
+ ch = getc
955
+ match = /[0-9_]/
956
+ when /[0-7]/
957
+ match = /[0-7_]/
958
+ when /[89]/
959
+ RubyLex.fail SyntaxError, "Invalid octal digit"
960
+ else
961
+ return Token(TkINTEGER)
962
+ end
963
+
964
+ len0 = true
965
+ non_digit = false
966
+ while ch = getc
967
+ if match =~ ch
968
+ if ch == "_"
969
+ if non_digit
970
+ RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
971
+ else
972
+ non_digit = ch
973
+ end
974
+ else
975
+ non_digit = false
976
+ len0 = false
977
+ end
978
+ else
979
+ ungetc
980
+ if len0
981
+ RubyLex.fail SyntaxError, "numeric literal without digits"
982
+ end
983
+ if non_digit
984
+ RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
413
985
  end
414
- when 'else', 'elsif', 'rescue', 'ensure', 'when', 'in'
415
- corresponding_token_depth = spaces_of_nest.last
416
- when 'end'
417
- if is_first_printable_of_line
418
- corresponding_token_depth = spaces_of_nest.pop
986
+ break
987
+ end
988
+ end
989
+ return Token(TkINTEGER)
990
+ end
991
+
992
+ type = TkINTEGER
993
+ allow_point = true
994
+ allow_e = true
995
+ non_digit = false
996
+ while ch = getc
997
+ case ch
998
+ when /[0-9]/
999
+ non_digit = false
1000
+ when "_"
1001
+ non_digit = ch
1002
+ when allow_point && "."
1003
+ if non_digit
1004
+ RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1005
+ end
1006
+ type = TkFLOAT
1007
+ if peek(0) !~ /[0-9]/
1008
+ type = TkINTEGER
1009
+ ungetc
1010
+ break
1011
+ end
1012
+ allow_point = false
1013
+ when allow_e && "e", allow_e && "E"
1014
+ if non_digit
1015
+ RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016
+ end
1017
+ type = TkFLOAT
1018
+ if peek(0) =~ /[+-]/
1019
+ getc
1020
+ end
1021
+ allow_e = false
1022
+ allow_point = false
1023
+ non_digit = ch
1024
+ else
1025
+ if non_digit
1026
+ RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027
+ end
1028
+ ungetc
1029
+ break
1030
+ end
1031
+ end
1032
+ Token(type)
1033
+ end
1034
+
1035
+ def identify_string(ltype, quoted = ltype)
1036
+ @ltype = ltype
1037
+ @quoted = quoted
1038
+ subtype = nil
1039
+ begin
1040
+ nest = 0
1041
+ while ch = getc
1042
+ if @quoted == ch and nest == 0
1043
+ break
1044
+ elsif @ltype != "'" && ch == "#" && peek(0) == "{"
1045
+ identify_string_dvar
1046
+ elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1047
+ subtype = true
1048
+ elsif ch == '\\' and @ltype == "'" #'
1049
+ case ch = getc
1050
+ when "\\", "\n", "'"
419
1051
  else
420
- spaces_of_nest.pop
421
- corresponding_token_depth = nil
1052
+ ungetc
1053
+ end
1054
+ elsif ch == '\\' #'
1055
+ read_escape
1056
+ end
1057
+ if PERCENT_PAREN.values.include?(@quoted)
1058
+ if PERCENT_PAREN[ch] == @quoted
1059
+ nest += 1
1060
+ elsif ch == @quoted
1061
+ nest -= 1
422
1062
  end
423
1063
  end
424
1064
  end
425
- is_first_spaces_of_line = false
426
- is_first_printable_of_line = false
1065
+ if @ltype == "/"
1066
+ while /[imxoesun]/ =~ peek(0)
1067
+ getc
1068
+ end
1069
+ end
1070
+ if subtype
1071
+ Token(DLtype2Token[ltype])
1072
+ else
1073
+ Token(Ltype2Token[ltype])
1074
+ end
1075
+ ensure
1076
+ @ltype = nil
1077
+ @quoted = nil
1078
+ @lex_state = EXPR_END
427
1079
  end
428
- corresponding_token_depth
429
1080
  end
430
1081
 
431
- def check_string_literal
432
- i = 0
433
- start_token = []
434
- end_type = []
435
- while i < @tokens.size
436
- t = @tokens[i]
437
- case t[1]
438
- when :on_tstring_beg
439
- start_token << t
440
- end_type << [:on_tstring_end, :on_label_end]
441
- when :on_regexp_beg
442
- start_token << t
443
- end_type << :on_regexp_end
444
- when :on_symbeg
445
- acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw}
446
- if (i + 1) < @tokens.size and acceptable_single_tokens.all?{ |t| @tokens[i + 1][1] != t }
447
- start_token << t
448
- end_type << :on_tstring_end
1082
+ def identify_string_dvar
1083
+ begin
1084
+ getc
1085
+
1086
+ reserve_continue = @continue
1087
+ reserve_ltype = @ltype
1088
+ reserve_indent = @indent
1089
+ reserve_indent_stack = @indent_stack
1090
+ reserve_state = @lex_state
1091
+ reserve_quoted = @quoted
1092
+
1093
+ @ltype = nil
1094
+ @quoted = nil
1095
+ @indent = 0
1096
+ @indent_stack = []
1097
+ @lex_state = EXPR_BEG
1098
+
1099
+ loop do
1100
+ @continue = false
1101
+ prompt
1102
+ tk = token
1103
+ if @ltype or @continue or @indent >= 0
1104
+ next
449
1105
  end
450
- when :on_backtick
451
- start_token << t
452
- end_type << :on_tstring_end
453
- when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
454
- start_token << t
455
- end_type << :on_tstring_end
456
- when :on_heredoc_beg
457
- start_token << t
458
- end_type << :on_heredoc_end
459
- when *end_type.last
460
- start_token.pop
461
- end_type.pop
462
- end
463
- i += 1
464
- end
465
- start_token.last.nil? ? '' : start_token.last
1106
+ break if tk.kind_of?(TkRBRACE)
1107
+ end
1108
+ ensure
1109
+ @continue = reserve_continue
1110
+ @ltype = reserve_ltype
1111
+ @indent = reserve_indent
1112
+ @indent_stack = reserve_indent_stack
1113
+ @lex_state = reserve_state
1114
+ @quoted = reserve_quoted
1115
+ end
466
1116
  end
467
1117
 
468
- def process_literal_type
469
- start_token = check_string_literal
470
- case start_token[1]
471
- when :on_tstring_beg
472
- case start_token[2]
473
- when ?" then ?"
474
- when /^%.$/ then ?"
475
- when /^%Q.$/ then ?"
476
- when ?' then ?'
477
- when /^%q.$/ then ?'
478
- end
479
- when :on_regexp_beg then ?/
480
- when :on_symbeg then ?:
481
- when :on_backtick then ?`
482
- when :on_qwords_beg then ?]
483
- when :on_words_beg then ?]
484
- when :on_qsymbols_beg then ?]
485
- when :on_symbols_beg then ?]
486
- when :on_heredoc_beg
487
- start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
488
- case $1
489
- when ?" then ?"
490
- when ?' then ?'
491
- when ?` then ?`
492
- else ?"
1118
+ def identify_comment
1119
+ @ltype = "#"
1120
+
1121
+ while ch = getc
1122
+ if ch == "\n"
1123
+ @ltype = nil
1124
+ ungetc
1125
+ break
1126
+ end
1127
+ end
1128
+ return Token(TkCOMMENT)
1129
+ end
1130
+
1131
+ def read_escape
1132
+ case ch = getc
1133
+ when "\n", "\r", "\f"
1134
+ when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1135
+ when /[0-7]/
1136
+ ungetc ch
1137
+ 3.times do
1138
+ case ch = getc
1139
+ when /[0-7]/
1140
+ when nil
1141
+ break
1142
+ else
1143
+ ungetc
1144
+ break
1145
+ end
1146
+ end
1147
+
1148
+ when "x"
1149
+ 2.times do
1150
+ case ch = getc
1151
+ when /[0-9a-fA-F]/
1152
+ when nil
1153
+ break
1154
+ else
1155
+ ungetc
1156
+ break
1157
+ end
1158
+ end
1159
+
1160
+ when "M"
1161
+ if (ch = getc) != '-'
1162
+ ungetc
1163
+ else
1164
+ if (ch = getc) == "\\" #"
1165
+ read_escape
1166
+ end
1167
+ end
1168
+
1169
+ when "C", "c" #, "^"
1170
+ if ch == "C" and (ch = getc) != "-"
1171
+ ungetc
1172
+ elsif (ch = getc) == "\\" #"
1173
+ read_escape
493
1174
  end
494
1175
  else
495
- nil
1176
+ # other characters
496
1177
  end
497
1178
  end
498
1179
  end