yard 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.

This version of yard might be problematic. Click here for more details.

Files changed (23) hide show
  1. data/bin/yardoc +30 -7
  2. data/lib/code_object.rb +16 -19
  3. data/lib/extra.rb +8 -0
  4. data/lib/formatter.rb +30 -18
  5. data/lib/handlers/attribute_handler.rb +9 -4
  6. data/lib/handlers/class_handler.rb +3 -2
  7. data/lib/handlers/constant_handler.rb +1 -1
  8. data/lib/handlers/exception_handler.rb +1 -1
  9. data/lib/handlers/method_handler.rb +2 -1
  10. data/lib/handlers/mixin_handler.rb +7 -1
  11. data/lib/handlers/module_handler.rb +1 -1
  12. data/lib/handlers/yield_handler.rb +24 -22
  13. data/lib/logger.rb +19 -0
  14. data/lib/ruby_lex.rb +1116 -1113
  15. data/lib/source_parser.rb +12 -5
  16. data/lib/tag_library.rb +24 -11
  17. data/lib/yard.rb +6 -1
  18. data/templates/default/html/_fulldoc.erb +64 -0
  19. data/templates/default/html/class.erb +226 -0
  20. data/templates/default/html/method.erb +20 -0
  21. data/templates/default/html/module.erb +126 -0
  22. metadata +66 -44
  23. data/templates/html_formatter.erb +0 -455
data/bin/yardoc CHANGED
@@ -1,9 +1,9 @@
1
- #!/usr/bin/ruby
1
+ #!/usr/bin/env ruby
2
2
  require File.dirname(__FILE__) + '/../lib/yard'
3
3
  include YARD
4
4
 
5
5
  if ARGV[0] == "-h"
6
- puts "yardoc 0.2.0"
6
+ puts "yardoc #{VERSION}"
7
7
  return
8
8
  end
9
9
 
@@ -18,6 +18,15 @@ ac.puts <<-eof
18
18
  <body>
19
19
  <h3>All Classes</h3>
20
20
  eof
21
+ am = File.open("doc/all-modules.html", "w")
22
+ am.puts <<-eof
23
+ <html>
24
+ <head>
25
+ <base target="main" />
26
+ </head>
27
+ <body>
28
+ <h3>All Modules</h3>
29
+ eof
21
30
  meths = []
22
31
  Namespace.all.sort.each do |path|
23
32
  object = Namespace.at(path)
@@ -25,12 +34,25 @@ Namespace.all.sort.each do |path|
25
34
  meths << [object.name, object]
26
35
  end
27
36
 
28
- next unless object.is_a? ClassObject
29
- ac.puts "<a href='" + path.gsub("::","_") + ".html'>" + path + "</a><br />"
30
- File.open("doc/#{path.gsub('::','_')}.html", "w") {|f| f.write(object.format) }
37
+ indexfile = nil
38
+
39
+ case object
40
+ when ClassObject
41
+ indexfile = ac
42
+ when ModuleObject
43
+ indexfile = am
44
+ else
45
+ next
46
+ end
47
+
48
+ indexfile.puts "<a href='" + path.gsub("::","_") + ".html'>" + path + "</a><br />"
49
+ puts "Generating " + (docfile = "doc/#{path.gsub('::','_')}.html") + "..."
50
+ File.open(docfile, "w") {|f| f.write(object.to_s) }
31
51
  end
32
52
  ac.puts "</body></html>"
33
53
  ac.close
54
+ am.puts "</body></html>"
55
+ am.close
34
56
 
35
57
  File.open("doc/all-methods.html", "w") do |f|
36
58
  f.puts <<-eof
@@ -40,7 +62,7 @@ File.open("doc/all-methods.html", "w") do |f|
40
62
  </head>
41
63
  <body>
42
64
  <h3>All Methods</h3>
43
- eof
65
+ eof
44
66
  meths.sort {|a,b| a.first <=> b.first }.each do |name, object|
45
67
  f.puts "<a href='" + object.parent.path.gsub("::", "_") + ".html##{object.scope}_method-#{name}'>#{name}</a><br />"
46
68
  end
@@ -62,8 +84,9 @@ File.open("doc/index.html", "w") do |f|
62
84
  <title>Ruby Classes</title>
63
85
  </head>
64
86
  <frameset cols="250,*">
65
- <frameset rows="*,40%">
87
+ <frameset rows="40%,40%,20%">
66
88
  <frame src="all-classes.html">
89
+ <frame src="all-modules.html">
67
90
  <frame src="all-methods.html">
68
91
  </frameset>
69
92
  <frame name="main" src="#{main_page}">
data/lib/code_object.rb CHANGED
@@ -8,6 +8,9 @@ module YARD #:nodoc:
8
8
  #
9
9
  # @author Loren Segal
10
10
  class CodeObject
11
+ SCOPES = [:class, :instance]
12
+ VISIBILITIES = [:private, :protected, :public]
13
+
11
14
  attr_reader :source, :full_source, :file, :line, :docstring, :attributes
12
15
 
13
16
  attr_reader :name, :type
@@ -42,10 +45,6 @@ module YARD #:nodoc:
42
45
  yield(self) if block_given?
43
46
  end
44
47
 
45
- def to_s
46
- "#{visibility} #{type} #{path}"
47
- end
48
-
49
48
  ##
50
49
  # Attaches source code to a code object with an optional file location
51
50
  #
@@ -132,7 +131,7 @@ module YARD #:nodoc:
132
131
  #
133
132
  #
134
133
  def path
135
- [(parent.path if parent && parent.type != :root), name].join(scope == :instance ? "#" : "::").gsub(/^::/, '')
134
+ [(parent.path if parent && parent.type != :root), name].join(scope == :instance ? "#" : "::").gsub(/^::/, '').trim
136
135
  end
137
136
 
138
137
  ##
@@ -170,16 +169,7 @@ module YARD #:nodoc:
170
169
  name = name.to_s
171
170
  @tags.any? {|tag| tag.tag_name == name }
172
171
  end
173
-
174
- ##
175
- # Returns a code object formatted as a given type, defaults to html.
176
- #
177
- # @param [Symbol] format the output format to generate
178
- # @return [String] the code object formatted by the specified +format+
179
- def format(type = :html)
180
- Formatter.new.format(self, type)
181
- end
182
-
172
+
183
173
  private
184
174
  ##
185
175
  # Parses out comments split by newlines into a new code object
@@ -231,6 +221,7 @@ module YARD #:nodoc:
231
221
  class CodeObjectWithMethods < CodeObject
232
222
  def initialize(name, type, parent = nil, comments = nil)
233
223
  super(name, type, :public, :class, parent, comments) do |obj|
224
+ obj[:attributes] = {}
234
225
  obj[:instance_methods] = {}
235
226
  obj[:class_methods] = {}
236
227
  obj[:constants] = {}
@@ -265,6 +256,11 @@ module YARD #:nodoc:
265
256
  yield(obj) if block_given?
266
257
  end
267
258
  end
259
+
260
+ def superclasses
261
+ #STDERR.puts "Warning: someone expected module #{path} to respond to #superclasses"
262
+ []
263
+ end
268
264
  end
269
265
 
270
266
  class ClassObject < CodeObjectWithMethods
@@ -272,7 +268,6 @@ module YARD #:nodoc:
272
268
 
273
269
  def initialize(name, superclass = BASE_OBJECT, *args)
274
270
  super(name, :class, *args) do |obj|
275
- obj[:attributes] = {}
276
271
  obj[:superclass] = superclass
277
272
  yield(obj) if block_given?
278
273
  end
@@ -292,8 +287,9 @@ module YARD #:nodoc:
292
287
 
293
288
  def superclasses
294
289
  superobject = Namespace.find_from_path(path, superclass)
295
- return [superclass] if superobject.nil?
296
- [superobject.path] + superobject.superclasses
290
+ return ["Object"] unless superobject
291
+ return [] if path == superobject.path
292
+ [superobject.path, *superobject.superclasses]
297
293
  end
298
294
 
299
295
  def inheritance_tree
@@ -308,7 +304,8 @@ module YARD #:nodoc:
308
304
  # @param [CodeObjectWithMethods] parent the object that holds this method
309
305
  def initialize(name, visibility, scope, parent, comments = nil)
310
306
  super(name, :method, visibility, scope, parent, comments) do |obj|
311
- parent["#{scope}_methods".to_sym].update(name.to_s => obj)
307
+ pmethods = parent["#{scope}_methods".to_sym]
308
+ pmethods.update(name.to_s => obj) if pmethods
312
309
  yield(obj) if block_given?
313
310
  end
314
311
  end
data/lib/extra.rb ADDED
@@ -0,0 +1,8 @@
1
+ class String
2
+ # Remove spaces on either end of text
3
+ # @return [String]
4
+ def trim
5
+ gsub(/^\s+|\s+$/, '')
6
+ end
7
+ end
8
+
data/lib/formatter.rb CHANGED
@@ -1,4 +1,5 @@
1
- require 'erb'
1
+ require 'rubygems'
2
+ require 'erubis'
2
3
  require 'rdoc/markup/simple_markup'
3
4
  require 'rdoc/markup/simple_markup/to_html'
4
5
 
@@ -12,21 +13,17 @@ module YARD
12
13
  # @author Loren Segal
13
14
  # @version 1.0
14
15
  class Formatter
15
- OUTPUT_FORMATS = [ :html, :xhtml, :xml ]
16
-
17
16
  ##
18
17
  # Formats an object as a specified output format. Default is +:html+.
19
18
  #
20
- # @param [String, CodeObject] object the code object to format or the path to the code object
21
- # @param [Symbol] format the output format to generate documentation in.
22
- # Defaults to +:html+, which is a synonym for +:xhtml+.
19
+ # @param format the output format to generate documentation in.
20
+ # Defaults to +:html+, which is a synonym for <tt>:xhtml</tt>
21
+ # @param template the template sub directory to use, default is <tt>:default</tt>
23
22
  # @see OUTPUT_FORMATS
24
- def format(object, format = :html)
23
+ def format(object, format, template)
25
24
  object = Namespace.at(object) if object.is_a? String
26
- erb = File.join(template_directory, "#{format}_formatter.erb")
27
-
28
- @object = object
29
- ERB.new(IO.read(erb), nil, ">").result(binding)
25
+ @object, @format, @template = object, format, template
26
+ render(@object.type)
30
27
  end
31
28
 
32
29
  ##
@@ -34,9 +31,21 @@ module YARD
34
31
  def template_directory
35
32
  File.join(File.dirname(__FILE__), '..', 'templates')
36
33
  end
34
+
35
+ def render(type, format = @format, template = @template)
36
+ formatter = self
37
+ _binding = @object ? @object.instance_eval("binding") : binding
38
+ filename = File.join(template_directory, template.to_s, format.to_s, "#{type}.erb")
39
+ Erubis::Eruby.new("<% extend #{format.to_s.capitalize}Formatter %>\n" +
40
+ IO.read(filename), :trim => true).result(_binding)
41
+ rescue => e
42
+ STDERR.puts "Could not render template #{filename}: #{e.message}"
43
+ STDERR.puts e.backtrace[0, 5].map {|x| "\t#{x}" }
44
+ STDERR.puts
45
+ end
37
46
  end
38
47
 
39
- protected
48
+ module HtmlFormatter
40
49
  def link_to_path(name, from_path = nil, label = nil)
41
50
  return "<a href='#instance_method-#{name[1..-1]}'>#{label || name}</a>" if name =~ /^\#/ && from_path.nil?
42
51
 
@@ -48,7 +57,7 @@ module YARD
48
57
 
49
58
  label = name if label.nil?
50
59
  if obj
51
- file = obj.parent.path.gsub("::","_") + ".html"
60
+ file = (obj.parent || obj).path.gsub("::","_") + ".html"
52
61
  case obj
53
62
  when ConstantObject
54
63
  "<a href='#{file}#const-#{obj.name}'>#{label}</a>"
@@ -69,10 +78,13 @@ module YARD
69
78
  end
70
79
 
71
80
  def resolve_links(text, path)
72
- t, re = text, /\{(.+?)\}/
73
- while t =~ re
74
- t.sub!(re, "<tt>" + link_to_path($1, path) + "</tt>")
75
- end
76
- t
81
+ text.gsub(/\{(.+?)\}/) {|match| "<tt>" + link_to_path(match, path) + "</tt>" }
77
82
  end
83
+ end
84
+
85
+ class CodeObject
86
+ def to_s(format = :html, template = :default)
87
+ Formatter.new.format(self, format, template)
88
+ end
89
+ end
78
90
  end
data/lib/handlers/attribute_handler.rb CHANGED
@@ -2,9 +2,14 @@ class YARD::AttributeHandler < YARD::CodeObjectHandler
2
2
  handles /\Aattr(_(reader|writer|accessor))?\b/
3
3
 
4
4
  def process
5
- attr_type = statement.tokens.first.text.to_sym
6
- symbols = eval("[" + statement.tokens[1..-1].to_s + "]")
7
- read, write = true, false
5
+ begin
6
+ attr_type = statement.tokens.first.text.to_sym
7
+ symbols = eval("[" + statement.tokens[1..-1].to_s + "]")
8
+ read, write = true, false
9
+ rescue SyntaxError
10
+ Logger.warning "in AttributeHandler: Undocumentable attribute statement: '#{statement.tokens.to_s}'"
11
+ return
12
+ end
8
13
 
9
14
  # Change read/write based on attr_reader/writer/accessor
10
15
  case attr_type
@@ -22,7 +27,7 @@ class YARD::AttributeHandler < YARD::CodeObjectHandler
22
27
  # Add all attributes
23
28
  symbols.each do |name|
24
29
  name = name.to_s
25
- object[:attributes].update(name.to_s => { :read => read, :write => write })
30
+ object[:attributes].update(name.to_s => { :read => read, :write => write }) if object.type == :class
26
31
 
27
32
  # Show their methods as well
28
33
  [name, "#{name}="].each do |method|
data/lib/handlers/class_handler.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  class YARD::ClassHandler < YARD::CodeObjectHandler
2
- handles RubyToken::TkCLASS
2
+ handles YARD::RubyToken::TkCLASS
3
3
 
4
4
  def process
5
5
  words = statement.tokens.to_s.strip.split(/\s+/)
@@ -22,7 +22,8 @@ class YARD::ClassHandler < YARD::CodeObjectHandler
22
22
  current_namespace.attributes[:scope], current_visibility = scope, vis
23
23
  else
24
24
  class_name = move_to_namespace(class_name)
25
- class_obj = YARD::ClassObject.new(class_name, superclass, object, statement.comments)
25
+ class_obj = Namespace.find_from_path(object, class_name)
26
+ class_obj ||= YARD::ClassObject.new(class_name, superclass, object, statement.comments)
26
27
  enter_namespace(class_obj) { parse_block }
27
28
  end
28
29
  end
data/lib/handlers/constant_handler.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  class YARD::ConstantHandler < YARD::CodeObjectHandler
2
- HANDLER_MATCH = /\A[^@\$]\S*\s*=\s*/m
2
+ HANDLER_MATCH = /\A[^@\$]\S*\s*=[^=]\s*/m
3
3
  handles HANDLER_MATCH
4
4
 
5
5
  def process
data/lib/handlers/exception_handler.rb CHANGED
@@ -7,7 +7,7 @@ class YARD::ExceptionHandler < YARD::CodeObjectHandler
7
7
  break index if token.class == RubyToken::TkIDENTIFIER && token.text == 'raise'
8
8
  end
9
9
  if from.is_a? Fixnum
10
- exception_class = tokens[(from+1)..-1].to_s[/^\s*(\S+?),?/, 1]
10
+ exception_class = tokens[(from+1)..-1].to_s[/^\W+(\w+)/, 1]
11
11
  # RuntimeError for Strings or no parameter
12
12
  exception_class = "RuntimeError" if exception_class =~ /^\"/ || exception_class.nil?
13
13
 
data/lib/handlers/method_handler.rb CHANGED
@@ -1,7 +1,8 @@
1
1
  class YARD::MethodHandler < YARD::CodeObjectHandler
2
- handles RubyToken::TkDEF
2
+ handles YARD::RubyToken::TkDEF
3
3
 
4
4
  def process
5
+ return unless object.is_a? YARD::CodeObjectWithMethods
5
6
  stmt_nospace = statement.tokens.reject {|t| t.is_a? RubyToken::TkSPACE }
6
7
  method_name, method_scope = stmt_nospace[1].text, current_scope
7
8
  holding_object = object
data/lib/handlers/mixin_handler.rb CHANGED
@@ -3,7 +3,13 @@ class YARD::MixinHandler < YARD::CodeObjectHandler
3
3
 
4
4
  def process
5
5
  return unless object.is_a? YARD::CodeObjectWithMethods
6
- object.mixins.push eval(statement.tokens[1..-1].to_s).to_s
6
+ begin
7
+ object.mixins.push eval("[ " + statement.tokens[1..-1].to_s + " ]").to_s
8
+ rescue NameError
9
+ object.mixins.push statement.tokens[1..-1].to_s
10
+ end
11
+ object.mixins.map! {|mixin| mixin.trim }
12
+ object.mixins.flatten!
7
13
  object.mixins.uniq!
8
14
  end
9
15
  end
data/lib/handlers/module_handler.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  class YARD::ModuleHandler < YARD::CodeObjectHandler
2
- handles RubyToken::TkMODULE
2
+ handles YARD::RubyToken::TkMODULE
3
3
 
4
4
  def process
5
5
  module_name = move_to_namespace(statement.tokens[2].text)
data/lib/handlers/yield_handler.rb CHANGED
@@ -1,29 +1,31 @@
1
- class YARD::YieldHandler < YARD::CodeObjectHandler
2
- handles 'yield'
1
+ module YARD
2
+ class YieldHandler < CodeObjectHandler
3
+ handles 'yield'
3
4
 
4
- def process
5
- tokens = statement.tokens.reject {|tk| [RubyToken::TkSPACE, RubyToken::TkLPAREN].include? tk.class }
6
- from = tokens.each_with_index do |token, index|
7
- break index if token.class == RubyToken::TkYIELD
8
- end
9
- if from.is_a? Fixnum
10
- params = []
11
- (from+1).step(tokens.size-1, 2) do |index|
12
- # FIXME: This won't work if the yield has a method call or complex constant name (A::B)
13
- params << tokens[index].text
14
- break unless tokens[index+1].is_a? RubyToken::TkCOMMA
5
+ def process
6
+ tokens = statement.tokens.reject {|tk| [RubyToken::TkSPACE, RubyToken::TkLPAREN].include? tk.class }
7
+ from = tokens.each_with_index do |token, index|
8
+ break index if token.class == RubyToken::TkYIELD
15
9
  end
10
+ if from.is_a? Fixnum
11
+ params = []
12
+ (from+1).step(tokens.size-1, 2) do |index|
13
+ # FIXME: This won't work if the yield has a method call or complex constant name (A::B)
14
+ params << tokens[index].text
15
+ break unless tokens[index+1].is_a? RubyToken::TkCOMMA
16
+ end
16
17
 
17
- # Only add the tags if none were added at all
18
- if object.tags("yieldparam").empty? && object.tags("yield").empty?
19
- params.each do |param|
20
- # TODO: We can technically introspect any constant to find out parameter types,
21
- # not just self.
22
- # If parameter is self, we have a chance to get some extra information
23
- if param == "self"
24
- param = "[#{object.parent.path}] _self the object that yields the value (self)"
18
+ # Only add the tags if none were added at all
19
+ if object.tags("yieldparam").empty? && object.tags("yield").empty?
20
+ params.each do |param|
21
+ # TODO: We can technically introspect any constant to find out parameter types,
22
+ # not just self.
23
+ # If parameter is self, we have a chance to get some extra information
24
+ if param == "self"
25
+ param = "[#{object.parent.path}] _self the object that yields the value (self)"
26
+ end
27
+ object.tags << YARD::TagLibrary.yieldparam_tag(param)
25
28
  end
26
- object.tags << YARD::TagLibrary.yieldparam_tag(param)
27
29
  end
28
30
  end
29
31
  end
data/lib/logger.rb ADDED
@@ -0,0 +1,19 @@
1
+ class Logger
2
+ class << self
3
+ attr_accessor :notices, :warnings, :errors
4
+ @notices, @warnings, @errors = true, true, true
5
+
6
+ def notice
7
+ puts "Notice: #{msg}"
8
+ end
9
+
10
+ def warning(msg)
11
+ puts "Warning: #{msg}"
12
+ end
13
+
14
+ def error(msg)
15
+ puts "Error: #{msg}"
16
+ end
17
+ end
18
+ end
19
+
data/lib/ruby_lex.rb CHANGED
@@ -1,1318 +1,1321 @@
1
1
  require "e2mmap"
2
2
  require "irb/slex"
3
3
 
4
- module RubyToken
5
- EXPR_BEG = :EXPR_BEG
6
- EXPR_MID = :EXPR_MID
7
- EXPR_END = :EXPR_END
8
- EXPR_ARG = :EXPR_ARG
9
- EXPR_FNAME = :EXPR_FNAME
10
- EXPR_DOT = :EXPR_DOT
11
- EXPR_CLASS = :EXPR_CLASS
4
+ module YARD
5
+ module RubyToken
6
+ EXPR_BEG = :EXPR_BEG
7
+ EXPR_MID = :EXPR_MID
8
+ EXPR_END = :EXPR_END
9
+ EXPR_ARG = :EXPR_ARG
10
+ EXPR_FNAME = :EXPR_FNAME
11
+ EXPR_DOT = :EXPR_DOT
12
+ EXPR_CLASS = :EXPR_CLASS
12
13
 
13
- class Token
14
- NO_TEXT = "??".freeze
15
- attr :text
16
-
17
- def initialize(line_no, char_no)
18
- @line_no = line_no
19
- @char_no = char_no
20
- @text = NO_TEXT
21
- end
14
+ class Token
15
+ NO_TEXT = "??".freeze
16
+ attr :text
17
+
18
+ def initialize(line_no, char_no)
19
+ @line_no = line_no
20
+ @char_no = char_no
21
+ @text = NO_TEXT
22
+ end
22
23
 
23
- # Because we're used in contexts that expect to return a token,
24
- # we set the text string and then return ourselves
25
- def set_text(text)
26
- @text = text
27
- self
28
- end
24
+ # Because we're used in contexts that expect to return a token,
25
+ # we set the text string and then return ourselves
26
+ def set_text(text)
27
+ @text = text
28
+ self
29
+ end
29
30
 
30
- attr_reader :line_no, :char_no, :text
31
- attr_accessor :lex_state
32
- end
31
+ attr_reader :line_no, :char_no, :text
32
+ attr_accessor :lex_state
33
+ end
33
34
 
34
- class TkNode < Token
35
- attr :node
36
- end
35
+ class TkNode < Token
36
+ attr :node
37
+ end
37
38
 
38
- class TkId < Token
39
- def initialize(line_no, char_no, name)
40
- super(line_no, char_no)
41
- @name = name
39
+ class TkId < Token
40
+ def initialize(line_no, char_no, name)
41
+ super(line_no, char_no)
42
+ @name = name
43
+ end
44
+ attr :name
42
45
  end
43
- attr :name
44
- end
45
46
 
46
- class TkKW < TkId
47
- end
47
+ class TkKW < TkId
48
+ end
48
49
 
49
- class TkVal < Token
50
- def initialize(line_no, char_no, value = nil)
51
- super(line_no, char_no)
52
- set_text(value)
50
+ class TkVal < Token
51
+ def initialize(line_no, char_no, value = nil)
52
+ super(line_no, char_no)
53
+ set_text(value)
54
+ end
53
55
  end
54
- end
55
56
 
56
- class TkOp < Token
57
- def name
58
- self.class.op_name
57
+ class TkOp < Token
58
+ def name
59
+ self.class.op_name
60
+ end
59
61
  end
60
- end
61
62
 
62
- class TkOPASGN < TkOp
63
- def initialize(line_no, char_no, op)
64
- super(line_no, char_no)
65
- op = TkReading2Token[op] unless op.kind_of?(Symbol)
66
- @op = op
63
+ class TkOPASGN < TkOp
64
+ def initialize(line_no, char_no, op)
65
+ super(line_no, char_no)
66
+ op = TkReading2Token[op] unless op.kind_of?(Symbol)
67
+ @op = op
68
+ end
69
+ attr :op
67
70
  end
68
- attr :op
69
- end
70
71
 
71
- class TkUnknownChar < Token
72
- def initialize(line_no, char_no, id)
73
- super(line_no, char_no)
74
- @name = char_no.chr
72
+ class TkUnknownChar < Token
73
+ def initialize(line_no, char_no, id)
74
+ super(line_no, char_no)
75
+ @name = char_no.chr
76
+ end
77
+ attr :name
75
78
  end
76
- attr :name
77
- end
78
79
 
79
- class TkError < Token
80
- end
80
+ class TkError < Token
81
+ end
81
82
 
82
- def set_token_position(line, char)
83
- @prev_line_no = line
84
- @prev_char_no = char
85
- end
83
+ def set_token_position(line, char)
84
+ @prev_line_no = line
85
+ @prev_char_no = char
86
+ end
86
87
 
87
- def Token(token, value = nil)
88
- tk = nil
89
- case token
90
- when String, Symbol
91
- source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
92
- if (tk = source[token]).nil?
93
- IRB.fail TkReading2TokenNoKey, token
94
- end
95
- tk = Token(tk[0], value)
96
- else
97
- tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
98
- token.new(@prev_line_no, @prev_char_no)
99
- else
100
- token.new(@prev_line_no, @prev_char_no, value)
101
- end
88
+ def Token(token, value = nil)
89
+ tk = nil
90
+ case token
91
+ when String, Symbol
92
+ source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
93
+ if (tk = source[token]).nil?
94
+ IRB.fail TkReading2TokenNoKey, token
95
+ end
96
+ tk = Token(tk[0], value)
97
+ else
98
+ tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
99
+ token.new(@prev_line_no, @prev_char_no)
100
+ else
101
+ token.new(@prev_line_no, @prev_char_no, value)
102
+ end
103
+ end
104
+ tk
102
105
  end
103
- tk
104
- end
105
106
 
106
- TokenDefinitions = [
107
- [:TkCLASS, TkKW, "class", EXPR_CLASS],
108
- [:TkMODULE, TkKW, "module", EXPR_BEG],
109
- [:TkDEF, TkKW, "def", EXPR_FNAME],
110
- [:TkUNDEF, TkKW, "undef", EXPR_FNAME],
111
- [:TkBEGIN, TkKW, "begin", EXPR_BEG],
112
- [:TkRESCUE, TkKW, "rescue", EXPR_MID],
113
- [:TkENSURE, TkKW, "ensure", EXPR_BEG],
114
- [:TkEND, TkKW, "end", EXPR_END],
115
- [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
116
- [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
117
- [:TkTHEN, TkKW, "then", EXPR_BEG],
118
- [:TkELSIF, TkKW, "elsif", EXPR_BEG],
119
- [:TkELSE, TkKW, "else", EXPR_BEG],
120
- [:TkCASE, TkKW, "case", EXPR_BEG],
121
- [:TkWHEN, TkKW, "when", EXPR_BEG],
122
- [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
123
- [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
124
- [:TkFOR, TkKW, "for", EXPR_BEG],
125
- [:TkBREAK, TkKW, "break", EXPR_END],
126
- [:TkNEXT, TkKW, "next", EXPR_END],
127
- [:TkREDO, TkKW, "redo", EXPR_END],
128
- [:TkRETRY, TkKW, "retry", EXPR_END],
129
- [:TkIN, TkKW, "in", EXPR_BEG],
130
- [:TkDO, TkKW, "do", EXPR_BEG],
131
- [:TkRETURN, TkKW, "return", EXPR_MID],
132
- [:TkYIELD, TkKW, "yield", EXPR_END],
133
- [:TkSUPER, TkKW, "super", EXPR_END],
134
- [:TkSELF, TkKW, "self", EXPR_END],
135
- [:TkNIL, TkKW, "nil", EXPR_END],
136
- [:TkTRUE, TkKW, "true", EXPR_END],
137
- [:TkFALSE, TkKW, "false", EXPR_END],
138
- [:TkAND, TkKW, "and", EXPR_BEG],
139
- [:TkOR, TkKW, "or", EXPR_BEG],
140
- [:TkNOT, TkKW, "not", EXPR_BEG],
141
- [:TkIF_MOD, TkKW],
142
- [:TkUNLESS_MOD, TkKW],
143
- [:TkWHILE_MOD, TkKW],
144
- [:TkUNTIL_MOD, TkKW],
145
- [:TkALIAS, TkKW, "alias", EXPR_FNAME],
146
- [:TkDEFINED, TkKW, "defined?", EXPR_END],
147
- [:TklBEGIN, TkKW, "BEGIN", EXPR_END],
148
- [:TklEND, TkKW, "END", EXPR_END],
149
- [:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
150
- [:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
151
-
152
- [:TkIDENTIFIER, TkId],
153
- [:TkFID, TkId],
154
- [:TkGVAR, TkId],
155
- [:TkIVAR, TkId],
156
- [:TkCONSTANT, TkId],
157
-
158
- [:TkINTEGER, TkVal],
159
- [:TkFLOAT, TkVal],
160
- [:TkSTRING, TkVal],
161
- [:TkXSTRING, TkVal],
162
- [:TkREGEXP, TkVal],
163
- [:TkCOMMENT, TkVal],
164
-
165
- [:TkDSTRING, TkNode],
166
- [:TkDXSTRING, TkNode],
167
- [:TkDREGEXP, TkNode],
168
- [:TkNTH_REF, TkId],
169
- [:TkBACK_REF, TkId],
170
-
171
- [:TkUPLUS, TkOp, "+@"],
172
- [:TkUMINUS, TkOp, "-@"],
173
- [:TkPOW, TkOp, "**"],
174
- [:TkCMP, TkOp, "<=>"],
175
- [:TkEQ, TkOp, "=="],
176
- [:TkEQQ, TkOp, "==="],
177
- [:TkNEQ, TkOp, "!="],
178
- [:TkGEQ, TkOp, ">="],
179
- [:TkLEQ, TkOp, "<="],
180
- [:TkANDOP, TkOp, "&&"],
181
- [:TkOROP, TkOp, "||"],
182
- [:TkMATCH, TkOp, "=~"],
183
- [:TkNMATCH, TkOp, "!~"],
184
- [:TkDOT2, TkOp, ".."],
185
- [:TkDOT3, TkOp, "..."],
186
- [:TkAREF, TkOp, "[]"],
187
- [:TkASET, TkOp, "[]="],
188
- [:TkLSHFT, TkOp, "<<"],
189
- [:TkRSHFT, TkOp, ">>"],
190
- [:TkCOLON2, TkOp],
191
- [:TkCOLON3, TkOp],
192
- [:OPASGN, TkOp], # +=, -= etc. #
193
- [:TkASSOC, TkOp, "=>"],
194
- [:TkQUESTION, TkOp, "?"], #?
195
- [:TkCOLON, TkOp, ":"], #:
107
+ TokenDefinitions = [
108
+ [:TkCLASS, TkKW, "class", EXPR_CLASS],
109
+ [:TkMODULE, TkKW, "module", EXPR_BEG],
110
+ [:TkDEF, TkKW, "def", EXPR_FNAME],
111
+ [:TkUNDEF, TkKW, "undef", EXPR_FNAME],
112
+ [:TkBEGIN, TkKW, "begin", EXPR_BEG],
113
+ [:TkRESCUE, TkKW, "rescue", EXPR_MID],
114
+ [:TkENSURE, TkKW, "ensure", EXPR_BEG],
115
+ [:TkEND, TkKW, "end", EXPR_END],
116
+ [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
117
+ [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
118
+ [:TkTHEN, TkKW, "then", EXPR_BEG],
119
+ [:TkELSIF, TkKW, "elsif", EXPR_BEG],
120
+ [:TkELSE, TkKW, "else", EXPR_BEG],
121
+ [:TkCASE, TkKW, "case", EXPR_BEG],
122
+ [:TkWHEN, TkKW, "when", EXPR_BEG],
123
+ [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
124
+ [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
125
+ [:TkFOR, TkKW, "for", EXPR_BEG],
126
+ [:TkBREAK, TkKW, "break", EXPR_END],
127
+ [:TkNEXT, TkKW, "next", EXPR_END],
128
+ [:TkREDO, TkKW, "redo", EXPR_END],
129
+ [:TkRETRY, TkKW, "retry", EXPR_END],
130
+ [:TkIN, TkKW, "in", EXPR_BEG],
131
+ [:TkDO, TkKW, "do", EXPR_BEG],
132
+ [:TkRETURN, TkKW, "return", EXPR_MID],
133
+ [:TkYIELD, TkKW, "yield", EXPR_END],
134
+ [:TkSUPER, TkKW, "super", EXPR_END],
135
+ [:TkSELF, TkKW, "self", EXPR_END],
136
+ [:TkNIL, TkKW, "nil", EXPR_END],
137
+ [:TkTRUE, TkKW, "true", EXPR_END],
138
+ [:TkFALSE, TkKW, "false", EXPR_END],
139
+ [:TkAND, TkKW, "and", EXPR_BEG],
140
+ [:TkOR, TkKW, "or", EXPR_BEG],
141
+ [:TkNOT, TkKW, "not", EXPR_BEG],
142
+ [:TkIF_MOD, TkKW],
143
+ [:TkUNLESS_MOD, TkKW],
144
+ [:TkWHILE_MOD, TkKW],
145
+ [:TkUNTIL_MOD, TkKW],
146
+ [:TkALIAS, TkKW, "alias", EXPR_FNAME],
147
+ [:TkDEFINED, TkKW, "defined?", EXPR_END],
148
+ [:TklBEGIN, TkKW, "BEGIN", EXPR_END],
149
+ [:TklEND, TkKW, "END", EXPR_END],
150
+ [:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
151
+ [:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
152
+
153
+ [:TkIDENTIFIER, TkId],
154
+ [:TkFID, TkId],
155
+ [:TkGVAR, TkId],
156
+ [:TkIVAR, TkId],
157
+ [:TkCONSTANT, TkId],
158
+
159
+ [:TkINTEGER, TkVal],
160
+ [:TkFLOAT, TkVal],
161
+ [:TkSTRING, TkVal],
162
+ [:TkXSTRING, TkVal],
163
+ [:TkREGEXP, TkVal],
164
+ [:TkCOMMENT, TkVal],
165
+
166
+ [:TkDSTRING, TkNode],
167
+ [:TkDXSTRING, TkNode],
168
+ [:TkDREGEXP, TkNode],
169
+ [:TkNTH_REF, TkId],
170
+ [:TkBACK_REF, TkId],
171
+
172
+ [:TkUPLUS, TkOp, "+@"],
173
+ [:TkUMINUS, TkOp, "-@"],
174
+ [:TkPOW, TkOp, "**"],
175
+ [:TkCMP, TkOp, "<=>"],
176
+ [:TkEQ, TkOp, "=="],
177
+ [:TkEQQ, TkOp, "==="],
178
+ [:TkNEQ, TkOp, "!="],
179
+ [:TkGEQ, TkOp, ">="],
180
+ [:TkLEQ, TkOp, "<="],
181
+ [:TkANDOP, TkOp, "&&"],
182
+ [:TkOROP, TkOp, "||"],
183
+ [:TkMATCH, TkOp, "=~"],
184
+ [:TkNMATCH, TkOp, "!~"],
185
+ [:TkDOT2, TkOp, ".."],
186
+ [:TkDOT3, TkOp, "..."],
187
+ [:TkAREF, TkOp, "[]"],
188
+ [:TkASET, TkOp, "[]="],
189
+ [:TkLSHFT, TkOp, "<<"],
190
+ [:TkRSHFT, TkOp, ">>"],
191
+ [:TkCOLON2, TkOp],
192
+ [:TkCOLON3, TkOp],
193
+ [:OPASGN, TkOp], # +=, -= etc. #
194
+ [:TkASSOC, TkOp, "=>"],
195
+ [:TkQUESTION, TkOp, "?"], #?
196
+ [:TkCOLON, TkOp, ":"], #:
196
197
 
197
- [:TkfLPAREN], # func( #
198
- [:TkfLBRACK], # func[ #
199
- [:TkfLBRACE], # func{ #
200
- [:TkSTAR], # *arg
201
- [:TkAMPER], # &arg #
202
- [:TkSYMBOL, TkId], # :SYMBOL
203
- [:TkSYMBEG, TkId],
204
- [:TkGT, TkOp, ">"],
205
- [:TkLT, TkOp, "<"],
206
- [:TkPLUS, TkOp, "+"],
207
- [:TkMINUS, TkOp, "-"],
208
- [:TkMULT, TkOp, "*"],
209
- [:TkDIV, TkOp, "/"],
210
- [:TkMOD, TkOp, "%"],
211
- [:TkBITOR, TkOp, "|"],
212
- [:TkBITXOR, TkOp, "^"],
213
- [:TkBITAND, TkOp, "&"],
214
- [:TkBITNOT, TkOp, "~"],
215
- [:TkNOTOP, TkOp, "!"],
216
-
217
- [:TkBACKQUOTE, TkOp, "`"],
218
-
219
- [:TkASSIGN, Token, "="],
220
- [:TkDOT, Token, "."],
221
- [:TkLPAREN, Token, "("], #(exp)
222
- [:TkLBRACK, Token, "["], #[arry]
223
- [:TkLBRACE, Token, "{"], #{hash}
224
- [:TkRPAREN, Token, ")"],
225
- [:TkRBRACK, Token, "]"],
226
- [:TkRBRACE, Token, "}"],
227
- [:TkCOMMA, Token, ","],
228
- [:TkSEMICOLON, Token, ";"],
229
-
230
- [:TkRD_COMMENT],
231
- [:TkSPACE],
232
- [:TkNL],
233
- [:TkEND_OF_SCRIPT],
234
-
235
- [:TkBACKSLASH, TkUnknownChar, "\\"],
236
- [:TkAT, TkUnknownChar, "@"],
237
- [:TkDOLLAR, TkUnknownChar, "\$"], #"
238
- ]
239
-
240
- # {reading => token_class}
241
- # {reading => [token_class, *opt]}
242
- TkReading2Token = {}
243
- TkSymbol2Token = {}
244
-
245
- def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
246
- token_n = token_n.id2name unless token_n.kind_of?(String)
247
- if RubyToken.const_defined?(token_n)
248
- #IRB.fail AlreadyDefinedToken, token_n
249
- end
198
+ [:TkfLPAREN], # func( #
199
+ [:TkfLBRACK], # func[ #
200
+ [:TkfLBRACE], # func{ #
201
+ [:TkSTAR], # *arg
202
+ [:TkAMPER], # &arg #
203
+ [:TkSYMBOL, TkId], # :SYMBOL
204
+ [:TkSYMBEG, TkId],
205
+ [:TkGT, TkOp, ">"],
206
+ [:TkLT, TkOp, "<"],
207
+ [:TkPLUS, TkOp, "+"],
208
+ [:TkMINUS, TkOp, "-"],
209
+ [:TkMULT, TkOp, "*"],
210
+ [:TkDIV, TkOp, "/"],
211
+ [:TkMOD, TkOp, "%"],
212
+ [:TkBITOR, TkOp, "|"],
213
+ [:TkBITXOR, TkOp, "^"],
214
+ [:TkBITAND, TkOp, "&"],
215
+ [:TkBITNOT, TkOp, "~"],
216
+ [:TkNOTOP, TkOp, "!"],
217
+
218
+ [:TkBACKQUOTE, TkOp, "`"],
219
+
220
+ [:TkASSIGN, Token, "="],
221
+ [:TkDOT, Token, "."],
222
+ [:TkLPAREN, Token, "("], #(exp)
223
+ [:TkLBRACK, Token, "["], #[arry]
224
+ [:TkLBRACE, Token, "{"], #{hash}
225
+ [:TkRPAREN, Token, ")"],
226
+ [:TkRBRACK, Token, "]"],
227
+ [:TkRBRACE, Token, "}"],
228
+ [:TkCOMMA, Token, ","],
229
+ [:TkSEMICOLON, Token, ";"],
230
+
231
+ [:TkRD_COMMENT],
232
+ [:TkSPACE],
233
+ [:TkNL],
234
+ [:TkEND_OF_SCRIPT],
235
+
236
+ [:TkBACKSLASH, TkUnknownChar, "\\"],
237
+ [:TkAT, TkUnknownChar, "@"],
238
+ [:TkDOLLAR, TkUnknownChar, "\$"], #"
239
+ ]
240
+
241
+ # {reading => token_class}
242
+ # {reading => [token_class, *opt]}
243
+ TkReading2Token = {}
244
+ TkSymbol2Token = {}
245
+
246
+ def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
247
+ token_n = token_n.id2name unless token_n.kind_of?(String)
248
+ if RubyToken.const_defined?(token_n)
249
+ #IRB.fail AlreadyDefinedToken, token_n
250
+ end
250
251
 
251
- token_c = Class.new super_token
252
- RubyToken.const_set token_n, token_c
253
- # token_c.inspect
252
+ token_c = Class.new super_token
253
+ RubyToken.const_set token_n, token_c
254
+ # token_c.inspect
254
255
 
255
- if reading
256
- if TkReading2Token[reading]
257
- IRB.fail TkReading2TokenDuplicateError, token_n, reading
256
+ if reading
257
+ if TkReading2Token[reading]
258
+ IRB.fail TkReading2TokenDuplicateError, token_n, reading
259
+ end
260
+ if opts.empty?
261
+ TkReading2Token[reading] = [token_c]
262
+ else
263
+ TkReading2Token[reading] = [token_c].concat(opts)
264
+ end
258
265
  end
259
- if opts.empty?
260
- TkReading2Token[reading] = [token_c]
261
- else
262
- TkReading2Token[reading] = [token_c].concat(opts)
266
+ TkSymbol2Token[token_n.intern] = token_c
267
+
268
+ if token_c <= TkOp
269
+ token_c.class_eval %{
270
+ def self.op_name; "#{reading}"; end
271
+ }
263
272
  end
264
273
  end
265
- TkSymbol2Token[token_n.intern] = token_c
266
274
 
267
- if token_c <= TkOp
268
- token_c.class_eval %{
269
- def self.op_name; "#{reading}"; end
270
- }
275
+ for defs in TokenDefinitions
276
+ def_token(*defs)
271
277
  end
272
- end
273
278
 
274
- for defs in TokenDefinitions
275
- def_token(*defs)
279
+ NEWLINE_TOKEN = TkNL.new(0,0)
280
+ NEWLINE_TOKEN.set_text("\n")
281
+
276
282
  end
277
283
 
278
- NEWLINE_TOKEN = TkNL.new(0,0)
279
- NEWLINE_TOKEN.set_text("\n")
280
-
281
- end
282
-
283
-
284
-
285
- # Lexical analyzer for Ruby source
286
-
287
- class RubyLex
288
-
289
- ######################################################################
290
- #
291
- # Read an input stream character by character. We allow for unlimited
292
- # ungetting of characters just read.
293
- #
294
- # We simplify the implementation greatly by reading the entire input
295
- # into a buffer initially, and then simply traversing it using
296
- # pointers.
297
- #
298
- # We also have to allow for the <i>here document diversion</i>. This
299
- # little gem comes about when the lexer encounters a here
300
- # document. At this point we effectively need to split the input
301
- # stream into two parts: one to read the body of the here document,
302
- # the other to read the rest of the input line where the here
303
- # document was initially encountered. For example, we might have
304
- #
305
- # do_something(<<-A, <<-B)
306
- # stuff
307
- # for
308
- # A
309
- # stuff
310
- # for
311
- # B
312
- #
313
- # When the lexer encounters the <<A, it reads until the end of the
314
- # line, and keeps it around for later. It then reads the body of the
315
- # here document. Once complete, it needs to read the rest of the
316
- # original line, but then skip the here document body.
317
- #
284
+
285
+
286
+ # Lexical analyzer for Ruby source
287
+
288
+ class RubyLex
289
+
290
+ ######################################################################
291
+ #
292
+ # Read an input stream character by character. We allow for unlimited
293
+ # ungetting of characters just read.
294
+ #
295
+ # We simplify the implementation greatly by reading the entire input
296
+ # into a buffer initially, and then simply traversing it using
297
+ # pointers.
298
+ #
299
+ # We also have to allow for the <i>here document diversion</i>. This
300
+ # little gem comes about when the lexer encounters a here
301
+ # document. At this point we effectively need to split the input
302
+ # stream into two parts: one to read the body of the here document,
303
+ # the other to read the rest of the input line where the here
304
+ # document was initially encountered. For example, we might have
305
+ #
306
+ # do_something(<<-A, <<-B)
307
+ # stuff
308
+ # for
309
+ # A
310
+ # stuff
311
+ # for
312
+ # B
313
+ #
314
+ # When the lexer encounters the <<A, it reads until the end of the
315
+ # line, and keeps it around for later. It then reads the body of the
316
+ # here document. Once complete, it needs to read the rest of the
317
+ # original line, but then skip the here document body.
318
+ #
318
319
 
319
- class BufferedReader
320
+ class BufferedReader
320
321
 
321
- attr_reader :line_num
322
+ attr_reader :line_num
322
323
 
323
- def initialize(content)
324
- if /\t/ =~ content
325
- tab_width = 2
326
- content = content.split(/\n/).map do |line|
327
- 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
328
- line
329
- end .join("\n")
330
- end
331
- @content = content
332
- @content << "\n" unless @content[-1,1] == "\n"
333
- @size = @content.size
334
- @offset = 0
335
- @hwm = 0
336
- @line_num = 1
337
- @read_back_offset = 0
338
- @last_newline = 0
339
- @newline_pending = false
340
- end
324
+ def initialize(content)
325
+ if /\t/ =~ content
326
+ tab_width = 2
327
+ content = content.split(/\n/).map do |line|
328
+ 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
329
+ line
330
+ end .join("\n")
331
+ end
332
+ @content = content
333
+ @content << "\n" unless @content[-1,1] == "\n"
334
+ @size = @content.size
335
+ @offset = 0
336
+ @hwm = 0
337
+ @line_num = 1
338
+ @read_back_offset = 0
339
+ @last_newline = 0
340
+ @newline_pending = false
341
+ end
341
342
 
342
- def column
343
- @offset - @last_newline
344
- end
343
+ def column
344
+ @offset - @last_newline
345
+ end
345
346
 
346
- def getc
347
- return nil if @offset >= @size
348
- ch = @content[@offset, 1]
347
+ def getc
348
+ return nil if @offset >= @size
349
+ ch = @content[@offset, 1]
349
350
 
350
- @offset += 1
351
- @hwm = @offset if @hwm < @offset
351
+ @offset += 1
352
+ @hwm = @offset if @hwm < @offset
352
353
 
353
- if @newline_pending
354
- @line_num += 1
355
- @last_newline = @offset - 1
356
- @newline_pending = false
357
- end
354
+ if @newline_pending
355
+ @line_num += 1
356
+ @last_newline = @offset - 1
357
+ @newline_pending = false
358
+ end
358
359
 
359
- if ch == "\n"
360
- @newline_pending = true
360
+ if ch == "\n"
361
+ @newline_pending = true
362
+ end
363
+ ch
361
364
  end
362
- ch
363
- end
364
365
 
365
- def getc_already_read
366
- getc
367
- end
366
+ def getc_already_read
367
+ getc
368
+ end
368
369
 
369
- def ungetc(ch)
370
- raise "unget past beginning of file" if @offset <= 0
371
- @offset -= 1
372
- if @content[@offset] == ?\n
373
- @newline_pending = false
370
+ def ungetc(ch)
371
+ raise "unget past beginning of file" if @offset <= 0
372
+ @offset -= 1
373
+ if @content[@offset] == ?\n
374
+ @newline_pending = false
375
+ end
374
376
  end
375
- end
376
377
 
377
- def get_read
378
- res = @content[@read_back_offset...@offset]
379
- @read_back_offset = @offset
380
- res
381
- end
378
+ def get_read
379
+ res = @content[@read_back_offset...@offset]
380
+ @read_back_offset = @offset
381
+ res
382
+ end
382
383
 
383
- def peek(at)
384
- pos = @offset + at
385
- if pos >= @size
386
- nil
387
- else
388
- @content[pos, 1]
384
+ def peek(at)
385
+ pos = @offset + at
386
+ if pos >= @size
387
+ nil
388
+ else
389
+ @content[pos, 1]
390
+ end
389
391
  end
390
- end
391
392
 
392
- def peek_equal(str)
393
- @content[@offset, str.length] == str
394
- end
393
+ def peek_equal(str)
394
+ @content[@offset, str.length] == str
395
+ end
395
396
 
396
- def divert_read_from(reserve)
397
- @content[@offset, 0] = reserve
398
- @size = @content.size
397
+ def divert_read_from(reserve)
398
+ @content[@offset, 0] = reserve
399
+ @size = @content.size
400
+ end
399
401
  end
400
- end
401
402
 
402
- # end of nested class BufferedReader
403
+ # end of nested class BufferedReader
403
404
 
404
- extend Exception2MessageMapper
405
- def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
406
- def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
407
- def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
408
- def_exception(:TkReading2TokenDuplicateError,
409
- "key duplicate(token_n='%s', key='%s')")
410
- def_exception(:SyntaxError, "%s")
405
+ extend Exception2MessageMapper
406
+ def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
407
+ def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
408
+ def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
409
+ def_exception(:TkReading2TokenDuplicateError,
410
+ "key duplicate(token_n='%s', key='%s')")
411
+ def_exception(:SyntaxError, "%s")
411
412
 
412
- include RubyToken
413
- include IRB
413
+ include RubyToken
414
+ include IRB
414
415
 
415
- attr_reader :continue
416
- attr_reader :lex_state
416
+ attr_reader :continue
417
+ attr_reader :lex_state
417
418
 
418
- def RubyLex.debug?
419
- false
420
- end
419
+ def RubyLex.debug?
420
+ false
421
+ end
421
422
 
422
- def initialize(content)
423
- lex_init
423
+ def initialize(content)
424
+ lex_init
424
425
 
425
- @reader = BufferedReader.new(content)
426
+ @reader = BufferedReader.new(content)
426
427
 
427
- @exp_line_no = @line_no = 1
428
- @base_char_no = 0
429
- @indent = 0
428
+ @exp_line_no = @line_no = 1
429
+ @base_char_no = 0
430
+ @indent = 0
430
431
 
431
- @ltype = nil
432
- @quoted = nil
433
- @lex_state = EXPR_BEG
434
- @space_seen = false
432
+ @ltype = nil
433
+ @quoted = nil
434
+ @lex_state = EXPR_BEG
435
+ @space_seen = false
435
436
 
436
- @continue = false
437
- @line = ""
438
-
439
- @skip_space = false
440
- @read_auto_clean_up = false
441
- @exception_on_syntax_error = true
442
- end
437
+ @continue = false
438
+ @line = ""
443
439
 
444
- attr :skip_space, true
445
- attr :read_auto_clean_up, true
446
- attr :exception_on_syntax_error, true
440
+ @skip_space = false
441
+ @read_auto_clean_up = false
442
+ @exception_on_syntax_error = true
443
+ end
447
444
 
448
- attr :indent
445
+ attr :skip_space, true
446
+ attr :read_auto_clean_up, true
447
+ attr :exception_on_syntax_error, true
449
448
 
450
- # io functions
451
- def line_no
452
- @reader.line_num
453
- end
449
+ attr :indent
454
450
 
455
- def char_no
456
- @reader.column
457
- end
451
+ # io functions
452
+ def line_no
453
+ @reader.line_num
454
+ end
458
455
 
459
- def get_read
460
- @reader.get_read
461
- end
456
+ def char_no
457
+ @reader.column
458
+ end
462
459
 
463
- def getc
464
- @reader.getc
465
- end
460
+ def get_read
461
+ @reader.get_read
462
+ end
466
463
 
467
- def getc_of_rests
468
- @reader.getc_already_read
469
- end
464
+ def getc
465
+ @reader.getc
466
+ end
470
467
 
471
- def gets
472
- c = getc or return
473
- l = ""
474
- begin
475
- l.concat c unless c == "\r"
476
- break if c == "\n"
477
- end while c = getc
478
- l
479
- end
468
+ def getc_of_rests
469
+ @reader.getc_already_read
470
+ end
480
471
 
472
+ def gets
473
+ c = getc or return
474
+ l = ""
475
+ begin
476
+ l.concat c unless c == "\r"
477
+ break if c == "\n"
478
+ end while c = getc
479
+ l
480
+ end
481
481
 
482
- def ungetc(c = nil)
483
- @reader.ungetc(c)
484
- end
485
482
 
486
- def peek_equal?(str)
487
- @reader.peek_equal(str)
488
- end
483
+ def ungetc(c = nil)
484
+ @reader.ungetc(c)
485
+ end
489
486
 
490
- def peek(i = 0)
491
- @reader.peek(i)
492
- end
487
+ def peek_equal?(str)
488
+ @reader.peek_equal(str)
489
+ end
493
490
 
494
- def lex
495
- until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
496
- !@continue or
497
- tk.nil?)
491
+ def peek(i = 0)
492
+ @reader.peek(i)
498
493
  end
499
- line = get_read
500
494
 
501
- if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
502
- nil
503
- else
504
- line
495
+ def lex
496
+ until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
497
+ !@continue or
498
+ tk.nil?)
499
+ end
500
+ line = get_read
501
+
502
+ if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
503
+ nil
504
+ else
505
+ line
506
+ end
505
507
  end
506
- end
507
508
 
508
- def token
509
- set_token_position(line_no, char_no)
510
- begin
509
+ def token
510
+ set_token_position(line_no, char_no)
511
511
  begin
512
- tk = @OP.match(self)
513
- @space_seen = tk.kind_of?(TkSPACE)
514
- rescue SyntaxError
515
- abort if @exception_on_syntax_error
516
- tk = TkError.new(line_no, char_no)
517
- end
518
- end while @skip_space and tk.kind_of?(TkSPACE)
519
- if @read_auto_clean_up
520
- get_read
512
+ begin
513
+ tk = @OP.match(self)
514
+ @space_seen = tk.kind_of?(TkSPACE)
515
+ rescue SyntaxError
516
+ abort if @exception_on_syntax_error
517
+ tk = TkError.new(line_no, char_no)
518
+ end
519
+ end while @skip_space and tk.kind_of?(TkSPACE)
520
+ if @read_auto_clean_up
521
+ get_read
522
+ end
523
+ # throw :eof unless tk
524
+ p tk if $DEBUG
525
+ tk.lex_state = lex_state if tk
526
+ tk
521
527
  end
522
- # throw :eof unless tk
523
- p tk if $DEBUG
524
- tk.lex_state = lex_state if tk
525
- tk
526
- end
527
528
 
528
- ENINDENT_CLAUSE = [
529
- "case", "class", "def", "do", "for", "if",
530
- "module", "unless", "until", "while", "begin" #, "when"
531
- ]
532
- DEINDENT_CLAUSE = ["end" #, "when"
533
- ]
534
-
535
- PERCENT_LTYPE = {
536
- "q" => "\'",
537
- "Q" => "\"",
538
- "x" => "\`",
539
- "r" => "/",
540
- "w" => "]"
541
- }
529
+ ENINDENT_CLAUSE = [
530
+ "case", "class", "def", "do", "for", "if",
531
+ "module", "unless", "until", "while", "begin" #, "when"
532
+ ]
533
+ DEINDENT_CLAUSE = ["end" #, "when"
534
+ ]
535
+
536
+ PERCENT_LTYPE = {
537
+ "q" => "\'",
538
+ "Q" => "\"",
539
+ "x" => "\`",
540
+ "r" => "/",
541
+ "w" => "]",
542
+ "W" => "]"
543
+ }
542
544
 
543
- PERCENT_PAREN = {
544
- "{" => "}",
545
- "[" => "]",
546
- "<" => ">",
547
- "(" => ")"
548
- }
549
-
550
- Ltype2Token = {
551
- "\'" => TkSTRING,
552
- "\"" => TkSTRING,
553
- "\`" => TkXSTRING,
554
- "/" => TkREGEXP,
555
- "]" => TkDSTRING
556
- }
557
- Ltype2Token.default = TkSTRING
558
-
559
- DLtype2Token = {
560
- "\"" => TkDSTRING,
561
- "\`" => TkDXSTRING,
562
- "/" => TkDREGEXP,
563
- }
564
-
565
- def lex_init()
566
- @OP = SLex.new
567
- @OP.def_rules("\0", "\004", "\032") do |chars, io|
568
- Token(TkEND_OF_SCRIPT).set_text(chars)
569
- end
545
+ PERCENT_PAREN = {
546
+ "{" => "}",
547
+ "[" => "]",
548
+ "<" => ">",
549
+ "(" => ")"
550
+ }
551
+
552
+ Ltype2Token = {
553
+ "\'" => TkSTRING,
554
+ "\"" => TkSTRING,
555
+ "\`" => TkXSTRING,
556
+ "/" => TkREGEXP,
557
+ "]" => TkDSTRING
558
+ }
559
+ Ltype2Token.default = TkSTRING
560
+
561
+ DLtype2Token = {
562
+ "\"" => TkDSTRING,
563
+ "\`" => TkDXSTRING,
564
+ "/" => TkDREGEXP,
565
+ }
566
+
567
+ def lex_init()
568
+ @OP = SLex.new
569
+ @OP.def_rules("\0", "\004", "\032") do |chars, io|
570
+ Token(TkEND_OF_SCRIPT).set_text(chars)
571
+ end
570
572
 
571
- @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
572
- @space_seen = TRUE
573
- while (ch = getc) =~ /[ \t\f\r\13]/
574
- chars << ch
573
+ @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
574
+ @space_seen = TRUE
575
+ while (ch = getc) =~ /[ \t\f\r\13]/
576
+ chars << ch
577
+ end
578
+ ungetc
579
+ Token(TkSPACE).set_text(chars)
575
580
  end
576
- ungetc
577
- Token(TkSPACE).set_text(chars)
578
- end
579
581
 
580
- @OP.def_rule("#") do
581
- |op, io|
582
- identify_comment
583
- end
582
+ @OP.def_rule("#") do
583
+ |op, io|
584
+ identify_comment
585
+ end
584
586
 
585
- @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
586
- |op, io|
587
- str = op
588
- @ltype = "="
587
+ @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
588
+ |op, io|
589
+ str = op
590
+ @ltype = "="
589
591
 
590
592
 
591
- begin
592
- line = ""
593
593
  begin
594
- ch = getc
595
- line << ch
596
- end until ch == "\n"
597
- str << line
598
- end until line =~ /^=end/
594
+ line = ""
595
+ begin
596
+ ch = getc
597
+ line << ch
598
+ end until ch == "\n"
599
+ str << line
600
+ end until line =~ /^=end/
601
+
602
+ ungetc
603
+
604
+ @ltype = nil
605
+
606
+ if str =~ /\A=begin\s+rdoc/i
607
+ str.sub!(/\A=begin.*\n/, '')
608
+ str.sub!(/^=end.*/m, '')
609
+ Token(TkCOMMENT).set_text(str)
610
+ else
611
+ Token(TkRD_COMMENT)#.set_text(str)
612
+ end
613
+ end
599
614
 
600
- ungetc
615
+ @OP.def_rule("\n") do
616
+ print "\\n\n" if RubyLex.debug?
617
+ case @lex_state
618
+ when EXPR_BEG, EXPR_FNAME, EXPR_DOT
619
+ @continue = TRUE
620
+ else
621
+ @continue = FALSE
622
+ @lex_state = EXPR_BEG
623
+ end
624
+ Token(TkNL).set_text("\n")
625
+ end
601
626
 
602
- @ltype = nil
627
+ @OP.def_rules("*", "**",
628
+ "!", "!=", "!~",
629
+ "=", "==", "===",
630
+ "=~", "<=>",
631
+ "<", "<=",
632
+ ">", ">=", ">>") do |op, io|
633
+ @lex_state = EXPR_BEG
634
+ Token(op).set_text(op)
635
+ end
603
636
 
604
- if str =~ /\A=begin\s+rdoc/i
605
- str.sub!(/\A=begin.*\n/, '')
606
- str.sub!(/^=end.*/m, '')
607
- Token(TkCOMMENT).set_text(str)
608
- else
609
- Token(TkRD_COMMENT)#.set_text(str)
637
+ @OP.def_rules("<<") do |op, io|
638
+ tk = nil
639
+ if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
640
+ (@lex_state != EXPR_ARG || @space_seen)
641
+ c = peek(0)
642
+ tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
643
+ end
644
+ if !tk
645
+ @lex_state = EXPR_BEG
646
+ tk = Token(op).set_text(op)
647
+ end
648
+ tk
610
649
  end
611
- end
612
650
 
613
- @OP.def_rule("\n") do
614
- print "\\n\n" if RubyLex.debug?
615
- case @lex_state
616
- when EXPR_BEG, EXPR_FNAME, EXPR_DOT
617
- @continue = TRUE
618
- else
619
- @continue = FALSE
620
- @lex_state = EXPR_BEG
651
+ @OP.def_rules("'", '"') do |op, io|
652
+ identify_string(op)
621
653
  end
622
- Token(TkNL).set_text("\n")
623
- end
624
654
 
625
- @OP.def_rules("*", "**",
626
- "!", "!=", "!~",
627
- "=", "==", "===",
628
- "=~", "<=>",
629
- "<", "<=",
630
- ">", ">=", ">>") do |op, io|
631
- @lex_state = EXPR_BEG
632
- Token(op).set_text(op)
633
- end
655
+ @OP.def_rules("`") do |op, io|
656
+ if @lex_state == EXPR_FNAME
657
+ Token(op).set_text(op)
658
+ else
659
+ identify_string(op)
660
+ end
661
+ end
634
662
 
635
- @OP.def_rules("<<") do |op, io|
636
- tk = nil
637
- if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
638
- (@lex_state != EXPR_ARG || @space_seen)
639
- c = peek(0)
640
- tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
663
+ @OP.def_rules('?') do |op, io|
664
+ if @lex_state == EXPR_END
665
+ @lex_state = EXPR_BEG
666
+ Token(TkQUESTION).set_text(op)
667
+ else
668
+ ch = getc
669
+ if @lex_state == EXPR_ARG && ch !~ /\s/
670
+ ungetc
671
+ @lex_state = EXPR_BEG
672
+ Token(TkQUESTION).set_text(op)
673
+ else
674
+ str = op
675
+ str << ch
676
+ if (ch == '\\') #'
677
+ str << read_escape
678
+ end
679
+ @lex_state = EXPR_END
680
+ Token(TkINTEGER).set_text(str)
681
+ end
682
+ end
641
683
  end
642
- if !tk
684
+
685
+ @OP.def_rules("&", "&&", "|", "||") do |op, io|
643
686
  @lex_state = EXPR_BEG
644
- tk = Token(op).set_text(op)
687
+ Token(op).set_text(op)
645
688
  end
646
- tk
647
- end
648
-
649
- @OP.def_rules("'", '"') do |op, io|
650
- identify_string(op)
651
- end
652
-
653
- @OP.def_rules("`") do |op, io|
654
- if @lex_state == EXPR_FNAME
655
- Token(op).set_text(op)
656
- else
657
- identify_string(op)
689
+
690
+ @OP.def_rules("+=", "-=", "*=", "**=",
691
+ "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
692
+ @lex_state = EXPR_BEG
693
+ op =~ /^(.*)=$/
694
+ Token(TkOPASGN, $1).set_text(op)
658
695
  end
659
- end
660
696
 
661
- @OP.def_rules('?') do |op, io|
662
- if @lex_state == EXPR_END
663
- @lex_state = EXPR_BEG
664
- Token(TkQUESTION).set_text(op)
665
- else
666
- ch = getc
667
- if @lex_state == EXPR_ARG && ch !~ /\s/
668
- ungetc
669
- @lex_state = EXPR_BEG
670
- Token(TkQUESTION).set_text(op)
671
- else
672
- str = op
673
- str << ch
674
- if (ch == '\\') #'
675
- str << read_escape
676
- end
677
- @lex_state = EXPR_END
678
- Token(TkINTEGER).set_text(str)
679
- end
697
+ @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
698
+ Token(TkUPLUS).set_text(op)
680
699
  end
681
- end
682
700
 
683
- @OP.def_rules("&", "&&", "|", "||") do |op, io|
684
- @lex_state = EXPR_BEG
685
- Token(op).set_text(op)
686
- end
687
-
688
- @OP.def_rules("+=", "-=", "*=", "**=",
689
- "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
690
- @lex_state = EXPR_BEG
691
- op =~ /^(.*)=$/
692
- Token(TkOPASGN, $1).set_text(op)
693
- end
694
-
695
- @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
696
- Token(TkUPLUS).set_text(op)
697
- end
701
+ @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
702
+ Token(TkUMINUS).set_text(op)
703
+ end
698
704
 
699
- @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
700
- Token(TkUMINUS).set_text(op)
701
- end
705
+ @OP.def_rules("+", "-") do |op, io|
706
+ catch(:RET) do
707
+ if @lex_state == EXPR_ARG
708
+ if @space_seen and peek(0) =~ /[0-9]/
709
+ throw :RET, identify_number(op)
710
+ else
711
+ @lex_state = EXPR_BEG
712
+ end
713
+ elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
714
+ throw :RET, identify_number(op)
715
+ else
716
+ @lex_state = EXPR_BEG
717
+ end
718
+ Token(op).set_text(op)
719
+ end
720
+ end
702
721
 
703
- @OP.def_rules("+", "-") do |op, io|
704
- catch(:RET) do
705
- if @lex_state == EXPR_ARG
706
- if @space_seen and peek(0) =~ /[0-9]/
707
- throw :RET, identify_number(op)
708
- else
709
- @lex_state = EXPR_BEG
710
- end
711
- elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
712
- throw :RET, identify_number(op)
713
- else
714
- @lex_state = EXPR_BEG
715
- end
716
- Token(op).set_text(op)
722
+ @OP.def_rule(".") do
723
+ @lex_state = EXPR_BEG
724
+ if peek(0) =~ /[0-9]/
725
+ ungetc
726
+ identify_number("")
727
+ else
728
+ # for obj.if
729
+ @lex_state = EXPR_DOT
730
+ Token(TkDOT).set_text(".")
731
+ end
717
732
  end
718
- end
719
733
 
720
- @OP.def_rule(".") do
721
- @lex_state = EXPR_BEG
722
- if peek(0) =~ /[0-9]/
723
- ungetc
724
- identify_number("")
725
- else
726
- # for obj.if
727
- @lex_state = EXPR_DOT
728
- Token(TkDOT).set_text(".")
734
+ @OP.def_rules("..", "...") do |op, io|
735
+ @lex_state = EXPR_BEG
736
+ Token(op).set_text(op)
729
737
  end
730
- end
731
738
 
732
- @OP.def_rules("..", "...") do |op, io|
733
- @lex_state = EXPR_BEG
734
- Token(op).set_text(op)
739
+ lex_int2
735
740
  end
736
-
737
- lex_int2
738
- end
739
741
 
740
- def lex_int2
741
- @OP.def_rules("]", "}", ")") do
742
- |op, io|
743
- @lex_state = EXPR_END
744
- @indent -= 1
745
- Token(op).set_text(op)
746
- end
742
+ def lex_int2
743
+ @OP.def_rules("]", "}", ")") do
744
+ |op, io|
745
+ @lex_state = EXPR_END
746
+ @indent -= 1
747
+ Token(op).set_text(op)
748
+ end
747
749
 
748
- @OP.def_rule(":") do
749
- if @lex_state == EXPR_END || peek(0) =~ /\s/
750
- @lex_state = EXPR_BEG
751
- tk = Token(TkCOLON)
752
- else
753
- @lex_state = EXPR_FNAME
754
- tk = Token(TkSYMBEG)
750
+ @OP.def_rule(":") do
751
+ if @lex_state == EXPR_END || peek(0) =~ /\s/
752
+ @lex_state = EXPR_BEG
753
+ tk = Token(TkCOLON)
754
+ else
755
+ @lex_state = EXPR_FNAME
756
+ tk = Token(TkSYMBEG)
757
+ end
758
+ tk.set_text(":")
755
759
  end
756
- tk.set_text(":")
757
- end
758
760
 
759
- @OP.def_rule("::") do
760
- # p @lex_state.id2name, @space_seen
761
- if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
762
- @lex_state = EXPR_BEG
763
- tk = Token(TkCOLON3)
764
- else
765
- @lex_state = EXPR_DOT
766
- tk = Token(TkCOLON2)
761
+ @OP.def_rule("::") do
762
+ # p @lex_state.id2name, @space_seen
763
+ if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
764
+ @lex_state = EXPR_BEG
765
+ tk = Token(TkCOLON3)
766
+ else
767
+ @lex_state = EXPR_DOT
768
+ tk = Token(TkCOLON2)
769
+ end
770
+ tk.set_text("::")
767
771
  end
768
- tk.set_text("::")
769
- end
770
772
 
771
- @OP.def_rule("/") do |op, io|
772
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
773
- identify_string(op)
774
- elsif peek(0) == '='
775
- getc
776
- @lex_state = EXPR_BEG
777
- Token(TkOPASGN, :/).set_text("/=") #")
778
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
779
- identify_string(op)
780
- else
781
- @lex_state = EXPR_BEG
782
- Token("/").set_text(op)
773
+ @OP.def_rule("/") do |op, io|
774
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
775
+ identify_string(op)
776
+ elsif peek(0) == '='
777
+ getc
778
+ @lex_state = EXPR_BEG
779
+ Token(TkOPASGN, :/).set_text("/=") #")
780
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
781
+ identify_string(op)
782
+ else
783
+ @lex_state = EXPR_BEG
784
+ Token("/").set_text(op)
785
+ end
783
786
  end
784
- end
785
787
 
786
- @OP.def_rules("^") do
787
- @lex_state = EXPR_BEG
788
- Token("^").set_text("^")
789
- end
788
+ @OP.def_rules("^") do
789
+ @lex_state = EXPR_BEG
790
+ Token("^").set_text("^")
791
+ end
790
792
 
791
- # @OP.def_rules("^=") do
792
- # @lex_state = EXPR_BEG
793
- # Token(TkOPASGN, :^)
794
- # end
793
+ # @OP.def_rules("^=") do
794
+ # @lex_state = EXPR_BEG
795
+ # Token(TkOPASGN, :^)
796
+ # end
795
797
 
796
- @OP.def_rules(",", ";") do |op, io|
797
- @lex_state = EXPR_BEG
798
- Token(op).set_text(op)
799
- end
798
+ @OP.def_rules(",", ";") do |op, io|
799
+ @lex_state = EXPR_BEG
800
+ Token(op).set_text(op)
801
+ end
800
802
 
801
- @OP.def_rule("~") do
802
- @lex_state = EXPR_BEG
803
- Token("~").set_text("~")
804
- end
803
+ @OP.def_rule("~") do
804
+ @lex_state = EXPR_BEG
805
+ Token("~").set_text("~")
806
+ end
805
807
 
806
- @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
807
- @lex_state = EXPR_BEG
808
- Token("~").set_text("~@")
809
- end
808
+ @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
809
+ @lex_state = EXPR_BEG
810
+ Token("~").set_text("~@")
811
+ end
810
812
 
811
- @OP.def_rule("(") do
812
- @indent += 1
813
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
814
- @lex_state = EXPR_BEG
815
- tk = Token(TkfLPAREN)
816
- else
817
- @lex_state = EXPR_BEG
818
- tk = Token(TkLPAREN)
813
+ @OP.def_rule("(") do
814
+ @indent += 1
815
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
816
+ @lex_state = EXPR_BEG
817
+ tk = Token(TkfLPAREN)
818
+ else
819
+ @lex_state = EXPR_BEG
820
+ tk = Token(TkLPAREN)
821
+ end
822
+ tk.set_text("(")
819
823
  end
820
- tk.set_text("(")
821
- end
822
824
 
823
- @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
824
- Token("[]").set_text("[]")
825
- end
825
+ @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
826
+ Token("[]").set_text("[]")
827
+ end
826
828
 
827
- @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
828
- Token("[]=").set_text("[]=")
829
- end
829
+ @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
830
+ Token("[]=").set_text("[]=")
831
+ end
830
832
 
831
- @OP.def_rule("[") do
832
- @indent += 1
833
- if @lex_state == EXPR_FNAME
834
- t = Token(TkfLBRACK)
835
- else
836
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
837
- t = Token(TkLBRACK)
838
- elsif @lex_state == EXPR_ARG && @space_seen
839
- t = Token(TkLBRACK)
840
- else
841
- t = Token(TkfLBRACK)
842
- end
843
- @lex_state = EXPR_BEG
844
- end
845
- t.set_text("[")
846
- end
833
+ @OP.def_rule("[") do
834
+ @indent += 1
835
+ if @lex_state == EXPR_FNAME
836
+ t = Token(TkfLBRACK)
837
+ else
838
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
839
+ t = Token(TkLBRACK)
840
+ elsif @lex_state == EXPR_ARG && @space_seen
841
+ t = Token(TkLBRACK)
842
+ else
843
+ t = Token(TkfLBRACK)
844
+ end
845
+ @lex_state = EXPR_BEG
846
+ end
847
+ t.set_text("[")
848
+ end
847
849
 
848
- @OP.def_rule("{") do
849
- @indent += 1
850
- if @lex_state != EXPR_END && @lex_state != EXPR_ARG
851
- t = Token(TkLBRACE)
852
- else
853
- t = Token(TkfLBRACE)
850
+ @OP.def_rule("{") do
851
+ @indent += 1
852
+ if @lex_state != EXPR_END && @lex_state != EXPR_ARG
853
+ t = Token(TkLBRACE)
854
+ else
855
+ t = Token(TkfLBRACE)
856
+ end
857
+ @lex_state = EXPR_BEG
858
+ t.set_text("{")
854
859
  end
855
- @lex_state = EXPR_BEG
856
- t.set_text("{")
857
- end
858
860
 
859
- @OP.def_rule('\\') do #'
860
- if getc == "\n"
861
- @space_seen = true
862
- @continue = true
863
- Token(TkSPACE).set_text("\\\n")
864
- else
865
- ungetc
866
- Token("\\").set_text("\\") #"
861
+ @OP.def_rule('\\') do #'
862
+ if getc == "\n"
863
+ @space_seen = true
864
+ @continue = true
865
+ Token(TkSPACE).set_text("\\\n")
866
+ else
867
+ ungetc
868
+ Token("\\").set_text("\\") #"
869
+ end
867
870
  end
868
- end
869
-
870
- @OP.def_rule('%') do
871
- |op, io|
872
- if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
873
- identify_quotation('%')
874
- elsif peek(0) == '='
875
- getc
876
- Token(TkOPASGN, "%").set_text("%=")
877
- elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
878
- identify_quotation('%')
879
- else
880
- @lex_state = EXPR_BEG
881
- Token("%").set_text("%")
871
+
872
+ @OP.def_rule('%') do
873
+ |op, io|
874
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
875
+ identify_quotation('%')
876
+ elsif peek(0) == '='
877
+ getc
878
+ Token(TkOPASGN, "%").set_text("%=")
879
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
880
+ identify_quotation('%')
881
+ else
882
+ @lex_state = EXPR_BEG
883
+ Token("%").set_text("%")
884
+ end
882
885
  end
883
- end
884
886
 
885
- @OP.def_rule('$') do #'
886
- identify_gvar
887
- end
887
+ @OP.def_rule('$') do #'
888
+ identify_gvar
889
+ end
888
890
 
889
- @OP.def_rule('@') do
890
- if peek(0) =~ /[@\w_]/
891
- ungetc
892
- identify_identifier
893
- else
894
- Token("@").set_text("@")
891
+ @OP.def_rule('@') do
892
+ if peek(0) =~ /[@\w_]/
893
+ ungetc
894
+ identify_identifier
895
+ else
896
+ Token("@").set_text("@")
897
+ end
895
898
  end
896
- end
897
899
 
898
- # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
899
- # |op, io|
900
- # @indent += 1
901
- # @lex_state = EXPR_FNAME
902
- # # @lex_state = EXPR_END
903
- # # until @rests[0] == "\n" or @rests[0] == ";"
904
- # # rests.shift
905
- # # end
906
- # end
907
-
908
- @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
909
- throw :eof
910
- end
900
+ # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
901
+ # |op, io|
902
+ # @indent += 1
903
+ # @lex_state = EXPR_FNAME
904
+ # # @lex_state = EXPR_END
905
+ # # until @rests[0] == "\n" or @rests[0] == ";"
906
+ # # rests.shift
907
+ # # end
908
+ # end
909
+
910
+ @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
911
+ throw :eof
912
+ end
911
913
 
912
- @OP.def_rule("") do
913
- |op, io|
914
- printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
915
- if peek(0) =~ /[0-9]/
916
- t = identify_number("")
917
- elsif peek(0) =~ /[\w_]/
918
- t = identify_identifier
914
+ @OP.def_rule("") do
915
+ |op, io|
916
+ printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
917
+ if peek(0) =~ /[0-9]/
918
+ t = identify_number("")
919
+ elsif peek(0) =~ /[\w_]/
920
+ t = identify_identifier
921
+ end
922
+ printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
923
+ t
919
924
  end
920
- printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
921
- t
922
- end
923
925
 
924
- p @OP if RubyLex.debug?
925
- end
926
+ p @OP if RubyLex.debug?
927
+ end
926
928
 
927
- def identify_gvar
928
- @lex_state = EXPR_END
929
- str = "$"
930
-
931
- tk = case ch = getc
932
- when /[~_*$?!@\/\\;,=:<>".]/ #"
933
- str << ch
934
- Token(TkGVAR, str)
929
+ def identify_gvar
930
+ @lex_state = EXPR_END
931
+ str = "$"
932
+
933
+ tk = case ch = getc
934
+ when /[~_*$?!@\/\\;,=:<>".]/ #"
935
+ str << ch
936
+ Token(TkGVAR, str)
935
937
 
936
- when "-"
937
- str << "-" << getc
938
- Token(TkGVAR, str)
938
+ when "-"
939
+ str << "-" << getc
940
+ Token(TkGVAR, str)
939
941
 
940
- when "&", "`", "'", "+"
941
- str << ch
942
- Token(TkBACK_REF, str)
942
+ when "&", "`", "'", "+"
943
+ str << ch
944
+ Token(TkBACK_REF, str)
943
945
 
944
- when /[1-9]/
945
- str << ch
946
- while (ch = getc) =~ /[0-9]/
946
+ when /[1-9]/
947
947
  str << ch
948
+ while (ch = getc) =~ /[0-9]/
949
+ str << ch
950
+ end
951
+ ungetc
952
+ Token(TkNTH_REF)
953
+ when /\w/
954
+ ungetc
955
+ ungetc
956
+ return identify_identifier
957
+ else
958
+ ungetc
959
+ Token("$")
948
960
  end
949
- ungetc
950
- Token(TkNTH_REF)
951
- when /\w/
952
- ungetc
953
- ungetc
954
- return identify_identifier
955
- else
956
- ungetc
957
- Token("$")
958
- end
959
- tk.set_text(str)
960
- end
961
-
962
- def identify_identifier
963
- token = ""
964
- token.concat getc if peek(0) =~ /[$@]/
965
- token.concat getc if peek(0) == "@"
966
-
967
- while (ch = getc) =~ /\w|_/
968
- print ":", ch, ":" if RubyLex.debug?
969
- token.concat ch
961
+ tk.set_text(str)
970
962
  end
971
- ungetc
963
+
964
+ def identify_identifier
965
+ token = ""
966
+ token.concat getc if peek(0) =~ /[$@]/
967
+ token.concat getc if peek(0) == "@"
968
+
969
+ while (ch = getc) =~ /\w|_/
970
+ print ":", ch, ":" if RubyLex.debug?
971
+ token.concat ch
972
+ end
973
+ ungetc
972
974
 
973
- if ch == "!" or ch == "?"
974
- token.concat getc
975
- end
976
- # fix token
975
+ if ch == "!" or ch == "?"
976
+ token.concat getc
977
+ end
978
+ # fix token
977
979
 
978
- # $stderr.puts "identifier - #{token}, state = #@lex_state"
980
+ # $stderr.puts "identifier - #{token}, state = #@lex_state"
979
981
 
980
- case token
981
- when /^\$/
982
- return Token(TkGVAR, token).set_text(token)
983
- when /^\@/
984
- @lex_state = EXPR_END
985
- return Token(TkIVAR, token).set_text(token)
986
- end
982
+ case token
983
+ when /^\$/
984
+ return Token(TkGVAR, token).set_text(token)
985
+ when /^\@/
986
+ @lex_state = EXPR_END
987
+ return Token(TkIVAR, token).set_text(token)
988
+ end
987
989
 
988
- if @lex_state != EXPR_DOT
989
- print token, "\n" if RubyLex.debug?
990
-
991
- token_c, *trans = TkReading2Token[token]
992
- if token_c
993
- # reserved word?
994
-
995
- if (@lex_state != EXPR_BEG &&
996
- @lex_state != EXPR_FNAME &&
997
- trans[1])
998
- # modifiers
999
- token_c = TkSymbol2Token[trans[1]]
1000
- @lex_state = trans[0]
1001
- else
1002
- if @lex_state != EXPR_FNAME
1003
- if ENINDENT_CLAUSE.include?(token)
1004
- @indent += 1
1005
- elsif DEINDENT_CLAUSE.include?(token)
1006
- @indent -= 1
1007
- end
1008
- @lex_state = trans[0]
1009
- else
1010
- @lex_state = EXPR_END
1011
- end
1012
- end
1013
- return Token(token_c, token).set_text(token)
990
+ if @lex_state != EXPR_DOT
991
+ print token, "\n" if RubyLex.debug?
992
+
993
+ token_c, *trans = TkReading2Token[token]
994
+ if token_c
995
+ # reserved word?
996
+
997
+ if (@lex_state != EXPR_BEG &&
998
+ @lex_state != EXPR_FNAME &&
999
+ trans[1])
1000
+ # modifiers
1001
+ token_c = TkSymbol2Token[trans[1]]
1002
+ @lex_state = trans[0]
1003
+ else
1004
+ if @lex_state != EXPR_FNAME
1005
+ if ENINDENT_CLAUSE.include?(token)
1006
+ @indent += 1
1007
+ elsif DEINDENT_CLAUSE.include?(token)
1008
+ @indent -= 1
1009
+ end
1010
+ @lex_state = trans[0]
1011
+ else
1012
+ @lex_state = EXPR_END
1013
+ end
1014
+ end
1015
+ return Token(token_c, token).set_text(token)
1016
+ end
1014
1017
  end
1015
- end
1016
1018
 
1017
- if @lex_state == EXPR_FNAME
1018
- @lex_state = EXPR_END
1019
- if peek(0) == '='
1020
- token.concat getc
1019
+ if @lex_state == EXPR_FNAME
1020
+ @lex_state = EXPR_END
1021
+ if peek(0) == '='
1022
+ token.concat getc
1023
+ end
1024
+ elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
1025
+ @lex_state = EXPR_ARG
1026
+ else
1027
+ @lex_state = EXPR_END
1021
1028
  end
1022
- elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
1023
- @lex_state = EXPR_ARG
1024
- else
1025
- @lex_state = EXPR_END
1026
- end
1027
1029
 
1028
- if token[0, 1] =~ /[A-Z]/
1029
- return Token(TkCONSTANT, token).set_text(token)
1030
- elsif token[token.size - 1, 1] =~ /[!?]/
1031
- return Token(TkFID, token).set_text(token)
1032
- else
1033
- return Token(TkIDENTIFIER, token).set_text(token)
1030
+ if token[0, 1] =~ /[A-Z]/
1031
+ return Token(TkCONSTANT, token).set_text(token)
1032
+ elsif token[token.size - 1, 1] =~ /[!?]/
1033
+ return Token(TkFID, token).set_text(token)
1034
+ else
1035
+ return Token(TkIDENTIFIER, token).set_text(token)
1036
+ end
1034
1037
  end
1035
- end
1036
1038
 
1037
- def identify_here_document
1038
- ch = getc
1039
- if ch == "-"
1039
+ def identify_here_document
1040
1040
  ch = getc
1041
- indent = true
1042
- end
1043
- if /['"`]/ =~ ch # '
1044
- lt = ch
1045
- quoted = ""
1046
- while (c = getc) && c != lt
1047
- quoted.concat c
1048
- end
1049
- else
1050
- lt = '"'
1051
- quoted = ch.dup
1052
- while (c = getc) && c =~ /\w/
1053
- quoted.concat c
1041
+ if ch == "-"
1042
+ ch = getc
1043
+ indent = true
1044
+ end
1045
+ if /['"`]/ =~ ch # '
1046
+ lt = ch
1047
+ quoted = ""
1048
+ while (c = getc) && c != lt
1049
+ quoted.concat c
1050
+ end
1051
+ else
1052
+ lt = '"'
1053
+ quoted = ch.dup
1054
+ while (c = getc) && c =~ /\w/
1055
+ quoted.concat c
1056
+ end
1057
+ ungetc
1054
1058
  end
1055
- ungetc
1056
- end
1057
1059
 
1058
- ltback, @ltype = @ltype, lt
1059
- reserve = ""
1060
+ ltback, @ltype = @ltype, lt
1061
+ reserve = ""
1060
1062
 
1061
- while ch = getc
1062
- reserve << ch
1063
- if ch == "\\" #"
1064
- ch = getc
1065
- reserve << ch
1066
- elsif ch == "\n"
1067
- break
1063
+ while ch = getc
1064
+ reserve << ch
1065
+ if ch == "\\" #"
1066
+ ch = getc
1067
+ reserve << ch
1068
+ elsif ch == "\n"
1069
+ break
1070
+ end
1068
1071
  end
1069
- end
1070
1072
 
1071
- str = ""
1072
- while (l = gets)
1073
- l.chomp!
1074
- l.strip! if indent
1075
- break if l == quoted
1076
- str << l.chomp << "\n"
1077
- end
1073
+ str = ""
1074
+ while (l = gets)
1075
+ l.chomp!
1076
+ l.strip! if indent
1077
+ break if l == quoted
1078
+ str << l.chomp << "\n"
1079
+ end
1078
1080
 
1079
- @reader.divert_read_from(reserve)
1081
+ @reader.divert_read_from(reserve)
1080
1082
 
1081
- @ltype = ltback
1082
- @lex_state = EXPR_END
1083
- Token(Ltype2Token[lt], str).set_text(str.dump)
1084
- end
1083
+ @ltype = ltback
1084
+ @lex_state = EXPR_END
1085
+ Token(Ltype2Token[lt], str).set_text(str.dump)
1086
+ end
1085
1087
 
1086
- def identify_quotation(initial_char)
1087
- ch = getc
1088
- if lt = PERCENT_LTYPE[ch]
1089
- initial_char += ch
1088
+ def identify_quotation(initial_char)
1090
1089
  ch = getc
1091
- elsif ch =~ /\W/
1092
- lt = "\""
1093
- else
1094
- RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
1090
+ if lt = PERCENT_LTYPE[ch]
1091
+ initial_char += ch
1092
+ ch = getc
1093
+ elsif ch =~ /\W/
1094
+ lt = "\""
1095
+ else
1096
+ RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
1097
+ end
1098
+ # if ch !~ /\W/
1099
+ # ungetc
1100
+ # next
1101
+ # end
1102
+ #@ltype = lt
1103
+ @quoted = ch unless @quoted = PERCENT_PAREN[ch]
1104
+ identify_string(lt, @quoted, ch, initial_char)
1095
1105
  end
1096
- # if ch !~ /\W/
1097
- # ungetc
1098
- # next
1099
- # end
1100
- #@ltype = lt
1101
- @quoted = ch unless @quoted = PERCENT_PAREN[ch]
1102
- identify_string(lt, @quoted, ch, initial_char)
1103
- end
1104
1106
 
1105
- def identify_number(start)
1106
- str = start.dup
1107
+ def identify_number(start)
1108
+ str = start.dup
1107
1109
 
1108
- if start == "+" or start == "-" or start == ""
1109
- start = getc
1110
- str << start
1111
- end
1110
+ if start == "+" or start == "-" or start == ""
1111
+ start = getc
1112
+ str << start
1113
+ end
1112
1114
 
1113
- @lex_state = EXPR_END
1115
+ @lex_state = EXPR_END
1114
1116
 
1115
- if start == "0"
1116
- if peek(0) == "x"
1117
- ch = getc
1118
- str << ch
1119
- match = /[0-9a-f_]/
1120
- else
1121
- match = /[0-7_]/
1122
- end
1123
- while ch = getc
1124
- if ch !~ match
1125
- ungetc
1126
- break
1127
- else
1117
+ if start == "0"
1118
+ if peek(0) == "x"
1119
+ ch = getc
1128
1120
  str << ch
1121
+ match = /[0-9a-f_]/
1122
+ else
1123
+ match = /[0-7_]/
1129
1124
  end
1125
+ while ch = getc
1126
+ if ch !~ match
1127
+ ungetc
1128
+ break
1129
+ else
1130
+ str << ch
1131
+ end
1132
+ end
1133
+ return Token(TkINTEGER).set_text(str)
1130
1134
  end
1131
- return Token(TkINTEGER).set_text(str)
1132
- end
1133
1135
 
1134
- type = TkINTEGER
1135
- allow_point = TRUE
1136
- allow_e = TRUE
1137
- while ch = getc
1138
- case ch
1139
- when /[0-9_]/
1140
- str << ch
1141
-
1142
- when allow_point && "."
1143
- type = TkFLOAT
1144
- if peek(0) !~ /[0-9]/
1145
- ungetc
1146
- break
1147
- end
1148
- str << ch
1149
- allow_point = false
1150
-
1151
- when allow_e && "e", allow_e && "E"
1152
- str << ch
1153
- type = TkFLOAT
1154
- if peek(0) =~ /[+-]/
1155
- str << getc
1156
- end
1157
- allow_e = false
1158
- allow_point = false
1159
- else
1160
- ungetc
1161
- break
1136
+ type = TkINTEGER
1137
+ allow_point = TRUE
1138
+ allow_e = TRUE
1139
+ while ch = getc
1140
+ case ch
1141
+ when /[0-9_]/
1142
+ str << ch
1143
+
1144
+ when allow_point && "."
1145
+ type = TkFLOAT
1146
+ if peek(0) !~ /[0-9]/
1147
+ ungetc
1148
+ break
1149
+ end
1150
+ str << ch
1151
+ allow_point = false
1152
+
1153
+ when allow_e && "e", allow_e && "E"
1154
+ str << ch
1155
+ type = TkFLOAT
1156
+ if peek(0) =~ /[+-]/
1157
+ str << getc
1158
+ end
1159
+ allow_e = false
1160
+ allow_point = false
1161
+ else
1162
+ ungetc
1163
+ break
1164
+ end
1162
1165
  end
1166
+ Token(type).set_text(str)
1163
1167
  end
1164
- Token(type).set_text(str)
1165
- end
1166
1168
 
1167
- def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
1168
- @ltype = ltype
1169
- @quoted = quoted
1170
- subtype = nil
1171
-
1172
- str = ""
1173
- str << initial_char if initial_char
1174
- str << (opener||quoted)
1175
-
1176
- nest = 0
1177
- begin
1178
- while ch = getc
1179
- str << ch
1180
- if @quoted == ch
1181
- if nest == 0
1182
- break
1183
- else
1184
- nest -= 1
1185
- end
1186
- elsif opener == ch
1187
- nest += 1
1188
- elsif @ltype != "'" && @ltype != "]" and ch == "#"
1189
- ch = getc
1190
- if ch == "{"
1191
- subtype = true
1192
- str << ch << skip_inner_expression
1193
- else
1194
- ungetc(ch)
1195
- end
1196
- elsif ch == '\\' #'
1197
- str << read_escape
1198
- end
1199
- end
1200
- if @ltype == "/"
1201
- if peek(0) =~ /i|o|n|e|s/
1202
- str << getc
1203
- end
1169
+ def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
1170
+ @ltype = ltype
1171
+ @quoted = quoted
1172
+ subtype = nil
1173
+
1174
+ str = ""
1175
+ str << initial_char if initial_char
1176
+ str << (opener||quoted)
1177
+
1178
+ nest = 0
1179
+ begin
1180
+ while ch = getc
1181
+ str << ch
1182
+ if @quoted == ch
1183
+ if nest == 0
1184
+ break
1185
+ else
1186
+ nest -= 1
1187
+ end
1188
+ elsif opener == ch
1189
+ nest += 1
1190
+ elsif @ltype != "'" && @ltype != "]" and ch == "#"
1191
+ ch = getc
1192
+ if ch == "{"
1193
+ subtype = true
1194
+ str << ch << skip_inner_expression
1195
+ else
1196
+ ungetc(ch)
1197
+ end
1198
+ elsif ch == '\\' #'
1199
+ str << read_escape
1200
+ end
1201
+ end
1202
+ if @ltype == "/"
1203
+ if peek(0) =~ /i|o|n|e|s/
1204
+ str << getc
1205
+ end
1206
+ end
1207
+ if subtype
1208
+ Token(DLtype2Token[ltype], str)
1209
+ else
1210
+ Token(Ltype2Token[ltype], str)
1211
+ end.set_text(str)
1212
+ ensure
1213
+ @ltype = nil
1214
+ @quoted = nil
1215
+ @lex_state = EXPR_END
1204
1216
  end
1205
- if subtype
1206
- Token(DLtype2Token[ltype], str)
1207
- else
1208
- Token(Ltype2Token[ltype], str)
1209
- end.set_text(str)
1210
- ensure
1211
- @ltype = nil
1212
- @quoted = nil
1213
- @lex_state = EXPR_END
1214
1217
  end
1215
- end
1216
1218
 
1217
- def skip_inner_expression
1218
- res = ""
1219
- nest = 0
1220
- while (ch = getc)
1221
- res << ch
1222
- if ch == '}'
1223
- break if nest.zero?
1224
- nest -= 1
1225
- elsif ch == '{'
1226
- nest += 1
1219
+ def skip_inner_expression
1220
+ res = ""
1221
+ nest = 0
1222
+ while (ch = getc)
1223
+ res << ch
1224
+ if ch == '}'
1225
+ break if nest.zero?
1226
+ nest -= 1
1227
+ elsif ch == '{'
1228
+ nest += 1
1229
+ end
1227
1230
  end
1231
+ res
1228
1232
  end
1229
- res
1230
- end
1231
1233
 
1232
- def identify_comment
1233
- @ltype = "#"
1234
- comment = "#"
1235
- while ch = getc
1236
- if ch == "\\"
1237
- ch = getc
1238
- if ch == "\n"
1239
- ch = " "
1234
+ def identify_comment
1235
+ @ltype = "#"
1236
+ comment = "#"
1237
+ while ch = getc
1238
+ if ch == "\\"
1239
+ ch = getc
1240
+ if ch == "\n"
1241
+ ch = " "
1242
+ else
1243
+ comment << "\\"
1244
+ end
1240
1245
  else
1241
- comment << "\\"
1242
- end
1243
- else
1244
- if ch == "\n"
1245
- @ltype = nil
1246
- ungetc
1247
- break
1246
+ if ch == "\n"
1247
+ @ltype = nil
1248
+ ungetc
1249
+ break
1250
+ end
1248
1251
  end
1252
+ comment << ch
1249
1253
  end
1250
- comment << ch
1254
+ return Token(TkCOMMENT).set_text(comment)
1251
1255
  end
1252
- return Token(TkCOMMENT).set_text(comment)
1253
- end
1254
1256
 
1255
- def read_escape
1256
- res = ""
1257
- case ch = getc
1258
- when /[0-7]/
1259
- ungetc ch
1260
- 3.times do
1261
- case ch = getc
1262
- when /[0-7]/
1263
- when nil
1264
- break
1265
- else
1266
- ungetc
1267
- break
1268
- end
1269
- res << ch
1270
- end
1257
+ def read_escape
1258
+ res = ""
1259
+ case ch = getc
1260
+ when /[0-7]/
1261
+ ungetc ch
1262
+ 3.times do
1263
+ case ch = getc
1264
+ when /[0-7]/
1265
+ when nil
1266
+ break
1267
+ else
1268
+ ungetc
1269
+ break
1270
+ end
1271
+ res << ch
1272
+ end
1271
1273
 
1272
- when "x"
1273
- res << ch
1274
- 2.times do
1275
- case ch = getc
1276
- when /[0-9a-fA-F]/
1277
- when nil
1278
- break
1279
- else
1280
- ungetc
1281
- break
1282
- end
1274
+ when "x"
1283
1275
  res << ch
1284
- end
1276
+ 2.times do
1277
+ case ch = getc
1278
+ when /[0-9a-fA-F]/
1279
+ when nil
1280
+ break
1281
+ else
1282
+ ungetc
1283
+ break
1284
+ end
1285
+ res << ch
1286
+ end
1285
1287
 
1286
- when "M"
1287
- res << ch
1288
- if (ch = getc) != '-'
1289
- ungetc
1290
- else
1288
+ when "M"
1291
1289
  res << ch
1292
- if (ch = getc) == "\\" #"
1293
- res << ch
1294
- res << read_escape
1290
+ if (ch = getc) != '-'
1291
+ ungetc
1295
1292
  else
1296
1293
  res << ch
1297
- end
1298
- end
1294
+ if (ch = getc) == "\\" #"
1295
+ res << ch
1296
+ res << read_escape
1297
+ else
1298
+ res << ch
1299
+ end
1300
+ end
1299
1301
 
1300
- when "C", "c" #, "^"
1301
- res << ch
1302
- if ch == "C" and (ch = getc) != "-"
1303
- ungetc
1304
- else
1302
+ when "C", "c" #, "^"
1305
1303
  res << ch
1306
- if (ch = getc) == "\\" #"
1307
- res << ch
1308
- res << read_escape
1304
+ if ch == "C" and (ch = getc) != "-"
1305
+ ungetc
1309
1306
  else
1310
1307
  res << ch
1308
+ if (ch = getc) == "\\" #"
1309
+ res << ch
1310
+ res << read_escape
1311
+ else
1312
+ res << ch
1313
+ end
1311
1314
  end
1315
+ else
1316
+ res << ch
1312
1317
  end
1313
- else
1314
- res << ch
1318
+ res
1315
1319
  end
1316
- res
1317
1320
  end
1318
- end
1321
+ end