sdl4r 0.9.9 → 0.9.11

data/lib/sdl4r/tag.rb

@@ -25,8 +25,8 @@ module SDL4R
   require 'open-uri'
   require 'stringio'
   
-  require File.dirname(__FILE__) + '/sdl4r'
-  require File.dirname(__FILE__) + '/parser'
+  require 'sdl4r/sdl4r'
+  require 'sdl4r/reader'
 
   # SDL documents are made of Tags.
   #
@@ -56,18 +56,54 @@ module SDL4R
     #
     # Use at the beginning of a method in order to have correctly defined parameters:
     #   def foo(namespace, name = nil)
-    #     namespace, name = to_nns namespace, name
+    #     namespace, name = to_ns_n namespace, name
     #   end
     #
-    def to_nns(namespace, name)
+    # Also converts Symbols to Strings.
+    #
+    def to_ns_n(namespace, name)
+      namespace = namespace.id2name if namespace.is_a? Symbol
+      name = name.id2name if name.is_a? Symbol
+      
       if name.nil? and not namespace.nil?
         name = namespace
         namespace = ""
       end
       return namespace, name
     end
-    private :to_nns
-    
+    private :to_ns_n
+
+    # Returns [recursive, namespace, name] based on the parameters.
+    # Allows:
+    #   to_rec_ns_n()
+    #   to_rec_ns_n(recursive)
+    #   to_rec_ns_n(recursive, name)
+    #   to_rec_ns_n(name)
+    #   to_rec_ns_n(namespace, name)
+    #   to_rec_ns_n(recursive, namespace, name)
+    #
+    def to_rec_ns_n(*args)
+      raise ArgumentError, "too many arguments: #{args.length}" if args.length > 3
+
+      recursive = false
+      if args.length > 0 and (args[0].is_a? TrueClass or args[0].is_a? FalseClass)
+        recursive = args.shift
+      elsif args.length > 2
+        raise ArgumentError, "too many or wrong arguments: #{args.length}"
+      end
+
+      namespace = nil # we look at any namespace by default
+      namespace = args.shift if args.length == 2
+      namespace = namespace.id2name if namespace.is_a? Symbol
+
+      name = nil
+      name = args.shift if args.length == 1
+      name = name.id2name if name.is_a? Symbol
+
+      return recursive, namespace, name
+    end
+    private :to_rec_ns_n
+
     # Creates an empty tag in the given namespace.  If the +namespace+ is nil
     # it will be coerced to an empty String.
     #
@@ -104,7 +140,8 @@ module SDL4R
     # and is not a legal SDL identifier.
     #
     def initialize(namespace, name = nil, &block)
-      namespace, name = to_nns namespace, name
+      namespace, name = to_ns_n namespace, name
+
 
       raise ArgumentError, "tag namespace must be a String" unless namespace.is_a? String
       raise ArgumentError, "tag name must be a String" unless name.is_a? String
@@ -165,6 +202,7 @@ module SDL4R
     # Returns the added child.
     #
     def add_child(child)
+      raise ArgumentError, "child is nil" unless child
       @children.push(child)
       return child
     end
@@ -200,6 +238,7 @@ module SDL4R
         add_child(o)
       elsif o.is_a?(Hash)
         o.each_pair { |key, value|
+          key = key.to_s if key.is_a? Symbol
           namespace, key = key.split(/:/) if key.match(/:/)
           namespace ||= ""
           set_attribute(namespace, key, value)
@@ -286,11 +325,8 @@ module SDL4R
     #   tag.children(false, "name") # => children of name "name"
     #   tag.children(false, "ns", nil) # => children of namespace "ns"
     #
-    def children(recursive = false, namespace = nil, name = :DEFAULT, &block) # :yields: child
-      if name == :DEFAULT
-        name = namespace
-        namespace = nil
-      end
+    def children(*args, &block) # :yields: child
+      recursive, namespace, name = to_rec_ns_n(*args)
 
       if block_given?
         each_child(recursive, namespace, name, &block)
@@ -316,6 +352,8 @@ module SDL4R
     #
     # _name_:: if nil, all children are considered (nil by default).
     def children_values(name = nil)
+      name = name.id2name if name.is_a? Symbol
+      
       children_values = []
       each_child(false, name) { |child|
         case child.values.size
@@ -330,9 +368,11 @@ module SDL4R
       return children_values
     end
 
-    #   child
-    #   child(name)
-    #   child(recursive, name)
+    #   child # => first child
+    #   child(name) # => first child of specified name (any namespace)
+    #   child(true, name) # => first child (or descendant) of specified name (any namspace)
+    #   child(namespace, name) # => first child of specified namespace and name
+    #   child(recursive, namespace, name)
     #
     # Get the first child with the given name, optionally using a recursive search.
     # 
@@ -341,16 +381,13 @@ module SDL4R
     # 
     # Returns the first child tag having the given name or +nil+ if no such child exists
     #
-    def child(recursive = false, name = nil)
-      if name.nil?
-        name = recursive
-        recursive = false
-      end
-      
-      unless name
+    def child(*args)
+      recursive, namespace, name = to_rec_ns_n(*args)
+
+      if namespace.nil? and name.nil?
         return @children.first
       else
-        each_child(recursive, name) { |child| return child }
+        each_child(recursive, namespace, name) { |child| return child }
       end
     end
 
@@ -358,8 +395,16 @@ module SDL4R
     #
     # _name_:: name of the searched child Tag
     #
-    def has_child?(name)
-      !child(name).nil?
+    #   has_child?(name)
+    #   has_child?(namespace, name)
+    #
+    def has_child?(namespace, name = :DEFAULT)
+      if name == :DEFAULT
+        name = namespace
+        namespace = nil
+      end
+      
+      !child(namespace, name).nil?
     end
 
     # Indicates whether there are children Tag.
@@ -492,7 +537,7 @@ module SDL4R
     #
     def values=(someValues)
       @values.clear()
-      someValues.to_a.each { |v|
+      someValues.each { |v|
         # this is required to ensure validation of types
         add_value(v)
       }
@@ -519,13 +564,14 @@ module SDL4R
     #
     def set_attribute(namespace, key, value = :default)
       if value == :default
-        value = key
-        key = namespace
-        namespace = ""
+        value, key, namespace = key, namespace, ""
       end
+      namespace = namespace.id2name if namespace.is_a? Symbol
+      key = key.id2name if key.is_a? Symbol
 
-      raise ArgumentError, "attribute namespace must be a String" unless namespace.is_a? String
-      raise ArgumentError, "attribute key must be a String" unless key.is_a? String
+      raise ArgumentError,
+        "attribute namespace must be a String or a Symbol" unless namespace.is_a? String
+      raise ArgumentError, "attribute key must be a String or a Symbol" unless key.is_a? String
       raise ArgumentError, "attribute key cannot be empty" if key.empty?
 
       SDL4R.validate_identifier(namespace) unless namespace.empty?
@@ -548,7 +594,7 @@ module SDL4R
     #
     #
     def attribute(namespace, key = nil)
-      namespace, key = to_nns namespace, key
+      namespace, key = to_ns_n namespace, key
       attributes = @attributesByNamespace[namespace]
       return attributes.nil? ? nil : attributes[key]
     end
@@ -561,7 +607,7 @@ module SDL4R
     #   has_attribute?(namespace, key)
     #
     def has_attribute?(namespace = nil, key = nil)
-      namespace, key = to_nns namespace, key
+      namespace, key = to_ns_n namespace, key
 
       if namespace or key
         attributes = @attributesByNamespace[namespace]
@@ -590,6 +636,8 @@ module SDL4R
     # qualified names (e.g. "meat:color"). If "", attributes of the default namespace are returned.
     #
     def attributes(namespace = nil, &block) # :yields: namespace, key, value
+      namespace = namespace.id2name if namespace.is_a? Symbol
+      
       if block_given?
         each_attribute(namespace, &block)
         
@@ -597,8 +645,8 @@ module SDL4R
         if namespace.nil?
           hash = {}
 
-          each_attribute do | namespace, key, value |
-            qualified_name = namespace.empty? ? key : namespace + ':' + key
+          each_attribute do | attr_namespace, key, value |
+            qualified_name = attr_namespace.empty? ? key : attr_namespace + ':' + key
             hash[qualified_name] = value
           end
 
@@ -621,7 +669,7 @@ module SDL4R
     # Returns the value of the removed attribute or +nil+ if it didn't exist.
     #
     def remove_attribute(namespace, key = nil)
-      namespace, key = to_nns namespace, key
+      namespace, key = to_ns_n namespace, key
       attributes = @attributesByNamespace[namespace]
       return attributes.nil? ? nil : attributes.delete(key)
     end
@@ -633,6 +681,7 @@ module SDL4R
       if namespace.nil?
         @attributesByNamespace.clear
       else
+        namespace = namespace.id2name if namespace.is_a? Symbol
         @attributesByNamespace.delete(namespace)
       end
     end
@@ -671,9 +720,9 @@ module SDL4R
     #
     def set_attributes(namespace, attribute_hash = nil)
       if attribute_hash.nil?
-        attribute_hash = namespace
-        namespace = ""
+        attribute_hash, namespace = namespace, ""
       end
+      namespace = namespace.id2name if namespace.is_a? Symbol
       
       raise ArgumentError, "namespace can't be nil" if namespace.nil?
       raise ArgumentError, "attribute_hash should be a Hash" unless attribute_hash.is_a? Hash
@@ -701,10 +750,10 @@ module SDL4R
     # Raises +ArgumentError+ if the name is not a legal SDL identifier
     # (see SDL4R#validate_identifier).
     #
-    def name=(a_name)
-      a_name = a_name.to_s
-      SDL4R.validate_identifier(a_name)
-      @name = a_name
+    def name=(name)
+      name = name.id2name if name.is_a? Symbol
+      SDL4R.validate_identifier(name)
+      @name = name
     end
   
     # The namespace to set. +nil+ will be coerced to the empty string.
@@ -712,10 +761,10 @@ module SDL4R
     # Raises +ArgumentError+ if the namespace is non-blank and is not
     # a legal SDL identifier (see SDL4R#validate_identifier)
     #
-    def namespace=(a_namespace)
-      a_namespace = a_namespace.to_s
-      SDL4R.validate_identifier(a_namespace) unless a_namespace.empty?
-      @namespace = a_namespace
+    def namespace=(namespace)
+      namespace = namespace.id2name if namespace.is_a? Symbol
+      SDL4R.validate_identifier(namespace) unless namespace.empty?
+      @namespace = namespace
     end
         
     # Adds all the tags specified in the given IO, String, Pathname or URI to this Tag.
@@ -745,7 +794,7 @@ module SDL4R
       io = yield
 
       begin
-        Parser.new(io).parse.each do |tag|
+        SDL4R::Reader.from_io(io).each_tag(true) do |tag|
           add_child(tag)
         end
 
@@ -867,7 +916,7 @@ module SDL4R
 
     # Returns a string representation of the children tags.
     #
-    # _linePrefix_:: A prefix to insert before every line.
+    # _linePrefix_:: a prefix to insert before every line.
     # _s_:: a String that receives the string representation
     #
     # TODO: break up long lines using the backslash
@@ -947,12 +996,18 @@ module SDL4R
           end
         end
       end
-      
+
       # output values
       unless @values.empty?
         i = 0
         @values.each do |value|
-          s << " _val" << i.to_s << "=\"" << SDL4R.format(value, false) << "\""
+          s << " _val" << i.to_s << "=\""
+          if value.is_a? String
+            s << value_string_to_xml(value, true)
+          else
+            s << SDL4R.format(value, false)
+          end
+          s << "\""
           i += 1
         end
       end
@@ -964,7 +1019,13 @@ module SDL4R
           unless omit_null_attributes and attribute_value.nil?
             s << " "
             s << "#{attribute_namespace}:" unless attribute_namespace.empty?
-            s << attribute_name << "=\"" << SDL4R.format(attribute_value, false) << ?"
+            s << attribute_name << "=\""
+            if attribute_value.is_a? String
+              s << value_string_to_xml(attribute_value, true)
+            else
+              s << SDL4R.format(attribute_value, false)
+            end
+            s << ?"
           end
         end
       end
@@ -984,5 +1045,19 @@ module SDL4R
   
       return s
     end
+
+    # @private
+    def value_string_to_xml(s, in_quotes = true)
+      s = s.clone
+      if in_quotes
+        s.gsub!(/"/, "&quot;")
+        s.gsub!(/'/, "&apos;")
+      end
+      s.gsub!(/</, "&gt;")
+      s.gsub!(/>/, "&lt;")
+      s.gsub!(/&/, "&amp;")
+      s
+    end
+    private :value_string_to_xml
   end
 end

data/lib/sdl4r/token.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+#--
+# Simple Declarative Language (SDL) for Ruby
+# Copyright 2005 Ikayzo, inc.
+#
+# This program is free software. You can distribute or modify it under the
+# terms of the GNU Lesser General Public License version 2.1 as published by
+# the Free Software Foundation.
+#
+# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
+# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, contact the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#++
+
+module SDL4R
+  # Used internally by Tokenizer for keeping track of its state.
+  # Don't use directly.
+  # @private
+  class Token
+
+    def initialize(text, type, matcher, line_no, pos)
+      @text = text
+      @type = type
+      @matcher = matcher
+      @line_no = line_no
+      @pos = pos
+    end
+
+    # Line number of the token
+    attr_accessor :line_no
+    # Position of the token in the line
+    attr_accessor :pos
+    # Type of token (e.g. :WHITESPACE)
+    attr_accessor :type
+    # Matcher object associated that discovered this Token
+    attr_accessor :matcher
+    # The token text
+    attr_accessor :text
+
+    self.freeze
+  end
+end
+

data/lib/sdl4r/tokenizer.rb

@@ -0,0 +1,431 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+#--
+# Simple Declarative Language (SDL) for Ruby
+# Copyright 2005 Ikayzo, inc.
+#
+# This program is free software. You can distribute or modify it under the
+# terms of the GNU Lesser General Public License version 2.1 as published by
+# the Free Software Foundation.
+#
+# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
+# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, contact the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#++
+
+module SDL4R
+
+  require 'strscan'
+
+  require 'sdl4r/sdl_parse_error'
+  require 'sdl4r/token'
+
+  # Tokenizer for SDL.
+  #
+  # As Ruby's IO standard libraries are not so much low-level, this class works on lines. This
+  # means that some token types reflect this line-oriented tokenizing.
+  #
+  # The other solution would be to implement a proper tokenizer natively, which I don't feel like
+  # doing right now.
+  #
+  #--
+  # FIXME: implement a way of stacking the errors without raising an error immediately
+  #++
+  #
+  class Tokenizer # :nodoc: all
+
+    class Matcher # :nodoc: all
+      def initialize(token_type, regex, options = {}, &block)
+        options = {
+          :next_mode => nil,
+          :push_back_eol => false,
+          :error => nil,
+        }.merge(options)
+
+        @token_type = token_type
+        @regex = regex
+        @next_mode = options[:next_mode]
+        @push_back_eol = options[:push_back_eol]
+        @error = options[:error]
+
+        if block_given?
+          instance_eval(&block)
+        end
+      end
+      attr_reader :token_type, :regex, :next_mode
+
+      # Indicates whether the matched token tends to match the end of line character and whether
+      # it should be pushed back in this cases.
+      attr_reader :push_back_eol
+
+      # If +nil+, this Matcher is normal, otherwise it is meant to detect errors and this
+      # returns a message.
+      attr_reader :error
+
+      # Called when a token is found in order to remove meaningless characters, etc.
+      def process_token(token)
+        token
+      end
+
+      self.freeze
+    end
+
+    # A string used at the end of each line in order to trigger the EOL token.
+    # @private
+    @@EOL_STRING = "\n"
+
+    # @private
+    @@matcher_sets = {
+      :top => [
+        Matcher.new(:EOL, /\A\n/),
+        Matcher.new(:WHITESPACE, /\A\s+/, :push_back_eol => true),
+        Matcher.new(:SEMICOLON, /\A;/),
+        Matcher.new(:COLON, /\A:/),
+        Matcher.new(:EQUAL, /\A=/),
+        Matcher.new(:BLOCK_START, /\A\{/),
+        Matcher.new(:BLOCK_END, /\A\}/),
+        Matcher.new(:BOOLEAN, /\Atrue|false|on|off/),
+        Matcher.new(:NULL, /\Anull/),
+        Matcher.new(:ONE_LINE_COMMENT, /\A(?:#|--|\/\/).*\Z/, :push_back_eol => true) do
+          def process_token(token)
+            token.gsub!(/\A(?:#|--|\/\/)/, "")
+          end
+        end,
+        Matcher.new(:INLINE_COMMENT, /\A\/\*[\s\S]*?\*\//) do
+          def process_token(token)
+            token.gsub!(/\A\/\*|\*\/\Z/, "")
+          end
+        end,
+        Matcher.new(
+          :MULTILINE_COMMENT_START,
+          /\A\/\*.*\Z/,
+          :next_mode => :multiline_comment,
+          :push_back_eol => true) do
+          def process_token(token)
+            token.gsub!(/\A\/\*/, "")
+          end
+        end,
+        Matcher.new(:CHARACTER, /\A'(?:[^\\']|\\.)'/) do
+          def process_token(token)
+            token.gsub!(/\A'|'\Z/, "")
+          end
+        end,
+        Matcher.new(:INLINE_BACKQUOTE_STRING, /\A`[^`]*`/, :is_node => true) do
+          def process_token(token)
+            token.gsub!(/\A`|`\Z/, "")
+          end
+        end,
+        Matcher.new(:INLINE_DOUBLE_QUOTE_STRING, /\A"(?:[^\\"]|\\.)*"/) do
+          def process_token(token)
+            token.gsub!(/\A"|"\Z/, "")
+          end
+        end,
+        Matcher.new(
+          :MULTILINE_BACKQUOTE_STRING_START,
+          /\A`[^`]*\Z/,
+          :next_mode => :multiline_backquote_string,
+          :is_node => true) do
+            def process_token(token)
+              token.gsub!(/\A`/, "")
+            end
+          end,
+        Matcher.new(
+          :MULTILINE_DOUBLE_QUOTE_STRING_START,
+          /\A"(?:[^\\"]|\\\S)*\\\s*\Z/,
+          :next_mode => :multiline_double_quote_string,
+          :push_back_eol => true) do
+            def process_token(token)
+              token.gsub!(/\A"|\\\s*\Z/, "")
+            end
+          end,
+        Matcher.new(:INLINE_BINARY, /\A\[[\sA-Za-z0-9\/=\+]*\]/) do
+            def process_token(token)
+              token.gsub!(/\A\[|\s+|\]\Z/, "")
+            end
+          end,
+        Matcher.new(
+          :MULTILINE_BINARY_START, /\A\[[\sA-Za-z0-9\/=\+]*\Z/,
+          :next_mode => :multiline_binary,
+          :push_back_eol => true) do
+            def process_token(token)
+              token.gsub!(/\A\[|\s+/, "")
+            end
+          end,
+        Matcher.new(
+          :IDENTIFIER, /\A#{SDL4R::IDENTIFIER_START_CLASS}#{SDL4R::IDENTIFIER_PART_CLASS}*/),
+        Matcher.new(:DATE, /\A-?\d+\/\d+\/\d+/, :is_node => true),
+        Matcher.new(
+          :TIME_OR_TIMESPAN,
+          /\A(?:-?\d+d:)?-?\d+:\d+(?::\d+(?:\.\d+)?)?
+            (?:-[a-zA-Z\/]+(?:[+-]\d+(?::\d+)?)?)?/ix),
+        Matcher.new(:INTEGER, /\A[\+\-]?\d+L/i), # takes precedence on floats
+        # the float regex is meant to also catch bad syntaxed floats like "1.2.2" (otherwise, we
+        # would not detect this kind of errors easily).
+        Matcher.new(
+          :FLOAT, /\A[\+\-]?(?:\d+(?:F|D|BD)|\d*\.[\d\.]+(?:F|D|BD)?)/i),
+        Matcher.new(:INTEGER, /\A[\+\-]?\d+L?/i),
+        Matcher.new(:LINE_CONTINUATION, /\A\\\s*\Z/), # outside of comments, strings, etc
+        Matcher.new(
+          :UNCLOSED_DOUBLE_QUOTE_STRING,
+          /\A"(?:[^\\"]|\\\S)*/,
+          :error => "unclosed string"),
+      ],
+
+      :multiline_comment => [
+        Matcher.new(:EOL, /\A\n/),
+        Matcher.new(:MULTILINE_COMMENT_END, /\A[\s\S]*?\*\//, :next_mode => :top) do
+          def process_token(token)
+            token.gsub!(/\*\/\Z/, "")
+          end
+        end,
+        Matcher.new(:MULTILINE_COMMENT_PART, /\A.+\Z/, :push_back_eol => true)
+      ],
+
+      :multiline_backquote_string => [
+        Matcher.new(:EOL, /\A\n/),
+        Matcher.new(:MULTILINE_BACKQUOTE_STRING_END, /\A[^`]*`/, :next_mode => :top) do
+          def process_token(token)
+            token.gsub!(/`\Z/, "")
+          end
+        end,
+        Matcher.new(:MULTILINE_BACKQUOTE_STRING_PART, /\A[^`]*\Z/)
+      ],
+
+      :multiline_double_quote_string => [
+        Matcher.new(:EOL, /\A\n/),
+        Matcher.new(
+          :MULTILINE_DOUBLE_QUOTE_STRING_END, /\A(?:[^\\"]|\\\S)*"/, :next_mode => :top) do
+            def process_token(token)
+              token.gsub!(/\A\s+|"\Z/, "")
+            end
+          end,
+        Matcher.new(
+          :MULTILINE_DOUBLE_QUOTE_STRING_PART,
+          /\A(?:[^\\"]|\\\S)*\\\s*\Z/,
+          :push_back_eol => true) do
+            def process_token(token)
+              token.gsub!(/\A\s+|\\\s*\Z/, "")
+            end
+          end,
+        Matcher.new(
+          :UNCLOSED_DOUBLE_QUOTE_STRING,
+          /\A(?:[^\\"]|\\\S)*\Z/,
+          :error => "unclosed multiline string")
+      ],
+
+      :multiline_binary => [
+        Matcher.new(:EOL, /\A\n/),
+        Matcher.new(:MULTILINE_BINARY_END, /\A[\sA-Za-z0-9\/=\+]*\]/, :next_mode => :top) do
+          def process_token(token)
+            token.gsub!(/\s+|\]\Z/, "")
+          end
+        end,
+        Matcher.new(:MULTILINE_BINARY_PART, /\A[\sA-Za-z0-9\/=\+]*\Z/, :push_back_eol => true) do
+          def process_token(token)
+            token.gsub!(/\s+/, "")
+          end
+        end
+      ]
+    }
+
+    # @param [IO] the IO to read from
+    # @raise [ArgumentError] if +io+ is +nil+.
+    def initialize io
+      raise ArgumentError, 'io' unless io
+      @io = io
+      @scanner = nil
+      @line_no = -1
+      set_mode(:top)
+
+      @token = nil
+      @pushed_back_token = nil
+      @previous_token = nil
+
+      @token_pool = [] # a pool of reusable Tokens
+    end
+
+    # @return [String] text of the current token.
+    def token
+      @token.text
+    end
+
+    # @return [Symbol] type of the current token (e.g. +:WHITESPACE+)
+    def token_type
+      @token.type
+    end
+
+    # @return [Integer] position of the current token (only meant for error tracking for the time
+    #   being)
+    def token_line_no
+      @token.line_no
+    end
+
+    # @return [Integer] position of the current token (only meant for error tracking for the time
+    #   being)
+    def token_pos
+      @token.pos
+    end
+
+    # Sets the current working mode of this Tokenizer.
+    #
+    # @param [Symbol] new mode
+    #   * +:top+ (normal default mode)
+    #   * +:multiline_comment+
+    #   * +:multiline_backquote_string+
+    #   * +:multiline_double_quote_string+
+    #   * +:multiline_binary+
+    #
+    # @return [self]
+    # @raise [ArgumentError] if the given mode is unknown.
+    #
+    def set_mode(mode)
+      ms = @@matcher_sets[mode]
+      raise ArgumentError, "unknown tokenizer mode #{mode.to_s}" unless ms
+      @matcher_set = ms
+      self
+    end
+
+    # Reads a token from the pushed back ones.
+    def read_pushed_back
+      record_previous_token
+
+      # Set the current state
+      @token = @pushed_back_token
+      @pushed_back_token = nil
+
+      if @token.matcher
+        next_mode = @token.matcher.next_mode
+        set_mode(next_mode) if next_mode
+      end
+    end
+    private :read_pushed_back
+
+    # Goes to the next token.
+    #
+    # @return [Symbol] +nil+ if eof has been reached, the current token type otherwise.
+    #
+    def read
+      if @pushed_back_token
+        read_pushed_back
+        return @token.type
+      end
+
+      record_previous_token
+      @token = nil
+
+      if @line_no < 0 or @scanner.eos? # fetch a line if beginning or at end of line
+        unless read_line
+          if previous_token_type == :EOF
+            return nil
+          else
+            @token = Token.new(nil, :EOF, nil, @line_no, @scanner ? @scanner.pos : 0)
+            return @token.type
+          end
+        end
+      end
+
+      pos = @scanner.pos
+      @matcher_set.each do |matcher|
+        if token_text = @scanner.scan(matcher.regex)
+          error = matcher.error
+          if error
+            raise_parse_error(error)
+
+          else
+            set_matcher_token(matcher, token_text, pos)
+            if matcher.push_back_eol and @scanner.eos?
+              @scanner.pos = @scanner.pos - @@EOL_STRING.size
+            end
+          end
+          break
+        end
+      end
+
+      raise_unexpected_char unless @token
+
+      return @token.type
+    end
+
+    def record_previous_token
+      @previous_token = @token
+    end
+    private :record_previous_token
+
+    # Sets the current Token using the Matcher that detected it
+    def set_matcher_token(matcher, token_text, pos)
+      @token = Token.new(
+        matcher.process_token(token_text), matcher.token_type, matcher, @line_no, pos)
+
+      next_mode = matcher.next_mode
+      set_mode(next_mode) if next_mode
+    end
+    private :set_matcher_token
+
+    # @return [Symbol] the type of the previous Token.
+    def previous_token_type
+      @previous_token ? @previous_token.type : nil
+    end
+
+    # Unreads the current token.
+    # The previous token becomes the current one
+    #
+    # @raise if #unread has been called twice in a row (no call to #read)
+    def unread
+      if @pushed_back_token
+        raise "only one token can be pushed back"
+      else
+        @pushed_back_token = @token
+        @token = @previous_token
+
+        # We have no memory of what happened before
+        @previous_token = nil
+
+        if @token.matcher
+          next_mode = @token.matcher.next_mode
+          set_mode(next_mode) if next_mode
+        end
+      end
+    end
+
+    # Raises a standard "unexpected character" error.
+    def raise_unexpected_char(msg = "unexpected char")
+      raise_parse_error "#{msg}: <#{@scanner.peek(1)}>"
+    end
+
+    def raise_parse_error(msg = "parse error", line_no = @line_no, pos = @scanner.pos)
+      line = (line_no == @line_no)? @scanner.string : nil
+      raise SdlParseError.new(msg, line_no + 1, pos + 1, line)
+    end
+
+    private
+
+    # Reads the next line of the IO.
+    # All lines are normalized to end with a single '\n'.
+    #
+    # @return [String] the new read line.
+    #
+    def read_line
+      line = @io.gets
+
+      if line
+        # Clean the line of its end characters
+        line.gsub!(/(?:\n|\r\n|\r)\Z/, @@EOL_STRING)
+
+        @line_no += 1
+
+        if @scanner
+          @scanner.string = line
+        else
+          @scanner = StringScanner.new(line)
+        end
+      end
+
+      line
+    end
+  end
+end