RbYAML 0.1.0 → 0.2.0

data/lib/rbyaml/scanner.rb

@@ -26,9 +26,10 @@
 require 'rbyaml/util'
 require 'rbyaml/error'
 require 'rbyaml/tokens'
+require 'rbyaml/constants'
 
 module RbYAML
-  class ScannerError < MarkedYAMLError
+  class ScannerError < YAMLError
   end
   class ReaderError < YAMLError
     def initialize(name, position, character, encoding, reason)
@@ -48,10 +49,10 @@ module RbYAML
     end
   end
 
-  SimpleKey = Struct.new(:token_number, :required, :index, :line, :column, :mark)
+  SimpleKey = Struct.new(:token_number, :required, :column)
 
   class Scanner
-    attr_reader :column, :stream, :stream_pointer, :eof, :buffer, :pointer, :index, :line
+    attr_reader :column, :stream, :stream_pointer, :eof, :buffer, :pointer
     def initialize(stream)
       # Had we reached the end of the stream?
       @done = false
@@ -185,8 +186,6 @@ module RbYAML
       update(1) unless @pointer1 < @buffer_length
     end
 
-    LINE_BR = "\n\x85"
-
     def forward1
       update(2) unless @pointer1+1 < @buffer_length
       buff = @buffer[@pointer...@pointer1+1]
@@ -241,7 +240,6 @@ module RbYAML
       @pointer1 += 6
     end
 
-    LINE_BR_REG = /[\n\x85]|(?:\r[^\n])/
     def forwardn(length)
       update(length + 1) unless @pointer1+length < @buffer_length
       buff = @buffer[@pointer...@pointer+length]
@@ -251,17 +249,8 @@ module RbYAML
       @pointer1 += length
     end
     
-    def get_mark
-      if @stream.nil?
-        Mark.new(@name,@column,@buffer,@pointer)
-      else
-        Mark.new(@name,@column,nil,nil)
-      end
-    end
-    
-    NON_PRINTABLE = /[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF]/
     def check_printable(data)
-      if NON_PRINTABLE =~ data
+      if NON_PRINTABLE_RE =~ data
         position = @buffer.length-@pointer+($~.offset(0)[0])
         raise ReaderError.new(@name, position, $&,"unicode","special characters are not allowed"),"special characters are not allowed"
       end
@@ -341,10 +330,6 @@ module RbYAML
       @tokens.empty? || next_possible_simple_key == @tokens_taken
     end
 
-    ENDING = /^---[\0 \t\r\n\x85]$/
-    START = /^\.\.\.[\0 \t\r\n\x85]$/
-    NULL_OR_OTHER = "\0 \t\r\n\x85"
-    BEG = /^([^\0 \t\r\n\x85\-?:,\[\]{}#&*!|>'"%@`]|([\-?:][^\0 \t\r\n\x85]))/
     def fetch_more_tokens
       # Eat whitespaces and comments until we reach the next token.
       scan_to_next_token
@@ -381,7 +366,7 @@ module RbYAML
       when ?>: if @flow_zero: return fetch_folded end
       end
       return fetch_plain if BEG =~ prefix(2)
-      raise ScannerError.new("while scanning for the next token", nil,"found character #{ch.chr}(#{ch}) that cannot start any token",get_mark)
+      raise ScannerError.new("while scanning for the next token","found character #{ch.chr}(#{ch}) that cannot start any token")
     end
 
     # Simple keys treatment.
@@ -399,7 +384,7 @@ module RbYAML
       #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
       # The next token might be a simple key. Let's save it's number and
       # position.
-      @possible_simple_keys[@flow_level] = SimpleKey.new(@tokens_taken+@tokens.length, @flow_zero && @indent == @column,index,line,column,get_mark) if @allow_simple_key
+      @possible_simple_keys[@flow_level] = SimpleKey.new(@tokens_taken+@tokens.length, @flow_zero && @indent == @column,column) if @allow_simple_key
     end
 
     # Indentation functions.
@@ -421,17 +406,16 @@ module RbYAML
       return nil if !@flow_zero
       # In block context, we may need to issue the BLOCK-END tokens.
       while @indent > col
-        mark = get_mark
         @indent = @indents.pop
-        @tokens << BlockEndToken.new(mark, mark)
+        @tokens << BLOCK_END
       end
     end
     
-    def add_indent(column)
+    def add_indent(col)
       # Check if we need to increase indentation.
-      if @indent < column
+      if @indent < col
         @indents << @indent
-        @indent = column
+        @indent = col
         return true
       end
       return false
@@ -443,9 +427,8 @@ module RbYAML
       # We always add STREAM-START as the first token and STREAM-END as the
       # last token.
       # Read the token.
-      mark = get_mark
       # Add STREAM-START.
-      @tokens << StreamStartToken.new(mark, mark, @encoding)
+      @tokens << STREAM_START
     end
 
 
@@ -456,9 +439,8 @@ module RbYAML
       @allow_simple_key = false
       @possible_simple_keys = {}
       # Read the token.
-      mark = get_mark
       # Add STREAM-END.
-      @tokens << StreamEndToken.new(mark, mark)
+      @tokens << STREAM_END
       # The stream is finished.
       @done = true
     end
@@ -473,11 +455,11 @@ module RbYAML
     end
 
     def fetch_document_start
-      fetch_document_indicator(DocumentStartToken)
+      fetch_document_indicator(DOCUMENT_START)
     end
 
     def fetch_document_end
-      fetch_document_indicator(DocumentEndToken)
+      fetch_document_indicator(DOCUMENT_END)
     end
 
     def fetch_document_indicator(token)
@@ -487,18 +469,16 @@ module RbYAML
       # after '---'.
       @allow_simple_key = false
       # Add DOCUMENT-START or DOCUMENT-END.
-      start_mark = get_mark
       forward3
-      end_mark = get_mark
-      @tokens << token.new(start_mark, end_mark)
+      @tokens << token
     end
 
     def fetch_flow_sequence_start
-      fetch_flow_collection_start(FlowSequenceStartToken)
+      fetch_flow_collection_start(FLOW_SEQUENCE_START)
     end
 
     def fetch_flow_mapping_start
-      fetch_flow_collection_start(FlowMappingStartToken)
+      fetch_flow_collection_start(FLOW_MAPPING_START)
     end
 
     def fetch_flow_collection_start(token)
@@ -510,18 +490,16 @@ module RbYAML
       # Simple keys are allowed after '[' and '{'.
       @allow_simple_key = true
       # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << token.new(start_mark, end_mark)
+      @tokens << token
     end
     
     def fetch_flow_sequence_end
-      fetch_flow_collection_end(FlowSequenceEndToken)
+      fetch_flow_collection_end(FLOW_SEQUENCE_END)
     end
     
     def fetch_flow_mapping_end
-      fetch_flow_collection_end(FlowMappingEndToken)
+      fetch_flow_collection_end(FLOW_MAPPING_END)
     end
     
     def fetch_flow_collection_end(token)
@@ -533,30 +511,25 @@ module RbYAML
       # No simple keys after ']' or '}'.
       @allow_simple_key = false
       # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << token.new(start_mark, end_mark)
+      @tokens << token
     end
 
     def fetch_flow_entry
       # Simple keys are allowed after ','.
       @allow_simple_key = true
       # Add FLOW-ENTRY.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << FlowEntryToken.new(start_mark, end_mark)
+      @tokens << FLOW_ENTRY
     end
 
     def fetch_block_entry
       # Block context needs additional checks.
       if @flow_zero
-        raise ScannerError.new(nil,nil,"sequence entries are not allowed here",get_mark) if !@allow_simple_key
+        raise ScannerError.new(nil,"sequence entries are not allowed here") if !@allow_simple_key
         # We may need to add BLOCK-SEQUENCE-START.
         if add_indent(column)
-          mark = get_mark
-          @tokens << BlockSequenceStartToken.new(mark, mark)
+          @tokens << BLOCK_SEQUENCE_START
         end
         # It's an error for the block entry to occur in the flow context,
         # but we let the parser detect this.
@@ -564,30 +537,25 @@ module RbYAML
       # Simple keys are allowed after '-'.
       @allow_simple_key = true
       # Add BLOCK-ENTRY.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << BlockEntryToken.new(start_mark, end_mark)
+      @tokens << BLOCK_ENTRY
     end
 
     def fetch_key
       # Block context needs additional checks.
       if @flow_zero
         # Are we allowed to start a key (not nessesary a simple)?
-        raise ScannerError.new(nil,nil,"mapping keys are not allowed here",get_mark) if !@allow_simple_key
+        raise ScannerError.new(nil,"mapping keys are not allowed here") if !@allow_simple_key
         # We may need to add BLOCK-MAPPING-START.
         if add_indent(column)
-          mark = get_mark
-          @tokens << BlockMappingStartToken.new(mark, mark)
+          @tokens << BLOCK_MAPPING_START
         end
       end
       # Simple keys are allowed after '?' in the block context.
       @allow_simple_key = @flow_zero
       # Add KEY.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << KeyToken.new(start_mark, end_mark)
+      @tokens << KEY
     end
     
     def fetch_value
@@ -600,7 +568,7 @@ module RbYAML
         if @flow_zero
           # We are allowed to start a complex value if and only if
           # we can start a simple key.
-          raise ScannerError.new(nil,nil,"mapping values are not allowed here",get_mark) if !@allow_simple_key
+          raise ScannerError.new(nil,"mapping values are not allowed here") if !@allow_simple_key
           # Simple keys are allowed after ':' in the block context.
           @allow_simple_key = true
         end
@@ -610,18 +578,16 @@ module RbYAML
 
         # If this key starts a new block mapping, we need to add
         # BLOCK-MAPPING-START.
-        se = (@flow_zero && add_indent(key.column)) ? [BlockMappingStartToken.new(key.mark, key.mark)] : []
-        se << KeyToken.new(key.mark, key.mark)
+        se = (@flow_zero && add_indent(key.column)) ? [BLOCK_MAPPING_START] : []
+        se << KEY
         @tokens.insert(key.token_number-@tokens_taken,*se)
         # There cannot be two simple keys one after another.
         @allow_simple_key = false
         # It must be a part of a complex key.
       end
       # Add VALUE.
-      start_mark = get_mark
       forward1
-      end_mark = get_mark
-      @tokens << ValueToken.new(start_mark, end_mark)
+      @tokens << VALUE
     end
 
     def fetch_alias
@@ -696,7 +662,6 @@ module RbYAML
 
     
     # Scanners.
-    NULL_OR_LINEBR = "\0\r\n\x85"
     def scan_to_next_token
       # We ignore spaces, line breaks and comments.
       # If we find a line break in the block context, we set the flag
@@ -713,15 +678,12 @@ module RbYAML
       # `unwind_indent` before issuing BLOCK-END.
       # Scanners for block, flow, and plain scalars need to be modified.
       while true
-        peek_0 = peek0
-        while peek_0 == 32
+        while peek0 == 32
           forward1
-          peek_0 = peek0
         end
-        if peek_0 == ?#
+        if peek0 == ?#
           while !NULL_OR_LINEBR.include?(peek0)
             forward1
-            peek_0 = peek0
           end
         end
 
@@ -735,60 +697,53 @@ module RbYAML
     
     def scan_directive
       # See the specification for details.
-      start_mark = get_mark
       forward1
-      name = scan_directive_name(start_mark)
+      name = scan_directive_name
       value = nil
       if name == "YAML"
-        value = scan_yaml_directive_value(start_mark)
-        end_mark = get_mark
+        value = scan_yaml_directive_value
       elsif name == "TAG"
-        value = scan_tag_directive_value(start_mark)
-        end_mark = get_mark
+        value = scan_tag_directive_value
       else
-        end_mark = get_mark
         forward1 while !NULL_OR_LINEBR.include?(peek0)
       end
-      scan_directive_ignored_line(start_mark)
-      DirectiveToken.new(name, value, start_mark, end_mark)
+      scan_directive_ignored_line
+      DirectiveToken.new(name, value)
     end
 
-    ALPHA_REG = /[-0-9A-Za-z_]/
-    NULL_BL_LINEBR = "\0 \r\n\x85"
-    NULL_BL_T_LINEBR = "\0 \t\r\n\x85"
-    def scan_directive_name(start_mark)
+    def scan_directive_name
       # See the specification for details.
       length = 0
       ch = peek(length)
-      zlen = false
+      zlen = true
       while ALPHA_REG  =~ ch.chr
-        zlen = true
+        zlen = false
         length += 1
         ch = peek(length)
       end
-      raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if zlen
+      raise ScannerError.new("while scanning a directive","expected alphabetic or numeric character, but found #{ch.to_s}") if zlen
       value = prefix(length)
       forward(length)
       ch = peek0
-      raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if !NULL_BL_LINEBR.include?(ch)
+      raise ScannerError.new("while scanning a directive","expected alphabetic or numeric character, but found #{ch.to_s}") if !NULL_BL_LINEBR.include?(ch)
       value
     end
 
-    def scan_yaml_directive_value(start_mark)
+    def scan_yaml_directive_value
       # See the specification for details.
       forward1 while peek0 == 32
-      major = scan_yaml_directive_number(start_mark)
-      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or '.', but found #{peek.to_s}",get_mark) if peek0 != ?.
+      major = scan_yaml_directive_number
+      raise ScannerError.new("while scanning a directive","expected a digit or '.', but found #{peek.to_s}") if peek0 != ?.
       forward1
-      minor = scan_yaml_directive_number(start_mark)
-      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or ' ', but found #{peek.to_s}",get_mark) if !NULL_BL_LINEBR.include?(peek0)
+      minor = scan_yaml_directive_number
+      raise ScannerError.new("while scanning a directive","expected a digit or ' ', but found #{peek.to_s}") if !NULL_BL_LINEBR.include?(peek0)
       [major, minor]
     end
 
-    def scan_yaml_directive_number(start_mark)
+    def scan_yaml_directive_number
       # See the specification for details.
       ch = peek0
-      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit, but found #{ch.to_s}",get_mark) if !(ch.__is_ascii_num)
+      raise ScannerError.new("while scanning a directive","expected a digit, but found #{ch.to_s}") if !(ch.__is_ascii_num)
       length = 0
       length += 1 while (peek(length).__is_ascii_num)
       value = prefix(length)
@@ -796,42 +751,40 @@ module RbYAML
       value
     end
     
-    def scan_tag_directive_value(start_mark)
+    def scan_tag_directive_value
       # See the specification for details.
       forward1 while peek0 == 32
-      handle = scan_tag_directive_handle(start_mark)
+      handle = scan_tag_directive_handle
       forward1 while peek0 == 32
-      prefix = scan_tag_directive_prefix(start_mark)
+      prefix = scan_tag_directive_prefix
       [handle, prefix]
     end
 
-    def scan_tag_directive_handle(start_mark)
+    def scan_tag_directive_handle
       # See the specification for details.
-      value = scan_tag_handle("directive", start_mark)
-      raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{peek0}",get_mark()) if peek0 != 32
+      value = scan_tag_handle("directive")
+      raise ScannerError.new("while scanning a directive","expected ' ', but found #{peek0}") if peek0 != 32
       value
     end
 
-    def scan_tag_directive_prefix(start_mark)
+    def scan_tag_directive_prefix
       # See the specification for details.
-      value = scan_tag_uri("directive", start_mark)
-      raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{peek0}",get_mark()) if !NULL_BL_LINEBR.include?(peek0)
+      value = scan_tag_uri("directive")
+      raise ScannerError.new("while scanning a directive","expected ' ', but found #{peek0}") if !NULL_BL_LINEBR.include?(peek0)
       value
     end
     
-    def scan_directive_ignored_line(start_mark)
+    def scan_directive_ignored_line
       # See the specification for details.
       forward1 while peek0 == 32
       if peek0 == ?#
           forward1 while !NULL_OR_LINEBR.include?(peek0)
       end
       ch = peek0
-      raise ScannerError.new("while scanning a directive", start_mark,"expected a comment or a line break, but found #{peek0.to_s}",get_mark()) if !NULL_OR_LINEBR.include?(peek0)
+      raise ScannerError.new("while scanning a directive","expected a comment or a line break, but found #{peek0.to_s}") if !NULL_OR_LINEBR.include?(peek0)
       scan_line_break
     end
  
-    NON_ALPHA = /[^-0-9A-Za-z_]/   
-    NON_ALPHA_OR_NUM = "\0 \t\r\n\x85?:,]}%@`"
     def scan_anchor(token)
       # The specification does not restrict characters for anchors and
       # aliases. This may lead to problems, for instance, the document:
@@ -841,9 +794,7 @@ module RbYAML
       # and
       #   [ *alias , "value" ]
       # Therefore we restrict aliases to numbers and ASCII letters.
-      start_mark = get_mark
-      indicator = peek0
-      name = (indicator == ?*) ? "alias":"anchor"
+      name = (peek0 == ?*) ? "alias":"anchor"
       forward1
       length = 0
       chunk_size = 16
@@ -854,26 +805,23 @@ module RbYAML
         end
         chunk_size += 16
       end
-      raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found something else...",get_mark) if length==0
+      raise ScannerError.new("while scanning an #{name}","expected alphabetic or numeric character, but found something else...") if length==0
       value = prefix(length)
       forward(length)
       if !NON_ALPHA_OR_NUM.include?(peek0)
-        raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found #{peek0}",get_mark)
+        raise ScannerError.new("while scanning an #{name}","expected alphabetic or numeric character, but found #{peek0}")
       end
-      end_mark = get_mark
-      token.new(value, start_mark, end_mark)
+      token.new(value)
     end
 
-    NULL_T_BL_LINEBR = "\0 \t\r\n\x85"
     def scan_tag
       # See the specification for details.
-      start_mark = get_mark
       ch = peek1
       if ch == ?<
         handle = nil
         forward2
-        suffix = scan_tag_uri("tag", start_mark)
-        raise ScannerError.new("while parsing a tag", start_mark,"expected '>', but found #{peek.to_s}",get_mark) if peek0 != ?>
+        suffix = scan_tag_uri("tag")
+        raise ScannerError.new("while parsing a tag","expected '>', but found #{peek.to_s}") if peek0 != ?>
         forward1
       elsif NULL_T_BL_LINEBR.include?(ch)
         handle = nil
@@ -892,38 +840,35 @@ module RbYAML
         end
         handle = "!"
         if use_handle
-          handle = scan_tag_handle("tag", start_mark)
+          handle = scan_tag_handle("tag")
         else
           handle = "!"
           forward1
         end
-        suffix = scan_tag_uri("tag", start_mark)
+        suffix = scan_tag_uri("tag")
       end
-      raise ScannerError.new("while scanning a tag",start_mark,"expected ' ', but found #{peek0}",get_mark) if !NULL_BL_LINEBR.include?(peek0)
+      raise ScannerError.new("while scanning a tag","expected ' ', but found #{peek0}") if !NULL_BL_LINEBR.include?(peek0)
       value = [handle, suffix]
-      end_mark = get_mark
-      TagToken.new(value, start_mark, end_mark)
+      TagToken.new(value)
     end
 
-    BLANK_T = " \t"
     def scan_block_scalar(style)
       # See the specification for details.
       folded = style== ?>
       chunks = []
-      start_mark = get_mark
       # Scan the header.
       forward1
-      chomping, increment = scan_block_scalar_indicators(start_mark)
-      scan_block_scalar_ignored_line(start_mark)
+      chomping, increment = scan_block_scalar_indicators
+      scan_block_scalar_ignored_line
       # Determine the indentation level and go to the first non-empty line.
       min_indent = @indent+1
       min_indent = 1 if min_indent < 1
       if increment.nil?
-        breaks, max_indent, end_mark = scan_block_scalar_indentation
+        breaks, max_indent = scan_block_scalar_indentation
         indent = [min_indent, max_indent].max
       else
         indent = min_indent+increment-1
-        breaks, end_mark = scan_block_scalar_breaks(indent)
+        breaks = scan_block_scalar_breaks(indent)
       end
       line_break = ''
       # Scan the inner part of the block scalar.
@@ -935,12 +880,12 @@ module RbYAML
         chunks << prefix(length)
         forward(length)
         line_break = scan_line_break
-        breaks, end_mark = scan_block_scalar_breaks(indent)
+        breaks = scan_block_scalar_breaks(indent)
         if column == indent && peek0 != 0
           # Unfortunately, folding rules are ambiguous.
           #
           # This is the folding according to the specification:
-          if folded && line_break == ?\n && leading_non_space && !BLANK_T.include?(peek0)
+          if folded && line_break == "\n" && leading_non_space && !BLANK_T.include?(peek0)
             chunks << ' ' if breaks.empty?
           else
             chunks << line_break
@@ -968,11 +913,10 @@ module RbYAML
       end
 
       # We are done.
-      ScalarToken.new(chunks.to_s, false, start_mark, end_mark,style)
+      ScalarToken.new(chunks.to_s, false, style)
     end
 
-    PLUS_MIN = /[+-]/
-    def scan_block_scalar_indicators(start_mark)
+    def scan_block_scalar_indicators
       # See the specification for details.
       chomping = nil
       increment = nil
@@ -982,13 +926,13 @@ module RbYAML
         forward1
         ch = peek0
         if ch.__is_ascii_num
-          increment = ch.to_i
-          raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0
+          increment = ch.chr.to_i
+          raise ScannerError.new("while scanning a block scalar","expected indentation indicator in the range 1-9, but found 0") if increment == 0
           forward1
         end
       elsif ch.__is_ascii_num
-        increment = ch
-        raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0
+        increment = ch.chr.to_i
+        raise ScannerError.new("while scanning a block scalar","expected indentation indicator in the range 1-9, but found 0") if increment == 0
         forward1
         ch = peek0
         if PLUS_MIN =~ ch.chr
@@ -996,50 +940,44 @@ module RbYAML
           forward1
         end
       end
-      raise ScannerError.new("while scanning a block scalar", start_mark,"expected chomping or indentation indicators, but found #{peek0}",get_mark) if !NULL_BL_LINEBR.include?(peek0)
+      raise ScannerError.new("while scanning a block scalar","expected chomping or indentation indicators, but found #{peek0}") if !NULL_BL_LINEBR.include?(peek0)
       [chomping, increment]
     end
 
-    def scan_block_scalar_ignored_line(start_mark)
+    def scan_block_scalar_ignored_line
       # See the specification for details.
       forward1 while peek0 == 32
       if peek0 == ?#
           forward1 while !NULL_OR_LINEBR.include?(peek0)
       end
-      raise ScannerError.new("while scanning a block scalar", start_mark,"expected a comment or a line break, but found #{peek0}",get_mark) if !NULL_OR_LINEBR.include?(peek0)
+      raise ScannerError.new("while scanning a block scalar","expected a comment or a line break, but found #{peek0}") if !NULL_OR_LINEBR.include?(peek0)
       scan_line_break
     end
 
-    BLANK_OR_LINEBR = " \r\n\x85"
     def scan_block_scalar_indentation
       # See the specification for details.
       chunks = []
       max_indent = 0
-      end_mark = get_mark
       while BLANK_OR_LINEBR.include?(peek0)
         if peek0 != 32
           chunks << scan_line_break
-          end_mark = get_mark
         else
           forward1
           max_indent = column if column > max_indent
         end
       end
-      [chunks, max_indent, end_mark]
+      [chunks, max_indent]
     end
 
-    FULL_LINEBR = "\r\n\x85"
     def scan_block_scalar_breaks(indent)
       # See the specification for details.
       chunks = []
-      end_mark = get_mark
       forward1 while @column < indent && peek0 == 32
       while FULL_LINEBR.include?(peek0)
         chunks << scan_line_break
-        end_mark = get_mark
         forward1 while @column < indent && peek0 == 32
       end
-      [chunks, end_mark]
+      chunks
     end
 
     def scan_flow_scalar(style)
@@ -1051,45 +989,18 @@ module RbYAML
       # that document separators are not included in scalars.
       double = style == ?"
       chunks = []
-      start_mark = get_mark
       quote = peek0
       forward1
-      chunks += scan_flow_scalar_non_spaces(double, start_mark)
+      chunks += scan_flow_scalar_non_spaces(double)
       while peek0 != quote
-        chunks += scan_flow_scalar_spaces(double, start_mark)
-        chunks += scan_flow_scalar_non_spaces(double, start_mark)
+        chunks += scan_flow_scalar_spaces(double)
+        chunks += scan_flow_scalar_non_spaces(double)
       end
       forward1
-      end_mark = get_mark
-      ScalarToken.new(chunks.to_s, false, start_mark, end_mark,style)
+      ScalarToken.new(chunks.to_s, false, style)
     end
     
-    ESCAPE_REPLACEMENTS = {
-      "0"  =>   "\0",
-      "a"  =>   "\x07",
-      "b"  =>   "\x08",
-      "t"  =>   "\x09",
-      "\t" =>   "\x09",
-      "n"  =>   "\x0A",
-      "v"  =>   "\x0B",
-      "f"  =>   "\x0C",
-      "r"  =>   "\x0D",
-      "e"  =>   "\x1B",
-      " "  =>   "\x20",
-      '"'  =>   '"',
-      "\\" =>   "\\",
-      "N"  =>   "\x85",
-      "_"  =>   "\xA0"
-    }
-    
-    ESCAPE_CODES = {
-      'x' =>   2
-    }
-    
-    SPACES_AND_STUFF = "'\"\\\0 \t\r\n\x85"
-    DOUBLE_ESC = "\"\\"
-    NOT_HEXA = /[^0-9A-Fa-f]/
-    def scan_flow_scalar_non_spaces(double, start_mark)
+    def scan_flow_scalar_non_spaces(double)
       # See the specification for details.
       chunks = []
       while true
@@ -1109,24 +1020,23 @@ module RbYAML
         elsif double && ch == ?\\
           forward1
           ch = peek0
-          if ESCAPE_REPLACEMENTS.member?(ch.chr)
-            chunks << ESCAPE_REPLACEMENTS[ch.chr]
+          if UNESCAPES.member?(ch.chr)
+            chunks << UNESCAPES[ch.chr]
             forward1
           elsif ESCAPE_CODES.member?(ch.chr)
             length = ESCAPE_CODES[ch.chr]
             forward1
             if NOT_HEXA =~ prefix(length)
-              raise ScannerError.new("while scanning a double-quoted scalar", start_mark,
-                                     "expected escape sequence of #{length} hexdecimal numbers, but found something else: #{prefix(length)}}",get_mark)
+              raise ScannerError.new("while scanning a double-quoted scalar","expected escape sequence of #{length} hexdecimal numbers, but found something else: #{prefix(length)}}")
             end
-            code = prefix(length).to_i.to_s(16)
+            code = prefix(length).to_i(16).to_s
             chunks << code
             forward(length)
           elsif FULL_LINEBR.include?(ch)
             scan_line_break
-            chunks += scan_flow_scalar_breaks(double, start_mark)
+            chunks += scan_flow_scalar_breaks(double)
           else
-            raise ScannerError.new("while scanning a double-quoted scalar", start_mark,"found unknown escape character #{ch}",get_mark)
+            raise ScannerError.new("while scanning a double-quoted scalar","found unknown escape character #{ch}")
           end
         else
           return chunks
@@ -1134,7 +1044,7 @@ module RbYAML
       end
     end
 
-    def scan_flow_scalar_spaces(double, start_mark)
+    def scan_flow_scalar_spaces(double)
       # See the specification for details.
       chunks = []
       length = 0
@@ -1143,11 +1053,11 @@ module RbYAML
       forward(length)
       ch = peek0
       if ch == ?\0
-        raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected end of stream",get_mark)
+        raise ScannerError.new("while scanning a quoted scalar","found unexpected end of stream")
       elsif FULL_LINEBR.include?(ch)
         line_break = scan_line_break
-        breaks = scan_flow_scalar_breaks(double, start_mark)
-        if line_break != ?\n
+        breaks = scan_flow_scalar_breaks(double)
+        if line_break != "\n"
           chunks << line_break
         elsif breaks.empty?
           chunks << ' '
@@ -1159,7 +1069,7 @@ module RbYAML
       chunks
     end
     
-    def scan_flow_scalar_breaks(double, start_mark)
+    def scan_flow_scalar_breaks(double)
       # See the specification for details.
       chunks = []
       while true
@@ -1167,7 +1077,7 @@ module RbYAML
         # separators.
         prefix = prefix(3)
         if (prefix == "---" || prefix == "...") &&NULL_BL_T_LINEBR.include?(peek3)
-          raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected document separator", get_mark)
+          raise ScannerError.new("while scanning a quoted scalar","found unexpected document separator")
         end
         forward1 while BLANK_T.include?(peek0)
         if FULL_LINEBR.include?(peek0)
@@ -1178,11 +1088,6 @@ module RbYAML
       end
     end
     
-
-    R_flowzero = /[\0 \t\r\n\x85]|(:[\0 \t\r\n\x28])/
-    R_flownonzero = /[\0 \t\r\n\x85\[\]{},:?]/
-    S4 = "\0 \t\r\n\x28[]{}"
-
     def scan_plain
       # See the specification for details.
       # We add an additional restriction for the flow context:
@@ -1190,7 +1095,6 @@ module RbYAML
       # We also keep track of the `allow_simple_key` flag here.
       # Indentation rules are loosed for the flow context.
       chunks = []
-      end_mark = start_mark = get_mark
       indent = @indent+1
       # We allow zero indentation for scalars, but then we need to check for
       # document separators at the beginning of the line.
@@ -1210,22 +1114,20 @@ module RbYAML
         ch = peek(length)
         if f_nzero && ch == ?: && !S4.include?(peek(length+1))
           forward(length)
-          raise ScannerError.new("while scanning a plain scalar",start_mark,"found unexpected ':'",get_mark,"Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
+          raise ScannerError.new("while scanning a plain scalar","found unexpected ':'","Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
         end
         break if length == 0
         @allow_simple_key = false
         chunks += spaces
         chunks << prefix(length)
         forward(length)
-        end_mark = get_mark
-        spaces = scan_plain_spaces(indent, start_mark)
+        spaces = scan_plain_spaces(indent)
         break if !spaces || (@flow_zero && @column < indent)
       end
-      return ScalarToken.new(chunks.to_s, true, start_mark, end_mark)
+      return ScalarToken.new(chunks.to_s, true)
     end
 
-    END_OR_START = /^(---|\.\.\.)[\0 \t\r\n\x85]$/
-    def scan_plain_spaces(indent, start_mark)
+    def scan_plain_spaces(indent)
       # See the specification for details.
       # The specification is really confusing about tabs in plain scalars.
       # We just forbid them completely. Do not use tabs in YAML!
@@ -1248,10 +1150,10 @@ module RbYAML
             return if END_OR_START =~ prefix(4)
           end
         end
-        if line_break != '\n'
+        if line_break != "\n"
           chunks << line_break
-        elsif breaks.empty?
-          chunks << ' '
+        elsif breaks.nil? || breaks.empty?
+          chunks << " "
         end
         chunks += breaks
       else
@@ -1261,12 +1163,12 @@ module RbYAML
     end
 
     
-    def scan_tag_handle(name, start_mark)
+    def scan_tag_handle(name)
       # See the specification for details.
       # For some strange reasons, the specification does not allow '_' in
       # tag handles. I have allowed it anyway.
       ch = peek0
-      raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark) if ch != ?!
+      raise ScannerError.new("while scanning a #{name}","expected '!', but found #{ch}") if ch != ?!
       length = 1
       ch = peek(length)
       if ch != 32
@@ -1276,7 +1178,7 @@ module RbYAML
         end
         if ch != ?!
           forward(length)
-          raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark)
+          raise ScannerError.new("while scanning a #{name}","expected '!', but found #{ch}")
         end
         length += 1
       end
@@ -1285,8 +1187,7 @@ module RbYAML
       value
     end
     
-    STRANGE_CHR = /[\]\[\-';\/?:@&=+$,.!~*()%\w]/
-    def scan_tag_uri(name, start_mark)
+    def scan_tag_uri(name)
       # See the specification for details.
       # Note: we do not check if URI is well-formed.
       chunks = []
@@ -1297,7 +1198,7 @@ module RbYAML
           chunks << prefix(length)
           forward(length)
           length = 0
-          chunks << scan_uri_escapes(name, start_mark)
+          chunks << scan_uri_escapes(name)
         else
           length += 1
         end
@@ -1306,22 +1207,19 @@ module RbYAML
       if length!=0
         chunks << prefix(length)
         forward(length)
-        length = 0
       end
       
-      raise ScannerError.new("while parsing a #{name}", start_mark,"expected URI, but found #{ch}",get_mark) if chunks.empty?
+      raise ScannerError.new("while parsing a #{name}","expected URI, but found #{ch}") if chunks.empty?
       chunks.to_s
     end
 
-    HEXA_REG = /[0-9A-Fa-f]/
-    def scan_uri_escapes(name, start_mark)
+    def scan_uri_escapes(name)
       # See the specification for details.
       bytes = []
-      mark = get_mark
       while peek0 == ?%
         forward1
-        raise ScannerError.new("while scanning a #{name}", start_mark,"expected URI escape sequence of 2 hexdecimal numbers, but found #{peek1} and #{peek2}",get_mark) if HEXA_REG !~ peek1.chr || HEXA_REG !~ peek2.chr
-        bytes << prefix(2).to_i.to_s(16)
+        raise ScannerError.new("while scanning a #{name}","expected URI escape sequence of 2 hexdecimal numbers, but found #{peek1} and #{peek2}") if HEXA_REG !~ peek1.chr || HEXA_REG !~ peek2.chr
+        bytes << prefix(2).to_i(16).to_s
         forward2
       end
       bytes.to_s