RbYAML 0.0.2 → 0.1.0

data/lib/rbyaml/scanner.rb.~1.2.~

@@ -0,0 +1,1259 @@
+# Scanner produces tokens of the following types:
+# STREAM-START
+# STREAM-END
+# DIRECTIVE(name, value)
+# DOCUMENT-START
+# DOCUMENT-END
+# BLOCK-SEQUENCE-START
+# BLOCK-MAPPING-START
+# BLOCK-END
+# FLOW-SEQUENCE-START
+# FLOW-MAPPING-START
+# FLOW-SEQUENCE-END
+# FLOW-MAPPING-END
+# BLOCK-ENTRY
+# FLOW-ENTRY
+# KEY
+# VALUE
+# ALIAS(value)
+# ANCHOR(value)
+# TAG(value)
+# SCALAR(value, plain)
+#
+# Read comments in the Scanner code for more details.
+#
+
+require 'rbyaml/error'
+require 'rbyaml/tokens'
+
+module RbYAML
+  class ScannerError < MarkedYAMLError
+  end
+  
+  class SimpleKey
+    attr_reader :token_number, :required, :index, :line, :column, :mark
+
+    def initialize(token_number,required,index,line,column,mark)
+      @token_number = token_number
+      @required = required
+      @index = index
+      @line = line
+      @column = column
+      @mark = mark
+    end
+  end
+
+  module Scanner
+    def initialize_scanner
+      # It is assumed that Scanner and Reader will mixin to the same point.
+      # Reader do the dirty work of checking for BOM. It also adds NUL to the end.
+      #
+      # Reader supports the following methods
+      #   self.peek(i=0)       # peek the next i-th character
+      #   self.prefix(l=1)     # peek the next l characters
+      #   self.forward(l=1)    # read the next l characters and move the pointer.
+
+      # Had we reached the end of the stream?
+      @done = false
+
+      # The number of unclosed '{' and '['. `flow_level == 0` means block
+      # context.
+      @flow_level = 0
+
+      # List of processed tokens that are not yet emitted.
+      @tokens = []
+
+      # Add the STREAM-START token.
+      fetch_stream_start
+
+      # Number of tokens that were emitted through the `get_token` method.
+      @tokens_taken = 0
+
+      # The current indentation level.
+      @indent = -1
+
+      # Past indentation levels.
+      @indents = []
+
+      # Variables related to simple keys treatment.
+
+      # A simple key is a key that is not denoted by the '?' indicator.
+      # Example of simple keys:
+      #   ---
+      #   block simple key: value
+      #   ? not a simple key:
+      #   : { flow simple key: value }
+      # We emit the KEY token before all keys, so when we find a potential
+      # simple key, we try to locate the corresponding ':' indicator.
+      # Simple keys should be limited to a single line and 1024 characters.
+
+      # Can a simple key start at the current position? A simple key may
+      # start:
+      # - at the beginning of the line, not counting indentation spaces
+      #       (in block context),
+      # - after '{', '[', ',' (in the flow context),
+      # - after '?', ':', '-' (in the block context).
+      # In the block context, this flag also signifies if a block collection
+      # may start at the current position.
+      @allow_simple_key = true
+
+      # Keep track of possible simple keys. This is a dictionary. The key
+      # is `flow_level`; there can be no more that one possible simple key
+      # for each level. The value is a SimpleKey record:
+      #   (token_number, required, index, line, column, mark)
+      # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
+      # '[', or '{' tokens.
+      @possible_simple_keys = {}
+    end
+
+    def check_token(*choices)
+      # Check if the next token is one of the given types.
+      fetch_more_tokens while need_more_tokens
+      unless @tokens.empty?
+        return true if choices.empty?
+        for choice in choices
+          return true if choice === @tokens[0]
+        end
+      end
+      return false
+    end
+    
+    def peek_token
+      # Return the next token, but do not delete if from the queue.
+      fetch_more_tokens while need_more_tokens
+      return @tokens[0] unless @tokens.empty?
+    end
+    
+    def get_token
+      # Return the next token.
+      fetch_more_tokens while need_more_tokens
+      unless @tokens.empty?
+        @tokens_taken += 1
+        @tokens.shift
+      end
+    end
+
+    def each_token
+      fetch_more_tokens while need_more_tokens
+      while !@tokens.empty?
+        @tokens_taken += 1
+        yield @tokens.shift
+        fetch_more_tokens while need_more_tokens
+      end
+    end
+    
+    def need_more_tokens
+      return false if @done
+      return true if @tokens.empty?
+      # The current token may be a potential simple key, so we
+      # need to look further.
+      stale_possible_simple_keys
+      return true if next_possible_simple_key == @tokens_taken
+    end
+
+    def fetch_more_tokens
+      # Eat whitespaces and comments until we reach the next token.
+      scan_to_next_token
+
+      # Remove obsolete possible simple keys.
+      stale_possible_simple_keys
+
+      # Compare the current indentation and column. It may add some tokens
+      # and decrease the current indentation level.
+      unwind_indent(@column)
+
+      # Peek the next character.
+      ch = peek
+
+      return case
+               # Is it the end of stream?
+             when ch == ?\0: fetch_stream_end
+               # Is it a directive?
+             when ch == ?% && check_directive: fetch_directive
+               # Is it the document start?
+             when ch == ?- && check_document_start: fetch_document_start
+               # Is it the document end?
+             when ch == ?. && check_document_end: fetch_document_end
+               # Is it the flow sequence start indicator?
+             when ch == ?[: fetch_flow_sequence_start
+               # Is it the flow mapping start indicator?
+             when ch == ?{: fetch_flow_mapping_start
+               # Is it the flow sequence end indicator?
+             when ch == ?]: fetch_flow_sequence_end
+               # Is it the flow mapping end indicator?
+             when ch == ?}: fetch_flow_mapping_end
+               # Is it the flow entry indicator?
+             when ch == ?,: fetch_flow_entry
+               # Is it the block entry indicator?
+             when ch == ?- && check_block_entry: fetch_block_entry
+               # Is it the key indicator?
+             when ch == ?? && check_key: fetch_key
+               # Is it the value indicator?
+             when ch == ?: && check_value: fetch_value
+               # Is it an alias?
+             when ch == ?*: fetch_alias
+               # Is it an anchor?
+             when ch == ?&: fetch_anchor
+               # Is it a tag?
+             when ch == ?!: fetch_tag
+               # Is it a literal scalar?
+             when ch == ?| && @flow_level==0: fetch_literal
+               # Is it a folded scalar?
+             when ch == ?> && @flow_level==0: fetch_folded
+               # Is it a single quoted scalar?
+             when ch == ?': fetch_single
+               # Is it a double quoted scalar?
+             when ch == ?": fetch_double
+               # It must be a plain scalar then.
+             when check_plain: fetch_plain
+             else raise ScannerError.new("while scanning for the next token", nil,"found character #{ch.chr}(#{ch}) that cannot start any token",get_mark)
+             end
+    end
+
+    # Simple keys treatment.
+    
+    def next_possible_simple_key
+      # Return the number of the nearest possible simple key. Actually we
+      # don't need to loop through the whole dictionary.
+      min_token_number = nil
+      for level in @possible_simple_keys.keys
+        key = @possible_simple_keys[level]
+        if min_token_number.nil? || key.token_number < min_token_number
+          min_token_number = key.token_number
+        end
+      end
+      min_token_number
+    end
+    
+    def stale_possible_simple_keys
+      # Remove entries that are no longer possible simple keys. According to
+      # the YAML specification, simple keys
+      # - should be limited to a single line,
+      # - should be no longer than 1024 characters.
+      # Disabling this procedure will allow simple keys of any length and
+      # height (may cause problems if indentation is broken though).
+      @possible_simple_keys.delete_if {|level,key| 
+        if key.line != @line || @index-key.index > 1024
+          raise ScannerError.new("while scanning a simple key", key.mark, "could not found expected ':'",get_mark) if key.required
+          return true
+        end
+        return false
+      }
+    end
+
+    def save_possible_simple_key
+      # The next token may start a simple key. We check if it's possible
+      # and save its position. This function is called for
+      #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
+      
+      # Check if a simple key is required at the current position.
+      required = @flow_level==0 && @indent == @column
+
+      # The next token might be a simple key. Let's save it's number and
+      # position.
+      if @allow_simple_key
+        remove_possible_simple_key
+        token_number = @tokens_taken+@tokens.length
+        key = SimpleKey.new(token_number, required,@index,@line,@column,get_mark)
+        @possible_simple_keys[@flow_level] = key
+      end
+    end
+
+    def remove_possible_simple_key
+      # Remove the saved possible key position at the current flow level.
+      key = @possible_simple_keys[@flow_level] if @possible_simple_keys.member?(@flow_level)
+    end            
+    
+    # Indentation functions.
+
+    def unwind_indent(column)
+      ## In flow context, tokens should respect indentation.
+      ## Actually the condition should be `@indent >= column` according to
+      ## the spec. But this condition will prohibit intuitively correct
+      ## constructions such as
+      ## key : {
+      ## }
+      #if @flow_level and @indent > column
+      #    raise ScannerError(nil, nil,
+      #            "invalid intendation or unclosed '[' or '{'",
+      #            get_mark)
+
+      # In the flow context, indentation is ignored. We make the scanner less
+      # restrictive then specification requires.
+      return nil if @flow_level != 0
+      # In block context, we may need to issue the BLOCK-END tokens.
+      while @indent > column
+        mark = get_mark
+        @indent = @indents.pop()
+        @tokens << BlockEndToken.new(mark, mark)
+      end
+    end
+    
+    def add_indent(column)
+      # Check if we need to increase indentation.
+      if @indent < column
+        @indents << @indent
+        @indent = column
+        return true
+      end
+      return false
+    end
+
+    # Fetchers.
+
+    def fetch_stream_start
+      # We always add STREAM-START as the first token and STREAM-END as the
+      # last token.
+      # Read the token.
+      mark = get_mark
+      # Add STREAM-START.
+      @tokens << StreamStartToken.new(mark, mark, @encoding)
+    end
+
+
+    def fetch_stream_end
+      # Set the current intendation to -1.
+      unwind_indent(-1)
+      # Reset everything (not really needed).
+      @allow_simple_key = false
+      @possible_simple_keys = {}
+      # Read the token.
+      mark = get_mark
+      # Add STREAM-END.
+      @tokens << StreamEndToken.new(mark, mark)
+      # The stream is finished.
+      @done = true
+    end
+
+    def fetch_directive
+      # Set the current intendation to -1.
+      unwind_indent(-1)
+      # Reset simple keys.
+      remove_possible_simple_key
+      @allow_simple_key = false
+      # Scan and add DIRECTIVE.
+      @tokens << scan_directive
+    end
+
+    def fetch_document_start
+      fetch_document_indicator(DocumentStartToken)
+    end
+
+    def fetch_document_end
+      fetch_document_indicator(DocumentEndToken)
+    end
+
+    def fetch_document_indicator(token)
+      # Set the current intendation to -1.
+      unwind_indent(-1)
+      # Reset simple keys. Note that there could not be a block collection
+      # after '---'.
+      remove_possible_simple_key
+      @allow_simple_key = false
+      # Add DOCUMENT-START or DOCUMENT-END.
+      start_mark = get_mark
+      forward(3)
+      end_mark = get_mark
+      @tokens << token.new(start_mark, end_mark)
+    end
+
+    def fetch_flow_sequence_start
+      fetch_flow_collection_start(FlowSequenceStartToken)
+    end
+
+    def fetch_flow_mapping_start
+      fetch_flow_collection_start(FlowMappingStartToken)
+    end
+
+    def fetch_flow_collection_start(token)
+      # '[' and '{' may start a simple key.
+      save_possible_simple_key
+      # Increase the flow level.
+      @flow_level += 1
+      # Simple keys are allowed after '[' and '{'.
+      @allow_simple_key = true
+      # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << token.new(start_mark, end_mark)
+    end
+    
+    def fetch_flow_sequence_end
+      fetch_flow_collection_end(FlowSequenceEndToken)
+    end
+    
+    def fetch_flow_mapping_end
+      fetch_flow_collection_end(FlowMappingEndToken)
+    end
+    
+    def fetch_flow_collection_end(token)
+      # Reset possible simple key on the current level.
+      remove_possible_simple_key
+      # Decrease the flow level.
+      @flow_level -= 1
+      # No simple keys after ']' or '}'.
+      @allow_simple_key = false
+      # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << token.new(start_mark, end_mark)
+    end
+
+    def fetch_flow_entry
+      # Simple keys are allowed after ','.
+      @allow_simple_key = true
+      # Reset possible simple key on the current level.
+      remove_possible_simple_key
+      # Add FLOW-ENTRY.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << FlowEntryToken.new(start_mark, end_mark)
+    end
+
+    def fetch_block_entry
+      # Block context needs additional checks.
+      if @flow_level==0
+        raise ScannerError.new(nil,nil,"sequence entries are not allowed here",get_mark) if !@allow_simple_key
+        # We may need to add BLOCK-SEQUENCE-START.
+        if add_indent(@column)
+          mark = get_mark
+          @tokens << BlockSequenceStartToken.new(mark, mark)
+        end
+        # It's an error for the block entry to occur in the flow context,
+        # but we let the parser detect this.
+      end
+      # Simple keys are allowed after '-'.
+      @allow_simple_key = true
+      # Reset possible simple key on the current level.
+      remove_possible_simple_key
+      # Add BLOCK-ENTRY.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << BlockEntryToken.new(start_mark, end_mark)
+    end
+
+    def fetch_key
+      # Block context needs additional checks.
+      if @flow_level==0
+        # Are we allowed to start a key (not nessesary a simple)?
+        raise ScannerError.new(nil,nil,"mapping keys are not allowed here",get_mark) if !@allow_simple_key
+        # We may need to add BLOCK-MAPPING-START.
+        if add_indent(@column)
+          mark = get_mark
+          @tokens << BlockMappingStartToken.new(mark, mark)
+        end
+      end
+      # Simple keys are allowed after '?' in the block context.
+      @allow_simple_key = @flow_level==0
+      # Reset possible simple key on the current level.
+      remove_possible_simple_key
+      # Add KEY.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << KeyToken.new(start_mark, end_mark)
+    end
+    
+    def fetch_value
+      # Do we determine a simple key?
+      if @possible_simple_keys.include?(@flow_level)
+        # Add KEY.
+        key = @possible_simple_keys[@flow_level]
+        @possible_simple_keys.delete(@flow_level)
+        @tokens.insert(key.token_number-@tokens_taken,KeyToken.new(key.mark, key.mark))
+        # If this key starts a new block mapping, we need to add
+        # BLOCK-MAPPING-START.
+        @tokens.insert(key.token_number-@tokens_taken,BlockMappingStartToken.new(key.mark, key.mark)) if @flow_level==0 && add_indent(key.column)
+        # There cannot be two simple keys one after another.
+        @allow_simple_key = false
+        # It must be a part of a complex key.
+      else
+        # Block context needs additional checks.
+        # (Do we really need them? They will be catched by the parser
+        # anyway.)
+        if @flow_level==0
+          # We are allowed to start a complex value if and only if
+          # we can start a simple key.
+          raise ScannerError.new(nil,nil,"mapping values are not allowed here",get_mark) if !@allow_simple_key
+          # Simple keys are allowed after ':' in the block context.
+          @allow_simple_key = @flow_level==0
+          # Reset possible simple key on the current level.
+          remove_possible_simple_key
+        end
+      end
+      # Add VALUE.
+      start_mark = get_mark
+      forward
+      end_mark = get_mark
+      @tokens << ValueToken.new(start_mark, end_mark)
+    end
+
+    def fetch_alias
+      # ALIAS could be a simple key.
+      save_possible_simple_key
+      # No simple keys after ALIAS.
+      @allow_simple_key = false
+      # Scan and add ALIAS.
+      @tokens << scan_anchor(AliasToken)
+    end
+
+    def fetch_anchor
+      # ANCHOR could start a simple key.
+      save_possible_simple_key
+      # No simple keys after ANCHOR.
+      @allow_simple_key = false
+      # Scan and add ANCHOR.
+      @tokens << scan_anchor(AnchorToken)
+    end
+
+    def fetch_tag
+      # TAG could start a simple key.
+      save_possible_simple_key
+      # No simple keys after TAG.
+      @allow_simple_key = false
+      # Scan and add TAG.
+      @tokens << scan_tag
+    end
+
+    def fetch_literal
+      fetch_block_scalar(?|)
+    end
+
+    def fetch_folded
+      fetch_block_scalar(?>)
+    end
+
+    def fetch_block_scalar(style)
+      # A simple key may follow a block scalar.
+      @allow_simple_key = true
+      # Reset possible simple key on the current level.
+      remove_possible_simple_key
+      # Scan and add SCALAR.
+      @tokens << scan_block_scalar(style)
+    end
+
+    def fetch_single
+      fetch_flow_scalar(?')
+    end
+    
+    def fetch_double
+      fetch_flow_scalar(?")
+    end
+
+    def fetch_flow_scalar(style)
+      # A flow scalar could be a simple key.
+      save_possible_simple_key
+      # No simple keys after flow scalars.
+      @allow_simple_key = false
+      # Scan and add SCALAR.
+      @tokens << scan_flow_scalar(style)
+    end
+    
+    def fetch_plain
+      # A plain scalar could be a simple key.
+      save_possible_simple_key
+      # No simple keys after plain scalars. But note that `scan_plain` will
+      # change this flag if the scan is finished at the beginning of the
+      # line.
+      @allow_simple_key = false
+      # Scan and add SCALAR. May change `allow_simple_key`.
+      @tokens << scan_plain
+    end
+
+    # Checkers.
+
+    def check_directive
+      # DIRECTIVE:        ^ '%' ...
+      # The '%' indicator is already checked.
+      @column == 0
+    end
+
+    def check_document_start
+      # DOCUMENT-START:   ^ '---' (' '|'\n')
+      @column == 0 && prefix(3) == "---" && "\0 \t\r\n\x85".include?(peek(3))
+    end
+
+    def check_document_end
+      # DOCUMENT-END:     ^ '...' (' '|'\n')
+      @column == 0 && prefix(3) == "..." && "\0 \t\r\n\x85".include?(peek(3))
+    end
+
+    def check_block_entry
+      # BLOCK-ENTRY:      '-' (' '|'\n')
+      "\0 \t\r\n\x85".include?(peek(1))
+    end
+    
+    def check_key
+      # KEY(flow context):    '?'
+      # KEY(block context):   '?' (' '|'\n')
+      @flow_level!=0 || "\0 \t\r\n\x85".include?(peek(1)) 
+    end
+
+    def check_value
+      # VALUE(flow context):  ':'
+      # VALUE(block context): ':' (' '|'\n')
+      @flow_level!=0 || "\0 \t\r\n\x85".include?(peek(1)) 
+    end
+    
+    def check_plain
+      # A plain scalar may start with any non-space character except:
+      #   '-', '?', ':', ',', '[', ']', '{', '}',
+      #   '#', '&', '*', '!', '|', '>', '\'', '\"',
+      #   '%', '@', '`'.
+      #
+      # It may also start with
+      #   '-', '?', ':'
+      # if it is followed by a non-space character.
+      #
+      # Note that we limit the last rule to the block context (except the
+      # '-' character) because we want the flow context to be space
+      # independent.
+      ch = peek
+      !("\0 \t\r\n\x85-?:,[]{}#&*!|>'\"%@`".include?(ch)) || (!("\0 \t\r\n\x85".include?(peek(1)) && (ch == ?- || (@flow_level==0 && "?:".include?(ch)))))
+    end
+
+
+
+
+
+    
+    # Scanners.
+    
+    def scan_to_next_token
+      # We ignore spaces, line breaks and comments.
+      # If we find a line break in the block context, we set the flag
+      # `allow_simple_key` on.
+      #
+      # TODO: We need to make tab handling rules more sane. A good rule is
+      #   Tabs cannot precede tokens
+      #   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
+      #   KEY(block), VALUE(block), BLOCK-ENTRY
+      # So the checking code is
+      #   if <TAB>:
+      #       @allow_simple_keys = false
+      # We also need to add the check for `allow_simple_keys == true` to
+      # `unwind_indent` before issuing BLOCK-END.
+      # Scanners for block, flow, and plain scalars need to be modified.
+      found = false
+      while !found
+        while peek == 32
+          forward
+        end
+        if peek == ?#
+            forward while !"\0\r\n\x85".include?(peek)
+        end
+        if !scan_line_break.empty?
+          @allow_simple_key = true if @flow_level==0
+        else
+          found = true
+        end
+      end
+    end
+    
+    def scan_directive
+      # See the specification for details.
+      start_mark = get_mark
+      forward
+      name = scan_directive_name(start_mark)
+      value = nil
+      if name == "YAML"
+        value = scan_yaml_directive_value(start_mark)
+        end_mark = get_mark
+      elsif name == "TAG"
+        value = scan_tag_directive_value(start_mark)
+        end_mark = get_mark
+      else
+        end_mark = get_mark
+        forward while !"\0\r\n\x85".include?(peek)
+      end
+      scan_directive_ignored_line(start_mark)
+      DirectiveToken.new(name, value, start_mark, end_mark)
+    end
+
+    def scan_directive_name(start_mark)
+      # See the specification for details.
+      length = 0
+      ch = peek(length)
+      while /[-0-9A-Za-z_]/ =~ ch.chr
+        length += 1
+        ch = peek(length)
+      end
+      raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if length==0
+      value = prefix(length)
+      forward(length)
+      ch = peek()
+      raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if !"\0 \r\n\x85".include?(ch)
+      value
+    end
+
+    def scan_yaml_directive_value(start_mark)
+      # See the specification for details.
+      forward while peek == 32
+      major = scan_yaml_directive_number(start_mark)
+      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or '.', but found #{peek.to_s}",get_mark) if peek != ?.
+      forward
+      minor = scan_yaml_directive_number(start_mark)
+      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or ' ', but found #{peek.to_s}",get_mark) if !"\0 \r\n\x85".include?(peek)
+      [major, minor]
+    end
+
+    def scan_yaml_directive_number(start_mark)
+      # See the specification for details.
+      ch = peek
+      raise ScannerError.new("while scanning a directive", start_mark,"expected a digit, but found #{ch.to_s}",get_mark) if !((?0..?9) === ch)
+      length = 0
+      length += 1 while ((?0..?9) === peek(length))
+      value = prefix(length)
+      forward(length)
+      value
+    end
+    
+    def scan_tag_directive_value(start_mark)
+      # See the specification for details.
+      forward while peek == 32
+      handle = scan_tag_directive_handle(start_mark)
+      forward while peek == 32
+      prefix = scan_tag_directive_prefix(start_mark)
+      [handle, prefix]
+    end
+
+    def scan_tag_directive_handle(start_mark)
+      # See the specification for details.
+      value = scan_tag_handle("directive", start_mark)
+      ch = peek
+      raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{ch}",get_mark()) if ch != 32
+      value
+    end
+
+    def scan_tag_directive_prefix(start_mark)
+      # See the specification for details.
+      value = scan_tag_uri("directive", start_mark)
+      ch = peek
+      raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{ch}",get_mark()) if !"\0 \r\n\x85".include?(ch)
+      value
+    end
+    
+    def scan_directive_ignored_line(start_mark)
+      # See the specification for details.
+      forward while peek == 32
+      if peek == ?#
+          forward while !"\0\r\n\x85".include?(peek)
+      end
+      ch = peek
+      raise ScannerError.new("while scanning a directive", start_mark,"expected a comment or a line break, but found #{ch.to_s}",get_mark()) if !"\0\r\n\x85".include?(ch)
+      scan_line_break
+    end
+    
+    def scan_anchor(token)
+      # The specification does not restrict characters for anchors and
+      # aliases. This may lead to problems, for instance, the document:
+      #   [ *alias, value ]
+      # can be interpteted in two ways, as
+      #   [ "value" ]
+      # and
+      #   [ *alias , "value" ]
+      # Therefore we restrict aliases to numbers and ASCII letters.
+      start_mark = get_mark
+      indicator = peek
+      name = (indicator == ?*) ? "alias":"anchor"
+      forward
+      length = 0
+      ch = peek(length)
+      while /[-0-9A-Za-z_]/ =~ ch.chr
+        length += 1
+        ch = peek(length)
+      end
+      raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found #{ch}",get_mark) if length==0
+      value = prefix(length)
+      forward(length)
+      ch = peek
+      if !"\0 \t\r\n\x85?:,]}%@`".include?(ch)
+        raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found #{ch}",get_mark)
+      end
+      end_mark = get_mark
+      token.new(value, start_mark, end_mark)
+    end
+
+
+    def scan_tag
+      # See the specification for details.
+      start_mark = get_mark
+      ch = peek(1)
+      if ch == ?<
+        handle = nil
+        forward(2)
+        suffix = scan_tag_uri("tag", start_mark)
+        raise ScannerError.new("while parsing a tag", start_mark,"expected '>', but found #{peek.to_s}",get_mark) if peek != ?>
+        forward
+      elsif "\0 \t\r\n\x85".include?(ch)
+        handle = nil
+        suffix = "!"
+        forward
+      else
+        length = 1
+        use_handle = false
+        while !"\0 \t\r\n\x85".include?(ch)
+          if ch == ?!
+            use_handle = true
+            break
+          end
+          length += 1
+          ch = peek(length)
+        end
+        handle = "!"
+        if use_handle
+          handle = scan_tag_handle("tag", start_mark)
+        else
+          handle = "!"
+          forward
+        end
+        suffix = scan_tag_uri("tag", start_mark)
+      end
+      ch = peek
+      raise ScannerError.new("while scanning a tag",start_mark,"expected ' ', but found #{ch}",get_mark) if !"\0 \r\n\x85".include?(ch)
+      value = [handle, suffix]
+      end_mark = get_mark
+      TagToken.new(value, start_mark, end_mark)
+    end
+
+    def scan_block_scalar(style)
+      # See the specification for details.
+      folded = style== ?>
+      chunks = []
+      start_mark = get_mark
+      # Scan the header.
+      forward
+      chomping, increment = scan_block_scalar_indicators(start_mark)
+      scan_block_scalar_ignored_line(start_mark)
+      # Determine the indentation level and go to the first non-empty line.
+      min_indent = @indent+1
+      min_indent = 1 if min_indent < 1
+      if increment.nil?
+        breaks, max_indent, end_mark = scan_block_scalar_indentation
+        indent = [min_indent, max_indent].max
+      else
+        indent = min_indent+increment-1
+        breaks, end_mark = scan_block_scalar_breaks(indent)
+      end
+      line_break = ''
+      # Scan the inner part of the block scalar.
+      while @column == indent and peek != ?\0
+        chunks += breaks
+        leading_non_space = !" \t".include?(peek)
+        length = 0
+        length += 1 while !"\0\r\n\x85".include?(peek(length))
+        chunks << prefix(length)
+        forward(length)
+        line_break = scan_line_break
+        breaks, end_mark = scan_block_scalar_breaks(indent)
+        if @column == indent && peek != 0
+          # Unfortunately, folding rules are ambiguous.
+          #
+          # This is the folding according to the specification:
+          if folded && line_break == ?\n && leading_non_space && !" \t".include?(peek())
+            chunks << ' ' if breaks.empty?
+          else
+            chunks << line_break
+          end 
+          # This is Clark Evans's interpretation (also in the spec
+          # examples):
+          #
+          #if folded and line_break == u'\n':
+          #    if not breaks:
+          #        if self.peek() not in ' \t':
+          #            chunks.append(u' ')
+          #        else:
+          #            chunks.append(line_break)
+          #else:
+          #    chunks.append(line_break)
+        else
+          break
+        end
+      end
+      
+      # Chomp the tail.
+      if chomping
+        chunks << line_break
+        chunks += breaks
+      end
+
+      # We are done.
+      ScalarToken.new(chunks.join(''), false, start_mark, end_mark,style)
+    end
+
+    def scan_block_scalar_indicators(start_mark)
+      # See the specification for details.
+      chomping = nil
+      increment = nil
+      ch = peek
+      if /[+-]/ =~ ch.chr
+        chomping = ch == ?+
+        forward
+        ch = peek
+        if (?0..?9) === ch
+          increment = ch.to_i
+          raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0
+          forward
+        end
+      elsif (?0..?9) === ch
+        increment = ch
+        raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0
+        forward
+        ch = peek
+        if /[+-]/ =~ ch.chr
+          chomping = ch == ?+
+          forward
+        end
+      end
+      ch = peek
+      raise ScannerError.new("while scanning a block scalar", start_mark,"expected chomping or indentation indicators, but found #{ch.to_s}",get_mark) if !"\0 \r\n\x85".include?(ch)
+      [chomping, increment]
+    end
+
+    def scan_block_scalar_ignored_line(start_mark)
+      # See the specification for details.
+      forward while peek == 32
+      if peek == ?#
+          forward while !"\0\r\n\x85".include?(peek)
+      end
+      ch = peek
+      
+      raise ScannerError.new("while scanning a block scalar", start_mark,"expected a comment or a line break, but found #{ch.to_s}",get_mark) if !"\0\r\n\x85".include?(ch)
+      scan_line_break
+    end
+
+    def scan_block_scalar_indentation
+      # See the specification for details.
+      chunks = []
+      max_indent = 0
+      end_mark = get_mark
+      while " \r\n\x85".include?(peek)
+        if peek != 32
+          chunks << scan_line_break
+          end_mark = get_mark
+        else
+          forward
+          max_indent = @column if @column > max_indent
+        end
+      end
+      [chunks, max_indent, end_mark]
+    end
+
+    def scan_block_scalar_breaks(indent)
+      # See the specification for details.
+      chunks = []
+      end_mark = get_mark
+      forward while @column < indent && peek == 32
+      while "\r\n\x85".include?(peek)
+        chunks << scan_line_break
+        end_mark = get_mark
+        forward while @column < indent && peek == 32
+      end
+      [chunks, end_mark]
+    end
+
+    def scan_flow_scalar(style)
+      # See the specification for details.
+      # Note that we loose indentation rules for quoted scalars. Quoted
+      # scalars don't need to adhere indentation because " and ' clearly
+      # mark the beginning and the end of them. Therefore we are less
+      # restrictive then the specification requires. We only need to check
+      # that document separators are not included in scalars.
+      double = style == ?"
+      chunks = []
+      start_mark = get_mark
+      quote = peek
+      forward
+      chunks += scan_flow_scalar_non_spaces(double, start_mark)
+      while peek != quote
+        chunks += scan_flow_scalar_spaces(double, start_mark)
+        chunks += scan_flow_scalar_non_spaces(double, start_mark)
+      end
+      forward
+      end_mark = get_mark
+      ScalarToken.new(chunks.join(''), false, start_mark, end_mark,style)
+    end
+    
+    ESCAPE_REPLACEMENTS = {
+      "0"  =>   "\0",
+      "a"  =>   "\x07",
+      "b"  =>   "\x08",
+      "t"  =>   "\x09",
+      "\t" =>   "\x09",
+      "n"  =>   "\x0A",
+      "v"  =>   "\x0B",
+      "f"  =>   "\x0C",
+      "r"  =>   "\x0D",
+      "e"  =>   "\x1B",
+      " "  =>   "\x20",
+      '"'  =>   '"',
+      "\\" =>   "\\",
+      "N"  =>   "\x85",
+      "_"  =>   "\xA0"
+    }
+    
+    ESCAPE_CODES = {
+      'x' =>   2
+    }
+    
+    def scan_flow_scalar_non_spaces(double, start_mark)
+      # See the specification for details.
+      chunks = []
+      while true
+        length = 0
+        length += 1 while !"'\"\\\0 \t\r\n\x85".include?(peek(length))
+        if length!=0
+          chunks << prefix(length)
+          forward(length)
+        end
+        ch = peek
+        if !double && ch == ?' && peek(1) == ?'
+          chunks << ?'
+          forward(2)
+        elsif (double && ch == ?') || (!double && "\"\\".include?(ch))
+          chunks << ch
+          forward
+        elsif double && ch == ?\\
+          forward
+          ch = peek
+          if ESCAPE_REPLACEMENTS.member?(ch.chr)
+            chunks << ESCAPE_REPLACEMENTS[ch.chr]
+            forward
+          elsif ESCAPE_CODES.member?(ch.chr)
+            length = ESCAPE_CODES[ch.chr]
+            forward
+            length.times do |k|
+              if /[0-9A-Fa-f]/ !~ peek(k).chr
+                raise ScannerError.new("while scanning a double-quoted scalar", start_mark,
+                                       "expected escape sequence of #{length} hexdecimal numbers, but found #{peek(k)}",get_mark)
+              end
+            end
+            code = prefix(length).to_i.to_s(16)
+            chunks << code
+            forward(length)
+          elsif "\r\n\x85".include?(ch)
+            scan_line_break
+            chunks += scan_flow_scalar_breaks(double, start_mark)
+          else
+            raise ScannerError.new("while scanning a double-quoted scalar", start_mark,"found unknown escape character #{ch}",get_mark)
+          end
+        else
+          return chunks
+        end
+      end
+    end
+
+    def scan_flow_scalar_spaces(double, start_mark)
+      # See the specification for details.
+      chunks = []
+      length = 0
+      length += 1 while /[ \t]/ =~ peek(length).chr
+      whitespaces = prefix(length)
+      forward(length)
+      ch = peek
+      if ch == ?\0
+        raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected end of stream",get_mark)
+      elsif "\r\n\x85".include?(ch)
+        line_break = scan_line_break
+        breaks = scan_flow_scalar_breaks(double, start_mark)
+        if line_break != ?\n
+          chunks << line_break
+        elsif breaks.empty?
+          chunks << ' '
+        end
+        chunks += breaks
+      else
+        chunks << whitespaces
+      end
+      chunks
+    end
+    
+    def scan_flow_scalar_breaks(double, start_mark)
+      # See the specification for details.
+      chunks = []
+      while true
+        # Instead of checking indentation, we check for document
+        # separators.
+        prefix = prefix(3)
+        if (prefix == "---" || prefix == "...") && "\0 \t\r\n\x85".include?(peek(3))
+          raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected document separator", get_mark)
+        end
+        forward while /[ \t]/ =~ peek.chr
+        if "\r\n\x85".include?(peek)
+          chunks << scan_line_break
+        else
+          return chunks
+        end
+      end
+    end
+
+    def scan_plain
+      # See the specification for details.
+      # We add an additional restriction for the flow context:
+      #   plain scalars in the flow context cannot contain ',', ':' and '?'.
+      # We also keep track of the `allow_simple_key` flag here.
+      # Indentation rules are loosed for the flow context.
+      chunks = []
+      start_mark = get_mark
+      end_mark = start_mark
+      indent = @indent+1
+      # We allow zero indentation for scalars, but then we need to check for
+      # document separators at the beginning of the line.
+      #if indent == 0
+      #    indent = 1
+      spaces = []
+      while true
+        length = 0
+        break if peek == ?#
+        while true
+          ch = peek(length)
+          if "\0 \t\r\n\x85".include?(ch) || (@flow_level==0 && ch == ?: && "\0 \t\r\n\x28".include?(peek(length+1))) || (@flow_level!=0 && ",:?[]{}".include?(ch))
+            break
+          end
+          length += 1
+        end
+        if @flow_level != 0 && ch == ?: && !"\0 \t\r\n\x28[]{}".include?(peek(length+1))
+          forward(length)
+          raise ScannerError.new("while scanning a plain scalar",start_mark,"found unexpected ':'",get_mark,"Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
+        end
+        break if length == 0
+        @allow_simple_key = false
+        chunks += spaces
+        chunks << prefix(length)
+        forward(length)
+        end_mark = get_mark
+        spaces = scan_plain_spaces(indent, start_mark)
+        break if spaces.nil? || spaces.empty? || peek == ?# || (@flow_level==0 && @column < indent)
+      end
+      return ScalarToken.new(chunks.join(''), true, start_mark, end_mark)
+    end
+
+    def scan_plain_spaces(indent, start_mark)
+      # See the specification for details.
+      # The specification is really confusing about tabs in plain scalars.
+      # We just forbid them completely. Do not use tabs in YAML!
+      chunks = []
+      length = 0
+      length += 1 while peek(length) == 32
+      whitespaces = prefix(length)
+      forward(length)
+      ch = peek
+      if "\r\n\x85".include?(ch)
+        line_break = scan_line_break
+        @allow_simple_key = true
+        prefix = prefix(3)
+        return if (prefix == "---" || prefix == "...") && "\0 \t\r\n\x85".include?(peek(3))
+        breaks = []
+        while " \r\n\x85".include?(peek)
+          if peek == 32
+            forward
+          else
+            breaks << scan_line_break
+            prefix = prefix(3)
+            return if (prefix == "---" || prefix == "...") && "\0 \t\r\n\x85".include?(peek(3))
+          end
+        end
+        if line_break != '\n'
+          chunks << line_break
+        elsif breaks.empty?
+          chunks << ' '
+        end
+        chunks += breaks
+      elsif !whitespaces.empty?
+        chunks << whitespaces
+      end
+      chunks
+    end
+
+    def scan_tag_handle(name, start_mark)
+      # See the specification for details.
+      # For some strange reasons, the specification does not allow '_' in
+      # tag handles. I have allowed it anyway.
+      ch = peek
+      raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark) if ch != ?!
+      length = 1
+      ch = peek(length)
+      if ch != 32
+        while /[-_0-9A-Za-z]/ =~ ch.chr
+          length += 1
+          ch = peek(length)
+        end
+        if ch != ?!
+          forward(length)
+          raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark)
+        end
+        length += 1
+      end
+      value = prefix(length)
+      forward(length)
+      value
+    end
+    
+    def scan_tag_uri(name, start_mark)
+      # See the specification for details.
+      # Note: we do not check if URI is well-formed.
+      chunks = []
+      length = 0
+      ch = peek(length)
+      while /[\]\[\-';\/?:@&=+$,.!~*()%\w]/ =~ ch.chr
+        if ch == ?%
+          chunks << prefix(length)
+          forward(length)
+          length = 0
+          chunks << scan_uri_escapes(name, start_mark)
+        else
+          length += 1
+        end
+        ch = peek(length)
+      end
+      if length!=0
+        chunks << prefix(length)
+        forward(length)
+        length = 0
+      end
+      
+      raise ScannerError.new("while parsing a #{name}", start_mark,"expected URI, but found #{ch}",get_mark) if chunks.empty?
+      chunks.join('')
+    end
+
+    def scan_uri_escapes(name, start_mark)
+      # See the specification for details.
+      bytes = []
+      mark = get_mark
+      while peek == ?%
+        forward
+        2.times do |k|
+          raise ScannerError.new("while scanning a #{name}", start_mark,"expected URI escape sequence of 2 hexdecimal numbers, but found #{peek(k)}", 
+                                 get_mark) if /[0-9A-Fa-f]/ !~ peek(k).chr
+        end
+        bytes << prefix(2).to_i.to_s(16)
+        forward(2)
+      end
+      bytes.join('')
+    end
+
+    def scan_line_break
+      # Transforms:
+      #   '\r\n'      :   '\n'
+      #   '\r'        :   '\n'
+      #   '\n'        :   '\n'
+      #   '\x85'      :   '\n'
+      #   default     :   ''
+      ch = peek
+      if "\r\n\x85".include?(ch)
+        if prefix(2) == "\r\n"
+          forward(2)
+        else
+          forward
+        end
+        return "\n"
+      end
+      ""
+    end
+  end
+end
+