RbYAML 0.0.2 → 0.1.0

data/lib/rbyaml/parser.rb

@@ -53,24 +53,28 @@ module RbYAML
   class ParserError < MarkedYAMLError
   end
 
-  module Parser
-
+  class Parser
     DEFAULT_TAGS = {
       '!' => '!',
       '!!' => 'tag:yaml.org,2002:'
     }
 
-    def initialize_parser
+    def initialize(scanner)
+      @scanner = scanner
       @current_event = nil
       @yaml_version = nil
       @events = nil
       @working_events = nil
       @tag_handles = { }
+      @parse_stack = nil
+      @start_mark = []
+      @tks = []
+
     end
 
     def check_event(*choices)
-      init_events
-      @current_event = @working_events.shift if @current_event.nil?
+      parse_stream
+      @current_event = parse_stream_next if @current_event.nil?
       if @current_event
         return true if choices.empty?
         for choice in choices
@@ -81,409 +85,543 @@ module RbYAML
     end
 
     def peek_event
-      init_events
-      @current_event = @working_events.shift if @current_event.nil?
+      parse_stream
+      @current_event = parse_stream_next unless @current_event
       @current_event
     end
 
     def get_event
-      init_events
-      @current_event = @working_events.shift if @current_event.nil?
+      parse_stream
+      @current_event = parse_stream_next unless @current_event
       value = @current_event
       @current_event = nil
       value
     end
 
-    def init_events
-      @events ||= parse_stream
-      @working_events ||= @events        
-    end
-    
-    def each_event(&block)
-      init_events
-      @events.each(&block)
+    def each_event
+      parse_stream
+      while @current_event = parse_stream_next
+        yield @current_event
+      end
     end
 
     def parse_stream
-      # STREAM-START implicit_document? explicit_document* STREAM-END
-      
-      # Parse start of stream.
-      events = []
-      token = get_token
-      events << StreamStartEvent.new(token.start_mark, token.end_mark,token.encoding)
-      
-      # Parse implicit document.
-      unless check_token(DirectiveToken, DocumentStartToken,StreamEndToken)
-        @tag_handles = DEFAULT_TAGS
-        token = peek_token
-        start_mark = end_mark = token.start_mark
-        events << DocumentStartEvent.new(start_mark, end_mark,false)
-        events += parse_block_node
-        token = peek_token
-        start_mark = end_mark = token.start_mark
-        explicit = false
-        while check_token(DocumentEndToken)
-          token = get_token
-          end_mark = token.end_mark
-          explicit = true
+      if !@parse_stack
+        @parse_stack = [:stream]
+        @tokens = nil
+        @tags = []
+        @anchors = []
+        @start_marks = []
+        @end_marks = []
+      end
+    end
+
+    def parse_stream_next
+      if !@parse_stack.empty?
+        while true
+          meth = @parse_stack.pop
+#puts "our method: :#{meth}"
+#puts "--- with peeked: :#{@scanner.peek_token.class} #{if @scanner.peek_token.respond_to?(:value): @scanner.peek_token.value.inspect; end}"
+          val = send(meth)
+          if !val.nil?
+#puts "returning: #{val}"
+            return val
+          end
         end
-        events << DocumentEndEvent.new(start_mark, end_mark,explicit)
+      else
+        @tokens = nil
+        @tags = []
+        @anchors = []
+        @start_marks = []
+        @end_marks = []
+        return nil
       end
+    end
 
-      # Parse explicit documents.
-      while !check_token(StreamEndToken)
-        token = peek_token
-        start_mark = token.start_mark
-        version, tags = process_directives
-        raise ParserError.new(nil, nil,"expected '<document start>', but found #{peek_token.id}",peek_token.start_mark) unless check_token(DocumentStartToken)
-        token = get_token
+#TERMINALS, definitions
+
+    def stream_start
+      token = @scanner.get_token
+      StreamStartEvent.new(token.start_mark, token.end_mark,token.encoding)
+    end
+
+    def stream_end
+      token = @scanner.get_token
+      StreamEndEvent.new(token.start_mark, token.end_mark)
+    end
+    
+    def document_start_implicit
+      token = @scanner.peek_token
+      version, tags = process_directives
+      DocumentStartEvent.new(token.start_mark,token.start_mark,false)
+    end
+
+    def document_start
+      token = @scanner.peek_token
+      start_mark = token.start_mark
+      version, tags = process_directives
+      raise ParserError.new(nil, nil,"expected '<document start>', but found #{token.tid}",token.start_mark) unless @scanner.peek_token.__is_document_start
+      @token = token = @scanner.get_token
+      end_mark = token.end_mark
+      DocumentStartEvent.new(start_mark, end_mark,true,version,tags)
+    end
+
+    def document_end
+      token = @scanner.peek_token
+      start_mark = end_mark = token.start_mark
+      explicit = false
+      while @scanner.peek_token.__is_document_end
+        @tokens = token = @scanner.get_token
         end_mark = token.end_mark
-        events << DocumentStartEvent.new(start_mark, end_mark,true,version,tags)
-        if check_token(DirectiveToken,DocumentStartToken, DocumentEndToken, StreamEndToken)
-          events << process_empty_scalar(token.end_mark)
-        else
-          events += parse_block_node
-        end
-        token = peek_token
-        start_mark = end_mark = token.start_mark
-        explicit = false
-        while check_token(DocumentEndToken)
-          token = get_token
-          end_mark = token.end_mark
-          explicit=true
-        end
-        events << DocumentEndEvent.new(start_mark, end_mark,explicit)
+        explicit = true
       end
-      # Parse end of stream.
-      token = get_token
-      events << StreamEndEvent.new(token.start_mark, token.end_mark)
-      events
+      DocumentEndEvent.new(start_mark, end_mark, explicit)
     end
 
-    def process_directives
-      # DIRECTIVE*
-      while check_token(DirectiveToken)
-        token = get_token
-        if token.name == "YAML"
-          raise ParserError.new(nil, nil,"found duplicate YAML directive", token.start_mark) if !@yaml_version.nil?
-          major, minor = token.value[0].to_i, token.value[1].to_i
-          raise ParserError.new(nil,nil,"found incompatible YAML document (version 1.* is required)",token.start_mark) if major != 1
-          @yaml_version = [major,minor]
-        elsif token.name == "TAG"
-          handle, prefix = token.value
-          raise ParserError.new(nil,nil,"duplicate tag handle #{handle}",token.start_mark) if @tag_handles.member?(handle)
-          @tag_handles[handle] = prefix
-        end
+    def _alias
+      token = @scanner.get_token
+      AliasEvent.new(token.value, token.start_mark, token.end_mark)
+    end
+
+    def block_sequence_start
+      end_mark = @scanner.peek_token.start_mark
+      implicit = @tags.last.nil? || @tags.last == ?!
+      @tokens = token = @scanner.get_token
+      SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false)
+    end
+
+    def block_indentless_sequence_start
+      end_mark = @scanner.peek_token.end_mark
+      implicit = @tags.last.nil? || @tags.last == ?!
+      SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false)
+    end
+
+    def block_sequence_end
+      if !@scanner.peek_token.__is_block_end
+        token = @scanner.peek_token
+        raise ParserError.new("while scanning a block collection", @start_marks.last,"expected <block end>, but found #{token.tid}: #{token.inspect}", token.start_mark)
       end
-      if !@tag_handles.empty?
-        value = @yaml_version, @tag_handles.dup
-      else
-        value = @yaml_version, nil
+      token = @scanner.get_token
+      SequenceEndEvent.new(token.start_mark, token.end_mark)
+    end
+
+    def block_indentless_sequence_end
+      @tokens = token = @scanner.peek_token
+      SequenceEndEvent.new(token.start_mark, token.end_mark)
+    end
+
+    def block_mapping_start
+      end_mark = @scanner.peek_token.start_mark
+      implicit = @tags.last.nil? || @tags.last == ?!
+      @tokens = token = @scanner.get_token
+      MappingStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false)
+    end
+
+    def block_mapping_end
+      if !@scanner.peek_token.__is_block_end
+        token = @scanner.peek_token
+        raise ParserError.new("while scanning a block mapping", @start_marks.last,"expected <block end>, but found #{token.tid}", token.start_mark)
       end
-      for key in DEFAULT_TAGS.keys
-        @tag_handles[key] = DEFAULT_TAGS[key] if !@tag_handles.include?(key)
+      @tokens = token = @scanner.get_token
+      MappingEndEvent.new(token.start_mark, token.end_mark)
+    end
+
+    def flow_sequence_start
+      end_mark = @scanner.peek_token.end_mark
+      implicit = @tags.last.nil? || @tags.last == ?!
+      @tokens = token = @scanner.get_token
+      SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,true)
+    end
+
+    def flow_sequence_end
+      @tokens = token = @scanner.get_token
+      SequenceEndEvent.new(token.start_mark, token.end_mark)
+    end
+
+    def flow_internal_mapping_start
+      @tokens = token = @scanner.get_token
+      MappingStartEvent.new(nil,nil,true,token.start_mark, token.end_mark,true)
+    end
+
+    def flow_internal_mapping_end
+      token = peek_token
+      MappingEndEvent.new(token.start_mark, token.start_mark)
+    end
+
+    def flow_mapping_start
+      end_mark = @scanner.peek_token.end_mark
+      implicit = @tags.last.nil? || @tags.last == ?!
+      @tokens = token = @scanner.get_token
+      MappingStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,true)
+    end
+
+    def flow_mapping_end
+      @tokens = token = @scanner.get_token
+      MappingEndEvent.new(token.start_mark, token.end_mark)
+    end
+
+    def scalar
+      token = @scanner.get_token
+      end_mark = token.end_mark
+      if (token.plain && @tags.last.nil?) || @tags.last == ?!
+        implicit = [true, false]
+      elsif @tags.last.nil?
+        implicit = [false, true]
+      else
+        implicit = [false, false]
       end
-      value
+      ScalarEvent.new(@anchors.last, @tags.last, implicit, token.value, @start_marks.last, end_mark, token.style)
     end
 
-    def parse_block_node
-      parse_node(true)
+    def empty_scalar
+      process_empty_scalar(@tokens.end_mark)
     end
     
-    def parse_flow_node
-      parse_node
+
+# PRODUCTIONS
+    def stream
+      @parse_stack += [:stream_end, :explicit_document, :implicit_document]
+      stream_start
     end
-    
-    def parse_block_node_or_indentless_sequence
-      parse_node(true, true)
-    end
-
-    def parse_node(block=false, indentless_sequence=false)
-      # block_node    ::= ALIAS | properties? block_content
-      # flow_node     ::= ALIAS | properties? flow_content
-      # properties    ::= TAG ANCHOR? | ANCHOR TAG?
-      # block_content     ::= block_collection | flow_collection | SCALAR
-      # flow_content      ::= flow_collection | SCALAR
-      # block_collection  ::= block_sequence | block_mapping
-      # block_node_or_indentless_sequence ::= ALIAS | properties?
-      #                                       (block_content | indentless_block_sequence)
-      events = []
-      if check_token(AliasToken)
-        token = get_token
-        events << AliasEvent.new(token.value, token.start_mark, token.end_mark)
+
+    def implicit_document
+      curr = @scanner.peek_token
+      unless curr.__is_directive || curr.__is_document_start || curr.__is_stream_end
+        @parse_stack += [:document_end, :block_node]
+        return document_start_implicit
+      end
+      nil
+    end
+
+    def explicit_document
+      if !@scanner.peek_token.__is_stream_end
+        @parse_stack += [:explicit_document, :document_end, :block_node]
+        return document_start
+      end
+      nil
+    end
+
+    def block_node
+      curr = @scanner.peek_token
+      if curr.__is_directive || curr.__is_document_start || curr.__is_document_end || curr.__is_stream_end
+        return empty_scalar
       else
-        anchor = nil
-        tag = nil
-        start_mark = end_mark = tag_mark = nil
-        if check_token(AnchorToken)
-          token = get_token
-          start_mark = token.start_mark
-          end_mark = token.end_mark
-          anchor = token.value
-          if check_token(TagToken)
-            token = get_token
-            tag_mark = token.start_mark
-            end_mark = token.end_mark
-            tag = token.value
-          end
-        elsif check_token(TagToken)
-          token = get_token
-          start_mark = tag_mark = token.start_mark
+        if curr.__is_alias
+          return _alias
+        else
+          @parse_stack << :un_properties
+          properties
+          return block_content
+        end
+      end
+    end
+
+    def flow_node
+      if @scanner.peek_token.__is_alias
+        return _alias
+      else
+        @parse_stack << :un_properties
+        properties
+        return flow_content
+      end
+    end
+
+    def properties
+      anchor = nil
+      tag = nil
+      start_mark = end_mark = tag_mark = nil
+      if @scanner.peek_token.__is_anchor
+        token = @scanner.get_token
+        start_mark = token.start_mark
+        end_mark = token.end_mark
+        anchor = token.value
+        if @scanner.peek_token.__is_tag
+          token = @scanner.get_token
+          tag_mark = token.start_mark
           end_mark = token.end_mark
           tag = token.value
-          if check_token(AnchorToken)
-            token = get_token
-            end_mark = token.end_mark
-            anchor = token.value
-          end
         end
+      elsif @scanner.peek_token.__is_tag
+        token = @scanner.get_token
+        start_mark = tag_mark = token.start_mark
+        end_mark = token.end_mark
+        tag = token.value
+        if @scanner.peek_token.__is_anchor
+          token = @scanner.get_token
+          end_mark = token.end_mark
+          anchor = token.value
+        end
+      end
+      
+      if !tag.nil? and tag != "!"
+        handle, suffix = tag
+        if !handle.nil?
+          raise ParserError.new("while parsing a node", start_mark,"found undefined tag handle #{handle}",tag_mark) if !@tag_handles.include?(handle)
+          tag = @tag_handles[handle]+suffix
+        else
+          tag = suffix
+        end
+      end
+      if start_mark.nil?
+        start_mark = end_mark = @scanner.peek_token.start_mark
+      end
+      @anchors << anchor
+      @tags << tag
+      @start_marks << start_mark
+      @end_marks << end_mark
+      nil
+    end
+
+    def un_properties
+      @anchors.pop
+      @tags.pop
+      @start_marks.pop
+      @end_marks.pop
+      nil
+    end
+
+    def block_content
+      token = @scanner.peek_token
+      if token.__is_block_sequence_start
+        return block_sequence
+      elsif token.__is_block_mapping_start
+        return block_mapping
+      elsif token.__is_flow_sequence_start
+        return flow_sequence
+      elsif token.__is_flow_mapping_start
+        return flow_mapping
+      elsif token.__is_scalar
+        return scalar
+      else
+        raise ParserError.new("while scanning a node", @start_marks.last,"expected the node content, but found #{token.tid}",token.start_mark)
+      end
+    end
 
-        if !tag.nil? and tag != "!"
-          handle, suffix = tag
-          if !handle.nil?
-            raise ParserError.new("while parsing a node", start_mark,"found undefined tag handle #{handle}",tag_mark) if !@tag_handles.include?(handle)
-            tag = @tag_handles[handle]+suffix
+    def flow_content
+      token = @scanner.peek_token
+      if token.__is_flow_sequence_start
+        return flow_sequence
+      elsif token.__is_flow_mapping_start
+        return flow_mapping
+      elsif token.__is_scalar
+        return scalar
+      else
+        raise ParserError.new("while scanning a flow node", @start_marks.last,"expected the node content, but found #{token.tid}",token.start_mark)
+      end
+    end
+
+    def block_sequence_entry
+      if @scanner.peek_token.__is_block_entry
+        @tokens = token = @scanner.get_token
+        if !(@scanner.peek_token.__is_block_entry || @scanner.peek_token.__is_block_end)
+          @parse_stack += [:block_sequence_entry]
+          return block_node
+        else
+          @parse_steck += [:block_sequence_entry]
+          return empty_scalar
+        end
+      end      
+      nil
+    end
+
+    def block_mapping_entry
+      #   ((KEY block_node_or_indentless_sequence?)? (VALUE block_node_or_indentless_sequence?)?)*
+      if @scanner.peek_token.__is_key || @scanner.peek_token.__is_value
+        if @scanner.check_token(KeyToken)
+          @tokens = token = @scanner.get_token
+          curr = @scanner.peek_token
+          if !(curr.__is_key || curr.__is_value || curr.__is_block_end)
+            @parse_stack += [:block_mapping_entry,:block_mapping_entry_value]
+            return block_node_or_indentless_sequence
           else
-            tag = suffix
+            @parse_stack += [:block_mapping_entry,:block_mapping_entry_value]
+            return empty_scalar
           end
+        else
+          @parse_stack += [:block_mapping_entry,:block_mapping_entry_value]
+          return empty_scalar
         end
+      end
+      nil
+    end
 
-        #if tag == u'!':
-        #    raise ParserError("while parsing a node", start_mark,
-        #            "found non-specific tag '!'", tag_mark,
-        #            "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
-        if start_mark.nil?
-          start_mark = end_mark = peek_token.start_mark
-        end
-        event = nil
-        collection_events = nil
-        implicit = tag.nil? || tag == ?!
-        if indentless_sequence && check_token(BlockEntryToken)
-          end_mark = peek_token.end_mark
-          event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark)
-          collection_events = parse_indentless_sequence
-        else
-          if check_token(ScalarToken)
-            token = get_token
-            end_mark = token.end_mark
-            if (token.plain && tag.nil?) || tag == ?!
-              implicit = [true, false]
-            elsif tag.nil?
-              implicit = [false, true]
-            else
-              implicit = [false, false]
-            end
-            event = ScalarEvent.new(anchor, tag, implicit, token.value,start_mark, end_mark,token.style)
-          elsif check_token(FlowSequenceStartToken)
-            end_mark = peek_token.end_mark
-            event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark,true)
-            collection_events = parse_flow_sequence
-          elsif check_token(FlowMappingStartToken)
-            end_mark = peek_token.end_mark
-            event = MappingStartEvent.new(anchor, tag, implicit, start_mark, end_mark,true)
-            collection_events = parse_flow_mapping
-          elsif block && check_token(BlockSequenceStartToken)
-            end_mark = peek_token.start_mark
-            event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark,false)
-            collection_events = parse_block_sequence
-          elsif block && check_token(BlockMappingStartToken)
-            end_mark = peek_token.start_mark
-            event = MappingStartEvent.new(anchor, tag, implicit, start_mark, end_mark,false)
-            collection_events = parse_block_mapping
-          elsif !anchor.nil? || !tag.nil?
-            # Empty scalars are allowed even if a tag or an anchor is
-            # specified.
-            event = ScalarEvent.new(anchor, tag, [implicit,false],"",start_mark, end_mark)
+    def block_mapping_entry_value
+      if @scanner.peek_token.__is_key || @scanner.peek_token.__is_value
+        if @scanner.peek_token.__is_value
+          @tokens = token = @scanner.get_token
+          curr = @scanner.peek_token
+          if !(curr.__is_key || curr.__is_value || curr.__is_block_end)
+            return block_node_or_indentless_sequence
           else
-            if block
-              node = "block"
-            else
-              node = "flow"
-            end
-            token = peek_token
-            raise ParserError.new("while scanning a #{node} node", start_mark,"expected the node content, but found #{token.tid}",token.start_mark)
+            return empty_scalar
           end
+        else
+          @tokens = token = @scanner.peek_token
+          return empty_scalar
         end
-        events << event
-        events += collection_events if collection_events
       end
-      events
+      nil
     end
 
-    def parse_block_sequence
-      # BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
-      events = []
-      token = get_token
-      start_mark = token.start_mark
-      while check_token(BlockEntryToken)
-        token = get_token
-        if !check_token(BlockEntryToken, BlockEndToken)
-          events += parse_block_node
+    def block_sequence
+      @parse_stack += [:block_sequence_end,:block_sequence_entry]
+      block_sequence_start
+    end
+
+    def block_mapping
+      @parse_stack += [:block_mapping_end,:block_mapping_entry]
+      block_mapping_start
+    end
+
+    def block_node_or_indentless_sequence
+      if @scanner.peek_token.__is_alias
+        return _alias
+      else
+        if @scanner.peek_token.__is_block_entry
+          properties
+          return indentless_block_sequence
         else
-          events << process_empty_scalar(token.end_mark)
+          properties
+          return block_content
         end
       end
-      if !check_token(BlockEndToken)
-        token = peek_token
-        raise ParserError.new("while scanning a block collection", start_mark,"expected <block end>, but found #{token.tid}", token.start_mark)
-      end
-      token = get_token
-      events << SequenceEndEvent.new(token.start_mark, token.end_mark)
-      events
-    end
-
-    def parse_indentless_sequence
-      # (BLOCK-ENTRY block_node?)+
-      events = []
-      while check_token(BlockEntryToken)
-        token = get_token
-        if !check_token(BlockEntryToken,KeyToken, ValueToken, BlockEndToken)
-          events += parse_block_node
+    end
+
+    def indentless_block_sequence
+      @parse_stack += [:block_indentless_sequence_end,:indentless_block_sequence_entry]
+      block_indentless_sequence_start
+    end
+
+    def indentless_block_sequence_entry
+      if @scanner.peek_token.__is_block_entry
+        @tokens = @scanner.get_token
+        curr = @scanner.peek_token
+        if !(curr.__is_block_entry || curr.__is_key || curr.__is_value || curr.__is_block_end)
+          @parse_stack << :indentless_block_sequence_entry
+          return block_node
         else
-          events << process_empty_scalar(token.end_mark)
+          @parse_stack << :indentless_block_sequence_entry
+          return empty_scalar
         end
       end
-      token = peek_token
-      events << SequenceEndEvent.new(token.start_mark, token.start_mark)
-      events
+      nil
     end
 
+    def flow_sequence
+      @parse_stack += [:flow_sequence_end,:flow_sequence_entry]
+      flow_sequence_start
+    end
 
-    def parse_block_mapping
-      # BLOCK-MAPPING_START
-      #   ((KEY block_node_or_indentless_sequence?)?
-      #   (VALUE block_node_or_indentless_sequence?)?)*
-      # BLOCK-END
-      events = []
-      token = get_token
-      start_mark = token.start_mark
-      while check_token(KeyToken, ValueToken)
-        if check_token(KeyToken)
-          token = get_token
-          if !check_token(KeyToken, ValueToken, BlockEndToken)
-            events += parse_block_node_or_indentless_sequence
-          else
-            events << process_empty_scalar(token.end_mark)
-          end
-        end
-        if check_token(ValueToken)
-          token = get_token
-          if !check_token(KeyToken, ValueToken, BlockEndToken)
-            events += parse_block_node_or_indentless_sequence
-          else
-            events << process_empty_scalar(token.end_mark)
-          end
+    def flow_mapping
+      @parse_stack += [:flow_mapping_end,:flow_mapping_entry]
+      flow_mapping_start
+    end
+
+    def flow_sequence_entry
+      if !@scanner.peek_token.__is_flow_sequence_end
+        if @scanner.peek_token.__is_key
+          @parse_stack += [:flow_sequence_entry,:flow_entry_marker,:flow_internal_mapping_end,:flow_internal_value,:flow_internal_content]
+          return flow_internal_mapping_start
         else
-          token = peek_token
-          events << process_empty_scalar(token.start_mark)
+          @parse_stack += [:flow_sequence_entry,:flow_node]
+          return flow_entry_marker
         end
       end
-      if !check_token(BlockEndToken)
-        token = peek_token
-        raise ParserError.new("while scanning a block mapping", start_mark,"expected <block end>, but found #{token.tid}", token.start_mark)
+      nil
+    end
+
+    def flow_internal_content
+      token = @scanner.peek_token
+      if !(token.__is_value || token.__is_flow_entry || token.__is_flow_sequence_end)
+        flow_node
+      else
+        empty_scalar
       end
-      token = get_token
-      events << MappingEndEvent.new(token.start_mark, token.end_mark)
-      events
-    end
-
-    def parse_flow_sequence
-      # flow_sequence     ::= FLOW-SEQUENCE-START
-      #                       (flow_sequence_entry FLOW-ENTRY)*
-      #                       flow_sequence_entry?
-      #                       FLOW-SEQUENCE-END
-      # flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
-      #
-      # Note that while production rules for both flow_sequence_entry and
-      # flow_mapping_entry are equal, their interpretations are different.
-      # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
-      # generate an inline mapping (set syntax).
-      events = []
-      token = get_token
-      start_mark = token.start_mark
-      while !check_token(FlowSequenceEndToken)
-        if check_token(KeyToken)
-          token = get_token
-          events << MappingStartEvent.new(nil,nil,true,token.start_mark, token.end_mark,true)
-          if !check_token(ValueToken,FlowEntryToken, FlowSequenceEndToken)
-            events += parse_flow_node
-          else
-            events << process_empty_scalar(token.end_mark)
-          end
-          if check_token(ValueToken)
-            token = get_token
-            if !check_token(FlowEntryToken, FlowSequenceEndToken)
-              events += parse_flow_node
-            else
-              events << process_empty_scalar(token.end_mark)
-            end
-          else
-            token = peek_token
-            events << process_empty_scalar(token.start_mark)
-          end
-          token = peek_token
-          events << MappingEndEvent.new(token.start_mark, token.start_mark)
+    end
+
+    def flow_internal_value
+      if @scanner.peek_token.__is_value
+        @tokens = token = @scanner.get_token
+        if !(@scanner.peek_token.__is_flow_entry || @scanner.peek_token.__is_flow_sequence_end)
+          flow_node
         else
-          events += parse_flow_node
-        end
-        if !check_token(FlowEntryToken, FlowSequenceEndToken)
-          token = peek_token
-          raise ParserError.new("while scanning a flow sequence", start_mark,"expected ',' or ']', but got #{token.tid}", token.start_mark)
+          empty_scalar
         end
-        if check_token(FlowEntryToken)
-          get_token
+      else
+        @tokens = token = @scanner.peek_token
+        empty_scalar
+      end
+    end
+
+    def flow_entry_marker
+      if @scanner.peek_token.__is_flow_entry
+        @scanner.get_token
+      end
+      nil
+    end
+
+    def flow_mapping_entry
+      if !@scanner.peek_token.__is_flow_mapping_end
+        if @scanner.peek_token.__is_key
+          @parse_stack += [:flow_mapping_entry,:flow_entry_marker,:flow_mapping_internal_value]
+          return flow_mapping_internal_content
+        else
+          @parse_stack += [:flow_mapping_entry,:flow_node]
+          return flow_entry_marker
         end
       end
-      token = get_token
-      events << SequenceEndEvent.new(token.start_mark, token.end_mark)
-      events
+      nil
     end
-    
-    def parse_flow_mapping
-      # flow_mapping      ::= FLOW-MAPPING-START
-      #                       (flow_mapping_entry FLOW-ENTRY)*
-      #                       flow_mapping_entry?
-      #                       FLOW-MAPPING-END
-      # flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
-      events = []
-      token = get_token
-      start_mark = token.start_mark
-      while !check_token(FlowMappingEndToken)
-        if check_token(KeyToken)
-          token = get_token
-          if !check_token(ValueToken,FlowEntryToken, FlowMappingEndToken)
-            events += parse_flow_node
-          else
-            events << process_empty_scalar(token.end_mark)
-          end
-          if check_token(ValueToken)
-            token = get_token
-            if !check_token(FlowEntryToken, FlowMappingEndToken)
-              events += parse_flow_node
-            else
-              events << process_empty_scalar(token.end_mark)
-            end
-          else
-            token = peek_token
-            events << process_empty_scalar(token.start_mark)
-          end
+
+    def flow_mapping_internal_content
+      curr = @scanner.peek_token
+      if !(curr.__is_value || curr.__is_flow_entry || curr.__is_flow_mapping_end)
+        @tokens = token = @scanner.get_token
+        flow_node
+      else
+        empty_scalar
+      end
+    end
+
+    def flow_mapping_internal_value
+      if @scanner.peek_token.__is_value
+        @tokens = token = @scanner.get_token
+        if !(@scanner.peek_token.__is_flow_entry || @scanner.peek_token.__is_flow_mapping_end)
+          flow_node
         else
-          events += parse_flow_node
-          events << process_empty_scalar(peek_token.start_mark)
+          empty_scalar
         end
-        if !check_token(FlowEntryToken, FlowMappingEndToken)
-          token = peek_token
-          raise ParserError.new("while scanning a flow mapping", start_mark,"expected ',' or '}', but got #{token.tid}", token.start_mark)
+      else
+        @tokens = token = @scanner.peek_token
+        empty_scalar
+      end
+    end
+   
+
+    def process_directives
+      # DIRECTIVE*
+      while @scanner.peek_token.__is_directive
+        token = @scanner.get_token
+        if token.name == "YAML"
+          raise ParserError.new(nil, nil,"found duplicate YAML directive", token.start_mark) if !@yaml_version.nil?
+          major, minor = token.value[0].to_i, token.value[1].to_i
+          raise ParserError.new(nil,nil,"found incompatible YAML document (version 1.* is required)",token.start_mark) if major != 1
+          @yaml_version = [major,minor]
+        elsif token.name == "TAG"
+          handle, prefix = token.value
+          raise ParserError.new(nil,nil,"duplicate tag handle #{handle}",token.start_mark) if @tag_handles.member?(handle)
+          @tag_handles[handle] = prefix
         end
-        get_token if check_token(FlowEntryToken)
       end
-      if !check_token(FlowMappingEndToken)
-        token = peek_token
-        raise ParserError.new("while scanning a flow mapping", start_mark,"expected '}', but found #{token.tid}", token.start_mark)
+      if !@tag_handles.empty?
+        value = @yaml_version, @tag_handles.dup
+      else
+        value = @yaml_version, nil
+      end
+      for key in DEFAULT_TAGS.keys
+        @tag_handles[key] = DEFAULT_TAGS[key] if !@tag_handles.include?(key)
       end
-      token = get_token
-      events << MappingEndEvent.new(token.start_mark, token.end_mark)
-      events
+      value
     end
 
     def process_empty_scalar(mark)