RbYAML 0.0.2 → 0.1.0

data/lib/rbyaml/parser.rb.~1.2.~

@@ -0,0 +1,494 @@
+
+# YAML can be parsed by an LL(1) parser!
+#
+# We use the following production rules:
+# stream            ::= STREAM-START implicit_document? explicit_document* STREAM-END
+# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END?
+# implicit_document ::= block_node DOCUMENT-END?
+# block_node    ::= ALIAS | properties? block_content
+# flow_node     ::= ALIAS | properties? flow_content
+# properties    ::= TAG ANCHOR? | ANCHOR TAG?
+# block_content     ::= block_collection | flow_collection | SCALAR
+# flow_content      ::= flow_collection | SCALAR
+# block_collection  ::= block_sequence | block_mapping
+# block_sequence    ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+# block_mapping     ::= BLOCK-MAPPING_START ((KEY block_node_or_indentless_sequence?)? (VALUE block_node_or_indentless_sequence?)?)* BLOCK-END
+# block_node_or_indentless_sequence ::= ALIAS | properties? (block_content | indentless_block_sequence)
+# indentless_block_sequence         ::= (BLOCK-ENTRY block_node?)+
+# flow_collection   ::= flow_sequence | flow_mapping
+# flow_sequence     ::= FLOW-SEQUENCE-START (flow_sequence_entry FLOW-ENTRY)* flow_sequence_entry? FLOW-SEQUENCE-END
+# flow_mapping      ::= FLOW-MAPPING-START (flow_mapping_entry FLOW-ENTRY)* flow_mapping_entry? FLOW-MAPPING-END
+# flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+# flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+
+# TODO: support for BOM within a stream.
+# stream ::= (BOM? implicit_document)? (BOM? explicit_document)* STREAM-END
+
+# FIRST sets:
+# stream: { STREAM-START }
+# explicit_document: { DIRECTIVE DOCUMENT-START }
+# implicit_document: FIRST(block_node)
+# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_sequence: { BLOCK-SEQUENCE-START }
+# block_mapping: { BLOCK-MAPPING-START }
+# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
+# indentless_sequence: { ENTRY }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_sequence: { FLOW-SEQUENCE-START }
+# flow_mapping: { FLOW-MAPPING-START }
+# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+
+require 'rbyaml/error'
+require 'rbyaml/tokens'
+require 'rbyaml/events'
+require 'rbyaml/scanner'
+
+module RbYAML
+  class ParserError < MarkedYAMLError
+  end
+
+  module Parser
+
+    DEFAULT_TAGS = {
+      '!' => '!',
+      '!!' => 'tag:yaml.org,2002:'
+    }
+
+    def initialize_parser
+      @current_event = nil
+      @yaml_version = nil
+      @events = nil
+      @working_events = nil
+      @tag_handles = { }
+    end
+
+    def check_event(*choices)
+      init_events
+      @current_event = @working_events.shift if @current_event.nil?
+      if @current_event
+        return true if choices.empty?
+        for choice in choices
+          return true if choice === @current_event
+        end
+      end
+      false
+    end
+
+    def peek_event
+      init_events
+      @current_event = @working_events.shift if @current_event.nil?
+      @current_event
+    end
+
+    def get_event
+      init_events
+      @current_event = @working_events.shift if @current_event.nil?
+      value = @current_event
+      @current_event = nil
+      value
+    end
+
+    def init_events
+      @events ||= parse_stream
+      @working_events ||= @events        
+    end
+    
+    def each_event(&block)
+      init_events
+      @events.each(&block)
+    end
+
+    def parse_stream
+      # STREAM-START implicit_document? explicit_document* STREAM-END
+      
+      # Parse start of stream.
+      events = []
+      token = get_token
+      events << StreamStartEvent.new(token.start_mark, token.end_mark,token.encoding)
+      
+      # Parse implicit document.
+      unless check_token(DirectiveToken, DocumentStartToken,StreamEndToken)
+        @tag_handles = DEFAULT_TAGS
+        token = peek_token
+        start_mark = end_mark = token.start_mark
+        events << DocumentStartEvent.new(start_mark, end_mark,false)
+        events += parse_block_node
+        token = peek_token
+        start_mark = end_mark = token.start_mark
+        explicit = false
+        while check_token(DocumentEndToken)
+          token = get_token
+          end_mark = token.end_mark
+          explicit = true
+        end
+        events << DocumentEndEvent.new(start_mark, end_mark,explicit)
+      end
+
+      # Parse explicit documents.
+      while !check_token(StreamEndToken)
+        token = peek_token
+        start_mark = token.start_mark
+        version, tags = process_directives
+        raise ParserError.new(nil, nil,"expected '<document start>', but found #{peek_token.id}",peek_token.start_mark) unless check_token(DocumentStartToken)
+        token = get_token
+        end_mark = token.end_mark
+        events << DocumentStartEvent.new(start_mark, end_mark,true,version,tags)
+        if check_token(DirectiveToken,DocumentStartToken, DocumentEndToken, StreamEndToken)
+          events << process_empty_scalar(token.end_mark)
+        else
+          events += parse_block_node
+        end
+        token = peek_token
+        start_mark = end_mark = token.start_mark
+        explicit = false
+        while check_token(DocumentEndToken)
+          token = get_token
+          end_mark = token.end_mark
+          explicit=true
+        end
+        events << DocumentEndEvent.new(start_mark, end_mark,explicit)
+      end
+      # Parse end of stream.
+      token = get_token
+      events << StreamEndEvent.new(token.start_mark, token.end_mark)
+      events
+    end
+
+    def process_directives
+      # DIRECTIVE*
+      while check_token(DirectiveToken)
+        token = get_token
+        if token.name == "YAML"
+          raise ParserError.new(nil, nil,"found duplicate YAML directive", token.start_mark) if !@yaml_version.nil?
+          major, minor = token.value[0].to_i, token.value[1].to_i
+          raise ParserError.new(nil,nil,"found incompatible YAML document (version 1.* is required)",token.start_mark) if major != 1
+          @yaml_version = [major,minor]
+        elsif token.name == "TAG"
+          handle, prefix = token.value
+          raise ParserError.new(nil,nil,"duplicate tag handle #{handle}",token.start_mark) if @tag_handles.member?(handle)
+          @tag_handles[handle] = prefix
+        end
+      end
+      if !@tag_handles.empty?
+        value = @yaml_version, @tag_handles.dup
+      else
+        value = @yaml_version, nil
+      end
+      for key in DEFAULT_TAGS.keys
+        @tag_handles[key] = DEFAULT_TAGS[key] if !@tag_handles.include?(key)
+      end
+      value
+    end
+
+    def parse_block_node
+      parse_node(true)
+    end
+    
+    def parse_flow_node
+      parse_node
+    end
+    
+    def parse_block_node_or_indentless_sequence
+      parse_node(true, true)
+    end
+
+    def parse_node(block=false, indentless_sequence=false)
+      # block_node    ::= ALIAS | properties? block_content
+      # flow_node     ::= ALIAS | properties? flow_content
+      # properties    ::= TAG ANCHOR? | ANCHOR TAG?
+      # block_content     ::= block_collection | flow_collection | SCALAR
+      # flow_content      ::= flow_collection | SCALAR
+      # block_collection  ::= block_sequence | block_mapping
+      # block_node_or_indentless_sequence ::= ALIAS | properties?
+      #                                       (block_content | indentless_block_sequence)
+      events = []
+      if check_token(AliasToken)
+        token = get_token
+        events << AliasEvent.new(token.value, token.start_mark, token.end_mark)
+      else
+        anchor = nil
+        tag = nil
+        start_mark = end_mark = tag_mark = nil
+        if check_token(AnchorToken)
+          token = get_token
+          start_mark = token.start_mark
+          end_mark = token.end_mark
+          anchor = token.value
+          if check_token(TagToken)
+            token = get_token
+            tag_mark = token.start_mark
+            end_mark = token.end_mark
+            tag = token.value
+          end
+        elsif check_token(TagToken)
+          token = get_token
+          start_mark = tag_mark = token.start_mark
+          end_mark = token.end_mark
+          tag = token.value
+          if check_token(AnchorToken)
+            token = get_token
+            end_mark = token.end_mark
+            anchor = token.value
+          end
+        end
+
+        if !tag.nil? and tag != "!"
+          handle, suffix = tag
+          if !handle.nil?
+            raise ParserError.new("while parsing a node", start_mark,"found undefined tag handle #{handle}",tag_mark) if !@tag_handles.include?(handle)
+            tag = @tag_handles[handle]+suffix
+          else
+            tag = suffix
+          end
+        end
+
+        #if tag == u'!':
+        #    raise ParserError("while parsing a node", start_mark,
+        #            "found non-specific tag '!'", tag_mark,
+        #            "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
+        if start_mark.nil?
+          start_mark = end_mark = peek_token.start_mark
+        end
+        event = nil
+        collection_events = nil
+        implicit = tag.nil? || tag == ?!
+        if indentless_sequence && check_token(BlockEntryToken)
+          end_mark = peek_token.end_mark
+          event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark)
+          collection_events = parse_indentless_sequence
+        else
+          if check_token(ScalarToken)
+            token = get_token
+            end_mark = token.end_mark
+            if (token.plain && tag.nil?) || tag == ?!
+              implicit = [true, false]
+            elsif tag.nil?
+              implicit = [false, true]
+            else
+              implicit = [false, false]
+            end
+            event = ScalarEvent.new(anchor, tag, implicit, token.value,start_mark, end_mark,token.style)
+          elsif check_token(FlowSequenceStartToken)
+            end_mark = peek_token.end_mark
+            event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark,true)
+            collection_events = parse_flow_sequence
+          elsif check_token(FlowMappingStartToken)
+            end_mark = peek_token.end_mark
+            event = MappingStartEvent.new(anchor, tag, implicit, start_mark, end_mark,true)
+            collection_events = parse_flow_mapping
+          elsif block && check_token(BlockSequenceStartToken)
+            end_mark = peek_token.start_mark
+            event = SequenceStartEvent.new(anchor, tag, implicit, start_mark, end_mark,false)
+            collection_events = parse_block_sequence
+          elsif block && check_token(BlockMappingStartToken)
+            end_mark = peek_token.start_mark
+            event = MappingStartEvent.new(anchor, tag, implicit, start_mark, end_mark,false)
+            collection_events = parse_block_mapping
+          elsif !anchor.nil? || !tag.nil?
+            # Empty scalars are allowed even if a tag or an anchor is
+            # specified.
+            event = ScalarEvent.new(anchor, tag, [implicit,false],"",start_mark, end_mark)
+          else
+            if block
+              node = "block"
+            else
+              node = "flow"
+            end
+            token = peek_token
+            raise ParserError.new("while scanning a #{node} node", start_mark,"expected the node content, but found #{token.tid}",token.start_mark)
+          end
+        end
+        events << event
+        events += collection_events if collection_events
+      end
+      events
+    end
+
+    def parse_block_sequence
+      # BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+      events = []
+      token = get_token
+      start_mark = token.start_mark
+      while check_token(BlockEntryToken)
+        token = get_token
+        if !check_token(BlockEntryToken, BlockEndToken)
+          events += parse_block_node
+        else
+          events << process_empty_scalar(token.end_mark)
+        end
+      end
+      if !check_token(BlockEndToken)
+        token = peek_token
+        raise ParserError.new("while scanning a block collection", start_mark,"expected <block end>, but found #{token.tid}", token.start_mark)
+      end
+      token = get_token
+      events << SequenceEndEvent.new(token.start_mark, token.end_mark)
+      events
+    end
+
+    def parse_indentless_sequence
+      # (BLOCK-ENTRY block_node?)+
+      events = []
+      while check_token(BlockEntryToken)
+        token = get_token
+        if !check_token(BlockEntryToken,KeyToken, ValueToken, BlockEndToken)
+          events += parse_block_node
+        else
+          events << process_empty_scalar(token.end_mark)
+        end
+      end
+      token = peek_token
+      events << SequenceEndEvent.new(token.start_mark, token.start_mark)
+      events
+    end
+
+
+    def parse_block_mapping
+      # BLOCK-MAPPING_START
+      #   ((KEY block_node_or_indentless_sequence?)?
+      #   (VALUE block_node_or_indentless_sequence?)?)*
+      # BLOCK-END
+      events = []
+      token = get_token
+      start_mark = token.start_mark
+      while check_token(KeyToken, ValueToken)
+        if check_token(KeyToken)
+          token = get_token
+          if !check_token(KeyToken, ValueToken, BlockEndToken)
+            events += parse_block_node_or_indentless_sequence
+          else
+            events << process_empty_scalar(token.end_mark)
+          end
+        end
+        if check_token(ValueToken)
+          token = get_token
+          if !check_token(KeyToken, ValueToken, BlockEndToken)
+            events += parse_block_node_or_indentless_sequence
+          else
+            events << process_empty_scalar(token.end_mark)
+          end
+        else
+          token = peek_token
+          events << process_empty_scalar(token.start_mark)
+        end
+      end
+      if !check_token(BlockEndToken)
+        token = peek_token
+        raise ParserError.new("while scanning a block mapping", start_mark,"expected <block end>, but found #{token.tid}", token.start_mark)
+      end
+      token = get_token
+      events << MappingEndEvent.new(token.start_mark, token.end_mark)
+      events
+    end
+
+    def parse_flow_sequence
+      # flow_sequence     ::= FLOW-SEQUENCE-START
+      #                       (flow_sequence_entry FLOW-ENTRY)*
+      #                       flow_sequence_entry?
+      #                       FLOW-SEQUENCE-END
+      # flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+      #
+      # Note that while production rules for both flow_sequence_entry and
+      # flow_mapping_entry are equal, their interpretations are different.
+      # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
+      # generate an inline mapping (set syntax).
+      events = []
+      token = get_token
+      start_mark = token.start_mark
+      while !check_token(FlowSequenceEndToken)
+        if check_token(KeyToken)
+          token = get_token
+          events << MappingStartEvent.new(nil,nil,true,token.start_mark, token.end_mark,true)
+          if !check_token(ValueToken,FlowEntryToken, FlowSequenceEndToken)
+            events += parse_flow_node
+          else
+            events << process_empty_scalar(token.end_mark)
+          end
+          if check_token(ValueToken)
+            token = get_token
+            if !check_token(FlowEntryToken, FlowSequenceEndToken)
+              events += parse_flow_node
+            else
+              events << process_empty_scalar(token.end_mark)
+            end
+          else
+            token = peek_token
+            events << process_empty_scalar(token.start_mark)
+          end
+          token = peek_token
+          events << MappingEndEvent.new(token.start_mark, token.start_mark)
+        else
+          events += parse_flow_node
+        end
+        if !check_token(FlowEntryToken, FlowSequenceEndToken)
+          token = peek_token
+          raise ParserError.new("while scanning a flow sequence", start_mark,"expected ',' or ']', but got #{token.tid}", token.start_mark)
+        end
+        if check_token(FlowEntryToken)
+          get_token
+        end
+      end
+      token = get_token
+      events << SequenceEndEvent.new(token.start_mark, token.end_mark)
+      events
+    end
+    
+    def parse_flow_mapping
+      # flow_mapping      ::= FLOW-MAPPING-START
+      #                       (flow_mapping_entry FLOW-ENTRY)*
+      #                       flow_mapping_entry?
+      #                       FLOW-MAPPING-END
+      # flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+      events = []
+      token = get_token
+      start_mark = token.start_mark
+      while !check_token(FlowMappingEndToken)
+        if check_token(KeyToken)
+          token = get_token
+          if !check_token(ValueToken,FlowEntryToken, FlowMappingEndToken)
+            events += parse_flow_node
+          else
+            events << process_empty_scalar(token.end_mark)
+          end
+          if check_token(ValueToken)
+            token = get_token
+            if !check_token(FlowEntryToken, FlowMappingEndToken)
+              events += parse_flow_node
+            else
+              events << process_empty_scalar(token.end_mark)
+            end
+          else
+            token = peek_token
+            events << process_empty_scalar(token.start_mark)
+          end
+        else
+          events += parse_flow_node
+          events << process_empty_scalar(peek_token.start_mark)
+        end
+        if !check_token(FlowEntryToken, FlowMappingEndToken)
+          token = peek_token
+          raise ParserError.new("while scanning a flow mapping", start_mark,"expected ',' or '}', but got #{token.tid}", token.start_mark)
+        end
+        get_token if check_token(FlowEntryToken)
+      end
+      if !check_token(FlowMappingEndToken)
+        token = peek_token
+        raise ParserError.new("while scanning a flow mapping", start_mark,"expected '}', but found #{token.tid}", token.start_mark)
+      end
+      token = get_token
+      events << MappingEndEvent.new(token.start_mark, token.end_mark)
+      events
+    end
+
+    def process_empty_scalar(mark)
+      ScalarEvent.new(nil, nil, [true, false], "", mark, mark)
+    end
+  end
+end
+