psych 1.2.2 → 1.3.0

data/lib/psych.rb

@@ -10,7 +10,10 @@ require 'psych/set'
 require 'psych/coder'
 require 'psych/core_ext'
 require 'psych/deprecated'
-require 'psych/json'
+require 'psych/stream'
+require 'psych/json/tree_builder'
+require 'psych/json/stream'
+require 'psych/handlers/document_stream'
 
 ###
 # = Overview
@@ -90,7 +93,7 @@ require 'psych/json'
 
 module Psych
   # The version is Psych you're using
-  VERSION         = '1.2.2'
+  VERSION         = '1.3.0'
 
   # The version of libyaml Psych is using
   LIBYAML_VERSION = Psych.libyaml_version.join '.'
@@ -101,39 +104,63 @@ module Psych
   class BadAlias < Exception
   end
 
-  autoload :Stream, 'psych/stream'
-
   ###
   # Load +yaml+ in to a Ruby data structure.  If multiple documents are
   # provided, the object contained in the first document will be returned.
+  # +filename+ will be used in the exception message if any exception is raised
+  # while parsing.
+  #
+  # Raises a Psych::SyntaxError when a YAML syntax error is detected.
   #
   # Example:
   #
-  #   Psych.load("--- a")           # => 'a'
-  #   Psych.load("---\n - a\n - b") # => ['a', 'b']
-  def self.load yaml
-    result = parse(yaml)
+  #   Psych.load("--- a")             # => 'a'
+  #   Psych.load("---\n - a\n - b")   # => ['a', 'b']
+  #
+  #   begin
+  #     Psych.load("--- `", "file.txt")
+  #   rescue Psych::SyntaxError => ex
+  #     ex.file    # => 'file.txt'
+  #     ex.message # => "(foo.txt): found character that cannot start any token"
+  #   end
+  def self.load yaml, filename = nil
+    result = parse(yaml, filename)
     result ? result.to_ruby : result
   end
 
   ###
   # Parse a YAML string in +yaml+.  Returns the first object of a YAML AST.
+  # +filename+ is used in the exception message if a Psych::SyntaxError is
+  # raised.
+  #
+  # Raises a Psych::SyntaxError when a YAML syntax error is detected.
   #
   # Example:
   #
   #   Psych.parse("---\n - a\n - b") # => #<Psych::Nodes::Sequence:0x00>
   #
+  #   begin
+  #     Psych.parse("--- `", "file.txt")
+  #   rescue Psych::SyntaxError => ex
+  #     ex.file    # => 'file.txt'
+  #     ex.message # => "(foo.txt): found character that cannot start any token"
+  #   end
+  #
   # See Psych::Nodes for more information about YAML AST.
-  def self.parse yaml
-    children = parse_stream(yaml).children
-    children.empty? ? false : children.first.children.first
+  def self.parse yaml, filename = nil
+    parse_stream(yaml, filename) do |node|
+      return node
+    end
+    false
   end
 
   ###
   # Parse a file at +filename+. Returns the YAML AST.
+  #
+  # Raises a Psych::SyntaxError when a YAML syntax error is detected.
   def self.parse_file filename
-    File.open filename do |f|
-      parse f
+    File.open filename, 'r:bom|utf-8' do |f|
+      parse f, filename
     end
   end
 
@@ -146,16 +173,39 @@ module Psych
   ###
   # Parse a YAML string in +yaml+.  Returns the full AST for the YAML document.
   # This method can handle multiple YAML documents contained in +yaml+.
+  # +filename+ is used in the exception message if a Psych::SyntaxError is
+  # raised.
+  #
+  # If a block is given, a Psych::Nodes::Document node will be yielded to the
+  # block as it's being parsed.
+  #
+  # Raises a Psych::SyntaxError when a YAML syntax error is detected.
   #
   # Example:
   #
   #   Psych.parse_stream("---\n - a\n - b") # => #<Psych::Nodes::Stream:0x00>
   #
+  #   Psych.parse_stream("--- a\n--- b") do |node|
+  #     node # => #<Psych::Nodes::Document:0x00>
+  #   end
+  #
+  #   begin
+  #     Psych.parse_stream("--- `", "file.txt")
+  #   rescue Psych::SyntaxError => ex
+  #     ex.file    # => 'file.txt'
+  #     ex.message # => "(foo.txt): found character that cannot start any token"
+  #   end
+  #
   # See Psych::Nodes for more information about YAML AST.
-  def self.parse_stream yaml
-    parser = self.parser
-    parser.parse yaml
-    parser.handler.root
+  def self.parse_stream yaml, filename = nil, &block
+    if block_given?
+      parser = Psych::Parser.new(Handlers::DocumentStream.new(&block))
+      parser.parse yaml, filename
+    else
+      parser = self.parser
+      parser.parse yaml, filename
+      parser.handler.root
+    end
   end
 
   ###
@@ -217,19 +267,34 @@ module Psych
 
   ###
   # Load multiple documents given in +yaml+.  Returns the parsed documents
-  # as a list.  For example:
+  # as a list.  If a block is given, each document will be converted to ruby
+  # and passed to the block during parsing
+  #
+  # Example:
   #
   #   Psych.load_stream("--- foo\n...\n--- bar\n...") # => ['foo', 'bar']
   #
-  def self.load_stream yaml
-    parse_stream(yaml).children.map { |child| child.to_ruby }
+  #   list = []
+  #   Psych.load_stream("--- foo\n...\n--- bar\n...") do |ruby|
+  #     list << ruby
+  #   end
+  #   list # => ['foo', 'bar']
+  #
+  def self.load_stream yaml, filename = nil
+    if block_given?
+      parse_stream(yaml, filename) do |node|
+        yield node.to_ruby
+      end
+    else
+      parse_stream(yaml, filename).children.map { |child| child.to_ruby }
+    end
   end
 
   ###
   # Load the document contained in +filename+.  Returns the yaml contained in
   # +filename+ as a ruby object
   def self.load_file filename
-    self.load File.open(filename)
+    File.open(filename, 'r:bom|utf-8') { |f| self.load f, filename }
   end
 
   # :stopdoc:

data/lib/psych/core_ext.rb

@@ -30,6 +30,7 @@ class Module
   alias :yaml_as :psych_yaml_as
 end
 
+if defined?(::IRB)
 module Kernel
   def psych_y *objects
     puts Psych.dump_stream(*objects)
@@ -38,3 +39,4 @@ module Kernel
   alias y psych_y
   private :y
 end
+end

data/lib/psych/handlers/document_stream.rb

@@ -0,0 +1,22 @@
+require 'psych/tree_builder'
+
+module Psych
+  module Handlers
+    class DocumentStream < Psych::TreeBuilder # :nodoc:
+      def initialize &block
+        super
+        @block = block
+      end
+
+      def start_document version, tag_directives, implicit
+        n = Nodes::Document.new version, tag_directives, implicit
+        push n
+      end
+
+      def end_document implicit_end = !streaming?
+        @last.implicit_end = implicit_end
+        @block.call pop
+      end
+    end
+  end
+end

data/lib/psych/parser.rb

@@ -36,12 +36,16 @@ module Psych
     # The handler on which events will be called
     attr_accessor :handler
 
+    # Set the encoding for this parser to +encoding+
+    attr_writer :external_encoding
+
     ###
     # Creates a new Psych::Parser instance with +handler+.  YAML events will
     # be called on +handler+.  See Psych::Parser for more details.
 
     def initialize handler = Handler.new
       @handler = handler
+      @external_encoding = ANY
     end
   end
 end

data/lib/psych/scalar_scanner.rb

@@ -46,9 +46,13 @@ module Psych
         end
       when TIME
         parse_time string
-      when /^\d{4}-\d{1,2}-\d{1,2}$/
+      when /^\d{4}-(?:1[012]|0\d|\d)-(?:[12]\d|3[01]|0\d|\d)$/
         require 'date'
-        Date.strptime(string, '%Y-%m-%d')
+        begin
+          Date.strptime(string, '%Y-%m-%d')
+        rescue ArgumentError
+          string
+        end
       when /^\.inf$/i
         1 / 0.0
       when /^-\.inf$/i
@@ -61,7 +65,7 @@ module Psych
         else
           string.sub(/^:/, '').to_sym
         end
-      when /^[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+$/
+      when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+$/
         i = 0
         string.split(':').each_with_index do |n,e|
           i += (n.to_i * 60 ** (e - 2).abs)
@@ -74,13 +78,19 @@ module Psych
         end
         i
       when FLOAT
-        return Float(string.gsub(/[,_]/, '')) rescue ArgumentError
+        begin
+          return Float(string.gsub(/[,_]/, ''))
+        rescue ArgumentError
+        end
 
         @string_cache[string] = true
         string
       else
         if string.count('.') < 2
-          return Integer(string.gsub(/[,_]/, '')) rescue ArgumentError
+          begin
+            return Integer(string.gsub(/[,_]/, ''))
+          rescue ArgumentError
+          end
         end
 
         @string_cache[string] = true

data/lib/psych/syntax_error.rb

@@ -0,0 +1,19 @@
+module Psych
+  class SyntaxError < ::SyntaxError
+    attr_reader :file, :line, :column, :offset, :problem, :context
+
+    def initialize file, line, col, offset, problem, context
+      err      = [problem, context].compact.join ' '
+      filename = file || '<unknown>'
+      message  = "(%s): %s at line %d column %d" % [filename, err, line, col]
+
+      @file    = file
+      @line    = line
+      @column  = col
+      @offset  = offset
+      @problem = problem
+      @context = context
+      super(message)
+    end
+  end
+end

data/lib/psych/tree_builder.rb

@@ -72,7 +72,9 @@ module Psych
     end
 
     def scalar value, anchor, tag, plain, quoted, style
-      @last.children << Nodes::Scalar.new(value,anchor,tag,plain,quoted,style)
+      s = Nodes::Scalar.new(value,anchor,tag,plain,quoted,style)
+      @last.children << s
+      s
     end
 
     def alias anchor

data/lib/psych/visitors/to_ruby.rb

@@ -31,9 +31,7 @@ module Psych
         result
       end
 
-      def visit_Psych_Nodes_Scalar o
-        @st[o.anchor] = o.value if o.anchor
-
+      def deserialize o
         if klass = Psych.load_tags[o.tag]
           instance = klass.allocate
 
@@ -52,8 +50,16 @@ module Psych
         case o.tag
         when '!binary', 'tag:yaml.org,2002:binary'
           o.value.unpack('m').first
-        when '!str', 'tag:yaml.org,2002:str'
-          o.value
+        when /^!(?:str|ruby\/string)(?::(.*))?/, 'tag:yaml.org,2002:str'
+          klass = resolve_class($1)
+          if klass
+            klass.allocate.replace o.value
+          else
+            o.value
+          end
+        when '!ruby/object:BigDecimal'
+          require 'bigdecimal'
+          BigDecimal._load o.value
         when "!ruby/object:DateTime"
           require 'date'
           @ss.parse_time(o.value).to_datetime
@@ -92,6 +98,11 @@ module Psych
           @ss.tokenize o.value
         end
       end
+      private :deserialize
+
+      def visit_Psych_Nodes_Scalar o
+        register o, deserialize(o)
+      end
 
       def visit_Psych_Nodes_Sequence o
         if klass = Psych.load_tags[o.tag]
@@ -108,15 +119,18 @@ module Psych
 
         case o.tag
         when '!omap', 'tag:yaml.org,2002:omap'
-          map = Psych::Omap.new
-          @st[o.anchor] = map if o.anchor
+          map = register(o, Psych::Omap.new)
           o.children.each { |a|
             map[accept(a.children.first)] = accept a.children.last
           }
           map
+        when /^!(?:seq|ruby\/array):(.*)$/
+          klass = resolve_class($1)
+          list  = register(o, klass.allocate)
+          o.children.each { |c| list.push accept c }
+          list
         else
-          list = []
-          @st[o.anchor] = list if o.anchor
+          list = register(o, [])
           o.children.each { |c| list.push accept c }
           list
         end
@@ -127,16 +141,33 @@ module Psych
         return revive_hash({}, o) unless o.tag
 
         case o.tag
-        when '!str', 'tag:yaml.org,2002:str'
+        when /^!(?:str|ruby\/string)(?::(.*))?/, 'tag:yaml.org,2002:str'
+          klass = resolve_class($1)
           members = Hash[*o.children.map { |c| accept c }]
           string = members.delete 'str'
+
+          if klass
+            string = klass.allocate
+            string.replace string
+          end
+
           init_with(string, members.map { |k,v| [k.to_s.sub(/^@/, ''),v] }, o)
+        when /^!ruby\/array:(.*)$/
+          klass = resolve_class($1)
+          list  = register(o, klass.allocate)
+
+          members = Hash[o.children.map { |c| accept c }.each_slice(2).to_a]
+          list.replace members['internal']
+
+          members['ivars'].each do |ivar, v|
+            list.instance_variable_set ivar, v
+          end
+          list
         when /^!ruby\/struct:?(.*)?$/
           klass = resolve_class($1)
 
           if klass
-            s = klass.allocate
-            @st[o.anchor] = s if o.anchor
+            s = register(o, klass.allocate)
 
             members = {}
             struct_members = s.members.map { |x| x.to_sym }
@@ -158,7 +189,7 @@ module Psych
 
         when '!ruby/range'
           h = Hash[*o.children.map { |c| accept c }]
-          Range.new(h['begin'], h['end'], h['excl'])
+          register o, Range.new(h['begin'], h['end'], h['excl'])
 
         when /^!ruby\/exception:?(.*)?$/
           h = Hash[*o.children.map { |c| accept c }]
@@ -177,11 +208,11 @@ module Psych
 
         when '!ruby/object:Complex'
           h = Hash[*o.children.map { |c| accept c }]
-          Complex(h['real'], h['image'])
+          register o, Complex(h['real'], h['image'])
 
         when '!ruby/object:Rational'
           h = Hash[*o.children.map { |c| accept c }]
-          Rational(h['numerator'], h['denominator'])
+          register o, Rational(h['numerator'], h['denominator'])
 
         when /^!ruby\/object:?(.*)?$/
           name = $1 || 'Object'
@@ -209,6 +240,11 @@ module Psych
       end
 
       private
+      def register node, object
+        @st[node.anchor] = object if node.anchor
+        object
+      end
+
       def revive_hash hash, o
         @st[o.anchor] = hash if o.anchor
 

data/lib/psych/visitors/yaml_tree.rb

@@ -159,13 +159,13 @@ module Psych
       end
 
       def visit_Regexp o
-        @emitter.scalar o.inspect, nil, '!ruby/regexp', false, false, Nodes::Scalar::ANY
+        register o, @emitter.scalar(o.inspect, nil, '!ruby/regexp', false, false, Nodes::Scalar::ANY)
       end
 
       def visit_DateTime o
         formatted = format_time o.to_time
         tag = '!ruby/object:DateTime'
-        @emitter.scalar formatted, nil, tag, false, false, Nodes::Scalar::ANY
+        register o, @emitter.scalar(formatted, nil, tag, false, false, Nodes::Scalar::ANY)
       end
 
       def visit_Time o
@@ -174,7 +174,7 @@ module Psych
       end
 
       def visit_Rational o
-        @emitter.start_mapping(nil, '!ruby/object:Rational', false, Nodes::Mapping::BLOCK)
+        register o, @emitter.start_mapping(nil, '!ruby/object:Rational', false, Nodes::Mapping::BLOCK)
 
         [
           'denominator', o.denominator.to_s,
@@ -187,7 +187,7 @@ module Psych
       end
 
       def visit_Complex o
-        @emitter.start_mapping(nil, '!ruby/object:Complex', false, Nodes::Mapping::BLOCK)
+        register o, @emitter.start_mapping(nil, '!ruby/object:Complex', false, Nodes::Mapping::BLOCK)
 
         ['real', o.real.to_s, 'image', o.imag.to_s].each do |m|
           @emitter.scalar m, nil, nil, true, false, Nodes::Scalar::ANY
@@ -214,6 +214,10 @@ module Psych
         end
       end
 
+      def visit_BigDecimal o
+        @emitter.scalar o._dump, nil, '!ruby/object:BigDecimal', false, false, Nodes::Scalar::ANY
+      end
+
       def binary? string
         string.encoding == Encoding::ASCII_8BIT ||
           string.index("\x00") ||
@@ -241,9 +245,15 @@ module Psych
         ivars = find_ivars o
 
         if ivars.empty?
+          unless o.class == ::String
+            tag = "!ruby/string:#{o.class}"
+          end
           @emitter.scalar str, nil, tag, plain, quote, style
         else
-          @emitter.start_mapping nil, '!str', false, Nodes::Mapping::BLOCK
+          maptag = '!ruby/string'
+          maptag << ":#{o.class}" unless o.class == ::String
+
+          @emitter.start_mapping nil, maptag, false, Nodes::Mapping::BLOCK
           @emitter.scalar 'str', nil, nil, true, false, Nodes::Scalar::ANY
           @emitter.scalar str, nil, tag, plain, quote, style
 
@@ -255,16 +265,16 @@ module Psych
 
       def visit_Module o
         raise TypeError, "can't dump anonymous module: #{o}" unless o.name
-        @emitter.scalar o.name, nil, '!ruby/module', false, false, Nodes::Scalar::SINGLE_QUOTED
+        register o, @emitter.scalar(o.name, nil, '!ruby/module', false, false, Nodes::Scalar::SINGLE_QUOTED)
       end
 
       def visit_Class o
         raise TypeError, "can't dump anonymous class: #{o}" unless o.name
-        @emitter.scalar o.name, nil, '!ruby/class', false, false, Nodes::Scalar::SINGLE_QUOTED
+        register o, @emitter.scalar(o.name, nil, '!ruby/class', false, false, Nodes::Scalar::SINGLE_QUOTED)
       end
 
       def visit_Range o
-        @emitter.start_mapping nil, '!ruby/range', false, Nodes::Mapping::BLOCK
+        register o, @emitter.start_mapping(nil, '!ruby/range', false, Nodes::Mapping::BLOCK)
         ['begin', o.begin, 'end', o.end, 'excl', o.exclude_end?].each do |m|
           accept m
         end
@@ -297,9 +307,13 @@ module Psych
       end
 
       def visit_Array o
-        register o, @emitter.start_sequence(nil, nil, true, Nodes::Sequence::BLOCK)
-        o.each { |c| accept c }
-        @emitter.end_sequence
+        if o.class == ::Array
+          register o, @emitter.start_sequence(nil, nil, true, Nodes::Sequence::BLOCK)
+          o.each { |c| accept c }
+          @emitter.end_sequence
+        else
+          visit_array_subclass o
+        end
       end
 
       def visit_NilClass o
@@ -311,6 +325,39 @@ module Psych
       end
 
       private
+      def visit_array_subclass o
+        tag = "!ruby/array:#{o.class}"
+        if o.instance_variables.empty?
+          node = @emitter.start_sequence(nil, tag, false, Nodes::Sequence::BLOCK)
+          register o, node
+          o.each { |c| accept c }
+          @emitter.end_sequence
+        else
+          node = @emitter.start_mapping(nil, tag, false, Nodes::Sequence::BLOCK)
+          register o, node
+
+          # Dump the internal list
+          accept 'internal'
+          @emitter.start_sequence(nil, nil, true, Nodes::Sequence::BLOCK)
+          o.each { |c| accept c }
+          @emitter.end_sequence
+
+          # Dump the ivars
+          accept 'ivars'
+          @emitter.start_mapping(nil, nil, true, Nodes::Sequence::BLOCK)
+          o.instance_variables.each do |ivar|
+            accept ivar
+            accept o.instance_variable_get ivar
+          end
+          @emitter.end_mapping
+
+          @emitter.end_mapping
+        end
+      end
+
+      def dump_list o
+      end
+
       # '%:z' was no defined until 1.9.3
       if RUBY_VERSION < '1.9.3'
         def format_time time