psych 1.2.2 → 1.3.0

data/.travis.yml

@@ -1,3 +1,9 @@
 rvm:
   - 1.9.2
   - 1.9.3
+  - ruby-head
+before_script:
+  - gem install isolate
+  - gem install hoe
+  - gem install rake-compiler
+script: rake isolate test

data/CHANGELOG.rdoc

@@ -1,3 +1,129 @@
+Fri Mar  9 06:29:22 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb (load, parse): stop parsing or loading after
+	  the first document has been parsed.
+
+	* test/psych/test_stream.rb: pertinent tests.
+
+Fri Mar  9 06:17:05 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb (parse_stream, load_stream): if a block is
+	  given, documents will be yielded to the block as they are parsed.
+	  [ruby-core:42404] [Bug #5978]
+
+	* ext/psych/lib/psych/handlers/document_stream.rb: add a handler that
+	  yields documents as they are parsed
+
+	* test/psych/test_stream.rb: corresponding tests.
+
+Tue Mar  6 02:31:20 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/core_ext.rb: only extend Kernel if IRB is loaded
+	  in order to stop method pollution.
+
+Tue Feb 28 10:28:51 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb: default open YAML files with utf8 external
+	  encoding. [ruby-core:42967]
+	* test/psych/test_tainted.rb: ditto
+
+Fri Feb 24 13:54:33 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/parser.c: prevent a memory leak by protecting calls to
+	  handler callbacks.
+	* test/psych/test_parser.rb: test to demonstrate leak.
+
+Fri Feb 24 08:08:38 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/parser.c: set parser encoding based on the YAML input
+	  rather than user configuration.
+	* test/psych/test_encoding.rb: corresponding tests.
+	* test/psych/test_parser.rb: ditto
+	* test/psych/test_tainted.rb: ditto
+
+Fri Feb 10 03:41:31 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/parser.c: removed external encoding setter, allow parser
+	  to be reused.
+	* ext/psych/lib/psych/parser.rb: added external encoding setter.
+	* test/psych/test_parser.rb: test parser reuse
+
+Wed Jan 18 12:49:15 2012  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/visitors/to_ruby.rb: Added support for loading
+	  subclasses of String with ivars
+	* ext/psych/lib/psych/visitors/yaml_tree.rb: Added support for dumping
+	  subclasses of String with ivars
+	* test/psych/test_string.rb: corresponding tests
+
+Sun Dec 18 12:42:48 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/visitors/to_ruby.rb: BigDecimals can be restored
+	  from YAML.
+	* ext/psych/lib/psych/visitors/yaml_tree.rb: BigDecimals can be dumped
+	  to YAML.
+	* test/psych/test_numeric.rb: tests for BigDecimal serialization
+
+Sun Dec 18 12:03:13 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/scalar_scanner.rb: Strings that look like dates
+	  should be treated as strings and not dates.
+
+	* test/psych/test_scalar_scanner.rb: corresponding tests.
+
+Wed Dec  7 08:04:31 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb (module Psych): parse and load methods take
+	  an optional file name that is used when raising Psych::SyntaxError
+	  exceptions
+	* ext/psych/lib/psych/syntax_error.rb (module Psych): allow nil file
+	  names and handle nil file names in the exception message
+	* test/psych/test_exception.rb (module Psych): Tests for changes.
+
+Wed Nov 30 09:09:37 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/parser.c (parse): parse method can take an option file
+	  name for use in exception messages.
+	* test/psych/test_parser.rb: corresponding tests.
+
+Tue Nov 22 04:46:22 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb: remove autoload from psych
+	* ext/psych/lib/psych/json.rb: ditto
+
+Thu Nov 17 10:36:46 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych.rb (load_file): make sure opened yaml files are
+	  also closed. [ruby-core:41088]
+
+Wed Nov  9 04:52:16 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/tree_builder.rb: dump complex numbers,
+	  rationals, etc with reference ids.
+	* ext/psych/lib/psych/visitors/yaml_tree.rb: ditto
+	* ext/psych/lib/psych/visitors/to_ruby.rb: loading complex numbers,
+	  rationals, etc with reference ids.
+	* test/psych/test_object_references.rb: corresponding tests
+
+Mon Nov  7 20:31:52 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/scalar_scanner.rb: make sure strings that look
+	  like base 60 numbers are serialized as quoted strings.
+	* test/psych/test_string.rb: test for change.
+
+Wed Oct  5 02:50:27 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/lib/psych/syntax_error.rb: Add file, line, offset, and
+	  message attributes during parse failure.
+	* ext/psych/parser.c: Update parser to raise exception with correct
+	  values.
+	* test/psych/test_exception.rb: corresponding tests.
+
+Wed Oct  5 01:52:16 2011  Aaron Patterson <aaron@tenderlovemaking.com>
+
+	* ext/psych/parser.c (parse): Use context_mark for indicating error
+	  line and column.
+
 Tue Oct  4 06:29:55 2011  Aaron Patterson <aaron@tenderlovemaking.com>
 
 	* ext/psych/lib/psych.rb: calling `yaml` rather than `to_yaml`.

data/Manifest.txt

@@ -1,7 +1,6 @@
 .autotest
 .travis.yml
 CHANGELOG.rdoc
-Gemfile
 Manifest.txt
 README.rdoc
 Rakefile
@@ -21,7 +20,7 @@ lib/psych/coder.rb
 lib/psych/core_ext.rb
 lib/psych/deprecated.rb
 lib/psych/handler.rb
-lib/psych/json.rb
+lib/psych/handlers/document_stream.rb
 lib/psych/json/ruby_events.rb
 lib/psych/json/stream.rb
 lib/psych/json/tree_builder.rb
@@ -40,6 +39,7 @@ lib/psych/scalar_scanner.rb
 lib/psych/set.rb
 lib/psych/stream.rb
 lib/psych/streaming.rb
+lib/psych/syntax_error.rb
 lib/psych/tree_builder.rb
 lib/psych/visitors.rb
 lib/psych/visitors/depth_first.rb
@@ -70,6 +70,7 @@ test/psych/test_nil.rb
 test/psych/test_null.rb
 test/psych/test_numeric.rb
 test/psych/test_object.rb
+test/psych/test_object_references.rb
 test/psych/test_omap.rb
 test/psych/test_parser.rb
 test/psych/test_psych.rb

data/README.rdoc

@@ -4,7 +4,7 @@
 
 == Description
 
-Psych is a YAML parser and emitter.  Psych leverages libyaml[http://libyaml.org]
+Psych is a YAML parser and emitter.  Psych leverages libyaml[http://pyyaml.org/wiki/LibYAML]
 for its YAML parsing and emitting capabilities.  In addition to wrapping
 libyaml, Psych also knows how to serialize and de-serialize most Ruby objects
 to and from the YAML format.

data/Rakefile

@@ -11,7 +11,7 @@ end
 gem 'rake-compiler', '>= 0.4.1'
 require "rake/extensiontask"
 
-Hoe.plugin :debugging, :doofus, :git, :gemspec, :bundler
+Hoe.plugin :doofus, :git, :gemspec, :isolate
 
 $hoe = Hoe.spec 'psych' do
   developer 'Aaron Patterson', 'aaron@tenderlovemaking.com'

data/ext/psych/emitter.c

@@ -351,7 +351,7 @@ static VALUE start_mapping(
 	    (yaml_char_t *)(NIL_P(anchor) ? NULL : StringValuePtr(anchor)),
 	    (yaml_char_t *)(NIL_P(tag) ? NULL : StringValuePtr(tag)),
 	    implicit ? 1 : 0,
-	    (yaml_sequence_style_t)NUM2INT(style)
+	    (yaml_mapping_style_t)NUM2INT(style)
 	    );
 
     emit(emitter, &event);

data/ext/psych/parser.c

@@ -59,6 +59,163 @@ static VALUE allocate(VALUE klass)
     return Data_Wrap_Struct(klass, 0, dealloc, parser);
 }
 
+static VALUE make_exception(yaml_parser_t * parser, VALUE path)
+{
+    size_t line, column;
+
+    line = parser->context_mark.line + 1;
+    column = parser->context_mark.column + 1;
+
+    return rb_funcall(ePsychSyntaxError, rb_intern("new"), 6,
+	    path,
+	    INT2NUM(line),
+	    INT2NUM(column),
+	    INT2NUM(parser->problem_offset),
+	    parser->problem ? rb_usascii_str_new2(parser->problem) : Qnil,
+	    parser->context ? rb_usascii_str_new2(parser->context) : Qnil);
+}
+
+#ifdef HAVE_RUBY_ENCODING_H
+static VALUE transcode_string(VALUE src, int * parser_encoding)
+{
+    int utf8    = rb_utf8_encindex();
+    int utf16le = rb_enc_find_index("UTF16_LE");
+    int utf16be = rb_enc_find_index("UTF16_BE");
+    int source_encoding = rb_enc_get_index(src);
+
+    if (source_encoding == utf8) {
+	*parser_encoding = YAML_UTF8_ENCODING;
+	return src;
+    }
+
+    if (source_encoding == utf16le) {
+	*parser_encoding = YAML_UTF16LE_ENCODING;
+	return src;
+    }
+
+    if (source_encoding == utf16be) {
+	*parser_encoding = YAML_UTF16BE_ENCODING;
+	return src;
+    }
+
+    src = rb_str_export_to_enc(src, rb_utf8_encoding());
+    RB_GC_GUARD(src);
+
+    *parser_encoding = YAML_UTF8_ENCODING;
+    return src;
+}
+
+static VALUE transcode_io(VALUE src, int * parser_encoding)
+{
+    VALUE io_external_encoding;
+    int io_external_enc_index;
+
+    io_external_encoding = rb_funcall(src, rb_intern("external_encoding"), 0);
+
+    /* if no encoding is returned, assume ascii8bit. */
+    if (NIL_P(io_external_encoding)) {
+	io_external_enc_index = rb_ascii8bit_encindex();
+    } else {
+	io_external_enc_index = rb_to_encoding_index(io_external_encoding);
+    }
+
+    /* Treat US-ASCII as utf_8 */
+    if (io_external_enc_index == rb_usascii_encindex()) {
+	*parser_encoding = YAML_UTF8_ENCODING;
+	return src;
+    }
+
+    if (io_external_enc_index == rb_utf8_encindex()) {
+	*parser_encoding = YAML_UTF8_ENCODING;
+	return src;
+    }
+
+    if (io_external_enc_index == rb_enc_find_index("UTF-16LE")) {
+	*parser_encoding = YAML_UTF16LE_ENCODING;
+	return src;
+    }
+
+    if (io_external_enc_index == rb_enc_find_index("UTF-16BE")) {
+	*parser_encoding = YAML_UTF16BE_ENCODING;
+	return src;
+    }
+
+    /* Just guess on ASCII-8BIT */
+    if (io_external_enc_index == rb_ascii8bit_encindex()) {
+	*parser_encoding = YAML_ANY_ENCODING;
+	return src;
+    }
+
+    rb_raise(rb_eArgError, "YAML file must be UTF-8, UTF-16LE, or UTF-16BE, not %s",
+	    rb_enc_name(rb_enc_from_index(io_external_enc_index)));
+
+    return Qnil;
+}
+
+#endif
+
+static VALUE protected_start_stream(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall(args[0], id_start_stream, 1, args[1]);
+}
+
+static VALUE protected_start_document(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall3(args[0], id_start_document, 3, args + 1);
+}
+
+static VALUE protected_end_document(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall(args[0], id_end_document, 1, args[1]);
+}
+
+static VALUE protected_alias(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall(args[0], id_alias, 1, args[1]);
+}
+
+static VALUE protected_scalar(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall3(args[0], id_scalar, 6, args + 1);
+}
+
+static VALUE protected_start_sequence(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall3(args[0], id_start_sequence, 4, args + 1);
+}
+
+static VALUE protected_end_sequence(VALUE handler)
+{
+    return rb_funcall(handler, id_end_sequence, 0);
+}
+
+static VALUE protected_start_mapping(VALUE pointer)
+{
+    VALUE *args = (VALUE *)pointer;
+    return rb_funcall3(args[0], id_start_mapping, 4, args + 1);
+}
+
+static VALUE protected_end_mapping(VALUE handler)
+{
+    return rb_funcall(handler, id_end_mapping, 0);
+}
+
+static VALUE protected_empty(VALUE handler)
+{
+    return rb_funcall(handler, id_empty, 0);
+}
+
+static VALUE protected_end_stream(VALUE handler)
+{
+    return rb_funcall(handler, id_end_stream, 0);
+}
+
 /*
  * call-seq:
  *    parser.parse(yaml)
@@ -68,27 +225,48 @@ static VALUE allocate(VALUE klass)
  *
  * See Psych::Parser and Psych::Parser#handler
  */
-static VALUE parse(VALUE self, VALUE yaml)
+static VALUE parse(int argc, VALUE *argv, VALUE self)
 {
+    VALUE yaml, path;
     yaml_parser_t * parser;
     yaml_event_t event;
     int done = 0;
     int tainted = 0;
+    int state = 0;
+    int parser_encoding = YAML_ANY_ENCODING;
 #ifdef HAVE_RUBY_ENCODING_H
     int encoding = rb_utf8_encindex();
     rb_encoding * internal_enc = rb_default_internal_encoding();
 #endif
     VALUE handler = rb_iv_get(self, "@handler");
 
+    if (rb_scan_args(argc, argv, "11", &yaml, &path) == 1) {
+	if(rb_respond_to(yaml, id_path))
+	    path = rb_funcall(yaml, id_path, 0);
+	else
+	    path = rb_str_new2("<unknown>");
+    }
+
     Data_Get_Struct(self, yaml_parser_t, parser);
 
+    yaml_parser_delete(parser);
+    yaml_parser_initialize(parser);
+
     if (OBJ_TAINTED(yaml)) tainted = 1;
 
-    if(rb_respond_to(yaml, id_read)) {
+    if (rb_respond_to(yaml, id_read)) {
+#ifdef HAVE_RUBY_ENCODING_H
+	yaml = transcode_io(yaml, &parser_encoding);
+	yaml_parser_set_encoding(parser, parser_encoding);
+#endif
 	yaml_parser_set_input(parser, io_reader, (void *)yaml);
 	if (RTEST(rb_obj_is_kind_of(yaml, rb_cIO))) tainted = 1;
     } else {
 	StringValue(yaml);
+#ifdef HAVE_RUBY_ENCODING_H
+	yaml = transcode_string(yaml, &parser_encoding);
+	yaml_parser_set_encoding(parser, parser_encoding);
+#endif
 	yaml_parser_set_input_string(
 		parser,
 		(const unsigned char *)RSTRING_PTR(yaml),
@@ -98,32 +276,28 @@ static VALUE parse(VALUE self, VALUE yaml)
 
     while(!done) {
 	if(!yaml_parser_parse(parser, &event)) {
-	    VALUE path;
-	    size_t line   = parser->mark.line;
-	    size_t column = parser->mark.column;
-
-	    if(rb_respond_to(yaml, id_path))
-		path = rb_funcall(yaml, id_path, 0);
-	    else
-		path = rb_str_new2("<unknown>");
+	    VALUE exception;
 
+	    exception = make_exception(parser, path);
 	    yaml_parser_delete(parser);
 	    yaml_parser_initialize(parser);
 
-	    rb_raise(ePsychSyntaxError, "(%s): couldn't parse YAML at line %d column %d",
-		    StringValuePtr(path),
-		    (int)line, (int)column);
+	    rb_exc_raise(exception);
 	}
 
 	switch(event.type) {
-	  case YAML_STREAM_START_EVENT:
-
-	    rb_funcall(handler, id_start_stream, 1,
-		       INT2NUM((long)event.data.stream_start.encoding)
-		);
-	    break;
+	    case YAML_STREAM_START_EVENT:
+	      {
+		  VALUE args[2];
+
+		  args[0] = handler;
+		  args[1] = INT2NUM((long)event.data.stream_start.encoding);
+		  rb_protect(protected_start_stream, (VALUE)args, &state);
+	      }
+	      break;
 	  case YAML_DOCUMENT_START_EVENT:
 	    {
+		VALUE args[4];
 		/* Get a list of tag directives (if any) */
 		VALUE tag_directives = rb_ary_new();
 		/* Grab the document version */
@@ -161,19 +335,25 @@ static VALUE parse(VALUE self, VALUE yaml)
 			rb_ary_push(tag_directives, rb_ary_new3((long)2, handle, prefix));
 		    }
 		}
-		rb_funcall(handler, id_start_document, 3,
-			   version, tag_directives,
-			   event.data.document_start.implicit == 1 ? Qtrue : Qfalse
-		    );
+		args[0] = handler;
+		args[1] = version;
+		args[2] = tag_directives;
+		args[3] = event.data.document_start.implicit == 1 ? Qtrue : Qfalse;
+		rb_protect(protected_start_document, (VALUE)args, &state);
 	    }
 	    break;
 	  case YAML_DOCUMENT_END_EVENT:
-	    rb_funcall(handler, id_end_document, 1,
-		       event.data.document_end.implicit == 1 ? Qtrue : Qfalse
-		);
+	    {
+		VALUE args[2];
+
+		args[0] = handler;
+		args[1] = event.data.document_end.implicit == 1 ? Qtrue : Qfalse;
+		rb_protect(protected_end_document, (VALUE)args, &state);
+	    }
 	    break;
 	  case YAML_ALIAS_EVENT:
 	    {
+		VALUE args[2];
 		VALUE alias = Qnil;
 		if(event.data.alias.anchor) {
 		    alias = rb_str_new2((const char *)event.data.alias.anchor);
@@ -183,11 +363,14 @@ static VALUE parse(VALUE self, VALUE yaml)
 #endif
 		}
 
-		rb_funcall(handler, id_alias, 1, alias);
+		args[0] = handler;
+		args[1] = alias;
+		rb_protect(protected_alias, (VALUE)args, &state);
 	    }
 	    break;
 	  case YAML_SCALAR_EVENT:
 	    {
+		VALUE args[7];
 		VALUE anchor = Qnil;
 		VALUE tag = Qnil;
 		VALUE plain_implicit, quoted_implicit, style;
@@ -225,12 +408,19 @@ static VALUE parse(VALUE self, VALUE yaml)
 
 		style = INT2NUM((long)event.data.scalar.style);
 
-		rb_funcall(handler, id_scalar, 6,
-			   val, anchor, tag, plain_implicit, quoted_implicit, style);
+		args[0] = handler;
+		args[1] = val;
+		args[2] = anchor;
+		args[3] = tag;
+		args[4] = plain_implicit;
+		args[5] = quoted_implicit;
+		args[6] = style;
+		rb_protect(protected_scalar, (VALUE)args, &state);
 	    }
 	    break;
 	  case YAML_SEQUENCE_START_EVENT:
 	    {
+		VALUE args[5];
 		VALUE anchor = Qnil;
 		VALUE tag = Qnil;
 		VALUE implicit, style;
@@ -256,15 +446,21 @@ static VALUE parse(VALUE self, VALUE yaml)
 
 		style = INT2NUM((long)event.data.sequence_start.style);
 
-		rb_funcall(handler, id_start_sequence, 4,
-			   anchor, tag, implicit, style);
+		args[0] = handler;
+		args[1] = anchor;
+		args[2] = tag;
+		args[3] = implicit;
+		args[4] = style;
+
+		rb_protect(protected_start_sequence, (VALUE)args, &state);
 	    }
 	    break;
 	  case YAML_SEQUENCE_END_EVENT:
-	    rb_funcall(handler, id_end_sequence, 0);
+	    rb_protect(protected_end_sequence, handler, &state);
 	    break;
 	  case YAML_MAPPING_START_EVENT:
 	    {
+		VALUE args[5];
 		VALUE anchor = Qnil;
 		VALUE tag = Qnil;
 		VALUE implicit, style;
@@ -289,50 +485,33 @@ static VALUE parse(VALUE self, VALUE yaml)
 
 		style = INT2NUM((long)event.data.mapping_start.style);
 
-		rb_funcall(handler, id_start_mapping, 4,
-			   anchor, tag, implicit, style);
+		args[0] = handler;
+		args[1] = anchor;
+		args[2] = tag;
+		args[3] = implicit;
+		args[4] = style;
+
+		rb_protect(protected_start_mapping, (VALUE)args, &state);
 	    }
 	    break;
 	  case YAML_MAPPING_END_EVENT:
-	    rb_funcall(handler, id_end_mapping, 0);
+	    rb_protect(protected_end_mapping, handler, &state);
 	    break;
 	  case YAML_NO_EVENT:
-	    rb_funcall(handler, id_empty, 0);
+	    rb_protect(protected_empty, handler, &state);
 	    break;
 	  case YAML_STREAM_END_EVENT:
-	    rb_funcall(handler, id_end_stream, 0);
+	    rb_protect(protected_end_stream, handler, &state);
 	    done = 1;
 	    break;
 	}
 	yaml_event_delete(&event);
+	if (state) rb_jump_tag(state);
     }
 
     return self;
 }
 
-/*
- * call-seq:
- *    parser.external_encoding=(encoding)
- *
- * Set the encoding for this parser to +encoding+
- */
-static VALUE set_external_encoding(VALUE self, VALUE encoding)
-{
-    yaml_parser_t * parser;
-    VALUE exception;
-
-    Data_Get_Struct(self, yaml_parser_t, parser);
-
-    if(parser->encoding) {
-	exception = rb_const_get_at(mPsych, rb_intern("Exception"));
-	rb_raise(exception, "don't set the encoding twice!");
-    }
-
-    yaml_parser_set_encoding(parser, NUM2INT(encoding));
-
-    return encoding;
-}
-
 /*
  * call-seq:
  *    parser.mark # => #<Psych::Parser::Mark>
@@ -376,11 +555,11 @@ void Init_psych_parser()
     /* UTF-16-BE Encoding with BOM */
     rb_define_const(cPsychParser, "UTF16BE", INT2NUM(YAML_UTF16BE_ENCODING));
 
+    rb_require("psych/syntax_error");
     ePsychSyntaxError = rb_define_class_under(mPsych, "SyntaxError", rb_eSyntaxError);
 
-    rb_define_method(cPsychParser, "parse", parse, 1);
+    rb_define_method(cPsychParser, "parse", parse, -1);
     rb_define_method(cPsychParser, "mark", mark, 0);
-    rb_define_method(cPsychParser, "external_encoding=", set_external_encoding, 1);
 
     id_read           = rb_intern("read");
     id_path           = rb_intern("path");