json_pure 1.1.9 → 1.2.0

data/ext/json/ext/parser/parser.rl

@@ -19,15 +19,19 @@
 #ifdef HAVE_RUBY_ENCODING_H
 #include "ruby/encoding.h"
 #define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
+static VALUE mEncoding_ASCII_8BIT, mEncoding_UTF_8, mEncoding_UTF_16BE,
+    mEncoding_UTF_16LE, mEncoding_UTF_32BE, mEncoding_UTF_32LE;
+static ID i_encoding, i_encode, i_encode_bang, i_force_encoding;
 #else
 #define FORCE_UTF8(obj)
+static ID i_iconv;
 #endif
 
 static VALUE mJSON, mExt, cParser, eParserError, eNestingError;
 static VALUE CNaN, CInfinity, CMinusInfinity;
 
 static ID i_json_creatable_p, i_json_create, i_create_id, i_create_additions,
-          i_chr, i_max_nesting, i_allow_nan, i_object_class, i_array_class; 
+          i_chr, i_max_nesting, i_allow_nan, i_object_class, i_array_class;
 
 #define MinusInfinity "-Infinity"
 
@@ -484,6 +488,54 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
  *
  */
 
+inline static VALUE convert_encoding(VALUE source)
+{
+    char *ptr = RSTRING_PTR(source);
+    long len = RSTRING_LEN(source);
+    if (len < 2) {
+        rb_raise(eParserError, "A JSON text must at least contain two octets!");
+    }
+#ifdef HAVE_RUBY_ENCODING_H
+    {
+        VALUE encoding = rb_funcall(source, i_encoding, 0);
+        if (encoding == mEncoding_ASCII_8BIT) {
+            if (len >= 4 &&  ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
+                source = rb_str_dup(source);
+                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_32BE);
+                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+            } else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
+                source = rb_str_dup(source);
+                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_16BE);
+                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+            } else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
+                source = rb_str_dup(source);
+                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_32LE);
+                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+            } else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
+                source = rb_str_dup(source);
+                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_16LE);
+                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+            } else {
+                source = rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_8);
+            }
+        } else {
+            source = rb_funcall(source, i_encode, 1, mEncoding_UTF_8);
+        }
+    }
+#else
+    if (len >= 4 &&  ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
+      source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-32be"), source);
+    } else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
+      source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-16be"), source);
+    } else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
+      source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-32le"), source);
+    } else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
+      source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-16le"), source);
+    }
+#endif
+    return source;
+}
+
 /*
  * call-seq: new(source, opts => {})
  *
@@ -514,12 +566,9 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
     VALUE source, opts;
     GET_STRUCT;
     rb_scan_args(argc, argv, "11", &source, &opts);
-    source = StringValue(source);
+    source = convert_encoding(StringValue(source));
     ptr = RSTRING_PTR(source);
     len = RSTRING_LEN(source);
-    if (len < 2) {
-        rb_raise(eParserError, "A JSON text must at least contain two octets!");
-    }
     if (!NIL_P(opts)) {
         opts = rb_convert_type(opts, T_HASH, "Hash", "to_hash");
         if (NIL_P(opts)) {
@@ -576,18 +625,6 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
         json->array_class = Qnil;
     }
     json->current_nesting = 0;
-    /*
-       Convert these?
-    if (len >= 4 &&  ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
-        rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
-    } else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
-        rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
-    } else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
-        rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
-    } else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
-        rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
-    }
-    */
     json->len = len;
     json->source = ptr;
     json->Vsource = source;
@@ -683,4 +720,18 @@ void Init_parser()
     i_allow_nan = rb_intern("allow_nan");
     i_object_class = rb_intern("object_class");
     i_array_class = rb_intern("array_class");
+#ifdef HAVE_RUBY_ENCODING_H
+    mEncoding_UTF_8 = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-8"));
+    mEncoding_UTF_16BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16be"));
+    mEncoding_UTF_16LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16le"));
+    mEncoding_UTF_32BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32be"));
+    mEncoding_UTF_32LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32le"));
+    mEncoding_ASCII_8BIT = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("ascii-8bit"));
+    i_encoding = rb_intern("encoding");
+    i_encode = rb_intern("encode");
+    i_encode_bang = rb_intern("encode!");
+    i_force_encoding = rb_intern("force_encoding");
+#else
+    i_iconv = rb_intern("iconv");
+#endif
 }

data/ext/json/ext/parser/unicode.c

@@ -105,7 +105,7 @@ char *JSON_convert_UTF16_to_UTF8 (
                 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
                     ruby_xfree(tmp);
                     rb_raise(rb_path2class("JSON::ParserError"),
-                            "source sequence is illegal/malformed near %s", source);
+                            "\\uXXXX is illegal/malformed utf-16 near %s", source);
                 }
             } else { /* We don't have the 16 bits following the high surrogate. */
                 ruby_xfree(tmp);
@@ -118,7 +118,7 @@ char *JSON_convert_UTF16_to_UTF8 (
             if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
                 ruby_xfree(tmp);
                 rb_raise(rb_path2class("JSON::ParserError"),
-                    "source sequence is illegal/malformed near %s", source);
+                    "\\uXXXX is illegal/malformed utf-16 near %s", source);
             }
         }
         /* Figure out how many bytes the result will require */

data/lib/json/common.rb

@@ -1,4 +1,5 @@
 require 'json/version'
+require 'iconv'
 
 module JSON
   class << self
@@ -74,7 +75,7 @@ module JSON
   end
   self.create_id = 'json_class'
 
-  NaN           = (-1.0) ** 0.5
+  NaN           = 0.0/0
 
   Infinity      = 1.0/0
 
@@ -105,7 +106,7 @@ module JSON
 
   module_function
 
-  # Parse the JSON string _source_ into a Ruby data structure and return it.
+  # Parse the JSON document _source_ into a Ruby data structure and return it.
   #
   # _opts_ can have the following
   # keys:
@@ -122,9 +123,9 @@ module JSON
     JSON.parser.new(source, opts).parse
   end
 
-  # Parse the JSON string _source_ into a Ruby data structure and return it.
+  # Parse the JSON document _source_ into a Ruby data structure and return it.
   # The bang version of the parse method, defaults to the more dangerous values
-  # for the _opts_ hash, so be sure only to parse trusted _source_ strings.
+  # for the _opts_ hash, so be sure only to parse trusted _source_ documents.
   #
   # _opts_ can have the following keys:
   # * *max_nesting*: The maximum depth of nesting allowed in the parsed data
@@ -145,9 +146,8 @@ module JSON
     JSON.parser.new(source, opts).parse
   end
 
-  # Unparse the Ruby data structure _obj_ into a single line JSON string and
-  # return it. _state_ is
-  # * a JSON::State object,
+  # Generate a JSON document from the Ruby data structure _obj_ and return
+  # it. _state_ is * a JSON::State object,
   # * or a Hash like object (responding to to_hash),
   # * an object convertible into a hash by a to_h method,
   # that is used as or to configure a State object.
@@ -180,7 +180,11 @@ module JSON
     else
       state = State.new
     end
-    obj.to_json(state)
+    result = obj.to_json(state)
+    if result !~ /\A\s*(?:\[.*\]|\{.*\})\s*\Z/m
+      raise GeneratorError, "only generation of JSON objects or arrays allowed"
+    end
+    result
   end
 
   # :stopdoc:
@@ -190,14 +194,17 @@ module JSON
   module_function :unparse
   # :startdoc:
 
-  # Unparse the Ruby data structure _obj_ into a single line JSON string and
-  # return it. This method disables the checks for circles in Ruby objects, and
-  # also generates NaN, Infinity, and, -Infinity float values.
+  # Generate a JSON document from the Ruby data structure _obj_ and return it.
+  # This method disables the checks for circles in Ruby objects.
   #
   # *WARNING*: Be careful not to pass any Ruby data structures with circles as
   # _obj_ argument, because this will cause JSON to go into an infinite loop.
   def fast_generate(obj)
-    obj.to_json(nil)
+    result = obj.to_json(nil)
+    if result !~ /\A(?:\[.*\]|\{.*\})\Z/
+      raise GeneratorError, "only generation of JSON objects or arrays allowed"
+    end
+    result
   end
 
   # :stopdoc:
@@ -206,8 +213,9 @@ module JSON
   module_function :fast_unparse
   # :startdoc:
 
-  # Unparse the Ruby data structure _obj_ into a JSON string and return it. The
-  # returned string is a prettier form of the string returned by #unparse.
+  # Generate a JSON document from the Ruby data structure _obj_ and return it.
+  # The returned document is a prettier form of the document returned by
+  # #unparse.
   #
   # The _opts_ argument can be used to configure the generator, see the
   # generate method for a more detailed explanation.
@@ -229,7 +237,11 @@ module JSON
       end
       state.configure(opts)
     end
-    obj.to_json(state)
+    result = obj.to_json(state)
+    if result !~ /\A\s*(?:\[.*\]|\{.*\})\s*\Z/m
+      raise GeneratorError, "only generation of JSON objects or arrays allowed"
+    end
+    result
   end
 
   # :stopdoc:
@@ -270,8 +282,6 @@ module JSON
       proc.call result
     end
   end
-  private :recurse_proc
-  module_function :recurse_proc
 
   alias restore load
   module_function :restore
@@ -307,6 +317,11 @@ module JSON
   rescue JSON::NestingError
     raise ArgumentError, "exceed depth limit"
   end
+
+  # Shortuct for iconv.
+  def self.iconv(to, from, string)
+    Iconv.iconv(to, from, string).first
+  end
 end
 
 module ::Kernel

data/lib/json/pure/generator.rb

@@ -38,11 +38,11 @@ module JSON
 
   # Convert a UTF8 encoded Ruby string _string_ to a JSON string, encoded with
   # UTF16 big endian characters as \u????, and return it.
-  if String.method_defined?(:force_encoding)
+  if defined?(::Encoding)
     def utf8_to_json(string) # :nodoc:
       string = string.dup
       string << '' # XXX workaround: avoid buffer sharing
-      string.force_encoding(Encoding::ASCII_8BIT)
+      string.force_encoding(::Encoding::ASCII_8BIT)
       string.gsub!(/["\\\x0-\x1f]/) { MAP[$&] }
       string.gsub!(/(
                       (?:
@@ -56,7 +56,7 @@ module JSON
                       s = JSON::UTF8toUTF16.iconv(c).unpack('H*')[0]
                       s.gsub!(/.{4}/n, '\\\\u\&')
                     }
-      string.force_encoding(Encoding::UTF_8)
+      string.force_encoding(::Encoding::UTF_8)
       string
     rescue Iconv::Failure => e
       raise GeneratorError, "Caught #{e.class}: #{e}"
@@ -351,13 +351,13 @@ module JSON
           def to_json(state = nil, *)
             case
             when infinite?
-              if !state || state.allow_nan?
+              if state && state.allow_nan?
                 to_s
               else
                 raise GeneratorError, "#{self} not allowed in JSON"
               end
             when nan?
-              if !state || state.allow_nan?
+              if state && state.allow_nan?
                 to_s
               else
                 raise GeneratorError, "#{self} not allowed in JSON"
@@ -369,11 +369,25 @@ module JSON
         end
 
         module String
-          # This string should be encoded with UTF-8 A call to this method
-          # returns a JSON string encoded with UTF16 big endian characters as
-          # \u????.
-          def to_json(*)
-            '"' << JSON.utf8_to_json(self) << '"'
+          if defined?(::Encoding)
+            # This string should be encoded with UTF-8 A call to this method
+            # returns a JSON string encoded with UTF16 big endian characters as
+            # \u????.
+            def to_json(*)
+              if encoding == ::Encoding::UTF_8
+                '"' << JSON.utf8_to_json(self) << '"'
+              else
+                string = encode(::Encoding::UTF_8)
+                '"' << JSON.utf8_to_json(string) << '"'
+              end
+            end
+          else
+            # This string should be encoded with UTF-8 A call to this method
+            # returns a JSON string encoded with UTF16 big endian characters as
+            # \u????.
+            def to_json(*)
+              '"' << JSON.utf8_to_json(self) << '"'
+            end
           end
 
           # Module that holds the extinding methods if, the String module is

data/lib/json/pure/parser.rb

@@ -66,7 +66,41 @@ module JSON
       # * *object_class*: Defaults to Hash
       # * *array_class*: Defaults to Array
       def initialize(source, opts = {})
-        super
+          if defined?(::Encoding)
+            if source.encoding == Encoding::ASCII_8BIT
+              b = source[0, 4].bytes.to_a
+              source = case
+                       when b.size >= 4 && b[0] == 0 && b[1] == 0 && b[2] == 0
+                         source.dup.force_encoding(Encoding::UTF_32BE).encode!(Encoding::UTF_8)
+                       when b.size >= 4 && b[0] == 0 && b[2] == 0
+                         source.dup.force_encoding(Encoding::UTF_16BE).encode!(Encoding::UTF_8)
+                       when b.size >= 4 && b[1] == 0 && b[2] == 0 && b[3] == 0
+                         source.dup.force_encoding(Encoding::UTF_32LE).encode!(Encoding::UTF_8)
+                       when b.size >= 4 && b[1] == 0 && b[3] == 0
+                         source.dup.force_encoding(Encoding::UTF_16LE).encode!(Encoding::UTF_8)
+                       else
+                         source.dup
+                       end
+            else
+              source = source.encode(Encoding::UTF_8)
+            end
+            source.force_encoding(Encoding::ASCII_8BIT)
+          else
+            b = source
+            source = case
+                     when b.size >= 4 && b[0] == 0 && b[1] == 0 && b[2] == 0
+                       JSON.iconv('utf-8', 'utf-32be', b)
+                     when b.size >= 4 && b[0] == 0 && b[2] == 0
+                       JSON.iconv('utf-8', 'utf-16be', b)
+                     when b.size >= 4 && b[1] == 0 && b[2] == 0 && b[3] == 0
+                       JSON.iconv('utf-8', 'utf-32le', b)
+                     when b.size >= 4 && b[1] == 0 && b[3] == 0
+                       JSON.iconv('utf-8', 'utf-16le', b)
+                     else
+                       b
+                     end
+          end
+        super source
         if !opts.key?(:max_nesting) # defaults to 19
           @max_nesting = 19
         elsif opts[:max_nesting]

data/lib/json/version.rb

@@ -1,6 +1,6 @@
 module JSON
   # JSON version
-  VERSION         = '1.1.9'
+  VERSION         = '1.2.0'
   VERSION_ARRAY   = VERSION.split(/\./).map { |x| x.to_i } # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/tests/test_json.rb

@@ -9,6 +9,20 @@ else             require 'json'
 end
 require 'stringio'
 
+unless Array.method_defined?(:permutation)
+  begin
+    require 'enumerator'
+    require 'permutation'
+    class Array
+      def permutation
+        Permutation.for(self).to_enum.map { |x| x.project }
+      end
+    end
+  rescue LoadError
+    warn "Skipping permutation tests."
+  end
+end
+
 class TC_JSON < Test::Unit::TestCase
   include JSON
 
@@ -94,30 +108,24 @@ class TC_JSON < Test::Unit::TestCase
     assert_equal({ "a" => 0.23 }, parse('  {  "a"  :  0.23  }  '))
   end
 
-  begin
-    require 'permutation'
+  if Array.method_defined?(:permutation)
     def test_parse_more_complex_arrays
       a = [ nil, false, true, "foßbar", [ "n€st€d", true ], { "nested" => true, "n€ßt€ð2" => {} }]
-      perms = Permutation.for a
-      perms.each do |perm|
-        orig_ary = perm.project
-        json = pretty_generate(orig_ary)
-        assert_equal orig_ary, parse(json)
+      a.permutation.each do |perm|
+        json = pretty_generate(perm)
+        assert_equal perm, parse(json)
       end
     end
 
     def test_parse_complex_objects
       a = [ nil, false, true, "foßbar", [ "n€st€d", true ], { "nested" => true, "n€ßt€ð2" => {} }]
-      perms = Permutation.for a
-      perms.each do |perm|
+      a.permutation.each do |perm|
         s = "a"
-        orig_obj = perm.project.inject({}) { |h, x| h[s.dup] = x; s = s.succ; h }
+        orig_obj = perm.inject({}) { |h, x| h[s.dup] = x; s = s.succ; h }
         json = pretty_generate(orig_obj)
         assert_equal orig_obj, parse(json)
       end
     end
-  rescue LoadError
-    warn "Skipping permutation tests."
   end
 
   def test_parse_arrays
@@ -222,27 +230,27 @@ EOT
   def test_backslash
     data = [ '\\.(?i:gif|jpe?g|png)$' ]
     json = '["\\\\.(?i:gif|jpe?g|png)$"]'
-    assert_equal json, JSON.unparse(data)
+    assert_equal json, JSON.generate(data)
     assert_equal data, JSON.parse(json)
     #
     data = [ '\\"' ]
     json = '["\\\\\""]'
-    assert_equal json, JSON.unparse(data)
+    assert_equal json, JSON.generate(data)
     assert_equal data, JSON.parse(json)
     #
     json = '["/"]'
     data = JSON.parse(json)
     assert_equal ['/'], data
-    assert_equal json, JSON.unparse(data)
+    assert_equal json, JSON.generate(data)
     #
     json = '["\""]'
     data = JSON.parse(json)
     assert_equal ['"'], data
-    assert_equal json, JSON.unparse(data)
+    assert_equal json, JSON.generate(data)
     json = '["\\\'"]'
     data = JSON.parse(json)
     assert_equal ["'"], data
-    assert_equal '["\'"]', JSON.unparse(data)
+    assert_equal '["\'"]', JSON.generate(data)
   end
 
   def test_wrong_inputs