oj 0.7.0 → 0.8.0

data/README.md

@@ -22,13 +22,9 @@ A fast JSON parser and Object marshaller as a Ruby gem.
 
 ## <a name="release">Release Notes</a>
 
-### Release 0.7.0
+### Release 0.8.0
 
-- changed the object JSON format
-
-- serialized Ruby Objects can now be deserialized
-
-- improved performance testing
+- Auto creation of data classes when unmarshalling Objects if the Class is not defined
 
 ## <a name="description">Description</a>
 
@@ -57,11 +53,13 @@ to_hash() is more flexible and produces more consistent output so it has a
 preference over the to_json() method. If neither the to_json() or to_hash()
 methods exist then the Oj internal Object variable encoding is used.
 
-Coming soon: As an Object marshaller with support for circular references.
+Oj is compatible with Ruby 1.8.7, 1.9.2, 1.9.3, JRuby, and RBX.
 
-Coming soon: A JSON stream parser.
+## <a name="plans">Planned Releases</a>
 
-Oj is compatible with Ruby 1.8.7, 1.9.2, 1.9.3, JRuby, and RBX.
+- Release 0.9: Support for circular references.
+
+- Release 1.0: A JSON stream parser.
 
 ## <a name="compare">Comparisons</a>
 
@@ -81,6 +79,9 @@ Object encoding and without Bignums.
 None of the packages except Oj were able to serialize Ruby Objects that did
 not have a to_json() method or were of the 7 native JSON types.
 
+A perf_obj.rb file was added for comparing different Object marshalling
+packages.
+
 It is also worth noting that although Oj is slightly behind MessagePack for
 parsing, Oj serialization is much faster than MessagePack even though Oj uses
 human readable JSON vs the binary MessagePack format.
@@ -227,3 +228,50 @@ without Objects or numbers (for JSON Pure) JSON:
     h2 = Oj.parse(json)
     puts "Same? #{h == h2}"
     # true
+
+### Object JSON format:
+
+In :object mode Oj generates JSON that follows conventions which allow Class
+and other information such as Object IDs for circular reference detection. The
+formating follows the following rules.
+
+1. JSON native types, true, false, nil, String, Hash, Array, and Number are
+encoded normally.
+
+2. If a Hash uses Symbols as keys those keys appear as Strings with a leading
+':' character.
+
+3. The '^' character denotes a special key value when in a JSON Object sequence.
+
+4. If a String begins with a ':' character such as ':abc' it is encoded as {"^s":":abc"}.
+
+5. If a Symbol begins with a ':' character such as :":abc" is is encoded as {"^m":":abc"}.
+
+6. A "^c" JSON Object key indicates the value should be converted to a Ruby
+class. The sequence {"^c":"Oj::Bag"} is read as the Oj::Bag class.
+
+7. A "^t" JSON Object key indicates the value should be converted to a Ruby
+Time. The sequence {"^t":1325775487.000000} is read as Jan 5, 2012 at
+23:58:07.
+
+8. A "^o" JSON Object key indicates the value should be converted to a Ruby
+Object. The first entry in the JSON Object must be a class with the "^o"
+key. After that each entry is treated as a variable of the Object where the
+key is the variable name without the preceeding '@'. An example is
+{"^o":"Oj::Bag","x":58,"y":"marbles"}.
+
+9. When encoding an Object, if the variable name does not begin with an '@'
+character then the name preceeded by a '~' character. This occurs in the
+Exception class. An example is {"^o":"StandardError","~mesg":"A
+Message","~bt":[".\/tests.rb:345:in `test_exception'"]}
+
+10. If a Hash entry has a key that is not a String or Symbol then the entry is
+encoded with a key of the form "^#n" where n is a hex number. The value that
+is an Array where the first element is the key in the Hash and the second is
+the value. An example is {"^#3":[2,5]}.
+
+11. A "^i" JSON entry in either an Object or Array is the ID of the Ruby
+Object being encoded. It is used when the :circular flag is set. It can appear
+in either a JSON Object or in a JSON Array. If alone it represented a link to
+the original Hash or JSON. If an added attribute it is the ID of the original
+Object or Array. Examples are TBD.

data/ext/oj/dump.c

@@ -83,7 +83,8 @@ static void	dump_bignum(VALUE obj, Out out);
 static void	dump_float(VALUE obj, Out out);
 static void	dump_cstr(const char *str, size_t cnt, int is_sym, Out out);
 static void	dump_hex(u_char c, Out out);
-static void	dump_str(VALUE obj, Out out);
+static void	dump_str_comp(VALUE obj, Out out);
+static void	dump_str_obj(VALUE obj, Out out);
 static void	dump_sym_comp(VALUE obj, Out out);
 static void	dump_sym_obj(VALUE obj, Out out);
 static void	dump_class_comp(VALUE obj, Out out);
@@ -350,10 +351,33 @@ dump_cstr(const char *str, size_t cnt, int is_sym, Out out) {
 }
 
 static void
-dump_str(VALUE obj, Out out) {
+dump_str_comp(VALUE obj, Out out) {
     dump_cstr(StringValuePtr(obj), RSTRING_LEN(obj), 0, out);
 }
 
+static void
+dump_str_obj(VALUE obj, Out out) {
+    const char	*s = StringValuePtr(obj);
+    size_t	len = RSTRING_LEN(obj);
+    
+    if (':' == *s) {
+	if (out->end - out->cur <= 6) {
+	    grow(out, 6);
+	}
+	*out->cur++ = '{';
+	*out->cur++ = '"';
+	*out->cur++ = '^';
+	*out->cur++ = 's';
+	*out->cur++ = '"';
+	*out->cur++ = ':';
+	dump_cstr(s, len, 0, out);
+	*out->cur++ = '}';
+	*out->cur = '\0';
+    } else {
+	dump_cstr(s, len, 0, out);
+    }
+}
+
 static void
 dump_sym_comp(VALUE obj, Out out) {
     const char	*sym = rb_id2name(SYM2ID(obj));
@@ -373,7 +397,7 @@ dump_sym_obj(VALUE obj, Out out) {
 	*out->cur++ = '{';
 	*out->cur++ = '"';
 	*out->cur++ = '^';
-	*out->cur++ = 's';
+	*out->cur++ = 'm';
 	*out->cur++ = '"';
 	*out->cur++ = ':';
 	dump_cstr(sym, len, 0, out);
@@ -456,7 +480,7 @@ hash_cb_strict(VALUE key, VALUE value, Out out) {
     }
     fill_indent(out, depth);
     if (rb_type(key) == T_STRING) {
-	dump_str(key, out);
+	dump_str_comp(key, out);
 	*out->cur++ = ':';
 	dump_val(value, depth, out);
     } else {
@@ -479,7 +503,7 @@ hash_cb_object(VALUE key, VALUE value, Out out) {
     fill_indent(out, depth);
     // TBD if key is a string else dump with unique key for and entry array
     if (rb_type(key) == T_STRING) {
-	dump_str(key, out);
+	dump_str_obj(key, out);
 	*out->cur++ = ':';
 	dump_val(value, depth, out);
     } else if (rb_type(key) == T_SYMBOL) {
@@ -778,7 +802,15 @@ dump_val(VALUE obj, int depth, Out out) {
     case T_FIXNUM:	dump_fixnum(obj, out);		break;
     case T_FLOAT:	dump_float(obj, out);		break;
     case T_BIGNUM:	dump_bignum(obj, out);		break;
-    case T_STRING:	dump_str(obj, out);		break;
+    case T_STRING:
+	switch (out->opts->mode) {
+	case StrictMode:
+	case NullMode:
+	case CompatMode:	dump_str_comp(obj, out);	break;
+	case ObjectMode:
+	default:		dump_str_obj(obj, out);		break;
+	}
+	break;
     case T_SYMBOL:
 	switch (out->opts->mode) {
 	case StrictMode:	raise_strict(obj);		break;

data/ext/oj/load.c

@@ -112,15 +112,14 @@ next_white(ParseInfo pi) {
 }
 
 inline static VALUE
-resolve_classname(VALUE mod, const char *class_name, int create) {
+resolve_classname(VALUE mod, const char *class_name, int auto_define) {
     VALUE       clas;
     ID          ci = rb_intern(class_name);
 
-    if (rb_const_defined_at(mod, ci) || !create) {
+    if (rb_const_defined_at(mod, ci) || !auto_define) {
 	clas = rb_const_get_at(mod, ci);
     } else {
-	//clas = rb_define_class_under(mod, class_name, oj_bag_clas);
-	clas = rb_const_get_at(mod, ci); // TBD temp
+	clas = rb_define_class_under(mod, class_name, oj_bag_class);
     }
     return clas;
 }
@@ -139,10 +138,8 @@ classname2obj(const char *name, ParseInfo pi) {
 static VALUE
 classname2class(const char *name, ParseInfo pi) {
     VALUE       clas;
-    int		create = 0; // TBD from options
-            
-#if 1
     VALUE       *slot;
+    int		auto_define = (Yes == pi->options->auto_define);
 
     if (Qundef == (clas = oj_cache_get(oj_class_cache, name, &slot))) {
         char            class_name[1024];
@@ -154,7 +151,10 @@ classname2class(const char *name, ParseInfo pi) {
             if (':' == *n) {
                 *s = '\0';
                 n++;
-                if (Qundef == (clas = resolve_classname(clas, class_name, create))) {
+		if (':' != *n) {
+                    raise_error("Invalid classname, expected another ':'", pi->str, pi->s);
+		}
+                if (Qundef == (clas = resolve_classname(clas, class_name, auto_define))) {
                     return Qundef;
                 }
                 s = class_name;
@@ -163,31 +163,10 @@ classname2class(const char *name, ParseInfo pi) {
             }
         }
         *s = '\0';
-        if (Qundef != (clas = resolve_classname(clas, class_name, create))) {
+        if (Qundef != (clas = resolve_classname(clas, class_name, auto_define))) {
             *slot = clas;
         }
     }
-#else
-    char            class_name[1024];
-    char            *s;
-    const char      *n = name;
-
-    clas = rb_cObject;
-    for (s = class_name; '\0' != *n; n++) {
-	if (':' == *n) {
-	    *s = '\0';
-	    n++;
-	    if (Qundef == (clas = resolve_classname(clas, class_name, create))) {
-		return Qundef;
-	    }
-	    s = class_name;
-	} else {
-	    *s++ = *n;
-	}
-    }
-    *s = '\0';
-    clas = resolve_classname(clas, class_name, create);
-#endif
     return clas;
 }
 
@@ -314,7 +293,11 @@ read_obj(ParseInfo pi) {
 		    obj = read_next(pi, T_CLASS);
 		    key = Qundef;
 		    break;
-		case 's': // Symbol
+		case 's': // String
+		    obj = read_next(pi, T_STRING);
+		    key = Qundef;
+		    break;
+		case 'm': // Symbol
 		    obj = read_next(pi, T_SYMBOL);
 		    key = Qundef;
 		    break;

data/ext/oj/oj.c

@@ -54,8 +54,10 @@ ID	oj_tv_nsec_id;
 ID	oj_tv_sec_id;
 ID	oj_tv_usec_id;
 
+VALUE	oj_bag_class;
 VALUE	oj_time_class;
 
+static VALUE	auto_define_sym;
 static VALUE	circular_sym;
 static VALUE	compat_sym;
 static VALUE	encoding_sym;
@@ -72,6 +74,7 @@ static struct _Options  default_options = {
     { '\0' },		// encoding
     0,			// indent
     No,			// circular
+    Yes,		// auto_define
     ObjectMode,		// mode
 };
 
@@ -81,6 +84,7 @@ static struct _Options  default_options = {
  * - indent: [Fixnum] number of spaces to indent each element in an XML document
  * - encoding: [String] character encoding for the JSON file
  * - circular: [true|false|nil] support circular references while dumping
+ * - auto_define: [true|false|nil] automatically define classes if they do not exist
  * - mode: [:object|:strict|:compat|:null] load and dump modes to use for JSON
  * @return [Hash] all current option settings.
  */
@@ -92,6 +96,7 @@ get_def_opts(VALUE self) {
     rb_hash_aset(opts, encoding_sym, (0 == elen) ? Qnil : rb_str_new(default_options.encoding, elen));
     rb_hash_aset(opts, indent_sym, INT2FIX(default_options.indent));
     rb_hash_aset(opts, circular_sym, (Yes == default_options.circular) ? Qtrue : ((No == default_options.circular) ? Qfalse : Qnil));
+    rb_hash_aset(opts, auto_define_sym, (Yes == default_options.auto_define) ? Qtrue : ((No == default_options.auto_define) ? Qfalse : Qnil));
     switch (default_options.mode) {
     case StrictMode:	rb_hash_aset(opts, mode_sym, strict_sym);	break;
     case CompatMode:	rb_hash_aset(opts, mode_sym, compat_sym);	break;
@@ -109,7 +114,8 @@ get_def_opts(VALUE self) {
  * @param [Fixnum] :indent number of spaces to indent each element in an XML document
  * @param [String] :encoding character encoding for the JSON file
  * @param [true|false|nil] :circular support circular references while dumping
- * @parsm [:object|:strict|:compat|:null] load and dump mode to use for JSON
+ * @param [true|false|nil] :auto_define automatically define classes if they do not exist
+ * @param [:object|:strict|:compat|:null] load and dump mode to use for JSON
  *        :strict raises an exception when a non-supported Object is
  *        encountered. :compat attempts to extract variable values from an
  *        Object using to_json() or to_hash() then it walks the Object's
@@ -122,6 +128,7 @@ static VALUE
 set_def_opts(VALUE self, VALUE opts) {
     struct _YesNoOpt    ynos[] = {
         { circular_sym, &default_options.circular },
+        { auto_define_sym, &default_options.auto_define },
         { Qnil, 0 }
     };
     YesNoOpt    o;
@@ -366,8 +373,10 @@ void Init_oj() {
     oj_tv_sec_id = rb_intern("tv_sec");
     oj_tv_usec_id = rb_intern("tv_usec");
     
+    oj_bag_class = rb_const_get_at(Oj, rb_intern("Bag"));
     oj_time_class = rb_const_get(rb_cObject, rb_intern("Time"));
 
+    auto_define_sym = ID2SYM(rb_intern("auto_define"));	rb_ary_push(keep, auto_define_sym);
     circular_sym = ID2SYM(rb_intern("circular"));	rb_ary_push(keep, circular_sym);
     compat_sym = ID2SYM(rb_intern("compat"));		rb_ary_push(keep, compat_sym);
     encoding_sym = ID2SYM(rb_intern("encoding"));	rb_ary_push(keep, encoding_sym);

data/ext/oj/oj.h

@@ -78,10 +78,11 @@ typedef enum {
 } Mode;
 
 typedef struct _Options {
-    char        encoding[64];   // encoding, stored in the option to avoid GC invalidation in default values
-    int         indent;         // indention for dump, default 2
-    char        circular;       // YesNo
-    char        mode;           // Mode
+    char        encoding[64];	// encoding, stored in the option to avoid GC invalidation in default values
+    int         indent;		// indention for dump, default 2
+    char        circular;	// YesNo
+    char        auto_define;	// YesNo
+    char        mode;		// Mode
 } *Options;
 
 extern VALUE	oj_parse(char *json, Options options);
@@ -92,6 +93,7 @@ extern void	_oj_raise_error(const char *msg, const char *xml, const char *curren
 
 extern VALUE    Oj;
 
+extern VALUE	oj_bag_class;
 extern VALUE	oj_time_class;
 
 extern ID	oj_at_id;

data/lib/oj.rb

@@ -31,6 +31,7 @@ module Oj
   @@keep = []
 end
 
-#require 'ox/version'
+require 'oj/version'
+require 'oj/bag'
 
 require 'oj/oj' # C extension

data/lib/oj/bag.rb

@@ -0,0 +1,92 @@
+
+module Oj
+
+  # A generic class that is used only for storing attributes. It is the base
+  # Class for auto-generated classes in the storage system. Instance variables
+  # are added using the instance_variable_set() method. All instance variables
+  # can be accessed using the variable name (without the @ prefix). No setters
+  # are provided as the Class is intended for reading only.
+  class Bag
+
+    # The initializer can take multiple arguments in the form of key values
+    # where the key is the variable name and the value is the variable
+    # value. This is intended for testing purposes only.
+    # @example Oj::Bag.new(:@x => 42, :@y => 57)
+    # @param [Hash] args instance variable symbols and their values
+    def initialize(args={ })
+      args.each do |k,v|
+        self.instance_variable_set(k, v)
+      end
+    end
+
+    # Replaces the Object.respond_to?() method.
+    # @param [Symbol] m method symbol
+    # @return [Boolean] true for any method that matches an instance
+    #                   variable reader, otherwise false.
+    def respond_to?(m)
+      return true if super
+      at_m = ('@' + m.to_s).to_sym
+      instance_variables.include?(at_m)
+    end
+
+    # Handles requests for variable values. Others cause an Exception to be
+    # raised.
+    # @param [Symbol] m method symbol
+    # @return [Boolean] the value of the specified instance variable.
+    # @raise [ArgumentError] if an argument is given. Zero arguments expected.
+    # @raise [NoMethodError] if the instance variable is not defined.
+    def method_missing(m, *args, &block)
+      raise ArgumentError.new("wrong number of arguments (#{args.size} for 0) to method #{m}") unless args.nil? or args.empty?
+      at_m = ('@' + m.to_s).to_sym
+      raise NoMethodError.new("undefined method #{m}", m) unless instance_variable_defined?(at_m)
+      instance_variable_get(at_m)
+    end
+
+    # Replaces eql?() with something more reasonable for this Class.
+    # @param [Object] other Object to compare self to
+    # @return [Boolean] true if each variable and value are the same, otherwise false.
+    def eql?(other)
+      return false if (other.nil? or self.class != other.class)
+      ova = other.instance_variables
+      iv = instance_variables
+      return false if ova.size != iv.size
+      iv.each do |vid|
+        return false if instance_variable_get(vid) != other.instance_variable_get(vid)
+      end
+      true
+    end
+    alias == eql?
+    
+    # Define a new class based on the Oj::Bag class. This is used internally in
+    # the Oj module and is available to service wrappers that receive XML
+    # requests that include Objects of Classes not defined in the storage
+    # process.
+    # @param [String] classname Class name or symbol that includes Module names.
+    # @return [Object] an instance of the specified Class.
+    # @raise [NameError] if the classname is invalid.
+    def self.define_class(classname)
+      classname = classname.to_s unless classname.is_a?(String)
+      tokens = classname.split('::').map { |n| n.to_sym }
+      raise NameError.new("Invalid classname '#{classname}") if tokens.empty?
+      m = Object
+      tokens[0..-2].each do |sym|
+        if m.const_defined?(sym)
+          m = m.const_get(sym)
+        else
+          c = Module.new
+          m.const_set(sym, c)
+          m = c
+        end
+      end
+      sym = tokens[-1]
+      if m.const_defined?(sym)
+        c = m.const_get(sym)
+      else
+        c = Class.new(Oj::Bag)
+        m.const_set(sym, c)
+      end
+      c
+    end
+
+  end # Bag
+end # Oj

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/test/tests.rb

@@ -50,6 +50,7 @@ class Juice < ::Test::Unit::TestCase
                    :encoding=>nil,
                    :indent=>0,
                    :circular=>false,
+                   :auto_define=>true,
                    :mode=>:object})
   end
 
@@ -58,11 +59,13 @@ class Juice < ::Test::Unit::TestCase
       :encoding=>nil,
       :indent=>0,
       :circular=>false,
+      :auto_define=>true,
       :mode=>:object}
     o2 = {
       :encoding=>"UTF-8",
       :indent=>4,
       :circular=>true,
+      :auto_define=>false,
       :mode=>:compat}
     o3 = { :indent => 4 }
     Oj.default_options = o2
@@ -109,6 +112,11 @@ class Juice < ::Test::Unit::TestCase
     dump_and_load("a\u0041", false)
   end
 
+  def test_string_object
+    dump_and_load('abc', false)
+    dump_and_load(':abc', false)
+  end
+
   def test_encode
     Oj.default_options = { :encoding => 'UTF-8' }
     dump_and_load("ぴーたー", false)
@@ -348,6 +356,17 @@ class Juice < ::Test::Unit::TestCase
     assert_equal(e, e2);
   end
 
+  def test_bag
+    json = %{{
+  "^o":"Jem",
+  "x":true,
+  "y":58 }}
+    obj = Oj.load(json, :mode => :object)
+    assert_equal('Jem', obj.class.name)
+    assert_equal(true, obj.x)
+    assert_equal(58, obj.y)
+  end
+
   def dump_and_load(obj, trace=false)
     json = Oj.dump(obj, :indent => 2)
     puts json if trace

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-26 00:00:00.000000000 Z
+date: 2012-02-27 00:00:00.000000000 Z
 dependencies: []
 description: ! 'The fastest JSON parser and object serializer. '
 email: peter@ohler.com
@@ -19,6 +19,7 @@ extensions:
 extra_rdoc_files:
 - README.md
 files:
+- lib/oj/bag.rb
 - lib/oj/version.rb
 - lib/oj.rb
 - ext/oj/extconf.rb