oj 3.1.4 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 168a9b04bcd9b3af98e7d7d8b04058391a900972
-  data.tar.gz: 44c13fcd7056ba300761012eb679817c1b0171a4
+  metadata.gz: af2435c13d3e1e32934f3743b0de0333093ec0bd
+  data.tar.gz: 05013dc900f2839c67a5d8aea480845686ac2bf9
 SHA512:
-  metadata.gz: 8efc86da94738b1c30eba69fe98d5b83151ed3a6c41d8d9ef5730160f5baf79ee2f4a2a063673f8da051832a8f472bba765d4f6722f4b3048c4986f895888c89
-  data.tar.gz: e9ccc9ac57b6fc3e772a7399b89c7aff64b1fefeef280a9a43125409e56c58b3a19b8aabbfa1973a1c2e6e51d6d2ba6cf8ebda6ca3034491760235f4d5b382b8
+  metadata.gz: 49af58a9f5747e506f972ca8a6a85d9463ae915194f230f2b64edb74f627c5d20f50ba38b9a07c5a9e56f774c3c5c1a58e45ec979151a35ac72e96cb0a77d2d6
+  data.tar.gz: 6c0fb738c3dd93610034550cb2ef123c75e5bb9cc2cf27f5effd7f97197f5747eac7dcce7542a5d1b0c9e6b7de8b25b9a6b914e77d710ebaf6514e98d3bb7a11

data/ext/oj/dump.c

@@ -552,7 +552,7 @@ oj_dump_obj_to_json_using_params(VALUE obj, Options copts, Out out, int argc, VA
     if (0 == out->buf) {
 	out->buf = ALLOC_N(char, 4096);
 	out->end = out->buf + 4095 - BUFFER_EXTRA; // 1 less than end plus extra for possible errors
-	out->allocated = 1;
+	out->allocated = true;
     }
     out->cur = out->buf;
     out->circ_cnt = 0;
@@ -600,7 +600,7 @@ oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
 
     out.buf = buf;
     out.end = buf + sizeof(buf) - BUFFER_EXTRA;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts->dump_opts.omit_nil;
     oj_dump_obj_to_json(obj, copts, &out);
     size = out.cur - out.buf;
@@ -635,7 +635,7 @@ oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
 
     out.buf = buf;
     out.end = buf + sizeof(buf) - BUFFER_EXTRA;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts->dump_opts.omit_nil;
     oj_dump_obj_to_json(obj, copts, &out);
     size = out.cur - out.buf;
@@ -873,7 +873,7 @@ oj_grow_out(Out out, size_t len) {
 	buf = REALLOC_N(out->buf, char, (size + BUFFER_EXTRA));
     } else {
 	buf = ALLOC_N(char, (size + BUFFER_EXTRA));
-	out->allocated = 1;
+	out->allocated = true;
 	memcpy(buf, out->buf, out->end - out->buf + BUFFER_EXTRA);
     }
     if (0 == buf) {

data/ext/oj/dump_leaf.c

@@ -24,7 +24,7 @@ grow(Out out, size_t len) {
 	buf = REALLOC_N(out->buf, char, (size + BUFFER_EXTRA));
     } else {
 	buf = ALLOC_N(char, (size + BUFFER_EXTRA));
-	out->allocated = 1;
+	out->allocated = true;
 	memcpy(buf, out->buf, out->end - out->buf + BUFFER_EXTRA);
     }
     if (0 == buf) {
@@ -214,7 +214,7 @@ oj_dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
     if (0 == out->buf) {
 	out->buf = ALLOC_N(char, 4096);
 	out->end = out->buf + 4095 - BUFFER_EXTRA; // 1 less than end plus extra for possible errors
-	out->allocated = 1;
+	out->allocated = true;
     }
     out->cur = out->buf;
     out->circ_cnt = 0;
@@ -233,7 +233,7 @@ oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
 
     out.buf = buf;
     out.end = buf + sizeof(buf) - BUFFER_EXTRA;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts->dump_opts.omit_nil;
     oj_dump_leaf_to_json(leaf, copts, &out);
     size = out.cur - out.buf;

data/ext/oj/fast.c

@@ -92,7 +92,7 @@ static char*	read_quoted_value(ParseInfo pi);
 static void	skip_comment(ParseInfo pi);
 
 static VALUE	protect_open_proc(VALUE x);
-static VALUE	parse_json(VALUE clas, char *json, int given, int allocated);
+static VALUE	parse_json(VALUE clas, char *json, bool given, bool allocated);
 static void	each_leaf(Doc doc, VALUE self);
 static int	move_step(Doc doc, const char *path, int loc);
 static Leaf	get_doc_leaf(Doc doc, const char *path);
@@ -779,7 +779,7 @@ free_doc_cb(void *x) {
 }
 
 static VALUE
-parse_json(VALUE clas, char *json, int given, int allocated) {
+parse_json(VALUE clas, char *json, bool given, bool allocated) {
     struct _ParseInfo	pi;
     VALUE		result = Qnil;
     Doc			doc;
@@ -1594,7 +1594,7 @@ doc_dump(int argc, VALUE *argv, VALUE self) {
 
 	    out.buf = buf;
 	    out.end = buf + sizeof(buf) - 10;
-	    out.allocated = 0;
+	    out.allocated = false;
 	    out.omit_nil = oj_default_options.dump_opts.omit_nil;
 	    oj_dump_leaf_to_json(leaf, &oj_default_options, &out);
 	    rjson = rb_str_new2(out.buf);

data/ext/oj/mimic_json.c

@@ -198,7 +198,7 @@ mimic_dump(int argc, VALUE *argv, VALUE self) {
     copts.str_rx.tail = NULL;
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.caller = CALLER_DUMP;
     copts.escape_mode = JXEsc;
     copts.mode = CompatMode;
@@ -357,7 +357,7 @@ mimic_generate_core(int argc, VALUE *argv, Options copts) {
 
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts->dump_opts.omit_nil;
     out.caller = CALLER_GENERATE;
     // For obj.to_json or generate nan is not allowed but if called from dump
@@ -720,7 +720,7 @@ mimic_object_to_json(int argc, VALUE *argv, VALUE self) {
     copts.str_rx.tail = NULL;
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts.dump_opts.omit_nil;
     copts.mode = CompatMode;
     copts.to_json = No;

data/ext/oj/oj.c

@@ -957,7 +957,7 @@ dump(int argc, VALUE *argv, VALUE self) {
     }
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts.dump_opts.omit_nil;
     out.caller = CALLER_DUMP;
     oj_dump_obj_to_json_using_params(*argv, &copts, &out, argc - 1,argv + 1);
@@ -1009,7 +1009,7 @@ to_json(int argc, VALUE *argv, VALUE self) {
     copts.to_json = Yes;
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts.dump_opts.omit_nil;
     // For obj.to_json or generate nan is not allowed but if called from dump
     // it is.

data/ext/oj/oj.h

@@ -188,7 +188,7 @@ typedef struct _Out {
     int			depth; // used by dump_hash
     Options		opts;
     uint32_t		hash_cnt;
-    int			allocated;
+    bool		allocated;
     bool		omit_nil;
     int			argc;
     VALUE		*argv;
@@ -203,6 +203,7 @@ typedef struct _StrWriter {
     char		*types;	// DumpType
     char		*types_end;
     int			keyWritten;
+    
 } *StrWriter;
 
 typedef struct _StreamWriter {
@@ -210,6 +211,7 @@ typedef struct _StreamWriter {
     StreamWriterType	type;
     VALUE		stream;
     int			fd;
+    int			flush_limit; // indicator of when to flush
 } *StreamWriter;
 
 enum {
@@ -269,7 +271,7 @@ extern void	oj_str_writer_pop_all(StrWriter sw);
 extern void	oj_init_doc(void);
 extern void	oj_string_writer_init();
 extern void	oj_stream_writer_init();
-extern void	oj_str_writer_init(StrWriter sw);
+extern void	oj_str_writer_init(StrWriter sw, int buf_size);
 extern VALUE	oj_define_mimic_json(int argc, VALUE *argv, VALUE self);
 extern VALUE	oj_mimic_generate(int argc, VALUE *argv, VALUE self);
 extern VALUE	oj_mimic_pretty_generate(int argc, VALUE *argv, VALUE self);

data/ext/oj/rails.c

@@ -658,7 +658,7 @@ encode(VALUE obj, ROptTable ropts, Options opts, int argc, VALUE *argv) {
     }
     out.buf = buf;
     out.end = buf + sizeof(buf) - 10;
-    out.allocated = 0;
+    out.allocated = false;
     out.omit_nil = copts.dump_opts.omit_nil;
     out.caller = 0;
     out.cur = out.buf;

data/ext/oj/stream_writer.c

@@ -24,6 +24,12 @@ stream_writer_free(void *ptr) {
     xfree(ptr);
 }
 
+static void
+stream_writer_reset_buf(StreamWriter sw) {
+    sw->sw.out.cur = sw->sw.out.buf;
+    *sw->sw.out.cur = '\0';
+}
+
 static void
 stream_writer_write(StreamWriter sw) {
     ssize_t	size = sw->sw.out.cur - sw->sw.out.buf;
@@ -43,18 +49,24 @@ stream_writer_write(StreamWriter sw) {
     default:
 	rb_raise(rb_eArgError, "expected an IO Object.");
     }
+    stream_writer_reset_buf(sw);
 }
 
-static void
-stream_writer_reset_buf(StreamWriter sw) {
-    sw->sw.out.cur = sw->sw.out.buf;
-    *sw->sw.out.cur = '\0';
-}
+static VALUE	buffer_size_sym = Qundef;
 
 /* Document-method: new
  * call-seq: new(io, options)
  *
- * Creates a new StreamWriter.
+ * Creates a new StreamWriter. Options are supported according the the
+ * specified mode or the mode in the default options. Note that if mimic_JSON
+ * or Oj.optimize_rails has not been called then the behavior of the modes may
+ * not be the same as if they were.
+ *
+ * In addition to the regular dump options for the various modes a
+ * _:buffer_size_ option is available. It should be set to a positive
+ * integer. It is considered a hint of how large the initial internal buffer
+ * should be and also a hint on when to flush.
+ *
  * - *io* [_IO_] stream to write to
  * - *options* [_Hash_] formating options
  */
@@ -83,9 +95,32 @@ stream_writer_new(int argc, VALUE *argv, VALUE self) {
 	rb_raise(rb_eArgError, "expected an IO Object.");
     }
     sw = ALLOC(struct _StreamWriter);
-    oj_str_writer_init(&sw->sw);
-    if (2 == argc) {
+    if (2 == argc && T_HASH == rb_type(argv[1])) {
+	volatile VALUE	v;
+	int		buf_size = 0;
+
+	if (Qundef == buffer_size_sym) {
+	    buffer_size_sym = ID2SYM(rb_intern("buffer_size"));	rb_gc_register_address(&buffer_size_sym);
+	    
+	}
+	if (Qnil != (v = rb_hash_lookup(argv[1], buffer_size_sym))) {
+#ifdef RUBY_INTEGER_UNIFICATION
+	    if (rb_cInteger != rb_obj_class(v)) {
+		rb_raise(rb_eArgError, ":buffer size must be a Integer.");
+	    }
+#else
+	    if (T_FIXNUM != rb_type(v)) {
+		rb_raise(rb_eArgError, ":buffer size must be a Integer.");
+	    }
+#endif
+	    buf_size = FIX2INT(v);
+	}
+	oj_str_writer_init(&sw->sw, buf_size);
 	oj_parse_options(argv[1], &sw->sw.opts);
+	sw->flush_limit = buf_size;
+    } else {
+	oj_str_writer_init(&sw->sw, 4096);
+	sw->flush_limit = 0;
     }
     sw->sw.out.indent = sw->sw.opts.indent;
     sw->stream = stream;
@@ -101,6 +136,7 @@ stream_writer_new(int argc, VALUE *argv, VALUE self) {
  * Pushes a key onto the JSON document. The key will be used for the next push
  * if currently in a JSON object and ignored otherwise. If a key is provided on
  * the next push then that new key will be ignored.
+ *
  * - *key* [_String_] the key pending for the next push
  */
 static VALUE
@@ -108,9 +144,10 @@ stream_writer_push_key(VALUE self, VALUE key) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
     rb_check_type(key, T_STRING);
-    stream_writer_reset_buf(sw);
     oj_str_writer_push_key(&sw->sw, StringValuePtr(key));
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -119,13 +156,13 @@ stream_writer_push_key(VALUE self, VALUE key) {
  *
  * Pushes an object onto the JSON document. Future pushes will be to this object
  * until a pop() is called.
+ *
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE
 stream_writer_push_object(int argc, VALUE *argv, VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
-    stream_writer_reset_buf(sw);
     switch (argc) {
     case 0:
 	oj_str_writer_push_object(&sw->sw, 0);
@@ -142,7 +179,9 @@ stream_writer_push_object(int argc, VALUE *argv, VALUE self) {
 	rb_raise(rb_eArgError, "Wrong number of argument to 'push_object'.");
 	break;
     }
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -151,13 +190,13 @@ stream_writer_push_object(int argc, VALUE *argv, VALUE self) {
  *
  * Pushes an array onto the JSON document. Future pushes will be to this object
  * until a pop() is called.
+ *
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE
 stream_writer_push_array(int argc, VALUE *argv, VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
-    stream_writer_reset_buf(sw);
     switch (argc) {
     case 0:
 	oj_str_writer_push_array(&sw->sw, 0);
@@ -174,7 +213,9 @@ stream_writer_push_array(int argc, VALUE *argv, VALUE self) {
 	rb_raise(rb_eArgError, "Wrong number of argument to 'push_object'.");
 	break;
     }
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -189,7 +230,6 @@ static VALUE
 stream_writer_push_value(int argc, VALUE *argv, VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
-    stream_writer_reset_buf(sw);
     switch (argc) {
     case 1:
 	oj_str_writer_push_value((StrWriter)DATA_PTR(self), *argv, 0);
@@ -206,7 +246,9 @@ stream_writer_push_value(int argc, VALUE *argv, VALUE self) {
 	rb_raise(rb_eArgError, "Wrong number of argument to 'push_value'.");
 	break;
     }
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -224,7 +266,6 @@ stream_writer_push_json(int argc, VALUE *argv, VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
     rb_check_type(argv[0], T_STRING);
-    stream_writer_reset_buf(sw);
     switch (argc) {
     case 1:
 	oj_str_writer_push_json((StrWriter)DATA_PTR(self), StringValuePtr(*argv), 0);
@@ -241,7 +282,9 @@ stream_writer_push_json(int argc, VALUE *argv, VALUE self) {
 	rb_raise(rb_eArgError, "Wrong number of argument to 'push_json'.");
 	break;
     }
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -255,9 +298,10 @@ static VALUE
 stream_writer_pop(VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
-    stream_writer_reset_buf(sw);
     oj_str_writer_pop(&sw->sw);
-    stream_writer_write(sw);
+    if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
+	stream_writer_write(sw);
+    }
     return Qnil;
 }
 
@@ -271,13 +315,24 @@ static VALUE
 stream_writer_pop_all(VALUE self) {
     StreamWriter	sw = (StreamWriter)DATA_PTR(self);
 
-    stream_writer_reset_buf(sw);
     oj_str_writer_pop_all(&sw->sw);
     stream_writer_write(sw);
 
     return Qnil;
 }
 
+/* Document-method: flush
+ * call-seq: flush()
+ *
+ * Flush any remaining characters in the buffer.
+ */
+static VALUE
+stream_writer_flush(VALUE self) {
+    stream_writer_write((StreamWriter)DATA_PTR(self));
+
+    return Qnil;
+}
+
 /* Document-class: Oj::StreamWriter
  * 
  * Supports building a JSON document one element at a time. Build the IO stream
@@ -296,6 +351,7 @@ oj_stream_writer_init() {
     rb_define_method(oj_stream_writer_class, "push_json", stream_writer_push_json, -1);
     rb_define_method(oj_stream_writer_class, "pop", stream_writer_pop, 0);
     rb_define_method(oj_stream_writer_class, "pop_all", stream_writer_pop_all, 0);
+    rb_define_method(oj_stream_writer_class, "flush", stream_writer_flush, 0);
 }
 
 

data/ext/oj/string_writer.c

@@ -48,7 +48,7 @@ maybe_comma(StrWriter sw) {
 
 // Used by stream writer also.
 void
-oj_str_writer_init(StrWriter sw) {
+oj_str_writer_init(StrWriter sw, int buf_size) {
     sw->opts = oj_default_options;
     sw->depth = 0;
     sw->types = ALLOC_N(char, 256);
@@ -56,9 +56,14 @@ oj_str_writer_init(StrWriter sw) {
     *sw->types = '\0';
     sw->keyWritten = 0;
 
-    sw->out.buf = ALLOC_N(char, 4096);
-    sw->out.end = sw->out.buf + 4086;
-    sw->out.allocated = 1;
+    if (0 == buf_size) {
+	buf_size = 4096;
+    } else if (buf_size < 1024) {
+	buf_size = 1024;
+    }
+    sw->out.buf = ALLOC_N(char, buf_size);
+    sw->out.end = sw->out.buf + buf_size - 10;
+    sw->out.allocated = true;
     sw->out.cur = sw->out.buf;
     *sw->out.cur = '\0';
     sw->out.circ_cnt = 0;
@@ -255,6 +260,11 @@ str_writer_free(void *ptr) {
  * or Oj.optimize_rails has not been called then the behavior of the modes may
  * not be the same as if they were.
  *
+ * In addition to the regular dump options for the various modes a
+ * _:buffer_size_ option is available. It should be set to a positive
+ * integer. It is considered a hint of how large the initial internal buffer
+ * should be.
+ *
  * - *io* [_IO_] stream to write to
  * - *options* [_Hash_] formating options
  */
@@ -262,7 +272,7 @@ static VALUE
 str_writer_new(int argc, VALUE *argv, VALUE self) {
     StrWriter	sw = ALLOC(struct _StrWriter);
     
-    oj_str_writer_init(sw);
+    oj_str_writer_init(sw, 0);
     if (1 == argc) {
 	oj_parse_options(argv[0], &sw->opts);
     }

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '3.1.4'
+  VERSION = '3.2.0'
 end

data/test/test_writer.rb

@@ -297,6 +297,61 @@ class OjWriter < Minitest::Test
     assert_equal(%|{"a1":{},"a2":{"b":{}},"a3":{"a4":37}}\n|, output.string())
   end
 
+  def push_stuff(w, pop_all=true)
+    w.push_object()
+    w.push_object("a1")
+    w.pop()
+    w.push_object("a2")
+    w.push_array("b")
+    w.push_value(7)
+    w.push_value(true)
+    w.push_value("string")
+    w.pop()
+    w.pop()
+    w.push_object("a3")
+    if pop_all
+      w.pop_all()
+    else
+      w.pop()
+      w.pop()
+    end
+  end
+
+  def test_stream_writer_buf_small
+    output = StringIO.open("", "w+")
+    w = Oj::StreamWriter.new(output, :indent => 0, :buffer_size => 20)
+    push_stuff(w)
+    assert_equal(%|{"a1":{},"a2":{"b":[7,true,"string"]},"a3":{}}\n|, output.string())
+  end
+
+  def test_stream_writer_buf_large
+    output = StringIO.open("", "w+")
+    w = Oj::StreamWriter.new(output, :indent => 0, :buffer_size => 16000)
+    push_stuff(w)
+    assert_equal(%|{"a1":{},"a2":{"b":[7,true,"string"]},"a3":{}}\n|, output.string())
+  end
+
+  def test_stream_writer_buf_flush
+    output = StringIO.open("", "w+")
+    w = Oj::StreamWriter.new(output, :indent => 0, :buffer_size => 4096)
+    push_stuff(w, false)
+    # no flush so nothing should be in the output yet
+    assert_equal("", output.string())
+    w.flush()
+    assert_equal(%|{"a1":{},"a2":{"b":[7,true,"string"]},"a3":{}}\n|, output.string())
+  end
+  
+  def test_stream_writer_buf_flush_small
+    output = StringIO.open("", "w+")
+    w = Oj::StreamWriter.new(output, :indent => 0, :buffer_size => 30)
+    push_stuff(w, false)
+    # flush should be called at 30 bytes but since the writes are chunky flush
+    # is called after adding "string".
+    assert_equal(%|{"a1":{},"a2":{"b":[7,true,"string"|, output.string())
+    w.flush()
+    assert_equal(%|{"a1":{},"a2":{"b":[7,true,"string"]},"a3":{}}\n|, output.string())
+  end
+
   def test_stream_writer_push_null_value_with_key
     output = StringIO.open("", "w+")
     w = Oj::StreamWriter.new(output, :indent => 0)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.2.0
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-16 00:00:00.000000000 Z
+date: 2017-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler