oj 3.13.10 → 3.13.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4f2c04b24ed0dc8c781036c17371378649d1aad9b927b402b2b0f3d67f9f94b
-  data.tar.gz: 65b50beb64e09569e7bcdcff9675a60b146de5ed78027b90de5def8eb6b57aa8
+  metadata.gz: dcb5f80ad6384640d44889424c39f91ee2db778b17c91b9505b5b3b4c280abf1
+  data.tar.gz: 85b70200f48f46019ecf1020ca8e1b8ea604bca1161120b3747a8aeac2903682
 SHA512:
-  metadata.gz: 62a4e36ea67596152899cf389502a69f4f19b354dc5980c66b31ad5163b932245030bfd338ad582ec101572b56fd280c0e784b1d20567f830933627274506503
-  data.tar.gz: 671d6a7066b4f74c164ce4af83acbab38fcb0743c97c13985ad4f8e7f1431f67288d1bb80f74b7e55d90d6c5e727e01dc72b138b7dfda3220cbc55ae3ebf2856
+  metadata.gz: f96776be4f9431b160f1d9f588e8688fb7e0c4a431c58a10fa43a71f5a6f95781456287f40db5dfba03bb69350946ad36f246d0b59c196f9dc95218f51005f7a
+  data.tar.gz: e106b88134f86d82b9740f3e836ecb2aaa4af7ad643b96a4600eaa09392ba5f630c402101a6fb476823c41bc127b5c3d9625c6900a033cf6f9830ca01e57d459

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 3.13.11 - 2022-01-05
+
+- Fixed write blocking failures on writes to a slow stream with larger writes.
 
 ## 3.13.10 - 2021-12-12
 

data/ext/oj/dump.c

@@ -10,6 +10,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#if !IS_WINDOWS
+#include <poll.h>
+#endif
 
 #include "cache8.h"
 #include "odd.h"
@@ -113,49 +116,39 @@ static char rails_friendly_chars[256] = "\
 11111111111111111111111111111111";
 
 static void raise_strict(VALUE obj) {
-    rb_raise(rb_eTypeError,
-             "Failed to dump %s Object to JSON in strict mode.",
-             rb_class2name(rb_obj_class(obj)));
+    rb_raise(rb_eTypeError, "Failed to dump %s Object to JSON in strict mode.", rb_class2name(rb_obj_class(obj)));
 }
 
-inline static size_t newline_friendly_size(const uint8_t *str, size_t len) {
+inline static size_t calculate_string_size(const uint8_t *str, size_t len, const char *table) {
     size_t size = 0;
     size_t i    = len;
 
-    for (; 0 < i; str++, i--) {
-        size += newline_friendly_chars[*str];
+    for (; 3 < i; i -= 4) {
+        size += table[*str++];
+        size += table[*str++];
+        size += table[*str++];
+        size += table[*str++];
+    }
+    for (; 0 < i; i--) {
+        size += table[*str++];
     }
     return size - len * (size_t)'0';
 }
 
-inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
-    size_t size = 0;
-    size_t i    = len;
+inline static size_t newline_friendly_size(const uint8_t *str, size_t len) {
+    return calculate_string_size(str, len, newline_friendly_chars);
+}
 
-    for (; 0 < i; str++, i--) {
-        size += hibit_friendly_chars[*str];
-    }
-    return size - len * (size_t)'0';
+inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
+    return calculate_string_size(str, len, hibit_friendly_chars);
 }
 
 inline static size_t ascii_friendly_size(const uint8_t *str, size_t len) {
-    size_t size = 0;
-    size_t i    = len;
-
-    for (; 0 < i; str++, i--) {
-        size += ascii_friendly_chars[*str];
-    }
-    return size - len * (size_t)'0';
+    return calculate_string_size(str, len, ascii_friendly_chars);
 }
 
 inline static size_t xss_friendly_size(const uint8_t *str, size_t len) {
-    size_t size = 0;
-    size_t i    = len;
-
-    for (; 0 < i; str++, i--) {
-        size += xss_friendly_chars[*str];
-    }
-    return size - len * (size_t)'0';
+    return calculate_string_size(str, len, xss_friendly_chars);
 }
 
 inline static size_t hixss_friendly_size(const uint8_t *str, size_t len) {
@@ -188,13 +181,7 @@ inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
 }
 
 inline static size_t rails_friendly_size(const uint8_t *str, size_t len) {
-    size_t size = 0;
-    size_t i    = len;
-
-    for (; 0 < i; str++, i--) {
-        size += rails_friendly_chars[*str];
-    }
-    return size - len * (size_t)'0';
+    return calculate_string_size(str, len, rails_friendly_chars);
 }
 
 const char *oj_nan_str(VALUE obj, int opt, int mode, bool plus, int *lenp) {
@@ -535,14 +522,7 @@ void oj_dump_xml_time(VALUE obj, Out out) {
     }
     if ((0 == nsec && !out->opts->sec_prec_set) || 0 == out->opts->sec_prec) {
         if (0 == tzsecs && rb_funcall2(obj, oj_utcq_id, 0, 0)) {
-            int len = sprintf(buf,
-                              "%04d-%02d-%02dT%02d:%02d:%02dZ",
-                              ti.year,
-                              ti.mon,
-                              ti.day,
-                              ti.hour,
-                              ti.min,
-                              ti.sec);
+            int len = sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02dZ", ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec);
             oj_dump_cstr(buf, len, 0, 0, out);
         } else {
             int len = sprintf(buf,
@@ -574,18 +554,7 @@ void oj_dump_xml_time(VALUE obj, Out out) {
         if (9 > out->opts->sec_prec) {
             format[32] = '0' + out->opts->sec_prec;
         }
-        len = sprintf(buf,
-                      format,
-                      ti.year,
-                      ti.mon,
-                      ti.day,
-                      ti.hour,
-                      ti.min,
-                      ti.sec,
-                      (long)nsec,
-                      tzsign,
-                      tzhour,
-                      tzmin);
+        len = sprintf(buf, format, ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec, (long)nsec, tzsign, tzhour, tzmin);
         oj_dump_cstr(buf, len, 0, 0, out);
     }
 }
@@ -666,6 +635,21 @@ void oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
     }
 }
 
+static void write_ready(int fd) {
+    struct pollfd pp;
+    int           i;
+
+    pp.fd      = fd;
+    pp.events  = POLLERR | POLLOUT;
+    pp.revents = 0;
+    if (0 >= (i = poll(&pp, 1, 5000))) {
+        if (0 == i || EAGAIN == errno) {
+            rb_raise(rb_eIOError, "write timed out");
+        }
+        rb_raise(rb_eIOError, "write failed. %d %s.", errno, strerror(errno));
+    }
+}
+
 void oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
     char        buf[4096];
     struct _out out;
@@ -685,13 +669,24 @@ void oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
     if (oj_stringio_class == clas) {
         rb_funcall(stream, oj_write_id, 1, rb_str_new(out.buf, size));
 #if !IS_WINDOWS
-    } else if (rb_respond_to(stream, oj_fileno_id) &&
-               Qnil != (s = rb_funcall(stream, oj_fileno_id, 0)) && 0 != (fd = FIX2INT(s))) {
-        if (size != write(fd, out.buf, size)) {
-            if (out.allocated) {
-                xfree(out.buf);
+    } else if (rb_respond_to(stream, oj_fileno_id) && Qnil != (s = rb_funcall(stream, oj_fileno_id, 0)) &&
+               0 != (fd = FIX2INT(s))) {
+        ssize_t cnt;
+        ssize_t total = 0;
+
+        while (true) {
+            if (0 > (cnt = write(fd, out.buf + total, size - total))) {
+                if (EAGAIN != errno) {
+                    rb_raise(rb_eIOError, "write failed. %d %s.", errno, strerror(errno));
+                    break;
+                }
+            }
+            total += cnt;
+            if (size <= total) {
+                // Completed
+                break;
             }
-            rb_raise(rb_eIOError, "Write failed. [%d:%s]", errno, strerror(errno));
+            write_ready(fd);
         }
 #endif
     } else if (rb_respond_to(stream, oj_write_id)) {
@@ -835,8 +830,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         for (; str < end; str++) {
             switch (cmap[(uint8_t)*str]) {
             case '1':
-                if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
-                    check_start <= str) {
+                if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && check_start <= str) {
                     if (0 != (0x80 & (uint8_t)*str)) {
                         if (0xC0 == (0xC0 & (uint8_t)*str)) {
                             check_start = check_unicode(str, end, orig);
@@ -860,11 +854,9 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                 }
                 break;
             case '3':  // Unicode
-                if (0xe2 == (uint8_t)*str &&
-                    (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
+                if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
                     2 <= end - str) {
-                    if (0x80 == (uint8_t)str[1] &&
-                        (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
+                    if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                         str = dump_unicode(str, end, out, orig);
                     } else {
                         check_start = check_unicode(str, end, orig);
@@ -883,10 +875,8 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                     dump_hex((uint8_t)*str, out);
                 } else {
                     if (0xe2 == (uint8_t)*str &&
-                        (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
-                        2 <= end - str) {
-                        if (0x80 == (uint8_t)str[1] &&
-                            (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
+                        (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
+                        if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                             str = dump_unicode(str, end, out, orig);
                         } else {
                             check_start = check_unicode(str, end, orig);
@@ -902,8 +892,8 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         }
         *out->cur++ = '"';
     }
-    if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
-        0 < str - orig && 0 != (0x80 & *(str - 1))) {
+    if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 0 < str - orig &&
+        0 != (0x80 & *(str - 1))) {
         uint8_t c = (uint8_t) * (str - 1);
         int     i;
         int     scnt = (int)(str - orig);
@@ -1064,8 +1054,7 @@ void oj_dump_bignum(VALUE obj, int depth, Out out, bool as_ok) {
     int            cnt            = (int)RSTRING_LEN(rs);
     bool           dump_as_string = false;
 
-    if (out->opts->int_range_max != 0 ||
-        out->opts->int_range_min != 0) {  // Bignum cannot be inside of Fixnum range
+    if (out->opts->int_range_max != 0 || out->opts->int_range_min != 0) {  // Bignum cannot be inside of Fixnum range
         dump_as_string = true;
         assure_size(out, cnt + 2);
         *out->cur++ = '"';

data/ext/oj/encoder.c

@@ -0,0 +1,43 @@
+// Copyright (c) 2011, 2022 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#include "oj.h"
+
+typedef struct _encoder {
+    int              indent;         // indention for dump, default 2
+    char             circular;       // YesNo
+    char             escape_mode;    // Escape_Mode
+    char             mode;           // Mode
+    char             time_format;    // TimeFormat
+    char             bigdec_as_num;  // YesNo
+    char             to_hash;        // YesNo
+    char             to_json;        // YesNo
+    char             as_json;        // YesNo
+    char             raw_json;       // YesNo
+    char             trace;          // YesNo
+    char             sec_prec_set;   // boolean (0 or 1)
+    char             ignore_under;   // YesNo - ignore attrs starting with _ if true in object and custom modes
+    int64_t          int_range_min;  // dump numbers below as string
+    int64_t          int_range_max;  // dump numbers above as string
+    const char*      create_id;      // 0 or string
+    size_t           create_id_len;  // length of create_id
+    int              sec_prec;       // second precision when dumping time
+    char             float_prec;     // float precision, linked to float_fmt
+    char             float_fmt[7];   // float format for dumping, if empty use Ruby
+    struct _dumpOpts dump_opts;
+    struct _rxClass  str_rx;
+    VALUE*           ignore;  // Qnil terminated array of classes or NULL
+} * Encoder;
+
+/*
+    rb_define_module_function(Oj, "encode", encode, -1);
+    rb_define_module_function(Oj, "to_file", to_file, -1); // or maybe just write
+    rb_define_module_function(Oj, "to_stream", to_stream, -1);
+*/
+
+// write(to, obj)
+//  if to is a string then open file
+//  else if stream then write to stream
+//  handle non-blocking
+
+// should each mode have a different encoder or use delegates like the parser?

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.13.10'
+  VERSION = '3.13.11'
 end

data/test/foo.rb

@@ -4,32 +4,74 @@ $: << '.'
 $: << File.join(File.dirname(__FILE__), "../lib")
 $: << File.join(File.dirname(__FILE__), "../ext")
 
-require 'rails'
-require 'oj'
+require "oj"
+require "socket"
+require 'io/nonblock'
 
-$data = {:ticker=>"ASAI3", :price=>18.7, :rate=>-0.8.to_d}
-
-def encode
-  p "JSON.generate: #{JSON.generate($data)}"
-  p "Oj.generate: #{Oj.generate($data)}"
-  p "Oj.dump: #{Oj.dump($data)}"
-  p "to_json: #{$data.to_json}"
-  p "ActiveSupport::JSON.encode: #{ActiveSupport::JSON.encode($data)}"
+=begin
+#pid = spawn("nc -d 0.1 -l 5000", out: "/dev/null")
+pid = spawn("nc -i 1 -l 7777", out: "/dev/null")
+at_exit { Process.kill 9, pid }
+sleep 0.2
+s = Socket.tcp("localhost", 7777)
+#s.nonblock = false
+1_000_000.times do |x|
+  Oj.to_stream(s, { x: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
 end
+=end
 
-puts "With Oj version (#{Oj::VERSION})"
-
-puts
-puts "Before optimizing"
-encode
-
-Oj.optimize_rails
-Oj.default_options = {
-  mode: :rails,
-  bigdecimal_as_decimal: true,
-  bigdecimal_load: true
-}
+=begin
+IO.pipe do |r, w|
+  if fork
+    r.close
+    #w.nonblock = false
+    1_000_000.times do |i|
+      begin
+	Oj.to_stream(w, { x: i})
+      rescue IOError => e
+	puts "*** #{i} raised #{e.class}: #{e}"
+	IO.select(nil, [w])
+	retry
+      end
+      w.puts
+    end
+  else
+    w.close
+    sleep(0.1)
+    r.each_line { |b|
+      #print b
+    }
+    r.close
+    Process.exit(0)
+  end
+end
+=end
 
-puts
-puts "After optimizing"
-encode
+IO.pipe do |r, w|
+  if fork
+    r.close
+    #w.nonblock = false
+    a = []
+    10_000.times do |i|
+      a << i
+    end
+    begin
+      Oj.to_stream(w, a, indent: 2)
+    rescue IOError => e
+      puts "*** raised #{e.class}: #{e}"
+      puts "*** fileno: #{w.fileno}"
+      puts "*** is an IO?: #{w.kind_of?(IO)}"
+      IO.select(nil, [w])
+      retry
+    end
+    w.puts
+  else
+    w.close
+    sleep(0.5)
+    r.each_line { |b|
+      #print b
+    }
+    r.close
+    Process.exit(0)
+  end
+end

data/test/test_saj.rb

@@ -180,7 +180,7 @@ class SajTest < Minitest::Test
     assert_equal([:add_value, 12345, nil], handler.calls.first)
     type, message, line, column = handler.calls.last
     assert_equal([:error, 1, 6], [type, line, column])
-    assert_match(%r{invalid format, extra characters at line 1, column 6 \[(?:[a-z\.]+/)*saj\.c:\d+\]}, message)
+    assert_match(%r{invalid format, extra characters at line 1, column 6 \[(?:[A-Za-z]:\/)?(?:[a-z\.]+/)*saj\.c:\d+\]}, message)
   end
 
 end

data/test/test_various.rb

@@ -528,7 +528,7 @@ class Juice < Minitest::Test
     assert_equal(58, obj.y)
   end
 
-# Stream Deeply Nested
+  # Stream Deeply Nested
   def test_deep_nest_dump
     begin
       a = []
@@ -541,7 +541,7 @@ class Juice < Minitest::Test
     assert(false, "*** expected an exception")
   end
 
-# Stream IO
+  # Stream IO
   def test_io_string
     src = { 'x' => true, 'y' => 58, 'z' => [1, 2, 3]}
     output = StringIO.open("", "w+")
@@ -553,6 +553,9 @@ class Juice < Minitest::Test
   end
 
   def test_io_file
+    # Windows does not support fork
+    return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
+
     src = { 'x' => true, 'y' => 58, 'z' => [1, 2, 3]}
     filename = File.join(File.dirname(__FILE__), 'open_file_test.json')
     File.open(filename, "w") { |f|
@@ -564,6 +567,28 @@ class Juice < Minitest::Test
     assert_equal(src, obj)
   end
 
+  def test_io_stream
+    IO.pipe do |r, w|
+      if fork
+	r.close
+	#w.nonblock = false
+	a = []
+	10_000.times do |i|
+	  a << i
+	end
+	Oj.to_stream(w, a, indent: 2)
+	w.close
+      else
+	w.close
+	sleep(0.1) # to force a busy
+	cnt = 0
+	r.each_line { cnt += 1 }
+	r.close
+	Process.exit(0)
+      end
+    end
+  end
+
 # comments
   def test_comment_slash
     json = %{{

data/test/tests.rb

@@ -18,7 +18,6 @@ require 'test_object'
 require 'test_saj'
 require 'test_scp'
 require 'test_strict'
-require 'test_various'
 require 'test_rails'
 require 'test_wab'
 require 'test_writer'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.13.10
+  version: 3.13.11
 platform: ruby
 authors:
 - Peter Ohler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-12 00:00:00.000000000 Z
+date: 2022-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -117,6 +117,7 @@ files:
 - ext/oj/dump_object.c
 - ext/oj/dump_strict.c
 - ext/oj/encode.h
+- ext/oj/encoder.c
 - ext/oj/err.c
 - ext/oj/err.h
 - ext/oj/extconf.rb