oj 3.13.10 → 3.13.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e4f2c04b24ed0dc8c781036c17371378649d1aad9b927b402b2b0f3d67f9f94b
4
- data.tar.gz: 65b50beb64e09569e7bcdcff9675a60b146de5ed78027b90de5def8eb6b57aa8
3
+ metadata.gz: dcb5f80ad6384640d44889424c39f91ee2db778b17c91b9505b5b3b4c280abf1
4
+ data.tar.gz: 85b70200f48f46019ecf1020ca8e1b8ea604bca1161120b3747a8aeac2903682
5
5
  SHA512:
6
- metadata.gz: 62a4e36ea67596152899cf389502a69f4f19b354dc5980c66b31ad5163b932245030bfd338ad582ec101572b56fd280c0e784b1d20567f830933627274506503
7
- data.tar.gz: 671d6a7066b4f74c164ce4af83acbab38fcb0743c97c13985ad4f8e7f1431f67288d1bb80f74b7e55d90d6c5e727e01dc72b138b7dfda3220cbc55ae3ebf2856
6
+ metadata.gz: f96776be4f9431b160f1d9f588e8688fb7e0c4a431c58a10fa43a71f5a6f95781456287f40db5dfba03bb69350946ad36f246d0b59c196f9dc95218f51005f7a
7
+ data.tar.gz: e106b88134f86d82b9740f3e836ecb2aaa4af7ad643b96a4600eaa09392ba5f630c402101a6fb476823c41bc127b5c3d9625c6900a033cf6f9830ca01e57d459
data/CHANGELOG.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 3.13.11 - 2022-01-05
4
+
5
+ - Fixed write blocking failures on writes to a slow stream with larger writes.
3
6
 
4
7
  ## 3.13.10 - 2021-12-12
5
8
 
data/ext/oj/dump.c CHANGED
@@ -10,6 +10,9 @@
10
10
  #include <stdlib.h>
11
11
  #include <string.h>
12
12
  #include <unistd.h>
13
+ #if !IS_WINDOWS
14
+ #include <poll.h>
15
+ #endif
13
16
 
14
17
  #include "cache8.h"
15
18
  #include "odd.h"
@@ -113,49 +116,39 @@ static char rails_friendly_chars[256] = "\
113
116
  11111111111111111111111111111111";
114
117
 
115
118
  static void raise_strict(VALUE obj) {
116
- rb_raise(rb_eTypeError,
117
- "Failed to dump %s Object to JSON in strict mode.",
118
- rb_class2name(rb_obj_class(obj)));
119
+ rb_raise(rb_eTypeError, "Failed to dump %s Object to JSON in strict mode.", rb_class2name(rb_obj_class(obj)));
119
120
  }
120
121
 
121
- inline static size_t newline_friendly_size(const uint8_t *str, size_t len) {
122
+ inline static size_t calculate_string_size(const uint8_t *str, size_t len, const char *table) {
122
123
  size_t size = 0;
123
124
  size_t i = len;
124
125
 
125
- for (; 0 < i; str++, i--) {
126
- size += newline_friendly_chars[*str];
126
+ for (; 3 < i; i -= 4) {
127
+ size += table[*str++];
128
+ size += table[*str++];
129
+ size += table[*str++];
130
+ size += table[*str++];
131
+ }
132
+ for (; 0 < i; i--) {
133
+ size += table[*str++];
127
134
  }
128
135
  return size - len * (size_t)'0';
129
136
  }
130
137
 
131
- inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
132
- size_t size = 0;
133
- size_t i = len;
138
+ inline static size_t newline_friendly_size(const uint8_t *str, size_t len) {
139
+ return calculate_string_size(str, len, newline_friendly_chars);
140
+ }
134
141
 
135
- for (; 0 < i; str++, i--) {
136
- size += hibit_friendly_chars[*str];
137
- }
138
- return size - len * (size_t)'0';
142
+ inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
143
+ return calculate_string_size(str, len, hibit_friendly_chars);
139
144
  }
140
145
 
141
146
  inline static size_t ascii_friendly_size(const uint8_t *str, size_t len) {
142
- size_t size = 0;
143
- size_t i = len;
144
-
145
- for (; 0 < i; str++, i--) {
146
- size += ascii_friendly_chars[*str];
147
- }
148
- return size - len * (size_t)'0';
147
+ return calculate_string_size(str, len, ascii_friendly_chars);
149
148
  }
150
149
 
151
150
  inline static size_t xss_friendly_size(const uint8_t *str, size_t len) {
152
- size_t size = 0;
153
- size_t i = len;
154
-
155
- for (; 0 < i; str++, i--) {
156
- size += xss_friendly_chars[*str];
157
- }
158
- return size - len * (size_t)'0';
151
+ return calculate_string_size(str, len, xss_friendly_chars);
159
152
  }
160
153
 
161
154
  inline static size_t hixss_friendly_size(const uint8_t *str, size_t len) {
@@ -188,13 +181,7 @@ inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
188
181
  }
189
182
 
190
183
  inline static size_t rails_friendly_size(const uint8_t *str, size_t len) {
191
- size_t size = 0;
192
- size_t i = len;
193
-
194
- for (; 0 < i; str++, i--) {
195
- size += rails_friendly_chars[*str];
196
- }
197
- return size - len * (size_t)'0';
184
+ return calculate_string_size(str, len, rails_friendly_chars);
198
185
  }
199
186
 
200
187
  const char *oj_nan_str(VALUE obj, int opt, int mode, bool plus, int *lenp) {
@@ -535,14 +522,7 @@ void oj_dump_xml_time(VALUE obj, Out out) {
535
522
  }
536
523
  if ((0 == nsec && !out->opts->sec_prec_set) || 0 == out->opts->sec_prec) {
537
524
  if (0 == tzsecs && rb_funcall2(obj, oj_utcq_id, 0, 0)) {
538
- int len = sprintf(buf,
539
- "%04d-%02d-%02dT%02d:%02d:%02dZ",
540
- ti.year,
541
- ti.mon,
542
- ti.day,
543
- ti.hour,
544
- ti.min,
545
- ti.sec);
525
+ int len = sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02dZ", ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec);
546
526
  oj_dump_cstr(buf, len, 0, 0, out);
547
527
  } else {
548
528
  int len = sprintf(buf,
@@ -574,18 +554,7 @@ void oj_dump_xml_time(VALUE obj, Out out) {
574
554
  if (9 > out->opts->sec_prec) {
575
555
  format[32] = '0' + out->opts->sec_prec;
576
556
  }
577
- len = sprintf(buf,
578
- format,
579
- ti.year,
580
- ti.mon,
581
- ti.day,
582
- ti.hour,
583
- ti.min,
584
- ti.sec,
585
- (long)nsec,
586
- tzsign,
587
- tzhour,
588
- tzmin);
557
+ len = sprintf(buf, format, ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec, (long)nsec, tzsign, tzhour, tzmin);
589
558
  oj_dump_cstr(buf, len, 0, 0, out);
590
559
  }
591
560
  }
@@ -666,6 +635,21 @@ void oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
666
635
  }
667
636
  }
668
637
 
638
+ static void write_ready(int fd) {
639
+ struct pollfd pp;
640
+ int i;
641
+
642
+ pp.fd = fd;
643
+ pp.events = POLLERR | POLLOUT;
644
+ pp.revents = 0;
645
+ if (0 >= (i = poll(&pp, 1, 5000))) {
646
+ if (0 == i || EAGAIN == errno) {
647
+ rb_raise(rb_eIOError, "write timed out");
648
+ }
649
+ rb_raise(rb_eIOError, "write failed. %d %s.", errno, strerror(errno));
650
+ }
651
+ }
652
+
669
653
  void oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
670
654
  char buf[4096];
671
655
  struct _out out;
@@ -685,13 +669,24 @@ void oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
685
669
  if (oj_stringio_class == clas) {
686
670
  rb_funcall(stream, oj_write_id, 1, rb_str_new(out.buf, size));
687
671
  #if !IS_WINDOWS
688
- } else if (rb_respond_to(stream, oj_fileno_id) &&
689
- Qnil != (s = rb_funcall(stream, oj_fileno_id, 0)) && 0 != (fd = FIX2INT(s))) {
690
- if (size != write(fd, out.buf, size)) {
691
- if (out.allocated) {
692
- xfree(out.buf);
672
+ } else if (rb_respond_to(stream, oj_fileno_id) && Qnil != (s = rb_funcall(stream, oj_fileno_id, 0)) &&
673
+ 0 != (fd = FIX2INT(s))) {
674
+ ssize_t cnt;
675
+ ssize_t total = 0;
676
+
677
+ while (true) {
678
+ if (0 > (cnt = write(fd, out.buf + total, size - total))) {
679
+ if (EAGAIN != errno) {
680
+ rb_raise(rb_eIOError, "write failed. %d %s.", errno, strerror(errno));
681
+ break;
682
+ }
683
+ }
684
+ total += cnt;
685
+ if (size <= total) {
686
+ // Completed
687
+ break;
693
688
  }
694
- rb_raise(rb_eIOError, "Write failed. [%d:%s]", errno, strerror(errno));
689
+ write_ready(fd);
695
690
  }
696
691
  #endif
697
692
  } else if (rb_respond_to(stream, oj_write_id)) {
@@ -835,8 +830,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
835
830
  for (; str < end; str++) {
836
831
  switch (cmap[(uint8_t)*str]) {
837
832
  case '1':
838
- if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
839
- check_start <= str) {
833
+ if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && check_start <= str) {
840
834
  if (0 != (0x80 & (uint8_t)*str)) {
841
835
  if (0xC0 == (0xC0 & (uint8_t)*str)) {
842
836
  check_start = check_unicode(str, end, orig);
@@ -860,11 +854,9 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
860
854
  }
861
855
  break;
862
856
  case '3': // Unicode
863
- if (0xe2 == (uint8_t)*str &&
864
- (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
857
+ if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
865
858
  2 <= end - str) {
866
- if (0x80 == (uint8_t)str[1] &&
867
- (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
859
+ if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
868
860
  str = dump_unicode(str, end, out, orig);
869
861
  } else {
870
862
  check_start = check_unicode(str, end, orig);
@@ -883,10 +875,8 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
883
875
  dump_hex((uint8_t)*str, out);
884
876
  } else {
885
877
  if (0xe2 == (uint8_t)*str &&
886
- (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
887
- 2 <= end - str) {
888
- if (0x80 == (uint8_t)str[1] &&
889
- (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
878
+ (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
879
+ if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
890
880
  str = dump_unicode(str, end, out, orig);
891
881
  } else {
892
882
  check_start = check_unicode(str, end, orig);
@@ -902,8 +892,8 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
902
892
  }
903
893
  *out->cur++ = '"';
904
894
  }
905
- if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
906
- 0 < str - orig && 0 != (0x80 & *(str - 1))) {
895
+ if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 0 < str - orig &&
896
+ 0 != (0x80 & *(str - 1))) {
907
897
  uint8_t c = (uint8_t) * (str - 1);
908
898
  int i;
909
899
  int scnt = (int)(str - orig);
@@ -1064,8 +1054,7 @@ void oj_dump_bignum(VALUE obj, int depth, Out out, bool as_ok) {
1064
1054
  int cnt = (int)RSTRING_LEN(rs);
1065
1055
  bool dump_as_string = false;
1066
1056
 
1067
- if (out->opts->int_range_max != 0 ||
1068
- out->opts->int_range_min != 0) { // Bignum cannot be inside of Fixnum range
1057
+ if (out->opts->int_range_max != 0 || out->opts->int_range_min != 0) { // Bignum cannot be inside of Fixnum range
1069
1058
  dump_as_string = true;
1070
1059
  assure_size(out, cnt + 2);
1071
1060
  *out->cur++ = '"';
data/ext/oj/encoder.c ADDED
@@ -0,0 +1,43 @@
1
+ // Copyright (c) 2011, 2022 Peter Ohler. All rights reserved.
2
+ // Licensed under the MIT License. See LICENSE file in the project root for license details.
3
+
4
+ #include "oj.h"
5
+
6
+ typedef struct _encoder {
7
+ int indent; // indention for dump, default 2
8
+ char circular; // YesNo
9
+ char escape_mode; // Escape_Mode
10
+ char mode; // Mode
11
+ char time_format; // TimeFormat
12
+ char bigdec_as_num; // YesNo
13
+ char to_hash; // YesNo
14
+ char to_json; // YesNo
15
+ char as_json; // YesNo
16
+ char raw_json; // YesNo
17
+ char trace; // YesNo
18
+ char sec_prec_set; // boolean (0 or 1)
19
+ char ignore_under; // YesNo - ignore attrs starting with _ if true in object and custom modes
20
+ int64_t int_range_min; // dump numbers below as string
21
+ int64_t int_range_max; // dump numbers above as string
22
+ const char* create_id; // 0 or string
23
+ size_t create_id_len; // length of create_id
24
+ int sec_prec; // second precision when dumping time
25
+ char float_prec; // float precision, linked to float_fmt
26
+ char float_fmt[7]; // float format for dumping, if empty use Ruby
27
+ struct _dumpOpts dump_opts;
28
+ struct _rxClass str_rx;
29
+ VALUE* ignore; // Qnil terminated array of classes or NULL
30
+ } * Encoder;
31
+
32
+ /*
33
+ rb_define_module_function(Oj, "encode", encode, -1);
34
+ rb_define_module_function(Oj, "to_file", to_file, -1); // or maybe just write
35
+ rb_define_module_function(Oj, "to_stream", to_stream, -1);
36
+ */
37
+
38
+ // write(to, obj)
39
+ // if to is a string then open file
40
+ // else if stream then write to stream
41
+ // handle non-blocking
42
+
43
+ // should each mode have a different encoder or use delegates like the parser?
data/lib/oj/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
 
2
2
  module Oj
3
3
  # Current version of the module.
4
- VERSION = '3.13.10'
4
+ VERSION = '3.13.11'
5
5
  end
data/test/foo.rb CHANGED
@@ -4,32 +4,74 @@ $: << '.'
4
4
  $: << File.join(File.dirname(__FILE__), "../lib")
5
5
  $: << File.join(File.dirname(__FILE__), "../ext")
6
6
 
7
- require 'rails'
8
- require 'oj'
7
+ require "oj"
8
+ require "socket"
9
+ require 'io/nonblock'
9
10
 
10
- $data = {:ticker=>"ASAI3", :price=>18.7, :rate=>-0.8.to_d}
11
-
12
- def encode
13
- p "JSON.generate: #{JSON.generate($data)}"
14
- p "Oj.generate: #{Oj.generate($data)}"
15
- p "Oj.dump: #{Oj.dump($data)}"
16
- p "to_json: #{$data.to_json}"
17
- p "ActiveSupport::JSON.encode: #{ActiveSupport::JSON.encode($data)}"
11
+ =begin
12
+ #pid = spawn("nc -d 0.1 -l 5000", out: "/dev/null")
13
+ pid = spawn("nc -i 1 -l 7777", out: "/dev/null")
14
+ at_exit { Process.kill 9, pid }
15
+ sleep 0.2
16
+ s = Socket.tcp("localhost", 7777)
17
+ #s.nonblock = false
18
+ 1_000_000.times do |x|
19
+ Oj.to_stream(s, { x: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
18
20
  end
21
+ =end
19
22
 
20
- puts "With Oj version (#{Oj::VERSION})"
21
-
22
- puts
23
- puts "Before optimizing"
24
- encode
25
-
26
- Oj.optimize_rails
27
- Oj.default_options = {
28
- mode: :rails,
29
- bigdecimal_as_decimal: true,
30
- bigdecimal_load: true
31
- }
23
+ =begin
24
+ IO.pipe do |r, w|
25
+ if fork
26
+ r.close
27
+ #w.nonblock = false
28
+ 1_000_000.times do |i|
29
+ begin
30
+ Oj.to_stream(w, { x: i})
31
+ rescue IOError => e
32
+ puts "*** #{i} raised #{e.class}: #{e}"
33
+ IO.select(nil, [w])
34
+ retry
35
+ end
36
+ w.puts
37
+ end
38
+ else
39
+ w.close
40
+ sleep(0.1)
41
+ r.each_line { |b|
42
+ #print b
43
+ }
44
+ r.close
45
+ Process.exit(0)
46
+ end
47
+ end
48
+ =end
32
49
 
33
- puts
34
- puts "After optimizing"
35
- encode
50
+ IO.pipe do |r, w|
51
+ if fork
52
+ r.close
53
+ #w.nonblock = false
54
+ a = []
55
+ 10_000.times do |i|
56
+ a << i
57
+ end
58
+ begin
59
+ Oj.to_stream(w, a, indent: 2)
60
+ rescue IOError => e
61
+ puts "*** raised #{e.class}: #{e}"
62
+ puts "*** fileno: #{w.fileno}"
63
+ puts "*** is an IO?: #{w.kind_of?(IO)}"
64
+ IO.select(nil, [w])
65
+ retry
66
+ end
67
+ w.puts
68
+ else
69
+ w.close
70
+ sleep(0.5)
71
+ r.each_line { |b|
72
+ #print b
73
+ }
74
+ r.close
75
+ Process.exit(0)
76
+ end
77
+ end
data/test/test_saj.rb CHANGED
@@ -180,7 +180,7 @@ class SajTest < Minitest::Test
180
180
  assert_equal([:add_value, 12345, nil], handler.calls.first)
181
181
  type, message, line, column = handler.calls.last
182
182
  assert_equal([:error, 1, 6], [type, line, column])
183
- assert_match(%r{invalid format, extra characters at line 1, column 6 \[(?:[a-z\.]+/)*saj\.c:\d+\]}, message)
183
+ assert_match(%r{invalid format, extra characters at line 1, column 6 \[(?:[A-Za-z]:\/)?(?:[a-z\.]+/)*saj\.c:\d+\]}, message)
184
184
  end
185
185
 
186
186
  end
data/test/test_various.rb CHANGED
@@ -528,7 +528,7 @@ class Juice < Minitest::Test
528
528
  assert_equal(58, obj.y)
529
529
  end
530
530
 
531
- # Stream Deeply Nested
531
+ # Stream Deeply Nested
532
532
  def test_deep_nest_dump
533
533
  begin
534
534
  a = []
@@ -541,7 +541,7 @@ class Juice < Minitest::Test
541
541
  assert(false, "*** expected an exception")
542
542
  end
543
543
 
544
- # Stream IO
544
+ # Stream IO
545
545
  def test_io_string
546
546
  src = { 'x' => true, 'y' => 58, 'z' => [1, 2, 3]}
547
547
  output = StringIO.open("", "w+")
@@ -553,6 +553,9 @@ class Juice < Minitest::Test
553
553
  end
554
554
 
555
555
  def test_io_file
556
+ # Windows does not support fork
557
+ return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
558
+
556
559
  src = { 'x' => true, 'y' => 58, 'z' => [1, 2, 3]}
557
560
  filename = File.join(File.dirname(__FILE__), 'open_file_test.json')
558
561
  File.open(filename, "w") { |f|
@@ -564,6 +567,28 @@ class Juice < Minitest::Test
564
567
  assert_equal(src, obj)
565
568
  end
566
569
 
570
+ def test_io_stream
571
+ IO.pipe do |r, w|
572
+ if fork
573
+ r.close
574
+ #w.nonblock = false
575
+ a = []
576
+ 10_000.times do |i|
577
+ a << i
578
+ end
579
+ Oj.to_stream(w, a, indent: 2)
580
+ w.close
581
+ else
582
+ w.close
583
+ sleep(0.1) # to force a busy
584
+ cnt = 0
585
+ r.each_line { cnt += 1 }
586
+ r.close
587
+ Process.exit(0)
588
+ end
589
+ end
590
+ end
591
+
567
592
  # comments
568
593
  def test_comment_slash
569
594
  json = %{{
data/test/tests.rb CHANGED
@@ -18,7 +18,6 @@ require 'test_object'
18
18
  require 'test_saj'
19
19
  require 'test_scp'
20
20
  require 'test_strict'
21
- require 'test_various'
22
21
  require 'test_rails'
23
22
  require 'test_wab'
24
23
  require 'test_writer'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: oj
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.13.10
4
+ version: 3.13.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Peter Ohler
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-12-12 00:00:00.000000000 Z
11
+ date: 2022-01-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake-compiler
@@ -117,6 +117,7 @@ files:
117
117
  - ext/oj/dump_object.c
118
118
  - ext/oj/dump_strict.c
119
119
  - ext/oj/encode.h
120
+ - ext/oj/encoder.c
120
121
  - ext/oj/err.c
121
122
  - ext/oj/err.h
122
123
  - ext/oj/extconf.rb