oj 2.9.9 → 2.10.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of oj might be problematic. Click here for more details.

@@ -49,7 +49,7 @@
49
49
  #define NUM_MAX (FIXNUM_MAX >> 8)
50
50
  #endif
51
51
  #define EXP_MAX 1023
52
- #define DEC_MAX 14
52
+ #define DEC_MAX 15
53
53
 
54
54
  static void
55
55
  skip_comment(ParseInfo pi) {
@@ -99,7 +99,7 @@ add_value(ParseInfo pi, VALUE rval) {
99
99
  parent->next = NEXT_ARRAY_COMMA;
100
100
  break;
101
101
  case NEXT_HASH_VALUE:
102
- pi->hash_set_value(pi, parent->key, parent->klen, rval);
102
+ pi->hash_set_value(pi, parent, rval);
103
103
  if (parent->kalloc) {
104
104
  xfree((char*)parent->key);
105
105
  }
@@ -134,7 +134,7 @@ add_num_value(ParseInfo pi, NumInfo ni) {
134
134
  parent->next = NEXT_ARRAY_COMMA;
135
135
  break;
136
136
  case NEXT_HASH_VALUE:
137
- pi->hash_set_num(pi, parent->key, parent->klen, ni);
137
+ pi->hash_set_num(pi, parent, ni);
138
138
  if (parent->kalloc) {
139
139
  xfree((char*)parent->key);
140
140
  }
@@ -302,13 +302,18 @@ read_escaped_str(ParseInfo pi) {
302
302
  break;
303
303
  case NEXT_HASH_NEW:
304
304
  case NEXT_HASH_KEY:
305
- parent->key = strdup(buf.head);
306
- parent->klen = buf_len(&buf);
305
+ if (Qundef == (parent->key_val = pi->hash_key(pi, buf.head, buf_len(&buf)))) {
306
+ parent->key = strdup(buf.head);
307
+ parent->klen = buf_len(&buf);
308
+ } else {
309
+ parent->key = "";
310
+ parent->klen = 0;
311
+ }
307
312
  parent->k1 = *pi->rd.str;
308
313
  parent->next = NEXT_HASH_COLON;
309
314
  break;
310
315
  case NEXT_HASH_VALUE:
311
- pi->hash_set_cstr(pi, parent->key, parent->klen, buf.head, buf_len(&buf), pi->rd.str);
316
+ pi->hash_set_cstr(pi, parent, buf.head, buf_len(&buf), pi->rd.str);
312
317
  if (parent->kalloc) {
313
318
  xfree((char*)parent->key);
314
319
  }
@@ -366,11 +371,12 @@ read_str(ParseInfo pi) {
366
371
  parent->key = parent->karray;
367
372
  parent->kalloc = 0;
368
373
  }
374
+ parent->key_val = pi->hash_key(pi, parent->key, parent->klen);
369
375
  parent->k1 = *pi->rd.str;
370
376
  parent->next = NEXT_HASH_COLON;
371
377
  break;
372
378
  case NEXT_HASH_VALUE:
373
- pi->hash_set_cstr(pi, parent->key, parent->klen, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
379
+ pi->hash_set_cstr(pi, parent, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
374
380
  if (parent->kalloc) {
375
381
  xfree((char*)parent->key);
376
382
  }
@@ -42,6 +42,11 @@ static void
42
42
  noop_end(struct _ParseInfo *pi) {
43
43
  }
44
44
 
45
+ static VALUE
46
+ noop_hash_key(struct _ParseInfo *pi, const char *key, size_t klen) {
47
+ return Qundef;
48
+ }
49
+
45
50
  static void
46
51
  add_value(ParseInfo pi, VALUE val) {
47
52
  pi->stack.head->val = val;
@@ -69,9 +74,12 @@ start_hash(ParseInfo pi) {
69
74
  }
70
75
 
71
76
  static VALUE
72
- hash_key(ParseInfo pi, const char *key, size_t klen) {
73
- volatile VALUE rkey = rb_str_new(key, klen);
77
+ calc_hash_key(ParseInfo pi, Val parent) {
78
+ volatile VALUE rkey = parent->key_val;
74
79
 
80
+ if (Qundef == rkey) {
81
+ rkey = rb_str_new(parent->key, parent->klen);
82
+ }
75
83
  rkey = oj_encode(rkey);
76
84
  if (Yes == pi->options.sym_key) {
77
85
  rkey = rb_str_intern(rkey);
@@ -80,24 +88,24 @@ hash_key(ParseInfo pi, const char *key, size_t klen) {
80
88
  }
81
89
 
82
90
  static void
83
- hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_t len, const char *orig) {
91
+ hash_set_cstr(ParseInfo pi, Val parent, const char *str, size_t len, const char *orig) {
84
92
  volatile VALUE rstr = rb_str_new(str, len);
85
93
 
86
94
  rstr = oj_encode(rstr);
87
- rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), rstr);
95
+ rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), rstr);
88
96
  }
89
97
 
90
98
  static void
91
- hash_set_num(struct _ParseInfo *pi, const char *key, size_t klen, NumInfo ni) {
99
+ hash_set_num(struct _ParseInfo *pi, Val parent, NumInfo ni) {
92
100
  if (ni->infinity || ni->nan) {
93
101
  oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
94
102
  }
95
- rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), oj_num_as_value(ni));
103
+ rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), oj_num_as_value(ni));
96
104
  }
97
105
 
98
106
  static void
99
- hash_set_value(ParseInfo pi, const char *key, size_t klen, VALUE value) {
100
- rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), value);
107
+ hash_set_value(ParseInfo pi, Val parent, VALUE value) {
108
+ rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), value);
101
109
  }
102
110
 
103
111
  static VALUE
@@ -130,6 +138,7 @@ void
130
138
  oj_set_strict_callbacks(ParseInfo pi) {
131
139
  pi->start_hash = start_hash;
132
140
  pi->end_hash = noop_end;
141
+ pi->hash_key = noop_hash_key;
133
142
  pi->hash_set_cstr = hash_set_cstr;
134
143
  pi->hash_set_num = hash_set_num;
135
144
  pi->hash_set_value = hash_set_value;
@@ -69,6 +69,7 @@ oj_stack_init(ValStack stack) {
69
69
  stack->tail = stack->head;
70
70
  stack->head->val = Qundef;
71
71
  stack->head->key = 0;
72
+ stack->head->key_val = Qundef;
72
73
  stack->head->classname = 0;
73
74
  stack->head->klen = 0;
74
75
  stack->head->clen = 0;
@@ -53,9 +53,10 @@ typedef enum {
53
53
  } ValNext;
54
54
 
55
55
  typedef struct _Val {
56
- VALUE val;
56
+ volatile VALUE val;
57
57
  const char *key;
58
58
  char karray[32];
59
+ volatile VALUE key_val;
59
60
  union {
60
61
  const char *classname;
61
62
  OddArgs odd_args;
@@ -127,6 +128,7 @@ stack_push(ValStack stack, VALUE val, ValNext next) {
127
128
  stack->tail->next = next;
128
129
  stack->tail->classname = 0;
129
130
  stack->tail->key = 0;
131
+ stack->tail->key_val = Qundef;
130
132
  stack->tail->clen = 0;
131
133
  stack->tail->klen = 0;
132
134
  stack->tail->kalloc = 0;
@@ -45,6 +45,7 @@ module Oj
45
45
  #
46
46
  # def hash_start(); end
47
47
  # def hash_end(); end
48
+ # def hash_key(key); end
48
49
  # def hash_set(h, key, value); end
49
50
  # def array_start(); end
50
51
  # def array_end(); end
@@ -63,6 +64,12 @@ module Oj
63
64
  #
64
65
  # hash_end
65
66
  #
67
+ # When a hash key is encountered the hash_key method is called with the parsed
68
+ # hash value key. The return value from the call is then used as the key in
69
+ # the key-value pair that follows.
70
+ #
71
+ # hash_key
72
+ #
66
73
  # At the end of a JSON object element the hash_end() callback is called if public.
67
74
  #
68
75
  # hash_set
@@ -112,6 +119,10 @@ module Oj
112
119
  def hash_end()
113
120
  end
114
121
 
122
+ def hash_key(key)
123
+ key
124
+ end
125
+
115
126
  def hash_set(h, key, value)
116
127
  end
117
128
 
@@ -1,5 +1,5 @@
1
1
 
2
2
  module Oj
3
3
  # Current version of the module.
4
- VERSION = '2.9.9'
4
+ VERSION = '2.10.0'
5
5
  end
@@ -1,3 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
1
3
  #!/usr/bin/env ruby
2
4
  # encoding: UTF-8
3
5
 
@@ -5,60 +7,47 @@ $: << File.dirname(__FILE__)
5
7
 
6
8
  require 'helper'
7
9
 
8
- require 'oj'
9
- require 'securerandom'
10
-
11
10
  class Handler
12
- def hash_start() {} end
13
- def hash_set(h,k,v) h.store(k,v) end
14
- def array_start() [] end
15
- def array_append(a,v) a << v end
16
- def error(message, line, column)
17
- raise Exception.new(message, line, column)
11
+ def initialize
12
+ @state = []
13
+ end
14
+
15
+ def hash_start
16
+ @state << {}
17
+ @state.last
18
+ end
19
+
20
+ def hash_end
21
+ @state.pop
22
+ end
23
+
24
+ def hash_set(h,k,v)
25
+ h.store(k,v)
18
26
  end
19
- end
20
27
 
21
- json = Oj.dump({"this"=>"object"})
22
-
23
- if true
24
- name = "/tmp/#{SecureRandom.uuid}"
25
- `mkfifo #{name}`
26
- if fork
27
- open(name, 'r+') do |read_io|
28
- p "start reading #{read_io.stat.ftype}"
29
- Oj.sc_parse(Handler.new, read_io) {|v| p v}
30
- p "stop reading"
31
- end
32
- else
33
- open(name, 'w+') do |write_io|
34
- p "start writing #{write_io.stat.ftype} autoclose: #{write_io.autoclose?}"
35
- write_io.write json
36
- write_io.write json
37
- p "stop writing"
38
- end
39
- sleep(1) # make it obvious that there are two threads
40
- open(name, 'w+') do |write_io|
41
- p "start writing #{write_io.stat.ftype}"
42
- write_io.write json
43
- write_io.write json
44
- p "stop writing"
45
- end
28
+ def array_start
29
+ @state << []
30
+ @state.last
46
31
  end
47
- else
48
- IO.pipe do |read_io, write_io|
49
- if fork
50
- write_io.close
51
- p "start reading #{read_io.stat.ftype}"
52
- Oj.sc_parse(Handler.new, read_io) {|v| p v}
53
- p "stop reading"
54
- read_io.close
55
- else
56
- read_io.close
57
- p "start writing #{write_io.stat.ftype}"
58
- write_io.write json
59
- write_io.write json
60
- p "stop writing"
61
- write_io.close
62
- end
32
+
33
+
34
+ def array_end
35
+ @state.pop
36
+ end
37
+
38
+ def array_append(a,v)
39
+ a << v
40
+ end
41
+
42
+ def add_value(v)
43
+ p v
63
44
  end
45
+
46
+ def error(message, line, column); p "ERROR: #{message}" end
64
47
  end
48
+
49
+ $handler = Handler.new
50
+
51
+ IO.popen("cat tst") { |p| puts Oj.sc_parse($handler, p) }
52
+
53
+ #File.open('tst', 'r') { |file| Oj.sc_parse($handler, file) }
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #!/usr/bin/env ruby
4
+ # encoding: UTF-8
5
+
6
+ $: << File.dirname(__FILE__)
7
+
8
+ require 'helper'
9
+
10
+
@@ -0,0 +1,48 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: UTF-8
3
+
4
+ $: << File.dirname(__FILE__)
5
+
6
+ require 'helper'
7
+
8
+ class Handler
9
+ def initialize
10
+ @state = []
11
+ end
12
+
13
+ def hash_start
14
+ @state << {}
15
+ @state.last
16
+ end
17
+
18
+ def hash_end
19
+ @state.pop
20
+ end
21
+
22
+ def hash_set(h,k,v)
23
+ h.store(k,v)
24
+ end
25
+
26
+ def array_start
27
+ @state << []
28
+ @state.last
29
+ end
30
+
31
+
32
+ def array_end
33
+ @state.pop
34
+ end
35
+
36
+ def array_append(a,v)
37
+ a << v
38
+ end
39
+
40
+ def error(message, line, column); p "ERROR: #{message}" end
41
+ end
42
+
43
+ handler = Handler.new
44
+ def handler.add_value(v)
45
+ p v
46
+ end
47
+
48
+ Oj.sc_parse(handler, StringIO.new('{"a":"b","c":[1,2,{"d":"e"}]}[4,5,6]'))
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: UTF-8
3
+
4
+ # Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
5
+ # required. That can be set in the RUBYOPT environment variable.
6
+ # export RUBYOPT=-w
7
+
8
+ $VERBOSE = true
9
+
10
+ $: << File.join(File.dirname(__FILE__), "../lib")
11
+ $: << File.join(File.dirname(__FILE__), "../ext")
12
+
13
+ require 'oj'
14
+
15
+ A = Struct.new(:a,:b,:c,:d)
16
+ B = Struct.new(:e,:f)
17
+
18
+ obj = [A.new(55, B.new(1, 'X'), B.new(2, 'Y'), 3)]
19
+
20
+ s = Oj.dump(obj, :mode => :object)
21
+
22
+ 100000.times do
23
+ Oj.load(s, :mode => :object)
24
+ # ds = Oj.dump(o, :mode => :object)
25
+ # if ds != s
26
+ # puts ds
27
+ # raise "holy crap"
28
+ # end
29
+ end
@@ -90,6 +90,8 @@ class CompatJuice < Minitest::Test
90
90
  dump_and_load(12345.6789, false)
91
91
  dump_and_load(70.35, false)
92
92
  dump_and_load(-54321.012, false)
93
+ dump_and_load(1.7775, false)
94
+ dump_and_load(2.5024, false)
93
95
  dump_and_load(2.48e16, false)
94
96
  dump_and_load(2.48e100 * 1.0e10, false)
95
97
  dump_and_load(-2.48e100 * 1.0e10, false)
@@ -85,6 +85,8 @@ class FileJuice < Minitest::Test
85
85
  dump_and_load(12345.6789, false)
86
86
  dump_and_load(70.35, false)
87
87
  dump_and_load(-54321.012, false)
88
+ dump_and_load(1.7775, false)
89
+ dump_and_load(2.5024, false)
88
90
  dump_and_load(2.48e16, false)
89
91
  dump_and_load(2.48e100 * 1.0e10, false)
90
92
  dump_and_load(-2.48e100 * 1.0e10, false)
@@ -163,6 +163,8 @@ class ObjectJuice < Minitest::Test
163
163
  dump_and_load(12345.6789, false)
164
164
  dump_and_load(70.35, false)
165
165
  dump_and_load(-54321.012, false)
166
+ dump_and_load(1.7775, false)
167
+ dump_and_load(2.5024, false)
166
168
  dump_and_load(2.48e16, false)
167
169
  dump_and_load(2.48e100 * 1.0e10, false)
168
170
  dump_and_load(-2.48e100 * 1.0e10, false)
@@ -41,6 +41,13 @@ class AllHandler < Oj::ScHandler
41
41
  @calls << [:hash_end]
42
42
  end
43
43
 
44
+ def hash_key(key)
45
+ @calls << [:hash_key, key]
46
+ return 'too' if 'two' == key
47
+ return :symbol if 'symbol' == key
48
+ key
49
+ end
50
+
44
51
  def array_start()
45
52
  @calls << [:array_start]
46
53
  []
@@ -159,8 +166,10 @@ class ScpTest < Minitest::Test
159
166
  json = %{{"one":true,"two":false}}
160
167
  Oj.sc_parse(handler, json)
161
168
  assert_equal([[:hash_start],
169
+ [:hash_key, 'one'],
162
170
  [:hash_set, 'one', true],
163
- [:hash_set, 'two', false],
171
+ [:hash_key, 'two'],
172
+ [:hash_set, 'too', false],
164
173
  [:hash_end],
165
174
  [:add_value, {}]], handler.calls)
166
175
  end
@@ -170,8 +179,23 @@ class ScpTest < Minitest::Test
170
179
  json = %{{"one":true,"two":false}}
171
180
  Oj.sc_parse(handler, json, :symbol_keys => true)
172
181
  assert_equal([[:hash_start],
173
- [:hash_set, :one, true],
174
- [:hash_set, :two, false],
182
+ [:hash_key, 'one'],
183
+ [:hash_set, 'one', true],
184
+ [:hash_key, 'two'],
185
+ [:hash_set, 'too', false],
186
+ [:hash_end],
187
+ [:add_value, {}]], handler.calls)
188
+ end
189
+
190
+ def test_symbol_hash_key_without_symbol_keys
191
+ handler = AllHandler.new()
192
+ json = %{{"one":true,"symbol":false}}
193
+ Oj.sc_parse(handler, json)
194
+ assert_equal([[:hash_start],
195
+ [:hash_key, 'one'],
196
+ [:hash_set, 'one', true],
197
+ [:hash_key, 'symbol'],
198
+ [:hash_set, :symbol, false],
175
199
  [:hash_end],
176
200
  [:add_value, {}]], handler.calls)
177
201
  end
@@ -180,12 +204,18 @@ class ScpTest < Minitest::Test
180
204
  handler = AllHandler.new()
181
205
  Oj.sc_parse(handler, $json)
182
206
  assert_equal([[:hash_start],
207
+ [:hash_key, 'array'],
183
208
  [:array_start],
184
209
  [:hash_start],
210
+ [:hash_key, 'num'],
185
211
  [:hash_set, "num", 3],
212
+ [:hash_key, 'string'],
186
213
  [:hash_set, "string", "message"],
214
+ [:hash_key, 'hash'],
187
215
  [:hash_start],
216
+ [:hash_key, 'h2'],
188
217
  [:hash_start],
218
+ [:hash_key, 'a'],
189
219
  [:array_start],
190
220
  [:array_append, 1],
191
221
  [:array_append, 2],
@@ -200,6 +230,7 @@ class ScpTest < Minitest::Test
200
230
  [:array_append, {}],
201
231
  [:array_end],
202
232
  [:hash_set, "array", []],
233
+ [:hash_key, 'boolean'],
203
234
  [:hash_set, "boolean", true],
204
235
  [:hash_end],
205
236
  [:add_value, {}]], handler.calls)
@@ -210,12 +241,16 @@ class ScpTest < Minitest::Test
210
241
  json = %{{"one":true,"two":false}{"three":true,"four":false}}
211
242
  Oj.sc_parse(handler, json)
212
243
  assert_equal([[:hash_start],
244
+ [:hash_key, 'one'],
213
245
  [:hash_set, 'one', true],
214
- [:hash_set, 'two', false],
246
+ [:hash_key, 'two'],
247
+ [:hash_set, 'too', false],
215
248
  [:hash_end],
216
249
  [:add_value, {}],
217
250
  [:hash_start],
251
+ [:hash_key, 'three'],
218
252
  [:hash_set, 'three', true],
253
+ [:hash_key, 'four'],
219
254
  [:hash_set, 'four', false],
220
255
  [:hash_end],
221
256
  [:add_value, {}]], handler.calls)
@@ -253,8 +288,10 @@ class ScpTest < Minitest::Test
253
288
  Oj.sc_parse(handler, read_io) {|v| p v}
254
289
  read_io.close
255
290
  assert_equal([[:hash_start],
291
+ [:hash_key, 'one'],
256
292
  [:hash_set, 'one', true],
257
- [:hash_set, 'two', false],
293
+ [:hash_key, 'two'],
294
+ [:hash_set, 'too', false],
258
295
  [:hash_end],
259
296
  [:add_value, {}]], handler.calls)
260
297
  else