oj 3.16.11 → 3.17.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +133 -0
- data/README.md +0 -16
- data/ext/oj/cache.c +17 -1
- data/ext/oj/cache.h +5 -0
- data/ext/oj/circarray.c +6 -2
- data/ext/oj/compat.c +5 -5
- data/ext/oj/custom.c +12 -5
- data/ext/oj/dump.c +433 -206
- data/ext/oj/dump.h +12 -0
- data/ext/oj/dump_compat.c +17 -5
- data/ext/oj/dump_object.c +25 -10
- data/ext/oj/dump_strict.c +13 -8
- data/ext/oj/extconf.rb +11 -7
- data/ext/oj/fast.c +123 -60
- data/ext/oj/intern.c +7 -2
- data/ext/oj/mimic_json.c +3 -0
- data/ext/oj/object.c +26 -15
- data/ext/oj/odd.c +6 -2
- data/ext/oj/oj.c +511 -22
- data/ext/oj/oj.h +61 -53
- data/ext/oj/parse.c +229 -30
- data/ext/oj/parse.h +0 -1
- data/ext/oj/parser.c +226 -65
- data/ext/oj/rails.c +197 -57
- data/ext/oj/reader.c +18 -6
- data/ext/oj/resolve.c +11 -2
- data/ext/oj/rxclass.c +17 -1
- data/ext/oj/rxclass.h +2 -1
- data/ext/oj/safe.c +244 -0
- data/ext/oj/safe.h +91 -0
- data/ext/oj/saj.c +56 -12
- data/ext/oj/simd.h +210 -1
- data/ext/oj/sparse.c +22 -9
- data/ext/oj/stream_writer.c +16 -1
- data/ext/oj/string_writer.c +23 -7
- data/ext/oj/usual.c +23 -6
- data/ext/oj/util.c +27 -0
- data/ext/oj/util.h +2 -1
- data/ext/oj/val_stack.h +24 -5
- data/ext/oj/wab.c +15 -27
- data/lib/oj/version.rb +1 -1
- data/pages/Compatibility.md +1 -1
- data/pages/Custom.md +2 -3
- data/pages/InstallOptions.md +8 -6
- data/pages/Modes.md +1 -1
- data/pages/Options.md +5 -6
- data/pages/Parser.md +2 -2
- data/pages/Rails.md +6 -0
- data/pages/Security.md +21 -7
- metadata +21 -19
data/ext/oj/safe.c
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
|
|
2
|
+
#include "safe.h"
|
|
3
|
+
|
|
4
|
+
static VALUE max_hash_size_sym, max_array_size_sym, max_depth_sym, max_total_elements_sym, max_hash_size_error_class,
|
|
5
|
+
max_array_size_error_class, max_depth_error_class, max_total_elements_error_class;
|
|
6
|
+
|
|
7
|
+
static void check_object_size(safe_T safe) {
|
|
8
|
+
if (!safe->max_hash_size_isset) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
struct _usual usual = safe->usual;
|
|
13
|
+
Col current_object_location = usual.ctail - 1;
|
|
14
|
+
|
|
15
|
+
long int number_of_items_in_stack = usual.vtail - usual.vhead;
|
|
16
|
+
long int number_of_items_in_hash = (number_of_items_in_stack - current_object_location->vi - 1) / 2;
|
|
17
|
+
|
|
18
|
+
if (safe->max_hash_size > number_of_items_in_hash) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
rb_raise(max_hash_size_error_class, "Too many object items!");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
static void check_array_size(safe_T safe) {
|
|
26
|
+
if (!safe->max_array_size_isset) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
struct _usual usual = safe->usual;
|
|
31
|
+
Col current_object_location = usual.ctail - 1;
|
|
32
|
+
|
|
33
|
+
long int number_of_items_in_stack = usual.vtail - usual.vhead;
|
|
34
|
+
long int number_of_items_in_array = number_of_items_in_stack - current_object_location->vi - 1;
|
|
35
|
+
|
|
36
|
+
if (safe->max_array_size > number_of_items_in_array) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
rb_raise(max_array_size_error_class, "Too many array items!");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
static void check_max_depth(safe_T safe, ojParser p) {
|
|
44
|
+
if (!safe->max_depth_isset || safe->max_depth >= (p->depth + 1)) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
rb_raise(max_depth_error_class, "JSON is too deep!");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
static void check_max_total_elements(safe_T safe) {
|
|
52
|
+
/*
|
|
53
|
+
* We check if `max_total_elements` is greater than `current_elements_count`
|
|
54
|
+
* (instead of greater than or equal) because top-level elements (e.g., [],
|
|
55
|
+
* null, true) are not counted. As a result, `current_elements_count`
|
|
56
|
+
* always holds one less than the actual total.
|
|
57
|
+
*/
|
|
58
|
+
if (!safe->max_total_elements_isset || safe->max_total_elements > safe->current_elements_count) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
rb_raise(max_total_elements_error_class, "Too many elements!");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
static void safe_start(ojParser p) {
|
|
66
|
+
safe_T safe = (safe_T)p->ctx;
|
|
67
|
+
|
|
68
|
+
safe->current_hash_size = 0;
|
|
69
|
+
safe->current_array_size = 0;
|
|
70
|
+
safe->current_elements_count = 0;
|
|
71
|
+
|
|
72
|
+
safe->delegated_start_func(p);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
static void safe_open_object(ojParser p) {
|
|
76
|
+
safe_T safe = (safe_T)p->ctx;
|
|
77
|
+
|
|
78
|
+
safe->current_hash_size++;
|
|
79
|
+
safe->current_elements_count++;
|
|
80
|
+
|
|
81
|
+
check_array_size(safe);
|
|
82
|
+
check_max_depth(safe, p);
|
|
83
|
+
check_max_total_elements(safe);
|
|
84
|
+
|
|
85
|
+
safe->delegated_open_object_func(p);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static void safe_open_array(ojParser p) {
|
|
89
|
+
safe_T safe = (safe_T)p->ctx;
|
|
90
|
+
|
|
91
|
+
safe->current_array_size++;
|
|
92
|
+
safe->current_elements_count++;
|
|
93
|
+
|
|
94
|
+
check_array_size(safe);
|
|
95
|
+
check_max_depth(safe, p);
|
|
96
|
+
check_max_total_elements(safe);
|
|
97
|
+
|
|
98
|
+
safe->delegated_open_array_func(p);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
DEFINE_DELEGATED_FUNCTION(add_null);
|
|
102
|
+
DEFINE_DELEGATED_FUNCTION(add_true);
|
|
103
|
+
DEFINE_DELEGATED_FUNCTION(add_false);
|
|
104
|
+
DEFINE_DELEGATED_FUNCTION(add_int);
|
|
105
|
+
DEFINE_DELEGATED_FUNCTION(add_float);
|
|
106
|
+
DEFINE_DELEGATED_FUNCTION(add_big);
|
|
107
|
+
DEFINE_DELEGATED_FUNCTION(add_str);
|
|
108
|
+
|
|
109
|
+
static void safe_open_object_key(ojParser p) {
|
|
110
|
+
safe_T safe = (safe_T)p->ctx;
|
|
111
|
+
|
|
112
|
+
safe->current_hash_size++;
|
|
113
|
+
safe->current_elements_count += 2;
|
|
114
|
+
|
|
115
|
+
check_object_size(safe);
|
|
116
|
+
check_max_depth(safe, p);
|
|
117
|
+
check_max_total_elements(safe);
|
|
118
|
+
|
|
119
|
+
safe->delegated_open_object_key_func(p);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
static void safe_open_array_key(ojParser p) {
|
|
123
|
+
safe_T safe = (safe_T)p->ctx;
|
|
124
|
+
|
|
125
|
+
safe->current_array_size++;
|
|
126
|
+
safe->current_elements_count += 2;
|
|
127
|
+
|
|
128
|
+
check_object_size(safe);
|
|
129
|
+
check_max_depth(safe, p);
|
|
130
|
+
check_max_total_elements(safe);
|
|
131
|
+
|
|
132
|
+
safe->delegated_open_array_key_func(p);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_null);
|
|
136
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_true);
|
|
137
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_false);
|
|
138
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_int);
|
|
139
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_float);
|
|
140
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_big);
|
|
141
|
+
DEFINE_DELEGATED_OBJECT_FUNCTION(add_str);
|
|
142
|
+
|
|
143
|
+
// The counting wrappers live in the parser function table, and the option
|
|
144
|
+
// setters of the usual parser write to the same slots, so an option set after
|
|
145
|
+
// the parser was built puts the unwrapped handler back and the limits stop
|
|
146
|
+
// being applied for that kind of value.
|
|
147
|
+
#define WRAP(funcs, slot, wrapper, saved) \
|
|
148
|
+
do { \
|
|
149
|
+
if (wrapper != (funcs)->slot) { \
|
|
150
|
+
safe->saved = (funcs)->slot; \
|
|
151
|
+
} \
|
|
152
|
+
(funcs)->slot = wrapper; \
|
|
153
|
+
} while (0)
|
|
154
|
+
|
|
155
|
+
static void wrap_funcs(ojParser p, safe_T safe) {
|
|
156
|
+
Funcs f = &p->funcs[ARRAY_FUN];
|
|
157
|
+
|
|
158
|
+
WRAP(f, open_object, safe_open_object, delegated_open_object_func);
|
|
159
|
+
WRAP(f, open_array, safe_open_array, delegated_open_array_func);
|
|
160
|
+
// The following overrides are done for counting objects
|
|
161
|
+
WRAP(f, add_null, safe_add_null, delegated_add_null_func);
|
|
162
|
+
WRAP(f, add_true, safe_add_true, delegated_add_true_func);
|
|
163
|
+
WRAP(f, add_false, safe_add_false, delegated_add_false_func);
|
|
164
|
+
WRAP(f, add_int, safe_add_int, delegated_add_int_func);
|
|
165
|
+
WRAP(f, add_float, safe_add_float, delegated_add_float_func);
|
|
166
|
+
WRAP(f, add_big, safe_add_big, delegated_add_big_func);
|
|
167
|
+
WRAP(f, add_str, safe_add_str, delegated_add_str_func);
|
|
168
|
+
|
|
169
|
+
f = &p->funcs[OBJECT_FUN];
|
|
170
|
+
WRAP(f, open_object, safe_open_object_key, delegated_open_object_key_func);
|
|
171
|
+
WRAP(f, open_array, safe_open_array_key, delegated_open_array_key_func);
|
|
172
|
+
WRAP(f, add_null, safe_add_null_key, delegated_add_null_key_func);
|
|
173
|
+
WRAP(f, add_true, safe_add_true_key, delegated_add_true_key_func);
|
|
174
|
+
WRAP(f, add_false, safe_add_false_key, delegated_add_false_key_func);
|
|
175
|
+
WRAP(f, add_int, safe_add_int_key, delegated_add_int_key_func);
|
|
176
|
+
WRAP(f, add_float, safe_add_float_key, delegated_add_float_key_func);
|
|
177
|
+
WRAP(f, add_big, safe_add_big_key, delegated_add_big_key_func);
|
|
178
|
+
WRAP(f, add_str, safe_add_str_key, delegated_add_str_key_func);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
static VALUE safe_option(ojParser p, const char *key, VALUE value) {
|
|
182
|
+
safe_T safe = (safe_T)p->ctx;
|
|
183
|
+
VALUE rv;
|
|
184
|
+
|
|
185
|
+
if (0 == strcmp("omit_null", key)) {
|
|
186
|
+
return safe->omit_null ? Qtrue : Qfalse;
|
|
187
|
+
}
|
|
188
|
+
rv = safe->delegated_option_func(p, key, value);
|
|
189
|
+
if (0 == strcmp("omit_null=", key)) {
|
|
190
|
+
safe->omit_null = (Qtrue == rv);
|
|
191
|
+
}
|
|
192
|
+
wrap_funcs(p, safe);
|
|
193
|
+
|
|
194
|
+
return rv;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
void oj_init_safe_parser(ojParser p, safe_T safe, VALUE options) {
|
|
198
|
+
// Safe parser inherits all members of usual parser
|
|
199
|
+
oj_init_usual(p, &safe->usual);
|
|
200
|
+
|
|
201
|
+
safe->delegated_start_func = p->start;
|
|
202
|
+
p->start = safe_start;
|
|
203
|
+
safe->omit_null = false;
|
|
204
|
+
|
|
205
|
+
wrap_funcs(p, safe);
|
|
206
|
+
|
|
207
|
+
safe->delegated_option_func = p->option;
|
|
208
|
+
p->option = safe_option;
|
|
209
|
+
|
|
210
|
+
SET_CONFIG(max_hash_size);
|
|
211
|
+
SET_CONFIG(max_array_size);
|
|
212
|
+
SET_CONFIG(max_depth);
|
|
213
|
+
SET_CONFIG(max_total_elements);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
void oj_set_parser_safe(ojParser p, VALUE options) {
|
|
217
|
+
safe_T s = OJ_R_ALLOC(struct _safe_S);
|
|
218
|
+
|
|
219
|
+
oj_init_safe_parser(p, s, options);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
void oj_safe_init(VALUE parser_class) {
|
|
223
|
+
VALUE validation_error_class = rb_define_class_under(parser_class, "ValidationError", rb_eRuntimeError);
|
|
224
|
+
|
|
225
|
+
max_hash_size_error_class = rb_define_class_under(parser_class, "HashSizeError", validation_error_class);
|
|
226
|
+
max_array_size_error_class = rb_define_class_under(parser_class, "ArraySizeError", validation_error_class);
|
|
227
|
+
max_depth_error_class = rb_define_class_under(parser_class, "DepthError", validation_error_class);
|
|
228
|
+
max_total_elements_error_class = rb_define_class_under(parser_class, "TotalElementsError", validation_error_class);
|
|
229
|
+
|
|
230
|
+
rb_gc_register_address(&max_hash_size_error_class);
|
|
231
|
+
rb_gc_register_address(&max_array_size_error_class);
|
|
232
|
+
rb_gc_register_address(&max_depth_error_class);
|
|
233
|
+
rb_gc_register_address(&max_total_elements_error_class);
|
|
234
|
+
|
|
235
|
+
max_hash_size_sym = ID2SYM(rb_intern("max_hash_size"));
|
|
236
|
+
max_array_size_sym = ID2SYM(rb_intern("max_array_size"));
|
|
237
|
+
max_depth_sym = ID2SYM(rb_intern("max_depth"));
|
|
238
|
+
max_total_elements_sym = ID2SYM(rb_intern("max_total_elements"));
|
|
239
|
+
|
|
240
|
+
rb_gc_register_address(&max_hash_size_sym);
|
|
241
|
+
rb_gc_register_address(&max_array_size_sym);
|
|
242
|
+
rb_gc_register_address(&max_depth_sym);
|
|
243
|
+
rb_gc_register_address(&max_total_elements_sym);
|
|
244
|
+
}
|
data/ext/oj/safe.h
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#include <ruby.h>
|
|
2
|
+
|
|
3
|
+
#include "parser.h"
|
|
4
|
+
#include "usual.h"
|
|
5
|
+
|
|
6
|
+
#define SET_CONFIG(config_name) \
|
|
7
|
+
do { \
|
|
8
|
+
VALUE rb_##config_name = rb_hash_aref(options, config_name##_sym); \
|
|
9
|
+
\
|
|
10
|
+
if (RB_INTEGER_TYPE_P(rb_##config_name)) { \
|
|
11
|
+
safe->config_name = NUM2LONG(rb_##config_name); \
|
|
12
|
+
safe->config_name##_isset = true; \
|
|
13
|
+
} else if (!NIL_P(rb_##config_name)) { \
|
|
14
|
+
rb_raise(rb_eArgError, "Incorrect value provided for `" #config_name "`"); \
|
|
15
|
+
} else { \
|
|
16
|
+
safe->config_name = 0; \
|
|
17
|
+
safe->config_name##_isset = false; \
|
|
18
|
+
} \
|
|
19
|
+
} while (0);
|
|
20
|
+
|
|
21
|
+
#define DEFINE_DELEGATED_FUNCTION(function_name) \
|
|
22
|
+
static void safe_##function_name(ojParser p) { \
|
|
23
|
+
safe_T safe = (safe_T)p->ctx; \
|
|
24
|
+
\
|
|
25
|
+
safe->current_elements_count++; \
|
|
26
|
+
\
|
|
27
|
+
check_array_size(safe); \
|
|
28
|
+
check_max_total_elements(safe); \
|
|
29
|
+
\
|
|
30
|
+
safe->delegated_##function_name##_func(p); \
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
#define DEFINE_DELEGATED_OBJECT_FUNCTION(function_name) \
|
|
34
|
+
static void safe_##function_name##_key(ojParser p) { \
|
|
35
|
+
safe_T safe = (safe_T)p->ctx; \
|
|
36
|
+
\
|
|
37
|
+
safe->current_elements_count += 2; \
|
|
38
|
+
\
|
|
39
|
+
check_object_size(safe); \
|
|
40
|
+
check_max_total_elements(safe); \
|
|
41
|
+
\
|
|
42
|
+
safe->delegated_##function_name##_key_func(p); \
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
typedef struct _safe_S {
|
|
46
|
+
struct _usual usual;
|
|
47
|
+
|
|
48
|
+
long int max_hash_size;
|
|
49
|
+
long int max_array_size;
|
|
50
|
+
long int max_depth;
|
|
51
|
+
long int max_total_elements;
|
|
52
|
+
long int max_json_size_bytes;
|
|
53
|
+
|
|
54
|
+
bool max_hash_size_isset;
|
|
55
|
+
bool max_array_size_isset;
|
|
56
|
+
bool max_depth_isset;
|
|
57
|
+
bool max_total_elements_isset;
|
|
58
|
+
|
|
59
|
+
long int current_hash_size;
|
|
60
|
+
long int current_array_size;
|
|
61
|
+
long int current_elements_count;
|
|
62
|
+
|
|
63
|
+
// omit_null is kept here because the usual parser reports it by looking at
|
|
64
|
+
// the add_null slot, which is one of the slots wrapped below.
|
|
65
|
+
bool omit_null;
|
|
66
|
+
|
|
67
|
+
VALUE (*delegated_option_func)(struct _ojParser *p, const char *key, VALUE value);
|
|
68
|
+
void (*delegated_start_func)(struct _ojParser *p);
|
|
69
|
+
|
|
70
|
+
// Array functions
|
|
71
|
+
void (*delegated_open_object_func)(struct _ojParser *p);
|
|
72
|
+
void (*delegated_open_array_func)(struct _ojParser *p);
|
|
73
|
+
void (*delegated_add_null_func)(struct _ojParser *p);
|
|
74
|
+
void (*delegated_add_true_func)(struct _ojParser *p);
|
|
75
|
+
void (*delegated_add_false_func)(struct _ojParser *p);
|
|
76
|
+
void (*delegated_add_int_func)(struct _ojParser *p);
|
|
77
|
+
void (*delegated_add_float_func)(struct _ojParser *p);
|
|
78
|
+
void (*delegated_add_big_func)(struct _ojParser *p);
|
|
79
|
+
void (*delegated_add_str_func)(struct _ojParser *p);
|
|
80
|
+
|
|
81
|
+
// Object functions
|
|
82
|
+
void (*delegated_open_object_key_func)(struct _ojParser *p);
|
|
83
|
+
void (*delegated_open_array_key_func)(struct _ojParser *p);
|
|
84
|
+
void (*delegated_add_null_key_func)(struct _ojParser *p);
|
|
85
|
+
void (*delegated_add_true_key_func)(struct _ojParser *p);
|
|
86
|
+
void (*delegated_add_false_key_func)(struct _ojParser *p);
|
|
87
|
+
void (*delegated_add_int_key_func)(struct _ojParser *p);
|
|
88
|
+
void (*delegated_add_float_key_func)(struct _ojParser *p);
|
|
89
|
+
void (*delegated_add_big_key_func)(struct _ojParser *p);
|
|
90
|
+
void (*delegated_add_str_key_func)(struct _ojParser *p);
|
|
91
|
+
} *safe_T;
|
data/ext/oj/saj.c
CHANGED
|
@@ -82,7 +82,14 @@ inline static void next_non_white(ParseInfo pi) {
|
|
|
82
82
|
case '\f':
|
|
83
83
|
case '\n':
|
|
84
84
|
case '\r': break;
|
|
85
|
-
case '/':
|
|
85
|
+
case '/':
|
|
86
|
+
skip_comment(pi);
|
|
87
|
+
/* A comment that is not terminated by a newline ends on the null
|
|
88
|
+
* terminator. Stop here so the loop does not step past it. */
|
|
89
|
+
if ('\0' == *pi->s) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
break;
|
|
86
93
|
default: return;
|
|
87
94
|
}
|
|
88
95
|
}
|
|
@@ -118,14 +125,15 @@ static void skip_comment(ParseInfo pi) {
|
|
|
118
125
|
if ('*' == *pi->s && '/' == *(pi->s + 1)) {
|
|
119
126
|
pi->s++;
|
|
120
127
|
return;
|
|
121
|
-
} else if ('\0' == *pi->s) {
|
|
122
|
-
if (pi->has_error) {
|
|
123
|
-
call_error("comment not terminated", pi, __FILE__, __LINE__);
|
|
124
|
-
} else {
|
|
125
|
-
raise_error("comment not terminated", pi->str, pi->s);
|
|
126
|
-
}
|
|
127
128
|
}
|
|
128
129
|
}
|
|
130
|
+
/* The loop only ends on the null terminator so the comment was never
|
|
131
|
+
* closed. */
|
|
132
|
+
if (pi->has_error) {
|
|
133
|
+
call_error("comment not terminated", pi, __FILE__, __LINE__);
|
|
134
|
+
} else {
|
|
135
|
+
raise_error("comment not terminated", pi->str, pi->s);
|
|
136
|
+
}
|
|
129
137
|
} else if ('/' == *pi->s) {
|
|
130
138
|
for (; 1; pi->s++) {
|
|
131
139
|
switch (*pi->s) {
|
|
@@ -190,6 +198,12 @@ static void read_hash(ParseInfo pi, const char *key) {
|
|
|
190
198
|
} else {
|
|
191
199
|
while (1) {
|
|
192
200
|
next_non_white(pi);
|
|
201
|
+
if ('"' != *pi->s) {
|
|
202
|
+
if (pi->has_error) {
|
|
203
|
+
call_error("invalid format, expected a key", pi, __FILE__, __LINE__);
|
|
204
|
+
}
|
|
205
|
+
raise_error("invalid format, expected a key", pi->str, pi->s);
|
|
206
|
+
}
|
|
193
207
|
ks = read_quoted_value(pi);
|
|
194
208
|
next_non_white(pi);
|
|
195
209
|
if (':' == *pi->s) {
|
|
@@ -606,6 +620,19 @@ static void saj_parse(VALUE handler, char *json) {
|
|
|
606
620
|
}
|
|
607
621
|
}
|
|
608
622
|
|
|
623
|
+
struct _sajArgs {
|
|
624
|
+
VALUE handler;
|
|
625
|
+
char *json;
|
|
626
|
+
};
|
|
627
|
+
|
|
628
|
+
static VALUE protect_saj_parse(VALUE x) {
|
|
629
|
+
struct _sajArgs *args = (struct _sajArgs *)x;
|
|
630
|
+
|
|
631
|
+
saj_parse(args->handler, args->json);
|
|
632
|
+
|
|
633
|
+
return Qnil;
|
|
634
|
+
}
|
|
635
|
+
|
|
609
636
|
/* call-seq: saj_parse(handler, io)
|
|
610
637
|
*
|
|
611
638
|
* Parses an IO stream or file containing an JSON document. Raises an exception
|
|
@@ -648,8 +675,16 @@ oj_saj_parse(int argc, VALUE *argv, VALUE self) {
|
|
|
648
675
|
len = lseek(fd, 0, SEEK_END);
|
|
649
676
|
lseek(fd, 0, SEEK_SET);
|
|
650
677
|
json = OJ_R_ALLOC_N(char, len + 1);
|
|
651
|
-
|
|
652
|
-
|
|
678
|
+
{
|
|
679
|
+
size_t total = 0;
|
|
680
|
+
|
|
681
|
+
while (total < len) {
|
|
682
|
+
cnt = read(fd, json + total, len - total);
|
|
683
|
+
if (cnt <= 0) {
|
|
684
|
+
rb_raise(rb_eIOError, "failed to read from IO Object.");
|
|
685
|
+
}
|
|
686
|
+
total += cnt;
|
|
687
|
+
}
|
|
653
688
|
}
|
|
654
689
|
json[len] = '\0';
|
|
655
690
|
#endif
|
|
@@ -662,8 +697,17 @@ oj_saj_parse(int argc, VALUE *argv, VALUE self) {
|
|
|
662
697
|
rb_raise(rb_eArgError, "saj_parse() expected a String or IO Object.");
|
|
663
698
|
}
|
|
664
699
|
}
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
700
|
+
{
|
|
701
|
+
// saj_parse() raises on a malformed document so the json buffer has to
|
|
702
|
+
// be freed even when the parse does not return normally.
|
|
703
|
+
struct _sajArgs args = {*argv, json};
|
|
704
|
+
int ex = 0;
|
|
705
|
+
|
|
706
|
+
rb_protect(protect_saj_parse, (VALUE)&args, &ex);
|
|
707
|
+
OJ_R_FREE(json);
|
|
708
|
+
if (0 != ex) {
|
|
709
|
+
rb_jump_tag(ex);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
668
712
|
return Qnil;
|
|
669
713
|
}
|
data/ext/oj/simd.h
CHANGED
|
@@ -1,10 +1,219 @@
|
|
|
1
1
|
#ifndef OJ_SIMD_H
|
|
2
2
|
#define OJ_SIMD_H
|
|
3
3
|
|
|
4
|
+
// SIMD architecture detection and configuration
|
|
5
|
+
// This header provides unified SIMD support across different CPU architectures
|
|
6
|
+
// with cross-platform runtime detection (Windows/Linux/Mac)
|
|
7
|
+
|
|
8
|
+
// SIMD implementation enum - used for runtime selection
|
|
9
|
+
typedef enum _simd_implementation { SIMD_NONE, SIMD_NEON, SIMD_SSE2, SIMD_SSE42 } SIMD_Implementation;
|
|
10
|
+
|
|
11
|
+
// Define in oj.c.
|
|
12
|
+
extern SIMD_Implementation SIMD_Impl;
|
|
13
|
+
|
|
14
|
+
// Runtime CPU detection function (implemented in oj.c)
|
|
15
|
+
SIMD_Implementation oj_get_simd_implementation(void);
|
|
16
|
+
|
|
17
|
+
// =============================================================================
|
|
18
|
+
// Compiler compatibility macros
|
|
19
|
+
// =============================================================================
|
|
20
|
+
|
|
21
|
+
// Branch prediction hints
|
|
22
|
+
#if defined(__GNUC__) || defined(__clang__)
|
|
23
|
+
#define OJ_LIKELY(x) __builtin_expect(!!(x), 1)
|
|
24
|
+
#define OJ_UNLIKELY(x) __builtin_expect(!!(x), 0)
|
|
25
|
+
#else
|
|
26
|
+
#define OJ_LIKELY(x) (x)
|
|
27
|
+
#define OJ_UNLIKELY(x) (x)
|
|
28
|
+
#endif
|
|
29
|
+
|
|
30
|
+
// Prefetch hints
|
|
31
|
+
#if defined(__GNUC__) || defined(__clang__)
|
|
32
|
+
#define OJ_PREFETCH(addr) __builtin_prefetch(addr, 0, 0)
|
|
33
|
+
#elif defined(_MSC_VER)
|
|
34
|
+
#include <intrin.h>
|
|
35
|
+
#define OJ_PREFETCH(addr) _mm_prefetch((const char *)(addr), _MM_HINT_T0)
|
|
36
|
+
#else
|
|
37
|
+
#define OJ_PREFETCH(addr) ((void)0)
|
|
38
|
+
#endif
|
|
39
|
+
|
|
40
|
+
// Count trailing zeros (for SSE2 mask scanning)
|
|
41
|
+
#if defined(__GNUC__) || defined(__clang__)
|
|
42
|
+
#define OJ_CTZ(x) __builtin_ctz(x)
|
|
43
|
+
#define OJ_CTZ64(x) __builtin_ctzll(x)
|
|
44
|
+
#elif defined(_MSC_VER)
|
|
45
|
+
#include <intrin.h>
|
|
46
|
+
static __inline int oj_ctz_msvc(unsigned int x) {
|
|
47
|
+
unsigned long index;
|
|
48
|
+
if (0 == x) {
|
|
49
|
+
return 32;
|
|
50
|
+
}
|
|
51
|
+
_BitScanForward(&index, x);
|
|
52
|
+
return (int)index;
|
|
53
|
+
}
|
|
54
|
+
static __inline int oj_ctz64_msvc(uint64_t x) {
|
|
55
|
+
unsigned long index;
|
|
56
|
+
if (_BitScanForward64(&index, x)) {
|
|
57
|
+
return (int)index;
|
|
58
|
+
}
|
|
59
|
+
return 64;
|
|
60
|
+
}
|
|
61
|
+
#define OJ_CTZ(x) oj_ctz_msvc(x)
|
|
62
|
+
#define OJ_CTZ64(x) oj_ctz64_msvc(x)
|
|
63
|
+
#else
|
|
64
|
+
// Fallback: naive implementation
|
|
65
|
+
static inline int oj_ctz_fallback(unsigned int x) {
|
|
66
|
+
int count = 0;
|
|
67
|
+
while ((x & 1) == 0 && count < 32) {
|
|
68
|
+
x >>= 1;
|
|
69
|
+
count++;
|
|
70
|
+
}
|
|
71
|
+
return count;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
static inline int oj_ctz64_fallback(uint64_t x) {
|
|
75
|
+
int count = 0;
|
|
76
|
+
while ((x & 1) == 0 && count < 64) {
|
|
77
|
+
x >>= 1;
|
|
78
|
+
count++;
|
|
79
|
+
}
|
|
80
|
+
return count;
|
|
81
|
+
}
|
|
82
|
+
#define OJ_CTZ(x) oj_ctz_fallback(x)
|
|
83
|
+
#define OJ_CTZ64(x) oj_ctz64_fallback(x)
|
|
84
|
+
#endif
|
|
85
|
+
|
|
86
|
+
// =============================================================================
|
|
87
|
+
// x86/x86_64 SIMD detection
|
|
88
|
+
// =============================================================================
|
|
89
|
+
#if defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
|
|
90
|
+
#define HAVE_SIMD_X86 1
|
|
91
|
+
|
|
92
|
+
// Include appropriate SIMD headers
|
|
93
|
+
#if defined(_MSC_VER)
|
|
94
|
+
// MSVC: use intrin.h for all intrinsics
|
|
95
|
+
#include <intrin.h>
|
|
96
|
+
#define HAVE_SIMD_SSE4_2 1
|
|
97
|
+
#define HAVE_SIMD_SSE2 1
|
|
98
|
+
#elif defined(__GNUC__) || defined(__clang__)
|
|
99
|
+
// GCC/Clang: check for header availability and include them
|
|
100
|
+
// We include headers but use target attributes to enable instructions per-function
|
|
101
|
+
// Include cpuid.h for __get_cpuid fallback when __builtin_cpu_supports is unavailable
|
|
102
|
+
#if __has_include(<cpuid.h>)
|
|
103
|
+
#include <cpuid.h>
|
|
104
|
+
#endif
|
|
105
|
+
#if defined(__SSE4_2__) || defined(__SSE2__)
|
|
106
|
+
// If any SSE is enabled globally, x86intrin.h should be available
|
|
107
|
+
#include <x86intrin.h>
|
|
108
|
+
#define HAVE_SIMD_SSE4_2 1
|
|
109
|
+
#define HAVE_SIMD_SSE2 1
|
|
110
|
+
#else
|
|
111
|
+
// Try to include headers anyway for target attribute functions
|
|
112
|
+
#if __has_include(<x86intrin.h>)
|
|
113
|
+
#include <x86intrin.h>
|
|
114
|
+
#define HAVE_SIMD_SSE4_2 1
|
|
115
|
+
#define HAVE_SIMD_SSE2 1
|
|
116
|
+
#elif __has_include(<nmmintrin.h>)
|
|
117
|
+
#include <nmmintrin.h>
|
|
118
|
+
#define HAVE_SIMD_SSE4_2 1
|
|
119
|
+
#define HAVE_SIMD_SSE2 1
|
|
120
|
+
#elif __has_include(<emmintrin.h>)
|
|
121
|
+
#include <emmintrin.h>
|
|
122
|
+
#define HAVE_SIMD_SSE2 1
|
|
123
|
+
#endif
|
|
124
|
+
#endif
|
|
125
|
+
#endif
|
|
126
|
+
|
|
127
|
+
// Target attribute macros for function-level SIMD enabling
|
|
128
|
+
#if defined(__clang__) || defined(__GNUC__)
|
|
129
|
+
#define OJ_TARGET_SSE42 __attribute__((target("sse4.2")))
|
|
130
|
+
#define OJ_TARGET_SSE2 __attribute__((target("sse2")))
|
|
131
|
+
#else
|
|
132
|
+
// MSVC doesn't need target attributes - intrinsics are always available
|
|
133
|
+
#define OJ_TARGET_SSE42
|
|
134
|
+
#define OJ_TARGET_SSE2
|
|
135
|
+
#endif
|
|
136
|
+
|
|
137
|
+
#endif // x86/x86_64
|
|
138
|
+
|
|
139
|
+
// =============================================================================
|
|
140
|
+
// ARM NEON detection
|
|
141
|
+
// =============================================================================
|
|
4
142
|
#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64)
|
|
5
143
|
#define HAVE_SIMD_NEON 1
|
|
6
144
|
#define SIMD_MINIMUM_THRESHOLD 6
|
|
7
145
|
#include <arm_neon.h>
|
|
8
146
|
#endif
|
|
9
147
|
|
|
10
|
-
|
|
148
|
+
// =============================================================================
|
|
149
|
+
// SIMD type string for debugging/logging
|
|
150
|
+
// =============================================================================
|
|
151
|
+
#if defined(HAVE_SIMD_SSE4_2) || defined(HAVE_SIMD_SSE2)
|
|
152
|
+
#define HAVE_SIMD_STRING_SCAN 1
|
|
153
|
+
#define SIMD_TYPE "x86 (runtime detected)"
|
|
154
|
+
#elif defined(HAVE_SIMD_NEON)
|
|
155
|
+
#define HAVE_SIMD_STRING_SCAN 1
|
|
156
|
+
#define SIMD_TYPE "NEON"
|
|
157
|
+
#else
|
|
158
|
+
#define SIMD_TYPE "none"
|
|
159
|
+
#endif
|
|
160
|
+
|
|
161
|
+
#if defined(HAVE_SIMD_SSE4_2)
|
|
162
|
+
|
|
163
|
+
#define SIMD_MINIMUM_THRESHOLD 6
|
|
164
|
+
|
|
165
|
+
extern void initialize_sse42(void);
|
|
166
|
+
|
|
167
|
+
static inline OJ_TARGET_SSE42 __m128i vector_lookup_sse42(__m128i input, __m128i *lookup_table, int tab_size) {
|
|
168
|
+
// Extract high 4 bits to determine which 16-byte chunk (0-15)
|
|
169
|
+
__m128i hi_index = _mm_and_si128(_mm_srli_epi32(input, 4), _mm_set1_epi8(0x0F));
|
|
170
|
+
|
|
171
|
+
// Extract low 4 bits for index within the chunk (0-15)
|
|
172
|
+
__m128i low_index = _mm_and_si128(input, _mm_set1_epi8(0x0F));
|
|
173
|
+
|
|
174
|
+
// Perform lookups in all 16 tables
|
|
175
|
+
__m128i results[16];
|
|
176
|
+
for (int i = 0; i < tab_size; i++) {
|
|
177
|
+
results[i] = _mm_shuffle_epi8(lookup_table[i], low_index);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Create masks for each chunk and blend results
|
|
181
|
+
__m128i final_result = _mm_setzero_si128();
|
|
182
|
+
|
|
183
|
+
for (int i = 0; i < tab_size; i++) {
|
|
184
|
+
__m128i mask = _mm_cmpeq_epi8(hi_index, _mm_set1_epi8(i));
|
|
185
|
+
__m128i masked_result = _mm_and_si128(mask, results[i]);
|
|
186
|
+
final_result = _mm_or_si128(final_result, masked_result);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return final_result;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
#endif
|
|
193
|
+
|
|
194
|
+
#ifndef __has_builtin
|
|
195
|
+
#define __has_builtin(x) 0
|
|
196
|
+
#endif
|
|
197
|
+
|
|
198
|
+
#if __has_builtin(__builtin_memcpy)
|
|
199
|
+
#define HAVE_FAST_MEMCPY 1
|
|
200
|
+
|
|
201
|
+
inline static void fast_memcpy16(void *dest, const void *src, size_t n) {
|
|
202
|
+
char *d = (char *)dest;
|
|
203
|
+
char *s = (char *)src;
|
|
204
|
+
if (n >= 8) {
|
|
205
|
+
__builtin_memcpy(d, s, 8);
|
|
206
|
+
__builtin_memcpy(d + n - 8, s + n - 8, 8);
|
|
207
|
+
} else if (n >= 4) {
|
|
208
|
+
__builtin_memcpy(d, s, 4);
|
|
209
|
+
__builtin_memcpy(d + n - 4, s + n - 4, 4);
|
|
210
|
+
} else if (n >= 2) {
|
|
211
|
+
__builtin_memcpy(d, s, 2);
|
|
212
|
+
__builtin_memcpy(d + n - 2, s + n - 2, 2);
|
|
213
|
+
} else if (n >= 1) {
|
|
214
|
+
*d = *s;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
#endif
|
|
218
|
+
|
|
219
|
+
#endif /* OJ_SIMD_H */
|