json 2.10.2 → 2.19.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +205 -7
- data/LEGAL +12 -0
- data/README.md +43 -1
- data/ext/json/ext/fbuffer/fbuffer.h +140 -86
- data/ext/json/ext/generator/extconf.rb +8 -0
- data/ext/json/ext/generator/generator.c +787 -616
- data/ext/json/ext/json.h +116 -0
- data/ext/json/ext/parser/extconf.rb +11 -3
- data/ext/json/ext/parser/parser.c +974 -703
- data/ext/json/ext/simd/conf.rb +24 -0
- data/ext/json/ext/simd/simd.h +208 -0
- data/ext/json/ext/vendor/fpconv.c +480 -0
- data/ext/json/ext/vendor/jeaiii-ltoa.h +267 -0
- data/ext/json/ext/vendor/ryu.h +819 -0
- data/json.gemspec +2 -3
- data/lib/json/add/core.rb +1 -0
- data/lib/json/add/string.rb +35 -0
- data/lib/json/common.rb +374 -188
- data/lib/json/ext/generator/state.rb +11 -14
- data/lib/json/ext.rb +2 -2
- data/lib/json/generic_object.rb +0 -8
- data/lib/json/truffle_ruby/generator.rb +137 -72
- data/lib/json/version.rb +1 -1
- data/lib/json.rb +90 -2
- metadata +66 -57
|
@@ -1,61 +1,22 @@
|
|
|
1
|
-
#include "
|
|
2
|
-
#include "
|
|
3
|
-
|
|
4
|
-
/* shims */
|
|
5
|
-
/* This is the fallback definition from Ruby 3.4 */
|
|
6
|
-
|
|
7
|
-
#ifndef RBIMPL_STDBOOL_H
|
|
8
|
-
#if defined(__cplusplus)
|
|
9
|
-
# if defined(HAVE_STDBOOL_H) && (__cplusplus >= 201103L)
|
|
10
|
-
# include <cstdbool>
|
|
11
|
-
# endif
|
|
12
|
-
#elif defined(HAVE_STDBOOL_H)
|
|
13
|
-
# include <stdbool.h>
|
|
14
|
-
#elif !defined(HAVE__BOOL)
|
|
15
|
-
typedef unsigned char _Bool;
|
|
16
|
-
# define bool _Bool
|
|
17
|
-
# define true ((_Bool)+1)
|
|
18
|
-
# define false ((_Bool)+0)
|
|
19
|
-
# define __bool_true_false_are_defined
|
|
20
|
-
#endif
|
|
21
|
-
#endif
|
|
22
|
-
|
|
23
|
-
#ifndef RB_UNLIKELY
|
|
24
|
-
#define RB_UNLIKELY(expr) expr
|
|
25
|
-
#endif
|
|
26
|
-
|
|
27
|
-
#ifndef RB_LIKELY
|
|
28
|
-
#define RB_LIKELY(expr) expr
|
|
29
|
-
#endif
|
|
1
|
+
#include "../json.h"
|
|
2
|
+
#include "../vendor/ryu.h"
|
|
3
|
+
#include "../simd/simd.h"
|
|
30
4
|
|
|
31
5
|
static VALUE mJSON, eNestingError, Encoding_UTF_8;
|
|
32
6
|
static VALUE CNaN, CInfinity, CMinusInfinity;
|
|
33
7
|
|
|
34
|
-
static ID
|
|
35
|
-
i_chr, i_deep_const_get, i_match, i_aset, i_aref,
|
|
36
|
-
i_leftshift, i_new, i_try_convert, i_uminus, i_encode;
|
|
8
|
+
static ID i_new, i_try_convert, i_uminus, i_encode;
|
|
37
9
|
|
|
38
|
-
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma,
|
|
39
|
-
|
|
40
|
-
|
|
10
|
+
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters,
|
|
11
|
+
sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load,
|
|
12
|
+
sym_allow_duplicate_key;
|
|
41
13
|
|
|
42
14
|
static int binary_encindex;
|
|
43
15
|
static int utf8_encindex;
|
|
44
16
|
|
|
45
|
-
#ifdef HAVE_RB_CATEGORY_WARN
|
|
46
|
-
# define json_deprecated(message) rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, message)
|
|
47
|
-
#else
|
|
48
|
-
# define json_deprecated(message) rb_warn(message)
|
|
49
|
-
#endif
|
|
50
|
-
|
|
51
|
-
static const char deprecated_create_additions_warning[] =
|
|
52
|
-
"JSON.load implicit support for `create_additions: true` is deprecated "
|
|
53
|
-
"and will be removed in 3.0, use JSON.unsafe_load or explicitly "
|
|
54
|
-
"pass `create_additions: true`";
|
|
55
|
-
|
|
56
17
|
#ifndef HAVE_RB_HASH_BULK_INSERT
|
|
57
18
|
// For TruffleRuby
|
|
58
|
-
void
|
|
19
|
+
static void
|
|
59
20
|
rb_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash)
|
|
60
21
|
{
|
|
61
22
|
long index = 0;
|
|
@@ -72,6 +33,12 @@ rb_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash)
|
|
|
72
33
|
#define rb_hash_new_capa(n) rb_hash_new()
|
|
73
34
|
#endif
|
|
74
35
|
|
|
36
|
+
#ifndef HAVE_RB_STR_TO_INTERNED_STR
|
|
37
|
+
static VALUE rb_str_to_interned_str(VALUE str)
|
|
38
|
+
{
|
|
39
|
+
return rb_funcall(rb_str_freeze(str), i_uminus, 0);
|
|
40
|
+
}
|
|
41
|
+
#endif
|
|
75
42
|
|
|
76
43
|
/* name cache */
|
|
77
44
|
|
|
@@ -117,116 +84,104 @@ static void rvalue_cache_insert_at(rvalue_cache *cache, int index, VALUE rstring
|
|
|
117
84
|
cache->entries[index] = rstring;
|
|
118
85
|
}
|
|
119
86
|
|
|
120
|
-
|
|
87
|
+
#define rstring_cache_memcmp memcmp
|
|
88
|
+
|
|
89
|
+
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
90
|
+
#if __has_builtin(__builtin_bswap64)
|
|
91
|
+
#undef rstring_cache_memcmp
|
|
92
|
+
ALWAYS_INLINE(static) int rstring_cache_memcmp(const char *str, const char *rptr, const long length)
|
|
93
|
+
{
|
|
94
|
+
// The libc memcmp has numerous complex optimizations, but in this particular case,
|
|
95
|
+
// we know the string is small (JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH), so being able to
|
|
96
|
+
// inline a simpler memcmp outperforms calling the libc version.
|
|
97
|
+
long i = 0;
|
|
98
|
+
|
|
99
|
+
for (; i + 8 <= length; i += 8) {
|
|
100
|
+
uint64_t a, b;
|
|
101
|
+
memcpy(&a, str + i, 8);
|
|
102
|
+
memcpy(&b, rptr + i, 8);
|
|
103
|
+
if (a != b) {
|
|
104
|
+
a = __builtin_bswap64(a);
|
|
105
|
+
b = __builtin_bswap64(b);
|
|
106
|
+
return (a < b) ? -1 : 1;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (; i < length; i++) {
|
|
111
|
+
if (str[i] != rptr[i]) {
|
|
112
|
+
return (str[i] < rptr[i]) ? -1 : 1;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return 0;
|
|
117
|
+
}
|
|
118
|
+
#endif
|
|
119
|
+
#endif
|
|
120
|
+
|
|
121
|
+
ALWAYS_INLINE(static) int rstring_cache_cmp(const char *str, const long length, VALUE rstring)
|
|
121
122
|
{
|
|
122
|
-
|
|
123
|
+
const char *rstring_ptr;
|
|
124
|
+
long rstring_length;
|
|
125
|
+
|
|
126
|
+
RSTRING_GETMEM(rstring, rstring_ptr, rstring_length);
|
|
127
|
+
|
|
123
128
|
if (length == rstring_length) {
|
|
124
|
-
return
|
|
129
|
+
return rstring_cache_memcmp(str, rstring_ptr, length);
|
|
125
130
|
} else {
|
|
126
131
|
return (int)(length - rstring_length);
|
|
127
132
|
}
|
|
128
133
|
}
|
|
129
134
|
|
|
130
|
-
static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length)
|
|
135
|
+
ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length)
|
|
131
136
|
{
|
|
132
|
-
if (RB_UNLIKELY(length > JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH)) {
|
|
133
|
-
// Common names aren't likely to be very long. So we just don't
|
|
134
|
-
// cache names above an arbitrary threshold.
|
|
135
|
-
return Qfalse;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
if (RB_UNLIKELY(!isalpha((unsigned char)str[0]))) {
|
|
139
|
-
// Simple heuristic, if the first character isn't a letter,
|
|
140
|
-
// we're much less likely to see this string again.
|
|
141
|
-
// We mostly want to cache strings that are likely to be repeated.
|
|
142
|
-
return Qfalse;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
137
|
int low = 0;
|
|
146
138
|
int high = cache->length - 1;
|
|
147
|
-
int mid = 0;
|
|
148
|
-
int last_cmp = 0;
|
|
149
139
|
|
|
150
140
|
while (low <= high) {
|
|
151
|
-
mid = (high + low) >> 1;
|
|
141
|
+
int mid = (high + low) >> 1;
|
|
152
142
|
VALUE entry = cache->entries[mid];
|
|
153
|
-
|
|
143
|
+
int cmp = rstring_cache_cmp(str, length, entry);
|
|
154
144
|
|
|
155
|
-
if (
|
|
145
|
+
if (cmp == 0) {
|
|
156
146
|
return entry;
|
|
157
|
-
} else if (
|
|
147
|
+
} else if (cmp > 0) {
|
|
158
148
|
low = mid + 1;
|
|
159
149
|
} else {
|
|
160
150
|
high = mid - 1;
|
|
161
151
|
}
|
|
162
152
|
}
|
|
163
153
|
|
|
164
|
-
if (RB_UNLIKELY(memchr(str, '\\', length))) {
|
|
165
|
-
// We assume the overwhelming majority of names don't need to be escaped.
|
|
166
|
-
// But if they do, we have to fallback to the slow path.
|
|
167
|
-
return Qfalse;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
154
|
VALUE rstring = build_interned_string(str, length);
|
|
171
155
|
|
|
172
156
|
if (cache->length < JSON_RVALUE_CACHE_CAPA) {
|
|
173
|
-
|
|
174
|
-
mid += 1;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
rvalue_cache_insert_at(cache, mid, rstring);
|
|
157
|
+
rvalue_cache_insert_at(cache, low, rstring);
|
|
178
158
|
}
|
|
179
159
|
return rstring;
|
|
180
160
|
}
|
|
181
161
|
|
|
182
162
|
static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const long length)
|
|
183
163
|
{
|
|
184
|
-
if (RB_UNLIKELY(length > JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH)) {
|
|
185
|
-
// Common names aren't likely to be very long. So we just don't
|
|
186
|
-
// cache names above an arbitrary threshold.
|
|
187
|
-
return Qfalse;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
if (RB_UNLIKELY(!isalpha((unsigned char)str[0]))) {
|
|
191
|
-
// Simple heuristic, if the first character isn't a letter,
|
|
192
|
-
// we're much less likely to see this string again.
|
|
193
|
-
// We mostly want to cache strings that are likely to be repeated.
|
|
194
|
-
return Qfalse;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
164
|
int low = 0;
|
|
198
165
|
int high = cache->length - 1;
|
|
199
|
-
int mid = 0;
|
|
200
|
-
int last_cmp = 0;
|
|
201
166
|
|
|
202
167
|
while (low <= high) {
|
|
203
|
-
mid = (high + low) >> 1;
|
|
168
|
+
int mid = (high + low) >> 1;
|
|
204
169
|
VALUE entry = cache->entries[mid];
|
|
205
|
-
|
|
170
|
+
int cmp = rstring_cache_cmp(str, length, rb_sym2str(entry));
|
|
206
171
|
|
|
207
|
-
if (
|
|
172
|
+
if (cmp == 0) {
|
|
208
173
|
return entry;
|
|
209
|
-
} else if (
|
|
174
|
+
} else if (cmp > 0) {
|
|
210
175
|
low = mid + 1;
|
|
211
176
|
} else {
|
|
212
177
|
high = mid - 1;
|
|
213
178
|
}
|
|
214
179
|
}
|
|
215
180
|
|
|
216
|
-
if (RB_UNLIKELY(memchr(str, '\\', length))) {
|
|
217
|
-
// We assume the overwhelming majority of names don't need to be escaped.
|
|
218
|
-
// But if they do, we have to fallback to the slow path.
|
|
219
|
-
return Qfalse;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
181
|
VALUE rsymbol = build_symbol(str, length);
|
|
223
182
|
|
|
224
183
|
if (cache->length < JSON_RVALUE_CACHE_CAPA) {
|
|
225
|
-
|
|
226
|
-
mid += 1;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
rvalue_cache_insert_at(cache, mid, rsymbol);
|
|
184
|
+
rvalue_cache_insert_at(cache, low, rsymbol);
|
|
230
185
|
}
|
|
231
186
|
return rsymbol;
|
|
232
187
|
}
|
|
@@ -286,17 +241,27 @@ static void rvalue_stack_mark(void *ptr)
|
|
|
286
241
|
{
|
|
287
242
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
288
243
|
long index;
|
|
289
|
-
|
|
290
|
-
|
|
244
|
+
if (stack && stack->ptr) {
|
|
245
|
+
for (index = 0; index < stack->head; index++) {
|
|
246
|
+
rb_gc_mark(stack->ptr[index]);
|
|
247
|
+
}
|
|
291
248
|
}
|
|
292
249
|
}
|
|
293
250
|
|
|
251
|
+
static void rvalue_stack_free_buffer(rvalue_stack *stack)
|
|
252
|
+
{
|
|
253
|
+
ruby_xfree(stack->ptr);
|
|
254
|
+
stack->ptr = NULL;
|
|
255
|
+
}
|
|
256
|
+
|
|
294
257
|
static void rvalue_stack_free(void *ptr)
|
|
295
258
|
{
|
|
296
259
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
297
260
|
if (stack) {
|
|
298
|
-
|
|
261
|
+
rvalue_stack_free_buffer(stack);
|
|
262
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
299
263
|
ruby_xfree(stack);
|
|
264
|
+
#endif
|
|
300
265
|
}
|
|
301
266
|
}
|
|
302
267
|
|
|
@@ -307,14 +272,13 @@ static size_t rvalue_stack_memsize(const void *ptr)
|
|
|
307
272
|
}
|
|
308
273
|
|
|
309
274
|
static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
|
|
310
|
-
"JSON::Ext::Parser/rvalue_stack",
|
|
311
|
-
{
|
|
275
|
+
.wrap_struct_name = "JSON::Ext::Parser/rvalue_stack",
|
|
276
|
+
.function = {
|
|
312
277
|
.dmark = rvalue_stack_mark,
|
|
313
278
|
.dfree = rvalue_stack_free,
|
|
314
279
|
.dsize = rvalue_stack_memsize,
|
|
315
280
|
},
|
|
316
|
-
|
|
317
|
-
RUBY_TYPED_FREE_IMMEDIATELY,
|
|
281
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
|
|
318
282
|
};
|
|
319
283
|
|
|
320
284
|
static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
@@ -336,85 +300,13 @@ static void rvalue_stack_eagerly_release(VALUE handle)
|
|
|
336
300
|
if (handle) {
|
|
337
301
|
rvalue_stack *stack;
|
|
338
302
|
TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack);
|
|
339
|
-
|
|
303
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
304
|
+
rvalue_stack_free_buffer(stack);
|
|
305
|
+
#else
|
|
340
306
|
rvalue_stack_free(stack);
|
|
341
|
-
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
#ifndef HAVE_STRNLEN
|
|
346
|
-
static size_t strnlen(const char *s, size_t maxlen)
|
|
347
|
-
{
|
|
348
|
-
char *p;
|
|
349
|
-
return ((p = memchr(s, '\0', maxlen)) ? p - s : maxlen);
|
|
350
|
-
}
|
|
351
|
-
#endif
|
|
352
|
-
|
|
353
|
-
#define PARSE_ERROR_FRAGMENT_LEN 32
|
|
354
|
-
#ifdef RBIMPL_ATTR_NORETURN
|
|
355
|
-
RBIMPL_ATTR_NORETURN()
|
|
307
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
356
308
|
#endif
|
|
357
|
-
static void raise_parse_error(const char *format, const char *start)
|
|
358
|
-
{
|
|
359
|
-
unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1];
|
|
360
|
-
|
|
361
|
-
size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0;
|
|
362
|
-
const char *ptr = start;
|
|
363
|
-
|
|
364
|
-
if (len == PARSE_ERROR_FRAGMENT_LEN) {
|
|
365
|
-
MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN);
|
|
366
|
-
|
|
367
|
-
while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte
|
|
368
|
-
len--;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
if (buffer[len - 1] >= 0xC0) { // multibyte character start
|
|
372
|
-
len--;
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
buffer[len] = '\0';
|
|
376
|
-
ptr = (const char *)buffer;
|
|
377
309
|
}
|
|
378
|
-
|
|
379
|
-
rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr);
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
/* unicode */
|
|
383
|
-
|
|
384
|
-
static const signed char digit_values[256] = {
|
|
385
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
386
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
387
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1,
|
|
388
|
-
-1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1,
|
|
389
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
390
|
-
10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
391
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
392
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
393
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
394
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
395
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
396
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
397
|
-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
398
|
-
-1, -1, -1, -1, -1, -1, -1
|
|
399
|
-
};
|
|
400
|
-
|
|
401
|
-
static uint32_t unescape_unicode(const unsigned char *p)
|
|
402
|
-
{
|
|
403
|
-
signed char b;
|
|
404
|
-
uint32_t result = 0;
|
|
405
|
-
b = digit_values[p[0]];
|
|
406
|
-
if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2);
|
|
407
|
-
result = (result << 4) | (unsigned char)b;
|
|
408
|
-
b = digit_values[p[1]];
|
|
409
|
-
if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2);
|
|
410
|
-
result = (result << 4) | (unsigned char)b;
|
|
411
|
-
b = digit_values[p[2]];
|
|
412
|
-
if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2);
|
|
413
|
-
result = (result << 4) | (unsigned char)b;
|
|
414
|
-
b = digit_values[p[3]];
|
|
415
|
-
if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2);
|
|
416
|
-
result = (result << 4) | (unsigned char)b;
|
|
417
|
-
return result;
|
|
418
310
|
}
|
|
419
311
|
|
|
420
312
|
static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
|
|
@@ -443,95 +335,276 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
|
|
|
443
335
|
return len;
|
|
444
336
|
}
|
|
445
337
|
|
|
338
|
+
enum duplicate_key_action {
|
|
339
|
+
JSON_DEPRECATED = 0,
|
|
340
|
+
JSON_IGNORE,
|
|
341
|
+
JSON_RAISE,
|
|
342
|
+
};
|
|
343
|
+
|
|
446
344
|
typedef struct JSON_ParserStruct {
|
|
447
|
-
VALUE
|
|
448
|
-
VALUE object_class;
|
|
449
|
-
VALUE array_class;
|
|
345
|
+
VALUE on_load_proc;
|
|
450
346
|
VALUE decimal_class;
|
|
451
347
|
ID decimal_method_id;
|
|
452
|
-
|
|
348
|
+
enum duplicate_key_action on_duplicate_key;
|
|
453
349
|
int max_nesting;
|
|
454
350
|
bool allow_nan;
|
|
455
351
|
bool allow_trailing_comma;
|
|
456
|
-
bool
|
|
352
|
+
bool allow_control_characters;
|
|
353
|
+
bool allow_invalid_escape;
|
|
457
354
|
bool symbolize_names;
|
|
458
355
|
bool freeze;
|
|
459
|
-
bool create_additions;
|
|
460
|
-
bool deprecated_create_additions;
|
|
461
356
|
} JSON_ParserConfig;
|
|
462
357
|
|
|
463
358
|
typedef struct JSON_ParserStateStruct {
|
|
464
|
-
VALUE stack_handle;
|
|
359
|
+
VALUE *stack_handle;
|
|
360
|
+
const char *start;
|
|
465
361
|
const char *cursor;
|
|
466
362
|
const char *end;
|
|
467
363
|
rvalue_stack *stack;
|
|
468
364
|
rvalue_cache name_cache;
|
|
469
365
|
int in_array;
|
|
470
366
|
int current_nesting;
|
|
367
|
+
unsigned int emitted_deprecations;
|
|
471
368
|
} JSON_ParserState;
|
|
472
369
|
|
|
370
|
+
static inline size_t rest(JSON_ParserState *state) {
|
|
371
|
+
return state->end - state->cursor;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
static inline bool eos(JSON_ParserState *state) {
|
|
375
|
+
return state->cursor >= state->end;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
static inline char peek(JSON_ParserState *state)
|
|
379
|
+
{
|
|
380
|
+
if (RB_UNLIKELY(eos(state))) {
|
|
381
|
+
return 0;
|
|
382
|
+
}
|
|
383
|
+
return *state->cursor;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
|
|
387
|
+
{
|
|
388
|
+
JSON_ASSERT(state->cursor <= state->end);
|
|
389
|
+
|
|
390
|
+
// Redundant but helpful for hardening
|
|
391
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
392
|
+
state->cursor = state->end;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const char *cursor = state->cursor;
|
|
396
|
+
long column = 0;
|
|
397
|
+
long line = 1;
|
|
398
|
+
|
|
399
|
+
while (cursor >= state->start) {
|
|
400
|
+
if (*cursor-- == '\n') {
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
403
|
+
column++;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
while (cursor >= state->start) {
|
|
407
|
+
if (*cursor-- == '\n') {
|
|
408
|
+
line++;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
*line_out = line;
|
|
412
|
+
*column_out = column;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
416
|
+
{
|
|
417
|
+
long line, column;
|
|
418
|
+
cursor_position(state, &line, &column);
|
|
419
|
+
|
|
420
|
+
VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
|
|
421
|
+
rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
#define PARSE_ERROR_FRAGMENT_LEN 32
|
|
425
|
+
|
|
426
|
+
static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column)
|
|
427
|
+
{
|
|
428
|
+
unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
|
|
429
|
+
|
|
430
|
+
const char *ptr = "EOF";
|
|
431
|
+
if (state->cursor && state->cursor < state->end) {
|
|
432
|
+
ptr = state->cursor;
|
|
433
|
+
size_t len = 0;
|
|
434
|
+
while (len < PARSE_ERROR_FRAGMENT_LEN) {
|
|
435
|
+
char ch = ptr[len];
|
|
436
|
+
if (!ch || ch == '\n' || ch == ' ' || ch == '\t' || ch == '\r') {
|
|
437
|
+
break;
|
|
438
|
+
}
|
|
439
|
+
len++;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
if (len) {
|
|
443
|
+
buffer[0] = '\'';
|
|
444
|
+
MEMCPY(buffer + 1, ptr, char, len);
|
|
445
|
+
|
|
446
|
+
while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte
|
|
447
|
+
len--;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
if (buffer[len] >= 0xC0) { // multibyte character start
|
|
451
|
+
len--;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
buffer[len + 1] = '\'';
|
|
455
|
+
buffer[len + 2] = '\0';
|
|
456
|
+
ptr = (const char *)buffer;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
VALUE message = rb_enc_sprintf(enc_utf8, format, ptr);
|
|
461
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
462
|
+
return message;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
static VALUE parse_error_new(VALUE message, long line, long column)
|
|
466
|
+
{
|
|
467
|
+
VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message);
|
|
468
|
+
rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line));
|
|
469
|
+
rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column));
|
|
470
|
+
return exc;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state)
|
|
474
|
+
{
|
|
475
|
+
long line, column;
|
|
476
|
+
cursor_position(state, &line, &column);
|
|
477
|
+
VALUE message = build_parse_error_message(format, state, line, column);
|
|
478
|
+
rb_exc_raise(parse_error_new(message, line, column));
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
482
|
+
{
|
|
483
|
+
state->cursor = at;
|
|
484
|
+
raise_parse_error(format, state);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/* unicode */
|
|
488
|
+
|
|
489
|
+
static const signed char digit_values[256] = {
|
|
490
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
491
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
492
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1,
|
|
493
|
+
-1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1,
|
|
494
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
495
|
+
10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
496
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
497
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
498
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
499
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
500
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
501
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
502
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
503
|
+
-1, -1, -1, -1, -1, -1, -1
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
|
|
507
|
+
{
|
|
508
|
+
if (RB_UNLIKELY(sp > spe - 4)) {
|
|
509
|
+
raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const unsigned char *p = (const unsigned char *)sp;
|
|
513
|
+
|
|
514
|
+
const signed char b0 = digit_values[p[0]];
|
|
515
|
+
const signed char b1 = digit_values[p[1]];
|
|
516
|
+
const signed char b2 = digit_values[p[2]];
|
|
517
|
+
const signed char b3 = digit_values[p[3]];
|
|
518
|
+
|
|
519
|
+
if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
|
|
520
|
+
raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
|
|
524
|
+
}
|
|
525
|
+
|
|
473
526
|
#define GET_PARSER_CONFIG \
|
|
474
527
|
JSON_ParserConfig *config; \
|
|
475
528
|
TypedData_Get_Struct(self, JSON_ParserConfig, &JSON_ParserConfig_type, config)
|
|
476
529
|
|
|
477
530
|
static const rb_data_type_t JSON_ParserConfig_type;
|
|
478
531
|
|
|
479
|
-
static const bool whitespace[256] = {
|
|
480
|
-
[' '] = 1,
|
|
481
|
-
['\t'] = 1,
|
|
482
|
-
['\n'] = 1,
|
|
483
|
-
['\r'] = 1,
|
|
484
|
-
['/'] = 1,
|
|
485
|
-
};
|
|
486
|
-
|
|
487
532
|
static void
|
|
488
533
|
json_eat_comments(JSON_ParserState *state)
|
|
489
534
|
{
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
535
|
+
const char *start = state->cursor;
|
|
536
|
+
state->cursor++;
|
|
537
|
+
|
|
538
|
+
switch (peek(state)) {
|
|
539
|
+
case '/': {
|
|
540
|
+
state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
|
|
541
|
+
if (!state->cursor) {
|
|
542
|
+
state->cursor = state->end;
|
|
543
|
+
} else {
|
|
544
|
+
state->cursor++;
|
|
500
545
|
}
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
546
|
+
break;
|
|
547
|
+
}
|
|
548
|
+
case '*': {
|
|
549
|
+
state->cursor++;
|
|
550
|
+
|
|
551
|
+
while (true) {
|
|
552
|
+
const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
|
|
553
|
+
if (!next_match) {
|
|
554
|
+
raise_parse_error_at("unterminated comment, expected closing '*/'", state, start);
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
state->cursor = next_match + 1;
|
|
558
|
+
if (peek(state) == '/') {
|
|
559
|
+
state->cursor++;
|
|
560
|
+
break;
|
|
515
561
|
}
|
|
516
|
-
break;
|
|
517
562
|
}
|
|
518
|
-
|
|
519
|
-
raise_parse_error("unexpected token at '%s'", state->cursor);
|
|
520
|
-
break;
|
|
563
|
+
break;
|
|
521
564
|
}
|
|
522
|
-
|
|
523
|
-
|
|
565
|
+
default:
|
|
566
|
+
raise_parse_error_at("unexpected token %s", state, start);
|
|
567
|
+
break;
|
|
524
568
|
}
|
|
525
569
|
}
|
|
526
570
|
|
|
527
|
-
static
|
|
571
|
+
ALWAYS_INLINE(static) void
|
|
528
572
|
json_eat_whitespace(JSON_ParserState *state)
|
|
529
573
|
{
|
|
530
|
-
while (
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
574
|
+
while (true) {
|
|
575
|
+
switch (peek(state)) {
|
|
576
|
+
case ' ':
|
|
577
|
+
state->cursor++;
|
|
578
|
+
break;
|
|
579
|
+
case '\n':
|
|
580
|
+
state->cursor++;
|
|
581
|
+
|
|
582
|
+
// Heuristic: if we see a newline, there is likely consecutive spaces after it.
|
|
583
|
+
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
584
|
+
while (rest(state) > 8) {
|
|
585
|
+
uint64_t chunk;
|
|
586
|
+
memcpy(&chunk, state->cursor, sizeof(uint64_t));
|
|
587
|
+
if (chunk == 0x2020202020202020) {
|
|
588
|
+
state->cursor += 8;
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
uint32_t consecutive_spaces = trailing_zeros64(chunk ^ 0x2020202020202020) / CHAR_BIT;
|
|
593
|
+
state->cursor += consecutive_spaces;
|
|
594
|
+
break;
|
|
595
|
+
}
|
|
596
|
+
#endif
|
|
597
|
+
break;
|
|
598
|
+
case '\t':
|
|
599
|
+
case '\r':
|
|
600
|
+
state->cursor++;
|
|
601
|
+
break;
|
|
602
|
+
case '/':
|
|
603
|
+
json_eat_comments(state);
|
|
604
|
+
break;
|
|
605
|
+
|
|
606
|
+
default:
|
|
607
|
+
return;
|
|
535
608
|
}
|
|
536
609
|
}
|
|
537
610
|
}
|
|
@@ -562,11 +635,22 @@ static inline VALUE build_string(const char *start, const char *end, bool intern
|
|
|
562
635
|
return result;
|
|
563
636
|
}
|
|
564
637
|
|
|
565
|
-
static inline
|
|
638
|
+
static inline bool json_string_cacheable_p(const char *string, size_t length)
|
|
566
639
|
{
|
|
640
|
+
// We mostly want to cache strings that are likely to be repeated.
|
|
641
|
+
// Simple heuristics:
|
|
642
|
+
// - Common names aren't likely to be very long. So we just don't cache names above an arbitrary threshold.
|
|
643
|
+
// - If the first character isn't a letter, we're much less likely to see this string again.
|
|
644
|
+
return length <= JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH && rb_isalpha(string[0]);
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
static inline VALUE json_string_fastpath(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name)
|
|
648
|
+
{
|
|
649
|
+
bool intern = is_name || config->freeze;
|
|
650
|
+
bool symbolize = is_name && config->symbolize_names;
|
|
567
651
|
size_t bufferSize = stringEnd - string;
|
|
568
652
|
|
|
569
|
-
if (is_name && state->in_array) {
|
|
653
|
+
if (is_name && state->in_array && RB_LIKELY(json_string_cacheable_p(string, bufferSize))) {
|
|
570
654
|
VALUE cached_key;
|
|
571
655
|
if (RB_UNLIKELY(symbolize)) {
|
|
572
656
|
cached_key = rsymbol_cache_fetch(&state->name_cache, string, bufferSize);
|
|
@@ -582,104 +666,129 @@ static inline VALUE json_string_fastpath(JSON_ParserState *state, const char *st
|
|
|
582
666
|
return build_string(string, stringEnd, intern, symbolize);
|
|
583
667
|
}
|
|
584
668
|
|
|
585
|
-
|
|
586
|
-
{
|
|
587
|
-
|
|
588
|
-
const char
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
char buf[4];
|
|
669
|
+
#define JSON_MAX_UNESCAPE_POSITIONS 16
|
|
670
|
+
typedef struct _json_unescape_positions {
|
|
671
|
+
long size;
|
|
672
|
+
const char **positions;
|
|
673
|
+
unsigned long additional_backslashes;
|
|
674
|
+
} JSON_UnescapePositions;
|
|
592
675
|
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
676
|
+
static inline const char *json_next_backslash(const char *pe, const char *stringEnd, JSON_UnescapePositions *positions)
|
|
677
|
+
{
|
|
678
|
+
while (positions->size) {
|
|
679
|
+
positions->size--;
|
|
680
|
+
const char *next_position = positions->positions[0];
|
|
681
|
+
positions->positions++;
|
|
682
|
+
if (next_position >= pe) {
|
|
683
|
+
return next_position;
|
|
599
684
|
}
|
|
685
|
+
}
|
|
600
686
|
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
687
|
+
if (positions->additional_backslashes) {
|
|
688
|
+
positions->additional_backslashes--;
|
|
689
|
+
return memchr(pe, '\\', stringEnd - pe);
|
|
604
690
|
}
|
|
605
691
|
|
|
692
|
+
return NULL;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name, JSON_UnescapePositions *positions)
|
|
696
|
+
{
|
|
697
|
+
bool intern = is_name || config->freeze;
|
|
698
|
+
bool symbolize = is_name && config->symbolize_names;
|
|
699
|
+
size_t bufferSize = stringEnd - string;
|
|
700
|
+
const char *p = string, *pe = string, *bufferStart;
|
|
701
|
+
char *buffer;
|
|
702
|
+
|
|
606
703
|
VALUE result = rb_str_buf_new(bufferSize);
|
|
607
704
|
rb_enc_associate_index(result, utf8_encindex);
|
|
608
705
|
buffer = RSTRING_PTR(result);
|
|
609
706
|
bufferStart = buffer;
|
|
610
707
|
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
708
|
+
#define APPEND_CHAR(chr) *buffer++ = chr; p = ++pe;
|
|
709
|
+
|
|
710
|
+
while (pe < stringEnd && (pe = json_next_backslash(pe, stringEnd, positions))) {
|
|
614
711
|
if (pe > p) {
|
|
615
712
|
MEMCPY(buffer, p, char, pe - p);
|
|
616
713
|
buffer += pe - p;
|
|
617
714
|
}
|
|
618
715
|
switch (*++pe) {
|
|
716
|
+
case '"':
|
|
717
|
+
case '/':
|
|
718
|
+
p = pe; // nothing to unescape just need to skip the backslash
|
|
719
|
+
break;
|
|
720
|
+
case '\\':
|
|
721
|
+
APPEND_CHAR('\\');
|
|
722
|
+
break;
|
|
619
723
|
case 'n':
|
|
620
|
-
|
|
724
|
+
APPEND_CHAR('\n');
|
|
621
725
|
break;
|
|
622
726
|
case 'r':
|
|
623
|
-
|
|
727
|
+
APPEND_CHAR('\r');
|
|
624
728
|
break;
|
|
625
729
|
case 't':
|
|
626
|
-
|
|
627
|
-
break;
|
|
628
|
-
case '"':
|
|
629
|
-
unescape = (char *) "\"";
|
|
630
|
-
break;
|
|
631
|
-
case '\\':
|
|
632
|
-
unescape = (char *) "\\";
|
|
730
|
+
APPEND_CHAR('\t');
|
|
633
731
|
break;
|
|
634
732
|
case 'b':
|
|
635
|
-
|
|
733
|
+
APPEND_CHAR('\b');
|
|
636
734
|
break;
|
|
637
735
|
case 'f':
|
|
638
|
-
|
|
736
|
+
APPEND_CHAR('\f');
|
|
639
737
|
break;
|
|
640
|
-
case 'u':
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
if (
|
|
659
|
-
|
|
660
|
-
}
|
|
661
|
-
if (pe[0] == '\\' && pe[1] == 'u') {
|
|
662
|
-
uint32_t sur = unescape_unicode((unsigned char *) pe + 2);
|
|
663
|
-
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16)
|
|
664
|
-
| (sur & 0x3FF));
|
|
665
|
-
pe += 5;
|
|
666
|
-
} else {
|
|
667
|
-
unescape = (char *) "?";
|
|
668
|
-
break;
|
|
738
|
+
case 'u': {
|
|
739
|
+
uint32_t ch = unescape_unicode(state, ++pe, stringEnd);
|
|
740
|
+
pe += 3;
|
|
741
|
+
/* To handle values above U+FFFF, we take a sequence of
|
|
742
|
+
* \uXXXX escapes in the U+D800..U+DBFF then
|
|
743
|
+
* U+DC00..U+DFFF ranges, take the low 10 bits from each
|
|
744
|
+
* to make a 20-bit number, then add 0x10000 to get the
|
|
745
|
+
* final codepoint.
|
|
746
|
+
*
|
|
747
|
+
* See Unicode 15: 3.8 "Surrogates", 5.3 "Handling
|
|
748
|
+
* Surrogate Pairs in UTF-16", and 23.6 "Surrogates
|
|
749
|
+
* Area".
|
|
750
|
+
*/
|
|
751
|
+
if ((ch & 0xFC00) == 0xD800) {
|
|
752
|
+
pe++;
|
|
753
|
+
if (RB_LIKELY((pe <= stringEnd - 6) && memcmp(pe, "\\u", 2) == 0)) {
|
|
754
|
+
uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
|
|
755
|
+
|
|
756
|
+
if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
|
|
757
|
+
raise_parse_error_at("invalid surrogate pair at %s", state, p);
|
|
669
758
|
}
|
|
759
|
+
|
|
760
|
+
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
|
|
761
|
+
pe += 5;
|
|
762
|
+
} else {
|
|
763
|
+
raise_parse_error_at("incomplete surrogate pair at %s", state, p);
|
|
764
|
+
break;
|
|
670
765
|
}
|
|
671
|
-
unescape_len = convert_UTF32_to_UTF8(buf, ch);
|
|
672
|
-
unescape = buf;
|
|
673
766
|
}
|
|
767
|
+
|
|
768
|
+
int unescape_len = convert_UTF32_to_UTF8(buffer, ch);
|
|
769
|
+
buffer += unescape_len;
|
|
770
|
+
p = ++pe;
|
|
674
771
|
break;
|
|
772
|
+
}
|
|
675
773
|
default:
|
|
676
|
-
|
|
677
|
-
|
|
774
|
+
if ((unsigned char)*pe < 0x20) {
|
|
775
|
+
if (!config->allow_control_characters) {
|
|
776
|
+
if (*pe == '\n') {
|
|
777
|
+
raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
|
|
778
|
+
}
|
|
779
|
+
raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
if (config->allow_invalid_escape) {
|
|
784
|
+
APPEND_CHAR(*pe);
|
|
785
|
+
} else {
|
|
786
|
+
raise_parse_error_at("invalid escape character in string: %s", state, pe - 1);
|
|
787
|
+
}
|
|
788
|
+
break;
|
|
678
789
|
}
|
|
679
|
-
MEMCPY(buffer, unescape, char, unescape_len);
|
|
680
|
-
buffer += unescape_len;
|
|
681
|
-
p = ++pe;
|
|
682
790
|
}
|
|
791
|
+
#undef APPEND_CHAR
|
|
683
792
|
|
|
684
793
|
if (stringEnd > p) {
|
|
685
794
|
MEMCPY(buffer, p, char, stringEnd - p);
|
|
@@ -690,97 +799,98 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c
|
|
|
690
799
|
if (symbolize) {
|
|
691
800
|
result = rb_str_intern(result);
|
|
692
801
|
} else if (intern) {
|
|
693
|
-
result =
|
|
802
|
+
result = rb_str_to_interned_str(result);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
return result;
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
#define MAX_FAST_INTEGER_SIZE 18
|
|
809
|
+
#define MAX_NUMBER_STACK_BUFFER 128
|
|
810
|
+
|
|
811
|
+
typedef VALUE (*json_number_decode_func_t)(const char *ptr);
|
|
812
|
+
|
|
813
|
+
static inline VALUE json_decode_large_number(const char *start, long len, json_number_decode_func_t func)
|
|
814
|
+
{
|
|
815
|
+
if (RB_LIKELY(len < MAX_NUMBER_STACK_BUFFER)) {
|
|
816
|
+
char buffer[MAX_NUMBER_STACK_BUFFER];
|
|
817
|
+
MEMCPY(buffer, start, char, len);
|
|
818
|
+
buffer[len] = '\0';
|
|
819
|
+
return func(buffer);
|
|
820
|
+
} else {
|
|
821
|
+
VALUE buffer_v = rb_str_tmp_new(len);
|
|
822
|
+
char *buffer = RSTRING_PTR(buffer_v);
|
|
823
|
+
MEMCPY(buffer, start, char, len);
|
|
824
|
+
buffer[len] = '\0';
|
|
825
|
+
VALUE number = func(buffer);
|
|
826
|
+
RB_GC_GUARD(buffer_v);
|
|
827
|
+
return number;
|
|
694
828
|
}
|
|
695
|
-
|
|
696
|
-
return result;
|
|
697
829
|
}
|
|
698
830
|
|
|
699
|
-
|
|
700
|
-
static inline VALUE fast_decode_integer(const char *p, const char *pe)
|
|
831
|
+
static VALUE json_decode_inum(const char *buffer)
|
|
701
832
|
{
|
|
702
|
-
|
|
703
|
-
if (*p == '-') {
|
|
704
|
-
negative = true;
|
|
705
|
-
p++;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
long long memo = 0;
|
|
709
|
-
while (p < pe) {
|
|
710
|
-
memo *= 10;
|
|
711
|
-
memo += *p - '0';
|
|
712
|
-
p++;
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
if (negative) {
|
|
716
|
-
memo = -memo;
|
|
717
|
-
}
|
|
718
|
-
return LL2NUM(memo);
|
|
833
|
+
return rb_cstr2inum(buffer, 10);
|
|
719
834
|
}
|
|
720
835
|
|
|
721
|
-
static VALUE json_decode_large_integer(const char *start, long len)
|
|
836
|
+
NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len)
|
|
722
837
|
{
|
|
723
|
-
|
|
724
|
-
char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1);
|
|
725
|
-
MEMCPY(buffer, start, char, len);
|
|
726
|
-
buffer[len] = '\0';
|
|
727
|
-
VALUE number = rb_cstr2inum(buffer, 10);
|
|
728
|
-
RB_ALLOCV_END(buffer_v);
|
|
729
|
-
return number;
|
|
838
|
+
return json_decode_large_number(start, len, json_decode_inum);
|
|
730
839
|
}
|
|
731
840
|
|
|
732
|
-
static inline VALUE
|
|
733
|
-
json_decode_integer(const char *start, const char *end)
|
|
841
|
+
static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end)
|
|
734
842
|
{
|
|
735
|
-
|
|
736
|
-
if (
|
|
737
|
-
return
|
|
843
|
+
if (RB_LIKELY(mantissa_digits < MAX_FAST_INTEGER_SIZE)) {
|
|
844
|
+
if (negative) {
|
|
845
|
+
return INT64T2NUM(-((int64_t)mantissa));
|
|
738
846
|
}
|
|
739
|
-
return
|
|
847
|
+
return UINT64T2NUM(mantissa);
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
return json_decode_large_integer(start, end - start);
|
|
740
851
|
}
|
|
741
852
|
|
|
742
|
-
static VALUE
|
|
853
|
+
static VALUE json_decode_dnum(const char *buffer)
|
|
743
854
|
{
|
|
744
|
-
|
|
745
|
-
char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1);
|
|
746
|
-
MEMCPY(buffer, start, char, len);
|
|
747
|
-
buffer[len] = '\0';
|
|
748
|
-
VALUE number = DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
749
|
-
RB_ALLOCV_END(buffer_v);
|
|
750
|
-
return number;
|
|
855
|
+
return DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
751
856
|
}
|
|
752
857
|
|
|
753
|
-
static VALUE
|
|
858
|
+
NOINLINE(static) VALUE json_decode_large_float(const char *start, long len)
|
|
754
859
|
{
|
|
755
|
-
|
|
860
|
+
return json_decode_large_number(start, len, json_decode_dnum);
|
|
861
|
+
}
|
|
756
862
|
|
|
863
|
+
/* Ruby JSON optimized float decoder using vendored Ryu algorithm
|
|
864
|
+
* Accepts pre-extracted mantissa and exponent from first-pass validation
|
|
865
|
+
*/
|
|
866
|
+
static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative,
|
|
867
|
+
const char *start, const char *end)
|
|
868
|
+
{
|
|
757
869
|
if (RB_UNLIKELY(config->decimal_class)) {
|
|
758
|
-
VALUE text = rb_str_new(start,
|
|
870
|
+
VALUE text = rb_str_new(start, end - start);
|
|
759
871
|
return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text);
|
|
760
|
-
} else if (RB_LIKELY(len < 64)) {
|
|
761
|
-
char buffer[64];
|
|
762
|
-
MEMCPY(buffer, start, char, len);
|
|
763
|
-
buffer[len] = '\0';
|
|
764
|
-
return DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
765
|
-
} else {
|
|
766
|
-
return json_decode_large_float(start, len);
|
|
767
872
|
}
|
|
873
|
+
|
|
874
|
+
if (RB_UNLIKELY(exponent > INT32_MAX)) {
|
|
875
|
+
return negative ? CMinusInfinity : CInfinity;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
if (RB_UNLIKELY(exponent < INT32_MIN)) {
|
|
879
|
+
return rb_float_new(negative ? -0.0 : 0.0);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case)
|
|
883
|
+
// Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308)
|
|
884
|
+
if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) {
|
|
885
|
+
return json_decode_large_float(start, end - start);
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, (int32_t)exponent, negative));
|
|
768
889
|
}
|
|
769
890
|
|
|
770
891
|
static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
|
|
771
892
|
{
|
|
772
|
-
VALUE array;
|
|
773
|
-
if (RB_UNLIKELY(config->array_class)) {
|
|
774
|
-
array = rb_class_new_instance(0, 0, config->array_class);
|
|
775
|
-
VALUE *items = rvalue_stack_peek(state->stack, count);
|
|
776
|
-
long index;
|
|
777
|
-
for (index = 0; index < count; index++) {
|
|
778
|
-
rb_funcall(array, i_leftshift, 1, items[index]);
|
|
779
|
-
}
|
|
780
|
-
} else {
|
|
781
|
-
array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count));
|
|
782
|
-
}
|
|
783
|
-
|
|
893
|
+
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count));
|
|
784
894
|
rvalue_stack_pop(state->stack, count);
|
|
785
895
|
|
|
786
896
|
if (config->freeze) {
|
|
@@ -790,43 +900,74 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig
|
|
|
790
900
|
return array;
|
|
791
901
|
}
|
|
792
902
|
|
|
793
|
-
static
|
|
903
|
+
static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs)
|
|
794
904
|
{
|
|
795
|
-
VALUE
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
905
|
+
VALUE set = rb_hash_new_capa(count / 2);
|
|
906
|
+
for (size_t index = 0; index < count; index += 2) {
|
|
907
|
+
size_t before = RHASH_SIZE(set);
|
|
908
|
+
VALUE key = pairs[index];
|
|
909
|
+
rb_hash_aset(set, key, Qtrue);
|
|
910
|
+
if (RHASH_SIZE(set) == before) {
|
|
911
|
+
if (RB_SYMBOL_P(key)) {
|
|
912
|
+
return rb_sym2str(key);
|
|
913
|
+
}
|
|
914
|
+
return key;
|
|
804
915
|
}
|
|
805
|
-
} else {
|
|
806
|
-
object = rb_hash_new_capa(count);
|
|
807
|
-
rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object);
|
|
808
916
|
}
|
|
917
|
+
return Qfalse;
|
|
918
|
+
}
|
|
809
919
|
|
|
810
|
-
|
|
920
|
+
NOINLINE(static) void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key)
|
|
921
|
+
{
|
|
922
|
+
VALUE message = rb_sprintf(
|
|
923
|
+
"detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`",
|
|
924
|
+
rb_inspect(duplicate_key)
|
|
925
|
+
);
|
|
811
926
|
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
927
|
+
emit_parse_warning(RSTRING_PTR(message), state);
|
|
928
|
+
RB_GC_GUARD(message);
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key)
|
|
932
|
+
{
|
|
933
|
+
VALUE message = rb_sprintf(
|
|
934
|
+
"duplicate key %"PRIsVALUE,
|
|
935
|
+
rb_inspect(duplicate_key)
|
|
936
|
+
);
|
|
937
|
+
|
|
938
|
+
long line, column;
|
|
939
|
+
cursor_position(state, &line, &column);
|
|
940
|
+
rb_str_concat(message, build_parse_error_message("", state, line, column)) ;
|
|
941
|
+
rb_exc_raise(parse_error_new(message, line, column));
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
|
|
945
|
+
{
|
|
946
|
+
size_t entries_count = count / 2;
|
|
947
|
+
VALUE object = rb_hash_new_capa(entries_count);
|
|
948
|
+
const VALUE *pairs = rvalue_stack_peek(state->stack, count);
|
|
949
|
+
rb_hash_bulk_insert(count, pairs, object);
|
|
950
|
+
|
|
951
|
+
if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
|
|
952
|
+
switch (config->on_duplicate_key) {
|
|
953
|
+
case JSON_IGNORE:
|
|
954
|
+
break;
|
|
955
|
+
case JSON_DEPRECATED:
|
|
956
|
+
// Only emit the first few deprecations to avoid spamming.
|
|
957
|
+
if (state->emitted_deprecations < 5) {
|
|
958
|
+
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
959
|
+
state->emitted_deprecations++;
|
|
824
960
|
}
|
|
825
|
-
|
|
826
|
-
|
|
961
|
+
|
|
962
|
+
break;
|
|
963
|
+
case JSON_RAISE:
|
|
964
|
+
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
965
|
+
break;
|
|
827
966
|
}
|
|
828
967
|
}
|
|
829
968
|
|
|
969
|
+
rvalue_stack_pop(state->stack, count);
|
|
970
|
+
|
|
830
971
|
if (config->freeze) {
|
|
831
972
|
RB_OBJ_FREEZE(object);
|
|
832
973
|
}
|
|
@@ -834,45 +975,16 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi
|
|
|
834
975
|
return object;
|
|
835
976
|
}
|
|
836
977
|
|
|
837
|
-
static
|
|
838
|
-
{
|
|
839
|
-
if (regexp == Qundef) return ST_STOP;
|
|
840
|
-
if (RTEST(rb_funcall(klass, i_json_creatable_p, 0)) &&
|
|
841
|
-
RTEST(rb_funcall(regexp, i_match, 1, rb_ary_entry(memo, 0)))) {
|
|
842
|
-
rb_ary_push(memo, klass);
|
|
843
|
-
return ST_STOP;
|
|
844
|
-
}
|
|
845
|
-
return ST_CONTINUE;
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
static inline VALUE json_decode_string(JSON_ParserState *state, JSON_ParserConfig *config, const char *start, const char *end, bool escaped, bool is_name)
|
|
978
|
+
static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *config, VALUE value)
|
|
849
979
|
{
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
bool symbolize = is_name && config->symbolize_names;
|
|
853
|
-
if (escaped) {
|
|
854
|
-
string = json_string_unescape(state, start, end, is_name, intern, symbolize);
|
|
855
|
-
} else {
|
|
856
|
-
string = json_string_fastpath(state, start, end, is_name, intern, symbolize);
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
if (RB_UNLIKELY(config->create_additions && RTEST(config->match_string))) {
|
|
860
|
-
VALUE klass;
|
|
861
|
-
VALUE memo = rb_ary_new2(2);
|
|
862
|
-
rb_ary_push(memo, string);
|
|
863
|
-
rb_hash_foreach(config->match_string, match_i, memo);
|
|
864
|
-
klass = rb_ary_entry(memo, 1);
|
|
865
|
-
if (RTEST(klass)) {
|
|
866
|
-
string = rb_funcall(klass, i_json_create, 1, string);
|
|
867
|
-
}
|
|
980
|
+
if (RB_UNLIKELY(config->on_load_proc)) {
|
|
981
|
+
value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
|
|
868
982
|
}
|
|
869
|
-
|
|
870
|
-
return
|
|
983
|
+
rvalue_stack_push(state->stack, value, state->stack_handle, &state->stack);
|
|
984
|
+
return value;
|
|
871
985
|
}
|
|
872
986
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
static const bool string_scan[256] = {
|
|
987
|
+
static const bool string_scan_table[256] = {
|
|
876
988
|
// ASCII Control Characters
|
|
877
989
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
878
990
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
@@ -885,157 +997,319 @@ static const bool string_scan[256] = {
|
|
|
885
997
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
886
998
|
};
|
|
887
999
|
|
|
888
|
-
|
|
1000
|
+
#ifdef HAVE_SIMD
|
|
1001
|
+
static SIMD_Implementation simd_impl = SIMD_NONE;
|
|
1002
|
+
#endif /* HAVE_SIMD */
|
|
1003
|
+
|
|
1004
|
+
ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
|
|
889
1005
|
{
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
bool escaped = false;
|
|
1006
|
+
#ifdef HAVE_SIMD
|
|
1007
|
+
#if defined(HAVE_SIMD_NEON)
|
|
893
1008
|
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
1009
|
+
uint64_t mask = 0;
|
|
1010
|
+
if (string_scan_simd_neon(&state->cursor, state->end, &mask)) {
|
|
1011
|
+
state->cursor += trailing_zeros64(mask) >> 2;
|
|
1012
|
+
return true;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
#elif defined(HAVE_SIMD_SSE2)
|
|
1016
|
+
if (simd_impl == SIMD_SSE2) {
|
|
1017
|
+
int mask = 0;
|
|
1018
|
+
if (string_scan_simd_sse2(&state->cursor, state->end, &mask)) {
|
|
1019
|
+
state->cursor += trailing_zeros(mask);
|
|
1020
|
+
return true;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
#endif /* HAVE_SIMD_NEON or HAVE_SIMD_SSE2 */
|
|
1024
|
+
#endif /* HAVE_SIMD */
|
|
1025
|
+
|
|
1026
|
+
while (!eos(state)) {
|
|
1027
|
+
if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) {
|
|
1028
|
+
return true;
|
|
1029
|
+
}
|
|
1030
|
+
state->cursor++;
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
// If the string ended with an unterminated escape sequence, we might
|
|
1034
|
+
// have gone past the end.
|
|
1035
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
1036
|
+
state->cursor = state->end;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
return false;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name, const char *start)
|
|
1043
|
+
{
|
|
1044
|
+
const char *backslashes[JSON_MAX_UNESCAPE_POSITIONS];
|
|
1045
|
+
JSON_UnescapePositions positions = {
|
|
1046
|
+
.size = 0,
|
|
1047
|
+
.positions = backslashes,
|
|
1048
|
+
.additional_backslashes = 0,
|
|
1049
|
+
};
|
|
1050
|
+
|
|
1051
|
+
do {
|
|
1052
|
+
switch (*state->cursor) {
|
|
1053
|
+
case '"': {
|
|
1054
|
+
VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
|
|
1055
|
+
state->cursor++;
|
|
1056
|
+
return json_push_value(state, config, string);
|
|
1057
|
+
}
|
|
1058
|
+
case '\\': {
|
|
1059
|
+
if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
|
|
1060
|
+
backslashes[positions.size] = state->cursor;
|
|
1061
|
+
positions.size++;
|
|
1062
|
+
} else {
|
|
1063
|
+
positions.additional_backslashes++;
|
|
909
1064
|
}
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
break;
|
|
1065
|
+
state->cursor++;
|
|
1066
|
+
break;
|
|
913
1067
|
}
|
|
1068
|
+
default:
|
|
1069
|
+
if (!config->allow_control_characters) {
|
|
1070
|
+
raise_parse_error("invalid ASCII control character in string: %s", state);
|
|
1071
|
+
}
|
|
1072
|
+
break;
|
|
914
1073
|
}
|
|
915
1074
|
|
|
916
1075
|
state->cursor++;
|
|
917
|
-
}
|
|
1076
|
+
} while (string_scan(state));
|
|
918
1077
|
|
|
919
|
-
raise_parse_error("unexpected end of input, expected closing \"", state
|
|
1078
|
+
raise_parse_error("unexpected end of input, expected closing \"", state);
|
|
920
1079
|
return Qfalse;
|
|
921
1080
|
}
|
|
922
1081
|
|
|
1082
|
+
ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
|
|
1083
|
+
{
|
|
1084
|
+
state->cursor++;
|
|
1085
|
+
const char *start = state->cursor;
|
|
1086
|
+
|
|
1087
|
+
if (RB_UNLIKELY(!string_scan(state))) {
|
|
1088
|
+
raise_parse_error("unexpected end of input, expected closing \"", state);
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
if (RB_LIKELY(*state->cursor == '"')) {
|
|
1092
|
+
VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name);
|
|
1093
|
+
state->cursor++;
|
|
1094
|
+
return json_push_value(state, config, string);
|
|
1095
|
+
}
|
|
1096
|
+
return json_parse_escaped_string(state, config, is_name, start);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
1100
|
+
// From: https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/
|
|
1101
|
+
// Additional References:
|
|
1102
|
+
// https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
|
1103
|
+
// http://0x80.pl/notesen/2014-10-12-parsing-decimal-numbers-part-1-swar.html
|
|
1104
|
+
static inline uint64_t decode_8digits_unrolled(uint64_t val) {
|
|
1105
|
+
const uint64_t mask = 0x000000FF000000FF;
|
|
1106
|
+
const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32)
|
|
1107
|
+
const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32)
|
|
1108
|
+
val -= 0x3030303030303030;
|
|
1109
|
+
val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8;
|
|
1110
|
+
val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32;
|
|
1111
|
+
return val;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
static inline uint64_t decode_4digits_unrolled(uint32_t val) {
|
|
1115
|
+
const uint32_t mask = 0x000000FF;
|
|
1116
|
+
const uint32_t mul1 = 100;
|
|
1117
|
+
val -= 0x30303030;
|
|
1118
|
+
val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8;
|
|
1119
|
+
val = ((val & mask) * mul1) + (((val >> 16) & mask));
|
|
1120
|
+
return val;
|
|
1121
|
+
}
|
|
1122
|
+
#endif
|
|
1123
|
+
|
|
1124
|
+
static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulator)
|
|
1125
|
+
{
|
|
1126
|
+
const char *start = state->cursor;
|
|
1127
|
+
|
|
1128
|
+
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
1129
|
+
while (rest(state) >= sizeof(uint64_t)) {
|
|
1130
|
+
uint64_t next_8bytes;
|
|
1131
|
+
memcpy(&next_8bytes, state->cursor, sizeof(uint64_t));
|
|
1132
|
+
|
|
1133
|
+
// From: https://github.com/simdjson/simdjson/blob/32b301893c13d058095a07d9868edaaa42ee07aa/include/simdjson/generic/numberparsing.h#L333
|
|
1134
|
+
// Branchless version of: http://0x80.pl/articles/swar-digits-validate.html
|
|
1135
|
+
uint64_t match = (next_8bytes & 0xF0F0F0F0F0F0F0F0) | (((next_8bytes + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4);
|
|
1136
|
+
|
|
1137
|
+
if (match == 0x3333333333333333) { // 8 consecutive digits
|
|
1138
|
+
*accumulator = (*accumulator * 100000000) + decode_8digits_unrolled(next_8bytes);
|
|
1139
|
+
state->cursor += 8;
|
|
1140
|
+
continue;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
uint32_t consecutive_digits = trailing_zeros64(match ^ 0x3333333333333333) / CHAR_BIT;
|
|
1144
|
+
|
|
1145
|
+
if (consecutive_digits >= 4) {
|
|
1146
|
+
*accumulator = (*accumulator * 10000) + decode_4digits_unrolled((uint32_t)next_8bytes);
|
|
1147
|
+
state->cursor += 4;
|
|
1148
|
+
consecutive_digits -= 4;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
while (consecutive_digits) {
|
|
1152
|
+
*accumulator = *accumulator * 10 + (*state->cursor - '0');
|
|
1153
|
+
consecutive_digits--;
|
|
1154
|
+
state->cursor++;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
return (int)(state->cursor - start);
|
|
1158
|
+
}
|
|
1159
|
+
#endif
|
|
1160
|
+
|
|
1161
|
+
char next_char;
|
|
1162
|
+
while (rb_isdigit(next_char = peek(state))) {
|
|
1163
|
+
*accumulator = *accumulator * 10 + (next_char - '0');
|
|
1164
|
+
state->cursor++;
|
|
1165
|
+
}
|
|
1166
|
+
return (int)(state->cursor - start);
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
|
|
1170
|
+
{
|
|
1171
|
+
bool integer = true;
|
|
1172
|
+
const char first_digit = *state->cursor;
|
|
1173
|
+
|
|
1174
|
+
// Variables for Ryu optimization - extract digits during parsing
|
|
1175
|
+
int64_t exponent = 0;
|
|
1176
|
+
int decimal_point_pos = -1;
|
|
1177
|
+
uint64_t mantissa = 0;
|
|
1178
|
+
|
|
1179
|
+
// Parse integer part and extract mantissa digits
|
|
1180
|
+
int mantissa_digits = json_parse_digits(state, &mantissa);
|
|
1181
|
+
|
|
1182
|
+
if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
|
|
1183
|
+
raise_parse_error_at("invalid number: %s", state, start);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
// Parse fractional part
|
|
1187
|
+
if (peek(state) == '.') {
|
|
1188
|
+
integer = false;
|
|
1189
|
+
decimal_point_pos = mantissa_digits; // Remember position of decimal point
|
|
1190
|
+
state->cursor++;
|
|
1191
|
+
|
|
1192
|
+
int fractional_digits = json_parse_digits(state, &mantissa);
|
|
1193
|
+
mantissa_digits += fractional_digits;
|
|
1194
|
+
|
|
1195
|
+
if (RB_UNLIKELY(!fractional_digits)) {
|
|
1196
|
+
raise_parse_error_at("invalid number: %s", state, start);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
// Parse exponent
|
|
1201
|
+
if (rb_tolower(peek(state)) == 'e') {
|
|
1202
|
+
integer = false;
|
|
1203
|
+
state->cursor++;
|
|
1204
|
+
|
|
1205
|
+
bool negative_exponent = false;
|
|
1206
|
+
const char next_char = peek(state);
|
|
1207
|
+
if (next_char == '-' || next_char == '+') {
|
|
1208
|
+
negative_exponent = next_char == '-';
|
|
1209
|
+
state->cursor++;
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
uint64_t abs_exponent = 0;
|
|
1213
|
+
int exponent_digits = json_parse_digits(state, &abs_exponent);
|
|
1214
|
+
|
|
1215
|
+
if (RB_UNLIKELY(!exponent_digits)) {
|
|
1216
|
+
raise_parse_error_at("invalid number: %s", state, start);
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
|
|
1220
|
+
exponent = negative_exponent ? INT64_MIN : INT64_MAX;
|
|
1221
|
+
} else {
|
|
1222
|
+
exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
if (integer) {
|
|
1227
|
+
return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor);
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
// Adjust exponent based on decimal point position
|
|
1231
|
+
if (decimal_point_pos >= 0) {
|
|
1232
|
+
exponent -= (mantissa_digits - decimal_point_pos);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
1239
|
+
{
|
|
1240
|
+
return json_parse_number(state, config, false, state->cursor);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
1244
|
+
{
|
|
1245
|
+
const char *start = state->cursor;
|
|
1246
|
+
state->cursor++;
|
|
1247
|
+
return json_parse_number(state, config, true, start);
|
|
1248
|
+
}
|
|
1249
|
+
|
|
923
1250
|
static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
924
1251
|
{
|
|
925
1252
|
json_eat_whitespace(state);
|
|
926
|
-
if (state->cursor >= state->end) {
|
|
927
|
-
raise_parse_error("unexpected end of input", state->cursor);
|
|
928
|
-
}
|
|
929
1253
|
|
|
930
|
-
switch (
|
|
1254
|
+
switch (peek(state)) {
|
|
931
1255
|
case 'n':
|
|
932
|
-
if ((state
|
|
1256
|
+
if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
|
|
933
1257
|
state->cursor += 4;
|
|
934
|
-
return
|
|
1258
|
+
return json_push_value(state, config, Qnil);
|
|
935
1259
|
}
|
|
936
1260
|
|
|
937
|
-
raise_parse_error("unexpected token
|
|
1261
|
+
raise_parse_error("unexpected token %s", state);
|
|
938
1262
|
break;
|
|
939
1263
|
case 't':
|
|
940
|
-
if ((state
|
|
1264
|
+
if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
|
|
941
1265
|
state->cursor += 4;
|
|
942
|
-
return
|
|
1266
|
+
return json_push_value(state, config, Qtrue);
|
|
943
1267
|
}
|
|
944
1268
|
|
|
945
|
-
raise_parse_error("unexpected token
|
|
1269
|
+
raise_parse_error("unexpected token %s", state);
|
|
946
1270
|
break;
|
|
947
1271
|
case 'f':
|
|
948
1272
|
// Note: memcmp with a small power of two compile to an integer comparison
|
|
949
|
-
if ((state
|
|
1273
|
+
if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) {
|
|
950
1274
|
state->cursor += 5;
|
|
951
|
-
return
|
|
1275
|
+
return json_push_value(state, config, Qfalse);
|
|
952
1276
|
}
|
|
953
1277
|
|
|
954
|
-
raise_parse_error("unexpected token
|
|
1278
|
+
raise_parse_error("unexpected token %s", state);
|
|
955
1279
|
break;
|
|
956
1280
|
case 'N':
|
|
957
1281
|
// Note: memcmp with a small power of two compile to an integer comparison
|
|
958
|
-
if (config->allow_nan && (state
|
|
1282
|
+
if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) {
|
|
959
1283
|
state->cursor += 3;
|
|
960
|
-
return
|
|
1284
|
+
return json_push_value(state, config, CNaN);
|
|
961
1285
|
}
|
|
962
1286
|
|
|
963
|
-
raise_parse_error("unexpected token
|
|
1287
|
+
raise_parse_error("unexpected token %s", state);
|
|
964
1288
|
break;
|
|
965
1289
|
case 'I':
|
|
966
|
-
if (config->allow_nan && (state
|
|
1290
|
+
if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
|
|
967
1291
|
state->cursor += 8;
|
|
968
|
-
return
|
|
1292
|
+
return json_push_value(state, config, CInfinity);
|
|
969
1293
|
}
|
|
970
1294
|
|
|
971
|
-
raise_parse_error("unexpected token
|
|
1295
|
+
raise_parse_error("unexpected token %s", state);
|
|
972
1296
|
break;
|
|
973
|
-
case '-':
|
|
1297
|
+
case '-': {
|
|
974
1298
|
// Note: memcmp with a small power of two compile to an integer comparison
|
|
975
|
-
if ((state
|
|
1299
|
+
if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
|
|
976
1300
|
if (config->allow_nan) {
|
|
977
1301
|
state->cursor += 9;
|
|
978
|
-
return
|
|
1302
|
+
return json_push_value(state, config, CMinusInfinity);
|
|
979
1303
|
} else {
|
|
980
|
-
raise_parse_error("unexpected token
|
|
981
|
-
}
|
|
982
|
-
}
|
|
983
|
-
// Fallthrough
|
|
984
|
-
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
|
|
985
|
-
bool integer = true;
|
|
986
|
-
|
|
987
|
-
// /\A-?(0|[1-9]\d*)(\.\d+)?([Ee][-+]?\d+)?/
|
|
988
|
-
const char *start = state->cursor;
|
|
989
|
-
state->cursor++;
|
|
990
|
-
|
|
991
|
-
while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) {
|
|
992
|
-
state->cursor++;
|
|
993
|
-
}
|
|
994
|
-
|
|
995
|
-
long integer_length = state->cursor - start;
|
|
996
|
-
|
|
997
|
-
if (RB_UNLIKELY(start[0] == '0' && integer_length > 1)) {
|
|
998
|
-
raise_parse_error("invalid number: %s", start);
|
|
999
|
-
} else if (RB_UNLIKELY(integer_length > 2 && start[0] == '-' && start[1] == '0')) {
|
|
1000
|
-
raise_parse_error("invalid number: %s", start);
|
|
1001
|
-
} else if (RB_UNLIKELY(integer_length == 1 && start[0] == '-')) {
|
|
1002
|
-
raise_parse_error("invalid number: %s", start);
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
if ((state->cursor < state->end) && (*state->cursor == '.')) {
|
|
1006
|
-
integer = false;
|
|
1007
|
-
state->cursor++;
|
|
1008
|
-
|
|
1009
|
-
if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') {
|
|
1010
|
-
raise_parse_error("invalid number: %s", state->cursor);
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) {
|
|
1014
|
-
state->cursor++;
|
|
1015
|
-
}
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
if ((state->cursor < state->end) && ((*state->cursor == 'e') || (*state->cursor == 'E'))) {
|
|
1019
|
-
integer = false;
|
|
1020
|
-
state->cursor++;
|
|
1021
|
-
if ((state->cursor < state->end) && ((*state->cursor == '+') || (*state->cursor == '-'))) {
|
|
1022
|
-
state->cursor++;
|
|
1304
|
+
raise_parse_error("unexpected token %s", state);
|
|
1023
1305
|
}
|
|
1024
|
-
|
|
1025
|
-
if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') {
|
|
1026
|
-
raise_parse_error("invalid number: %s", state->cursor);
|
|
1027
|
-
}
|
|
1028
|
-
|
|
1029
|
-
while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) {
|
|
1030
|
-
state->cursor++;
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
if (integer) {
|
|
1035
|
-
return PUSH(json_decode_integer(start, state->cursor));
|
|
1036
1306
|
}
|
|
1037
|
-
return
|
|
1307
|
+
return json_push_value(state, config, json_parse_negative_number(state, config));
|
|
1308
|
+
break;
|
|
1038
1309
|
}
|
|
1310
|
+
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
|
|
1311
|
+
return json_push_value(state, config, json_parse_positive_number(state, config));
|
|
1312
|
+
break;
|
|
1039
1313
|
case '"': {
|
|
1040
1314
|
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1041
1315
|
return json_parse_string(state, config, false);
|
|
@@ -1046,9 +1320,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
|
1046
1320
|
json_eat_whitespace(state);
|
|
1047
1321
|
long stack_head = state->stack->head;
|
|
1048
1322
|
|
|
1049
|
-
if ((state
|
|
1323
|
+
if (peek(state) == ']') {
|
|
1050
1324
|
state->cursor++;
|
|
1051
|
-
return
|
|
1325
|
+
return json_push_value(state, config, json_decode_array(state, config, 0));
|
|
1052
1326
|
} else {
|
|
1053
1327
|
state->current_nesting++;
|
|
1054
1328
|
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
@@ -1061,54 +1335,56 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
|
1061
1335
|
while (true) {
|
|
1062
1336
|
json_eat_whitespace(state);
|
|
1063
1337
|
|
|
1064
|
-
|
|
1065
|
-
if (*state->cursor == ']') {
|
|
1066
|
-
state->cursor++;
|
|
1067
|
-
long count = state->stack->head - stack_head;
|
|
1068
|
-
state->current_nesting--;
|
|
1069
|
-
state->in_array--;
|
|
1070
|
-
return PUSH(json_decode_array(state, config, count));
|
|
1071
|
-
}
|
|
1338
|
+
const char next_char = peek(state);
|
|
1072
1339
|
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
}
|
|
1340
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1341
|
+
state->cursor++;
|
|
1342
|
+
if (config->allow_trailing_comma) {
|
|
1343
|
+
json_eat_whitespace(state);
|
|
1344
|
+
if (peek(state) == ']') {
|
|
1345
|
+
continue;
|
|
1080
1346
|
}
|
|
1081
|
-
json_parse_any(state, config);
|
|
1082
|
-
continue;
|
|
1083
1347
|
}
|
|
1348
|
+
json_parse_any(state, config);
|
|
1349
|
+
continue;
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
if (next_char == ']') {
|
|
1353
|
+
state->cursor++;
|
|
1354
|
+
long count = state->stack->head - stack_head;
|
|
1355
|
+
state->current_nesting--;
|
|
1356
|
+
state->in_array--;
|
|
1357
|
+
return json_push_value(state, config, json_decode_array(state, config, count));
|
|
1084
1358
|
}
|
|
1085
1359
|
|
|
1086
|
-
raise_parse_error("expected ',' or ']' after array value", state
|
|
1360
|
+
raise_parse_error("expected ',' or ']' after array value", state);
|
|
1087
1361
|
}
|
|
1088
1362
|
break;
|
|
1089
1363
|
}
|
|
1090
1364
|
case '{': {
|
|
1365
|
+
const char *object_start_cursor = state->cursor;
|
|
1366
|
+
|
|
1091
1367
|
state->cursor++;
|
|
1092
1368
|
json_eat_whitespace(state);
|
|
1093
1369
|
long stack_head = state->stack->head;
|
|
1094
1370
|
|
|
1095
|
-
if ((state
|
|
1371
|
+
if (peek(state) == '}') {
|
|
1096
1372
|
state->cursor++;
|
|
1097
|
-
return
|
|
1373
|
+
return json_push_value(state, config, json_decode_object(state, config, 0));
|
|
1098
1374
|
} else {
|
|
1099
1375
|
state->current_nesting++;
|
|
1100
1376
|
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1101
1377
|
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1102
1378
|
}
|
|
1103
1379
|
|
|
1104
|
-
if (
|
|
1105
|
-
raise_parse_error("expected object key, got
|
|
1380
|
+
if (peek(state) != '"') {
|
|
1381
|
+
raise_parse_error("expected object key, got %s", state);
|
|
1106
1382
|
}
|
|
1107
1383
|
json_parse_string(state, config, true);
|
|
1108
1384
|
|
|
1109
1385
|
json_eat_whitespace(state);
|
|
1110
|
-
if ((state
|
|
1111
|
-
raise_parse_error("expected ':' after object key", state
|
|
1386
|
+
if (peek(state) != ':') {
|
|
1387
|
+
raise_parse_error("expected ':' after object key", state);
|
|
1112
1388
|
}
|
|
1113
1389
|
state->cursor++;
|
|
1114
1390
|
|
|
@@ -1118,59 +1394,70 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
|
1118
1394
|
while (true) {
|
|
1119
1395
|
json_eat_whitespace(state);
|
|
1120
1396
|
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
return PUSH(json_decode_object(state, config, count));
|
|
1127
|
-
}
|
|
1397
|
+
const char next_char = peek(state);
|
|
1398
|
+
if (next_char == '}') {
|
|
1399
|
+
state->cursor++;
|
|
1400
|
+
state->current_nesting--;
|
|
1401
|
+
size_t count = state->stack->head - stack_head;
|
|
1128
1402
|
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1403
|
+
// Temporary rewind cursor in case an error is raised
|
|
1404
|
+
const char *final_cursor = state->cursor;
|
|
1405
|
+
state->cursor = object_start_cursor;
|
|
1406
|
+
VALUE object = json_decode_object(state, config, count);
|
|
1407
|
+
state->cursor = final_cursor;
|
|
1132
1408
|
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
continue;
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1409
|
+
return json_push_value(state, config, object);
|
|
1410
|
+
}
|
|
1138
1411
|
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
json_parse_string(state, config, true);
|
|
1412
|
+
if (next_char == ',') {
|
|
1413
|
+
state->cursor++;
|
|
1414
|
+
json_eat_whitespace(state);
|
|
1143
1415
|
|
|
1144
|
-
|
|
1145
|
-
if ((state
|
|
1146
|
-
|
|
1416
|
+
if (config->allow_trailing_comma) {
|
|
1417
|
+
if (peek(state) == '}') {
|
|
1418
|
+
continue;
|
|
1147
1419
|
}
|
|
1148
|
-
|
|
1420
|
+
}
|
|
1149
1421
|
|
|
1150
|
-
|
|
1422
|
+
if (RB_UNLIKELY(peek(state) != '"')) {
|
|
1423
|
+
raise_parse_error("expected object key, got: %s", state);
|
|
1424
|
+
}
|
|
1425
|
+
json_parse_string(state, config, true);
|
|
1151
1426
|
|
|
1152
|
-
|
|
1427
|
+
json_eat_whitespace(state);
|
|
1428
|
+
if (RB_UNLIKELY(peek(state) != ':')) {
|
|
1429
|
+
raise_parse_error("expected ':' after object key, got: %s", state);
|
|
1153
1430
|
}
|
|
1431
|
+
state->cursor++;
|
|
1432
|
+
|
|
1433
|
+
json_parse_any(state, config);
|
|
1434
|
+
|
|
1435
|
+
continue;
|
|
1154
1436
|
}
|
|
1155
1437
|
|
|
1156
|
-
raise_parse_error("expected ',' or '}' after object value, got:
|
|
1438
|
+
raise_parse_error("expected ',' or '}' after object value, got: %s", state);
|
|
1157
1439
|
}
|
|
1158
1440
|
break;
|
|
1159
1441
|
}
|
|
1160
1442
|
|
|
1443
|
+
case 0:
|
|
1444
|
+
raise_parse_error("unexpected end of input", state);
|
|
1445
|
+
break;
|
|
1446
|
+
|
|
1161
1447
|
default:
|
|
1162
|
-
raise_parse_error("unexpected character:
|
|
1448
|
+
raise_parse_error("unexpected character: %s", state);
|
|
1163
1449
|
break;
|
|
1164
1450
|
}
|
|
1165
1451
|
|
|
1166
|
-
raise_parse_error("
|
|
1452
|
+
raise_parse_error("unreachable: %s", state);
|
|
1453
|
+
return Qundef;
|
|
1167
1454
|
}
|
|
1168
1455
|
|
|
1169
1456
|
static void json_ensure_eof(JSON_ParserState *state)
|
|
1170
1457
|
{
|
|
1171
1458
|
json_eat_whitespace(state);
|
|
1172
|
-
if (state
|
|
1173
|
-
raise_parse_error("unexpected token at end of stream
|
|
1459
|
+
if (!eos(state)) {
|
|
1460
|
+
raise_parse_error("unexpected token at end of stream %s", state);
|
|
1174
1461
|
}
|
|
1175
1462
|
}
|
|
1176
1463
|
|
|
@@ -1188,40 +1475,56 @@ static void json_ensure_eof(JSON_ParserState *state)
|
|
|
1188
1475
|
|
|
1189
1476
|
static VALUE convert_encoding(VALUE source)
|
|
1190
1477
|
{
|
|
1191
|
-
|
|
1478
|
+
StringValue(source);
|
|
1479
|
+
int encindex = RB_ENCODING_GET(source);
|
|
1480
|
+
|
|
1481
|
+
if (RB_LIKELY(encindex == utf8_encindex)) {
|
|
1482
|
+
return source;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
if (encindex == binary_encindex) {
|
|
1486
|
+
// For historical reason, we silently reinterpret binary strings as UTF-8
|
|
1487
|
+
return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
|
|
1488
|
+
}
|
|
1192
1489
|
|
|
1193
|
-
|
|
1490
|
+
source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
|
|
1491
|
+
StringValue(source);
|
|
1194
1492
|
return source;
|
|
1195
|
-
|
|
1493
|
+
}
|
|
1196
1494
|
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1495
|
+
struct parser_config_init_args {
|
|
1496
|
+
JSON_ParserConfig *config;
|
|
1497
|
+
VALUE self;
|
|
1498
|
+
};
|
|
1201
1499
|
|
|
1202
|
-
|
|
1500
|
+
static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
|
|
1501
|
+
{
|
|
1502
|
+
*dest = val;
|
|
1503
|
+
if (self) RB_OBJ_WRITTEN(self, Qundef, val);
|
|
1203
1504
|
}
|
|
1204
1505
|
|
|
1205
1506
|
static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
1206
1507
|
{
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
else if (key ==
|
|
1213
|
-
else if (key ==
|
|
1214
|
-
else if (key ==
|
|
1215
|
-
else if (key ==
|
|
1216
|
-
else if (key ==
|
|
1217
|
-
else if (key ==
|
|
1218
|
-
else if (key ==
|
|
1508
|
+
struct parser_config_init_args *args = (struct parser_config_init_args *)data;
|
|
1509
|
+
JSON_ParserConfig *config = args->config;
|
|
1510
|
+
VALUE self = args->self;
|
|
1511
|
+
|
|
1512
|
+
if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
|
|
1513
|
+
else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
|
|
1514
|
+
else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
|
|
1515
|
+
else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
|
|
1516
|
+
else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
|
|
1517
|
+
else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
|
|
1518
|
+
else if (key == sym_freeze) { config->freeze = RTEST(val); }
|
|
1519
|
+
else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
|
|
1520
|
+
else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1521
|
+
else if (key == sym_decimal_class) {
|
|
1219
1522
|
if (RTEST(val)) {
|
|
1220
1523
|
if (rb_respond_to(val, i_try_convert)) {
|
|
1221
|
-
config->decimal_class
|
|
1524
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1222
1525
|
config->decimal_method_id = i_try_convert;
|
|
1223
1526
|
} else if (rb_respond_to(val, i_new)) {
|
|
1224
|
-
config->decimal_class
|
|
1527
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1225
1528
|
config->decimal_method_id = i_new;
|
|
1226
1529
|
} else if (RB_TYPE_P(val, T_CLASS)) {
|
|
1227
1530
|
VALUE name = rb_class_name(val);
|
|
@@ -1230,7 +1533,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1230
1533
|
if (last_colon) {
|
|
1231
1534
|
const char *mod_path_end = last_colon - 1;
|
|
1232
1535
|
VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
|
|
1233
|
-
config->decimal_class
|
|
1536
|
+
parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
|
|
1234
1537
|
|
|
1235
1538
|
const char *method_name_beg = last_colon + 1;
|
|
1236
1539
|
long before_len = method_name_beg - name_cstr;
|
|
@@ -1238,45 +1541,31 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1238
1541
|
VALUE method_name = rb_str_substr(name, before_len, len);
|
|
1239
1542
|
config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
|
|
1240
1543
|
} else {
|
|
1241
|
-
config->decimal_class
|
|
1544
|
+
parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
|
|
1242
1545
|
config->decimal_method_id = SYM2ID(rb_str_intern(name));
|
|
1243
1546
|
}
|
|
1244
1547
|
}
|
|
1245
1548
|
}
|
|
1246
1549
|
}
|
|
1247
|
-
else if (key == sym_create_additions) {
|
|
1248
|
-
if (NIL_P(val)) {
|
|
1249
|
-
config->create_additions = true;
|
|
1250
|
-
config->deprecated_create_additions = true;
|
|
1251
|
-
} else {
|
|
1252
|
-
config->create_additions = RTEST(val);
|
|
1253
|
-
config->deprecated_create_additions = false;
|
|
1254
|
-
}
|
|
1255
|
-
}
|
|
1256
1550
|
|
|
1257
1551
|
return ST_CONTINUE;
|
|
1258
1552
|
}
|
|
1259
1553
|
|
|
1260
|
-
static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
1554
|
+
static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self)
|
|
1261
1555
|
{
|
|
1262
1556
|
config->max_nesting = 100;
|
|
1263
1557
|
|
|
1558
|
+
struct parser_config_init_args args = {
|
|
1559
|
+
.config = config,
|
|
1560
|
+
.self = self,
|
|
1561
|
+
};
|
|
1562
|
+
|
|
1264
1563
|
if (!NIL_P(opts)) {
|
|
1265
1564
|
Check_Type(opts, T_HASH);
|
|
1266
1565
|
if (RHASH_SIZE(opts) > 0) {
|
|
1267
1566
|
// We assume in most cases few keys are set so it's faster to go over
|
|
1268
1567
|
// the provided keys than to check all possible keys.
|
|
1269
|
-
rb_hash_foreach(opts, parser_config_init_i, (VALUE)
|
|
1270
|
-
|
|
1271
|
-
if (config->symbolize_names && config->create_additions) {
|
|
1272
|
-
rb_raise(rb_eArgError,
|
|
1273
|
-
"options :symbolize_names and :create_additions cannot be "
|
|
1274
|
-
" used in conjunction");
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
if (config->create_additions && !config->create_id) {
|
|
1278
|
-
config->create_id = rb_funcall(mJSON, i_create_id, 0);
|
|
1279
|
-
}
|
|
1568
|
+
rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
|
|
1280
1569
|
}
|
|
1281
1570
|
|
|
1282
1571
|
}
|
|
@@ -1301,38 +1590,30 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
|
1301
1590
|
* (keys) in a JSON object. Otherwise strings are returned, which is
|
|
1302
1591
|
* also the default. It's not possible to use this option in
|
|
1303
1592
|
* conjunction with the *create_additions* option.
|
|
1304
|
-
* * *create_additions*: If set to false, the Parser doesn't create
|
|
1305
|
-
* additions even if a matching class and create_id was found. This option
|
|
1306
|
-
* defaults to false.
|
|
1307
|
-
* * *object_class*: Defaults to Hash. If another type is provided, it will be used
|
|
1308
|
-
* instead of Hash to represent JSON objects. The type must respond to
|
|
1309
|
-
* +new+ without arguments, and return an object that respond to +[]=+.
|
|
1310
|
-
* * *array_class*: Defaults to Array If another type is provided, it will be used
|
|
1311
|
-
* instead of Hash to represent JSON arrays. The type must respond to
|
|
1312
|
-
* +new+ without arguments, and return an object that respond to +<<+.
|
|
1313
1593
|
* * *decimal_class*: Specifies which class to use instead of the default
|
|
1314
1594
|
* (Float) when parsing decimal numbers. This class must accept a single
|
|
1315
1595
|
* string argument in its constructor.
|
|
1316
1596
|
*/
|
|
1317
1597
|
static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
|
|
1318
1598
|
{
|
|
1599
|
+
rb_check_frozen(self);
|
|
1319
1600
|
GET_PARSER_CONFIG;
|
|
1320
1601
|
|
|
1321
|
-
parser_config_init(config, opts);
|
|
1322
|
-
|
|
1323
|
-
RB_OBJ_WRITTEN(self, Qundef, config->create_id);
|
|
1324
|
-
RB_OBJ_WRITTEN(self, Qundef, config->object_class);
|
|
1325
|
-
RB_OBJ_WRITTEN(self, Qundef, config->array_class);
|
|
1326
|
-
RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
|
|
1327
|
-
RB_OBJ_WRITTEN(self, Qundef, config->match_string);
|
|
1602
|
+
parser_config_init(config, opts, self);
|
|
1328
1603
|
|
|
1329
1604
|
return self;
|
|
1330
1605
|
}
|
|
1331
1606
|
|
|
1332
|
-
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE
|
|
1607
|
+
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
|
|
1333
1608
|
{
|
|
1334
|
-
Vsource = convert_encoding(
|
|
1335
|
-
|
|
1609
|
+
VALUE Vsource = convert_encoding(src);
|
|
1610
|
+
|
|
1611
|
+
// Ensure the string isn't mutated under us.
|
|
1612
|
+
// The classic API to use is `rb_str_locktmp`, but then we'd
|
|
1613
|
+
// need to use `rb_protect` to make sure we always unlock.
|
|
1614
|
+
if (Vsource == src) {
|
|
1615
|
+
Vsource = rb_str_new_frozen(Vsource);
|
|
1616
|
+
}
|
|
1336
1617
|
|
|
1337
1618
|
VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
|
|
1338
1619
|
rvalue_stack stack = {
|
|
@@ -1341,10 +1622,18 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource)
|
|
|
1341
1622
|
.capa = RVALUE_STACK_INITIAL_CAPA,
|
|
1342
1623
|
};
|
|
1343
1624
|
|
|
1625
|
+
long len;
|
|
1626
|
+
const char *start;
|
|
1627
|
+
|
|
1628
|
+
RSTRING_GETMEM(Vsource, start, len);
|
|
1629
|
+
|
|
1630
|
+
VALUE stack_handle = 0;
|
|
1344
1631
|
JSON_ParserState _state = {
|
|
1345
|
-
.
|
|
1346
|
-
.
|
|
1632
|
+
.start = start,
|
|
1633
|
+
.cursor = start,
|
|
1634
|
+
.end = start + len,
|
|
1347
1635
|
.stack = &stack,
|
|
1636
|
+
.stack_handle = &stack_handle,
|
|
1348
1637
|
};
|
|
1349
1638
|
JSON_ParserState *state = &_state;
|
|
1350
1639
|
|
|
@@ -1352,8 +1641,9 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource)
|
|
|
1352
1641
|
|
|
1353
1642
|
// This may be skipped in case of exception, but
|
|
1354
1643
|
// it won't cause a leak.
|
|
1355
|
-
rvalue_stack_eagerly_release(
|
|
1356
|
-
|
|
1644
|
+
rvalue_stack_eagerly_release(stack_handle);
|
|
1645
|
+
RB_GC_GUARD(stack_handle);
|
|
1646
|
+
RB_GC_GUARD(Vsource);
|
|
1357
1647
|
json_ensure_eof(state);
|
|
1358
1648
|
|
|
1359
1649
|
return result;
|
|
@@ -1374,12 +1664,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
|
|
|
1374
1664
|
|
|
1375
1665
|
static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
1376
1666
|
{
|
|
1377
|
-
Vsource = convert_encoding(StringValue(Vsource));
|
|
1378
|
-
StringValue(Vsource);
|
|
1379
|
-
|
|
1380
1667
|
JSON_ParserConfig _config = {0};
|
|
1381
1668
|
JSON_ParserConfig *config = &_config;
|
|
1382
|
-
parser_config_init(config, opts);
|
|
1669
|
+
parser_config_init(config, opts, false);
|
|
1383
1670
|
|
|
1384
1671
|
return cParser_parse(config, Vsource);
|
|
1385
1672
|
}
|
|
@@ -1387,17 +1674,8 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
|
1387
1674
|
static void JSON_ParserConfig_mark(void *ptr)
|
|
1388
1675
|
{
|
|
1389
1676
|
JSON_ParserConfig *config = ptr;
|
|
1390
|
-
rb_gc_mark(config->
|
|
1391
|
-
rb_gc_mark(config->object_class);
|
|
1392
|
-
rb_gc_mark(config->array_class);
|
|
1677
|
+
rb_gc_mark(config->on_load_proc);
|
|
1393
1678
|
rb_gc_mark(config->decimal_class);
|
|
1394
|
-
rb_gc_mark(config->match_string);
|
|
1395
|
-
}
|
|
1396
|
-
|
|
1397
|
-
static void JSON_ParserConfig_free(void *ptr)
|
|
1398
|
-
{
|
|
1399
|
-
JSON_ParserConfig *config = ptr;
|
|
1400
|
-
ruby_xfree(config);
|
|
1401
1679
|
}
|
|
1402
1680
|
|
|
1403
1681
|
static size_t JSON_ParserConfig_memsize(const void *ptr)
|
|
@@ -1406,14 +1684,13 @@ static size_t JSON_ParserConfig_memsize(const void *ptr)
|
|
|
1406
1684
|
}
|
|
1407
1685
|
|
|
1408
1686
|
static const rb_data_type_t JSON_ParserConfig_type = {
|
|
1409
|
-
"JSON::Ext::Parser/ParserConfig",
|
|
1410
|
-
{
|
|
1687
|
+
.wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
|
|
1688
|
+
.function = {
|
|
1411
1689
|
JSON_ParserConfig_mark,
|
|
1412
|
-
|
|
1690
|
+
RUBY_DEFAULT_FREE,
|
|
1413
1691
|
JSON_ParserConfig_memsize,
|
|
1414
1692
|
},
|
|
1415
|
-
|
|
1416
|
-
RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED,
|
|
1693
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
|
|
1417
1694
|
};
|
|
1418
1695
|
|
|
1419
1696
|
static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
@@ -1457,24 +1734,14 @@ void Init_parser(void)
|
|
|
1457
1734
|
sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
|
|
1458
1735
|
sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
|
|
1459
1736
|
sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
|
|
1737
|
+
sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
|
|
1738
|
+
sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
|
|
1460
1739
|
sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
|
|
1461
1740
|
sym_freeze = ID2SYM(rb_intern("freeze"));
|
|
1462
|
-
|
|
1463
|
-
sym_create_id = ID2SYM(rb_intern("create_id"));
|
|
1464
|
-
sym_object_class = ID2SYM(rb_intern("object_class"));
|
|
1465
|
-
sym_array_class = ID2SYM(rb_intern("array_class"));
|
|
1741
|
+
sym_on_load = ID2SYM(rb_intern("on_load"));
|
|
1466
1742
|
sym_decimal_class = ID2SYM(rb_intern("decimal_class"));
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
i_create_id = rb_intern("create_id");
|
|
1470
|
-
i_json_creatable_p = rb_intern("json_creatable?");
|
|
1471
|
-
i_json_create = rb_intern("json_create");
|
|
1472
|
-
i_chr = rb_intern("chr");
|
|
1473
|
-
i_match = rb_intern("match");
|
|
1474
|
-
i_deep_const_get = rb_intern("deep_const_get");
|
|
1475
|
-
i_aset = rb_intern("[]=");
|
|
1476
|
-
i_aref = rb_intern("[]");
|
|
1477
|
-
i_leftshift = rb_intern("<<");
|
|
1743
|
+
sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
|
|
1744
|
+
|
|
1478
1745
|
i_new = rb_intern("new");
|
|
1479
1746
|
i_try_convert = rb_intern("try_convert");
|
|
1480
1747
|
i_uminus = rb_intern("-@");
|
|
@@ -1483,4 +1750,8 @@ void Init_parser(void)
|
|
|
1483
1750
|
binary_encindex = rb_ascii8bit_encindex();
|
|
1484
1751
|
utf8_encindex = rb_utf8_encindex();
|
|
1485
1752
|
enc_utf8 = rb_utf8_encoding();
|
|
1753
|
+
|
|
1754
|
+
#ifdef HAVE_SIMD
|
|
1755
|
+
simd_impl = find_simd_implementation();
|
|
1756
|
+
#endif
|
|
1486
1757
|
}
|