nosj 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +61 -0
- data/Cargo.lock +1 -1
- data/README.md +30 -14
- data/ext/nosj/Cargo.toml +1 -1
- data/ext/nosj/fuzz/src/prelude.rb +20 -44
- data/ext/nosj/src/gen/opts.rs +76 -141
- data/ext/nosj/src/gen/ruby.rs +5 -0
- data/ext/nosj/src/gen/walker.rs +166 -32
- data/ext/nosj/src/lib.rs +4 -0
- data/ext/nosj/src/locate.rs +162 -0
- data/ext/nosj/src/opt_reader.rs +170 -0
- data/ext/nosj/src/parse.rs +160 -93
- data/ext/nosj/src/reformat.rs +54 -125
- data/ext/nosj/src/sink.rs +180 -44
- data/ext/nosj/src/state.rs +5 -0
- data/ext/nosj/src/stats.rs +33 -30
- data/lib/nosj/json.rb +105 -39
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +34 -21
- data/sig/nosj.rbs +4 -3
- metadata +3 -1
data/ext/nosj/src/sink.rs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
//! The nosj sinks: `RubyValueSink` builds Ruby VALUEs directly during
|
|
2
2
|
//! the parse (with interned-key caches and gem-compatible option
|
|
3
3
|
//! handling); `NullSink` powers `NOSJ.valid?` by discarding every
|
|
4
|
-
//! event
|
|
4
|
+
//! event; `DupKeys` gives hash-less sinks duplicate-key detection. Raw
|
|
5
|
+
//! VALUE construction helpers live here too.
|
|
5
6
|
|
|
6
7
|
use ahash::AHashMap;
|
|
7
8
|
|
|
@@ -17,17 +18,20 @@ pub(crate) const MAX_NESTING: usize = 100;
|
|
|
17
18
|
const KEY_CACHE_CAP: usize = 2048;
|
|
18
19
|
|
|
19
20
|
/// Why a sink stopped the drive; mapped onto the gem's exceptions in
|
|
20
|
-
/// [`crate::parse::
|
|
21
|
+
/// [`crate::parse::drive_error`]. Sinks see no offsets, so the two
|
|
22
|
+
/// document refusals get their position from a cold-path re-walk
|
|
23
|
+
/// (`crate::locate`).
|
|
21
24
|
pub(crate) enum SinkAbort {
|
|
22
25
|
Overflow,
|
|
23
26
|
BadBigint,
|
|
24
27
|
TooDeep,
|
|
25
|
-
///
|
|
26
|
-
///
|
|
27
|
-
///
|
|
28
|
-
|
|
29
|
-
///
|
|
30
|
-
|
|
28
|
+
/// An object repeats a key and `allow_duplicate_key` is off (json
|
|
29
|
+
/// 3 semantics). From `DupKeys` it may be a fingerprint collision,
|
|
30
|
+
/// which the exact cold-path check rules out.
|
|
31
|
+
DuplicateKey,
|
|
32
|
+
/// A string or key decodes to a lone UTF-16 surrogate (json 3
|
|
33
|
+
/// rejects trailing ones too, not only leading ones).
|
|
34
|
+
LoneSurrogate,
|
|
31
35
|
/// The reformat pipe met a non-finite float without `allow_nan`.
|
|
32
36
|
/// Parsing accepts huge-exponent literals like 1e999 as Infinity
|
|
33
37
|
/// even in strict mode (gem parity), but generation refuses them;
|
|
@@ -35,6 +39,145 @@ pub(crate) enum SinkAbort {
|
|
|
35
39
|
NonFiniteFloat(&'static str),
|
|
36
40
|
}
|
|
37
41
|
|
|
42
|
+
/// Duplicate-key detection for sinks that build no Hash (validation and
|
|
43
|
+
/// the reformat pipe): each key is remembered as a 64-bit fingerprint,
|
|
44
|
+
/// `mark()` at a container's start is the fingerprint count, and an
|
|
45
|
+
/// object's close checks only its own keys. Integers, because comparing
|
|
46
|
+
/// them is what keeps this cheap: comparing the key bytes themselves
|
|
47
|
+
/// measured up to 93% slower (twitter's 40-key objects). A fingerprint
|
|
48
|
+
/// collision reads as a duplicate, so callers confirm a hit exactly on
|
|
49
|
+
/// the cold path; a collision can only cost time.
|
|
50
|
+
///
|
|
51
|
+
/// Measured split (twitter `valid?`): hashing and pushing each key is
|
|
52
|
+
/// ~0-2%; the close-time check is the cost, hence [`SeenTable`].
|
|
53
|
+
pub(crate) struct DupKeys<'a> {
|
|
54
|
+
scratch: &'a mut DupScratch,
|
|
55
|
+
enabled: bool,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/// [`DupKeys`]' pooled buffers (per thread, in `PullState`): the key
|
|
59
|
+
/// fingerprints of every open object, and the close-time set.
|
|
60
|
+
#[derive(Default)]
|
|
61
|
+
pub(crate) struct DupScratch {
|
|
62
|
+
fingerprints: Vec<u64>,
|
|
63
|
+
table: SeenTable,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/// Fixed seeds: fingerprints only need to spread keys, and a collision
|
|
67
|
+
/// merely sends the document to the exact cold-path check. (A leaner
|
|
68
|
+
/// hand-rolled hash measured no faster: hashing is not the cost.)
|
|
69
|
+
const FINGERPRINT: ahash::RandomState = ahash::RandomState::with_seeds(
|
|
70
|
+
0x243f_6a88_85a3_08d3,
|
|
71
|
+
0x1319_8a2e_0370_7344,
|
|
72
|
+
0xa409_3822_299f_31d0,
|
|
73
|
+
0x082e_fa98_ec4e_6c89,
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
/// Objects up to this many keys compare pairwise (a few compares beat
|
|
77
|
+
/// any table traffic).
|
|
78
|
+
const PAIRWISE_MAX: usize = 8;
|
|
79
|
+
/// Table slots; a power of two, so a fingerprint's low bits index it.
|
|
80
|
+
const TABLE_SLOTS: usize = 256;
|
|
81
|
+
/// Objects up to this many keys use the table, which then stays at
|
|
82
|
+
/// least half empty; larger ones sort.
|
|
83
|
+
const TABLE_MAX_KEYS: usize = TABLE_SLOTS / 2;
|
|
84
|
+
|
|
85
|
+
/// Open-addressing set for one object's close: fingerprints are already
|
|
86
|
+
/// uniformly mixed, so their low bits index it directly, and one probe
|
|
87
|
+
/// per key usually decides. Each close claims a fresh epoch, and a slot
|
|
88
|
+
/// is empty unless it carries the current one, so nothing is cleared
|
|
89
|
+
/// between objects.
|
|
90
|
+
struct SeenTable {
|
|
91
|
+
slots: Box<[(u64, u32)]>,
|
|
92
|
+
epoch: u32,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
impl Default for SeenTable {
|
|
96
|
+
fn default() -> Self {
|
|
97
|
+
SeenTable {
|
|
98
|
+
slots: vec![(0, 0); TABLE_SLOTS].into_boxed_slice(),
|
|
99
|
+
epoch: 0,
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
impl SeenTable {
|
|
105
|
+
/// Whether `keys` repeats a fingerprint. Callers keep
|
|
106
|
+
/// `keys.len() <= TABLE_MAX_KEYS`, so a free slot always exists.
|
|
107
|
+
#[inline(always)]
|
|
108
|
+
fn any_repeat(&mut self, keys: &[u64]) -> bool {
|
|
109
|
+
const MASK: usize = TABLE_SLOTS - 1;
|
|
110
|
+
self.epoch = self.epoch.wrapping_add(1);
|
|
111
|
+
if self.epoch == 0 {
|
|
112
|
+
// Epoch 0 marks an empty slot; after a full wrap, really clear.
|
|
113
|
+
self.slots.fill((0, 0));
|
|
114
|
+
self.epoch = 1;
|
|
115
|
+
}
|
|
116
|
+
let epoch = self.epoch;
|
|
117
|
+
for &fp in keys {
|
|
118
|
+
let mut at = fp as usize & MASK;
|
|
119
|
+
loop {
|
|
120
|
+
let slot = &mut self.slots[at];
|
|
121
|
+
if slot.1 != epoch {
|
|
122
|
+
*slot = (fp, epoch);
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
if slot.0 == fp {
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
at = (at + 1) & MASK;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
false
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
impl<'a> DupKeys<'a> {
|
|
136
|
+
pub(crate) fn new(scratch: &'a mut DupScratch, enabled: bool) -> Self {
|
|
137
|
+
scratch.fingerprints.clear();
|
|
138
|
+
DupKeys { scratch, enabled }
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
#[inline(always)]
|
|
142
|
+
pub(crate) fn mark(&self) -> usize {
|
|
143
|
+
self.scratch.fingerprints.len()
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
#[inline(always)]
|
|
147
|
+
pub(crate) fn key(&mut self, key: &[u8]) {
|
|
148
|
+
if self.enabled {
|
|
149
|
+
self.scratch.fingerprints.push(FINGERPRINT.hash_one(key));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/// Close the object whose keys start at `mark`.
|
|
154
|
+
#[inline(always)]
|
|
155
|
+
pub(crate) fn close(&mut self, mark: usize) -> Result<(), SinkAbort> {
|
|
156
|
+
if !self.enabled {
|
|
157
|
+
return Ok(());
|
|
158
|
+
}
|
|
159
|
+
let DupScratch {
|
|
160
|
+
fingerprints,
|
|
161
|
+
table,
|
|
162
|
+
} = &mut *self.scratch;
|
|
163
|
+
let keys = &mut fingerprints[mark..];
|
|
164
|
+
let repeated = match keys.len() {
|
|
165
|
+
n if n <= PAIRWISE_MAX => (1..n).any(|i| keys[..i].contains(&keys[i])),
|
|
166
|
+
n if n <= TABLE_MAX_KEYS => table.any_repeat(keys),
|
|
167
|
+
_ => {
|
|
168
|
+
keys.sort_unstable();
|
|
169
|
+
keys.windows(2).any(|w| w[0] == w[1])
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
fingerprints.truncate(mark);
|
|
173
|
+
if repeated {
|
|
174
|
+
Err(SinkAbort::DuplicateKey)
|
|
175
|
+
} else {
|
|
176
|
+
Ok(())
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
38
181
|
/// Integer VALUE for `i` via rb-sys's inline `LONG2NUM` (the header
|
|
39
182
|
/// macro: a Fixnum tagged inline when it fits, else a Bignum), which
|
|
40
183
|
/// saves the FFI call per integer that `rb_ll2inum` costs. The fixable
|
|
@@ -143,6 +286,7 @@ pub(crate) struct RubyValueSink<'a> {
|
|
|
143
286
|
pub(crate) symbolize: bool,
|
|
144
287
|
pub(crate) freeze: bool,
|
|
145
288
|
pub(crate) max_nesting: usize,
|
|
289
|
+
pub(crate) allow_duplicate_key: bool,
|
|
146
290
|
}
|
|
147
291
|
|
|
148
292
|
// Tried and rejected (2026-07-10): a jiter-style cache of repeated VALUE
|
|
@@ -225,35 +369,14 @@ impl nosj::Sink for RubyValueSink<'_> {
|
|
|
225
369
|
self.push_raw(raw)
|
|
226
370
|
}
|
|
227
371
|
|
|
228
|
-
/// Lone-
|
|
229
|
-
///
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
let raw = unsafe {
|
|
233
|
-
let s = rb_sys::rb_utf8_str_new(
|
|
234
|
-
value.as_ptr() as *const std::os::raw::c_char,
|
|
235
|
-
value.len() as std::os::raw::c_long,
|
|
236
|
-
);
|
|
237
|
-
if self.freeze {
|
|
238
|
-
rb_sys::rb_str_freeze(s)
|
|
239
|
-
} else {
|
|
240
|
-
s
|
|
241
|
-
}
|
|
242
|
-
};
|
|
243
|
-
self.push_raw(raw)
|
|
372
|
+
/// Lone-surrogate content (the crate hands it over as WTF-8): json 3
|
|
373
|
+
/// rejects it, trailing surrogates included.
|
|
374
|
+
fn str_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
|
|
375
|
+
Err(SinkAbort::LoneSurrogate)
|
|
244
376
|
}
|
|
245
377
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
// Interning is skipped: these keys are pathological, not hot.
|
|
249
|
-
let raw = unsafe {
|
|
250
|
-
rb_sys::rb_utf8_str_new(
|
|
251
|
-
key.as_ptr() as *const std::os::raw::c_char,
|
|
252
|
-
key.len() as std::os::raw::c_long,
|
|
253
|
-
)
|
|
254
|
-
};
|
|
255
|
-
let frozen = unsafe { rb_sys::rb_str_freeze(raw) };
|
|
256
|
-
self.push_raw(frozen)
|
|
378
|
+
fn key_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
|
|
379
|
+
Err(SinkAbort::LoneSurrogate)
|
|
257
380
|
}
|
|
258
381
|
|
|
259
382
|
#[inline(always)]
|
|
@@ -311,19 +434,28 @@ impl nosj::Sink for RubyValueSink<'_> {
|
|
|
311
434
|
}
|
|
312
435
|
}
|
|
313
436
|
self.stack.truncate(mark);
|
|
437
|
+
// A repeated key collapses into one entry: the hash comes out
|
|
438
|
+
// smaller than the pair count (one size read per object).
|
|
439
|
+
if !self.allow_duplicate_key
|
|
440
|
+
&& (unsafe { rb_sys::macros::RHASH_SIZE(hash_raw) } as usize) < pairs
|
|
441
|
+
{
|
|
442
|
+
return Err(SinkAbort::DuplicateKey);
|
|
443
|
+
}
|
|
314
444
|
self.push_raw(hash_raw)
|
|
315
445
|
}
|
|
316
446
|
}
|
|
317
447
|
|
|
318
448
|
/// Validation-only sink: every event is a no-op except nesting-depth
|
|
319
|
-
/// tracking, so `NOSJ.valid?` runs the
|
|
320
|
-
/// string decode, number validation) without
|
|
321
|
-
|
|
449
|
+
/// tracking and duplicate-key fingerprints, so `NOSJ.valid?` runs the
|
|
450
|
+
/// full parser (tokenizers, string decode, number validation) without
|
|
451
|
+
/// allocating a single VALUE.
|
|
452
|
+
pub(crate) struct NullSink<'a> {
|
|
322
453
|
pub(crate) depth: usize,
|
|
323
454
|
pub(crate) max_nesting: usize,
|
|
455
|
+
pub(crate) dup_keys: DupKeys<'a>,
|
|
324
456
|
}
|
|
325
457
|
|
|
326
|
-
impl nosj::Sink for NullSink {
|
|
458
|
+
impl nosj::Sink for NullSink<'_> {
|
|
327
459
|
type Error = SinkAbort;
|
|
328
460
|
|
|
329
461
|
fn null(&mut self) -> Result<(), SinkAbort> {
|
|
@@ -344,14 +476,18 @@ impl nosj::Sink for NullSink {
|
|
|
344
476
|
fn str(&mut self, _: &str) -> Result<(), SinkAbort> {
|
|
345
477
|
Ok(())
|
|
346
478
|
}
|
|
347
|
-
fn key(&mut self,
|
|
479
|
+
fn key(&mut self, key: &str) -> Result<(), SinkAbort> {
|
|
480
|
+
self.dup_keys.key(key.as_bytes());
|
|
348
481
|
Ok(())
|
|
349
482
|
}
|
|
350
483
|
fn str_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
|
|
351
|
-
|
|
484
|
+
Err(SinkAbort::LoneSurrogate)
|
|
485
|
+
}
|
|
486
|
+
fn key_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
|
|
487
|
+
Err(SinkAbort::LoneSurrogate)
|
|
352
488
|
}
|
|
353
489
|
fn mark(&self) -> usize {
|
|
354
|
-
|
|
490
|
+
self.dup_keys.mark()
|
|
355
491
|
}
|
|
356
492
|
fn begin_array(&mut self) -> Result<(), SinkAbort> {
|
|
357
493
|
self.depth += 1;
|
|
@@ -371,8 +507,8 @@ impl nosj::Sink for NullSink {
|
|
|
371
507
|
self.depth -= 1;
|
|
372
508
|
Ok(())
|
|
373
509
|
}
|
|
374
|
-
fn end_object(&mut self,
|
|
510
|
+
fn end_object(&mut self, mark: usize, _: usize) -> Result<(), SinkAbort> {
|
|
375
511
|
self.depth -= 1;
|
|
376
|
-
|
|
512
|
+
self.dup_keys.close(mark)
|
|
377
513
|
}
|
|
378
514
|
}
|
data/ext/nosj/src/state.rs
CHANGED
|
@@ -13,6 +13,8 @@ use nosj::Buffers;
|
|
|
13
13
|
use std::cell::Cell;
|
|
14
14
|
use std::thread::LocalKey;
|
|
15
15
|
|
|
16
|
+
use crate::sink::DupScratch;
|
|
17
|
+
|
|
16
18
|
/// Run `f` on a pooled thread-local value, taken OUT of its cell for the
|
|
17
19
|
/// call and stored back afterwards. Pooled state is never borrowed
|
|
18
20
|
/// across a call that can reach Ruby: any allocation can raise
|
|
@@ -45,6 +47,8 @@ pub(crate) struct PullState {
|
|
|
45
47
|
/// Marked shadow holding the cached key VALUEs; keys are kept alive by
|
|
46
48
|
/// this (collectable on epoch clear), NOT by per-key eternal GC pins.
|
|
47
49
|
pub(crate) key_shadow: Option<&'static mut VStackShadow>,
|
|
50
|
+
/// Duplicate-key detection buffers for hash-less sinks.
|
|
51
|
+
pub(crate) dup: DupScratch,
|
|
48
52
|
}
|
|
49
53
|
|
|
50
54
|
impl PullState {
|
|
@@ -56,6 +60,7 @@ impl PullState {
|
|
|
56
60
|
sym_keys: AHashMap::new(),
|
|
57
61
|
vstack: None,
|
|
58
62
|
key_shadow: None,
|
|
63
|
+
dup: DupScratch::default(),
|
|
59
64
|
})
|
|
60
65
|
}
|
|
61
66
|
}
|
data/ext/nosj/src/stats.rs
CHANGED
|
@@ -8,9 +8,9 @@ use ahash::AHashMap;
|
|
|
8
8
|
use magnus::value::ReprValue;
|
|
9
9
|
use magnus::{Error, RHash, RString, Ruby, Value};
|
|
10
10
|
|
|
11
|
-
use crate::errors::{nesting_error, parser_error, parser_error_at};
|
|
12
11
|
use crate::files::with_mapped_file;
|
|
13
|
-
use crate::
|
|
12
|
+
use crate::opt_reader::{Opt, OptReader};
|
|
13
|
+
use crate::parse::{drive_error, max_nesting_of, options_hash, utf8_input, ParseNativeOpts};
|
|
14
14
|
use crate::sink::SinkAbort;
|
|
15
15
|
use crate::state::with_pull_state;
|
|
16
16
|
|
|
@@ -239,40 +239,41 @@ fn stats_to_hash(ruby: &Ruby, s: &StatsSink, byte_size: usize) -> Result<Value,
|
|
|
239
239
|
Ok(out.as_value())
|
|
240
240
|
}
|
|
241
241
|
|
|
242
|
-
///
|
|
243
|
-
///
|
|
244
|
-
///
|
|
245
|
-
///
|
|
246
|
-
fn
|
|
247
|
-
let o =
|
|
248
|
-
|
|
249
|
-
|
|
242
|
+
/// Stats' options: `max_nesting`, which here defaults to UNLIMITED (a
|
|
243
|
+
/// deep blob is exactly what a diagnostic should describe, not refuse),
|
|
244
|
+
/// and the grammar extensions. Decoded before the source is borrowed:
|
|
245
|
+
/// a hash lookup can run a key's own `hash`/`eql?`.
|
|
246
|
+
fn stats_opts(ruby: &Ruby, opts: Value) -> Result<ParseNativeOpts, Error> {
|
|
247
|
+
let mut o = ParseNativeOpts {
|
|
248
|
+
max_nesting: usize::MAX,
|
|
249
|
+
..ParseNativeOpts::default()
|
|
250
|
+
};
|
|
251
|
+
let Some(h) = options_hash(ruby, opts)? else {
|
|
252
|
+
return Ok(o);
|
|
253
|
+
};
|
|
254
|
+
let mut r = OptReader::new(ruby, h);
|
|
255
|
+
if let Some(v) = r.get(Opt::MaxNesting) {
|
|
256
|
+
o.max_nesting = max_nesting_of(v);
|
|
257
|
+
}
|
|
258
|
+
o.popts.allow_nan = r.truthy(Opt::AllowNan);
|
|
259
|
+
o.popts.allow_trailing_comma = r.truthy(Opt::AllowTrailingComma);
|
|
260
|
+
r.finish()?;
|
|
261
|
+
Ok(o)
|
|
262
|
+
}
|
|
250
263
|
|
|
264
|
+
/// Run the counting pass over already-UTF-8-vouched bytes and build
|
|
265
|
+
/// the result.
|
|
266
|
+
fn stats_over(ruby: &Ruby, input: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
|
|
251
267
|
let mut sink = StatsSink {
|
|
252
|
-
max_nesting:
|
|
253
|
-
o.max_nesting
|
|
254
|
-
} else {
|
|
255
|
-
usize::MAX
|
|
256
|
-
},
|
|
268
|
+
max_nesting: o.max_nesting,
|
|
257
269
|
..StatsSink::default()
|
|
258
270
|
};
|
|
259
271
|
let result = with_pull_state(|state| {
|
|
260
272
|
// Safety: callers verified UTF-8 (coderange or a full scan).
|
|
261
273
|
unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, o.popts) }
|
|
262
274
|
});
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
|
|
266
|
-
ruby,
|
|
267
|
-
format!("nesting of {} is too deep", o.max_nesting.saturating_add(1)),
|
|
268
|
-
)),
|
|
269
|
-
// The other aborts cannot happen (this sink never raises them),
|
|
270
|
-
// but the match must be total.
|
|
271
|
-
Err(nosj::DriveError::Sink(_)) => Err(parser_error(ruby, "stats pass aborted".into())),
|
|
272
|
-
Err(nosj::DriveError::Parse(e)) => {
|
|
273
|
-
Err(parser_error_at(ruby, input, e.offset, e.to_string()))
|
|
274
|
-
}
|
|
275
|
-
}
|
|
275
|
+
result.map_err(|failure| drive_error(ruby, failure, o, input, (0, input.len())))?;
|
|
276
|
+
stats_to_hash(ruby, &sink, input.len())
|
|
276
277
|
}
|
|
277
278
|
|
|
278
279
|
/// `NOSJ.stats(source, opts)`: document statistics from one null-sink
|
|
@@ -283,8 +284,9 @@ pub fn stats_native(
|
|
|
283
284
|
data: RString,
|
|
284
285
|
opts: Value,
|
|
285
286
|
) -> Result<Value, Error> {
|
|
287
|
+
let o = stats_opts(ruby, opts)?;
|
|
286
288
|
let input = utf8_input(ruby, &data)?;
|
|
287
|
-
stats_over(ruby, input,
|
|
289
|
+
stats_over(ruby, input, &o)
|
|
288
290
|
}
|
|
289
291
|
|
|
290
292
|
/// `NOSJ.stats_file(path, opts)`: `NOSJ.stats` against a memory-mapped
|
|
@@ -295,6 +297,7 @@ pub fn stats_file_native(
|
|
|
295
297
|
path: RString,
|
|
296
298
|
opts: Value,
|
|
297
299
|
) -> Result<Value, Error> {
|
|
300
|
+
let o = stats_opts(ruby, opts)?;
|
|
298
301
|
let p = path.to_string()?;
|
|
299
|
-
with_mapped_file(ruby, &p, |map| stats_over(ruby, &map,
|
|
302
|
+
with_mapped_file(ruby, &p, |map| stats_over(ruby, &map, &o))
|
|
300
303
|
}
|
data/lib/nosj/json.rb
CHANGED
|
@@ -12,12 +12,18 @@
|
|
|
12
12
|
# Entry points built on JSON.parse (JSON.load, JSON.parse!,
|
|
13
13
|
# JSON.load_file, JSON.unsafe_load) pick up the fast path automatically
|
|
14
14
|
# and keep their exact legacy behavior when they need unsupported options
|
|
15
|
-
# (JSON.load
|
|
15
|
+
# (json 2's JSON.load passes create_additions, so it always takes the
|
|
16
|
+
# fallback).
|
|
16
17
|
#
|
|
17
|
-
#
|
|
18
|
-
# (
|
|
19
|
-
#
|
|
20
|
-
#
|
|
18
|
+
# The drop-in follows whichever json is installed, 2.x or 3.x: its
|
|
19
|
+
# calling conventions (json 3's keyword-only parse, its dump defaults)
|
|
20
|
+
# and its semantics. NOSJ implements json 3's (duplicate keys and lone
|
|
21
|
+
# surrogates are errors), so whenever the fast path refuses a call, the
|
|
22
|
+
# original gem runs it again and has the last word: json 2 accepts what
|
|
23
|
+
# it always accepted, and every exception is the gem's own, message,
|
|
24
|
+
# json_path and invalid_object included. That second pass happens on
|
|
25
|
+
# failures only; a generate run twice this way calls to_json on the
|
|
26
|
+
# objects visited before the refusal twice.
|
|
21
27
|
#
|
|
22
28
|
# Not rerouted: obj.to_json (core extensions drive the gem's generator
|
|
23
29
|
# directly), and objects with a custom to_json inside a rerouted
|
|
@@ -30,15 +36,26 @@ module NOSJ
|
|
|
30
36
|
# Implementation detail of `require "nosj/json"`.
|
|
31
37
|
# @private
|
|
32
38
|
module JSONDropIn
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
|
|
39
|
+
# json 3 made parse's options keyword-only, fixed dump's defaults and
|
|
40
|
+
# raises for options json 2 ignored or aliased.
|
|
41
|
+
JSON3 = ::JSON::VERSION.to_i >= 3
|
|
42
|
+
|
|
36
43
|
PARSE_OPTS = %i[symbolize_names freeze max_nesting allow_nan
|
|
37
|
-
allow_trailing_comma
|
|
44
|
+
allow_trailing_comma allow_duplicate_key].freeze
|
|
45
|
+
# json 2 ignores quirks_mode, which Rails 7.x passes from
|
|
46
|
+
# ActiveSupport::JSON.decode, so its fast path takes the key and
|
|
47
|
+
# drops it (NOSJ.parse always parses top-level scalars, and refuses
|
|
48
|
+
# unknown keys). json 3 raises for it, so there it reaches the gem
|
|
49
|
+
# like any other unknown option.
|
|
50
|
+
QUIRKS_MODE = :quirks_mode
|
|
51
|
+
JSON2_PARSE_OPTS = (PARSE_OPTS + [QUIRKS_MODE]).freeze
|
|
38
52
|
GENERATE_OPTS = %i[indent space space_before object_nl array_nl
|
|
39
|
-
max_nesting allow_nan ascii_only script_safe
|
|
40
|
-
|
|
41
|
-
|
|
53
|
+
max_nesting allow_nan ascii_only script_safe strict depth
|
|
54
|
+
buffer_initial_length allow_duplicate_key].freeze
|
|
55
|
+
JSON3_DUMP_DEFAULTS = {allow_nan: true}.freeze
|
|
56
|
+
# json 2.10 and older accept only strict: in dump's options hash,
|
|
57
|
+
# through this private helper; later versions merge any option.
|
|
58
|
+
DUMP_MERGES_OPTIONS = !::JSON.respond_to?(:merge_dump_options, true)
|
|
42
59
|
|
|
43
60
|
module_function
|
|
44
61
|
|
|
@@ -52,6 +69,13 @@ module NOSJ
|
|
|
52
69
|
true
|
|
53
70
|
end
|
|
54
71
|
|
|
72
|
+
# Generate options the fast path handles: supported keys, minus the
|
|
73
|
+
# one combination NOSJ refuses (ascii_only with script_safe, which
|
|
74
|
+
# json supports).
|
|
75
|
+
def generate_supported?(opts)
|
|
76
|
+
supported?(opts, GENERATE_OPTS) && !(opts && opts[:ascii_only] && opts[:script_safe])
|
|
77
|
+
end
|
|
78
|
+
|
|
55
79
|
def parse(source, opts)
|
|
56
80
|
# NOSJ.parse is deliberately strict about encodings (json-3.0
|
|
57
81
|
# semantics), but the drop-in must match the installed gem, which
|
|
@@ -61,30 +85,60 @@ module NOSJ
|
|
|
61
85
|
# (copy-on-write bytes), and the validity scan is memoized
|
|
62
86
|
# coderange the parse would compute anyway. Anything else
|
|
63
87
|
# non-UTF-8 (UTF-16, ...) belongs to gem json, which transcodes.
|
|
88
|
+
input = source
|
|
64
89
|
if source.is_a?(String)
|
|
65
90
|
case source.encoding
|
|
66
91
|
when Encoding::UTF_8, Encoding::US_ASCII
|
|
67
92
|
# the fast path as-is
|
|
68
93
|
when Encoding::BINARY
|
|
69
94
|
utf8 = source.dup.force_encoding(Encoding::UTF_8)
|
|
70
|
-
|
|
95
|
+
input = utf8 if utf8.valid_encoding?
|
|
71
96
|
else
|
|
72
|
-
return
|
|
97
|
+
return original_parse(source, opts)
|
|
73
98
|
end
|
|
74
99
|
end
|
|
75
|
-
NOSJ.parse(
|
|
76
|
-
rescue NOSJ::NestingError
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
100
|
+
NOSJ.parse(input, opts)
|
|
101
|
+
rescue NOSJ::ParserError, NOSJ::NestingError
|
|
102
|
+
original_parse(source, opts)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# json 2's options without quirks_mode (see QUIRKS_MODE), allocating
|
|
106
|
+
# nothing for Rails' lone `quirks_mode: true`.
|
|
107
|
+
def without_quirks_mode(opts)
|
|
108
|
+
return opts unless opts&.key?(QUIRKS_MODE)
|
|
109
|
+
if opts.size == 1
|
|
110
|
+
nil
|
|
111
|
+
else
|
|
112
|
+
opts.except(QUIRKS_MODE)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The installed gem's parse. json 3 takes keywords only; json 2's
|
|
117
|
+
# positional options hash receives them just the same.
|
|
118
|
+
def original_parse(source, opts)
|
|
119
|
+
::JSON.nosj_original_parse(source, **(opts || {}))
|
|
80
120
|
end
|
|
81
121
|
|
|
82
122
|
def generate(obj, opts, pretty)
|
|
83
123
|
pretty ? NOSJ.pretty_generate(obj, opts) : NOSJ.generate(obj, opts)
|
|
84
|
-
rescue NOSJ::NestingError
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
124
|
+
rescue NOSJ::GeneratorError, NOSJ::NestingError
|
|
125
|
+
if pretty
|
|
126
|
+
::JSON.nosj_original_pretty_generate(obj, opts)
|
|
127
|
+
else
|
|
128
|
+
::JSON.nosj_original_generate(obj, opts)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# The options JSON.dump generates with before the caller's own: fixed
|
|
133
|
+
# in json 3; json 2 reads its user-settable dump_default_options,
|
|
134
|
+
# through the internal reader 2.11 added when it deprecated the
|
|
135
|
+
# public one. Which one is decided here, once.
|
|
136
|
+
if JSON3
|
|
137
|
+
def dump_defaults = JSON3_DUMP_DEFAULTS
|
|
138
|
+
elsif ::JSON.respond_to?(:_dump_default_options)
|
|
139
|
+
def dump_defaults = ::JSON._dump_default_options
|
|
140
|
+
else
|
|
141
|
+
def dump_defaults = ::JSON.dump_default_options
|
|
88
142
|
end
|
|
89
143
|
end
|
|
90
144
|
end
|
|
@@ -101,16 +155,26 @@ module JSON
|
|
|
101
155
|
alias_method :nosj_original_pretty_generate, :pretty_generate
|
|
102
156
|
alias_method :nosj_original_dump, :dump
|
|
103
157
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
NOSJ::JSONDropIn.
|
|
107
|
-
|
|
108
|
-
|
|
158
|
+
if NOSJ::JSONDropIn::JSON3
|
|
159
|
+
def parse(source, **opts)
|
|
160
|
+
if NOSJ::JSONDropIn.supported?(opts, NOSJ::JSONDropIn::PARSE_OPTS)
|
|
161
|
+
NOSJ::JSONDropIn.parse(source, opts)
|
|
162
|
+
else
|
|
163
|
+
nosj_original_parse(source, **opts)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
else
|
|
167
|
+
def parse(source, opts = nil)
|
|
168
|
+
if NOSJ::JSONDropIn.supported?(opts, NOSJ::JSONDropIn::JSON2_PARSE_OPTS)
|
|
169
|
+
NOSJ::JSONDropIn.parse(source, NOSJ::JSONDropIn.without_quirks_mode(opts))
|
|
170
|
+
else
|
|
171
|
+
nosj_original_parse(source, opts)
|
|
172
|
+
end
|
|
109
173
|
end
|
|
110
174
|
end
|
|
111
175
|
|
|
112
176
|
def generate(obj, opts = nil)
|
|
113
|
-
if NOSJ::JSONDropIn.
|
|
177
|
+
if NOSJ::JSONDropIn.generate_supported?(opts)
|
|
114
178
|
NOSJ::JSONDropIn.generate(obj, opts, false)
|
|
115
179
|
else
|
|
116
180
|
nosj_original_generate(obj, opts)
|
|
@@ -118,7 +182,7 @@ module JSON
|
|
|
118
182
|
end
|
|
119
183
|
|
|
120
184
|
def pretty_generate(obj, opts = nil)
|
|
121
|
-
if NOSJ::JSONDropIn.
|
|
185
|
+
if NOSJ::JSONDropIn.generate_supported?(opts)
|
|
122
186
|
NOSJ::JSONDropIn.generate(obj, opts, true)
|
|
123
187
|
else
|
|
124
188
|
nosj_original_pretty_generate(obj, opts)
|
|
@@ -127,17 +191,19 @@ module JSON
|
|
|
127
191
|
|
|
128
192
|
def dump(obj, an_io = nil, limit = nil, kwargs = nil)
|
|
129
193
|
# Fast path for the common shapes, dump(obj) and dump(obj, opts
|
|
130
|
-
# hash)
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
|
|
134
|
-
|
|
194
|
+
# hash): the installed gem's dump defaults merged under the
|
|
195
|
+
# caller's options. IO and limit arguments, and anything the
|
|
196
|
+
# fast path refuses, take gem json's own dump, which also turns
|
|
197
|
+
# json 2's NestingError into its ArgumentError.
|
|
198
|
+
if limit.nil? && kwargs.nil? &&
|
|
199
|
+
(an_io.nil? || NOSJ::JSONDropIn::DUMP_MERGES_OPTIONS && an_io.instance_of?(Hash))
|
|
200
|
+
opts = NOSJ::JSONDropIn.dump_defaults
|
|
135
201
|
opts = opts.merge(an_io) if an_io
|
|
136
|
-
if NOSJ::JSONDropIn.
|
|
202
|
+
if NOSJ::JSONDropIn.generate_supported?(opts)
|
|
137
203
|
begin
|
|
138
|
-
return NOSJ
|
|
139
|
-
rescue ::
|
|
140
|
-
|
|
204
|
+
return NOSJ.generate(obj, opts)
|
|
205
|
+
rescue NOSJ::GeneratorError, NOSJ::NestingError
|
|
206
|
+
# the gem's own dump below runs it again and decides
|
|
141
207
|
end
|
|
142
208
|
end
|
|
143
209
|
end
|
data/lib/nosj/version.rb
CHANGED