nosj 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -1
- data/Cargo.lock +11 -1
- data/README.md +63 -11
- data/ext/nosj/Cargo.toml +4 -1
- data/ext/nosj/src/files.rs +218 -0
- data/ext/nosj/src/gen/mod.rs +44 -10
- data/ext/nosj/src/lazy.rs +402 -0
- data/ext/nosj/src/lib.rs +30 -0
- data/ext/nosj/src/pointer.rs +14 -12
- data/lib/nosj/lazy.rb +165 -0
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +85 -2
- data/sig/nosj.rbs +54 -0
- metadata +7 -2
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
//! Lazy documents: `NOSJ.lazy` wraps a JSON document and resolves
|
|
2
|
+
//! access on demand through the crate's pointer skipper. Every node is
|
|
3
|
+
//! a byte span into a shared, immutable copy of the document; container
|
|
4
|
+
//! children come back as further lazy nodes, scalars materialize
|
|
5
|
+
//! immediately through the same sink machinery as a full parse. Nothing
|
|
6
|
+
//! outside the touched path is ever parsed, so pulling a few fields out
|
|
7
|
+
//! of a large document costs microseconds, not a full parse.
|
|
8
|
+
//!
|
|
9
|
+
//! Validation is as-you-go (the crate's skipper checks bracket balance
|
|
10
|
+
//! over skipped content and fully validates resolved targets), so a
|
|
11
|
+
//! malformed region raises when an access first walks it, not at
|
|
12
|
+
//! `NOSJ.lazy` time.
|
|
13
|
+
|
|
14
|
+
use std::sync::Arc;
|
|
15
|
+
|
|
16
|
+
use magnus::typed_data::Obj;
|
|
17
|
+
use magnus::value::ReprValue;
|
|
18
|
+
use magnus::{DataTypeFunctions, Error, RArray, RString, Ruby, TypedData, Value};
|
|
19
|
+
|
|
20
|
+
use crate::parse::{err, materialize, parse_native_opts, utf8_input, ParseNativeOpts};
|
|
21
|
+
use crate::pointer::{path_to_pointer, push_escaped_token};
|
|
22
|
+
use crate::state::PULL_STATE;
|
|
23
|
+
|
|
24
|
+
/// The document bytes behind a node tree. A frozen Ruby source is
|
|
25
|
+
/// borrowed zero-copy: freezing rules out mutation, and every node
|
|
26
|
+
/// GC-marks the string with `rb_gc_mark` semantics, which both keeps it
|
|
27
|
+
/// alive and pins it against compaction, so the captured pointer stays
|
|
28
|
+
/// valid for as long as any node exists. Anything else is copied once.
|
|
29
|
+
pub(crate) enum DocBytes {
|
|
30
|
+
Owned(Vec<u8>),
|
|
31
|
+
Frozen {
|
|
32
|
+
source: rb_sys::VALUE,
|
|
33
|
+
ptr: *const u8,
|
|
34
|
+
len: usize,
|
|
35
|
+
},
|
|
36
|
+
/// A read-only file mapping (`NOSJ.load_lazy_file`): pages never
|
|
37
|
+
/// touched are never read off disk. Concurrent modification of the
|
|
38
|
+
/// mapped file by another process is documented as unsupported
|
|
39
|
+
/// (the standard mmap caveat).
|
|
40
|
+
Mmap(memmap2::Mmap),
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/// The shared document: stable bytes plus the parse options every
|
|
44
|
+
/// materialization from this document uses.
|
|
45
|
+
struct DocInner {
|
|
46
|
+
bytes: DocBytes,
|
|
47
|
+
opts: ParseNativeOpts,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// SAFETY: the bytes are immutable for the document's whole life (an
|
|
51
|
+
// owned Vec, or a frozen Ruby string pinned and kept alive by every
|
|
52
|
+
// node's GC mark), so cross-thread reads are plain shared reads; the
|
|
53
|
+
// raw VALUE is only dereferenced by the GC mark, which runs at
|
|
54
|
+
// safepoints (the ShadowHandle contract in state.rs). There is no
|
|
55
|
+
// interior mutability anywhere in the type.
|
|
56
|
+
unsafe impl Send for DocInner {}
|
|
57
|
+
unsafe impl Sync for DocInner {}
|
|
58
|
+
|
|
59
|
+
impl DocInner {
|
|
60
|
+
fn bytes(&self) -> &[u8] {
|
|
61
|
+
match &self.bytes {
|
|
62
|
+
DocBytes::Owned(v) => v,
|
|
63
|
+
// SAFETY: the source string is frozen (no mutation, no
|
|
64
|
+
// buffer reallocation) and pinned+kept alive by every
|
|
65
|
+
// node's GC mark; see DocBytes.
|
|
66
|
+
DocBytes::Frozen { ptr, len, .. } => unsafe { std::slice::from_raw_parts(*ptr, *len) },
|
|
67
|
+
DocBytes::Mmap(m) => m,
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const KIND_OBJECT: u8 = b'{';
|
|
73
|
+
const KIND_ARRAY: u8 = b'[';
|
|
74
|
+
|
|
75
|
+
/// One lazy container node: a byte span into its document. Spans always
|
|
76
|
+
/// come from the crate's resolver (token edges within the doc bytes),
|
|
77
|
+
/// and never cross the Ruby boundary, so they cannot be forged from
|
|
78
|
+
/// Ruby.
|
|
79
|
+
#[derive(TypedData)]
|
|
80
|
+
#[magnus(class = "NOSJ::Lazy", mark)]
|
|
81
|
+
pub struct LazyNode {
|
|
82
|
+
doc: Arc<DocInner>,
|
|
83
|
+
start: usize,
|
|
84
|
+
end: usize,
|
|
85
|
+
kind: u8,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
impl DataTypeFunctions for LazyNode {
|
|
89
|
+
fn mark(&self, marker: &magnus::gc::Marker) {
|
|
90
|
+
if let DocBytes::Frozen { source, .. } = self.doc.bytes {
|
|
91
|
+
use magnus::rb_sys::FromRawValue;
|
|
92
|
+
// SAFETY: the VALUE was a live, frozen string at node
|
|
93
|
+
// creation and this mark is what keeps it that way.
|
|
94
|
+
marker.mark(unsafe { Value::from_raw(source) });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
impl LazyNode {
|
|
100
|
+
fn span(&self) -> &[u8] {
|
|
101
|
+
&self.doc.bytes()[self.start..self.end]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Wrap a resolved raw-value slice: containers become new lazy nodes,
|
|
106
|
+
/// scalars materialize now. `sub` must be a subslice of `doc.bytes`.
|
|
107
|
+
fn resolved_to_value(ruby: &Ruby, doc: &Arc<DocInner>, sub: &[u8]) -> Result<Value, Error> {
|
|
108
|
+
match sub.first().copied() {
|
|
109
|
+
Some(k) if k == KIND_OBJECT || k == KIND_ARRAY => {
|
|
110
|
+
let base = doc.bytes().as_ptr() as usize;
|
|
111
|
+
let start = sub.as_ptr() as usize - base;
|
|
112
|
+
let node = LazyNode {
|
|
113
|
+
doc: Arc::clone(doc),
|
|
114
|
+
start,
|
|
115
|
+
end: start + sub.len(),
|
|
116
|
+
kind: k,
|
|
117
|
+
};
|
|
118
|
+
let obj: Obj<LazyNode> = ruby.obj_wrap(node);
|
|
119
|
+
Ok(obj.as_value())
|
|
120
|
+
}
|
|
121
|
+
_ => materialize(ruby, sub, &doc.opts),
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/// Resolve `pointer` within `node`'s span. Shared by `__get` and
|
|
126
|
+
/// `__at_pointer`; both misses and negative-index paths return nil.
|
|
127
|
+
fn resolve_in_span(ruby: &Ruby, node: &LazyNode, pointer: &str) -> Result<Value, Error> {
|
|
128
|
+
// Resolve first (one PULL_STATE borrow, slice borrows the doc, not
|
|
129
|
+
// the buffers), then materialize (which re-borrows internally).
|
|
130
|
+
let resolved = PULL_STATE.with(|cell| {
|
|
131
|
+
let mut state = cell.borrow_mut();
|
|
132
|
+
// SAFETY: doc bytes were coderange-gated at NOSJ.lazy creation,
|
|
133
|
+
// and spans lie on token edges, so the span is valid UTF-8.
|
|
134
|
+
unsafe { nosj::pointer_utf8_unchecked(node.span(), pointer, &mut state.bufs) }
|
|
135
|
+
});
|
|
136
|
+
match resolved {
|
|
137
|
+
Ok(None) => Ok(ruby.qnil().as_value()),
|
|
138
|
+
Ok(Some(sub)) => resolved_to_value(ruby, &node.doc, sub.as_bytes()),
|
|
139
|
+
Err(e) if matches!(e.kind, nosj::ErrorKind::InvalidPointer) => {
|
|
140
|
+
Err(Error::new(ruby.exception_arg_error(), e.to_string()))
|
|
141
|
+
}
|
|
142
|
+
Err(e) => Err(err(ruby, e.to_string())),
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/// `NOSJ.lazy(source, opts)`: gate the encoding, copy the bytes, locate
|
|
147
|
+
/// the root value. Container roots wrap as lazy nodes; a scalar root has
|
|
148
|
+
/// nothing to defer and materializes immediately.
|
|
149
|
+
///
|
|
150
|
+
/// The root span is the input trimmed of surrounding whitespace; no
|
|
151
|
+
/// byte of content is walked here (resolving via the "" pointer would
|
|
152
|
+
/// bracket-skip the whole document just to find the same end), so
|
|
153
|
+
/// malformation anywhere, including root-level trailing garbage,
|
|
154
|
+
/// surfaces on first access, per the module's lazy-validation contract.
|
|
155
|
+
pub fn lazy_native(
|
|
156
|
+
ruby: &Ruby,
|
|
157
|
+
_rb_self: Value,
|
|
158
|
+
data: RString,
|
|
159
|
+
opts: Value,
|
|
160
|
+
) -> Result<Value, Error> {
|
|
161
|
+
let o = parse_native_opts(ruby, opts)?;
|
|
162
|
+
let input = utf8_input(ruby, &data)?;
|
|
163
|
+
|
|
164
|
+
// Frozen sources are borrowed zero-copy (see DocBytes); `data` is
|
|
165
|
+
// on the caller's machine stack, so it stays pinned through this
|
|
166
|
+
// call, and the node's mark takes over from the first GC on.
|
|
167
|
+
let bytes = if data.as_value().is_frozen() {
|
|
168
|
+
use magnus::rb_sys::AsRawValue;
|
|
169
|
+
DocBytes::Frozen {
|
|
170
|
+
source: data.as_raw(),
|
|
171
|
+
ptr: input.as_ptr(),
|
|
172
|
+
len: input.len(),
|
|
173
|
+
}
|
|
174
|
+
} else {
|
|
175
|
+
DocBytes::Owned(input.to_vec())
|
|
176
|
+
};
|
|
177
|
+
wrap_root(ruby, bytes, o)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/// Wrap document bytes as the root lazy value. Shared by `NOSJ.lazy`
|
|
181
|
+
/// and `NOSJ.load_lazy_file`; the bytes must already be known UTF-8.
|
|
182
|
+
pub(crate) fn wrap_root(
|
|
183
|
+
ruby: &Ruby,
|
|
184
|
+
bytes: DocBytes,
|
|
185
|
+
opts: ParseNativeOpts,
|
|
186
|
+
) -> Result<Value, Error> {
|
|
187
|
+
const WS: [u8; 4] = *b" \t\n\r";
|
|
188
|
+
let doc = Arc::new(DocInner { bytes, opts });
|
|
189
|
+
let input = doc.bytes();
|
|
190
|
+
let Some(start) = input.iter().position(|b| !WS.contains(b)) else {
|
|
191
|
+
return Err(err(ruby, "unexpected end of input".into()));
|
|
192
|
+
};
|
|
193
|
+
let end = input.iter().rposition(|b| !WS.contains(b)).unwrap() + 1;
|
|
194
|
+
let sub = &doc.bytes()[start..end];
|
|
195
|
+
resolved_to_value(ruby, &doc, sub)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/// `__get(token)`: one path step. Integer tokens are JSON Pointer
|
|
199
|
+
/// indices (negative ones resolve to nil, as in NOSJ.dig); String and
|
|
200
|
+
/// Symbol tokens are keys, `~`/`/`-escaped.
|
|
201
|
+
pub fn lazy_get(ruby: &Ruby, rb_self: Obj<LazyNode>, token: Value) -> Result<Value, Error> {
|
|
202
|
+
let mut ptr = String::new();
|
|
203
|
+
if let Some(int) = magnus::Integer::from_value(token) {
|
|
204
|
+
let idx = int.to_i64()?;
|
|
205
|
+
if idx < 0 {
|
|
206
|
+
return Ok(ruby.qnil().as_value());
|
|
207
|
+
}
|
|
208
|
+
ptr.push('/');
|
|
209
|
+
ptr.push_str(&idx.to_string());
|
|
210
|
+
} else if let Some(s) = RString::from_value(token) {
|
|
211
|
+
push_escaped_token(&mut ptr, &s.to_string()?);
|
|
212
|
+
} else if let Some(sym) = magnus::Symbol::from_value(token) {
|
|
213
|
+
push_escaped_token(&mut ptr, &sym.name()?);
|
|
214
|
+
} else {
|
|
215
|
+
return Err(Error::new(
|
|
216
|
+
ruby.exception_arg_error(),
|
|
217
|
+
"keys must be Strings, Symbols, or Integers",
|
|
218
|
+
));
|
|
219
|
+
}
|
|
220
|
+
resolve_in_span(ruby, &rb_self, &ptr)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/// `__dig(path)`: the whole dig path fused into ONE pointer resolution
|
|
224
|
+
/// within this node's span, instead of one resolve (and one cached
|
|
225
|
+
/// node) per step. Semantics match `NOSJ.dig`: negative indices and
|
|
226
|
+
/// steps into scalars resolve to nil.
|
|
227
|
+
pub fn lazy_dig(ruby: &Ruby, rb_self: Obj<LazyNode>, path: RArray) -> Result<Value, Error> {
|
|
228
|
+
match path_to_pointer(ruby, path)? {
|
|
229
|
+
Some(ptr) => resolve_in_span(ruby, &rb_self, &ptr),
|
|
230
|
+
None => Ok(ruby.qnil().as_value()),
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/// `__at_pointer(pointer)`: a full RFC 6901 pointer, resolved within
|
|
235
|
+
/// this node's subtree.
|
|
236
|
+
pub fn lazy_at_pointer(
|
|
237
|
+
ruby: &Ruby,
|
|
238
|
+
rb_self: Obj<LazyNode>,
|
|
239
|
+
pointer: RString,
|
|
240
|
+
) -> Result<Value, Error> {
|
|
241
|
+
let ptr = pointer.to_string()?;
|
|
242
|
+
resolve_in_span(ruby, &rb_self, &ptr)
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/// `__materialize`: the whole span as plain Ruby values, under the
|
|
246
|
+
/// document's parse options.
|
|
247
|
+
pub fn lazy_materialize(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<Value, Error> {
|
|
248
|
+
materialize(ruby, rb_self.span(), &rb_self.doc.opts)
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/// `__kind`: `:object` or `:array`.
|
|
252
|
+
pub fn lazy_kind(ruby: &Ruby, rb_self: Obj<LazyNode>) -> magnus::Symbol {
|
|
253
|
+
match rb_self.kind {
|
|
254
|
+
KIND_OBJECT => ruby.to_symbol("object"),
|
|
255
|
+
_ => ruby.to_symbol("array"),
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/// `__byte_size`: span length in bytes (cheap; used by #inspect).
|
|
260
|
+
pub fn lazy_byte_size(_ruby: &Ruby, rb_self: Obj<LazyNode>) -> usize {
|
|
261
|
+
rb_self.end - rb_self.start
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
fn reader_err(ruby: &Ruby, e: nosj::ParseError) -> Error {
|
|
265
|
+
err(ruby, e.to_string())
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/// `__keys`: the object's decoded keys, one Reader walk, values skipped.
|
|
269
|
+
pub fn lazy_keys(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
|
|
270
|
+
if rb_self.kind != KIND_OBJECT {
|
|
271
|
+
return Err(Error::new(
|
|
272
|
+
ruby.exception_type_error(),
|
|
273
|
+
"keys on a JSON array",
|
|
274
|
+
));
|
|
275
|
+
}
|
|
276
|
+
let out = ruby.ary_new();
|
|
277
|
+
PULL_STATE.with(|cell| -> Result<(), Error> {
|
|
278
|
+
let mut state = cell.borrow_mut();
|
|
279
|
+
// SAFETY: spans are valid UTF-8 (see resolve_in_span).
|
|
280
|
+
let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
|
|
281
|
+
r.next_node().map_err(|e| reader_err(ruby, e))?;
|
|
282
|
+
let mut has = match r.object_first_key().map_err(|e| reader_err(ruby, e))? {
|
|
283
|
+
Some(k) => {
|
|
284
|
+
out.push(ruby.str_new(k))?;
|
|
285
|
+
true
|
|
286
|
+
}
|
|
287
|
+
None => false,
|
|
288
|
+
};
|
|
289
|
+
while has {
|
|
290
|
+
r.skip_value().map_err(|e| reader_err(ruby, e))?;
|
|
291
|
+
has = match r.object_next_key().map_err(|e| reader_err(ruby, e))? {
|
|
292
|
+
Some(k) => {
|
|
293
|
+
out.push(ruby.str_new(k))?;
|
|
294
|
+
true
|
|
295
|
+
}
|
|
296
|
+
None => false,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
Ok(())
|
|
300
|
+
})?;
|
|
301
|
+
Ok(out)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/// `__size`: entry count (object pairs or array elements), one walk,
|
|
305
|
+
/// nothing materialized.
|
|
306
|
+
pub fn lazy_size(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<usize, Error> {
|
|
307
|
+
PULL_STATE.with(|cell| {
|
|
308
|
+
let mut state = cell.borrow_mut();
|
|
309
|
+
// SAFETY: spans are valid UTF-8 (see resolve_in_span).
|
|
310
|
+
let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
|
|
311
|
+
r.next_node().map_err(|e| reader_err(ruby, e))?;
|
|
312
|
+
let mut n = 0usize;
|
|
313
|
+
if rb_self.kind == KIND_OBJECT {
|
|
314
|
+
let mut has = r
|
|
315
|
+
.object_first_key()
|
|
316
|
+
.map_err(|e| reader_err(ruby, e))?
|
|
317
|
+
.is_some();
|
|
318
|
+
while has {
|
|
319
|
+
n += 1;
|
|
320
|
+
r.skip_value().map_err(|e| reader_err(ruby, e))?;
|
|
321
|
+
has = r
|
|
322
|
+
.object_next_key()
|
|
323
|
+
.map_err(|e| reader_err(ruby, e))?
|
|
324
|
+
.is_some();
|
|
325
|
+
}
|
|
326
|
+
} else {
|
|
327
|
+
let mut has = r.array_first().map_err(|e| reader_err(ruby, e))?;
|
|
328
|
+
while has {
|
|
329
|
+
n += 1;
|
|
330
|
+
r.skip_value().map_err(|e| reader_err(ruby, e))?;
|
|
331
|
+
has = r.array_next().map_err(|e| reader_err(ruby, e))?;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
Ok(n)
|
|
335
|
+
})
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/// A child discovered during a container walk: a doc-relative span,
|
|
339
|
+
/// plus the owned key for object entries (decoded keys borrow the
|
|
340
|
+
/// reader's scratch, so they are copied out before phase two).
|
|
341
|
+
struct ChildDesc {
|
|
342
|
+
key: Option<String>,
|
|
343
|
+
start: usize,
|
|
344
|
+
end: usize,
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/// `__children`: every direct child in ONE walk. Objects yield
|
|
348
|
+
/// `[key, child]` pairs, arrays yield children; containers wrap lazily,
|
|
349
|
+
/// scalars materialize. Two phases so the Reader's buffer borrow ends
|
|
350
|
+
/// before materialization re-borrows the thread state.
|
|
351
|
+
pub fn lazy_children(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
|
|
352
|
+
let base = rb_self.doc.bytes().as_ptr() as usize;
|
|
353
|
+
let descs: Result<Vec<ChildDesc>, nosj::ParseError> = PULL_STATE.with(|cell| {
|
|
354
|
+
let mut state = cell.borrow_mut();
|
|
355
|
+
// SAFETY: spans are valid UTF-8 (see resolve_in_span).
|
|
356
|
+
let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
|
|
357
|
+
r.next_node()?;
|
|
358
|
+
let mut out = Vec::new();
|
|
359
|
+
if rb_self.kind == KIND_OBJECT {
|
|
360
|
+
let mut key = r.object_first_key()?.map(String::from);
|
|
361
|
+
while let Some(k) = key {
|
|
362
|
+
let sub = r.skip_value()?;
|
|
363
|
+
let start = sub.as_ptr() as usize - base;
|
|
364
|
+
out.push(ChildDesc {
|
|
365
|
+
key: Some(k),
|
|
366
|
+
start,
|
|
367
|
+
end: start + sub.len(),
|
|
368
|
+
});
|
|
369
|
+
key = r.object_next_key()?.map(String::from);
|
|
370
|
+
}
|
|
371
|
+
} else {
|
|
372
|
+
let mut has = r.array_first()?;
|
|
373
|
+
while has {
|
|
374
|
+
let sub = r.skip_value()?;
|
|
375
|
+
let start = sub.as_ptr() as usize - base;
|
|
376
|
+
out.push(ChildDesc {
|
|
377
|
+
key: None,
|
|
378
|
+
start,
|
|
379
|
+
end: start + sub.len(),
|
|
380
|
+
});
|
|
381
|
+
has = r.array_next()?;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
Ok(out)
|
|
385
|
+
});
|
|
386
|
+
let descs = descs.map_err(|e| reader_err(ruby, e))?;
|
|
387
|
+
|
|
388
|
+
let out = ruby.ary_new_capa(descs.len());
|
|
389
|
+
for d in descs {
|
|
390
|
+
let child = resolved_to_value(ruby, &rb_self.doc, &rb_self.doc.bytes()[d.start..d.end])?;
|
|
391
|
+
match d.key {
|
|
392
|
+
Some(k) => {
|
|
393
|
+
let pair = ruby.ary_new_capa(2);
|
|
394
|
+
pair.push(ruby.str_new(&k))?;
|
|
395
|
+
pair.push(child)?;
|
|
396
|
+
out.push(pair)?;
|
|
397
|
+
}
|
|
398
|
+
None => out.push(child)?,
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
Ok(out)
|
|
402
|
+
}
|
data/ext/nosj/src/lib.rs
CHANGED
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
//! GVL-releasing indexed parse) plus shared option decoding and
|
|
6
6
|
//! input gating.
|
|
7
7
|
//! - `pointer.rs`: partial parsing (dig, at_pointer, batch forms).
|
|
8
|
+
//! - `lazy.rs`: lazy documents (NOSJ.lazy nodes resolving access on
|
|
9
|
+
//! demand over shared document bytes).
|
|
10
|
+
//! - `files.rs`: file entry points (load_file, write_file, and the
|
|
11
|
+
//! mmap-backed load_lazy_file / at_pointer_file / dig_file).
|
|
8
12
|
//! - `sink.rs`: the VALUE-building and validation sinks with their
|
|
9
13
|
//! interned-key caches.
|
|
10
14
|
//! - `state.rs`: per-thread reusable state and the GC-marked value
|
|
@@ -12,7 +16,9 @@
|
|
|
12
16
|
//! - `gen/`: generation (JSON.generate-compatible walker, options,
|
|
13
17
|
//! key cache, protect shims, error mapping).
|
|
14
18
|
|
|
19
|
+
pub mod files;
|
|
15
20
|
pub mod gen;
|
|
21
|
+
pub mod lazy;
|
|
16
22
|
pub mod parse;
|
|
17
23
|
pub mod pointer;
|
|
18
24
|
pub mod sink;
|
|
@@ -39,6 +45,30 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
39
45
|
"at_pointers_native",
|
|
40
46
|
method!(pointer::at_pointers_native, 3),
|
|
41
47
|
)?;
|
|
48
|
+
module.define_singleton_method("lazy_native", method!(lazy::lazy_native, 2))?;
|
|
49
|
+
module.define_singleton_method("load_file_native", method!(files::load_file_native, 2))?;
|
|
50
|
+
module.define_singleton_method("write_file_native", method!(files::write_file_native, 3))?;
|
|
51
|
+
module.define_singleton_method(
|
|
52
|
+
"load_lazy_file_native",
|
|
53
|
+
method!(files::load_lazy_file_native, 2),
|
|
54
|
+
)?;
|
|
55
|
+
module.define_singleton_method(
|
|
56
|
+
"at_pointer_file_native",
|
|
57
|
+
method!(files::at_pointer_file_native, 3),
|
|
58
|
+
)?;
|
|
59
|
+
module.define_singleton_method("dig_file_native", method!(files::dig_file_native, 2))?;
|
|
60
|
+
let lazy_class = module.define_class("Lazy", ruby.class_object())?;
|
|
61
|
+
// Nodes are only born from NOSJ.lazy / lazy resolution.
|
|
62
|
+
lazy_class.undef_default_alloc_func();
|
|
63
|
+
lazy_class.define_method("__get", method!(lazy::lazy_get, 1))?;
|
|
64
|
+
lazy_class.define_method("__dig", method!(lazy::lazy_dig, 1))?;
|
|
65
|
+
lazy_class.define_method("__at_pointer", method!(lazy::lazy_at_pointer, 1))?;
|
|
66
|
+
lazy_class.define_method("__materialize", method!(lazy::lazy_materialize, 0))?;
|
|
67
|
+
lazy_class.define_method("__kind", method!(lazy::lazy_kind, 0))?;
|
|
68
|
+
lazy_class.define_method("__byte_size", method!(lazy::lazy_byte_size, 0))?;
|
|
69
|
+
lazy_class.define_method("__keys", method!(lazy::lazy_keys, 0))?;
|
|
70
|
+
lazy_class.define_method("__size", method!(lazy::lazy_size, 0))?;
|
|
71
|
+
lazy_class.define_method("__children", method!(lazy::lazy_children, 0))?;
|
|
42
72
|
module.define_singleton_method("generate_native", method!(gen::generate_native, 2))?;
|
|
43
73
|
// `generate` itself is native and variadic: the json gem routes
|
|
44
74
|
// its `generate` through a Ruby frame into C, so skipping our own
|
data/ext/nosj/src/pointer.rs
CHANGED
|
@@ -49,21 +49,23 @@ pub fn at_pointer_native(
|
|
|
49
49
|
at_pointer_impl(ruby, data, &ptr, &o)
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
///
|
|
53
|
-
///
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
'/' => ptr.push_str("~1"),
|
|
62
|
-
c => ptr.push(c),
|
|
63
|
-
}
|
|
52
|
+
/// Append one RFC 6901 reference token (`/` prefix, `~`/`/` escaped).
|
|
53
|
+
/// Shared with the lazy-document single-step resolver.
|
|
54
|
+
pub(crate) fn push_escaped_token(ptr: &mut String, key: &str) {
|
|
55
|
+
ptr.push('/');
|
|
56
|
+
for c in key.chars() {
|
|
57
|
+
match c {
|
|
58
|
+
'~' => ptr.push_str("~0"),
|
|
59
|
+
'/' => ptr.push_str("~1"),
|
|
60
|
+
c => ptr.push(c),
|
|
64
61
|
}
|
|
65
62
|
}
|
|
63
|
+
}
|
|
66
64
|
|
|
65
|
+
/// Convert one dig path (String/Symbol keys, Integer indices) into a
|
|
66
|
+
/// JSON Pointer. `None` = a negative index: no pointer equivalent, the
|
|
67
|
+
/// path resolves to `nil` by definition. Shared with `NOSJ::Lazy#dig`.
|
|
68
|
+
pub(crate) fn path_to_pointer(ruby: &Ruby, path: magnus::RArray) -> Result<Option<String>, Error> {
|
|
67
69
|
let mut ptr = String::new();
|
|
68
70
|
for i in 0..path.len() {
|
|
69
71
|
let item: Value = path.entry(i as isize)?;
|
data/lib/nosj/lazy.rb
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module NOSJ
|
|
4
|
+
# A lazy view of one JSON container inside a document created with
|
|
5
|
+
# {NOSJ.lazy}. Nothing is parsed until it is touched: indexing walks
|
|
6
|
+
# the raw bytes to the requested child (skipping siblings at SIMD
|
|
7
|
+
# block speed), returns another Lazy node for containers, and
|
|
8
|
+
# materializes plain Ruby values for scalars. Resolved children are
|
|
9
|
+
# cached, so repeated access is free and object identity is stable.
|
|
10
|
+
#
|
|
11
|
+
# Validation is as-you-go: skipped content is bracket-balance
|
|
12
|
+
# checked and resolved targets fully validated, so a malformed region
|
|
13
|
+
# raises when an access first walks it, not at {NOSJ.lazy} time.
|
|
14
|
+
#
|
|
15
|
+
# Nodes hold their own stable copy of the document (shared across the
|
|
16
|
+
# whole node tree), so mutating the source string later is harmless.
|
|
17
|
+
#
|
|
18
|
+
# @example Pull two fields out of a big document
|
|
19
|
+
# doc = NOSJ.lazy(huge_json)
|
|
20
|
+
# doc["users"][3]["name"] # only this path is parsed
|
|
21
|
+
# doc.dig("meta", "count")
|
|
22
|
+
#
|
|
23
|
+
# @example Materialize a subtree
|
|
24
|
+
# doc["users"][3].value #=> {"name" => ..., ...}
|
|
25
|
+
class Lazy
|
|
26
|
+
include Enumerable
|
|
27
|
+
|
|
28
|
+
# Resolves one child. String and Symbol keys index objects; Integer
|
|
29
|
+
# indices index arrays (negative indices resolve to +nil+, as in
|
|
30
|
+
# {NOSJ.dig}). Containers come back as further Lazy nodes, scalars
|
|
31
|
+
# as plain Ruby values, misses as +nil+. Results are cached on the
|
|
32
|
+
# node.
|
|
33
|
+
#
|
|
34
|
+
# @param token [String, Symbol, Integer]
|
|
35
|
+
# @return [NOSJ::Lazy, Object, nil]
|
|
36
|
+
def [](token)
|
|
37
|
+
cache = (@children ||= {})
|
|
38
|
+
cache.fetch(token) { cache[token] = __get(token) }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Hash#dig-shaped access, fused into a single resolution: the whole
|
|
42
|
+
# path resolves in one walk of this node's bytes, no intermediate
|
|
43
|
+
# nodes. Semantics match {NOSJ.dig}: misses, negative indices, and
|
|
44
|
+
# steps into scalars resolve to +nil+. Not cached.
|
|
45
|
+
#
|
|
46
|
+
# @param path [Array<String, Symbol, Integer>]
|
|
47
|
+
# @return [NOSJ::Lazy, Object, nil]
|
|
48
|
+
def dig(first, *rest)
|
|
49
|
+
__dig([first, *rest])
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Resolves an RFC 6901 JSON Pointer within this node's subtree,
|
|
53
|
+
# with the same lazy/materialize behavior as {#[]}. Not cached.
|
|
54
|
+
#
|
|
55
|
+
# @param pointer [String] e.g. <tt>"/users/3/name"</tt>
|
|
56
|
+
# @return [NOSJ::Lazy, Object, nil]
|
|
57
|
+
def at_pointer(pointer)
|
|
58
|
+
__at_pointer(pointer)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Materializes this node's whole subtree as plain Ruby values,
|
|
62
|
+
# under the options given to {NOSJ.lazy} (+symbolize_names+,
|
|
63
|
+
# +freeze+, ...).
|
|
64
|
+
#
|
|
65
|
+
# @return [Hash, Array]
|
|
66
|
+
def value
|
|
67
|
+
__materialize
|
|
68
|
+
end
|
|
69
|
+
alias_method :materialize, :value
|
|
70
|
+
|
|
71
|
+
# @return [Hash] the materialized subtree
|
|
72
|
+
# @raise [TypeError] on an array node
|
|
73
|
+
def to_h
|
|
74
|
+
raise TypeError, "to_h on a JSON array" unless object?
|
|
75
|
+
|
|
76
|
+
__materialize
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @return [Array] the materialized subtree
|
|
80
|
+
# @raise [TypeError] on an object node
|
|
81
|
+
def to_a
|
|
82
|
+
raise TypeError, "to_a on a JSON object" unless array?
|
|
83
|
+
|
|
84
|
+
__materialize
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# @return [Boolean] whether this node is a JSON object
|
|
88
|
+
def object?
|
|
89
|
+
__kind == :object
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# @return [Boolean] whether this node is a JSON array
|
|
93
|
+
def array?
|
|
94
|
+
__kind == :array
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The object's keys (always Strings), read in one walk without
|
|
98
|
+
# materializing any values.
|
|
99
|
+
#
|
|
100
|
+
# @return [Array<String>]
|
|
101
|
+
# @raise [TypeError] on an array node
|
|
102
|
+
def keys
|
|
103
|
+
__keys
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Entry count (object pairs or array elements), read in one walk
|
|
107
|
+
# without materializing anything.
|
|
108
|
+
#
|
|
109
|
+
# @return [Integer]
|
|
110
|
+
def size
|
|
111
|
+
__size
|
|
112
|
+
end
|
|
113
|
+
alias_method :length, :size
|
|
114
|
+
alias_method :count, :size
|
|
115
|
+
|
|
116
|
+
# @return [Boolean]
|
|
117
|
+
def empty?
|
|
118
|
+
size.zero?
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Iterates direct children, resolved in one walk: object nodes
|
|
122
|
+
# yield +[key, child]+ pairs (like Hash), array nodes yield
|
|
123
|
+
# children. Containers arrive as Lazy nodes, scalars as values.
|
|
124
|
+
#
|
|
125
|
+
# @return [Enumerator] when no block is given
|
|
126
|
+
def each(&block)
|
|
127
|
+
return enum_for(:each) unless block
|
|
128
|
+
|
|
129
|
+
if object?
|
|
130
|
+
__children.each { |pair| yield pair }
|
|
131
|
+
else
|
|
132
|
+
__children.each { |child| yield child }
|
|
133
|
+
end
|
|
134
|
+
self
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @return [String] kind and span size, without touching content
|
|
138
|
+
def inspect
|
|
139
|
+
"#<NOSJ::Lazy #{__kind} (#{__byte_size} bytes)>"
|
|
140
|
+
end
|
|
141
|
+
alias_method :to_s, :inspect
|
|
142
|
+
|
|
143
|
+
private :__get, :__dig, :__at_pointer, :__materialize, :__kind,
|
|
144
|
+
:__byte_size, :__keys, :__size, :__children
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Wraps a JSON document for lazy, on-demand access: returns a
|
|
148
|
+
# {NOSJ::Lazy} node for a container root, or the materialized value
|
|
149
|
+
# for a scalar root. The document bytes are copied once; no parsing
|
|
150
|
+
# happens beyond locating the root value.
|
|
151
|
+
#
|
|
152
|
+
# @example
|
|
153
|
+
# doc = NOSJ.lazy('{"users":[{"name":"ada"},{"name":"grace"}]}')
|
|
154
|
+
# doc["users"][1]["name"] #=> "grace" — the rest is never parsed
|
|
155
|
+
#
|
|
156
|
+
# @param source [String] the JSON document (UTF-8 or US-ASCII)
|
|
157
|
+
# @param opts [Hash, nil] {NOSJ.parse} options applied whenever a
|
|
158
|
+
# value materializes: +symbolize_names+, +freeze+, +max_nesting+,
|
|
159
|
+
# +allow_nan+, +allow_trailing_comma+
|
|
160
|
+
# @return [NOSJ::Lazy, Object]
|
|
161
|
+
# @raise [RuntimeError] when the document root is malformed
|
|
162
|
+
def self.lazy(source, opts = nil)
|
|
163
|
+
lazy_native(source, opts)
|
|
164
|
+
end
|
|
165
|
+
end
|
data/lib/nosj/version.rb
CHANGED