nosj 0.4.0 → 0.5.0

data/ext/nosj/src/gen/walker.rs

@@ -3,6 +3,7 @@
 //! Compact and pretty modes are one const-generic body, so the compact
 //! hot path carries no formatting branches.
 
+use magnus::Error;
 use nosj::emit::{self, copy_short_raw, EscapeMode};
 use rb_sys::macros::{
     FIX2LONG, FIXNUM_P, FLONUM_P, RARRAY_CONST_PTR, RARRAY_LEN, RB_BUILTIN_TYPE, RHASH_SIZE,
@@ -15,8 +16,9 @@ use super::keys::GenKeyCache;
 use super::opts::GenConfig;
 use super::ruby::{
     is_json_fragment, is_special_const, protected_as_json, protected_encode_utf8,
-    protected_to_json, protected_to_s, rstring_bytes, str_coderange, str_enc_index, to_json_id,
-    utf8_encindexes, CR_7BIT, CR_VALID, QFALSE, QNIL, QTRUE,
+    protected_inspect, protected_to_json, protected_to_json_if_responds, protected_to_s,
+    rstring_bytes, str_coderange, str_enc_index, utf8_encindexes, CR_7BIT, CR_VALID, QFALSE, QNIL,
+    QTRUE,
 };
 
 /// Whether keys escaped under `mode` may be cached: the cached bytes
@@ -26,6 +28,84 @@ pub(super) fn mode_cacheable(mode: EscapeMode) -> bool {
     matches!(mode, EscapeMode::Standard | EscapeMode::HtmlSafe)
 }
 
+/// A hash key's kind: how it renders (see [`key_string`]) and json 3's
+/// duplicate-key rule.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum KeyKind {
+    String,
+    Symbol,
+    Other,
+}
+
+#[inline(always)]
+fn key_kind(k: VALUE) -> KeyKind {
+    if !is_special_const(k) {
+        match unsafe { RB_BUILTIN_TYPE(k) } {
+            ruby_value_type::RUBY_T_STRING => KeyKind::String,
+            ruby_value_type::RUBY_T_SYMBOL => KeyKind::Symbol,
+            _ => KeyKind::Other,
+        }
+    } else if STATIC_SYM_P(k) {
+        KeyKind::Symbol
+    } else {
+        KeyKind::Other
+    }
+}
+
+/// The String a key renders as, the gem's key coercion: Strings as they
+/// are, Symbols by name, anything else through `to_s`.
+fn key_string(k: VALUE, kind: KeyKind) -> Result<VALUE, Error> {
+    match kind {
+        KeyKind::String => Ok(k),
+        KeyKind::Symbol => Ok(unsafe { rb_sys::rb_sym2str(k) }),
+        KeyKind::Other => protected_to_s(k),
+    }
+}
+
+/// json 3's cheap trigger for its duplicate-key check, per hash: keys of
+/// one kind cannot render alike, so only a String or Symbol key in a
+/// hash whose first key was of another kind runs the full check (once).
+/// A key of the first key's kind costs one byte compare (the Option
+/// packs into one byte), a hash's first key one more; only actual
+/// mixing goes out of line.
+struct MixedKeys {
+    first: Option<KeyKind>,
+    /// Checked already, or `allow_duplicate_key`: nothing left to do.
+    done: bool,
+}
+
+impl MixedKeys {
+    #[inline(always)]
+    fn new(allow_duplicate_key: bool) -> Self {
+        MixedKeys {
+            first: None,
+            done: allow_duplicate_key,
+        }
+    }
+
+    #[inline(always)]
+    fn needs_check(&mut self, kind: KeyKind) -> bool {
+        match self.first {
+            Some(first) if first == kind => false,
+            None => {
+                self.first = Some(kind);
+                false
+            }
+            Some(_) => self.mixed(kind),
+        }
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn mixed(&mut self, kind: KeyKind) -> bool {
+        if self.done || kind == KeyKind::Other {
+            return false;
+        }
+        self.done = true;
+        true
+    }
+}
+
 pub(super) struct Gen<'a> {
     /// The pooled per-thread output buffer, borrowed for the call.
     pub(super) out: &'a mut Vec<u8>,
@@ -163,13 +243,19 @@ impl Gen<'_> {
     /// The pre-escaped bytes for `k` when the cache may serve it:
     /// frozen string key, cacheable escape mode (see
     /// [`Gen::emit_key_cached`]). An associated fn over the split-out
-    /// fields so callers keep `self.out` free.
+    /// fields so callers keep `self.out` free. `kind` is the caller's
+    /// [`key_kind`] of `k`, computed once per pair for both this and
+    /// the duplicate-key tracking.
     #[inline(always)]
-    fn cached_key_bytes<'k>(cfg: &GenConfig, keys: &'k GenKeyCache, k: VALUE) -> Option<&'k [u8]> {
+    fn cached_key_bytes<'k>(
+        cfg: &GenConfig,
+        keys: &'k GenKeyCache,
+        k: VALUE,
+        kind: KeyKind,
+    ) -> Option<&'k [u8]> {
         const FL_FREEZE: u64 = rb_sys::ruby_fl_type::RUBY_FL_FREEZE as u64;
-        if mode_cacheable(cfg.mode)
-            && !is_special_const(k)
-            && unsafe { RB_BUILTIN_TYPE(k) } == ruby_value_type::RUBY_T_STRING
+        if kind == KeyKind::String
+            && mode_cacheable(cfg.mode)
             && unsafe { (*(k as *const rb_sys::RBasic)).flags } & FL_FREEZE != 0
         {
             keys.get(k)
@@ -186,8 +272,8 @@ impl Gen<'_> {
     /// all through the cache-hit path here. Misses take the plain path
     /// below, which also populates the cache.
     #[inline(always)]
-    fn emit_pair_prefix_compact(&mut self, k: VALUE, comma: bool) -> Result<(), ()> {
-        if let Some(bytes) = Self::cached_key_bytes(self.cfg, self.keys, k) {
+    fn emit_pair_prefix_compact(&mut self, k: VALUE, kind: KeyKind, comma: bool) -> Result<(), ()> {
+        if let Some(bytes) = Self::cached_key_bytes(self.cfg, self.keys, k, kind) {
             let n = bytes.len();
             self.out.reserve(n + 2);
             // SAFETY: `n + 2` bytes reserved above; `bytes` borrows
@@ -210,7 +296,7 @@ impl Gen<'_> {
         if comma {
             self.out.push(b',');
         }
-        self.emit_key(k)?;
+        self.emit_key(k, kind)?;
         self.out.push(b':');
         Ok(())
     }
@@ -220,8 +306,14 @@ impl Gen<'_> {
     /// separator, cached key, colon, and digits in one reservation and
     /// one raw cursor.
     #[inline(always)]
-    fn emit_pair_int_compact(&mut self, k: VALUE, comma: bool, value: i64) -> Result<(), ()> {
-        if let Some(bytes) = Self::cached_key_bytes(self.cfg, self.keys, k) {
+    fn emit_pair_int_compact(
+        &mut self,
+        k: VALUE,
+        kind: KeyKind,
+        comma: bool,
+        value: i64,
+    ) -> Result<(), ()> {
+        if let Some(bytes) = Self::cached_key_bytes(self.cfg, self.keys, k, kind) {
             let n = bytes.len();
             self.out.reserve(n + 2 + emit::I64_MAX_LEN);
             // SAFETY: the reservation covers separator + key + colon +
@@ -244,34 +336,26 @@ impl Gen<'_> {
             }
             return Ok(());
         }
-        self.emit_pair_prefix_compact(k, comma)?;
+        self.emit_pair_prefix_compact(k, kind, comma)?;
         emit::write_i64(&mut *self.out, value);
         Ok(())
     }
 
-    /// Object/hash key: String and Symbol direct, anything else via to_s
-    /// (the gem's key coercion).
-    fn emit_key(&mut self, k: VALUE) -> Result<(), ()> {
-        if !is_special_const(k) {
-            match unsafe { RB_BUILTIN_TYPE(k) } {
-                ruby_value_type::RUBY_T_STRING => return self.emit_key_cached(k),
-                ruby_value_type::RUBY_T_SYMBOL => {
-                    let s = unsafe { rb_sys::rb_sym2str(k) };
-                    return self.emit_rstring_quoted(s);
-                }
-                _ => {}
-            }
-        } else if STATIC_SYM_P(k) {
-            let s = unsafe { rb_sys::rb_sym2str(k) };
-            return self.emit_rstring_quoted(s);
-        }
-        match protected_to_s(k) {
-            Ok(s) => self.emit_rstring_quoted(s),
-            Err(exc) => {
-                self.fail = Some(GenFail::Reraise(exc));
-                Err(())
-            }
-        }
+    /// Object/hash key of `kind` (the caller's [`key_kind`]), rendered
+    /// by [`key_string`]; String keys go through the pre-escaped cache.
+    fn emit_key(&mut self, k: VALUE, kind: KeyKind) -> Result<(), ()> {
+        if kind == KeyKind::String {
+            return self.emit_key_cached(k);
+        }
+        let s = self.reraise(key_string(k, kind))?;
+        self.emit_rstring_quoted(s)
+    }
+
+    /// A protected Ruby call's outcome inside the walk: a raised
+    /// exception becomes this walk's failure, re-raised unchanged once
+    /// the walk has unwound.
+    fn reraise<T>(&mut self, r: Result<T, Error>) -> Result<T, ()> {
+        r.map_err(|exc| self.fail = Some(GenFail::Reraise(exc)))
     }
 
     /// Non-native type: strict raises (except `JSON::Fragment`, which
@@ -285,7 +369,7 @@ impl Gen<'_> {
             return self.emit_rails_fallback::<PRETTY>(raw, depth);
         }
         if self.cfg.strict {
-            if is_json_fragment(raw) {
+            if self.reraise(is_json_fragment(raw))? {
                 return self.splice_to_json(raw);
             }
             let name = unsafe {
@@ -296,53 +380,76 @@ impl Gen<'_> {
             self.fail = Some(GenFail::Generator(format!("{name} not allowed in JSON")));
             return Err(());
         }
-        if unsafe { rb_sys::rb_respond_to(raw, to_json_id()) } != 0 {
-            match protected_to_json(raw) {
-                Ok(json) => {
-                    if !is_special_const(json)
-                        && unsafe { RB_BUILTIN_TYPE(json) } == ruby_value_type::RUBY_T_STRING
-                    {
-                        self.append_rstring_raw(json);
-                        return Ok(());
-                    }
-                }
-                Err(exc) => {
-                    self.fail = Some(GenFail::Reraise(exc));
-                    return Err(());
-                }
+        if let Some(json) = self.reraise(protected_to_json_if_responds(raw))? {
+            if !is_special_const(json)
+                && unsafe { RB_BUILTIN_TYPE(json) } == ruby_value_type::RUBY_T_STRING
+            {
+                self.append_rstring_raw(json);
+                return Ok(());
             }
         }
-        match protected_to_s(raw) {
-            Ok(s) => self.emit_rstring_quoted(s),
-            Err(exc) => {
-                self.fail = Some(GenFail::Reraise(exc));
-                Err(())
-            }
+        let s = self.reraise(protected_to_s(raw))?;
+        self.emit_rstring_quoted(s)
+    }
+
+    /// json 3's full duplicate check, run once for a hash whose keys
+    /// mixed kinds: every key through `to_s`, first repeat raises the
+    /// gem's exact message. Cold: only mixed-kind hashes reach it.
+    #[cold]
+    fn check_duplicate_keys(&mut self, hash: VALUE) -> Result<(), ()> {
+        use super::hash_iter::{foreach_raw, Step};
+        let mut seen = std::collections::HashSet::new();
+        let mut outcome: Result<Option<VALUE>, Error> = Ok(None);
+        // SAFETY: only reached for T_HASH values.
+        unsafe {
+            foreach_raw(hash, |k, _| {
+                let key_str = match key_string(k, key_kind(k)) {
+                    Ok(s) => s,
+                    Err(exc) => {
+                        outcome = Err(exc);
+                        return Step::Stop;
+                    }
+                };
+                if seen.insert(rstring_bytes(key_str).to_vec()) {
+                    Step::Continue
+                } else {
+                    outcome = Ok(Some(key_str));
+                    Step::Stop
+                }
+            });
         }
+        let Some(key_str) = self.reraise(outcome)? else {
+            return Ok(());
+        };
+        let key_inspect = self.reraise(protected_inspect(key_str))?;
+        let hash_inspect = self.reraise(protected_inspect(hash))?;
+        // SAFETY: rb_inspect returns Strings; both are copied out here.
+        let (key_inspect, hash_inspect) = unsafe {
+            (
+                String::from_utf8_lossy(rstring_bytes(key_inspect)).into_owned(),
+                String::from_utf8_lossy(rstring_bytes(hash_inspect)).into_owned(),
+            )
+        };
+        self.fail = Some(GenFail::Generator(format!(
+            "detected duplicate key {key_inspect} in {hash_inspect}"
+        )));
+        Err(())
     }
 
     /// Splice `raw`'s `to_json` result verbatim: the JSON::Fragment
     /// path (pre-rendered JSON, trusted like the gem trusts it).
     fn splice_to_json(&mut self, raw: VALUE) -> Result<(), ()> {
-        match protected_to_json(raw) {
-            Ok(json)
-                if !is_special_const(json)
-                    && unsafe { RB_BUILTIN_TYPE(json) } == ruby_value_type::RUBY_T_STRING =>
-            {
-                self.append_rstring_raw(json);
-                Ok(())
-            }
-            Ok(_) => {
-                self.fail = Some(GenFail::Generator(
-                    "JSON::Fragment#to_json did not return a String".to_string(),
-                ));
-                Err(())
-            }
-            Err(exc) => {
-                self.fail = Some(GenFail::Reraise(exc));
-                Err(())
-            }
+        let json = self.reraise(protected_to_json(raw))?;
+        if is_special_const(json)
+            || unsafe { RB_BUILTIN_TYPE(json) } != ruby_value_type::RUBY_T_STRING
+        {
+            self.fail = Some(GenFail::Generator(
+                "JSON::Fragment#to_json did not return a String".to_string(),
+            ));
+            return Err(());
         }
+        self.append_rstring_raw(json);
+        Ok(())
     }
 
     /// Rails-mode fallback, mirroring JSONGemEncoder#jsonify:
@@ -356,27 +463,22 @@ impl Gen<'_> {
         raw: VALUE,
         depth: usize,
     ) -> Result<(), ()> {
-        if is_json_fragment(raw) {
+        if self.reraise(is_json_fragment(raw))? {
             return self.splice_to_json(raw);
         }
-        match protected_as_json(raw) {
-            Ok(json) if json == raw => {
-                let name = unsafe {
-                    std::ffi::CStr::from_ptr(rb_sys::rb_obj_classname(raw))
-                        .to_string_lossy()
-                        .into_owned()
-                };
-                self.fail = Some(GenFail::Generator(format!(
-                    "{name}#as_json returned the receiver"
-                )));
-                Err(())
-            }
-            Ok(json) => self.emit_value::<PRETTY>(json, depth),
-            Err(exc) => {
-                self.fail = Some(GenFail::Reraise(exc));
-                Err(())
-            }
+        let json = self.reraise(protected_as_json(raw))?;
+        if json == raw {
+            let name = unsafe {
+                std::ffi::CStr::from_ptr(rb_sys::rb_obj_classname(raw))
+                    .to_string_lossy()
+                    .into_owned()
+            };
+            self.fail = Some(GenFail::Generator(format!(
+                "{name}#as_json returned the receiver"
+            )));
+            return Err(());
         }
+        self.emit_value::<PRETTY>(json, depth)
     }
 
     fn nesting_check(&mut self, inner: usize) -> Result<(), ()> {
@@ -390,14 +492,23 @@ impl Gen<'_> {
     fn emit_array<const PRETTY: bool>(&mut self, ary: VALUE, depth: usize) -> Result<(), ()> {
         let inner = depth + 1;
         self.nesting_check(inner)?;
-        let len = unsafe { RARRAY_LEN(ary) } as usize;
         self.out.push(b'[');
-        if len == 0 {
+        if unsafe { RARRAY_LEN(ary) } == 0 {
             self.out.push(b']');
             return Ok(());
         }
         let mut i = 0usize;
-        while i < len {
+        loop {
+            // Length and pointer are re-read every element, like the
+            // json gem: a user callback inside the recursion (to_json,
+            // to_s, as_json) may shrink the array (slots past the live
+            // length hold freed or reused VALUEs; growth is emitted),
+            // and any allocation may compact it elsewhere.
+            let len = unsafe { RARRAY_LEN(ary) } as usize;
+            if i >= len {
+                break;
+            }
+            let elem = unsafe { *RARRAY_CONST_PTR(ary).add(i) };
             if i > 0 {
                 self.out.push(b',');
             }
@@ -405,9 +516,6 @@ impl Gen<'_> {
                 self.out.extend_from_slice(&self.cfg.array_nl);
                 self.push_indent(inner);
             }
-            // Re-read the pointer every element: an allocation inside the
-            // recursion may trigger GC compaction and move the array.
-            let elem = unsafe { *RARRAY_CONST_PTR(ary).add(i) };
             // Numeric runs (compact mode) emit through a raw local
             // cursor under one chunked reservation: per-element Vec
             // operations round-trip length and pointer through memory
@@ -523,18 +631,23 @@ impl Gen<'_> {
             self.out.push(b'}');
             return Ok(());
         }
+        let mut mixed = MixedKeys::new(self.cfg.allow_duplicate_key);
         // SAFETY: emit_object is only reached for T_HASH values.
         if PRETTY {
             let mut first = true;
             unsafe {
                 foreach_raw(hash, |k, v| {
+                    let kind = key_kind(k);
+                    if mixed.needs_check(kind) && self.check_duplicate_keys(hash).is_err() {
+                        return Step::Stop;
+                    }
                     if !first {
                         self.out.push(b',');
                     }
                     first = false;
                     self.out.extend_from_slice(&self.cfg.object_nl);
                     self.push_indent(inner);
-                    if self.emit_key(k).is_err() {
+                    if self.emit_key(k, kind).is_err() {
                         return Step::Stop;
                     }
                     self.out.extend_from_slice(&self.cfg.space_before);
@@ -549,20 +662,24 @@ impl Gen<'_> {
         } else {
             unsafe {
                 foreach_raw(hash, |k, v| {
+                    let kind = key_kind(k);
+                    if mixed.needs_check(kind) && self.check_duplicate_keys(hash).is_err() {
+                        return Step::Stop;
+                    }
                     // `out` always ends with '{' (just pushed) or the
                     // previous pair.
                     let comma = *self.out.last().unwrap_unchecked() != b'{';
                     // Int values fuse with their key into one write.
                     if FIXNUM_P(v) {
                         if self
-                            .emit_pair_int_compact(k, comma, FIX2LONG(v) as i64)
+                            .emit_pair_int_compact(k, kind, comma, FIX2LONG(v) as i64)
                             .is_err()
                         {
                             return Step::Stop;
                         }
                         return Step::Continue;
                     }
-                    if self.emit_pair_prefix_compact(k, comma).is_err() {
+                    if self.emit_pair_prefix_compact(k, kind, comma).is_err() {
                         return Step::Stop;
                     }
                     // Value fast arms, mirroring emit_value's: one

data/ext/nosj/src/lazy.rs

@@ -20,20 +20,20 @@ use magnus::{DataTypeFunctions, Error, RArray, RString, Ruby, TypedData, Value};
 use crate::errors::parser_error_at;
 use crate::parse::{materialize_at, parse_native_opts, span_of, utf8_input, ParseNativeOpts};
 use crate::pointer::{path_to_pointer, push_escaped_token};
-use crate::state::PULL_STATE;
+use crate::state::with_pull_state;
 
 /// The document bytes behind a node tree. A frozen Ruby source is
-/// borrowed zero-copy: freezing rules out mutation, and every node
-/// GC-marks the string with `rb_gc_mark` semantics, which both keeps it
-/// alive and pins it against compaction, so the captured pointer stays
-/// valid for as long as any node exists. Anything else is copied once.
+/// borrowed zero-copy: freezing rules out any change to its CONTENT,
+/// and every node GC-marks the string with `rb_gc_mark` semantics
+/// (alive, and pinned against compaction). Freezing does NOT pin the
+/// buffer, though: deduplicating a frozen String subclass or a string
+/// carrying ivars (`-str`) swaps in an identical shared buffer and
+/// frees the old one. So the bytes are re-read from the string on every
+/// access; same content means every span stays valid. Anything else is
+/// copied once.
 pub(crate) enum DocBytes {
     Owned(Vec<u8>),
-    Frozen {
-        source: rb_sys::VALUE,
-        ptr: *const u8,
-        len: usize,
-    },
+    Frozen(RString),
     /// A read-only file mapping (`NOSJ.load_lazy_file`): pages never
     /// touched are never read off disk. Concurrent modification of the
     /// mapped file by another process is documented as unsupported
@@ -48,23 +48,24 @@ struct DocInner {
     opts: ParseNativeOpts,
 }
 
-// SAFETY: the bytes are immutable for the document's whole life (an
+// SAFETY: the content is immutable for the document's whole life (an
 // owned Vec, or a frozen Ruby string pinned and kept alive by every
-// node's GC mark), so cross-thread reads are plain shared reads; the
-// raw VALUE is only dereferenced by the GC mark, which runs at
-// safepoints (the ShadowHandle contract in state.rs). There is no
-// interior mutability anywhere in the type.
+// node's GC mark), so cross-thread reads are plain shared reads of it.
+// There is no interior mutability anywhere in the type.
 unsafe impl Send for DocInner {}
 unsafe impl Sync for DocInner {}
 
 impl DocInner {
+    /// The document bytes. For a frozen source the slice is valid only
+    /// until the next Ruby call (which could swap the string's buffer;
+    /// see DocBytes): callers finish with it, or call this again,
+    /// before running Ruby code.
     fn bytes(&self) -> &[u8] {
         match &self.bytes {
             DocBytes::Owned(v) => v,
-            // SAFETY: the source string is frozen (no mutation, no
-            // buffer reallocation) and pinned+kept alive by every
-            // node's GC mark; see DocBytes.
-            DocBytes::Frozen { ptr, len, .. } => unsafe { std::slice::from_raw_parts(*ptr, *len) },
+            // SAFETY: a live string (kept alive and pinned by every
+            // node's GC mark), read fresh on each call; see DocBytes.
+            DocBytes::Frozen(source) => unsafe { source.as_slice() },
             DocBytes::Mmap(m) => m,
         }
     }
@@ -77,8 +78,16 @@ const KIND_ARRAY: u8 = b'[';
 /// come from the crate's resolver (token edges within the doc bytes),
 /// and never cross the Ruby boundary, so they cannot be forged from
 /// Ruby.
+///
+/// `wb_protected`: a node's only Ruby reference (a frozen source, in
+/// the shared `DocInner`) is set before the root node is wrapped and
+/// never written again, so there is no write to put a barrier on, and
+/// nodes promote to the old generation instead of being rescanned by
+/// every minor GC (children cache Ruby-side, in barrier-protected
+/// ivars). `free_immediately`: dropping a node calls no Ruby API (it
+/// releases a Vec, an mmap, or nothing).
 #[derive(TypedData)]
-#[magnus(class = "NOSJ::Lazy", mark)]
+#[magnus(class = "NOSJ::Lazy", free_immediately, mark, wb_protected)]
 pub struct LazyNode {
     doc: Arc<DocInner>,
     start: usize,
@@ -88,11 +97,10 @@ pub struct LazyNode {
 
 impl DataTypeFunctions for LazyNode {
     fn mark(&self, marker: &magnus::gc::Marker) {
-        if let DocBytes::Frozen { source, .. } = self.doc.bytes {
-            use magnus::rb_sys::FromRawValue;
-            // SAFETY: the VALUE was a live, frozen string at node
-            // creation and this mark is what keeps it that way.
-            marker.mark(unsafe { Value::from_raw(source) });
+        // The source was a live, frozen string at node creation, and
+        // this mark is what keeps it that way.
+        if let DocBytes::Frozen(source) = self.doc.bytes {
+            marker.mark(source);
         }
     }
 }
@@ -101,6 +109,13 @@ impl LazyNode {
     fn span(&self) -> &[u8] {
         &self.doc.bytes()[self.start..self.end]
     }
+
+    /// A Reader over this node's span, walking the grammar the document
+    /// was opened with (trailing commas, NaN keywords).
+    fn reader<'a, 'b>(&'a self, bufs: &'b mut nosj::Buffers) -> nosj::Reader<'a, 'b> {
+        // SAFETY: spans are valid UTF-8 (see resolve_in_span).
+        unsafe { nosj::Reader::from_utf8_unchecked_with(self.span(), bufs, self.doc.opts.popts) }
+    }
 }
 
 /// Wrap a resolved raw-value slice: containers become new lazy nodes,
@@ -125,13 +140,19 @@ fn resolved_to_value(ruby: &Ruby, doc: &Arc<DocInner>, sub: &[u8]) -> Result<Val
 /// Resolve `pointer` within `node`'s span. Shared by `__get` and
 /// `__at_pointer`; both misses and negative-index paths return nil.
 fn resolve_in_span(ruby: &Ruby, node: &LazyNode, pointer: &str) -> Result<Value, Error> {
-    // Resolve first (one PULL_STATE borrow, slice borrows the doc, not
-    // the buffers), then materialize (which re-borrows internally).
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    // Resolve, then materialize, as two separate uses of the parse
+    // state; the resolved slice borrows the doc, not the state.
+    let resolved = with_pull_state(|state| {
         // SAFETY: doc bytes were coderange-gated at NOSJ.lazy creation,
         // and spans lie on token edges, so the span is valid UTF-8.
-        unsafe { nosj::pointer_utf8_unchecked(node.span(), pointer, &mut state.bufs) }
+        unsafe {
+            nosj::pointer_utf8_unchecked_with(
+                node.span(),
+                pointer,
+                &mut state.bufs,
+                node.doc.opts.popts,
+            )
+        }
     });
     match resolved {
         Ok(None) => Ok(ruby.qnil().as_value()),
@@ -165,12 +186,7 @@ pub fn lazy_native(
     // on the caller's machine stack, so it stays pinned through this
     // call, and the node's mark takes over from the first GC on.
     let bytes = if data.as_value().is_frozen() {
-        use magnus::rb_sys::AsRawValue;
-        DocBytes::Frozen {
-            source: data.as_raw(),
-            ptr: input.as_ptr(),
-            len: input.len(),
-        }
+        DocBytes::Frozen(data)
     } else {
         DocBytes::Owned(input.to_vec())
     };
@@ -288,10 +304,8 @@ pub fn lazy_keys(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
         ));
     }
     let out = ruby.ary_new();
-    PULL_STATE.with(|cell| -> Result<(), Error> {
-        let mut state = cell.borrow_mut();
-        // SAFETY: spans are valid UTF-8 (see resolve_in_span).
-        let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
+    with_pull_state(|state| -> Result<(), Error> {
+        let mut r = rb_self.reader(&mut state.bufs);
         r.next_node().map_err(|e| reader_err(ruby, &rb_self, e))?;
         let mut has = match r
             .object_first_key()
@@ -324,10 +338,8 @@ pub fn lazy_keys(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
 /// `__size`: entry count (object pairs or array elements), one walk,
 /// nothing materialized.
 pub fn lazy_size(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<usize, Error> {
-    PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
-        // SAFETY: spans are valid UTF-8 (see resolve_in_span).
-        let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
+    with_pull_state(|state| {
+        let mut r = rb_self.reader(&mut state.bufs);
         r.next_node().map_err(|e| reader_err(ruby, &rb_self, e))?;
         let mut n = 0usize;
         if rb_self.kind == KIND_OBJECT {
@@ -366,14 +378,12 @@ struct ChildDesc {
 
 /// `__children`: every direct child in ONE walk. Objects yield
 /// `[key, child]` pairs, arrays yield children; containers wrap lazily,
-/// scalars materialize. Two phases so the Reader's buffer borrow ends
-/// before materialization re-borrows the thread state.
+/// scalars materialize. Two phases so the walk's use of the parse state
+/// ends before materialization needs it (nested, it would start fresh).
 pub fn lazy_children(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
     let base = rb_self.doc.bytes().as_ptr() as usize;
-    let descs: Result<Vec<ChildDesc>, nosj::ParseError> = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
-        // SAFETY: spans are valid UTF-8 (see resolve_in_span).
-        let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
+    let descs: Result<Vec<ChildDesc>, nosj::ParseError> = with_pull_state(|state| {
+        let mut r = rb_self.reader(&mut state.bufs);
         r.next_node()?;
         let mut out = Vec::new();
         if rb_self.kind == KIND_OBJECT {