nosj 0.4.0 → 0.4.1

data/ext/nosj/src/pointer.rs

@@ -7,7 +7,7 @@ use magnus::{Error, RString, Ruby, Value};
 
 use crate::errors::parser_error_at;
 use crate::parse::{materialize_at, parse_native_opts, span_of, utf8_input, ParseNativeOpts};
-use crate::state::PULL_STATE;
+use crate::state::with_pull_state;
 
 /// Resolve one JSON Pointer against `data`, materializing the matched
 /// subtree; `nil` when the pointer misses.
@@ -19,12 +19,11 @@ fn at_pointer_impl(
 ) -> Result<Value, Error> {
     use magnus::value::ReprValue;
     let input = utf8_input(ruby, &data)?;
-    // Resolve first (one PULL_STATE borrow), then materialize (a fresh
-    // borrow); the resolved slice borrows `input`, not the buffers.
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    // Resolve, then materialize, as two separate uses of the parse
+    // state; the resolved slice borrows `input`, not the state.
+    let resolved = with_pull_state(|state| {
         // Safety: coderange verified above.
-        unsafe { nosj::pointer_utf8_unchecked(input, pointer, &mut state.bufs) }
+        unsafe { nosj::pointer_utf8_unchecked_with(input, pointer, &mut state.bufs, o.popts) }
     });
     match resolved {
         Ok(None) => Ok(ruby.qnil().as_value()),
@@ -125,12 +124,11 @@ fn at_pointers_impl(
     let input = utf8_input(ruby, &data)?;
     let live: Vec<&str> = pointers.iter().flatten().map(String::as_str).collect();
 
-    // Resolve first (one PULL_STATE borrow); the resolved slices borrow
-    // `input`, not the buffers, so materializing can re-borrow freely.
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    // Resolve first (one use of the parse state); the resolved slices
+    // borrow `input`, not the state, so each materializes separately.
+    let resolved = with_pull_state(|state| {
         // Safety: coderange verified by utf8_input.
-        unsafe { nosj::pointers_utf8_unchecked(input, &live, &mut state.bufs) }
+        unsafe { nosj::pointers_utf8_unchecked_with(input, &live, &mut state.bufs, o.popts) }
     });
     let mut hits = match resolved {
         Ok(hits) => hits.into_iter(),

data/ext/nosj/src/reformat.rs

@@ -13,7 +13,7 @@
 //! canonical spelling (`1.50` becomes `1.5`), and string escapes are
 //! normalized by the emission kernels.
 
-use std::cell::RefCell;
+use std::cell::Cell;
 
 use magnus::value::ReprValue;
 use magnus::{Error, RString, Ruby, Value};
@@ -23,16 +23,15 @@ use nosj::{FloatFormat, WriteOptions, Writer};
 use crate::errors::{nesting_error, nosj_exception, parser_error, parser_error_at};
 use crate::files::with_mapped_file;
 use crate::gen::opts::{parse_gen_opts, GenConfig, DEFAULT_CONFIG};
-use crate::parse::{parse_native_opts, utf8_input};
+use crate::parse::{parse_native_opts, utf8_input, ParseNativeOpts};
 use crate::patch::finish_string;
 use crate::sink::SinkAbort;
-use crate::state::PULL_STATE;
+use crate::state::{with_pull_state, with_taken};
 
 thread_local! {
-    /// Pooled output buffer: capacity survives across calls. The pipe
-    /// never calls back into Ruby, so the borrow spans the whole drive
-    /// without any reentrancy concern.
-    static PIPE_BUF: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
+    /// Pooled output buffer: capacity survives across calls (see
+    /// `state::with_taken`).
+    static PIPE_BUF: Cell<Vec<u8>> = const { Cell::new(Vec::new()) };
 }
 
 /// The event-to-Writer pipe. Structure events forward to the Writer's
@@ -230,37 +229,50 @@ fn write_options(cfg: &GenConfig) -> WriteOptions {
     w
 }
 
+/// Decoded reformat options: parse acceptance plus generate formatting.
+/// Decoded before the source is borrowed, since decoding can run Ruby
+/// (an option value's `to_int`; see `parse::utf8_input`).
+struct ReformatOpts {
+    parse: ParseNativeOpts,
+    generate: Option<GenConfig>,
+}
+
+impl ReformatOpts {
+    fn decode(ruby: &Ruby, opts: Value) -> Result<Self, Error> {
+        Ok(Self {
+            parse: parse_native_opts(ruby, opts)?,
+            generate: if opts.is_nil() {
+                None
+            } else {
+                Some(parse_gen_opts(ruby, opts)?.0)
+            },
+        })
+    }
+}
+
 /// Run the pipe over already-UTF-8-vouched bytes.
-fn reformat_over(ruby: &Ruby, input: &[u8], opts: Value) -> Result<RString, Error> {
-    let po = parse_native_opts(ruby, opts)?;
-    let built;
-    let gcfg: &GenConfig = if opts.is_nil() {
-        &DEFAULT_CONFIG
-    } else {
-        built = parse_gen_opts(ruby, opts)?.0;
-        &built
-    };
+fn reformat_over(ruby: &Ruby, input: &[u8], opts: &ReformatOpts) -> Result<RString, Error> {
+    let po = &opts.parse;
+    let gcfg = opts.generate.as_ref().unwrap_or(&DEFAULT_CONFIG);
     let wopts = write_options(gcfg);
 
-    PIPE_BUF.with(|cell| {
-        let mut buf = cell.borrow_mut();
+    with_taken(&PIPE_BUF, |buf| {
         buf.clear();
         // The output is at least input-sized for minify-shaped runs.
         buf.reserve(input.len());
         let mut sink = PipeSink {
-            w: Writer::new(&mut buf, &wopts),
+            w: Writer::new(buf, &wopts),
             depth: 0,
             max_nesting: po.max_nesting,
             mode: gcfg.mode,
             allow_nan: gcfg.allow_nan,
         };
-        let result = PULL_STATE.with(|state_cell| {
-            let mut state = state_cell.borrow_mut();
+        let result = with_pull_state(|state| {
             // Safety: callers verified UTF-8 (coderange or full scan).
             unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, po.popts) }
         });
         match result {
-            Ok(()) => finish_string(&buf),
+            Ok(()) => finish_string(buf),
             Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
                 ruby,
                 format!(
@@ -296,8 +308,9 @@ pub fn reformat_native(
     data: RString,
     opts: Value,
 ) -> Result<RString, Error> {
+    let opts = ReformatOpts::decode(ruby, opts)?;
     let input = utf8_input(ruby, &data)?;
-    reformat_over(ruby, input, opts)
+    reformat_over(ruby, input, &opts)
 }
 
 /// `NOSJ.reformat_file_native(path, opts)`: the pipe over a read-only
@@ -309,12 +322,13 @@ pub fn reformat_file_native(
     opts: Value,
 ) -> Result<RString, Error> {
     let p = path.to_string()?;
+    let opts = ReformatOpts::decode(ruby, opts)?;
     // Mapping a zero-length file fails with EINVAL on Linux; route an
     // empty file to the parser's own "unexpected end of input" so the
     // error class is deterministic across platforms. Metadata failures
     // fall through for the mapper's Errno.
     if std::fs::metadata(&p).is_ok_and(|m| m.len() == 0) {
-        return reformat_over(ruby, &[], opts);
+        return reformat_over(ruby, &[], &opts);
     }
-    with_mapped_file(ruby, &p, |map| reformat_over(ruby, &map, opts))
+    with_mapped_file(ruby, &p, |map| reformat_over(ruby, &map, &opts))
 }

data/ext/nosj/src/sink.rs

@@ -35,24 +35,21 @@ pub(crate) enum SinkAbort {
     NonFiniteFloat(&'static str),
 }
 
-/// `RB_INT2FIX` ported from Ruby's public inline headers: fixnums are
-/// `(i << 1) + 1` for `|i| <= LONG_MAX / 2`, and the range is defined by
-/// the C `long`, which is 32-bit on Windows (LLP64): fixnums there hold
-/// only 31 bits, and tagging anything wider crashes Ruby with
-/// "Unnormalized Fixnum value". C extensions get the check inlined by the
-/// header; going through the extern `rb_ll2inum` costs an FFI call per
-/// integer. Non-fixable values still take the call.
-// The widening is an identity on LP64 hosts (clippy flags it there) but
-// required on Windows, where c_long is 32-bit.
-#[allow(clippy::unnecessary_cast)]
+/// Integer VALUE for `i` via rb-sys's inline `LONG2NUM` (the header
+/// macro: a Fixnum tagged inline when it fits, else a Bignum), which
+/// saves the FFI call per integer that `rb_ll2inum` costs. The fixable
+/// range is defined by the C `long`, which is 32-bit on Windows (LLP64):
+/// fixnums there hold only 31 bits, and tagging anything wider crashes
+/// Ruby with "Unnormalized Fixnum value"; values beyond `long` take
+/// `rb_ll2inum`.
+// The conversion is an identity on LP64 hosts (clippy flags it there)
+// but narrows on Windows, where c_long is 32-bit.
+#[allow(clippy::useless_conversion, clippy::unnecessary_fallible_conversions)]
 #[inline(always)]
 fn int_to_raw(i: i64) -> rb_sys::VALUE {
-    const FIXABLE_MIN: i64 = (std::os::raw::c_long::MIN / 2) as i64;
-    const FIXABLE_MAX: i64 = (std::os::raw::c_long::MAX / 2) as i64;
-    if (FIXABLE_MIN..=FIXABLE_MAX).contains(&i) {
-        ((i as u64) << 1).wrapping_add(1) as rb_sys::VALUE
-    } else {
-        unsafe { rb_sys::rb_ll2inum(i) }
+    match std::os::raw::c_long::try_from(i) {
+        Ok(l) => rb_sys::macros::LONG2NUM(l),
+        Err(_) => unsafe { rb_sys::rb_ll2inum(i) },
     }
 }
 

data/ext/nosj/src/state.rs

@@ -10,18 +10,36 @@ use ahash::AHashMap;
 use magnus::typed_data::Obj;
 use magnus::{DataTypeFunctions, TypedData};
 use nosj::Buffers;
-use std::cell::RefCell;
+use std::cell::Cell;
+use std::thread::LocalKey;
 
-/// Everything a parse touches, allocated once per thread and reused:
-/// nosj's scratch buffers, the interned-key caches, and the GC-marked
-/// stacks.
+/// Run `f` on a pooled thread-local value, taken OUT of its cell for the
+/// call and stored back afterwards. Pooled state is never borrowed
+/// across a call that can reach Ruby: any allocation can raise
+/// NoMemoryError, whose longjmp skips these frames, and a RefCell borrow
+/// held across it would stay borrowed for good (under panic=abort, the
+/// next borrow kills the process). A value lost that way is only leaked;
+/// the next call, like a nested one finding the cell empty, starts from
+/// `T::default()`. The outermost call's value is the one stored back.
+pub(crate) fn with_taken<T: Default, R>(
+    key: &'static LocalKey<Cell<T>>,
+    f: impl FnOnce(&mut T) -> R,
+) -> R {
+    let mut value = key.take();
+    let result = f(&mut value);
+    key.set(value);
+    result
+}
+
+/// Everything a parse touches, reused across calls: nosj's scratch
+/// buffers, the interned-key caches, and the GC-marked stacks.
 pub(crate) struct PullState {
     pub(crate) bufs: Buffers,
     pub(crate) keys: AHashMap<Box<str>, rb_sys::VALUE>,
     /// Separate cache for symbolize_names mode: symbol and string VALUEs
     /// must never share a map.
     pub(crate) sym_keys: AHashMap<Box<str>, rb_sys::VALUE>,
-    /// Leaked once per thread; kept alive + GC-marked via the wrapped
+    /// Leaked once per state; kept alive + GC-marked via the wrapped
     /// handle.
     pub(crate) vstack: Option<&'static mut VStackShadow>,
     /// Marked shadow holding the cached key VALUEs; keys are kept alive by
@@ -29,14 +47,36 @@ pub(crate) struct PullState {
     pub(crate) key_shadow: Option<&'static mut VStackShadow>,
 }
 
+impl PullState {
+    #[cold]
+    fn fresh() -> Box<Self> {
+        Box::new(PullState {
+            bufs: Buffers::new(),
+            keys: AHashMap::with_capacity(256),
+            sym_keys: AHashMap::new(),
+            vstack: None,
+            key_shadow: None,
+        })
+    }
+}
+
 thread_local! {
-    pub(crate) static PULL_STATE: RefCell<PullState> = RefCell::new(PullState {
-        bufs: Buffers::new(),
-        keys: AHashMap::with_capacity(256),
-        sym_keys: AHashMap::new(),
-        vstack: None,
-        key_shadow: None,
-    });
+    static PULL_STATE: Cell<Option<Box<PullState>>> = const { Cell::new(None) };
+}
+
+/// Run `f` on this thread's parse state, taken out like [`with_taken`]
+/// (a state lost to a longjmp leaks its shadows' last VALUEs; a nested
+/// call would start a fresh one). Unlike the generate scratch, parse
+/// bodies never run Ruby code, so the thread cannot hop native threads
+/// mid-call and one thread-local access serves both the take and the
+/// put-back: measured ~5ns per call on tiny documents against two.
+pub(crate) fn with_pull_state<R>(f: impl FnOnce(&mut PullState) -> R) -> R {
+    PULL_STATE.with(|cell| {
+        let mut state = cell.take().unwrap_or_else(PullState::fresh);
+        let result = f(&mut state);
+        cell.set(Some(state));
+        result
+    })
 }
 
 /// GC-marked holder for pending VALUEs.
@@ -48,6 +88,12 @@ pub(crate) struct VStackShadow {
 /// through [`DataTypeFunctions::mark`] (its trampoline, not ours),
 /// pinning every pending VALUE with `rb_gc_mark` semantics. The class
 /// is defined (and made a private constant) at init.
+///
+/// Deliberately NOT `wb_protected`: parses push VALUEs into the shadow
+/// with plain stores, no write barriers, so an old protected handle
+/// would let the GC miss young values it holds (a use-after-free).
+/// Staying write-barrier-unprotected makes every GC rescan the handle,
+/// which costs nothing measurable: there are one to three per thread.
 #[derive(TypedData)]
 #[magnus(class = "NOSJ::ValueStackShadow", mark)]
 pub(crate) struct ShadowHandle(*const VStackShadow);
@@ -74,7 +120,7 @@ impl DataTypeFunctions for ShadowHandle {
     }
 }
 
-/// Create (once per thread) a leaked, GC-marked VStackShadow.
+/// Create (once per owning state) a leaked, GC-marked VStackShadow.
 pub(crate) fn ensure_marked_shadow(slot: &mut Option<&'static mut VStackShadow>) {
     if slot.is_none() {
         let ruby = magnus::Ruby::get().expect("called on a Ruby thread");
@@ -83,7 +129,7 @@ pub(crate) fn ensure_marked_shadow(slot: &mut Option<&'static mut VStackShadow>)
         }));
         let ptr = std::ptr::from_mut::<VStackShadow>(shadow).cast_const();
         let handle: Obj<ShadowHandle> = ruby.obj_wrap(ShadowHandle(ptr));
-        magnus::gc::register_mark_object(handle);
+        ruby.gc_register_mark_object(handle);
         *slot = Some(shadow);
     }
 }

data/ext/nosj/src/stats.rs

@@ -12,7 +12,7 @@ use crate::errors::{nesting_error, parser_error, parser_error_at};
 use crate::files::with_mapped_file;
 use crate::parse::{parse_native_opts, utf8_input};
 use crate::sink::SinkAbort;
-use crate::state::PULL_STATE;
+use crate::state::with_pull_state;
 
 /// What the document's root value was; reported as a Symbol. The
 /// Default is never observable (a successful pass always saw a root
@@ -256,8 +256,7 @@ fn stats_over(ruby: &Ruby, input: &[u8], opts: Value) -> Result<Value, Error> {
         },
         ..StatsSink::default()
     };
-    let result = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    let result = with_pull_state(|state| {
         // Safety: callers verified UTF-8 (coderange or a full scan).
         unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, o.popts) }
     });

data/lib/nosj/version.rb

@@ -2,5 +2,5 @@
 
 module NOSJ
   # The gem version.
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/lib/nosj.rb

@@ -205,7 +205,8 @@ module NOSJ
 
   # Partial parsing by JSON Pointer (with the standard +~0+/+~1+
   # escapes). The matched subtree materializes under the same options
-  # as {.parse}.
+  # as {.parse}; +allow_nan+ and +allow_trailing_comma+ also govern the
+  # walk to it.
   #
   # @example
   #   NOSJ.at_pointer(json, "/users/3/name")  #=> "grace" or nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nosj
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Yaroslav Markin
@@ -104,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 3.3.11
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: An extremely fast JSON parser and generator for Ruby, written in Rust and
   SIMD-accelerated on every platform.