nosj 0.4.0 → 0.5.0

data/ext/nosj/src/parse.rs

@@ -6,9 +6,11 @@
 use magnus::rb_sys::{AsRawValue, FromRawValue};
 use magnus::{Error, RString, Ruby, Value};
 
-use crate::errors::{nesting_error, parser_error, parser_error_at};
-use crate::sink::{NullSink, RubyValueSink, SinkAbort, MAX_NESTING};
-use crate::state::{ensure_marked_shadow, PullState, PULL_STATE};
+use crate::errors::{nesting_error, nosj_exception, parser_error, parser_error_at};
+use crate::locate::Repeat;
+use crate::opt_reader::{Opt, OptReader};
+use crate::sink::{DupKeys, NullSink, RubyValueSink, SinkAbort, MAX_NESTING};
+use crate::state::{ensure_marked_shadow, with_pull_state, PullState};
 
 pub(crate) use crate::errors::parser_error as err;
 
@@ -22,6 +24,12 @@ pub(crate) fn span_of(source: &[u8], sub: &[u8]) -> (usize, usize) {
 
 /// Validate that `data` is UTF-8 (or US-ASCII) with intact coderange and
 /// hand out its byte slice.
+///
+/// The slice borrows the string's buffer, which Ruby may reallocate or
+/// swap (even for a frozen string: see `lazy::DocBytes`), so it must
+/// not be held across anything that can run Ruby code: user callbacks,
+/// yields, or option decoding (`to_int` and friends). Decode options
+/// first; re-borrow after callbacks.
 pub(crate) fn utf8_input<'a>(ruby: &Ruby, data: &'a RString) -> Result<&'a [u8], Error> {
     let raw = data.as_raw();
     unsafe {
@@ -45,6 +53,8 @@ pub(crate) struct ParseNativeOpts {
     pub(crate) symbolize: bool,
     pub(crate) freeze: bool,
     pub(crate) max_nesting: usize,
+    /// json 3 default: a repeated key raises.
+    pub(crate) allow_duplicate_key: bool,
     pub(crate) popts: nosj::ParseOptions,
 }
 
@@ -54,113 +64,173 @@ impl Default for ParseNativeOpts {
             symbolize: false,
             freeze: false,
             max_nesting: MAX_NESTING,
+            allow_duplicate_key: false,
             popts: nosj::ParseOptions::default(),
         }
     }
 }
 
-/// Decode a JSON.parse-compatible options hash: symbolize_names, freeze,
-/// max_nesting, allow_nan, allow_trailing_comma. Unsupported gem options
-/// (object_class, array_class, decimal_class, create_additions) raise.
+/// Decode a JSON.parse-compatible options hash (see [`read_parse_opts`]);
+/// keys it does not read raise ArgumentError, like json 3.
 pub(crate) fn parse_native_opts(ruby: &Ruby, opts: Value) -> Result<ParseNativeOpts, Error> {
-    use magnus::value::ReprValue;
-    use magnus::RHash;
+    let Some(h) = options_hash(ruby, opts)? else {
+        return Ok(ParseNativeOpts::default());
+    };
+    let mut reader = OptReader::new(ruby, h);
+    let out = read_parse_opts(&mut reader)?;
+    reader.finish()?;
+    Ok(out)
+}
 
-    let mut out = ParseNativeOpts::default();
+/// The non-empty options Hash in `opts`, or None for nil and `{}`
+/// (json 3's keyword-only parse hands the drop-in an empty hash per
+/// call).
+pub(crate) fn options_hash(ruby: &Ruby, opts: Value) -> Result<Option<magnus::RHash>, Error> {
+    use magnus::value::ReprValue;
     if opts.is_nil() {
-        return Ok(out);
+        return Ok(None);
     }
-    let h = RHash::from_value(opts)
+    let h = magnus::RHash::from_value(opts)
         .ok_or_else(|| Error::new(ruby.exception_arg_error(), "options must be a Hash"))?;
+    Ok((!h.is_empty()).then_some(h))
+}
 
-    let truthy = |name: &str| -> bool {
-        h.get(ruby.to_symbol(name))
-            .is_some_and(|v: Value| v.to_bool())
+/// Read symbolize_names, freeze, max_nesting, allow_nan,
+/// allow_trailing_comma and allow_duplicate_key. The json options NOSJ
+/// does not implement (object_class, array_class, decimal_class,
+/// on_load, create_additions, allow_comments, allow_control_characters,
+/// allow_invalid_escape) raise unless falsy.
+pub(crate) fn read_parse_opts(r: &mut OptReader) -> Result<ParseNativeOpts, Error> {
+    let mut out = ParseNativeOpts {
+        symbolize: r.truthy(Opt::SymbolizeNames),
+        freeze: r.truthy(Opt::Freeze),
+        allow_duplicate_key: r.truthy(Opt::AllowDuplicateKey),
+        ..ParseNativeOpts::default()
     };
+    out.popts.allow_nan = r.truthy(Opt::AllowNan);
+    out.popts.allow_trailing_comma = r.truthy(Opt::AllowTrailingComma);
 
-    out.symbolize = truthy("symbolize_names");
-    out.freeze = truthy("freeze");
-    out.popts.allow_nan = truthy("allow_nan");
-    out.popts.allow_trailing_comma = truthy("allow_trailing_comma");
-
-    if let Some(mn) = h.get(ruby.to_symbol("max_nesting")) {
-        out.max_nesting =
-            if mn.is_nil() || mn.to_bool() && magnus::Integer::from_value(mn).is_none() {
-                MAX_NESTING // nil / true: gem default
-            } else if !mn.to_bool() {
-                usize::MAX // false: unlimited
-            } else {
-                magnus::Integer::from_value(mn)
-                    .and_then(|i| i.to_u64().ok())
-                    .map_or(MAX_NESTING, |n| n as usize)
-            };
+    if let Some(mn) = r.get(Opt::MaxNesting) {
+        out.max_nesting = max_nesting_of(mn);
     }
 
-    for unsupported in [
-        "object_class",
-        "array_class",
-        "decimal_class",
-        "create_additions",
-    ] {
-        if truthy(unsupported) {
-            return Err(Error::new(
-                ruby.exception_arg_error(),
-                format!("NOSJ.parse does not support the {unsupported} option"),
-            ));
-        }
-    }
+    r.tolerate(&[
+        Opt::ObjectClass,
+        Opt::ArrayClass,
+        Opt::DecimalClass,
+        Opt::OnLoad,
+        Opt::CreateAdditions,
+        Opt::AllowComments,
+        Opt::AllowControlCharacters,
+        Opt::AllowInvalidEscape,
+    ]);
     Ok(out)
 }
 
-/// Pop the root value off the sink stack, or map a drive failure onto
-/// the gem's exceptions. Shared by every driver. `source`/`base` locate
-/// the driven bytes within the full document, so ParserError positions
-/// stay absolute when a subtree slice was parsed.
-fn finish_drive(
+/// A given `max_nesting` value as a limit: nil or true keep the gem's
+/// default, false is unlimited, an Integer is the limit.
+pub(crate) fn max_nesting_of(value: Value) -> usize {
+    use magnus::value::ReprValue;
+    if value.is_nil() || value.to_bool() && magnus::Integer::from_value(value).is_none() {
+        MAX_NESTING
+    } else if !value.to_bool() {
+        usize::MAX
+    } else {
+        magnus::Integer::from_value(value)
+            .and_then(|i| i.to_u64().ok())
+            .map_or(MAX_NESTING, |n| n as usize)
+    }
+}
+
+pub(crate) type DriveResult = Result<(), nosj::DriveError<SinkAbort>>;
+
+/// A drive's failure as the gem's exception; shared by every driver.
+/// `source[start..end]` is what was driven, within the full document,
+/// so ParserError positions stay absolute when a subtree slice was
+/// parsed. Sinks see no offsets, so their two document refusals are
+/// positioned by a cold-path re-walk (`crate::locate`).
+pub(crate) fn drive_error(
     ruby: &Ruby,
-    result: Result<(), nosj::DriveError<SinkAbort>>,
-    stack: &mut Vec<rb_sys::VALUE>,
-    max_nesting: usize,
+    failure: nosj::DriveError<SinkAbort>,
+    o: &ParseNativeOpts,
     source: &[u8],
-    base: usize,
-) -> Result<Value, Error> {
-    match result {
-        Ok(()) => {
-            let raw = stack
-                .pop()
-                .unwrap_or(rb_sys::special_consts::Qnil as rb_sys::VALUE);
-            Ok(unsafe { Value::from_raw(raw) })
-        }
-        Err(nosj::DriveError::Sink(SinkAbort::Overflow)) => {
-            Err(parser_error(ruby, "document too large".into()))
-        }
-        Err(nosj::DriveError::Sink(SinkAbort::BadBigint)) => {
-            Err(parser_error(ruby, "invalid bignum".into()))
-        }
-        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
-            ruby,
-            format!("nesting of {} is too deep", max_nesting.saturating_add(1)),
-        )),
-        // Raised only by the reformat pipe's sink, which never drives
-        // through here; the match must stay total.
-        Err(nosj::DriveError::Sink(SinkAbort::BrokenUtf8Output)) => Err(parser_error(
-            ruby,
-            "source sequence is illegal/malformed utf-8".into(),
-        )),
-        // Also reformat-pipe-only, kept total for the same reason.
-        Err(nosj::DriveError::Sink(SinkAbort::NonFiniteFloat(spelling))) => Err(parser_error(
+    (start, end): (usize, usize),
+) -> Error {
+    use magnus::value::ReprValue;
+    use nosj::DriveError::{Parse, Sink};
+
+    let driven = &source[start..end];
+    match failure {
+        Sink(SinkAbort::Overflow) => parser_error(ruby, "document too large".into()),
+        Sink(SinkAbort::BadBigint) => parser_error(ruby, "invalid bignum".into()),
+        Sink(SinkAbort::TooDeep) => nesting_error(
             ruby,
+            format!("nesting of {} is too deep", o.max_nesting.saturating_add(1)),
+        ),
+        // Positioned like json 3's: at the `{` of the object repeating
+        // the key.
+        Sink(SinkAbort::DuplicateKey) => match crate::locate::duplicate_key(driven, o.popts) {
+            Repeat::Found { at, key } => parser_error_at(
+                ruby,
+                source,
+                start + at,
+                format!(
+                    "duplicate key {} at byte {at}",
+                    ruby.str_new(&key).inspect()
+                ),
+            ),
+            Repeat::Absent | Repeat::Undecided => parser_error(ruby, "duplicate key".into()),
+        },
+        Sink(SinkAbort::LoneSurrogate) => match crate::locate::first_walk_error(driven, o.popts) {
+            Some(e) => parser_error_at(ruby, source, start + e.offset, e.to_string()),
+            None => parser_error(ruby, "lone UTF-16 surrogate".into()),
+        },
+        // Only the reformat pipe raises it: generation refuses a
+        // non-finite float, with the gem's GeneratorError.
+        Sink(SinkAbort::NonFiniteFloat(spelling)) => Error::new(
+            nosj_exception(ruby, "GeneratorError"),
             format!("{spelling} not allowed in JSON"),
-        )),
-        Err(nosj::DriveError::Parse(e)) => Err(parser_error_at(
-            ruby,
-            source,
-            base + e.offset,
-            e.to_string(),
-        )),
+        ),
+        Parse(e) => parser_error_at(ruby, source, start + e.offset, e.to_string()),
     }
 }
 
+/// Drive a sink that builds no Hash (`valid?`, the reformat pipe) under
+/// json 3's duplicate-key rule. `drive(check_dups)` runs the parse. With
+/// the check on, `DupKeys` compares key fingerprints, so a refusal is
+/// confirmed exactly, and a collision (no object really repeats a key)
+/// drives again with the check off.
+pub(crate) fn drive_hashless(
+    input: &[u8],
+    o: &ParseNativeOpts,
+    mut drive: impl FnMut(bool) -> DriveResult,
+) -> DriveResult {
+    let result = drive(!o.allow_duplicate_key);
+    if matches!(result, Err(nosj::DriveError::Sink(SinkAbort::DuplicateKey)))
+        && matches!(crate::locate::duplicate_key(input, o.popts), Repeat::Absent)
+    {
+        return drive(false);
+    }
+    result
+}
+
+/// Pop the root value off the sink stack, or raise the drive's failure
+/// (see [`drive_error`]).
+fn finish_drive(
+    ruby: &Ruby,
+    result: DriveResult,
+    stack: &mut Vec<rb_sys::VALUE>,
+    o: &ParseNativeOpts,
+    source: &[u8],
+    span: (usize, usize),
+) -> Result<Value, Error> {
+    result.map_err(|failure| drive_error(ruby, failure, o, source, span))?;
+    let raw = stack
+        .pop()
+        .unwrap_or(rb_sys::special_consts::Qnil as rb_sys::VALUE);
+    Ok(unsafe { Value::from_raw(raw) })
+}
+
 /// Drive the fused cursor over the whole of `source`. See
 /// [`materialize_at`].
 pub(crate) fn materialize(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
@@ -178,8 +248,7 @@ pub(crate) fn materialize_at(
     end: usize,
     o: &ParseNativeOpts,
 ) -> Result<Value, Error> {
-    PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    with_pull_state(|state| {
         ensure_marked_shadow(&mut state.vstack);
         ensure_marked_shadow(&mut state.key_shadow);
 
@@ -190,7 +259,7 @@ pub(crate) fn materialize_at(
             vstack,
             key_shadow,
             ..
-        } = &mut *state;
+        } = state;
         let stack = &mut vstack.as_mut().unwrap().values;
         stack.clear();
 
@@ -202,13 +271,14 @@ pub(crate) fn materialize_at(
             symbolize: o.symbolize,
             freeze: o.freeze,
             max_nesting: o.max_nesting,
+            allow_duplicate_key: o.allow_duplicate_key,
         };
 
         // Safety: callers verified UTF-8 (coderange or nosj slice).
         let result = unsafe {
             nosj::parse_utf8_unchecked_with(&source[start..end], bufs, &mut sink, o.popts)
         };
-        finish_drive(ruby, result, sink.stack, o.max_nesting, source, start)
+        finish_drive(ruby, result, sink.stack, o, source, (start, end))
     })
 }
 
@@ -238,15 +308,16 @@ pub fn valid_native(
     let Ok(input) = utf8_input(ruby, &data) else {
         return Ok(false);
     };
-    let ok = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
-        let mut sink = NullSink {
-            depth: 0,
-            max_nesting: o.max_nesting,
-        };
-        // Safety: coderange verified by utf8_input.
-        unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, o.popts) }
-            .is_ok()
+    let result = drive_hashless(input, &o, |check_dups| {
+        with_pull_state(|state| {
+            let mut sink = NullSink {
+                depth: 0,
+                max_nesting: o.max_nesting,
+                dup_keys: DupKeys::new(&mut state.dup, check_dups),
+            };
+            // Safety: coderange verified by utf8_input.
+            unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, o.popts) }
+        })
     });
-    Ok(ok)
+    Ok(result.is_ok())
 }

data/ext/nosj/src/patch.rs

@@ -16,7 +16,7 @@ use magnus::{Error, ExceptionClass, RArray, RHash, RString, Ruby, Value};
 use crate::errors::{nosj_exception, parser_error_at};
 use crate::gen::{self, opts::GenConfig};
 use crate::parse::{materialize_at, span_of, utf8_input, ParseNativeOpts};
-use crate::state::PULL_STATE;
+use crate::state::with_pull_state;
 
 const WS: [u8; 4] = *b" \t\n\r";
 
@@ -46,8 +46,7 @@ fn gen_config<'a>(
 /// Parse failures raise the rich ParserError (absolute positions);
 /// pointer syntax errors raise ArgumentError, like `at_pointer`.
 fn span_at(ruby: &Ruby, doc: &[u8], pointer: &str) -> Result<Option<(usize, usize)>, Error> {
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    let resolved = with_pull_state(|state| {
         // SAFETY: every entry validated the document bytes as UTF-8.
         unsafe { nosj::pointer_utf8_unchecked(doc, pointer, &mut state.bufs) }
     });
@@ -82,8 +81,7 @@ fn container_children(
 ) -> Result<(u8, Vec<ChildSpan>), Error> {
     let span = &doc[start..end];
     let kind = span.first().copied().unwrap_or(0);
-    let walk: Result<Vec<ChildSpan>, nosj::ParseError> = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    let walk: Result<Vec<ChildSpan>, nosj::ParseError> = with_pull_state(|state| {
         // SAFETY: doc validated UTF-8 by the entry; spans lie on token
         // edges.
         let mut r = unsafe { nosj::Reader::from_utf8_unchecked(span, &mut state.bufs) };
@@ -360,6 +358,12 @@ fn op_test(ruby: &Ruby, doc: &[u8], path: &str, expected: Value) -> Result<(), E
 /// `NOSJ.splice(json, edits, opts)`: batch pointer replacement. All
 /// targets resolve in ONE forward pass; the output is built in one
 /// sweep copying every byte outside the target spans untouched.
+///
+/// Every value is generated BEFORE the source bytes are borrowed:
+/// generation runs user code (`to_json`, `to_s`) that may mutate or
+/// reallocate the source, or drop the only Ruby reference to a later
+/// value. Rendering inside the edits iteration keeps each value a live
+/// argument while it runs, and nothing after the borrow calls Ruby.
 pub fn splice_native(
     ruby: &Ruby,
     _rb_self: Value,
@@ -369,21 +373,26 @@ pub fn splice_native(
 ) -> Result<RString, Error> {
     let mut slot = None;
     let cfg = gen_config(ruby, opts, &mut slot)?;
-    let input = utf8_input(ruby, &data)?;
+    // Validate the encoding up front (error precedence); the bytes are
+    // borrowed again only after every callback has run.
+    utf8_input(ruby, &data)?;
 
     let mut pointers: Vec<String> = Vec::with_capacity(edits.len());
-    let mut values: Vec<Value> = Vec::with_capacity(edits.len());
+    let mut rendered: Vec<u8> = Vec::new();
+    let mut ranges: Vec<(usize, usize)> = Vec::with_capacity(edits.len());
     edits.foreach(|k: Value, v: Value| {
         let ptr = RString::from_value(k)
             .ok_or_else(|| arg_error(ruby, "splice pointers must be Strings".into()))?;
         pointers.push(ptr.to_string()?);
-        values.push(v);
+        let start = rendered.len();
+        gen::emit_into(ruby, v, cfg, &mut rendered)?;
+        ranges.push((start, rendered.len()));
         Ok(magnus::r_hash::ForEach::Continue)
     })?;
 
+    let input = utf8_input(ruby, &data)?;
     let refs: Vec<&str> = pointers.iter().map(String::as_str).collect();
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    let resolved = with_pull_state(|state| {
         // SAFETY: coderange verified by utf8_input.
         unsafe { nosj::pointers_utf8_unchecked(input, &refs, &mut state.bufs) }
     });
@@ -395,7 +404,7 @@ pub fn splice_native(
         Err(e) => return Err(parser_error_at(ruby, input, e.offset, e.to_string())),
     };
 
-    let mut spans: Vec<(usize, usize, Value)> = Vec::with_capacity(pointers.len());
+    let mut spans: Vec<(usize, usize, usize)> = Vec::with_capacity(pointers.len());
     for (i, hit) in hits.into_iter().enumerate() {
         let Some(slice) = hit else {
             let exc: ExceptionClass = ruby.exception_key_error();
@@ -405,7 +414,7 @@ pub fn splice_native(
             ));
         };
         let (s, e) = span_of(input, slice.as_bytes());
-        spans.push((s, e, values[i]));
+        spans.push((s, e, i));
     }
     spans.sort_unstable_by_key(|&(s, _, _)| s);
     for pair in spans.windows(2) {
@@ -417,11 +426,12 @@ pub fn splice_native(
         }
     }
 
-    let mut out = Vec::with_capacity(input.len() + 64);
+    let mut out = Vec::with_capacity(input.len() + rendered.len());
     let mut pos = 0;
-    for &(s, e, v) in &spans {
+    for &(s, e, edit) in &spans {
+        let (rs, re) = ranges[edit];
         out.extend_from_slice(&input[pos..s]);
-        gen::emit_into(ruby, v, cfg, &mut out)?;
+        out.extend_from_slice(&rendered[rs..re]);
         pos = e;
     }
     out.extend_from_slice(&input[pos..]);

data/ext/nosj/src/pointer.rs

@@ -7,7 +7,7 @@ use magnus::{Error, RString, Ruby, Value};
 
 use crate::errors::parser_error_at;
 use crate::parse::{materialize_at, parse_native_opts, span_of, utf8_input, ParseNativeOpts};
-use crate::state::PULL_STATE;
+use crate::state::with_pull_state;
 
 /// Resolve one JSON Pointer against `data`, materializing the matched
 /// subtree; `nil` when the pointer misses.
@@ -19,12 +19,11 @@ fn at_pointer_impl(
 ) -> Result<Value, Error> {
     use magnus::value::ReprValue;
     let input = utf8_input(ruby, &data)?;
-    // Resolve first (one PULL_STATE borrow), then materialize (a fresh
-    // borrow); the resolved slice borrows `input`, not the buffers.
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    // Resolve, then materialize, as two separate uses of the parse
+    // state; the resolved slice borrows `input`, not the state.
+    let resolved = with_pull_state(|state| {
         // Safety: coderange verified above.
-        unsafe { nosj::pointer_utf8_unchecked(input, pointer, &mut state.bufs) }
+        unsafe { nosj::pointer_utf8_unchecked_with(input, pointer, &mut state.bufs, o.popts) }
     });
     match resolved {
         Ok(None) => Ok(ruby.qnil().as_value()),
@@ -125,12 +124,11 @@ fn at_pointers_impl(
     let input = utf8_input(ruby, &data)?;
     let live: Vec<&str> = pointers.iter().flatten().map(String::as_str).collect();
 
-    // Resolve first (one PULL_STATE borrow); the resolved slices borrow
-    // `input`, not the buffers, so materializing can re-borrow freely.
-    let resolved = PULL_STATE.with(|cell| {
-        let mut state = cell.borrow_mut();
+    // Resolve first (one use of the parse state); the resolved slices
+    // borrow `input`, not the state, so each materializes separately.
+    let resolved = with_pull_state(|state| {
         // Safety: coderange verified by utf8_input.
-        unsafe { nosj::pointers_utf8_unchecked(input, &live, &mut state.bufs) }
+        unsafe { nosj::pointers_utf8_unchecked_with(input, &live, &mut state.bufs, o.popts) }
     });
     let mut hits = match resolved {
         Ok(hits) => hits.into_iter(),