nosj 0.4.0 → 0.5.0

data/ext/nosj/src/lib.rs

@@ -4,6 +4,8 @@
 //! - `parse.rs`: whole-document entry points (parse, valid?, the
 //!   GVL-releasing indexed parse) plus shared option decoding and
 //!   input gating.
+//! - `opt_reader.rs`: options-hash reading with json 3's unknown-key
+//!   rule, shared by the parse and generate decoders.
 //! - `pointer.rs`: partial parsing (dig, at_pointer, batch forms).
 //! - `lazy.rs`: lazy documents (NOSJ.lazy nodes resolving access on
 //!   demand over shared document bytes).
@@ -23,6 +25,8 @@ pub mod files;
 pub mod gen;
 pub mod lazy;
 pub mod lines;
+pub mod locate;
+pub mod opt_reader;
 pub mod parse;
 pub mod patch;
 pub mod pointer;

data/ext/nosj/src/lines.rs

@@ -24,22 +24,31 @@ fn blank(line: &[u8]) -> bool {
 /// Yield one parsed value per non-blank line. Each line parses through
 /// the shared sink machinery against the FULL source, so a malformed
 /// line raises the rich ParserError whose `#line` is the physical
-/// NDJSON line number. The thread state is borrowed per line, never
-/// across a yield: the block is free to call back into NOSJ.
-fn walk_lines(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<(), Error> {
+/// NDJSON line number. No slice is held across a yield: the block runs
+/// arbitrary Ruby (even deduplicating a frozen source swaps its buffer,
+/// see `lazy::DocBytes`), so `source` hands out the bytes afresh for
+/// every line, with identical content and so identical offsets.
+fn walk_lines<'s>(
+    ruby: &Ruby,
+    source: impl Fn() -> &'s [u8],
+    o: &ParseNativeOpts,
+) -> Result<(), Error> {
     let mut pos = 0;
-    while pos < source.len() {
-        let line_end = source[pos..]
+    loop {
+        let bytes = source();
+        if pos >= bytes.len() {
+            return Ok(());
+        }
+        let line_end = bytes[pos..]
             .iter()
             .position(|&b| b == b'\n')
-            .map_or(source.len(), |p| pos + p);
-        if !blank(&source[pos..line_end]) {
-            let value = materialize_at(ruby, source, pos, line_end, o)?;
+            .map_or(bytes.len(), |p| pos + p);
+        if !blank(&bytes[pos..line_end]) {
+            let value = materialize_at(ruby, bytes, pos, line_end, o)?;
             let _: Value = ruby.yield_value(value)?;
         }
         pos = line_end + 1;
     }
-    Ok(())
 }
 
 /// `NOSJ.each_line(source, opts) { |value| }`: the Ruby wrapper
@@ -53,15 +62,16 @@ pub fn each_line_native(
     let o = parse_native_opts(ruby, opts)?;
     let input = utf8_input(ruby, &data)?;
     if data.as_value().is_frozen() {
-        // A frozen source cannot be mutated by the block, and `data`
-        // lives on this frame, so the borrow stays valid across yields.
-        walk_lines(ruby, input, &o)?;
+        // A frozen source keeps its content (never its buffer, see
+        // walk_lines), so it is re-read per line, zero-copy.
+        // SAFETY: validated UTF-8 above; `data` lives on this frame.
+        walk_lines(ruby, || unsafe { data.as_slice() }, &o)?;
     } else {
-        // The block could mutate (or free the buffer of) an unfrozen
-        // source mid-iteration; walk a private copy. Same policy as
-        // NOSJ.lazy: pass a frozen string for zero-copy.
+        // The block could rewrite an unfrozen source mid-iteration;
+        // walk a private copy. Same policy as NOSJ.lazy: pass a frozen
+        // string for zero-copy.
         let owned = input.to_vec();
-        walk_lines(ruby, &owned, &o)?;
+        walk_lines(ruby, || &owned, &o)?;
     }
     Ok(ruby.qnil().as_value())
 }
@@ -84,7 +94,7 @@ pub fn each_line_file_native(
         return Ok(ruby.qnil().as_value());
     }
     with_mapped_file(ruby, &p, |map| {
-        walk_lines(ruby, &map, &o)?;
+        walk_lines(ruby, || &map, &o)?;
         Ok(ruby.qnil().as_value())
     })
 }

data/ext/nosj/src/locate.rs

@@ -0,0 +1,162 @@
+//! Cold-path positions for the refusals a sink detects without offsets.
+//!
+//! Sinks see events, not byte positions, so a duplicate key (found by
+//! hash size or key fingerprints) or a lone surrogate (delivered as
+//! WTF-8) aborts the drive with no location. These walks re-read the
+//! same bytes with the crate's pull `Reader`, under the same grammar
+//! options, to find where. Each is linear and iterative: one `Reader`
+//! over the whole document, an explicit stack instead of recursion, so
+//! no nesting depth is out of reach.
+
+use std::collections::HashSet;
+
+use nosj::{Buffers, Node, ParseError, ParseOptions, Reader};
+
+use crate::parse::span_of;
+
+/// What the exact check found behind a sink's duplicate-key refusal.
+pub(crate) enum Repeat {
+    /// The first object, in the order objects close (as the drive meets
+    /// them), that repeats a key: the offset of its `{` and the key.
+    Found { at: usize, key: String },
+    /// No object repeats a key: the refusal was a fingerprint collision.
+    Absent,
+    /// The walk stopped on a Reader error before deciding. Callers
+    /// treat it as a refusal.
+    Undecided,
+}
+
+pub(crate) fn duplicate_key(doc: &[u8], popts: ParseOptions) -> Repeat {
+    match scan(doc, popts, true) {
+        Ok(None) => Repeat::Absent,
+        Ok(Some((path, key))) => match container_offset(doc, &path, popts) {
+            Ok(at) => Repeat::Found { at, key },
+            Err(_) => Repeat::Undecided,
+        },
+        Err(_) => Repeat::Undecided,
+    }
+}
+
+/// The first error a full walk of `doc` meets, decoding every string
+/// and key: where a lone surrogate the sink refused sits (the Reader
+/// rejects them with a position).
+pub(crate) fn first_walk_error(doc: &[u8], popts: ParseOptions) -> Option<ParseError> {
+    scan(doc, popts, false).err()
+}
+
+/// One open container of [`scan`]'s walk.
+struct Frame {
+    object: bool,
+    /// The member or element being walked.
+    index: usize,
+    seen: HashSet<String>,
+    repeated: Option<String>,
+}
+
+impl Frame {
+    fn open(object: bool) -> Self {
+        Frame {
+            object,
+            index: 0,
+            seen: HashSet::new(),
+            repeated: None,
+        }
+    }
+
+    fn note_key(&mut self, key: &str) {
+        if self.repeated.is_none() && !self.seen.insert(key.to_owned()) {
+            self.repeated = Some(key.to_owned());
+        }
+    }
+}
+
+/// Walk all of `doc`, depth first. With `find_repeats`, stop at the
+/// first object to close with a repeated key: the member/element
+/// indices from the root down to it, and the key.
+fn scan(
+    doc: &[u8],
+    popts: ParseOptions,
+    find_repeats: bool,
+) -> Result<Option<(Vec<usize>, String)>, ParseError> {
+    let mut bufs = Buffers::new();
+    // SAFETY: `doc` is validated UTF-8.
+    let mut r = unsafe { Reader::from_utf8_unchecked_with(doc, &mut bufs, popts) };
+    let mut stack: Vec<Frame> = Vec::new();
+    loop {
+        // The cursor is at a value: open it, or step past it.
+        let opened = match r.next_node()? {
+            Node::ObjectStart => match r.object_first_key()? {
+                Some(key) => {
+                    let mut frame = Frame::open(true);
+                    if find_repeats {
+                        frame.note_key(key);
+                    }
+                    Some(frame)
+                }
+                None => None,
+            },
+            Node::ArrayStart => r.array_first()?.then(|| Frame::open(false)),
+            _ => None,
+        };
+        if let Some(frame) = opened {
+            stack.push(frame);
+            continue;
+        }
+        // The value is complete: advance its container, closing every
+        // container it completes on the way up.
+        loop {
+            let Some(top) = stack.last_mut() else {
+                return Ok(None);
+            };
+            let more = if top.object {
+                match r.object_next_key()? {
+                    Some(key) => {
+                        if find_repeats {
+                            top.note_key(key);
+                        }
+                        true
+                    }
+                    None => false,
+                }
+            } else {
+                r.array_next()?
+            };
+            if more {
+                top.index += 1;
+                break;
+            }
+            if let Some(key) = stack.pop().and_then(|closed| closed.repeated) {
+                return Ok(Some((stack.iter().map(|f| f.index).collect(), key)));
+            }
+        }
+    }
+}
+
+/// The offset of the container at `path`, member/element indices from
+/// the root: walk down skipping earlier siblings, then skip the
+/// container itself, whose slice starts at its bracket.
+fn container_offset(doc: &[u8], path: &[usize], popts: ParseOptions) -> Result<usize, ParseError> {
+    let mut bufs = Buffers::new();
+    // SAFETY: `doc` is validated UTF-8.
+    let mut r = unsafe { Reader::from_utf8_unchecked_with(doc, &mut bufs, popts) };
+    // Every index on the path was walked by `scan`, so each step lands
+    // on an existing member.
+    for &index in path {
+        let object = matches!(r.next_node()?, Node::ObjectStart);
+        if object {
+            r.object_first_key()?;
+        } else {
+            r.array_first()?;
+        }
+        for _ in 0..index {
+            r.skip_value()?;
+            if object {
+                r.object_next_key()?;
+            } else {
+                r.array_next()?;
+            }
+        }
+    }
+    let container = r.skip_value()?;
+    Ok(span_of(doc, container.as_bytes()).0)
+}

data/ext/nosj/src/opt_reader.rs

@@ -0,0 +1,170 @@
+//! Options-hash reading under json 3's unknown-key rule: every key of an
+//! options hash must be one the entry point reads, else ArgumentError
+//! in json 3's wording ("unknown keyword: foo"). A reader counts the
+//! keys it finds, so a clean hash costs one length compare; only a
+//! shortfall walks the hash to name the keys nobody read.
+
+use magnus::r_hash::ForEach;
+use magnus::value::ReprValue;
+use magnus::{Error, RHash, Ruby, Symbol, Value};
+
+macro_rules! options {
+    ($($variant:ident => $name:literal,)*) => {
+        /// Every option any entry point reads. The discriminant is the
+        /// option's bit in [`OptReader`]'s masks, so a key two decoders
+        /// read from one hash (reformat takes parse and generate
+        /// options together) counts once.
+        #[derive(Clone, Copy)]
+        pub(crate) enum Opt {
+            $($variant,)*
+        }
+
+        /// Ruby option names, indexed by [`Opt`] discriminant.
+        const NAMES: &[&str] = &[$($name,)*];
+    };
+}
+
+options! {
+    SymbolizeNames => "symbolize_names",
+    Freeze => "freeze",
+    MaxNesting => "max_nesting",
+    AllowNan => "allow_nan",
+    AllowTrailingComma => "allow_trailing_comma",
+    AllowDuplicateKey => "allow_duplicate_key",
+    ObjectClass => "object_class",
+    ArrayClass => "array_class",
+    DecimalClass => "decimal_class",
+    OnLoad => "on_load",
+    CreateAdditions => "create_additions",
+    AllowComments => "allow_comments",
+    AllowControlCharacters => "allow_control_characters",
+    AllowInvalidEscape => "allow_invalid_escape",
+    Indent => "indent",
+    Space => "space",
+    SpaceBefore => "space_before",
+    ObjectNl => "object_nl",
+    ArrayNl => "array_nl",
+    AsciiOnly => "ascii_only",
+    ScriptSafe => "script_safe",
+    Strict => "strict",
+    Depth => "depth",
+    BufferInitialLength => "buffer_initial_length",
+    SortKeys => "sort_keys",
+    AsJson => "as_json",
+}
+
+const _: () = assert!(NAMES.len() <= u64::BITS as usize);
+
+impl Opt {
+    fn bit(self) -> u64 {
+        1 << self as u32
+    }
+
+    pub(crate) fn name(self) -> &'static str {
+        NAMES[self as usize]
+    }
+}
+
+pub(crate) struct OptReader<'a> {
+    ruby: &'a Ruby,
+    hash: RHash,
+    /// The hash's key count, read once.
+    len: usize,
+    /// Options read and present in the hash.
+    found: u64,
+    /// Options accepted only while falsy (see [`OptReader::tolerate`]).
+    tolerated: u64,
+}
+
+impl<'a> OptReader<'a> {
+    pub(crate) fn new(ruby: &'a Ruby, hash: RHash) -> Self {
+        Self {
+            ruby,
+            hash,
+            len: hash.len(),
+            found: 0,
+            tolerated: 0,
+        }
+    }
+
+    pub(crate) fn ruby(&self) -> &'a Ruby {
+        self.ruby
+    }
+
+    fn all_found(&self) -> bool {
+        self.found.count_ones() as usize == self.len
+    }
+
+    /// The value under `opt`'s Symbol key; an explicit nil is present.
+    /// Once every key is found, the rest are absent without a lookup
+    /// (`{symbolize_names: true}` costs one lookup, not six). An option
+    /// read twice (reformat reads a few for both parsing and
+    /// generating) is looked up again.
+    pub(crate) fn get(&mut self, opt: Opt) -> Option<Value> {
+        if self.all_found() && self.found & opt.bit() == 0 {
+            return None;
+        }
+        // An interned StaticSymbol: no String allocation per lookup.
+        let value = self.hash.get(self.ruby.sym_new(opt.name()));
+        if value.is_some() {
+            self.found |= opt.bit();
+        }
+        value
+    }
+
+    pub(crate) fn truthy(&mut self, opt: Opt) -> bool {
+        self.get(opt).is_some_and(|v| v.to_bool())
+    }
+
+    /// Accept these json options, which NOSJ does not implement, only
+    /// while falsy: their default behavior is NOSJ's, anything else
+    /// would be silently ignored. No lookup happens here: such a key can
+    /// only be present when the reads leave keys unfound, so
+    /// [`OptReader::finish`] checks them on its cold path.
+    pub(crate) fn tolerate(&mut self, opts: &[Opt]) {
+        for opt in opts {
+            self.tolerated |= opt.bit();
+        }
+    }
+
+    /// Raise for keys no read asked for, and for tolerated options set
+    /// to something truthy.
+    pub(crate) fn finish(self) -> Result<(), Error> {
+        if self.all_found() {
+            return Ok(());
+        }
+        self.leftover_keys()
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn leftover_keys(self) -> Result<(), Error> {
+        let (found, tolerated) = (self.found, self.tolerated);
+        let option_of = |key: Value| {
+            let name = Symbol::from_value(key)?.name().ok()?;
+            NAMES.iter().position(|known| *known == name)
+        };
+        let mut unknown = Vec::new();
+        let mut unsupported = None;
+        self.hash.foreach(|key: Value, value: Value| {
+            match option_of(key).map(|index| 1u64 << index) {
+                Some(bit) if found & bit != 0 => {}
+                Some(bit) if tolerated & bit != 0 => {
+                    if value.to_bool() {
+                        unsupported = Some(key.to_string());
+                        return Ok(ForEach::Stop);
+                    }
+                }
+                _ => unknown.push(key.to_string()),
+            }
+            Ok(ForEach::Continue)
+        })?;
+        let message = match (unsupported, unknown.as_slice()) {
+            (Some(name), _) => format!("NOSJ does not support the {name} option"),
+            (None, []) => return Ok(()),
+            (None, [key]) => format!("unknown keyword: {key}"),
+            (None, keys) => format!("unknown keywords: {}", keys.join(", ")),
+        };
+        Err(Error::new(self.ruby.exception_arg_error(), message))
+    }
+}