nosj 0.2.0 → 0.3.1

data/ext/nosj/src/lazy.rs

@@ -17,7 +17,8 @@ use magnus::typed_data::Obj;
 use magnus::value::ReprValue;
 use magnus::{DataTypeFunctions, Error, RArray, RString, Ruby, TypedData, Value};
 
-use crate::parse::{err, materialize, parse_native_opts, utf8_input, ParseNativeOpts};
+use crate::errors::parser_error_at;
+use crate::parse::{materialize_at, parse_native_opts, span_of, utf8_input, ParseNativeOpts};
 use crate::pointer::{path_to_pointer, push_escaped_token};
 use crate::state::PULL_STATE;
 
@@ -105,20 +106,19 @@ impl LazyNode {
 /// Wrap a resolved raw-value slice: containers become new lazy nodes,
 /// scalars materialize now. `sub` must be a subslice of `doc.bytes`.
 fn resolved_to_value(ruby: &Ruby, doc: &Arc<DocInner>, sub: &[u8]) -> Result<Value, Error> {
+    let (start, end) = span_of(doc.bytes(), sub);
     match sub.first().copied() {
         Some(k) if k == KIND_OBJECT || k == KIND_ARRAY => {
-            let base = doc.bytes().as_ptr() as usize;
-            let start = sub.as_ptr() as usize - base;
             let node = LazyNode {
                 doc: Arc::clone(doc),
                 start,
-                end: start + sub.len(),
+                end,
                 kind: k,
             };
             let obj: Obj<LazyNode> = ruby.obj_wrap(node);
             Ok(obj.as_value())
         }
-        _ => materialize(ruby, sub, &doc.opts),
+        _ => materialize_at(ruby, doc.bytes(), start, end, &doc.opts),
     }
 }
 
@@ -139,7 +139,7 @@ fn resolve_in_span(ruby: &Ruby, node: &LazyNode, pointer: &str) -> Result<Value,
         Err(e) if matches!(e.kind, nosj::ErrorKind::InvalidPointer) => {
             Err(Error::new(ruby.exception_arg_error(), e.to_string()))
         }
-        Err(e) => Err(err(ruby, e.to_string())),
+        Err(e) => Err(reader_err(ruby, node, e)),
     }
 }
 
@@ -188,7 +188,12 @@ pub(crate) fn wrap_root(
     let doc = Arc::new(DocInner { bytes, opts });
     let input = doc.bytes();
     let Some(start) = input.iter().position(|b| !WS.contains(b)) else {
-        return Err(err(ruby, "unexpected end of input".into()));
+        return Err(parser_error_at(
+            ruby,
+            input,
+            input.len(),
+            "unexpected end of input".into(),
+        ));
     };
     let end = input.iter().rposition(|b| !WS.contains(b)).unwrap() + 1;
     let sub = &doc.bytes()[start..end];
@@ -245,7 +250,13 @@ pub fn lazy_at_pointer(
 /// `__materialize`: the whole span as plain Ruby values, under the
 /// document's parse options.
 pub fn lazy_materialize(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<Value, Error> {
-    materialize(ruby, rb_self.span(), &rb_self.doc.opts)
+    materialize_at(
+        ruby,
+        rb_self.doc.bytes(),
+        rb_self.start,
+        rb_self.end,
+        &rb_self.doc.opts,
+    )
 }
 
 /// `__kind`: `:object` or `:array`.
@@ -261,8 +272,11 @@ pub fn lazy_byte_size(_ruby: &Ruby, rb_self: Obj<LazyNode>) -> usize {
     rb_self.end - rb_self.start
 }
 
-fn reader_err(ruby: &Ruby, e: nosj::ParseError) -> Error {
-    err(ruby, e.to_string())
+/// A walk failure inside `node`'s span: the reader's offset is
+/// span-relative, so shift by the node's start to report an absolute
+/// document position.
+fn reader_err(ruby: &Ruby, node: &LazyNode, e: nosj::ParseError) -> Error {
+    parser_error_at(ruby, node.doc.bytes(), node.start + e.offset, e.to_string())
 }
 
 /// `__keys`: the object's decoded keys, one Reader walk, values skipped.
@@ -278,8 +292,11 @@ pub fn lazy_keys(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
         let mut state = cell.borrow_mut();
         // SAFETY: spans are valid UTF-8 (see resolve_in_span).
         let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
-        r.next_node().map_err(|e| reader_err(ruby, e))?;
-        let mut has = match r.object_first_key().map_err(|e| reader_err(ruby, e))? {
+        r.next_node().map_err(|e| reader_err(ruby, &rb_self, e))?;
+        let mut has = match r
+            .object_first_key()
+            .map_err(|e| reader_err(ruby, &rb_self, e))?
+        {
             Some(k) => {
                 out.push(ruby.str_new(k))?;
                 true
@@ -287,8 +304,11 @@ pub fn lazy_keys(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Error> {
             None => false,
         };
         while has {
-            r.skip_value().map_err(|e| reader_err(ruby, e))?;
-            has = match r.object_next_key().map_err(|e| reader_err(ruby, e))? {
+            r.skip_value().map_err(|e| reader_err(ruby, &rb_self, e))?;
+            has = match r
+                .object_next_key()
+                .map_err(|e| reader_err(ruby, &rb_self, e))?
+            {
                 Some(k) => {
                     out.push(ruby.str_new(k))?;
                     true
@@ -308,27 +328,27 @@ pub fn lazy_size(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<usize, Error> {
         let mut state = cell.borrow_mut();
         // SAFETY: spans are valid UTF-8 (see resolve_in_span).
         let mut r = unsafe { nosj::Reader::from_utf8_unchecked(rb_self.span(), &mut state.bufs) };
-        r.next_node().map_err(|e| reader_err(ruby, e))?;
+        r.next_node().map_err(|e| reader_err(ruby, &rb_self, e))?;
         let mut n = 0usize;
         if rb_self.kind == KIND_OBJECT {
             let mut has = r
                 .object_first_key()
-                .map_err(|e| reader_err(ruby, e))?
+                .map_err(|e| reader_err(ruby, &rb_self, e))?
                 .is_some();
             while has {
                 n += 1;
-                r.skip_value().map_err(|e| reader_err(ruby, e))?;
+                r.skip_value().map_err(|e| reader_err(ruby, &rb_self, e))?;
                 has = r
                     .object_next_key()
-                    .map_err(|e| reader_err(ruby, e))?
+                    .map_err(|e| reader_err(ruby, &rb_self, e))?
                     .is_some();
             }
         } else {
-            let mut has = r.array_first().map_err(|e| reader_err(ruby, e))?;
+            let mut has = r.array_first().map_err(|e| reader_err(ruby, &rb_self, e))?;
             while has {
                 n += 1;
-                r.skip_value().map_err(|e| reader_err(ruby, e))?;
-                has = r.array_next().map_err(|e| reader_err(ruby, e))?;
+                r.skip_value().map_err(|e| reader_err(ruby, &rb_self, e))?;
+                has = r.array_next().map_err(|e| reader_err(ruby, &rb_self, e))?;
             }
         }
         Ok(n)
@@ -383,7 +403,7 @@ pub fn lazy_children(ruby: &Ruby, rb_self: Obj<LazyNode>) -> Result<RArray, Erro
         }
         Ok(out)
     });
-    let descs = descs.map_err(|e| reader_err(ruby, e))?;
+    let descs = descs.map_err(|e| reader_err(ruby, &rb_self, e))?;
 
     let out = ruby.ary_new_capa(descs.len());
     for d in descs {

data/ext/nosj/src/lib.rs

@@ -9,6 +9,8 @@
 //!   demand over shared document bytes).
 //! - `files.rs`: file entry points (load_file, write_file, and the
 //!   mmap-backed load_lazy_file / at_pointer_file / dig_file).
+//! - `stats.rs`: document statistics (NOSJ.stats / stats_file) from a
+//!   counting sink pass.
 //! - `sink.rs`: the VALUE-building and validation sinks with their
 //!   interned-key caches.
 //! - `state.rs`: per-thread reusable state and the GC-marked value
@@ -16,13 +18,18 @@
 //! - `gen/`: generation (JSON.generate-compatible walker, options,
 //!   key cache, protect shims, error mapping).
 
+pub mod errors;
 pub mod files;
 pub mod gen;
 pub mod lazy;
+pub mod lines;
 pub mod parse;
+pub mod patch;
 pub mod pointer;
+pub mod reformat;
 pub mod sink;
 pub mod state;
+pub mod stats;
 
 use magnus::{method, prelude::*, Error, Ruby};
 
@@ -57,6 +64,25 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
         method!(files::at_pointer_file_native, 3),
     )?;
     module.define_singleton_method("dig_file_native", method!(files::dig_file_native, 2))?;
+    module.define_singleton_method("stats_native", method!(stats::stats_native, 2))?;
+    module.define_singleton_method("stats_file_native", method!(stats::stats_file_native, 2))?;
+    module.define_singleton_method("each_line_native", method!(lines::each_line_native, 2))?;
+    module.define_singleton_method(
+        "each_line_file_native",
+        method!(lines::each_line_file_native, 2),
+    )?;
+    module.define_singleton_method(
+        "generate_lines_native",
+        method!(gen::generate_lines_native, 2),
+    )?;
+    module.define_singleton_method("write_lines_native", method!(files::write_lines_native, 3))?;
+    module.define_singleton_method("splice_native", method!(patch::splice_native, 3))?;
+    module.define_singleton_method("patch_native", method!(patch::patch_native, 3))?;
+    module.define_singleton_method("reformat_native", method!(reformat::reformat_native, 2))?;
+    module.define_singleton_method(
+        "reformat_file_native",
+        method!(reformat::reformat_file_native, 2),
+    )?;
     let lazy_class = module.define_class("Lazy", ruby.class_object())?;
     // Nodes are only born from NOSJ.lazy / lazy resolution.
     lazy_class.undef_default_alloc_func();
@@ -70,6 +96,10 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     lazy_class.define_method("__size", method!(lazy::lazy_size, 0))?;
     lazy_class.define_method("__children", method!(lazy::lazy_children, 0))?;
     module.define_singleton_method("generate_native", method!(gen::generate_native, 2))?;
+    module.define_singleton_method(
+        "generate_rails_native",
+        method!(gen::generate_rails_native, 3),
+    )?;
     // `generate` itself is native and variadic: the json gem routes
     // its `generate` through a Ruby frame into C, so skipping our own
     // forwarder frame is a straight per-call win on small documents.

data/ext/nosj/src/lines.rs

@@ -0,0 +1,90 @@
+//! NDJSON / JSON Lines: `NOSJ.each_line` walks a newline-delimited
+//! document yielding one parsed value per line, and the file form does
+//! it over a read-only memory map. A raw newline can never occur
+//! INSIDE a JSON value (control characters must be escaped), so line
+//! splitting is exact framing, and one-value-per-line is what the
+//! format requires: a second value on a line fails as trailing
+//! garbage, with an absolute document position on the error.
+
+use magnus::value::ReprValue;
+use magnus::{Error, RString, Ruby, Value};
+
+use crate::files::with_mapped_file;
+use crate::parse::{materialize_at, parse_native_opts, utf8_input, ParseNativeOpts};
+
+/// Blank-line whitespace (the newline itself is the separator). Lines
+/// holding only this are skipped, per the NDJSON convention that
+/// parsers ignore empty lines (trailing newlines at EOF are universal).
+const LINE_WS: [u8; 3] = *b" \t\r";
+
+fn blank(line: &[u8]) -> bool {
+    line.iter().all(|b| LINE_WS.contains(b))
+}
+
+/// Yield one parsed value per non-blank line. Each line parses through
+/// the shared sink machinery against the FULL source, so a malformed
+/// line raises the rich ParserError whose `#line` is the physical
+/// NDJSON line number. The thread state is borrowed per line, never
+/// across a yield: the block is free to call back into NOSJ.
+fn walk_lines(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<(), Error> {
+    let mut pos = 0;
+    while pos < source.len() {
+        let line_end = source[pos..]
+            .iter()
+            .position(|&b| b == b'\n')
+            .map_or(source.len(), |p| pos + p);
+        if !blank(&source[pos..line_end]) {
+            let value = materialize_at(ruby, source, pos, line_end, o)?;
+            let _: Value = ruby.yield_value(value)?;
+        }
+        pos = line_end + 1;
+    }
+    Ok(())
+}
+
+/// `NOSJ.each_line(source, opts) { |value| }`: the Ruby wrapper
+/// guarantees a block (and builds the Enumerator otherwise).
+pub fn each_line_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    data: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let input = utf8_input(ruby, &data)?;
+    if data.as_value().is_frozen() {
+        // A frozen source cannot be mutated by the block, and `data`
+        // lives on this frame, so the borrow stays valid across yields.
+        walk_lines(ruby, input, &o)?;
+    } else {
+        // The block could mutate (or free the buffer of) an unfrozen
+        // source mid-iteration; walk a private copy. Same policy as
+        // NOSJ.lazy: pass a frozen string for zero-copy.
+        let owned = input.to_vec();
+        walk_lines(ruby, &owned, &o)?;
+    }
+    Ok(ruby.qnil().as_value())
+}
+
+/// `NOSJ.each_line_file(path, opts) { |value| }`: NDJSON over a
+/// read-only memory map; the file never becomes a Ruby String.
+pub fn each_line_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let p = path.to_string()?;
+    // An empty file is a valid, zero-event NDJSON stream, but mapping
+    // a zero-length file fails with EINVAL on Linux; answer before the
+    // mapper. Metadata failures (ENOENT, ...) fall through so the
+    // mapper raises the same Errno an open would.
+    if std::fs::metadata(&p).is_ok_and(|m| m.len() == 0) {
+        return Ok(ruby.qnil().as_value());
+    }
+    with_mapped_file(ruby, &p, |map| {
+        walk_lines(ruby, &map, &o)?;
+        Ok(ruby.qnil().as_value())
+    })
+}

data/ext/nosj/src/parse.rs

@@ -6,12 +6,18 @@
 use magnus::rb_sys::{AsRawValue, FromRawValue};
 use magnus::{Error, RString, Ruby, Value};
 
+use crate::errors::{nesting_error, parser_error, parser_error_at};
 use crate::sink::{NullSink, RubyValueSink, SinkAbort, MAX_NESTING};
 use crate::state::{ensure_marked_shadow, PullState, PULL_STATE};
 
-#[cold]
-pub(crate) fn err(ruby: &Ruby, msg: String) -> Error {
-    Error::new(ruby.exception_runtime_error(), msg)
+pub(crate) use crate::errors::parser_error as err;
+
+/// Byte range of `sub` within `source`. `sub` must be a subslice of
+/// `source` (it always is here: pointer resolution and lazy spans hand
+/// out slices borrowed from the document they resolved against).
+pub(crate) fn span_of(source: &[u8], sub: &[u8]) -> (usize, usize) {
+    let start = sub.as_ptr() as usize - source.as_ptr() as usize;
+    (start, start + sub.len())
 }
 
 /// Validate that `data` is UTF-8 (or US-ASCII) with intact coderange and
@@ -107,12 +113,16 @@ pub(crate) fn parse_native_opts(ruby: &Ruby, opts: Value) -> Result<ParseNativeO
 }
 
 /// Pop the root value off the sink stack, or map a drive failure onto
-/// the gem's exceptions. Shared by every driver.
+/// the gem's exceptions. Shared by every driver. `source`/`base` locate
+/// the driven bytes within the full document, so ParserError positions
+/// stay absolute when a subtree slice was parsed.
 fn finish_drive(
     ruby: &Ruby,
     result: Result<(), nosj::DriveError<SinkAbort>>,
     stack: &mut Vec<rb_sys::VALUE>,
     max_nesting: usize,
+    source: &[u8],
+    base: usize,
 ) -> Result<Value, Error> {
     match result {
         Ok(()) => {
@@ -122,23 +132,52 @@ fn finish_drive(
             Ok(unsafe { Value::from_raw(raw) })
         }
         Err(nosj::DriveError::Sink(SinkAbort::Overflow)) => {
-            Err(err(ruby, "document too large".into()))
+            Err(parser_error(ruby, "document too large".into()))
         }
         Err(nosj::DriveError::Sink(SinkAbort::BadBigint)) => {
-            Err(err(ruby, "invalid bignum".into()))
+            Err(parser_error(ruby, "invalid bignum".into()))
         }
-        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(err(
+        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
             ruby,
             format!("nesting of {} is too deep", max_nesting.saturating_add(1)),
         )),
-        Err(nosj::DriveError::Parse(e)) => Err(err(ruby, e.to_string())),
+        // Raised only by the reformat pipe's sink, which never drives
+        // through here; the match must stay total.
+        Err(nosj::DriveError::Sink(SinkAbort::BrokenUtf8Output)) => Err(parser_error(
+            ruby,
+            "source sequence is illegal/malformed utf-8".into(),
+        )),
+        // Also reformat-pipe-only, kept total for the same reason.
+        Err(nosj::DriveError::Sink(SinkAbort::NonFiniteFloat(spelling))) => Err(parser_error(
+            ruby,
+            format!("{spelling} not allowed in JSON"),
+        )),
+        Err(nosj::DriveError::Parse(e)) => Err(parser_error_at(
+            ruby,
+            source,
+            base + e.offset,
+            e.to_string(),
+        )),
     }
 }
 
-/// Drive the fused cursor over `input`, building Ruby values through the
-/// shared thread-local sink machinery. `input` must be valid UTF-8 (see
-/// [`utf8_input`]).
-pub(crate) fn materialize(ruby: &Ruby, input: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
+/// Drive the fused cursor over the whole of `source`. See
+/// [`materialize_at`].
+pub(crate) fn materialize(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
+    materialize_at(ruby, source, 0, source.len(), o)
+}
+
+/// Drive the fused cursor over `source[start..end]`, building Ruby
+/// values through the shared thread-local sink machinery. `source` must
+/// be valid UTF-8 (see [`utf8_input`]); the full document is passed so
+/// error positions come out absolute.
+pub(crate) fn materialize_at(
+    ruby: &Ruby,
+    source: &[u8],
+    start: usize,
+    end: usize,
+    o: &ParseNativeOpts,
+) -> Result<Value, Error> {
     PULL_STATE.with(|cell| {
         let mut state = cell.borrow_mut();
         ensure_marked_shadow(&mut state.vstack);
@@ -166,8 +205,10 @@ pub(crate) fn materialize(ruby: &Ruby, input: &[u8], o: &ParseNativeOpts) -> Res
         };
 
         // Safety: callers verified UTF-8 (coderange or nosj slice).
-        let result = unsafe { nosj::parse_utf8_unchecked_with(input, bufs, &mut sink, o.popts) };
-        finish_drive(ruby, result, sink.stack, o.max_nesting)
+        let result = unsafe {
+            nosj::parse_utf8_unchecked_with(&source[start..end], bufs, &mut sink, o.popts)
+        };
+        finish_drive(ruby, result, sink.stack, o.max_nesting, source, start)
     })
 }