nosj 0.1.0 → 0.3.0

data/ext/nosj/src/lib.rs

@@ -5,6 +5,12 @@
 //!   GVL-releasing indexed parse) plus shared option decoding and
 //!   input gating.
 //! - `pointer.rs`: partial parsing (dig, at_pointer, batch forms).
+//! - `lazy.rs`: lazy documents (NOSJ.lazy nodes resolving access on
+//!   demand over shared document bytes).
+//! - `files.rs`: file entry points (load_file, write_file, and the
+//!   mmap-backed load_lazy_file / at_pointer_file / dig_file).
+//! - `stats.rs`: document statistics (NOSJ.stats / stats_file) from a
+//!   counting sink pass.
 //! - `sink.rs`: the VALUE-building and validation sinks with their
 //!   interned-key caches.
 //! - `state.rs`: per-thread reusable state and the GC-marked value
@@ -12,11 +18,18 @@
 //! - `gen/`: generation (JSON.generate-compatible walker, options,
 //!   key cache, protect shims, error mapping).
 
+pub mod errors;
+pub mod files;
 pub mod gen;
+pub mod lazy;
+pub mod lines;
 pub mod parse;
+pub mod patch;
 pub mod pointer;
+pub mod reformat;
 pub mod sink;
 pub mod state;
+pub mod stats;
 
 use magnus::{method, prelude::*, Error, Ruby};
 
@@ -39,7 +52,54 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
         "at_pointers_native",
         method!(pointer::at_pointers_native, 3),
     )?;
+    module.define_singleton_method("lazy_native", method!(lazy::lazy_native, 2))?;
+    module.define_singleton_method("load_file_native", method!(files::load_file_native, 2))?;
+    module.define_singleton_method("write_file_native", method!(files::write_file_native, 3))?;
+    module.define_singleton_method(
+        "load_lazy_file_native",
+        method!(files::load_lazy_file_native, 2),
+    )?;
+    module.define_singleton_method(
+        "at_pointer_file_native",
+        method!(files::at_pointer_file_native, 3),
+    )?;
+    module.define_singleton_method("dig_file_native", method!(files::dig_file_native, 2))?;
+    module.define_singleton_method("stats_native", method!(stats::stats_native, 2))?;
+    module.define_singleton_method("stats_file_native", method!(stats::stats_file_native, 2))?;
+    module.define_singleton_method("each_line_native", method!(lines::each_line_native, 2))?;
+    module.define_singleton_method(
+        "each_line_file_native",
+        method!(lines::each_line_file_native, 2),
+    )?;
+    module.define_singleton_method(
+        "generate_lines_native",
+        method!(gen::generate_lines_native, 2),
+    )?;
+    module.define_singleton_method("write_lines_native", method!(files::write_lines_native, 3))?;
+    module.define_singleton_method("splice_native", method!(patch::splice_native, 3))?;
+    module.define_singleton_method("patch_native", method!(patch::patch_native, 3))?;
+    module.define_singleton_method("reformat_native", method!(reformat::reformat_native, 2))?;
+    module.define_singleton_method(
+        "reformat_file_native",
+        method!(reformat::reformat_file_native, 2),
+    )?;
+    let lazy_class = module.define_class("Lazy", ruby.class_object())?;
+    // Nodes are only born from NOSJ.lazy / lazy resolution.
+    lazy_class.undef_default_alloc_func();
+    lazy_class.define_method("__get", method!(lazy::lazy_get, 1))?;
+    lazy_class.define_method("__dig", method!(lazy::lazy_dig, 1))?;
+    lazy_class.define_method("__at_pointer", method!(lazy::lazy_at_pointer, 1))?;
+    lazy_class.define_method("__materialize", method!(lazy::lazy_materialize, 0))?;
+    lazy_class.define_method("__kind", method!(lazy::lazy_kind, 0))?;
+    lazy_class.define_method("__byte_size", method!(lazy::lazy_byte_size, 0))?;
+    lazy_class.define_method("__keys", method!(lazy::lazy_keys, 0))?;
+    lazy_class.define_method("__size", method!(lazy::lazy_size, 0))?;
+    lazy_class.define_method("__children", method!(lazy::lazy_children, 0))?;
     module.define_singleton_method("generate_native", method!(gen::generate_native, 2))?;
+    module.define_singleton_method(
+        "generate_rails_native",
+        method!(gen::generate_rails_native, 3),
+    )?;
     // `generate` itself is native and variadic: the json gem routes
     // its `generate` through a Ruby frame into C, so skipping our own
     // forwarder frame is a straight per-call win on small documents.

data/ext/nosj/src/lines.rs

@@ -0,0 +1,90 @@
+//! NDJSON / JSON Lines: `NOSJ.each_line` walks a newline-delimited
+//! document yielding one parsed value per line, and the file form does
+//! it over a read-only memory map. A raw newline can never occur
+//! INSIDE a JSON value (control characters must be escaped), so line
+//! splitting is exact framing, and one-value-per-line is what the
+//! format requires: a second value on a line fails as trailing
+//! garbage, with an absolute document position on the error.
+
+use magnus::value::ReprValue;
+use magnus::{Error, RString, Ruby, Value};
+
+use crate::files::with_mapped_file;
+use crate::parse::{materialize_at, parse_native_opts, utf8_input, ParseNativeOpts};
+
+/// Blank-line whitespace (the newline itself is the separator). Lines
+/// holding only this are skipped, per the NDJSON convention that
+/// parsers ignore empty lines (trailing newlines at EOF are universal).
+const LINE_WS: [u8; 3] = *b" \t\r";
+
+fn blank(line: &[u8]) -> bool {
+    line.iter().all(|b| LINE_WS.contains(b))
+}
+
+/// Yield one parsed value per non-blank line. Each line parses through
+/// the shared sink machinery against the FULL source, so a malformed
+/// line raises the rich ParserError whose `#line` is the physical
+/// NDJSON line number. The thread state is borrowed per line, never
+/// across a yield: the block is free to call back into NOSJ.
+fn walk_lines(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<(), Error> {
+    let mut pos = 0;
+    while pos < source.len() {
+        let line_end = source[pos..]
+            .iter()
+            .position(|&b| b == b'\n')
+            .map_or(source.len(), |p| pos + p);
+        if !blank(&source[pos..line_end]) {
+            let value = materialize_at(ruby, source, pos, line_end, o)?;
+            let _: Value = ruby.yield_value(value)?;
+        }
+        pos = line_end + 1;
+    }
+    Ok(())
+}
+
+/// `NOSJ.each_line(source, opts) { |value| }`: the Ruby wrapper
+/// guarantees a block (and builds the Enumerator otherwise).
+pub fn each_line_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    data: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let input = utf8_input(ruby, &data)?;
+    if data.as_value().is_frozen() {
+        // A frozen source cannot be mutated by the block, and `data`
+        // lives on this frame, so the borrow stays valid across yields.
+        walk_lines(ruby, input, &o)?;
+    } else {
+        // The block could mutate (or free the buffer of) an unfrozen
+        // source mid-iteration; walk a private copy. Same policy as
+        // NOSJ.lazy: pass a frozen string for zero-copy.
+        let owned = input.to_vec();
+        walk_lines(ruby, &owned, &o)?;
+    }
+    Ok(ruby.qnil().as_value())
+}
+
+/// `NOSJ.each_line_file(path, opts) { |value| }`: NDJSON over a
+/// read-only memory map; the file never becomes a Ruby String.
+pub fn each_line_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let p = path.to_string()?;
+    // An empty file is a valid, zero-event NDJSON stream, but mapping
+    // a zero-length file fails with EINVAL on Linux; answer before the
+    // mapper. Metadata failures (ENOENT, ...) fall through so the
+    // mapper raises the same Errno an open would.
+    if std::fs::metadata(&p).is_ok_and(|m| m.len() == 0) {
+        return Ok(ruby.qnil().as_value());
+    }
+    with_mapped_file(ruby, &p, |map| {
+        walk_lines(ruby, &map, &o)?;
+        Ok(ruby.qnil().as_value())
+    })
+}

data/ext/nosj/src/parse.rs

@@ -6,12 +6,18 @@
 use magnus::rb_sys::{AsRawValue, FromRawValue};
 use magnus::{Error, RString, Ruby, Value};
 
+use crate::errors::{nesting_error, parser_error, parser_error_at};
 use crate::sink::{NullSink, RubyValueSink, SinkAbort, MAX_NESTING};
 use crate::state::{ensure_marked_shadow, PullState, PULL_STATE};
 
-#[cold]
-pub(crate) fn err(ruby: &Ruby, msg: String) -> Error {
-    Error::new(ruby.exception_runtime_error(), msg)
+pub(crate) use crate::errors::parser_error as err;
+
+/// Byte range of `sub` within `source`. `sub` must be a subslice of
+/// `source` (it always is here: pointer resolution and lazy spans hand
+/// out slices borrowed from the document they resolved against).
+pub(crate) fn span_of(source: &[u8], sub: &[u8]) -> (usize, usize) {
+    let start = sub.as_ptr() as usize - source.as_ptr() as usize;
+    (start, start + sub.len())
 }
 
 /// Validate that `data` is UTF-8 (or US-ASCII) with intact coderange and
@@ -107,12 +113,16 @@ pub(crate) fn parse_native_opts(ruby: &Ruby, opts: Value) -> Result<ParseNativeO
 }
 
 /// Pop the root value off the sink stack, or map a drive failure onto
-/// the gem's exceptions. Shared by every driver.
+/// the gem's exceptions. Shared by every driver. `source`/`base` locate
+/// the driven bytes within the full document, so ParserError positions
+/// stay absolute when a subtree slice was parsed.
 fn finish_drive(
     ruby: &Ruby,
     result: Result<(), nosj::DriveError<SinkAbort>>,
     stack: &mut Vec<rb_sys::VALUE>,
     max_nesting: usize,
+    source: &[u8],
+    base: usize,
 ) -> Result<Value, Error> {
     match result {
         Ok(()) => {
@@ -122,23 +132,47 @@ fn finish_drive(
             Ok(unsafe { Value::from_raw(raw) })
         }
         Err(nosj::DriveError::Sink(SinkAbort::Overflow)) => {
-            Err(err(ruby, "document too large".into()))
+            Err(parser_error(ruby, "document too large".into()))
         }
         Err(nosj::DriveError::Sink(SinkAbort::BadBigint)) => {
-            Err(err(ruby, "invalid bignum".into()))
+            Err(parser_error(ruby, "invalid bignum".into()))
         }
-        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(err(
+        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
             ruby,
             format!("nesting of {} is too deep", max_nesting.saturating_add(1)),
         )),
-        Err(nosj::DriveError::Parse(e)) => Err(err(ruby, e.to_string())),
+        // Raised only by the reformat pipe's sink, which never drives
+        // through here; the match must stay total.
+        Err(nosj::DriveError::Sink(SinkAbort::BrokenUtf8Output)) => Err(parser_error(
+            ruby,
+            "source sequence is illegal/malformed utf-8".into(),
+        )),
+        Err(nosj::DriveError::Parse(e)) => Err(parser_error_at(
+            ruby,
+            source,
+            base + e.offset,
+            e.to_string(),
+        )),
     }
 }
 
-/// Drive the fused cursor over `input`, building Ruby values through the
-/// shared thread-local sink machinery. `input` must be valid UTF-8 (see
-/// [`utf8_input`]).
-pub(crate) fn materialize(ruby: &Ruby, input: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
+/// Drive the fused cursor over the whole of `source`. See
+/// [`materialize_at`].
+pub(crate) fn materialize(ruby: &Ruby, source: &[u8], o: &ParseNativeOpts) -> Result<Value, Error> {
+    materialize_at(ruby, source, 0, source.len(), o)
+}
+
+/// Drive the fused cursor over `source[start..end]`, building Ruby
+/// values through the shared thread-local sink machinery. `source` must
+/// be valid UTF-8 (see [`utf8_input`]); the full document is passed so
+/// error positions come out absolute.
+pub(crate) fn materialize_at(
+    ruby: &Ruby,
+    source: &[u8],
+    start: usize,
+    end: usize,
+    o: &ParseNativeOpts,
+) -> Result<Value, Error> {
     PULL_STATE.with(|cell| {
         let mut state = cell.borrow_mut();
         ensure_marked_shadow(&mut state.vstack);
@@ -166,8 +200,10 @@ pub(crate) fn materialize(ruby: &Ruby, input: &[u8], o: &ParseNativeOpts) -> Res
         };
 
         // Safety: callers verified UTF-8 (coderange or nosj slice).
-        let result = unsafe { nosj::parse_utf8_unchecked_with(input, bufs, &mut sink, o.popts) };
-        finish_drive(ruby, result, sink.stack, o.max_nesting)
+        let result = unsafe {
+            nosj::parse_utf8_unchecked_with(&source[start..end], bufs, &mut sink, o.popts)
+        };
+        finish_drive(ruby, result, sink.stack, o.max_nesting, source, start)
     })
 }