nosj 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87c27bdaeada8a26a98f378d168befec0d1b50b497663bf09ea58b133358b98f
-  data.tar.gz: 0756f4a05107fabcffe58c889c7a40487c7f060d944d9b5bce161430d4e0c70f
+  metadata.gz: 78314f13c659fff6d0e146ac0b951564e66b7b10deeb439e317703754a0100ef
+  data.tar.gz: 993f8a38c4e04dcd68a6ec8ee2dc62c611d2a04c1bc0ff23fc31d67ad69dd585
 SHA512:
-  metadata.gz: ff14cc073152752000f9ad0102222c69e3a63ec561b8411886164c5e1c85fb30044d31e6286074169e1cc531a533b8d59d5b103473b4ae413c32c4e0d565b7f9
-  data.tar.gz: 673019e943af6446a6bb63dacaaaee002f11c07e450154abd5b0842deffc1f798e924ec3531366d2717a71391bf630d98b83b1e2320e90aa3790c83cb2cd93dd
+  metadata.gz: 6b5f09e43ed85c2e345a35d0fd681900e205d4743161542d82ba5c3d1c80879c62bff3acf88f12dac5836334c67d933cc6241ead7dbe6a7a820d5f672bccd3d4
+  data.tar.gz: 0b48922585bb4863f9ed71fb9500a9a60b4c1f857baeaaf79136ed4c9e752aa71c9c23235e85d3bab38b1f0967b7b7212ed6d771be57d62d87fcd39e91666cd8

data/CHANGELOG.md

@@ -1,4 +1,25 @@
-## [0.1.0] - 2026-07-15
+## [0.2.0] - 2026-07-16
+
+- File APIs. `NOSJ.load_file(path, opts)` parses a file directly
+  (~1.3× faster than `parse(File.read(path))`—no file-sized Ruby
+  String is created), and `NOSJ.write_file(path, obj, opts)` generates
+  straight to disk, returning the byte count like `File.write`.
+  `NOSJ.load_lazy_file(path, opts)` wraps a file as a lazy document
+  over a read-only memory map, and `NOSJ.at_pointer_file` /
+  `NOSJ.dig_file` pull single values out of a file without reading the
+  rest into Ruby. Missing files raise the usual `Errno` exceptions.
+- `NOSJ.lazy`: lazy documents. Wrap a document once, then read only
+  what you need: `doc["users"][3]["name"]` parses just that path, `#dig`
+  and `#at_pointer` resolve whole paths, and `#keys`, `#size`, and
+  `#each` inspect a node without parsing its values. Containers come
+  back lazy, scalars come back as plain Ruby values, and repeated
+  reads are cached. `#value` (also `#to_h` / `#to_a`) materializes a
+  subtree under the usual parse options (`symbolize_names`, `freeze`,
+  ...). Pass a frozen string and creating the view is practically
+  free, even on megabyte documents. Malformed content raises on first
+  read, not at wrap time.
+
+## [0.1.0] - 2026-07-16
 
 Initial release.
 

data/Cargo.lock

@@ -162,6 +162,15 @@ version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
+[[package]]
+name = "memmap2"
+version = "0.9.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -189,10 +198,11 @@ dependencies = [
 
 [[package]]
 name = "nosj_native"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "ahash",
  "magnus",
+ "memmap2",
  "nosj",
  "rb-sys",
 ]

data/README.md

@@ -10,10 +10,12 @@
 - It is **faster** than gem json and every
 third-party parser, including Oj, RapidJSON, FastJsonparser, Yajl. 1.0–1.8× faster than the bundled json gem, 1.3–11× faster than Oj, and up to 17×
 faster than Yajl—[see Benchmarks](#benchmarks).
+- It has **lazy documents**: `NOSJ.lazy` wraps a document and parses a value only when you touch it—repeated access costs nanoseconds, and everything you never read is never parsed.
+- It has a **partial parsing mode**: JSON Pointer lookups that pull single values out of big documents in microseconds, skipping everything else.
+- It has **file APIs**: parse, generate, dig, and lazy-wrap files directly—no throwaway file-sized Ruby String, and the partial modes memory-map the file so unread pages never even leave the disk.
 - It comes **precompiled** (platform gems built with per-platform optimizations,
 nothing to compile on install).
-- It has a **partial parsing mode**: JSON Pointer lookups that pull single values out of big documents in microseconds, skipping everything else.
-- Same API and option names as gem json.
+- Otherwise, same API and option names as gem json.
 
 **And there's more**: validate documents without building a single Ruby object, resolve whole batches of paths in one pass, and accelerate an entire application with a one-line drop-in.
 
@@ -83,17 +85,23 @@ NOSJ.generate(obj)                       # indent, space, object_nl, ...,
 NOSJ.pretty_generate(obj)                # ascii_only, script_safe, strict
 ```
 
-**Validation without parsing.** `NOSJ.valid?` runs the full
-parser—tokenizers, string decode, number validation—into a null sink
-and allocates no Ruby objects at all. It is 2-4× faster than
-`NOSJ.parse`, which already leads every parser above:
+**Lazy documents.** `NOSJ.lazy` wraps a document in a lazy view: read
+a field and only that path is parsed—containers stay lazy, scalars
+arrive as plain Ruby values, and repeated reads are cached:
 
 ```ruby
-NOSJ.valid?('{"a":1}')                #=> true
-NOSJ.valid?('{"a":}')                 #=> false
-NOSJ.valid?(src, max_nesting: false)  # same options as parse
+doc = NOSJ.lazy(json)
+doc["users"][3]["name"]   # parses only this path
+doc.dig("meta", "count")  # a whole path in one fused resolution
+doc["users"].size         # counted without materializing anything
+doc["users"][3].value     # materialize one subtree (parse options apply)
 ```
 
+Pass a frozen string and creating the view is practically free—
+nanoseconds, even on a megabyte document. Malformed content raises
+when it is first read, not at wrap time.
+
+
 **Partial parsing.** Pull values out of a document without
 materializing the rest—skipped content is stepped over at SIMD block
 speed, so a lookup costs what it skips, not what the document weighs:
@@ -114,11 +122,38 @@ at the far end of a 570 KB document costs ~71µs, still 13× faster
 than parse-then-dig. Misses return nil; matched subtrees materialize
 with the same options as `parse` (`symbolize_names:`, `freeze:`).
 
+**Files.** Every mode has a file-native form, so a document never
+round-trips through a throwaway Ruby String:
+
+```ruby
+NOSJ.load_file("config.json")                 # 1.3× File.read + parse
+NOSJ.write_file("out.json", obj)              # generate straight to disk
+NOSJ.dig_file("huge.json", "users", 3, "name")   # never reads the rest
+NOSJ.at_pointer_file("huge.json", "/meta/count")
+doc = NOSJ.load_lazy_file("huge.json")        # lazy view over a memory map
+```
+
+The partial and lazy forms memory-map the file, so pages you never
+read are never loaded from disk. Missing files raise the usual
+`Errno` exceptions. Measured numbers live in
+[Benchmarks → File APIs](#file-apis).
+
+**Validation without parsing.** `NOSJ.valid?` runs the full
+parser—tokenizers, string decode, number validation—into a null sink
+and allocates no Ruby objects at all. It is 2-4× faster than
+`NOSJ.parse`, which already leads every parser above:
+
+```ruby
+NOSJ.valid?('{"a":1}')                #=> true
+NOSJ.valid?('{"a":}')                 #=> false
+NOSJ.valid?(src, max_nesting: false)  # same options as parse
+```
+
 ## Benchmarks
 
 Every installed JSON gem, benchmark-ips: AWS EC2 c7a.2xlarge (AMD EPYC 9R14, Zen 4), Ruby 4.0.6 + YJIT, json 2.21.1, Oj 3.17.4, RapidJSON 0.4.0, FastJsonparser 0.6.0, Yajl 1.4.3, PGO build, 2026-07-16. `×N` = times slower than nosj. 
 
-Parse:
+### Parse
 
 | file | nosj (i/s) | json | Oj | FastJsonparser | RapidJSON | Yajl |
 |---|---:|---:|---:|---:|---:|---:|
@@ -136,7 +171,7 @@ Parse:
 | tolstoy | **8.9k** | ×1.79 | ×1.96 | ×2.29 | ×2.10 | ×17.31 |
 | twitter | **1.1k** | ×1.09 | ×1.83 | ×2.25 | ×2.61 | ×5.40 |
 
-Generate:
+### Generate
 
 | file | nosj (i/s) | json | Oj | RapidJSON | Yajl |
 |---|---:|---:|---:|---:|---:|
@@ -157,6 +192,23 @@ Generate:
 \* canada-generate is a statistical tie with the json gem (within
 measurement error).
 
+### File APIs
+
+twitter.json (570 KB) from a warm page cache, medians of 7 alternating
+rounds against the plain-Ruby composition on the same parser (Apple
+Silicon dev box, Ruby 4.0.6 + YJIT, PGO build, 2026-07-16):
+
+| operation | µs/op | vs the Ruby way |
+|---|---:|---|
+| `NOSJ.load_file` (parse the whole file) | 948 | ×1.33 vs `NOSJ.parse(File.read(path))` |
+| `NOSJ.dig_file` (one deep field) | 246 | ×5.2 vs read + parse + dig |
+| `NOSJ.load_lazy_file` + one field | 257 | ×5.0 vs read + parse + dig |
+
+`NOSJ.write_file` measures at parity with
+`File.write(NOSJ.generate(obj))` on this box—file-write timings swing
+too much for an honest multiplier; what it saves is the intermediate
+file-sized Ruby String.
+
 Reproduce with `rake bench` (the parity-gated comparison, after a PGO retrain—the shipping configuration) or `rake bench:ips` (the multi-gem shoot-out).
 
 ## Switching from the json gem

data/ext/nosj/Cargo.toml

@@ -5,7 +5,7 @@
 # required by lib/nosj.rb as "nosj/nosj" (Init_nosj set explicitly in
 # lib.rs). Gem name, module, and API are plain nosj.
 name = "nosj_native"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["Yaroslav Markin <yaroslav@markin.net>"]
 license = "MIT"
@@ -19,6 +19,9 @@ crate-type = ["cdylib", "rlib"]
 magnus = { git = "https://github.com/matsadler/magnus", features = ["embed", "rb-sys"] }
 rb-sys = { version = "0.9.124", default-features = false }
 ahash = "0.8.11"
+# Read-only file mapping for the file entry points (load_lazy_file,
+# at_pointer_file, dig_file): pages never touched are never read.
+memmap2 = "0.9"
 # First-party SIMD JSON parse/generate library (github.com/yaroslav/nosj).
 # For coordinated crate+gem work, temporarily flip to
 # { path = "../../../nosj" } and restore before any commit or release.

data/ext/nosj/src/files.rs

@@ -0,0 +1,218 @@
+//! File entry points. `load_file` reads natively into a reused Rust
+//! buffer, so a document never round-trips through a throwaway Ruby
+//! String (the string allocation and its GC churn are ~37% of
+//! `parse(File.read(path))` on a 570 KB document). `write_file`
+//! streams the generator's pooled buffer straight to disk. The mmap
+//! trio (`load_lazy_file`, `at_pointer_file`, `dig_file`) maps the
+//! file read-only, so partial access never reads unused pages off
+//! disk.
+//!
+//! I/O failures raise the mapped `Errno::*` exception, like `File`
+//! methods do.
+
+use std::cell::RefCell;
+use std::fs;
+use std::io::Read;
+
+use magnus::value::ReprValue;
+use magnus::{Error, RArray, RString, Ruby, Value};
+
+use crate::gen;
+use crate::lazy::{self, DocBytes};
+use crate::parse::{err, materialize, parse_native_opts, ParseNativeOpts};
+use crate::pointer::path_to_pointer;
+use crate::state::PULL_STATE;
+
+const NOT_UTF8: &str = "input is not valid UTF-8";
+
+thread_local! {
+    /// Reused read buffer for `load_file`: capacity survives across
+    /// calls, so a hot loop over files allocates nothing.
+    static FILE_BUF: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
+}
+
+/// On Unix the raw OS error IS the errno `rb_syserr_new` expects.
+#[cfg(unix)]
+fn errno_of(e: &std::io::Error) -> Option<i32> {
+    e.raw_os_error()
+}
+
+/// On Windows the raw OS error is a Win32 code, not a POSIX errno
+/// (ERROR_PATH_NOT_FOUND = 3 read as errno 3 raised Errno::ESRCH on
+/// CI), so map the portable ErrorKind onto the classic CRT errno
+/// values Ruby's Errno classes use. Unmapped kinds fall back to a
+/// plain RuntimeError with the message.
+#[cfg(windows)]
+fn errno_of(e: &std::io::Error) -> Option<i32> {
+    const ENOENT: i32 = 2;
+    const EACCES: i32 = 13;
+    const EEXIST: i32 = 17;
+    match e.kind() {
+        std::io::ErrorKind::NotFound => Some(ENOENT),
+        std::io::ErrorKind::PermissionDenied => Some(EACCES),
+        std::io::ErrorKind::AlreadyExists => Some(EEXIST),
+        _ => None,
+    }
+}
+
+/// Map an I/O failure onto the matching `Errno::*` exception (class
+/// parity with `File.read`/`File.write`; the message carries the path).
+fn io_error(ruby: &Ruby, path: &str, e: &std::io::Error) -> Error {
+    use magnus::rb_sys::FromRawValue;
+    let Some(errno) = errno_of(e) else {
+        return err(ruby, format!("{e} - {path}"));
+    };
+    let Ok(cpath) = std::ffi::CString::new(path) else {
+        return err(ruby, format!("{e} - {path}"));
+    };
+    // SAFETY: rb_syserr_new returns a live Errno exception instance.
+    let exc = unsafe { Value::from_raw(rb_sys::rb_syserr_new(errno, cpath.as_ptr())) };
+    match magnus::Exception::from_value(exc) {
+        Some(exc) => exc.into(),
+        None => err(ruby, format!("{e} - {path}")),
+    }
+}
+
+/// `NOSJ.load_file(path, opts)`: parse a file without ever creating a
+/// file-sized Ruby String.
+pub fn load_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let p = path.to_string()?;
+    FILE_BUF.with(|cell| {
+        let mut buf = cell
+            .try_borrow_mut()
+            .map_or_else(|_| Vec::new(), |mut b| std::mem::take(&mut *b));
+        buf.clear();
+        let result = read_into(&mut buf, &p)
+            .map_err(|e| io_error(ruby, &p, &e))
+            .and_then(|()| {
+                if std::str::from_utf8(&buf).is_err() {
+                    return Err(err(ruby, NOT_UTF8.into()));
+                }
+                materialize(ruby, &buf, &o)
+            });
+        if let Ok(mut slot) = cell.try_borrow_mut() {
+            *slot = buf;
+        }
+        result
+    })
+}
+
+fn read_into(buf: &mut Vec<u8>, path: &str) -> std::io::Result<()> {
+    let mut f = fs::File::open(path)?;
+    let hint = f.metadata().map(|m| m.len() as usize).unwrap_or(0);
+    buf.reserve(hint.saturating_add(1));
+    f.read_to_end(buf)?;
+    Ok(())
+}
+
+/// `NOSJ.write_file(path, obj, opts)`: generate under the usual
+/// options and write the bytes straight from the generator's pooled
+/// buffer. Returns the byte count, like `File.write`.
+pub fn write_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    obj: Value,
+    opts: Value,
+) -> Result<usize, Error> {
+    let p = path.to_string()?;
+    gen::generate_bytes_into(ruby, obj, opts, |ruby, bytes| {
+        fs::write(&p, bytes).map_err(|e| io_error(ruby, &p, &e))?;
+        Ok(bytes.len())
+    })
+}
+
+/// Map `path` read-only and hand a UTF-8-checked view to `f`.
+fn with_mapped_file<R>(
+    ruby: &Ruby,
+    path: &str,
+    f: impl FnOnce(memmap2::Mmap) -> Result<R, Error>,
+) -> Result<R, Error> {
+    let file = fs::File::open(path).map_err(|e| io_error(ruby, path, &e))?;
+    // SAFETY: the mapping is read-only; concurrent modification of the
+    // file by another process is documented as unsupported (the
+    // standard mmap caveat, shared with every mmap-based parser).
+    let map = unsafe { memmap2::Mmap::map(&file) }.map_err(|e| io_error(ruby, path, &e))?;
+    if std::str::from_utf8(&map).is_err() {
+        return Err(err(ruby, NOT_UTF8.into()));
+    }
+    f(map)
+}
+
+/// `NOSJ.load_lazy_file(path, opts)`: a lazy document over a read-only
+/// file mapping. Creation costs one sequential UTF-8 scan; access is
+/// the usual lazy resolution, and pages outside the touched paths are
+/// never read.
+pub fn load_lazy_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let p = path.to_string()?;
+    with_mapped_file(ruby, &p, |map| {
+        lazy::wrap_root(ruby, DocBytes::Mmap(map), o)
+    })
+}
+
+/// Resolve one pointer against a mapped file and materialize the
+/// matched subtree; the mapping is dropped before returning.
+fn resolve_file_pointer(
+    ruby: &Ruby,
+    path: &str,
+    pointer: &str,
+    o: &ParseNativeOpts,
+) -> Result<Value, Error> {
+    with_mapped_file(ruby, path, |map| {
+        let resolved = PULL_STATE.with(|cell| {
+            let mut state = cell.borrow_mut();
+            // SAFETY: UTF-8 checked by with_mapped_file.
+            unsafe { nosj::pointer_utf8_unchecked(&map, pointer, &mut state.bufs) }
+        });
+        match resolved {
+            Ok(None) => Ok(ruby.qnil().as_value()),
+            Ok(Some(slice)) => materialize(ruby, slice.as_bytes(), o),
+            Err(e) if matches!(e.kind, nosj::ErrorKind::InvalidPointer) => {
+                Err(Error::new(ruby.exception_arg_error(), e.to_string()))
+            }
+            Err(e) => Err(err(ruby, e.to_string())),
+        }
+    })
+}
+
+/// `NOSJ.at_pointer_file(path, pointer, opts)`: `NOSJ.at_pointer`
+/// against a file, without reading it into Ruby.
+pub fn at_pointer_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    pointer: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let p = path.to_string()?;
+    let ptr = pointer.to_string()?;
+    resolve_file_pointer(ruby, &p, &ptr, &o)
+}
+
+/// `NOSJ.dig_file(path, *dig_path)`: `NOSJ.dig` against a file.
+/// Negative indices resolve to nil, as everywhere else.
+pub fn dig_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    dig_path: RArray,
+) -> Result<Value, Error> {
+    let p = path.to_string()?;
+    match path_to_pointer(ruby, dig_path)? {
+        Some(ptr) => resolve_file_pointer(ruby, &p, &ptr, &ParseNativeOpts::default()),
+        None => Ok(ruby.qnil().as_value()),
+    }
+}

data/ext/nosj/src/gen/mod.rs

@@ -102,12 +102,50 @@ pub fn generate_native(
     generate_scratched(ruby, obj, &cfg, cap_hint)
 }
 
+/// Generate `obj` under a Ruby options hash (or nil) and hand the
+/// finished bytes to `finish` instead of building a Ruby String: the
+/// entry `NOSJ.write_file` uses to stream straight to disk.
+pub(crate) fn generate_bytes_into<R>(
+    ruby: &Ruby,
+    obj: Value,
+    opts: Value,
+    finish: impl FnOnce(&Ruby, &[u8]) -> Result<R, Error>,
+) -> Result<R, Error> {
+    use magnus::value::ReprValue;
+    if opts.is_nil() {
+        return generate_scratched_into(ruby, obj, &opts::DEFAULT_CONFIG, 0, finish);
+    }
+    let (cfg, cap_hint) = opts::parse_gen_opts(ruby, opts)?;
+    generate_scratched_into(ruby, obj, &cfg, cap_hint, finish)
+}
+
 fn generate_scratched(
     ruby: &Ruby,
     obj: Value,
     cfg: &opts::GenConfig,
     cap_hint: usize,
 ) -> Result<RString, Error> {
+    generate_scratched_into(ruby, obj, cfg, cap_hint, finish_rstring)
+}
+
+/// The default finisher: the generated bytes as a Ruby String.
+fn finish_rstring(_ruby: &Ruby, out: &[u8]) -> Result<RString, Error> {
+    Ok(unsafe {
+        RString::from_value(Value::from_raw(rb_sys::rb_utf8_str_new(
+            out.as_ptr().cast(),
+            out.len() as std::os::raw::c_long,
+        )))
+        .expect("rb_utf8_str_new returns a String")
+    })
+}
+
+fn generate_scratched_into<R>(
+    ruby: &Ruby,
+    obj: Value,
+    cfg: &opts::GenConfig,
+    cap_hint: usize,
+    finish: impl FnOnce(&Ruby, &[u8]) -> Result<R, Error>,
+) -> Result<R, Error> {
     GEN_SCRATCH.with(|cell| match cell.try_borrow_mut() {
         Ok(mut scratch) => {
             let scratch = &mut *scratch;
@@ -118,24 +156,26 @@ fn generate_scratched(
                 cap_hint,
                 &mut scratch.buf,
                 &mut scratch.keys,
+                finish,
             )
         }
         Err(_) => {
             let mut buf = Vec::new();
             let mut keys = GenKeyCache::default();
-            generate_with(ruby, obj, cfg, cap_hint, &mut buf, &mut keys)
+            generate_with(ruby, obj, cfg, cap_hint, &mut buf, &mut keys, finish)
         }
     })
 }
 
-fn generate_with(
+fn generate_with<R>(
     ruby: &Ruby,
     obj: Value,
     cfg: &opts::GenConfig,
     cap_hint: usize,
     out: &mut Vec<u8>,
     keys: &mut GenKeyCache,
-) -> Result<RString, Error> {
+    finish: impl FnOnce(&Ruby, &[u8]) -> Result<R, Error>,
+) -> Result<R, Error> {
     out.clear();
     if out.capacity() < cap_hint {
         out.reserve(cap_hint);
@@ -155,13 +195,7 @@ fn generate_with(
 
     let Gen { out, fail, .. } = g;
     match (result, fail) {
-        (Ok(()), None) => Ok(unsafe {
-            RString::from_value(Value::from_raw(rb_sys::rb_utf8_str_new(
-                out.as_ptr().cast(),
-                out.len() as std::os::raw::c_long,
-            )))
-            .expect("rb_utf8_str_new returns a String")
-        }),
+        (Ok(()), None) => finish(ruby, out),
         (_, Some(fail)) => Err(raise_fail(ruby, fail)),
         (Err(()), None) => Err(Error::new(
             ruby.exception_runtime_error(),