nosj 0.2.0 → 0.3.1

data/ext/nosj/src/reformat.rs

@@ -0,0 +1,320 @@
+//! Reformat without building values: `NOSJ.minify` / `NOSJ.reformat`
+//! drive the full parser straight into the crate's `Writer`, a pure
+//! event-to-bytes pipe. No Ruby object is allocated for the document,
+//! only the result String; between the SIMD scan on the way in and the
+//! SIMD escape kernels on the way out there is nothing else.
+//!
+//! Output is exactly what `NOSJ.generate(NOSJ.parse(json), opts)`
+//! would produce, with two deliberate differences: duplicate object
+//! keys pass through (a reformatter must not silently drop data the
+//! way parse's last-key-wins materialization does), and lone-surrogate
+//! string values re-escape as `\uXXXX` instead of raising (the output
+//! must reparse; raw WTF-8 would not). Numbers come out in the gem's
+//! canonical spelling (`1.50` becomes `1.5`), and string escapes are
+//! normalized by the emission kernels.
+
+use std::cell::RefCell;
+
+use magnus::value::ReprValue;
+use magnus::{Error, RString, Ruby, Value};
+use nosj::emit::EscapeMode;
+use nosj::{FloatFormat, WriteOptions, Writer};
+
+use crate::errors::{nesting_error, nosj_exception, parser_error, parser_error_at};
+use crate::files::with_mapped_file;
+use crate::gen::opts::{parse_gen_opts, GenConfig, DEFAULT_CONFIG};
+use crate::parse::{parse_native_opts, utf8_input};
+use crate::patch::finish_string;
+use crate::sink::SinkAbort;
+use crate::state::PULL_STATE;
+
+thread_local! {
+    /// Pooled output buffer: capacity survives across calls. The pipe
+    /// never calls back into Ruby, so the borrow spans the whole drive
+    /// without any reentrancy concern.
+    static PIPE_BUF: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
+}
+
+/// The event-to-Writer pipe. Structure events forward to the Writer's
+/// grammar state (separators, layout, indentation); scalar events
+/// forward to its emission kernels.
+struct PipeSink<'a> {
+    w: Writer<'a>,
+    depth: usize,
+    max_nesting: usize,
+    /// For re-escaping WTF-8 string content (see [`quote_wtf8`]).
+    mode: EscapeMode,
+    /// Non-finite floats pass through as literals only when the
+    /// generate side allows them; see [`PipeSink::float`].
+    allow_nan: bool,
+}
+
+const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef";
+
+/// WTF-8 lone surrogates arrive as one 3-byte sequence with this lead
+/// byte (the only ill-formed runs the parser ever emits).
+const WTF8_SURROGATE_LEAD: u8 = 0xED;
+const WTF8_SURROGATE_LEN: usize = 3;
+/// Payload bits of a UTF-8 lead / continuation byte.
+const UTF8_LEAD3_BITS: u32 = 0x0F;
+const UTF8_CONT_BITS: u32 = 0x3F;
+
+/// Quote and escape WTF-8 content: valid UTF-8 runs go through the
+/// configured escape kernel, and lone-surrogate sequences re-escape as
+/// `\uXXXX`, so the output reparses to the identical string in every
+/// mode (raw WTF-8 bytes would not: the parser requires UTF-8 input).
+/// This deliberately diverges from `generate`, which refuses
+/// broken-coderange strings: a reformatter must accept everything the
+/// parser accepts.
+fn quote_wtf8(out: &mut Vec<u8>, bytes: &[u8], mode: EscapeMode) {
+    out.push(b'"');
+    let mut rest = bytes;
+    loop {
+        match std::str::from_utf8(rest) {
+            Ok(s) => {
+                nosj::emit::escape_into(out, s.as_bytes(), mode);
+                break;
+            }
+            Err(e) => {
+                let valid = e.valid_up_to();
+                nosj::emit::escape_into(out, &rest[..valid], mode);
+                let sur = &rest[valid..];
+                debug_assert!(
+                    sur.len() >= WTF8_SURROGATE_LEN && sur[0] == WTF8_SURROGATE_LEAD,
+                    "parser only emits lone-surrogate WTF-8"
+                );
+                // Standard 3-byte UTF-8 decode of the surrogate
+                // codepoint (U+D800..U+DFFF), re-emitted as \uXXXX.
+                let cp = ((u32::from(sur[0]) & UTF8_LEAD3_BITS) << 12)
+                    | ((u32::from(sur[1]) & UTF8_CONT_BITS) << 6)
+                    | (u32::from(sur[2]) & UTF8_CONT_BITS);
+                out.extend_from_slice(b"\\u");
+                for shift in [12, 8, 4, 0] {
+                    out.push(HEX_DIGITS[((cp >> shift) & 0xF) as usize]);
+                }
+                rest = &sur[WTF8_SURROGATE_LEN..];
+            }
+        }
+    }
+    out.push(b'"');
+}
+
+impl PipeSink<'_> {
+    fn enter(&mut self) -> Result<(), SinkAbort> {
+        self.depth += 1;
+        if self.depth > self.max_nesting {
+            return Err(SinkAbort::TooDeep);
+        }
+        Ok(())
+    }
+}
+
+impl nosj::Sink for PipeSink<'_> {
+    type Error = SinkAbort;
+
+    fn null(&mut self) -> Result<(), SinkAbort> {
+        self.w.null();
+        Ok(())
+    }
+
+    fn boolean(&mut self, value: bool) -> Result<(), SinkAbort> {
+        self.w.boolean(value);
+        Ok(())
+    }
+
+    fn int(&mut self, value: i64) -> Result<(), SinkAbort> {
+        self.w.int(value);
+        Ok(())
+    }
+
+    fn float(&mut self, value: f64) -> Result<(), SinkAbort> {
+        if value.is_finite() {
+            self.w.float(value);
+            return Ok(());
+        }
+        let spelling = if value.is_nan() {
+            "NaN"
+        } else if value > 0.0 {
+            "Infinity"
+        } else {
+            "-Infinity"
+        };
+        // Not just the allow_nan keywords: a huge-exponent literal
+        // (1e999) parses to Infinity in strict mode too, and generate
+        // refuses to emit it. Gem parity either way. The parser only
+        // delivers the f64, never the original digits, so passing the
+        // source spelling through is not an option. One asymmetry
+        // follows: the pipe streams duplicate-key entries that
+        // parse's last-key-wins would discard, so a document with an
+        // overflowing literal shadowed by a duplicate key raises here
+        // even though generate(parse(x)) would succeed.
+        if !self.allow_nan {
+            return Err(SinkAbort::NonFiniteFloat(spelling));
+        }
+        self.w.value_raw(spelling.as_bytes());
+        Ok(())
+    }
+
+    fn big_int(&mut self, digits: &str) -> Result<(), SinkAbort> {
+        // Verbatim digit passthrough: no bignum is ever built.
+        self.w.value_raw(digits.as_bytes());
+        Ok(())
+    }
+
+    fn str(&mut self, value: &str) -> Result<(), SinkAbort> {
+        self.w.str(value);
+        Ok(())
+    }
+
+    fn str_bytes(&mut self, value: &[u8]) -> Result<(), SinkAbort> {
+        // Rare path (lone-surrogate content); a per-call buffer is fine.
+        let mut quoted = Vec::with_capacity(value.len() + 8);
+        quote_wtf8(&mut quoted, value, self.mode);
+        self.w.value_raw(&quoted);
+        Ok(())
+    }
+
+    fn key(&mut self, key: &str) -> Result<(), SinkAbort> {
+        self.w.key(key);
+        Ok(())
+    }
+
+    fn key_bytes(&mut self, _key: &[u8]) -> Result<(), SinkAbort> {
+        // A lone-surrogate KEY has no pre-serialized escape hatch in
+        // the Writer (values have value_raw; a key_raw is on the crate
+        // wishlist), so this pathological case keeps generate's
+        // refusal semantics.
+        Err(SinkAbort::BrokenUtf8Output)
+    }
+
+    fn begin_array(&mut self) -> Result<(), SinkAbort> {
+        self.enter()?;
+        self.w.begin_array();
+        Ok(())
+    }
+
+    fn begin_object(&mut self) -> Result<(), SinkAbort> {
+        self.enter()?;
+        self.w.begin_object();
+        Ok(())
+    }
+
+    fn mark(&self) -> usize {
+        0
+    }
+
+    fn end_array(&mut self, _: usize, _: usize) -> Result<(), SinkAbort> {
+        self.depth -= 1;
+        self.w.end_array();
+        Ok(())
+    }
+
+    fn end_object(&mut self, _: usize, _: usize) -> Result<(), SinkAbort> {
+        self.depth -= 1;
+        self.w.end_object();
+        Ok(())
+    }
+}
+
+/// Map a GenConfig (the gem's generate-option decoding) onto the
+/// crate's WriteOptions.
+fn write_options(cfg: &GenConfig) -> WriteOptions {
+    let mut w = WriteOptions::COMPACT;
+    w.indent = cfg.indent.clone();
+    w.space = cfg.space.clone();
+    w.space_before = cfg.space_before.clone();
+    w.object_nl = cfg.object_nl.clone();
+    w.array_nl = cfg.array_nl.clone();
+    w.escape = cfg.mode;
+    w.float = FloatFormat::Fpconv;
+    w
+}
+
+/// Run the pipe over already-UTF-8-vouched bytes.
+fn reformat_over(ruby: &Ruby, input: &[u8], opts: Value) -> Result<RString, Error> {
+    let po = parse_native_opts(ruby, opts)?;
+    let built;
+    let gcfg: &GenConfig = if opts.is_nil() {
+        &DEFAULT_CONFIG
+    } else {
+        built = parse_gen_opts(ruby, opts)?.0;
+        &built
+    };
+    let wopts = write_options(gcfg);
+
+    PIPE_BUF.with(|cell| {
+        let mut buf = cell.borrow_mut();
+        buf.clear();
+        // The output is at least input-sized for minify-shaped runs.
+        buf.reserve(input.len());
+        let mut sink = PipeSink {
+            w: Writer::new(&mut buf, &wopts),
+            depth: 0,
+            max_nesting: po.max_nesting,
+            mode: gcfg.mode,
+            allow_nan: gcfg.allow_nan,
+        };
+        let result = PULL_STATE.with(|state_cell| {
+            let mut state = state_cell.borrow_mut();
+            // Safety: callers verified UTF-8 (coderange or full scan).
+            unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, po.popts) }
+        });
+        match result {
+            Ok(()) => finish_string(&buf),
+            Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
+                ruby,
+                format!(
+                    "nesting of {} is too deep",
+                    po.max_nesting.saturating_add(1)
+                ),
+            )),
+            Err(nosj::DriveError::Sink(SinkAbort::BrokenUtf8Output)) => Err(Error::new(
+                // Gem parity: generate raises GeneratorError for a
+                // string ascii_only cannot represent.
+                nosj_exception(ruby, "GeneratorError"),
+                "source sequence is illegal/malformed utf-8",
+            )),
+            Err(nosj::DriveError::Sink(SinkAbort::NonFiniteFloat(spelling))) => Err(Error::new(
+                nosj_exception(ruby, "GeneratorError"),
+                format!("{spelling} not allowed in JSON"),
+            )),
+            Err(nosj::DriveError::Sink(_)) => {
+                Err(parser_error(ruby, "reformat pass aborted".into()))
+            }
+            Err(nosj::DriveError::Parse(e)) => {
+                Err(parser_error_at(ruby, input, e.offset, e.to_string()))
+            }
+        }
+    })
+}
+
+/// `NOSJ.reformat_native(source, opts)`: minify and reformat share
+/// this entry; the formatting defaults are compact.
+pub fn reformat_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    data: RString,
+    opts: Value,
+) -> Result<RString, Error> {
+    let input = utf8_input(ruby, &data)?;
+    reformat_over(ruby, input, opts)
+}
+
+/// `NOSJ.reformat_file_native(path, opts)`: the pipe over a read-only
+/// memory map; the input document never becomes a Ruby String.
+pub fn reformat_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<RString, Error> {
+    let p = path.to_string()?;
+    // Mapping a zero-length file fails with EINVAL on Linux; route an
+    // empty file to the parser's own "unexpected end of input" so the
+    // error class is deterministic across platforms. Metadata failures
+    // fall through for the mapper's Errno.
+    if std::fs::metadata(&p).is_ok_and(|m| m.len() == 0) {
+        return reformat_over(ruby, &[], opts);
+    }
+    with_mapped_file(ruby, &p, |map| reformat_over(ruby, &map, opts))
+}

data/ext/nosj/src/sink.rs

@@ -22,6 +22,17 @@ pub(crate) enum SinkAbort {
     Overflow,
     BadBigint,
     TooDeep,
+    /// The reformat pipe met a WTF-8 (lone-surrogate) object KEY,
+    /// which the Writer has no pre-serialized escape hatch for; gem
+    /// parity is the GeneratorError `generate` raises on the
+    /// equivalent broken-coderange string. (String VALUES re-escape
+    /// as \uXXXX instead.)
+    BrokenUtf8Output,
+    /// The reformat pipe met a non-finite float without `allow_nan`.
+    /// Parsing accepts huge-exponent literals like 1e999 as Infinity
+    /// even in strict mode (gem parity), but generation refuses them;
+    /// carries the JSON spelling for the gem-exact error message.
+    NonFiniteFloat(&'static str),
 }
 
 /// `RB_INT2FIX` ported from Ruby's public inline headers: fixnums are

data/ext/nosj/src/stats.rs

@@ -0,0 +1,301 @@
+//! `NOSJ.stats` / `NOSJ.stats_file`: one full-parser pass into a
+//! counting sink (the `NOSJ.valid?` machinery with counters instead of
+//! discards), answering "what is this 40 MB blob" without building a
+//! single Ruby value for the document. Only the small result Hash is
+//! allocated, after the pass.
+
+use ahash::AHashMap;
+use magnus::value::ReprValue;
+use magnus::{Error, RHash, RString, Ruby, Value};
+
+use crate::errors::{nesting_error, parser_error, parser_error_at};
+use crate::files::with_mapped_file;
+use crate::parse::{parse_native_opts, utf8_input};
+use crate::sink::SinkAbort;
+use crate::state::PULL_STATE;
+
+/// What the document's root value was; reported as a Symbol. The
+/// Default is never observable (a successful pass always saw a root
+/// event); it only makes the sink derivable.
+#[derive(Clone, Copy, Default)]
+enum RootKind {
+    Object,
+    Array,
+    String,
+    Integer,
+    Float,
+    Boolean,
+    #[default]
+    Null,
+}
+
+impl RootKind {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Object => "object",
+            Self::Array => "array",
+            Self::String => "string",
+            Self::Integer => "integer",
+            Self::Float => "float",
+            Self::Boolean => "boolean",
+            Self::Null => "null",
+        }
+    }
+}
+
+/// Counting sink: every event increments counters; nothing else is
+/// retained except the key histogram (memory proportional to the
+/// number of UNIQUE keys, unbounded by design: this is a diagnostic
+/// pass over a document the caller already holds).
+#[derive(Default)]
+struct StatsSink {
+    depth: usize,
+    max_nesting: usize,
+    root: Option<RootKind>,
+    max_depth: usize,
+    objects: u64,
+    arrays: u64,
+    max_object_entries: u64,
+    max_array_length: u64,
+    nulls: u64,
+    booleans: u64,
+    integers: u64,
+    floats: u64,
+    strings: u64,
+    string_bytes: u64,
+    max_string_bytes: u64,
+    keys: u64,
+    histogram: AHashMap<Box<str>, u64>,
+}
+
+impl StatsSink {
+    fn root(&mut self, kind: RootKind) {
+        if self.root.is_none() {
+            self.root = Some(kind);
+        }
+    }
+
+    fn string(&mut self, bytes: u64) {
+        self.strings += 1;
+        self.string_bytes += bytes;
+        self.max_string_bytes = self.max_string_bytes.max(bytes);
+    }
+
+    fn count_key(&mut self, key: &str) {
+        self.keys += 1;
+        *self.histogram.entry(Box::from(key)).or_insert(0) += 1;
+    }
+
+    fn enter_container(&mut self, kind: RootKind) -> Result<(), SinkAbort> {
+        self.root(kind);
+        self.depth += 1;
+        if self.depth > self.max_nesting {
+            return Err(SinkAbort::TooDeep);
+        }
+        self.max_depth = self.max_depth.max(self.depth);
+        Ok(())
+    }
+
+    fn total_values(&self) -> u64 {
+        self.objects
+            + self.arrays
+            + self.strings
+            + self.integers
+            + self.floats
+            + self.booleans
+            + self.nulls
+    }
+}
+
+impl nosj::Sink for StatsSink {
+    type Error = SinkAbort;
+
+    fn null(&mut self) -> Result<(), SinkAbort> {
+        self.root(RootKind::Null);
+        self.nulls += 1;
+        Ok(())
+    }
+
+    fn boolean(&mut self, _: bool) -> Result<(), SinkAbort> {
+        self.root(RootKind::Boolean);
+        self.booleans += 1;
+        Ok(())
+    }
+
+    fn int(&mut self, _: i64) -> Result<(), SinkAbort> {
+        self.root(RootKind::Integer);
+        self.integers += 1;
+        Ok(())
+    }
+
+    fn float(&mut self, _: f64) -> Result<(), SinkAbort> {
+        self.root(RootKind::Float);
+        self.floats += 1;
+        Ok(())
+    }
+
+    fn big_int(&mut self, _: &str) -> Result<(), SinkAbort> {
+        self.root(RootKind::Integer);
+        self.integers += 1;
+        Ok(())
+    }
+
+    fn str(&mut self, value: &str) -> Result<(), SinkAbort> {
+        self.root(RootKind::String);
+        self.string(value.len() as u64);
+        Ok(())
+    }
+
+    fn str_bytes(&mut self, value: &[u8]) -> Result<(), SinkAbort> {
+        self.root(RootKind::String);
+        self.string(value.len() as u64);
+        Ok(())
+    }
+
+    fn key(&mut self, key: &str) -> Result<(), SinkAbort> {
+        self.count_key(key);
+        Ok(())
+    }
+
+    fn key_bytes(&mut self, key: &[u8]) -> Result<(), SinkAbort> {
+        // Broken-WTF-8 keys are pathological; a lossy conversion keeps
+        // the histogram total consistent with the key count.
+        self.count_key(&String::from_utf8_lossy(key));
+        Ok(())
+    }
+
+    fn begin_array(&mut self) -> Result<(), SinkAbort> {
+        self.enter_container(RootKind::Array)
+    }
+
+    fn begin_object(&mut self) -> Result<(), SinkAbort> {
+        self.enter_container(RootKind::Object)
+    }
+
+    fn mark(&self) -> usize {
+        0
+    }
+
+    fn end_array(&mut self, _: usize, len: usize) -> Result<(), SinkAbort> {
+        self.depth -= 1;
+        self.arrays += 1;
+        self.max_array_length = self.max_array_length.max(len as u64);
+        Ok(())
+    }
+
+    fn end_object(&mut self, _: usize, pairs: usize) -> Result<(), SinkAbort> {
+        self.depth -= 1;
+        self.objects += 1;
+        self.max_object_entries = self.max_object_entries.max(pairs as u64);
+        Ok(())
+    }
+}
+
+/// Assemble the result Hash. Sub-hashes group related counters; the
+/// histogram is sorted by count (descending), ties by key, so
+/// `.first(10)` reads as a top-10.
+fn stats_to_hash(ruby: &Ruby, s: &StatsSink, byte_size: usize) -> Result<Value, Error> {
+    let set = |h: &RHash, name: &str, v: u64| h.aset(ruby.to_symbol(name), v);
+
+    let values = ruby.hash_new();
+    set(&values, "total", s.total_values())?;
+    set(&values, "objects", s.objects)?;
+    set(&values, "arrays", s.arrays)?;
+    set(&values, "strings", s.strings)?;
+    set(&values, "integers", s.integers)?;
+    set(&values, "floats", s.floats)?;
+    set(&values, "booleans", s.booleans)?;
+    set(&values, "nulls", s.nulls)?;
+
+    let keys = ruby.hash_new();
+    set(&keys, "total", s.keys)?;
+    set(&keys, "unique", s.histogram.len() as u64)?;
+
+    let containers = ruby.hash_new();
+    set(&containers, "max_object_entries", s.max_object_entries)?;
+    set(&containers, "max_array_length", s.max_array_length)?;
+
+    let strings = ruby.hash_new();
+    set(&strings, "bytes", s.string_bytes)?;
+    set(&strings, "max_bytes", s.max_string_bytes)?;
+
+    let mut sorted: Vec<(&str, u64)> = s.histogram.iter().map(|(k, &n)| (&**k, n)).collect();
+    sorted.sort_unstable_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
+    let histogram = ruby.hash_new_capa(sorted.len());
+    for (key, count) in sorted {
+        histogram.aset(ruby.str_new(key), count)?;
+    }
+
+    let out = ruby.hash_new();
+    set(&out, "byte_size", byte_size as u64)?;
+    let root = s.root.unwrap_or_default();
+    out.aset(ruby.to_symbol("root"), ruby.to_symbol(root.name()))?;
+    set(&out, "max_depth", s.max_depth as u64)?;
+    out.aset(ruby.to_symbol("values"), values)?;
+    out.aset(ruby.to_symbol("keys"), keys)?;
+    out.aset(ruby.to_symbol("key_histogram"), histogram)?;
+    out.aset(ruby.to_symbol("containers"), containers)?;
+    out.aset(ruby.to_symbol("strings"), strings)?;
+    Ok(out.as_value())
+}
+
+/// Run the counting pass over already-UTF-8-vouched bytes and build
+/// the result. `max_nesting` here defaults to UNLIMITED (a deep blob
+/// is exactly what a diagnostic should describe, not refuse), unless
+/// the caller passes the option explicitly.
+fn stats_over(ruby: &Ruby, input: &[u8], opts: Value) -> Result<Value, Error> {
+    let o = parse_native_opts(ruby, opts)?;
+    let nesting_given =
+        RHash::from_value(opts).is_some_and(|h| h.get(ruby.to_symbol("max_nesting")).is_some());
+
+    let mut sink = StatsSink {
+        max_nesting: if nesting_given {
+            o.max_nesting
+        } else {
+            usize::MAX
+        },
+        ..StatsSink::default()
+    };
+    let result = PULL_STATE.with(|cell| {
+        let mut state = cell.borrow_mut();
+        // Safety: callers verified UTF-8 (coderange or a full scan).
+        unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, o.popts) }
+    });
+    match result {
+        Ok(()) => stats_to_hash(ruby, &sink, input.len()),
+        Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
+            ruby,
+            format!("nesting of {} is too deep", o.max_nesting.saturating_add(1)),
+        )),
+        // The other aborts cannot happen (this sink never raises them),
+        // but the match must be total.
+        Err(nosj::DriveError::Sink(_)) => Err(parser_error(ruby, "stats pass aborted".into())),
+        Err(nosj::DriveError::Parse(e)) => {
+            Err(parser_error_at(ruby, input, e.offset, e.to_string()))
+        }
+    }
+}
+
+/// `NOSJ.stats(source, opts)`: document statistics from one null-sink
+/// parser pass.
+pub fn stats_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    data: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let input = utf8_input(ruby, &data)?;
+    stats_over(ruby, input, opts)
+}
+
+/// `NOSJ.stats_file(path, opts)`: `NOSJ.stats` against a memory-mapped
+/// file; `byte_size` is the file's size.
+pub fn stats_file_native(
+    ruby: &Ruby,
+    _rb_self: Value,
+    path: RString,
+    opts: Value,
+) -> Result<Value, Error> {
+    let p = path.to_string()?;
+    with_mapped_file(ruby, &p, |map| stats_over(ruby, &map, opts))
+}