nosj 0.4.0 → 0.5.0

data/ext/nosj/src/reformat.rs

@@ -5,34 +5,32 @@
 //! SIMD escape kernels on the way out there is nothing else.
 //!
 //! Output is exactly what `NOSJ.generate(NOSJ.parse(json), opts)`
-//! would produce, with two deliberate differences: duplicate object
-//! keys pass through (a reformatter must not silently drop data the
-//! way parse's last-key-wins materialization does), and lone-surrogate
-//! string values re-escape as `\uXXXX` instead of raising (the output
-//! must reparse; raw WTF-8 would not). Numbers come out in the gem's
-//! canonical spelling (`1.50` becomes `1.5`), and string escapes are
-//! normalized by the emission kernels.
+//! would produce, and the pipe accepts exactly what parse accepts:
+//! duplicate keys raise unless `allow_duplicate_key` (then they pass
+//! through: a reformatter must not silently drop data the way parse's
+//! last-key-wins materialization does), and lone surrogates raise.
+//! Numbers come out in the gem's canonical spelling (`1.50` becomes
+//! `1.5`), and string escapes are normalized by the emission kernels.
 
-use std::cell::RefCell;
+use std::cell::Cell;
 
-use magnus::value::ReprValue;
 use magnus::{Error, RString, Ruby, Value};
-use nosj::emit::EscapeMode;
 use nosj::{FloatFormat, WriteOptions, Writer};
 
-use crate::errors::{nesting_error, nosj_exception, parser_error, parser_error_at};
 use crate::files::with_mapped_file;
-use crate::gen::opts::{parse_gen_opts, GenConfig, DEFAULT_CONFIG};
-use crate::parse::{parse_native_opts, utf8_input};
+use crate::gen::opts::{read_gen_opts, GenConfig, DEFAULT_CONFIG};
+use crate::opt_reader::OptReader;
+use crate::parse::{
+    drive_error, drive_hashless, options_hash, read_parse_opts, utf8_input, ParseNativeOpts,
+};
 use crate::patch::finish_string;
-use crate::sink::SinkAbort;
-use crate::state::PULL_STATE;
+use crate::sink::{DupKeys, SinkAbort};
+use crate::state::{with_pull_state, with_taken};
 
 thread_local! {
-    /// Pooled output buffer: capacity survives across calls. The pipe
-    /// never calls back into Ruby, so the borrow spans the whole drive
-    /// without any reentrancy concern.
-    static PIPE_BUF: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
+    /// Pooled output buffer: capacity survives across calls (see
+    /// `state::with_taken`).
+    static PIPE_BUF: Cell<Vec<u8>> = const { Cell::new(Vec::new()) };
 }
 
 /// The event-to-Writer pipe. Structure events forward to the Writer's
@@ -42,61 +40,10 @@ struct PipeSink<'a> {
     w: Writer<'a>,
     depth: usize,
     max_nesting: usize,
-    /// For re-escaping WTF-8 string content (see [`quote_wtf8`]).
-    mode: EscapeMode,
     /// Non-finite floats pass through as literals only when the
     /// generate side allows them; see [`PipeSink::float`].
     allow_nan: bool,
-}
-
-const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef";
-
-/// WTF-8 lone surrogates arrive as one 3-byte sequence with this lead
-/// byte (the only ill-formed runs the parser ever emits).
-const WTF8_SURROGATE_LEAD: u8 = 0xED;
-const WTF8_SURROGATE_LEN: usize = 3;
-/// Payload bits of a UTF-8 lead / continuation byte.
-const UTF8_LEAD3_BITS: u32 = 0x0F;
-const UTF8_CONT_BITS: u32 = 0x3F;
-
-/// Quote and escape WTF-8 content: valid UTF-8 runs go through the
-/// configured escape kernel, and lone-surrogate sequences re-escape as
-/// `\uXXXX`, so the output reparses to the identical string in every
-/// mode (raw WTF-8 bytes would not: the parser requires UTF-8 input).
-/// This deliberately diverges from `generate`, which refuses
-/// broken-coderange strings: a reformatter must accept everything the
-/// parser accepts.
-fn quote_wtf8(out: &mut Vec<u8>, bytes: &[u8], mode: EscapeMode) {
-    out.push(b'"');
-    let mut rest = bytes;
-    loop {
-        match std::str::from_utf8(rest) {
-            Ok(s) => {
-                nosj::emit::escape_into(out, s.as_bytes(), mode);
-                break;
-            }
-            Err(e) => {
-                let valid = e.valid_up_to();
-                nosj::emit::escape_into(out, &rest[..valid], mode);
-                let sur = &rest[valid..];
-                debug_assert!(
-                    sur.len() >= WTF8_SURROGATE_LEN && sur[0] == WTF8_SURROGATE_LEAD,
-                    "parser only emits lone-surrogate WTF-8"
-                );
-                // Standard 3-byte UTF-8 decode of the surrogate
-                // codepoint (U+D800..U+DFFF), re-emitted as \uXXXX.
-                let cp = ((u32::from(sur[0]) & UTF8_LEAD3_BITS) << 12)
-                    | ((u32::from(sur[1]) & UTF8_CONT_BITS) << 6)
-                    | (u32::from(sur[2]) & UTF8_CONT_BITS);
-                out.extend_from_slice(b"\\u");
-                for shift in [12, 8, 4, 0] {
-                    out.push(HEX_DIGITS[((cp >> shift) & 0xF) as usize]);
-                }
-                rest = &sur[WTF8_SURROGATE_LEN..];
-            }
-        }
-    }
-    out.push(b'"');
+    dup_keys: DupKeys<'a>,
 }
 
 impl PipeSink<'_> {
@@ -166,25 +113,18 @@ impl nosj::Sink for PipeSink<'_> {
         Ok(())
     }
 
-    fn str_bytes(&mut self, value: &[u8]) -> Result<(), SinkAbort> {
-        // Rare path (lone-surrogate content); a per-call buffer is fine.
-        let mut quoted = Vec::with_capacity(value.len() + 8);
-        quote_wtf8(&mut quoted, value, self.mode);
-        self.w.value_raw(&quoted);
-        Ok(())
+    fn str_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
+        Err(SinkAbort::LoneSurrogate)
     }
 
     fn key(&mut self, key: &str) -> Result<(), SinkAbort> {
+        self.dup_keys.key(key.as_bytes());
         self.w.key(key);
         Ok(())
     }
 
-    fn key_bytes(&mut self, _key: &[u8]) -> Result<(), SinkAbort> {
-        // A lone-surrogate KEY has no pre-serialized escape hatch in
-        // the Writer (values have value_raw; a key_raw is on the crate
-        // wishlist), so this pathological case keeps generate's
-        // refusal semantics.
-        Err(SinkAbort::BrokenUtf8Output)
+    fn key_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
+        Err(SinkAbort::LoneSurrogate)
     }
 
     fn begin_array(&mut self) -> Result<(), SinkAbort> {
@@ -200,7 +140,7 @@ impl nosj::Sink for PipeSink<'_> {
     }
 
     fn mark(&self) -> usize {
-        0
+        self.dup_keys.mark()
     }
 
     fn end_array(&mut self, _: usize, _: usize) -> Result<(), SinkAbort> {
@@ -209,8 +149,9 @@ impl nosj::Sink for PipeSink<'_> {
         Ok(())
     }
 
-    fn end_object(&mut self, _: usize, _: usize) -> Result<(), SinkAbort> {
+    fn end_object(&mut self, mark: usize, _: usize) -> Result<(), SinkAbort> {
         self.depth -= 1;
+        self.dup_keys.close(mark)?;
         self.w.end_object();
         Ok(())
     }
@@ -230,61 +171,61 @@ fn write_options(cfg: &GenConfig) -> WriteOptions {
     w
 }
 
+/// Decoded reformat options: parse acceptance plus generate formatting.
+/// Decoded before the source is borrowed, since decoding can run Ruby
+/// (an option value's `to_int`; see `parse::utf8_input`).
+struct ReformatOpts {
+    parse: ParseNativeOpts,
+    generate: Option<GenConfig>,
+}
+
+impl ReformatOpts {
+    /// One reader over both option sets, so a key either reads is known.
+    fn decode(ruby: &Ruby, opts: Value) -> Result<Self, Error> {
+        let Some(h) = options_hash(ruby, opts)? else {
+            return Ok(Self {
+                parse: ParseNativeOpts::default(),
+                generate: None,
+            });
+        };
+        let mut reader = OptReader::new(ruby, h);
+        let parse = read_parse_opts(&mut reader)?;
+        let (generate, _) = read_gen_opts(&mut reader)?;
+        reader.finish()?;
+        Ok(Self {
+            parse,
+            generate: Some(generate),
+        })
+    }
+}
+
 /// Run the pipe over already-UTF-8-vouched bytes.
-fn reformat_over(ruby: &Ruby, input: &[u8], opts: Value) -> Result<RString, Error> {
-    let po = parse_native_opts(ruby, opts)?;
-    let built;
-    let gcfg: &GenConfig = if opts.is_nil() {
-        &DEFAULT_CONFIG
-    } else {
-        built = parse_gen_opts(ruby, opts)?.0;
-        &built
-    };
+fn reformat_over(ruby: &Ruby, input: &[u8], opts: &ReformatOpts) -> Result<RString, Error> {
+    let po = &opts.parse;
+    let gcfg = opts.generate.as_ref().unwrap_or(&DEFAULT_CONFIG);
     let wopts = write_options(gcfg);
 
-    PIPE_BUF.with(|cell| {
-        let mut buf = cell.borrow_mut();
-        buf.clear();
-        // The output is at least input-sized for minify-shaped runs.
-        buf.reserve(input.len());
-        let mut sink = PipeSink {
-            w: Writer::new(&mut buf, &wopts),
-            depth: 0,
-            max_nesting: po.max_nesting,
-            mode: gcfg.mode,
-            allow_nan: gcfg.allow_nan,
-        };
-        let result = PULL_STATE.with(|state_cell| {
-            let mut state = state_cell.borrow_mut();
-            // Safety: callers verified UTF-8 (coderange or full scan).
-            unsafe { nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, po.popts) }
-        });
-        match result {
-            Ok(()) => finish_string(&buf),
-            Err(nosj::DriveError::Sink(SinkAbort::TooDeep)) => Err(nesting_error(
-                ruby,
-                format!(
-                    "nesting of {} is too deep",
-                    po.max_nesting.saturating_add(1)
-                ),
-            )),
-            Err(nosj::DriveError::Sink(SinkAbort::BrokenUtf8Output)) => Err(Error::new(
-                // Gem parity: generate raises GeneratorError for a
-                // string ascii_only cannot represent.
-                nosj_exception(ruby, "GeneratorError"),
-                "source sequence is illegal/malformed utf-8",
-            )),
-            Err(nosj::DriveError::Sink(SinkAbort::NonFiniteFloat(spelling))) => Err(Error::new(
-                nosj_exception(ruby, "GeneratorError"),
-                format!("{spelling} not allowed in JSON"),
-            )),
-            Err(nosj::DriveError::Sink(_)) => {
-                Err(parser_error(ruby, "reformat pass aborted".into()))
-            }
-            Err(nosj::DriveError::Parse(e)) => {
-                Err(parser_error_at(ruby, input, e.offset, e.to_string()))
-            }
-        }
+    with_taken(&PIPE_BUF, |buf| {
+        drive_hashless(input, po, |check_dups| {
+            buf.clear();
+            // The output is at least input-sized for minify-shaped runs.
+            buf.reserve(input.len());
+            with_pull_state(|state| {
+                let mut sink = PipeSink {
+                    w: Writer::new(buf, &wopts),
+                    depth: 0,
+                    max_nesting: po.max_nesting,
+                    allow_nan: gcfg.allow_nan,
+                    dup_keys: DupKeys::new(&mut state.dup, check_dups),
+                };
+                // Safety: callers verified UTF-8 (coderange or full scan).
+                unsafe {
+                    nosj::parse_utf8_unchecked_with(input, &mut state.bufs, &mut sink, po.popts)
+                }
+            })
+        })
+        .map_err(|failure| drive_error(ruby, failure, po, input, (0, input.len())))?;
+        finish_string(buf)
     })
 }
 
@@ -296,8 +237,9 @@ pub fn reformat_native(
     data: RString,
     opts: Value,
 ) -> Result<RString, Error> {
+    let opts = ReformatOpts::decode(ruby, opts)?;
     let input = utf8_input(ruby, &data)?;
-    reformat_over(ruby, input, opts)
+    reformat_over(ruby, input, &opts)
 }
 
 /// `NOSJ.reformat_file_native(path, opts)`: the pipe over a read-only
@@ -309,12 +251,13 @@ pub fn reformat_file_native(
     opts: Value,
 ) -> Result<RString, Error> {
     let p = path.to_string()?;
+    let opts = ReformatOpts::decode(ruby, opts)?;
     // Mapping a zero-length file fails with EINVAL on Linux; route an
     // empty file to the parser's own "unexpected end of input" so the
     // error class is deterministic across platforms. Metadata failures
     // fall through for the mapper's Errno.
     if std::fs::metadata(&p).is_ok_and(|m| m.len() == 0) {
-        return reformat_over(ruby, &[], opts);
+        return reformat_over(ruby, &[], &opts);
     }
-    with_mapped_file(ruby, &p, |map| reformat_over(ruby, &map, opts))
+    with_mapped_file(ruby, &p, |map| reformat_over(ruby, &map, &opts))
 }

data/ext/nosj/src/sink.rs

@@ -1,7 +1,8 @@
 //! The nosj sinks: `RubyValueSink` builds Ruby VALUEs directly during
 //! the parse (with interned-key caches and gem-compatible option
 //! handling); `NullSink` powers `NOSJ.valid?` by discarding every
-//! event. Raw VALUE construction helpers live here too.
+//! event; `DupKeys` gives hash-less sinks duplicate-key detection. Raw
+//! VALUE construction helpers live here too.
 
 use ahash::AHashMap;
 
@@ -17,17 +18,20 @@ pub(crate) const MAX_NESTING: usize = 100;
 const KEY_CACHE_CAP: usize = 2048;
 
 /// Why a sink stopped the drive; mapped onto the gem's exceptions in
-/// [`crate::parse::finish_drive`].
+/// [`crate::parse::drive_error`]. Sinks see no offsets, so the two
+/// document refusals get their position from a cold-path re-walk
+/// (`crate::locate`).
 pub(crate) enum SinkAbort {
     Overflow,
     BadBigint,
     TooDeep,
-    /// The reformat pipe met a WTF-8 (lone-surrogate) object KEY,
-    /// which the Writer has no pre-serialized escape hatch for; gem
-    /// parity is the GeneratorError `generate` raises on the
-    /// equivalent broken-coderange string. (String VALUES re-escape
-    /// as \uXXXX instead.)
-    BrokenUtf8Output,
+    /// An object repeats a key and `allow_duplicate_key` is off (json
+    /// 3 semantics). From `DupKeys` it may be a fingerprint collision,
+    /// which the exact cold-path check rules out.
+    DuplicateKey,
+    /// A string or key decodes to a lone UTF-16 surrogate (json 3
+    /// rejects trailing ones too, not only leading ones).
+    LoneSurrogate,
     /// The reformat pipe met a non-finite float without `allow_nan`.
     /// Parsing accepts huge-exponent literals like 1e999 as Infinity
     /// even in strict mode (gem parity), but generation refuses them;
@@ -35,24 +39,160 @@ pub(crate) enum SinkAbort {
     NonFiniteFloat(&'static str),
 }
 
-/// `RB_INT2FIX` ported from Ruby's public inline headers: fixnums are
-/// `(i << 1) + 1` for `|i| <= LONG_MAX / 2`, and the range is defined by
-/// the C `long`, which is 32-bit on Windows (LLP64): fixnums there hold
-/// only 31 bits, and tagging anything wider crashes Ruby with
-/// "Unnormalized Fixnum value". C extensions get the check inlined by the
-/// header; going through the extern `rb_ll2inum` costs an FFI call per
-/// integer. Non-fixable values still take the call.
-// The widening is an identity on LP64 hosts (clippy flags it there) but
-// required on Windows, where c_long is 32-bit.
-#[allow(clippy::unnecessary_cast)]
+/// Duplicate-key detection for sinks that build no Hash (validation and
+/// the reformat pipe): each key is remembered as a 64-bit fingerprint,
+/// `mark()` at a container's start is the fingerprint count, and an
+/// object's close checks only its own keys. Integers, because comparing
+/// them is what keeps this cheap: comparing the key bytes themselves
+/// measured up to 93% slower (twitter's 40-key objects). A fingerprint
+/// collision reads as a duplicate, so callers confirm a hit exactly on
+/// the cold path; a collision can only cost time.
+///
+/// Measured split (twitter `valid?`): hashing and pushing each key is
+/// ~0-2%; the close-time check is the cost, hence [`SeenTable`].
+pub(crate) struct DupKeys<'a> {
+    scratch: &'a mut DupScratch,
+    enabled: bool,
+}
+
+/// [`DupKeys`]' pooled buffers (per thread, in `PullState`): the key
+/// fingerprints of every open object, and the close-time set.
+#[derive(Default)]
+pub(crate) struct DupScratch {
+    fingerprints: Vec<u64>,
+    table: SeenTable,
+}
+
+/// Fixed seeds: fingerprints only need to spread keys, and a collision
+/// merely sends the document to the exact cold-path check. (A leaner
+/// hand-rolled hash measured no faster: hashing is not the cost.)
+const FINGERPRINT: ahash::RandomState = ahash::RandomState::with_seeds(
+    0x243f_6a88_85a3_08d3,
+    0x1319_8a2e_0370_7344,
+    0xa409_3822_299f_31d0,
+    0x082e_fa98_ec4e_6c89,
+);
+
+/// Objects up to this many keys compare pairwise (a few compares beat
+/// any table traffic).
+const PAIRWISE_MAX: usize = 8;
+/// Table slots; a power of two, so a fingerprint's low bits index it.
+const TABLE_SLOTS: usize = 256;
+/// Objects up to this many keys use the table, which then stays at
+/// least half empty; larger ones sort.
+const TABLE_MAX_KEYS: usize = TABLE_SLOTS / 2;
+
+/// Open-addressing set for one object's close: fingerprints are already
+/// uniformly mixed, so their low bits index it directly, and one probe
+/// per key usually decides. Each close claims a fresh epoch, and a slot
+/// is empty unless it carries the current one, so nothing is cleared
+/// between objects.
+struct SeenTable {
+    slots: Box<[(u64, u32)]>,
+    epoch: u32,
+}
+
+impl Default for SeenTable {
+    fn default() -> Self {
+        SeenTable {
+            slots: vec![(0, 0); TABLE_SLOTS].into_boxed_slice(),
+            epoch: 0,
+        }
+    }
+}
+
+impl SeenTable {
+    /// Whether `keys` repeats a fingerprint. Callers keep
+    /// `keys.len() <= TABLE_MAX_KEYS`, so a free slot always exists.
+    #[inline(always)]
+    fn any_repeat(&mut self, keys: &[u64]) -> bool {
+        const MASK: usize = TABLE_SLOTS - 1;
+        self.epoch = self.epoch.wrapping_add(1);
+        if self.epoch == 0 {
+            // Epoch 0 marks an empty slot; after a full wrap, really clear.
+            self.slots.fill((0, 0));
+            self.epoch = 1;
+        }
+        let epoch = self.epoch;
+        for &fp in keys {
+            let mut at = fp as usize & MASK;
+            loop {
+                let slot = &mut self.slots[at];
+                if slot.1 != epoch {
+                    *slot = (fp, epoch);
+                    break;
+                }
+                if slot.0 == fp {
+                    return true;
+                }
+                at = (at + 1) & MASK;
+            }
+        }
+        false
+    }
+}
+
+impl<'a> DupKeys<'a> {
+    pub(crate) fn new(scratch: &'a mut DupScratch, enabled: bool) -> Self {
+        scratch.fingerprints.clear();
+        DupKeys { scratch, enabled }
+    }
+
+    #[inline(always)]
+    pub(crate) fn mark(&self) -> usize {
+        self.scratch.fingerprints.len()
+    }
+
+    #[inline(always)]
+    pub(crate) fn key(&mut self, key: &[u8]) {
+        if self.enabled {
+            self.scratch.fingerprints.push(FINGERPRINT.hash_one(key));
+        }
+    }
+
+    /// Close the object whose keys start at `mark`.
+    #[inline(always)]
+    pub(crate) fn close(&mut self, mark: usize) -> Result<(), SinkAbort> {
+        if !self.enabled {
+            return Ok(());
+        }
+        let DupScratch {
+            fingerprints,
+            table,
+        } = &mut *self.scratch;
+        let keys = &mut fingerprints[mark..];
+        let repeated = match keys.len() {
+            n if n <= PAIRWISE_MAX => (1..n).any(|i| keys[..i].contains(&keys[i])),
+            n if n <= TABLE_MAX_KEYS => table.any_repeat(keys),
+            _ => {
+                keys.sort_unstable();
+                keys.windows(2).any(|w| w[0] == w[1])
+            }
+        };
+        fingerprints.truncate(mark);
+        if repeated {
+            Err(SinkAbort::DuplicateKey)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Integer VALUE for `i` via rb-sys's inline `LONG2NUM` (the header
+/// macro: a Fixnum tagged inline when it fits, else a Bignum), which
+/// saves the FFI call per integer that `rb_ll2inum` costs. The fixable
+/// range is defined by the C `long`, which is 32-bit on Windows (LLP64):
+/// fixnums there hold only 31 bits, and tagging anything wider crashes
+/// Ruby with "Unnormalized Fixnum value"; values beyond `long` take
+/// `rb_ll2inum`.
+// The conversion is an identity on LP64 hosts (clippy flags it there)
+// but narrows on Windows, where c_long is 32-bit.
+#[allow(clippy::useless_conversion, clippy::unnecessary_fallible_conversions)]
 #[inline(always)]
 fn int_to_raw(i: i64) -> rb_sys::VALUE {
-    const FIXABLE_MIN: i64 = (std::os::raw::c_long::MIN / 2) as i64;
-    const FIXABLE_MAX: i64 = (std::os::raw::c_long::MAX / 2) as i64;
-    if (FIXABLE_MIN..=FIXABLE_MAX).contains(&i) {
-        ((i as u64) << 1).wrapping_add(1) as rb_sys::VALUE
-    } else {
-        unsafe { rb_sys::rb_ll2inum(i) }
+    match std::os::raw::c_long::try_from(i) {
+        Ok(l) => rb_sys::macros::LONG2NUM(l),
+        Err(_) => unsafe { rb_sys::rb_ll2inum(i) },
     }
 }
 
@@ -146,6 +286,7 @@ pub(crate) struct RubyValueSink<'a> {
     pub(crate) symbolize: bool,
     pub(crate) freeze: bool,
     pub(crate) max_nesting: usize,
+    pub(crate) allow_duplicate_key: bool,
 }
 
 // Tried and rejected (2026-07-10): a jiter-style cache of repeated VALUE
@@ -228,35 +369,14 @@ impl nosj::Sink for RubyValueSink<'_> {
         self.push_raw(raw)
     }
 
-    /// Lone-low-surrogate content: gem parity is a UTF-8-encoded Ruby string
-    /// carrying the raw WTF-8 bytes (broken coderange, like the gem's).
-    #[inline(always)]
-    fn str_bytes(&mut self, value: &[u8]) -> Result<(), SinkAbort> {
-        let raw = unsafe {
-            let s = rb_sys::rb_utf8_str_new(
-                value.as_ptr() as *const std::os::raw::c_char,
-                value.len() as std::os::raw::c_long,
-            );
-            if self.freeze {
-                rb_sys::rb_str_freeze(s)
-            } else {
-                s
-            }
-        };
-        self.push_raw(raw)
+    /// Lone-surrogate content (the crate hands it over as WTF-8): json 3
+    /// rejects it, trailing surrogates included.
+    fn str_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
+        Err(SinkAbort::LoneSurrogate)
     }
 
-    #[inline(always)]
-    fn key_bytes(&mut self, key: &[u8]) -> Result<(), SinkAbort> {
-        // Interning is skipped: these keys are pathological, not hot.
-        let raw = unsafe {
-            rb_sys::rb_utf8_str_new(
-                key.as_ptr() as *const std::os::raw::c_char,
-                key.len() as std::os::raw::c_long,
-            )
-        };
-        let frozen = unsafe { rb_sys::rb_str_freeze(raw) };
-        self.push_raw(frozen)
+    fn key_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
+        Err(SinkAbort::LoneSurrogate)
     }
 
     #[inline(always)]
@@ -314,19 +434,28 @@ impl nosj::Sink for RubyValueSink<'_> {
             }
         }
         self.stack.truncate(mark);
+        // A repeated key collapses into one entry: the hash comes out
+        // smaller than the pair count (one size read per object).
+        if !self.allow_duplicate_key
+            && (unsafe { rb_sys::macros::RHASH_SIZE(hash_raw) } as usize) < pairs
+        {
+            return Err(SinkAbort::DuplicateKey);
+        }
         self.push_raw(hash_raw)
     }
 }
 
 /// Validation-only sink: every event is a no-op except nesting-depth
-/// tracking, so `NOSJ.valid?` runs the full parser (tokenizers,
-/// string decode, number validation) without allocating a single VALUE.
-pub(crate) struct NullSink {
+/// tracking and duplicate-key fingerprints, so `NOSJ.valid?` runs the
+/// full parser (tokenizers, string decode, number validation) without
+/// allocating a single VALUE.
+pub(crate) struct NullSink<'a> {
     pub(crate) depth: usize,
     pub(crate) max_nesting: usize,
+    pub(crate) dup_keys: DupKeys<'a>,
 }
 
-impl nosj::Sink for NullSink {
+impl nosj::Sink for NullSink<'_> {
     type Error = SinkAbort;
 
     fn null(&mut self) -> Result<(), SinkAbort> {
@@ -347,14 +476,18 @@ impl nosj::Sink for NullSink {
     fn str(&mut self, _: &str) -> Result<(), SinkAbort> {
         Ok(())
     }
-    fn key(&mut self, _: &str) -> Result<(), SinkAbort> {
+    fn key(&mut self, key: &str) -> Result<(), SinkAbort> {
+        self.dup_keys.key(key.as_bytes());
         Ok(())
     }
     fn str_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
-        Ok(())
+        Err(SinkAbort::LoneSurrogate)
+    }
+    fn key_bytes(&mut self, _: &[u8]) -> Result<(), SinkAbort> {
+        Err(SinkAbort::LoneSurrogate)
     }
     fn mark(&self) -> usize {
-        0
+        self.dup_keys.mark()
     }
     fn begin_array(&mut self) -> Result<(), SinkAbort> {
         self.depth += 1;
@@ -374,8 +507,8 @@ impl nosj::Sink for NullSink {
         self.depth -= 1;
         Ok(())
     }
-    fn end_object(&mut self, _: usize, _: usize) -> Result<(), SinkAbort> {
+    fn end_object(&mut self, mark: usize, _: usize) -> Result<(), SinkAbort> {
         self.depth -= 1;
-        Ok(())
+        self.dup_keys.close(mark)
     }
 }