smarter_json 0.9.2 → 0.9.9

data/ext/smarter_json/smarter_json.c

@@ -4,7 +4,7 @@
 #ifdef __ARM_NEON
 #include <arm_neon.h>
 #endif
-#include "vendor/ryu.h" /* Ryū string->double, correctly rounded (Ulf Adams, Apache-2.0) */
+#include "vendor/eisel_lemire.h" /* Eisel-Lemire decimal->double, correctly rounded (fast_float) */
 
 /* Branch hints / prefetch on the hot scan loops. No-ops on compilers without the
  * builtins (the code is correct either way; these only steer code layout). */
@@ -12,10 +12,12 @@
 #  define FJ_LIKELY(x)   __builtin_expect(!!(x), 1)
 #  define FJ_UNLIKELY(x) __builtin_expect(!!(x), 0)
 #  define FJ_PREFETCH(p) __builtin_prefetch(p)
+#  define FJ_ALWAYS_INLINE inline __attribute__((always_inline))
 #else
 #  define FJ_LIKELY(x)   (x)
 #  define FJ_UNLIKELY(x) (x)
 #  define FJ_PREFETCH(p) ((void)0)
+#  define FJ_ALWAYS_INLINE inline
 #endif
 
 /*
@@ -48,8 +50,7 @@ static ID    fj_name_id;
 static VALUE fj_sym_encoding;
 static VALUE fj_sym_symbolize_keys;
 static VALUE fj_sym_first_wins;
-static VALUE fj_sym_raise;
-static VALUE fj_sym_bigdecimal_load;
+static VALUE fj_sym_decimal_precision;
 static VALUE fj_sym_float;
 static VALUE fj_sym_bigdecimal;
 static VALUE fj_sym_on_warning;
@@ -70,8 +71,7 @@ typedef struct {
   int depth;
   int symbolize_keys;
   int dup_first_wins;
-  int dup_raise;
-  int bigdecimal_load;  /* 0 = float, 1 = auto, 2 = bigdecimal */
+  int decimal_precision;  /* 0 = float, 1 = auto, 2 = bigdecimal */
   fj_kc_slot *kcache;   /* per-parse key cache (NULL when interning unavailable) */
   VALUE on_warning;     /* on_warning: callable invoked per non-fatal lenient fix, else Qnil */
 } fj_state;
@@ -168,20 +168,39 @@ static long fj_mbws(const char *p, long n) {
   return 0;
 }
 
+/* Skip a run of whitespace. This is hot on pretty-printed input, where most of
+ * the bytes are indentation. Indentation is homogeneous — all spaces OR all tabs,
+ * the two common styles — so a run of it is skipped 8 bytes at a time with a
+ * single 64-bit compare (the uniform-byte patterns read the same regardless of
+ * endianness). Everything else — newlines, CR, short/partial runs, and Unicode
+ * whitespace — falls to the tight byte loop, which also avoids the per-byte helper
+ * calls (fj_byte / fj_is_ws / fj_advance) the previous byte-at-a-time version paid.
+ * The set of bytes treated as whitespace is unchanged. */
 static void fj_skip_pure_ws(fj_state *st) {
+  const char *p   = st->buf + st->pos;
+  const char *end = st->buf + st->len;
   for (;;) {
-    int b = fj_byte(st);
-    if (b == -1) break;
-    if (fj_is_ws(b)) {
-      fj_advance(st, 1);
-    } else if (b >= 0x80) {
-      long m = fj_mbws(st->buf + st->pos, st->len - st->pos);
-      if (m == 0) break;
-      st->pos += m;
-    } else {
+    while (end - p >= 8) {
+      uint64_t w;
+      memcpy(&w, p, 8);
+      if (w == 0x2020202020202020ULL || w == 0x0909090909090909ULL) { p += 8; continue; }
       break;
     }
+    if (p >= end) break;
+    {
+      unsigned char b = (unsigned char)*p;
+      if (b == 0x20 || (b >= 0x09 && b <= 0x0D)) {
+        p++;
+      } else if (b >= 0x80) {
+        long m = fj_mbws(p, end - p);
+        if (m == 0) break;
+        p += m;
+      } else {
+        break;
+      }
+    }
   }
+  st->pos = p - st->buf;
 }
 
 /* A comment marker only starts a comment when preceded by whitespace or at the
@@ -228,6 +247,18 @@ static void fj_skip_ws_comments(fj_state *st) {
   }
 }
 
+/* Cheap guard for the hot loop: could the current byte begin whitespace or a
+ * comment marker, so the (otherwise no-op) fj_skip_ws_comments call is actually
+ * needed? Compact data — the next byte is already a structural char or a value
+ * start — answers no, and we elide both the call and its memcpy/lookahead. ASCII
+ * whitespace, '#', '/', and possible multibyte-ws lead bytes (>=0x80) answer yes;
+ * EOF (-1) answers no (the caller's existing -1 checks handle it). Behaviour is
+ * identical to always calling fj_skip_ws_comments — this only skips a known no-op. */
+static inline int fj_needs_ws_skip(int b) {
+  if (b < 0) return 0;
+  return b == 0x20 || (b >= 0x09 && b <= 0x0D) || b == '#' || b == '/' || b >= 0x80;
+}
+
 /* forward declarations (mutual recursion) */
 static VALUE fj_parse_value(fj_state *st);
 static VALUE fj_parse_member_value(fj_state *st);
@@ -471,7 +502,7 @@ static VALUE fj_to_bigdecimal_token(const char *p, long n) {
  * (quoteless path) call these, so the Integer/Float a token produces is identical
  * no matter which path scanned it. [p, n) is the raw token slice (with any sign),
  * needed only by the bignum / strtod fallbacks. */
-static VALUE fj_int_from_parts(uint64_t m, int digits, int neg, int overflow, const char *p, long n) {
+static FJ_ALWAYS_INLINE VALUE fj_int_from_parts(uint64_t m, int digits, int neg, int overflow, const char *p, long n) {
   if (!overflow && digits >= 1 && digits <= 18) {
     int64_t v = (int64_t)m;
     return LL2NUM(neg ? -v : v);
@@ -481,16 +512,87 @@ static VALUE fj_int_from_parts(uint64_t m, int digits, int neg, int overflow, co
   return rb_str_to_inum(fj_strip_underscores(p, n), 10, 0);
 }
 
+/* Convert a >17-digit / subnormal float token to a double. A double resolves ~17
+ * significant decimals; the digits past that affect only the final round-to-nearest-
+ * even, which a single sticky marker ("was any dropped digit nonzero?") captures. So
+ * we keep FJ_FLOAT_ODD_DIGITS significant digits and, if more nonzero digits follow,
+ * force the last kept digit odd (round-to-odd). strtod's round-to-nearest of that
+ * shorter mantissa then equals round-to-nearest of the full value — but strtod grinds
+ * far fewer digits. The kept count is well above 2x double's ~16 significant decimals,
+ * which is what round-to-odd needs to be exact (verified bit-for-bit against
+ * JSON.parse on the high-precision corpus). The token is rebuilt into a NUL-terminated
+ * "<digits>e<exp>" buffer (passing the raw input slice would make rb_cstr_to_dbl treat
+ * the trailing delimiter as garbage and re-run strtod a second time). */
+#define FJ_FLOAT_ODD_DIGITS 40
+static VALUE fj_float_strtod(const char *p, long n) {
+  char    digits[FJ_FLOAT_ODD_DIGITS];
+  char    out[FJ_FLOAT_ODD_DIGITS + 40];
+  long    i = 0, ow = 0, kept = 0, point_pos = 0, lead_frac_zeros = 0;
+  int     neg = 0, after_point = 0, seen_sig = 0, sticky = 0, esign = 0;
+  int64_t expl_exp = 0, x;
+
+  if (i < n && (p[i] == '+' || p[i] == '-')) { neg = (p[i] == '-'); i++; }
+
+  for (; i < n; i++) {
+    char c = p[i];
+    if (c == '_') continue;
+    if (c == '.') { after_point = 1; continue; }
+    if (c == 'e' || c == 'E') { i++; break; }
+    if (!seen_sig && c == '0') { if (after_point) lead_frac_zeros++; continue; }
+    seen_sig = 1;
+    if (!after_point) point_pos++;
+    if (kept < FJ_FLOAT_ODD_DIGITS) digits[kept++] = c;
+    else if (c != '0') sticky = 1;
+  }
+
+  if (i < n && (p[i] == '+' || p[i] == '-')) { esign = (p[i] == '-'); i++; }
+  for (; i < n; i++) {
+    char c = p[i];
+    if (c == '_') continue;
+    if (c < '0' || c > '9') break;
+    expl_exp = expl_exp * 10 + (c - '0');
+  }
+  if (esign) expl_exp = -expl_exp;
+
+  if (kept == 0) return rb_float_new(neg ? -0.0 : 0.0);
+
+  /* round-to-odd: a dropped nonzero tail forces the last kept digit odd. */
+  if (sticky && ((digits[kept - 1] - '0') % 2) == 0) digits[kept - 1]++;
+
+  x = expl_exp + point_pos - lead_frac_zeros - kept;
+  if (neg) out[ow++] = '-';
+  memcpy(out + ow, digits, (size_t)kept);
+  ow += kept;
+  /* Append "e<exp>" by hand. snprintf here showed up as BSD_vfprintf in profiling —
+     a full printf formatter per number is absurdly heavy for one integer. */
+  out[ow++] = 'e';
+  if (x < 0) { out[ow++] = '-'; x = -x; }
+  {
+    char ex[24];
+    int  en = 0;
+    if (x == 0) ex[en++] = '0';
+    else while (x > 0) { ex[en++] = (char)('0' + (int)(x % 10)); x /= 10; }
+    while (en > 0) out[ow++] = ex[--en];
+  }
+  out[ow] = '\0';
+  return rb_float_new(rb_cstr_to_dbl(out, 0));
+}
+
 /* e10 is the final base-10 exponent (already adjusted by the fraction length). */
-static VALUE fj_float_from_parts(uint64_t m10, int m10digits, int64_t e10, int neg, int overflow, const char *p, long n) {
-  /* Ryū fast path: <=17 mantissa digits and not in the subnormal range. */
-  if (!overflow && m10digits >= 1 && m10digits <= 17 && (long)m10digits + e10 >= -307) {
+static FJ_ALWAYS_INLINE VALUE fj_float_from_parts(uint64_t m10, int m10digits, int64_t e10, int neg, int overflow, const char *p, long n) {
+  /* Fast path by mantissa width (our scanner accumulates m10 exactly up to 18
+     digits, flagging overflow beyond):
+       1..18 digits -> Eisel-Lemire, correctly-rounded for any exact uint64 mantissa
+                       (Mushtak-Lemire). This pulls full-double-precision data (e.g.
+                       citylots coordinates, 18 sig digits) off the slow strtod
+                       fallback — the stdlib json gem still strtods it.
+     >18 digits / overflow / extreme exponent -> strtod (round-to-odd). */
+  if (!overflow && m10digits >= 1 && m10digits <= 18 && (long)m10digits + e10 >= -307) {
     if (m10 == 0) return rb_float_new(neg ? -0.0 : 0.0);
-    return rb_float_new(ryu_s2d_from_parts(m10, m10digits, (int32_t)e10, neg != 0));
+    return rb_float_new(fj_eisel_lemire_s2d(e10, m10, neg));
   }
-  /* Fallback for >17 digits / extreme or subnormal exponents. */
-  if (memchr(p, '_', (size_t)n) == NULL) return rb_float_new(rb_cstr_to_dbl(p, 0));
-  return rb_float_new(rb_str_to_dbl(fj_strip_underscores(p, n), 0));
+  /* Fallback for >18 digits / extreme or subnormal exponents. */
+  return fj_float_strtod(p, n);
 }
 
 /* Scan an already-bounded quoteless token [p, p+n) exactly once: validate it as a
@@ -571,8 +673,8 @@ static int fj_try_decimal(fj_state *st, const char *p, long n, VALUE *out) {
   e10 -= frac;
   /* :bigdecimal always; :auto only when significant digits > 16. m10digits is >=
    * the significant-digit count, so m10digits <= 16 skips the fj_sig_digits scan. */
-  if (st->bigdecimal_load == 2 ||
-      (st->bigdecimal_load == 1 && m10digits > 16 && fj_sig_digits(p, n) > 16)) {
+  if (st->decimal_precision == 2 ||
+      (st->decimal_precision == 1 && m10digits > 16 && fj_sig_digits(p, n) > 16)) {
     *out = fj_to_bigdecimal_token(p, n);
   } else {
     *out = fj_float_from_parts(m10, m10digits, e10, neg, overflow, p, n);
@@ -596,7 +698,7 @@ static VALUE fj_parse_number(fj_state *st) {
   long   nlen;
   int    is_float = 0, neg = 0, overflow = 0;
   uint64_t m10 = 0;                 /* mantissa: integer + fraction digits */
-  int    m10digits = 0;             /* mantissa digit chars (caps the Ryū fast path at 17) */
+  int    m10digits = 0;             /* mantissa digit chars (caps the Eisel-Lemire fast path at 18) */
   int    frac = 0;                  /* fraction digit chars: e10 -= frac */
   int64_t e10 = 0;
 
@@ -683,8 +785,8 @@ static VALUE fj_parse_number(fj_state *st) {
    * when significant digits > 16. Since m10digits >= significant digits, m10digits
    * <= 16 guarantees not-BigDecimal and lets us skip the fj_sig_digits scan
    * entirely (the common case — e.g. every coordinate in canada.json). */
-  if (st->bigdecimal_load == 2 ||
-      (st->bigdecimal_load == 1 && m10digits > 16 && fj_sig_digits(np, nlen) > 16)) {
+  if (st->decimal_precision == 2 ||
+      (st->decimal_precision == 1 && m10digits > 16 && fj_sig_digits(np, nlen) > 16)) {
     return fj_to_bigdecimal_token(np, nlen);
   }
   return fj_float_from_parts(m10, m10digits, e10, neg, overflow, np, nlen);
@@ -851,7 +953,8 @@ static VALUE fj_classify_quoteless(fj_state *st, const char *p0, long n0) {
  * before the whitespace check. */
 enum { FJ_QL_ORD = 0, FJ_QL_TERM, FJ_QL_WS, FJ_QL_CMT };
 static const unsigned char fj_ql_class[256] = {
-  [','] = FJ_QL_TERM, ['}'] = FJ_QL_TERM, [']'] = FJ_QL_TERM,
+  [','] = FJ_QL_TERM, ['{'] = FJ_QL_TERM, ['}'] = FJ_QL_TERM,
+  ['['] = FJ_QL_TERM, [']'] = FJ_QL_TERM,
   [0x0A] = FJ_QL_TERM, [0x0D] = FJ_QL_TERM,
   [0x09] = FJ_QL_WS, [0x0B] = FJ_QL_WS, [0x0C] = FJ_QL_WS, [' '] = FJ_QL_WS,
   ['#'] = FJ_QL_CMT, ['/'] = FJ_QL_CMT,
@@ -1078,7 +1181,8 @@ static int fj_try_member_number(fj_state *st, VALUE *out) {
   /* Commit only if the number abuts a value terminator; otherwise (whitespace,
    * letters, a second '.', "0x…", …) leave it to the quoteless scanner. */
   t = (unsigned char)*p;
-  if (!(t == ',' || t == '}' || t == ']' || t == 0x0A || t == 0x0D || p == buf + st->len)) {
+  if (!(t == ',' || t == '{' || t == '}' || t == '[' || t == ']' ||
+        t == 0x0A || t == 0x0D || p == buf + st->len)) {
     return 0;
   }
 
@@ -1089,8 +1193,8 @@ static int fj_try_member_number(fj_state *st, VALUE *out) {
     return 1;
   }
   e10 -= frac;
-  if (st->bigdecimal_load == 2 ||
-      (st->bigdecimal_load == 1 && m10digits > 16 && fj_sig_digits(np, nlen) > 16)) {
+  if (st->decimal_precision == 2 ||
+      (st->decimal_precision == 1 && m10digits > 16 && fj_sig_digits(np, nlen) > 16)) {
     *out = fj_to_bigdecimal_token(np, nlen);
   } else {
     *out = fj_float_from_parts(m10, m10digits, e10, neg, overflow, np, nlen);
@@ -1164,19 +1268,9 @@ static void fj_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash) {
 void rb_hash_bulk_insert(long, const VALUE *, VALUE);
 #endif
 
-/* Hash entry count as a C long. RHASH_SIZE is not part of the public C API on
- * older Ruby (< ~2.7), but rb_hash_size (Hash#size's implementation) is available
- * everywhere. Only used on the rare :raise duplicate-key path, so the boxing cost
- * is irrelevant — and it keeps the extension buildable down to Ruby 2.5. */
-static inline long fj_hash_len(VALUE hash) {
-  return NUM2LONG(rb_hash_size(hash));
-}
-
 /* Build a Hash from `count` interleaved key,value slots. Fast path (String keys,
- * default :last_wins or :raise): pre-size + bulk insert, detecting duplicates by
- * comparing the resulting size to the pair count — free unless a collision
- * actually happened. symbolize_keys / :first_wins use a per-member loop into the
- * same pre-sized hash. */
+ * default :last_wins): pre-size + bulk insert. symbolize_keys / :first_wins use a
+ * per-member loop into the same pre-sized hash. */
 static VALUE fj_build_object(fj_state *st, const VALUE *pairs, long count) {
   long  entries = count / 2, i;
   VALUE hash    = rb_hash_new_capa(entries);
@@ -1185,22 +1279,13 @@ static VALUE fj_build_object(fj_state *st, const VALUE *pairs, long count) {
    * the per-member loop below to report each dropped duplicate key. */
   if (!st->symbolize_keys && !st->dup_first_wins && st->on_warning == Qnil) {
     rb_hash_bulk_insert(count, pairs, hash);
-    if (st->dup_raise && fj_hash_len(hash) < entries) {
-      VALUE seen = rb_hash_new_capa(entries);
-      for (i = 0; i + 1 < count; i += 2) {
-        long before = fj_hash_len(seen);
-        rb_hash_aset(seen, pairs[i], Qtrue);
-        if (fj_hash_len(seen) == before) fj_error(st, "duplicate key");
-      }
-    }
     return hash;
   }
 
   for (i = 0; i + 1 < count; i += 2) {
     VALUE k = st->symbolize_keys ? rb_funcall(pairs[i], fj_to_sym_id, 0) : pairs[i];
-    if (st->dup_first_wins || st->dup_raise || st->on_warning != Qnil) {
+    if (st->dup_first_wins || st->on_warning != Qnil) {
       if (RTEST(rb_funcall(hash, fj_key_p_id, 1, k))) {
-        if (st->dup_raise) fj_error(st, "duplicate key");
         fj_warn(st, fj_sym_duplicate_key, "duplicate key");
         if (st->dup_first_wins) continue;
       }
@@ -1274,11 +1359,14 @@ static VALUE fj_parse_iter(fj_state *st, int implicit_root) {
     int  is_obj;
 
     if (ps->fhead == 0) { /* top level: parse exactly one value */
-      fj_skip_ws_comments(st);
       b = fj_byte(st);
+      if (FJ_UNLIKELY(fj_needs_ws_skip(b))) { fj_skip_ws_comments(st); b = fj_byte(st); }
       if (b == '{') { fj_advance(st, 1); fj_fpush(ps, ps->vhead, 1); vss = 0; continue; }
       if (b == '[') { fj_advance(st, 1); fj_fpush(ps, ps->vhead, 0); vss = 0; continue; }
       if (b == -1) fj_error(st, "unexpected end of input");
+      /* Top-level scalar: must be a recognized JSON value (number / literal / quoted
+       * string). A bare word raises — no top-level quoteless strings (B-broad). The
+       * scalar-vs-separator boundary is enforced in fj_parse_c. */
       result = fj_parse_value(st);
       break;
     }
@@ -1288,8 +1376,8 @@ static VALUE fj_parse_iter(fj_state *st, int implicit_root) {
 
     if (is_obj) {
       VALUE key;
-      fj_skip_ws_comments(st);
       b = fj_byte(st);
+      if (FJ_UNLIKELY(fj_needs_ws_skip(b))) { fj_skip_ws_comments(st); b = fj_byte(st); }
       if (b == ',') { /* collapsing separator: skip empty member */
         if (st->on_warning != Qnil && !vss) fj_warn(st, fj_sym_empty_slot, "extra comma, collapsed an empty slot");
         vss = 0;
@@ -1316,11 +1404,12 @@ static VALUE fj_parse_iter(fj_state *st, int implicit_root) {
       }
       if (b == ']') fj_error(st, "unexpected ']' — expected a key or '}'");
       key = fj_parse_object_key(st);
-      fj_skip_ws_comments(st);
-      if (fj_byte(st) != ':') fj_error(st, "expected ':' after object key");
+      b = fj_byte(st);
+      if (FJ_UNLIKELY(fj_needs_ws_skip(b))) { fj_skip_ws_comments(st); b = fj_byte(st); }
+      if (b != ':') fj_error(st, "expected ':' after object key");
       fj_advance(st, 1);
-      fj_skip_ws_comments(st);
       b = fj_byte(st);
+      if (FJ_UNLIKELY(fj_needs_ws_skip(b))) { fj_skip_ws_comments(st); b = fj_byte(st); }
       if (b == '{' || b == '[') {
         fj_vpush(ps, key);
         fj_advance(st, 1);
@@ -1340,8 +1429,8 @@ static VALUE fj_parse_iter(fj_state *st, int implicit_root) {
       fj_vpush(ps, fj_parse_member_value(st));
       vss = 1;
     } else { /* array */
-      fj_skip_ws_comments(st);
       b = fj_byte(st);
+      if (FJ_UNLIKELY(fj_needs_ws_skip(b))) { fj_skip_ws_comments(st); b = fj_byte(st); }
       if (b == ',') { /* collapsing separator: skip empty slot */
         if (st->on_warning != Qnil && !vss) fj_warn(st, fj_sym_empty_slot, "extra comma, collapsed an empty slot");
         vss = 0;
@@ -1367,6 +1456,15 @@ static VALUE fj_parse_iter(fj_state *st, int implicit_root) {
         vss = 0;
         continue;
       }
+      /* Strict hot path: inline the two commonest element types — a number and a
+         plain double-quoted string — so they skip fj_parse_member_value's byte
+         re-read + switch. Everything else (quoteless, single/triple-quote,
+         smart-quote, literals) falls through to the full dispatch below. */
+      if (b == '-' || b == '+' || b == '.' || (b >= '0' && b <= '9')) {
+        VALUE num;
+        if (fj_try_member_number(st, &num)) { fj_vpush(ps, num); vss = 1; continue; }
+      }
+      if (b == '"') { fj_vpush(ps, fj_parse_string(st, '"')); vss = 1; continue; }
       fj_vpush(ps, fj_parse_member_value(st));
       vss = 1;
     }
@@ -1391,9 +1489,46 @@ static int fj_implicit_root_ahead(fj_state *st) {
   return result;
 }
 
+/* Between top-level documents, whitespace, comments, AND commas all separate
+ * (commas collapse like the in-container lenient-comma rule). A space alone never
+ * separates — that is handled inside the document by the quoteless run. Mirrors
+ * the Ruby Parser#skip_document_separators. */
+static void fj_skip_document_separators(fj_state *st) {
+  for (;;) {
+    fj_skip_ws_comments(st);
+    if (fj_byte(st) != ',') break;
+    fj_advance(st, 1);
+  }
+}
+
+static int fj_is_hws(int b) { return b == ' ' || b == '\t' || b == 0x0B || b == 0x0C; }
+
+/* After a top-level value: a self-delimiting value (object / array / string) may be
+ * followed by anything, but a bare scalar (number / keyword) must be followed by a
+ * real separator — a newline, ',', a comment, or EOF. A space is NOT a separator, so
+ * `1 2 3` and `42 "x" true` raise. Mirrors the Ruby Parser#enforce_scalar_boundary. */
+static void fj_enforce_scalar_boundary(fj_state *st, VALUE value) {
+  int b, nx;
+  if (RB_TYPE_P(value, T_STRING) || RB_TYPE_P(value, T_HASH) || RB_TYPE_P(value, T_ARRAY)) return;
+  for (;;) {
+    b = fj_byte(st);
+    if (b != -1 && fj_is_hws(b)) { fj_advance(st, 1); continue; }
+    if (b != -1 && b >= 0x80) {
+      long m = fj_mbws(st->buf + st->pos, st->len - st->pos);
+      if (m > 0) { st->pos += m; continue; }  /* multibyte horizontal whitespace (NBSP, …) */
+    }
+    break;
+  }
+  b = fj_byte(st);
+  if (b == -1 || b == 0x0A || b == 0x0D || b == ',') return;
+  if (b == '#') return;
+  if (b == '/') { nx = fj_byte_at(st, 1); if (nx == '/' || nx == '*') return; }
+  fj_error(st, "a top-level number or keyword must be followed by a newline, ',', or end of input");
+}
+
 static VALUE fj_parse_c(VALUE self, VALUE input, VALUE opts) {
   fj_state st;
-  VALUE value, enc_opt, dk;
+  VALUE enc_opt, dk;
 
   Check_Type(input, T_STRING);
 
@@ -1423,13 +1558,12 @@ static VALUE fj_parse_c(VALUE self, VALUE input, VALUE opts) {
   st.symbolize_keys = RTEST(rb_hash_aref(opts, fj_sym_symbolize_keys));
   dk = rb_hash_aref(opts, fj_sym_duplicate_key);
   st.dup_first_wins = (dk == fj_sym_first_wins);
-  st.dup_raise = (dk == fj_sym_raise);
 
   {
-    VALUE bd = rb_hash_aref(opts, fj_sym_bigdecimal_load);
-    if (bd == fj_sym_float) st.bigdecimal_load = 0;
-    else if (bd == fj_sym_bigdecimal) st.bigdecimal_load = 2;
-    else st.bigdecimal_load = 1; /* :auto (default), including nil */
+    VALUE bd = rb_hash_aref(opts, fj_sym_decimal_precision);
+    if (bd == fj_sym_float) st.decimal_precision = 0;
+    else if (bd == fj_sym_bigdecimal) st.decimal_precision = 2;
+    else st.decimal_precision = 1; /* :auto (default), including nil */
   }
 
   st.on_warning = rb_hash_aref(opts, fj_sym_on_warning); /* Qnil when absent */
@@ -1439,36 +1573,37 @@ static VALUE fj_parse_c(VALUE self, VALUE input, VALUE opts) {
     st.pos = 3;
   }
 
-  /* With a block: yield each top-level value until EOF (JSONL / NDJSON /
-   * concatenated). Same loop as the Ruby each_value path, on the C parser. */
+  /* With a block: yield each top-level document until EOF and return the document
+   * count (NDJSON / JSONL / concatenated). Same loop as the Ruby each_value path. */
   if (rb_block_given_p()) {
+    long count = 0;
     for (;;) {
-      fj_skip_ws_comments(&st);
+      VALUE v;
+      fj_skip_document_separators(&st);
       if (fj_eof(&st)) break;
-      rb_yield(fj_parse_iter(&st, fj_implicit_root_ahead(&st)));
+      v = fj_parse_iter(&st, fj_implicit_root_ahead(&st));
+      fj_enforce_scalar_boundary(&st, v);
+      rb_yield(v);
+      count++;
     }
-    return Qnil;
+    return LONG2NUM(count);
   }
 
-  /* No block: auto-detect the document count for free — it is the same "is there
-   * trailing content after the first value?" check that used to raise. 0 documents
-   * -> nil; 1 document -> the value itself (single-document hot path, no Array
-   * allocated); 2+ documents (NDJSON / JSONL / concatenated / whitespace-separated)
-   * -> an Array of every top-level value. Commas do NOT separate documents (only
-   * whitespace / newline / concatenation do), so a bracketless comma list still
-   * raises in fj_parse_iter — the unsupported implicit-root array. */
-  fj_skip_ws_comments(&st);
-  if (fj_eof(&st)) return Qnil;
-  value = fj_parse_iter(&st, fj_implicit_root_ahead(&st));
-  fj_skip_ws_comments(&st);
-  if (fj_eof(&st)) return value;
+  /* No block: always return an Array of every top-level document (0 -> [], 1 ->
+   * [doc], 2+ -> [d1, d2, …]) — the always-array contract. Documents are separated by
+   * newline / comma / concatenation (self-delimiting values); a space alone never
+   * separates, and a bare scalar must be followed by a real separator, so `1 2 3`
+   * raises while `1\n2\n3` and `1, 2, 3` are three documents. */
   {
     VALUE arr = rb_ary_new();
-    rb_ary_push(arr, value);
-    do {
-      rb_ary_push(arr, fj_parse_iter(&st, fj_implicit_root_ahead(&st)));
-      fj_skip_ws_comments(&st);
-    } while (!fj_eof(&st));
+    for (;;) {
+      VALUE v;
+      fj_skip_document_separators(&st);
+      if (fj_eof(&st)) break;
+      v = fj_parse_iter(&st, fj_implicit_root_ahead(&st));
+      fj_enforce_scalar_boundary(&st, v);
+      rb_ary_push(arr, v);
+    }
     return arr;
   }
 }
@@ -1493,8 +1628,7 @@ void Init_smarter_json(void) {
   fj_sym_encoding = ID2SYM(rb_intern("encoding"));
   fj_sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
   fj_sym_first_wins = ID2SYM(rb_intern("first_wins"));
-  fj_sym_raise = ID2SYM(rb_intern("raise"));
-  fj_sym_bigdecimal_load = ID2SYM(rb_intern("bigdecimal_load"));
+  fj_sym_decimal_precision = ID2SYM(rb_intern("decimal_precision"));
   fj_sym_float = ID2SYM(rb_intern("float"));
   fj_sym_bigdecimal = ID2SYM(rb_intern("bigdecimal"));
   fj_sym_on_warning = ID2SYM(rb_intern("on_warning"));

data/ext/smarter_json/vendor/LICENSE-fast_float-MIT

@@ -0,0 +1,27 @@
+MIT License
+
+Copyright (c) 2021 The fast_float authors
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

data/ext/smarter_json/vendor/eisel_lemire.h

@@ -0,0 +1,117 @@
+/* Eisel-Lemire decimal->double, ported from fast_float:
+ * include/fast_float/decimal_to_binary.h, the compute_float<binary_format<double>>
+ * + compute_product_approximation routines.
+ *
+ * Algorithm authors: Michael Eisel (original approach) and Daniel Lemire
+ * (formalization, proof, and the fast_float implementation) — hence "Eisel-Lemire".
+ *
+ * Copyright (c) 2021 The fast_float authors. Tri-licensed Apache-2.0 / MIT / BSL-1.0;
+ * used here under MIT — see LICENSE-fast_float-MIT in this directory.
+ *
+ * This is the "without fallback" variant (Noble Mushtak & Daniel Lemire, "Fast
+ * Number Parsing Without Fallback"): for ANY nonzero mantissa w that fits exactly
+ * in a uint64 (i.e. <= 19 significant digits, not truncated) and decimal exponent
+ * q, it returns the correctly-rounded binary64 with no slow-path needed.
+ *
+ * smarter_json uses it as THE decimal->double path for mantissas up to 18 digits
+ * (everything wider / overflowed / with an extreme exponent goes to the strtod
+ * round-to-odd fallback). It is correctly-rounded across that whole range, with no
+ * round-to-even tie loss, and is fast on the common short-mantissa case.
+ * Verified bit-for-bit vs JSON.parse. See eisel_lemire.md for provenance. */
+#ifndef FJ_EISEL_LEMIRE_H
+#define FJ_EISEL_LEMIRE_H
+
+#include <stdint.h>
+#include <string.h>
+#include "eisel_lemire_powers.h"
+
+/* binary_format<double> constants from fast_float. */
+#define FJ_EL_MANTISSA_BITS   52
+#define FJ_EL_MIN_EXPONENT   (-1023)
+#define FJ_EL_INFINITE_POWER  0x7FF
+#define FJ_EL_SMALLEST_POW10 (-342)
+#define FJ_EL_LARGEST_POW10   308
+#define FJ_EL_MIN_RTE        (-4)   /* min_exponent_round_to_even */
+#define FJ_EL_MAX_RTE         23    /* max_exponent_round_to_even */
+
+/* (((152170 + 65536) * q) >> 16) + 63 == floor(log2(10^q)) + q + 63, see paper. */
+static inline int32_t fj_el_power(int32_t q) {
+  return (((152170 + 65536) * q) >> 16) + 63;
+}
+
+static inline void fj_el_mul128(uint64_t a, uint64_t b, uint64_t *hi, uint64_t *lo) {
+#if defined(__SIZEOF_INT128__)
+  __uint128_t p = (__uint128_t)a * (__uint128_t)b;
+  *lo = (uint64_t)p;
+  *hi = (uint64_t)(p >> 64);
+#else
+  uint64_t a0 = (uint32_t)a, a1 = a >> 32, b0 = (uint32_t)b, b1 = b >> 32;
+  uint64_t p00 = a0 * b0, p01 = a0 * b1, p10 = a1 * b0, p11 = a1 * b1;
+  uint64_t mid = p10 + (p00 >> 32) + (uint32_t)p01;
+  *hi = p11 + (mid >> 32) + (p01 >> 32);
+  *lo = (mid << 32) | (uint32_t)p00;
+#endif
+}
+
+static inline double fj_el_bits2double(uint64_t bits) {
+  double d;
+  memcpy(&d, &bits, sizeof(d));
+  return d;
+}
+
+/* q = power of ten, w = mantissa (NONZERO, exact, fits in uint64). neg = sign. */
+static inline double fj_eisel_lemire_s2d(int64_t q, uint64_t w, int neg) {
+  const uint64_t sign = (uint64_t)(neg != 0) << 63;
+  uint64_t mantissa, prod_hi, prod_lo, sp_hi, sp_lo;
+  int32_t  power2;
+  int      lz, upperbit, shift, index;
+
+  if (q < FJ_EL_SMALLEST_POW10) return fj_el_bits2double(sign); /* underflow -> 0 */
+  if (q > FJ_EL_LARGEST_POW10)
+    return fj_el_bits2double(sign | ((uint64_t)FJ_EL_INFINITE_POWER << FJ_EL_MANTISSA_BITS));
+
+  lz = __builtin_clzll(w);
+  w <<= lz;
+
+  /* compute_product_approximation<mantissa_bits + 3 = 55>: precision_mask = 0x1FF. */
+  index = 2 * (int)(q - FJ_EL_SMALLEST_POWER_OF_FIVE);
+  fj_el_mul128(w, fj_power_of_five_128[index], &prod_hi, &prod_lo);
+  if ((prod_hi & 0x1FF) == 0x1FF) {
+    fj_el_mul128(w, fj_power_of_five_128[index + 1], &sp_hi, &sp_lo);
+    prod_lo += sp_hi;
+    if (sp_hi > prod_lo) prod_hi++;
+  }
+
+  upperbit = (int)(prod_hi >> 63);
+  shift = upperbit + 64 - FJ_EL_MANTISSA_BITS - 3; /* upperbit + 9 */
+  mantissa = prod_hi >> shift;
+  power2 = (int32_t)(fj_el_power((int32_t)q) + upperbit - lz - FJ_EL_MIN_EXPONENT);
+
+  if (power2 <= 0) { /* subnormal */
+    if (-power2 + 1 >= 64) return fj_el_bits2double(sign); /* far below min -> 0 */
+    mantissa >>= (-power2 + 1);
+    mantissa += (mantissa & 1);
+    mantissa >>= 1;
+    power2 = (mantissa < ((uint64_t)1 << FJ_EL_MANTISSA_BITS)) ? 0 : 1;
+    return fj_el_bits2double(sign | ((uint64_t)power2 << FJ_EL_MANTISSA_BITS) | mantissa);
+  }
+
+  /* round-to-even: if we land exactly between two doubles, round down. */
+  if ((prod_lo <= 1) && (q >= FJ_EL_MIN_RTE) && (q <= FJ_EL_MAX_RTE) &&
+      ((mantissa & 3) == 1) && ((mantissa << shift) == prod_hi)) {
+    mantissa &= ~(uint64_t)1;
+  }
+
+  mantissa += (mantissa & 1);
+  mantissa >>= 1;
+  if (mantissa >= ((uint64_t)2 << FJ_EL_MANTISSA_BITS)) {
+    mantissa = (uint64_t)1 << FJ_EL_MANTISSA_BITS;
+    power2++;
+  }
+  mantissa &= ~((uint64_t)1 << FJ_EL_MANTISSA_BITS); /* drop implicit bit */
+  if (power2 >= FJ_EL_INFINITE_POWER)
+    return fj_el_bits2double(sign | ((uint64_t)FJ_EL_INFINITE_POWER << FJ_EL_MANTISSA_BITS));
+  return fj_el_bits2double(sign | ((uint64_t)power2 << FJ_EL_MANTISSA_BITS) | mantissa);
+}
+
+#endif