smarter_json 1.2.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf6191c05bd9049082a1362f4bfa5ab3240c4690bc59c1675c716c0901d5c6cb
-  data.tar.gz: 56f1d5418b20d8bad23694f7831dfb335ab4737dd34e9184b9c20113c41fd3aa
+  metadata.gz: a31a7485e53042c06cb28c92f23e24cb55067fd312704c0c9ac0776868f20293
+  data.tar.gz: 531187f8f7ea38f573785f09fa6e217f63a4393fd537d80c92f00973aeb2b375
 SHA512:
-  metadata.gz: 64c3511d1f21662b703ee1a02876ba8f05401ebcd48b1025e7290a50c49a5fc1d74623c94477b6fc7006ea782028698e298010b3e827f5ed0315fa1f7e88f595
-  data.tar.gz: c2801263204013c23954d7f4489f7f4f38f74802c42da5cf2ba2e5965decfc6c00dd304a7ff16f9f96741e4103fd499f36efa8a8ba5ec7abfdb40dded1c996a9
+  metadata.gz: f88266d06416277355f770645d694be6e81a8faef22a86806cd1e4620e773dee50aeeeeb09cfbc0de47354972250c2654e599c5c03ba0926ca129d9d6a638b0c
+  data.tar.gz: c8bdd4a34d4617c897c815f2526c0f5555459b35bad47106a1db2b71dedac5d7a2605d0a5f92676735ae4895208fcdf1272f05dce09026aa696b84f143693ee5

data/CHANGELOG.md

@@ -13,6 +13,18 @@
 > ⚠️ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents
 >    Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs.
 
+## 1.2.2 (unreleased)
+
+RSpec tests: 1,167
+
+- The Eisel-Lemire fast path for `decimal_precision: :float` now covers decimals with **up to 19 significant digits** (was 18). 19 digits is the most that fits exactly in a `uint64` (max 19-digit ≈ 1.0e19 < `UINT64_MAX` ≈ 1.8e19), so these no longer fall back to the slower `strtod`. Still correctly rounded, bit-for-bit identical to the stdlib — verified across 18/19-digit round-to-even tie shapes.
+
+## 1.2.1 (2026-06-17)
+
+RSpec tests: 1,165
+
+- Performance improvements
+
 ## 1.2.0 (2026-06-16)
 
 RSpec tests: 1,097 → 1,165

data/README.md

@@ -130,10 +130,10 @@ SmarterJSON.process_file("config.json5")            # read a file, then process
 
 ## Usage in APIs
 
-At an API boundary the JSON comes from someone you don't control — a client POSTing a request body to *your* service, or an upstream service answering a call *you* made — and it isn't always clean: a stray trailing comma, a `NaN`, a payload wrapped in prose, or a quiet change to the format. A strict parser turns any of those into an exception (a request you reject, or a failed call chain). SmarterJSON extracts the data that's there instead, so one formatting quirk doesn't sink the whole request:
+At an API boundary the JSON comes from someone you don't control — a client sending a request body to *your* service, or an upstream service answering a call *you* made — and it isn't always clean: a stray trailing comma, a `NaN`, a payload wrapped in prose, or a quiet change to the format. A strict parser turns any of those into an exception (a request you reject, or a failed call chain). SmarterJSON extracts the data that's there instead, so one formatting quirk doesn't sink the whole request:
 
 ```ruby
-# Inbound — JSON a caller sent to your endpoint:
+# Inbound — JSON a caller sent to your service:
 data = SmarterJSON.process(request.body)
 
 # Outbound — JSON from a service you called:

data/ext/smarter_json/smarter_json.c

@@ -629,14 +629,15 @@ static VALUE fj_float_strtod(const char *p, long n) {
 /* e10 is the final base-10 exponent (already adjusted by the fraction length). */
 static FJ_ALWAYS_INLINE VALUE fj_float_from_parts(fj_state *st, uint64_t m10, int m10digits, int64_t e10, int neg, int overflow, const char *p, long n) {
   double d;
-  /* Fast path by mantissa width (our scanner accumulates m10 exactly up to 18
+  /* Fast path by mantissa width (our scanner accumulates m10 exactly up to 19
      digits, flagging overflow beyond):
-       1..18 digits -> Eisel-Lemire, correctly-rounded for any exact uint64 mantissa
-                       (Mushtak-Lemire). This pulls full-double-precision data (e.g.
-                       citylots coordinates, 18 sig digits) off the slow strtod
-                       fallback — the stdlib json gem still strtods it.
-     >18 digits / overflow / extreme exponent -> strtod (round-to-odd). */
-  if (!overflow && m10digits >= 1 && m10digits <= 18 && (long)m10digits + e10 >= -307) {
+       1..19 digits -> Eisel-Lemire, correctly-rounded for any exact uint64 mantissa
+                       (Mushtak-Lemire). 19 digits is the most that fits exactly in a
+                       uint64 (max 19-digit ~1.0e19 < UINT64_MAX ~1.8e19); this pulls
+                       full-double-precision data (e.g. citylots coordinates) off the
+                       slow strtod fallback — the stdlib json gem still strtods it.
+     >19 digits / overflow / extreme exponent -> strtod (round-to-odd). */
+  if (!overflow && m10digits >= 1 && m10digits <= 19 && (long)m10digits + e10 >= -307) {
     if (m10 == 0) return rb_float_new(neg ? -0.0 : 0.0);
     d = fj_eisel_lemire_s2d(e10, m10, neg);
   } else {
@@ -683,7 +684,7 @@ static int fj_try_decimal(fj_state *st, const char *p, long n, VALUE *out) {
       for (;;) {
         while (i < n && p[i] >= '0' && p[i] <= '9') {
           had_leading_zero = 1;
-          if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; }
+          if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; }
           else overflow = 1;
           i++;
         }
@@ -695,7 +696,7 @@ static int fj_try_decimal(fj_state *st, const char *p, long n, VALUE *out) {
     has_digit = 1;
     for (;;) {
       while (i < n && p[i] >= '0' && p[i] <= '9') {
-        if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; }
+        if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; }
         else overflow = 1;
         i++;
       }
@@ -710,7 +711,7 @@ static int fj_try_decimal(fj_state *st, const char *p, long n, VALUE *out) {
     for (;;) {
       while (i < n && p[i] >= '0' && p[i] <= '9') {
         has_digit = 1;
-        if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; frac++; }
+        if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(p[i] - '0'); m10digits++; frac++; }
         else overflow = 1;
         i++;
       }
@@ -774,7 +775,7 @@ static VALUE fj_parse_number(fj_state *st) {
   long   nlen;
   int    is_float = 0, neg = 0, overflow = 0, has_sign = 0, had_leading_zero = 0;
   uint64_t m10 = 0;                 /* mantissa: integer + fraction digits */
-  int    m10digits = 0;             /* mantissa digit chars (caps the Eisel-Lemire fast path at 18) */
+  int    m10digits = 0;             /* mantissa digit chars (caps the Eisel-Lemire fast path at 19) */
   int    frac = 0;                  /* fraction digit chars: e10 -= frac */
   int64_t e10 = 0;
 
@@ -810,7 +811,7 @@ static VALUE fj_parse_number(fj_state *st) {
       for (;;) {
         while (*p >= '0' && *p <= '9') {
           had_leading_zero = 1;
-          if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
+          if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
           else overflow = 1;
           p++;
         }
@@ -821,7 +822,7 @@ static VALUE fj_parse_number(fj_state *st) {
   } else if (*p >= '1' && *p <= '9') {
     for (;;) {
       while (*p >= '0' && *p <= '9') {
-        if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
+        if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
         else overflow = 1;
         p++;
       }
@@ -841,7 +842,7 @@ static VALUE fj_parse_number(fj_state *st) {
     p++;
     for (;;) {
       while (*p >= '0' && *p <= '9') {
-        if (m10digits < 18) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; frac++; }
+        if (m10digits < 19) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; frac++; }
         else overflow = 1;
         p++;
       }
@@ -1245,7 +1246,7 @@ static int fj_try_member_number(fj_state *st, VALUE *out) {
   } else if (*p >= '1' && *p <= '9') {
     for (;;) {
       while (*p >= '0' && *p <= '9') {
-        if (FJ_LIKELY(m10digits < 18)) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
+        if (FJ_LIKELY(m10digits < 19)) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; }
         else overflow = 1;
         p++;
       }
@@ -1259,7 +1260,7 @@ static int fj_try_member_number(fj_state *st, VALUE *out) {
     is_float = 1; p++;
     for (;;) {
       while (*p >= '0' && *p <= '9') {
-        if (FJ_LIKELY(m10digits < 18)) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; frac++; }
+        if (FJ_LIKELY(m10digits < 19)) { m10 = m10 * 10 + (uint64_t)(*p - '0'); m10digits++; frac++; }
         else overflow = 1;
         p++;
       }

data/ext/smarter_json/vendor/eisel_lemire.md

@@ -5,7 +5,7 @@
 
 ## What smarter_json uses it for
 
-`fj_float_from_parts` (in `smarter_json.c`) routes `m10digits ≤ 18 → fj_eisel_lemire_s2d`, and `> 18 / overflow / extreme exponent → strtod` (round-to-odd). Eisel-Lemire is correctly-rounded across the whole ≤18-digit range — no round-to-even tie loss — **and** fast on the common short-mantissa case.
+`fj_float_from_parts` (in `smarter_json.c`) routes `m10digits ≤ 19 → fj_eisel_lemire_s2d`, and `> 19 / overflow / extreme exponent → strtod` (round-to-odd). Eisel-Lemire is correctly-rounded across the whole ≤19-digit range — every mantissa that fits exactly in a uint64, no round-to-even tie loss — **and** fast on the common short-mantissa case.
 
 ## These two files are DERIVED, not verbatim copies
 

data/lib/smarter_json/parser.rb

@@ -757,8 +757,10 @@ module SmarterJSON
     # incl. LF/CR which also terminate). Stopping at a terminator/EOF means the run had no
     # interior whitespace, so there's nothing to trim and no comment marker can apply.
     #
-    # U+FEFF is JSON5/ES5 whitespace but not in [[:space:]], so we need to add it:
-    QL_BREAK = /[,{}\[\]]|[[:space:]]|#{[0xFEFF].pack("U")}/.freeze
+    # U+FEFF is JSON5/ES5 whitespace but NOT in [[:space:]]. It is deliberately kept OUT of
+    # this regex: a multibyte alternative defeats byteindex's fast byte-search (~3.3x slower
+    # on number-dense input). A trailing U+FEFF is trimmed cheaply in the fast path below.
+    QL_BREAK = /[,{}\[\]]|[[:space:]]/.freeze
 
     # The defaults live centrally in SmarterJSON::Options (lib/smarter_json/options.rb).
     DEFAULT_OPTIONS = Options::DEFAULT_OPTIONS
@@ -1350,7 +1352,14 @@ module SmarterJSON
         b = hit < @bytesize ? input.getbyte(hit) : nil
         if b.nil? || b == COMMA || b == RBRACE || b == RBRACKET || b == LBRACE || b == LBRACKET || b == LF || b == CR
           @pos = hit
-          return hit
+          # A trailing U+FEFF (EF BB BF) is JSON5/ES5 whitespace but not in QL_BREAK, so
+          # byteindex scanned past it into the run — trim it (and a run of them). On the
+          # common path the last byte is a digit/letter, so the first compare fails at once.
+          fin = hit
+          while fin - 3 >= pos && input.getbyte(fin - 1) == 0xBF && input.getbyte(fin - 2) == 0xBB && input.getbyte(fin - 3) == 0xEF
+            fin -= 3
+          end
+          return fin
         end
       end
 

data/lib/smarter_json/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterJSON
-  VERSION = "1.2.0"
+  VERSION = "1.2.2"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_json
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: exe
 cert_chain: []
-date: 2026-06-16 00:00:00.000000000 Z
+date: 2026-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal