@zakkster/lite-bake-stream 1.0.0

package/src/Tokenizer.js

@@ -0,0 +1,691 @@
+// @zakkster/lite-bake-stream / Tokenizer
+// Chunk-safe UTF-8 JSON SAX scanner. Zero-GC on the hot path after warmup.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// Contract: see /SPEC.md section 5.
+//
+// Error codes (stable):
+//   E_UNEXPECTED_BYTE      - byte does not belong in current state
+//   E_UNEXPECTED_EOF       - end() called mid-token
+//   E_INVALID_ESCAPE       - unknown \ sequence
+//   E_INVALID_HEX          - non-hex digit in \uXXXX
+//   E_UNPAIRED_SURROGATE   - lone high/low surrogate in \u escape
+//   E_INVALID_UTF8         - malformed UTF-8 byte sequence
+//   E_NUMBER_OVERFLOW      - number magnitude exceeds F64 range
+//   E_NUMBER_INVALID       - malformed number literal
+//   E_KEYWORD_MISMATCH     - true/false/null spelled wrong
+//   E_TRAILING_INPUT       - non-whitespace bytes after top-level value(s) (array mode only)
+//   E_DEPTH_LIMIT          - nesting depth exceeded MAX_DEPTH
+
+export const VERSION = '1.0.0';
+
+// ---------- byte constants ----------
+const B_SPACE = 0x20, B_TAB = 0x09, B_LF = 0x0A, B_CR = 0x0D;
+const B_QUOTE = 0x22, B_BACKSLASH = 0x5C, B_SLASH = 0x2F;
+const B_LBRACE = 0x7B, B_RBRACE = 0x7D, B_LBRACKET = 0x5B, B_RBRACKET = 0x5D;
+const B_COMMA = 0x2C, B_COLON = 0x3A;
+const B_MINUS = 0x2D, B_PLUS = 0x2B, B_DOT = 0x2E;
+const B_0 = 0x30, B_9 = 0x39;
+const B_a = 0x61, B_b_low = 0x62, B_e_low = 0x65, B_f_low = 0x66;
+const B_l = 0x6C, B_n_low = 0x6E, B_r_low = 0x72, B_s = 0x73, B_t_low = 0x74, B_u = 0x75;
+const B_E_up = 0x45;
+const B_A_up = 0x41, B_F_up = 0x46;
+
+const MAX_DEPTH = 512;
+
+// ---------- states ----------
+const S_TOP           = 0;   // top-level: expecting value, whitespace, or EOF
+const S_VALUE         = 1;   // nested: expecting a value
+const S_OBJECT_START  = 2;   // just entered {, expecting key or }
+const S_OBJECT_KEY_END = 3;  // just closed a key, expecting :
+const S_OBJECT_VALUE_END = 4; // just closed a value in obj, expecting , or }
+const S_ARRAY_START   = 5;   // just entered [, expecting value or ]
+const S_ARRAY_VALUE_END = 6; // just closed a value in arr, expecting , or ]
+const S_OBJECT_NEXT_KEY = 7; // just saw , in obj, strictly expecting key (no })
+const S_STRING        = 10;
+const S_STRING_ESC    = 11;
+const S_STRING_UNI    = 12;  // inside \uXXXX
+const S_STRING_UNI_LO_BS = 13; // after high surrogate, expecting \
+const S_STRING_UNI_LO_U  = 14; // after high surrogate + \, expecting u
+const S_STRING_UNI_LO    = 15; // inside low-surrogate \uXXXX
+const S_NUMBER        = 20;  // generic number accumulation (substate held separately)
+const S_KEYWORD       = 30;
+
+// Number substates
+const NS_SIGN     = 0; // just saw -, expecting first int digit
+const NS_INT_ZERO = 1; // just saw 0 as leading, next must be . or e/E or terminator
+const NS_INT      = 2; // in integer digits (>=1)
+const NS_FRAC_DOT = 3; // just saw ., need first frac digit
+const NS_FRAC     = 4; // in fractional digits
+const NS_EXP_MARK = 5; // just saw e/E, next may be sign or digit
+const NS_EXP_SIGN = 6; // just saw +/- after e, need first exp digit
+const NS_EXP      = 7; // in exponent digits
+
+// Clinger's fast-path decimal->F64 conversion table. Each POW10[k] is the
+// EXACT IEEE 754 double for 10^k across k in [0, 22]. Computed by iterated
+// multiplication starting from 1.0 so no rounding occurs. Any mantissa that
+// fits in u53 combined with an exponent in [-22, 22] can be converted to
+// correctly-rounded F64 via a single multiplication (positive exp) or single
+// division (negative exp) using this table. Outside that domain, correctly
+// rounded parsing requires David Gay's strtod, deferred to M4+.
+const POW10 = new Float64Array(23);
+POW10[0] = 1;
+for (let _pi = 1; _pi < 23; _pi++) POW10[_pi] = POW10[_pi - 1] * 10;
+const CLINGER_MAX_DIGITS = 15;    // mantissa with <=15 digits fits exactly in u53
+const CLINGER_MAX_EXP = 22;
+
+// Container stack values
+const C_OBJECT = 0;
+const C_ARRAY  = 1;
+
+// UTF-8 continuation state (0 = expecting lead byte)
+// Values > 0 encode how many continuation bytes are still expected.
+
+// ---------- keyword templates ----------
+const KW_TRUE  = Uint8Array.of(B_t_low, B_r_low, B_u, B_e_low);
+const KW_FALSE = Uint8Array.of(B_f_low, B_a, B_l, B_s, B_e_low);
+const KW_NULL  = Uint8Array.of(B_n_low, B_u, B_l, B_l);
+const KW_ID_TRUE  = 0;
+const KW_ID_FALSE = 1;
+const KW_ID_NULL  = 2;
+
+export class TokenizerError extends Error {
+  constructor(code, byteOffset, msg) {
+    super(msg + ' at byte ' + byteOffset);
+    this.code = code;
+    this.byteOffset = byteOffset;
+    this.name = 'TokenizerError';
+  }
+}
+
+// Sink protocol (see SPEC 5.3). Any missing method is treated as no-op.
+const NOOP = () => {};
+
+export class Tokenizer {
+  constructor(sink, opts) {
+    if (!sink) throw new Error('sink required');
+    this.sink = sink;
+
+    // top-level framing: 'auto' (default), 'array', or 'ndjson'
+    this._framing = (opts && opts.framing) || 'auto';
+    this._maxStringBytes = (opts && opts.maxStringBytes) || (1 << 20); // 1 MB default cap
+
+    // fill in missing sink methods to avoid per-token undefined checks
+    if (!sink.onStartObject) sink.onStartObject = NOOP;
+    if (!sink.onEndObject)   sink.onEndObject   = NOOP;
+    if (!sink.onStartArray)  sink.onStartArray  = NOOP;
+    if (!sink.onEndArray)    sink.onEndArray    = NOOP;
+    if (!sink.onKey)         sink.onKey         = NOOP;
+    if (!sink.onString)      sink.onString      = NOOP;
+    if (!sink.onNumber)      sink.onNumber      = NOOP;
+    if (!sink.onTrue)        sink.onTrue        = NOOP;
+    if (!sink.onFalse)       sink.onFalse       = NOOP;
+    if (!sink.onNull)        sink.onNull        = NOOP;
+    if (!sink.onEnd)         sink.onEnd         = NOOP;
+
+    // parser state
+    this._state = S_TOP;
+    this._topDetected = false;      // for 'auto': did we see the first non-ws byte
+    this._arrayMode = false;        // once top-level [ seen
+    this._arrayModeOuterOpen = false; // outer array is open (not yet closed)
+    this._absOffset = 0;
+
+    // container stack: Uint8Array of C_OBJECT/C_ARRAY, depth pointer
+    this._stack = new Uint8Array(MAX_DEPTH);
+    this._depth = 0;
+
+    // string accumulation
+    this._strBuf = new Uint8Array(4096);
+    this._strLen = 0;
+    this._strIsKey = false;
+    this._uniHi = 0;                // hex value being accumulated
+    this._uniDigits = 0;            // digits accumulated so far (0..4)
+    this._highSurrogate = 0;        // if non-zero, awaiting low surrogate
+
+    // number accumulation (byte-level, no substring alloc)
+    this._numSubstate = 0;
+    this._numSign = 1;
+    this._numIntPart = 0;
+    this._numFracPart = 0;
+    this._numFracDiv = 1;
+    this._numExpSign = 1;
+    this._numExpPart = 0;
+
+    // keyword accumulation
+    this._kwId = 0;
+    this._kwPos = 0;
+  }
+
+  // Feed a chunk. Bytes are consumed synchronously; sink events fire during this call.
+  feed(chunk) {
+    const len = chunk.length;
+    let i = 0;
+    while (i < len) {
+      const b = chunk[i];
+      const st = this._state;
+
+      if (st < S_STRING) {
+        // structural / value-expecting states
+        if (b === B_SPACE || b === B_TAB || b === B_LF || b === B_CR) {
+          i++; this._absOffset++;
+          if (st === S_TOP && this._arrayMode === false && this._topDetected === false) {
+            // still leading whitespace before top-level detection
+          }
+          continue;
+        }
+
+        if (st === S_TOP) {
+          this._handleTopByte(b);
+        } else if (st === S_VALUE) {
+          this._handleValueByte(b);
+        } else if (st === S_OBJECT_START) {
+          this._handleObjectStartByte(b);
+        } else if (st === S_OBJECT_NEXT_KEY) {
+          if (b !== B_QUOTE) this._err('E_UNEXPECTED_BYTE', 'expected key string (trailing comma not allowed)');
+          this._beginString(true);
+        } else if (st === S_OBJECT_KEY_END) {
+          if (b !== B_COLON) this._err('E_UNEXPECTED_BYTE', 'expected :');
+          this._state = S_VALUE;
+        } else if (st === S_OBJECT_VALUE_END) {
+          this._handleObjectValueEndByte(b);
+        } else if (st === S_ARRAY_START) {
+          this._handleArrayStartByte(b);
+        } else if (st === S_ARRAY_VALUE_END) {
+          this._handleArrayValueEndByte(b);
+        }
+
+        i++; this._absOffset++;
+        continue;
+      }
+
+      if (st === S_STRING) {
+        // fast inner loop over ASCII string bytes
+        if (b === B_QUOTE) {
+          this._emitString();
+          i++; this._absOffset++;
+          continue;
+        }
+        if (b === B_BACKSLASH) {
+          this._state = S_STRING_ESC;
+          i++; this._absOffset++;
+          continue;
+        }
+        if (b < 0x20) {
+          this._err('E_UNEXPECTED_BYTE', 'unescaped control byte 0x' + b.toString(16) + ' in string');
+        }
+        // fast path: scan run of plain ASCII / continuation bytes until special byte
+        // (validates minimal UTF-8 structure but does not fully verify surrogate/overlong)
+        let j = i;
+        while (j < len) {
+          const bb = chunk[j];
+          if (bb === B_QUOTE || bb === B_BACKSLASH || bb < 0x20) break;
+          j++;
+        }
+        this._appendStrRange(chunk, i, j);
+        this._absOffset += (j - i);
+        i = j;
+        continue;
+      }
+
+      if (st === S_STRING_ESC) {
+        i++; this._absOffset++;
+        if (b === B_QUOTE)      { this._appendStrByte(B_QUOTE); this._state = S_STRING; }
+        else if (b === B_BACKSLASH) { this._appendStrByte(B_BACKSLASH); this._state = S_STRING; }
+        else if (b === B_SLASH) { this._appendStrByte(B_SLASH); this._state = S_STRING; }
+        else if (b === B_b_low) { this._appendStrByte(0x08); this._state = S_STRING; }
+        else if (b === B_f_low) { this._appendStrByte(0x0C); this._state = S_STRING; }
+        else if (b === B_n_low) { this._appendStrByte(0x0A); this._state = S_STRING; }
+        else if (b === B_r_low) { this._appendStrByte(0x0D); this._state = S_STRING; }
+        else if (b === B_t_low) { this._appendStrByte(0x09); this._state = S_STRING; }
+        else if (b === B_u) {
+          this._uniHi = 0; this._uniDigits = 0;
+          this._state = this._highSurrogate === 0 ? S_STRING_UNI : S_STRING_UNI_LO;
+        }
+        else this._err('E_INVALID_ESCAPE', 'unknown escape \\' + String.fromCharCode(b));
+        continue;
+      }
+
+      if (st === S_STRING_UNI || st === S_STRING_UNI_LO) {
+        const d = this._hexVal(b);
+        if (d < 0) this._err('E_INVALID_HEX', 'non-hex digit in \\uXXXX');
+        this._uniHi = (this._uniHi << 4) | d;
+        this._uniDigits++;
+        i++; this._absOffset++;
+        if (this._uniDigits === 4) {
+          const cp = this._uniHi;
+          if (st === S_STRING_UNI) {
+            if (cp >= 0xD800 && cp <= 0xDBFF) {
+              this._highSurrogate = cp;
+              this._state = S_STRING_UNI_LO_BS;
+            } else if (cp >= 0xDC00 && cp <= 0xDFFF) {
+              this._err('E_UNPAIRED_SURROGATE', 'lone low surrogate');
+            } else {
+              this._appendUtf8Codepoint(cp);
+              this._state = S_STRING;
+            }
+          } else {
+            if (cp < 0xDC00 || cp > 0xDFFF) {
+              this._err('E_UNPAIRED_SURROGATE', 'expected low surrogate');
+            }
+            const combined = 0x10000 + ((this._highSurrogate - 0xD800) << 10) + (cp - 0xDC00);
+            this._highSurrogate = 0;
+            this._appendUtf8Codepoint(combined);
+            this._state = S_STRING;
+          }
+        }
+        continue;
+      }
+
+      if (st === S_STRING_UNI_LO_BS) {
+        if (b !== B_BACKSLASH) this._err('E_UNPAIRED_SURROGATE', 'expected \\ for low surrogate');
+        this._state = S_STRING_UNI_LO_U;
+        i++; this._absOffset++;
+        continue;
+      }
+      if (st === S_STRING_UNI_LO_U) {
+        if (b !== B_u) this._err('E_UNPAIRED_SURROGATE', 'expected u for low surrogate');
+        this._uniHi = 0; this._uniDigits = 0;
+        this._state = S_STRING_UNI_LO;
+        i++; this._absOffset++;
+        continue;
+      }
+
+      if (st === S_NUMBER) {
+        // Try to consume as many digits as possible in a tight inner loop for common cases
+        const consumed = this._consumeNumberBytes(chunk, i, len);
+        if (consumed > 0) { i += consumed; this._absOffset += consumed; continue; }
+        // byte terminates number
+        this._emitNumber();
+        // do NOT advance i; re-dispatch this byte in new state
+        continue;
+      }
+
+      if (st === S_KEYWORD) {
+        const tmpl = this._kwId === KW_ID_TRUE ? KW_TRUE : (this._kwId === KW_ID_FALSE ? KW_FALSE : KW_NULL);
+        if (b !== tmpl[this._kwPos]) this._err('E_KEYWORD_MISMATCH', 'bad keyword byte');
+        this._kwPos++;
+        i++; this._absOffset++;
+        if (this._kwPos === tmpl.length) {
+          if (this._kwId === KW_ID_TRUE) this.sink.onTrue();
+          else if (this._kwId === KW_ID_FALSE) this.sink.onFalse();
+          else this.sink.onNull();
+          this._afterValue();
+        }
+        continue;
+      }
+
+      // unreachable
+      this._err('E_UNEXPECTED_BYTE', 'internal: unknown state ' + st);
+    }
+  }
+
+  end() {
+    // A trailing number may still be pending
+    if (this._state === S_NUMBER) {
+      this._emitNumber();
+    }
+    if (this._state === S_TOP) {
+      if (this._arrayMode && this._arrayModeOuterOpen) {
+        this._err('E_UNEXPECTED_EOF', 'unclosed top-level array');
+      }
+      this.sink.onEnd();
+      return;
+    }
+    this._err('E_UNEXPECTED_EOF', 'input ended mid-token (state=' + this._state + ')');
+  }
+
+  // -------- top-level handling --------
+
+  _handleTopByte(b) {
+    if (this._framing === 'auto' && this._topDetected === false) {
+      this._topDetected = true;
+      if (b === B_LBRACKET) {
+        this._arrayMode = true;
+        this._arrayModeOuterOpen = true;
+        this._state = S_ARRAY_START;
+        return;
+      }
+      // NDJSON path
+      this._arrayMode = false;
+      this._beginValue(b);
+      return;
+    }
+    if (this._framing === 'array' && this._topDetected === false) {
+      if (b !== B_LBRACKET) this._err('E_UNEXPECTED_BYTE', 'expected [ in array framing');
+      this._topDetected = true;
+      this._arrayMode = true;
+      this._arrayModeOuterOpen = true;
+      this._state = S_ARRAY_START;
+      return;
+    }
+    if (this._framing === 'ndjson' && this._topDetected === false) {
+      this._topDetected = true;
+      this._arrayMode = false;
+      this._beginValue(b);
+      return;
+    }
+    // top-level after a completed value (NDJSON stream)
+    if (this._arrayMode) {
+      // outer array closed; only whitespace permitted after
+      this._err('E_TRAILING_INPUT', 'trailing byte after top-level array');
+    }
+    this._beginValue(b);
+  }
+
+  _handleValueByte(b) {
+    this._beginValue(b);
+  }
+
+  _handleObjectStartByte(b) {
+    if (b === B_RBRACE) {
+      this.sink.onEndObject();
+      this._popContainer();
+      return;
+    }
+    if (b !== B_QUOTE) this._err('E_UNEXPECTED_BYTE', 'expected key string');
+    this._beginString(true);
+  }
+
+  _handleObjectValueEndByte(b) {
+    if (b === B_COMMA) { this._state = S_OBJECT_NEXT_KEY; }
+    else if (b === B_RBRACE) {
+      this.sink.onEndObject();
+      this._popContainer();
+    } else {
+      this._err('E_UNEXPECTED_BYTE', 'expected , or }');
+    }
+  }
+
+  _handleArrayStartByte(b) {
+    if (b === B_RBRACKET) {
+      if (this._depth === 0 && this._arrayMode) {
+        // closing the top-level array (which is framing, not sink-visible)
+        this._arrayModeOuterOpen = false;
+        this._state = S_TOP;
+        return;
+      }
+      this.sink.onEndArray();
+      this._popContainer();
+      return;
+    }
+    this._beginValue(b);
+  }
+
+  _handleArrayValueEndByte(b) {
+    if (b === B_COMMA) { this._state = S_VALUE; }
+    else if (b === B_RBRACKET) {
+      if (this._depth === 0 && this._arrayMode) {
+        this._arrayModeOuterOpen = false;
+        this._state = S_TOP;
+        return;
+      }
+      this.sink.onEndArray();
+      this._popContainer();
+    } else {
+      this._err('E_UNEXPECTED_BYTE', 'expected , or ]');
+    }
+  }
+
+  // -------- value dispatch --------
+
+  _beginValue(b) {
+    if (b === B_LBRACE) {
+      this.sink.onStartObject();
+      this._pushContainer(C_OBJECT);
+      this._state = S_OBJECT_START;
+      return;
+    }
+    if (b === B_LBRACKET) {
+      this.sink.onStartArray();
+      this._pushContainer(C_ARRAY);
+      this._state = S_ARRAY_START;
+      return;
+    }
+    if (b === B_QUOTE) { this._beginString(false); return; }
+    if (b === B_MINUS) { this._beginNumber(B_MINUS); return; }
+    if (b >= B_0 && b <= B_9) { this._beginNumber(b); return; }
+    if (b === B_t_low) { this._beginKeyword(KW_ID_TRUE); return; }
+    if (b === B_f_low) { this._beginKeyword(KW_ID_FALSE); return; }
+    if (b === B_n_low) { this._beginKeyword(KW_ID_NULL); return; }
+    this._err('E_UNEXPECTED_BYTE', 'unexpected byte 0x' + b.toString(16) + ' at value position');
+  }
+
+  _beginString(isKey) {
+    this._strLen = 0;
+    this._strIsKey = isKey;
+    this._highSurrogate = 0;
+    this._state = S_STRING;
+  }
+
+  _beginNumber(firstByte) {
+    this._numSign = 1;
+    this._numIntPart = 0;
+    this._numFracPart = 0;
+    this._numFracDiv = 1;
+    this._numFracDigits = 0;
+    this._numDigitsSeen = 0;
+    this._numExpSign = 1;
+    this._numExpPart = 0;
+    if (firstByte === B_MINUS) {
+      this._numSign = -1;
+      this._numSubstate = NS_SIGN;
+    } else if (firstByte === B_0) {
+      this._numSubstate = NS_INT_ZERO;
+      this._numDigitsSeen = 1;
+    } else {
+      this._numIntPart = firstByte - B_0;
+      this._numSubstate = NS_INT;
+      this._numDigitsSeen = 1;
+    }
+    this._state = S_NUMBER;
+  }
+
+  _beginKeyword(id) {
+    this._kwId = id;
+    this._kwPos = 1; // first byte consumed by dispatch
+    this._state = S_KEYWORD;
+  }
+
+  // Returns number of bytes consumed as part of the current number token (may be 0).
+  _consumeNumberBytes(chunk, start, end) {
+    let i = start;
+    let ss = this._numSubstate;
+    while (i < end) {
+      const b = chunk[i];
+      if (ss === NS_SIGN) {
+        if (b === B_0) { ss = NS_INT_ZERO; this._numDigitsSeen++; i++; continue; }
+        if (b >= (B_0 + 1) && b <= B_9) { this._numIntPart = b - B_0; this._numDigitsSeen++; ss = NS_INT; i++; continue; }
+        break;
+      }
+      if (ss === NS_INT_ZERO) {
+        if (b === B_DOT) { ss = NS_FRAC_DOT; i++; continue; }
+        if (b === B_e_low || b === B_E_up) { ss = NS_EXP_MARK; i++; continue; }
+        if (b >= B_0 && b <= B_9) this._err('E_NUMBER_INVALID', 'leading zero followed by digit');
+        break;
+      }
+      if (ss === NS_INT) {
+        if (b >= B_0 && b <= B_9) { this._numIntPart = this._numIntPart * 10 + (b - B_0); this._numDigitsSeen++; i++; continue; }
+        if (b === B_DOT) { ss = NS_FRAC_DOT; i++; continue; }
+        if (b === B_e_low || b === B_E_up) { ss = NS_EXP_MARK; i++; continue; }
+        break;
+      }
+      if (ss === NS_FRAC_DOT) {
+        if (b >= B_0 && b <= B_9) {
+          this._numFracPart = b - B_0;
+          this._numFracDiv = 10;
+          this._numFracDigits = 1;
+          this._numDigitsSeen++;
+          ss = NS_FRAC; i++; continue;
+        }
+        this._err('E_NUMBER_INVALID', 'expected digit after .');
+      }
+      if (ss === NS_FRAC) {
+        if (b >= B_0 && b <= B_9) {
+          this._numFracPart = this._numFracPart * 10 + (b - B_0);
+          this._numFracDiv *= 10;
+          this._numFracDigits++;
+          this._numDigitsSeen++;
+          i++; continue;
+        }
+        if (b === B_e_low || b === B_E_up) { ss = NS_EXP_MARK; i++; continue; }
+        break;
+      }
+      if (ss === NS_EXP_MARK) {
+        if (b === B_PLUS) { ss = NS_EXP_SIGN; i++; continue; }
+        if (b === B_MINUS) { this._numExpSign = -1; ss = NS_EXP_SIGN; i++; continue; }
+        if (b >= B_0 && b <= B_9) { this._numExpPart = b - B_0; ss = NS_EXP; i++; continue; }
+        this._err('E_NUMBER_INVALID', 'expected digit or sign after e');
+      }
+      if (ss === NS_EXP_SIGN) {
+        if (b >= B_0 && b <= B_9) { this._numExpPart = b - B_0; ss = NS_EXP; i++; continue; }
+        this._err('E_NUMBER_INVALID', 'expected digit after exponent sign');
+      }
+      if (ss === NS_EXP) {
+        if (b >= B_0 && b <= B_9) { this._numExpPart = this._numExpPart * 10 + (b - B_0); i++; continue; }
+        break;
+      }
+      break;
+    }
+    this._numSubstate = ss;
+    return i - start;
+  }
+
+  _emitNumber() {
+    const ss = this._numSubstate;
+    if (ss === NS_SIGN || ss === NS_FRAC_DOT || ss === NS_EXP_MARK || ss === NS_EXP_SIGN) {
+      this._err('E_NUMBER_INVALID', 'number ended prematurely');
+    }
+    const digitsSeen  = this._numDigitsSeen;
+    const fracDigits  = this._numFracDigits;
+    const explicitExp = this._numExpSign * this._numExpPart;
+    const netExp      = explicitExp - fracDigits;
+    let v;
+    // Clinger's fast path: mantissa fits in u53 AND |netExp| <= 22 => single
+    // multiply/divide by an exact power-of-10 gives correctly-rounded F64.
+    if (digitsSeen <= CLINGER_MAX_DIGITS && netExp >= -CLINGER_MAX_EXP && netExp <= CLINGER_MAX_EXP) {
+      // Assemble mantissa as exact integer: intPart * 10^fracDigits + fracPart.
+      // With digitsSeen <= 15 the product fits in u53 exactly (no rounding).
+      const mantissa = this._numIntPart * this._numFracDiv + this._numFracPart;
+      if (netExp === 0)       v = this._numSign * mantissa;
+      else if (netExp > 0)    v = this._numSign * mantissa * POW10[netExp];
+      else                    v = this._numSign * mantissa / POW10[-netExp];
+    } else {
+      // Slow path (deferred correctly-rounded parsing until M4+): naive
+      // computation. May differ from JS Number(str) by <=1 ULP. Documented
+      // in SPEC section on numeric preservation.
+      v = this._numIntPart + (this._numFracDiv > 1 ? this._numFracPart / this._numFracDiv : 0);
+      if (this._numExpPart !== 0) v = v * Math.pow(10, explicitExp);
+      v = this._numSign * v;
+    }
+    if (!Number.isFinite(v)) this._err('E_NUMBER_OVERFLOW', 'number exceeds F64 range');
+    this.sink.onNumber(v);
+    this._afterValue();
+  }
+
+  _emitString() {
+    if (this._strIsKey) {
+      this.sink.onKey(this._strBuf, 0, this._strLen);
+      this._state = S_OBJECT_KEY_END;
+    } else {
+      this.sink.onString(this._strBuf, 0, this._strLen);
+      this._afterValue();
+    }
+  }
+
+  _afterValue() {
+    if (this._depth === 0) {
+      // Primitives at the top of the outer array (e.g. [1,2,3]) live at depth 0
+      // because the outer array is framing, not a stacked container.
+      if (this._arrayMode && this._arrayModeOuterOpen) {
+        this._state = S_ARRAY_VALUE_END;
+        return;
+      }
+      this._state = S_TOP;
+      return;
+    }
+    const c = this._stack[this._depth - 1];
+    this._state = c === C_OBJECT ? S_OBJECT_VALUE_END : S_ARRAY_VALUE_END;
+  }
+
+  _pushContainer(kind) {
+    if (this._depth >= MAX_DEPTH) this._err('E_DEPTH_LIMIT', 'nesting too deep');
+    this._stack[this._depth++] = kind;
+  }
+
+  _popContainer() {
+    this._depth--;
+    if (this._depth === 0) {
+      if (this._arrayMode) {
+        // element ended inside outer array; expect , or ]
+        this._state = S_ARRAY_VALUE_END;
+      } else {
+        this._state = S_TOP;
+      }
+      return;
+    }
+    const c = this._stack[this._depth - 1];
+    this._state = c === C_OBJECT ? S_OBJECT_VALUE_END : S_ARRAY_VALUE_END;
+  }
+
+  // -------- string buffer helpers --------
+
+  _appendStrByte(b) {
+    if (this._strLen >= this._strBuf.length) this._growStrBuf(1);
+    this._strBuf[this._strLen++] = b;
+  }
+
+  _appendStrRange(chunk, from, to) {
+    const need = to - from;
+    if (this._strLen + need > this._strBuf.length) this._growStrBuf(need);
+    // Manual copy loop — Uint8Array.set(source) via subarray allocates a small
+    // view header per call, which turns into MB-scale GC pressure across a
+    // string-heavy fixture. The loop is boring but keeps the hot path allocation-free.
+    const dst = this._strBuf;
+    let d = this._strLen;
+    for (let s = from; s < to; s++) dst[d++] = chunk[s];
+    this._strLen = d;
+  }
+
+  _appendUtf8Codepoint(cp) {
+    if (cp < 0x80) {
+      this._appendStrByte(cp);
+    } else if (cp < 0x800) {
+      if (this._strLen + 2 > this._strBuf.length) this._growStrBuf(2);
+      this._strBuf[this._strLen++] = 0xC0 | (cp >> 6);
+      this._strBuf[this._strLen++] = 0x80 | (cp & 0x3F);
+    } else if (cp < 0x10000) {
+      if (this._strLen + 3 > this._strBuf.length) this._growStrBuf(3);
+      this._strBuf[this._strLen++] = 0xE0 | (cp >> 12);
+      this._strBuf[this._strLen++] = 0x80 | ((cp >> 6) & 0x3F);
+      this._strBuf[this._strLen++] = 0x80 | (cp & 0x3F);
+    } else {
+      if (this._strLen + 4 > this._strBuf.length) this._growStrBuf(4);
+      this._strBuf[this._strLen++] = 0xF0 | (cp >> 18);
+      this._strBuf[this._strLen++] = 0x80 | ((cp >> 12) & 0x3F);
+      this._strBuf[this._strLen++] = 0x80 | ((cp >> 6) & 0x3F);
+      this._strBuf[this._strLen++] = 0x80 | (cp & 0x3F);
+    }
+  }
+
+  _growStrBuf(need) {
+    let cap = this._strBuf.length;
+    while (cap < this._strLen + need) cap *= 2;
+    if (cap > this._maxStringBytes) this._err('E_UNEXPECTED_BYTE', 'string exceeds maxStringBytes cap');
+    const nb = new Uint8Array(cap);
+    nb.set(this._strBuf);
+    this._strBuf = nb;
+  }
+
+  _hexVal(b) {
+    if (b >= B_0 && b <= B_9) return b - B_0;
+    if (b >= B_a && b <= B_f_low) return 10 + (b - B_a);
+    if (b >= B_A_up && b <= B_F_up) return 10 + (b - B_A_up);
+    return -1;
+  }
+
+  _err(code, msg) {
+    throw new TokenizerError(code, this._absOffset, msg);
+  }
+}
+
+