@jarenjs/josl 0.72.0 → 0.72.2

package/README.md

@@ -147,7 +147,10 @@ for await (const record of iterateJoslStream(response.body, { signal }))
 Every reader refuses a document by size only when asked: the limits
 default to `Infinity`, count **UTF-8 bytes** (never code units), and are
 checked while the offending text is still in cutter, token or container
-state — before a concatenation or a link could cross them. A crossing
+state. Limited streams inspect bounded slices and reject an oversized pending
+span before accepting another slice; complete spans are checked before decoding
+or linking values. Total bytes include a leading BOM, and splitting a surrogate
+pair across chunks does not change the byte count. A crossing
 throws `JoslLimitError` with a stable code and the limit; it is never a
 repair, because a document that is too large is not damaged.
 
@@ -159,7 +162,7 @@ repair, because a document that is too large is not damaged.
 | CSV | `maxColumns` | `CSV2004` |
 | JOSL / TOML | `maxTotalBytes` | `JOSL2001` |
 | JOSL / TOML | `maxRecordBytes` (one logical line) | `JOSL2002` |
-| JOSL / TOML | `maxTokenBytes` (a string or key as written, quotes included) | `JOSL2003` |
+| JOSL / TOML | `maxTokenBytes` (a key or scalar token as written, quotes and regexp flags included) | `JOSL2003` |
 | JOSL / TOML | `maxDepth` (inline nesting, header path depth) | `JOSL2004` |
 | JOSL / TOML | `maxRetainedValues` (values the root holds; starts over per detached `[[]]` item) | `JOSL2005` |
 | JSONX / JSON stream | `maxTotalBytes` | `JSONX2001` |
@@ -172,6 +175,11 @@ the same machines; the JSONX limits are the stream reader's, which is
 where a document arrives a chunk at a time. `CSV_LIMIT_CODES`,
 `JOSL_LIMIT_CODES` and `JSONX_LIMIT_CODES` are the tables as data.
 
+Field and token limits bound those spans, not an entire record containing many
+small spans. Use `maxRecordBytes` or `maxTotalBytes` as well when the buffered
+source itself needs a hard ceiling. Open strings, bare keys and scalar tokens
+are checked as they arrive, including in the first chunk.
+
 Measured on a synthetic OpenStreetMap-shaped extract (`node --expose-gc
 benchmark/jsonx-stream.js`), reading 20 000 features of 40 vertices each:
 
@@ -457,12 +465,12 @@ acceptance corpus, and on a scorecard of damaged documents.
 <!--fact:csv.table-->
 | engine | csv-spectrum | 10k×6 plain | 10k×3 quoted | 1k×50 wide |
 | --- | --- | --- | --- | --- |
-| **jaren** | **11/11** | 2.3 ms | 3.4 ms | 1.4 ms |
-| udsv | 11/11 | **1.9 ms** | **3.1 ms** | **1.0 ms** |
-| papaparse | 11/11 | 6.8 ms | 8.0 ms | 2.1 ms |
-| csv-parse | 11/11 | 18.8 ms | 12.1 ms | 11.0 ms |
-| d3-dsv | 11/11 | 2.8 ms | 4.3 ms | 1.7 ms |
-| @vanillaes/csv | n/a | 7.9 ms | 7.9 ms | 4.8 ms |
+| **jaren** | **11/11** | 2.2 ms | 3.0 ms | 1.3 ms |
+| udsv | 11/11 | **1.8 ms** | **2.9 ms** | **1.0 ms** |
+| papaparse | 11/11 | 5.8 ms | 7.5 ms | 2.1 ms |
+| csv-parse | 11/11 | 17.7 ms | 11.9 ms | 11.0 ms |
+| d3-dsv | 11/11 | 2.9 ms | 3.9 ms | 1.7 ms |
+| @vanillaes/csv | n/a | 5.9 ms | 7.3 ms | 4.6 ms |
 <!--/fact-->
 
 (The suite's twelfth fixture, `location_coordinates`, is excluded: its

package/dist/types/csv-machine.d.ts

@@ -45,6 +45,8 @@ export declare class CsvMachine {
     maxColumns: number;
     limited: boolean;
     totalBytes: number;
+    byteTail: any;
+    scanFieldStart: number;
     wantHeader: boolean;
     headerFields: any;
     objectRows: boolean;
@@ -76,7 +78,6 @@ export declare class CsvMachine {
      */
     feed(chunk: string): this;
     count(text: any): void;
-    endsRecordWithin(text: any): boolean;
     /**
      * Finish the document, flushing any pending record.
      * @returns {Array} The completed rows
@@ -101,6 +102,8 @@ export declare class CsvMachine {
     heal(code: any, line: any, column: any, detail?: undefined): void;
     scan(): void;
     compact(start: any): void;
+    /** Check a field span while its record is still incomplete. */
+    checkFieldBytes(text: any, start: any, end: any, recordStart?: number): void;
     readSpan(text: any, pos: any, end: any): any;
     admitCell(cells: any, text: any, start: any, stop: any): void;
     parseRecord(text: any, pos: any, end: any, cells: any): any;

package/dist/types/jsonx-stream.d.ts

@@ -8,6 +8,7 @@ export declare class JsonxMachine {
     maxDepth: number;
     maxRetainedValues: number;
     totalBytes: number;
+    byteTail: any;
     retained: number;
     partialFrom: number;
     partialHold: string;
@@ -21,6 +22,7 @@ export declare class JsonxMachine {
     curLine: number;
     lineStart: number;
     scanPos: number;
+    partialBoundary: boolean;
     scanInFlags: boolean;
     scanInClass: boolean;
     scanDtSpace: boolean;

package/dist/types/limits.d.ts

@@ -1,3 +1,11 @@
+/** Count a string stream without charging split surrogate pairs as two replacements.
+ * @param {{ byteTail?: number }} state @param {string} text @returns {number} */
+export declare function chunkByteLength(state: {
+    byteTail?: number;
+}, text: string): number;
+/** Let a cutter inspect bounded additions, including the first incomplete token.
+ * @param {string} chunk @param {number} limit @param {(part: string) => void} consume */
+export declare function feedBounded(chunk: string, limit: number, consume: (part: string) => void): void;
 /**
  * The CSV limits, by option name.
  * @type {Readonly<Record<'CSV2001' | 'CSV2002' | 'CSV2003' | 'CSV2004', string>>}

package/dist/types/machine.d.ts

@@ -9,6 +9,12 @@ export declare class JoslMachine {
     maxDepth: number;
     maxRetainedValues: number;
     totalBytes: number;
+    byteTail: any;
+    scanTokenStart: number;
+    scanBareStart: number;
+    scanKeyMode: boolean;
+    scanContainers: any[];
+    scanRegExpClass: boolean;
     retained: number;
     depth: number;
     onLine: any;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@jarenjs/josl",
   "private": false,
-  "version": "0.72.0",
+  "version": "0.72.2",
   "type": "module",
   "main": "./src/index.js",
   "types": "./dist/types/index.d.ts",
@@ -99,6 +99,6 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@jarenjs/core": "^0.72.0"
+    "@jarenjs/core": "^0.72.2"
   }
 }

package/src/csv-machine.js

@@ -48,7 +48,7 @@ import {
 } from '@jarenjs/core/scan';
 
 import { CsvSyntaxError } from './errors.js';
-import { JoslLimitError, limitOption } from './limits.js';
+import { JoslLimitError, limitOption, chunkByteLength, feedBounded } from './limits.js';
 import { columnOf, feedMachine, beginParseAll, offsetDateTime } from './util.js';
 import { setObjectMember } from '@jarenjs/core/object';
 import { utf8ByteLength } from '@jarenjs/core/string';
@@ -253,6 +253,8 @@ export class CsvMachine {
     this.limited = this.maxTotalBytes !== Infinity || this.maxRecordBytes !== Infinity
       || this.maxFieldBytes !== Infinity || this.maxColumns !== Infinity;
     this.totalBytes = 0;
+    this.byteTail = undefined;
+    this.scanFieldStart = 0;
 
     const headers = options.headers;
     this.wantHeader = headers === true;
@@ -299,44 +301,31 @@ export class CsvMachine {
    * @returns {this} The machine, for chaining
    */
   feed(chunk) {
-    // Growth is append-only, so a found cursor position stays valid; but
-    // a cursor that had run off the end must look again in the new data.
-    if (this.nextDelim === -1)
-      this.nextDelim = -2;
-    if (this.nextLf === -1)
-      this.nextLf = -2;
-    if (this.nextCr === -1)
-      this.nextCr = -2;
     if (this.limited)
       this.count(chunk);
-    return feedMachine(this, chunk);
+    feedBounded(chunk, Math.min(this.maxRecordBytes, this.maxFieldBytes), (part) => {
+      // Every bounded addition can extend a delimiter search that ran
+      // off the previous buffer, just as a caller's next chunk can.
+      if (this.nextDelim === -1) this.nextDelim = -2;
+      if (this.nextLf === -1) this.nextLf = -2;
+      if (this.nextCr === -1) this.nextCr = -2;
+      feedMachine(this, part);
+      if (this.maxRecordBytes !== Infinity && utf8ByteLength(this.buf) > this.maxRecordBytes)
+        throw new JoslLimitError('CSV2002', 'a record exceeds maxRecordBytes', this.maxRecordBytes, this.recordOrigin);
+      const end = this.scanState !== S_QUOTED && this.buf.endsWith('\r') ? this.buf.length - 1 : this.buf.length;
+      this.checkFieldBytes(this.buf, this.scanFieldStart, end);
+    });
+    return this;
   }
 
-  // The byte limits, checked BEFORE the chunk is buffered: a chunk that
-  // takes the document past maxTotalBytes is refused whole, and a record
-  // still being cut — the unconsumed tail plus this chunk — that would
-  // pass maxRecordBytes is refused before the concatenation that would
-  // hold it. `parseAll` counts its one text the same way.
+  // Whole-input accounting precedes buffering. Pending field and record
+  // spans are checked between bounded additions by feed().
   count(text) {
     if (this.maxTotalBytes !== Infinity) {
-      this.totalBytes += utf8ByteLength(text);
+      this.totalBytes += chunkByteLength(this, text);
       if (this.totalBytes > this.maxTotalBytes)
         throw new JoslLimitError('CSV2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, this.line);
     }
-    if (this.maxRecordBytes !== Infinity) {
-      // the pending record is what the cutter has not handed off yet;
-      // a fresh chunk extends it
-      const pending = utf8ByteLength(this.buf) + utf8ByteLength(text);
-      if (pending > this.maxRecordBytes && !this.endsRecordWithin(text))
-        throw new JoslLimitError('CSV2002', 'a record exceeds maxRecordBytes', this.maxRecordBytes, this.recordOrigin);
-    }
-  }
-
-  // Whether a chunk can end the pending record within the bound: cheap
-  // and permissive — a terminator anywhere in the chunk means the cutter
-  // gets its chance; the span check in readSpan is the exact judge.
-  endsRecordWithin(text) {
-    return text.indexOf('\n') >= 0 || text.indexOf('\r') >= 0;
   }
 
   /**
@@ -367,9 +356,9 @@ export class CsvMachine {
    * @returns {Array} The completed rows
    */
   parseAll(text) {
-    text = beginParseAll(this, text);
     if (this.maxTotalBytes !== Infinity && utf8ByteLength(text) > this.maxTotalBytes)
       throw new JoslLimitError('CSV2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, 1);
+    text = beginParseAll(this, text);
     this.readSpan(text, 0, text.length);
     return this.outRows;
   }
@@ -439,7 +428,7 @@ export class CsvMachine {
       // per-character machine. A record that does contain a quote is
       // handed to the machine below, one record at a time; one that
       // visibly STARTS with a quote skips the lookups outright.
-      if (state === S_PLAIN || (state === S_START && buf.charCodeAt(pos) !== quote)) {
+      if (this.maxFieldBytes === Infinity && (state === S_PLAIN || (state === S_START && buf.charCodeAt(pos) !== quote))) {
         if (qi !== -1 && qi < pos)
           qi = quote < 0 ? -1 : buf.indexOf(this.quoteChar, pos);
         const qlimit = qi < 0 ? len : qi;
@@ -509,6 +498,8 @@ export class CsvMachine {
             continue;
           case S_PLAIN: {
             if (c === delim) {
+              this.checkFieldBytes(buf, this.scanFieldStart, pos, start);
+              this.scanFieldStart = pos + 1;
               state = S_START;
               pos++;
               continue;
@@ -517,6 +508,7 @@ export class CsvMachine {
               pos++;
               this.readSpan(buf, start, pos);
               start = pos;
+              this.scanFieldStart = pos;
               state = S_START;
               continue outer;
             }
@@ -539,6 +531,7 @@ export class CsvMachine {
                 pos++;
               this.readSpan(buf, start, pos);
               start = pos;
+              this.scanFieldStart = pos;
               state = S_START;
               continue outer;
             }
@@ -597,6 +590,7 @@ export class CsvMachine {
       return;
     this.buf = this.buf.slice(start);
     this.scanPos -= start;
+    this.scanFieldStart = Math.max(0, this.scanFieldStart - start);
     // The scanner cursors are absolute in the buffer, so they shift with
     // it; one already consumed has no meaning in the new buffer.
     if (this.nextDelim >= 0)
@@ -611,6 +605,13 @@ export class CsvMachine {
 
   //#region record parsing
 
+  /** Check a field span while its record is still incomplete. */
+  checkFieldBytes(text, start, end, recordStart = 0) {
+    if (this.comment >= 0 && text.charCodeAt(recordStart) === this.comment) return;
+    if (this.maxFieldBytes !== Infinity && utf8ByteLength(text, start, end) > this.maxFieldBytes)
+      throw new JoslLimitError('CSV2003', 'a field exceeds maxFieldBytes', this.maxFieldBytes, this.line);
+  }
+
   // Read every record in `text[pos, end)`. A cut span always carries its
   // own terminator, so this is the same loop the whole-document form runs
   // and a blank line is a record of one empty field in both.

package/src/csv.js

@@ -166,7 +166,8 @@ export function sniffCsvDialect(text, options = undefined) {
   let bestWidth = 0;
   let bestRows = null;
 
-  for (const candidate of CANDIDATES) {
+  const candidates = opts.delimiter && opts.delimiter !== 'auto' ? [opts.delimiter] : CANDIDATES;
+  for (const candidate of candidates) {
     const probe = probeDelimiter(sample, candidate, opts);
     if (probe.score > bestScore || (probe.score === bestScore && probe.width > bestWidth)) {
       bestScore = probe.score;

package/src/jsonx-stream.js

@@ -79,7 +79,7 @@ import {
 } from '@jarenjs/core/scan';
 
 import { JsonxSyntaxError } from './errors.js';
-import { JoslLimitError, limitOption } from './limits.js';
+import { JoslLimitError, limitOption, chunkByteLength, feedBounded } from './limits.js';
 import { setObjectMember } from '@jarenjs/core/object';
 import { utf8ByteLength } from '@jarenjs/core/string';
 import {
@@ -162,6 +162,7 @@ export class JsonxMachine {
     this.maxDepth = limitOption(options, 'maxDepth');
     this.maxRetainedValues = limitOption(options, 'maxRetainedValues');
     this.totalBytes = 0;
+    this.byteTail = undefined;
     this.retained = 0; // values linked into the tree
     this.partialFrom = -1; // body offset the next text-partial delta starts at
     this.partialHold = ''; // lone high surrogate held back for the next delta
@@ -179,6 +180,7 @@ export class JsonxMachine {
     this.lineStart = 0; // buf offset of the current line start (may go negative after compaction)
     // resumable token-scan state
     this.scanPos = -1; // where the pending token's scan left off
+    this.partialBoundary = true;
     this.scanInFlags = false; // regexp scan: past the closing '/'
     this.scanInClass = false; // regexp scan: inside [...]
     this.scanDtSpace = false; // scalar scan: crossed a date-time space separator
@@ -198,19 +200,21 @@ export class JsonxMachine {
       throw new Error('cannot feed after end()');
     if (chunk.length !== 0) {
       if (this.maxTotalBytes !== Infinity) {
-        this.totalBytes += utf8ByteLength(chunk);
+        this.totalBytes += chunkByteLength(this, chunk);
         if (this.totalBytes > this.maxTotalBytes)
           throw new JoslLimitError('JSONX2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, this.curLine);
       }
-      if (this.maxTokenBytes !== Infinity && this.scanPos >= 0) {
-        // a token still being scanned grows by this chunk: refused
-        // before the concatenation that would hold it
-        const pending = utf8ByteLength(this.buf, this.pos, this.buf.length) + utf8ByteLength(chunk);
-        if (pending > this.maxTokenBytes)
-          throw new JoslLimitError('JSONX2002', 'a token exceeds maxTokenBytes', this.maxTokenBytes, this.curLine);
+      let remaining = chunk.length;
+      try {
+        feedBounded(chunk, this.maxTokenBytes, (part) => {
+          remaining -= part.length;
+          this.partialBoundary = remaining === 0;
+          this.buf += part;
+          this.pump();
+          if (this.scanPos >= 0) this.token(this.buf, this.pos, this.buf.length);
+        });
       }
-      this.buf += chunk;
-      this.pump();
+      finally { this.partialBoundary = true; }
     }
     return this;
   }
@@ -426,7 +430,8 @@ export class JsonxMachine {
     if (c === CC_DQUOTE) {
       const end = this.scanString(buf, pos);
       if (end < 0) {
-        if (this.partialText)
+        this.token(buf, pos, buf.length);
+        if (this.partialText && this.partialBoundary)
           this.emitPartialText(buf, pos, buf.length, true);
         return -1;
       }

package/src/limits.js

@@ -1,9 +1,32 @@
+//@ts-check
+import { utf8ByteLength } from '@jarenjs/core/string';
+
+/** Count a string stream without charging split surrogate pairs as two replacements.
+ * @param {{ byteTail?: number }} state @param {string} text @returns {number} */
+export function chunkByteLength(state, text) {
+  if (text.length === 0) return 0;
+  const first = text.charCodeAt(0);
+  const joined = state.byteTail >= 0xD800 && state.byteTail <= 0xDBFF
+    && first >= 0xDC00 && first <= 0xDFFF;
+  state.byteTail = text.charCodeAt(text.length - 1);
+  return utf8ByteLength(text) - (joined ? 2 : 0);
+}
+
+/** Let a cutter inspect bounded additions, including the first incomplete token.
+ * @param {string} chunk @param {number} limit @param {(part: string) => void} consume */
+export function feedBounded(chunk, limit, consume) {
+  if (limit === Infinity || chunk.length === 0) { consume(chunk); return; }
+  const size = Math.max(1, Math.min(16384, limit));
+  for (let offset = 0; offset < chunk.length; offset += size) consume(chunk.slice(offset, offset + size));
+}
+
 //#region hostile-input limits
 // The three readers and the CSV machine take the same shape of guard:
 // a limit that defaults to Infinity — nothing in this package refuses a
 // document by size unless a caller asks — and, once asked, is checked
 // while the offending text is still in cutter, token or container
-// state, before a concatenation or a link could cross it. Every limit
+// state. Limited streams inspect bounded additions before accepting another;
+// complete spans are checked before decoding or linking values. Every limit
 // counts UTF-8 bytes, never JavaScript code units: a limit stated in
 // bytes is the one an HTTP body limit, a disk quota or a proxy speaks.
 //

package/src/machine.js

@@ -53,7 +53,7 @@ import {
 } from '@jarenjs/core/scan';
 
 import { JoslSyntaxError } from './errors.js';
-import { JoslLimitError, limitOption } from './limits.js';
+import { JoslLimitError, limitOption, chunkByteLength, feedBounded } from './limits.js';
 import {
   LocalDate,
   LocalTime,
@@ -81,6 +81,8 @@ const S_LITERAL = 2; // '...'  (single line)
 const S_ML_BASIC = 3; // """..."""
 const S_ML_LITERAL = 4; // '''...'''
 const S_COMMENT = 5;
+const S_REGEXP = 6;
+const S_REGEXP_FLAGS = 7;
 
 // Runs of characters that cannot end a logical line or change the cutter's
 // state. The cutter skips them with the regex engine rather than stepping
@@ -89,7 +91,7 @@ const S_COMMENT = 5;
 // logical line spans without a separate walk over it.
 // The basic/literal classes serve both the single- and multi-line states:
 // a `'` is inert inside a basic string and a `"` inside a literal one.
-const RUN_NONE = /[^\n"'#[\]]*/y;
+const RUN_NONE = /[^\n"'#[\]/]*/y;
 const RUN_BASIC = /[^\n"\\]*/y;
 const RUN_LITERAL = /[^\n']*/y;
 
@@ -146,6 +148,12 @@ export class JoslMachine {
     this.maxDepth = limitOption(options, 'maxDepth');
     this.maxRetainedValues = limitOption(options, 'maxRetainedValues');
     this.totalBytes = 0;
+    this.byteTail = undefined;
+    this.scanTokenStart = -1;
+    this.scanBareStart = -1;
+    this.scanKeyMode = true;
+    this.scanContainers = [];
+    this.scanRegExpClass = false;
     this.retained = 0; // values linked into the root since the last detachment
     this.depth = 0; // inline container nesting while a value is parsed
     // Logical-line sink used by the CST layer: reports each line's source
@@ -185,18 +193,20 @@ export class JoslMachine {
    */
   feed(chunk) {
     if (this.maxTotalBytes !== Infinity) {
-      this.totalBytes += utf8ByteLength(chunk);
+      this.totalBytes += chunkByteLength(this, chunk);
       if (this.totalBytes > this.maxTotalBytes)
         throw new JoslLimitError('JOSL2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, this.startLine);
     }
-    if (this.maxRecordBytes !== Infinity && chunk.indexOf('\n') < 0
-      && utf8ByteLength(this.buf) + utf8ByteLength(chunk) > this.maxRecordBytes) {
-      // the logical line still being cut, plus this chunk, would pass the
-      // bound before any newline could end it: refused before the
-      // concatenation that would hold it
-      throw new JoslLimitError('JOSL2002', 'a logical line exceeds maxRecordBytes', this.maxRecordBytes, this.startLine);
-    }
-    return feedMachine(this, chunk);
+    feedBounded(chunk, Math.min(this.maxRecordBytes, this.maxTokenBytes), (part) => {
+      feedMachine(this, part);
+      const end = this.scanDepth === 0 && (this.scanState === S_NONE || this.scanState === S_COMMENT)
+        && this.buf.endsWith('\r') ? this.buf.length - 1 : this.buf.length;
+      if (this.maxRecordBytes !== Infinity && utf8ByteLength(this.buf, 0, end) > this.maxRecordBytes)
+        throw new JoslLimitError('JOSL2002', 'a logical line exceeds maxRecordBytes', this.maxRecordBytes, this.startLine);
+      if (this.scanTokenStart >= 0) this.token(this.buf, this.scanTokenStart, this.buf.length);
+      if (this.scanBareStart >= 0) this.token(this.buf, this.scanBareStart, this.buf.length);
+    });
+    return this;
   }
 
   /**
@@ -232,9 +242,9 @@ export class JoslMachine {
    * @returns {*} The completed root value
    */
   parseAll(text) {
-    text = beginParseAll(this, text);
     if (this.maxTotalBytes !== Infinity && utf8ByteLength(text) > this.maxTotalBytes)
       throw new JoslLimitError('JOSL2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, 1);
+    text = beginParseAll(this, text);
     // positions are offsets into the whole source, which starts at line 1
     this.lineOrigin = 1;
     const tracking = this.onEvent !== null;
@@ -327,20 +337,54 @@ export class JoslMachine {
     let state = this.scanState;
     let depth = this.scanDepth;
     let nl = this.scanNl;
+    let tokenStart = this.scanTokenStart;
+    let bareStart = this.scanBareStart;
+    let keyMode = this.scanKeyMode;
+    const containers = this.scanContainers;
+    const limited = this.maxTokenBytes !== Infinity;
     const ended = this.ended;
     outer:
     while (pos < buf.length) {
       switch (state) {
         case S_NONE: {
-          pos = skipRun(RUN_NONE, buf, pos);
+          if (!limited) pos = skipRun(RUN_NONE, buf, pos);
           if (pos >= buf.length)
             break outer;
           const c = buf.charCodeAt(pos);
+          if (limited) {
+            const boundary = c === CC_SPACE || c === CC_TAB || c === CC_CR || c === CC_LF
+              || c === CC_EQ || c === CC_COMMA || c === CC_LBRACE || c === CC_RBRACE
+              || c === CC_LBRACKET || c === CC_RBRACKET || c === CC_HASH
+              || c === CC_DQUOTE || c === CC_SQUOTE || (keyMode && c === CC_DOT)
+              || (!keyMode && c === CC_SLASH);
+            if (boundary) {
+              if (bareStart >= 0) this.token(buf, bareStart, pos);
+              bareStart = -1;
+            }
+            else if (bareStart < 0) bareStart = pos;
+            if (c === CC_EQ) keyMode = false;
+            else if (c === CC_LBRACE) { containers.push('table'); keyMode = true; }
+            else if (c === CC_LBRACKET) {
+              const header = keyMode && (containers.length === 0 || containers.at(-1) === 'header');
+              containers.push(header ? 'header' : 'array');
+              keyMode = header;
+            }
+            else if (c === CC_RBRACKET || c === CC_RBRACE) { containers.pop(); keyMode = false; }
+            else if (c === CC_COMMA) keyMode = containers.at(-1) === 'table';
+          }
+          if (c === CC_SLASH) {
+            tokenStart = pos++;
+            this.scanRegExpClass = false;
+            state = S_REGEXP;
+            break;
+          }
           if (c === CC_LF) {
             if (depth === 0) {
               this.cutLine(buf, lineStart, pos, nl);
               lineStart = pos + 1;
               nl = 0;
+              keyMode = true;
+              containers.length = 0;
             }
             else
               nl++;
@@ -348,6 +392,7 @@ export class JoslMachine {
             break;
           }
           if (c === CC_DQUOTE || c === CC_SQUOTE) {
+            tokenStart = pos;
             if (pos + 2 >= buf.length && !ended)
               break outer; // may be a triple delimiter split across chunks
             if (buf.charCodeAt(pos + 1) === c && buf.charCodeAt(pos + 2) === c) {
@@ -383,6 +428,8 @@ export class JoslMachine {
             this.cutLine(buf, lineStart, at, nl);
             lineStart = at + 1;
             nl = 0;
+            keyMode = true;
+            containers.length = 0;
           }
           else
             nl++;
@@ -390,6 +437,28 @@ export class JoslMachine {
           pos = at + 1;
           break;
         }
+        case S_REGEXP: {
+          const c = buf.charCodeAt(pos);
+          if (c === CC_BACKSLASH) {
+            if (pos + 1 >= buf.length && !ended) break outer;
+            pos += 2;
+            break;
+          }
+          if (c === CC_LF) this.cutLine(buf, lineStart, pos, nl);
+          if (c === CC_LBRACKET) this.scanRegExpClass = true;
+          else if (c === CC_RBRACKET) this.scanRegExpClass = false;
+          else if (c === CC_SLASH && !this.scanRegExpClass) state = S_REGEXP_FLAGS;
+          pos++;
+          break;
+        }
+        case S_REGEXP_FLAGS:
+          if (isAsciiLetterCode(buf.charCodeAt(pos))) pos++;
+          else {
+            this.token(buf, tokenStart, pos);
+            tokenStart = -1;
+            state = S_NONE;
+          }
+          break;
         case S_BASIC:
         case S_LITERAL: {
           const basic = state === S_BASIC;
@@ -418,6 +487,8 @@ export class JoslMachine {
           }
           state = S_NONE; // the run only stops on the closing quote
           pos++;
+          this.token(buf, tokenStart, pos);
+          tokenStart = -1;
           break;
         }
         case S_ML_BASIC:
@@ -444,8 +515,11 @@ export class JoslMachine {
             run++;
           if (run === buf.length && run - pos < 3 && !ended)
             break outer; // quote run may continue in the next chunk
-          if (run - pos >= 3)
+          if (run - pos >= 3) {
             state = S_NONE;
+            this.token(buf, tokenStart, run);
+            tokenStart = -1;
+          }
           pos = run;
           break;
         }
@@ -460,6 +534,9 @@ export class JoslMachine {
     this.scanState = state;
     this.scanDepth = depth;
     this.scanNl = nl;
+    this.scanTokenStart = tokenStart < 0 ? -1 : tokenStart - lineStart;
+    this.scanBareStart = bareStart < 0 ? -1 : bareStart - lineStart;
+    this.scanKeyMode = keyMode;
   }
 
   // `innerNl` is how many newlines the cutter already counted inside this
@@ -876,6 +953,7 @@ export class JoslMachine {
     const start = pos;
     while (pos < line.length && isAsciiLetterCode(line.charCodeAt(pos)))
       pos++;
+    this.token(line, start, pos);
     const word = line.slice(start, pos);
     switch (word) {
       case 'true': return [true, this.checkValueEnd(line, pos)];
@@ -899,6 +977,7 @@ export class JoslMachine {
     const start = p;
     while (p < line.length && isAsciiLetterCode(line.charCodeAt(p)))
       p++;
+    this.token(line, pos, p);
     const word = line.slice(start, p);
     if (word === 'inf')
       return [neg ? -Infinity : Infinity, this.checkValueEnd(line, p)];
@@ -1216,11 +1295,13 @@ export class JoslMachine {
         break;
       pos++;
     }
+    this.token(line, start, pos + 1);
     const body = line.slice(start + 1, pos);
     pos++; // consume '/'
     const flagStart = pos;
     while (pos < line.length && isAsciiLetterCode(line.charCodeAt(pos)))
       pos++;
+    this.token(line, start, pos);
     const flags = line.slice(flagStart, pos);
     try {
       return [new RegExp(body, flags), this.checkValueEnd(line, pos)];
@@ -1237,6 +1318,7 @@ export class JoslMachine {
       return this.parseNumber(line, pos);
     let m = stickyExec(RE_DATETIME, line, pos);
     if (m !== null) {
+      this.token(line, pos, pos + m[0].length);
       const year = Number(m[1]);
       const month = Number(m[2]);
       const day = Number(m[3]);
@@ -1261,6 +1343,7 @@ export class JoslMachine {
     }
     m = stickyExec(RE_TIMEONLY, line, pos);
     if (m !== null) {
+      this.token(line, pos, pos + m[0].length);
       const hour = Number(m[1]);
       const minute = Number(m[2]);
       const second = Number(m[3]);
@@ -1309,6 +1392,7 @@ export class JoslMachine {
       while (p < line.length && isDigitCode(line.charCodeAt(p)))
         p++;
       if ((p - pos === 1 || c0 !== CC_0) && atValueEnd(line, p)) {
+        this.token(line, pos, p);
         const source = line.slice(pos, p);
         return [this.intValue(pos, source, false, source), p];
       }
@@ -1325,6 +1409,7 @@ export class JoslMachine {
         m = stickyExec(RE_BIN, line, pos);
     }
     if (m !== null) {
+      this.token(line, pos, pos + m[0].length);
       const big = this.bigIntCheck(pos, m[1]);
       const stripped = (big ? m[0].slice(0, -1) : m[0]).replace(/_/g, '');
       const end = this.checkValueEnd(line, pos + m[0].length);
@@ -1333,6 +1418,7 @@ export class JoslMachine {
     m = stickyExec(RE_NUM, line, pos);
     if (m === null)
       this.err(pos, 'invalid number');
+    this.token(line, pos, pos + m[0].length);
     const big = this.bigIntCheck(pos, m[1]);
     const token = big ? m[0].slice(0, -1) : m[0];
     const isFloat = /[.eE]/.test(token);