@jarenjs/josl 0.56.0 → 0.67.0

package/src/limits.js

@@ -0,0 +1,96 @@
+//#region hostile-input limits
+// The three readers and the CSV machine take the same shape of guard:
+// a limit that defaults to Infinity — nothing in this package refuses a
+// document by size unless a caller asks — and, once asked, is checked
+// while the offending text is still in cutter, token or container
+// state, before a concatenation or a link could cross it. Every limit
+// counts UTF-8 bytes, never JavaScript code units: a limit stated in
+// bytes is the one an HTTP body limit, a disk quota or a proxy speaks.
+//
+// A crossing is a `JoslLimitError` with a stable code — `CSV2xxx`,
+// `JOSL2xxx`, `JSONX2xxx` — and the limit that was crossed. It is never
+// a repair: repair mode heals damaged syntax, and a document that is
+// too large is not damaged, it is refused.
+
+/**
+ * The CSV limits, by option name.
+ * @type {Readonly<Record<'CSV2001' | 'CSV2002' | 'CSV2003' | 'CSV2004', string>>}
+ */
+export const CSV_LIMIT_CODES = Object.freeze({
+  CSV2001: 'the document exceeds maxTotalBytes',
+  CSV2002: 'a record exceeds maxRecordBytes',
+  CSV2003: 'a field exceeds maxFieldBytes',
+  CSV2004: 'a record has more than maxColumns fields',
+});
+
+/**
+ * The JOSL/TOML limits, by option name. A "record" is one logical line —
+ * the unit the reader buffers before it parses anything.
+ * @type {Readonly<Record<'JOSL2001' | 'JOSL2002' | 'JOSL2003' | 'JOSL2004' | 'JOSL2005', string>>}
+ */
+export const JOSL_LIMIT_CODES = Object.freeze({
+  JOSL2001: 'the document exceeds maxTotalBytes',
+  JOSL2002: 'a logical line exceeds maxRecordBytes',
+  JOSL2003: 'a token exceeds maxTokenBytes',
+  JOSL2004: 'the document nests deeper than maxDepth',
+  JOSL2005: 'the root retains more than maxRetainedValues values',
+});
+
+/**
+ * The JSONX/JSON limits, by option name.
+ * @type {Readonly<Record<'JSONX2001' | 'JSONX2002' | 'JSONX2003' | 'JSONX2004', string>>}
+ */
+export const JSONX_LIMIT_CODES = Object.freeze({
+  JSONX2001: 'the document exceeds maxTotalBytes',
+  JSONX2002: 'a token exceeds maxTokenBytes',
+  JSONX2003: 'the document nests deeper than maxDepth',
+  JSONX2004: 'the root retains more than maxRetainedValues values',
+});
+
+/** The option name each code guards. */
+const OPTION_OF = Object.freeze({
+  CSV2001: 'maxTotalBytes', CSV2002: 'maxRecordBytes', CSV2003: 'maxFieldBytes', CSV2004: 'maxColumns',
+  JOSL2001: 'maxTotalBytes', JOSL2002: 'maxRecordBytes', JOSL2003: 'maxTokenBytes', JOSL2004: 'maxDepth', JOSL2005: 'maxRetainedValues',
+  JSONX2001: 'maxTotalBytes', JSONX2002: 'maxTokenBytes', JSONX2003: 'maxDepth', JSONX2004: 'maxRetainedValues',
+});
+
+/**
+ * Error thrown when a document crosses a limit a caller set. Never a
+ * repair, never healed: the text is refused where it stands.
+ */
+export class JoslLimitError extends Error {
+  /**
+   * @param {string} code - The stable `CSV2xxx` / `JOSL2xxx` / `JSONX2xxx` code
+   * @param {string} message - What was crossed
+   * @param {number} limit - The limit the option set
+   * @param {number} [line] - 1-based physical line where the crossing was met, when known
+   */
+  constructor(code, message, limit, line = undefined) {
+    super(`${code}: ${message} (${OPTION_OF[code]} ${limit})${line === undefined ? '' : ` at line ${line}`}`);
+    this.name = 'JoslLimitError';
+    this.code = code;
+    this.limit = limit;
+    this.line = line;
+  }
+}
+
+/**
+ * Read one limit option: absent is `Infinity`; anything but a positive
+ * integer or `Infinity` is a `TypeError` — a limit of zero refuses every
+ * document, and a fraction or a string is a caller's mistake.
+ * @param {object} options - The reader/writer options
+ * @param {string} name - The option name
+ * @returns {number} The limit, `Infinity` when unset
+ */
+export function limitOption(options, name) {
+  const value = options[name];
+  if (value === undefined || value === null)
+    return Infinity;
+  if (value === Infinity)
+    return Infinity;
+  if (typeof value !== 'number' || !Number.isInteger(value) || value < 1)
+    throw new TypeError(`options.${name} must be a positive integer or Infinity`);
+  return value;
+}
+
+//#endregion

package/src/machine.js

@@ -53,6 +53,7 @@ import {
 } from '@jarenjs/core/scan';
 
 import { JoslSyntaxError } from './errors.js';
+import { JoslLimitError, limitOption } from './limits.js';
 import {
   LocalDate,
   LocalTime,
@@ -61,10 +62,10 @@ import {
   isValidTimeParts,
 } from './values.js';
 import {
-  getOwn, columnOf, feedMachine, beginParseAll, stickyExec, RE_DATETIME, RE_TIMEONLY,
+  getOwn, columnOf, feedMachine, beginParseAll, stickyExec, RE_DATETIME, RE_TIMEONLY, offsetDateTime,
 } from './util.js';
 import { setObjectMember } from '@jarenjs/core/object';
-import { countCharCode } from '@jarenjs/core/string';
+import { countCharCode, utf8ByteLength } from '@jarenjs/core/string';
 
 function isBareKeyCode(c) {
   return isAsciiLetterCode(c)
@@ -133,6 +134,20 @@ export class JoslMachine {
   constructor(options = {}) {
     this.mode = options.mode === 'toml' ? 'toml' : 'josl';
     this.onEvent = options.onEvent ?? null;
+    // Root-item detachment: when set, a completed `[[]]` item is handed
+    // to it and dropped from the root, so the root never holds more
+    // than the item in progress (iterateJoslStream's reason to exist).
+    this.detachRoot = options.detachRoot ?? null;
+    this.rootCount = 0; // root items opened so far (indices survive detachment)
+    // the hostile-input limits: Infinity unless asked for (limits.js)
+    this.maxTotalBytes = limitOption(options, 'maxTotalBytes');
+    this.maxRecordBytes = limitOption(options, 'maxRecordBytes');
+    this.maxTokenBytes = limitOption(options, 'maxTokenBytes');
+    this.maxDepth = limitOption(options, 'maxDepth');
+    this.maxRetainedValues = limitOption(options, 'maxRetainedValues');
+    this.totalBytes = 0;
+    this.retained = 0; // values linked into the root since the last detachment
+    this.depth = 0; // inline container nesting while a value is parsed
     // Logical-line sink used by the CST layer: reports each line's source
     // span, and the span of a pair's value inside it, so a rewriter can
     // replace a value without disturbing the bytes around it.
@@ -169,6 +184,18 @@ export class JoslMachine {
    * @returns {this} The machine, for chaining
    */
   feed(chunk) {
+    if (this.maxTotalBytes !== Infinity) {
+      this.totalBytes += utf8ByteLength(chunk);
+      if (this.totalBytes > this.maxTotalBytes)
+        throw new JoslLimitError('JOSL2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, this.startLine);
+    }
+    if (this.maxRecordBytes !== Infinity && chunk.indexOf('\n') < 0
+      && utf8ByteLength(this.buf) + utf8ByteLength(chunk) > this.maxRecordBytes) {
+      // the logical line still being cut, plus this chunk, would pass the
+      // bound before any newline could end it: refused before the
+      // concatenation that would hold it
+      throw new JoslLimitError('JOSL2002', 'a logical line exceeds maxRecordBytes', this.maxRecordBytes, this.startLine);
+    }
     return feedMachine(this, chunk);
   }
 
@@ -206,6 +233,8 @@ export class JoslMachine {
    */
   parseAll(text) {
     text = beginParseAll(this, text);
+    if (this.maxTotalBytes !== Infinity && utf8ByteLength(text) > this.maxTotalBytes)
+      throw new JoslLimitError('JOSL2001', 'the document exceeds maxTotalBytes', this.maxTotalBytes, 1);
     // positions are offsets into the whole source, which starts at line 1
     this.lineOrigin = 1;
     const tracking = this.onEvent !== null;
@@ -215,6 +244,11 @@ export class JoslMachine {
       this.lineValueStart = -1;
       this.lineValueEnd = -1;
       const end = this.parseLine(text, pos);
+      // the whole document is already in memory here: a logical line is
+      // judged by its extent once the parser found it (the streaming
+      // path refuses it earlier, while it is still being cut)
+      if (this.maxRecordBytes !== Infinity && utf8ByteLength(text, pos, end) > this.maxRecordBytes)
+        throw new JoslLimitError('JOSL2002', 'a logical line exceeds maxRecordBytes', this.maxRecordBytes, this.startLine);
       const next = end < len && text.charCodeAt(end) === CC_LF ? end + 1 : end;
       if (this.onLine !== null)
         this.onLine(pos, next, this.lineValueStart, this.lineValueEnd);
@@ -235,14 +269,47 @@ export class JoslMachine {
   /**
    * The (possibly still growing) root value: `{}`-rooted for documents,
    * `[]`-rooted after a `[[]]` header. Undefined content yields `{}`.
+   * Under `detachRoot` a finished document hands its last item over
+   * here, so the root ends empty.
    * @returns {*} Current root value
    */
   root() {
     if (this.rootValue === undefined)
       this.rootValue = {};
+    if (this.ended && this.detachRoot !== null && this.rootIsArray && this.rootValue.length !== 0)
+      this.detachLast();
     return this.rootValue;
   }
 
+  // Hand the last root item to the detach sink and drop it: the root
+  // keeps at most the item in progress, and the retained-value count
+  // starts over with it.
+  detachLast() {
+    const item = this.rootValue.pop();
+    this.retained = 0;
+    this.detachRoot(item);
+  }
+
+  // One more value linked into the root: a pair, an array element, an
+  // inline-table member, a table or a root item. Judged before the link.
+  retain() {
+    if (++this.retained > this.maxRetainedValues)
+      throw new JoslLimitError('JOSL2005', 'the root retains more than maxRetainedValues values', this.maxRetainedValues, this.startLine);
+  }
+
+  // One nesting level deeper — an inline array or table, or a header
+  // path segment — judged before it opens.
+  deeper(depth) {
+    if (depth > this.maxDepth)
+      throw new JoslLimitError('JOSL2004', 'the document nests deeper than maxDepth', this.maxDepth, this.startLine);
+  }
+
+  // A token about to be sliced: its byte size, judged first.
+  token(line, start, end) {
+    if (this.maxTokenBytes !== Infinity && utf8ByteLength(line, start, end) > this.maxTokenBytes)
+      throw new JoslLimitError('JOSL2003', 'a token exceeds maxTokenBytes', this.maxTokenBytes, this.startLine);
+  }
+
   //#endregion
 
   //#region chunk cutter
@@ -402,6 +469,8 @@ export class JoslMachine {
       ? nlPos - 1
       : nlPos;
     if (end > start) {
+      if (this.maxRecordBytes !== Infinity && utf8ByteLength(buf, start, end) > this.maxRecordBytes)
+        throw new JoslLimitError('JOSL2002', 'a logical line exceeds maxRecordBytes', this.maxRecordBytes, this.startLine);
       this.lineOrigin = this.startLine;
       this.parseLine(buf.slice(start, end));
     }
@@ -544,7 +613,7 @@ export class JoslMachine {
 
   headerBase() {
     if (this.rootIsArray)
-      return [this.rootValue[this.rootValue.length - 1], [this.rootValue.length - 1]];
+      return [this.rootValue[this.rootValue.length - 1], [this.rootCount - 1]];
     if (this.rootValue === undefined)
       this.rootValue = {};
     return [this.rootValue, []];
@@ -554,6 +623,7 @@ export class JoslMachine {
   // descending into the last element of arrays-of-tables
   navigate(keys, pos) {
     let [t, path] = this.headerBase();
+    this.deeper(keys.length);
     for (let i = 0; i < keys.length - 1; ++i) {
       const k = keys[i];
       const ex = getOwn(t, k);
@@ -593,6 +663,7 @@ export class JoslMachine {
     const k = keys[keys.length - 1];
     const ex = getOwn(t, k);
     if (ex === undefined) {
+      this.retain();
       const nt = {};
       this.meta.set(nt, { explicit: true });
       setObjectMember(t, k, nt);
@@ -622,6 +693,7 @@ export class JoslMachine {
     }
     else if (!Array.isArray(arr) || this.meta.get(arr)?.aot !== true)
       this.err(pos, `key '${keys.join('.')}' is not an array of tables`);
+    this.retain();
     const el = {};
     arr.push(el);
     this.current = el;
@@ -641,14 +713,20 @@ export class JoslMachine {
     else if (!this.rootIsArray)
       this.err(pos, 'cannot mix a root table and a root array',
         'a document that starts with key-value pairs has an object root');
+    // the previous item is complete the moment the next header opens:
+    // under detachment it leaves the root here
+    if (this.detachRoot !== null && this.rootValue.length !== 0)
+      this.detachLast();
+    this.retain();
     const el = {};
     this.rootValue.push(el);
+    this.rootCount++;
     this.current = el;
-    this.currentPath = [this.rootValue.length - 1];
+    this.currentPath = [this.rootCount - 1];
     this.emit({
       type: 'root-item',
       path: this.currentPath,
-      index: this.rootValue.length - 1,
+      index: this.rootCount - 1,
       line: this.startLine,
     });
   }
@@ -710,6 +788,7 @@ export class JoslMachine {
     const k = keys[keys.length - 1];
     if (Object.hasOwn(t, k))
       this.err(pos, `duplicate key '${k}'`);
+    this.retain();
     setObjectMember(t, k, value);
   }
 
@@ -736,6 +815,7 @@ export class JoslMachine {
           pos++;
         if (pos === start)
           this.err(pos, 'expected a key');
+        this.token(line, start, pos);
         keys.push(line.slice(start, pos));
       }
       pos = this.skipWs(line, pos);
@@ -875,13 +955,16 @@ export class JoslMachine {
   }
 
   parseBasicString(line, pos) {
+    const opened = pos;
     pos++; // consume '"'
     let out = '';
     let chunk = pos;
     while (pos < line.length) {
       const c = line.charCodeAt(pos);
-      if (c === CC_DQUOTE)
+      if (c === CC_DQUOTE) {
+        this.token(line, opened, pos + 1);
         return [out + line.slice(chunk, pos), pos + 1];
+      }
       if (c === CC_BACKSLASH) {
         out += line.slice(chunk, pos);
         const [dec, p] = this.decodeEscape(line, pos);
@@ -903,8 +986,10 @@ export class JoslMachine {
     const start = pos;
     while (pos < line.length) {
       const c = line.charCodeAt(pos);
-      if (c === CC_SQUOTE)
+      if (c === CC_SQUOTE) {
+        this.token(line, start - 1, pos + 1);
         return [line.slice(start, pos), pos + 1];
+      }
       if (c === CC_LF)
         break;
       this.checkStringChar(line, pos, false);
@@ -914,6 +999,7 @@ export class JoslMachine {
   }
 
   parseMlBasicString(line, pos) {
+    const mlStart = pos;
     pos += 3; // consume '"""'
     if (line.charCodeAt(pos) === CC_CR && line.charCodeAt(pos + 1) === CC_LF)
       pos += 2;
@@ -931,6 +1017,7 @@ export class JoslMachine {
         if (n >= 3) {
           if (n > 5)
             this.err(pos, 'too many quotes closing a multi-line string');
+          this.token(line, mlStart, run);
           return [out + line.slice(chunk, pos) + '"'.repeat(n - 3), run];
         }
         pos = run;
@@ -991,6 +1078,7 @@ export class JoslMachine {
         if (n >= 3) {
           if (n > 5)
             this.err(pos, 'too many quotes closing a multi-line string');
+          this.token(line, start - 3, run);
           return [line.slice(start, pos) + "'".repeat(n - 3), run];
         }
         pos = run;
@@ -1008,15 +1096,19 @@ export class JoslMachine {
 
   parseArray(line, pos) {
     pos++; // consume '['
+    this.deeper(++this.depth);
     const arr = [];
     this.meta.set(arr, { aot: false });
     for (;;) {
       pos = this.skipWsNlComment(line, pos);
       if (pos >= line.length)
         this.err(pos, 'unterminated array', "close the array with ']'");
-      if (line.charCodeAt(pos) === CC_RBRACKET)
+      if (line.charCodeAt(pos) === CC_RBRACKET) {
+        this.depth--;
         return [arr, pos + 1];
+      }
       const [v, p] = this.parseValue(line, pos);
+      this.retain();
       arr.push(v);
       pos = this.skipWsNlComment(line, p);
       if (pos >= line.length)
@@ -1026,19 +1118,24 @@ export class JoslMachine {
         pos++;
         continue;
       }
-      if (c === CC_RBRACKET)
+      if (c === CC_RBRACKET) {
+        this.depth--;
         return [arr, pos + 1];
+      }
       this.err(pos, "expected ',' or ']' in array");
     }
   }
 
   parseInlineTable(line, pos) {
     pos++; // consume '{'
+    this.deeper(++this.depth);
     const obj = {};
     this.meta.set(obj, { inline: true });
     pos = this.skipWs(line, pos);
-    if (pos < line.length && line.charCodeAt(pos) === CC_RBRACE)
+    if (pos < line.length && line.charCodeAt(pos) === CC_RBRACE) {
+      this.depth--;
       return [obj, pos + 1];
+    }
     for (;;) {
       if (pos < line.length && line.charCodeAt(pos) === CC_LF)
         this.err(pos, 'newlines are not allowed inside inline tables',
@@ -1054,8 +1151,10 @@ export class JoslMachine {
       if (pos >= line.length)
         this.err(pos, 'unterminated inline table', "close the table with '}'");
       const c = line.charCodeAt(pos);
-      if (c === CC_RBRACE)
+      if (c === CC_RBRACE) {
+        this.depth--;
         return [obj, pos + 1];
+      }
       if (c === CC_COMMA) {
         pos = this.skipWs(line, pos + 1);
         continue;
@@ -1086,6 +1185,7 @@ export class JoslMachine {
     const k = keys[keys.length - 1];
     if (Object.hasOwn(t, k))
       this.err(pos, `duplicate key '${k}'`);
+    this.retain();
     setObjectMember(t, k, value);
   }
 
@@ -1154,8 +1254,7 @@ export class JoslMachine {
       const end = this.checkValueEnd(line, pos + m[0].length);
       if (m[8] === undefined)
         return [new LocalDateTime(date, time), end];
-      const offset = m[8] === 'z' || m[8] === 'Z' ? 'Z' : m[8];
-      const instant = new Date(`${date.toString()}T${time.toString()}${offset}`);
+      const instant = offsetDateTime(date, time, m[8]);
       if (Number.isNaN(instant.getTime()))
         this.err(pos, `invalid date-time '${m[0]}'`);
       return [instant, end];

package/src/pull.js

@@ -0,0 +1,40 @@
+//#region pull helpers shared by the async serializers and readers
+// The two facts every pull pipeline in this package relies on, written
+// once: closing an upstream iterator exactly once (a consumer that stops
+// early, an abort, a throw — never a source read to its end, whose
+// `return()` would be a second close), and the rejection an abort
+// carries (the signal's own reason, so a caller's `AbortError` is the
+// one it sees).
+
+/**
+ * Close an iterator once, swallowing what the close throws: a source
+ * that refuses its cancel is already gone.
+ * @param {Iterator<any> | AsyncIterator<any>} iterator
+ * @returns {Promise<void>}
+ */
+export async function closeIterator(iterator) {
+  if (typeof iterator.return !== 'function')
+    return;
+  try {
+    await iterator.return(undefined);
+  }
+  catch {
+    // the upstream refused its close; nothing more can be done for it
+  }
+}
+
+/**
+ * The rejection of an aborted pull: the signal's reason when it has one,
+ * else an `AbortError`-named error, as the platform spells it.
+ * @param {AbortSignal} signal
+ * @returns {unknown}
+ */
+export function abortedError(signal) {
+  if (signal.reason !== undefined)
+    return signal.reason;
+  const error = new Error('The operation was aborted.');
+  error.name = 'AbortError';
+  return error;
+}
+
+//#endregion

package/src/stream.js

@@ -7,6 +7,8 @@
 // array-of-tables indices), so events are directly JSON-Pointer-able.
 
 import { JoslMachine } from './machine.js';
+import { JoslSyntaxError } from './errors.js';
+import { closeIterator, abortedError } from './pull.js';
 
 /**
  * Create an incremental JOSL/TOML reader.
@@ -52,6 +54,62 @@ export async function parseJoslStream(chunks, options = undefined) {
   return machine.end();
 }
 
+/**
+ * Yield the `[[]]` root items of a JOSL document as they complete,
+ * without ever holding the whole document: each item is DETACHED from
+ * the root the moment the next `[[]]` header (or the end) completes it,
+ * so `root()`-style retention never grows past one record — the JOSL
+ * twin of the JSONX reader's `detach`. A document with a table root is
+ * refused by name (`JoslSyntaxError`): a table root is one retained
+ * value, not a stream of records. `options.signal` aborts between
+ * chunks; an abort, a consumer that stops early or a throw closes the
+ * chunk source exactly once.
+ * @param {AsyncIterable<string>|Iterable<string>} chunks - Source chunks
+ * @param {object} [options] - Reader options; see `createStreamReader`,
+ *  plus `signal` and the `JOSL2xxx` limits
+ * @yields {object} One completed root item at a time
+ * @example
+ * for await (const record of iterateJoslStream(response.body))
+ *   await save(record);
+ */
+export async function* iterateJoslStream(chunks, options = undefined) {
+  /** @type {object[]} */
+  const completed = [];
+  const machine = new JoslMachine({
+    ...(options ?? {}),
+    detachRoot: (item) => { completed.push(item); },
+  });
+  const signal = options?.signal ?? null;
+  const iterator = chunks[Symbol.asyncIterator]?.() ?? chunks[Symbol.iterator]();
+  let finished = false;
+  try {
+    for (;;) {
+      if (signal !== null && signal.aborted)
+        throw abortedError(signal);
+      const step = await iterator.next();
+      if (step.done) {
+        finished = true;
+        break;
+      }
+      machine.feed(step.value);
+      if (machine.rootValue !== undefined && !machine.rootIsArray) {
+        throw new JoslSyntaxError('iterateJoslStream reads a [[]] root array; this document has a table root',
+          machine.lineOrigin, 1, 'stream a document of [[]] records, or read a table root with parseJoslStream');
+      }
+      while (completed.length !== 0)
+        yield /** @type {object} */ (completed.shift());
+    }
+  }
+  finally {
+    if (!finished)
+      await closeIterator(iterator);
+  }
+  machine.end();
+  while (completed.length !== 0)
+    yield /** @type {object} */ (completed.shift());
+}
+
 export { JoslSyntaxError } from './errors.js';
+export { JoslLimitError, JOSL_LIMIT_CODES } from './limits.js';
 
 //#endregion

package/src/util.js

@@ -17,6 +17,21 @@ export const RE_DATETIME = /(\d{4})-(\d{2})-(\d{2})(?:[Tt ](\d{2}):(\d{2}):(\d{2
 /** A bare `HH:MM:SS` with an optional fraction. @type {RegExp} */
 export const RE_TIMEONLY = /(\d{2}):(\d{2}):(\d{2})(\.\d+)?/y;
 
+/**
+ * Convert a local date and time with an offset to their native instant.
+ * ISO parsing preserves years 0000-0099 and truncates sub-millisecond
+ * precision consistently for JOSL and typed CSV. An invalid offset (or
+ * an instant Date cannot represent) returns an invalid Date for the
+ * caller's own refusal or text-preservation policy.
+ * @param {import('./values.js').LocalDate} date - Validated calendar date
+ * @param {import('./values.js').LocalTime} time - Validated local time
+ * @param {string} offset - Z/z or a signed HH:MM offset
+ * @returns {Date}
+ */
+export function offsetDateTime(date, time, offset) {
+  return new Date(`${date.toString()}T${time.toString()}${offset.toUpperCase()}`);
+}
+
 /**
  * Run a sticky regex at `pos` and return its match (or null).
  * @param {RegExp} re - A sticky (`y`) pattern