@jarenjs/josl 0.34.0

package/src/parse.js

@@ -0,0 +1,37 @@
+//#region JOSL parse
+// Whole-document parsing and streaming share one grammar implementation
+// (machine.js) and differ only in how a logical line's end is found: with
+// the whole text in hand the parser discovers it as it goes, while a chunk
+// stream needs the cutter's side-effect-free pre-pass first.
+
+import { JoslMachine } from './machine.js';
+
+/**
+ * Parse a complete JOSL document.
+ * @param {string} text - JOSL source text
+ * @param {object} [options] - Reader options
+ * @param {'josl'|'toml'} [options.mode] - 'toml' rejects JOSL extensions
+ *  (null, bigint, regexp literals, root arrays) for strict TOML 1.0 input
+ * @param {(event: object) => void} [options.onEvent] - Document-order
+ *  event sink; see `createStreamReader`
+ * @returns {object|Array} The root table, or root array for [[]] documents
+ * @throws {import('./errors.js').JoslSyntaxError} On invalid input
+ */
+export function parseJosl(text, options = undefined) {
+  return new JoslMachine(options).parseAll(text);
+}
+
+/**
+ * Parse a complete document in strict TOML 1.0 mode.
+ * @param {string} text - TOML source text
+ * @param {object} [options] - Reader options minus `mode`
+ * @returns {object} The root table
+ */
+export function parseToml(text, options = undefined) {
+  return new JoslMachine({ ...options, mode: 'toml' }).parseAll(text);
+}
+
+export { JoslSyntaxError } from './errors.js';
+export { LocalDate, LocalTime, LocalDateTime } from './values.js';
+
+//#endregion

package/src/stream.js

@@ -0,0 +1,57 @@
+//#region JOSL streaming reader
+// Chunk-feedable reader for incremental (e.g. LLM token) output. Events
+// fire in document order the moment each construct completes — the
+// opposite of `JSON.parse(text, reviver)`, which visits leaves bottom-up
+// after the whole text has arrived and never tells you where you are.
+// Every event carries an absolute `path` (strings for keys, numbers for
+// array-of-tables indices), so events are directly JSON-Pointer-able.
+
+import { JoslMachine } from './machine.js';
+
+/**
+ * Create an incremental JOSL/TOML reader.
+ *
+ * Events, in document order:
+ *   {type:'table',       path, line}          - a [header] opened
+ *   {type:'table-array', path, line}          - a [[header]] appended
+ *   {type:'root-item',   path, index, line}   - a [[]] element started
+ *   {type:'pair',        path, key, value, line} - a key-value completed
+ *
+ * @param {object} [options] - Reader options
+ * @param {'josl'|'toml'} [options.mode] - 'toml' rejects JOSL extensions
+ * @param {(event: object) => void} [options.onEvent] - Event sink
+ * @returns {{feed(chunk: string): void, end(): *, root(): *}} The reader:
+ *  `feed` accepts chunks that may split any token, `end` flushes and
+ *  returns the completed root, `root` peeks at the partial result.
+ */
+export function createStreamReader(options = undefined) {
+  const machine = new JoslMachine(options);
+  return {
+    feed(chunk) {
+      machine.feed(chunk);
+    },
+    end() {
+      return machine.end();
+    },
+    root() {
+      return machine.root();
+    },
+  };
+}
+
+/**
+ * Parse an async iterable of string chunks (e.g. an LLM output stream).
+ * @param {AsyncIterable<string>|Iterable<string>} chunks - Source chunks
+ * @param {object} [options] - Reader options; see `createStreamReader`
+ * @returns {Promise<*>} The completed root value
+ */
+export async function parseJoslStream(chunks, options = undefined) {
+  const machine = new JoslMachine(options);
+  for await (const chunk of chunks)
+    machine.feed(chunk);
+  return machine.end();
+}
+
+export { JoslSyntaxError } from './errors.js';
+
+//#endregion

package/src/stringify.js

@@ -0,0 +1,341 @@
+//#region JOSL writer
+// Serializes a JS value to JOSL text, or to strict TOML 1.0 with
+// `mode: 'toml'` — in which case the JOSL-only types must be downleveled:
+// null is omitted or an error (`onNull`), regexps become strings or an
+// error (`onRegExp`), and bigints are emitted plain while they fit TOML's
+// 64-bit integer range. Round-trips are faithful for data, not for
+// formatting: key order is preserved, comments do not exist in the value
+// model, and any array whose elements are all plain objects is emitted in
+// array-of-tables form.
+
+import { JoslStringifyError } from './errors.js';
+import { LocalDate, LocalTime, LocalDateTime } from './values.js';
+
+const RE_BARE_KEY = /^[A-Za-z0-9_-]+$/;
+const TOML_INT_MIN = -(2n ** 63n);
+const TOML_INT_MAX = 2n ** 63n - 1n;
+
+/**
+ * Whether a value serializes as a table (a plain object).
+ * @param {*} v - The value
+ * @returns {boolean} True for plain objects
+ */
+export function isPlainTable(v) {
+  return isPlainObject(v);
+}
+
+function isPlainObject(v) {
+  if (v === null || typeof v !== 'object' || Array.isArray(v))
+    return false;
+  const proto = Object.getPrototypeOf(v);
+  return proto === Object.prototype || proto === null;
+}
+
+// eslint-disable-next-line no-control-regex
+const RE_NEEDS_ESCAPE = /["\\\u0000-\u001F\u007F]/;
+
+function escapeChar(c) {
+  if (c === 0x22)
+    return '\\"';
+  if (c === 0x5C)
+    return '\\\\';
+  if (c === 0x08)
+    return '\\b';
+  if (c === 0x09)
+    return '\\t';
+  if (c === 0x0A)
+    return '\\n';
+  if (c === 0x0C)
+    return '\\f';
+  if (c === 0x0D)
+    return '\\r';
+  return `\\u${c.toString(16).toUpperCase().padStart(4, '0')}`;
+}
+
+function quoteString(s) {
+  if (!RE_NEEDS_ESCAPE.test(s))
+    return `"${s}"`;
+  // Copy unescaped runs whole; only escape characters break the run.
+  let out = '"';
+  let start = 0;
+  for (let i = 0; i < s.length; ++i) {
+    const c = s.charCodeAt(i);
+    if (c === 0x22 || c === 0x5C || c < 0x20 || c === 0x7F) {
+      if (start < i)
+        out += s.slice(start, i);
+      out += escapeChar(c);
+      start = i + 1;
+    }
+  }
+  if (start < s.length)
+    out += s.slice(start);
+  return out + '"';
+}
+
+function fmtKey(k) {
+  return RE_BARE_KEY.test(k) ? k : quoteString(k);
+}
+
+function fmtPath(path) {
+  let out = '';
+  for (let i = 0; i < path.length; ++i)
+    out += (i === 0 ? '' : '.') + fmtKey(path[i]);
+  return out;
+}
+
+function fmtNumber(v) {
+  if (Number.isFinite(v))
+    return Object.is(v, -0) ? '-0.0' : String(v);
+  if (Number.isNaN(v))
+    return 'nan';
+  return v === Infinity ? 'inf' : '-inf';
+}
+
+// The error path lives on `ctx.path` as a mutable stack (pushed/popped
+// around each descent) so the happy path allocates no per-key arrays; it
+// is sliced only at a throw site.
+function fmtValue(ctx, v) {
+  switch (typeof v) {
+    case 'string':
+      return quoteString(v);
+    case 'number':
+      return fmtNumber(v);
+    case 'boolean':
+      return v ? 'true' : 'false';
+    case 'bigint':
+      if (ctx.mode === 'toml') {
+        if (v < TOML_INT_MIN || v > TOML_INT_MAX)
+          throw new JoslStringifyError('bigint exceeds the TOML 64-bit integer range', ctx.path.slice());
+        return String(v);
+      }
+      return `${v}n`;
+    case 'object':
+      break;
+    default:
+      throw new JoslStringifyError(`cannot represent a ${typeof v} value`, ctx.path.slice());
+  }
+  if (v === null) {
+    if (ctx.mode === 'toml')
+      throw new JoslStringifyError('TOML cannot represent null', ctx.path.slice());
+    return 'null';
+  }
+  if (v instanceof Date)
+    return v.toISOString();
+  if (v instanceof LocalDate || v instanceof LocalTime || v instanceof LocalDateTime)
+    return v.toString();
+  if (v instanceof RegExp) {
+    if (ctx.mode === 'toml') {
+      if (ctx.onRegExp === 'string')
+        return quoteString(`/${v.source}/${v.flags}`);
+      throw new JoslStringifyError('TOML cannot represent a RegExp', ctx.path.slice());
+    }
+    return `/${v.source}/${v.flags}`;
+  }
+  if (Array.isArray(v)) {
+    if (ctx.seen.has(v))
+      throw new JoslStringifyError('circular reference', ctx.path.slice());
+    ctx.seen.add(v);
+    let out = '[';
+    for (let i = 0; i < v.length; ++i) {
+      ctx.path.push(i);
+      out += (i === 0 ? ' ' : ', ') + fmtValue(ctx, v[i]);
+      ctx.path.pop();
+    }
+    ctx.seen.delete(v);
+    return v.length === 0 ? '[]' : out + ' ]';
+  }
+  if (isPlainObject(v)) {
+    if (ctx.seen.has(v))
+      throw new JoslStringifyError('circular reference', ctx.path.slice());
+    ctx.seen.add(v);
+    let out = '{';
+    let first = true;
+    const keys = Object.keys(v);
+    for (let i = 0; i < keys.length; ++i) {
+      const k = keys[i];
+      const e = v[k];
+      if (e === null && ctx.mode === 'toml' && ctx.onNull === 'omit')
+        continue;
+      ctx.path.push(k);
+      out += (first ? ' ' : ', ') + `${fmtKey(k)} = ${fmtValue(ctx, e)}`;
+      ctx.path.pop();
+      first = false;
+    }
+    ctx.seen.delete(v);
+    return first ? '{}' : out + ' }';
+  }
+  throw new JoslStringifyError('cannot represent this object type', ctx.path.slice());
+}
+
+// `header` is the already-formatted dotted header prefix ('' at the root),
+// so a nested section formats only its own key instead of re-walking every
+// ancestor. Values are read exactly once (a getter must not fire twice):
+// deferred tables/arrays-of-tables are kept as flat [key, value, ...] runs.
+function emitTable(ctx, header, obj) {
+  if (ctx.seen.has(obj))
+    throw new JoslStringifyError('circular reference', ctx.path.slice());
+  ctx.seen.add(obj);
+  const keys = Object.keys(obj);
+  let tables = null;
+  let aots = null;
+  for (let i = 0; i < keys.length; ++i) {
+    const k = keys[i];
+    const v = obj[k];
+    if (v === undefined)
+      continue;
+    if (v === null && ctx.mode === 'toml' && ctx.onNull === 'omit')
+      continue;
+    if (isPlainObject(v))
+      (tables ??= []).push(k, v);
+    else if (Array.isArray(v) && v.length !== 0 && v.every(isPlainObject))
+      (aots ??= []).push(k, v);
+    else {
+      ctx.path.push(k);
+      ctx.out.push(`${fmtKey(k)} = ${fmtValue(ctx, v)}`);
+      ctx.path.pop();
+    }
+  }
+  if (tables !== null) {
+    for (let i = 0; i < tables.length; i += 2) {
+      const k = tables[i];
+      const h = header === '' ? fmtKey(k) : `${header}.${fmtKey(k)}`;
+      if (ctx.out.length !== 0)
+        ctx.out.push('');
+      ctx.out.push(`[${h}]`);
+      ctx.path.push(k);
+      emitTable(ctx, h, tables[i + 1]);
+      ctx.path.pop();
+    }
+  }
+  if (aots !== null) {
+    for (let i = 0; i < aots.length; i += 2) {
+      const k = aots[i];
+      const arr = aots[i + 1];
+      const h = header === '' ? fmtKey(k) : `${header}.${fmtKey(k)}`;
+      const line = `[[${h}]]`;
+      ctx.path.push(k);
+      for (let j = 0; j < arr.length; ++j) {
+        if (ctx.out.length !== 0)
+          ctx.out.push('');
+        ctx.out.push(line);
+        ctx.path.push(j);
+        emitTable(ctx, h, arr[j]);
+        ctx.path.pop();
+      }
+      ctx.path.pop();
+    }
+  }
+  ctx.seen.delete(obj);
+}
+
+function makeCtx(options) {
+  return {
+    mode: options.mode === 'toml' ? 'toml' : 'josl',
+    onNull: options.onNull === 'omit' ? 'omit' : 'error',
+    onRegExp: options.onRegExp === 'string' ? 'string' : 'error',
+    out: [],
+    seen: new Set(),
+    path: [],
+  };
+}
+
+/**
+ * Format a single key (bare when possible, quoted otherwise).
+ * @param {string} key - The key
+ * @returns {string} JOSL/TOML key text
+ */
+export function formatKey(key) {
+  return fmtKey(key);
+}
+
+/**
+ * Format a dotted key path.
+ * @param {string[]} path - Key path segments
+ * @returns {string} Dotted key path text
+ */
+export function formatKeyPath(path) {
+  return fmtPath(path);
+}
+
+/**
+ * Format a single value (scalars, inline arrays, inline tables).
+ * @param {*} value - The value
+ * @param {object} [options] - Writer options; see `stringifyJosl`
+ * @param {(string|number)[]} [path] - Error-reporting path
+ * @returns {string} JOSL/TOML value text
+ * @throws {JoslStringifyError} When the value cannot be represented
+ */
+export function formatValue(value, options = {}, path = []) {
+  const ctx = makeCtx(options);
+  ctx.path = path.slice();
+  return fmtValue(ctx, value);
+}
+
+/**
+ * Format a table body: its pairs followed by nested `[header]` /
+ * `[[header]]` sections, with headers made relative to `headerPath`.
+ * @param {object} obj - A plain object table
+ * @param {object} [options] - Writer options; see `stringifyJosl`
+ * @param {string[]} [headerPath] - Prefix for nested section headers
+ * @returns {string} Section text (newline terminated, may be empty)
+ * @throws {JoslStringifyError} When a value cannot be represented
+ */
+export function formatSection(obj, options = {}, headerPath = []) {
+  const ctx = makeCtx(options);
+  emitTable(ctx, fmtPath(headerPath), obj);
+  return ctx.out.length === 0 ? '' : ctx.out.join('\n') + '\n';
+}
+
+/**
+ * Serialize a value to JOSL (or strict TOML) text.
+ * @param {object|Array} value - A plain object root, or (JOSL mode only)
+ *  an array of plain objects for a [[]] root-array document
+ * @param {object} [options] - Writer options
+ * @param {'josl'|'toml'} [options.mode] - 'toml' emits strict TOML 1.0
+ * @param {'error'|'omit'} [options.onNull] - TOML mode: what to do with
+ *  null table values (array elements always error)
+ * @param {'error'|'string'} [options.onRegExp] - TOML mode: represent
+ *  regexps as strings, or error
+ * @returns {string} The serialized document, newline terminated
+ * @throws {JoslStringifyError} When the value cannot be represented
+ */
+export function stringifyJosl(value, options = {}) {
+  const ctx = makeCtx(options);
+  if (Array.isArray(value)) {
+    if (ctx.mode === 'toml')
+      throw new JoslStringifyError('a TOML root must be a table; root arrays are a JOSL extension');
+    for (let i = 0; i < value.length; ++i) {
+      if (!isPlainObject(value[i]))
+        throw new JoslStringifyError('root array elements must be tables', [i]);
+      if (ctx.out.length !== 0)
+        ctx.out.push('');
+      ctx.out.push('[[]]');
+      ctx.path.push(i);
+      emitTable(ctx, '', value[i]);
+      ctx.path.pop();
+    }
+    if (value.length === 0)
+      return '';
+  }
+  else if (isPlainObject(value))
+    emitTable(ctx, '', value);
+  else
+    throw new JoslStringifyError(ctx.mode === 'toml'
+      ? 'a TOML root must be a table'
+      : 'a JOSL root must be a table or an array of tables');
+  return ctx.out.length === 0 ? '' : ctx.out.join('\n') + '\n';
+}
+
+/**
+ * Serialize a value to strict TOML 1.0 text.
+ * @param {object} value - A plain object root
+ * @param {object} [options] - Writer options minus `mode`
+ * @returns {string} The serialized document
+ */
+export function stringifyToml(value, options = {}) {
+  return stringifyJosl(value, { ...options, mode: 'toml' });
+}
+
+export { JoslStringifyError } from './errors.js';
+
+//#endregion

package/src/util.js

@@ -0,0 +1,96 @@
+//#region internal helpers shared by the JOSL and JSONX readers
+
+// The date-time token patterns of both readers. Sticky (`y`) so they
+// match in place at the current position without slicing the logical
+// line. They live here because the JOSL machine and the JSONX scalar
+// reader lex the same tokens: two copies drifting apart would give one
+// syntax two date grammars.
+//
+// The capture groups are the readers' contract: 1-3 date, 4-6 clock,
+// 7 the fraction INCLUDING its leading dot, 8 the offset. The fraction
+// is captured as text on purpose - JOSL round-trips a document, so
+// `00.100` must not come back `00.1`.
+
+/** `YYYY-MM-DD` with an optional time half. @type {RegExp} */
+export const RE_DATETIME = /(\d{4})-(\d{2})-(\d{2})(?:[Tt ](\d{2}):(\d{2}):(\d{2})(\.\d+)?([Zz]|[+-]\d{2}:\d{2})?)?/y;
+
+/** A bare `HH:MM:SS` with an optional fraction. @type {RegExp} */
+export const RE_TIMEONLY = /(\d{2}):(\d{2}):(\d{2})(\.\d+)?/y;
+
+/**
+ * Run a sticky regex at `pos` and return its match (or null).
+ * @param {RegExp} re - A sticky (`y`) pattern
+ * @param {string} text - The text to match against
+ * @param {number} pos - Position the match must start at
+ * @returns {RegExpExecArray | null} The match, anchored at `pos`
+ */
+export function stickyExec(re, text, pos) {
+  re.lastIndex = pos;
+  return re.exec(text);
+}
+
+/**
+ * Shared `feed(chunk)` body for the buffering stream machines (JOSL and
+ * CSV): guard against feeding after `end()`, strip a leading BOM on the
+ * first non-empty chunk, then buffer and scan. The JSONX stream reader
+ * has its own `feed` on purpose - it pumps a token loop and models BOM
+ * handling differently.
+ * @template {{ ended: boolean, started: boolean, buf: string, scan: () => void }} T
+ * @param {T} machine - The stream machine (`this` of its `feed`)
+ * @param {string} chunk - Next piece of the document
+ * @returns {T} The machine, for chaining
+ */
+export function feedMachine(machine, chunk) {
+  if (machine.ended)
+    throw new Error('cannot feed after end()');
+  if (!machine.started && chunk.length !== 0) {
+    machine.started = true;
+    if (chunk.charCodeAt(0) === 0xFEFF)
+      chunk = chunk.slice(1); // strip a leading BOM
+  }
+  if (chunk.length !== 0) {
+    machine.buf += chunk;
+    machine.scan();
+  }
+  return machine;
+}
+
+/**
+ * Shared `parseAll(text)` prelude for the buffering stream machines:
+ * reject mixing with `feed()`/`end()`, mark the machine started and
+ * ended, and strip a leading BOM.
+ * @param {{ ended: boolean, started: boolean }} machine - The stream machine
+ * @param {string} text - The entire document
+ * @returns {string} The text with any leading BOM removed
+ */
+export function beginParseAll(machine, text) {
+  if (machine.started || machine.ended)
+    throw new Error('parseAll cannot be mixed with feed()/end()');
+  machine.started = true;
+  machine.ended = true;
+  return text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text;
+}
+
+/**
+ * Read an own property, ignoring the prototype chain.
+ * @param {object} obj - Source object
+ * @param {string} key - Member name
+ * @returns {*} The own value or undefined
+ */
+export function getOwn(obj, key) {
+  return Object.hasOwn(obj, key) ? obj[key] : undefined;
+}
+
+/**
+ * Compute the 1-based column of `pos` inside `str` (columns restart after
+ * every newline; multi-line logical lines report positions within them).
+ * @param {string} str - Input text
+ * @param {number} pos - Offset into the text
+ * @returns {number} 1-based column number
+ */
+export function columnOf(str, pos) {
+  const nl = str.lastIndexOf('\n', pos - 1);
+  return pos - nl;
+}
+
+//#endregion

package/src/values.js

@@ -0,0 +1,104 @@
+//#region JOSL first-class value types
+// TOML distinguishes four date-time flavours; JavaScript's `Date` can only
+// faithfully hold the offset one. JOSL keeps the local flavours in small
+// immutable value classes that round-trip through `toString()` and behave
+// under `JSON.stringify` via `toJSON()`. An offset date-time parses to a
+// native `Date` (the original offset is normalized to the instant).
+
+function pad(n, w) {
+  return String(n).padStart(w, '0');
+}
+
+/**
+ * A TOML/JOSL local date (no time, no offset), e.g. `1979-05-27`.
+ */
+export class LocalDate {
+  /**
+   * @param {number} year - Full year
+   * @param {number} month - 1-based month
+   * @param {number} day - 1-based day of month
+   */
+  constructor(year, month, day) {
+    this.year = year;
+    this.month = month;
+    this.day = day;
+    Object.freeze(this);
+  }
+  toString() {
+    return `${pad(this.year, 4)}-${pad(this.month, 2)}-${pad(this.day, 2)}`;
+  }
+  toJSON() {
+    return this.toString();
+  }
+}
+
+/**
+ * A TOML/JOSL local time (no date, no offset), e.g. `07:32:00.999`.
+ * The sub-second part is kept as the literal fraction string (including
+ * the leading dot, or `''`) so precision round-trips exactly.
+ */
+export class LocalTime {
+  /**
+   * @param {number} hour - 0-23
+   * @param {number} minute - 0-59
+   * @param {number} second - 0-60 (60 allows leap seconds)
+   * @param {string} [fraction] - Literal fraction incl. leading dot, or ''
+   */
+  constructor(hour, minute, second, fraction = '') {
+    this.hour = hour;
+    this.minute = minute;
+    this.second = second;
+    this.fraction = fraction;
+    Object.freeze(this);
+  }
+  toString() {
+    return `${pad(this.hour, 2)}:${pad(this.minute, 2)}:${pad(this.second, 2)}${this.fraction}`;
+  }
+  toJSON() {
+    return this.toString();
+  }
+}
+
+/**
+ * A TOML/JOSL local date-time (no offset), e.g. `1979-05-27T07:32:00`.
+ */
+export class LocalDateTime {
+  /**
+   * @param {LocalDate} date - The date part
+   * @param {LocalTime} time - The time part
+   */
+  constructor(date, time) {
+    this.date = date;
+    this.time = time;
+    Object.freeze(this);
+  }
+  toString() {
+    return `${this.date.toString()}T${this.time.toString()}`;
+  }
+  toJSON() {
+    return this.toString();
+  }
+}
+
+// Whether year/month/day form an existing calendar date: core's
+// `isDateOnlyInRange`, re-exported under this package's established
+// name (the pair of `isValidTimeParts` below). Using the shared
+// calendar kernel means TOML's four date flavours and the rest of the
+// suite cannot disagree about whether a date exists.
+export { isDateOnlyInRange as isValidDateParts } from '@jarenjs/core/dates';
+
+/**
+ * Whether hour/minute/second form a valid time-of-day (second 60 is
+ * accepted for leap seconds).
+ * @param {number} hour - Hours
+ * @param {number} minute - Minutes
+ * @param {number} second - Seconds
+ * @returns {boolean} True when in range
+ */
+export function isValidTimeParts(hour, minute, second) {
+  return hour >= 0 && hour <= 23
+    && minute >= 0 && minute <= 59
+    && second >= 0 && second <= 60;
+}
+
+//#endregion