@jarenjs/josl 0.34.0

package/src/gbnf.js

@@ -0,0 +1,179 @@
+//#region GBNF grammar for raw JOSL / TOML text
+// A character-level grammar for engines that constrain token sampling
+// directly (the llama.cpp family), as opposed to the hosted `json_schema`
+// providers the data-model twin in schemas/ serves. The two are answers to
+// the same question at different layers: this one constrains the *text* a
+// model may emit, so the output is syntactically valid JOSL by
+// construction rather than by a post-hoc parse.
+//
+// A context-free grammar can only carry syntax. Duplicate keys, a header
+// that reopens a table, a dotted key that collides with a section — every
+// rule that needs to remember what the document already said stays the
+// parser's job. Constrained sampling narrows the model to well-formed
+// text; it does not make `parseJosl` unnecessary.
+//
+// The rules follow the TOML 1.0 ABNF closely enough to be read next to it.
+// Where JOSL adds a value form (null, bigint, regexp, a `[[]]` root array)
+// the extra rule is gated on `mode`.
+
+// One deliberate narrowing: `regexp-body` treats an unescaped '/' as the
+// closing delimiter unless it sits inside a character class, so a pattern
+// that would need deeper nesting is not expressible here. Constrained
+// output is a subset of what the parser accepts, never a superset.
+
+const CORE = [
+  ['root', 'expression ( newline expression )*'],
+  ['expression', 'ws ( keyval ws | table ws )? comment?'],
+
+  ['ws', 'wschar*'],
+  ['wschar', '[ \\t]'],
+  ['newline', '"\\r"? "\\n"'],
+  ['comment', '"#" non-eol*'],
+  ['non-eol', '[\\t\\u0020-\\u007E] | non-ascii'],
+  // U+007F and the surrogate block are not scalar values TOML admits
+  // anywhere, so no character rule may reach them
+  ['non-ascii', '[\\u0080-\\uD7FF\\uE000-\\U0010FFFF]'],
+
+  ['keyval', 'key ws "=" ws val'],
+  ['key', 'simple-key ( ws "." ws simple-key )*'],
+  ['simple-key', 'quoted-key | unquoted-key'],
+  ['unquoted-key', '[A-Za-z0-9_-]+'],
+  ['quoted-key', 'basic-string | literal-string'],
+
+  ['string', 'ml-basic-string | basic-string | ml-literal-string | literal-string'],
+  ['basic-string', '"\\"" basic-char* "\\""'],
+  ['basic-char', 'basic-unescaped | escaped'],
+  ['basic-unescaped', '[\\t\\u0020-\\u0021\\u0023-\\u005B\\u005D-\\u007E] | non-ascii'],
+  ['escaped', '"\\\\" escape-seq-char'],
+  // no "\\/": JSON allows it, TOML does not
+  ['escape-seq-char', '[\\"\\\\bfnrt] | "u" unicode-scalar-4 | "U" unicode-scalar-8'],
+  // Only escapes that name a Unicode scalar value. Spelling the excluded
+  // ranges out in hex digits is what keeps a constrained sampler from
+  // being steered into a surrogate or a code point past U+10FFFF.
+  ['unicode-scalar-4', '[0-9A-CEFa-cef] HEXDIG HEXDIG HEXDIG | [Dd] [0-7] HEXDIG HEXDIG'],
+  ['unicode-scalar-8', '"0000" unicode-scalar-4 | "000" HEXNZ hex4 | "0010" hex4'],
+  ['hex4', 'HEXDIG HEXDIG HEXDIG HEXDIG'],
+  ['HEXDIG', '[0-9A-Fa-f]'],
+  ['HEXNZ', '[1-9A-Fa-f]'],
+
+  ['ml-basic-string', '"\\"\\"\\"" newline? ml-basic-body "\\"\\"\\""'],
+  ['ml-basic-body', 'mlb-content* ( mlb-quotes mlb-content+ )* mlb-quotes?'],
+  ['mlb-content', 'mlb-char | newline | mlb-escaped-nl'],
+  ['mlb-char', 'mlb-unescaped | escaped'],
+  ['mlb-quotes', '"\\"" "\\""?'],
+  ['mlb-unescaped', '[\\t\\u0020-\\u0021\\u0023-\\u005B\\u005D-\\u007E] | non-ascii'],
+  ['mlb-escaped-nl', '"\\\\" ws newline ( wschar | newline )*'],
+
+  ['literal-string', '"\'" literal-char* "\'"'],
+  ['literal-char', '[\\t\\u0020-\\u0026\\u0028-\\u007E] | non-ascii'],
+  ['ml-literal-string', '"\'\'\'" newline? ml-literal-body "\'\'\'"'],
+  ['ml-literal-body', 'mll-content* ( mll-quotes mll-content+ )* mll-quotes?'],
+  ['mll-content', 'literal-char | newline'],
+  ['mll-quotes', '"\'" "\'"?'],
+
+  ['boolean', '"true" | "false"'],
+
+  ['integer', 'hex-int | oct-int | bin-int | dec-int'],
+  ['dec-int', '( "+" | "-" )? unsigned-dec-int'],
+  ['unsigned-dec-int', '[1-9] ( digit | "_" digit )+ | digit'],
+  ['digit', '[0-9]'],
+  ['hex-int', '"0x" HEXDIG ( HEXDIG | "_" HEXDIG )*'],
+  ['oct-int', '"0o" [0-7] ( [0-7] | "_" [0-7] )*'],
+  ['bin-int', '"0b" [01] ( [01] | "_" [01] )*'],
+
+  ['float', 'dec-int ( exp | frac exp? ) | special-float'],
+  ['frac', '"." zero-prefixable-int'],
+  ['zero-prefixable-int', 'digit ( digit | "_" digit )*'],
+  ['exp', '( "e" | "E" ) ( "+" | "-" )? zero-prefixable-int'],
+  ['special-float', '( "+" | "-" )? ( "inf" | "nan" )'],
+
+  ['date-time', 'offset-date-time | local-date-time | local-date | local-time'],
+  ['date-fullyear', 'digit digit digit digit'],
+  ['date-month', 'digit digit'],
+  ['date-mday', 'digit digit'],
+  ['time-delim', '"T" | "t" | " "'],
+  ['time-hour', 'digit digit'],
+  ['time-minute', 'digit digit'],
+  ['time-second', 'digit digit'],
+  ['time-secfrac', '"." digit+'],
+  ['time-numoffset', '( "+" | "-" ) time-hour ":" time-minute'],
+  ['time-offset', '"Z" | "z" | time-numoffset'],
+  ['partial-time', 'time-hour ":" time-minute ":" time-second time-secfrac?'],
+  ['full-date', 'date-fullyear "-" date-month "-" date-mday'],
+  ['full-time', 'partial-time time-offset'],
+  ['offset-date-time', 'full-date time-delim full-time'],
+  ['local-date-time', 'full-date time-delim partial-time'],
+  ['local-date', 'full-date'],
+  ['local-time', 'partial-time'],
+
+  ['array', '"[" array-values? ws-comment-newline "]"'],
+  ['array-values',
+    'ws-comment-newline val ws-comment-newline "," array-values'
+    + ' | ws-comment-newline val ws-comment-newline ","?'],
+  ['ws-comment-newline', '( wschar | comment? newline )*'],
+
+  ['inline-table', '"{" ws inline-table-keyvals? ws "}"'],
+  ['inline-table-keyvals', 'keyval ( ws "," ws keyval )*'],
+
+  ['std-table', '"[" ws key ws "]"'],
+  ['array-table', '"[[" ws key ws "]]"'],
+];
+
+// Value alternatives and table forms, in the order a reader should try
+// them: the longest, most specific token first.
+const VAL_CORE = 'string | boolean | array | inline-table | date-time | float | integer';
+const VAL_JOSL = `${VAL_CORE} | null-lit | bigint | regexp`;
+
+const JOSL_ONLY = [
+  ['null-lit', '"null"'],
+  ['bigint', '( hex-int | oct-int | bin-int | unsigned-dec-int ) "n"'],
+  ['regexp', '"/" regexp-body "/" regexp-flags'],
+  ['regexp-body', 'regexp-atom+'],
+  ['regexp-atom', 'regexp-char | "[" regexp-class-char* "]"'],
+  ['regexp-char', '[^/\\\\\\[\\n] | "\\\\" [^\\n]'],
+  ['regexp-class-char', '[^\\]\\\\\\n] | "\\\\" [^\\n]'],
+  ['regexp-flags', '[a-z]*'],
+  ['root-item-table', '"[[" ws "]]"'],
+];
+
+/**
+ * Build a GBNF grammar for raw JOSL or TOML text.
+ *
+ * The result is a complete grammar string with `root` as its entry rule,
+ * ready to hand to a llama.cpp-family sampler. It constrains syntax only —
+ * a document it accepts still has to go through `parseJosl` for the rules
+ * a context-free grammar cannot express (duplicate keys, table conflicts).
+ * @param {object} [options] - Grammar options
+ * @param {'josl'|'toml'} [options.mode] - 'toml' omits the JOSL-only value
+ *  forms (null, bigint, regexp) and the `[[]]` root-array header
+ * @returns {string} The GBNF grammar
+ */
+export function toGbnf(options = undefined) {
+  const toml = options?.mode === 'toml';
+  const rules = [
+    ...CORE,
+    ['val', toml ? VAL_CORE : VAL_JOSL],
+    ['table', toml ? 'array-table | std-table' : 'root-item-table | array-table | std-table'],
+    ...(toml ? [] : JOSL_ONLY),
+  ];
+  const order = new Map(rules.map(([name], i) => [name, i]));
+  const header = toml
+    ? '# GBNF grammar for TOML 1.0 text (@jarenjs/josl)'
+    : '# GBNF grammar for JOSL text (@jarenjs/josl)';
+  const body = rules
+    .slice()
+    .sort((a, b) => order.get(a[0]) - order.get(b[0]))
+    .map(([name, def]) => `${name} ::= ${def}`)
+    .join('\n');
+  return `${header}\n# syntax only; semantic rules stay with the parser\n\n${body}\n`;
+}
+
+/**
+ * Build a GBNF grammar for strict TOML 1.0 text.
+ * @returns {string} The GBNF grammar
+ */
+export function tomlToGbnf() {
+  return toGbnf({ mode: 'toml' });
+}
+
+//#endregion

package/src/index.js

@@ -0,0 +1,50 @@
+//#region @jarenjs/josl
+// JOSL - JavaScript Obvious Streaming Language.
+//
+// A TOML 1.0 backward-compatible data language with JavaScript's obvious
+// value types as first-class citizens (null, bigint, regexp, datetimes)
+// plus a streamable root array, and JSONX, the same extensions over JSON.
+// See FORMAT.md for the language delta and design rationale.
+
+export { parseJosl, parseToml } from './parse.js';
+export { parseJoslCst, parseTomlCst, JoslCstDocument } from './cst.js';
+export { createStreamReader, parseJoslStream } from './stream.js';
+export {
+  stringifyJosl,
+  stringifyToml,
+  formatKey,
+  formatKeyPath,
+  formatValue,
+  formatSection,
+} from './stringify.js';
+export { createStreamWriter, stringifyJoslChunks } from './write.js';
+export { toGbnf, tomlToGbnf } from './gbnf.js';
+export { parseJsonx, stringifyJsonx } from './jsonx.js';
+export { createJsonxStreamReader, parseJsonxStream } from './jsonx-stream.js';
+export {
+  parseCsv,
+  parseCsvDocument,
+  stringifyCsv,
+  stringifyCsvChunks,
+  sniffCsvDialect,
+  formatCsvValue,
+  coerceCsvValue,
+  CSV_CODES,
+} from './csv.js';
+export {
+  createCsvStreamReader,
+  parseCsvStream,
+  iterateCsvStream,
+  createCsvStreamWriter,
+  CsvStreamWriter,
+} from './csv-stream.js';
+export { JoslSyntaxError, JoslStringifyError, JsonxSyntaxError, CsvSyntaxError } from './errors.js';
+export {
+  LocalDate,
+  LocalTime,
+  LocalDateTime,
+  isValidDateParts,
+  isValidTimeParts,
+} from './values.js';
+
+//#endregion

package/src/jsonx-scalar.js

@@ -0,0 +1,326 @@
+//#region JSONX scalar decoders
+// Pure, position-based scalar decoders shared by the full-text parser
+// (jsonx.js) and the incremental reader (jsonx-stream.js). Every function
+// takes the source text plus an offset and reports errors through an
+// `err(pos, message, hint)` callback that must throw; none of them keep
+// state, so both parsers decode every scalar through a single code path.
+//
+// The `checkEnd(pos)` callback lets the caller assert that the character
+// after a match is a legal value terminator *at the same point in the
+// grammar* where the full-text parser checks it — error ordering between
+// 'unexpected character after value' and the value-specific validations
+// is part of the observable behavior and must match in both parsers.
+
+import {
+  CC_LF,
+  CC_MINUS,
+  CC_PLUS,
+  CC_SLASH,
+  CC_COMMA,
+  CC_SPACE,
+  CC_TAB,
+  CC_CR,
+  CC_DQUOTE,
+  CC_BACKSLASH,
+  CC_LBRACKET,
+  CC_RBRACKET,
+  CC_RBRACE,
+  isAsciiLetterCode,
+} from '@jarenjs/core/scan';
+
+import {
+  LocalDate,
+  LocalTime,
+  LocalDateTime,
+  isValidDateParts,
+  isValidTimeParts,
+} from './values.js';
+// the date-time token patterns are shared with the JOSL machine
+import { RE_DATETIME, RE_TIMEONLY, stickyExec } from './util.js';
+
+const RE_NUM_JSON = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
+const RE_NUM_JSONX = /[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?(n?)/y;
+
+/**
+ * @callback JsonxErrCallback
+ * @param {number} pos - Offset of the error in the source text
+ * @param {string} message - What is wrong
+ * @param {string} [hint] - Repair suggestion
+ * @returns {never} Must throw
+ */
+
+/**
+ * @callback JsonxCheckEndCallback
+ * @param {number} pos - Offset of the first character after the value
+ * @returns {void} Must throw when the character cannot follow a value
+ */
+
+/**
+ * Whether a char code may legally follow a completed JSON/JSONX value
+ * (whitespace, `,`, `]` or `}`).
+ * @param {number} c - The char code
+ * @returns {boolean}
+ */
+export function isValueEndCode(c) {
+  return c === CC_SPACE || c === CC_TAB || c === CC_LF || c === CC_CR
+    || c === CC_COMMA || c === CC_RBRACKET || c === CC_RBRACE;
+}
+
+/**
+ * Decode one escape sequence.
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the backslash
+ * @param {JsonxErrCallback} err - Error reporter
+ * @returns {[string, number]} The decoded text and the offset after it
+ */
+export function decodeEscape(text, pos, err) {
+  if (pos + 1 >= text.length)
+    err(pos, 'unterminated escape sequence');
+  const c = text.charCodeAt(pos + 1);
+  switch (c) {
+    case CC_DQUOTE: return ['"', pos + 2];
+    case CC_BACKSLASH: return ['\\', pos + 2];
+    case CC_SLASH: return ['/', pos + 2];
+    case 0x62: return ['\b', pos + 2];
+    case 0x66: return ['\f', pos + 2];
+    case 0x6E: return ['\n', pos + 2];
+    case 0x72: return ['\r', pos + 2];
+    case 0x74: return ['\t', pos + 2];
+    case 0x75: {
+      const hex = text.slice(pos + 2, pos + 6);
+      if (hex.length !== 4 || !/^[0-9a-fA-F]{4}$/.test(hex))
+        err(pos, "expected 4 hex digits after '\\u'");
+      return [String.fromCharCode(parseInt(hex, 16)), pos + 6];
+    }
+    default:
+      err(pos, `invalid escape '\\${text[pos + 1]}'`);
+  }
+}
+
+/**
+ * Decode a double-quoted string.
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the opening quote
+ * @param {JsonxErrCallback} err - Error reporter
+ * @returns {[string, number]} The value and the offset after the close quote
+ */
+export function decodeString(text, pos, err) {
+  const [out, p] = decodeStringSpan(text, pos + 1, text.length, err, false);
+  if (p < text.length) // stopped on the closing quote
+    return [out, p + 1];
+  err(p, 'unterminated string', "close the string with '\"'");
+}
+
+/**
+ * Decode the body of a string between two offsets, stopping at the closing
+ * quote (which it does not consume) or at `stop`. In `partial` mode it also
+ * stops before an escape that is not complete within the span, so a
+ * still-arriving string can be decoded as far as it is safe to.
+ * @param {string} text - Source text
+ * @param {number} from - Offset of the first body character
+ * @param {number} stop - Exclusive end offset
+ * @param {JsonxErrCallback} err - Error reporter
+ * @param {boolean} partial - Whether the span may end mid-escape
+ * @returns {[string, number]} The decoded text and the offset reached
+ */
+export function decodeStringSpan(text, from, stop, err, partial) {
+  let p = from;
+  let out = '';
+  let chunk = p;
+  while (p < stop) {
+    const c = text.charCodeAt(p);
+    if (c === CC_DQUOTE)
+      break;
+    if (c === CC_BACKSLASH) {
+      // the longest escape is \uXXXX; anything shorter than that near the
+      // span's end may simply not have arrived yet
+      if (partial && p + (text.charCodeAt(p + 1) === 0x75 ? 6 : 2) > stop)
+        break;
+      out += text.slice(chunk, p);
+      const [dec, np] = decodeEscape(text, p, err);
+      out += dec;
+      p = np;
+      chunk = p;
+      continue;
+    }
+    if (c < 0x20)
+      err(p, 'control characters must be escaped in strings');
+    p++;
+  }
+  return [out + text.slice(chunk, p), p];
+}
+
+/**
+ * Try to match an RFC 3339 datetime token (date, time, local or offset
+ * date-time; `T`, `t` or a single space may separate date and time).
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the first digit
+ * @param {JsonxErrCallback} err - Error reporter
+ * @param {JsonxCheckEndCallback} checkEnd - Value-terminator check
+ * @returns {[*, number]|null} The value and end offset, or null if the
+ *  text at `pos` is not a datetime
+ */
+export function matchDateTime(text, pos, err, checkEnd) {
+  let m = stickyExec(RE_DATETIME, text, pos);
+  if (m !== null) {
+    const year = Number(m[1]);
+    const month = Number(m[2]);
+    const day = Number(m[3]);
+    if (!isValidDateParts(year, month, day))
+      err(pos, `invalid date '${m[0]}'`);
+    const date = new LocalDate(year, month, day);
+    const end = pos + m[0].length;
+    checkEnd(end);
+    if (m[4] === undefined)
+      return [date, end];
+    const hour = Number(m[4]);
+    const minute = Number(m[5]);
+    const second = Number(m[6]);
+    if (!isValidTimeParts(hour, minute, second))
+      err(end, `invalid time '${m[0]}'`);
+    const time = new LocalTime(hour, minute, second, m[7] ?? '');
+    if (m[8] === undefined)
+      return [new LocalDateTime(date, time), end];
+    const offset = m[8] === 'z' || m[8] === 'Z' ? 'Z' : m[8];
+    const instant = new Date(`${date.toString()}T${time.toString()}${offset}`);
+    if (Number.isNaN(instant.getTime()))
+      err(end, `invalid date-time '${m[0]}'`);
+    return [instant, end];
+  }
+  m = stickyExec(RE_TIMEONLY, text, pos);
+  if (m !== null) {
+    const hour = Number(m[1]);
+    const minute = Number(m[2]);
+    const second = Number(m[3]);
+    if (!isValidTimeParts(hour, minute, second))
+      err(pos, `invalid time '${m[0]}'`);
+    const end = pos + m[0].length;
+    checkEnd(end);
+    return [new LocalTime(hour, minute, second, m[4] ?? ''), end];
+  }
+  return null;
+}
+
+/**
+ * Match a number token. Strict JSON follows RFC 8259; JSONX adds a
+ * leading `+`, `_` separators, an `n` bigint suffix, and auto-promotes
+ * unsafe integers to BigInt.
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the first sign or digit
+ * @param {'jsonx'|'json'} mode - Dialect
+ * @param {JsonxErrCallback} err - Error reporter
+ * @param {JsonxCheckEndCallback} checkEnd - Value-terminator check
+ * @returns {[number|bigint, number]} The value and end offset
+ */
+export function matchNumber(text, pos, mode, err, checkEnd) {
+  const m = stickyExec(mode === 'json' ? RE_NUM_JSON : RE_NUM_JSONX, text, pos);
+  if (m === null || m[0].length === 0)
+    err(pos, 'invalid number');
+  const end = pos + m[0].length;
+  checkEnd(end);
+  if (mode === 'json')
+    return [Number(m[0]), end];
+  const big = m[1] === 'n';
+  const token = big ? m[0].slice(0, -1) : m[0];
+  const isFloat = /[.eE]/.test(token);
+  if (isFloat) {
+    if (big)
+      err(pos, 'bigint literals cannot have a fraction or exponent');
+    return [Number(token.replace(/_/g, '')), end];
+  }
+  const stripped = token.replace(/[_+]/g, '');
+  if (big)
+    return [BigInt(stripped), end];
+  const value = Number(stripped);
+  return [Number.isSafeInteger(value) ? value : BigInt(stripped), end];
+}
+
+/**
+ * Match a word token (`true`, `false`, `null`, and in JSONX the
+ * optionally signed non-finite spellings `inf`/`Infinity`/`nan`/`NaN`).
+ * The caller performs the value-terminator check.
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the first sign or letter
+ * @param {'jsonx'|'json'} mode - Dialect
+ * @param {JsonxErrCallback} err - Error reporter
+ * @returns {[boolean|null|number, number]} The value and end offset
+ */
+export function matchWord(text, pos, mode, err) {
+  let p = pos;
+  const c0 = text.charCodeAt(p);
+  if (c0 === CC_MINUS || c0 === CC_PLUS)
+    p++;
+  const neg = c0 === CC_MINUS;
+  const wordStart = p;
+  while (p < text.length && isAsciiLetterCode(text.charCodeAt(p)))
+    p++;
+  const word = text.slice(wordStart, p);
+  const signed = wordStart !== pos;
+  switch (word) {
+    case 'true':
+      if (signed)
+        break;
+      return [true, p];
+    case 'false':
+      if (signed)
+        break;
+      return [false, p];
+    case 'null':
+      if (signed)
+        break;
+      return [null, p];
+    case 'inf':
+    case 'Infinity':
+      if (mode === 'json')
+        err(pos, `'${word}' is a JSONX extension`, 'JSON cannot represent non-finite numbers');
+      return [neg ? -Infinity : Infinity, p];
+    case 'nan':
+    case 'NaN':
+      if (mode === 'json')
+        err(pos, `'${word}' is a JSONX extension`, 'JSON cannot represent non-finite numbers');
+      return [NaN, p];
+  }
+  err(pos, `invalid value '${text.slice(pos, p)}'`);
+}
+
+/**
+ * Match a regexp literal. The caller performs the value-terminator check.
+ * @param {string} text - Source text
+ * @param {number} pos - Offset of the opening slash
+ * @param {JsonxErrCallback} err - Error reporter
+ * @returns {[RegExp, number]} The value and end offset
+ */
+export function matchRegExp(text, pos, err) {
+  let p = pos + 1;
+  let inClass = false;
+  for (;;) {
+    if (p >= text.length || text.charCodeAt(p) === CC_LF)
+      err(pos, 'unterminated regexp literal', "close the regexp with '/'");
+    const c = text.charCodeAt(p);
+    if (c === CC_BACKSLASH) {
+      p += 2;
+      continue;
+    }
+    if (c === CC_LBRACKET)
+      inClass = true;
+    else if (c === CC_RBRACKET)
+      inClass = false;
+    else if (c === CC_SLASH && !inClass)
+      break;
+    p++;
+  }
+  const body = text.slice(pos + 1, p);
+  p++;
+  const flagStart = p;
+  while (p < text.length && isAsciiLetterCode(text.charCodeAt(p)))
+    p++;
+  const flags = text.slice(flagStart, p);
+  try {
+    return [new RegExp(body, flags), p];
+  }
+  catch (e) {
+    err(pos, `invalid regexp literal: ${e.message}`);
+  }
+}
+
+//#endregion