@hiveku-apps/sync 0.3.5 → 0.4.0

package/src/toml.mjs

@@ -0,0 +1,666 @@
+/**
+ * A small TOML reader for hiveku-sync: it says whether a file parses, and
+ * where each table and key is declared. It keeps no values. hiveku-sync has no
+ * dependencies, hence a reader of its own.
+ *
+ * Two ways to read:
+ *
+ * - checkToml reads TOML 1.0 and follows Python's tomllib (CPython
+ *   Lib/tomllib/_parser.py) rule for rule: the same grammar, the same checks
+ *   for a table or key declared twice, and the same "frozen" rule for inline
+ *   tables and arrays. test/toml.test.mjs compares the two.
+ *
+ * - readTomlDeclarations is what hiveku-sync uses on .codex/config.toml. Codex
+ *   (0.131) reads more than TOML 1.0, so this reads that too: a byte-order mark
+ *   at the start of the file, and the TOML 1.1 additions (newlines, comments
+ *   and a trailing comma inside an inline table, the \e and \xHH escapes,
+ *   times without seconds). It never throws on a file. A line it cannot read
+ *   is noted by its line number and skipped, and a table or key declared twice
+ *   is noted with both of its declarations, so hiveku-sync can tell whether its
+ *   own block is one of them. The rules for what counts as declared twice are
+ *   the same in both.
+ */
+
+export class TomlError extends Error {
+  constructor(problem, line, pos) {
+    super(`${problem} (line ${line})`);
+    this.name = 'TomlError';
+    this.problem = problem;
+    this.line = line;
+    this.pos = pos;
+  }
+}
+
+// Read the TOML 1.1 syntax Codex reads. Set by parseDocument for one
+// synchronous parse (readTomlDeclarations); false for checkToml.
+let toml11 = false;
+
+const WS = ' \t';
+const WS_AND_NEWLINE = ' \t\n';
+const BARE_KEY = /^[A-Za-z0-9_-]$/;
+const HEX = /^[0-9A-Fa-f]+$/;
+const SCALAR = 0; // any value that is not an inline table or an array
+
+const ESCAPES = { '\\b': '\b', '\\t': '\t', '\\n': '\n', '\\f': '\f', '\\r': '\r', '\\"': '"', '\\\\': '\\' };
+
+const TIME = '([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\\.([0-9]{1,6})[0-9]*)?';
+// TOML 1.1: the seconds (and so the fraction) are optional.
+const TIME_11 = '([01][0-9]|2[0-3]):([0-5][0-9])(?::([0-5][0-9])(?:\\.([0-9]{1,6})[0-9]*)?)?';
+const datetimeRegex = (time) =>
+  new RegExp(
+    `([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])(?:[Tt ]${time}(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))?)?`,
+    'y',
+  );
+const RE_LOCALTIME = new RegExp(TIME, 'y');
+const RE_DATETIME = datetimeRegex(TIME);
+const RE_LOCALTIME_11 = new RegExp(TIME_11, 'y');
+const RE_DATETIME_11 = datetimeRegex(TIME_11);
+const RE_NUMBER = new RegExp(
+  '0(?:x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*|b[01](?:_?[01])*|o[0-7](?:_?[0-7])*)' +
+    '|[+-]?(?:0|[1-9](?:_?[0-9])*)(?:\\.[0-9](?:_?[0-9])*)?(?:[eE][+-]?[0-9](?:_?[0-9])*)?',
+  'y',
+);
+
+/** A sticky regex matched exactly at `pos`, or null. */
+function matchAt(regex, src, pos) {
+  regex.lastIndex = pos;
+  return src.match(regex);
+}
+
+/** ASCII control characters, less the ones a context allows. */
+function isControl(ch, allowed = '') {
+  const code = ch.charCodeAt(0);
+  return (code < 32 || code === 127) && !allowed.includes(ch);
+}
+
+// Where each line starts, for the text read last: readTomlDeclarations can
+// note many lines in one file, so a line number is a search, not a count.
+let lineStartsFor = null;
+let lineStarts = [];
+
+function lineAt(src, pos) {
+  if (lineStartsFor !== src) {
+    lineStarts = [0];
+    for (let i = src.indexOf('\n'); i !== -1; i = src.indexOf('\n', i + 1)) lineStarts.push(i + 1);
+    lineStartsFor = src;
+  }
+  let low = 0;
+  let high = lineStarts.length - 1;
+  while (low < high) {
+    const mid = (low + high + 1) >> 1;
+    if (lineStarts[mid] <= pos) low = mid;
+    else high = mid - 1;
+  }
+  return low + 1;
+}
+
+function fail(src, pos, problem) {
+  throw new TomlError(problem, lineAt(src, pos), pos);
+}
+
+function describeChar(ch) {
+  const code = ch.charCodeAt(0);
+  return code < 32 || code === 127 ? `control character U+${code.toString(16).toUpperCase().padStart(4, '0')}` : `'${ch}'`;
+}
+
+/** A key as TOML would write it: dotted, each part bare when it can be. */
+export function formatKey(key) {
+  return key.map((part) => (/^[A-Za-z0-9_-]+$/.test(part) ? part : JSON.stringify(part))).join('.');
+}
+
+function skipChars(src, pos, chars) {
+  while (pos < src.length && chars.includes(src[pos])) pos += 1;
+  return pos;
+}
+
+function skipUntil(src, pos, expect, allowedControls, errorOnEof) {
+  let end = src.indexOf(expect, pos);
+  if (end === -1) {
+    end = src.length;
+    if (errorOnEof) fail(src, end, `expected ${JSON.stringify(expect)} before the end of the file`);
+  }
+  for (let i = pos; i < end; i += 1) {
+    if (isControl(src[i], allowedControls)) fail(src, i, `${describeChar(src[i])} is not allowed here`);
+  }
+  return end;
+}
+
+function skipComment(src, pos) {
+  if (src[pos] === '#') return skipUntil(src, pos + 1, '\n', '\t', false);
+  return pos;
+}
+
+function skipCommentsAndArrayWs(src, pos) {
+  for (;;) {
+    const before = pos;
+    pos = skipComment(src, skipChars(src, pos, WS_AND_NEWLINE));
+    if (pos === before) return pos;
+  }
+}
+
+// ---------------------------------------------------------------- keys
+
+function parseKeyPart(src, pos) {
+  const ch = src[pos];
+  if (ch !== undefined && BARE_KEY.test(ch)) {
+    const start = pos;
+    while (pos < src.length && BARE_KEY.test(src[pos])) pos += 1;
+    return [pos, src.slice(start, pos)];
+  }
+  if (ch === "'") return parseLiteralStr(src, pos);
+  if (ch === '"') return parseBasicStr(src, pos + 1, false);
+  return fail(src, pos, 'a key must start with a letter, digit, "-", "_" or a quote');
+}
+
+function parseKey(src, pos) {
+  let part;
+  [pos, part] = parseKeyPart(src, pos);
+  const key = [part];
+  pos = skipChars(src, pos, WS);
+  while (src[pos] === '.') {
+    pos = skipChars(src, pos + 1, WS);
+    [pos, part] = parseKeyPart(src, pos);
+    key.push(part);
+    pos = skipChars(src, pos, WS);
+  }
+  return [pos, key];
+}
+
+// ---------------------------------------------------------------- strings
+
+function parseLiteralStr(src, pos) {
+  const start = pos + 1;
+  const end = skipUntil(src, start, "'", '\t', true);
+  return [end + 1, src.slice(start, end)];
+}
+
+function parseHexChar(src, pos, length) {
+  const hex = src.slice(pos, pos + length);
+  if (hex.length !== length || !HEX.test(hex)) fail(src, pos, 'a \\x, \\u or \\U escape needs 2, 4 or 8 hex digits');
+  const code = parseInt(hex, 16);
+  if (!((code >= 0 && code <= 0xd7ff) || (code >= 0xe000 && code <= 0x10ffff))) {
+    fail(src, pos, `the escape \\${length === 4 ? 'u' : 'U'}${hex} is not a Unicode scalar value`);
+  }
+  return [pos + length, String.fromCodePoint(code)];
+}
+
+function parseEscape(src, pos, multiline) {
+  const id = src.slice(pos, pos + 2);
+  pos += 2;
+  if (multiline && (id === '\\ ' || id === '\\\t' || id === '\\\n')) {
+    // A line-ending backslash: skip the whitespace and newlines after it.
+    if (id !== '\\\n') {
+      pos = skipChars(src, pos, WS);
+      if (pos >= src.length) return [pos, ''];
+      if (src[pos] !== '\n') fail(src, pos, 'a backslash in a string must start an escape');
+      pos += 1;
+    }
+    return [skipChars(src, pos, WS_AND_NEWLINE), ''];
+  }
+  if (id === '\\u') return parseHexChar(src, pos, 4);
+  if (id === '\\U') return parseHexChar(src, pos, 8);
+  if (Object.hasOwn(ESCAPES, id)) return [pos, ESCAPES[id]];
+  if (toml11 && id === '\\e') return [pos, '\x1b'];
+  if (toml11 && id === '\\x') return parseHexChar(src, pos, 2);
+  return fail(src, pos, 'a backslash in a string must start an escape');
+}
+
+/** A basic string, from just after its opening quote(s). */
+function parseBasicStr(src, pos, multiline) {
+  const allowed = multiline ? '\t\n' : '\t';
+  let result = '';
+  let start = pos;
+  for (;;) {
+    if (pos >= src.length) fail(src, pos, 'a string is not closed');
+    const ch = src[pos];
+    if (ch === '"') {
+      if (!multiline) return [pos + 1, result + src.slice(start, pos)];
+      if (src.startsWith('"""', pos)) return [pos + 3, result + src.slice(start, pos)];
+      pos += 1;
+    } else if (ch === '\\') {
+      result += src.slice(start, pos);
+      let piece;
+      [pos, piece] = parseEscape(src, pos, multiline);
+      result += piece;
+      start = pos;
+    } else if (isControl(ch, allowed)) {
+      fail(src, pos, `${describeChar(ch)} is not allowed in a string`);
+    } else {
+      pos += 1;
+    }
+  }
+}
+
+function parseMultilineStr(src, pos, literal) {
+  pos += 3;
+  if (src[pos] === '\n') pos += 1;
+  const delim = literal ? "'" : '"';
+  let result;
+  if (literal) {
+    const end = skipUntil(src, pos, "'''", '\t\n', true);
+    result = src.slice(pos, end);
+    pos = end + 3;
+  } else {
+    [pos, result] = parseBasicStr(src, pos, true);
+  }
+  // Up to two more quotes right before the closing three belong to the string.
+  if (src[pos] !== delim) return [pos, result];
+  pos += 1;
+  if (src[pos] !== delim) return [pos, result + delim];
+  return [pos + 1, result + delim + delim];
+}
+
+// ---------------------------------------------------------------- values
+
+function daysInMonth(year, month) {
+  if (month === 2) return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0 ? 29 : 28;
+  return [4, 6, 9, 11].includes(month) ? 30 : 31;
+}
+
+function parseArray(src, pos) {
+  pos = skipCommentsAndArrayWs(src, pos + 1);
+  const array = [];
+  if (src[pos] === ']') return [pos + 1, array];
+  for (;;) {
+    let value;
+    [pos, value] = parseValue(src, pos);
+    array.push(value);
+    pos = skipCommentsAndArrayWs(src, pos);
+    if (src[pos] === ']') return [pos + 1, array];
+    if (src[pos] !== ',') fail(src, pos, 'an array is not closed (expected "," or "]")');
+    pos = skipCommentsAndArrayWs(src, pos + 1);
+    if (src[pos] === ']') return [pos + 1, array];
+  }
+}
+
+function parseInlineTable(src, pos) {
+  // TOML 1.1 allows newlines and comments between the entries, and a comma after the last one.
+  const skip = (at) => (toml11 ? skipCommentsAndArrayWs(src, at) : skipChars(src, at, WS));
+  pos = skip(pos + 1);
+  const data = new NestedDict();
+  const flags = new Flags();
+  if (src[pos] === '}') return [pos + 1, data.root];
+  for (;;) {
+    let key;
+    let value;
+    [pos, key, value] = parseKeyValuePair(src, pos);
+    const stem = key[key.length - 1];
+    if (flags.is(key, Flags.FROZEN)) fail(src, pos, 'a key is set twice in an inline table');
+    const nest = data.getOrCreateNest(key.slice(0, -1), false);
+    if (!nest || nest.has(stem)) fail(src, pos, 'a key is set twice in an inline table');
+    nest.set(stem, value);
+    pos = skip(pos);
+    if (src[pos] === '}') return [pos + 1, data.root];
+    if (src[pos] !== ',') fail(src, pos, 'an inline table is not closed (expected "," or "}")');
+    if (value instanceof Map || Array.isArray(value)) flags.set(key, Flags.FROZEN, true);
+    pos = skip(pos + 1);
+    if (toml11 && src[pos] === '}') return [pos + 1, data.root];
+  }
+}
+
+function parseValue(src, pos) {
+  const ch = src[pos];
+  if (ch === '"') return src.startsWith('"""', pos) ? parseMultilineStr(src, pos, false) : parseBasicStr(src, pos + 1, false);
+  if (ch === "'") return src.startsWith("'''", pos) ? parseMultilineStr(src, pos, true) : parseLiteralStr(src, pos);
+  if (ch === 't' && src.startsWith('true', pos)) return [pos + 4, SCALAR];
+  if (ch === 'f' && src.startsWith('false', pos)) return [pos + 5, SCALAR];
+  if (ch === '[') return parseArray(src, pos);
+  if (ch === '{') return parseInlineTable(src, pos);
+
+  const datetime = matchAt(toml11 ? RE_DATETIME_11 : RE_DATETIME, src, pos);
+  if (datetime) {
+    const year = Number(datetime[1]);
+    const month = Number(datetime[2]);
+    const day = Number(datetime[3]);
+    if (year < 1 || day > daysInMonth(year, month)) fail(src, pos, `${datetime[0]} is not a real date`);
+    return [pos + datetime[0].length, SCALAR];
+  }
+  const time = matchAt(toml11 ? RE_LOCALTIME_11 : RE_LOCALTIME, src, pos);
+  if (time) return [pos + time[0].length, SCALAR];
+  const number = matchAt(RE_NUMBER, src, pos);
+  if (number) return [pos + number[0].length, SCALAR];
+  if (['inf', 'nan'].includes(src.slice(pos, pos + 3))) return [pos + 3, SCALAR];
+  if (['+inf', '-inf', '+nan', '-nan'].includes(src.slice(pos, pos + 4))) return [pos + 4, SCALAR];
+  return fail(src, pos, 'this is not a valid TOML value');
+}
+
+function parseKeyValuePair(src, pos) {
+  let key;
+  [pos, key] = parseKey(src, pos);
+  // The message never repeats the key: a line the parser takes for a key may be a pasted secret.
+  if (src[pos] !== '=') fail(src, pos, 'expected "=" after a key');
+  pos = skipChars(src, pos + 1, WS);
+  let value;
+  [pos, value] = parseValue(src, pos);
+  return [pos, key, value];
+}
+
+// ---------------------------------------------------------------- document
+
+function newFlagEntry() {
+  return { flags: new Set(), recursive: new Set(), nested: new Map() };
+}
+
+/** Which keys are tables, and which of them may no longer change. */
+class Flags {
+  static FROZEN = 0; // an inline table or array: nothing may be added under it
+  static EXPLICIT_NEST = 1; // declared by a [header], or by dotted keys in a closed section
+
+  constructor() {
+    this.flags = new Map();
+    this.pending = [];
+  }
+
+  addPending(key, flag) {
+    this.pending.push([key, flag]);
+  }
+
+  finalizePending() {
+    for (const [key, flag] of this.pending) this.set(key, flag, false);
+    this.pending = [];
+  }
+
+  unsetAll(key) {
+    let cont = this.flags;
+    for (const part of key.slice(0, -1)) {
+      if (!cont.has(part)) return;
+      cont = cont.get(part).nested;
+    }
+    cont.delete(key[key.length - 1]);
+  }
+
+  set(key, flag, recursive) {
+    let cont = this.flags;
+    for (const part of key.slice(0, -1)) {
+      if (!cont.has(part)) cont.set(part, newFlagEntry());
+      cont = cont.get(part).nested;
+    }
+    const stem = key[key.length - 1];
+    if (!cont.has(stem)) cont.set(stem, newFlagEntry());
+    (recursive ? cont.get(stem).recursive : cont.get(stem).flags).add(flag);
+  }
+
+  is(key, flag) {
+    if (key.length === 0) return false;
+    let cont = this.flags;
+    for (const part of key.slice(0, -1)) {
+      if (!cont.has(part)) return false;
+      const inner = cont.get(part);
+      if (inner.recursive.has(flag)) return true;
+      cont = inner.nested;
+    }
+    const stem = key[key.length - 1];
+    if (!cont.has(stem)) return false;
+    const entry = cont.get(stem);
+    return entry.flags.has(flag) || entry.recursive.has(flag);
+  }
+}
+
+/** The document's shape: tables are Maps, arrays are Arrays, other values SCALAR. */
+class NestedDict {
+  constructor() {
+    this.root = new Map();
+  }
+
+  /** The table at `key`, created when missing; null when a value is in the way. */
+  getOrCreateNest(key, accessLists = true) {
+    let cont = this.root;
+    for (const part of key) {
+      if (!cont.has(part)) cont.set(part, new Map());
+      cont = cont.get(part);
+      if (accessLists && Array.isArray(cont)) cont = cont[cont.length - 1];
+      if (!(cont instanceof Map)) return null;
+    }
+    return cont;
+  }
+
+  appendNestToList(key) {
+    const cont = this.getOrCreateNest(key.slice(0, -1));
+    if (!cont) return false;
+    const last = key[key.length - 1];
+    if (!cont.has(last)) {
+      cont.set(last, [new Map()]);
+      return true;
+    }
+    const list = cont.get(last);
+    if (!Array.isArray(list)) return false;
+    list.push(new Map());
+    return true;
+  }
+}
+
+const pathId = (key) => JSON.stringify(key);
+
+/** The statement that first declared or created `key` or, failing that, the nearest table above it. */
+function originOf(out, key) {
+  for (let n = key.length; n > 0; n -= 1) {
+    const statement = out.origin.get(pathId(key.slice(0, n)));
+    if (statement) return statement;
+  }
+  return null;
+}
+
+/** Note `statement` as where `key`, and each table above it not seen before, came from. */
+function touch(out, key, statement) {
+  for (let n = 1; n <= key.length; n += 1) {
+    const id = pathId(key.slice(0, n));
+    if (!out.origin.has(id)) out.origin.set(id, statement);
+  }
+}
+
+/** The tables that dotted keys declared in the section that just ended can no longer change. */
+function finalizePending(out) {
+  out.flags.finalizePending();
+  for (const [key, statement] of out.pendingDeclared) {
+    if (!out.declared.has(pathId(key))) out.declared.set(pathId(key), statement);
+  }
+  out.pendingDeclared = [];
+}
+
+/**
+ * A table or key declared twice. checkToml throws; readTomlDeclarations notes
+ * it with the earlier declaration it collides with (`prior`) and reads on.
+ */
+function declaredTwice(src, out, statement, problem, prior) {
+  if (!out.recover) fail(src, statement.pos, problem);
+  out.conflicts.push({ line: lineAt(src, statement.pos), problem, statement, prior });
+}
+
+/** A [table] header. Returns [pos, key, declared]: `declared` is false when it was declared twice. */
+function tableRule(src, pos, out) {
+  const start = pos;
+  let key;
+  [pos, key] = parseKey(src, skipChars(src, pos + 1, WS));
+  if (src[pos] !== ']') fail(src, pos, 'expected "]" at the end of a table header');
+  const statement = { type: 'table', key, header: key, pos: start };
+  out.statements.push(statement);
+  const name = `[${formatKey(key)}]`;
+  if (out.flags.is(key, Flags.FROZEN)) {
+    declaredTwice(src, out, statement, `${name} cannot be declared: it is already set as an inline table or array`, originOf(out, key));
+    return [pos + 1, key, false];
+  }
+  if (out.flags.is(key, Flags.EXPLICIT_NEST)) {
+    const prior = out.declared.get(pathId(key)) ?? originOf(out, key);
+    declaredTwice(src, out, statement, `the table ${name} is declared twice`, prior);
+    return [pos + 1, key, false];
+  }
+  out.flags.set(key, Flags.EXPLICIT_NEST, false);
+  if (!out.data.getOrCreateNest(key)) {
+    declaredTwice(src, out, statement, `${name} cannot be declared: a key on its path already holds a value`, originOf(out, key));
+    return [pos + 1, key, false];
+  }
+  if (!out.declared.has(pathId(key))) out.declared.set(pathId(key), statement);
+  touch(out, key, statement);
+  return [pos + 1, key, true];
+}
+
+/** A [[table]] header, as tableRule. */
+function arrayTableRule(src, pos, out) {
+  const start = pos;
+  let key;
+  [pos, key] = parseKey(src, skipChars(src, pos + 2, WS));
+  if (!src.startsWith(']]', pos)) fail(src, pos, 'expected "]]" at the end of an array-of-tables header');
+  const statement = { type: 'arrayTable', key, header: key, pos: start };
+  out.statements.push(statement);
+  const name = `[[${formatKey(key)}]]`;
+  if (out.flags.is(key, Flags.FROZEN)) {
+    declaredTwice(src, out, statement, `${name} cannot be declared: it is already set as an inline table or array`, originOf(out, key));
+    return [pos + 2, key, false];
+  }
+  // The new element starts empty, but the key itself still may not become a [table].
+  out.flags.unsetAll(key);
+  out.flags.set(key, Flags.EXPLICIT_NEST, false);
+  if (!out.data.appendNestToList(key)) {
+    declaredTwice(src, out, statement, `${name} cannot be declared: that key, or one on its path, already holds a value`, originOf(out, key));
+    return [pos + 2, key, false];
+  }
+  if (!out.declared.has(pathId(key))) out.declared.set(pathId(key), statement);
+  touch(out, key, statement);
+  return [pos + 2, key, true];
+}
+
+/**
+ * A key = value line in the table `header` (null: a header that could not be
+ * read, so the line is read but not kept). `headerDeclared` false: the header
+ * was declared twice, so the line is kept but not applied.
+ */
+function keyValueRule(src, pos, out, header, headerDeclared) {
+  const start = pos;
+  let key;
+  let value;
+  [pos, key, value] = parseKeyValuePair(src, pos);
+  if (header === null) return pos;
+  const statement = {
+    type: 'keyval',
+    key,
+    header,
+    pos: start,
+    value: value instanceof Map ? 'table' : Array.isArray(value) ? 'array' : 'other',
+    tableKeys: value instanceof Map ? [...value.keys()] : [],
+  };
+  out.statements.push(statement);
+  if (!headerDeclared) return pos;
+
+  const full = [...header, ...key];
+  const parent = full.slice(0, -1);
+  const stem = full[full.length - 1];
+  const containers = [];
+  for (let i = 1; i < key.length; i += 1) containers.push([...header, ...key.slice(0, i)]);
+  const redeclared = containers.find((container) => out.flags.is(container, Flags.EXPLICIT_NEST));
+  if (redeclared) {
+    const prior = out.declared.get(pathId(redeclared)) ?? originOf(out, redeclared);
+    declaredTwice(src, out, statement, `the key ${formatKey(full)} cannot add to [${formatKey(redeclared)}], which is declared elsewhere`, prior);
+    return pos;
+  }
+  if (out.flags.is(parent, Flags.FROZEN)) {
+    const problem = `the key ${formatKey(full)} cannot be added: ${formatKey(parent)} is already set as an inline table or array`;
+    declaredTwice(src, out, statement, problem, originOf(out, parent));
+    return pos;
+  }
+  const nest = out.data.getOrCreateNest(parent);
+  if (!nest || nest.has(stem)) {
+    declaredTwice(src, out, statement, `the key ${formatKey(full)} is set twice`, originOf(out, nest ? full : parent));
+    return pos;
+  }
+  // A table made by dotted keys cannot be reopened once its section ends.
+  for (const container of containers) {
+    out.flags.addPending(container, Flags.EXPLICIT_NEST);
+    out.pendingDeclared.push([container, statement]);
+  }
+  if (value instanceof Map || Array.isArray(value)) out.flags.set(full, Flags.FROZEN, true);
+  nest.set(stem, value);
+  touch(out, full, statement);
+  return pos;
+}
+
+/**
+ * `recover`: note a table or key declared twice, and a line that cannot be
+ * read, and read on (readTomlDeclarations); otherwise throw (checkToml).
+ * `syntax11`: read the TOML 1.1 syntax and a leading byte-order mark.
+ */
+function parseDocument(text, { recover, syntax11 }) {
+  const src = text.replace(/\r\n/g, '\n');
+  const out = {
+    recover,
+    data: new NestedDict(),
+    flags: new Flags(),
+    statements: [],
+    conflicts: [],
+    unreadLines: [],
+    origin: new Map(),
+    declared: new Map(),
+    pendingDeclared: [],
+  };
+  let header = [];
+  let headerDeclared = true;
+  // Codex reads a file that starts with a byte-order mark (Windows PowerShell 5.1 writes one); TOML 1.0 does not.
+  let pos = syntax11 && src.startsWith('\ufeff') ? 1 : 0;
+  toml11 = syntax11;
+  try {
+    for (;;) {
+      pos = skipChars(src, pos, WS);
+      if (pos >= src.length) break;
+      const ch = src[pos];
+      if (ch === '\n') {
+        pos += 1;
+        continue;
+      }
+      try {
+        if (BARE_KEY.test(ch) || ch === '"' || ch === "'") {
+          pos = skipChars(src, keyValueRule(src, pos, out, header, headerDeclared), WS);
+        } else if (ch === '[') {
+          finalizePending(out);
+          header = null; // until the new header has been read
+          [pos, header, headerDeclared] = src[pos + 1] === '[' ? arrayTableRule(src, pos, out) : tableRule(src, pos, out);
+          pos = skipChars(src, pos, WS);
+        } else if (ch !== '#') {
+          fail(src, pos, `a line must hold a key = value, a [table] header or a comment, not ${describeChar(ch)}`);
+        }
+        pos = skipComment(src, pos);
+        if (pos >= src.length) break;
+        if (src[pos] !== '\n') fail(src, pos, 'expected the end of the line after a statement');
+        pos += 1;
+      } catch (err) {
+        if (!recover || !(err instanceof TomlError)) throw err;
+        // Note the line, and read on from the next one.
+        if (out.unreadLines[out.unreadLines.length - 1] !== err.line) out.unreadLines.push(err.line);
+        const next = src.indexOf('\n', err.pos);
+        if (next === -1) break;
+        pos = next + 1;
+      }
+    }
+  } finally {
+    toml11 = false;
+  }
+  return out;
+}
+
+/** Throw a TomlError (with .problem, .line and .pos) unless `text` is a valid TOML 1.0 document. */
+export function checkToml(text) {
+  parseDocument(text, { recover: false, syntax11: false });
+}
+
+/**
+ * Read a config.toml the way Codex does (TOML 1.1 syntax and a leading
+ * byte-order mark allowed) and never throw for anything in it. Positions are
+ * offsets in the text with CRLF read as LF.
+ *
+ * - statements: every [table] header, [[array]] header and key = value line it
+ *   read, in order, as { type: 'table' | 'arrayTable' | 'keyval', key, header,
+ *   pos }. `key` is the declared key (relative to `header` for a keyval),
+ *   `header` the table it sits in. A keyval also has `value` ('table', 'array'
+ *   or 'other') and, for an inline table, `tableKeys`: the keys set directly
+ *   in it. Lines under a header it could not read are left out.
+ * - conflicts: each table or key declared twice, in file order, as { line,
+ *   problem, statement, prior }: `statement` is the later declaration and
+ *   `prior` the earlier one it collides with (null when not known). `problem`
+ *   names keys from the file, so it is for tests, never for printing.
+ * - unreadLines: the numbers of the lines it could not read, and skipped.
+ *
+ * `toml11: false` reads TOML 1.0 syntax only; the tests use it to compare the
+ * conflicts noted here with what checkToml throws.
+ */
+export function readTomlDeclarations(text, { toml11: syntax11 = true } = {}) {
+  const { statements, conflicts, unreadLines } = parseDocument(text, { recover: true, syntax11 });
+  return { statements, conflicts, unreadLines };
+}