links-notation 0.27.1 → 0.30.0

package/index.d.ts

@@ -24,6 +24,22 @@ export class Link {
    */
   values: Link[];
 
+  /**
+   * The ID exactly as the parsed document wrote it, quotes, escapes and
+   * literal syntax included: `'1'` for a reference written `'1'`, `1` for one
+   * written `1`. Null for a link without an ID, a link built in code, a link
+   * made by indentation path combination, or a parser in AST mode
+   * (`syntax: false`). Equality, formatting and serialization ignore it.
+   * @throws {TypeError} When set to anything but a string or null
+   */
+  rawId: string | null;
+
+  /**
+   * The delimiter the ID was written between (`'`, `"` or `` ` ``), read from
+   * `rawId`; null for a bare or literal reference, or no `rawId`.
+   */
+  readonly quote: "'" | '"' | '`' | null;
+
   /**
    * Create a new Link
    * @param id - Optional identifier for the link
@@ -123,6 +139,56 @@ export function escapeReference(
 /** Decode a complete escaped reference, including n-quotes and versioned literals. */
 export function unescapeReference(reference: string): string;
 
+/**
+ * The delimiter one reference was written between, read from the reference as
+ * written: `'`, `"` or `` ` `` for a complete quoted reference, otherwise null.
+ */
+export function referenceQuote(
+  raw: string | null | undefined
+): "'" | '"' | '`' | null;
+
+/** The kinds of nodes and tokens of a concrete syntax tree. */
+export const CstKind: Readonly<{
+  DOCUMENT: 'document';
+  LINE: 'line';
+  GROUP: 'group';
+  WHITESPACE: 'whitespace';
+  NEWLINE: 'newline';
+  COMMENT: 'comment';
+  OPEN_PAREN: 'open_paren';
+  CLOSE_PAREN: 'close_paren';
+  COLON: 'colon';
+  REFERENCE: 'reference';
+}>;
+
+export type CstKindName = (typeof CstKind)[keyof typeof CstKind];
+
+/**
+ * One node or token of a lossless concrete syntax tree: every character of
+ * the document belongs to exactly one token, so the document is the text of
+ * the tree.
+ */
+export class CstNode {
+  /** The kind of this node or token. */
+  kind: CstKindName;
+  /** The children of a `document`, `line` or `group` node; empty for a token. */
+  children: CstNode[];
+  /** What a reference token means; null for every other kind. */
+  value: string | null;
+  /** The delimiter of a quoted reference token; null otherwise. */
+  quote: "'" | '"' | '`' | null;
+  /** Whether this is a token rather than a node. */
+  readonly isToken: boolean;
+  /** The exact text this node or token was read from. */
+  readonly text: string;
+  /** The exact text this node or token was read from. */
+  toString(): string;
+  /** Every token of this subtree, in document order. */
+  tokens(): CstNode[];
+  /** The tree as an S-expression: `(kind child ...)` and `kind:"text"`. */
+  toSExpression(): string;
+}
+
 /**
  * Parser options for configuring the parser behavior
  */
@@ -150,6 +216,12 @@ export interface ParserOptions {
    * comment (default: true)
    */
   comments?: boolean;
+
+  /**
+   * If false (AST mode), parsed links do not keep how their IDs were written:
+   * `rawId` and `quote` stay null (default: true)
+   */
+  syntax?: boolean;
 }
 
 /**
@@ -173,6 +245,11 @@ export class Parser {
    */
   comments: boolean;
 
+  /**
+   * Whether parsed links keep how their IDs were written (`rawId`)
+   */
+  syntax: boolean;
+
   /**
    * Create a new Parser instance
    * @param options - Parser options
@@ -189,6 +266,13 @@ export class Parser {
 
   /** Read the indentation tree without expanding it into path combinations. */
   parseGroups(input: string): IndentedGroup[];
+
+  /**
+   * Read the document as a lossless concrete syntax tree, so that
+   * `parseCst(input).toString() === input`.
+   * @throws {ParseError} When `parse(input)` would throw
+   */
+  parseCst(input: string): CstNode;
 }
 
 /** A parsed line and the immediately indented lines below it. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "links-notation",
-  "version": "0.27.1",
+  "version": "0.30.0",
   "description": "Links Notation parser for JavaScript",
   "main": "dist/index.js",
   "types": "index.d.ts",
@@ -32,10 +32,13 @@
   },
   "homepage": "https://github.com/link-foundation/links-notation#readme",
   "devDependencies": {
-    "bun-types": "^1.4.3",
-    "eslint": "^10.13.0",
-    "peggy": "^5.1.0",
+    "bun-types": "^1.4.4",
+    "eslint": "^10.14.0",
+    "peggy": "^5.1.1",
     "prettier": "^3.9.10",
     "typescript": "^7.0.2"
+  },
+  "overrides": {
+    "minimatch": "^11.0.4"
   }
 }

package/src/Cst.js

@@ -0,0 +1,246 @@
+/**
+ * The concrete syntax tree of a Links Notation document.
+ *
+ * Every character of a document belongs to exactly one token, so the text of
+ * the tree is the document itself, comments, blank lines, indentation, quotes
+ * and line endings included. A document is a list of `line` nodes; a line holds
+ * the tokens up to and including its line ending, and a parenthesized `group`
+ * node holds its own parentheses and the lines written between them. A group is
+ * part of the line it opens on, however many lines it spans.
+ *
+ * The tree records how a document was written; `Parser.parse()` reads what it
+ * means. They agree because a document is parsed before its tree is built, and
+ * the tokens are read with the same rules the grammar uses: comments are found
+ * by {@link stripComments}, delimited references by {@link DelimitedReferences}
+ * and versioned literals by the same pattern.
+ */
+
+import { stripComments } from './comments.js';
+import { DelimitedReferences, QUOTES } from './quotes.js';
+import {
+  decodeReferenceLiteral,
+  referenceLiteralEnd as literalEnd,
+} from './ReferenceLiteral.js';
+
+/** The kinds of nodes and tokens of a concrete syntax tree. */
+export const CstKind = Object.freeze({
+  /** The whole document: its lines. */
+  DOCUMENT: 'document',
+  /** One line: its tokens and groups, up to and including its line ending. */
+  LINE: 'line',
+  /** A parenthesized group: `(`, the lines inside it, and `)`. */
+  GROUP: 'group',
+  /** A run of spaces and tabs. */
+  WHITESPACE: 'whitespace',
+  /** One line ending: `\n`, `\r\n` or `\r`. */
+  NEWLINE: 'newline',
+  /** A comment, from `#` up to the line ending. */
+  COMMENT: 'comment',
+  /** `(` */
+  OPEN_PAREN: 'open_paren',
+  /** `)` */
+  CLOSE_PAREN: 'close_paren',
+  /** `:` */
+  COLON: 'colon',
+  /** A bare, quoted or versioned literal reference. */
+  REFERENCE: 'reference',
+});
+
+const NODE_KINDS = new Set([CstKind.DOCUMENT, CstKind.LINE, CstKind.GROUP]);
+
+/** One node or token of a concrete syntax tree. */
+export class CstNode {
+  #text;
+
+  /**
+   * @param {string} kind - One of {@link CstKind}
+   * @param {Object} parts - The token's text, or the node's children
+   * @param {string} [parts.text] - A token's text exactly as written
+   * @param {CstNode[]} [parts.children] - A node's children
+   * @param {string|null} [parts.value] - What a reference token means
+   * @param {string|null} [parts.quote] - The delimiter of a quoted reference
+   */
+  constructor(
+    kind,
+    { text = '', children = [], value = null, quote = null } = {}
+  ) {
+    /** One of {@link CstKind}. */
+    this.kind = kind;
+    /** The children of a node, in document order; empty for a token. */
+    this.children = children;
+    /**
+     * What a reference token means, its delimiters and escapes resolved (the
+     * ID a parsed link has); null for every other node and token.
+     */
+    this.value = value;
+    /**
+     * The delimiter a quoted reference token was written between (`'`, `"` or
+     * `` ` ``); null for a bare or literal reference and every other kind.
+     */
+    this.quote = quote;
+    this.#text = text;
+  }
+
+  /** Whether this is a token rather than a node with children. */
+  get isToken() {
+    return !NODE_KINDS.has(this.kind);
+  }
+
+  /** The exact text this node or token was read from. */
+  get text() {
+    if (this.isToken) return this.#text;
+    return this.children.map((child) => child.text).join('');
+  }
+
+  /** The exact text this node or token was read from. */
+  toString() {
+    return this.text;
+  }
+
+  /**
+   * Every token of this subtree, in document order.
+   * @returns {CstNode[]}
+   */
+  tokens() {
+    if (this.isToken) return [this];
+    return this.children.flatMap((child) => child.tokens());
+  }
+
+  /**
+   * The tree as an S-expression: `(kind child ...)` for a node and
+   * `kind:"text"` for a token, the text escaped as a JSON string would escape
+   * `\`, `"`, line feeds, carriage returns and tabs. Two trees are the same
+   * exactly when their S-expressions are.
+   * @returns {string}
+   * @example
+   * new Parser().parseCst("a '1'").toSExpression();
+   * // (document (line reference:"a" whitespace:" " reference:"'1'"))
+   */
+  toSExpression() {
+    if (this.isToken) return `${this.kind}:${quoteText(this.#text)}`;
+    return `(${[this.kind, ...this.children.map((child) => child.toSExpression())].join(' ')})`;
+  }
+}
+
+const ESCAPES = {
+  '\\': '\\\\',
+  '"': '\\"',
+  '\n': '\\n',
+  '\r': '\\r',
+  '\t': '\\t',
+};
+
+function quoteText(text) {
+  return `"${text.replace(/[\\"\n\r\t]/g, (character) => ESCAPES[character])}"`;
+}
+
+/** What ends a bare reference. */
+const BARE_REFERENCE_END = new Set([' ', '\t', '\r', '\n', '(', ':', ')']);
+
+/**
+ * The concrete syntax tree of a document that has already been parsed.
+ * @param {string} input - The document as written
+ * @param {Object} options
+ * @param {boolean} [options.comments=true] - Whether `#` starts a comment
+ * @returns {CstNode} The `document` node
+ */
+export function buildCst(input, { comments = true } = {}) {
+  // Tokens end where they end in the document the grammar reads: the one with
+  // its comments blanked, which keeps every position of the one written.
+  // Delimited references are read, as the grammar reads them, in the document
+  // as written, so a comment never changes what a reference holds.
+  const prepared = comments ? stripComments(input) : input;
+  const references = new DelimitedReferences(input);
+  const document = new CstNode(CstKind.DOCUMENT);
+  // The open containers, innermost last, each with the line being filled.
+  const containers = [{ node: document, line: null }];
+
+  const add = (token) => {
+    const container = containers[containers.length - 1];
+    if (container.line === null) {
+      container.line = new CstNode(CstKind.LINE);
+      container.node.children.push(container.line);
+    }
+    container.line.children.push(token);
+  };
+  const token = (kind, start, end, parts = {}) =>
+    new CstNode(kind, { text: input.slice(start, end), ...parts });
+
+  let position = 0;
+  while (position < input.length) {
+    const character = input[position];
+    let end = position + 1;
+
+    if (comments && character === '#' && prepared[position] !== '#') {
+      // A blanked `#` opens a comment, which runs to the line ending.
+      while (end < input.length && input[end] !== '\n' && input[end] !== '\r') {
+        end++;
+      }
+      add(token(CstKind.COMMENT, position, end));
+    } else if (character === '\r' || character === '\n') {
+      if (character === '\r' && input[end] === '\n') end++;
+      add(token(CstKind.NEWLINE, position, end));
+      containers[containers.length - 1].line = null;
+    } else if (character === ' ' || character === '\t') {
+      while (
+        end < input.length &&
+        (input[end] === ' ' || input[end] === '\t')
+      ) {
+        end++;
+      }
+      add(token(CstKind.WHITESPACE, position, end));
+    } else if (character === '(') {
+      const group = new CstNode(CstKind.GROUP, {
+        children: [token(CstKind.OPEN_PAREN, position, end)],
+      });
+      add(group);
+      containers.push({ node: group, line: null });
+    } else if (character === ')') {
+      if (containers.length === 1) {
+        throw new SyntaxError(`Unbalanced ")" at offset ${position}`);
+      }
+      containers
+        .pop()
+        .node.children.push(token(CstKind.CLOSE_PAREN, position, end));
+    } else if (character === ':') {
+      add(token(CstKind.COLON, position, end));
+    } else {
+      end = referenceEnd(prepared, position, references);
+      const text = input.slice(position, end);
+      const quoted = QUOTES.includes(character) && references.readAt(position);
+      add(
+        token(CstKind.REFERENCE, position, end, {
+          value: quoted
+            ? quoted.value
+            : character === '~' && isLiteral(prepared, position, end)
+              ? decodeReferenceLiteral(text)
+              : text,
+          quote: quoted ? character : null,
+        })
+      );
+    }
+    position = end;
+  }
+
+  if (containers.length !== 1) {
+    throw new SyntaxError('Unbalanced "(" at the end of the document');
+  }
+  return document;
+}
+
+/** Where the reference that starts at `start` ends, read as the grammar reads it. */
+function referenceEnd(prepared, start, references) {
+  const literal = literalEnd(prepared, start);
+  if (literal !== -1) return literal;
+  if (QUOTES.includes(prepared[start])) {
+    const end = references.endAt(start);
+    if (end !== null) return end;
+  }
+  let end = start + 1;
+  while (end < prepared.length && !BARE_REFERENCE_END.has(prepared[end])) end++;
+  return end;
+}
+
+function isLiteral(prepared, start, end) {
+  return literalEnd(prepared, start) === end;
+}

package/src/Link.js

@@ -1,6 +1,16 @@
-import { escapeReference, unescapeReference } from './references.js';
+import {
+  escapeReference,
+  referenceQuote,
+  unescapeReference,
+} from './references.js';
 
 export class Link {
+  /**
+   * The ID exactly as the parsed document wrote it, kept out of the link's own
+   * enumerable properties so that comparing or serializing links is unchanged.
+   */
+  #rawId = null;
+
   /**
    * Create a new Link
    * @param {string|null} id - Optional identifier for the link
@@ -21,6 +31,36 @@ export class Link {
     }
   }
 
+  /**
+   * The ID exactly as the parsed document wrote it: `'1'` for a quoted `1`,
+   * `1` for a bare one, `~1{31}` for a versioned literal. Null for a link
+   * without an ID, a link built in code, a link made by combining the lines of
+   * an indented document, and every link read by a parser created with
+   * `syntax: false`. It takes no part in `equals()` or `format()`.
+   * @type {string|null}
+   */
+  get rawId() {
+    return this.#rawId;
+  }
+
+  set rawId(raw) {
+    if (raw !== null && raw !== undefined && typeof raw !== 'string') {
+      throw new TypeError('rawId must be a string or null');
+    }
+    this.#rawId = raw ?? null;
+  }
+
+  /**
+   * The delimiter the ID was written between: `'`, `"` or `` ` `` when it was
+   * quoted, and null when it was bare, a versioned literal, or not read from a
+   * document (see {@link Link#rawId}). `1` and `'1'` stay equal links; this
+   * only says how each was written.
+   * @type {string|null}
+   */
+  get quote() {
+    return referenceQuote(this.#rawId);
+  }
+
   /**
    * Convert link to string representation
    * @returns {string} String representation of the link

package/src/Parser.js

@@ -2,6 +2,7 @@ import { Link } from './Link.js';
 import { LinksGroup } from './LinksGroup.js';
 import { ParseError } from './ParseError.js';
 import { stripComments } from './comments.js';
+import { buildCst } from './Cst.js';
 import * as parserModule from './parser-generated.js';
 
 /**
@@ -18,11 +19,13 @@ export class Parser {
    * @param {number} options.maxInputSize - Maximum input size in bytes (default: 10MB)
    * @param {number} options.maxDepth - How deep links may nest: every parenthesized group and every indentation level is one level (default: 64)
    * @param {boolean} options.comments - If false, read `#` as an ordinary character instead of the start of a comment (default: true)
+   * @param {boolean} options.syntax - If false, read links as a bare abstract syntax tree: they do not record how their IDs were written, so `rawId` and `quote` are null (default: true)
    */
   constructor(options = {}) {
     this.maxInputSize = options.maxInputSize || 10 * 1024 * 1024; // 10MB default
     this.maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
     this.comments = options.comments ?? true;
+    this.syntax = options.syntax ?? true;
   }
 
   /**
@@ -48,6 +51,20 @@ export class Parser {
     );
   }
 
+  /**
+   * Read the document as a lossless concrete syntax tree: every character of
+   * the input, comments and whitespace included, belongs to exactly one token,
+   * so `parseCst(input).toString() === input`. The document is first parsed
+   * with this parser's options, so it is refused exactly when `parse()` would
+   * refuse it.
+   * @param {string} input - The Lino document
+   * @returns {CstNode} The `document` node
+   */
+  parseCst(input) {
+    this.parse(input);
+    return buildCst(input, { comments: this.comments });
+  }
+
   _transformGroup(item) {
     return new LinksGroup(
       this.transformLink(item),
@@ -74,6 +91,7 @@ export class Parser {
     try {
       const rawResult = parserModule.parse(prepared, {
         maxDepth: this.maxDepth,
+        source: input,
       });
       return transform(rawResult);
     } catch (error) {
@@ -119,6 +137,7 @@ export class Parser {
         );
         const linkWithChildren = {
           id: item.id,
+          rawId: item.rawId,
           values: childValues,
         };
         const currentLink = this.transformLink(linkWithChildren);
@@ -163,18 +182,24 @@ export class Parser {
   transformIndentedValue(item) {
     const children = item.children || [];
     if (children.length && item.id != null && !item.values?.length) {
-      return new Link(
-        item.id,
-        children.map((child) => this.transformIndentedValue(child))
+      return this._withRawId(
+        new Link(
+          item.id,
+          children.map((child) => this.transformIndentedValue(child))
+        ),
+        item
       );
     }
 
     const current = this.transformLink(item);
     if (children.length) {
-      return new Link(current.id, [
-        ...current.values,
-        ...children.map((child) => this.transformIndentedValue(child)),
-      ]);
+      return this._withRawId(
+        new Link(current.id, [
+          ...current.values,
+          ...children.map((child) => this.transformIndentedValue(child)),
+        ]),
+        item
+      );
     }
     if (
       item.id == null &&
@@ -252,18 +277,32 @@ export class Parser {
 
     // Handle simple reference objects like {id: 'a'}
     if (item.id !== undefined && !item.values && !item.children) {
-      return new Link(item.id);
+      return this._withRawId(new Link(item.id), item);
     }
 
     // For items with values, create a link with those values
     if (item.values && Array.isArray(item.values)) {
       // Create a link with id (if present) and transformed values
-      const link = new Link(item.id ?? null, []);
+      const link = this._withRawId(new Link(item.id ?? null, []), item);
       link.values = item.values.map((v) => this.transformLink(v));
       return link;
     }
 
     // Default case
-    return new Link(item.id ?? null, []);
+    return this._withRawId(new Link(item.id ?? null, []), item);
+  }
+
+  /**
+   * Record how the parsed item wrote its ID, unless this parser reads a bare
+   * abstract syntax tree.
+   * @param {Link} link - The link made from the item
+   * @param {Object} item - The parsed item, with the `rawId` the grammar read
+   * @returns {Link} The same link
+   */
+  _withRawId(link, item) {
+    if (this.syntax && link.id !== null && typeof item.rawId === 'string') {
+      link.rawId = item.rawId;
+    }
+    return link;
   }
 }

package/src/ReferenceLiteral.js

@@ -5,6 +5,28 @@ export function hasReferenceLiteralPrefix(text, start = 0) {
   return /^~[0-9]+\{/.test(text.slice(start));
 }
 
+/** What a versioned literal's payload cannot contain. */
+const LITERAL_PAYLOAD_END = new Set(['}', ' ', '\t', '\r', '\n', '(', ')']);
+
+/**
+ * Where the versioned literal `~<digits>{<payload>}` that starts at `start`
+ * ends, or -1 if none starts there. A single forward scan, so it stays linear
+ * on any input (a regular expression here was flagged as a ReDoS risk).
+ */
+export function referenceLiteralEnd(text, start) {
+  if (text[start] !== '~') return -1;
+  let index = start + 1;
+  while (index < text.length && text[index] >= '0' && text[index] <= '9') {
+    index++;
+  }
+  if (index === start + 1 || text[index] !== '{') return -1;
+  index++;
+  while (index < text.length && !LITERAL_PAYLOAD_END.has(text[index])) {
+    index++;
+  }
+  return text[index] === '}' ? index + 1 : -1;
+}
+
 /** Encode exact Unicode text as a version 1 UTF-8 hexadecimal reference. */
 export function encodeReferenceLiteral(text) {
   if (typeof text !== 'string' || !text.isWellFormed()) {

package/src/StreamParser.js

@@ -1,6 +1,7 @@
 import { Emitter } from './Emitter.js';
 import { Parser } from './Parser.js';
 import { DelimitedReferences } from './quotes.js';
+import { referenceLiteralEnd } from './ReferenceLiteral.js';
 
 const DEFAULT_MAX_BUFFER_SIZE = 10 * 1024 * 1024;
 
@@ -246,26 +247,20 @@ function countNewlines(value) {
 }
 
 function structurallyComplete(document, comments) {
+  // Tokens are read as the comment reader and the grammar read them: a
+  // reference may start after a separator, and right after a delimited
+  // reference or a literal (issue #377).
   const quotes = ['"', "'", '`'];
-  const beforeReference = [' ', '\t', '\n', '\r', '(', ':'];
+  const separators = [' ', '\t', '\n', '\r', '(', ':', ')'];
   const beforeComment = [' ', '\t', '\n', '\r'];
   const references = new DelimitedReferences(document);
   let depth = 0;
+  let tokenStart = true;
 
   for (let position = 0; position < document.length; position++) {
     const character = document[position];
     const previous = position === 0 ? null : document[position - 1];
 
-    if (
-      quotes.includes(character) &&
-      (previous === null || beforeReference.includes(previous))
-    ) {
-      const end = references.endAt(position);
-      if (end === null) return false;
-      position = end - 1;
-      continue;
-    }
-
     if (
       comments &&
       character === '#' &&
@@ -274,11 +269,36 @@ function structurallyComplete(document, comments) {
       const newline = document.indexOf('\n', position);
       if (newline === -1) return true;
       position = newline;
+      tokenStart = true;
+      continue;
+    }
+
+    if (separators.includes(character)) {
+      if (character === '(') depth += 1;
+      if (character === ')') depth -= 1;
+      tokenStart = true;
+      continue;
+    }
+
+    if (tokenStart && quotes.includes(character)) {
+      // A reference nothing closes yet may be closed by a later line, which
+      // changes what it holds: an even run is the empty reference only when
+      // no run in the whole document closes it.
+      const reading = references.readAt(position);
+      if (reading === null || !reading.closed) return false;
+      position += reading.length - 1;
       continue;
     }
 
-    if (character === '(') depth += 1;
-    if (character === ')') depth -= 1;
+    if (tokenStart) {
+      const end = referenceLiteralEnd(document, position);
+      if (end !== -1) {
+        position = end - 1;
+        continue;
+      }
+    }
+
+    tokenStart = false;
   }
 
   return depth === 0;