links-notation 0.28.0 → 0.30.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "links-notation",
-  "version": "0.28.0",
+  "version": "0.30.0",
   "description": "Links Notation parser for JavaScript",
   "main": "dist/index.js",
   "types": "index.d.ts",
@@ -32,10 +32,13 @@
   },
   "homepage": "https://github.com/link-foundation/links-notation#readme",
   "devDependencies": {
-    "bun-types": "^1.4.3",
-    "eslint": "^10.13.0",
-    "peggy": "^5.1.0",
+    "bun-types": "^1.4.4",
+    "eslint": "^10.14.0",
+    "peggy": "^5.1.1",
     "prettier": "^3.9.10",
     "typescript": "^7.0.2"
+  },
+  "overrides": {
+    "minimatch": "^11.0.4"
   }
 }

package/src/Cst.js

@@ -17,7 +17,10 @@
 
 import { stripComments } from './comments.js';
 import { DelimitedReferences, QUOTES } from './quotes.js';
-import { decodeReferenceLiteral } from './ReferenceLiteral.js';
+import {
+  decodeReferenceLiteral,
+  referenceLiteralEnd as literalEnd,
+} from './ReferenceLiteral.js';
 
 /** The kinds of nodes and tokens of a concrete syntax tree. */
 export const CstKind = Object.freeze({
@@ -131,28 +134,6 @@ function quoteText(text) {
   return `"${text.replace(/[\\"\n\r\t]/g, (character) => ESCAPES[character])}"`;
 }
 
-/** What a versioned literal's payload cannot contain. */
-const LITERAL_PAYLOAD_END = new Set(['}', ' ', '\t', '\r', '\n', '(', ')']);
-
-/**
- * Where the versioned literal `~<digits>{<payload>}` that starts at `start`
- * ends, or -1 if none starts there. A single forward scan, so it stays linear
- * on any input (a regular expression here was flagged as a ReDoS risk).
- */
-function literalEnd(text, start) {
-  if (text[start] !== '~') return -1;
-  let index = start + 1;
-  while (index < text.length && text[index] >= '0' && text[index] <= '9') {
-    index++;
-  }
-  if (index === start + 1 || text[index] !== '{') return -1;
-  index++;
-  while (index < text.length && !LITERAL_PAYLOAD_END.has(text[index])) {
-    index++;
-  }
-  return text[index] === '}' ? index + 1 : -1;
-}
-
 /** What ends a bare reference. */
 const BARE_REFERENCE_END = new Set([' ', '\t', '\r', '\n', '(', ':', ')']);
 
@@ -164,10 +145,12 @@ const BARE_REFERENCE_END = new Set([' ', '\t', '\r', '\n', '(', ':', ')']);
  * @returns {CstNode} The `document` node
  */
 export function buildCst(input, { comments = true } = {}) {
-  // References are read in the document the grammar reads: the one with its
-  // comments blanked, which keeps every position of the one written.
+  // Tokens end where they end in the document the grammar reads: the one with
+  // its comments blanked, which keeps every position of the one written.
+  // Delimited references are read, as the grammar reads them, in the document
+  // as written, so a comment never changes what a reference holds.
   const prepared = comments ? stripComments(input) : input;
-  const references = new DelimitedReferences(prepared);
+  const references = new DelimitedReferences(input);
   const document = new CstNode(CstKind.DOCUMENT);
   // The open containers, innermost last, each with the line being filled.
   const containers = [{ node: document, line: null }];

package/src/ReferenceLiteral.js

@@ -5,6 +5,28 @@ export function hasReferenceLiteralPrefix(text, start = 0) {
   return /^~[0-9]+\{/.test(text.slice(start));
 }
 
+/** What a versioned literal's payload cannot contain. */
+const LITERAL_PAYLOAD_END = new Set(['}', ' ', '\t', '\r', '\n', '(', ')']);
+
+/**
+ * Where the versioned literal `~<digits>{<payload>}` that starts at `start`
+ * ends, or -1 if none starts there. A single forward scan, so it stays linear
+ * on any input (a regular expression here was flagged as a ReDoS risk).
+ */
+export function referenceLiteralEnd(text, start) {
+  if (text[start] !== '~') return -1;
+  let index = start + 1;
+  while (index < text.length && text[index] >= '0' && text[index] <= '9') {
+    index++;
+  }
+  if (index === start + 1 || text[index] !== '{') return -1;
+  index++;
+  while (index < text.length && !LITERAL_PAYLOAD_END.has(text[index])) {
+    index++;
+  }
+  return text[index] === '}' ? index + 1 : -1;
+}
+
 /** Encode exact Unicode text as a version 1 UTF-8 hexadecimal reference. */
 export function encodeReferenceLiteral(text) {
   if (typeof text !== 'string' || !text.isWellFormed()) {

package/src/StreamParser.js

@@ -1,6 +1,7 @@
 import { Emitter } from './Emitter.js';
 import { Parser } from './Parser.js';
 import { DelimitedReferences } from './quotes.js';
+import { referenceLiteralEnd } from './ReferenceLiteral.js';
 
 const DEFAULT_MAX_BUFFER_SIZE = 10 * 1024 * 1024;
 
@@ -246,26 +247,20 @@ function countNewlines(value) {
 }
 
 function structurallyComplete(document, comments) {
+  // Tokens are read as the comment reader and the grammar read them: a
+  // reference may start after a separator, and right after a delimited
+  // reference or a literal (issue #377).
   const quotes = ['"', "'", '`'];
-  const beforeReference = [' ', '\t', '\n', '\r', '(', ':'];
+  const separators = [' ', '\t', '\n', '\r', '(', ':', ')'];
   const beforeComment = [' ', '\t', '\n', '\r'];
   const references = new DelimitedReferences(document);
   let depth = 0;
+  let tokenStart = true;
 
   for (let position = 0; position < document.length; position++) {
     const character = document[position];
     const previous = position === 0 ? null : document[position - 1];
 
-    if (
-      quotes.includes(character) &&
-      (previous === null || beforeReference.includes(previous))
-    ) {
-      const end = references.endAt(position);
-      if (end === null) return false;
-      position = end - 1;
-      continue;
-    }
-
     if (
       comments &&
       character === '#' &&
@@ -274,11 +269,36 @@ function structurallyComplete(document, comments) {
       const newline = document.indexOf('\n', position);
       if (newline === -1) return true;
       position = newline;
+      tokenStart = true;
+      continue;
+    }
+
+    if (separators.includes(character)) {
+      if (character === '(') depth += 1;
+      if (character === ')') depth -= 1;
+      tokenStart = true;
+      continue;
+    }
+
+    if (tokenStart && quotes.includes(character)) {
+      // A reference nothing closes yet may be closed by a later line, which
+      // changes what it holds: an even run is the empty reference only when
+      // no run in the whole document closes it.
+      const reading = references.readAt(position);
+      if (reading === null || !reading.closed) return false;
+      position += reading.length - 1;
       continue;
     }
 
-    if (character === '(') depth += 1;
-    if (character === ')') depth -= 1;
+    if (tokenStart) {
+      const end = referenceLiteralEnd(document, position);
+      if (end !== -1) {
+        position = end - 1;
+        continue;
+      }
+    }
+
+    tokenStart = false;
   }
 
   return depth === 0;

package/src/comments.js

@@ -13,23 +13,35 @@
  */
 
 import { DelimitedReferences, QUOTES, readReference } from './quotes.js';
+import { referenceLiteralEnd } from './ReferenceLiteral.js';
 
 /** The character that starts a comment. */
 export const COMMENT = '#';
 
-/** What a delimited reference may start after. */
-const BEFORE_REFERENCE = [' ', '\t', '\n', '\r', '(', ':'];
+/**
+ * What ends a bare reference. A token starts after one of these, at the start
+ * of the document, and right after a delimited reference or a versioned
+ * literal: the places the grammar reads a reference from.
+ */
+const SEPARATORS = [' ', '\t', '\n', '\r', '(', ':', ')'];
 
 /** What a comment may start after. */
 const BEFORE_COMMENT = [' ', '\t', '\n', '\r'];
 
 /**
  * The document with every comment blanked out.
+ *
+ * The document is read token by token, the way the grammar reads it, so a `#`
+ * inside a delimited reference that starts right after a group, another
+ * reference or a literal is content, as it is in one that starts after a
+ * space. References are read in the document as written, as the grammar
+ * reads them.
  * @param {string} document - The document to read
  * @returns {string} The document, of the same length, without its comments
  * @example
  * stripComments('a: b # why\n'); // 'a: b      \n'
  * stripComments('"# kept"\n'); // '"# kept"\n'
+ * stripComments('(a)" #kept"\n'); // '(a)" #kept"\n'
  * stripComments('issue#1047\n'); // 'issue#1047\n'
  */
 export function stripComments(document) {
@@ -37,20 +49,13 @@ export function stripComments(document) {
   // parse error reports are both counted in.
   let blanked = null;
   let position = 0;
+  // Whether a token may start at `position`.
+  let tokenStart = true;
   const references = new DelimitedReferences(document);
 
   while (position < document.length) {
     const character = document[position];
 
-    if (
-      QUOTES.includes(character) &&
-      follows(document, position, BEFORE_REFERENCE)
-    ) {
-      const end = references.endAt(position);
-      position = end === null ? position + 1 : end;
-      continue;
-    }
-
     if (character === COMMENT && follows(document, position, BEFORE_COMMENT)) {
       blanked = blanked ?? document.split('');
       while (
@@ -64,12 +69,46 @@ export function stripComments(document) {
       continue;
     }
 
+    if (SEPARATORS.includes(character)) {
+      tokenStart = true;
+      position++;
+      continue;
+    }
+
+    if (tokenStart) {
+      const end = delimitedTokenEnd(document, position, references);
+      if (end !== null) {
+        position = end;
+        continue;
+      }
+    }
+
+    // Anything else is part of a bare reference, which runs to a separator.
+    tokenStart = false;
     position++;
   }
 
   return blanked === null ? document : blanked.join('');
 }
 
+/**
+ * The position just past the delimited reference or versioned literal that
+ * starts the token at `start`, or `null` when the token there is a bare
+ * reference: a delimiter run that never closes is read as one, as the grammar
+ * reads it.
+ * @param {string} document - The document being read
+ * @param {number} start - Position where a token starts
+ * @param {DelimitedReferences} references - The references of the document
+ * @returns {number|null} The position just past the token
+ */
+export function delimitedTokenEnd(document, start, references) {
+  const literal = referenceLiteralEnd(document, start);
+  if (literal !== -1) {
+    return literal;
+  }
+  return QUOTES.includes(document[start]) ? references.endAt(start) : null;
+}
+
 /**
  * Whether the character before the given position is one the caller allows,
  * the start of the document counting as allowed.

package/src/grammar.pegjs

@@ -117,8 +117,10 @@
   }
 
   // Delimited references of this document, each read once however many
-  // alternatives ask for it (see quotes.js for how a reference is read)
-  const delimitedReferences = new DelimitedReferences(input);
+  // alternatives ask for it (see quotes.js for how a reference is read). They
+  // are read in the document as written, as the comment reader reads them, so
+  // blanking a comment never changes how far a reference reaches (issue #377).
+  const delimitedReferences = new DelimitedReferences(source);
 
   // Universal procedural parser for N-quote strings (any N >= 1)
   // Parses from the given position in the input string