links-notation 0.18.0 → 0.20.0

package/src/Parser.js

@@ -1,5 +1,6 @@
 import { Link } from './Link.js';
 import { ParseError } from './ParseError.js';
+import { stripComments } from './comments.js';
 import * as parserModule from './parser-generated.js';
 
 export class Parser {
@@ -8,10 +9,12 @@ export class Parser {
    * @param {Object} options - Parser options
    * @param {number} options.maxInputSize - Maximum input size in bytes (default: 10MB)
    * @param {number} options.maxDepth - Maximum nesting depth (default: 1000)
+   * @param {boolean} options.comments - If false, read `#` as an ordinary character instead of the start of a comment (default: true)
    */
   constructor(options = {}) {
     this.maxInputSize = options.maxInputSize || 10 * 1024 * 1024; // 10MB default
     this.maxDepth = options.maxDepth || 1000;
+    this.comments = options.comments ?? true;
   }
 
   /**
@@ -32,8 +35,12 @@ export class Parser {
       );
     }
 
+    // Comments are blanked rather than removed, so a position reported for the
+    // prepared document is the same position in the document the caller wrote.
+    const prepared = this.comments ? stripComments(input) : input;
+
     try {
-      const rawResult = parserModule.parse(input);
+      const rawResult = parserModule.parse(prepared);
       return this.transformResult(rawResult);
     } catch (error) {
       // A syntax error knows where it stopped; anything else is passed on with

package/src/StreamParser.js

@@ -0,0 +1,284 @@
+import { EventEmitter } from 'events';
+import { Parser } from './Parser.js';
+import { quotedReferenceEnd } from './comments.js';
+
+const DEFAULT_MAX_BUFFER_SIZE = 10 * 1024 * 1024;
+
+/**
+ * An error raised while finishing a streamed document.
+ *
+ * The canonical parser reports positions relative to the currently buffered
+ * record. This wrapper translates them back to the complete stream.
+ */
+export class StreamParseError extends Error {
+  constructor(error, startOffset, startLine) {
+    const localOffset = error?.offset ?? 0;
+    const localLine = error?.line ?? 1;
+    const column = error?.column ?? 1;
+    const line = startLine + localLine - 1;
+    super(
+      `Stream parse error at line ${line}, column ${column}: ${error.message}`
+    );
+    this.name = 'StreamParseError';
+    this.cause = error;
+    this.offset = startOffset + localOffset;
+    this.line = line;
+    this.column = column;
+    this.found = error?.found ?? null;
+    this.lineText = error?.lineText ?? '';
+    this.snippet = error?.snippet ?? '';
+  }
+}
+
+/**
+ * Incrementally parses complete top-level Links Notation records.
+ *
+ * A line can acquire indented children, so its terminating newline alone does
+ * not prove that the record is complete. The parser commits a buffered record
+ * as soon as the first non-indented character of the next record arrives. It
+ * uses the regular Parser as the source of truth, which keeps quoted strings,
+ * nested contexts, comments, and future grammar changes in one place.
+ */
+export class StreamParser extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.parser = options.parser ?? new Parser(options);
+    this.comments = options.comments ?? this.parser.comments ?? true;
+    this.maxBufferSize =
+      options.maxBufferSize ?? options.maxInputSize ?? DEFAULT_MAX_BUFFER_SIZE;
+    this.collect = options.collect ?? true;
+    this.reset();
+  }
+
+  /** Feed a string chunk and return the links made complete by that chunk. */
+  write(chunk) {
+    if (typeof chunk !== 'string') {
+      return this._fail(new TypeError('Input must be a string'));
+    }
+    if (this.ended) {
+      return this._fail(new Error('Cannot write after end()'));
+    }
+
+    const emitted = [];
+    for (let index = 0; index < chunk.length; index++) {
+      const character = chunk[index];
+      this.currentLine += character;
+      this.offset += 1;
+
+      if (character === '\n') {
+        this.buffer += this.currentLine;
+        this.currentLine = '';
+        this.lineClassified = false;
+        this.line += 1;
+        this.column = 1;
+      } else {
+        if (
+          !this.lineClassified &&
+          character !== ' ' &&
+          character !== '\t' &&
+          character !== '\r'
+        ) {
+          this.lineClassified = true;
+          const isComment = this.comments && character === '#';
+          if (!isComment) {
+            const indentation = leadingSpaces(this.currentLine);
+            this._startContentLine(indentation, emitted);
+          }
+        }
+        this.column += 1;
+      }
+
+      if (this.buffer.length + this.currentLine.length > this.maxBufferSize) {
+        return this._fail(
+          new RangeError(
+            `Buffered record exceeds maximum size of ${this.maxBufferSize} characters`
+          )
+        );
+      }
+    }
+    return emitted;
+  }
+
+  /**
+   * Finish the stream. With collection enabled (the default), returns every
+   * undrained link; otherwise returns only links completed by this call.
+   */
+  end(chunk = '') {
+    const emitted = chunk === '' ? [] : this.write(chunk);
+    if (this.ended) {
+      return this.collect ? [...this.links] : emitted;
+    }
+
+    const document = this.buffer + this.currentLine;
+    if (document.length > 0) {
+      let links;
+      try {
+        links = this.parser.parse(document);
+      } catch (error) {
+        const streamError = new StreamParseError(
+          error,
+          this.segmentOffset,
+          this.segmentLine
+        );
+        return this._fail(streamError);
+      }
+      this._publish(links, emitted);
+      this._advanceSegment(document);
+    }
+
+    this.buffer = '';
+    this.currentLine = '';
+    this.baseIndentation = null;
+    this.ended = true;
+    const result = this.collect ? [...this.links] : emitted;
+    this.emit('end', result);
+    return result;
+  }
+
+  /** Return and forget links retained since the previous drain. */
+  drain() {
+    const links = this.links;
+    this.links = [];
+    return links;
+  }
+
+  /** Reuse this parser for a new stream while keeping listeners and options. */
+  reset() {
+    this.buffer = '';
+    this.currentLine = '';
+    this.baseIndentation = null;
+    this.lineClassified = false;
+    this.links = [];
+    this.offset = 0;
+    this.line = 1;
+    this.column = 1;
+    this.segmentOffset = 0;
+    this.segmentLine = 1;
+    this.ended = false;
+    return this;
+  }
+
+  /** Position immediately after the last character written. */
+  position() {
+    return {
+      offset: this.offset,
+      line: this.line,
+      column: this.column,
+      buffered: this.buffer.length + this.currentLine.length,
+    };
+  }
+
+  /** Lazily parse a synchronous iterable of chunks. */
+  static *parse(chunks, options = {}) {
+    const parser = new StreamParser({ ...options, collect: false });
+    for (const chunk of chunks) {
+      yield* parser.write(chunk);
+    }
+    yield* parser.end();
+  }
+
+  /** Lazily parse an async iterable of chunks. */
+  static async *parseAsync(chunks, options = {}) {
+    const parser = new StreamParser({ ...options, collect: false });
+    for await (const chunk of chunks) {
+      yield* parser.write(chunk);
+    }
+    yield* parser.end();
+  }
+
+  _startContentLine(indentation, emitted) {
+    if (
+      this.buffer.length > 0 &&
+      this.baseIndentation !== null &&
+      indentation <= this.baseIndentation &&
+      structurallyComplete(this.buffer, this.comments)
+    ) {
+      let links;
+      try {
+        links = this.parser.parse(this.buffer);
+      } catch {
+        links = null;
+      }
+      if (links !== null) {
+        this._publish(links, emitted);
+        this._advanceSegment(this.buffer);
+        this.buffer = '';
+        this.baseIndentation = null;
+      }
+    }
+
+    if (this.baseIndentation === null) {
+      this.baseIndentation = indentation;
+    }
+  }
+
+  _publish(links, emitted) {
+    for (const link of links) {
+      emitted.push(link);
+      if (this.collect) this.links.push(link);
+      this.emit('link', link);
+    }
+  }
+
+  _advanceSegment(document) {
+    this.segmentOffset += document.length;
+    this.segmentLine += countNewlines(document);
+  }
+
+  _fail(error) {
+    this.emit('error', error);
+    throw error;
+  }
+}
+
+function leadingSpaces(line) {
+  let indentation = 0;
+  while (line[indentation] === ' ') indentation += 1;
+  return indentation;
+}
+
+function countNewlines(value) {
+  let count = 0;
+  for (const character of value) {
+    if (character === '\n') count += 1;
+  }
+  return count;
+}
+
+function structurallyComplete(document, comments) {
+  const quotes = ['"', "'", '`'];
+  const beforeReference = [' ', '\t', '\n', '\r', '(', ':'];
+  const beforeComment = [' ', '\t', '\n', '\r'];
+  let depth = 0;
+
+  for (let position = 0; position < document.length; position++) {
+    const character = document[position];
+    const previous = position === 0 ? null : document[position - 1];
+
+    if (
+      quotes.includes(character) &&
+      (previous === null || beforeReference.includes(previous))
+    ) {
+      const end = quotedReferenceEnd(document, position);
+      if (end === null) return false;
+      position = end - 1;
+      continue;
+    }
+
+    if (
+      comments &&
+      character === '#' &&
+      (previous === null || beforeComment.includes(previous))
+    ) {
+      const newline = document.indexOf('\n', position);
+      if (newline === -1) return true;
+      position = newline;
+      continue;
+    }
+
+    if (character === '(') depth += 1;
+    if (character === ')') depth -= 1;
+  }
+
+  return depth === 0;
+}

package/src/comments.js

@@ -0,0 +1,164 @@
+/**
+ * Comments, and the rule that decides where one starts.
+ *
+ * `#` starts a comment when it opens a token: at the start of the document or
+ * after a space, a tab or a line break. A `#` written inside a token
+ * (`issue#1047`) or inside a delimited reference (`"# not a comment"`) is an
+ * ordinary character.
+ *
+ * {@link stripComments} replaces every character of every comment with a space
+ * rather than removing it, so every later character keeps the offset it had in
+ * the document the caller wrote, and a parse error still points at the position
+ * the reader can see.
+ */
+
+/** The character that starts a comment. */
+export const COMMENT = '#';
+
+/** The delimiters a reference can be written between. */
+const QUOTES = ['"', "'", '`'];
+
+/** What a delimited reference may start after. */
+const BEFORE_REFERENCE = [' ', '\t', '\n', '\r', '(', ':'];
+
+/** What a comment may start after. */
+const BEFORE_COMMENT = [' ', '\t', '\n', '\r'];
+
+/**
+ * The document with every comment blanked out.
+ * @param {string} document - The document to read
+ * @returns {string} The document, of the same length, without its comments
+ * @example
+ * stripComments('a: b # why\n'); // 'a: b      \n'
+ * stripComments('"# kept"\n'); // '"# kept"\n'
+ * stripComments('issue#1047\n'); // 'issue#1047\n'
+ */
+export function stripComments(document) {
+  // Split into UTF-16 units, the units the positions here and the positions a
+  // parse error reports are both counted in.
+  let blanked = null;
+  let position = 0;
+
+  while (position < document.length) {
+    const character = document[position];
+
+    if (
+      QUOTES.includes(character) &&
+      follows(document, position, BEFORE_REFERENCE)
+    ) {
+      const end = quotedReferenceEnd(document, position);
+      position = end === null ? position + 1 : end;
+      continue;
+    }
+
+    if (character === COMMENT && follows(document, position, BEFORE_COMMENT)) {
+      blanked = blanked ?? document.split('');
+      while (
+        position < document.length &&
+        document[position] !== '\n' &&
+        document[position] !== '\r'
+      ) {
+        blanked[position] = ' ';
+        position++;
+      }
+      continue;
+    }
+
+    position++;
+  }
+
+  return blanked === null ? document : blanked.join('');
+}
+
+/**
+ * Whether the character before the given position is one the caller allows,
+ * the start of the document counting as allowed.
+ * @param {string} document - The document being read
+ * @param {number} position - Position of the character being judged
+ * @param {string[]} allowed - The characters that may stand before it
+ * @returns {boolean} True when the position opens a token
+ */
+function follows(document, position, allowed) {
+  return position === 0 || allowed.includes(document[position - 1]);
+}
+
+/**
+ * The position just past the delimited reference that starts at `start`, or
+ * `null` when nothing there opens one.
+ *
+ * This follows the same rules as the `parseQuotedStringAt` of the grammar: a
+ * run of N delimiters closes at the next run of exactly N, a run of 2N is that
+ * delimiter escaped, and an even run that encloses nothing substantive is the
+ * empty reference.
+ * @param {string} document - The document being read
+ * @param {number} start - Position of the opening delimiter
+ * @returns {number|null} The position just past the reference
+ */
+export function quotedReferenceEnd(document, start) {
+  const quote = document[start];
+  if (!QUOTES.includes(quote)) {
+    return null;
+  }
+
+  let position = start;
+  while (position < document.length && document[position] === quote) {
+    position++;
+  }
+  const count = position - start;
+  const isEvenRun = count % 2 === 0;
+  const emptyReference = isEvenRun ? start + count : null;
+
+  const closing = quote.repeat(count);
+  const escape = quote.repeat(count * 2);
+  let content = '';
+
+  while (position < document.length) {
+    if (document.startsWith(escape, position)) {
+      content += closing;
+      position += escape.length;
+      continue;
+    }
+
+    if (document.startsWith(closing, position)) {
+      const afterClosing = position + count;
+      if (afterClosing >= document.length || document[afterClosing] !== quote) {
+        if (isEvenRun && !isSubstantiveBody(content)) {
+          return emptyReference;
+        }
+        return afterClosing;
+      }
+    }
+
+    content += document[position];
+    position++;
+  }
+
+  return emptyReference;
+}
+
+/**
+ * Whether a body written between an even run of delimiters carries something a
+ * pair of delimiters enclosing nothing cannot.
+ * @param {string} content - The body between the delimiters
+ * @returns {boolean} True when the n-quote reading is the one to take
+ */
+function isSubstantiveBody(content) {
+  let depth = 0;
+  let hasVisible = false;
+
+  for (const character of content) {
+    if (character === '(') {
+      depth++;
+    } else if (character === ')') {
+      depth--;
+      if (depth < 0) {
+        return false;
+      }
+    }
+    if (!/\s/.test(character)) {
+      hasVisible = true;
+    }
+  }
+
+  return hasVisible && depth === 0;
+}

package/src/grammar.pegjs

@@ -264,7 +264,13 @@ PUSH_INDENTATION = spaces:" "* &{ return normalizeIndentation(spaces) > getCurre
 
 CHECK_INDENTATION = spaces:" "* &{ return checkIndentation(spaces); }
 
-eol = __ ([\r\n]+ / eof / nestedGroupEnd)
+eol = __ (lineBreaks / eof / nestedGroupEnd)
+
+// A line ends at the first line break, and the blank lines that follow it
+// belong to that ending: a line holding nothing but spaces or tabs separates
+// links exactly the way an empty line does. Blanking a comment leaves such a
+// line behind, so this is also what lets a comment stand on a line of its own.
+lineBreaks = [\r\n]+ ([ \t]+ [\r\n]+)*
 
 // Inside a parenthesised group the closing parenthesis terminates a line the
 // same way the end of the input does at the root.

package/src/index.js

@@ -1,6 +1,8 @@
 export { Link, formatLinks } from './Link.js';
 export { LinksGroup } from './LinksGroup.js';
 export { Parser } from './Parser.js';
+export { StreamParseError, StreamParser } from './StreamParser.js';
 export { ParseError } from './ParseError.js';
 export { FormatConfig } from './FormatConfig.js';
 export { FormatOptions } from './FormatOptions.js';
+export { stripComments } from './comments.js';

package/src/parser-generated.js

@@ -1487,39 +1487,173 @@ function peg$parse(input, options) {
   }
 
   function peg$parseeol() {
-    let s0, s1, s2, s3;
+    let s0, s1, s2;
 
     s0 = peg$currPos;
     s1 = peg$parse__();
-    s2 = [];
-    s3 = input.charAt(peg$currPos);
-    if (peg$r1.test(s3)) {
+    s2 = peg$parselineBreaks();
+    if (s2 === peg$FAILED) {
+      s2 = peg$parseeof();
+      if (s2 === peg$FAILED) {
+        s2 = peg$parsenestedGroupEnd();
+      }
+    }
+    if (s2 !== peg$FAILED) {
+      s1 = [s1, s2];
+      s0 = s1;
+    } else {
+      peg$currPos = s0;
+      s0 = peg$FAILED;
+    }
+
+    return s0;
+  }
+
+  function peg$parselineBreaks() {
+    let s0, s1, s2, s3, s4, s5, s6;
+
+    s0 = peg$currPos;
+    s1 = [];
+    s2 = input.charAt(peg$currPos);
+    if (peg$r1.test(s2)) {
       peg$currPos++;
     } else {
-      s3 = peg$FAILED;
+      s2 = peg$FAILED;
       if (peg$silentFails === 0) { peg$fail(peg$e1); }
     }
-    if (s3 !== peg$FAILED) {
-      while (s3 !== peg$FAILED) {
-        s2.push(s3);
-        s3 = input.charAt(peg$currPos);
-        if (peg$r1.test(s3)) {
+    if (s2 !== peg$FAILED) {
+      while (s2 !== peg$FAILED) {
+        s1.push(s2);
+        s2 = input.charAt(peg$currPos);
+        if (peg$r1.test(s2)) {
           peg$currPos++;
         } else {
-          s3 = peg$FAILED;
+          s2 = peg$FAILED;
           if (peg$silentFails === 0) { peg$fail(peg$e1); }
         }
       }
     } else {
-      s2 = peg$FAILED;
+      s1 = peg$FAILED;
     }
-    if (s2 === peg$FAILED) {
-      s2 = peg$parseeof();
-      if (s2 === peg$FAILED) {
-        s2 = peg$parsenestedGroupEnd();
+    if (s1 !== peg$FAILED) {
+      s2 = [];
+      s3 = peg$currPos;
+      s4 = [];
+      s5 = input.charAt(peg$currPos);
+      if (peg$r0.test(s5)) {
+        peg$currPos++;
+      } else {
+        s5 = peg$FAILED;
+        if (peg$silentFails === 0) { peg$fail(peg$e0); }
+      }
+      if (s5 !== peg$FAILED) {
+        while (s5 !== peg$FAILED) {
+          s4.push(s5);
+          s5 = input.charAt(peg$currPos);
+          if (peg$r0.test(s5)) {
+            peg$currPos++;
+          } else {
+            s5 = peg$FAILED;
+            if (peg$silentFails === 0) { peg$fail(peg$e0); }
+          }
+        }
+      } else {
+        s4 = peg$FAILED;
+      }
+      if (s4 !== peg$FAILED) {
+        s5 = [];
+        s6 = input.charAt(peg$currPos);
+        if (peg$r1.test(s6)) {
+          peg$currPos++;
+        } else {
+          s6 = peg$FAILED;
+          if (peg$silentFails === 0) { peg$fail(peg$e1); }
+        }
+        if (s6 !== peg$FAILED) {
+          while (s6 !== peg$FAILED) {
+            s5.push(s6);
+            s6 = input.charAt(peg$currPos);
+            if (peg$r1.test(s6)) {
+              peg$currPos++;
+            } else {
+              s6 = peg$FAILED;
+              if (peg$silentFails === 0) { peg$fail(peg$e1); }
+            }
+          }
+        } else {
+          s5 = peg$FAILED;
+        }
+        if (s5 !== peg$FAILED) {
+          s4 = [s4, s5];
+          s3 = s4;
+        } else {
+          peg$currPos = s3;
+          s3 = peg$FAILED;
+        }
+      } else {
+        peg$currPos = s3;
+        s3 = peg$FAILED;
+      }
+      while (s3 !== peg$FAILED) {
+        s2.push(s3);
+        s3 = peg$currPos;
+        s4 = [];
+        s5 = input.charAt(peg$currPos);
+        if (peg$r0.test(s5)) {
+          peg$currPos++;
+        } else {
+          s5 = peg$FAILED;
+          if (peg$silentFails === 0) { peg$fail(peg$e0); }
+        }
+        if (s5 !== peg$FAILED) {
+          while (s5 !== peg$FAILED) {
+            s4.push(s5);
+            s5 = input.charAt(peg$currPos);
+            if (peg$r0.test(s5)) {
+              peg$currPos++;
+            } else {
+              s5 = peg$FAILED;
+              if (peg$silentFails === 0) { peg$fail(peg$e0); }
+            }
+          }
+        } else {
+          s4 = peg$FAILED;
+        }
+        if (s4 !== peg$FAILED) {
+          s5 = [];
+          s6 = input.charAt(peg$currPos);
+          if (peg$r1.test(s6)) {
+            peg$currPos++;
+          } else {
+            s6 = peg$FAILED;
+            if (peg$silentFails === 0) { peg$fail(peg$e1); }
+          }
+          if (s6 !== peg$FAILED) {
+            while (s6 !== peg$FAILED) {
+              s5.push(s6);
+              s6 = input.charAt(peg$currPos);
+              if (peg$r1.test(s6)) {
+                peg$currPos++;
+              } else {
+                s6 = peg$FAILED;
+                if (peg$silentFails === 0) { peg$fail(peg$e1); }
+              }
+            }
+          } else {
+            s5 = peg$FAILED;
+          }
+          if (s5 !== peg$FAILED) {
+            s4 = [s4, s5];
+            s3 = s4;
+          } else {
+            peg$currPos = s3;
+            s3 = peg$FAILED;
+          }
+        } else {
+          peg$currPos = s3;
+          s3 = peg$FAILED;
+        }
       }
-    }
-    if (s2 !== peg$FAILED) {
       s1 = [s1, s2];
       s0 = s1;
     } else {