links-notation 0.27.1 → 0.30.0

package/src/quotes.js

@@ -43,8 +43,10 @@ export class DelimitedReferences {
   /**
    * The reference opened at `start`.
    * @param {number} start - Position of the opening delimiter
-   * @returns {{value: string, length: number}|null} What the reference holds
-   *   and how many characters it takes, or null when nothing opens there
+   * @returns {{value: string, length: number, closed: boolean}|null} What the
+   *   reference holds, how many characters it takes and whether a run closes
+   *   it (an even run nothing closes is the empty reference, until a closing
+   *   run is written after it), or null when nothing opens there
    */
   readAt(start) {
     if (!this.readings.has(start)) {
@@ -103,8 +105,9 @@ export class DelimitedReferences {
  * reference that is as quick, since it is read once.
  * @param {string} document - The document being read
  * @param {number} start - Position of the opening delimiter
- * @returns {{value: string, length: number}|null} What the reference holds
- *   and how many characters it takes, or null when nothing opens there
+ * @returns {{value: string, length: number, closed: boolean}|null} What the
+ *   reference holds, how many characters it takes and whether a run closes it,
+ *   or null when nothing opens there
  */
 export function readReference(document, start) {
   const quote = document[start];
@@ -131,10 +134,13 @@ export function readReference(document, start) {
  * by the run that ends at `end` when there is one.
  */
 function reading(document, start, quote, count, end) {
-  const emptyReference = count % 2 === 0 ? { value: '', length: count } : null;
   if (end === null) {
-    return emptyReference;
+    // Nothing closes the run: an even one is the empty reference, which a
+    // later closing run would still turn into one holding what follows.
+    return count % 2 === 0 ? { value: '', length: count, closed: false } : null;
   }
+  const emptyReference =
+    count % 2 === 0 ? { value: '', length: count, closed: true } : null;
 
   const parts = [];
   let position = start + count;
@@ -154,7 +160,7 @@ function reading(document, start, quote, count, end) {
   if (emptyReference !== null && !isSubstantiveBody(value)) {
     return emptyReference;
   }
-  return { value, length: end - start };
+  return { value, length: end - start, closed: true };
 }
 
 /** The number of `quote` delimiters in a row from `start` of `text`. */

package/src/references.js

@@ -61,3 +61,21 @@ export function unescapeReference(reference) {
   }
   return reading.value;
 }
+
+/**
+ * The delimiter a reference was written between, read from the reference as it
+ * was written: `'`, `"` or `` ` `` for a quoted reference (however many of them
+ * open it), and null for a bare reference, a versioned literal, or no text.
+ * @param {string|null|undefined} raw - One reference exactly as written
+ * @returns {string|null} The delimiter character, or null
+ * @example
+ * referenceQuote("'1'"); // "'"
+ * referenceQuote('""a b""'); // '"'
+ * referenceQuote('1'); // null
+ * referenceQuote("'unclosed"); // null: a bare reference that starts with '
+ */
+export function referenceQuote(raw) {
+  if (typeof raw !== 'string' || !QUOTES.includes(raw[0])) return null;
+  const reading = readReference(raw, 0);
+  return reading !== null && reading.length === raw.length ? raw[0] : null;
+}

package/tests/Comments.test.js

@@ -158,3 +158,34 @@ test('a hash that cannot open a comment is left unquoted', () => {
   expect(Link.escapeReference('#')).toBe("'#'");
   expect(Link.escapeReference('#ff0000')).toBe("'#ff0000'");
 });
+
+// Issue #377: the comment reader and the grammar must agree on where every
+// delimited reference starts and how far it reaches, or a comment is blanked
+// inside what the grammar then reads as a reference.
+
+test('a hash inside a reference that follows a group is content', () => {
+  expect(parsesAs('(a)" #b"\n')).toBe('((<a>) < #b>)');
+});
+
+test('a hash inside a reference that follows another reference is content', () => {
+  expect(parsesAs('"x"\' #y\'\n')).toBe('(<x> < #y>)');
+  expect(parsesAs('```q```"" #c\n"""\n')).toBe('(<q> < #c\n">)');
+});
+
+test('a hash inside a reference that follows a literal is content', () => {
+  expect(parsesAs('~1{62}" #b"\n')).toBe('(<b> < #b>)');
+});
+
+test('a reference is read in the document as written, comments included', () => {
+  // The body of the first `""` straddles a parenthesis, so it is the empty
+  // reference and the `#` after it opens a comment. Blanking that comment
+  // must not turn the `""` into a reference that holds the blanked text.
+  expect(parsesAs('"" a #)b\n""\n')).toBe('(<> <a>)\n(<>)');
+});
+
+test('the comment reader skips the references the grammar reads', () => {
+  expect(stripComments('(a)" #b"\n')).toBe('(a)" #b"\n');
+  expect(stripComments('"x"\' #y\'\n')).toBe('"x"\' #y\'\n');
+  expect(stripComments('~1{62}" #b"\n')).toBe('~1{62}" #b"\n');
+  expect(stripComments('a"b #c"\n')).toBe('a"b    \n');
+});

package/tests/Cst.test.js

@@ -0,0 +1,109 @@
+import { describe, expect, test } from 'bun:test';
+import { CstKind, CstNode, Parser } from '../src/index.js';
+
+const roundTrip = (input, options) =>
+  new Parser(options).parseCst(input).toString();
+
+describe('concrete syntax tree (issue #364)', () => {
+  test('is the default lossless view of a document', () => {
+    const input = "(t 1 '1')";
+    const cst = new Parser().parseCst(input);
+    expect(cst).toBeInstanceOf(CstNode);
+    expect(cst.kind).toBe(CstKind.DOCUMENT);
+    expect(cst.toString()).toBe(input);
+    expect(cst.toSExpression()).toBe(
+      '(document (line (group open_paren:"(" (line reference:"t" whitespace:" " reference:"1" whitespace:" " reference:"\'1\'") close_paren:")")))'
+    );
+  });
+
+  test('reference tokens carry their value and delimiter', () => {
+    const references = new Parser()
+      .parseCst("(a \"b c\" ~1{64} ''e''''f'')")
+      .tokens()
+      .filter((token) => token.kind === CstKind.REFERENCE);
+    expect(
+      references.map((token) => [token.text, token.value, token.quote])
+    ).toEqual([
+      ['a', 'a', null],
+      ['"b c"', 'b c', '"'],
+      ['~1{64}', 'd', null],
+      ["''e''''f''", "e''f", "'"],
+    ]);
+  });
+
+  test.each([
+    ['empty', ''],
+    ['only whitespace', ' \t '],
+    ['blank lines', '\n\n'],
+    ['CRLF and CR', 'a\r\nb\rc\r\n'],
+    ['tabs and trailing spaces', 'a\t b  \n'],
+    ['comments everywhere', '# head\na: b # tail\n  # inner\n  c\n# end'],
+    ['multiline quotes', '(note: "one\n  two\r\nthree")'],
+    ['nested groups over lines', '(a (b\n  (c d)\n) e)\n'],
+    ['indentation', 'root:\n  child:\n    leaf\n  tail'],
+    ['literals', '(~1{} ~1{e6a0b9})'],
+    ['comment marks inside quotes', '(\'#a\' "b # c" `#`)'],
+    ['unicode', '(根: café 🙂)\n'],
+  ])('recovers %s exactly', (_, input) => {
+    expect(roundTrip(input)).toBe(input);
+  });
+
+  test('without comment support, # is part of references', () => {
+    const input = 'a #b';
+    const cst = new Parser({ comments: false }).parseCst(input);
+    expect(cst.toString()).toBe(input);
+    expect(cst.tokens().map((token) => token.kind)).toEqual([
+      'reference',
+      'whitespace',
+      'reference',
+    ]);
+  });
+
+  test('a group belongs to the line it opens on', () => {
+    const cst = new Parser().parseCst('a (b\nc)\nd');
+    expect(cst.children.map((line) => line.toString())).toEqual([
+      'a (b\nc)\n',
+      'd',
+    ]);
+    const group = cst.children[0].children[2];
+    expect(group.kind).toBe(CstKind.GROUP);
+    expect(group.children.map((child) => child.kind)).toEqual([
+      'open_paren',
+      'line',
+      'line',
+      'close_paren',
+    ]);
+  });
+
+  test('reads versioned literals in linear time', () => {
+    // CodeQL js/polynomial-redos flagged the regular expression that used to
+    // find where a literal ends; the scanner that replaced it is one pass.
+    const input = [
+      `~1{${'61'.repeat(50000)}}`,
+      `~${'1'.repeat(50000)}x`,
+      '~1{61} '.repeat(20000),
+    ].join('\n');
+    const started = Date.now();
+    const cst = new Parser().parseCst(input);
+    expect(Date.now() - started).toBeLessThan(5000);
+    expect(cst.toString()).toBe(input);
+    expect(
+      new Parser()
+        .parseCst('(~1{64} ~{} ~a{b} ~1x)')
+        .tokens()
+        .filter((token) => token.kind === CstKind.REFERENCE)
+        .map((token) => [token.text, token.value])
+    ).toEqual([
+      ['~1{64}', 'd'],
+      ['~{}', '~{}'],
+      ['~a{b}', '~a{b}'],
+      ['~1x', '~1x'],
+    ]);
+  });
+
+  test('rejects what parse() rejects', () => {
+    expect(() => new Parser().parseCst('(a')).toThrow();
+    expect(() => new Parser().parseCst('a)')).toThrow();
+    expect(() => new Parser({ maxDepth: 1 }).parseCst('((a))')).toThrow();
+  });
+});

package/tests/SharedSyntaxParity.test.js

@@ -0,0 +1,47 @@
+import { readFileSync } from 'node:fs';
+import { URL } from 'node:url';
+import { expect, test } from 'bun:test';
+import { Parser, StreamParser } from '../src/index.js';
+
+const read = (name) =>
+  JSON.parse(
+    readFileSync(new URL(`../../parity/${name}`, import.meta.url), 'utf8')
+  );
+const syntaxFixtures = read('syntax-fixtures.json');
+const textFixtures = read('text-fixtures.json');
+const tree = (links) =>
+  links.map((link) => ({
+    id: link.id,
+    raw: link.rawId,
+    quote: link.quote,
+    values: tree(link.values),
+  }));
+
+test('shared syntax parity: raw references and concrete syntax trees', () => {
+  for (const fixture of syntaxFixtures) {
+    const parser = new Parser();
+    expect(tree(parser.parse(fixture.input)), fixture.id).toEqual(
+      fixture.links
+    );
+    const cst = parser.parseCst(fixture.input);
+    expect(cst.toSExpression(), fixture.id).toBe(fixture.cst);
+    expect(cst.toString(), fixture.id).toBe(fixture.input);
+    const stream = new StreamParser();
+    for (const character of fixture.input) stream.write(character);
+    expect(tree(stream.end()), fixture.id).toEqual(fixture.links);
+  }
+});
+
+test('every shared input is recovered exactly from its concrete syntax tree', () => {
+  for (const { id, input } of [...textFixtures, ...syntaxFixtures]) {
+    const cst = new Parser().parseCst(input);
+    expect(cst.toString(), id).toBe(input);
+    expect(
+      cst
+        .tokens()
+        .map((token) => token.text)
+        .join(''),
+      id
+    ).toBe(input);
+  }
+});

package/tests/StreamParser.test.js

@@ -162,4 +162,17 @@ describe('StreamParser', () => {
       buffered: 2,
     });
   });
+
+  test('waits for the line that closes an even run of quotes (issue #377)', () => {
+    // `""` is the empty reference only when nothing closes it, and here the
+    // next line does, so the first line cannot be emitted on its own.
+    const input = 'a "" b\n"" c\n';
+    const stream = new StreamParser();
+    for (const character of input) {
+      expect(stream.write(character)).toEqual([]);
+    }
+    expect(formatLinks(stream.end())).toBe(
+      formatLinks(new Parser().parse(input))
+    );
+  });
 });

package/tests/SyntaxMetadata.test.js

@@ -0,0 +1,102 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  Link,
+  Parser,
+  StreamParser,
+  formatLinks,
+  referenceQuote,
+} from '../src/index.js';
+
+describe('syntax metadata on parsed links (issue #364)', () => {
+  test("1 and '1' are the same reference but keep how they were written", () => {
+    const [link] = new Parser().parse("(t 1 '1')");
+    const [, bare, quoted] = link.values;
+    expect(bare.id).toBe('1');
+    expect(quoted.id).toBe('1');
+    expect(bare.rawId).toBe('1');
+    expect(quoted.rawId).toBe("'1'");
+    expect(bare.quote).toBeNull();
+    expect(quoted.quote).toBe("'");
+    // Equality and formatting ignore the metadata.
+    expect(bare.equals(quoted)).toBe(true);
+    expect(link.format()).toBe('(t 1 1)');
+  });
+
+  test('records every quote kind and run length', () => {
+    const [link] = new Parser().parse('("a": \'b\' `c` ""d"" ~1{65} f)');
+    expect(link.rawId).toBe('"a"');
+    expect(link.quote).toBe('"');
+    expect(link.values.map((value) => [value.rawId, value.quote])).toEqual([
+      ["'b'", "'"],
+      ['`c`', '`'],
+      ['""d""', '"'],
+      ['~1{65}', null],
+      ['f', null],
+    ]);
+  });
+
+  test('keeps raw text from the source, not from comment stripping', () => {
+    const [link] = new Parser().parse("(a 'b # c') # d");
+    expect(link.values[1].rawId).toBe("'b # c'");
+  });
+
+  test('leaves links without written IDs without raw text', () => {
+    const [link] = new Parser().parse('(a b)');
+    expect(link.rawId).toBeNull();
+    expect(link.quote).toBeNull();
+    expect(new Link('x').rawId).toBeNull();
+    expect(new Link('x').quote).toBeNull();
+  });
+
+  test('does not change equality, formatting or serialization', () => {
+    const quoted = new Parser().parse("('a': 'b')");
+    const bare = new Parser().parse('(a: b)');
+    expect(quoted[0].equals(bare[0])).toBe(true);
+    expect(formatLinks(quoted)).toBe(formatLinks(bare));
+    expect(quoted[0].toString()).toBe(bare[0].toString());
+    expect(JSON.stringify(quoted)).toBe(JSON.stringify(bare));
+    expect(quoted).toEqual(bare);
+  });
+
+  test('AST mode (syntax: false) drops the metadata', () => {
+    const [link] = new Parser({ syntax: false }).parse("('a': 'b')");
+    expect(link.rawId).toBeNull();
+    expect(link.values[0].rawId).toBeNull();
+    expect(link.values[0].quote).toBeNull();
+    expect(link.equals(new Parser().parse("('a': 'b')")[0])).toBe(true);
+  });
+
+  test('parseGroups and StreamParser keep the metadata', () => {
+    const groups = new Parser().parseGroups('\'root\':\n  "child"');
+    expect(groups[0].element.rawId).toBe("'root'");
+    expect(groups[0].children[0].element.values[0].rawId).toBe('"child"');
+    const stream = new StreamParser();
+    stream.write("(a '");
+    stream.write("b')");
+    const [link] = stream.end();
+    expect(link.values[1].rawId).toBe("'b'");
+    expect(link.values[1].quote).toBe("'");
+  });
+
+  test('rawId can be set on links built in code and must be text or null', () => {
+    const link = new Link('a');
+    link.rawId = '"a"';
+    expect(link.quote).toBe('"');
+    link.rawId = null;
+    expect(link.quote).toBeNull();
+    expect(() => {
+      link.rawId = 1;
+    }).toThrow(TypeError);
+  });
+
+  test('referenceQuote reads the delimiter of one complete reference', () => {
+    expect(referenceQuote("'1'")).toBe("'");
+    expect(referenceQuote('""a b""')).toBe('"');
+    expect(referenceQuote('`x`')).toBe('`');
+    expect(referenceQuote('1')).toBeNull();
+    expect(referenceQuote('~1{31}')).toBeNull();
+    expect(referenceQuote("'unclosed")).toBeNull();
+    expect(referenceQuote("'a' b")).toBeNull();
+    expect(referenceQuote(null)).toBeNull();
+  });
+});