@outputai/llm 0.11.0 → 0.11.1-next.2223fa5.0

package/src/prompt/markup/attributes.spec.js

@@ -0,0 +1,132 @@
+import { describe, expect, it } from 'vitest';
+import { extractAttributes } from './attributes.js';
+
+describe( 'extractAttributes', () => {
+  it( 'returns no attributes for empty input', () => {
+    expect( extractAttributes( '' ) ).toEqual( {} );
+  } );
+
+  it( 'returns no attributes for whitespace-only input', () => {
+    expect( extractAttributes( ' \n\t ' ) ).toEqual( {} );
+  } );
+
+  it( 'parses bare attributes as true', () => {
+    expect( extractAttributes( 'cached pinned' ) ).toEqual( {
+      cached: true,
+      pinned: true
+    } );
+  } );
+
+  it( 'lowercases attribute names', () => {
+    expect( extractAttributes( 'OPTIONS=cached PINNED' ) ).toEqual( {
+      options: 'cached',
+      pinned: true
+    } );
+  } );
+
+  it( 'preserves unquoted values as strings', () => {
+    expect( extractAttributes( 'enabled=false count=001 ratio=1.25' ) ).toEqual( {
+      enabled: 'false',
+      count: '001',
+      ratio: '1.25'
+    } );
+  } );
+
+  it( 'removes matching double quotes', () => {
+    expect( extractAttributes( 'options="cached shared"' ) ).toEqual( {
+      options: 'cached shared'
+    } );
+  } );
+
+  it( 'removes matching single quotes', () => {
+    expect( extractAttributes( 'options=\'cached shared\'' ) ).toEqual( {
+      options: 'cached shared'
+    } );
+  } );
+
+  it( 'allows the opposite quote type inside quoted values', () => {
+    expect( extractAttributes( 'single="it\'s ready" double=\'say "ready"\'' ) ).toEqual( {
+      single: 'it\'s ready',
+      double: 'say "ready"'
+    } );
+  } );
+
+  it( 'preserves equals and greater-than characters inside quoted values', () => {
+    expect( extractAttributes( 'condition="score >= 10" expression=\'a = b\'' ) ).toEqual( {
+      condition: 'score >= 10',
+      expression: 'a = b'
+    } );
+  } );
+
+  it( 'supports whitespace around equals signs', () => {
+    expect( extractAttributes( 'options = "cached shared" mode = fast pinned' ) ).toEqual( {
+      options: 'cached shared',
+      mode: 'fast',
+      pinned: true
+    } );
+  } );
+
+  it( 'supports tabs and newlines between attributes', () => {
+    expect( extractAttributes( 'options="cached"\nmode=fast\tpinned' ) ).toEqual( {
+      options: 'cached',
+      mode: 'fast',
+      pinned: true
+    } );
+  } );
+
+  it( 'ignores surrounding whitespace', () => {
+    expect( extractAttributes( '  options = "cached" pinned  ' ) ).toEqual( {
+      options: 'cached',
+      pinned: true
+    } );
+  } );
+
+  it( 'parses an explicitly empty value', () => {
+    expect( extractAttributes( 'options=' ) ).toEqual( { options: '' } );
+  } );
+
+  it( 'uses the last value when an attribute is repeated', () => {
+    expect( extractAttributes( 'mode=slow mode=fast' ) ).toEqual( { mode: 'fast' } );
+  } );
+
+  it( 'rejects an unterminated single-quoted value', () => {
+    expect( () => extractAttributes( 'color=\'red' ) )
+      .toThrow( /^Invalid attribute "color": quoted value is missing its closing quote \('\)\.$/ );
+  } );
+
+  it( 'rejects an unterminated double-quoted value', () => {
+    expect( () => extractAttributes( 'color="red' ) )
+      .toThrow( /^Invalid attribute "color": quoted value is missing its closing quote \("\)\.$/ );
+  } );
+
+  it( 'rejects values closed by a different quote type', () => {
+    expect( () => extractAttributes( 'color=\'red"' ) ).toThrow();
+    expect( () => extractAttributes( 'color="red\'' ) ).toThrow();
+  } );
+
+  it( 'rejects stray quotes in unquoted values', () => {
+    expect( () => extractAttributes( 'color=red\'' ) ).toThrow();
+    expect( () => extractAttributes( 'color=red"' ) ).toThrow();
+    expect( () => extractAttributes( 'color=arby\'s' ) ).toThrow();
+    expect( () => extractAttributes( 'color=jack"madman"silva' ) ).toThrow();
+  } );
+
+  it( 'rejects matching quote characters inside quoted values', () => {
+    expect( () => extractAttributes( 'message="say "hello""' ) )
+      .toThrow( /^Invalid attribute "message": quoted value contains an unexpected quote \("\)\.$/ );
+    expect( () => extractAttributes( 'message=\'say \'hello\'\'' ) ).toThrow();
+    expect( () => extractAttributes( 'color="red"dark"' ) ).toThrow();
+    expect( () => extractAttributes( 'color=\'red\'dark\'' ) ).toThrow();
+  } );
+
+  it( 'rejects an empty attribute name', () => {
+    expect( () => extractAttributes( '=value' ) )
+      .toThrow( /^Invalid attribute "=value": name is missing\.$/ );
+  } );
+
+  it( 'rejects invalid valued and bare attribute names', () => {
+    expect( () => extractAttributes( 'bad/name=value' ) )
+      .toThrow( /^Invalid attribute name "bad\/name"\.$/ );
+    expect( () => extractAttributes( 'bad/name' ) ).toThrow();
+  } );
+} );

package/src/prompt/markup/nodes.js

@@ -0,0 +1,71 @@
+import { isOpeningTag, isClosingTag, isEmptyText, isSelfClosingTag, isComment, extractTag } from './tokens.js' ;
+import { tokenize } from './tokenizer.js';
+import { ValidationError } from '@outputai/core';
+
+class Node {
+  constructor( tagName, attributes, content = '' ) {
+    this.tagName = tagName;
+    this.attributes = attributes;
+    this.content = content;
+  }
+}
+
+export const parseMarkup = content => {
+  const tokens = tokenize( content );
+  if ( !tokens ) {
+    return [];
+  }
+
+  const state = {
+    currentNode: null
+  };
+
+  const nodes = [];
+
+  // If the first ever valid text isnt a markup or comment, don't try to parse and return no nodes
+  const firstContentToken = tokens.find( token => !isComment( token ) && !isEmptyText( token ) );
+  if ( firstContentToken && !isOpeningTag( firstContentToken ) && !isClosingTag( firstContentToken ) && !isSelfClosingTag( firstContentToken ) ) {
+    return [];
+  }
+
+  for ( const token of tokens ) {
+    if ( isOpeningTag( token ) ) {
+      const { tagName, attributes } = extractTag( token );
+      if ( !state.currentNode ) {
+        state.currentNode = new Node( tagName, attributes );
+      } else {
+        if ( tagName === state.currentNode.tagName ) {
+          throw new ValidationError(
+            `Invalid child tag: <${tagName}> cannot appear inside another <${tagName}>. Escape it as "&lt;${tagName}&gt;".`
+          );
+        }
+        state.currentNode.content += token;
+      }
+    } else if ( isClosingTag( token ) ) {
+      if ( !state.currentNode ) {
+        throw new ValidationError( `Invalid root tag: closing tag "${token}" has no matching opening tag.` );
+      }
+      if ( extractTag( token ).tagName === state.currentNode.tagName ) {
+        nodes.push( state.currentNode );
+        state.currentNode = null;
+      } else {
+        state.currentNode.content += token;
+      }
+    } else {
+      if ( state.currentNode ) {
+        state.currentNode.content += token;
+      } else {
+        if ( isSelfClosingTag( token ) ) {
+          throw new ValidationError( `Invalid root tag: self-closing tag "${token}" is not allowed.` );
+        } else if ( !isComment( token ) && !isEmptyText( token ) ) {
+          throw new ValidationError( 'Invalid root text: text is not allowed outside message tags.' );
+        }
+      }
+    }
+  }
+
+  if ( state.currentNode ) {
+    throw new ValidationError( `Invalid tag: <${state.currentNode.tagName}> is missing its closing tag.` );
+  }
+  return nodes;
+};

package/src/prompt/markup/nodes.spec.js

@@ -0,0 +1,333 @@
+import { describe, expect, it } from 'vitest';
+import { parseMarkup } from './nodes.js';
+
+const blocks = content =>
+  parseMarkup( content ).map( ( { tagName, content: blockContent } ) => ( {
+    tagName,
+    content: blockContent
+  } ) );
+
+describe( 'parseMarkup', () => {
+  describe( 'top-level blocks', () => {
+    it( 'parses one generic block', () => {
+      expect( blocks( '<user>Hello</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Hello' }
+      ] );
+    } );
+
+    it( 'parses multiple sibling blocks', () => {
+      expect( blocks( '<system>Rules</system>\n\n<user>Question</user>' ) ).toEqual( [
+        { tagName: 'system', content: 'Rules' },
+        { tagName: 'user', content: 'Question' }
+      ] );
+    } );
+
+    it( 'supports generic and hyphenated tag names', () => {
+      expect( blocks( '<custom-block>Value</custom-block>' ) ).toEqual( [
+        { tagName: 'custom-block', content: 'Value' }
+      ] );
+    } );
+
+    it( 'matches tag names case-insensitively', () => {
+      expect( blocks( '<User>Hello</USER>' ) ).toEqual( [
+        { tagName: 'user', content: 'Hello' }
+      ] );
+    } );
+
+    it( 'parses an empty block', () => {
+      expect( blocks( '<user></user>' ) ).toEqual( [
+        { tagName: 'user', content: '' }
+      ] );
+    } );
+
+    it( 'rejects a top-level self-closing block', () => {
+      expect( () => parseMarkup( '<user />' ) )
+        .toThrow( /^Invalid root tag: self-closing tag "<user \/>" is not allowed\.$/ );
+    } );
+
+    it( 'returns no blocks for empty content', () => {
+      expect( parseMarkup( '' ) ).toEqual( [] );
+    } );
+
+    it( 'returns no blocks for plain text', () => {
+      expect( parseMarkup( 'Generate a cinematic poster.' ) ).toEqual( [] );
+    } );
+
+    it( 'returns no blocks for text mixed with whitespace and comments', () => {
+      expect( parseMarkup( '<!-- before -->\nGenerate a poster.\n<!-- after -->' ) ).toEqual( [] );
+    } );
+
+    it( 'returns no blocks when text precedes otherwise valid markup', () => {
+      expect( parseMarkup( 'Note\n<user>Hello</user>\nTail' ) ).toEqual( [] );
+    } );
+
+    it( 'uses the first meaningful token after leading comments', () => {
+      expect( parseMarkup( '<!-- note -->\nExplain <user>Hello</user> literally.' ) ).toEqual( [] );
+    } );
+
+    it( 'does not validate later tag-shaped content after instruction mode starts', () => {
+      expect( parseMarkup( 'Explain </user>, <image />, and <user> examples.' ) ).toEqual( [] );
+    } );
+
+    it( 'returns no blocks for tag-shaped text that is not a valid tag', () => {
+      expect( parseMarkup( 'Use a<b, b>c, and c < d.' ) ).toEqual( [] );
+    } );
+
+    it( 'treats an unterminated comment marker as plain text', () => {
+      expect( parseMarkup( '<!-- unfinished' ) ).toEqual( [] );
+    } );
+
+    it( 'returns no blocks for whitespace and comments only', () => {
+      expect( parseMarkup( '\n<!-- first -->\n<!-- second\nnote -->\n' ) ).toEqual( [] );
+    } );
+
+    it( 'ignores surrounding whitespace and comments', () => {
+      expect( blocks( '<!-- prompt note -->\n\n<user>Hello</user>\n' ) ).toEqual( [
+        { tagName: 'user', content: 'Hello' }
+      ] );
+    } );
+
+    it( 'ignores multiline comments outside blocks', () => {
+      expect( blocks( '<!-- prompt\nnote -->\n<user>Hello</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Hello' }
+      ] );
+    } );
+
+    it( 'ignores comments before, between, and after blocks', () => {
+      expect( blocks(
+        '<!-- before --><system>Rules</system><!-- between --><user>Question</user><!-- after -->'
+      ) ).toEqual( [
+        { tagName: 'system', content: 'Rules' },
+        { tagName: 'user', content: 'Question' }
+      ] );
+    } );
+
+    it( 'supports whitespace inside a closing tag', () => {
+      expect( blocks( '<user>Hello</ user >' ) ).toEqual( [
+        { tagName: 'user', content: 'Hello' }
+      ] );
+    } );
+  } );
+
+  describe( 'opaque block content', () => {
+    it( 'preserves a closed nested tag with a different name', () => {
+      expect( blocks( '<user>Use <title>Example</title> here.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Use <title>Example</title> here.' }
+      ] );
+    } );
+
+    it( 'preserves an open nested tag with a different name', () => {
+      expect( blocks( '<user>Use the <title> tag here.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Use the <title> tag here.' }
+      ] );
+    } );
+
+    it( 'preserves several unclosed nested tags with different names', () => {
+      expect( blocks( '<user><article>Use <title>Example.</user>' ) ).toEqual( [
+        { tagName: 'user', content: '<article>Use <title>Example.' }
+      ] );
+    } );
+
+    it( 'preserves a nested tag matching another supported role', () => {
+      expect( blocks( '<user>Describe the <system> tag.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Describe the <system> tag.' }
+      ] );
+    } );
+
+    it( 'preserves angle-bracket TypeScript syntax', () => {
+      expect( blocks( '<user>Explain Array<string> and Promise<Result>.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Explain Array<string> and Promise<Result>.' }
+      ] );
+    } );
+
+    it( 'preserves unmatched angle brackets in text', () => {
+      expect( blocks( '<user>Use a<b, b>c, c < d and d > e.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Use a<b, b>c, c < d and d > e.' }
+      ] );
+    } );
+
+    it( 'preserves a different unmatched closing tag', () => {
+      expect( blocks( '<user>Show </title> literally.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Show </title> literally.' }
+      ] );
+    } );
+
+    it( 'preserves comments inside a block', () => {
+      expect( blocks( '<user>Before<!-- hidden > note -->After</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Before<!-- hidden > note -->After' }
+      ] );
+    } );
+
+    it( 'preserves multiline comments inside a block', () => {
+      expect( blocks( '<user>Before<!-- hidden\nnote -->After</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Before<!-- hidden\nnote -->After' }
+      ] );
+    } );
+
+    it( 'preserves a closing-tag-shaped value inside a comment', () => {
+      expect( blocks( '<user>Before<!-- </user> -->After</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Before<!-- </user> -->After' }
+      ] );
+    } );
+
+    it( 'preserves empty comments inside a block', () => {
+      expect( blocks( '<user>Before<!---->After</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Before<!---->After' }
+      ] );
+    } );
+
+    it( 'preserves whitespace, line endings, and Unicode content exactly', () => {
+      expect( blocks( '<user>\r\n  Olá 👋\t \r\n</user>' ) ).toEqual( [
+        { tagName: 'user', content: '\r\n  Olá 👋\t \r\n' }
+      ] );
+    } );
+
+    it( 'preserves self-closing tags inside a block', () => {
+      expect( blocks( '<user>Before<image source="avatar.png" />After</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Before<image source="avatar.png" />After' }
+      ] );
+    } );
+
+    it( 'preserves a self-closing tag matching the block name', () => {
+      expect( blocks( '<user>Write <user /> literally.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Write <user /> literally.' }
+      ] );
+    } );
+
+    it( 'allows escaped same-name tags as text', () => {
+      expect( blocks( '<user>Use &lt;user&gt; and &lt;/user&gt;.</user>' ) ).toEqual( [
+        { tagName: 'user', content: 'Use &lt;user&gt; and &lt;/user&gt;.' }
+      ] );
+    } );
+
+  } );
+
+  describe( 'attributes', () => {
+    it( 'parses quoted, unquoted, and boolean attributes', () => {
+      const [ node ] = parseMarkup( '<user options="cached" mode=fast pinned>Hello</user>' );
+
+      expect( node.attributes ).toEqual( {
+        options: 'cached',
+        mode: 'fast',
+        pinned: true
+      } );
+    } );
+
+    it( 'supports whitespace around attribute equals signs', () => {
+      const [ node ] = parseMarkup( '<user options = "cached shared" mode = fast pinned>Hello</user>' );
+
+      expect( node.attributes ).toEqual( {
+        options: 'cached shared',
+        mode: 'fast',
+        pinned: true
+      } );
+    } );
+
+    it( 'lowercases attribute names and preserves values as strings', () => {
+      const [ node ] = parseMarkup( '<user ENABLED=false COUNT=001 RATIO=1.25>Hello</user>' );
+
+      expect( node.attributes ).toEqual( {
+        enabled: 'false',
+        count: '001',
+        ratio: '1.25'
+      } );
+    } );
+
+    it( 'parses empty explicit attribute values', () => {
+      const [ node ] = parseMarkup( '<user first="" second=\'\' third=>Hello</user>' );
+
+      expect( node.attributes ).toEqual( {
+        first: '',
+        second: '',
+        third: ''
+      } );
+    } );
+
+    it( 'allows the opposite quote type inside quoted values', () => {
+      const [ node ] = parseMarkup( '<user single="it\'s ready" double=\'say "ready"\'>Hello</user>' );
+
+      expect( node.attributes ).toEqual( {
+        single: 'it\'s ready',
+        double: 'say "ready"'
+      } );
+    } );
+
+    it( 'allows greater-than characters inside double-quoted attributes', () => {
+      const [ node ] = parseMarkup( '<user condition="score > 10">Hello</user>' );
+
+      expect( node.attributes ).toEqual( { condition: 'score > 10' } );
+      expect( node.content ).toBe( 'Hello' );
+    } );
+
+    it( 'allows greater-than characters inside single-quoted attributes', () => {
+      const [ node ] = parseMarkup( '<user condition=\'score > 10\'>Hello</user>' );
+
+      expect( node.attributes ).toEqual( { condition: 'score > 10' } );
+      expect( node.content ).toBe( 'Hello' );
+    } );
+
+    it( 'rejects on single-quoted attributes missing end quote', () => {
+      expect( () => parseMarkup( '<user color=\'red>Hello</user>' ) ).toThrow();
+    } );
+
+    it( 'rejects on double-quoted attributes missing end quote', () => {
+      expect( () => parseMarkup( '<user color="red>Hello</user>' ) ).toThrow();
+    } );
+
+    it( 'rejects attributes closed by a different quote type', () => {
+      expect( () => parseMarkup( '<user color=\'red">Hello</user>' ) ).toThrow();
+      expect( () => parseMarkup( '<user color="red\'>Hello</user>' ) ).toThrow();
+    } );
+
+    it( 'rejects stray quotes in unquoted attribute values', () => {
+      expect( () => parseMarkup( '<user color=red\'>Hello</user>' ) ).toThrow();
+      expect( () => parseMarkup( '<user color=red">Hello</user>' ) ).toThrow();
+    } );
+  } );
+
+  describe( 'invalid markup', () => {
+    it( 'rejects a closed nested tag with the same name', () => {
+      expect( () => parseMarkup( '<user>Outer <user>Inner</user> tail</user>' ) )
+        .toThrow( /^Invalid child tag: <user> cannot appear inside another <user>\. Escape it as "&lt;user&gt;"\.$/ );
+    } );
+
+    it( 'rejects an open nested tag with the same name', () => {
+      expect( () => parseMarkup( '<user>Use the <user> tag.</user>' ) ).toThrow();
+    } );
+
+    it( 'rejects a same-name nested tag with different casing', () => {
+      expect( () => parseMarkup( '<User>Use the <user> tag.</USER>' ) ).toThrow();
+    } );
+
+    it( 'rejects a missing top-level closing tag', () => {
+      expect( () => parseMarkup( '<user>Hello' ) )
+        .toThrow( /^Invalid tag: <user> is missing its closing tag\.$/ );
+    } );
+
+    it( 'rejects a mismatched top-level closing tag', () => {
+      expect( () => parseMarkup( '<user>Hello</system>' ) ).toThrow();
+    } );
+
+    it( 'rejects an unmatched closing tag outside a block', () => {
+      expect( () => parseMarkup( '</user>' ) )
+        .toThrow( /^Invalid root tag: closing tag "<\/user>" has no matching opening tag\.$/ );
+    } );
+
+    it( 'treats non-whitespace text before a block as instruction mode', () => {
+      expect( parseMarkup( 'Prefix <user>Hello</user>' ) ).toEqual( [] );
+    } );
+
+    it( 'rejects non-whitespace text between blocks', () => {
+      expect( () => parseMarkup( '<system>Rules</system> stray <user>Hello</user>' ) ).toThrow();
+    } );
+
+    it( 'rejects non-whitespace text after a block', () => {
+      expect( () => parseMarkup( '<user>Hello</user> suffix' ) ).toThrow();
+    } );
+
+    it( 'rejects root text immediately after markup mode starts', () => {
+      expect( () => parseMarkup( '<system>Rules</system> stray <user>Hello' ) )
+        .toThrow( /^Invalid root text: text is not allowed outside message tags\.$/ );
+    } );
+  } );
+} );

package/src/prompt/markup/tokenizer.js

@@ -0,0 +1,3 @@
+const tokenizer = /<!--[\s\S]*?-->|<\/\s*[A-Za-z][\w-]*\s*>|<[A-Za-z][\w-]*(?:"[^"]*"|'[^']*'|[^'"<>])*>|<[A-Za-z][\w-]*[^<>]*>|[^<]+|</g;
+
+export const tokenize = value => value.match( tokenizer );

package/src/prompt/markup/tokenizer.spec.js

@@ -0,0 +1,149 @@
+import { describe, expect, it } from 'vitest';
+import { tokenize } from './tokenizer.js';
+
+describe( 'tokenize', () => {
+  it( 'returns null for empty input', () => {
+    expect( tokenize( '' ) ).toBeNull();
+  } );
+
+  it( 'returns plain text as one token', () => {
+    expect( tokenize( 'plain text' ) ).toEqual( [ 'plain text' ] );
+  } );
+
+  it( 'separates opening tags, content, and closing tags', () => {
+    expect( tokenize( '<user>Hello</user>' ) ).toEqual( [
+      '<user>',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'tokenizes multiple sibling blocks and their whitespace', () => {
+    expect( tokenize( '<system>Rules</system>\n\n<user>Question</user>' ) ).toEqual( [
+      '<system>',
+      'Rules',
+      '</system>',
+      '\n\n',
+      '<user>',
+      'Question',
+      '</user>'
+    ] );
+  } );
+
+  it( 'keeps opening tags with attributes intact', () => {
+    expect( tokenize( '<user options="cached shared" mode=fast>Hello</user>' ) ).toEqual( [
+      '<user options="cached shared" mode=fast>',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'keeps greater-than characters inside quoted attributes', () => {
+    expect( tokenize( '<user condition="score > 10">Hello</user>' ) ).toEqual( [
+      '<user condition="score > 10">',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'keeps self-closing tags intact', () => {
+    expect( tokenize( '<user>Before<image source="avatar.png" />After</user>' ) ).toEqual( [
+      '<user>',
+      'Before',
+      '<image source="avatar.png" />',
+      'After',
+      '</user>'
+    ] );
+  } );
+
+  it( 'supports whitespace in closing tags', () => {
+    expect( tokenize( '<user>Hello</ user >' ) ).toEqual( [
+      '<user>',
+      'Hello',
+      '</ user >'
+    ] );
+  } );
+
+  it( 'keeps single-line comments intact', () => {
+    expect( tokenize( '<!-- note --><user>Hello</user>' ) ).toEqual( [
+      '<!-- note -->',
+      '<user>',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'keeps multiline comments intact', () => {
+    expect( tokenize( '<!-- first\nsecond --><user>Hello</user>' ) ).toEqual( [
+      '<!-- first\nsecond -->',
+      '<user>',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'tokenizes adjacent comments separately', () => {
+    expect( tokenize( '<!-- first --><!-- second -->' ) ).toEqual( [
+      '<!-- first -->',
+      '<!-- second -->'
+    ] );
+  } );
+
+  it( 'preserves a lone less-than character', () => {
+    expect( tokenize( '<' ) ).toEqual( [ '<' ] );
+  } );
+
+  it( 'preserves an unterminated comment losslessly', () => {
+    const input = '<!-- unfinished';
+    const tokens = tokenize( input );
+
+    expect( tokens ).toEqual( [ '<', '!-- unfinished' ] );
+    expect( tokens.join( '' ) ).toBe( input );
+  } );
+
+  it( 'keeps a tag with an unterminated attribute quote intact for later validation', () => {
+    const input = '<user color=\'red>Hello</user>';
+
+    expect( tokenize( input ) ).toEqual( [
+      '<user color=\'red>',
+      'Hello',
+      '</user>'
+    ] );
+  } );
+
+  it( 'preserves TypeScript angle-bracket syntax losslessly', () => {
+    const input = 'Explain Array<string> and Promise<Result>.';
+
+    expect( tokenize( input ).join( '' ) ).toBe( input );
+  } );
+
+  it( 'preserves comparison operators and tag-shaped text losslessly', () => {
+    const input = 'Use 1<2, a<b, b>c, c < d, and d > e.';
+
+    expect( tokenize( input ).join( '' ) ).toBe( input );
+  } );
+
+  it( 'preserves every character in mixed markup', () => {
+    const input = '<!-- note -->\n<user condition="a > b">Use x<y and <title>.</user>';
+
+    expect( tokenize( input ).join( '' ) ).toBe( input );
+  } );
+
+  it.each( [
+    '',
+    '<',
+    '<<<user>',
+    '<!-- first --><!-- second -->',
+    '<!-- unfinished',
+    '<user>Hello</user>',
+    '<user />',
+    '<user condition="score > 10">Hello</user>',
+    '<user color=\'red>Hello</user>',
+    '<user color=\'red">Hello</user>',
+    '<b,b>',
+    '<user>Use 1<2, a<b, and b>c.</user>',
+    '<user>\r\nUnicode: Olá 👋\r\n</user>'
+  ] )( 'is lossless for %s', input => {
+    expect( ( tokenize( input ) ?? [] ).join( '' ) ).toBe( input );
+  } );
+} );