@lithent/lithent-template-parser 0.1.0 → 0.1.1

package/src/parser/lexer.ts

@@ -1,3 +1,4 @@
+import { skipJsExpression } from '../docPipe/scan';
 import {
   Token,
   TokenType,
@@ -139,16 +140,23 @@ export class Lexer {
     // Consume <!--
     value += this.advance() + this.advance() + this.advance() + this.advance();
 
+    let closed = false;
     // Read until -->
     while (!this.isEOF()) {
       if (this.peek() === '-' && this.peek(1) === '-' && this.peek(2) === '>') {
         value += this.advance() + this.advance() + this.advance();
+        closed = true;
         break;
       }
       value += this.advance();
     }
 
     const end = this.getCurrentPosition();
+    if (!closed)
+      throw Object.assign(new Error('Unclosed comment'), {
+        line: start.line,
+        column: start.column,
+      });
     this.tokens.push(createToken(TokenType.COMMENT, value, start, end));
   }
 
@@ -324,6 +332,7 @@ export class Lexer {
     const quote = this.advance(); // " or '
     let value = quote;
     let escaped = false;
+    let closed = false;
 
     while (!this.isEOF()) {
       const char = this.peek();
@@ -342,6 +351,7 @@ export class Lexer {
 
       if (char === quote) {
         value += this.advance();
+        closed = true;
         break;
       }
 
@@ -349,6 +359,11 @@ export class Lexer {
     }
 
     const end = this.getCurrentPosition();
+    if (!closed)
+      throw Object.assign(new Error('Unclosed attribute string'), {
+        line: start.line,
+        column: start.column,
+      });
     this.tokens.push(createToken(TokenType.STRING_LITERAL, value, start, end));
   }
 
@@ -386,100 +401,35 @@ export class Lexer {
       createToken(TokenType.EXPRESSION_START, '{', startPos, openEnd)
     );
 
-    let value = '';
-    let depth = 1;
-    let inString: string | null = null;
-    let escaped = false;
-    let contentStart: Position | null = null;
-    let contentEnd: Position = openEnd;
-
-    const consumeChar = (): string => {
-      const charStart = this.getCurrentPosition();
-      const char = this.advance();
-      const charEnd = this.getCurrentPosition();
-
-      value += char;
-      if (!contentStart) {
-        contentStart = charStart;
-      }
-      contentEnd = charEnd;
-
-      return char;
-    };
-
-    while (!this.isEOF() && depth > 0) {
-      const char = this.peek();
-
-      // Handle string literals inside expressions
-      if (inString) {
-        if (escaped) {
-          consumeChar();
-          escaped = false;
-          continue;
-        }
-        if (char === '\\') {
-          escaped = true;
-          consumeChar();
-          continue;
-        }
-        if (char === inString) {
-          consumeChar();
-          inString = null;
-          continue;
-        }
-        consumeChar();
-        continue;
-      }
-
-      // Not in string
-      if (char === '"' || char === "'" || char === '`') {
-        inString = char;
-        consumeChar();
-        continue;
-      }
-
-      if (char === '{') {
-        depth++;
-        consumeChar();
-        continue;
-      }
-
-      if (char === '}') {
-        depth--;
-        if (depth === 0) {
-          // End of expression
-          if (value.trim()) {
-            const start = contentStart ?? openEnd;
-            const end = contentEnd;
-            this.tokens.push(
-              createToken(TokenType.EXPRESSION_CONTENT, value, start, end)
-            );
-          }
-
-          const closeStart = this.getCurrentPosition();
-          this.advance(); // consume }
-          const closeEnd = this.getCurrentPosition();
-          this.tokens.push(
-            createToken(TokenType.EXPRESSION_END, '}', closeStart, closeEnd)
-          );
-          return;
-        }
-        consumeChar();
-        continue;
-      }
-
-      consumeChar();
+    const contentStart = this.getCurrentPosition();
+    const end = skipJsExpression(this.input, this.position);
+    if (end === null) {
+      throw Object.assign(new Error('Unclosed expression'), {
+        line: startPos.line,
+        column: startPos.column,
+      });
     }
-
-    // If we get here, expression wasn't closed properly
-    // Still add the content we have
-    if (value.trim()) {
-      const start = contentStart ?? openEnd;
-      const end = contentStart ? contentEnd : this.getCurrentPosition();
+    const value = this.input.slice(this.position, end - 1);
+    while (this.position < end - 1) this.advance();
+    const contentEnd = this.getCurrentPosition();
+    if (value.trim())
       this.tokens.push(
-        createToken(TokenType.EXPRESSION_CONTENT, value, start, end)
+        createToken(
+          TokenType.EXPRESSION_CONTENT,
+          value,
+          contentStart,
+          contentEnd
+        )
       );
-    }
+    this.advance();
+    this.tokens.push(
+      createToken(
+        TokenType.EXPRESSION_END,
+        '}',
+        contentEnd,
+        this.getCurrentPosition()
+      )
+    );
   }
 
   /**

package/src/parser/parser.ts

@@ -25,9 +25,11 @@ export class Parser {
   private tokens: Token[];
   private position: number = 0;
 
-  constructor(tokens: Token[]) {
+  constructor(tokens: Token[], preserveWhitespace = false) {
     // Filter out whitespace tokens for easier parsing
-    this.tokens = tokens.filter(t => t.type !== TokenType.WHITESPACE);
+    this.tokens = preserveWhitespace
+      ? tokens
+      : tokens.filter(t => t.type !== TokenType.WHITESPACE);
   }
 
   /**
@@ -67,18 +69,16 @@ export class Parser {
       case TokenType.FRAGMENT_OPEN:
         return this.parseFragment();
       case TokenType.TEXT:
+      case TokenType.WHITESPACE:
         return this.parseText();
       case TokenType.EXPRESSION_START:
         return this.parseInterpolation();
       case TokenType.COMMENT:
         return this.parseComment();
       case TokenType.TAG_CLOSE_START:
-        // Unexpected closing tag - skip it
-        this.advance();
-        return null;
+        throw this.error('Unexpected closing tag', token);
       case TokenType.FRAGMENT_CLOSE:
-        this.advance();
-        return null;
+        throw this.error('Unexpected fragment closing tag', token);
       case TokenType.EOF:
         return null;
       default:
@@ -108,9 +108,10 @@ export class Parser {
       return createFragmentNode(children, start, closeToken.end);
     }
 
-    throw new Error(
-      `Unclosed fragment starting at line ${start.line}, column ${start.column}`
-    );
+    throw Object.assign(new Error('Unclosed fragment'), {
+      line: start.line,
+      column: start.column,
+    });
   }
 
   /**
@@ -204,17 +205,7 @@ export class Parser {
       );
     }
 
-    // No closing tag found
-    const end = children.length > 0 ? children[children.length - 1].end : start;
-    return createElementNode(
-      tag,
-      attributes,
-      directives,
-      children,
-      false,
-      start,
-      end
-    );
+    throw this.error(`Unclosed tag <${tag}>`, tagToken);
   }
 
   /**
@@ -298,17 +289,14 @@ export class Parser {
         isDynamic = true;
         this.advance();
       } else {
-        value = { expression: '' };
-        isDynamic = true;
-      }
-
-      if (this.check(TokenType.EXPRESSION_END)) {
-        end = this.peek().end;
-        this.advance(); // consume }
+        throw this.error(
+          'Expected a non-empty attribute expression',
+          exprToken
+        );
       }
+      end = this.expect(TokenType.EXPRESSION_END).end;
     } else {
-      // Unexpected token
-      return createAttributeNode(name, null, false, start, nameToken.end);
+      throw this.error('Expected a quoted attribute or expression', valueToken);
     }
 
     return createAttributeNode(name, value, isDynamic, start, end);
@@ -335,6 +323,8 @@ export class Parser {
           this.advance();
         }
 
+        if (!condition)
+          throw this.error('Expected a non-empty condition', exprToken);
         const end = this.peek().end;
         this.expect(TokenType.EXPRESSION_END);
 
@@ -364,7 +354,10 @@ export class Parser {
         this.expect(TokenType.EXPRESSION_END);
 
         // Parse for expression: "item in list" or "(item, index) in list"
-        const { item, index, list } = this.parseForExpression(expression);
+        const { item, index, list } = this.parseForExpression(
+          expression,
+          exprToken
+        );
 
         return createDirectiveForNode(item, list, index, start, end);
       }
@@ -377,7 +370,10 @@ export class Parser {
   /**
    * Parse l-for expression
    */
-  private parseForExpression(expression: string): {
+  private parseForExpression(
+    expression: string,
+    token: Token
+  ): {
     item: string;
     index?: string;
     list: string;
@@ -408,11 +404,10 @@ export class Parser {
       };
     }
 
-    // Fallback
-    return {
-      item: 'item',
-      list: expression,
-    };
+    throw this.error(
+      'Invalid l-for: expected item in list or (item, index) in list',
+      token
+    );
   }
 
   /**
@@ -439,6 +434,8 @@ export class Parser {
       this.advance();
     }
 
+    if (!expression)
+      throw this.error('Expected a non-empty interpolation', exprToken);
     const end = this.peek().end;
     this.expect(TokenType.EXPRESSION_END);
 
@@ -455,6 +452,15 @@ export class Parser {
     return createCommentNode(content, token.start, token.end);
   }
 
+  private error(message: string, token: Token): Error {
+    return Object.assign(
+      new Error(
+        `${message} at line ${token.start.line}, column ${token.start.column}`
+      ),
+      { line: token.start.line, column: token.start.column }
+    );
+  }
+
   /**
    * Peek at current token without consuming
    */
@@ -508,7 +514,7 @@ export class Parser {
 /**
  * Parse template string into AST
  */
-export function parse(tokens: Token[]): RootNode {
-  const parser = new Parser(tokens);
+export function parse(tokens: Token[], preserveWhitespace = false): RootNode {
+  const parser = new Parser(tokens, preserveWhitespace);
   return parser.parse();
 }

package/src/tests/compiler.test.ts

@@ -351,7 +351,7 @@ describe('Compiler', () => {
       </ul>
     </article>
   </main>
-  <footer l-else-if={items.length === 0}>
+  <footer l-if={items.length === 0}>
     <p>No items</p>
   </footer>
 </section>
@@ -383,7 +383,7 @@ describe('Compiler', () => {
       </>
     </section>
   </div>
-  <footer l-else>
+  <footer l-if={!sidebar}>
     <p>Sidebar disabled</p>
   </footer>
 </>

package/src/tests/parser.test.ts

@@ -622,7 +622,7 @@ describe('Parser', () => {
       </ul>
     </article>
   </main>
-  <footer l-else-if={items.length === 0}>
+  <footer l-if={items.length === 0}>
     <p>No items</p>
   </footer>
 </section>
@@ -654,7 +654,7 @@ describe('Parser', () => {
       </>
     </section>
   </div>
-  <footer l-else>
+  <footer l-if={!sidebar}>
     <p>Sidebar disabled</p>
   </footer>
 </>

package/src/tests/regressions.test.ts

@@ -0,0 +1,200 @@
+import { describe, expect, it } from 'vitest';
+import { h, Fragment } from 'lithent';
+import { renderToString } from '../../../../ssr/src/renderToString';
+import { compile, type CompileOptions } from '../compiler';
+import { transformDocument } from '../docPipe';
+
+function evaluate(
+  template: string,
+  scope: Record<string, unknown> = {},
+  options?: CompileOptions
+) {
+  const result = compile(template, options);
+  expect(result.errors).toEqual([]);
+  return new Function(
+    'h',
+    'Fragment',
+    ...Object.keys(scope),
+    `return (${result.code});`
+  )(h, Fragment, ...Object.values(scope));
+}
+
+function render(
+  template: string,
+  scope: Record<string, unknown> = {},
+  options?: CompileOptions
+) {
+  return renderToString(evaluate(template, scope, options));
+}
+
+describe('executable template regressions', () => {
+  it('renders hyphenated and namespaced attributes', () => {
+    expect(render('<div data-id="x" aria-label="hello" />')).toBe(
+      '<div data-id="x" aria-label="hello"></div>'
+    );
+    expect(() => evaluate('<svg xml:lang="en" />')).not.toThrow();
+  });
+
+  it('preserves backslashes and line breaks in static properties', () => {
+    expect(render(String.raw`<div title="C:\temp" />`)).toContain(
+      String.raw`title="C:\temp"`
+    );
+    expect(render('<div title="hello\nworld" />')).toContain(
+      'title="hello\nworld"'
+    );
+  });
+
+  it('loops before evaluating conditions that use the loop variable', () => {
+    expect(
+      render(
+        '<ul><li l-for={item in items} l-if={item.visible}>{item.label}</li></ul>',
+        {
+          items: [
+            { visible: true, label: 'yes' },
+            { visible: false, label: 'no' },
+          ],
+        }
+      )
+    ).toBe('<ul><li>yes</li></ul>');
+  });
+
+  it('preserves loops on each branch of a conditional chain', () => {
+    const template =
+      '<ul><li l-if={ok}>primary</li><li l-else l-for={other in fallback}>{other}</li></ul>';
+    expect(
+      render(template, { ok: true, items: ['a', 'b'], fallback: ['c'] })
+    ).toBe('<ul><li>primary</li></ul>');
+    expect(
+      render(template, { ok: false, items: ['a', 'b'], fallback: ['c'] })
+    ).toBe('<ul><li>c</li></ul>');
+  });
+
+  it('allows comments and whitespace between conditional branches', () => {
+    const template =
+      '<main><div l-if={ok}>yes</div>\n<!-- comment --><div l-else>no</div></main>';
+    expect(render(template, { ok: true })).toBe('<main><div>yes</div></main>');
+    expect(render(template, { ok: false })).toBe('<main><div>no</div></main>');
+  });
+
+  it('preserves meaningful spaces around interpolations and inline elements', () => {
+    expect(render('<p>Hello {name}, welcome!</p>', { name: 'Ada' })).toBe(
+      '<p>Hello Ada, welcome!</p>'
+    );
+    expect(render('<p><b>Hello</b> <i>world</i></p>')).toBe(
+      '<p><b>Hello</b> <i>world</i></p>'
+    );
+    expect(
+      render('<p> Hello</p>', {}, { transform: { optimize: false } })
+    ).toBe('<p> Hello</p>');
+  });
+
+  it('preserves preformatted content and explicitly disabled whitespace trimming', () => {
+    expect(render('<pre>\n  Hello {name}\n</pre>', { name: 'Ada' })).toBe(
+      '<pre>\n  Hello Ada\n</pre>'
+    );
+    expect(
+      render(
+        '<p>  Hello  </p>',
+        {},
+        {
+          transform: {
+            optimizeOptions: { trimText: false, removeWhitespace: false },
+          },
+        }
+      )
+    ).toBe('<p>  Hello  </p>');
+  });
+
+  it('preserves comma-expression values in children and properties', () => {
+    expect(render('<div>{a, b}</div>', { a: 'first', b: 'second' })).toBe(
+      '<div>second</div>'
+    );
+    expect(render('<div title={a, b} />', { a: 'first', b: 'second' })).toBe(
+      '<div title="second"></div>'
+    );
+  });
+
+  it.each([
+    ['1 /* } */ + 2', '3'],
+    ['1 // }\n + 2', '3'],
+    ['String(/}/.test(value))', 'true'],
+    ['String(/[/{}]/.test(value))', 'true'],
+    ['8 / 2 / 2', '2'],
+    ['`hello ${`${value}`}`', 'hello }'],
+  ])('handles JavaScript lexical contexts in %s', (expression, expected) => {
+    expect(render(`<p>{${expression}}</p>`, { value: '}' })).toBe(
+      `<p>${expected}</p>`
+    );
+  });
+
+  it('compiles embedded templates after return', () => {
+    expect(
+      render(
+        '<div>{items.map(item => { return <span>{item}</span>; })}</div>',
+        { items: ['a', 'b'] }
+      )
+    ).toBe('<div><span>a</span><span>b</span></div>');
+  });
+
+  it.each([
+    '<div>hello',
+    '</div>',
+    '<!-- unclosed',
+    '<div title="unclosed',
+    '<div l-else>fallback</div>',
+    '<div l-else-if={ok} />',
+    '<li l-for={item of items}>{item}</li>',
+    '<div value={} />',
+    '<div l-if={} />',
+    '<div value={foo />',
+    '<div l-if={ok} /><span /><div l-else />',
+    '<><div /></div>',
+    '<>',
+  ])('reports positioned errors for malformed input %s', template => {
+    const result = compile(template);
+    expect(result.code).toBe('');
+    expect(result.errors.length).toBeGreaterThan(0);
+    expect(result.errors[0].line).toBeGreaterThan(0);
+    expect(result.errors[0].column).toBeGreaterThan(0);
+  });
+});
+
+describe('document scanning regressions', () => {
+  it('transforms bare return templates', () => {
+    const result = transformDocument('function View() { return <div />; }');
+    expect(result.errors).toEqual([]);
+    expect(result.transformed).toBe(true);
+    expect(new Function('h', `${result.code}; return View();`)(h).tag).toBe(
+      'div'
+    );
+  });
+
+  it('does not interpret comment-looking template text as JavaScript', () => {
+    const result = transformDocument(
+      'const View=()=> <p>https://example.com /* hello */</p>;'
+    );
+    expect(result.transformed).toBe(true);
+    expect(
+      renderToString(new Function('h', `${result.code}; return View();`)(h))
+    ).toBe('<p>https://example.com /* hello */</p>');
+  });
+
+  it('preserves regexes, strings, comments, comparisons and TypeScript generics', () => {
+    const source = String.raw`const re = /<img \/>/; const text = '<div />'; // <a />
+const less = a < b; const identity = <T>(value: T) => value;`;
+    expect(transformDocument(source).code).toBe(source);
+  });
+
+  it('preserves regex expression statements after control parentheses and blocks', () => {
+    const source = String.raw`if (ok) /<img \/>/.test(html); if (ok) {} /<div \/>/.test(html);`;
+    expect(transformDocument(source).code).toBe(source);
+  });
+
+  it('transforms templates after division and preserves regexes elsewhere', () => {
+    const source = String.raw`const re = /<img \/>/; const n = 8 / 2; const View = () => <p>{n}</p>;`;
+    const result = transformDocument(source);
+    expect(result.transformed).toBe(true);
+    const view = new Function('h', `${result.code}; return View();`)(h);
+    expect(renderToString(view)).toBe('<p>4</p>');
+  });
+});

package/src/transform/directives.ts

@@ -46,6 +46,18 @@ function transformNodeList(nodes: TemplateNode[]): TemplateNode[] {
         children: transformNodeList(node.children),
       };
 
+      const orphan = node.directives.find(
+        d =>
+          d.type === NodeType.DIRECTIVE_ELSE_IF ||
+          d.type === NodeType.DIRECTIVE_ELSE
+      );
+      if (orphan) {
+        throw Object.assign(
+          new Error('l-else-if/l-else must follow l-if or l-else-if'),
+          { line: orphan.start.line, column: orphan.start.column }
+        );
+      }
+
       // Check if this starts a conditional chain
       const ifDirective = node.directives.find(
         d => d.type === NodeType.DIRECTIVE_IF
@@ -54,15 +66,24 @@ function transformNodeList(nodes: TemplateNode[]): TemplateNode[] {
       if (ifDirective) {
         // This starts a conditional chain, look ahead for else-if/else
         const conditionalGroup: ConditionalGroup = {
-          if: transformedElement,
+          if: { ...transformedElement },
           elseIfs: [],
         };
 
         // Look ahead for else-if and else
         let j = i + 1;
+        let consumed = i;
         while (j < nodes.length) {
           const nextNode = nodes[j];
 
+          if (
+            nextNode.type === NodeType.COMMENT ||
+            (nextNode.type === NodeType.TEXT && !nextNode.content.trim())
+          ) {
+            j++;
+            continue;
+          }
+
           if (nextNode.type === NodeType.ELEMENT) {
             const elseIfDirective = nextNode.directives.find(
               d => d.type === NodeType.DIRECTIVE_ELSE_IF
@@ -77,6 +98,7 @@ function transformNodeList(nodes: TemplateNode[]): TemplateNode[] {
                 ...nextNode,
                 children: transformNodeList(nextNode.children),
               });
+              consumed = j;
               j++;
               continue;
             }
@@ -86,6 +108,7 @@ function transformNodeList(nodes: TemplateNode[]): TemplateNode[] {
                 ...nextNode,
                 children: transformNodeList(nextNode.children),
               };
+              consumed = j;
               j++;
               break;
             }
@@ -100,7 +123,7 @@ function transformNodeList(nodes: TemplateNode[]): TemplateNode[] {
         (transformedElement as any).__conditionalGroup = conditionalGroup;
 
         // Skip the else-if/else nodes we've processed
-        i = j - 1;
+        i = consumed;
       }
 
       result.push(transformedElement);