@miliastry/quasar 1.0.0

package/src/Markdown/MarkdownLexer.ts

@@ -0,0 +1,203 @@
+/**
+ * DocumentEngine — MarkdownLexer
+ *
+ * Tokenizes standard Markdown into a flat token stream.
+ * Produces explicit newline tokens for the parser to make semantic
+ * decisions about paragraph breaks.
+ */
+
+export interface MarkdownHashToken { kind: 'hash'; value: string; start: number; end: number }
+export interface MarkdownGtToken { kind: 'gt'; start: number; end: number }
+export interface MarkdownStarToken { kind: 'star'; value: string; start: number; end: number }
+export interface MarkdownUnderscoreToken { kind: 'underscore'; value: string; start: number; end: number }
+export interface MarkdownTildeToken { kind: 'tilde'; value: string; start: number; end: number }
+export interface MarkdownBacktickToken { kind: 'backtick'; value: string; start: number; end: number }
+export interface MarkdownBracketOpenToken { kind: 'bracket_open'; start: number; end: number }
+export interface MarkdownBracketCloseToken { kind: 'bracket_close'; start: number; end: number }
+export interface MarkdownParenOpenToken { kind: 'paren_open'; start: number; end: number }
+export interface MarkdownParenCloseToken { kind: 'paren_close'; start: number; end: number }
+export interface MarkdownBangToken { kind: 'bang'; start: number; end: number }
+export interface MarkdownDashToken { kind: 'dash'; value: string; start: number; end: number }
+export interface MarkdownPlusToken { kind: 'plus'; start: number; end: number }
+export interface MarkdownDotToken { kind: 'dot'; start: number; end: number }
+export interface MarkdownTextToken { kind: 'text'; value: string; start: number; end: number }
+export interface MarkdownNewlineToken { kind: 'newline'; value: string; start: number; end: number }
+export interface MarkdownFenceToken { kind: 'fence'; value: string; lang: string; start: number; end: number }
+export interface MarkdownHrToken { kind: 'hr'; start: number; end: number }
+export interface MarkdownEscapeToken { kind: 'escape'; value: string; start: number; end: number }
+export interface MarkdownAngleToken { kind: 'angle'; value: string; start: number; end: number }
+
+export type MarkdownToken =
+  | MarkdownHashToken | MarkdownGtToken | MarkdownStarToken
+  | MarkdownUnderscoreToken | MarkdownTildeToken | MarkdownBacktickToken
+  | MarkdownBracketOpenToken | MarkdownBracketCloseToken
+  | MarkdownParenOpenToken | MarkdownParenCloseToken
+  | MarkdownBangToken | MarkdownDashToken | MarkdownPlusToken
+  | MarkdownDotToken | MarkdownTextToken | MarkdownNewlineToken
+  | MarkdownFenceToken | MarkdownHrToken | MarkdownEscapeToken
+  | MarkdownAngleToken;
+
+function pushToken(tokens: MarkdownToken[], kind: MarkdownToken['kind'], start: number, end: number, extra?: Record<string, string>) {
+  switch (kind) {
+    case 'hash': tokens.push({ kind, value: extra?.value ?? '', start, end }); break;
+    case 'gt': case 'bracket_open': case 'bracket_close':
+    case 'paren_open': case 'paren_close': case 'bang':
+    case 'plus': case 'dot': case 'hr':
+      tokens.push({ kind, start, end } as MarkdownToken); break;
+    case 'star': case 'underscore': case 'tilde':
+    case 'backtick': case 'dash': case 'text':
+    case 'newline': case 'escape': case 'angle':
+      tokens.push({ kind, value: extra?.value ?? '', start, end } as MarkdownToken); break;
+    case 'fence':
+      tokens.push({ kind, value: extra?.value ?? '', lang: extra?.lang ?? '', start, end }); break;
+  }
+}
+
+function isMarkdownSpecial(ch: string): boolean {
+  return '\\`*_{}[]()#+-.!|~\'"<>'.includes(ch) || ch === '\n' || ch === '\r';
+}
+
+/**
+ * Scan Markdown source text into tokens.
+ * Pure function: input → output, no state.
+ */
+export function scanMarkdown(source: string): MarkdownToken[] {
+  const tokens: MarkdownToken[] = [];
+  let i = 0;
+
+  while (i < source.length) {
+    const ch = source[i];
+    const start = i;
+
+    // Newline
+    if (ch === '\n') {
+      i++;
+      pushToken(tokens, 'newline', start, i, { value: '\n' });
+      continue;
+    }
+    if (ch === '\r') {
+      if (i + 1 < source.length && source[i + 1] === '\n') i++;
+      i++;
+      pushToken(tokens, 'newline', start, i, { value: '\r\n' });
+      continue;
+    }
+
+    // Escape character
+    if (ch === '\\' && i + 1 < source.length) {
+      i += 2;
+      pushToken(tokens, 'escape', start, i, { value: source.slice(start, i) });
+      continue;
+    }
+
+    // Hash (heading)
+    if (ch === '#') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '#' && count < 6) { count++; i++; }
+      pushToken(tokens, 'hash', start, i, { value: source.slice(start, i) });
+      continue;
+    }
+
+    // GT (blockquote)
+    if (ch === '>') {
+      i++;
+      pushToken(tokens, 'gt', start, i);
+      continue;
+    }
+
+    // Bang (image)
+    if (ch === '!' && i + 1 < source.length && source[i + 1] === '[') {
+      i++;
+      pushToken(tokens, 'bang', start, i);
+      continue;
+    }
+
+    // Brackets and parens
+    if (ch === '[') { i++; pushToken(tokens, 'bracket_open', start, i); continue; }
+    if (ch === ']') { i++; pushToken(tokens, 'bracket_close', start, i); continue; }
+    if (ch === '(') { i++; pushToken(tokens, 'paren_open', start, i); continue; }
+    if (ch === ')') { i++; pushToken(tokens, 'paren_close', start, i); continue; }
+
+    // Star (*, **, ***)
+    if (ch === '*') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '*' && count < 3) { count++; i++; }
+      pushToken(tokens, 'star', start, i, { value: source.slice(start, i) });
+      continue;
+    }
+
+    // Underscore (_, __, ___)
+    if (ch === '_') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '_' && count < 3) { count++; i++; }
+      pushToken(tokens, 'underscore', start, i, { value: source.slice(start, i) });
+      continue;
+    }
+
+    // Tilde (~, ~~)
+    if (ch === '~') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '~' && count < 2) { count++; i++; }
+      pushToken(tokens, 'tilde', start, i, { value: source.slice(start, i) });
+      continue;
+    }
+
+    // Backtick (inline or fence)
+    if (ch === '`') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '`' && count < 3) { count++; i++; }
+      if (count === 3) {
+        const langStart = i;
+        while (i < source.length && source[i] !== '\n' && source[i] !== '\r') i++;
+        pushToken(tokens, 'fence', start, i, { value: '```', lang: source.slice(langStart, i).trim() });
+      } else {
+        pushToken(tokens, 'backtick', start, i, { value: source.slice(start, i) });
+      }
+      continue;
+    }
+
+    // Dash (-, --, ---)
+    if (ch === '-') {
+      let count = 1;
+      i++;
+      while (i < source.length && source[i] === '-' && count < 3) { count++; i++; }
+      // Hr if --- and alone on line (simplified, normally requires parser validation)
+      if (count === 3) {
+        pushToken(tokens, 'dash', start, i, { value: source.slice(start, i) });
+      } else {
+        pushToken(tokens, 'dash', start, i, { value: source.slice(start, i) });
+      }
+      continue;
+    }
+
+    // Plus
+    if (ch === '+') { i++; pushToken(tokens, 'plus', start, i); continue; }
+
+    // Dot
+    if (ch === '.') { i++; pushToken(tokens, 'dot', start, i); continue; }
+    
+    // Angle
+    if (ch === '<' || ch === '>') {
+      i++;
+      pushToken(tokens, 'angle', start, i, { value: ch });
+      continue;
+    }
+
+    // Default: accumulate text
+    const textStart = i;
+    while (i < source.length && !isMarkdownSpecial(source[i])) i++;
+    if (i > textStart) {
+      pushToken(tokens, 'text', textStart, i, { value: source.slice(textStart, i) });
+    } else {
+      // If it's a special char not handled above, treat as text to avoid infinite loop
+      pushToken(tokens, 'text', textStart, i + 1, { value: source[i] });
+      i++; 
+    }
+  }
+
+  return tokens;
+}

package/src/Markdown/MarkdownParser.ts

@@ -0,0 +1,455 @@
+import { MarkdownToken, MarkdownFenceToken } from './MarkdownLexer';
+import { MarkdownNode, MarkdownDocument, MarkdownParagraph, MarkdownText, MarkdownHeading } from './MarkdownAST';
+
+/**
+ * DocumentEngine — MarkdownParser
+ * 
+ * Basic implementation to parse a token stream into an AST.
+ */
+export class MarkdownParser {
+  private tokens: MarkdownToken[];
+  private current: number = 0;
+
+  constructor(tokens: MarkdownToken[]) {
+    this.tokens = tokens;
+  }
+
+  public parse(): MarkdownDocument {
+    const doc: MarkdownDocument = {
+      type: 'document',
+      children: []
+    };
+
+    while (!this.isAtEnd()) {
+      let newlines = 0;
+      while (this.match('newline')) newlines++;
+
+      if (newlines > 0) {
+        // Since blocks like parseParagraph consume their trailing newline, 
+        // seeing 1 newline here means there were 2 newlines in the source (a blank line).
+        // 1 newline here -> 1 empty_line
+        // 2 newlines here -> 2 empty_lines
+        for (let i = 0; i < newlines; i++) {
+          doc.children.push({ type: 'empty_line' } as any);
+        }
+      }
+
+      if (this.isAtEnd()) break;
+
+      const node = this.parseBlock();
+      if (node) {
+        doc.children.push(node);
+      }
+    }
+
+    return doc;
+  }
+
+  private parseBlock(): MarkdownNode | null {
+    if (this.isAtEnd()) return null;
+
+    const token = this.peek();
+
+    if (token.kind === 'hash') {
+      return this.parseHeading();
+    }
+    
+    if (token.kind === 'gt') {
+      const saved = this.current;
+      this.advance(); // consume gt
+      const next1 = this.peek();
+      if (next1 && next1.kind === 'text' && ('value' in next1) && (next1.value as string).trim() === '') {
+        this.advance(); // consume space
+      }
+      const next2 = this.peek();
+      if (next2 && next2.kind === 'bracket_open') {
+        const next3 = this.tokens[this.current + 1];
+        if (next3 && next3.kind === 'bang') {
+           this.current = saved; // restore
+           return this.parseNotice();
+        }
+      }
+      this.current = saved; // restore
+      return this.parseBlockquote();
+    }
+    
+    if (token.kind === 'fence') {
+      return this.parseFence();
+    }
+    
+    if (token.kind === 'dash' || token.kind === 'star') {
+       // Must be followed by a space to be a list!
+       const next = this.tokens[this.current + 1];
+       if (next && next.kind === 'text' && ('value' in next) && (next.value as string).startsWith(' ')) {
+         return this.parseList();
+       }
+    }
+
+    // Default to paragraph
+    return this.parseParagraph();
+  }
+
+  private parseHeading(): MarkdownHeading {
+    const hash = this.advance();
+    const level = hash.kind === 'hash' ? Math.min(hash.value.length, 6) : 1;
+    
+    // We don't want to blindly consume a text token if it contains actual text.
+    // Let's just let parseInline handle it. The renderer can deal with leading spaces,
+    // or we can trim the first child if it's text.
+    const children: MarkdownNode[] = [];
+    while (!this.isAtEnd() && !this.check('newline')) {
+      const inlineNode = this.parseInline();
+      if (inlineNode) {
+        children.push(inlineNode);
+      }
+    }
+    
+    // Trim leading space from the first child if it's text
+    if (children.length > 0 && children[0].type === 'text') {
+      children[0].value = (children[0] as any).value.replace(/^\\s+/, '');
+    }
+
+    this.match('newline');
+
+    return {
+      type: 'heading',
+      level,
+      children
+    };
+  }
+
+  private parseBlockquote(): MarkdownNode {
+    const children: MarkdownNode[] = [];
+    
+    while (!this.isAtEnd()) {
+      if (this.match('gt')) {
+        // blockquote line prefix
+      }
+      
+      if (this.check('newline')) {
+        this.advance();
+        if (this.check('newline') || this.isAtEnd()) {
+          break; // Empty line ends blockquote
+        }
+        children.push({ type: 'text', value: '\n' });
+      } else {
+        const inlineNode = this.parseInline();
+        if (inlineNode) children.push(inlineNode);
+      }
+    }
+
+    return {
+      type: 'blockquote',
+      children
+    };
+  }
+  
+  private parseNotice(): MarkdownNode {
+    const children: MarkdownNode[] = [];
+    this.advance(); // consume gt
+    
+    // consume the [!NOTE] part
+    while (!this.isAtEnd() && !this.check('newline')) {
+      if (this.match('bracket_close')) break;
+      this.advance();
+    }
+    
+    while (!this.isAtEnd()) {
+      if (this.match('gt')) {
+        // blockquote line prefix
+      }
+      if (this.check('newline')) {
+        this.advance();
+        if (this.check('newline') || this.isAtEnd()) break;
+        children.push({ type: 'text', value: '\n' });
+      } else {
+        const inlineNode = this.parseInline();
+        if (inlineNode) children.push(inlineNode);
+      }
+    }
+    
+    return { type: 'notice', children };
+  }
+  
+  private tokenValue(token: MarkdownToken): string {
+    if ('value' in token) return (token as any).value;
+    switch (token.kind) {
+      case 'bracket_open': return '[';
+      case 'bracket_close': return ']';
+      case 'paren_open': return '(';
+      case 'paren_close': return ')';
+      case 'bang': return '!';
+      case 'gt': return '>';
+      case 'plus': return '+';
+      case 'dot': return '.';
+      case 'hr': return '---';
+      default: return '';
+    }
+  }
+
+  private parseFence(): MarkdownNode {
+    const token = this.advance() as MarkdownFenceToken;
+    const value = token.lang;
+    let code = '';
+    while (!this.isAtEnd()) {
+      if (this.check('fence')) {
+        this.advance(); // close fence
+        break;
+      }
+      const p = this.advance();
+      code += this.tokenValue(p);
+    }
+    return { type: 'code_block', lang: value, value: code.trim() };
+  }
+  
+  private parseList(): MarkdownNode {
+    const items: any[] = [];
+    while (!this.isAtEnd() && (this.check('dash') || this.check('star'))) {
+      this.advance(); // consume dash/star
+      const children: MarkdownNode[] = [];
+      while (!this.isAtEnd() && !this.check('newline')) {
+        const inlineNode = this.parseInline();
+        if (inlineNode) children.push(inlineNode);
+      }
+      // Trim leading space from the first child if it's text
+      if (children.length > 0 && children[0].type === 'text') {
+        children[0].value = (children[0] as any).value.replace(/^\\s+/, '');
+      }
+      items.push({ type: 'list_item', children });
+      if (this.match('newline')) {
+        if (this.check('newline')) break; // End of list on double newline
+      }
+    }
+    return { type: 'list', ordered: false, children: items };
+  }
+
+  private parseParagraph(): MarkdownParagraph {
+    const children: MarkdownNode[] = [];
+    
+    while (!this.isAtEnd()) {
+      if (this.check('newline')) {
+        this.advance();
+        // Two newlines mean end of paragraph
+        if (this.check('newline') || this.isAtEnd()) {
+          break;
+        } else {
+          // A single newline could mean lazy continuation, BUT we must check if a new block starts!
+          if (this.check('hash') || this.check('gt') || this.check('fence') || this.check('hr') || this.check('dash')) {
+            break; // Paragraph interrupted by block
+          }
+          // Single newline treated as space in paragraph
+          children.push({ type: 'text', value: '\n' });
+        }
+      } else {
+        const inlineNode = this.parseInline();
+        if (inlineNode) {
+          children.push(inlineNode);
+        }
+      }
+    }
+
+    return {
+      type: 'paragraph',
+      children
+    };
+  }
+
+  private parseInline(terminator?: { kind: string, value?: string }): MarkdownNode | null {
+    if (this.isAtEnd()) return null;
+
+    if (terminator) {
+      const peekToken = this.peek();
+      if (peekToken.kind === terminator.kind && (!terminator.value || ('value' in peekToken && peekToken.value === terminator.value))) {
+        return null;
+      }
+    }
+
+    const token = this.advance();
+    
+    // Bold and Italic
+    if (token.kind === 'star' || token.kind === 'underscore') {
+      const val = ('value' in token) ? token.value as string : '';
+      if (val === '**' || val === '__' || val === '*' || val === '_') {
+        const type = (val.length === 2) ? 'strong' : 'emphasis';
+        
+        const savedCurrent = this.current;
+        const children: MarkdownNode[] = [];
+        let closed = false;
+
+        while (!this.isAtEnd()) {
+          const next = this.peek();
+          if (next.kind === 'newline') break; // Don't span across lines in this basic parser
+
+          if (next.kind === token.kind && ('value' in next && next.value === val)) {
+            this.advance(); // consume closing token
+            closed = true;
+            break;
+          }
+
+          const child = this.parseInline({ kind: token.kind, value: val });
+          if (child) {
+            children.push(child);
+          } else {
+            break;
+          }
+        }
+
+        if (closed) {
+          return { type, children };
+        } else {
+          // Backtrack and treat as text
+          this.current = savedCurrent;
+          return { type: 'text', value: val };
+        }
+      }
+    }
+
+    // Strikethrough
+    if (token.kind === 'tilde') {
+      const val = ('value' in token) ? token.value as string : '';
+      if (val === '~~') {
+        const savedCurrent = this.current;
+        const children: MarkdownNode[] = [];
+        let closed = false;
+
+        while (!this.isAtEnd()) {
+          const next = this.peek();
+          if (next.kind === 'newline') break;
+
+          if (next.kind === 'tilde' && ('value' in next && next.value === val)) {
+            this.advance();
+            closed = true;
+            break;
+          }
+
+          const child = this.parseInline({ kind: 'tilde', value: val });
+          if (child) children.push(child);
+          else break;
+        }
+
+        if (closed) {
+          return { type: 'strikethrough', children } as any; 
+        } else {
+          this.current = savedCurrent;
+          return { type: 'text', value: val };
+        }
+      }
+    }
+    
+    // Spoiler (||)
+    if (token.kind === 'text' && token.value === '||') {
+      const savedCurrent = this.current;
+      const children: MarkdownNode[] = [];
+      let closed = false;
+      while (!this.isAtEnd()) {
+        const next = this.peek();
+        if (next.kind === 'newline') break;
+        if (next.kind === 'text' && next.value === '||') {
+          this.advance();
+          closed = true;
+          break;
+        }
+        const child = this.parseInline({ kind: 'text', value: '||' });
+        if (child) children.push(child);
+        else break;
+      }
+      if (closed) {
+        return { type: 'spoiler', children };
+      } else {
+        this.current = savedCurrent;
+        return { type: 'text', value: '||' };
+      }
+    }
+
+    // Link or Image
+    if (token.kind === 'bracket_open' || token.kind === 'bang') {
+      const isImage = token.kind === 'bang';
+      if (isImage) {
+        if (!this.match('bracket_open')) {
+          return { type: 'text', value: '!' };
+        }
+      }
+      
+      const savedCurrent = this.current;
+      let text = '';
+      let closedBracket = false;
+      
+      while (!this.isAtEnd() && !this.check('newline')) {
+        if (this.match('bracket_close')) {
+          closedBracket = true;
+          break;
+        }
+        const p = this.advance();
+        text += ('value' in p) ? (p as any).value : '';
+      }
+
+      if (closedBracket && this.match('paren_open')) {
+        let url = '';
+        let closedParen = false;
+        while (!this.isAtEnd() && !this.check('newline')) {
+          if (this.match('paren_close')) {
+            closedParen = true;
+            break;
+          }
+          const p = this.advance();
+          url += ('value' in p) ? (p as any).value : '';
+        }
+
+        if (closedParen) {
+          if (isImage) {
+            return { type: 'image', url, alt: text };
+          } else {
+            return { type: 'link', url, children: [{ type: 'text', value: text }] };
+          }
+        }
+      }
+      
+      // Fallback
+      this.current = savedCurrent;
+      return { type: 'text', value: isImage ? '![' : '[' };
+    }
+
+    return {
+      type: 'text',
+      value: this.tokenValue(token)
+    };
+  }
+
+  // --- Helpers ---
+
+  private peek(): MarkdownToken {
+    return this.tokens[this.current];
+  }
+
+  private previous(): MarkdownToken {
+    return this.tokens[this.current - 1];
+  }
+
+  private isAtEnd(): boolean {
+    return this.current >= this.tokens.length;
+  }
+
+  private advance(): MarkdownToken {
+    if (!this.isAtEnd()) this.current++;
+    return this.previous();
+  }
+
+  private check(kind: MarkdownToken['kind']): boolean {
+    if (this.isAtEnd()) return false;
+    return this.peek().kind === kind;
+  }
+
+  private match(kind: MarkdownToken['kind']): boolean {
+    if (this.check(kind)) {
+      this.advance();
+      return true;
+    }
+    return false;
+
+  }
+}
+
+export function parseMarkdown(tokens: MarkdownToken[]): MarkdownDocument {
+  const parser = new MarkdownParser(tokens);
+  return parser.parse();
+}