@miliastry/quasar 1.0.0

package/src/Lexer/Lexer.ts

@@ -0,0 +1,181 @@
+/**
+ * DocumentEngine — Lexer
+ *
+ * Tokenizes source text into a stream of Tokens with Trivia.
+ * Trivia includes whitespace, newlines, and formatting that is
+ * preserved in the token stream but not semantically significant.
+ *
+ * This is a LANGUAGE-INDEPENDENT lexer framework.
+ * Language-specific tokenization is provided via the LexerOptions.
+ *
+ * Inspired by Roslyn's lexer architecture.
+ */
+
+import type { Token, TokenStream, Trivia } from '../Types/tokens'
+import { createToken, createTrivia, range, ArrayTokenStream } from '../Types/tokens'
+import type { TokenKind } from '../Types/tokens'
+
+// ─── Lexer Interface ───────────────────────────────────────────
+
+export interface LexerRule {
+  /** Pattern to match at the current position */
+  pattern: RegExp
+  /** Token kind to produce */
+  kind: TokenKind
+  /** Optional: extract a name from the match (for tags) */
+  name?: (match: RegExpExecArray) => string
+  /** Optional: extract attributes from the match */
+  attrs?: (match: RegExpExecArray) => string
+}
+
+export interface LexerOptions {
+  /** Language-specific tokenization rules */
+  rules: LexerRule[]
+  /** Characters that count as whitespace trivia */
+  whitespaceChars?: RegExp
+  /** Characters that start a newline trivia */
+  newlineChars?: RegExp
+  /** Whether to collect trivia */
+  collectTrivia?: boolean
+  /** Lexer hooks: called before each token */
+  onToken?: (token: Token) => void
+}
+
+// ─── Default Lexer ─────────────────────────────────────────────
+
+export class Lexer {
+  private options: LexerOptions
+  private source: string
+  private pos: number
+  private tokens: Token[]
+
+  constructor(options: LexerOptions) {
+    this.options = {
+      collectTrivia: true,
+      whitespaceChars: /[ \t]+/,
+      newlineChars: /\r?\n/,
+      ...options,
+    }
+    this.source = ''
+    this.pos = 0
+    this.tokens = []
+  }
+
+  /**
+   * Tokenize source text into a TokenStream.
+   */
+  tokenize(source: string): TokenStream {
+    this.source = source
+    this.pos = 0
+    this.tokens = []
+
+    while (this.pos < source.length) {
+      const leadingTrivia = this.options.collectTrivia ? this.scanTrivia() : []
+
+      // Try each rule in order
+      const token = this.tryRules(leadingTrivia)
+
+      if (token) {
+        this.tokens.push(token)
+      } else {
+        // No rule matched — produce a text token for the character
+        const start = this.pos
+        this.pos++
+        const text = source[start]
+        const tok = createToken('text', text, start, this.pos, {
+          leadingTrivia,
+        })
+        this.tokens.push(tok)
+      }
+    }
+
+    return new ArrayTokenStream(this.tokens) as TokenStream
+  }
+
+  /**
+   * Re-tokenize a portion of the source (for incremental parsing).
+   * Returns just the new tokens for the affected range.
+   */
+  retokenize(source: string, start: number, end: number): Token[] {
+    const oldSource = this.source
+    const oldPos = this.pos
+
+    this.source = source
+    this.pos = start
+    this.tokens = []
+
+    while (this.pos < end && this.pos < source.length) {
+      const leadingTrivia = this.scanTrivia()
+      const token = this.tryRules(leadingTrivia)
+      if (token) {
+        this.tokens.push(token)
+      } else {
+        const s = this.pos
+        this.pos++
+        this.tokens.push(createToken('text', source[s], s, this.pos, { leadingTrivia }))
+      }
+    }
+
+    const result = [...this.tokens]
+    this.source = oldSource
+    this.pos = oldPos
+    this.tokens = []
+
+    return result
+  }
+
+  // ── Private ─────────────────────────────────────────────
+
+  private tryRules(leadingTrivia: Trivia[]): Token | null {
+    for (const rule of this.options.rules) {
+      const match = rule.pattern.exec(this.source.slice(this.pos))
+      if (match && match.index === 0) {
+        const start = this.pos
+        const text = match[0]
+        this.pos += text.length
+
+        const trailingTrivia = this.options.collectTrivia ? this.scanTrivia() : []
+
+        const token = createToken(rule.kind, text, start, this.pos, {
+          name: rule.name?.(match),
+          attrs: rule.attrs?.(match),
+          tagText: text,
+          leadingTrivia,
+          trailingTrivia,
+        })
+
+        this.options.onToken?.(token)
+        return token
+      }
+    }
+    return null
+  }
+
+  private scanTrivia(): Trivia[] {
+    const trivia: Trivia[] = []
+
+    while (this.pos < this.source.length) {
+      const rest = this.source.slice(this.pos)
+
+      const newlineMatch = this.options.newlineChars!.exec(rest)
+      if (newlineMatch && newlineMatch.index === 0) {
+        const start = this.pos
+        this.pos += newlineMatch[0].length
+        trivia.push(createTrivia('newline', newlineMatch[0], start, this.pos))
+        continue
+      }
+
+      const wsMatch = this.options.whitespaceChars!.exec(rest)
+      if (wsMatch && wsMatch.index === 0) {
+        const start = this.pos
+        this.pos += wsMatch[0].length
+        trivia.push(createTrivia('whitespace', wsMatch[0], start, this.pos))
+        continue
+      }
+
+      break
+    }
+
+    return trivia
+  }
+}

package/src/Lexer/index.ts

@@ -0,0 +1,10 @@
+export { Lexer } from './Lexer'
+export type { LexerOptions, LexerRule } from './Lexer'
+export { scanBBCode, formatTokens } from './BBCodeLexer'
+export type {
+  BBCodeToken,
+  BBCodeOpenToken,
+  BBCodeCloseToken,
+  BBCodeTextToken,
+  BBCodeNewlineToken,
+} from './BBCodeLexer'

package/src/Linter/Linter.ts

@@ -0,0 +1,193 @@
+/**
+ * DocumentEngine — Linter
+ *
+ * Lints the document tree for issues.
+ * Similar to ESLint but for BBCode documents.
+ *
+ * Rules can be added via the Plugin API.
+ */
+
+import { RedNode } from '../Syntax/RedNode'
+import type { SemanticAnalyzer, Validator } from '../Semantic/SemanticAnalyzer'
+
+export type LintSeverity = 'error' | 'warning' | 'info' | 'hint'
+
+export interface LintRule {
+  code: string
+  severity: LintSeverity
+  description: string
+  validate(node: RedNode, context: LintContext): LintIssue | LintIssue[] | null
+}
+
+export interface LintContext {
+  source: string
+  allNodes: Map<string, RedNode>
+}
+
+export interface LintIssue {
+  code: string
+  message: string
+  severity: LintSeverity
+  nodeId: string
+  range: { start: number; end: number } | null
+  fix?: {
+    description: string
+    apply: () => void
+  }
+}
+
+export interface LintResult {
+  issues: LintIssue[]
+  errorCount: number
+  warningCount: number
+  infoCount: number
+  hintCount: number
+}
+
+export class Linter {
+  private rules: Map<string, LintRule> = new Map()
+
+  constructor() {
+    this.registerBuiltinRules()
+  }
+
+  /**
+   * Register a lint rule.
+   */
+  register(rule: LintRule): void {
+    this.rules.set(rule.code, rule)
+  }
+
+  /**
+   * Unregister a lint rule.
+   */
+  unregister(code: string): void {
+    this.rules.delete(code)
+  }
+
+  /**
+   * Lint a RedNode tree.
+   */
+  lint(root: RedNode, source: string): LintResult {
+    const issues: LintIssue[] = []
+    const result = { issues, errorCount: 0, warningCount: 0, infoCount: 0, hintCount: 0 }
+
+    // Build node index
+    const allNodes = new Map<string, RedNode>()
+    root.walk(node => { allNodes.set(node.id, node) })
+
+    const context: LintContext = { source, allNodes }
+
+    // Walk tree and apply rules
+    root.walk(node => {
+      for (const [, rule] of this.rules) {
+        try {
+          const r = rule.validate(node, context)
+          if (r) {
+            const items = Array.isArray(r) ? r : [r]
+            for (const issue of items) {
+              issues.push(issue)
+              switch (issue.severity) {
+                case 'error': result.errorCount++; break
+                case 'warning': result.warningCount++; break
+                case 'info': result.infoCount++; break
+                case 'hint': result.hintCount++; break
+              }
+            }
+          }
+        } catch {
+          // Rule error shouldn't break linting
+        }
+      }
+    })
+
+    return result
+  }
+
+  private registerBuiltinRules(): void {
+    this.register({
+      code: 'no-nested-bold',
+      severity: 'warning',
+      description: 'Nested bold tags are redundant',
+      validate: (node) => {
+        if (node.kind === 'bold' && node.parent?.kind === 'bold') {
+          return {
+            code: 'no-nested-bold',
+            message: 'Nested [b] tags are redundant',
+            severity: 'warning' as LintSeverity,
+            nodeId: node.id,
+            range: null,
+          }
+        }
+        return null
+      },
+    })
+
+    this.register({
+      code: 'no-empty-tags',
+      severity: 'hint',
+      description: 'Empty tags have no effect',
+      validate: (node) => {
+        if (['bold', 'italic', 'underline', 'strikethrough', 'color', 'font_size', 'spoiler'].includes(node.kind)) {
+          if (node.children.length === 0 && (node.text === '' || !node.text)) {
+            return {
+              code: 'no-empty-tags',
+              message: `Empty [${node.kind}] tag has no effect`,
+              severity: 'hint' as LintSeverity,
+              nodeId: node.id,
+              range: null,
+            }
+          }
+        }
+        return null
+      },
+    })
+
+    this.register({
+      code: 'max-quote-depth',
+      severity: 'warning',
+      description: 'Quote blocks should not be nested more than 3 levels deep',
+      validate: (node) => {
+        if (node.kind === 'quote') {
+          let depth = 0
+          let current = node.parent
+          while (current) {
+            if (current.kind === 'quote') depth++
+            current = current.parent
+          }
+          if (depth >= 3) {
+            return {
+              code: 'max-quote-depth',
+              message: 'Quote blocks should not be nested more than 3 levels deep',
+              severity: 'warning' as LintSeverity,
+              nodeId: node.id,
+              range: null,
+            }
+          }
+        }
+        return null
+      },
+    })
+
+    this.register({
+      code: 'invalid-url-protocol',
+      severity: 'error',
+      description: 'URLs must start with http:// or https:// (javascript:, file:, etc. are not allowed)',
+      validate: (node) => {
+        if (node.kind === 'url') {
+          const destination = String(node.metadata?.href ?? '') || node.text
+          if (destination && !destination.startsWith('http://') && !destination.startsWith('https://') && !destination.startsWith('mailto:')) {
+            return {
+              code: 'invalid-url-protocol',
+              message: 'URLs must start with http:// or https:// (javascript:, file:, etc. are not allowed)',
+              severity: 'error' as LintSeverity,
+              nodeId: node.id,
+              range: null,
+            }
+          }
+        }
+        return null
+      },
+    })
+  }
+}

package/src/Linter/index.ts

@@ -0,0 +1,2 @@
+export { Linter } from './Linter'
+export type { LintRule, LintIssue, LintResult, LintContext, LintSeverity } from './Linter'

package/src/Markdown/MarkdownAST.ts

@@ -0,0 +1,112 @@
+/**
+ * DocumentEngine — Markdown AST Nodes
+ */
+
+export type MarkdownNode = 
+  | MarkdownDocument
+  | MarkdownParagraph
+  | MarkdownHeading
+  | MarkdownText
+  | MarkdownCodeBlock
+  | MarkdownCodeInline
+  | MarkdownStrong
+  | MarkdownEmphasis
+  | MarkdownLink
+  | MarkdownImage
+  | MarkdownList
+  | MarkdownListItem
+  | MarkdownBlockquote
+  | MarkdownSpoiler
+  | MarkdownNotice
+  | MarkdownSpacing
+  | MarkdownEmptyLine;
+
+export interface MarkdownDocument {
+  type: 'document';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownParagraph {
+  type: 'paragraph';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownHeading {
+  type: 'heading';
+  level: number;
+  children: MarkdownNode[];
+}
+
+export interface MarkdownText {
+  type: 'text';
+  value: string;
+}
+
+export interface MarkdownCodeBlock {
+  type: 'code_block';
+  lang: string;
+  value: string;
+}
+
+export interface MarkdownCodeInline {
+  type: 'code_inline';
+  value: string;
+}
+
+export interface MarkdownStrong {
+  type: 'strong';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownEmphasis {
+  type: 'emphasis';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownLink {
+  type: 'link';
+  url: string;
+  children: MarkdownNode[];
+}
+
+export interface MarkdownImage {
+  type: 'image';
+  url: string;
+  alt: string;
+}
+
+export interface MarkdownList {
+  type: 'list';
+  ordered: boolean;
+  children: MarkdownListItem[];
+}
+
+export interface MarkdownListItem {
+  type: 'list_item';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownBlockquote {
+  type: 'blockquote';
+  children: MarkdownNode[];
+}
+
+/** `>!` spoiler block — produced by MarkdownParser, maps to BBCode [spoiler]. */
+export interface MarkdownSpoiler {
+  type: 'spoiler';
+  children: MarkdownNode[];
+}
+
+/** `> [!NOTE]` callout — produced by MarkdownParser, maps to BBCode [notice]. */
+export interface MarkdownNotice {
+  type: 'notice';
+  children: MarkdownNode[];
+}
+
+export interface MarkdownSpacing {
+  type: 'spacing';
+}
+
+export interface MarkdownEmptyLine {
+  type: 'empty_line';
+}

package/src/Markdown/MarkdownDocumentModel.ts

@@ -0,0 +1,55 @@
+import { DocumentModel, type DocumentModelOptions } from '../Model/DocumentModel';
+import { GreenNode, greenLeaf } from '../Syntax/GreenNode';
+import { RedNode } from '../Syntax/RedNode';
+import { scanMarkdown } from './MarkdownLexer';
+import { parseMarkdown } from './MarkdownParser';
+import { markdownAstToGreenTree, greenToRedNode } from './MarkdownToGreenNode';
+
+export interface MarkdownDocumentModelOptions extends DocumentModelOptions {
+  source?: string;
+  language?: string;
+}
+
+export class MarkdownDocumentModel extends DocumentModel {
+  constructor(options: MarkdownDocumentModelOptions = {}) {
+    super({ source: '', language: options.language ?? 'markdown' });
+    if (options.source) {
+      this.rebuild(options.source);
+    }
+  }
+
+  static fromMarkdown(source: string): MarkdownDocumentModel {
+    return new MarkdownDocumentModel({ source });
+  }
+
+  protected parseToGreen(source: string): GreenNode {
+    try {
+      const tokens = scanMarkdown(source);
+      const ast = parseMarkdown(tokens);
+      return markdownAstToGreenTree(ast);
+    } catch (error) {
+      console.warn('[MarkdownDocumentModel] Parse error:', error);
+      return greenLeaf('text', source);
+    }
+  }
+
+  protected buildRedFromGreen(green: GreenNode): RedNode {
+    return greenToRedNode(green);
+  }
+
+  // Override text updates to bypass incremental parsing completely.
+  // The Markdown AST currently strips markup tokens (like **, [, ]), 
+  // so GreenTree lengths do not match the raw text lengths. 
+  // Incremental parsing relies on exact byte offsets, so it fails destructively here.
+  applyTextUpdate(newSource: string): void {
+    if (this.source === newSource) return;
+    this.rebuild(newSource);
+  }
+
+  applyChange(change: any): void {
+    // Fallback to full rebuild using the text change
+    const before = this.source.slice(0, change.start);
+    const after = this.source.slice(change.end);
+    this.rebuild(before + change.text + after);
+  }
+}