apexfile 1.1.0

package/src/tokenizer/index.js

@@ -0,0 +1,518 @@
+'use strict';
+
+/**
+ * ApexDoc Tokenizer
+ * Converts raw .apx source text into a flat array of tokens.
+ * Each token = { type, value, props, line, col }
+ */
+
+const T = require('./tokens');
+
+// Top-level sections (%%meta, %%style, %%body)
+const TOP_SECTIONS = new Set(['meta', 'style', 'body']);
+
+// Blocks that self-close (no %%end needed when written as %%name {props}%%end inline)
+const SELF_CLOSE_BLOCKS = new Set([
+  'chart', 'image', 'video', 'audio', 'clock', 'countdown', 'divider',
+  'spacer', 'progress', 'qr', 'map', 'lottie', 'icon', 'rating',
+  'back-to-top', 'breadcrumbs', 'sidenav', 'toc', 'progress-bar',
+  'theme', 'theme-switcher', 'stat', 'signature', 'feed',
+  'slider', 'toggle', 'unit',
+]);
+
+class Tokenizer {
+  constructor(source) {
+    this.source = source;
+    this.pos    = 0;
+    this.line   = 1;
+    this.col    = 1;
+    this.tokens = [];
+  }
+
+  // ── Main Entry ─────────────────────────────────────────────────
+
+  tokenize() {
+    const lines = this.source.split('\n');
+
+    for (let i = 0; i < lines.length; i++) {
+      this.line = i + 1;
+      const raw  = lines[i];
+      const line = raw.trimEnd();
+
+      // Empty line
+      if (line.trim() === '') {
+        this.push(T.NEWLINE, '', {}, i + 1);
+        continue;
+      }
+
+      // ── Block directives %%name {props} ──────────────────────
+      if (line.trimStart().startsWith('%%')) {
+        this._tokenizeBlock(line.trim(), i + 1);
+        continue;
+      }
+
+      // ── Context-dependent: inside %%meta or %%style ──────────
+      // MUST come before @ check so @theme/@import inside %%style
+      // are handled by the style tokenizer, not the logic tokenizer.
+      const ctx = this._currentContext();
+
+      if (ctx === 'meta') {
+        this._tokenizeMetaField(line, i + 1);
+        continue;
+      }
+
+      if (ctx === 'style') {
+        this._tokenizeStyleLine(line, i + 1);
+        continue;
+      }
+
+      // ── Logic directives @set / @if / @each / @end ───────────
+      if (line.trimStart().startsWith('@')) {
+        this._tokenizeLogic(line.trim(), i + 1);
+        continue;
+      }
+
+      // ── Body content ─────────────────────────────────────────
+      this._tokenizeBodyLine(line, i + 1);
+    }
+
+    this.push(T.EOF, '', {}, this.line);
+    return this.tokens;
+  }
+
+  // ── Block Tokenizer ────────────────────────────────────────────
+
+  _tokenizeBlock(line, lineNum) {
+    // %%end — closes current block or section
+    if (line === '%%end') {
+      const ctx = this._currentContext();
+      if (TOP_SECTIONS.has(ctx)) {
+        this.push(T.SECTION_CLOSE, ctx, {}, lineNum);
+      } else {
+        this.push(T.BLOCK_CLOSE, ctx, {}, lineNum);
+      }
+      return;
+    }
+
+    // %%name {props}%%end — self-closing inline
+    const selfCloseMatch = line.match(/^%%([a-z0-9-]+)(?:\s*(\{[^}]*\}))?%%end$/i);
+    if (selfCloseMatch) {
+      const name  = selfCloseMatch[1].toLowerCase();
+      const props = selfCloseMatch[2] ? this._parseProps(selfCloseMatch[2]) : {};
+      this.push(T.BLOCK_SELF_CLOSE, name, props, lineNum);
+      return;
+    }
+
+    // %%name {props}   or   %%name
+    const blockMatch = line.match(/^%%([a-z0-9-]+)(?:\s+(\{[\s\S]*\}))?$/i);
+    if (blockMatch) {
+      const name  = blockMatch[1].toLowerCase();
+      const props = blockMatch[2] ? this._parseProps(blockMatch[2]) : {};
+
+      if (TOP_SECTIONS.has(name)) {
+        this.push(T.SECTION_OPEN, name, props, lineNum);
+      } else {
+        this.push(T.BLOCK_OPEN, name, props, lineNum);
+      }
+      return;
+    }
+
+    // Fallback — treat as text
+    this.push(T.TEXT, line, {}, lineNum);
+  }
+
+  // ── Logic Tokenizer ────────────────────────────────────────────
+
+  _tokenizeLogic(line, lineNum) {
+    // @set name = value
+    const setMatch = line.match(/^@set\s+(\w+)\s*=\s*(.+)$/);
+    if (setMatch) {
+      this.push(T.SET, setMatch[1], { value: setMatch[2].trim() }, lineNum);
+      return;
+    }
+
+    // @if condition
+    const ifMatch = line.match(/^@if\s+(.+)$/);
+    if (ifMatch) {
+      this.push(T.IF, ifMatch[1].trim(), {}, lineNum);
+      return;
+    }
+
+    // @elseif condition
+    const elseifMatch = line.match(/^@elseif\s+(.+)$/);
+    if (elseifMatch) {
+      this.push(T.ELSEIF, elseifMatch[1].trim(), {}, lineNum);
+      return;
+    }
+
+    // @else
+    if (line === '@else') {
+      this.push(T.ELSE, '', {}, lineNum);
+      return;
+    }
+
+    // @end
+    if (line === '@end') {
+      this.push(T.END_LOGIC, '', {}, lineNum);
+      return;
+    }
+
+    // @each item in list
+    const eachMatch = line.match(/^@each\s+(\w+)\s+in\s+(.+)$/);
+    if (eachMatch) {
+      this.push(T.EACH, eachMatch[1], { source: eachMatch[2].trim() }, lineNum);
+      return;
+    }
+
+    // @keyframe name { ... } (single-line — multi-line handled in parser)
+    const kfMatch = line.match(/^@keyframe\s+(\w+)/);
+    if (kfMatch) {
+      this.push(T.KEYFRAME, kfMatch[1], {}, lineNum);
+      return;
+    }
+
+    // Unknown @ directive — treat as text
+    this.push(T.TEXT, line, {}, lineNum);
+  }
+
+  // ── Meta Field Tokenizer ───────────────────────────────────────
+
+  _tokenizeMetaField(line, lineNum) {
+    // key: value  or  key:   "value"
+    const fieldMatch = line.match(/^\s*([\w-]+)\s*:\s*(.*)$/);
+    if (fieldMatch) {
+      const key   = fieldMatch[1].trim();
+      const value = this._parseMetaValue(fieldMatch[2].trim());
+      this.push(T.META_FIELD, key, { value }, lineNum);
+    }
+  }
+
+  // ── Style Line Tokenizer ───────────────────────────────────────
+
+  _tokenizeStyleLine(line, lineNum) {
+    const trimmed = line.trim();
+
+    // @import / @media / @breakpoint  (without a block {)
+    if (trimmed.startsWith('@') && !trimmed.includes('{')) {
+      this.push(T.STYLE_DIRECTIVE, trimmed, {}, lineNum);
+      return;
+    }
+
+    // @theme name {  /  @media (...) {  /  @breakpoint name {
+    if (trimmed.startsWith('@') && trimmed.endsWith('{')) {
+      const directiveMatch = trimmed.match(/^(@\w+)\s+([^{]+)\s*\{$/);
+      if (directiveMatch) {
+        this.push(T.STYLE_BLOCK_OPEN, directiveMatch[1], { name: directiveMatch[2].trim() }, lineNum);
+        return;
+      }
+    }
+
+    // Closing brace of a style block
+    if (trimmed === '}') {
+      this.push(T.STYLE_BLOCK_CLOSE, '', {}, lineNum);
+      return;
+    }
+
+    // --variable: value
+    const varMatch = trimmed.match(/^(--[\w-]+)\s*:\s*(.+)$/);
+    if (varMatch) {
+      this.push(T.STYLE_VAR, varMatch[1], { value: varMatch[2].trim() }, lineNum);
+      return;
+    }
+
+    // Regular CSS property inside a style block
+    const propMatch = trimmed.match(/^([\w-]+)\s*:\s*(.+)$/);
+    if (propMatch) {
+      this.push(T.STYLE_VAR, propMatch[1], { value: propMatch[2].trim() }, lineNum);
+    }
+  }
+
+  // ── Body Line Tokenizer ────────────────────────────────────────
+
+  _tokenizeBodyLine(line, lineNum) {
+    // Heading  # ## ### ####
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+    if (headingMatch) {
+      const level = headingMatch[1].length;
+      const rest  = headingMatch[2];
+      // Extract inline props from {props} at end of heading
+      const { text, props } = this._extractInlineProps(rest);
+      this.push(T.HEADING, text, { level, ...props }, lineNum);
+      return;
+    }
+
+    // Horizontal rule
+    if (/^(---|===|~~~)$/.test(line.trim())) {
+      this.push(T.HR, line.trim(), {}, lineNum);
+      return;
+    }
+
+    // Blockquote  > text
+    if (line.trimStart().startsWith('>')) {
+      const level   = (line.match(/^(>+)/)?.[1] || '>').length;
+      const content = line.replace(/^>+\s?/, '');
+      this.push(T.BLOCKQUOTE, content, { level }, lineNum);
+      return;
+    }
+
+    // Unordered list item
+    const ulMatch = line.match(/^(\s*)-\s+(.+)$/);
+    if (ulMatch) {
+      const indent  = ulMatch[1].length;
+      const content = ulMatch[2];
+      // Task list  - [x] / - [ ] / - [~]
+      const taskMatch = content.match(/^\[([x ~])\]\s+(.+)$/i);
+      if (taskMatch) {
+        const state = taskMatch[1] === 'x' ? 'done' : taskMatch[1] === '~' ? 'progress' : 'todo';
+        this.push(T.LIST_ITEM_TASK, taskMatch[2], { indent, state }, lineNum);
+      } else {
+        this.push(T.LIST_ITEM, content, { indent }, lineNum);
+      }
+      return;
+    }
+
+    // Ordered list item  1. text
+    const olMatch = line.match(/^(\s*)(\d+)\.\s+(.+)$/);
+    if (olMatch) {
+      const indent = olMatch[1].length;
+      const num    = parseInt(olMatch[2], 10);
+      this.push(T.LIST_ITEM_ORDERED, olMatch[3], { indent, num }, lineNum);
+      return;
+    }
+
+    // Footnote definition  [^1]: content
+    const fnDefMatch = line.match(/^\[\^(\w+)\]:\s+(.+)$/);
+    if (fnDefMatch) {
+      this.push(T.FOOTNOTE_DEF, fnDefMatch[1], { content: fnDefMatch[2] }, lineNum);
+      return;
+    }
+
+    // Regular paragraph — tokenize inline syntax
+    const inlineTokens = this._tokenizeInline(line, lineNum);
+    for (const tok of inlineTokens) {
+      this.tokens.push(tok);
+    }
+  }
+
+  // ── Inline Tokenizer ───────────────────────────────────────────
+
+  _tokenizeInline(text, lineNum) {
+    const tokens = [];
+    let remaining = text;
+    let buffer    = '';
+
+    const flush = () => {
+      if (buffer.length > 0) {
+        tokens.push(this._tok(T.TEXT, buffer, {}, lineNum));
+        buffer = '';
+      }
+    };
+
+    const patterns = [
+      // Math inline  $...$
+      { re: /^\$([^$]+)\$/, type: T.MATH_INLINE, val: m => m[1] },
+
+      // Inline styled  [text]{props}
+      { re: /^\[([^\]]+)\]\{([^}]*)\}/, type: T.INLINE_STYLED,
+        val: m => m[1], extra: m => ({ props: this._parseInlineProps(m[2]) }) },
+
+      // Link  [text](url)
+      { re: /^\[([^\]]+)\]\(([^)]+)\)/, type: T.LINK,
+        val: m => m[1], extra: m => ({ href: m[2] }) },
+
+      // Footnote ref  [^1]
+      { re: /^\[\^(\w+)\]/, type: T.FOOTNOTE_REF, val: m => m[1] },
+
+      // Bold  **text**
+      { re: /^\*\*([^*]+)\*\*/, type: T.BOLD, val: m => m[1] },
+
+      // Italic  _text_
+      { re: /^_([^_]+)_/, type: T.ITALIC, val: m => m[1] },
+
+      // Underline  __text__
+      { re: /^__([^_]+)__/, type: T.UNDERLINE, val: m => m[1] },
+
+      // Strikethrough  ~~text~~
+      { re: /^~~([^~]+)~~/, type: T.STRIKETHROUGH, val: m => m[1] },
+
+      // Superscript  ^^text^^
+      { re: /^\^\^([^^]+)\^\^/, type: T.SUPERSCRIPT, val: m => m[1] },
+
+      // Subscript  ,,text,,
+      { re: /^,,([^,]+),,/, type: T.SUBSCRIPT, val: m => m[1] },
+
+      // Code inline  `text`
+      { re: /^`([^`]+)`/, type: T.CODE_INLINE, val: m => m[1] },
+
+      // Highlight  ==text==
+      { re: /^==([^=]+)==/, type: T.HIGHLIGHT, val: m => m[1] },
+
+      // Expression  {expr}
+      { re: /^\{([^}]+)\}/, type: T.EXPRESSION, val: m => m[1] },
+    ];
+
+    while (remaining.length > 0) {
+      let matched = false;
+
+      for (const pat of patterns) {
+        const m = remaining.match(pat.re);
+        if (m) {
+          flush();
+          const extra = pat.extra ? pat.extra(m) : {};
+          tokens.push(this._tok(pat.type, pat.val(m), extra, lineNum));
+          remaining = remaining.slice(m[0].length);
+          matched = true;
+          break;
+        }
+      }
+
+      if (!matched) {
+        buffer += remaining[0];
+        remaining = remaining.slice(1);
+      }
+    }
+
+    flush();
+    return tokens;
+  }
+
+  // ── Helpers ────────────────────────────────────────────────────
+
+  /**
+   * Parse a props string like {type: bar, data: sales, animated: true}
+   * into a plain object.
+   */
+  _parseProps(str) {
+    const result = {};
+    // Remove outer braces
+    const inner = str.replace(/^\{|\}$/g, '').trim();
+    if (!inner) return result;
+
+    // Split on commas that are NOT inside quotes or brackets
+    const parts = this._splitProps(inner);
+
+    for (const part of parts) {
+      const colonIdx = part.indexOf(':');
+      if (colonIdx === -1) continue;
+      const key   = part.slice(0, colonIdx).trim();
+      const value = part.slice(colonIdx + 1).trim();
+      result[key] = this._parseMetaValue(value);
+    }
+
+    return result;
+  }
+
+  _parseInlineProps(str) {
+    return this._parseProps(`{${str}}`);
+  }
+
+  /**
+   * Split a props string on commas, respecting nested brackets and quotes.
+   */
+  _splitProps(str) {
+    const parts = [];
+    let depth  = 0;
+    let inQuote = false;
+    let quoteChar = '';
+    let current = '';
+
+    for (let i = 0; i < str.length; i++) {
+      const ch = str[i];
+
+      if (!inQuote && (ch === '"' || ch === "'")) {
+        inQuote = true;
+        quoteChar = ch;
+        current += ch;
+      } else if (inQuote && ch === quoteChar) {
+        inQuote = false;
+        current += ch;
+      } else if (!inQuote && (ch === '(' || ch === '[' || ch === '{')) {
+        depth++;
+        current += ch;
+      } else if (!inQuote && (ch === ')' || ch === ']' || ch === '}')) {
+        depth--;
+        current += ch;
+      } else if (!inQuote && depth === 0 && ch === ',') {
+        parts.push(current.trim());
+        current = '';
+      } else {
+        current += ch;
+      }
+    }
+
+    if (current.trim()) parts.push(current.trim());
+    return parts;
+  }
+
+  /**
+   * Parse a meta value — handles strings, booleans, numbers, arrays
+   */
+  _parseMetaValue(raw) {
+    if (!raw) return '';
+
+    // String with quotes
+    if ((raw.startsWith('"') && raw.endsWith('"')) ||
+        (raw.startsWith("'") && raw.endsWith("'"))) {
+      return raw.slice(1, -1);
+    }
+
+    // Array  [a, b, c]
+    if (raw.startsWith('[') && raw.endsWith(']')) {
+      const inner = raw.slice(1, -1);
+      return this._splitProps(inner).map(v => this._parseMetaValue(v.trim()));
+    }
+
+    // Boolean
+    if (raw === 'true')  return true;
+    if (raw === 'false') return false;
+
+    // Null / undefined
+    if (raw === 'null' || raw === 'never' || raw === '') return null;
+
+    // Number
+    if (!isNaN(raw) && raw !== '') return parseFloat(raw);
+
+    // Semver / date / plain string
+    return raw;
+  }
+
+  /**
+   * Extract {props} from end of a heading line
+   * "My Heading {color: red}" → { text: "My Heading", props: { color: "red" } }
+   */
+  _extractInlineProps(text) {
+    const match = text.match(/^(.*?)\s*(\{[^}]*\})\s*$/);
+    if (match) {
+      return {
+        text:  match[1].trim(),
+        props: this._parseProps(match[2]),
+      };
+    }
+    return { text, props: {} };
+  }
+
+  /**
+   * Returns the current open context (meta/style/body/blockname)
+   */
+  _currentContext() {
+    // Walk backwards through tokens to find last open section/block
+    for (let i = this.tokens.length - 1; i >= 0; i--) {
+      const tok = this.tokens[i];
+      if (tok.type === T.SECTION_OPEN)  return tok.value;
+      if (tok.type === T.SECTION_CLOSE) return 'root';
+      if (tok.type === T.BLOCK_OPEN)    return tok.value;
+      if (tok.type === T.BLOCK_CLOSE)   continue;
+    }
+    return 'root';
+  }
+
+  _tok(type, value, props, line) {
+    return { type, value, props: props || {}, line };
+  }
+
+  push(type, value, props, line) {
+    this.tokens.push(this._tok(type, value, props, line));
+  }
+}
+
+module.exports = Tokenizer;

package/src/tokenizer/tokens.js

@@ -0,0 +1,75 @@
+/**
+ * ApexDoc Token Types
+ * Every piece of an .apx file becomes one of these tokens
+ */
+
+const T = {
+  // ── Document Structure ──────────────────────────────────────────
+  SECTION_OPEN:      'SECTION_OPEN',       // %%meta  %%style  %%body
+  SECTION_CLOSE:     'SECTION_CLOSE',      // %%end (closing a top-level section)
+  BLOCK_OPEN:        'BLOCK_OPEN',         // %%box {props}
+  BLOCK_CLOSE:       'BLOCK_CLOSE',        // %%end (closing a block)
+  BLOCK_SELF_CLOSE:  'BLOCK_SELF_CLOSE',   // %%chart {props}%%end
+
+  // ── Headings ────────────────────────────────────────────────────
+  HEADING:           'HEADING',            // # ## ### ####
+
+  // ── Inline Text ─────────────────────────────────────────────────
+  TEXT:              'TEXT',               // plain text
+  BOLD:              'BOLD',               // **text**
+  ITALIC:            'ITALIC',             // _text_
+  UNDERLINE:         'UNDERLINE',          // __text__
+  STRIKETHROUGH:     'STRIKETHROUGH',      // ~~text~~
+  SUPERSCRIPT:       'SUPERSCRIPT',        // ^^text^^
+  SUBSCRIPT:         'SUBSCRIPT',          // ,,text,,
+  CODE_INLINE:       'CODE_INLINE',        // `text`
+  HIGHLIGHT:         'HIGHLIGHT',          // ==text==
+  INLINE_STYLED:     'INLINE_STYLED',      // [text]{props}
+  LINK:              'LINK',               // [text](url)
+
+  // ── Math ────────────────────────────────────────────────────────
+  MATH_INLINE:       'MATH_INLINE',        // $...$
+  MATH_BLOCK:        'MATH_BLOCK',         // %%math...%%end
+
+  // ── Lists ───────────────────────────────────────────────────────
+  LIST_ITEM:         'LIST_ITEM',          // - item
+  LIST_ITEM_ORDERED: 'LIST_ITEM_ORDERED',  // 1. item
+  LIST_ITEM_TASK:    'LIST_ITEM_TASK',     // - [x] / - [ ] / - [~]
+
+  // ── Logic ───────────────────────────────────────────────────────
+  SET:               'SET',               // @set x = value
+  IF:                'IF',                // @if condition
+  ELSEIF:            'ELSEIF',            // @elseif condition
+  ELSE:              'ELSE',              // @else
+  END_LOGIC:         'END_LOGIC',         // @end
+  EACH:              'EACH',              // @each item in list
+  KEYFRAME:          'KEYFRAME',          // @keyframe name { }
+
+  // ── Variables & Expressions ─────────────────────────────────────
+  EXPRESSION:        'EXPRESSION',        // {expr}  {name}  {fn()}
+
+  // ── Metadata ────────────────────────────────────────────────────
+  META_FIELD:        'META_FIELD',        // key: value inside %%meta
+
+  // ── Style ───────────────────────────────────────────────────────
+  STYLE_VAR:         'STYLE_VAR',         // --primary: #00f5d4
+  STYLE_DIRECTIVE:   'STYLE_DIRECTIVE',   // @theme / @import / @media / @breakpoint
+  STYLE_BLOCK_OPEN:  'STYLE_BLOCK_OPEN',  // @theme name {
+  STYLE_BLOCK_CLOSE: 'STYLE_BLOCK_CLOSE', // }
+
+  // ── Blockquote ──────────────────────────────────────────────────
+  BLOCKQUOTE:        'BLOCKQUOTE',        // > text
+
+  // ── Horizontal Rule ─────────────────────────────────────────────
+  HR:                'HR',               // --- === ~~~
+
+  // ── Footnote ────────────────────────────────────────────────────
+  FOOTNOTE_REF:      'FOOTNOTE_REF',     // [^1]
+  FOOTNOTE_DEF:      'FOOTNOTE_DEF',     // [^1]: content
+
+  // ── Special ─────────────────────────────────────────────────────
+  NEWLINE:           'NEWLINE',
+  EOF:               'EOF',
+};
+
+module.exports = T;