@fragments-sdk/cli 0.2.2

package/src/service/token-parser.ts

@@ -0,0 +1,504 @@
+/**
+ * Token Parser
+ *
+ * Parses CSS/SCSS files to extract CSS custom properties (design tokens).
+ * Handles:
+ * - CSS custom property declarations (--token-name: value)
+ * - Reference resolution (var(--other-token))
+ * - Theme detection via selectors
+ * - Category inference from naming conventions
+ */
+
+import { readFile } from "node:fs/promises";
+import { resolve, relative } from "node:path";
+import fastGlob from "fast-glob";
+import type {
+  DesignToken,
+  TokenCategory,
+  TokenConfig,
+  TokenParseResult,
+  TokenParseError,
+} from "../core/index.js";
+
+/**
+ * Pattern to match CSS custom property declarations
+ * Captures: [full match, property name, value]
+ * Example: "--color-primary: var(--color-cobalt-50);"
+ */
+const TOKEN_DECLARATION_PATTERN =
+  /--([a-zA-Z0-9_-]+)\s*:\s*([^;]+);/g;
+
+/**
+ * Pattern to match var() references
+ * Captures: [full match, token name, fallback value?]
+ * Example: "var(--color-cobalt-50)" or "var(--color-primary, #fff)"
+ */
+const VAR_REFERENCE_PATTERN =
+  /var\(\s*--([a-zA-Z0-9_-]+)(?:\s*,\s*([^)]+))?\s*\)/g;
+
+/**
+ * Pattern to match CSS selectors (to detect theme blocks)
+ */
+const SELECTOR_PATTERN = /([^{]+)\s*\{/g;
+
+/**
+ * Category inference patterns - maps naming conventions to categories
+ */
+const CATEGORY_PATTERNS: Array<{
+  pattern: RegExp;
+  category: TokenCategory;
+}> = [
+  { pattern: /color|bg|background|border-color|fill|stroke/i, category: "color" },
+  { pattern: /spacing|margin|padding|gap|space|inset/i, category: "spacing" },
+  { pattern: /font|text|line-height|letter-spacing|typography/i, category: "typography" },
+  { pattern: /radius|rounded|corner/i, category: "radius" },
+  { pattern: /shadow|elevation/i, category: "shadow" },
+  { pattern: /size|width|height|min|max/i, category: "sizing" },
+  { pattern: /border(?!-color)|stroke-width|outline/i, category: "border" },
+  { pattern: /animation|transition|duration|timing|delay/i, category: "animation" },
+  { pattern: /z-index|layer|stack/i, category: "z-index" },
+];
+
+/**
+ * Parse a single CSS/SCSS file for tokens
+ */
+export async function parseTokenFile(
+  filePath: string,
+  themeSelectors: Record<string, string> = { ":root": "default" },
+  projectRoot?: string
+): Promise<TokenParseResult> {
+  const startTime = performance.now();
+  const tokens: DesignToken[] = [];
+  const errors: TokenParseError[] = [];
+  const warnings: string[] = [];
+
+  try {
+    const content = await readFile(filePath, "utf-8");
+    const relativePath = projectRoot
+      ? relative(projectRoot, filePath)
+      : filePath;
+
+    // Track which tokens we've seen for reference resolution
+    const tokensByName = new Map<string, { rawValue: string; line?: number }>();
+
+    // First pass: collect all token declarations
+    let lineNumber = 1;
+    const lines = content.split("\n");
+
+    // Track current selector/theme context
+    let currentSelector = ":root";
+    let braceDepth = 0;
+    const selectorStack: string[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      lineNumber = i + 1;
+
+      // Track brace depth for selector context
+      const openBraces = (line.match(/\{/g) || []).length;
+      const closeBraces = (line.match(/\}/g) || []).length;
+
+      // Check for selector at start of line (simple heuristic)
+      if (openBraces > closeBraces) {
+        // Entering a new block
+        const selectorMatch = line.match(/^\s*([^{]+)\s*\{/);
+        if (selectorMatch) {
+          selectorStack.push(selectorMatch[1].trim());
+          currentSelector = selectorStack[selectorStack.length - 1];
+        }
+        braceDepth += openBraces - closeBraces;
+      } else if (closeBraces > openBraces) {
+        braceDepth -= closeBraces - openBraces;
+        if (braceDepth >= 0 && selectorStack.length > 0) {
+          selectorStack.pop();
+          currentSelector = selectorStack.length > 0
+            ? selectorStack[selectorStack.length - 1]
+            : ":root";
+        }
+      }
+
+      // Find token declarations in this line
+      const tokenMatches = [...line.matchAll(TOKEN_DECLARATION_PATTERN)];
+      for (const match of tokenMatches) {
+        const [, name, rawValue] = match;
+        const fullName = `--${name}`;
+        tokensByName.set(fullName, { rawValue: rawValue.trim(), line: lineNumber });
+      }
+    }
+
+    // Second pass: resolve references and build token objects
+    for (const [name, { rawValue, line }] of tokensByName) {
+      // Find the selector context for this token
+      // Re-parse to get correct selector (simplified - uses last found)
+      const selector = findSelectorForLine(content, line || 1);
+      const theme = themeSelectors[selector] || "default";
+
+      // Resolve the value
+      const { resolvedValue, chain, hasCircular, unresolvedRef } = resolveValue(
+        rawValue,
+        tokensByName
+      );
+
+      if (hasCircular) {
+        warnings.push(
+          `Circular reference detected for ${name} at line ${line}`
+        );
+      }
+
+      if (unresolvedRef) {
+        warnings.push(
+          `Unresolved reference in ${name}: ${unresolvedRef}`
+        );
+      }
+
+      // Infer category from name
+      const category = inferCategory(name);
+
+      // Infer token level
+      const level = inferLevel(name, rawValue, chain);
+
+      // Extract description from preceding comment (if any)
+      const description = extractDescription(content, line || 1);
+
+      tokens.push({
+        name,
+        rawValue,
+        resolvedValue,
+        category,
+        level,
+        referenceChain: chain,
+        sourceFile: relativePath,
+        lineNumber: line,
+        theme,
+        selector,
+        description,
+      });
+    }
+  } catch (error) {
+    errors.push({
+      message: error instanceof Error ? error.message : "Unknown error",
+      file: filePath,
+    });
+  }
+
+  return {
+    tokens,
+    errors,
+    warnings,
+    parseTimeMs: performance.now() - startTime,
+  };
+}
+
+/**
+ * Parse multiple files based on config
+ */
+export async function parseTokenFiles(
+  config: TokenConfig,
+  projectRoot: string
+): Promise<TokenParseResult> {
+  const startTime = performance.now();
+  const allTokens: DesignToken[] = [];
+  const allErrors: TokenParseError[] = [];
+  const allWarnings: string[] = [];
+
+  // Discover files
+  const files = await fastGlob(config.include, {
+    cwd: projectRoot,
+    ignore: config.exclude || ["**/node_modules/**"],
+    absolute: true,
+  });
+
+  if (files.length === 0) {
+    allWarnings.push(
+      `No token files found matching: ${config.include.join(", ")}`
+    );
+  }
+
+  // Parse each file
+  for (const file of files) {
+    const result = await parseTokenFile(
+      file,
+      config.themeSelectors,
+      projectRoot
+    );
+
+    allTokens.push(...result.tokens);
+    allErrors.push(...result.errors);
+    allWarnings.push(...result.warnings);
+  }
+
+  return {
+    tokens: allTokens,
+    errors: allErrors,
+    warnings: allWarnings,
+    parseTimeMs: performance.now() - startTime,
+  };
+}
+
+/**
+ * Resolve a token value, following var() references
+ */
+function resolveValue(
+  rawValue: string,
+  tokensByName: Map<string, { rawValue: string; line?: number }>,
+  visited = new Set<string>()
+): {
+  resolvedValue: string;
+  chain: string[];
+  hasCircular: boolean;
+  unresolvedRef?: string;
+} {
+  const chain: string[] = [];
+  let current = rawValue;
+  let hasCircular = false;
+  let unresolvedRef: string | undefined;
+
+  // Maximum iterations to prevent infinite loops
+  const maxIterations = 20;
+  let iterations = 0;
+
+  while (iterations < maxIterations) {
+    iterations++;
+
+    // Check for var() reference
+    const varMatch = current.match(/var\(\s*--([a-zA-Z0-9_-]+)(?:\s*,\s*([^)]+))?\s*\)/);
+
+    if (!varMatch) {
+      // No more var() references, we have the resolved value
+      break;
+    }
+
+    const [, refName, fallback] = varMatch;
+    const fullRefName = `--${refName}`;
+
+    // Check for circular reference
+    if (visited.has(fullRefName)) {
+      hasCircular = true;
+      break;
+    }
+
+    visited.add(fullRefName);
+    chain.push(fullRefName);
+
+    // Look up the referenced token
+    const refToken = tokensByName.get(fullRefName);
+
+    if (refToken) {
+      // Replace var() with the referenced value
+      current = current.replace(
+        varMatch[0],
+        refToken.rawValue
+      );
+    } else if (fallback) {
+      // Use fallback value
+      current = current.replace(varMatch[0], fallback.trim());
+    } else {
+      // Unresolved reference
+      unresolvedRef = fullRefName;
+      break;
+    }
+  }
+
+  return {
+    resolvedValue: normalizeValue(current.trim()),
+    chain,
+    hasCircular,
+    unresolvedRef,
+  };
+}
+
+/**
+ * Normalize a CSS value for consistent comparison
+ */
+function normalizeValue(value: string): string {
+  // Lowercase hex colors
+  value = value.replace(/#[0-9a-fA-F]+/g, (match) => match.toLowerCase());
+
+  // Normalize whitespace
+  value = value.replace(/\s+/g, " ").trim();
+
+  // Normalize rgb/rgba spacing
+  value = value.replace(
+    /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*([\d.]+))?\s*\)/g,
+    (_, r, g, b, a) => a !== undefined ? `rgba(${r}, ${g}, ${b}, ${a})` : `rgb(${r}, ${g}, ${b})`
+  );
+
+  return value;
+}
+
+/**
+ * Infer token category from name
+ */
+function inferCategory(name: string): TokenCategory {
+  const lowerName = name.toLowerCase();
+
+  for (const { pattern, category } of CATEGORY_PATTERNS) {
+    if (pattern.test(lowerName)) {
+      return category;
+    }
+  }
+
+  return "other";
+}
+
+/**
+ * Infer token level (1=base, 2=semantic, 3=component)
+ */
+function inferLevel(
+  name: string,
+  rawValue: string,
+  referenceChain: string[]
+): 1 | 2 | 3 {
+  const lowerName = name.toLowerCase();
+
+  // Component-specific tokens (often contain component names)
+  if (
+    /btn|button|input|card|modal|dialog|menu|nav|header|footer|table|form/i.test(
+      lowerName
+    )
+  ) {
+    return 3;
+  }
+
+  // Semantic tokens (references other tokens)
+  if (referenceChain.length > 0) {
+    return 2;
+  }
+
+  // Base tokens (raw values like hex colors, numbers)
+  if (
+    rawValue.match(/^#[0-9a-fA-F]+$/) ||
+    rawValue.match(/^\d+(\.\d+)?(px|rem|em|%|vh|vw)?$/)
+  ) {
+    return 1;
+  }
+
+  // Default to semantic if unclear
+  return 2;
+}
+
+/**
+ * Find the CSS selector that contains a given line
+ */
+function findSelectorForLine(content: string, targetLine: number): string {
+  const lines = content.split("\n");
+  let currentSelector = ":root";
+  let braceDepth = 0;
+
+  for (let i = 0; i < Math.min(targetLine, lines.length); i++) {
+    const line = lines[i];
+
+    // Check for selector at start of block
+    const selectorMatch = line.match(/^\s*([^{]+)\s*\{/);
+    if (selectorMatch) {
+      const selector = selectorMatch[1].trim();
+      // Only update if entering a new block
+      if ((line.match(/\{/g) || []).length > (line.match(/\}/g) || []).length) {
+        currentSelector = selector;
+      }
+    }
+
+    // Track brace depth
+    braceDepth += (line.match(/\{/g) || []).length;
+    braceDepth -= (line.match(/\}/g) || []).length;
+
+    // If we close all braces, reset to root
+    if (braceDepth === 0) {
+      currentSelector = ":root";
+    }
+  }
+
+  return currentSelector;
+}
+
+/**
+ * Extract description from comment preceding the token
+ */
+function extractDescription(content: string, line: number): string | undefined {
+  const lines = content.split("\n");
+
+  // Look at the line before
+  if (line <= 1) return undefined;
+
+  const prevLine = lines[line - 2]?.trim();
+
+  // Check for single-line comment
+  const singleLineMatch = prevLine?.match(/\/\/\s*(.+)$/);
+  if (singleLineMatch) {
+    return singleLineMatch[1].trim();
+  }
+
+  // Check for multi-line comment ending
+  const multiLineMatch = prevLine?.match(/\*\s*(.+)\s*\*\//);
+  if (multiLineMatch) {
+    return multiLineMatch[1].trim();
+  }
+
+  // Check for simple /* comment */
+  const inlineMatch = prevLine?.match(/\/\*\s*(.+)\s*\*\//);
+  if (inlineMatch) {
+    return inlineMatch[1].trim();
+  }
+
+  return undefined;
+}
+
+/**
+ * Convert a hex color to RGB
+ */
+export function hexToRgb(hex: string): { r: number; g: number; b: number } | null {
+  const match = hex.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
+  if (!match) return null;
+
+  return {
+    r: parseInt(match[1], 16),
+    g: parseInt(match[2], 16),
+    b: parseInt(match[3], 16),
+  };
+}
+
+/**
+ * Convert RGB to hex
+ */
+export function rgbToHex(r: number, g: number, b: number): string {
+  return `#${[r, g, b]
+    .map((x) => Math.round(x).toString(16).padStart(2, "0"))
+    .join("")}`;
+}
+
+/**
+ * Parse an RGB/RGBA string
+ */
+export function parseRgb(
+  color: string
+): { r: number; g: number; b: number; a?: number } | null {
+  const match = color.match(
+    /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*([\d.]+))?\s*\)/
+  );
+  if (!match) return null;
+
+  return {
+    r: parseInt(match[1], 10),
+    g: parseInt(match[2], 10),
+    b: parseInt(match[3], 10),
+    a: match[4] ? parseFloat(match[4]) : undefined,
+  };
+}
+
+/**
+ * Normalize a color value to lowercase hex
+ */
+export function normalizeColor(color: string): string {
+  // Already hex
+  if (color.startsWith("#")) {
+    return color.toLowerCase();
+  }
+
+  // RGB/RGBA to hex
+  const rgb = parseRgb(color);
+  if (rgb) {
+    return rgbToHex(rgb.r, rgb.g, rgb.b);
+  }
+
+  return color.toLowerCase();
+}