@fragments-sdk/core 0.1.0

package/src/token-parser.ts

@@ -0,0 +1,321 @@
+/**
+ * Token Parser — extracts CSS custom property declarations from SCSS/CSS files.
+ *
+ * Parses files for `--prefix-*: value;` declarations and groups them
+ * by SCSS comment sections (e.g., `// Typography`, `// Colors`).
+ * Falls back to naming-convention-based categorization when comments
+ * are absent.
+ */
+
+export interface ParsedToken {
+  /** Full CSS variable name (e.g., "--fui-color-accent") */
+  name: string;
+  /** Raw value from the declaration (e.g., "#{$fui-space-4}" or "16px") */
+  value?: string;
+  /** Resolved value after SCSS variable substitution (e.g., "16px") */
+  resolvedValue?: string;
+  /** Category inferred from SCSS comment or naming convention */
+  category: string;
+  /** Description from inline comment, if any */
+  description?: string;
+}
+
+export interface TokenParseOutput {
+  /** Detected prefix (e.g., "--fui-") */
+  prefix: string;
+  /** Tokens grouped by category */
+  categories: Record<string, ParsedToken[]>;
+  /** Total number of tokens found */
+  total: number;
+}
+
+/**
+ * Category inference from naming conventions.
+ * Order matters — first match wins.
+ */
+const NAMING_RULES: Array<{ pattern: RegExp; category: string }> = [
+  { pattern: /--\w+-font-/, category: 'typography' },
+  { pattern: /--\w+-line-height-/, category: 'typography' },
+  { pattern: /--\w+-space-/, category: 'spacing' },
+  { pattern: /--\w+-padding-/, category: 'spacing' },
+  { pattern: /--\w+-radius-/, category: 'radius' },
+  { pattern: /--\w+-color-/, category: 'colors' },
+  { pattern: /--\w+-bg-/, category: 'surfaces' },
+  { pattern: /--\w+-text-/, category: 'text' },
+  { pattern: /--\w+-border/, category: 'borders' },
+  { pattern: /--\w+-shadow-/, category: 'shadows' },
+  { pattern: /--\w+-focus-/, category: 'focus' },
+  { pattern: /--\w+-transition-/, category: 'transitions' },
+  { pattern: /--\w+-scrollbar-/, category: 'scrollbar' },
+  { pattern: /--\w+-z-index/, category: 'z-index' },
+  { pattern: /--\w+-(button|input|touch)-/, category: 'component-sizing' },
+  { pattern: /--\w+-appshell-/, category: 'layout' },
+  { pattern: /--\w+-header-/, category: 'layout' },
+  { pattern: /--\w+-code-/, category: 'code' },
+  { pattern: /--\w+-tooltip-/, category: 'tooltip' },
+  { pattern: /--\w+-hero-/, category: 'marketing' },
+];
+
+/**
+ * Infer category from a CSS variable name using naming conventions.
+ */
+function inferCategory(name: string): string {
+  for (const rule of NAMING_RULES) {
+    if (rule.pattern.test(name)) {
+      return rule.category;
+    }
+  }
+  return 'other';
+}
+
+/**
+ * Detect the most common prefix from a list of CSS variable names.
+ * E.g., given ["--fui-color-accent", "--fui-bg-primary"] → "--fui-"
+ */
+function detectPrefix(names: string[]): string {
+  if (names.length === 0) return '--';
+
+  // Find common prefix after "--"
+  const stripped = names.map((n) => n.slice(2)); // remove "--"
+  let prefix = '';
+  const first = stripped[0];
+
+  for (let i = 0; i < first.length; i++) {
+    const ch = first[i];
+    if (stripped.every((s) => s[i] === ch)) {
+      prefix += ch;
+    } else {
+      break;
+    }
+  }
+
+  // Trim to last hyphen to get clean prefix
+  const lastHyphen = prefix.lastIndexOf('-');
+  if (lastHyphen > 0) {
+    prefix = prefix.slice(0, lastHyphen + 1);
+  }
+
+  return `--${prefix}`;
+}
+
+/**
+ * Normalize a SCSS comment into a category name.
+ * "// Typography" → "typography"
+ * "// Component heights" → "component-sizing"
+ * "// Hero/Marketing gradient" → "marketing"
+ */
+function normalizeCategory(comment: string): string {
+  const text = comment
+    .trim()
+    .replace(/^\/\/\s*/, '')
+    .replace(/^\/\*+\s*/, '')
+    .replace(/\s*\*+\/$/, '')
+    .trim()
+    .toLowerCase();
+
+  // Map common comment headings to clean category names
+  const mappings: Record<string, string> = {
+    'base configuration': 'base',
+    'typography': 'typography',
+    'spacing (micro)': 'spacing',
+    'spacing': 'spacing',
+    'density padding': 'spacing',
+    'border radius': 'radius',
+    'transitions': 'transitions',
+    'colors': 'colors',
+    'surfaces': 'surfaces',
+    'text': 'text',
+    'borders': 'borders',
+    'shadows': 'shadows',
+    'focus': 'focus',
+    'scrollbar': 'scrollbar',
+    'component heights': 'component-sizing',
+    'appshell layout': 'layout',
+    'codeblock': 'code',
+    'tooltip': 'tooltip',
+    'hero/marketing gradient': 'marketing',
+  };
+
+  return mappings[text] ?? text.replace(/\s+/g, '-');
+}
+
+/**
+ * Extract SCSS variable declarations ($name: value;) from file content.
+ * Returns a map of variable name → value.
+ */
+function extractScssVariables(content: string): Map<string, string> {
+  const vars = new Map<string, string>();
+  // Match: $var-name: value; (handles multi-word values, stops at semicolon)
+  const scssVarRegex = /^\s*(\$[\w-]+)\s*:\s*(.+?)\s*(?:!default\s*)?;/gm;
+
+  let match: RegExpExecArray | null;
+  while ((match = scssVarRegex.exec(content)) !== null) {
+    const name = match[1];
+    const value = match[2].replace(/\s*\/\/.*$/, '').trim();
+    // Only store the first occurrence (canonical definition)
+    if (!vars.has(name)) {
+      vars.set(name, value);
+    }
+  }
+
+  return vars;
+}
+
+/**
+ * Resolve SCSS interpolations and variable references in a token value.
+ *
+ * Handles:
+ * - `#{$var}` → looks up $var in scssVars map
+ * - `$var` standalone → looks up in scssVars map
+ * - `var(--other-token, fallback)` → returns fallback if provided
+ * - Recursive resolution up to 5 levels deep
+ */
+function resolveTokenValue(
+  rawValue: string,
+  scssVars: Map<string, string>,
+  cssVarValues: Map<string, string>,
+  depth = 0
+): string {
+  if (depth > 5) return rawValue; // Prevent infinite recursion
+
+  let resolved = rawValue;
+
+  // Resolve #{$var} interpolations
+  resolved = resolved.replace(/#\{(\$[\w-]+)\}/g, (_, varName) => {
+    const val = scssVars.get(varName);
+    return val !== undefined
+      ? resolveTokenValue(val, scssVars, cssVarValues, depth + 1)
+      : `#{${varName}}`;
+  });
+
+  // Resolve standalone $var references (not inside #{})
+  resolved = resolved.replace(/(?<![#\{])(\$[\w-]+)/g, (_, varName) => {
+    const val = scssVars.get(varName);
+    return val !== undefined
+      ? resolveTokenValue(val, scssVars, cssVarValues, depth + 1)
+      : varName;
+  });
+
+  // Resolve var(--token, fallback) — use the referenced token value or fallback
+  resolved = resolved.replace(
+    /var\((--[\w-]+)(?:\s*,\s*(.+?))?\)/g,
+    (original, tokenName, fallback) => {
+      const tokenVal = cssVarValues.get(tokenName);
+      if (tokenVal !== undefined) {
+        return resolveTokenValue(tokenVal, scssVars, cssVarValues, depth + 1);
+      }
+      if (fallback) {
+        return resolveTokenValue(fallback.trim(), scssVars, cssVarValues, depth + 1);
+      }
+      return original;
+    }
+  );
+
+  return resolved;
+}
+
+/**
+ * Parse a SCSS or CSS file and extract CSS custom property declarations.
+ *
+ * Handles two grouping strategies:
+ * 1. Comment-based: Uses `// Category` comments above groups of declarations
+ * 2. Naming-based: Falls back to inferring category from variable name patterns
+ *
+ * Also resolves SCSS variable interpolations (e.g., `#{$fui-space-4}` → `16px`)
+ * when the SCSS variable definitions are found in the same file content.
+ */
+export function parseTokenFile(content: string, filePath: string): TokenParseOutput {
+  const lines = content.split('\n');
+  const tokens: ParsedToken[] = [];
+  const seenNames = new Set<string>();
+  let currentCategory = 'other';
+  let hasCommentCategories = false;
+
+  // First pass: extract SCSS variable declarations for resolution
+  const scssVars = extractScssVariables(content);
+
+  // Regex for CSS custom property declarations
+  // Matches: --name: value; (with optional SCSS interpolation)
+  // Captures both the variable name and its value
+  const varDeclRegex = /^\s*(--[\w-]+)\s*:\s*(.+?)\s*;/;
+  // Regex for section comments (// Category or /* Category */)
+  // Allow any characters after uppercase start (including / for "Hero/Marketing")
+  const sectionCommentRegex = /^\s*\/\/\s+([A-Z].+)$/;
+
+  for (const line of lines) {
+    // Check for section comment
+    const commentMatch = line.match(sectionCommentRegex);
+    if (commentMatch) {
+      const normalized = normalizeCategory(commentMatch[0]);
+      if (normalized) {
+        currentCategory = normalized;
+        hasCommentCategories = true;
+      }
+      continue;
+    }
+
+    // Check for CSS variable declaration
+    const varMatch = line.match(varDeclRegex);
+    if (varMatch) {
+      const name = varMatch[1];
+      const rawValue = varMatch[2];
+
+      // Deduplicate: keep only the first occurrence of each variable.
+      // Dark mode and high contrast blocks redefine the same variables
+      // with different values — we only want the canonical list.
+      if (seenNames.has(name)) continue;
+      seenNames.add(name);
+
+      // Extract inline comment if present
+      const inlineComment = line.match(/\/\/\s*(.+)$/);
+      const description = inlineComment ? inlineComment[1].trim() : undefined;
+
+      // Clean the value: strip trailing inline comments
+      const cleanValue = rawValue.replace(/\s*\/\/.*$/, '').trim();
+
+      tokens.push({
+        name,
+        value: cleanValue || undefined,
+        category: hasCommentCategories ? currentCategory : inferCategory(name),
+        description,
+      });
+    }
+  }
+
+  // Second pass: build a CSS custom property → raw value map for cross-references
+  const cssVarValues = new Map<string, string>();
+  for (const token of tokens) {
+    if (token.value) {
+      cssVarValues.set(token.name, token.value);
+    }
+  }
+
+  // Third pass: resolve SCSS interpolations and var() references
+  for (const token of tokens) {
+    if (token.value) {
+      const resolved = resolveTokenValue(token.value, scssVars, cssVarValues);
+      // Only set resolvedValue if it's different from raw and doesn't still contain unresolved refs
+      if (resolved !== token.value && !resolved.includes('#{') && !resolved.includes('$')) {
+        token.resolvedValue = resolved;
+      }
+    }
+  }
+
+  // Group by category
+  const categories: Record<string, ParsedToken[]> = {};
+  for (const token of tokens) {
+    if (!categories[token.category]) {
+      categories[token.category] = [];
+    }
+    categories[token.category].push(token);
+  }
+
+  // Detect prefix
+  const prefix = detectPrefix(tokens.map((t) => t.name));
+
+  return {
+    prefix,
+    categories,
+    total: tokens.length,
+  };
+}

package/src/token-types.ts

@@ -0,0 +1,287 @@
+/**
+ * Design Token Types for Fragments
+ *
+ * These types define the structure for CSS custom property (CSS variable) discovery,
+ * parsing, and reverse lookup capabilities. The token system enables:
+ *
+ * 1. Automatic discovery of design tokens from CSS/SCSS files
+ * 2. Reverse lookup: given a computed value, find which token(s) produce it
+ * 3. Detection of hardcoded values vs token usage
+ * 4. AI-friendly fix suggestions
+ */
+
+/**
+ * Token categories for grouping and filtering
+ */
+export type TokenCategory =
+  | "color"
+  | "spacing"
+  | "typography"
+  | "radius"
+  | "shadow"
+  | "sizing"
+  | "border"
+  | "animation"
+  | "z-index"
+  | "other";
+
+/**
+ * A single design token (CSS custom property)
+ */
+export interface DesignToken {
+  /** Token name with leading dashes (e.g., "--color-primary") */
+  name: string;
+
+  /** Raw value as written in CSS (e.g., "var(--color-cobalt-50)") */
+  rawValue: string;
+
+  /** Fully resolved value (e.g., "#0051c2") */
+  resolvedValue: string;
+
+  /** Inferred category based on naming convention */
+  category: TokenCategory;
+
+  /**
+   * Token level in the design system hierarchy:
+   * - 1 = Base/primitive tokens (raw values like colors, sizes)
+   * - 2 = Semantic tokens (references to base tokens with meaning)
+   * - 3 = Component tokens (component-specific tokens)
+   */
+  level: 1 | 2 | 3;
+
+  /**
+   * Reference chain showing how the value was resolved
+   * e.g., ["--color-primary", "--color-cobalt-50"] means
+   * --color-primary references --color-cobalt-50
+   */
+  referenceChain: string[];
+
+  /** Source file where this token was defined */
+  sourceFile: string;
+
+  /** Line number in source file */
+  lineNumber?: number;
+
+  /** Theme this token belongs to (e.g., "default", "dark", "light") */
+  theme: string;
+
+  /** CSS selector where this token is defined (e.g., ":root", "[data-theme='dark']") */
+  selector: string;
+
+  /** Optional description from comments */
+  description?: string;
+}
+
+/**
+ * Token registry for fast lookups
+ */
+export interface TokenRegistry {
+  /** Lookup by token name (e.g., "--color-primary") */
+  byName: Map<string, DesignToken>;
+
+  /**
+   * REVERSE lookup: resolved value -> token names
+   * Key is normalized value (e.g., "#0051c2" lowercase)
+   * Value is array of token names that resolve to this value
+   */
+  byValue: Map<string, string[]>;
+
+  /** Tokens grouped by theme */
+  byTheme: Map<string, DesignToken[]>;
+
+  /** Tokens grouped by category */
+  byCategory: Map<TokenCategory, DesignToken[]>;
+
+  /** Registry metadata */
+  meta: TokenRegistryMeta;
+}
+
+/**
+ * Token registry metadata
+ */
+export interface TokenRegistryMeta {
+  /** When tokens were discovered */
+  discoveredAt: Date;
+
+  /** Source files that were parsed */
+  sourceFiles: string[];
+
+  /** Total number of tokens discovered */
+  totalTokens: number;
+
+  /** Time taken to parse (ms) */
+  parseTimeMs: number;
+
+  /** Number of circular references detected */
+  circularRefs: number;
+
+  /** Number of unresolved references */
+  unresolvedRefs: number;
+}
+
+/**
+ * Enhanced style diff item with token information
+ */
+export interface EnhancedStyleDiffItem {
+  /** CSS property name (e.g., "backgroundColor") */
+  property: string;
+
+  /** Value from Figma design */
+  figma: string;
+
+  /** Value from rendered component */
+  rendered: string;
+
+  /** Whether values match (within tolerance) */
+  match: boolean;
+
+  /** Token name if Figma value matches a known token */
+  figmaToken?: string;
+
+  /** Token name if rendered value uses a token */
+  renderedToken?: string;
+
+  /**
+   * True if rendered value doesn't use a token but should
+   * (i.e., Figma uses a token but code uses hardcoded value)
+   */
+  isHardcoded: boolean;
+
+  /** Suggested fix if hardcoded */
+  suggestedFix?: TokenFix;
+}
+
+/**
+ * Token-based fix suggestion
+ */
+export interface TokenFix {
+  /** Token name to use (e.g., "--color-primary") */
+  tokenName: string;
+
+  /** Token's resolved value */
+  tokenValue: string;
+
+  /** Code snippet to fix the issue */
+  codeFix: string;
+
+  /** Confidence score 0-1 */
+  confidence: number;
+
+  /** Human-readable explanation */
+  reason: string;
+}
+
+/**
+ * Configuration for token discovery
+ */
+export interface TokenConfig {
+  /**
+   * Glob patterns for files to scan for tokens
+   * e.g., ["src/styles/theme.scss", "src/styles/variables.css"]
+   */
+  include: string[];
+
+  /**
+   * Glob patterns to exclude
+   * @example ["node_modules"]
+   */
+  exclude?: string[];
+
+  /**
+   * Map CSS selectors to theme names
+   * @example { ":root": "default", "[data-theme='dark']": "dark" }
+   */
+  themeSelectors?: Record<string, string>;
+
+  /** Enable token comparison in style diffs (default: true) */
+  enabled?: boolean;
+}
+
+/**
+ * Result of parsing a CSS/SCSS file for tokens
+ */
+export interface TokenParseResult {
+  /** Tokens discovered in the file */
+  tokens: DesignToken[];
+
+  /** Errors encountered during parsing */
+  errors: TokenParseError[];
+
+  /** Warnings (non-fatal issues) */
+  warnings: string[];
+
+  /** Parse time in ms */
+  parseTimeMs: number;
+}
+
+/**
+ * Error during token parsing
+ */
+export interface TokenParseError {
+  /** Error message */
+  message: string;
+
+  /** File where error occurred */
+  file: string;
+
+  /** Line number if known */
+  line?: number;
+
+  /** The problematic content if available */
+  content?: string;
+}
+
+/**
+ * Request to match a value to tokens
+ */
+export interface TokenMatchRequest {
+  /** The value to find tokens for (e.g., "#0051c2") */
+  value: string;
+
+  /** Property type hint for better matching (e.g., "color") */
+  propertyType?: "color" | "spacing" | "typography" | "other";
+
+  /** Specific theme to search in */
+  theme?: string;
+}
+
+/**
+ * Result of token matching
+ */
+export interface TokenMatchResult {
+  /** Exact matches (same resolved value) */
+  exactMatches: DesignToken[];
+
+  /** Close matches (similar value, useful for colors) */
+  closeMatches: Array<{
+    token: DesignToken;
+    /** How close the match is (0-1, 1 = exact) */
+    similarity: number;
+  }>;
+
+  /** Whether any match was found */
+  found: boolean;
+}
+
+/**
+ * Summary of token usage in a component
+ */
+export interface TokenUsageSummary {
+  /** Total CSS properties checked */
+  totalProperties: number;
+
+  /** Properties using design tokens */
+  usingTokens: number;
+
+  /** Properties with hardcoded values */
+  hardcoded: number;
+
+  /** Properties matching but not using tokens explicitly */
+  implicitMatches: number;
+
+  /** Compliance percentage (usingTokens / totalProperties * 100) */
+  compliancePercent: number;
+
+  /** List of hardcoded properties with fix suggestions */
+  hardcodedProperties: EnhancedStyleDiffItem[];
+}