@usefragments/core 1.0.0

package/src/tokens/design-token-parser.test.ts

@@ -0,0 +1,159 @@
+import { describe, expect, it } from "vitest";
+
+import { parseDesignTokenContent } from "./design-token-parser.js";
+
+/**
+ * Token-drift robustness for the SINGLE producer the CLI's
+ * buildContractVocabularyFacts consumes (parseDesignTokenContent). Each format
+ * that previously yielded an EMPTY vocabulary (→ FUI2015 silently disabled) is
+ * locked here, plus a NON-SILENT 0-token warning so an empty result is never
+ * quiet.
+ */
+
+function names(content: string, filePath: string): string[] {
+  return parseDesignTokenContent(content, { filePath }).tokens.map((t) => t.name);
+}
+
+describe("parseDesignTokenContent — DTCG/JSON format dispatch (#10)", () => {
+  it("parses a DTCG token file into --kebab CSS names", () => {
+    const content = JSON.stringify({
+      color: { accent: { $value: "#fff", $type: "color" } },
+    });
+    const tokens = names(content, "tokens.tokens.json");
+    expect(tokens).toContain("--color-accent");
+  });
+
+  it("preserves DTCG token type/category before name heuristics", () => {
+    const content = JSON.stringify({
+      brand: {
+        primary: { $value: "#39594d", $type: "color" },
+        corner: { $value: "8px", $type: "dimension" },
+      },
+    });
+    const res = parseDesignTokenContent(content, { filePath: "tokens.tokens.json" });
+
+    expect(res.tokens.find((token) => token.name === "--brand-primary")?.category).toBe("color");
+    expect(res.tokens.find((token) => token.name === "--brand-corner")?.category).toBe("spacing");
+  });
+
+  it("parses DTCG even when the path is generic but content is JSON", () => {
+    // `looksDtcg` also triggers on `{`-leading content, so an unknown extension
+    // with JSON content still routes to the DTCG parser.
+    const content = JSON.stringify({
+      space: { $type: "dimension", small: { $value: "8px" } },
+    });
+    const tokens = names(content, "weird-name.config");
+    expect(tokens).toContain("--space-small");
+  });
+
+  it("surfaces a NON-SILENT warning for an empty DTCG file (#10)", () => {
+    const res = parseDesignTokenContent("{}", { filePath: "x.tokens.json" });
+    expect(res.tokens).toHaveLength(0);
+    expect(res.warnings).toHaveLength(1);
+    expect(res.warnings[0]).toMatch(/produced no token names/);
+  });
+
+  it("fails soft on malformed JSON instead of throwing", () => {
+    // Malformed `.json`: must NOT throw, must fall through to the CSS path and
+    // still emit a non-silent warning (no tokens found anywhere).
+    const res = parseDesignTokenContent("{ not valid json", {
+      filePath: "broken.tokens.json",
+    });
+    expect(res.tokens).toHaveLength(0);
+    expect(res.warnings.some((w) => /produced no token names/.test(w))).toBe(true);
+  });
+});
+
+describe("parseDesignTokenContent — SCSS $-variable format dispatch (#32)", () => {
+  it("parses SCSS $vars, normalizing $name to --name", () => {
+    const content = "$fui-color-accent: #fff;\n$fui-space-2: 8px;";
+    const tokens = names(content, "tokens.scss");
+    expect(tokens).toContain("--fui-color-accent");
+    expect(tokens).toContain("--fui-space-2");
+  });
+
+  it("dedupes a file declaring BOTH --fui-x and $fui-x to one --fui-x", () => {
+    const content = ":root {\n  --fui-x: red;\n}\n$fui-x: red;";
+    const tokens = names(content, "tokens.scss");
+    expect(tokens.filter((n) => n === "--fui-x")).toHaveLength(1);
+  });
+
+  it("surfaces a NON-SILENT warning for a comment-only SCSS file (#32)", () => {
+    const res = parseDesignTokenContent("// just a comment\n", {
+      filePath: "empty.scss",
+    });
+    expect(res.tokens).toHaveLength(0);
+    expect(res.warnings).toHaveLength(1);
+    expect(res.warnings[0]).toMatch(/produced no token names/);
+  });
+
+  it("routes $-only content to SCSS even without a .scss extension", () => {
+    // No `--name:` declarations + has `$name:` declarations → SCSS branch.
+    const content = "$accent: #fff;";
+    const tokens = names(content, "_vars");
+    expect(tokens).toContain("--accent");
+  });
+});
+
+describe("parseDesignTokenContent — CSS terminator robustness (#34/#35)", () => {
+  it("captures a final-in-block declaration with NO trailing semicolon (#34)", () => {
+    const content = ":root {\n  --fui-color-accent: #fff\n}";
+    const tokens = names(content, "tokens.css");
+    expect(tokens).toContain("--fui-color-accent");
+  });
+
+  it("captures both declarations in a block where the last has no semicolon (#34)", () => {
+    const content = ":root { --a: #000; --b: #fff }";
+    const tokens = names(content, "tokens.css");
+    expect(tokens).toContain("--a");
+    expect(tokens).toContain("--b");
+  });
+
+  it("captures a MULTI-LINE declaration value (#35)", () => {
+    const content = ":root {\n  --fui-shadow: 0 1px 2px\n    rgba(0,0,0,.1);\n}";
+    const res = parseDesignTokenContent(content, { filePath: "tokens.css" });
+    const shadow = res.tokens.find((t) => t.name === "--fui-shadow");
+    expect(shadow).toBeDefined();
+    // Line attribution survives the index-based recompute (declaration starts L2).
+    expect(shadow?.lineNumber).toBe(2);
+    // Internal whitespace (incl. the newline) is collapsed.
+    expect(shadow?.rawValue).toBe("0 1px 2px rgba(0,0,0,.1)");
+  });
+
+  it("captures a multi-line value that is also the final unterminated decl (#34+#35)", () => {
+    const content = ":root {\n  --fui-shadow: 0 1px 2px\n    rgba(0,0,0,.1)\n}";
+    const tokens = names(content, "tokens.css");
+    expect(tokens).toContain("--fui-shadow");
+  });
+
+  it("uses value shape before name heuristics for CSS custom properties", () => {
+    const res = parseDesignTokenContent(":root { --brand-primary: #39594d; }", {
+      filePath: "tokens.css",
+    });
+
+    expect(res.tokens.find((token) => token.name === "--brand-primary")?.category).toBe("color");
+  });
+
+  it("surfaces a NON-SILENT warning for a CSS file with no token names", () => {
+    const res = parseDesignTokenContent(".foo { color: red; }", {
+      filePath: "styles.css",
+    });
+    expect(res.tokens).toHaveLength(0);
+    expect(res.warnings.some((w) => /produced no token names/.test(w))).toBe(true);
+  });
+
+  it("still parses a normal CSS token block (regression)", () => {
+    const content = ":root {\n  --fui-color-accent: #ff0000;\n  --fui-space-2: 8px;\n}";
+    const tokens = names(content, "tokens.css");
+    expect(tokens).toContain("--fui-color-accent");
+    expect(tokens).toContain("--fui-space-2");
+  });
+
+  it("categorizes abbreviated --sp-* tokens as spacing", () => {
+    const content = ":root {\n  --sp-1: 4px;\n  --fui-sp-2: 8px;\n}";
+    const res = parseDesignTokenContent(content, { filePath: "tokens.css" });
+
+    expect(res.tokens.find((token) => token.name === "--sp-1")?.category).toBe("spacing");
+    expect(res.tokens.find((token) => token.name === "--fui-sp-2")?.category).toBe("spacing");
+  });
+});

package/src/tokens/design-token-parser.ts

@@ -0,0 +1,402 @@
+import type {
+  DesignToken,
+  TokenCategory,
+  TokenParseError,
+  TokenParseResult,
+} from "../token-types.js";
+import { inferTokenCategory } from "./categories.js";
+import { isDTCGFile, parseDtcgTokens } from "./dtcg.js";
+import { parseScssTokens } from "./scss.js";
+import { containsTailwindV4Theme, parseTailwindV4Theme } from "./tailwind-v4.js";
+import type { DesignTokenParseOptions, ParsedToken, TokenParseOutput } from "./types.js";
+
+// Token declaration value class excludes `;` (the normal terminator), and `{`/`}`
+// so the final declaration in a block (no trailing `;` — valid CSS, #34) and a
+// value that wraps across newlines (#35) are both captured without swallowing a
+// following block. Matched over the FULL content string (not per-line) so the
+// `[^;{}]+?` class can span `\n`. Terminates on `;`, the next `}`, or EOF.
+const TOKEN_DECLARATION_PATTERN = /--([a-zA-Z0-9_-]+)\s*:\s*([^;{}]+?)\s*(?:;|(?=\})|$)/g;
+const VAR_REFERENCE_PATTERN = /var\(\s*--([a-zA-Z0-9_-]+)(?:\s*,\s*([^)]+))?\s*\)/;
+
+export function parseDesignTokenContent(
+  content: string,
+  options: DesignTokenParseOptions = {}
+): TokenParseResult {
+  const startTime = performance.now();
+  const tokens: DesignToken[] = [];
+  const errors: TokenParseError[] = [];
+  const warnings: string[] = [];
+  const filePath = options.filePath ?? "unknown";
+  const themeSelectors = options.themeSelectors ?? { ":root": "default" };
+  const includeTailwindV4 = options.includeTailwindV4 ?? true;
+
+  // Format dispatch (single chokepoint for the CLI's buildContractVocabularyFacts).
+  // DTCG/JSON and SCSS files produce NO `--name:` CSS declarations, so the CSS
+  // line-loop below yields an empty vocabulary and FUI2015 silently disables — the
+  // worst failure mode (#10 DTCG, #32 SCSS). Route those through the sibling
+  // parsers (both already emit `--kebab` CSS names the rule consumes unchanged),
+  // each returning early. A 0-token result surfaces a NON-SILENT warning rather
+  // than a quietly-empty vocabulary. The CSS path (with the #34/#35 robustness
+  // fixes) is the fallthrough default.
+  const trimmed = content.trimStart();
+  const looksDtcg =
+    isDTCGFile(filePath) || /\.jsonc?$|\.tokens$/i.test(filePath) || trimmed.startsWith("{");
+  if (looksDtcg) {
+    try {
+      // Fail-soft per the architecture spine: a malformed .json must never throw
+      // or break the scan — fall through to the CSS path on any parse error.
+      const out = parseDtcgTokens(content, filePath);
+      const dtTokens = parsedOutputToDesignTokens(out, filePath);
+      const dtcgWarnings =
+        dtTokens.length === 0
+          ? [
+              `Token source file ${filePath}: DTCG/JSON file produced no token names; FUI2015 token-drift is inactive for it.`,
+            ]
+          : [];
+      return {
+        tokens: dtTokens,
+        errors: [],
+        warnings: dtcgWarnings,
+        parseTimeMs: performance.now() - startTime,
+      };
+    } catch {
+      // Fall through to the CSS path; if nothing matches there, the 0-token
+      // warning at the end of the CSS path still fires (non-silent).
+    }
+  }
+
+  const isScss = /\.(scss|sass)$/i.test(filePath);
+  const hasCssVar = /--[\w-]+\s*:/.test(content);
+  const hasScssVar = /(^|\n)\s*\$[\w-]+\s*:/.test(content);
+  if (isScss || (!hasCssVar && hasScssVar)) {
+    // parseScssTokens emits BOTH `--css-var` and `$scss-var` names. Normalize
+    // every `$name` to `--name`: the mandated dual-fallback convention
+    // `var(--fui-x, $fui-x)` guarantees a 1:1 `$`↔`--` correspondence, and `--x`
+    // is exactly the name the rule judges (it reads `var(--x)` references).
+    const out = parseScssTokens(content, filePath);
+    const scssTokens = scssOutputToDesignTokens(out, filePath);
+    const scssWarnings =
+      scssTokens.length === 0
+        ? [
+            `Token source file ${filePath}: SCSS file produced no token names; FUI2015 token-drift is inactive for it.`,
+          ]
+        : [];
+    return {
+      tokens: scssTokens,
+      errors: [],
+      warnings: scssWarnings,
+      parseTimeMs: performance.now() - startTime,
+    };
+  }
+
+  if (includeTailwindV4 && containsTailwindV4Theme(content)) {
+    const v4Result = parseTailwindV4Theme(content, filePath);
+    tokens.push(...v4Result.tokens);
+    warnings.push(...v4Result.warnings);
+    for (const err of v4Result.errors) {
+      errors.push({ message: err, file: filePath });
+    }
+  }
+
+  const tokensByName = new Map<string, { rawValue: string; line?: number }>();
+
+  // Match over the FULL content so a value spanning newlines (#35) is captured.
+  // The line number is recomputed from the match index because findSelectorForLine
+  // and extractDescription still attribute by line. Internal whitespace (incl.
+  // newlines) is collapsed so a multi-line value stores cleanly.
+  for (const match of content.matchAll(TOKEN_DECLARATION_PATTERN)) {
+    const [, name, rawValue] = match;
+    const fullName = `--${name}`;
+    const lineNumber = countNewlines(content, match.index ?? 0) + 1;
+    tokensByName.set(fullName, {
+      rawValue: rawValue.trim().replace(/\s+/g, " "),
+      line: lineNumber,
+    });
+  }
+
+  for (const [name, { rawValue, line }] of tokensByName) {
+    const selector = findSelectorForLine(content, line ?? 1);
+    const theme = themeSelectors[selector] || "default";
+    const { resolvedValue, chain, hasCircular, unresolvedRef } = resolveDesignTokenValue(
+      rawValue,
+      tokensByName
+    );
+
+    if (hasCircular) {
+      warnings.push(`Circular reference detected for ${name} at line ${line}`);
+    }
+
+    if (unresolvedRef) {
+      warnings.push(`Unresolved reference in ${name}: ${unresolvedRef}`);
+    }
+
+    tokens.push({
+      name,
+      rawValue,
+      resolvedValue,
+      category: inferTokenCategory(name, resolvedValue || rawValue),
+      level: inferTokenLevel(name, rawValue, chain),
+      referenceChain: chain,
+      sourceFile: filePath,
+      lineNumber: line,
+      theme,
+      selector,
+      description: extractDescription(content, line ?? 1),
+    });
+  }
+
+  // NON-SILENT invariant: a non-empty CSS file that yields no token names means
+  // FUI2015 has no vocabulary from it — surface a warning instead of a quietly
+  // empty result (mirrors the DTCG/SCSS 0-token warnings above).
+  if (tokens.length === 0 && trimmed.length > 0) {
+    warnings.push(
+      `Token source file ${filePath}: CSS file produced no token names; FUI2015 token-drift is inactive for it.`
+    );
+  }
+
+  return {
+    tokens,
+    errors,
+    warnings,
+    parseTimeMs: performance.now() - startTime,
+  };
+}
+
+/** Count the `\n` characters in `content` up to (not including) `index`. */
+function countNewlines(content: string, index: number): number {
+  let count = 0;
+  for (let i = 0; i < index && i < content.length; i++) {
+    if (content.charCodeAt(i) === 10) count++;
+  }
+  return count;
+}
+
+/**
+ * Flatten a DTCG/JSON `TokenParseOutput` into `DesignToken[]`. Names pass through
+ * verbatim (parseDtcgTokens already emits `--kebab` CSS names via
+ * tokenPathToCSSName). Determinism: walkTokenTree preserves JSON object key order.
+ */
+function parsedOutputToDesignTokens(output: TokenParseOutput, filePath: string): DesignToken[] {
+  const tokens: DesignToken[] = [];
+  for (const cat of Object.values(output.categories)) {
+    for (const t of cat) {
+      tokens.push(designTokenFromParsed(t.name, t, filePath));
+    }
+  }
+  return tokens;
+}
+
+/**
+ * Flatten an SCSS `TokenParseOutput` into `DesignToken[]`, normalizing every
+ * `$name` to `--name` (the mandated dual-fallback `var(--fui-x, $fui-x)` makes
+ * this 1:1 and deterministic) and de-duping so a file declaring both `--fui-x`
+ * and `$fui-x` yields one `--fui-x`.
+ */
+function scssOutputToDesignTokens(output: TokenParseOutput, filePath: string): DesignToken[] {
+  const names = new Set<string>();
+  const tokens: DesignToken[] = [];
+  for (const cat of Object.values(output.categories)) {
+    for (const t of cat) {
+      const cssName = t.name.startsWith("$") ? `--${t.name.slice(1)}` : t.name;
+      if (!cssName.startsWith("--") || names.has(cssName)) continue;
+      names.add(cssName);
+      tokens.push(designTokenFromParsed(cssName, t, filePath));
+    }
+  }
+  return tokens;
+}
+
+/** Build a `DesignToken` from a sibling-parser `ParsedToken` under a CSS name. */
+function designTokenFromParsed(name: string, parsed: ParsedToken, filePath: string): DesignToken {
+  return {
+    name,
+    rawValue: parsed.value ?? "",
+    resolvedValue: parsed.resolvedValue ?? parsed.value ?? "",
+    category: normalizeParsedTokenCategory(
+      parsed.category,
+      name,
+      parsed.resolvedValue ?? parsed.value ?? ""
+    ),
+    level: 1,
+    referenceChain: [],
+    sourceFile: filePath,
+    theme: "default",
+    selector: ":root",
+    description: parsed.description,
+  };
+}
+
+function normalizeParsedTokenCategory(
+  category: string | undefined,
+  name: string,
+  value: string
+): TokenCategory {
+  switch (category?.toLowerCase()) {
+    case "colors":
+    case "color":
+    case "surfaces":
+    case "surface":
+    case "text":
+      return "color";
+    case "spacing":
+    case "space":
+      return "spacing";
+    case "typography":
+    case "font":
+      return "typography";
+    case "radius":
+    case "radii":
+      return "radius";
+    case "shadows":
+    case "shadow":
+      return "shadow";
+    case "borders":
+    case "border":
+      return "border";
+    case "animation":
+    case "animations":
+    case "transitions":
+    case "transition":
+      return "animation";
+    case "sizing":
+    case "size":
+      return "sizing";
+    case "z-index":
+    case "zindex":
+      return "z-index";
+    default:
+      return inferTokenCategory(name, value);
+  }
+}
+
+export function resolveDesignTokenValue(
+  rawValue: string,
+  tokensByName: Map<string, { rawValue: string; line?: number }>,
+  visited = new Set<string>()
+): {
+  resolvedValue: string;
+  chain: string[];
+  hasCircular: boolean;
+  unresolvedRef?: string;
+} {
+  const chain: string[] = [];
+  let current = rawValue;
+  let hasCircular = false;
+  let unresolvedRef: string | undefined;
+
+  const maxIterations = 20;
+  let iterations = 0;
+
+  while (iterations < maxIterations) {
+    iterations++;
+
+    const varMatch = current.match(VAR_REFERENCE_PATTERN);
+    if (!varMatch) break;
+
+    const [, refName, fallback] = varMatch;
+    const fullRefName = `--${refName}`;
+
+    if (visited.has(fullRefName)) {
+      hasCircular = true;
+      break;
+    }
+
+    visited.add(fullRefName);
+    chain.push(fullRefName);
+
+    const refToken = tokensByName.get(fullRefName);
+    if (refToken) {
+      current = current.replace(varMatch[0], refToken.rawValue);
+    } else if (fallback) {
+      current = current.replace(varMatch[0], fallback.trim());
+    } else {
+      unresolvedRef = fullRefName;
+      break;
+    }
+  }
+
+  return {
+    resolvedValue: normalizeTokenValue(current.trim()),
+    chain,
+    hasCircular,
+    unresolvedRef,
+  };
+}
+
+export function normalizeTokenValue(value: string): string {
+  value = value.replace(/#[0-9a-fA-F]+/g, (match) => match.toLowerCase());
+  value = value.replace(/\s+/g, " ").trim();
+  value = value.replace(
+    /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*([\d.]+))?\s*\)/g,
+    (_, r, g, b, a) => (a !== undefined ? `rgba(${r}, ${g}, ${b}, ${a})` : `rgb(${r}, ${g}, ${b})`)
+  );
+
+  return value;
+}
+
+function inferTokenLevel(name: string, rawValue: string, referenceChain: string[]): 1 | 2 | 3 {
+  const lowerName = name.toLowerCase();
+
+  if (/btn|button|input|card|modal|dialog|menu|nav|header|footer|table|form/i.test(lowerName)) {
+    return 3;
+  }
+
+  if (referenceChain.length > 0) {
+    return 2;
+  }
+
+  if (rawValue.match(/^#[0-9a-fA-F]+$/) || rawValue.match(/^\d+(\.\d+)?(px|rem|em|%|vh|vw)?$/)) {
+    return 1;
+  }
+
+  return 2;
+}
+
+function findSelectorForLine(content: string, targetLine: number): string {
+  const lines = content.split("\n");
+  let currentSelector = ":root";
+  let braceDepth = 0;
+
+  for (let i = 0; i < Math.min(targetLine, lines.length); i++) {
+    const line = lines[i];
+    const selectorMatch = line.match(/^\s*([^{]+)\s*\{/);
+    if (selectorMatch) {
+      const selector = selectorMatch[1].trim();
+      if ((line.match(/\{/g) || []).length > (line.match(/\}/g) || []).length) {
+        currentSelector = selector;
+      }
+    }
+
+    braceDepth += (line.match(/\{/g) || []).length;
+    braceDepth -= (line.match(/\}/g) || []).length;
+
+    if (braceDepth === 0) {
+      currentSelector = ":root";
+    }
+  }
+
+  return currentSelector;
+}
+
+function extractDescription(content: string, line: number): string | undefined {
+  const lines = content.split("\n");
+  if (line <= 1) return undefined;
+
+  const prevLine = lines[line - 2]?.trim();
+  const singleLineMatch = prevLine?.match(/\/\/\s*(.+)$/);
+  if (singleLineMatch) {
+    return singleLineMatch[1].trim();
+  }
+
+  const multiLineMatch = prevLine?.match(/\*\s*(.+)\s*\*\//);
+  if (multiLineMatch) {
+    return multiLineMatch[1].trim();
+  }
+
+  const inlineMatch = prevLine?.match(/\/\*\s*(.+)\s*\*\//);
+  if (inlineMatch) {
+    return inlineMatch[1].trim();
+  }
+
+  return undefined;
+}