@usefragments/core 1.5.2 → 1.7.0

package/src/tokens/design-token-parser.test.ts

@@ -175,4 +175,135 @@ describe("parseDesignTokenContent — CSS terminator robustness (#34/#35)", () =
     expect(res.tokens.find((token) => token.name === "--sp-1")?.category).toBe("spacing");
     expect(res.tokens.find((token) => token.name === "--fui-sp-2")?.category).toBe("spacing");
   });
+
+  it("preserves the same custom property across configured theme selectors", () => {
+    const content = [
+      ":root { --surface: #ffffff; --text: #111111; }",
+      '[data-theme="dark"] { --surface: #111111; --text: var(--surface); }',
+      ".high-contrast { --surface: #000000; }",
+    ].join("\n");
+    const res = parseDesignTokenContent(content, {
+      filePath: "tokens.css",
+      themeSelectors: {
+        ":root": "default",
+        '[data-theme="dark"]': "dark",
+        ".high-contrast": "high-contrast",
+      },
+    });
+
+    expect(
+      res.tokens
+        .filter((token) => token.name === "--surface")
+        .map((token) => [token.theme, token.resolvedValue, token.selector])
+    ).toEqual([
+      ["default", "#ffffff", ":root"],
+      ["dark", "#111111", '[data-theme="dark"]'],
+      ["high-contrast", "#000000", ".high-contrast"],
+    ]);
+    expect(
+      res.tokens.find((token) => token.name === "--text" && token.theme === "dark")?.resolvedValue
+    ).toBe("#111111");
+  });
+
+  it("preserves selector-specific values in compact single-line theme CSS", () => {
+    const content =
+      ':root{--surface:#ffffff;--text:#111111}[data-theme="dark"]{--surface:#111111;--text:var(--surface)}.high-contrast{--surface:#000000}';
+    const res = parseDesignTokenContent(content, {
+      filePath: "tokens.min.css",
+      themeSelectors: {
+        ":root": "default",
+        '[data-theme="dark"]': "dark",
+        ".high-contrast": "high-contrast",
+      },
+    });
+
+    expect(
+      res.tokens
+        .filter((token) => token.name === "--surface")
+        .map((token) => [token.theme, token.resolvedValue, token.selector])
+    ).toEqual([
+      ["default", "#ffffff", ":root"],
+      ["dark", "#111111", '[data-theme="dark"]'],
+      ["high-contrast", "#000000", ".high-contrast"],
+    ]);
+    expect(
+      res.tokens.find((token) => token.name === "--text" && token.theme === "dark")?.resolvedValue
+    ).toBe("#111111");
+  });
+
+  it("uses the nearest selector through compact at-rules and ignores structural text in comments", () => {
+    const content =
+      ':root{/* fake } [data-theme="dark"] { */--surface:#fff}@media(prefers-color-scheme:dark){[data-theme="dark"]{--surface:#111}}';
+    const res = parseDesignTokenContent(content, {
+      filePath: "tokens.min.css",
+      themeSelectors: {
+        ":root": "default",
+        '[data-theme="dark"]': "dark",
+      },
+    });
+
+    expect(
+      res.tokens
+        .filter((token) => token.name === "--surface")
+        .map((token) => [token.theme, token.resolvedValue, token.selector])
+    ).toEqual([
+      ["default", "#fff", ":root"],
+      ["dark", "#111", '[data-theme="dark"]'],
+    ]);
+  });
+
+  it("ignores declaration-shaped text in comments and strings while preserving quoted delimiters", () => {
+    const content =
+      ':root{/* --comment-token:red; } */--label:"literal; {  value }";--image:url("data:image/svg+xml;utf8,<svg>{}</svg>");--raw-image:url(data:image/svg+xml;utf8,<svg>{}</svg>);--real:#fff;content:"--string-token:blue;"}';
+    const res = parseDesignTokenContent(content, { filePath: "tokens.min.css" });
+
+    expect(res.tokens.map((token) => [token.name, token.rawValue])).toEqual([
+      ["--label", '"literal; {  value }"'],
+      ["--image", 'url("data:image/svg+xml;utf8,<svg>{}</svg>")'],
+      ["--raw-image", "url(data:image/svg+xml;utf8,<svg>{}</svg>)"],
+      ["--real", "#fff"],
+    ]);
+  });
+
+  it("keeps the nearest theme selector through multiple nested compact at-rules", () => {
+    const content =
+      '@layer theme{@supports(--feature-probe:#fff){:root{--surface:#fff}@media(prefers-color-scheme:dark){[data-theme="dark"]{--surface:#111;--text:var(--surface)}}}}';
+    const res = parseDesignTokenContent(content, {
+      filePath: "tokens.min.css",
+      themeSelectors: {
+        ":root": "default",
+        '[data-theme="dark"]': "dark",
+      },
+    });
+
+    expect(
+      res.tokens.map((token) => [token.name, token.theme, token.selector, token.resolvedValue])
+    ).toEqual([
+      ["--surface", "default", ":root", "#fff"],
+      ["--surface", "dark", '[data-theme="dark"]', "#111"],
+      ["--text", "dark", '[data-theme="dark"]', "#111"],
+    ]);
+  });
+
+  it("handles thousands of compact declarations without losing selector context", () => {
+    const declarationCount = 4_000;
+    const declarations = Array.from(
+      { length: declarationCount },
+      (_, index) => `--token-${index}:${index}px`
+    ).join(";");
+    const res = parseDesignTokenContent(
+      `@layer theme{@media(min-width:0){[data-theme="dense"]{${declarations}}}}`,
+      {
+        filePath: "tokens.min.css",
+        themeSelectors: { '[data-theme="dense"]': "dense" },
+      }
+    );
+
+    expect(res.tokens).toHaveLength(declarationCount);
+    expect(res.tokens.at(-1)).toMatchObject({
+      name: `--token-${declarationCount - 1}`,
+      theme: "dense",
+      selector: '[data-theme="dense"]',
+    });
+  });
 });

package/src/tokens/design-token-parser.ts

@@ -10,12 +10,7 @@ import { parseScssTokens } from "./scss.js";
 import { containsTailwindV4Theme, parseTailwindV4Theme } from "./tailwind-v4.js";
 import type { DesignTokenParseOptions, ParsedToken, TokenParseOutput } from "./types.js";
 
-// Token declaration value class excludes `;` (the normal terminator), and `{`/`}`
-// so the final declaration in a block (no trailing `;` — valid CSS, #34) and a
-// value that wraps across newlines (#35) are both captured without swallowing a
-// following block. Matched over the FULL content string (not per-line) so the
-// `[^;{}]+?` class can span `\n`. Terminates on `;`, the next `}`, or EOF.
-const TOKEN_DECLARATION_PATTERN = /--([a-zA-Z0-9_-]+)\s*:\s*([^;{}]+?)\s*(?:;|(?=\})|$)/g;
+const CSS_CUSTOM_PROPERTY_NAME_CHARACTER = /[a-zA-Z0-9_-]/u;
 const VAR_REFERENCE_PATTERN = /var\(\s*--([a-zA-Z0-9_-]+)(?:\s*,\s*([^)]+))?\s*\)/;
 
 export function parseDesignTokenContent(
@@ -98,25 +93,64 @@ export function parseDesignTokenContent(
     }
   }
 
-  const tokensByName = new Map<string, { rawValue: string; line?: number }>();
-
-  // Match over the FULL content so a value spanning newlines (#35) is captured.
-  // The line number is recomputed from the match index because findSelectorForLine
-  // and extractDescription still attribute by line. Internal whitespace (incl.
-  // newlines) is collapsed so a multi-line value stores cleanly.
-  for (const match of content.matchAll(TOKEN_DECLARATION_PATTERN)) {
-    const [, name, rawValue] = match;
-    const fullName = `--${name}`;
-    const lineNumber = countNewlines(content, match.index ?? 0) + 1;
-    tokensByName.set(fullName, {
-      rawValue: rawValue.trim().replace(/\s+/g, " "),
+  const declarationsByThemeAndName = new Map<
+    string,
+    {
+      name: string;
+      rawValue: string;
+      line?: number;
+      selector: string;
+      theme: string;
+    }
+  >();
+
+  // Scan once over the full source. Besides resolving compact single-line
+  // selectors, the scanner distinguishes declarations from declaration-shaped
+  // text in comments/strings and respects delimiters inside CSS values.
+  const declarationMatches = scanCssCustomPropertyDeclarations(content);
+  for (const match of declarationMatches) {
+    const fullName = match.name;
+    const lineNumber = match.line;
+    const selector = match.selector;
+    const theme = themeSelectors[selector] || "default";
+    declarationsByThemeAndName.set(`${theme}\0${fullName}`, {
+      name: fullName,
+      rawValue: normalizeCssCustomPropertyValue(match.rawValue),
       line: lineNumber,
+      selector,
+      theme,
     });
   }
 
-  for (const [name, { rawValue, line }] of tokensByName) {
-    const selector = findSelectorForLine(content, line ?? 1);
-    const theme = themeSelectors[selector] || "default";
+  const declarations = [...declarationsByThemeAndName.values()];
+  const defaults = new Map<string, { rawValue: string; line?: number }>();
+  const declarationsByTheme = new Map<string, Map<string, { rawValue: string; line?: number }>>();
+  for (const declaration of declarations) {
+    if (declaration.theme === "default") defaults.set(declaration.name, declaration);
+    let themeDeclarations = declarationsByTheme.get(declaration.theme);
+    if (!themeDeclarations) {
+      themeDeclarations = new Map();
+      declarationsByTheme.set(declaration.theme, themeDeclarations);
+    }
+    themeDeclarations.set(declaration.name, declaration);
+  }
+
+  const tokensByTheme = new Map<string, Map<string, { rawValue: string; line?: number }>>();
+  for (const [theme, themeDeclarations] of declarationsByTheme) {
+    if (theme === "default") {
+      tokensByTheme.set(theme, defaults);
+      continue;
+    }
+    const tokensByName = new Map(defaults);
+    for (const [name, declaration] of themeDeclarations) {
+      tokensByName.set(name, declaration);
+    }
+    tokensByTheme.set(theme, tokensByName);
+  }
+
+  const contentLines = content.split("\n");
+  for (const { name, rawValue, line, selector, theme } of declarations) {
+    const tokensByName = tokensByTheme.get(theme) ?? defaults;
     const { resolvedValue, chain, hasCircular, unresolvedRef } = resolveDesignTokenValue(
       rawValue,
       tokensByName
@@ -141,7 +175,7 @@ export function parseDesignTokenContent(
       lineNumber: line,
       theme,
       selector,
-      description: extractDescription(content, line ?? 1),
+      description: extractDescription(contentLines, line ?? 1),
     });
   }
 
@@ -162,15 +196,6 @@ export function parseDesignTokenContent(
   };
 }
 
-/** Count the `\n` characters in `content` up to (not including) `index`. */
-function countNewlines(content: string, index: number): number {
-  let count = 0;
-  for (let i = 0; i < index && i < content.length; i++) {
-    if (content.charCodeAt(i) === 10) count++;
-  }
-  return count;
-}
-
 /**
  * Flatten a DTCG/JSON `TokenParseOutput` into `DesignToken[]`. Names pass through
  * verbatim (parseDtcgTokens already emits `--kebab` CSS names via
@@ -384,34 +409,322 @@ function inferTokenLevel(name: string, rawValue: string, referenceChain: string[
   return 2;
 }
 
-function findSelectorForLine(content: string, targetLine: number): string {
-  const lines = content.split("\n");
+interface ScannedCssCustomProperty {
+  name: string;
+  rawValue: string;
+  line: number;
+  selector: string;
+}
+
+/**
+ * Extract custom-property declarations and their nearest non-at-rule selector
+ * in one pass. Comments and strings are lexical, not structural; declaration
+ * values may contain quoted or nested `;`, `{`, and `}` delimiters.
+ */
+function scanCssCustomPropertyDeclarations(content: string): ScannedCssCustomProperty[] {
+  const declarations: ScannedCssCustomProperty[] = [];
+  const parentSelectors: string[] = [];
   let currentSelector = ":root";
-  let braceDepth = 0;
-
-  for (let i = 0; i < Math.min(targetLine, lines.length); i++) {
-    const line = lines[i];
-    const selectorMatch = line.match(/^\s*([^{]+)\s*\{/);
-    if (selectorMatch) {
-      const selector = selectorMatch[1].trim();
-      if ((line.match(/\{/g) || []).length > (line.match(/\}/g) || []).length) {
-        currentSelector = selector;
+  let boundary = 0;
+  let statementHasContent = false;
+  let line = 1;
+  let quote: "'" | '"' | null = null;
+  let escaped = false;
+  let inComment = false;
+  let parentheses = 0;
+  let brackets = 0;
+
+  for (let index = 0; index < content.length; index++) {
+    const character = content[index];
+    const next = content[index + 1];
+    if (character === "\n") line++;
+
+    if (inComment) {
+      if (character === "*" && next === "/") {
+        inComment = false;
+        index++;
       }
+      continue;
+    }
+    if (quote) {
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === quote) {
+        quote = null;
+      }
+      continue;
+    }
+    if (character === "/" && next === "*") {
+      inComment = true;
+      index++;
+      continue;
+    }
+    if (character === "'" || character === '"') {
+      quote = character;
+      statementHasContent = true;
+      continue;
+    }
+    if (character === "\\") {
+      statementHasContent = true;
+      if (next === "\n") line++;
+      index++;
+      continue;
+    }
+    if (character === "(") {
+      parentheses++;
+      statementHasContent = true;
+      continue;
+    }
+    if (character === ")") {
+      parentheses = Math.max(0, parentheses - 1);
+      statementHasContent = true;
+      continue;
+    }
+    if (character === "[") {
+      brackets++;
+      statementHasContent = true;
+      continue;
     }
+    if (character === "]") {
+      brackets = Math.max(0, brackets - 1);
+      statementHasContent = true;
+      continue;
+    }
+    if (parentheses > 0 || brackets > 0) {
+      if (!/\s/u.test(character)) statementHasContent = true;
+      continue;
+    }
+
+    if (character === "{") {
+      const header = stripCssComments(content.slice(boundary, index)).trim();
+      parentSelectors.push(currentSelector);
+      if (header && !header.startsWith("@")) currentSelector = header;
+      boundary = index + 1;
+      statementHasContent = false;
+    } else if (character === "}") {
+      currentSelector = parentSelectors.pop() ?? ":root";
+      boundary = index + 1;
+      statementHasContent = false;
+    } else if (character === ";") {
+      boundary = index + 1;
+      statementHasContent = false;
+    } else if (character === "-" && next === "-" && !statementHasContent) {
+      const declaration = scanCssCustomPropertyAt(content, index);
+      if (!declaration) {
+        statementHasContent = true;
+        continue;
+      }
+
+      const rawValue = content.slice(declaration.valueStart, declaration.end).trim();
+      if (rawValue) {
+        declarations.push({
+          name: declaration.name,
+          rawValue,
+          line,
+          selector: currentSelector,
+        });
+      }
+      line += countNewlinesInRange(content, index + 1, declaration.end);
+      statementHasContent = true;
+      index = declaration.end - 1;
+    } else if (!/\s/u.test(character)) {
+      statementHasContent = true;
+    }
+  }
 
-    braceDepth += (line.match(/\{/g) || []).length;
-    braceDepth -= (line.match(/\}/g) || []).length;
+  return declarations;
+}
 
-    if (braceDepth === 0) {
-      currentSelector = ":root";
+function scanCssCustomPropertyAt(
+  content: string,
+  start: number
+): { name: string; valueStart: number; end: number } | null {
+  let cursor = start + 2;
+  while (
+    cursor < content.length &&
+    CSS_CUSTOM_PROPERTY_NAME_CHARACTER.test(content[cursor] ?? "")
+  ) {
+    cursor++;
+  }
+  if (cursor === start + 2) return null;
+
+  const nameEnd = cursor;
+  cursor = skipCssWhitespaceAndComments(content, cursor);
+  if (content[cursor] !== ":") return null;
+
+  const valueStart = cursor + 1;
+  return {
+    name: content.slice(start, nameEnd),
+    valueStart,
+    end: findCssCustomPropertyValueEnd(content, valueStart),
+  };
+}
+
+function findCssCustomPropertyValueEnd(content: string, start: number): number {
+  let quote: "'" | '"' | null = null;
+  let escaped = false;
+  let inComment = false;
+  let parentheses = 0;
+  let brackets = 0;
+  let braces = 0;
+
+  for (let index = start; index < content.length; index++) {
+    const character = content[index];
+    const next = content[index + 1];
+    if (inComment) {
+      if (character === "*" && next === "/") {
+        inComment = false;
+        index++;
+      }
+      continue;
+    }
+    if (quote) {
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === quote) {
+        quote = null;
+      }
+      continue;
+    }
+    if (character === "/" && next === "*") {
+      inComment = true;
+      index++;
+      continue;
+    }
+    if (character === "'" || character === '"') {
+      quote = character;
+      continue;
+    }
+    if (character === "\\") {
+      index++;
+      continue;
+    }
+    if (character === "(") {
+      parentheses++;
+    } else if (character === ")") {
+      parentheses = Math.max(0, parentheses - 1);
+    } else if (character === "[") {
+      brackets++;
+    } else if (character === "]") {
+      brackets = Math.max(0, brackets - 1);
+    } else if (character === "{") {
+      braces++;
+    } else if (character === "}") {
+      if (braces > 0) {
+        braces--;
+      } else if (parentheses === 0 && brackets === 0) {
+        return index;
+      }
+    } else if (character === ";" && parentheses === 0 && brackets === 0 && braces === 0) {
+      return index;
+    }
+  }
+
+  return content.length;
+}
+
+function skipCssWhitespaceAndComments(content: string, start: number): number {
+  let cursor = start;
+  while (cursor < content.length) {
+    if (/\s/u.test(content[cursor] ?? "")) {
+      cursor++;
+      continue;
+    }
+    if (content[cursor] !== "/" || content[cursor + 1] !== "*") break;
+    const commentEnd = content.indexOf("*/", cursor + 2);
+    if (commentEnd < 0) return content.length;
+    cursor = commentEnd + 2;
+  }
+  return cursor;
+}
+
+function stripCssComments(content: string): string {
+  let output = "";
+  let quote: "'" | '"' | null = null;
+  let escaped = false;
+  let inComment = false;
+
+  for (let index = 0; index < content.length; index++) {
+    const character = content[index];
+    const next = content[index + 1];
+    if (inComment) {
+      if (character === "*" && next === "/") {
+        inComment = false;
+        index++;
+      }
+      continue;
+    }
+    if (quote) {
+      output += character;
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === quote) {
+        quote = null;
+      }
+      continue;
+    }
+    if (character === "/" && next === "*") {
+      inComment = true;
+      index++;
+      continue;
+    }
+    output += character;
+    if (character === "'" || character === '"') quote = character;
+  }
+
+  return output;
+}
+
+function countNewlinesInRange(content: string, start: number, end: number): number {
+  let count = 0;
+  for (let index = start; index < end; index++) {
+    if (content.charCodeAt(index) === 10) count++;
+  }
+  return count;
+}
+
+function normalizeCssCustomPropertyValue(value: string): string {
+  let output = "";
+  let quote: "'" | '"' | null = null;
+  let escaped = false;
+  let pendingWhitespace = false;
+
+  for (const character of value.trim()) {
+    if (quote) {
+      output += character;
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === quote) {
+        quote = null;
+      }
+      continue;
+    }
+    if (character === "'" || character === '"') {
+      if (pendingWhitespace && output) output += " ";
+      pendingWhitespace = false;
+      quote = character;
+      output += character;
+    } else if (/\s/u.test(character)) {
+      pendingWhitespace = true;
+    } else {
+      if (pendingWhitespace && output) output += " ";
+      pendingWhitespace = false;
+      output += character;
     }
   }
 
-  return currentSelector;
+  return output;
 }
 
-function extractDescription(content: string, line: number): string | undefined {
-  const lines = content.split("\n");
+function extractDescription(lines: readonly string[], line: number): string | undefined {
   if (line <= 1) return undefined;
 
   const prevLine = lines[line - 2]?.trim();

package/src/types.ts

@@ -892,7 +892,7 @@ export type Theme = "light" | "dark";
  */
 export interface Screenshot {
   /** PNG image data */
-  data: Buffer;
+  data: Uint8Array;
 
   /** SHA-256 hash of image data (for change detection) */
   hash: string;
@@ -944,7 +944,7 @@ export interface DiffResult {
   totalPixels: number;
 
   /** PNG image highlighting differences */
-  diffImage?: Buffer;
+  diffImage?: Uint8Array;
 
   /** Bounding boxes of changed regions */
   changedRegions: BoundingBox[];