@thejaredwilcurt/csslop 0.0.38 → 0.0.39

package/src/parser/tokenizer.js

@@ -0,0 +1,396 @@
+/**
+ * @file Reads a CSS value as the sequence of tokens that the CSS Syntax tokenizer reads out of it, so that what a value is made of can be judged token by token rather than character by character.
+ */
+
+/**
+ * Matches one CSS whitespace character. CSS counts a form feed and a carriage
+ * return as whitespace, so the shorter `\s` class would be too broad.
+ *
+ * @type {RegExp}
+ */
+const WHITESPACE_CHARACTER = /[ \t\n\r\f]/;
+
+/**
+ * Matches one decimal digit, which is what starts the numeric tokens.
+ *
+ * @type {RegExp}
+ */
+const DIGIT_CHARACTER = /[0-9]/;
+
+/**
+ * Matches one hexadecimal digit, which is what a unicode escape holds.
+ *
+ * @type {RegExp}
+ */
+const HEX_DIGIT_CHARACTER = /[0-9a-fA-F]/;
+
+/**
+ * Matches one character that may start an identifier: a letter, an underscore,
+ * or anything outside ASCII.
+ *
+ * @type {RegExp}
+ */
+const IDENT_START_CHARACTER = /[a-zA-Z_\u0080-\uFFFF]/;
+
+/**
+ * Matches one character that may continue an identifier, which is everything an
+ * identifier may start with plus the digits and the hyphen.
+ *
+ * @type {RegExp}
+ */
+const NAME_CHARACTER = /[-a-zA-Z0-9_\u0080-\uFFFF]/;
+
+/**
+ * @typedef  {object} CssToken
+ * @property {string} type      The kind of token, such as `ident`, `number`, or `whitespace`.
+ * @property {string} text      The exact slice of the value that the token spans.
+ */
+
+/**
+ * Reports whether the two characters at an index form a valid escape sequence.
+ * A backslash escapes the character after it, unless that character is the
+ * newline that ends the line.
+ *
+ * @param  {string}  text   The CSS value being read.
+ * @param  {number}  index  The index of the possible backslash.
+ * @return {boolean}        Whether an escape sequence starts at the index.
+ */
+function startsEscape (text, index) {
+  return text[index] === '\\' && text[index + 1] !== '\n';
+}
+
+/**
+ * Reads past an escape sequence, which is either up to six hexadecimal digits
+ * and the single whitespace character that may end them, or one escaped
+ * character.
+ *
+ * @param  {string} text   The CSS value being read.
+ * @param  {number} index  The index of the backslash that starts the escape.
+ * @return {number}        The index just past the escape sequence.
+ */
+function readPastEscape (text, index) {
+  let end = index + 1;
+  let hexDigitCount = 0;
+  while (end < text.length && hexDigitCount < 6 && HEX_DIGIT_CHARACTER.test(text[end])) {
+    end++;
+    hexDigitCount++;
+  }
+  if (!hexDigitCount) {
+    return Math.min(end + 1, text.length);
+  }
+  if (end < text.length && WHITESPACE_CHARACTER.test(text[end])) {
+    end++;
+  }
+  return end;
+}
+
+/**
+ * Reads past a name, the run of identifier characters and escapes that makes up
+ * an identifier, the body of a hash token, or the unit of a dimension.
+ *
+ * @param  {string} text   The CSS value being read.
+ * @param  {number} index  The index the name starts at.
+ * @return {number}        The index just past the name.
+ */
+function readPastName (text, index) {
+  let end = index;
+  while (end < text.length) {
+    if (NAME_CHARACTER.test(text[end])) {
+      end++;
+      continue;
+    }
+    if (startsEscape(text, end)) {
+      end = readPastEscape(text, end);
+      continue;
+    }
+    break;
+  }
+  return end;
+}
+
+/**
+ * Reports whether an identifier starts at an index. A hyphen starts one only
+ * when a second hyphen, an identifier character, or an escape follows it, which
+ * is what makes `--custom` an identifier and a lone `-` a delimiter.
+ *
+ * @param  {string}  text   The CSS value being read.
+ * @param  {number}  index  The index to test.
+ * @return {boolean}        Whether an identifier starts at the index.
+ */
+function startsIdent (text, index) {
+  const character = text[index];
+  if (character === undefined) {
+    return false;
+  }
+  if (character === '-') {
+    const nextCharacter = text[index + 1];
+    if (nextCharacter === '-') {
+      return true;
+    }
+    return Boolean(nextCharacter) && (IDENT_START_CHARACTER.test(nextCharacter) || startsEscape(text, index + 1));
+  }
+  if (IDENT_START_CHARACTER.test(character)) {
+    return true;
+  }
+  return startsEscape(text, index);
+}
+
+/**
+ * Reports whether a number starts at an index. A sign or a decimal point only
+ * starts one when a digit follows close enough behind it.
+ *
+ * @param  {string}  text   The CSS value being read.
+ * @param  {number}  index  The index to test.
+ * @return {boolean}        Whether a number starts at the index.
+ */
+function startsNumber (text, index) {
+  const character = text[index];
+  if (character === undefined) {
+    return false;
+  }
+  if (character === '+' || character === '-') {
+    const nextCharacter = text[index + 1] || '';
+    const followingCharacter = text[index + 2] || '';
+    return DIGIT_CHARACTER.test(nextCharacter) || (nextCharacter === '.' && DIGIT_CHARACTER.test(followingCharacter));
+  }
+  if (character === '.') {
+    return DIGIT_CHARACTER.test(text[index + 1] || '');
+  }
+  return DIGIT_CHARACTER.test(character);
+}
+
+/**
+ * Reads past the run of digits that starts at an index.
+ *
+ * @param  {string} text   The CSS value being read.
+ * @param  {number} index  The index the digits start at.
+ * @return {number}        The index just past the digits.
+ */
+function readPastDigits (text, index) {
+  let end = index;
+  while (end < text.length && DIGIT_CHARACTER.test(text[end])) {
+    end++;
+  }
+  return end;
+}
+
+/**
+ * Reads past a number: an optional sign, an integer part, an optional fraction,
+ * and an optional scientific exponent.
+ *
+ * @param  {string} text   The CSS value being read.
+ * @param  {number} index  The index the number starts at.
+ * @return {number}        The index just past the number.
+ */
+function readPastNumber (text, index) {
+  let end = index;
+  if (text[end] === '+' || text[end] === '-') {
+    end++;
+  }
+  end = readPastDigits(text, end);
+  if (text[end] === '.' && DIGIT_CHARACTER.test(text[end + 1] || '')) {
+    end = readPastDigits(text, end + 1);
+  }
+  const exponentCharacter = text[end];
+  if (exponentCharacter === 'e' || exponentCharacter === 'E') {
+    let exponentEnd = end + 1;
+    if (text[exponentEnd] === '+' || text[exponentEnd] === '-') {
+      exponentEnd++;
+    }
+    if (DIGIT_CHARACTER.test(text[exponentEnd] || '')) {
+      end = readPastDigits(text, exponentEnd);
+    }
+  }
+  return end;
+}
+
+/**
+ * Reads the numeric token at an index. A number takes a unit when an identifier
+ * follows it and becomes a percentage when a percent sign does.
+ *
+ * @param  {string}   text   The CSS value being read.
+ * @param  {number}   index  The index the number starts at.
+ * @return {CssToken}        The dimension, percentage, or number token.
+ */
+function readNumericToken (text, index) {
+  const numberEnd = readPastNumber(text, index);
+  if (startsIdent(text, numberEnd)) {
+    const unitEnd = readPastName(text, numberEnd);
+    return { type: 'dimension', text: text.slice(index, unitEnd) };
+  }
+  if (text[numberEnd] === '%') {
+    return { type: 'percentage', text: text.slice(index, numberEnd + 1) };
+  }
+  return { type: 'number', text: text.slice(index, numberEnd) };
+}
+
+/**
+ * Reports whether the contents of a `url()` are quoted, which makes it an
+ * ordinary function token holding a string rather than a single url token.
+ *
+ * @param  {string}  text   The CSS value being read.
+ * @param  {number}  index  The index just past the opening parenthesis.
+ * @return {boolean}        Whether a quoted string opens the parentheses.
+ */
+function holdsQuotedUrl (text, index) {
+  let end = index;
+  while (end < text.length && WHITESPACE_CHARACTER.test(text[end])) {
+    end++;
+  }
+  return text[end] === '"' || text[end] === '\'';
+}
+
+/**
+ * Reads past the body of an unquoted url token, which runs to the parenthesis
+ * that closes it. An unquoted url holds no nested parentheses, so the first
+ * closing one ends the token.
+ *
+ * @param  {string} text   The CSS value being read.
+ * @param  {number} index  The index just past the opening parenthesis.
+ * @return {number}        The index just past the closing parenthesis.
+ */
+function readPastUrlBody (text, index) {
+  let end = index;
+  while (end < text.length) {
+    if (startsEscape(text, end)) {
+      end = readPastEscape(text, end);
+      continue;
+    }
+    if (text[end] === ')') {
+      return end + 1;
+    }
+    end++;
+  }
+  return end;
+}
+
+/**
+ * Reads the token that an identifier starts. An identifier directly followed by
+ * an opening parenthesis is a function token instead, and the `url()` written
+ * without quotes is a single token that swallows its own contents.
+ *
+ * @param  {string}   text   The CSS value being read.
+ * @param  {number}   index  The index the identifier starts at.
+ * @return {CssToken}        The ident, function, or url token.
+ */
+function readIdentLikeToken (text, index) {
+  const nameEnd = readPastName(text, index);
+  const name = text.slice(index, nameEnd);
+  if (text[nameEnd] !== '(') {
+    return { type: 'ident', text: name };
+  }
+  if (name.toLowerCase() === 'url' && !holdsQuotedUrl(text, nameEnd + 1)) {
+    const urlEnd = readPastUrlBody(text, nameEnd + 1);
+    return { type: 'url', text: text.slice(index, urlEnd) };
+  }
+  return { type: 'function', text: text.slice(index, nameEnd + 1) };
+}
+
+/**
+ * Reads the string token at an index, which runs to the matching quote and
+ * takes escaped quotes in its stride.
+ *
+ * @param  {string}   text   The CSS value being read.
+ * @param  {number}   index  The index of the opening quote.
+ * @return {CssToken}        The string token.
+ */
+function readStringToken (text, index) {
+  const quote = text[index];
+  let end = index + 1;
+  while (end < text.length) {
+    if (startsEscape(text, end)) {
+      end = readPastEscape(text, end);
+      continue;
+    }
+    if (text[end] === quote) {
+      end++;
+      break;
+    }
+    if (text[end] === '\n') {
+      break;
+    }
+    end++;
+  }
+  return { type: 'string', text: text.slice(index, end) };
+}
+
+/**
+ * Reads the run of whitespace at an index as the single token that CSS treats
+ * it as.
+ *
+ * @param  {string}   text   The CSS value being read.
+ * @param  {number}   index  The index the whitespace starts at.
+ * @return {CssToken}        The whitespace token.
+ */
+function readWhitespaceToken (text, index) {
+  let end = index;
+  while (end < text.length && WHITESPACE_CHARACTER.test(text[end])) {
+    end++;
+  }
+  return { type: 'whitespace', text: text.slice(index, end) };
+}
+
+/**
+ * The characters that stand on their own as structural punctuation, none of
+ * which can ever merge with the token beside it.
+ *
+ * @type {Set<string>}
+ */
+const PUNCTUATION_CHARACTERS = new Set(['(', ')', '[', ']', '{', '}', ',', ':', ';']);
+
+/**
+ * Reads the single token that starts at an index.
+ *
+ * @param  {string}   text   The CSS value being read.
+ * @param  {number}   index  The index the token starts at.
+ * @return {CssToken}        The token found at the index.
+ */
+function readToken (text, index) {
+  const character = text[index];
+
+  if (WHITESPACE_CHARACTER.test(character)) {
+    return readWhitespaceToken(text, index);
+  }
+  if (character === '"' || character === '\'') {
+    return readStringToken(text, index);
+  }
+  if (PUNCTUATION_CHARACTERS.has(character)) {
+    return { type: 'punctuation', text: character };
+  }
+  if (character === '#') {
+    const startsHash = NAME_CHARACTER.test(text[index + 1] || '') || startsEscape(text, index + 1);
+    if (startsHash) {
+      return { type: 'hash', text: text.slice(index, readPastName(text, index + 1)) };
+    }
+  }
+  if (character === '@' && startsIdent(text, index + 1)) {
+    return { type: 'at-keyword', text: text.slice(index, readPastName(text, index + 1)) };
+  }
+  if (startsNumber(text, index)) {
+    return readNumericToken(text, index);
+  }
+  if (startsIdent(text, index)) {
+    return readIdentLikeToken(text, index);
+  }
+  return { type: 'delim', text: character };
+}
+
+/**
+ * Splits a CSS value into the tokens that the CSS Syntax tokenizer would read
+ * out of it.
+ *
+ * @param  {string} value  The CSS value to tokenize.
+ * @return {Array}         The tokens the value holds, in the order they appear.
+ */
+function tokenizeCssValue (value) {
+  const tokens = [];
+  let index = 0;
+  while (index < value.length) {
+    const token = readToken(value, index);
+    tokens.push(token);
+    index += Math.max(token.text.length, 1);
+  }
+  return tokens;
+}
+
+export { tokenizeCssValue };

package/src/preprocess.js

@@ -6,239 +6,31 @@ import { isUnicodeCharset } from './context.js';
 import { resolveUnicodeEscape } from './utilities.js';
 
 /**
- * Automatically fixes common CSS syntax errors that would cause parsing to fail.
- * Handles missing closing braces, unmatched quotes, and other structural issues.
- *
- * @param  {string} css  The raw CSS string to fix.
- * @return {string}      The CSS string with common syntax errors corrected.
- */
-function fixCommonSyntaxErrors (css) {
-  let fixed = css;
-
-  // Fix missing closing braces by analyzing brace balance
-  // Count braces while ignoring those inside comments and strings
-  let openBraces = 0;
-  let inString = false;
-  let stringChar = '';
-  let inComment = false;
-  let commentType = '';
-
-  for (let i = 0; i < fixed.length; i++) {
-    const char = fixed[i];
-    const prevChar = i > 0 ? fixed[i - 1] : '';
-
-    // Handle string state
-    if (!inComment && (char === '"' || char === '\'') && prevChar !== '\\') {
-      if (!inString) {
-        inString = true;
-        stringChar = char;
-      } else if (char === stringChar) {
-        inString = false;
-        stringChar = '';
-      }
-    }
-
-    // Handle comment state
-    if (!inString) {
-      if (!inComment && char === '/' && i + 1 < fixed.length && fixed[i + 1] === '*') {
-        inComment = true;
-        commentType = '/*';
-        i++; // Skip the next character
-      } else if (inComment && commentType === '/*' && char === '*' && i + 1 < fixed.length && fixed[i + 1] === '/') {
-        inComment = false;
-        commentType = '';
-        i++; // Skip the next character
-      }
-    }
-
-    // Count braces only when not in strings or comments
-    if (!inString && !inComment) {
-      if (char === '{') {
-        openBraces++;
-      } else if (char === '}') {
-        openBraces--;
-      }
-    }
-  }
-
-  // Add missing closing braces
-  if (openBraces > 0) {
-    fixed += '}'.repeat(openBraces);
-  }
-
-  // Fix unclosed strings (common in malformed CSS)
-  if (inString) {
-    fixed += stringChar;
-  }
-
-  // Fix unclosed comments
-  if (inComment && commentType === '/*') {
-    fixed += '*/';
-  }
-
-  return fixed;
-}
-
-/**
- * Converts CSS Unicode escape sequences inside declaration blocks to their literal character equivalents, while preserving control characters that must remain escaped.
- * Also cleans up malformed CSS syntax that would cause parser failures.
+ * Converts CSS Unicode escape sequences inside declaration blocks to their
+ * literal character equivalents, while preserving the control characters that
+ * must remain escaped. A property name is read as written, so an escape in one
+ * is resolved before the parse rather than after it.
  *
  * @param  {string} css  The raw CSS string to preprocess.
- * @return {string}      The CSS string with printable Unicode escapes resolved inside declaration blocks and syntax errors fixed.
+ * @return {string}      The CSS string with printable Unicode escapes resolved inside declaration blocks.
  */
 function preprocessDeclarationBlocks (css) {
-  // First, fix common syntax errors that would cause parsing to fail
-  let processed = fixCommonSyntaxErrors(css);
-
-  // Match top-level declaration blocks (non-nested { ... })
-  return processed.replace(/\{([^{}]*)\}/g, (match, content) => {
-    // Remove semicolons after standalone comments (between declarations) which cause parser errors.
-    // Only match when preceded by a semicolon, so property values like --foo: /*...*/; keep their terminator.
-    let processed = content.replace(/(?<=;)\s*\/\*.*?\*\/\s*;/g, (commentMatch) => {
-      // Remove the trailing semicolon from comment+semicolon combinations
-      return commentMatch.replace(/;$/, '');
+  if (!isUnicodeCharset()) {
+    return css;
+  }
+  // Match the innermost declaration blocks, which hold no block of their own
+  return css.replace(/\{([^{}]*)\}/g, (blockMatch, blockContents) => {
+    // Match a quoted string, whose escapes are data, or a unicode escape
+    // outside one: a backslash, one to six hexadecimal digits, and the
+    // optional whitespace that ends the escape
+    const resolved = blockContents.replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\\([0-9a-fA-F]{1,6})\s?/g, (escapeMatch, hex) => {
+      if (!hex) {
+        return escapeMatch;
+      }
+      return resolveUnicodeEscape(hex) ?? escapeMatch;
     });
-    
-    // Then, skip quoted strings and match CSS unicode escapes (backslash + 1-6 hex digits + optional whitespace).
-    // Only resolve when the charset is unicode-compatible.
-    if (isUnicodeCharset()) {
-      processed = processed.replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\\([0-9a-fA-F]{1,6})\s?/g, (fullMatch, hex) => {
-        if (!hex) {
-          return fullMatch;
-        }
-        return resolveUnicodeEscape(hex) ?? fullMatch;
-      });
-    }
-    
-    return '{' + processed + '}';
+    return '{' + resolved + '}';
   });
 }
 
-/**
- * Unicode Private Use Area characters used as temporary placeholders
- * for CSS escape sequences during preprocessing, preventing the parser
- * from exceeding its internal escape-counting limit on large files.
- */
-const ESCAPED_COLON_PLACEHOLDER = '\uE001';
-const ESCAPED_DOT_PLACEHOLDER = '\uE002';
-const ESCAPED_SLASH_PLACEHOLDER = '\uE003';
-
-/**
- * Replaces common single-character CSS escape sequences with Unicode Private
- * Use Area placeholder characters to prevent the parser from exceeding its
- * internal escape-counting limit on files with many escaped selectors
- * (e.g. Tailwind-style utility classes like `.sm\:p-0`).
- *
- * Only neutralizes escapes outside of attribute selector brackets `[...]`,
- * quoted strings, and comments so the minifier's quote-vs-escape length
- * comparisons in attribute selectors remain accurate.
- *
- * @param  {string} css  The raw CSS string to preprocess.
- * @return {string}      The CSS with escape sequences replaced by placeholders.
- */
-function neutralizeEscapeSequences (css) {
-  let result = '';
-  let insideBrackets = false;
-  let insideString = false;
-  let stringDelimiter = '';
-  let insideComment = false;
-
-  for (let i = 0; i < css.length; i++) {
-    const character = css[i];
-    const nextCharacter = i + 1 < css.length ? css[i + 1] : '';
-
-    // Track block comment state
-    if (!insideString && !insideComment && character === '/' && nextCharacter === '*') {
-      insideComment = true;
-      result += '/*';
-      i++;
-      continue;
-    }
-    if (insideComment) {
-      if (character === '*' && nextCharacter === '/') {
-        insideComment = false;
-        result += '*/';
-        i++;
-        continue;
-      }
-      result += character;
-      continue;
-    }
-
-    // Track string state (preserve escapes inside strings)
-    if (!insideString && (character === '"' || character === '\'')) {
-      insideString = true;
-      stringDelimiter = character;
-      result += character;
-      continue;
-    }
-    if (insideString) {
-      if (character === '\\' && nextCharacter) {
-        result += character + nextCharacter;
-        i++;
-        continue;
-      }
-      if (character === stringDelimiter) {
-        insideString = false;
-        stringDelimiter = '';
-      }
-      result += character;
-      continue;
-    }
-
-    // Track attribute selector bracket state
-    if (character === '[') {
-      insideBrackets = true;
-      result += character;
-      continue;
-    }
-    if (character === ']') {
-      insideBrackets = false;
-      result += character;
-      continue;
-    }
-
-    // Replace escape sequences only outside brackets, strings, and comments
-    if (!insideBrackets && character === '\\') {
-      if (nextCharacter === ':') {
-        result += ESCAPED_COLON_PLACEHOLDER;
-        i++;
-        continue;
-      }
-      if (nextCharacter === '.') {
-        result += ESCAPED_DOT_PLACEHOLDER;
-        i++;
-        continue;
-      }
-      if (nextCharacter === '/') {
-        result += ESCAPED_SLASH_PLACEHOLDER;
-        i++;
-        continue;
-      }
-    }
-
-    result += character;
-  }
-
-  return result;
-}
-
-/**
- * Restores the original CSS escape sequences from their Unicode Private Use
- * Area placeholder characters after minification is complete.
- *
- * @param  {string} css  The minified CSS string with placeholders.
- * @return {string}      The CSS with original escape sequences restored.
- */
-function restoreEscapeSequences (css) {
-  return css
-    .replaceAll(ESCAPED_COLON_PLACEHOLDER, '\\:')
-    .replaceAll(ESCAPED_DOT_PLACEHOLDER, '\\.')
-    .replaceAll(ESCAPED_SLASH_PLACEHOLDER, '\\/');
-}
-
-export {
-  neutralizeEscapeSequences,
-  preprocessDeclarationBlocks,
-  restoreEscapeSequences
-};
+export { preprocessDeclarationBlocks };