@eslint/css-tree 3.3.4 → 3.5.0

package/dist/version.cjs

@@ -1 +1 @@
-module.exports = "3.3.4";
+module.exports = "3.5.0";

package/dist/version.js

@@ -1 +1 @@
-export const version = "3.3.4";
+export const version = "3.5.0";

package/lib/generator/create.js

@@ -1,4 +1,5 @@
-import { tokenize, Delim, WhiteSpace } from '../tokenizer/index.js';
+import { Delim, WhiteSpace } from '../tokenizer/index.js';
+import { getTokenizer } from '../utils/get-tokenizer.js';
 import { generateSourceMap } from './sourceMap.js';
 import * as tokenBefore from './token-before.js';
 
@@ -23,12 +24,6 @@ function processChildren(node, delimeter) {
     node.children.forEach(this.node, this);
 }
 
-function processChunk(chunk) {
-    tokenize(chunk, (type, start, end) => {
-        this.token(type, chunk.slice(start, end));
-    });
-}
-
 export function createGenerator(config) {
     const types = new Map();
 
@@ -87,7 +82,13 @@ export function createGenerator(config) {
             node: (node) => handlers.node(node),
             children: processChildren,
             token: (type, value) => handlers.token(type, value),
-            tokenize: processChunk
+            tokenize: function (chunk) {
+                const tokenize = getTokenizer(config);
+
+                return tokenize(chunk, (type, start, end) => {
+                    this.token(type, chunk.slice(start, end));
+                });
+            }
         };
 
         handlers.node(node);

package/lib/index.d.ts

@@ -2693,19 +2693,21 @@ type ParserContext = TokenStream
         [key: string]: unknown;
     };
 
-interface StructureDescriptor {
-    type: string;
-    optional?: boolean;
-    parse?: (this: ParserContext) => CssNodeCommon;
-    generate?: (this: ParserContext, node: CssNodeCommon) => void;
+type StructureValue = string | Function | null;
+type StructureDescriptor =  StructureValue | Array<StructureValue> | Array<Array<StructureValue>>;
+
+interface StructureDefinition {
+    children?: Array<string>;
+
+    [key: string]: StructureDescriptor | undefined;
 }
 
 interface NodeSyntaxConfig<T extends CssNodeCommon = CssNodeCommon> {
     name: string;
-    structure: Record<string, StructureDescriptor>;
+    structure: StructureDefinition;
     parse(this: ParserContext): T;
     generate(this: ParserContext, node: T): void;
-    walkContext: WalkContext;
+    walkContext?: string;
 }
 
 interface AtruleSyntax {
@@ -2736,6 +2738,7 @@ export interface SyntaxConfig<T extends CssNodeCommon = CssNodeCommon> {
     scope: Record<string, Recognizer>;
     features: Record<string, Recognizer>;
     parseContext: ParseContext;
+    tokenize: TokenizeFunction;
 }
 
 interface LexerStructureWarning {

package/lib/lexer/prepare-tokens.js

@@ -1,4 +1,4 @@
-import { tokenize } from '../tokenizer/index.js';
+import { getTokenizer } from '../utils/get-tokenizer.js';
 
 const astToTokens = {
     decorator(handlers) {
@@ -27,8 +27,9 @@ const astToTokens = {
     }
 };
 
-function stringToTokens(str) {
+function stringToTokens(str, syntax) {
     const tokens = [];
+    const tokenize = getTokenizer(syntax);
 
     tokenize(str, (type, start, end) =>
         tokens.push({
@@ -43,7 +44,7 @@ function stringToTokens(str) {
 
 export default function(value, syntax) {
     if (typeof value === 'string') {
-        return stringToTokens(value);
+        return stringToTokens(value, syntax);
     }
 
     return syntax.generate(value, astToTokens);

package/lib/parser/create.js

@@ -1,7 +1,6 @@
 import { List } from '../utils/List.js';
 import { SyntaxError } from './SyntaxError.js';
 import {
-    tokenize,
     OffsetToLocation,
     TokenStream,
     tokenNames,
@@ -21,6 +20,7 @@ import {
     Number as NumberToken
 } from '../tokenizer/index.js';
 import { readSequence } from './sequence.js';
+import { getTokenizer } from '../utils/get-tokenizer.js';
 
 const NOOP = () => {};
 const EXCLAMATIONMARK = 0x0021;  // U+0021 EXCLAMATION MARK (!)
@@ -57,7 +57,8 @@ function processConfig(config) {
         scope: Object.assign(Object.create(null), config.scope),
         atrule: fetchParseValues(config.atrule),
         pseudo: fetchParseValues(config.pseudo),
-        node: fetchParseValues(config.node)
+        node: fetchParseValues(config.node),
+        tokenize: getTokenizer(config)
     };
 
     for (const [name, context] of Object.entries(config.parseContext)) {
@@ -327,7 +328,7 @@ export function createParser(config) {
         source = source_;
         options = options || {};
 
-        parser.setSource(source, tokenize);
+        parser.setSource(source, parser.tokenize);
         locationMap.setSource(
             source,
             options.offset,

package/lib/syntax/config/mix.js

@@ -115,6 +115,12 @@ export default function mix(dest, src) {
             case 'node':
                 result[prop] = mergeDicts(dest[prop], value, ['name', 'structure', 'parse', 'generate', 'walkContext']);
                 break;
+
+            case 'tokenize':
+                if (typeof value === 'function') {
+                    result[prop] = value;
+                }
+                break;
         }
     }
 

package/lib/syntax/config/parser.js

@@ -2,6 +2,7 @@ import * as scope from '../scope/index.js';
 import atrule from '../atrule/index.js';
 import pseudo from '../pseudo/index.js';
 import * as node from '../node/index-parse.js';
+import { tokenize } from '../../tokenizer/index.js';
 
 export default {
     parseContext: {
@@ -41,5 +42,6 @@ export default {
     scope,
     atrule,
     pseudo,
-    node
+    node,
+    tokenize
 };

package/lib/syntax/create.js

@@ -1,10 +1,10 @@
-import { tokenize } from '../tokenizer/index.js';
 import { createParser } from '../parser/create.js';
 import { createGenerator } from '../generator/create.js';
 import { createConvertor } from '../convertor/create.js';
 import { createWalker } from '../walker/create.js';
 import { Lexer } from '../lexer/Lexer.js';
 import mix from './config/mix.js';
+import { getTokenizer } from '../utils/get-tokenizer.js';
 
 function createSyntax(config) {
     const parse = createParser(config);
@@ -16,7 +16,7 @@ function createSyntax(config) {
         lexer: null,
         createLexer: config => new Lexer(config, syntax, syntax.lexer.structure),
 
-        tokenize,
+        tokenize: getTokenizer(config),
         parse,
         generate,
 

package/lib/syntax/node/Block.js

@@ -1,13 +1,25 @@
 import {
     WhiteSpace,
     Comment,
+    Delim,
     Semicolon,
+    Hash,
+    Colon,
     AtKeyword,
+    LeftSquareBracket,
     LeftCurlyBracket,
     RightCurlyBracket
 } from '../../tokenizer/index.js';
 
 const AMPERSAND = 0x0026;       // U+0026 AMPERSAND (&)
+const DOT = 0x002E;             // U+002E FULL STOP (.)
+const STAR = 0x002A;            // U+002A ASTERISK (*);
+
+const selectorStarts = new Set([
+    AMPERSAND,
+    DOT,
+    STAR
+]);
 
 function consumeRaw() {
     return this.Raw(null, true);
@@ -32,6 +44,12 @@ function consumeDeclaration() {
     return node;
 }
 
+function isSelectorStart() {
+    return this.tokenType === Delim && selectorStarts.has(this.source.charCodeAt(this.tokenStart)) ||
+        this.tokenType === Hash || this.tokenType === LeftSquareBracket ||
+        this.tokenType === Colon;
+}
+
 export const name = 'Block';
 export const walkContext = 'block';
 export const structure = {
@@ -65,7 +83,7 @@ export function parse(isStyleBlock) {
                 break;
 
             default:
-                if (isStyleBlock && this.isDelim(AMPERSAND))  {
+                if (isStyleBlock && isSelectorStart.call(this)) {
                     children.push(consumeRule.call(this));
                 } else {
                     children.push(consumer.call(this));

package/lib/syntax/node/CDC.js

@@ -1,7 +1,7 @@
 import { CDC } from '../../tokenizer/index.js';
 
 export const name = 'CDC';
-export const structure = [];
+export const structure = {};
 
 export function parse() {
     const start = this.tokenStart;

package/lib/syntax/node/CDO.js

@@ -1,7 +1,7 @@
 import { CDO } from '../../tokenizer/index.js';
 
 export const name = 'CDO';
-export const structure = [];
+export const structure = {};
 
 export function parse() {
     const start = this.tokenStart;

package/lib/utils/get-tokenizer.js

@@ -0,0 +1,18 @@
+import { tokenize } from '../tokenizer/index.js';
+
+const FUNCTION_TYPE = 'function';
+
+/**
+ * Gets the tokenizer function from the configuration object or returns the default tokenizer
+ *
+ * @param config Configuration object
+ * @returns Corresponding tokenizer function
+ */
+export function getTokenizer(config) {
+    if (config && typeof config.tokenize === FUNCTION_TYPE) {
+        return config.tokenize;
+    }
+
+    // Fallback to the default tokenizer
+    return tokenize;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eslint/css-tree",
-  "version": "3.3.4",
+  "version": "3.5.0",
   "description": "A tool set for CSS: fast detailed parser (CSS → AST), walker (AST traversal), generator (AST → CSS) and lexer (validation and matching) based on specs and browser implementations",
   "author": "Roman Dvornov <rdvornov@gmail.com> (https://github.com/lahmatiy)",
   "license": "MIT",