readability_js 0.0.1

data/lib/readability_js/node/node_modules/@csstools/css-tokenizer/dist/index.d.ts

@@ -0,0 +1,593 @@
+/**
+ * Tokenize CSS following the {@link https://drafts.csswg.org/css-syntax/#tokenization | CSS Syntax Level 3 specification}.
+ *
+ * @remarks
+ * The tokenizing and parsing tools provided by CSS Tools are designed to be low level and generic with strong ties to their respective specifications.
+ *
+ * Any analysis or mutation of CSS source code should be done with the least powerful tool that can accomplish the task.
+ * For many applications it is sufficient to work with tokens.
+ * For others you might need to use {@link https://github.com/csstools/postcss-plugins/tree/main/packages/css-parser-algorithms | @csstools/css-parser-algorithms} or a more specific parser.
+ *
+ * @example
+ * Tokenize a string of CSS into an array of tokens:
+ * ```js
+ * import { tokenize } from '@csstools/css-tokenizer';
+ *
+ * const myCSS = `@media only screen and (min-width: 768rem) {
+ * 	.foo {
+ * 		content: 'Some content!' !important;
+ * 	}
+ * }
+ * `;
+ *
+ * const tokens = tokenize({
+ * 	css: myCSS,
+ * });
+ *
+ * console.log(tokens);
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Deep clone a list of tokens.
+ * Useful for mutations without altering the original list.
+ */
+export declare function cloneTokens(tokens: Array<CSSToken>): Array<CSSToken>;
+
+/**
+ * The union of all possible CSS tokens
+ */
+export declare type CSSToken = TokenAtKeyword | TokenBadString | TokenBadURL | TokenCDC | TokenCDO | TokenColon | TokenComma | TokenComment | TokenDelim | TokenDimension | TokenEOF | TokenFunction | TokenHash | TokenIdent | TokenNumber | TokenPercentage | TokenSemicolon | TokenString | TokenURL | TokenWhitespace | TokenOpenParen | TokenCloseParen | TokenOpenSquare | TokenCloseSquare | TokenOpenCurly | TokenCloseCurly | TokenUnicodeRange;
+
+/**
+ * The type of hash token
+ */
+export declare enum HashType {
+    /**
+     * The hash token did not start with an ident sequence (e.g. `#-2`)
+     */
+    Unrestricted = "unrestricted",
+    /**
+     * The hash token started with an ident sequence (e.g. `#foo`)
+     * Only hash tokens with the "id" type are valid ID selectors.
+     */
+    ID = "id"
+}
+
+/**
+ * Assert that a given value has the general structure of a CSS token:
+ * 1. is an array.
+ * 2. has at least four items.
+ * 3. has a known token type.
+ * 4. has a string representation.
+ * 5. has a start position.
+ * 6. has an end position.
+ */
+export declare function isToken(x: any): x is CSSToken;
+
+export declare function isTokenAtKeyword(x?: CSSToken | null): x is TokenAtKeyword;
+
+export declare function isTokenBadString(x?: CSSToken | null): x is TokenBadString;
+
+export declare function isTokenBadURL(x?: CSSToken | null): x is TokenBadURL;
+
+export declare function isTokenCDC(x?: CSSToken | null): x is TokenCDC;
+
+export declare function isTokenCDO(x?: CSSToken | null): x is TokenCDO;
+
+export declare function isTokenCloseCurly(x?: CSSToken | null): x is TokenCloseCurly;
+
+export declare function isTokenCloseParen(x?: CSSToken | null): x is TokenCloseParen;
+
+export declare function isTokenCloseSquare(x?: CSSToken | null): x is TokenCloseSquare;
+
+export declare function isTokenColon(x?: CSSToken | null): x is TokenColon;
+
+export declare function isTokenComma(x?: CSSToken | null): x is TokenComma;
+
+export declare function isTokenComment(x?: CSSToken | null): x is TokenComment;
+
+export declare function isTokenDelim(x?: CSSToken | null): x is TokenDelim;
+
+export declare function isTokenDimension(x?: CSSToken | null): x is TokenDimension;
+
+export declare function isTokenEOF(x?: CSSToken | null): x is TokenEOF;
+
+export declare function isTokenFunction(x?: CSSToken | null): x is TokenFunction;
+
+export declare function isTokenHash(x?: CSSToken | null): x is TokenHash;
+
+export declare function isTokenIdent(x?: CSSToken | null): x is TokenIdent;
+
+export declare function isTokenNumber(x?: CSSToken | null): x is TokenNumber;
+
+/**
+ * Assert that a token is a numeric token
+ */
+export declare function isTokenNumeric(x?: CSSToken | null): x is NumericToken;
+
+export declare function isTokenOpenCurly(x?: CSSToken | null): x is TokenOpenCurly;
+
+export declare function isTokenOpenParen(x?: CSSToken | null): x is TokenOpenParen;
+
+export declare function isTokenOpenSquare(x?: CSSToken | null): x is TokenOpenSquare;
+
+export declare function isTokenPercentage(x?: CSSToken | null): x is TokenPercentage;
+
+export declare function isTokenSemicolon(x?: CSSToken | null): x is TokenSemicolon;
+
+export declare function isTokenString(x?: CSSToken | null): x is TokenString;
+
+export declare function isTokenUnicodeRange(x?: CSSToken | null): x is TokenUnicodeRange;
+
+export declare function isTokenURL(x?: CSSToken | null): x is TokenURL;
+
+export declare function isTokenWhitespace(x?: CSSToken | null): x is TokenWhitespace;
+
+/**
+ * Assert that a token is a whitespace or comment token
+ */
+export declare function isTokenWhiteSpaceOrComment(x?: CSSToken | null): x is TokenWhitespace | TokenComment;
+
+/**
+ * Get the mirror variant of a given token
+ *
+ * @example
+ *
+ * ```js
+ * const input = [TokenType.OpenParen, '(', 0, 1, undefined];
+ * const output = mirrorVariant(input);
+ *
+ * console.log(output); // [TokenType.CloseParen, ')', -1, -1, undefined]
+ * ```
+ */
+export declare function mirrorVariant(token: CSSToken): CSSToken | null;
+
+/**
+ * Get the mirror variant type of a given token type
+ *
+ * @example
+ *
+ * ```js
+ * const input = TokenType.OpenParen;
+ * const output = mirrorVariantType(input);
+ *
+ * console.log(output); // TokenType.CloseParen
+ * ```
+ */
+export declare function mirrorVariantType(type: TokenType): TokenType | null;
+
+/**
+ * Set the ident value and update the string representation.
+ * This handles escaping.
+ */
+export declare function mutateIdent(ident: TokenIdent, newValue: string): void;
+
+/**
+ * Set the unit and update the string representation.
+ * This handles escaping.
+ */
+export declare function mutateUnit(ident: TokenDimension, newUnit: string): void;
+
+/**
+ * The type of number token
+ * Either `integer` or `number`
+ */
+export declare enum NumberType {
+    Integer = "integer",
+    Number = "number"
+}
+
+/**
+ * The union of all possible CSS tokens that represent a numeric value
+ */
+export declare type NumericToken = TokenDimension | TokenNumber | TokenPercentage;
+
+/**
+ * The CSS Tokenizer is forgiving and will never throw on invalid input.
+ * Any errors are reported through the `onParseError` callback.
+ */
+export declare class ParseError extends Error {
+    /** The index of the start character of the current token. */
+    sourceStart: number;
+    /** The index of the end character of the current token. */
+    sourceEnd: number;
+    /** The parser steps that preceded the error. */
+    parserState: Array<string>;
+    constructor(message: string, sourceStart: number, sourceEnd: number, parserState: Array<string>);
+}
+
+export declare const ParseErrorMessage: {
+    UnexpectedNewLineInString: string;
+    UnexpectedEOFInString: string;
+    UnexpectedEOFInComment: string;
+    UnexpectedEOFInURL: string;
+    UnexpectedEOFInEscapedCodePoint: string;
+    UnexpectedCharacterInURL: string;
+    InvalidEscapeSequenceInURL: string;
+    InvalidEscapeSequenceAfterBackslash: string;
+};
+
+export declare class ParseErrorWithToken extends ParseError {
+    /** The associated token. */
+    token: CSSToken;
+    constructor(message: string, sourceStart: number, sourceEnd: number, parserState: Array<string>, token: CSSToken);
+}
+
+/**
+ * Concatenate the string representation of a list of tokens.
+ * This is not a proper serializer that will handle escaping and whitespace.
+ * It only produces valid CSS for a token list that is also valid.
+ */
+export declare function stringify(...tokens: Array<CSSToken>): string;
+
+/**
+ * The CSS Token interface
+ *
+ * @remarks
+ * CSS Tokens are fully typed and have a strict structure.
+ * This makes it easier to iterate and analyze a token stream.
+ *
+ * The string representation and the parsed value are stored separately for many token types.
+ * It is always assumed that the string representation will be used when stringifying, while the parsed value should be used when analyzing tokens.
+ */
+export declare interface Token<T extends TokenType, U> extends Array<T | string | number | U> {
+    /**
+     * The type of token
+     */
+    0: T;
+    /**
+     * The token representation
+     *
+     * @remarks
+     * This field will be used when stringifying the token.
+     * Any stored value is assumed to be valid CSS.
+     *
+     * You should never use this field when analyzing the token when there is a parsed value available.
+     * But you must store mutated values here.
+     */
+    1: string;
+    /**
+     * Start position of representation
+     */
+    2: number;
+    /**
+     * End position of representation
+     */
+    3: number;
+    /**
+     * Extra data
+     *
+     * @remarks
+     * This holds the parsed value of each token.
+     * These values are unescaped, unquoted, converted to numbers, etc.
+     *
+     * You should always use this field when analyzing the token.
+     * But you must not assume that mutating only this field will have any effect.
+     */
+    4: U;
+}
+
+export declare interface TokenAtKeyword extends Token<TokenType.AtKeyword, {
+    /**
+     * The unescaped at-keyword name without the leading `@`.
+     */
+    value: string;
+}> {
+}
+
+export declare interface TokenBadString extends Token<TokenType.BadString, undefined> {
+}
+
+export declare interface TokenBadURL extends Token<TokenType.BadURL, undefined> {
+}
+
+export declare interface TokenCDC extends Token<TokenType.CDC, undefined> {
+}
+
+export declare interface TokenCDO extends Token<TokenType.CDO, undefined> {
+}
+
+export declare interface TokenCloseCurly extends Token<TokenType.CloseCurly, undefined> {
+}
+
+export declare interface TokenCloseParen extends Token<TokenType.CloseParen, undefined> {
+}
+
+export declare interface TokenCloseSquare extends Token<TokenType.CloseSquare, undefined> {
+}
+
+export declare interface TokenColon extends Token<TokenType.Colon, undefined> {
+}
+
+export declare interface TokenComma extends Token<TokenType.Comma, undefined> {
+}
+
+export declare interface TokenComment extends Token<TokenType.Comment, undefined> {
+}
+
+export declare interface TokenDelim extends Token<TokenType.Delim, {
+    /**
+     * The delim character.
+     */
+    value: string;
+}> {
+}
+
+export declare interface TokenDimension extends Token<TokenType.Dimension, {
+    /**
+     * The numeric value.
+     */
+    value: number;
+    /**
+     * The unescaped unit name.
+     */
+    unit: string;
+    /**
+     * `integer` or `number`
+     */
+    type: NumberType;
+    /**
+     * The sign character as it appeared in the source.
+     * This is only useful if you need to determine if a value was written as "2px" or "+2px".
+     */
+    signCharacter?: '+' | '-';
+}> {
+}
+
+export declare interface TokenEOF extends Token<TokenType.EOF, undefined> {
+}
+
+export declare interface TokenFunction extends Token<TokenType.Function, {
+    /**
+     * The unescaped function name without the trailing `(`.
+     */
+    value: string;
+}> {
+}
+
+export declare interface TokenHash extends Token<TokenType.Hash, {
+    /**
+     * The unescaped hash value without the leading `#`.
+     */
+    value: string;
+    /**
+     * The hash type.
+     */
+    type: HashType;
+}> {
+}
+
+export declare interface TokenIdent extends Token<TokenType.Ident, {
+    /**
+     * The unescaped ident value.
+     */
+    value: string;
+}> {
+}
+
+/**
+ * Tokenize a CSS string into a list of tokens.
+ */
+export declare function tokenize(input: {
+    css: {
+        valueOf(): string;
+    };
+    unicodeRangesAllowed?: boolean;
+}, options?: {
+    onParseError?: (error: ParseError) => void;
+}): Array<CSSToken>;
+
+/**
+ * Create a tokenizer for a CSS string.
+ */
+export declare function tokenizer(input: {
+    css: {
+        valueOf(): string;
+    };
+    unicodeRangesAllowed?: boolean;
+}, options?: {
+    onParseError?: (error: ParseError) => void;
+}): {
+    nextToken: () => CSSToken;
+    endOfFile: () => boolean;
+};
+
+export declare interface TokenNumber extends Token<TokenType.Number, {
+    /**
+     * The numeric value.
+     */
+    value: number;
+    /**
+     * `integer` or `number`
+     */
+    type: NumberType;
+    /**
+     * The sign character as it appeared in the source.
+     * This is only useful if you need to determine if a value was written as "2" or "+2".
+     */
+    signCharacter?: '+' | '-';
+}> {
+}
+
+export declare interface TokenOpenCurly extends Token<TokenType.OpenCurly, undefined> {
+}
+
+export declare interface TokenOpenParen extends Token<TokenType.OpenParen, undefined> {
+}
+
+export declare interface TokenOpenSquare extends Token<TokenType.OpenSquare, undefined> {
+}
+
+export declare interface TokenPercentage extends Token<TokenType.Percentage, {
+    /**
+     * The numeric value.
+     */
+    value: number;
+    /**
+     * The sign character as it appeared in the source.
+     * This is only useful if you need to determine if a value was written as "2%" or "+2%".
+     */
+    signCharacter?: '+' | '-';
+}> {
+}
+
+export declare interface TokenSemicolon extends Token<TokenType.Semicolon, undefined> {
+}
+
+export declare interface TokenString extends Token<TokenType.String, {
+    /**
+     * The unescaped string value without the leading and trailing quotes.
+     */
+    value: string;
+}> {
+}
+
+/**
+ * All possible CSS token types
+ */
+export declare enum TokenType {
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#comment-diagram}
+     */
+    Comment = "comment",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-at-keyword-token}
+     */
+    AtKeyword = "at-keyword-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-string-token}
+     */
+    BadString = "bad-string-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-url-token}
+     */
+    BadURL = "bad-url-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdc-token}
+     */
+    CDC = "CDC-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdo-token}
+     */
+    CDO = "CDO-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-colon-token}
+     */
+    Colon = "colon-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-comma-token}
+     */
+    Comma = "comma-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-delim-token}
+     */
+    Delim = "delim-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-dimension-token}
+     */
+    Dimension = "dimension-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-eof-token}
+     */
+    EOF = "EOF-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-function-token}
+     */
+    Function = "function-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-hash-token}
+     */
+    Hash = "hash-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-ident-token}
+     */
+    Ident = "ident-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token}
+     */
+    Number = "number-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token}
+     */
+    Percentage = "percentage-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-semicolon-token}
+     */
+    Semicolon = "semicolon-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-string-token}
+     */
+    String = "string-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-url-token}
+     */
+    URL = "url-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-whitespace-token}
+     */
+    Whitespace = "whitespace-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-paren}
+     */
+    OpenParen = "(-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-paren}
+     */
+    CloseParen = ")-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-square}
+     */
+    OpenSquare = "[-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-square}
+     */
+    CloseSquare = "]-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-curly}
+     */
+    OpenCurly = "{-token",
+    /**
+     * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-curly}
+     */
+    CloseCurly = "}-token",
+    /**
+     * Only appears in the token stream when the `unicodeRangesAllowed` option is set to true.
+     *
+     * @example
+     * ```js
+     * import { tokenize } from '@csstools/css-tokenizer';
+     *
+     * const tokens = tokenize({
+     * 	css: `U+0025-00FF, U+4??`,
+     * 	unicodeRangesAllowed: true,
+     * });
+     *
+     * console.log(tokens);
+     * ```
+     *
+     * @see {@link https://drafts.csswg.org/css-syntax/#typedef-unicode-range-token}
+     */
+    UnicodeRange = "unicode-range-token"
+}
+
+export declare interface TokenUnicodeRange extends Token<TokenType.UnicodeRange, {
+    startOfRange: number;
+    endOfRange: number;
+}> {
+}
+
+export declare interface TokenURL extends Token<TokenType.URL, {
+    /**
+     * The unescaped URL value without the leading `url(` and trailing `)`.
+     */
+    value: string;
+}> {
+}
+
+export declare interface TokenWhitespace extends Token<TokenType.Whitespace, undefined> {
+}
+
+export { }

data/lib/readability_js/node/node_modules/@csstools/css-tokenizer/dist/index.mjs

@@ -0,0 +1 @@
+class ParseError extends Error{sourceStart;sourceEnd;parserState;constructor(e,n,t,o){super(e),this.name="ParseError",this.sourceStart=n,this.sourceEnd=t,this.parserState=o}}class ParseErrorWithToken extends ParseError{token;constructor(e,n,t,o,r){super(e,n,t,o),this.token=r}}const e={UnexpectedNewLineInString:"Unexpected newline while consuming a string token.",UnexpectedEOFInString:"Unexpected EOF while consuming a string token.",UnexpectedEOFInComment:"Unexpected EOF while consuming a comment.",UnexpectedEOFInURL:"Unexpected EOF while consuming a url token.",UnexpectedEOFInEscapedCodePoint:"Unexpected EOF while consuming an escaped code point.",UnexpectedCharacterInURL:"Unexpected character while consuming a url token.",InvalidEscapeSequenceInURL:"Invalid escape sequence while consuming a url token.",InvalidEscapeSequenceAfterBackslash:'Invalid escape sequence after "\\"'},n="undefined"!=typeof globalThis&&"structuredClone"in globalThis;function cloneTokens(e){return n?structuredClone(e):JSON.parse(JSON.stringify(e))}function stringify(...e){let n="";for(let t=0;t<e.length;t++)n+=e[t][1];return n}const t=13,o=45,r=10,i=43,s=65533;function checkIfFourCodePointsWouldStartCDO(e){return 60===e.source.codePointAt(e.cursor)&&33===e.source.codePointAt(e.cursor+1)&&e.source.codePointAt(e.cursor+2)===o&&e.source.codePointAt(e.cursor+3)===o}function isDigitCodePoint(e){return e>=48&&e<=57}function isUppercaseLetterCodePoint(e){return e>=65&&e<=90}function isLowercaseLetterCodePoint(e){return e>=97&&e<=122}function isHexDigitCodePoint(e){return e>=48&&e<=57||e>=97&&e<=102||e>=65&&e<=70}function isLetterCodePoint(e){return isLowercaseLetterCodePoint(e)||isUppercaseLetterCodePoint(e)}function isIdentStartCodePoint(e){return isLetterCodePoint(e)||isNonASCII_IdentCodePoint(e)||95===e}function isIdentCodePoint(e){return isIdentStartCodePoint(e)||isDigitCodePoint(e)||e===o}function isNonASCII_IdentCodePoint(e){return 183===e||8204===e||8205===e||8255===e||8256===e||8204===e||(192<=e&&e<=214||216<=e&&e<=246||248<=e&&e<=893||895<=e&&e<=8191||8304<=e&&e<=8591||11264<=e&&e<=12271||12289<=e&&e<=55295||63744<=e&&e<=64975||65008<=e&&e<=65533||(0===e||(!!isSurrogate(e)||e>=65536)))}function isNewLine(e){return e===r||e===t||12===e}function isWhitespace(e){return 32===e||e===r||9===e||e===t||12===e}function isSurrogate(e){return e>=55296&&e<=57343}function checkIfTwoCodePointsAreAValidEscape(e){return 92===e.source.codePointAt(e.cursor)&&!isNewLine(e.source.codePointAt(e.cursor+1)??-1)}function checkIfThreeCodePointsWouldStartAnIdentSequence(e,n){return n.source.codePointAt(n.cursor)===o?n.source.codePointAt(n.cursor+1)===o||(!!isIdentStartCodePoint(n.source.codePointAt(n.cursor+1)??-1)||92===n.source.codePointAt(n.cursor+1)&&!isNewLine(n.source.codePointAt(n.cursor+2)??-1)):!!isIdentStartCodePoint(n.source.codePointAt(n.cursor)??-1)||checkIfTwoCodePointsAreAValidEscape(n)}function checkIfThreeCodePointsWouldStartANumber(e){return e.source.codePointAt(e.cursor)===i||e.source.codePointAt(e.cursor)===o?!!isDigitCodePoint(e.source.codePointAt(e.cursor+1)??-1)||46===e.source.codePointAt(e.cursor+1)&&isDigitCodePoint(e.source.codePointAt(e.cursor+2)??-1):46===e.source.codePointAt(e.cursor)?isDigitCodePoint(e.source.codePointAt(e.cursor+1)??-1):isDigitCodePoint(e.source.codePointAt(e.cursor)??-1)}function checkIfTwoCodePointsStartAComment(e){return 47===e.source.codePointAt(e.cursor)&&42===e.source.codePointAt(e.cursor+1)}function checkIfThreeCodePointsWouldStartCDC(e){return e.source.codePointAt(e.cursor)===o&&e.source.codePointAt(e.cursor+1)===o&&62===e.source.codePointAt(e.cursor+2)}var c,a,u;function mirrorVariantType(e){switch(e){case c.OpenParen:return c.CloseParen;case c.CloseParen:return c.OpenParen;case c.OpenCurly:return c.CloseCurly;case c.CloseCurly:return c.OpenCurly;case c.OpenSquare:return c.CloseSquare;case c.CloseSquare:return c.OpenSquare;default:return null}}function mirrorVariant(e){switch(e[0]){case c.OpenParen:return[c.CloseParen,")",-1,-1,void 0];case c.CloseParen:return[c.OpenParen,"(",-1,-1,void 0];case c.OpenCurly:return[c.CloseCurly,"}",-1,-1,void 0];case c.CloseCurly:return[c.OpenCurly,"{",-1,-1,void 0];case c.OpenSquare:return[c.CloseSquare,"]",-1,-1,void 0];case c.CloseSquare:return[c.OpenSquare,"[",-1,-1,void 0];default:return null}}function consumeComment(n,t){for(t.advanceCodePoint(2);;){const o=t.readCodePoint();if(void 0===o){const o=[c.Comment,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInComment,t.representationStart,t.representationEnd,["4.3.2. Consume comments","Unexpected EOF"],o)),o}if(42===o&&(void 0!==t.source.codePointAt(t.cursor)&&47===t.source.codePointAt(t.cursor))){t.advanceCodePoint();break}}return[c.Comment,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0]}function consumeEscapedCodePoint(n,o){const i=o.readCodePoint();if(void 0===i)return n.onParseError(new ParseError(e.UnexpectedEOFInEscapedCodePoint,o.representationStart,o.representationEnd,["4.3.7. Consume an escaped code point","Unexpected EOF"])),s;if(isHexDigitCodePoint(i)){const e=[i];let n;for(;void 0!==(n=o.source.codePointAt(o.cursor))&&isHexDigitCodePoint(n)&&e.length<6;)e.push(n),o.advanceCodePoint();isWhitespace(o.source.codePointAt(o.cursor)??-1)&&(o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r&&o.advanceCodePoint(),o.advanceCodePoint());const c=parseInt(String.fromCodePoint(...e),16);return 0===c||isSurrogate(c)||c>1114111?s:c}return 0===i||isSurrogate(i)?s:i}function consumeIdentSequence(e,n){const t=[];for(;;){const o=n.source.codePointAt(n.cursor)??-1;if(0===o||isSurrogate(o))t.push(s),n.advanceCodePoint(+(o>65535)+1);else if(isIdentCodePoint(o))t.push(o),n.advanceCodePoint(+(o>65535)+1);else{if(!checkIfTwoCodePointsAreAValidEscape(n))return t;n.advanceCodePoint(),t.push(consumeEscapedCodePoint(e,n))}}}function consumeHashToken(e,n){n.advanceCodePoint();const t=n.source.codePointAt(n.cursor);if(void 0!==t&&(isIdentCodePoint(t)||checkIfTwoCodePointsAreAValidEscape(n))){let t=u.Unrestricted;checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)&&(t=u.ID);const o=consumeIdentSequence(e,n);return[c.Hash,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...o),type:t}]}return[c.Delim,"#",n.representationStart,n.representationEnd,{value:"#"}]}function consumeNumber(e,n){let t=a.Integer;for(n.source.codePointAt(n.cursor)!==i&&n.source.codePointAt(n.cursor)!==o||n.advanceCodePoint();isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint();if(46===n.source.codePointAt(n.cursor)&&isDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))for(n.advanceCodePoint(2),t=a.Number;isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint();if(101===n.source.codePointAt(n.cursor)||69===n.source.codePointAt(n.cursor)){if(isDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))n.advanceCodePoint(2);else{if(n.source.codePointAt(n.cursor+1)!==o&&n.source.codePointAt(n.cursor+1)!==i||!isDigitCodePoint(n.source.codePointAt(n.cursor+2)??-1))return t;n.advanceCodePoint(3)}for(t=a.Number;isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint()}return t}function consumeNumericToken(e,n){let t;{const e=n.source.codePointAt(n.cursor);e===o?t="-":e===i&&(t="+")}const r=consumeNumber(0,n),s=parseFloat(n.source.slice(n.representationStart,n.representationEnd+1));if(checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)){const o=consumeIdentSequence(e,n);return[c.Dimension,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t,type:r,unit:String.fromCodePoint(...o)}]}return 37===n.source.codePointAt(n.cursor)?(n.advanceCodePoint(),[c.Percentage,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t}]):[c.Number,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t,type:r}]}function consumeWhiteSpace(e){for(;isWhitespace(e.source.codePointAt(e.cursor)??-1);)e.advanceCodePoint();return[c.Whitespace,e.source.slice(e.representationStart,e.representationEnd+1),e.representationStart,e.representationEnd,void 0]}!function(e){e.Comment="comment",e.AtKeyword="at-keyword-token",e.BadString="bad-string-token",e.BadURL="bad-url-token",e.CDC="CDC-token",e.CDO="CDO-token",e.Colon="colon-token",e.Comma="comma-token",e.Delim="delim-token",e.Dimension="dimension-token",e.EOF="EOF-token",e.Function="function-token",e.Hash="hash-token",e.Ident="ident-token",e.Number="number-token",e.Percentage="percentage-token",e.Semicolon="semicolon-token",e.String="string-token",e.URL="url-token",e.Whitespace="whitespace-token",e.OpenParen="(-token",e.CloseParen=")-token",e.OpenSquare="[-token",e.CloseSquare="]-token",e.OpenCurly="{-token",e.CloseCurly="}-token",e.UnicodeRange="unicode-range-token"}(c||(c={})),function(e){e.Integer="integer",e.Number="number"}(a||(a={})),function(e){e.Unrestricted="unrestricted",e.ID="id"}(u||(u={}));class Reader{cursor=0;source="";representationStart=0;representationEnd=-1;constructor(e){this.source=e}advanceCodePoint(e=1){this.cursor=this.cursor+e,this.representationEnd=this.cursor-1}readCodePoint(){const e=this.source.codePointAt(this.cursor);if(void 0!==e)return this.cursor=this.cursor+1,this.representationEnd=this.cursor-1,e}unreadCodePoint(e=1){this.cursor=this.cursor-e,this.representationEnd=this.cursor-1}resetRepresentation(){this.representationStart=this.cursor,this.representationEnd=-1}}function consumeStringToken(n,o){let i="";const a=o.readCodePoint();for(;;){const u=o.readCodePoint();if(void 0===u){const t=[c.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:i}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInString,o.representationStart,o.representationEnd,["4.3.5. Consume a string token","Unexpected EOF"],t)),t}if(isNewLine(u)){o.unreadCodePoint();const i=[c.BadString,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedNewLineInString,o.representationStart,o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r?o.representationEnd+2:o.representationEnd+1,["4.3.5. Consume a string token","Unexpected newline"],i)),i}if(u===a)return[c.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:i}];if(92!==u)0===u||isSurrogate(u)?i+=String.fromCodePoint(s):i+=String.fromCodePoint(u);else{if(void 0===o.source.codePointAt(o.cursor))continue;if(isNewLine(o.source.codePointAt(o.cursor)??-1)){o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r&&o.advanceCodePoint(),o.advanceCodePoint();continue}i+=String.fromCodePoint(consumeEscapedCodePoint(n,o))}}}function checkIfCodePointsMatchURLIdent(e){return!(3!==e.length||117!==e[0]&&85!==e[0]||114!==e[1]&&82!==e[1]||108!==e[2]&&76!==e[2])}function consumeBadURL(e,n){for(;;){const t=n.source.codePointAt(n.cursor);if(void 0===t)return;if(41===t)return void n.advanceCodePoint();checkIfTwoCodePointsAreAValidEscape(n)?(n.advanceCodePoint(),consumeEscapedCodePoint(e,n)):n.advanceCodePoint()}}function consumeUrlToken(n,t){for(;isWhitespace(t.source.codePointAt(t.cursor)??-1);)t.advanceCodePoint();let o="";for(;;){if(void 0===t.source.codePointAt(t.cursor)){const r=[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected EOF"],r)),r}if(41===t.source.codePointAt(t.cursor))return t.advanceCodePoint(),[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];if(isWhitespace(t.source.codePointAt(t.cursor)??-1)){for(t.advanceCodePoint();isWhitespace(t.source.codePointAt(t.cursor)??-1);)t.advanceCodePoint();if(void 0===t.source.codePointAt(t.cursor)){const r=[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Consume as much whitespace as possible","Unexpected EOF"],r)),r}return 41===t.source.codePointAt(t.cursor)?(t.advanceCodePoint(),[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}]):(consumeBadURL(n,t),[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0])}const i=t.source.codePointAt(t.cursor);if(34===i||39===i||40===i||(11===(r=i??-1)||127===r||0<=r&&r<=8||14<=r&&r<=31)){consumeBadURL(n,t);const o=[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedCharacterInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected U+0022 QUOTATION MARK (\"), U+0027 APOSTROPHE ('), U+0028 LEFT PARENTHESIS (() or non-printable code point"],o)),o}if(92===i){if(checkIfTwoCodePointsAreAValidEscape(t)){t.advanceCodePoint(),o+=String.fromCodePoint(consumeEscapedCodePoint(n,t));continue}consumeBadURL(n,t);const r=[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],r)),r}0===t.source.codePointAt(t.cursor)||isSurrogate(t.source.codePointAt(t.cursor)??-1)?(o+=String.fromCodePoint(s),t.advanceCodePoint()):(o+=t.source[t.cursor],t.advanceCodePoint())}var r}function consumeIdentLikeToken(e,n){const t=consumeIdentSequence(e,n);if(40!==n.source.codePointAt(n.cursor))return[c.Ident,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];if(checkIfCodePointsMatchURLIdent(t)){n.advanceCodePoint();let o=0;for(;;){const e=isWhitespace(n.source.codePointAt(n.cursor)??-1),r=isWhitespace(n.source.codePointAt(n.cursor+1)??-1);if(e&&r){o+=1,n.advanceCodePoint(1);continue}const i=e?n.source.codePointAt(n.cursor+1):n.source.codePointAt(n.cursor);if(34===i||39===i)return o>0&&n.unreadCodePoint(o),[c.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];break}return consumeUrlToken(e,n)}return n.advanceCodePoint(),[c.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}]}function checkIfThreeCodePointsWouldStartAUnicodeRange(e){return!(117!==e.source.codePointAt(e.cursor)&&85!==e.source.codePointAt(e.cursor)||e.source.codePointAt(e.cursor+1)!==i||63!==e.source.codePointAt(e.cursor+2)&&!isHexDigitCodePoint(e.source.codePointAt(e.cursor+2)??-1))}function consumeUnicodeRangeToken(e,n){n.advanceCodePoint(2);const t=[],r=[];let i;for(;void 0!==(i=n.source.codePointAt(n.cursor))&&t.length<6&&isHexDigitCodePoint(i);)t.push(i),n.advanceCodePoint();for(;void 0!==(i=n.source.codePointAt(n.cursor))&&t.length<6&&63===i;)0===r.length&&r.push(...t),t.push(48),r.push(70),n.advanceCodePoint();if(!r.length&&n.source.codePointAt(n.cursor)===o&&isHexDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))for(n.advanceCodePoint();void 0!==(i=n.source.codePointAt(n.cursor))&&r.length<6&&isHexDigitCodePoint(i);)r.push(i),n.advanceCodePoint();if(!r.length){const e=parseInt(String.fromCodePoint(...t),16);return[c.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:e,endOfRange:e}]}const s=parseInt(String.fromCodePoint(...t),16),a=parseInt(String.fromCodePoint(...r),16);return[c.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:s,endOfRange:a}]}function tokenize(e,n){const t=tokenizer(e,n),o=[];for(;!t.endOfFile();)o.push(t.nextToken());return o.push(t.nextToken()),o}function tokenizer(n,s){const a=n.css.valueOf(),u=n.unicodeRangesAllowed??!1,d=new Reader(a),p={onParseError:s?.onParseError??noop};return{nextToken:function nextToken(){d.resetRepresentation();const n=d.source.codePointAt(d.cursor);if(void 0===n)return[c.EOF,"",-1,-1,void 0];if(47===n&&checkIfTwoCodePointsStartAComment(d))return consumeComment(p,d);if(u&&(117===n||85===n)&&checkIfThreeCodePointsWouldStartAUnicodeRange(d))return consumeUnicodeRangeToken(0,d);if(isIdentStartCodePoint(n))return consumeIdentLikeToken(p,d);if(isDigitCodePoint(n))return consumeNumericToken(p,d);switch(n){case 44:return d.advanceCodePoint(),[c.Comma,",",d.representationStart,d.representationEnd,void 0];case 58:return d.advanceCodePoint(),[c.Colon,":",d.representationStart,d.representationEnd,void 0];case 59:return d.advanceCodePoint(),[c.Semicolon,";",d.representationStart,d.representationEnd,void 0];case 40:return d.advanceCodePoint(),[c.OpenParen,"(",d.representationStart,d.representationEnd,void 0];case 41:return d.advanceCodePoint(),[c.CloseParen,")",d.representationStart,d.representationEnd,void 0];case 91:return d.advanceCodePoint(),[c.OpenSquare,"[",d.representationStart,d.representationEnd,void 0];case 93:return d.advanceCodePoint(),[c.CloseSquare,"]",d.representationStart,d.representationEnd,void 0];case 123:return d.advanceCodePoint(),[c.OpenCurly,"{",d.representationStart,d.representationEnd,void 0];case 125:return d.advanceCodePoint(),[c.CloseCurly,"}",d.representationStart,d.representationEnd,void 0];case 39:case 34:return consumeStringToken(p,d);case 35:return consumeHashToken(p,d);case i:case 46:return checkIfThreeCodePointsWouldStartANumber(d)?consumeNumericToken(p,d):(d.advanceCodePoint(),[c.Delim,d.source[d.representationStart],d.representationStart,d.representationEnd,{value:d.source[d.representationStart]}]);case r:case t:case 12:case 9:case 32:return consumeWhiteSpace(d);case o:return checkIfThreeCodePointsWouldStartANumber(d)?consumeNumericToken(p,d):checkIfThreeCodePointsWouldStartCDC(d)?(d.advanceCodePoint(3),[c.CDC,"--\x3e",d.representationStart,d.representationEnd,void 0]):checkIfThreeCodePointsWouldStartAnIdentSequence(0,d)?consumeIdentLikeToken(p,d):(d.advanceCodePoint(),[c.Delim,"-",d.representationStart,d.representationEnd,{value:"-"}]);case 60:return checkIfFourCodePointsWouldStartCDO(d)?(d.advanceCodePoint(4),[c.CDO,"\x3c!--",d.representationStart,d.representationEnd,void 0]):(d.advanceCodePoint(),[c.Delim,"<",d.representationStart,d.representationEnd,{value:"<"}]);case 64:if(d.advanceCodePoint(),checkIfThreeCodePointsWouldStartAnIdentSequence(0,d)){const e=consumeIdentSequence(p,d);return[c.AtKeyword,d.source.slice(d.representationStart,d.representationEnd+1),d.representationStart,d.representationEnd,{value:String.fromCodePoint(...e)}]}return[c.Delim,"@",d.representationStart,d.representationEnd,{value:"@"}];case 92:{if(checkIfTwoCodePointsAreAValidEscape(d))return consumeIdentLikeToken(p,d);d.advanceCodePoint();const n=[c.Delim,"\\",d.representationStart,d.representationEnd,{value:"\\"}];return p.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceAfterBackslash,d.representationStart,d.representationEnd,["4.3.1. Consume a token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],n)),n}}return d.advanceCodePoint(),[c.Delim,d.source[d.representationStart],d.representationStart,d.representationEnd,{value:d.source[d.representationStart]}]},endOfFile:function endOfFile(){return void 0===d.source.codePointAt(d.cursor)}}}function noop(){}function mutateIdent(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=String.fromCodePoint(...serializeIdent(t));e[1]=o,e[4].value=n}function mutateUnit(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=serializeIdent(t);101===o[0]&&insertEscapedCodePoint(o,0,o[0]);const r=String.fromCodePoint(...o),i="+"===e[4].signCharacter?e[4].signCharacter:"",s=e[4].value.toString();e[1]=`${i}${s}${r}`,e[4].unit=n}function serializeIdent(e){let n=0;if(0===e[0])e.splice(0,1,s),n=1;else if(e[0]===o&&e[1]===o)n=2;else if(e[0]===o&&e[1])n=2,isIdentStartCodePoint(e[1])||(n+=insertEscapedCodePoint(e,1,e[1]));else{if(e[0]===o&&!e[1])return[92,e[0]];isIdentStartCodePoint(e[0])?n=1:(n=1,n+=insertEscapedCodePoint(e,0,e[0]))}for(let t=n;t<e.length;t++)0!==e[t]?isIdentCodePoint(e[t])||(t+=insertEscapedCharacter(e,t,e[t])):(e.splice(t,1,s),t++);return e}function insertEscapedCharacter(e,n,t){return e.splice(n,1,92,t),1}function insertEscapedCodePoint(e,n,t){const o=t.toString(16),r=[];for(const e of o)r.push(e.codePointAt(0));return e.splice(n,1,92,...r,32),1+r.length}const d=Object.values(c);function isToken(e){return!!Array.isArray(e)&&(!(e.length<4)&&(!!d.includes(e[0])&&("string"==typeof e[1]&&("number"==typeof e[2]&&"number"==typeof e[3]))))}function isTokenNumeric(e){if(!e)return!1;switch(e[0]){case c.Dimension:case c.Number:case c.Percentage:return!0;default:return!1}}function isTokenWhiteSpaceOrComment(e){if(!e)return!1;switch(e[0]){case c.Whitespace:case c.Comment:return!0;default:return!1}}function isTokenAtKeyword(e){return!!e&&e[0]===c.AtKeyword}function isTokenBadString(e){return!!e&&e[0]===c.BadString}function isTokenBadURL(e){return!!e&&e[0]===c.BadURL}function isTokenCDC(e){return!!e&&e[0]===c.CDC}function isTokenCDO(e){return!!e&&e[0]===c.CDO}function isTokenColon(e){return!!e&&e[0]===c.Colon}function isTokenComma(e){return!!e&&e[0]===c.Comma}function isTokenComment(e){return!!e&&e[0]===c.Comment}function isTokenDelim(e){return!!e&&e[0]===c.Delim}function isTokenDimension(e){return!!e&&e[0]===c.Dimension}function isTokenEOF(e){return!!e&&e[0]===c.EOF}function isTokenFunction(e){return!!e&&e[0]===c.Function}function isTokenHash(e){return!!e&&e[0]===c.Hash}function isTokenIdent(e){return!!e&&e[0]===c.Ident}function isTokenNumber(e){return!!e&&e[0]===c.Number}function isTokenPercentage(e){return!!e&&e[0]===c.Percentage}function isTokenSemicolon(e){return!!e&&e[0]===c.Semicolon}function isTokenString(e){return!!e&&e[0]===c.String}function isTokenURL(e){return!!e&&e[0]===c.URL}function isTokenWhitespace(e){return!!e&&e[0]===c.Whitespace}function isTokenOpenParen(e){return!!e&&e[0]===c.OpenParen}function isTokenCloseParen(e){return!!e&&e[0]===c.CloseParen}function isTokenOpenSquare(e){return!!e&&e[0]===c.OpenSquare}function isTokenCloseSquare(e){return!!e&&e[0]===c.CloseSquare}function isTokenOpenCurly(e){return!!e&&e[0]===c.OpenCurly}function isTokenCloseCurly(e){return!!e&&e[0]===c.CloseCurly}function isTokenUnicodeRange(e){return!!e&&e[0]===c.UnicodeRange}export{u as HashType,a as NumberType,ParseError,e as ParseErrorMessage,ParseErrorWithToken,c as TokenType,cloneTokens,isToken,isTokenAtKeyword,isTokenBadString,isTokenBadURL,isTokenCDC,isTokenCDO,isTokenCloseCurly,isTokenCloseParen,isTokenCloseSquare,isTokenColon,isTokenComma,isTokenComment,isTokenDelim,isTokenDimension,isTokenEOF,isTokenFunction,isTokenHash,isTokenIdent,isTokenNumber,isTokenNumeric,isTokenOpenCurly,isTokenOpenParen,isTokenOpenSquare,isTokenPercentage,isTokenSemicolon,isTokenString,isTokenURL,isTokenUnicodeRange,isTokenWhiteSpaceOrComment,isTokenWhitespace,mirrorVariant,mirrorVariantType,mutateIdent,mutateUnit,stringify,tokenize,tokenizer};