@guojc1/sql-formatter-compact 1.0.0

package/dist/esm/languages/trino/trino.keywords.js

@@ -0,0 +1,279 @@
+export const keywords = [
+    // https://github.com/trinodb/trino/blob/432d2897bdef99388c1a47188743a061c4ac1f34/core/trino-parser/src/main/antlr4/io/trino/sql/parser/SqlBase.g4#L858-L1128
+    'ABSENT',
+    'ADD',
+    'ADMIN',
+    'AFTER',
+    'ALL',
+    'ALTER',
+    'ANALYZE',
+    'AND',
+    'ANY',
+    'AS',
+    'ASC',
+    'AT',
+    'AUTHORIZATION',
+    'BERNOULLI',
+    'BETWEEN',
+    'BOTH',
+    'BY',
+    'CALL',
+    'CASCADE',
+    'CASE',
+    'CATALOGS',
+    'COLUMN',
+    'COLUMNS',
+    'COMMENT',
+    'COMMIT',
+    'COMMITTED',
+    'CONDITIONAL',
+    'CONSTRAINT',
+    'COPARTITION',
+    'CREATE',
+    'CROSS',
+    'CUBE',
+    'CURRENT',
+    'CURRENT_PATH',
+    'CURRENT_ROLE',
+    'DATA',
+    'DEALLOCATE',
+    'DEFAULT',
+    'DEFINE',
+    'DEFINER',
+    'DELETE',
+    'DENY',
+    'DESC',
+    'DESCRIBE',
+    'DESCRIPTOR',
+    'DISTINCT',
+    'DISTRIBUTED',
+    'DOUBLE',
+    'DROP',
+    'ELSE',
+    'EMPTY',
+    'ENCODING',
+    'END',
+    'ERROR',
+    'ESCAPE',
+    'EXCEPT',
+    'EXCLUDING',
+    'EXECUTE',
+    'EXISTS',
+    'EXPLAIN',
+    'FALSE',
+    'FETCH',
+    'FINAL',
+    'FIRST',
+    'FOLLOWING',
+    'FOR',
+    'FROM',
+    'FULL',
+    'FUNCTIONS',
+    'GRANT',
+    'GRANTED',
+    'GRANTS',
+    'GRAPHVIZ',
+    'GROUP',
+    'GROUPING',
+    'GROUPS',
+    'HAVING',
+    'IGNORE',
+    'IN',
+    'INCLUDING',
+    'INITIAL',
+    'INNER',
+    'INPUT',
+    'INSERT',
+    'INTERSECT',
+    'INTERVAL',
+    'INTO',
+    'INVOKER',
+    'IO',
+    'IS',
+    'ISOLATION',
+    'JOIN',
+    'JSON',
+    'JSON_ARRAY',
+    'JSON_OBJECT',
+    'KEEP',
+    'KEY',
+    'KEYS',
+    'LAST',
+    'LATERAL',
+    'LEADING',
+    'LEFT',
+    'LEVEL',
+    'LIKE',
+    'LIMIT',
+    'LOCAL',
+    'LOGICAL',
+    'MATCH',
+    'MATCHED',
+    'MATCHES',
+    'MATCH_RECOGNIZE',
+    'MATERIALIZED',
+    'MEASURES',
+    'NATURAL',
+    'NEXT',
+    'NFC',
+    'NFD',
+    'NFKC',
+    'NFKD',
+    'NO',
+    'NONE',
+    'NOT',
+    'NULL',
+    'NULLS',
+    'OBJECT',
+    'OF',
+    'OFFSET',
+    'OMIT',
+    'ON',
+    'ONE',
+    'ONLY',
+    'OPTION',
+    'OR',
+    'ORDER',
+    'ORDINALITY',
+    'OUTER',
+    'OUTPUT',
+    'OVER',
+    'OVERFLOW',
+    'PARTITION',
+    'PARTITIONS',
+    'PASSING',
+    'PAST',
+    'PATH',
+    'PATTERN',
+    'PER',
+    'PERMUTE',
+    'PRECEDING',
+    'PRECISION',
+    'PREPARE',
+    'PRIVILEGES',
+    'PROPERTIES',
+    'PRUNE',
+    'QUOTES',
+    'RANGE',
+    'READ',
+    'RECURSIVE',
+    'REFRESH',
+    'RENAME',
+    'REPEATABLE',
+    'RESET',
+    'RESPECT',
+    'RESTRICT',
+    'RETURNING',
+    'REVOKE',
+    'RIGHT',
+    'ROLE',
+    'ROLES',
+    'ROLLBACK',
+    'ROLLUP',
+    'ROW',
+    'ROWS',
+    'RUNNING',
+    'SCALAR',
+    'SCHEMA',
+    'SCHEMAS',
+    'SECURITY',
+    'SEEK',
+    'SELECT',
+    'SERIALIZABLE',
+    'SESSION',
+    'SET',
+    'SETS',
+    'SHOW',
+    'SKIP',
+    'SOME',
+    'START',
+    'STATS',
+    'STRING',
+    'SUBSET',
+    'SYSTEM',
+    'TABLE',
+    'TABLES',
+    'TABLESAMPLE',
+    'TEXT',
+    'THEN',
+    'TIES',
+    'TIME',
+    'TIMESTAMP',
+    'TO',
+    'TRAILING',
+    'TRANSACTION',
+    'TRUE',
+    'TYPE',
+    'UESCAPE',
+    'UNBOUNDED',
+    'UNCOMMITTED',
+    'UNCONDITIONAL',
+    'UNION',
+    'UNIQUE',
+    'UNKNOWN',
+    'UNMATCHED',
+    'UNNEST',
+    'UPDATE',
+    'USE',
+    'USER',
+    'USING',
+    'UTF16',
+    'UTF32',
+    'UTF8',
+    'VALIDATE',
+    'VALUE',
+    'VALUES',
+    'VERBOSE',
+    'VIEW',
+    'WHEN',
+    'WHERE',
+    'WINDOW',
+    'WITH',
+    'WITHIN',
+    'WITHOUT',
+    'WORK',
+    'WRAPPER',
+    'WRITE',
+    'ZONE',
+];
+export const dataTypes = [
+    // https://github.com/trinodb/trino/blob/432d2897bdef99388c1a47188743a061c4ac1f34/core/trino-main/src/main/java/io/trino/metadata/TypeRegistry.java#L131-L168
+    // or https://trino.io/docs/current/language/types.html
+    'BIGINT',
+    'INT',
+    'INTEGER',
+    'SMALLINT',
+    'TINYINT',
+    'BOOLEAN',
+    'DATE',
+    'DECIMAL',
+    'REAL',
+    'DOUBLE',
+    'HYPERLOGLOG',
+    'QDIGEST',
+    'TDIGEST',
+    'P4HYPERLOGLOG',
+    'INTERVAL',
+    'TIMESTAMP',
+    'TIME',
+    'VARBINARY',
+    'VARCHAR',
+    'CHAR',
+    'ROW',
+    'ARRAY',
+    'MAP',
+    'JSON',
+    'JSON2016',
+    'IPADDRESS',
+    'GEOMETRY',
+    'UUID',
+    'SETDIGEST',
+    'JONIREGEXP',
+    'RE2JREGEXP',
+    'LIKEPATTERN',
+    'COLOR',
+    'CODEPOINTS',
+    'FUNCTION',
+    'JSONPATH',
+];
+//# sourceMappingURL=trino.keywords.js.map

package/dist/esm/languages/trino/trino.keywords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trino.keywords.js","sourceRoot":"","sources":["../../../../src/languages/trino/trino.keywords.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC,6JAA6J;IAC7J,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO;IACP,KAAK;IACL,OAAO;IACP,SAAS;IACT,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,eAAe;IACf,WAAW;IACX,SAAS;IACT,MAAM;IACN,IAAI;IACJ,MAAM;IACN,SAAS;IACT,MAAM;IACN,UAAU;IACV,QAAQ;IACR,SAAS;IACT,SAAS;IACT,QAAQ;IACR,WAAW;IACX,aAAa;IACb,YAAY;IACZ,aAAa;IACb,QAAQ;IACR,OAAO;IACP,MAAM;IACN,SAAS;IACT,cAAc;IACd,cAAc;IACd,MAAM;IACN,YAAY;IACZ,SAAS;IACT,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,MAAM;IACN,MAAM;IACN,UAAU;IACV,YAAY;IACZ,UAAU;IACV,aAAa;IACb,QAAQ;IACR,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,KAAK;IACL,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,SAAS;IACT,QAAQ;IACR,SAAS;IACT,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,WAAW;IACX,KAAK;IACL,MAAM;IACN,MAAM;IACN,WAAW;IACX,OAAO;IACP,SAAS;IACT,QAAQ;IACR,UAAU;IACV,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,WAAW;IACX,SAAS;IACT,OAAO;IACP,OAAO;IACP,QAAQ;IACR,WAAW;IACX,UAAU;IACV,MAAM;IACN,SAAS;IACT,IAAI;IACJ,IAAI;IACJ,WAAW;IACX,MAAM;IACN,MAAM;IACN,YAAY;IACZ,aAAa;IACb,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,SAAS;IACT,SAAS;IACT,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,SAAS;IACT,OAAO;IACP,SAAS;IACT,SAAS;IACT,iBAAiB;IACjB,cAAc;IACd,UAAU;IACV,SAAS;IACT,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,QAAQ;IACR,IAAI;IACJ,QAAQ;IACR,MAAM;IACN,IAAI;IACJ,KAAK;IACL,MAAM;IACN,QAAQ;IACR,IAAI;IACJ,OAAO;IACP,YAAY;IACZ,OAAO;IACP,QAAQ;IACR,MAAM;IACN,UAAU;IACV,WAAW;IACX,YAAY;IACZ,SAAS;IACT,MAAM;IACN,MAAM;IACN,SAAS;IACT,KAAK;IACL,SAAS;IACT,WAAW;IACX,WAAW;IACX,SAAS;IACT,YAAY;IACZ,YAAY;IACZ,OAAO;IACP,QAAQ;IACR,OAAO;IACP,MAAM;IACN,WAAW;IACX,SAAS;IACT,QAAQ;IACR,YAAY;IACZ,OAAO;IACP,SAAS;IACT,UAAU;IACV,WAAW;IACX,QAAQ;IACR,OAAO;IACP,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,KAAK;IACL,MAAM;IACN,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,UAAU;IACV,MAAM;IACN,QAAQ;IACR,cAAc;IACd,SAAS;IACT,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,aAAa;IACb,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,WAAW;IACX,IAAI;IACJ,UAAU;IACV,aAAa;IACb,MAAM;IACN,MAAM;IACN,SAAS;IACT,WAAW;IACX,aAAa;IACb,eAAe;IACf,OAAO;IACP,QAAQ;IACR,SAAS;IACT,WAAW;IACX,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,MAAM;IACN,OAAO;IACP,OAAO;IACP,OAAO;IACP,MAAM;IACN,UAAU;IACV,OAAO;IACP,QAAQ;IACR,SAAS;IACT,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,SAAS;IACT,MAAM;IACN,SAAS;IACT,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAa;IACjC,6JAA6J;IAC7J,uDAAuD;IACvD,QAAQ;IACR,KAAK;IACL,SAAS;IACT,UAAU;IACV,SAAS;IACT,SAAS;IACT,MAAM;IACN,SAAS;IACT,MAAM;IACN,QAAQ;IACR,aAAa;IACb,SAAS;IACT,SAAS;IACT,eAAe;IACf,UAAU;IACV,WAAW;IACX,MAAM;IACN,WAAW;IACX,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,KAAK;IACL,MAAM;IACN,UAAU;IACV,WAAW;IACX,UAAU;IACV,MAAM;IACN,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,OAAO;IACP,YAAY;IACZ,UAAU;IACV,UAAU;CACX,CAAC"}

package/dist/esm/lexer/NestedComment.d.ts

@@ -0,0 +1,10 @@
+import { RegExpLike } from './TokenizerEngine.js';
+/**
+ * An object mimicking a regular expression,
+ * for matching nested block-comments.
+ */
+export declare class NestedComment implements RegExpLike {
+    lastIndex: number;
+    exec(input: string): string[] | null;
+    private matchSection;
+}

package/dist/esm/lexer/NestedComment.js

@@ -0,0 +1,50 @@
+const START = /\/\*/uy; // matches: /*
+const ANY_CHAR = /[\s\S]/uy; // matches single character
+const END = /\*\//uy; // matches: */
+/**
+ * An object mimicking a regular expression,
+ * for matching nested block-comments.
+ */
+export class NestedComment {
+    constructor() {
+        this.lastIndex = 0;
+    }
+    exec(input) {
+        let result = '';
+        let match;
+        let nestLevel = 0;
+        if ((match = this.matchSection(START, input))) {
+            result += match;
+            nestLevel++;
+        }
+        else {
+            return null;
+        }
+        while (nestLevel > 0) {
+            if ((match = this.matchSection(START, input))) {
+                result += match;
+                nestLevel++;
+            }
+            else if ((match = this.matchSection(END, input))) {
+                result += match;
+                nestLevel--;
+            }
+            else if ((match = this.matchSection(ANY_CHAR, input))) {
+                result += match;
+            }
+            else {
+                return null;
+            }
+        }
+        return [result];
+    }
+    matchSection(regex, input) {
+        regex.lastIndex = this.lastIndex;
+        const matches = regex.exec(input);
+        if (matches) {
+            this.lastIndex += matches[0].length;
+        }
+        return matches ? matches[0] : null;
+    }
+}
+//# sourceMappingURL=NestedComment.js.map

package/dist/esm/lexer/NestedComment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NestedComment.js","sourceRoot":"","sources":["../../../src/lexer/NestedComment.ts"],"names":[],"mappings":"AAGA,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,cAAc;AACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,2BAA2B;AACxD,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,cAAc;AAEpC;;;GAGG;AACH,MAAM,OAAO,aAAa;IAA1B;QACS,cAAS,GAAW,CAAC,CAAC;IAuC/B,CAAC;IArCQ,IAAI,CAAC,KAAa;QACvB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,KAAoB,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC;YAChB,SAAS,EAAE,CAAC;SACb;aAAM;YACL,OAAO,IAAI,CAAC;SACb;QAED,OAAO,SAAS,GAAG,CAAC,EAAE;YACpB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE;gBAC7C,MAAM,IAAI,KAAK,CAAC;gBAChB,SAAS,EAAE,CAAC;aACb;iBAAM,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE;gBAClD,MAAM,IAAI,KAAK,CAAC;gBAChB,SAAS,EAAE,CAAC;aACb;iBAAM,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE;gBACvD,MAAM,IAAI,KAAK,CAAC;aACjB;iBAAM;gBACL,OAAO,IAAI,CAAC;aACb;SACF;QAED,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAEO,YAAY,CAAC,KAAa,EAAE,KAAa;QAC/C,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;SACrC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;CACF"}

package/dist/esm/lexer/Tokenizer.d.ts

@@ -0,0 +1,14 @@
+import { Token } from './token.js';
+import { ParamTypes, TokenizerOptions } from './TokenizerOptions.js';
+export default class Tokenizer {
+    private cfg;
+    private dialectName;
+    private rulesBeforeParams;
+    private rulesAfterParams;
+    constructor(cfg: TokenizerOptions, dialectName: string);
+    tokenize(input: string, paramTypesOverrides: ParamTypes): Token[];
+    private buildRulesBeforeParams;
+    private buildRulesAfterParams;
+    private buildParamRules;
+    private validRules;
+}

package/dist/esm/lexer/Tokenizer.js

@@ -0,0 +1,264 @@
+import { TokenType } from './token.js';
+import * as regex from './regexFactory.js';
+import TokenizerEngine from './TokenizerEngine.js';
+import { escapeRegExp, patternToRegex } from './regexUtil.js';
+import { equalizeWhitespace } from '../utils.js';
+import { NestedComment } from './NestedComment.js';
+export default class Tokenizer {
+    constructor(cfg, dialectName) {
+        this.cfg = cfg;
+        this.dialectName = dialectName;
+        this.rulesBeforeParams = this.buildRulesBeforeParams(cfg);
+        this.rulesAfterParams = this.buildRulesAfterParams(cfg);
+    }
+    tokenize(input, paramTypesOverrides) {
+        const rules = [
+            ...this.rulesBeforeParams,
+            ...this.buildParamRules(this.cfg, paramTypesOverrides),
+            ...this.rulesAfterParams,
+        ];
+        const tokens = new TokenizerEngine(rules, this.dialectName).tokenize(input);
+        return this.cfg.postProcess ? this.cfg.postProcess(tokens) : tokens;
+    }
+    // These rules can be cached as they only depend on
+    // the Tokenizer config options specified for each SQL dialect
+    buildRulesBeforeParams(cfg) {
+        var _a, _b, _c;
+        return this.validRules([
+            {
+                type: TokenType.DISABLE_COMMENT,
+                regex: /(\/\* *sql-formatter-disable *\*\/[\s\S]*?(?:\/\* *sql-formatter-enable *\*\/|$))/uy,
+            },
+            {
+                type: TokenType.BLOCK_COMMENT,
+                regex: cfg.nestedBlockComments ? new NestedComment() : /(\/\*[^]*?\*\/)/uy,
+            },
+            {
+                type: TokenType.LINE_COMMENT,
+                regex: regex.lineComment((_a = cfg.lineCommentTypes) !== null && _a !== void 0 ? _a : ['--']),
+            },
+            {
+                type: TokenType.QUOTED_IDENTIFIER,
+                regex: regex.string(cfg.identTypes),
+            },
+            {
+                type: TokenType.NUMBER,
+                regex: cfg.underscoresInNumbers
+                    ? /(?:0x[0-9a-fA-F_]+|0b[01_]+|(?:-\s*)?(?:[0-9_]*\.[0-9_]+|[0-9_]+(?:\.[0-9_]*)?)(?:[eE][-+]?[0-9_]+(?:\.[0-9_]+)?)?)(?![\w\p{Alphabetic}])/uy
+                    : /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?(?:[0-9]*\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)(?![\w\p{Alphabetic}])/uy,
+            },
+            // RESERVED_KEYWORD_PHRASE and RESERVED_DATA_TYPE_PHRASE  is matched before all other keyword tokens
+            // to e.g. prioritize matching "TIMESTAMP WITH TIME ZONE" phrase over "WITH" clause.
+            {
+                type: TokenType.RESERVED_KEYWORD_PHRASE,
+                regex: regex.reservedWord((_b = cfg.reservedKeywordPhrases) !== null && _b !== void 0 ? _b : [], cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_DATA_TYPE_PHRASE,
+                regex: regex.reservedWord((_c = cfg.reservedDataTypePhrases) !== null && _c !== void 0 ? _c : [], cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.CASE,
+                regex: /CASE\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.END,
+                regex: /END\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.BETWEEN,
+                regex: /BETWEEN\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.LIMIT,
+                regex: cfg.reservedClauses.includes('LIMIT') ? /LIMIT\b/iuy : undefined,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_CLAUSE,
+                regex: regex.reservedWord(cfg.reservedClauses, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_SELECT,
+                regex: regex.reservedWord(cfg.reservedSelect, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_SET_OPERATION,
+                regex: regex.reservedWord(cfg.reservedSetOperations, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.WHEN,
+                regex: /WHEN\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.ELSE,
+                regex: /ELSE\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.THEN,
+                regex: /THEN\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_JOIN,
+                regex: regex.reservedWord(cfg.reservedJoins, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.AND,
+                regex: /AND\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.OR,
+                regex: /OR\b/iuy,
+                text: toCanonical,
+            },
+            {
+                type: TokenType.XOR,
+                regex: cfg.supportsXor ? /XOR\b/iuy : undefined,
+                text: toCanonical,
+            },
+            ...(cfg.operatorKeyword
+                ? [
+                    {
+                        type: TokenType.OPERATOR,
+                        regex: /OPERATOR *\([^)]+\)/iuy,
+                    },
+                ]
+                : []),
+            {
+                type: TokenType.RESERVED_FUNCTION_NAME,
+                regex: regex.reservedWord(cfg.reservedFunctionNames, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_DATA_TYPE,
+                regex: regex.reservedWord(cfg.reservedDataTypes, cfg.identChars),
+                text: toCanonical,
+            },
+            {
+                type: TokenType.RESERVED_KEYWORD,
+                regex: regex.reservedWord(cfg.reservedKeywords, cfg.identChars),
+                text: toCanonical,
+            },
+        ]);
+    }
+    // These rules can also be cached as they only depend on
+    // the Tokenizer config options specified for each SQL dialect
+    buildRulesAfterParams(cfg) {
+        var _a, _b;
+        return this.validRules([
+            {
+                type: TokenType.VARIABLE,
+                regex: cfg.variableTypes ? regex.variable(cfg.variableTypes) : undefined,
+            },
+            { type: TokenType.STRING, regex: regex.string(cfg.stringTypes) },
+            {
+                type: TokenType.IDENTIFIER,
+                regex: regex.identifier(cfg.identChars),
+            },
+            { type: TokenType.DELIMITER, regex: /[;]/uy },
+            { type: TokenType.COMMA, regex: /[,]/y },
+            {
+                type: TokenType.OPEN_PAREN,
+                regex: regex.parenthesis('open', cfg.extraParens),
+            },
+            {
+                type: TokenType.CLOSE_PAREN,
+                regex: regex.parenthesis('close', cfg.extraParens),
+            },
+            {
+                type: TokenType.OPERATOR,
+                regex: regex.operator([
+                    // standard operators
+                    '+',
+                    '-',
+                    '/',
+                    '>',
+                    '<',
+                    '=',
+                    '<>',
+                    '<=',
+                    '>=',
+                    '!=',
+                    ...((_a = cfg.operators) !== null && _a !== void 0 ? _a : []),
+                ]),
+            },
+            { type: TokenType.ASTERISK, regex: /[*]/uy },
+            {
+                type: TokenType.PROPERTY_ACCESS_OPERATOR,
+                regex: regex.operator(['.', ...((_b = cfg.propertyAccessOperators) !== null && _b !== void 0 ? _b : [])]),
+            },
+        ]);
+    }
+    // These rules can't be blindly cached as the paramTypesOverrides object
+    // can differ on each invocation of the format() function.
+    buildParamRules(cfg, paramTypesOverrides) {
+        var _a, _b, _c, _d, _e;
+        // Each dialect has its own default parameter types (if any),
+        // but these can be overriden by the user of the library.
+        const paramTypes = {
+            named: (paramTypesOverrides === null || paramTypesOverrides === void 0 ? void 0 : paramTypesOverrides.named) || ((_a = cfg.paramTypes) === null || _a === void 0 ? void 0 : _a.named) || [],
+            quoted: (paramTypesOverrides === null || paramTypesOverrides === void 0 ? void 0 : paramTypesOverrides.quoted) || ((_b = cfg.paramTypes) === null || _b === void 0 ? void 0 : _b.quoted) || [],
+            numbered: (paramTypesOverrides === null || paramTypesOverrides === void 0 ? void 0 : paramTypesOverrides.numbered) || ((_c = cfg.paramTypes) === null || _c === void 0 ? void 0 : _c.numbered) || [],
+            positional: typeof (paramTypesOverrides === null || paramTypesOverrides === void 0 ? void 0 : paramTypesOverrides.positional) === 'boolean'
+                ? paramTypesOverrides.positional
+                : (_d = cfg.paramTypes) === null || _d === void 0 ? void 0 : _d.positional,
+            custom: (paramTypesOverrides === null || paramTypesOverrides === void 0 ? void 0 : paramTypesOverrides.custom) || ((_e = cfg.paramTypes) === null || _e === void 0 ? void 0 : _e.custom) || [],
+        };
+        return this.validRules([
+            {
+                type: TokenType.NAMED_PARAMETER,
+                regex: regex.parameter(paramTypes.named, regex.identifierPattern(cfg.paramChars || cfg.identChars)),
+                key: v => v.slice(1),
+            },
+            {
+                type: TokenType.QUOTED_PARAMETER,
+                regex: regex.parameter(paramTypes.quoted, regex.stringPattern(cfg.identTypes)),
+                key: v => (({ tokenKey, quoteChar }) => tokenKey.replace(new RegExp(escapeRegExp('\\' + quoteChar), 'gu'), quoteChar))({
+                    tokenKey: v.slice(2, -1),
+                    quoteChar: v.slice(-1),
+                }),
+            },
+            {
+                type: TokenType.NUMBERED_PARAMETER,
+                regex: regex.parameter(paramTypes.numbered, '[0-9]+'),
+                key: v => v.slice(1),
+            },
+            {
+                type: TokenType.POSITIONAL_PARAMETER,
+                regex: paramTypes.positional ? /[?]/y : undefined,
+            },
+            ...paramTypes.custom.map((customParam) => {
+                var _a;
+                return ({
+                    type: TokenType.CUSTOM_PARAMETER,
+                    regex: patternToRegex(customParam.regex),
+                    key: (_a = customParam.key) !== null && _a !== void 0 ? _a : (v => v),
+                });
+            }),
+        ]);
+    }
+    // filters out rules for token types whose regex is undefined
+    validRules(rules) {
+        return rules.filter((rule) => Boolean(rule.regex));
+    }
+}
+/**
+ * Converts keywords (and keyword sequences) to canonical form:
+ * - in uppercase
+ * - single spaces between words
+ */
+const toCanonical = (v) => equalizeWhitespace(v.toUpperCase());
+//# sourceMappingURL=Tokenizer.js.map

package/dist/esm/lexer/Tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Tokenizer.js","sourceRoot":"","sources":["../../../src/lexer/Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,SAAS,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,eAA8B,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAY,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAInD,MAAM,CAAC,OAAO,OAAO,SAAS;IAI5B,YAAoB,GAAqB,EAAU,WAAmB;QAAlD,QAAG,GAAH,GAAG,CAAkB;QAAU,gBAAW,GAAX,WAAW,CAAQ;QACpE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;IAEM,QAAQ,CAAC,KAAa,EAAE,mBAA+B;QAC5D,MAAM,KAAK,GAAG;YACZ,GAAG,IAAI,CAAC,iBAAiB;YACzB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,mBAAmB,CAAC;YACtD,GAAG,IAAI,CAAC,gBAAgB;SACzB,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACtE,CAAC;IAED,mDAAmD;IACnD,8DAA8D;IACtD,sBAAsB,CAAC,GAAqB;;QAClD,OAAO,IAAI,CAAC,UAAU,CAAC;YACrB;gBACE,IAAI,EAAE,SAAS,CAAC,eAAe;gBAC/B,KAAK,EACH,qFAAqF;aACxF;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,aAAa;gBAC7B,KAAK,EAAE,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC,CAAC,mBAAmB;aAC3E;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,YAAY;gBAC5B,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,MAAA,GAAG,CAAC,gBAAgB,mCAAI,CAAC,IAAI,CAAC,CAAC;aACzD;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,iBAAiB;gBACjC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;aACpC;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,MAAM;gBACtB,KAAK,EAAE,GAAG,CAAC,oBAAoB;oBAC7B,CAAC,CAAC,6IAA6I;oBAC/I,CAAC,CAAC,qIAAqI;aAC1I;YACD,oGAAoG;YACpG,oFAAoF;YACpF;gBACE,IAAI,EAAE,SAAS,CAAC,uBAAuB;gBACvC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,MAAA,GAAG,CAAC,sBAAsB,mCAAI,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC3E,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,yBAAyB;gBACzC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,MAAA,GAAG,CAAC,uBAAuB,mCAAI,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC5E,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,KAAK,EAAE,WAAW;gBAClB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,GAAG;gBACnB,KAAK,EAAE,UAAU;gBACjB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,OAAO;gBACvB,KAAK,EAAE,cAAc;gBACrB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,KAAK;gBACrB,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;gBACvE,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,eAAe;gBAC/B,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC9D,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,eAAe;gBAC/B,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC7D,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,sBAAsB;gBACtC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,qBAAqB,EAAE,GAAG,CAAC,UAAU,CAAC;gBACpE,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,KAAK,EAAE,WAAW;gBAClB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,KAAK,EAAE,WAAW;gBAClB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,KAAK,EAAE,WAAW;gBAClB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,aAAa;gBAC7B,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC5D,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,GAAG;gBACnB,KAAK,EAAE,UAAU;gBACjB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,EAAE;gBAClB,KAAK,EAAE,SAAS;gBAChB,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,GAAG;gBACnB,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC/C,IAAI,EAAE,WAAW;aAClB;YACD,GAAG,CAAC,GAAG,CAAC,eAAe;gBACrB,CAAC,CAAC;oBACE;wBACE,IAAI,EAAE,SAAS,CAAC,QAAQ;wBACxB,KAAK,EAAE,wBAAwB;qBAChC;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;YACP;gBACE,IAAI,EAAE,SAAS,CAAC,sBAAsB;gBACtC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,qBAAqB,EAAE,GAAG,CAAC,UAAU,CAAC;gBACpE,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,kBAAkB;gBAClC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,EAAE,GAAG,CAAC,UAAU,CAAC;gBAChE,IAAI,EAAE,WAAW;aAClB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,gBAAgB;gBAChC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,UAAU,CAAC;gBAC/D,IAAI,EAAE,WAAW;aAClB;SACF,CAAC,CAAC;IACL,CAAC;IAED,wDAAwD;IACxD,8DAA8D;IACtD,qBAAqB,CAAC,GAAqB;;QACjD,OAAO,IAAI,CAAC,UAAU,CAAC;YACrB;gBACE,IAAI,EAAE,SAAS,CAAC,QAAQ;gBACxB,KAAK,EAAE,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;aACzE;YACD,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE;YAChE;gBACE,IAAI,EAAE,SAAS,CAAC,UAAU;gBAC1B,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC;aACxC;YACD,EAAE,IAAI,EAAE,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;YAC7C,EAAE,IAAI,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;YACxC;gBACE,IAAI,EAAE,SAAS,CAAC,UAAU;gBAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;aAClD;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;aACnD;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,QAAQ;gBACxB,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC;oBACpB,qBAAqB;oBACrB,GAAG;oBACH,GAAG;oBACH,GAAG;oBACH,GAAG;oBACH,GAAG;oBACH,GAAG;oBACH,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,GAAG,CAAC,MAAA,GAAG,CAAC,SAAS,mCAAI,EAAE,CAAC;iBACzB,CAAC;aACH;YACD,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE;YAC5C;gBACE,IAAI,EAAE,SAAS,CAAC,wBAAwB;gBACxC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,MAAA,GAAG,CAAC,uBAAuB,mCAAI,EAAE,CAAC,CAAC,CAAC;aACrE;SACF,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,0DAA0D;IAClD,eAAe,CAAC,GAAqB,EAAE,mBAA+B;;QAC5E,6DAA6D;QAC7D,yDAAyD;QACzD,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,KAAK,MAAI,MAAA,GAAG,CAAC,UAAU,0CAAE,KAAK,CAAA,IAAI,EAAE;YAChE,MAAM,EAAE,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,MAAM,MAAI,MAAA,GAAG,CAAC,UAAU,0CAAE,MAAM,CAAA,IAAI,EAAE;YACnE,QAAQ,EAAE,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,QAAQ,MAAI,MAAA,GAAG,CAAC,UAAU,0CAAE,QAAQ,CAAA,IAAI,EAAE;YACzE,UAAU,EACR,OAAO,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,UAAU,CAAA,KAAK,SAAS;gBAClD,CAAC,CAAC,mBAAmB,CAAC,UAAU;gBAChC,CAAC,CAAC,MAAA,GAAG,CAAC,UAAU,0CAAE,UAAU;YAChC,MAAM,EAAE,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,MAAM,MAAI,MAAA,GAAG,CAAC,UAAU,0CAAE,MAAM,CAAA,IAAI,EAAE;SACpE,CAAC;QAEF,OAAO,IAAI,CAAC,UAAU,CAAC;YACrB;gBACE,IAAI,EAAE,SAAS,CAAC,eAAe;gBAC/B,KAAK,EAAE,KAAK,CAAC,SAAS,CACpB,UAAU,CAAC,KAAK,EAChB,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC,CAC1D;gBACD,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACrB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,gBAAgB;gBAChC,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9E,GAAG,EAAE,CAAC,CAAC,EAAE,CACP,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAC3B,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC/E,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBACxB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;iBACvB,CAAC;aACL;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,kBAAkB;gBAClC,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC;gBACrD,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACrB;YACD;gBACE,IAAI,EAAE,SAAS,CAAC,oBAAoB;gBACpC,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;aAClD;YACD,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CACtB,CAAC,WAAW,EAAa,EAAE;;gBAAC,OAAA,CAAC;oBAC3B,IAAI,EAAE,SAAS,CAAC,gBAAgB;oBAChC,KAAK,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,CAAC;oBACxC,GAAG,EAAE,MAAA,WAAW,CAAC,GAAG,mCAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBACjC,CAAC,CAAA;aAAA,CACH;SACF,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IACrD,UAAU,CAAC,KAA0B;QAC3C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAqB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC"}

package/dist/esm/lexer/TokenizerEngine.d.ts

@@ -0,0 +1,31 @@
+import { Token, TokenType } from './token.js';
+export interface RegExpLike {
+    lastIndex: number;
+    exec(input: string): string[] | null;
+}
+export interface TokenRule {
+    type: TokenType;
+    regex: RegExpLike;
+    text?: (rawText: string) => string;
+    key?: (rawText: string) => string;
+}
+export default class TokenizerEngine {
+    private rules;
+    private dialectName;
+    private input;
+    private index;
+    constructor(rules: TokenRule[], dialectName: string);
+    /**
+     * Takes a SQL string and breaks it into tokens.
+     * Each token is an object with type and value.
+     *
+     * @param {string} input - The SQL string
+     * @returns {Token[]} output token stream
+     */
+    tokenize(input: string): Token[];
+    private createParseError;
+    private dialectInfo;
+    private getWhitespace;
+    private getNextToken;
+    private match;
+}