@malloydata/malloy-filter 0.0.237-dev250221201621

package/dist/tokenizer.js

@@ -0,0 +1,263 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Tokenizer = void 0;
+class Tokenizer {
+    constructor(input, params) {
+        this.index = 0;
+        this.specialSubstrings = params.specialSubstrings;
+        this.specialWords = params.specialWords;
+        this.params = params;
+        this.input = input;
+    }
+    tokenize() {
+        let tokens = [];
+        let wordStart = -1;
+        let wordEnd = -1;
+        let special;
+        while (this.hasMoreInput()) {
+            if (this.params.splitOnWhitespace && this.isWhitespace(this.index)) {
+                this.maybeConsumeWord(wordStart, wordEnd, tokens);
+                wordStart = -1;
+                this.consumeWhitespace();
+            }
+            else if (this.input[this.index] === '\\') {
+                if (wordStart === -1) {
+                    wordStart = this.index;
+                }
+                wordEnd = this.index;
+                this.index++;
+                if (this.hasMoreInput()) {
+                    // Unless backslash is at the end, handle next char.
+                    wordEnd = this.index;
+                    this.index++;
+                }
+            }
+            else if ((special = this.isSpecialSubstring())) {
+                this.maybeConsumeWord(wordStart, wordEnd, tokens);
+                wordStart = -1;
+                this.consumeSpecialSubstring(special, tokens);
+            }
+            else {
+                if (wordStart === -1) {
+                    wordStart = this.index;
+                }
+                wordEnd = this.index;
+                this.index++;
+            }
+        }
+        this.maybeConsumeWord(wordStart, wordEnd, tokens);
+        tokens = Tokenizer.convertSpecialWords(tokens, this.specialWords);
+        if (this.params.combineAdjacentWords) {
+            tokens = Tokenizer.combineAdjacentWords(tokens);
+        }
+        if (this.params.trimWordWhitespace) {
+            tokens = Tokenizer.trimWordWhitespace(tokens);
+        }
+        return tokens;
+    }
+    parse() {
+        let tokens = this.tokenize();
+        if (this.params.trimWordWhitespace) {
+            tokens = Tokenizer.trimWordWhitespace(tokens);
+        }
+        return tokens;
+    }
+    hasMoreInput() {
+        return this.index < this.input.length;
+    }
+    isWhitespace(idx) {
+        return /\s/.test(this.input[idx]);
+    }
+    consumeWhitespace() {
+        while (this.hasMoreInput() && this.isWhitespace(this.index)) {
+            this.index++;
+        }
+    }
+    maybeConsumeWord(wordStart, wordEnd, tokens) {
+        if (wordStart >= 0 && wordEnd >= wordStart) {
+            tokens.push({
+                type: 'word',
+                value: this.input.substring(wordStart, wordEnd + 1),
+                startIndex: wordStart,
+                endIndex: wordEnd + 1,
+            });
+        }
+    }
+    isSpecialSubstring() {
+        for (const special of this.specialSubstrings) {
+            if (special.value instanceof RegExp) {
+                const shifted = this.input.substring(this.index); // create a substring starting at index.
+                const matcher = special.value.exec(shifted);
+                if (matcher) {
+                    return {
+                        type: special.type,
+                        value: special.value,
+                        matchedString: matcher[0],
+                    };
+                }
+            }
+            else {
+                const subString = this.input.slice(this.index, this.index + special.value.length);
+                const matches = special.ignoreCase
+                    ? subString.toLowerCase() === special.value.toLowerCase()
+                    : subString === special.value;
+                if (matches) {
+                    const value = special.ignoreCase
+                        ? subString.toUpperCase()
+                        : subString;
+                    return {
+                        type: special.type,
+                        value: special.value,
+                        matchedString: value,
+                    };
+                }
+            }
+        }
+        return undefined;
+    }
+    consumeSpecialSubstring(special, tokens) {
+        tokens.push({
+            type: special.type,
+            value: special.matchedString,
+            startIndex: this.index,
+            endIndex: this.index + special.matchedString.length,
+        });
+        this.index += special.matchedString.length;
+    }
+    static combineAdjacentWords(tokens) {
+        const output = [];
+        let previousToken = undefined;
+        for (let i = 0; i < tokens.length; i++) {
+            const currentToken = tokens[i];
+            if (currentToken.type === 'word' &&
+                previousToken &&
+                previousToken.type === 'word') {
+                previousToken.value += currentToken.value;
+                previousToken.endIndex = currentToken.endIndex;
+            }
+            else {
+                output.push(currentToken);
+            }
+            previousToken = currentToken;
+        }
+        return output;
+    }
+    static trimWordWhitespace(tokens) {
+        const output = [];
+        for (const token of tokens) {
+            if (token.type === 'word') {
+                token.value = token.value.trim();
+            }
+            output.push(token);
+        }
+        return output;
+    }
+    static isSpecialWord(token, specials) {
+        for (const special of specials) {
+            if (special.value instanceof RegExp) {
+                const regexp = special.value;
+                regexp.lastIndex = 0; // Set the starting index for the search
+                if (regexp.test(token.value)) {
+                    const value = special.ignoreCase
+                        ? token.value.toUpperCase()
+                        : token.value;
+                    return {
+                        type: special.type,
+                        value: value,
+                        startIndex: token.startIndex,
+                        endIndex: token.endIndex,
+                    };
+                }
+            }
+            else {
+                const matches = special.ignoreCase
+                    ? token.value.toLowerCase() === special.value.toLowerCase()
+                    : token.value === special.value;
+                if (matches) {
+                    const value = special.ignoreCase
+                        ? token.value.toUpperCase()
+                        : token.value;
+                    return {
+                        type: special.type,
+                        value: value,
+                        startIndex: token.startIndex,
+                        endIndex: token.endIndex,
+                    };
+                }
+            }
+        }
+        return undefined;
+    }
+    static convertSpecialWords(tokens, specials) {
+        const output = [];
+        let special = undefined;
+        for (const token of tokens) {
+            if (token.type === 'word' &&
+                (special = Tokenizer.isSpecialWord(token, specials))) {
+                output.push(special);
+            }
+            else {
+                output.push(token);
+            }
+        }
+        return output;
+    }
+    /**
+     * Checks if the token types starting at the given index match the types in the input string.
+     *
+     * @param types - A string of types separated by '|'.  Eg 'TYPEA|TYPEB|TYPEC'
+     * @param tokens - An array of tokens.
+     * @param index - The index into the token array to start checking from.
+     * @returns True if the token types match, false otherwise.
+     */
+    static matchTypes(types, tokens, index) {
+        const typeArray = types.split('|');
+        if (index < 0 || index + typeArray.length > tokens.length) {
+            return undefined;
+        }
+        // Iterate over the types and check if they match the token types
+        for (let i = 0; i < typeArray.length; i++) {
+            if (index + i >= tokens.length) {
+                return undefined;
+            }
+            if (tokens[index + i].type !== typeArray[i]) {
+                return undefined;
+            }
+        }
+        return tokens.slice(index, index + typeArray.length);
+    }
+    /**
+     * Merges tokens that match a given type string into a single token with a specified merge type.
+     *
+     * @param types - A string of types separated by '|'.
+     * @param tokens - An array of tokens.
+     * @param mergeType - The type to use for the merged token.
+     * @returns The updated token list with merged tokens.
+     */
+    static mergeTypes(types, tokens, mergeType) {
+        const output = [];
+        let i = 0;
+        while (i < tokens.length) {
+            // Check if the current token matches the type string
+            const matchedTokens = Tokenizer.matchTypes(types, tokens, i);
+            if (matchedTokens && matchedTokens.length > 0) {
+                const mergedToken = {
+                    type: mergeType,
+                    value: '',
+                    values: matchedTokens,
+                    startIndex: matchedTokens[0].startIndex,
+                    endIndex: matchedTokens[matchedTokens.length - 1].endIndex,
+                };
+                output.push(mergedToken);
+                i += matchedTokens.length;
+            }
+            else {
+                output.push(tokens[i]);
+                i++;
+            }
+        }
+        return output;
+    }
+}
+exports.Tokenizer = Tokenizer;
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":";;;AAqBA,MAAa,SAAS;IAOpB,YAAY,KAAa,EAAE,MAAuB;QAChD,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAEO,QAAQ;QACd,IAAI,MAAM,GAAY,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;QACjB,IAAI,OAAsC,CAAC;QAC3C,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE;YAC1B,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAClE,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAClD,SAAS,GAAG,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,iBAAiB,EAAE,CAAC;aAC1B;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBAC1C,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE;oBACpB,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;iBACxB;gBACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;gBACrB,IAAI,CAAC,KAAK,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE;oBACvB,oDAAoD;oBACpD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;oBACrB,IAAI,CAAC,KAAK,EAAE,CAAC;iBACd;aACF;iBAAM,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,EAAE;gBAChD,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAClD,SAAS,GAAG,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;aAC/C;iBAAM;gBACL,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE;oBACpB,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;iBACxB;gBACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;gBACrB,IAAI,CAAC,KAAK,EAAE,CAAC;aACd;SACF;QACD,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,GAAG,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAElE,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE;YACpC,MAAM,GAAG,SAAS,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;SACjD;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClC,MAAM,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;SAC/C;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,KAAK;QACV,IAAI,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClC,MAAM,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;SAC/C;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,YAAY;QAClB,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IACxC,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;SACd;IACH,CAAC;IACO,gBAAgB,CACtB,SAAiB,EACjB,OAAe,EACf,MAAe;QAEf,IAAI,SAAS,IAAI,CAAC,IAAI,OAAO,IAAI,SAAS,EAAE;YAC1C,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,CAAC;gBACnD,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,OAAO,GAAG,CAAC;aACtB,CAAC,CAAC;SACJ;IACH,CAAC;IAEO,kBAAkB;QACxB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC5C,IAAI,OAAO,CAAC,KAAK,YAAY,MAAM,EAAE;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,wCAAwC;gBAC1F,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,EAAE;oBACX,OAAO;wBACL,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC;qBAC1B,CAAC;iBACH;aACF;iBAAM;gBACL,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAChC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAClC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU;oBAChC,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;oBACzD,CAAC,CAAC,SAAS,KAAK,OAAO,CAAC,KAAK,CAAC;gBAChC,IAAI,OAAO,EAAE;oBACX,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU;wBAC9B,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE;wBACzB,CAAC,CAAC,SAAS,CAAC;oBACd,OAAO;wBACL,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,aAAa,EAAE,KAAK;qBACrB,CAAC;iBACH;aACF;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,uBAAuB,CAC7B,OAA0B,EAC1B,MAAe;QAEf,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,aAAa;YAC5B,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM;SACpD,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC;IAC7C,CAAC;IAEM,MAAM,CAAC,oBAAoB,CAAC,MAAe;QAChD,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,aAAa,GAAsB,SAAS,CAAC;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,YAAY,GAAU,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC,IACE,YAAY,CAAC,IAAI,KAAK,MAAM;gBAC5B,aAAa;gBACb,aAAa,CAAC,IAAI,KAAK,MAAM,EAC7B;gBACA,aAAa,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,CAAC;gBAC1C,aAAa,CAAC,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC;aAChD;iBAAM;gBACL,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aAC3B;YACD,aAAa,GAAG,YAAY,CAAC;SAC9B;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAC,MAAe;QAC9C,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE;gBACzB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;aAClC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACpB;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,MAAM,CAAC,aAAa,CACzB,KAAY,EACZ,QAAwB;QAExB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC9B,IAAI,OAAO,CAAC,KAAK,YAAY,MAAM,EAAE;gBACnC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;gBAC7B,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,wCAAwC;gBAC9D,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU;wBAC9B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;wBAC3B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;oBAChB,OAAO;wBACL,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,KAAK,EAAE,KAAK;wBACZ,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;qBACzB,CAAC;iBACH;aACF;iBAAM;gBACL,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU;oBAChC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;oBAC3D,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC;gBAClC,IAAI,OAAO,EAAE;oBACX,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU;wBAC9B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;wBAC3B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;oBAChB,OAAO;wBACL,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,KAAK,EAAE,KAAK;wBACZ,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;qBACzB,CAAC;iBACH;aACF;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,MAAM,CAAC,mBAAmB,CAC/B,MAAe,EACf,QAAwB;QAExB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,OAAO,GAAsB,SAAS,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,IACE,KAAK,CAAC,IAAI,KAAK,MAAM;gBACrB,CAAC,OAAO,GAAG,SAAS,CAAC,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,EACpD;gBACA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aACtB;iBAAM;gBACL,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACpB;SACF;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,UAAU,CACtB,KAAa,EACb,MAAe,EACf,KAAa;QAEb,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE;YACzD,OAAO,SAAS,CAAC;SAClB;QACD,iEAAiE;QACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;gBAC9B,OAAO,SAAS,CAAC;aAClB;YACD,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE;gBAC3C,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,UAAU,CACtB,KAAa,EACb,MAAe,EACf,SAAiB;QAEjB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE;YACxB,qDAAqD;YACrD,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YAC7D,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC7C,MAAM,WAAW,GAAU;oBACzB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU;oBACvC,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ;iBAC3D,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACzB,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC;aAC3B;iBAAM;gBACL,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvB,CAAC,EAAE,CAAC;aACL;SACF;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA1SD,8BA0SC"}

package/dist/tokenizer.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tokenizer.spec.js

@@ -0,0 +1,255 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tokenizer_1 = require("./tokenizer");
+function makeParams() {
+    const specialSubstrings = [
+        { type: ',', value: ',' },
+        { type: 'VARIABLE', value: /^\$\{[^}]+\}/ },
+    ];
+    const specialWords = [
+        { type: 'NULL', value: 'null', ignoreCase: true },
+        { type: 'EMPTY', value: 'empty', ignoreCase: true },
+        { type: 'NOTNULL', value: '-null', ignoreCase: true },
+        { type: 'NOTEMPTY', value: '-empty', ignoreCase: true },
+        {
+            type: 'DAYOFWEEK',
+            value: /^(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$/i,
+            ignoreCase: true,
+        },
+        { type: 'STATE', value: /^(California|Washington)$/i },
+        { type: 'DATE', value: /^\d\d\d\d-\d\d-\d\d$/ },
+        { type: 'DATE', value: /^\d\d\d\d-\d\d$/ },
+        { type: 'DATE', value: /^\d\d\d\d$/ },
+    ];
+    return {
+        splitOnWhitespace: true,
+        trimWordWhitespace: true,
+        specialSubstrings,
+        specialWords,
+    };
+}
+describe('Tokenizer', () => {
+    const makeToken = (type, value, startIndex, endIndex) => ({ type, value, startIndex, endIndex });
+    it('should tokenize a simple string', () => {
+        const input = 'hello world';
+        const expectedTokens = [
+            makeToken('word', 'hello', 0, 5),
+            makeToken('word', 'world', 6, 11),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should split adjacent characters when no whitespace', () => {
+        const specialSubstrings = [
+            { type: ',', value: ',' },
+            { type: 'exclamation', value: '!' },
+            { type: 'bracket', value: '[' },
+        ];
+        const input = '[hello],big,world!';
+        const expectedTokens = [
+            makeToken('bracket', '[', 0, 1),
+            makeToken('word', 'hello]', 1, 7),
+            makeToken(',', ',', 7, 8),
+            makeToken('word', 'big', 8, 11),
+            makeToken(',', ',', 11, 12),
+            makeToken('word', 'world', 12, 17),
+            makeToken('exclamation', '!', 17, 18),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, { ...makeParams(), specialSubstrings }).parse()).toEqual(expectedTokens);
+    });
+    it('should match special tokens', () => {
+        const input = 'hello NULL world,-Null,-\'NULL" ,NULL, NULL ,  ';
+        const expectedTokens = [
+            makeToken('word', 'hello', 0, 5),
+            makeToken('NULL', 'NULL', 6, 10),
+            makeToken('word', 'world', 11, 16),
+            makeToken(',', ',', 16, 17),
+            makeToken('NOTNULL', '-NULL', 17, 22),
+            makeToken(',', ',', 22, 23),
+            makeToken('word', '-\'NULL"', 23, 30),
+            makeToken(',', ',', 31, 32),
+            makeToken('NULL', 'NULL', 32, 36),
+            makeToken(',', ',', 36, 37),
+            makeToken('NULL', 'NULL', 38, 42),
+            makeToken(',', ',', 43, 44),
+        ];
+        const params = makeParams();
+        expect(new tokenizer_1.Tokenizer(input, params).parse()).toEqual(expectedTokens);
+    });
+    it('should not combine adjacent words', () => {
+        const input = 'ABC DEF';
+        const expectedTokens = [
+            makeToken('word', 'ABC', 0, 3),
+            makeToken('word', 'DEF', 4, 7),
+        ];
+        const params = makeParams();
+        expect(new tokenizer_1.Tokenizer(input, params).parse()).toEqual(expectedTokens);
+    });
+    it('should combine adjacent words when combineAdjacentWords', () => {
+        const input = 'ABC DEF';
+        const expectedTokens = [makeToken('word', 'ABCDEF', 0, 7)];
+        const params = { ...makeParams(), combineAdjacentWords: true };
+        expect(new tokenizer_1.Tokenizer(input, params).parse()).toEqual(expectedTokens);
+    });
+    it('escaping should prevent special token matching', () => {
+        const input = 'N\\ULL';
+        const expectedTokens = [makeToken('word', 'N\\ULL', 0, 5)];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should match escaped characters', () => {
+        const input = 'hello \\n world';
+        const expectedTokens = [
+            makeToken('word', 'hello', 0, 5),
+            makeToken('word', '\\n', 6, 8),
+            makeToken('word', 'world', 9, 14),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should preserve all escaped characters', () => {
+        const input = "he'llo \\t \\${w}or\\,ld";
+        const expectedTokens = [
+            makeToken('word', "he'llo", 0, 6),
+            makeToken('word', '\\t', 7, 9),
+            makeToken('word', '\\${w}or\\,ld', 10, 21),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should match regexp and capitalize special matches', () => {
+        const input = "hello tuesDAY,ttuesday, tuesdayy ,Tuesday , ttuesday, 'TUESday' ";
+        const expectedTokens = [
+            makeToken('word', 'hello', 0, 5),
+            makeToken('DAYOFWEEK', 'TUESDAY', 6, 13),
+            makeToken(',', ',', 13, 14),
+            makeToken('word', 'ttuesday', 14, 22),
+            makeToken(',', ',', 22, 23),
+            makeToken('word', 'tuesdayy', 24, 32),
+            makeToken(',', ',', 33, 34),
+            makeToken('DAYOFWEEK', 'TUESDAY', 34, 41),
+            makeToken(',', ',', 42, 43),
+            makeToken('word', 'ttuesday', 44, 52),
+            makeToken(',', ',', 52, 53),
+            makeToken('word', "'TUESday'", 54, 63),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should match regexp and not capitalize', () => {
+        const input = 'Washington, Washingo,washington,wWashington ';
+        const expectedTokens = [
+            makeToken('STATE', 'Washington', 0, 10),
+            makeToken(',', ',', 10, 11),
+            makeToken('word', 'Washingo', 12, 20),
+            makeToken(',', ',', 20, 21),
+            makeToken('STATE', 'washington', 21, 31),
+            makeToken(',', ',', 31, 32),
+            makeToken('word', 'wWashington', 32, 43),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    it('should match substring regexp', () => {
+        const input = 'hello  \\${var1},aa${var2}bb, cc${var3} dd';
+        const expectedTokens = [
+            makeToken('word', 'hello', 0, 5),
+            makeToken('word', '\\${var1}', 7, 15),
+            makeToken(',', ',', 15, 16),
+            makeToken('word', 'aa', 16, 18),
+            makeToken('VARIABLE', '${var2}', 18, 25),
+            makeToken('word', 'bb', 25, 27),
+            makeToken(',', ',', 27, 28),
+            makeToken('word', 'cc', 29, 31),
+            makeToken('VARIABLE', '${var3}', 31, 38),
+            makeToken('word', 'dd', 39, 41),
+        ];
+        expect(new tokenizer_1.Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+    });
+    describe('mergeTypes', () => {
+        it('should return an empty array when input is empty', () => {
+            const result = tokenizer_1.Tokenizer.mergeTypes('', [], 'merged');
+            expect(result).toEqual([]);
+        });
+        it('should not merge tokens when no match is found', () => {
+            const tokens = [
+                { type: 'x', value: '1', startIndex: 0, endIndex: 1 },
+                { type: 'y', value: '2', startIndex: 1, endIndex: 2 },
+            ];
+            const result = tokenizer_1.Tokenizer.mergeTypes('a|b', tokens, 'merged');
+            expect(result).toEqual(tokens);
+        });
+        it('should merge tokens when a single match is found', () => {
+            const tokens = [
+                { type: 'a', value: '1', startIndex: 0, endIndex: 1 },
+                { type: 'b', value: '2', startIndex: 1, endIndex: 2 },
+            ];
+            const expected = [
+                {
+                    type: 'merged',
+                    value: '',
+                    values: [tokens[0], tokens[1]],
+                    startIndex: 0,
+                    endIndex: 2,
+                },
+            ];
+            const result = tokenizer_1.Tokenizer.mergeTypes('a|b', tokens, 'merged');
+            expect(result).toEqual(expected);
+        });
+        it('should merge multiple matches', () => {
+            const tokens = [
+                { type: 'a', value: '1', startIndex: 0, endIndex: 1 },
+                { type: 'b', value: '2', startIndex: 1, endIndex: 2 },
+                { type: 'a', value: '3', startIndex: 2, endIndex: 3 },
+                { type: 'b', value: '4', startIndex: 3, endIndex: 4 },
+            ];
+            const expected = [
+                {
+                    type: 'merged',
+                    value: '',
+                    values: [tokens[0], tokens[1]],
+                    startIndex: 0,
+                    endIndex: 2,
+                },
+                {
+                    type: 'merged',
+                    value: '',
+                    values: [tokens[2], tokens[3]],
+                    startIndex: 2,
+                    endIndex: 4,
+                },
+            ];
+            const result = tokenizer_1.Tokenizer.mergeTypes('a|b', tokens, 'merged');
+            expect(result).toEqual(expected);
+        });
+        it('should not merge partial matches', () => {
+            const tokens = [
+                { type: 'a', value: '1', startIndex: 0, endIndex: 1 },
+                { type: 'b', value: '2', startIndex: 1, endIndex: 2 },
+            ];
+            const result = tokenizer_1.Tokenizer.mergeTypes('a|b|c', tokens, 'merged');
+            expect(result).toEqual(tokens);
+        });
+    });
+    describe('matchTypes', () => {
+        it('should return tokens when types match', () => {
+            const tokens = [makeToken('a', '1', 0, 1), makeToken('b', '2', 1, 2)];
+            expect(tokenizer_1.Tokenizer.matchTypes('a|b', tokens, 0)).toEqual(tokens);
+        });
+        it('should return undefined when types do not match', () => {
+            const tokens = [makeToken('x', '1', 0, 1), makeToken('y', '2', 1, 2)];
+            expect(tokenizer_1.Tokenizer.matchTypes('a|b', tokens, 0)).toBeUndefined();
+        });
+        it('should return undefined when index is out of range', () => {
+            const tokens = [makeToken('a', '1', 0, 1)];
+            expect(tokenizer_1.Tokenizer.matchTypes('a|b', tokens, 1)).toBeUndefined();
+        });
+        it('should return all matching tokens when multiple types match', () => {
+            const tokens = [
+                makeToken('a', '1', 0, 1),
+                makeToken('b', '2', 1, 2),
+                makeToken('c', '3', 2, 3),
+            ];
+            expect(tokenizer_1.Tokenizer.matchTypes('a|b|c', tokens, 0)).toEqual(tokens);
+        });
+        it('should return only matching tokens when partial match occurs', () => {
+            const tokens = [makeToken('a', '1', 0, 1), makeToken('b', '2', 1, 2)];
+            expect(tokenizer_1.Tokenizer.matchTypes('a|b|c', tokens, 0)).toBeUndefined();
+        });
+    });
+});
+//# sourceMappingURL=tokenizer.spec.js.map

package/dist/tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.spec.js","sourceRoot":"","sources":["../src/tokenizer.spec.ts"],"names":[],"mappings":";;AAAA,2CAAqE;AAGrE,SAAS,UAAU;IACjB,MAAM,iBAAiB,GAAmB;QACxC,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAC;QACvB,EAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,cAAc,EAAC;KAC1C,CAAC;IACF,MAAM,YAAY,GAAmB;QACnC,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAC;QAC/C,EAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAC;QACjD,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAC;QACnD,EAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAC;QACrD;YACE,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,+DAA+D;YACtE,UAAU,EAAE,IAAI;SACjB;QACD,EAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,4BAA4B,EAAC;QACpD,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,sBAAsB,EAAC;QAC7C,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAC;QACxC,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAC;KACpC,CAAC;IACF,OAAO;QACL,iBAAiB,EAAE,IAAI;QACvB,kBAAkB,EAAE,IAAI;QACxB,iBAAiB;QACjB,YAAY;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,MAAM,SAAS,GAAG,CAChB,IAAY,EACZ,KAAa,EACb,UAAkB,EAClB,QAAgB,EACT,EAAE,CAAC,CAAC,EAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAC,CAAC,CAAC;IAClD,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,KAAK,GAAG,aAAa,CAAC;QAC5B,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;SAClC,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,iBAAiB,GAAmB;YACxC,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAC;YACvB,EAAC,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAC;YACjC,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAC;SAC9B,CAAC;QACF,MAAM,KAAK,GAAG,oBAAoB,CAAC;QACnC,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/B,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;YACjC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACzB,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC;YAClC,SAAS,CAAC,aAAa,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;SACtC,CAAC;QACF,MAAM,CACJ,IAAI,qBAAS,CAAC,KAAK,EAAE,EAAC,GAAG,UAAU,EAAE,EAAE,iBAAiB,EAAC,CAAC,CAAC,KAAK,EAAE,CACnE,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,iDAAiD,CAAC;QAChE,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC;YAClC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACjC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACjC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;SAC5B,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,SAAS,CAAC;QACxB,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9B,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SAC/B,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,KAAK,GAAG,SAAS,CAAC;QACxB,MAAM,cAAc,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,EAAC,GAAG,UAAU,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAC,CAAC;QAC7D,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,KAAK,GAAG,QAAQ,CAAC;QACvB,MAAM,cAAc,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9B,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;SAClC,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAG,0BAA0B,CAAC;QACzC,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;YACjC,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9B,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE,CAAC;SAC3C,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,KAAK,GACT,kEAAkE,CAAC;QACrE,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;YACzC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC;SACvC,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAG,8CAA8C,CAAC;QAC7D,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,CAAC;YACxC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,EAAE,CAAC;SACzC,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,KAAK,GAAG,4CAA4C,CAAC;QAC3D,MAAM,cAAc,GAAG;YACrB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC;YAC/B,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;YACxC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC;YAC/B,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3B,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC;YAC/B,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;YACxC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC;SAChC,CAAC;QACF,MAAM,CAAC,IAAI,qBAAS,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,MAAM,GAAG,qBAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,MAAM,GAAY;gBACtB,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;aACpD,CAAC;YACF,MAAM,MAAM,GAAG,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,MAAM,GAAY;gBACtB,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;aACpD,CAAC;YACF,MAAM,QAAQ,GAAY;gBACxB;oBACE,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC9B,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;iBACZ;aACF,CAAC;YACF,MAAM,MAAM,GAAG,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAY;gBACtB,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;aACpD,CAAC;YACF,MAAM,QAAQ,GAAY;gBACxB;oBACE,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC9B,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;iBACZ;gBACD;oBACE,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC9B,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;iBACZ;aACF,CAAC;YACF,MAAM,MAAM,GAAG,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAY;gBACtB,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;gBACnD,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAC;aACpD,CAAC;YACF,MAAM,MAAM,GAAG,qBAAS,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,CAAC,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,CAAC,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,qBAAS,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;YACrE,MAAM,MAAM,GAAG;gBACb,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBACzB,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBACzB,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;aAC1B,CAAC;YACF,MAAM,CAAC,qBAAS,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;YACtE,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,CAAC,qBAAS,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  moduleFileExtensions: ['js'],
+};

package/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@malloydata/malloy-filter",
+  "version": "0.0.237-dev250221201621",
+  "license": "MIT",
+  "description": "",
+  "main": "dist/string_parser.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/malloydata/malloy"
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "samples": "node dist/generate_samples.js",
+    "asimpleparser": "node dist/a_simple_parser.js",
+    "asimpleserializer": "node dist/a_simple_serializer.js",
+    "test": "jest --runInBand",
+    "test:tokenizer": "jest tokenizer.spec.ts"
+  },
+  "keywords": [],
+  "author": ""
+}

package/src/DEVELOPING.md

@@ -0,0 +1,26 @@
+# Testing
+
+The `malloy-filter` tests use Jest.  Install Jest using:
+
+```bash
+npm install --save-dev jest @types/jest ts-jest
+```
+
+## Tokenizer Testing
+
+```bash
+npm run test:tokenizer
+```
+
+# generate_samples
+
+The `generate_samples.ts` script creates the suite of examples in ../SAMPLES.md.  As such, it is also a terrific tool for testing and validating changes to any of the parsers.  Feel free to add new examples to `generate_samples.ts` as new features are added to the parsers.
+
+To create the samples:
+
+```bash
+npm run build
+npm run samples
+```
+
+Comment/uncomment individual lines in `generateSamples()` to enable/disable sample generation for the different parsers.

package/src/a_simple_parser.ts

@@ -0,0 +1,22 @@
+import {FilterParser} from './filter_parser';
+
+/* eslint-disable no-console */
+function aSimpleParser() {
+  let str = 'CAT,DOG';
+  let response = new FilterParser(str, 'string').parse();
+  console.log(str, '\n', ...response.clauses, '\n');
+
+  str = '-5.5, 10, 2.3e7';
+  response = new FilterParser(str, 'number').parse();
+  console.log(str, '\n', ...response.clauses, '\n');
+
+  str = 'null, false';
+  response = new FilterParser(str, 'boolean').parse();
+  console.log(str, '\n', ...response.clauses, '\n');
+
+  str = 'after 2025-10-05';
+  response = new FilterParser(str, 'date').parse();
+  console.log(str, '\n', ...response.clauses, '\n');
+}
+
+aSimpleParser();