@malloydata/malloy-filter 0.0.237-dev250221201621

package/src/number_serializer.ts

@@ -0,0 +1,96 @@
+import {
+  NumberCondition,
+  NumberRange,
+  NumberOperator,
+  NumberRangeOperator,
+  Clause,
+} from './clause_types';
+import {BaseSerializer} from './base_serializer';
+
+export class NumberSerializer extends BaseSerializer {
+  constructor(clauses: Clause[]) {
+    super(clauses);
+  }
+
+  public serialize(): string {
+    const result = NumberSerializer.clauseToString(this.clauses);
+    return result.trim().replace(/,$/, '');
+  }
+
+  // NumberOperator = '<=' | '>=' | '!=' | '=' | '>' | '<';
+  private static numberConditionToString(
+    operator: NumberOperator,
+    value: number | null
+  ): string {
+    if (value === null) {
+      return operator === '=' ? 'NULL' : '-NULL';
+    }
+    const operatorString = operator === '=' ? '' : operator; // Remove operator for eg "5, 7, 9"
+    return operatorString + value;
+  }
+
+  private static getNegatedType(
+    operator: NumberRangeOperator
+  ): NumberRangeOperator {
+    switch (operator) {
+      case '<':
+        return '>=';
+      case '<=':
+        return '>';
+      case '>':
+        return '<=';
+      case '>=':
+        return '<';
+    }
+  }
+
+  private static isNumberOperator(value: string): value is NumberOperator {
+    return ['<=', '>=', '!=', '=', '>', '<'].includes(value);
+  }
+
+  private static rangeToString(clause: NumberRange): string {
+    const negated: string =
+      clause.startOperator === '<' || clause.startOperator === '<=' ? '!=' : '';
+    const startOperator = negated
+      ? NumberSerializer.getNegatedType(clause.startOperator)
+      : clause.startOperator;
+    const endOperator = negated
+      ? NumberSerializer.getNegatedType(clause.endOperator)
+      : clause.endOperator;
+    const leftBracket: string = startOperator === '>' ? '(' : '[';
+    const rightBracket: string = endOperator === '<' ? ')' : ']';
+    return (
+      negated +
+      leftBracket +
+      clause.startValue +
+      ', ' +
+      clause.endValue +
+      rightBracket
+    );
+  }
+
+  private static clauseToString(clauses: Clause[]): string {
+    let result = '';
+    for (const clause of clauses) {
+      if ('operator' in clause && clause.operator === 'range') {
+        result += NumberSerializer.rangeToString(clause);
+        result += ', ';
+      } else if (
+        'operator' in clause &&
+        NumberSerializer.isNumberOperator(clause.operator)
+      ) {
+        const numberClause: NumberCondition = clause as NumberCondition;
+        for (const value of numberClause.values) {
+          result += NumberSerializer.numberConditionToString(
+            numberClause.operator,
+            value
+          );
+          result += ', ';
+        }
+      } else {
+        throw new Error('Invalid number clause ' + JSON.stringify(clause));
+      }
+    }
+    return result;
+  }
+}

package/src/string_parser.ts

@@ -0,0 +1,193 @@
+import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
+import {StringCondition, StringOperator, QuoteType} from './clause_types';
+import {BaseParser} from './base_parser';
+import {FilterParserResponse, FilterError} from './filter_types';
+
+export class StringParser extends BaseParser {
+  private static readonly percentRegex: RegExp = /(?<!\\)%/;
+  private static readonly underscoreRegex: RegExp = /(?<!\\)_/;
+  private static readonly percentStartRegex: RegExp = /^%/;
+  private static readonly percentEndRegex: RegExp = /(?<!\\)%$/;
+  private static readonly negatedStartRegex: RegExp = /^-(.+)$/;
+  private static readonly singleBackslashRegex: RegExp = /(?<!\\)\\(?!\\)/g;
+
+  constructor(input: string) {
+    super(input);
+  }
+
+  private tokenize(): void {
+    const specialSubstrings: SpecialToken[] = [{type: ',', value: ','}];
+    const specialWords: SpecialToken[] = [
+      {type: 'NULL', value: 'null', ignoreCase: true},
+      {type: 'EMPTY', value: 'empty', ignoreCase: true},
+      {type: 'NOTNULL', value: '-null', ignoreCase: true},
+      {type: 'NOTEMPTY', value: '-empty', ignoreCase: true},
+    ];
+    const params: TokenizerParams = {
+      trimWordWhitespace: true,
+      combineAdjacentWords: true,
+      specialSubstrings,
+      specialWords: specialWords,
+    };
+
+    const tokenizer = new Tokenizer(this.inputString, params);
+    this.tokens = tokenizer.parse();
+    this.tokens = Tokenizer.convertSpecialWords(this.tokens, specialWords);
+  }
+
+  public parse(): FilterParserResponse {
+    this.index = 0;
+    this.tokenize();
+    const clauses: StringCondition[] = [];
+    const errors: FilterError[] = [];
+    while (this.index < this.tokens.length) {
+      const token = this.getNext();
+      if (token.type === ',') {
+        this.index++;
+      } else if (token.type === 'NULL') {
+        clauses.push({operator: '=', values: [null]});
+        this.index++;
+      } else if (token.type === 'EMPTY') {
+        clauses.push({operator: 'EMPTY', values: [null]});
+        this.index++;
+      } else if (token.type === 'NOTNULL') {
+        clauses.push({operator: '!=', values: [null]});
+        this.index++;
+      } else if (token.type === 'NOTEMPTY') {
+        clauses.push({operator: 'NOTEMPTY', values: [null]});
+        this.index++;
+      } else if (this.checkSimpleWord(clauses)) {
+        this.index++;
+      } else {
+        errors.push({
+          message: 'Invalid expression',
+          startIndex: token.startIndex,
+          endIndex: token.endIndex,
+        });
+        this.index++;
+      }
+    }
+    return {clauses: StringParser.groupClauses(clauses), errors};
+  }
+
+  private static findQuotes(str: string): QuoteType[] {
+    const quotes: Set<QuoteType> = new Set();
+    let i = 0;
+
+    while (i < str.length) {
+      // Check for triple quotes first to avoid false positives
+      if (str.slice(i, i + 3) === "'''") {
+        quotes.add('TRIPLESINGLE');
+        i += 3;
+      } else if (str.slice(i, i + 3) === '"""') {
+        quotes.add('TRIPLEDOUBLE');
+        i += 3;
+      } else if (str[i] === '\\') {
+        // Check for escaped quotes
+        if (i + 1 < str.length) {
+          switch (str[i + 1]) {
+            case "'":
+              quotes.add('ESCAPEDSINGLE');
+              break;
+            case '"':
+              quotes.add('ESCAPEDDOUBLE');
+              break;
+            case '`':
+              quotes.add('ESCAPEDBACKTICK');
+              break;
+          }
+          i += 2;
+        } else {
+          i++;
+        }
+      } else {
+        // Check for single quotes
+        switch (str[i]) {
+          case "'":
+            quotes.add('SINGLE');
+            break;
+          case '"':
+            quotes.add('DOUBLE');
+            break;
+          case '`':
+            quotes.add('BACKTICK');
+            break;
+        }
+        i++;
+      }
+    }
+    return Array.from(quotes);
+  }
+
+  private static groupClauses(clauses: StringCondition[]): StringCondition[] {
+    if (clauses.length < 2) {
+      return clauses;
+    }
+    let previous: StringCondition = clauses[0];
+    const outputs: StringCondition[] = [previous];
+    for (let i = 1; i < clauses.length; i++) {
+      if (previous.operator === clauses[i].operator) {
+        previous.values.push(...clauses[i].values);
+      } else {
+        previous = clauses[i];
+        outputs.push(previous);
+      }
+    }
+    return outputs;
+  }
+
+  private static percentInMiddle(word: string): boolean {
+    if (word.length < 3) return false;
+    word = word.substring(1, word.length - 1);
+    return StringParser.percentRegex.test(word);
+  }
+
+  private static removeBackslashes(word: string): string {
+    StringParser.singleBackslashRegex.lastIndex = 0;
+    return word.replace(StringParser.singleBackslashRegex, _match => '');
+  }
+
+  private checkSimpleWord(clauses: StringCondition[]): boolean {
+    const token = this.getNext();
+    if (token.type !== 'word') {
+      return false;
+    }
+    const negatedMatch = StringParser.negatedStartRegex.exec(token.value);
+    let word = negatedMatch ? negatedMatch[1] : token.value;
+
+    const isPercentStart = StringParser.percentStartRegex.test(word);
+    const isPercentEnd = StringParser.percentEndRegex.test(word);
+    const isPercentBoth = isPercentStart && isPercentEnd;
+    const isUnderscore = StringParser.underscoreRegex.test(word);
+    const isPercentMiddle = StringParser.percentInMiddle(word);
+
+    let operator: StringOperator = negatedMatch ? '!=' : '=';
+    if (isUnderscore || isPercentMiddle || (isPercentBoth && word.length < 3)) {
+      operator = negatedMatch ? '!~' : '~';
+    } else if (isPercentBoth && word.length > 2) {
+      operator = negatedMatch ? 'notContains' : 'contains';
+      word = word.substring(1, word.length - 1);
+      word = StringParser.removeBackslashes(word);
+    } else if (isPercentStart) {
+      operator = negatedMatch ? 'notEnds' : 'ends';
+      word = word.substring(1, word.length);
+      word = StringParser.removeBackslashes(word);
+    } else if (isPercentEnd) {
+      operator = negatedMatch ? 'notStarts' : 'starts';
+      word = word.substring(0, word.length - 1);
+      word = StringParser.removeBackslashes(word);
+    } else {
+      // = or !=
+      word = StringParser.removeBackslashes(word);
+    }
+    if (word.length === 0) {
+      return false;
+    }
+
+    const clause: StringCondition = {operator: operator, values: [word]};
+    //const quotes: QuoteType[] = StringParser.findQuotes(word);
+    //if (quotes.length > 0) { clause.quotes = quotes; }
+    clauses.push(clause);
+    return true;
+  }
+}

package/src/string_serializer.ts

@@ -0,0 +1,87 @@
+import {StringCondition, StringOperator, Clause} from './clause_types';
+import {BaseSerializer} from './base_serializer';
+
+export class StringSerializer extends BaseSerializer {
+  constructor(clauses: Clause[]) {
+    super(clauses);
+  }
+
+  public serialize(): string {
+    const result = StringSerializer.clauseToString(this.clauses);
+    return result.trim().replace(/,$/, '');
+  }
+
+  private static isNegated(operator: StringOperator): boolean {
+    return (
+      operator === 'NOTEMPTY' ||
+      operator === '!~' ||
+      operator === '!=' ||
+      operator === 'notStarts' ||
+      operator === 'notEnds' ||
+      operator === 'notContains'
+    );
+  }
+
+  private static escapeSpecialCharacters(input: string): string {
+    return input.replace(/[,\\]/g, match => `\\${match}`);
+  }
+
+  private static escapeWildcardCharacters(input: string): string {
+    return input.replace(/[_%]/g, match => `\\${match}`);
+  }
+
+  // export type StringOperator = 'EMPTY' | 'NOTEMPTY' | 'starts' | 'ends' | 'contains' | 'notStarts' |
+  // 'notEnds' | 'notContains' | '~' | '=' | '!~' | '!=';
+  private static stringConditionToString(
+    operator: StringOperator,
+    value: string | null
+  ): string {
+    if (operator === 'EMPTY') {
+      return 'EMPTY';
+    } else if (operator === 'NOTEMPTY') {
+      return '-EMPTY';
+    }
+
+    const negated: boolean = StringSerializer.isNegated(operator);
+    if (value === null) {
+      return negated ? '-NULL' : 'NULL';
+    }
+    if (value === 'NULL' || value === '-NULL') {
+      return (negated ? '-' : '') + '\\' + value;
+    }
+
+    value = StringSerializer.escapeSpecialCharacters(value);
+    if (operator === 'starts' || operator === 'notStarts') {
+      value = StringSerializer.escapeWildcardCharacters(value);
+      return (negated ? '-' : '') + value + '%';
+    } else if (operator === 'ends' || operator === 'notEnds') {
+      value = StringSerializer.escapeWildcardCharacters(value);
+      return (negated ? '-' : '') + '%' + value;
+    } else if (operator === 'contains' || operator === 'notContains') {
+      value = StringSerializer.escapeWildcardCharacters(value);
+      return (negated ? '-' : '') + '%' + value + '%';
+    } else if (operator === '=' || operator === '!=') {
+      value = StringSerializer.escapeWildcardCharacters(value);
+      return (negated ? '-' : '') + value;
+    }
+
+    return (negated ? '-' : '') + value;
+  }
+
+  private static clauseToString(clauses: Clause[]): string {
+    let result = '';
+    for (const genericClause of clauses) {
+      const clause: StringCondition = genericClause as StringCondition;
+      for (const value of clause.values) {
+        const word = StringSerializer.stringConditionToString(
+          clause.operator,
+          value
+        );
+        if (word) {
+          result += word + ', ';
+        }
+      }
+    }
+    return result;
+  }
+}

package/src/token_types.ts

@@ -0,0 +1,7 @@
+export interface Token {
+  type: string;
+  value: string;
+  startIndex: number; // The start index of this token in the original string.
+  endIndex: number; // The end index of this token in the original string.
+  values?: Token[]; // Merged tokens can contain tokens.  Otherwise undefined.
+}

package/src/tokenizer.spec.ts

@@ -0,0 +1,273 @@
+import {Tokenizer, SpecialToken, TokenizerParams} from './tokenizer';
+import {Token} from './token_types';
+
+function makeParams(): TokenizerParams {
+  const specialSubstrings: SpecialToken[] = [
+    {type: ',', value: ','},
+    {type: 'VARIABLE', value: /^\$\{[^}]+\}/},
+  ];
+  const specialWords: SpecialToken[] = [
+    {type: 'NULL', value: 'null', ignoreCase: true},
+    {type: 'EMPTY', value: 'empty', ignoreCase: true},
+    {type: 'NOTNULL', value: '-null', ignoreCase: true},
+    {type: 'NOTEMPTY', value: '-empty', ignoreCase: true},
+    {
+      type: 'DAYOFWEEK',
+      value: /^(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$/i,
+      ignoreCase: true,
+    },
+    {type: 'STATE', value: /^(California|Washington)$/i},
+    {type: 'DATE', value: /^\d\d\d\d-\d\d-\d\d$/},
+    {type: 'DATE', value: /^\d\d\d\d-\d\d$/},
+    {type: 'DATE', value: /^\d\d\d\d$/},
+  ];
+  return {
+    splitOnWhitespace: true,
+    trimWordWhitespace: true,
+    specialSubstrings,
+    specialWords,
+  };
+}
+
+describe('Tokenizer', () => {
+  const makeToken = (
+    type: string,
+    value: string,
+    startIndex: number,
+    endIndex: number
+  ): Token => ({type, value, startIndex, endIndex});
+  it('should tokenize a simple string', () => {
+    const input = 'hello world';
+    const expectedTokens = [
+      makeToken('word', 'hello', 0, 5),
+      makeToken('word', 'world', 6, 11),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should split adjacent characters when no whitespace', () => {
+    const specialSubstrings: SpecialToken[] = [
+      {type: ',', value: ','},
+      {type: 'exclamation', value: '!'},
+      {type: 'bracket', value: '['},
+    ];
+    const input = '[hello],big,world!';
+    const expectedTokens = [
+      makeToken('bracket', '[', 0, 1),
+      makeToken('word', 'hello]', 1, 7),
+      makeToken(',', ',', 7, 8),
+      makeToken('word', 'big', 8, 11),
+      makeToken(',', ',', 11, 12),
+      makeToken('word', 'world', 12, 17),
+      makeToken('exclamation', '!', 17, 18),
+    ];
+    expect(
+      new Tokenizer(input, {...makeParams(), specialSubstrings}).parse()
+    ).toEqual(expectedTokens);
+  });
+  it('should match special tokens', () => {
+    const input = 'hello NULL world,-Null,-\'NULL" ,NULL, NULL ,  ';
+    const expectedTokens = [
+      makeToken('word', 'hello', 0, 5),
+      makeToken('NULL', 'NULL', 6, 10),
+      makeToken('word', 'world', 11, 16),
+      makeToken(',', ',', 16, 17),
+      makeToken('NOTNULL', '-NULL', 17, 22),
+      makeToken(',', ',', 22, 23),
+      makeToken('word', '-\'NULL"', 23, 30),
+      makeToken(',', ',', 31, 32),
+      makeToken('NULL', 'NULL', 32, 36),
+      makeToken(',', ',', 36, 37),
+      makeToken('NULL', 'NULL', 38, 42),
+      makeToken(',', ',', 43, 44),
+    ];
+    const params = makeParams();
+    expect(new Tokenizer(input, params).parse()).toEqual(expectedTokens);
+  });
+  it('should not combine adjacent words', () => {
+    const input = 'ABC DEF';
+    const expectedTokens = [
+      makeToken('word', 'ABC', 0, 3),
+      makeToken('word', 'DEF', 4, 7),
+    ];
+    const params = makeParams();
+    expect(new Tokenizer(input, params).parse()).toEqual(expectedTokens);
+  });
+  it('should combine adjacent words when combineAdjacentWords', () => {
+    const input = 'ABC DEF';
+    const expectedTokens = [makeToken('word', 'ABCDEF', 0, 7)];
+    const params = {...makeParams(), combineAdjacentWords: true};
+    expect(new Tokenizer(input, params).parse()).toEqual(expectedTokens);
+  });
+  it('escaping should prevent special token matching', () => {
+    const input = 'N\\ULL';
+    const expectedTokens = [makeToken('word', 'N\\ULL', 0, 5)];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should match escaped characters', () => {
+    const input = 'hello \\n world';
+    const expectedTokens = [
+      makeToken('word', 'hello', 0, 5),
+      makeToken('word', '\\n', 6, 8),
+      makeToken('word', 'world', 9, 14),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should preserve all escaped characters', () => {
+    const input = "he'llo \\t \\${w}or\\,ld";
+    const expectedTokens = [
+      makeToken('word', "he'llo", 0, 6),
+      makeToken('word', '\\t', 7, 9),
+      makeToken('word', '\\${w}or\\,ld', 10, 21),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should match regexp and capitalize special matches', () => {
+    const input =
+      "hello tuesDAY,ttuesday, tuesdayy ,Tuesday , ttuesday, 'TUESday' ";
+    const expectedTokens = [
+      makeToken('word', 'hello', 0, 5),
+      makeToken('DAYOFWEEK', 'TUESDAY', 6, 13),
+      makeToken(',', ',', 13, 14),
+      makeToken('word', 'ttuesday', 14, 22),
+      makeToken(',', ',', 22, 23),
+      makeToken('word', 'tuesdayy', 24, 32),
+      makeToken(',', ',', 33, 34),
+      makeToken('DAYOFWEEK', 'TUESDAY', 34, 41),
+      makeToken(',', ',', 42, 43),
+      makeToken('word', 'ttuesday', 44, 52),
+      makeToken(',', ',', 52, 53),
+      makeToken('word', "'TUESday'", 54, 63),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should match regexp and not capitalize', () => {
+    const input = 'Washington, Washingo,washington,wWashington ';
+    const expectedTokens = [
+      makeToken('STATE', 'Washington', 0, 10),
+      makeToken(',', ',', 10, 11),
+      makeToken('word', 'Washingo', 12, 20),
+      makeToken(',', ',', 20, 21),
+      makeToken('STATE', 'washington', 21, 31),
+      makeToken(',', ',', 31, 32),
+      makeToken('word', 'wWashington', 32, 43),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+  it('should match substring regexp', () => {
+    const input = 'hello  \\${var1},aa${var2}bb, cc${var3} dd';
+    const expectedTokens = [
+      makeToken('word', 'hello', 0, 5),
+      makeToken('word', '\\${var1}', 7, 15),
+      makeToken(',', ',', 15, 16),
+      makeToken('word', 'aa', 16, 18),
+      makeToken('VARIABLE', '${var2}', 18, 25),
+      makeToken('word', 'bb', 25, 27),
+      makeToken(',', ',', 27, 28),
+      makeToken('word', 'cc', 29, 31),
+      makeToken('VARIABLE', '${var3}', 31, 38),
+      makeToken('word', 'dd', 39, 41),
+    ];
+    expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
+  });
+
+  describe('mergeTypes', () => {
+    it('should return an empty array when input is empty', () => {
+      const result = Tokenizer.mergeTypes('', [], 'merged');
+      expect(result).toEqual([]);
+    });
+
+    it('should not merge tokens when no match is found', () => {
+      const tokens: Token[] = [
+        {type: 'x', value: '1', startIndex: 0, endIndex: 1},
+        {type: 'y', value: '2', startIndex: 1, endIndex: 2},
+      ];
+      const result = Tokenizer.mergeTypes('a|b', tokens, 'merged');
+      expect(result).toEqual(tokens);
+    });
+
+    it('should merge tokens when a single match is found', () => {
+      const tokens: Token[] = [
+        {type: 'a', value: '1', startIndex: 0, endIndex: 1},
+        {type: 'b', value: '2', startIndex: 1, endIndex: 2},
+      ];
+      const expected: Token[] = [
+        {
+          type: 'merged',
+          value: '',
+          values: [tokens[0], tokens[1]],
+          startIndex: 0,
+          endIndex: 2,
+        },
+      ];
+      const result = Tokenizer.mergeTypes('a|b', tokens, 'merged');
+      expect(result).toEqual(expected);
+    });
+
+    it('should merge multiple matches', () => {
+      const tokens: Token[] = [
+        {type: 'a', value: '1', startIndex: 0, endIndex: 1},
+        {type: 'b', value: '2', startIndex: 1, endIndex: 2},
+        {type: 'a', value: '3', startIndex: 2, endIndex: 3},
+        {type: 'b', value: '4', startIndex: 3, endIndex: 4},
+      ];
+      const expected: Token[] = [
+        {
+          type: 'merged',
+          value: '',
+          values: [tokens[0], tokens[1]],
+          startIndex: 0,
+          endIndex: 2,
+        },
+        {
+          type: 'merged',
+          value: '',
+          values: [tokens[2], tokens[3]],
+          startIndex: 2,
+          endIndex: 4,
+        },
+      ];
+      const result = Tokenizer.mergeTypes('a|b', tokens, 'merged');
+      expect(result).toEqual(expected);
+    });
+
+    it('should not merge partial matches', () => {
+      const tokens: Token[] = [
+        {type: 'a', value: '1', startIndex: 0, endIndex: 1},
+        {type: 'b', value: '2', startIndex: 1, endIndex: 2},
+      ];
+      const result = Tokenizer.mergeTypes('a|b|c', tokens, 'merged');
+      expect(result).toEqual(tokens);
+    });
+  });
+
+  describe('matchTypes', () => {
+    it('should return tokens when types match', () => {
+      const tokens = [makeToken('a', '1', 0, 1), makeToken('b', '2', 1, 2)];
+      expect(Tokenizer.matchTypes('a|b', tokens, 0)).toEqual(tokens);
+    });
+
+    it('should return undefined when types do not match', () => {
+      const tokens = [makeToken('x', '1', 0, 1), makeToken('y', '2', 1, 2)];
+      expect(Tokenizer.matchTypes('a|b', tokens, 0)).toBeUndefined();
+    });
+
+    it('should return undefined when index is out of range', () => {
+      const tokens = [makeToken('a', '1', 0, 1)];
+      expect(Tokenizer.matchTypes('a|b', tokens, 1)).toBeUndefined();
+    });
+
+    it('should return all matching tokens when multiple types match', () => {
+      const tokens = [
+        makeToken('a', '1', 0, 1),
+        makeToken('b', '2', 1, 2),
+        makeToken('c', '3', 2, 3),
+      ];
+      expect(Tokenizer.matchTypes('a|b|c', tokens, 0)).toEqual(tokens);
+    });
+
+    it('should return only matching tokens when partial match occurs', () => {
+      const tokens = [makeToken('a', '1', 0, 1), makeToken('b', '2', 1, 2)];
+      expect(Tokenizer.matchTypes('a|b|c', tokens, 0)).toBeUndefined();
+    });
+  });
+});