@malloydata/malloy-filter 0.0.237-dev250221201621

package/src/generate_samples.ts

@@ -0,0 +1,387 @@
+import {Clause} from './clause_types';
+import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
+import {BooleanParser} from './boolean_parser';
+import {StringParser} from './string_parser';
+import {NumberParser} from './number_parser';
+import {DateParser} from './date_parser';
+import {BaseParser} from './base_parser';
+import {BooleanSerializer} from './boolean_serializer';
+import {StringSerializer} from './string_serializer';
+import {NumberSerializer} from './number_serializer';
+import {DateSerializer} from './date_serializer';
+import {BaseSerializer} from './base_serializer';
+
+const numberExamples = [
+  '5',
+  '!=5',
+  '1, 3, null , 7',
+  '<1, >=100 ',
+  '>=1',
+  ' <= 10 ',
+  'NULL',
+  ' -NULL',
+  '(1, 7)',
+  '[-5, 90]',
+  ' != ( 12, 20 ] ',
+  '[.12e-20, 20.0e3)',
+  '[0,9],[20,29]',
+  '[0,10], 20, NULL, ( 72, 82 ] ',
+  ', notanumber,, "null", apple pear orange, nulle, nnull, >=,',
+  '[cat, 100], <cat',
+  '-5.5 to 10',
+];
+
+const stringExamples = [
+  'CAT, DOG,mouse ',
+  '-CAT,-DOG , -mouse',
+  ' CAT,-"DOG",m o u s e',
+  '-CAT,-DOG,mouse, bird, zebra, -horse, -goat',
+  'Missing ,NULL',
+  'CAT%, D%OG, %ous%, %ira_f%, %_oat, ',
+  '-CAT%,-D%OG,-%mouse,-%zebra%',
+  'CAT%,-CATALOG',
+  '%,_,%%,%a%',
+  '%\\_X',
+  '_\\_X',
+  '_CAT,D_G,mouse_',
+  '\\_CAT,D\\%G,\\mouse',
+  'CAT,-NULL',
+  'CAT,-"NULL"',
+  'CAT,NULL',
+  'EMPTY',
+  '-EMPTY',
+  'CAT,-EMPTY',
+  '"CAT,DOG\',mo`use,zeb\'\'\'ra,g"""t,g\\"ir\\`af\\\'e',
+  'CAT\\,DOG',
+  'CAT,DOG,-, - ',
+  '--CAT,DOG,\\',
+  'CAT\\ DOG',
+  '_\\_CAT',
+  '\\NULL',
+  '\\-NULL',
+  '-N\\ULL',
+  'CA--,D-G', // _ = 'CA--' OR _ = 'D-G'
+  ' hello world, foo="bar baz" , qux=quux',
+  'one ,Null ,  Empty,E M P T Y Y,EEmpty,        emptIEs',
+  '',
+];
+
+const booleanExamples = [
+  'true',
+  'FALSE',
+  'null',
+  '-NULL',
+  ' True , faLSE,NULl,-null',
+  "-'null'",
+  '10',
+  'nnull',
+  ' truee ',
+];
+
+const dateExamples = [
+  'this month',
+  '3 days',
+  '3 days ago',
+  '3 months ago for 2 days',
+  'after 2025 seconds',
+  '2025 weeks ago',
+  'before 3 days ago',
+  'before 2025-08-30 08:30:20',
+  'after 2025-10-05',
+  '2025-08-30 12:00:00 to 2025-09-18 14:00:00',
+  'this year',
+  'next tuesday',
+  '7 years from now',
+  '2025-01-01 12:00:00 for 3 days',
+  '2020-08-12',
+  '2020-08',
+  'today',
+  'yesterday',
+  'tomorrow',
+  'TODay,Yesterday, TOMORROW , ,TODay ,,',
+  '2010 to 2011, 2015 to 2016 , 2018, 2020 ',
+  'next week',
+  'now',
+  'now to next month',
+  ' yyesterday ', // Typo
+  'before', // Bad syntax
+  'for', // Bad syntax
+  '7', // Bad syntax
+  'from now', // Bad syntax
+  '',
+];
+
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable no-console */
+
+function testTokenizerSingle(str: string, params: TokenizerParams): void {
+  const tokenizer = new Tokenizer(str, params);
+  console.log(str, ' -> ', tokenizer.parse(), '\n');
+}
+
+function testTokenizerString() {
+  // Test string parser tokenizer.  Do not split on whitespace
+  const specialSubstrings: SpecialToken[] = [{type: ',', value: ','}];
+  const specialWords: SpecialToken[] = [];
+  const params: TokenizerParams = {
+    trimWordWhitespace: true,
+    combineAdjacentWords: true,
+    specialSubstrings,
+    specialWords,
+  };
+  const examples = [
+    'CAT, DOG,mouse ',
+    '-CAT,-DOG , -mouse',
+    ' CAT,-"DOG",m o u s e',
+    '-CAT,-DOG,mouse, bird, zebra, -horse, -goat',
+    'Missing ,NULL',
+    'CAT%,D%OG',
+    'CAT%,-CATALOG',
+    '_CAT,D_G',
+    'CAT,-NULL',
+    'CAT,-"NULL"',
+    'EMPTY',
+    '-EMPTY',
+    'CAT,-EMPTY',
+    '"CAT,DOG"',
+    'CAT\\,DOG',
+    'CAT,DOG,-, - ',
+    '--CAT,DOG,\\',
+    'CA--,D-G', // _ = 'CA--' OR _ = 'D-G'
+    ' hello world, foo="bar baz" , qux=quux',
+    'one ,Null ,  Empty,E M P T Y Y,EEmpty,        emptIEs',
+    '',
+  ];
+  for (const example of examples) {
+    testTokenizerSingle(example, params);
+  }
+}
+
+function testTokenizerNumber() {
+  // Test number parser tokenizer, split on whitespace
+  const specialSubstrings: SpecialToken[] = [
+    {type: ',', value: ','},
+    {type: '[', value: '['},
+    {type: ']', value: ']'},
+    {type: '(', value: '('},
+    {type: ')', value: ')'},
+    {type: '<=', value: '<='},
+    {type: '>=', value: '>='},
+    {type: '!=', value: '!='},
+    {type: '=', value: '='},
+    {type: '>', value: '>'},
+    {type: '<', value: '<'},
+  ];
+  const specialWords: SpecialToken[] = [
+    {type: 'TO', value: 'to', ignoreCase: true},
+    {type: '-NULL', value: '-null', ignoreCase: true},
+    {type: 'NULL', value: 'null', ignoreCase: true},
+  ];
+  const params: TokenizerParams = {
+    trimWordWhitespace: true,
+    combineAdjacentWords: false,
+    splitOnWhitespace: true,
+    specialSubstrings,
+    specialWords,
+  };
+  const examples = [
+    '5',
+    '!=5',
+    '1, 3, null , 7',
+    '<1, >=100 ',
+    '-5.5 to 10',
+    '>=1',
+    ' <= 10 ',
+    'NULL',
+    ' -NULL',
+    '(1, 7)',
+    '[-5, 90]',
+    ' ( 12, 20 ] ',
+    '[.12e-20, 20.0e3)',
+    '[0,9],[20,29]',
+    '[0,10], 20, NULL, ( 72, 82 ] ',
+    ', notanumber,, -"-null", apple pear orange, nulle, nnull',
+  ];
+  for (const example of examples) {
+    testTokenizerSingle(example, params);
+  }
+}
+
+function testParserSingle(
+  str: string,
+  parser: BaseParser,
+  outputFormatter?: (clauses: Clause[]) => string
+): void {
+  console.log('Input: ', str);
+  try {
+    const response = parser.parse();
+    // console.log('Tokens: ', parser.getTokens());
+    if (response.clauses && response.clauses.length > 0) {
+      if (outputFormatter) {
+        console.log('Output: ', outputFormatter(response.clauses));
+      } else {
+        console.log('Output: ', ...response.clauses);
+      }
+    }
+    if (response.errors && response.errors.length > 0) {
+      console.log('Errors: ', ...response.errors);
+    }
+  } catch (ex: Error | unknown) {
+    if (ex instanceof Error) console.error('Thrown Error: ', ex.message);
+    else {
+      console.error('Thrown Unknown error: ', ex);
+    }
+  }
+  console.log('');
+}
+
+function testSerializerRoundtrip(
+  str: string,
+  parser: BaseParser,
+  serializerFunc: (clauses: Clause[]) => BaseSerializer
+): void {
+  console.log('Input:  ' + str);
+  try {
+    const response = parser.parse();
+    // console.log('Clause: ', ...response.clauses, '\n');
+    if (response.clauses && response.clauses.length > 0) {
+      const result = serializerFunc(response.clauses || []).serialize();
+      console.log('Output: ' + result);
+    }
+    if (response.errors && response.errors.length > 0) {
+      console.log('Errors: ', ...response.errors);
+    }
+  } catch (ex: Error | unknown) {
+    if (ex instanceof Error) console.error('Thrown Error: ', ex.message);
+    else {
+      console.error('Thrown Unknown error: ', ex);
+    }
+  }
+  console.log('');
+}
+
+function testNumberParser() {
+  for (const example of numberExamples) {
+    const parser = new NumberParser(example);
+    testParserSingle(example, parser);
+  }
+}
+
+function testStringParser() {
+  for (const example of stringExamples) {
+    const parser = new StringParser(example);
+    testParserSingle(example, parser);
+  }
+}
+
+function testBooleanParser() {
+  for (const example of booleanExamples) {
+    const parser = new BooleanParser(example);
+    testParserSingle(example, parser);
+  }
+}
+
+function jsonFormatter(clauses: Clause[]): string {
+  let str = '';
+  for (const clause of clauses) {
+    str += JSON.stringify(clause, null, ' ');
+  }
+  return str;
+}
+
+function testDateParser() {
+  for (const example of dateExamples) {
+    const parser = new DateParser(example);
+    testParserSingle(example, parser);
+    //testParserSingle(example, parser, jsonFormatter);
+  }
+}
+
+function testNumberSerializer(): void {
+  const examples = [
+    [{operator: '>', value: 10}],
+    [{startOperator: '>=', startValue: 20, endOperator: '<=', endValue: 30}],
+  ];
+  for (const example of numberExamples) {
+    testSerializerRoundtrip(
+      example,
+      new NumberParser(example),
+      clauses => new NumberSerializer(clauses)
+    );
+  }
+}
+
+function testStringSerializer(): void {
+  for (const example of stringExamples) {
+    testSerializerRoundtrip(
+      example,
+      new StringParser(example),
+      clauses => new StringSerializer(clauses)
+    );
+  }
+}
+
+function testBooleanSerializer(): void {
+  const examples = [[{operator: 'TRUE'}]];
+  for (const example of booleanExamples) {
+    testSerializerRoundtrip(
+      example,
+      new BooleanParser(example),
+      clauses => new BooleanSerializer(clauses)
+    );
+  }
+}
+
+function testDateSerializer(): void {
+  const examples = [
+    [{prefix: 'BEFORE', values: [{type: 'day', value: 'yesterday'}]}],
+    [
+      {
+        start: [{type: 'day', value: 'today'}],
+        operator: 'TO',
+        end: [{type: 'day', value: 'tomorrow'}],
+      },
+    ],
+    [{type: 'day', value: 'today'}],
+  ];
+  for (const example of dateExamples) {
+    testSerializerRoundtrip(
+      example,
+      new DateParser(example),
+      clauses => new DateSerializer(clauses)
+    );
+  }
+}
+
+function printHeader(title: string): void {
+  console.log(
+    '\n-------------------------------------------------------------------------'
+  );
+  console.log('## ', title, '\n');
+}
+
+// Comment or uncomment the following function calls to disable/enable examples.
+function generateSamples() {
+  //printHeader('Tokenizer');
+  //testTokenizerString();
+  //testTokenizerNumber();
+  //testTokenizerMatchTypes();
+  //printHeader('Numbers');
+  //testNumberParser();
+  //printHeader('Strings');
+  //testStringParser();
+  //printHeader('Booleans');
+  //testBooleanParser();
+  //printHeader('Dates and Times');
+  //testDateParser();
+  printHeader('Number Serializer');
+  testNumberSerializer();
+  printHeader('String Serializer');
+  testStringSerializer();
+  printHeader('Boolean Serializer');
+  testBooleanSerializer();
+  printHeader('Date and Time Serializer');
+  testDateSerializer();
+}
+
+generateSamples();

package/src/number_parser.ts

@@ -0,0 +1,308 @@
+import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
+import {
+  NumberCondition,
+  NumberRange,
+  NumberOperator,
+  NumberRangeOperator,
+  NumberClause,
+  Clause,
+} from './clause_types';
+import {BaseParser} from './base_parser';
+import {Token} from './token_types';
+import {FilterParserResponse, FilterError} from './filter_types';
+
+export class NumberParser extends BaseParser {
+  constructor(input: string) {
+    super(input);
+  }
+
+  private tokenize(): void {
+    const specialSubstrings: SpecialToken[] = [
+      {type: ',', value: ','},
+      {type: '[', value: '['},
+      {type: ']', value: ']'},
+      {type: '(', value: '('},
+      {type: ')', value: ')'},
+      {type: '<=', value: '<='},
+      {type: '>=', value: '>='},
+      {type: '!=', value: '!='},
+      {type: '=', value: '='},
+      {type: '>', value: '>'},
+      {type: '<', value: '<'},
+    ];
+    const specialWords = [
+      {type: 'NOTNULL', value: '-null', ignoreCase: true},
+      {type: 'NULL', value: 'null', ignoreCase: true},
+    ];
+    const params: TokenizerParams = {
+      trimWordWhitespace: true,
+      combineAdjacentWords: false,
+      splitOnWhitespace: true,
+      specialSubstrings,
+      specialWords,
+    };
+
+    const tokenizer = new Tokenizer(this.inputString, params);
+    this.tokens = tokenizer.parse();
+  }
+
+  public parse(): FilterParserResponse {
+    this.index = 0;
+    this.tokenize();
+    let clauses: NumberClause[] = [];
+    const errors: FilterError[] = [];
+    while (this.index < this.tokens.length) {
+      const token = this.getNext();
+      if (token.type === ',') {
+        this.index++;
+      } else if (this.isRangeStart(this.index)) {
+        clauses = this.checkRange(token, false, clauses, errors);
+      } else if (token.type === '!=' && this.isRangeStart(this.index + 1)) {
+        this.index++;
+        clauses = this.checkRange(token, true, clauses, errors);
+      } else if (this.checkNull(clauses)) {
+        this.index++;
+      } else if (
+        this.checkNumericExpression('<=', clauses) ||
+        this.checkNumericExpression('>=', clauses) ||
+        this.checkNumericExpression('<', clauses) ||
+        this.checkNumericExpression('>', clauses) ||
+        this.checkNumericExpression('!=', clauses) ||
+        this.checkNumericExpression('=', clauses)
+      ) {
+        this.index += 2;
+      } else if (this.checkSimpleNumber(clauses)) {
+        this.index++;
+      } else {
+        errors.push({
+          message: 'Invalid expression',
+          startIndex: token.startIndex,
+          endIndex: token.endIndex,
+        });
+        this.index++;
+      }
+    }
+    return {clauses: NumberParser.groupClauses(clauses), errors};
+  }
+
+  private static groupClauses(clauses: NumberClause[]): NumberClause[] {
+    if (clauses.length < 2) {
+      return clauses;
+    }
+    let previous: NumberClause | undefined = undefined;
+    const outputs: NumberClause[] = [];
+    for (let i = 0; i < clauses.length; i++) {
+      const clause: NumberClause = clauses[i];
+      if (clause.operator === 'range') {
+        previous = undefined;
+        outputs.push(clause);
+      } else if (
+        previous !== undefined &&
+        previous.operator === clause.operator
+      ) {
+        previous.values.push(...clause.values);
+      } else {
+        previous = clause;
+        outputs.push(clause);
+      }
+    }
+    return outputs;
+  }
+
+  private matchTokens(candidates: string[]): boolean {
+    return BaseParser.matchTokenTypes(candidates, this.index, this.tokens);
+  }
+
+  private checkRange(
+    token: Token,
+    negated: boolean,
+    clauses: NumberClause[],
+    errors: FilterError[]
+  ): NumberClause[] {
+    if (this.matchTokens(['[', 'word', ',', 'word', ']'])) {
+      return negated
+        ? this.consumeRange('<', '>', clauses, errors)
+        : this.consumeRange('>=', '<=', clauses, errors);
+    } else if (this.matchTokens(['[', 'word', ',', 'word', ')'])) {
+      return negated
+        ? this.consumeRange('<', '>=', clauses, errors)
+        : this.consumeRange('>=', '<', clauses, errors);
+    } else if (this.matchTokens(['(', 'word', ',', 'word', ']'])) {
+      return negated
+        ? this.consumeRange('<=', '>', clauses, errors)
+        : this.consumeRange('>', '<=', clauses, errors);
+    } else if (this.matchTokens(['(', 'word', ',', 'word', ')'])) {
+      return negated
+        ? this.consumeRange('<=', '>=', clauses, errors)
+        : this.consumeRange('>', '<', clauses, errors);
+    } else {
+      errors.push({
+        message: 'Invalid range expression',
+        startIndex: token.startIndex,
+        endIndex: token.endIndex,
+      });
+      this.index++;
+    }
+    return clauses;
+  }
+
+  private isRangeStart(position: number): boolean {
+    if (position < 0 || position >= this.tokens.length) {
+      return false;
+    }
+    const tokenType = this.tokens[position].type;
+    return tokenType === '[' || tokenType === '(';
+  }
+
+  private static parseNumber(value: string): number {
+    if (value.toUpperCase() === 'inf') {
+      return Infinity;
+    } else if (value.toUpperCase() === '-inf') {
+      return -Infinity;
+    } else {
+      return Number(value);
+    }
+  }
+
+  private static isValidNumber(value: number): boolean {
+    return Number.isNaN(value) === false;
+  }
+
+  private consumeRange(
+    startOperator: NumberRangeOperator,
+    endOperator: NumberRangeOperator,
+    clauses: NumberClause[],
+    errors: FilterError[]
+  ): NumberClause[] {
+    const startToken = this.getAt(this.index + 1);
+    const endToken = this.getAt(this.index + 3);
+    const startValue: number = NumberParser.parseNumber(startToken.value);
+    const endValue: number = NumberParser.parseNumber(endToken.value);
+    if (!NumberParser.isValidNumber(startValue)) {
+      errors.push({
+        message: 'Invalid number',
+        startIndex: startToken.startIndex,
+        endIndex: startToken.endIndex,
+      });
+    } else if (!NumberParser.isValidNumber(endValue)) {
+      errors.push({
+        message: 'Invalid number',
+        startIndex: endToken.startIndex,
+        endIndex: endToken.endIndex,
+      });
+    } else {
+      const clause: NumberRange = {
+        operator: 'range',
+        startOperator: startOperator,
+        startValue: startValue,
+        endOperator: endOperator,
+        endValue: endValue,
+      };
+      clauses.push(clause);
+    }
+    this.index += 5;
+    return clauses;
+  }
+
+  private static getNegatedType(tokenType: NumberOperator): NumberOperator {
+    switch (tokenType) {
+      case '<=':
+        return '>';
+      case '>=':
+        return '<';
+      case '<':
+        return '>';
+      case '>':
+        return '<';
+      case '=':
+        return '!=';
+      case '!=':
+        return '=';
+    }
+  }
+
+  private static getNumberOperator(
+    tokenType: string
+  ): NumberOperator | undefined {
+    switch (tokenType) {
+      case '<=':
+        return '<=';
+      case '>=':
+        return '>=';
+      case '<':
+        return '<';
+      case '>':
+        return '>';
+      case '=':
+        return '=';
+      case '!=':
+        return '!=';
+    }
+    return undefined;
+  }
+
+  private static getNumberRnageOperator(
+    tokenType: string
+  ): NumberRangeOperator | undefined {
+    switch (tokenType) {
+      case '<=':
+        return '<=';
+      case '>=':
+        return '>=';
+      case '<':
+        return '<';
+      case '>':
+        return '>';
+    }
+    return undefined;
+  }
+
+  private checkNumericExpression(
+    tokenType: NumberOperator,
+    clauses: NumberClause[]
+  ): boolean {
+    if (
+      this.getNext().type === tokenType &&
+      this.index < this.tokens.length - 1
+    ) {
+      const numericValue: number = NumberParser.parseNumber(
+        this.getAt(this.index + 1).value
+      );
+      if (!NumberParser.isValidNumber(numericValue)) {
+        return false;
+      }
+      const clause: NumberCondition = {
+        operator: tokenType,
+        values: [numericValue],
+      };
+      clauses.push(clause);
+      return true;
+    }
+    return false;
+  }
+
+  private checkSimpleNumber(clauses: Clause[]): boolean {
+    if (this.getNext().type === 'word') {
+      const numericValue: number = NumberParser.parseNumber(
+        this.getAt(this.index).value
+      );
+      if (!NumberParser.isValidNumber(numericValue)) {
+        return false;
+      }
+      const clause: NumberCondition = {operator: '=', values: [numericValue]};
+      clauses.push(clause);
+      return true;
+    }
+    return false;
+  }
+
+  private checkNull(clauses: Clause[]): boolean {
+    if (this.getNext().type === 'NULL' || this.getNext().type === 'NOTNULL') {
+      const tokenType = this.getNext().type === 'NULL' ? '=' : '!=';
+      const clause: NumberCondition = {operator: tokenType, values: [null]};
+      clauses.push(clause);
+      return true;
+    }
+    return false;
+  }
+}