@malloydata/malloy-filter 0.0.237-dev250222025222 → 0.0.237-dev250222205057

package/src/generate_samples.ts

@@ -1,20 +1,21 @@
-import {Clause} from './clause_types';
 import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
 import {BooleanParser} from './boolean_parser';
 import {StringParser} from './string_parser';
 import {NumberParser} from './number_parser';
 import {DateParser} from './date_parser';
-import {BaseParser} from './base_parser';
 import {BooleanSerializer} from './boolean_serializer';
 import {StringSerializer} from './string_serializer';
 import {NumberSerializer} from './number_serializer';
 import {DateSerializer} from './date_serializer';
-import {BaseSerializer} from './base_serializer';
+import {BooleanClause, NumberClause, StringClause} from './clause_types';
+import {DateClause} from './date_types';
+
+type Clause = BooleanClause | DateClause | NumberClause | StringClause;
 
 const numberExamples = [
   '5',
   '!=5',
-  '1, 3, null , 7',
+  '1, 3, 5, null',
   '<1, >=100 ',
   '>=1',
   ' <= 10 ',
@@ -69,9 +70,10 @@ const stringExamples = [
 const booleanExamples = [
   'true',
   'FALSE',
+  '=false',
   'null',
   '-NULL',
-  ' True , faLSE,NULl,-null',
+  ' True , faLSE,=false,NULl,-null',
   "-'null'",
   '10',
   'nnull',
@@ -88,7 +90,7 @@ const dateExamples = [
   'before 3 days ago',
   'before 2025-08-30 08:30:20',
   'after 2025-10-05',
-  '2025-08-30 12:00:00 to 2025-09-18 14:00:00',
+  '2025-08-30 12:00 to 2025-09-18 14:30',
   'this year',
   'next tuesday',
   '7 years from now',
@@ -103,11 +105,14 @@ const dateExamples = [
   'next week',
   'now',
   'now to next month',
+  'null',
+  '-null,',
   ' yyesterday ', // Typo
   'before', // Bad syntax
   'for', // Bad syntax
   '7', // Bad syntax
   'from now', // Bad syntax
+  '2025-12-25 12:32:', // Bad syntax
   '',
 ];
 
@@ -207,77 +212,89 @@ function testTokenizerNumber() {
   }
 }
 
-function testParserSingle(
+function testNumberParserSingle(
   str: string,
-  parser: BaseParser,
   outputFormatter?: (clauses: Clause[]) => string
 ): void {
   console.log('Input: ', str);
-  try {
-    const response = parser.parse();
-    // console.log('Tokens: ', parser.getTokens());
-    if (response.clauses && response.clauses.length > 0) {
-      if (outputFormatter) {
-        console.log('Output: ', outputFormatter(response.clauses));
-      } else {
-        console.log('Output: ', ...response.clauses);
-      }
-    }
-    if (response.errors && response.errors.length > 0) {
-      console.log('Errors: ', ...response.errors);
-    }
-  } catch (ex: Error | unknown) {
-    if (ex instanceof Error) console.error('Thrown Error: ', ex.message);
-    else {
-      console.error('Thrown Unknown error: ', ex);
+  const parser = new NumberParser(str);
+  const response = parser.parse();
+  // console.log('Tokens: ', parser.getTokens());
+  if (response.clauses && response.clauses.length > 0) {
+    if (outputFormatter) {
+      console.log('Output: ', outputFormatter(response.clauses));
+    } else {
+      console.log('Output: ', ...response.clauses);
     }
   }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
+  }
   console.log('');
 }
 
-function testSerializerRoundtrip(
+function testNumberParser() {
+  for (const example of numberExamples) {
+    testNumberParserSingle(example);
+  }
+}
+
+function testStringParserSingle(
   str: string,
-  parser: BaseParser,
-  serializerFunc: (clauses: Clause[]) => BaseSerializer
+  outputFormatter?: (clauses: Clause[]) => string
 ): void {
-  console.log('Input:  ' + str);
-  try {
-    const response = parser.parse();
-    // console.log('Clause: ', ...response.clauses, '\n');
-    if (response.clauses && response.clauses.length > 0) {
-      const result = serializerFunc(response.clauses || []).serialize();
-      console.log('Output: ' + result);
-    }
-    if (response.errors && response.errors.length > 0) {
-      console.log('Errors: ', ...response.errors);
-    }
-  } catch (ex: Error | unknown) {
-    if (ex instanceof Error) console.error('Thrown Error: ', ex.message);
-    else {
-      console.error('Thrown Unknown error: ', ex);
+  console.log('Input: ', str);
+  const parser = new StringParser(str);
+  const response = parser.parse();
+  // console.log('Tokens: ', parser.getTokens());
+  if (response.clauses && response.clauses.length > 0) {
+    if (outputFormatter) {
+      console.log('Output: ', outputFormatter(response.clauses));
+    } else {
+      console.log('Output: ', ...response.clauses);
     }
   }
-  console.log('');
-}
-
-function testNumberParser() {
-  for (const example of numberExamples) {
-    const parser = new NumberParser(example);
-    testParserSingle(example, parser);
+  if (response.quotes) {
+    console.log('Quotes: ', ...response.quotes);
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
   }
+  console.log('');
 }
 
 function testStringParser() {
   for (const example of stringExamples) {
     const parser = new StringParser(example);
-    testParserSingle(example, parser);
+    testStringParserSingle(example);
+  }
+}
+
+function testBooleanParserSingle(
+  str: string,
+  outputFormatter?: (clauses: Clause[]) => string
+): void {
+  console.log('Input: ', str);
+  const parser = new BooleanParser(str);
+  const response = parser.parse();
+  // console.log('Tokens: ', parser.getTokens());
+  if (response.clauses && response.clauses.length > 0) {
+    if (outputFormatter) {
+      console.log('Output: ', outputFormatter(response.clauses));
+    } else {
+      console.log('Output: ', ...response.clauses);
+    }
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
   }
+  console.log('');
 }
 
 function testBooleanParser() {
   for (const example of booleanExamples) {
     const parser = new BooleanParser(example);
-    testParserSingle(example, parser);
+    testBooleanParserSingle(example);
   }
 }
 
@@ -289,67 +306,112 @@ function jsonFormatter(clauses: Clause[]): string {
   return str;
 }
 
+function testDateParserSingle(
+  str: string,
+  outputFormatter?: (clauses: Clause[]) => string
+): void {
+  console.log('Input: ', str);
+  const parser = new DateParser(str);
+  const response = parser.parse();
+  // console.log('Tokens: ', parser.getTokens());
+  if (response.clauses && response.clauses.length > 0) {
+    if (outputFormatter) {
+      console.log('Output: ', outputFormatter(response.clauses));
+    } else {
+      console.log('Output: ', ...response.clauses);
+    }
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
+  }
+  console.log('');
+}
+
 function testDateParser() {
   for (const example of dateExamples) {
     const parser = new DateParser(example);
-    testParserSingle(example, parser);
-    //testParserSingle(example, parser, jsonFormatter);
+    testDateParserSingle(example);
+  }
+}
+
+function testNumberRoundtrip(str: string): void {
+  console.log('Input:  ' + str);
+  const response = new NumberParser(str).parse();
+  // console.log('Clause: ', ...response.clauses, '\n');
+  if (response.clauses && response.clauses.length > 0) {
+    const result = new NumberSerializer(response.clauses || []).serialize();
+    console.log('Output: ' + result);
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
   }
+  console.log('');
 }
 
 function testNumberSerializer(): void {
-  const examples = [
-    [{operator: '>', value: 10}],
-    [{startOperator: '>=', startValue: 20, endOperator: '<=', endValue: 30}],
-  ];
   for (const example of numberExamples) {
-    testSerializerRoundtrip(
-      example,
-      new NumberParser(example),
-      clauses => new NumberSerializer(clauses)
-    );
+    testNumberRoundtrip(example);
   }
 }
 
+function testStringRoundtrip(str: string): void {
+  console.log('Input:  ' + str);
+  const response = new StringParser(str).parse();
+  // console.log('Clause: ', ...response.clauses, '\n');
+  if (response.clauses && response.clauses.length > 0) {
+    const result = new StringSerializer(response.clauses || []).serialize();
+    console.log('Output: ' + result);
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
+  }
+  console.log('');
+}
+
 function testStringSerializer(): void {
   for (const example of stringExamples) {
-    testSerializerRoundtrip(
-      example,
-      new StringParser(example),
-      clauses => new StringSerializer(clauses)
-    );
+    testStringRoundtrip(example);
   }
 }
 
+function testBooleanRoundtrip(str: string): void {
+  console.log('Input:  ' + str);
+  const response = new BooleanParser(str).parse();
+  // console.log('Clause: ', ...response.clauses, '\n');
+  if (response.clauses && response.clauses.length > 0) {
+    const result = new BooleanSerializer(response.clauses || []).serialize();
+    console.log('Output: ' + result);
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
+  }
+  console.log('');
+}
+
 function testBooleanSerializer(): void {
   const examples = [[{operator: 'TRUE'}]];
   for (const example of booleanExamples) {
-    testSerializerRoundtrip(
-      example,
-      new BooleanParser(example),
-      clauses => new BooleanSerializer(clauses)
-    );
+    testBooleanRoundtrip(example);
   }
 }
 
+function testDateRoundtrip(str: string): void {
+  console.log('Input:  ' + str);
+  const response = new DateParser(str).parse();
+  // console.log('Clause: ', ...response.clauses, '\n');
+  if (response.clauses && response.clauses.length > 0) {
+    const result = new DateSerializer(response.clauses || []).serialize();
+    console.log('Output: ' + result);
+  }
+  if (response.errors && response.errors.length > 0) {
+    console.log('Errors: ', ...response.errors);
+  }
+  console.log('');
+}
+
 function testDateSerializer(): void {
-  const examples = [
-    [{prefix: 'BEFORE', values: [{type: 'day', value: 'yesterday'}]}],
-    [
-      {
-        start: [{type: 'day', value: 'today'}],
-        operator: 'TO',
-        end: [{type: 'day', value: 'tomorrow'}],
-      },
-    ],
-    [{type: 'day', value: 'today'}],
-  ];
   for (const example of dateExamples) {
-    testSerializerRoundtrip(
-      example,
-      new DateParser(example),
-      clauses => new DateSerializer(clauses)
-    );
+    testDateRoundtrip(example);
   }
 }
 
@@ -362,26 +424,33 @@ function printHeader(title: string): void {
 
 // Comment or uncomment the following function calls to disable/enable examples.
 function generateSamples() {
-  //printHeader('Tokenizer');
-  //testTokenizerString();
-  //testTokenizerNumber();
-  //testTokenizerMatchTypes();
-  //printHeader('Numbers');
-  //testNumberParser();
-  //printHeader('Strings');
-  //testStringParser();
-  //printHeader('Booleans');
-  //testBooleanParser();
-  //printHeader('Dates and Times');
-  //testDateParser();
-  printHeader('Number Serializer');
-  testNumberSerializer();
-  printHeader('String Serializer');
-  testStringSerializer();
-  printHeader('Boolean Serializer');
-  testBooleanSerializer();
-  printHeader('Date and Time Serializer');
-  testDateSerializer();
+  try {
+    //printHeader('Tokenizer');
+    //testTokenizerString();
+    //testTokenizerNumber();
+    //testTokenizerMatchTypes();
+    printHeader('Numbers');
+    testNumberParser();
+    printHeader('Strings');
+    testStringParser();
+    printHeader('Booleans');
+    testBooleanParser();
+    printHeader('Dates and Times');
+    testDateParser();
+    printHeader('Number Serializer');
+    testNumberSerializer();
+    printHeader('String Serializer');
+    testStringSerializer();
+    printHeader('Boolean Serializer');
+    testBooleanSerializer();
+    printHeader('Date and Time Serializer');
+    testDateSerializer();
+  } catch (ex: Error | unknown) {
+    if (ex instanceof Error) console.error('Thrown Error: ', ex.message);
+    else {
+      console.error('Thrown Unknown error: ', ex);
+    }
+  }
 }
 
 generateSamples();

package/src/number_parser.ts

@@ -5,11 +5,11 @@ import {
   NumberOperator,
   NumberRangeOperator,
   NumberClause,
-  Clause,
+  NumberParserResponse,
+  FilterError,
 } from './clause_types';
 import {BaseParser} from './base_parser';
 import {Token} from './token_types';
-import {FilterParserResponse, FilterError} from './filter_types';
 
 export class NumberParser extends BaseParser {
   constructor(input: string) {
@@ -46,7 +46,7 @@ export class NumberParser extends BaseParser {
     this.tokens = tokenizer.parse();
   }
 
-  public parse(): FilterParserResponse {
+  public parse(): NumberParserResponse {
     this.index = 0;
     this.tokenize();
     let clauses: NumberClause[] = [];
@@ -98,7 +98,9 @@ export class NumberParser extends BaseParser {
         outputs.push(clause);
       } else if (
         previous !== undefined &&
-        previous.operator === clause.operator
+        previous.operator === clause.operator &&
+        'values' in previous &&
+        'values' in clause
       ) {
         previous.values.push(...clause.values);
       } else {
@@ -204,59 +206,6 @@ export class NumberParser extends BaseParser {
     return clauses;
   }
 
-  private static getNegatedType(tokenType: NumberOperator): NumberOperator {
-    switch (tokenType) {
-      case '<=':
-        return '>';
-      case '>=':
-        return '<';
-      case '<':
-        return '>';
-      case '>':
-        return '<';
-      case '=':
-        return '!=';
-      case '!=':
-        return '=';
-    }
-  }
-
-  private static getNumberOperator(
-    tokenType: string
-  ): NumberOperator | undefined {
-    switch (tokenType) {
-      case '<=':
-        return '<=';
-      case '>=':
-        return '>=';
-      case '<':
-        return '<';
-      case '>':
-        return '>';
-      case '=':
-        return '=';
-      case '!=':
-        return '!=';
-    }
-    return undefined;
-  }
-
-  private static getNumberRnageOperator(
-    tokenType: string
-  ): NumberRangeOperator | undefined {
-    switch (tokenType) {
-      case '<=':
-        return '<=';
-      case '>=':
-        return '>=';
-      case '<':
-        return '<';
-      case '>':
-        return '>';
-    }
-    return undefined;
-  }
-
   private checkNumericExpression(
     tokenType: NumberOperator,
     clauses: NumberClause[]
@@ -281,7 +230,7 @@ export class NumberParser extends BaseParser {
     return false;
   }
 
-  private checkSimpleNumber(clauses: Clause[]): boolean {
+  private checkSimpleNumber(clauses: NumberClause[]): boolean {
     if (this.getNext().type === 'word') {
       const numericValue: number = NumberParser.parseNumber(
         this.getAt(this.index).value
@@ -296,11 +245,13 @@ export class NumberParser extends BaseParser {
     return false;
   }
 
-  private checkNull(clauses: Clause[]): boolean {
-    if (this.getNext().type === 'NULL' || this.getNext().type === 'NOTNULL') {
-      const tokenType = this.getNext().type === 'NULL' ? '=' : '!=';
-      const clause: NumberCondition = {operator: tokenType, values: [null]};
-      clauses.push(clause);
+  private checkNull(clauses: NumberClause[]): boolean {
+    const type = this.getNext().type;
+    if (type === 'NULL') {
+      clauses.push({operator: 'NULL'});
+      return true;
+    } else if (type === 'NOTNULL') {
+      clauses.push({operator: 'NOTNULL'});
       return true;
     }
     return false;

package/src/number_serializer.ts

@@ -3,13 +3,12 @@ import {
   NumberRange,
   NumberOperator,
   NumberRangeOperator,
-  Clause,
+  NumberClause,
 } from './clause_types';
-import {BaseSerializer} from './base_serializer';
 
-export class NumberSerializer extends BaseSerializer {
-  constructor(clauses: Clause[]) {
-    super(clauses);
+export class NumberSerializer {
+  constructor(private clauses: NumberClause[]) {
+    this.clauses = clauses;
   }
 
   public serialize(): string {
@@ -17,15 +16,13 @@ export class NumberSerializer extends BaseSerializer {
     return result.trim().replace(/,$/, '');
   }
 
-  // NumberOperator = '<=' | '>=' | '!=' | '=' | '>' | '<';
+  // NumberOperator = '<=' | '>=' | '!=' | '=' | '>' | '<'
   private static numberConditionToString(
     operator: NumberOperator,
-    value: number | null
+    value: number
   ): string {
-    if (value === null) {
-      return operator === '=' ? 'NULL' : '-NULL';
-    }
-    const operatorString = operator === '=' ? '' : operator; // Remove operator for eg "5, 7, 9"
+    // Remove operator for eg "5, 7, 9".
+    const operatorString = operator === '=' ? '' : operator;
     return operatorString + value;
   }
 
@@ -44,7 +41,7 @@ export class NumberSerializer extends BaseSerializer {
     }
   }
 
-  private static isNumberOperator(value: string): value is NumberOperator {
+  private static isNumberOperator(value: string): boolean {
     return ['<=', '>=', '!=', '=', '>', '<'].includes(value);
   }
 
@@ -69,16 +66,20 @@ export class NumberSerializer extends BaseSerializer {
     );
   }
 
-  private static clauseToString(clauses: Clause[]): string {
+  private static clauseToString(clauses: NumberClause[]): string {
     let result = '';
     for (const clause of clauses) {
-      if ('operator' in clause && clause.operator === 'range') {
+      if (!('operator' in clause)) {
+        throw new Error('Invalid number clause ' + JSON.stringify(clause));
+      }
+      if (clause.operator === 'range') {
         result += NumberSerializer.rangeToString(clause);
         result += ', ';
-      } else if (
-        'operator' in clause &&
-        NumberSerializer.isNumberOperator(clause.operator)
-      ) {
+      } else if (clause.operator === 'NULL') {
+        result += 'NULL, ';
+      } else if (clause.operator === 'NOTNULL') {
+        result += '-NULL, ';
+      } else if (NumberSerializer.isNumberOperator(clause.operator)) {
         const numberClause: NumberCondition = clause as NumberCondition;
         for (const value of numberClause.values) {
           result += NumberSerializer.numberConditionToString(
@@ -87,8 +88,6 @@ export class NumberSerializer extends BaseSerializer {
           );
           result += ', ';
         }
-      } else {
-        throw new Error('Invalid number clause ' + JSON.stringify(clause));
       }
     }
     return result;