@malloydata/malloy-filter 0.0.237-dev250224215546 → 0.0.237-dev250225015031

package/src/string_parser.ts

@@ -1,10 +1,17 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
 import {
   StringClause,
   StringCondition,
+  StringMatchOperator,
   StringConditionOperator,
-  QuoteType,
-  FilterError,
+  FilterLog,
   StringParserResponse,
 } from './clause_types';
 import {BaseParser} from './base_parser';
@@ -24,10 +31,10 @@ export class StringParser extends BaseParser {
   private tokenize(): void {
     const specialSubstrings: SpecialToken[] = [{type: ',', value: ','}];
     const specialWords: SpecialToken[] = [
-      {type: 'NULL', value: 'null', ignoreCase: true},
-      {type: 'EMPTY', value: 'empty', ignoreCase: true},
-      {type: 'NOTNULL', value: '-null', ignoreCase: true},
-      {type: 'NOTEMPTY', value: '-empty', ignoreCase: true},
+      {type: 'null', value: 'null', ignoreCase: true},
+      {type: 'empty', value: 'empty', ignoreCase: true},
+      {type: 'not_null', value: '-null', ignoreCase: true},
+      {type: 'not_empty', value: '-empty', ignoreCase: true},
     ];
     const params: TokenizerParams = {
       trimWordWhitespace: true,
@@ -44,89 +51,45 @@ export class StringParser extends BaseParser {
   public parse(): StringParserResponse {
     this.index = 0;
     this.tokenize();
-    const clauses: StringClause[] = [];
-    const errors: FilterError[] = [];
+    let clauses: StringClause[] = [];
+    const logs: FilterLog[] = [];
     while (this.index < this.tokens.length) {
       const token = this.getNext();
       if (token.type === ',') {
+        if (this.index > 0 && this.tokens[this.index - 1].type === ',') {
+          logs.push({
+            severity: 'warn',
+            message: 'Empty clause',
+            startIndex: token.startIndex,
+            endIndex: token.endIndex,
+          });
+        }
         this.index++;
       } else if (
-        token.type === 'NULL' ||
-        token.type === 'NOTNULL' ||
-        token.type === 'EMPTY' ||
-        token.type === 'NOTEMPTY'
+        token.type === 'null' ||
+        token.type === 'not_null' ||
+        token.type === 'empty' ||
+        token.type === 'not_empty'
       ) {
         clauses.push({operator: token.type});
         this.index++;
       } else if (this.checkSimpleWord(clauses)) {
         this.index++;
       } else {
-        errors.push({
-          message: 'Invalid expression',
+        logs.push({
+          severity: 'warn',
+          message: 'Empty clause',
           startIndex: token.startIndex,
           endIndex: token.endIndex,
         });
         this.index++;
       }
     }
-    const response: StringParserResponse = {
-      clauses: StringParser.groupClauses(clauses),
-      errors,
+    clauses = StringParser.groupClauses(clauses);
+    return {
+      clauses,
+      logs,
     };
-    // const quotes: QuoteType[] = StringParser.findQuotes(this.inputString);
-    // if (quotes.length > 0) {
-    //   response.quotes = quotes;
-    // }
-    return response;
-  }
-
-  private static findQuotes(str: string): QuoteType[] {
-    const quotes: Set<QuoteType> = new Set();
-    let i = 0;
-
-    while (i < str.length) {
-      // Check for triple quotes first to avoid false positives
-      if (str.slice(i, i + 3) === "'''") {
-        quotes.add('TRIPLESINGLE');
-        i += 3;
-      } else if (str.slice(i, i + 3) === '"""') {
-        quotes.add('TRIPLEDOUBLE');
-        i += 3;
-      } else if (str[i] === '\\') {
-        // Check for escaped quotes
-        if (i + 1 < str.length) {
-          switch (str[i + 1]) {
-            case "'":
-              quotes.add('ESCAPEDSINGLE');
-              break;
-            case '"':
-              quotes.add('ESCAPEDDOUBLE');
-              break;
-            case '`':
-              quotes.add('ESCAPEDBACKTICK');
-              break;
-          }
-          i += 2;
-        } else {
-          i++;
-        }
-      } else {
-        // Check for single quotes
-        switch (str[i]) {
-          case "'":
-            quotes.add('SINGLE');
-            break;
-          case '"':
-            quotes.add('DOUBLE');
-            break;
-          case '`':
-            quotes.add('BACKTICK');
-            break;
-        }
-        i++;
-      }
-    }
-    return Array.from(quotes);
   }
 
   private static groupClauses(clauses: StringClause[]): StringClause[] {
@@ -143,6 +106,12 @@ export class StringParser extends BaseParser {
         'values' in current
       ) {
         previous.values.push(...current.values);
+      } else if (
+        previous.operator === current.operator &&
+        'escaped_values' in previous &&
+        'escaped_values' in current
+      ) {
+        previous.escaped_values.push(...current.escaped_values);
       } else {
         previous = current;
         outputs.push(current);
@@ -178,23 +147,26 @@ export class StringParser extends BaseParser {
 
     let operator: StringConditionOperator = negatedMatch ? '!=' : '=';
     if (isUnderscore || isPercentMiddle || (isPercentBoth && word.length < 3)) {
-      operator = negatedMatch ? '!~' : '~';
+      // Special handling for string match
+      const matchOperator: StringMatchOperator = negatedMatch ? '!~' : '~';
+      if (word.length === 0) {
+        return false;
+      }
+      clauses.push({operator: matchOperator, escaped_values: [word]});
+      return true;
     } else if (isPercentBoth && word.length > 2) {
-      operator = negatedMatch ? 'notContains' : 'contains';
+      operator = negatedMatch ? 'not_contains' : 'contains';
       word = word.substring(1, word.length - 1);
-      word = StringParser.removeBackslashes(word);
     } else if (isPercentStart) {
-      operator = negatedMatch ? 'notEnds' : 'ends';
+      operator = negatedMatch ? 'not_ends' : 'ends';
       word = word.substring(1, word.length);
-      word = StringParser.removeBackslashes(word);
     } else if (isPercentEnd) {
-      operator = negatedMatch ? 'notStarts' : 'starts';
+      operator = negatedMatch ? 'not_starts' : 'starts';
       word = word.substring(0, word.length - 1);
-      word = StringParser.removeBackslashes(word);
     } else {
       // = or !=
-      word = StringParser.removeBackslashes(word);
     }
+    word = StringParser.removeBackslashes(word);
     if (word.length === 0) {
       return false;
     }

package/src/string_serializer.ts

@@ -1,7 +1,16 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {
   StringClause,
   StringCondition,
   StringConditionOperator,
+  StringMatch,
+  StringMatchOperator,
 } from './clause_types';
 
 export class StringSerializer {
@@ -16,11 +25,10 @@ export class StringSerializer {
 
   private static isNegated(operator: StringConditionOperator): boolean {
     return (
-      operator === '!~' ||
       operator === '!=' ||
-      operator === 'notStarts' ||
-      operator === 'notEnds' ||
-      operator === 'notContains'
+      operator === 'not_starts' ||
+      operator === 'not_ends' ||
+      operator === 'not_contains'
     );
   }
 
@@ -32,66 +40,84 @@ export class StringSerializer {
     return input.replace(/[_%]/g, match => `\\${match}`);
   }
 
-  // export type StringOperator =
+  // export type StringConditionOperator =
   //  | 'starts' | 'ends' | 'contains' | 'notStarts' | 'notEnds' | 'notContains'
-  //  | '~' | '=' | '!~' | '!=';
+  //  | '='| '!=';
   private static StringConditionWordToString(
     operator: StringConditionOperator,
     value: string
   ): string {
     const negated: boolean = StringSerializer.isNegated(operator);
-    if (value === 'NULL' || value === '-NULL') {
+    if (value === 'null' || value === '-null') {
       return (negated ? '-' : '') + '\\' + value;
     }
 
     value = StringSerializer.escapeSpecialCharacters(value);
-    if (operator === 'starts' || operator === 'notStarts') {
-      value = StringSerializer.escapeWildcardCharacters(value);
+    value = StringSerializer.escapeWildcardCharacters(value);
+    if (operator === 'starts' || operator === 'not_starts') {
       return (negated ? '-' : '') + value + '%';
-    } else if (operator === 'ends' || operator === 'notEnds') {
-      value = StringSerializer.escapeWildcardCharacters(value);
+    } else if (operator === 'ends' || operator === 'not_ends') {
       return (negated ? '-' : '') + '%' + value;
-    } else if (operator === 'contains' || operator === 'notContains') {
-      value = StringSerializer.escapeWildcardCharacters(value);
+    } else if (operator === 'contains' || operator === 'not_contains') {
       return (negated ? '-' : '') + '%' + value + '%';
-    } else if (operator === '=' || operator === '!=') {
-      value = StringSerializer.escapeWildcardCharacters(value);
-      return (negated ? '-' : '') + value;
     }
+    return (negated ? '-' : '') + value;
+  }
 
+  // export type StringMatchOperator = '~' | '!~';
+  private static StringMatchWordToString(
+    operator: StringMatchOperator,
+    value: string
+  ): string {
+    const negated: boolean = operator === '!~' ? true : false;
+    if (value === 'null' || value === '-null') {
+      return (negated ? '-' : '') + '\\' + value;
+    }
+    value = StringSerializer.escapeSpecialCharacters(value);
     return (negated ? '-' : '') + value;
   }
 
   private static StringClauseToString(
     operator:
       | StringConditionOperator
-      | 'EMPTY'
-      | 'NOTEMPTY'
-      | 'NULL'
-      | 'NOTNULL',
+      | StringMatchOperator
+      | 'empty'
+      | 'not_empty'
+      | 'null'
+      | 'not_null',
     clause: StringClause
   ): string {
-    if (operator === 'EMPTY') {
-      return 'EMPTY';
-    } else if (operator === 'NOTEMPTY') {
-      return '-EMPTY';
-    } else if (operator === 'NULL') {
-      return 'NULL';
-    } else if (operator === 'NOTNULL') {
-      return '-NULL';
-    }
-    if (!('values' in clause) || clause.values.length === 0) {
-      return '';
+    if (operator === 'empty') {
+      return 'empty';
+    } else if (operator === 'not_empty') {
+      return '-empty';
+    } else if (operator === 'null') {
+      return 'null';
+    } else if (operator === 'not_null') {
+      return '-null';
     }
     let result = '';
-    const condition: StringCondition = clause;
-    for (const value of condition.values) {
-      const word = StringSerializer.StringConditionWordToString(
-        condition.operator,
-        value
-      );
-      if (word) {
-        result += word + ', ';
+    if ('values' in clause && clause.values.length > 0) {
+      const condition: StringCondition = clause;
+      for (const value of condition.values) {
+        const word = StringSerializer.StringConditionWordToString(
+          condition.operator,
+          value
+        );
+        if (word) {
+          result += word + ', ';
+        }
+      }
+    } else if ('escaped_values' in clause && clause.escaped_values.length > 0) {
+      const condition: StringMatch = clause;
+      for (const value of condition.escaped_values) {
+        const word = StringSerializer.StringMatchWordToString(
+          condition.operator,
+          value
+        );
+        if (word) {
+          result += word + ', ';
+        }
       }
     }
     return result.trim().replace(/,$/, '');

package/src/token_types.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 export interface Token {
   type: string;
   value: string;

package/src/tokenizer.spec.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {Tokenizer, SpecialToken, TokenizerParams} from './tokenizer';
 import {Token} from './token_types';
 
@@ -68,16 +75,16 @@ describe('Tokenizer', () => {
     const input = 'hello NULL world,-Null,-\'NULL" ,NULL, NULL ,  ';
     const expectedTokens = [
       makeToken('word', 'hello', 0, 5),
-      makeToken('NULL', 'NULL', 6, 10),
+      makeToken('NULL', 'null', 6, 10),
       makeToken('word', 'world', 11, 16),
       makeToken(',', ',', 16, 17),
-      makeToken('NOTNULL', '-NULL', 17, 22),
+      makeToken('NOTNULL', '-null', 17, 22),
       makeToken(',', ',', 22, 23),
       makeToken('word', '-\'NULL"', 23, 30),
       makeToken(',', ',', 31, 32),
-      makeToken('NULL', 'NULL', 32, 36),
+      makeToken('NULL', 'null', 32, 36),
       makeToken(',', ',', 36, 37),
-      makeToken('NULL', 'NULL', 38, 42),
+      makeToken('NULL', 'null', 38, 42),
       makeToken(',', ',', 43, 44),
     ];
     const params = makeParams();
@@ -121,18 +128,18 @@ describe('Tokenizer', () => {
     ];
     expect(new Tokenizer(input, makeParams()).parse()).toEqual(expectedTokens);
   });
-  it('should match regexp and capitalize special matches', () => {
+  it('should match regexp and lowercase special matches', () => {
     const input =
       "hello tuesDAY,ttuesday, tuesdayy ,Tuesday , ttuesday, 'TUESday' ";
     const expectedTokens = [
       makeToken('word', 'hello', 0, 5),
-      makeToken('DAYOFWEEK', 'TUESDAY', 6, 13),
+      makeToken('DAYOFWEEK', 'tuesday', 6, 13),
       makeToken(',', ',', 13, 14),
       makeToken('word', 'ttuesday', 14, 22),
       makeToken(',', ',', 22, 23),
       makeToken('word', 'tuesdayy', 24, 32),
       makeToken(',', ',', 33, 34),
-      makeToken('DAYOFWEEK', 'TUESDAY', 34, 41),
+      makeToken('DAYOFWEEK', 'tuesday', 34, 41),
       makeToken(',', ',', 42, 43),
       makeToken('word', 'ttuesday', 44, 52),
       makeToken(',', ',', 52, 53),

package/src/tokenizer.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {Token} from './token_types';
 
 export interface SpecialToken {
@@ -138,7 +145,7 @@ export class Tokenizer {
           : subString === special.value;
         if (matches) {
           const value = special.ignoreCase
-            ? subString.toUpperCase()
+            ? subString.toLowerCase()
             : subString;
           return {
             type: special.type,
@@ -205,7 +212,7 @@ export class Tokenizer {
         regexp.lastIndex = 0; // Set the starting index for the search
         if (regexp.test(token.value)) {
           const value = special.ignoreCase
-            ? token.value.toUpperCase()
+            ? token.value.toLowerCase()
             : token.value;
           return {
             type: special.type,
@@ -220,7 +227,7 @@ export class Tokenizer {
           : token.value === special.value;
         if (matches) {
           const value = special.ignoreCase
-            ? token.value.toUpperCase()
+            ? token.value.toLowerCase()
             : token.value;
           return {
             type: special.type,