@malloydata/malloy-filter 0.0.237-dev250224203840 → 0.0.237-dev250225015031

package/src/date_types.ts

@@ -1,23 +1,31 @@
-import {FilterError} from './clause_types';
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {FilterLog} from './clause_types';
 
 export type DateTimeUnit =
-  | 'YEAR'
-  | 'QUARTER'
-  | 'MONTH'
-  | 'WEEK'
-  | 'DAY'
-  | 'HOUR'
-  | 'MINUTE'
-  | 'SECOND';
+  | 'year'
+  | 'quarter'
+  | 'month'
+  | 'week'
+  | 'day'
+  | 'hour'
+  | 'minute'
+  | 'second'
+  | 'instant';
 
 export type DateWeekday =
-  | 'MONDAY'
-  | 'TUESDAY'
-  | 'WEDNESDAY'
-  | 'THURSDAY'
-  | 'FRIDAY'
-  | 'SATURDAY'
-  | 'SUNDAY';
+  | 'monday'
+  | 'tuesday'
+  | 'wednesday'
+  | 'thursday'
+  | 'friday'
+  | 'saturday'
+  | 'sunday';
 
 // 7 weeks, 32 hours
 export interface Duration {
@@ -25,45 +33,45 @@ export interface Duration {
   unit: DateTimeUnit;
 }
 
-export type DateMomentName = 'NOW' | 'TODAY' | 'YESTERDAY' | 'TOMORROW';
+export type DateMomentName = 'now' | 'today' | 'yesterday' | 'tomorrow';
 
 // now, today, yesterday, tomorrow
 export interface NamedMoment {
-  type: 'NAMED';
+  type: 'named';
   name: DateMomentName;
 }
 
-export type DateMomentIntervalOperator = 'LAST' | 'THIS' | 'NEXT';
+export type DateMomentIntervalOperator = 'last' | 'this' | 'next';
 
 // LAST|UNITOFTIME, LAST|DAYOFWEEK
 // THIS|UNITOFTIME
 // NEXT|UNITOFTIME, NEXT|DAYOFWEEK
 // last month, next tuesday, this month
 export interface IntervalMoment {
-  type: 'INTERVAL';
+  type: 'interval';
   kind: DateMomentIntervalOperator;
   unit: DateTimeUnit | DateWeekday;
 }
 
-export type DateMomentOffsetFromNowDirection = 'AGO' | 'FROMNOW';
+export type DateMomentOffsetFromNowDirection = 'ago' | 'from_now';
 
 // NUMBER|UNITOFTIME|AGO
 // NUMBER|UNITOFTIME|FROM|NOW
 // 3 hours ago, 6 weeks from now
 export interface OffsetMoment {
-  type: 'OFFSET_FROM_NOW';
+  type: 'offset_from_now';
   direction: DateMomentOffsetFromNowDirection;
   unit: DateTimeUnit;
   amount: number;
 }
 
-export type DateMomentSpanFromNowDirection = 'LAST' | 'NEXT';
+export type DateMomentSpanFromNowDirection = 'last' | 'next';
 
 // LAST|NUMBER|UNITOFTIME
 // NEXT|NUMBER|UNITOFTIME
 // last 3 hours, next 2025 seconds, last 6 weeks
 export interface SpanMoment {
-  type: 'SPAN_FROM_NOW';
+  type: 'span_from_now';
   direction: DateMomentSpanFromNowDirection;
   unit: DateTimeUnit;
   amount: number;
@@ -71,7 +79,7 @@ export interface SpanMoment {
 
 // 2005, 2005-01, 2005-01-01, 2005-01-01 00:00, 2005-01-01 00:00:01
 export interface AbsoluteMoment {
-  type: 'ABSOLUTE';
+  type: 'absolute';
   date: string;
   unit: DateTimeUnit;
 }
@@ -99,47 +107,47 @@ export type DateMoment =
 
 // after 2005-01, after 3 hours ago
 export interface DateAfterClause {
-  operator: 'AFTER';
+  operator: 'after';
   moment: DateMoment;
 }
 
 // before 2005-01, before 3 hours ago
 export interface DateBeforeClause {
-  operator: 'BEFORE';
+  operator: 'before';
   moment: DateMoment;
 }
 
 // next week, last 3 weeks, 3 days ago
 export interface DateOnClause {
-  operator: 'ON';
+  operator: 'on';
   moment: DateMoment;
 }
 
 // 2015 to next month
 export interface DateBetweenClause {
-  operator: 'TO_RANGE';
+  operator: 'to_range';
   from: DateMoment;
   to: DateMoment;
 }
 
 // 2025-01-01 12:00:00 for 3 days
 export interface DateForClause {
-  operator: 'FOR_RANGE';
+  operator: 'for_range';
   from: DateMoment;
   duration: Duration;
 }
 
 export interface DateNullClause {
-  operator: 'NULL';
+  operator: 'null';
 }
 
 export interface DateNotNullClause {
-  operator: 'NOTNULL';
+  operator: 'not_null';
 }
 
 // 3 days
 export interface DateDurationClause {
-  operator: 'DURATION';
+  operator: 'duration';
   duration: Duration;
 }
 
@@ -155,5 +163,5 @@ export type DateClause =
 
 export interface DateParserResponse {
   clauses: DateClause[];
-  errors: FilterError[];
+  logs: FilterLog[];
 }

package/src/generate_samples.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import * as fs from 'fs';
 import {BooleanParser} from './boolean_parser';
 import {StringParser} from './string_parser';
@@ -12,6 +19,7 @@ const numberExamples = [
   '5',
   '!=5',
   '1, 3, 5, null',
+  '1, 3, , 5,',
   '<1, >=100 ',
   '>=1',
   ' <= 10 ',
@@ -45,6 +53,8 @@ const stringExamples = [
   'CAT,-NULL',
   'CAT,-"NULL"',
   'CAT,NULL',
+  'CAT,,',
+  'CAT, , DOG',
   'EMPTY',
   '-EMPTY',
   'CAT,-EMPTY',
@@ -69,7 +79,8 @@ const booleanExamples = [
   '=false',
   'null',
   '-NULL',
-  ' True , faLSE,=false,NULl,-null',
+  'null,',
+  ' True , , faLSE,=false,NULl,-null',
   "-'null'",
   '10',
   'nnull',
@@ -83,14 +94,21 @@ const dateExamples = [
   '3 months ago for 2 days',
   '2025 weeks ago',
   'before 3 days ago',
-  'before 2025-08-30 08:30:20',
-  'after 2025-10-05',
+  'Before 2025-08-30 08:30:20',
+  'AFTER 2025-10-05',
   '2025-08-30 12:00 to 2025-09-18 14:30',
-  'this year',
-  'next tuesday',
-  '7 years from now',
+  'this YEAR',
+  'Next Tuesday',
+  '7 years from Now',
   '2025-01-01 12:00:00 for 3 days',
+  '2020-08-12 03:12:56.57',
+  '2020-08-12T03:12:56[PST]',
+  '2020-08-12 03:12:56',
+  '2020-08-12 03:22',
+  '2020-08-12 03',
   '2020-08-12',
+  '2020-Q3',
+  '2020-08-07-wK',
   '2020-08',
   'today',
   'yesterday',
@@ -105,9 +123,10 @@ const dateExamples = [
   ' yyesterday ', // Typo
   'before', // Bad syntax
   'for', // Bad syntax
-  '7', // Bad syntax
+  '12', // Bad syntax
   'from now', // Bad syntax
   '2025-12-25 12:32:', // Bad syntax
+  '12:22',
   'after 2025 seconds', // Bad syntax
   '',
 ];
@@ -159,8 +178,8 @@ class GenerateSamples {
     if (response.clauses && response.clauses.length > 0) {
       GenerateSamples.writeJson(fp, 'Output: ', ...response.clauses);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -173,8 +192,8 @@ class GenerateSamples {
     if (response.clauses && response.clauses.length > 0) {
       GenerateSamples.writeJson(fp, 'Output: ', ...response.clauses);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -187,8 +206,8 @@ class GenerateSamples {
     if (response.clauses && response.clauses.length > 0) {
       GenerateSamples.writeJson(fp, 'Output: ', ...response.clauses);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -201,11 +220,8 @@ class GenerateSamples {
     if (response.clauses && response.clauses.length > 0) {
       GenerateSamples.writeJson(fp, 'Output: ', ...response.clauses);
     }
-    if (response.quotes) {
-      GenerateSamples.writeJson(fp, 'Quotes: ', ...response.quotes);
-    }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -218,8 +234,8 @@ class GenerateSamples {
       const result = new BooleanSerializer(response.clauses || []).serialize();
       GenerateSamples.writeRaw(fp, 'Output: ' + result);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -232,8 +248,8 @@ class GenerateSamples {
       const result = new DateSerializer(response.clauses || []).serialize();
       GenerateSamples.writeRaw(fp, 'Output: ' + result);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -246,8 +262,8 @@ class GenerateSamples {
       const result = new NumberSerializer(response.clauses || []).serialize();
       GenerateSamples.writeRaw(fp, 'Output: ' + result);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }
@@ -260,8 +276,8 @@ class GenerateSamples {
       const result = new StringSerializer(response.clauses || []).serialize();
       GenerateSamples.writeRaw(fp, 'Output: ' + result);
     }
-    if (response.errors && response.errors.length > 0) {
-      GenerateSamples.writeJson(fp, 'Errors: ', ...response.errors);
+    if (response.logs && response.logs.length > 0) {
+      GenerateSamples.writeJson(fp, 'Logs:   ', ...response.logs);
     }
     GenerateSamples.writeRaw(fp, '');
   }

package/src/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export {BooleanParser} from './boolean_parser';
+export {DateParser} from './date_parser';
+export {NumberParser} from './number_parser';
+export {StringParser} from './string_parser';
+export {BooleanSerializer} from './boolean_serializer';
+export {DateSerializer} from './date_serializer';
+export {NumberSerializer} from './number_serializer';
+export {StringSerializer} from './string_serializer';
+export * from './clause_types';
+export * from './date_types';

package/src/number_parser.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {SpecialToken, Tokenizer, TokenizerParams} from './tokenizer';
 import {
   NumberCondition,
@@ -6,7 +13,7 @@ import {
   NumberRangeOperator,
   NumberClause,
   NumberParserResponse,
-  FilterError,
+  FilterLog,
 } from './clause_types';
 import {BaseParser} from './base_parser';
 import {Token} from './token_types';
@@ -31,8 +38,8 @@ export class NumberParser extends BaseParser {
       {type: '<', value: '<'},
     ];
     const specialWords = [
-      {type: 'NOTNULL', value: '-null', ignoreCase: true},
-      {type: 'NULL', value: 'null', ignoreCase: true},
+      {type: 'not_null', value: '-null', ignoreCase: true},
+      {type: 'null', value: 'null', ignoreCase: true},
     ];
     const params: TokenizerParams = {
       trimWordWhitespace: true,
@@ -50,16 +57,24 @@ export class NumberParser extends BaseParser {
     this.index = 0;
     this.tokenize();
     let clauses: NumberClause[] = [];
-    const errors: FilterError[] = [];
+    const logs: FilterLog[] = [];
     while (this.index < this.tokens.length) {
       const token = this.getNext();
       if (token.type === ',') {
+        if (this.index > 0 && this.tokens[this.index - 1].type === ',') {
+          logs.push({
+            severity: 'warn',
+            message: 'Empty clause',
+            startIndex: token.startIndex,
+            endIndex: token.endIndex,
+          });
+        }
         this.index++;
       } else if (this.isRangeStart(this.index)) {
-        clauses = this.checkRange(token, false, clauses, errors);
+        clauses = this.checkRange(token, false, clauses, logs);
       } else if (token.type === '!=' && this.isRangeStart(this.index + 1)) {
         this.index++;
-        clauses = this.checkRange(token, true, clauses, errors);
+        clauses = this.checkRange(token, true, clauses, logs);
       } else if (this.checkNull(clauses)) {
         this.index++;
       } else if (
@@ -74,15 +89,16 @@ export class NumberParser extends BaseParser {
       } else if (this.checkSimpleNumber(clauses)) {
         this.index++;
       } else {
-        errors.push({
-          message: 'Invalid expression',
+        logs.push({
+          severity: 'error',
+          message: 'Invalid expression: ' + token.value,
           startIndex: token.startIndex,
           endIndex: token.endIndex,
         });
         this.index++;
       }
     }
-    return {clauses: NumberParser.groupClauses(clauses), errors};
+    return {clauses: NumberParser.groupClauses(clauses), logs};
   }
 
   private static groupClauses(clauses: NumberClause[]): NumberClause[] {
@@ -119,26 +135,27 @@ export class NumberParser extends BaseParser {
     token: Token,
     negated: boolean,
     clauses: NumberClause[],
-    errors: FilterError[]
+    logs: FilterLog[]
   ): NumberClause[] {
     if (this.matchTokens(['[', 'word', ',', 'word', ']'])) {
       return negated
-        ? this.consumeRange('<', '>', clauses, errors)
-        : this.consumeRange('>=', '<=', clauses, errors);
+        ? this.consumeRange('<', '>', clauses, logs)
+        : this.consumeRange('>=', '<=', clauses, logs);
     } else if (this.matchTokens(['[', 'word', ',', 'word', ')'])) {
       return negated
-        ? this.consumeRange('<', '>=', clauses, errors)
-        : this.consumeRange('>=', '<', clauses, errors);
+        ? this.consumeRange('<', '>=', clauses, logs)
+        : this.consumeRange('>=', '<', clauses, logs);
     } else if (this.matchTokens(['(', 'word', ',', 'word', ']'])) {
       return negated
-        ? this.consumeRange('<=', '>', clauses, errors)
-        : this.consumeRange('>', '<=', clauses, errors);
+        ? this.consumeRange('<=', '>', clauses, logs)
+        : this.consumeRange('>', '<=', clauses, logs);
     } else if (this.matchTokens(['(', 'word', ',', 'word', ')'])) {
       return negated
-        ? this.consumeRange('<=', '>=', clauses, errors)
-        : this.consumeRange('>', '<', clauses, errors);
+        ? this.consumeRange('<=', '>=', clauses, logs)
+        : this.consumeRange('>', '<', clauses, logs);
     } else {
-      errors.push({
+      logs.push({
+        severity: 'error',
         message: 'Invalid range expression',
         startIndex: token.startIndex,
         endIndex: token.endIndex,
@@ -174,20 +191,22 @@ export class NumberParser extends BaseParser {
     startOperator: NumberRangeOperator,
     endOperator: NumberRangeOperator,
     clauses: NumberClause[],
-    errors: FilterError[]
+    logs: FilterLog[]
   ): NumberClause[] {
     const startToken = this.getAt(this.index + 1);
     const endToken = this.getAt(this.index + 3);
     const startValue: number = NumberParser.parseNumber(startToken.value);
     const endValue: number = NumberParser.parseNumber(endToken.value);
     if (!NumberParser.isValidNumber(startValue)) {
-      errors.push({
+      logs.push({
+        severity: 'error',
         message: 'Invalid number',
         startIndex: startToken.startIndex,
         endIndex: startToken.endIndex,
       });
     } else if (!NumberParser.isValidNumber(endValue)) {
-      errors.push({
+      logs.push({
+        severity: 'error',
         message: 'Invalid number',
         startIndex: endToken.startIndex,
         endIndex: endToken.endIndex,
@@ -247,11 +266,11 @@ export class NumberParser extends BaseParser {
 
   private checkNull(clauses: NumberClause[]): boolean {
     const type = this.getNext().type;
-    if (type === 'NULL') {
-      clauses.push({operator: 'NULL'});
+    if (type === 'null') {
+      clauses.push({operator: 'null'});
       return true;
-    } else if (type === 'NOTNULL') {
-      clauses.push({operator: 'NOTNULL'});
+    } else if (type === 'not_null') {
+      clauses.push({operator: 'not_null'});
       return true;
     }
     return false;

package/src/number_serializer.ts

@@ -1,3 +1,10 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 import {
   NumberCondition,
   NumberRange,
@@ -75,10 +82,10 @@ export class NumberSerializer {
       if (clause.operator === 'range') {
         result += NumberSerializer.rangeToString(clause);
         result += ', ';
-      } else if (clause.operator === 'NULL') {
-        result += 'NULL, ';
-      } else if (clause.operator === 'NOTNULL') {
-        result += '-NULL, ';
+      } else if (clause.operator === 'null') {
+        result += 'null, ';
+      } else if (clause.operator === 'not_null') {
+        result += '-null, ';
       } else if (NumberSerializer.isNumberOperator(clause.operator)) {
         const numberClause: NumberCondition = clause as NumberCondition;
         for (const value of numberClause.values) {