@mastra/libsql 0.0.4-alpha.4 → 0.0.5-alpha.0

package/src/vector/sql-builder.ts

@@ -1,58 +1,93 @@
 import type { InValue } from '@libsql/client';
+import { parseFieldKey } from '@mastra/core/utils';
 import type {
-  VectorFilter,
   BasicOperator,
   NumericOperator,
   ArrayOperator,
   ElementOperator,
   LogicalOperator,
-  RegexOperator,
+  VectorFilter,
 } from '@mastra/core/vector/filter';
 
-export type OperatorType =
+type OperatorType =
   | BasicOperator
   | NumericOperator
   | ArrayOperator
   | ElementOperator
   | LogicalOperator
   | '$contains'
-  | Exclude<RegexOperator, '$options'>;
+  | '$size';
 
 type FilterOperator = {
   sql: string;
   needsValue: boolean;
-  transformValue?: (value: any) => any;
+  transformValue?: () => any;
 };
 
 type OperatorFn = (key: string, value?: any) => FilterOperator;
 
 // Helper functions to create operators
 const createBasicOperator = (symbol: string) => {
-  return (key: string): FilterOperator => ({
-    sql: `CASE 
-      WHEN ? IS NULL THEN json_extract(metadata, '$."${handleKey(key)}"') IS ${symbol === '=' ? '' : 'NOT'} NULL
-      ELSE json_extract(metadata, '$."${handleKey(key)}"') ${symbol} ?
-    END`,
-    needsValue: true,
-    transformValue: (value: any) => {
-      // Return the values directly, not in an object
-      return [value, value];
-    },
-  });
+  return (key: string, value: any): FilterOperator => {
+    const jsonPathKey = parseJsonPathKey(key);
+    return {
+      sql: `CASE 
+        WHEN ? IS NULL THEN json_extract(metadata, '$."${jsonPathKey}"') IS ${symbol === '=' ? '' : 'NOT'} NULL
+        ELSE json_extract(metadata, '$."${jsonPathKey}"') ${symbol} ?
+      END`,
+      needsValue: true,
+      transformValue: () => {
+        // Return the values directly, not in an object
+        return [value, value];
+      },
+    };
+  };
 };
 const createNumericOperator = (symbol: string) => {
-  return (key: string): FilterOperator => ({
-    sql: `CAST(json_extract(metadata, '$."${handleKey(key)}"') AS NUMERIC) ${symbol} ?`,
-    needsValue: true,
-  });
+  return (key: string): FilterOperator => {
+    const jsonPathKey = parseJsonPathKey(key);
+    return {
+      sql: `CAST(json_extract(metadata, '$."${jsonPathKey}"') AS NUMERIC) ${symbol} ?`,
+      needsValue: true,
+    };
+  };
 };
 
 const validateJsonArray = (key: string) =>
-  `json_valid(json_extract(metadata, '$."${handleKey(key)}"'))
-   AND json_type(json_extract(metadata, '$."${handleKey(key)}"')) = 'array'`;
+  `json_valid(json_extract(metadata, '$."${key}"'))
+   AND json_type(json_extract(metadata, '$."${key}"')) = 'array'`;
+
+const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
+
+function buildElemMatchConditions(value: any) {
+  const conditions = Object.entries(value).map(([field, fieldValue]) => {
+    if (field.startsWith('$')) {
+      // Direct operators on array elements ($in, $gt, etc)
+      const { sql, values } = buildCondition('elem.value', { [field]: fieldValue }, '');
+      // Replace the metadata path with elem.value
+      const elemSql = sql.replace(pattern, 'elem.value');
+      return { sql: elemSql, values };
+    } else if (typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
+      // Nested field with operators (count: { $gt: 20 })
+      const { sql, values } = buildCondition(field, fieldValue, '');
+      // Replace the field path with elem.value path
+      const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
+      return { sql: elemSql, values };
+    } else {
+      const parsedFieldKey = parseFieldKey(field);
+      // Simple field equality (warehouse: 'A')
+      return {
+        sql: `json_extract(elem.value, '$."${parsedFieldKey}"') = ?`,
+        values: [fieldValue],
+      };
+    }
+  });
+
+  return conditions;
+}
 
 // Define all filter operators
-export const FILTER_OPERATORS: Record<string, OperatorFn> = {
+const FILTER_OPERATORS: Record<OperatorType, OperatorFn> = {
   $eq: createBasicOperator('='),
   $ne: createBasicOperator('!='),
   $gt: createNumericOperator('>'),
@@ -61,101 +96,121 @@ export const FILTER_OPERATORS: Record<string, OperatorFn> = {
   $lte: createNumericOperator('<='),
 
   // Array Operators
-  $in: (key: string, value: any) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') IN (${value.map(() => '?').join(',')})`,
-    needsValue: true,
-  }),
-
-  $nin: (key: string, value: any) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') NOT IN (${value.map(() => '?').join(',')})`,
-    needsValue: true,
-  }),
-  $all: (key: string) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') = ?`,
-    needsValue: true,
-    transformValue: (value: any) => {
-      const arrayValue = Array.isArray(value) ? value : [value];
-      if (arrayValue.length === 0) {
-        return {
-          sql: '1 = 0',
-          values: [],
-        };
-      }
+  $in: (key: string, value: any) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    const arr = Array.isArray(value) ? value : [value];
+    if (arr.length === 0) {
+      return { sql: '1 = 0', needsValue: true, transformValue: () => [] };
+    }
+    const paramPlaceholders = arr.map(() => '?').join(',');
+    return {
+      sql: `(
+      CASE
+        WHEN ${validateJsonArray(jsonPathKey)} THEN
+          EXISTS (
+            SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
+            WHERE elem.value IN (SELECT value FROM json_each(?))
+          )
+        ELSE json_extract(metadata, '$."${jsonPathKey}"') IN (${paramPlaceholders})
+      END
+    )`,
+      needsValue: true,
+      transformValue: () => [JSON.stringify(arr), ...arr],
+    };
+  },
 
-      return {
-        sql: `(
-          CASE
-            WHEN ${validateJsonArray(key)} THEN
-                NOT EXISTS (
-                    SELECT value 
-                    FROM json_each(?) 
-                    WHERE value NOT IN (
-                    SELECT value 
-                    FROM json_each(json_extract(metadata, '$."${handleKey(key)}"'))
-                )
+  $nin: (key: string, value: any) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    const arr = Array.isArray(value) ? value : [value];
+    if (arr.length === 0) {
+      return { sql: '1 = 1', needsValue: true, transformValue: () => [] };
+    }
+    const paramPlaceholders = arr.map(() => '?').join(',');
+    return {
+      sql: `(
+      CASE
+        WHEN ${validateJsonArray(jsonPathKey)} THEN
+          NOT EXISTS (
+            SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
+            WHERE elem.value IN (SELECT value FROM json_each(?))
+          )
+        ELSE json_extract(metadata, '$."${jsonPathKey}"') NOT IN (${paramPlaceholders})
+      END
+    )`,
+      needsValue: true,
+      transformValue: () => [JSON.stringify(arr), ...arr],
+    };
+  },
+  $all: (key: string, value: any) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    let sql: string;
+    const arrayValue = Array.isArray(value) ? value : [value];
+
+    if (arrayValue.length === 0) {
+      // If the array is empty, always return false (no matches)
+      sql = '1 = 0';
+    } else {
+      sql = `(
+      CASE
+        WHEN ${validateJsonArray(jsonPathKey)} THEN
+          NOT EXISTS (
+            SELECT value
+            FROM json_each(?)
+            WHERE value NOT IN (
+              SELECT value
+              FROM json_each(json_extract(metadata, '$."${jsonPathKey}"'))
             )
-            ELSE FALSE
-          END
-        )`,
-        values: [JSON.stringify(arrayValue)],
-      };
-    },
-  }),
-  $elemMatch: (key: string) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') = ?`,
-    needsValue: true,
-    transformValue: (value: any) => {
-      if (typeof value !== 'object' || Array.isArray(value)) {
-        throw new Error('$elemMatch requires an object with conditions');
-      }
-
-      // For nested object conditions
-      const conditions = Object.entries(value).map(([field, fieldValue]) => {
-        if (field.startsWith('$')) {
-          // Direct operators on array elements ($in, $gt, etc)
-          const { sql, values } = buildCondition('elem.value', { [field]: fieldValue }, '');
-          // Replace the metadata path with elem.value
-          const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
-          const elemSql = sql.replace(pattern, 'elem.value');
-          return { sql: elemSql, values };
-        } else if (typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
-          // Nested field with operators (count: { $gt: 20 })
-          const { sql, values } = buildCondition(field, fieldValue, '');
-          // Replace the field path with elem.value path
-          const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
-          const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
-          return { sql: elemSql, values };
-        } else {
-          // Simple field equality (warehouse: 'A')
-          return {
-            sql: `json_extract(elem.value, '$."${field}"') = ?`,
-            values: [fieldValue],
-          };
+          )
+        ELSE FALSE
+      END
+    )`;
+    }
+
+    return {
+      sql,
+      needsValue: true,
+      transformValue: () => {
+        if (arrayValue.length === 0) {
+          return [];
         }
-      });
+        return [JSON.stringify(arrayValue)];
+      },
+    };
+  },
+  $elemMatch: (key: string, value: any) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    if (typeof value !== 'object' || Array.isArray(value)) {
+      throw new Error('$elemMatch requires an object with conditions');
+    }
 
-      return {
-        sql: `(
-          CASE
-            WHEN ${validateJsonArray(key)} THEN
-              EXISTS (
-                SELECT 1 
-                FROM json_each(json_extract(metadata, '$."${handleKey(key)}"')) as elem
-                WHERE ${conditions.map(c => c.sql).join(' AND ')}
-              )
-            ELSE FALSE
-          END
-        )`,
-        values: conditions.flatMap(c => c.values),
-      };
-    },
-  }),
+    // For nested object conditions
+    const conditions = buildElemMatchConditions(value);
+
+    return {
+      sql: `(
+        CASE
+          WHEN ${validateJsonArray(jsonPathKey)} THEN
+            EXISTS (
+              SELECT 1
+              FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
+              WHERE ${conditions.map(c => c.sql).join(' AND ')}
+            )
+          ELSE FALSE
+        END
+      )`,
+      needsValue: true,
+      transformValue: () => conditions.flatMap(c => c.values),
+    };
+  },
 
   // Element Operators
-  $exists: (key: string) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') IS NOT NULL`,
-    needsValue: false,
-  }),
+  $exists: (key: string) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    return {
+      sql: `json_extract(metadata, '$."${jsonPathKey}"') IS NOT NULL`,
+      needsValue: false,
+    };
+  },
 
   // Logical Operators
   $and: (key: string) => ({
@@ -171,27 +226,30 @@ export const FILTER_OPERATORS: Record<string, OperatorFn> = {
     sql: `NOT (${key})`,
     needsValue: false,
   }),
-  $size: (key: string, paramIndex: number) => ({
-    sql: `(
+  $size: (key: string, paramIndex: number) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    return {
+      sql: `(
     CASE
-      WHEN json_type(json_extract(metadata, '$."${handleKey(key)}"')) = 'array' THEN 
-        json_array_length(json_extract(metadata, '$."${handleKey(key)}"')) = $${paramIndex}
+      WHEN json_type(json_extract(metadata, '$."${jsonPathKey}"')) = 'array' THEN 
+        json_array_length(json_extract(metadata, '$."${jsonPathKey}"')) = $${paramIndex}
       ELSE FALSE
     END
   )`,
-    needsValue: true,
-  }),
+      needsValue: true,
+    };
+  },
   //   /**
   //    * Regex Operators
   //    * Supports case insensitive and multiline
   //    */
   //   $regex: (key: string): FilterOperator => ({
-  //     sql: `json_extract(metadata, '$."${handleKey(key)}"') = ?`,
+  //     sql: `json_extract(metadata, '$."${toJsonPathKey(key)}"') = ?`,
   //     needsValue: true,
   //     transformValue: (value: any) => {
   //       const pattern = typeof value === 'object' ? value.$regex : value;
   //       const options = typeof value === 'object' ? value.$options || '' : '';
-  //       let sql = `json_extract(metadata, '$."${handleKey(key)}"')`;
+  //       let sql = `json_extract(metadata, '$."${toJsonPathKey(key)}"')`;
 
   //       // Handle multiline
   //       //   if (options.includes('m')) {
@@ -247,63 +305,73 @@ export const FILTER_OPERATORS: Record<string, OperatorFn> = {
   //       };
   //     },
   //   }),
-  $contains: (key: string) => ({
-    sql: `json_extract(metadata, '$."${handleKey(key)}"') = ?`,
-    needsValue: true,
-    transformValue: (value: any) => {
-      // Array containment
-      if (Array.isArray(value)) {
-        return {
-          sql: `(
-            SELECT ${validateJsonArray(key)}
-            AND EXISTS (
-              SELECT 1 
-              FROM json_each(json_extract(metadata, '$."${handleKey(key)}"')) as m
-              WHERE m.value IN (SELECT value FROM json_each(?))
-            )
-          )`,
-          values: [JSON.stringify(value)],
-        };
-      }
-
-      // Nested object traversal
-      if (value && typeof value === 'object') {
-        const paths: string[] = [];
-        const values: any[] = [];
-
-        function traverse(obj: any, path: string[] = []) {
-          for (const [k, v] of Object.entries(obj)) {
-            const currentPath = [...path, k];
-            if (v && typeof v === 'object' && !Array.isArray(v)) {
-              traverse(v, currentPath);
-            } else {
-              paths.push(currentPath.join('.'));
-              values.push(v);
-            }
-          }
+  $contains: (key: string, value: any) => {
+    const jsonPathKey = parseJsonPathKey(key);
+    let sql;
+    if (Array.isArray(value)) {
+      sql = `(
+        SELECT ${validateJsonArray(jsonPathKey)}
+        AND EXISTS (
+          SELECT 1
+          FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as m
+          WHERE m.value IN (SELECT value FROM json_each(?))
+        )
+      )`;
+    } else if (typeof value === 'string') {
+      sql = `lower(json_extract(metadata, '$."${jsonPathKey}"')) LIKE '%' || lower(?) || '%' ESCAPE '\\'`;
+    } else {
+      sql = `json_extract(metadata, '$."${jsonPathKey}"') = ?`;
+    }
+    return {
+      sql,
+      needsValue: true,
+      transformValue: () => {
+        if (Array.isArray(value)) {
+          return [JSON.stringify(value)];
         }
-
-        traverse(value);
-        return {
-          sql: `(${paths.map(path => `json_extract(metadata, '$."${handleKey(key)}"."${path}"') = ?`).join(' AND ')})`,
-          values,
-        };
-      }
-
-      return value;
-    },
-  }),
+        if (typeof value === 'object' && value !== null) {
+          return [JSON.stringify(value)];
+        }
+        if (typeof value === 'string') {
+          return [escapeLikePattern(value)];
+        }
+        return [value];
+      },
+    };
+  },
+  /**
+   * $objectContains: True JSON containment for advanced use (deep sub-object match).
+   * Usage: { field: { $objectContains: { ...subobject } } }
+   */
+  // $objectContains: (key: string) => ({
+  //   sql: '', // Will be overridden by transformValue
+  //   needsValue: true,
+  //   transformValue: (value: any) => ({
+  //     sql: `json_type(json_extract(metadata, '$."${toJsonPathKey(key)}"')) = 'object'
+  //         AND json_patch(json_extract(metadata, '$."${toJsonPathKey(key)}"'), ?) = json_extract(metadata, '$."${toJsonPathKey(key)}"')`,
+  //     values: [JSON.stringify(value)],
+  //   }),
+  // }),
 };
 
-export interface FilterResult {
+interface FilterResult {
   sql: string;
   values: InValue[];
 }
 
-export const handleKey = (key: string) => {
-  return key.replace(/\./g, '"."');
+function isFilterResult(obj: any): obj is FilterResult {
+  return obj && typeof obj === 'object' && typeof obj.sql === 'string' && Array.isArray(obj.values);
+}
+
+const parseJsonPathKey = (key: string) => {
+  const parsedKey = parseFieldKey(key);
+  return parsedKey.replace(/\./g, '"."');
 };
 
+function escapeLikePattern(str: string): string {
+  return str.replace(/([%_\\])/g, '\\$1');
+}
+
 export function buildFilterQuery(filter: VectorFilter): FilterResult {
   if (!filter) {
     return { sql: '', values: [] };
@@ -418,13 +486,13 @@ function handleOperator(key: string, value: any): FilterResult {
       operator === '$not'
         ? {
             sql: `NOT (${Object.entries(operatorValue as Record<string, any>)
-              .map(([op, val]) => processOperator(key, op, val).sql)
+              .map(([op, val]) => processOperator(key, op as OperatorType, val).sql)
               .join(' AND ')})`,
             values: Object.entries(operatorValue as Record<string, any>).flatMap(
-              ([op, val]) => processOperator(key, op, val).values,
+              ([op, val]) => processOperator(key, op as OperatorType, val).values,
             ),
           }
-        : processOperator(key, operator, operatorValue),
+        : processOperator(key, operator as OperatorType, operatorValue),
     );
 
     return {
@@ -435,10 +503,10 @@ function handleOperator(key: string, value: any): FilterResult {
 
   // Handle single operator
   const [[operator, operatorValue] = []] = Object.entries(value);
-  return processOperator(key, operator as string, operatorValue);
+  return processOperator(key, operator as OperatorType, operatorValue);
 }
 
-const processOperator = (key: string, operator: string, operatorValue: any): FilterResult => {
+const processOperator = (key: string, operator: OperatorType, operatorValue: any): FilterResult => {
   if (!operator.startsWith('$') || !FILTER_OPERATORS[operator]) {
     throw new Error(`Invalid operator: ${operator}`);
   }
@@ -449,9 +517,9 @@ const processOperator = (key: string, operator: string, operatorValue: any): Fil
     return { sql: operatorResult.sql, values: [] };
   }
 
-  const transformed = operatorResult.transformValue ? operatorResult.transformValue(operatorValue) : operatorValue;
+  const transformed = operatorResult.transformValue ? operatorResult.transformValue() : operatorValue;
 
-  if (transformed && typeof transformed === 'object' && 'sql' in transformed) {
+  if (isFilterResult(transformed)) {
     return transformed;
   }