@stamhoofd/sql 2.89.2 → 2.90.1

package/src/filters/SQLFilter.ts

@@ -1,551 +1,269 @@
 import { SimpleError } from '@simonbackx/simple-errors';
-import { StamhoofdCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
-import { SQL } from '../SQL';
-import { SQLExpression } from '../SQLExpression';
-import { SQLArray, SQLCast, SQLColumnExpression, SQLLower, SQLNull, SQLSafeValue, SQLScalarValue, scalarToSQLExpression } from '../SQLExpressions';
-import { SQLJsonContains, SQLJsonOverlaps, SQLJsonSearch, SQLJsonUnquote, scalarToSQLJSONExpression } from '../SQLJsonExpressions';
-import { SQLSelect } from '../SQLSelect';
-import { SQLWhere, SQLWhereAnd, SQLWhereEqual, SQLWhereExists, SQLWhereJoin, SQLWhereLike, SQLWhereNot, SQLWhereOr, SQLWhereSign } from '../SQLWhere';
+import { compileFilter, FilterCompiler, FilterDefinitions, filterDefinitionsToCompiler, RequiredFilterCompiler, StamhoofdFilter } from '@stamhoofd/structures';
+import { SQLExpression, SQLExpressionOptions, SQLQuery } from '../SQLExpression';
 import { SQLJoin } from '../SQLJoin';
+import { SQLJsonValue } from '../SQLJsonExpressions';
+import { SQLSelect } from '../SQLSelect';
+import { SQLWhere, SQLWhereAnd, SQLWhereExists, SQLWhereJoin, SQLWhereNot, SQLWhereOr } from '../SQLWhere';
+import { $equalsSQLFilterCompiler, $greaterThanSQLFilterCompiler, $inSQLFilterCompiler, $lessThanSQLFilterCompiler } from './compilers';
+import { $containsSQLFilterCompiler } from './compilers/contains';
 
-export type SQLFilterCompiler = (filter: StamhoofdFilter, filters: SQLFilterDefinitions) => Promise<SQLWhere | null> | SQLWhere | null;
-export type SQLFilterDefinitions = Record<string, SQLFilterCompiler>;
-
-export async function andSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterDefinitions): Promise<SQLWhere> {
-    const runners = await compileSQLFilter(filter, filters);
-    return new SQLWhereAnd(runners);
-}
-
-export async function orSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterDefinitions): Promise<SQLWhere> {
-    const runners = await compileSQLFilter(filter, filters);
-    return new SQLWhereOr(runners);
-}
-
-export async function notSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterDefinitions): Promise<SQLWhere> {
-    const andRunner = await andSQLFilterCompiler(filter, filters);
-    return new SQLWhereNot(andRunner);
-}
-
-function guardFilterCompareValue(val: any): StamhoofdCompareValue {
-    if (val instanceof Date) {
-        return val;
-    }
-
-    if (typeof val === 'string') {
-        return val;
-    }
-
-    if (typeof val === 'number') {
-        return val;
-    }
-
-    if (typeof val === 'boolean') {
-        return val;
-    }
-
-    if (val === null) {
-        return null;
-    }
-
-    if (typeof val === 'object' && '$' in val) {
-        if (val['$'] === '$now') {
-            return val;
-        }
-    }
-
-    throw new Error('Invalid compare value. Expected a string, number, boolean, date or null.');
-}
-
-function doNormalizeValue(val: StamhoofdCompareValue, options?: SQLExpressionFilterOptions): string | number | Date | null | boolean {
-    if (val instanceof Date) {
-        return val;
-    }
-
-    if (typeof val === 'string') {
-        if (options?.isJSONObject) {
-            return val;
-        }
-        return val.toLocaleLowerCase();
-    }
-
-    if (typeof val === 'boolean') {
-        if (options?.type === SQLValueType.JSONBoolean) {
-            return val;
-        }
-        return val === true ? 1 : 0;
-    }
-
-    if (typeof val === 'number') {
-        if (options?.type === SQLValueType.JSONBoolean) {
-            return val === 1 ? true : false;
-        }
-
-        return val;
-    }
+export type SQLSyncFilterRunner = (column: SQLCurrentColumn) => SQLWhere;
+export type SQLFilterRunner = (column: SQLCurrentColumn) => Promise<SQLWhere> | SQLWhere;
+export type SQLFilterCompiler = FilterCompiler<SQLFilterRunner>;
+export type SQLRequiredFilterCompiler = RequiredFilterCompiler<SQLFilterRunner>;
+export type SQLFilterDefinitions = FilterDefinitions<SQLFilterRunner>;
 
-    if (val === null) {
-        return null;
-    }
+export enum SQLValueType {
+    /** At the root of a select */
+    Table = 'Table',
 
-    if (typeof val === 'object' && '$' in val) {
-        const specialValue = val['$'];
+    /** Column with type string */
+    String = 'String',
 
-        switch (specialValue) {
-            case '$now':
-                return doNormalizeValue(new Date());
-            default:
-                throw new Error('Unsupported magic value ' + specialValue);
-        }
-    }
+    /** MySQL Datetime */
+    Datetime = 'Datetime',
 
-    return val;
-}
+    /** Column with type number */
+    Number = 'Number',
 
-/**
- * WARNING: only use this on one-to-one relations. Using it on one-to-many relations will result in duplicate results.
- */
-export function createSQLJoinedRelationFilterCompiler(join: SQLJoin, definitions: SQLFilterDefinitions): SQLFilterCompiler {
-    return async (filter: StamhoofdFilter) => {
-        const f = filter as any;
-
-        if ('$elemMatch' in f) {
-            // $elemMatch is also supported but not required (since this is a one-to-one relation)
-            const w = await compileToSQLFilter(f['$elemMatch'] as StamhoofdFilter, definitions);
-            return new SQLWhereJoin(join, w);
-        }
+    /** Column with type boolean, meaning 1 or 0 */
+    Boolean = 'Boolean',
 
-        const w = await compileToSQLFilter(filter, definitions);
-        return new SQLWhereJoin(join, w);
-    };
-}
-
-export function createSQLRelationFilterCompiler(baseSelect: InstanceType<typeof SQLSelect> & SQLExpression, definitions: SQLFilterDefinitions): SQLFilterCompiler {
-    return async (filter: StamhoofdFilter) => {
-        const f = filter as any;
-
-        if ('$elemMatch' in f) {
-            const w = await compileToSQLFilter(f['$elemMatch'], definitions);
-            const q = baseSelect.clone().where(w);
-            return new SQLWhereExists(q);
-        }
+    /** True or false in JSON */
+    JSONBoolean = 'JSONBoolean',
+    JSONString = 'JSONString',
 
-        throw new SimpleError({
-            code: 'invalid_filter',
-            message: 'Invalid filter',
-            human: $t('a5c30846-b8ae-410d-8fcd-bfc3f127623d'),
-        });
-    };
-}
+    JSONNumber = 'JSONNumber',
 
-export function createSQLOneToOneRelationFilterCompiler(baseSelect: InstanceType<typeof SQLSelect> & SQLExpression, definitions: SQLFilterDefinitions): SQLFilterCompiler {
-    return async (filter: StamhoofdFilter) => {
-        const w = await compileToSQLFilter(filter, definitions);
-        const q = baseSelect.clone().where(w);
-        return new SQLWhereExists(q);
-    };
-}
+    /** [...] */
+    JSONArray = 'JSONArray',
 
-// Already joined, but creates a namespace
-export function createSQLFilterNamespace(definitions: SQLFilterDefinitions): SQLFilterCompiler {
-    return (filter: StamhoofdFilter) => {
-        return andSQLFilterCompiler(filter, definitions);
-    };
+    /** {...} */
+    JSONObject = 'JSONObject',
 }
 
-export enum SQLValueType {
-    JSONBoolean = 'JSONBoolean',
-    JSONString = 'JSONString',
-}
+export type SQLCurrentColumn = {
+    expression: SQLExpression;
 
-type SQLExpressionFilterOptions = {
-    normalizeValue?: (v: SQLScalarValue | null) => SQLScalarValue | null;
-    isJSONValue?: boolean;
-    isJSONObject?: boolean;
+    /**
+     * MySQL nullable. Please fill this in correctly! If a value can be null, can not exist (=mysql null), or can be JSONNull, set this to true.
+     *
+     * Mainly > and < operators will make sure the behaviour is consistent with MySQL sorting (normally comparing with null will always return false in MySQL)
+     */
     nullable?: boolean;
 
+    /**
+     * JSON nullable
+     */
+
     /**
      * Type of this column, use to normalize values received from filters
      */
-    type?: SQLValueType;
+    type: SQLValueType;
     checkPermission?: () => Promise<void>;
 };
 
-export function createSQLExpressionFilterCompiler(sqlExpression: SQLExpression, options: SQLExpressionFilterOptions = {}): SQLFilterCompiler {
-    const { isJSONObject = false, isJSONValue = false, nullable = false } = options;
-    let normalizeValue = options.normalizeValue;
-    normalizeValue = normalizeValue ?? (v => v);
-    const norm = (val: any) => {
-        const n = doNormalizeValue(guardFilterCompareValue(val), options);
-        return normalizeValue(n);
-    };
-    let convertToExpression = scalarToSQLExpression;
-
-    if (isJSONValue) {
-        const castJsonType = (expression: SQLExpression, type: SQLValueType | undefined): SQLExpression => {
-            if (type === undefined) {
-                return expression;
-            }
+export function createColumnFilter(column: SQLCurrentColumn, childDefinitions?: SQLFilterDefinitions): SQLFilterCompiler {
+    return (filter: StamhoofdFilter) => {
+        const compiler = childDefinitions ? filterDefinitionsToCompiler(childDefinitions) : filterDefinitionsToCompiler(baseSQLFilterCompilers);
+        const runner = $andSQLFilterCompiler(filter, compiler);
 
-            switch (type) {
-                case SQLValueType.JSONBoolean: {
-                    return expression;
-                }
-                case SQLValueType.JSONString: {
-                    return new SQLCast(new SQLJsonUnquote(expression), 'CHAR');
-                }
+        return async (_: SQLCurrentColumn) => {
+            if (column.checkPermission) {
+                await column.checkPermission();
             }
+            return await runner({
+                nullable: false,
+                ...column,
+            });
         };
+    };
+}
 
-        sqlExpression = castJsonType(sqlExpression, options.type);
-        convertToExpression = (q: SQLScalarValue | null) => {
-            const base = scalarToSQLJSONExpression(q);
-            return castJsonType(base, options.type);
-        };
-    }
-
-    return async (filter: StamhoofdFilter, filters: SQLFilterDefinitions) => {
-        if (options.checkPermission) {
-            // throws error if no permissions
-            await options.checkPermission();
-        }
-
-        if (typeof filter === 'string' || typeof filter === 'number' || typeof filter === 'boolean' || filter === null || filter === undefined || filter instanceof Date) {
-            filter = {
-                $eq: filter,
-            };
-        }
-
-        if (Array.isArray(filter)) {
-            throw new Error('Unexpected array in filter');
-        }
-
-        const f = filter;
-
-        if ('$eq' in f) {
-            if (isJSONObject) {
-                const v = norm(f.$eq);
-
-                if (typeof v === 'string') {
-                    // Custom query to support case insensitive comparing
-
-                    return new SQLWhereEqual(
-                        new SQLJsonSearch(
-                            new SQLLower(sqlExpression),
-                            'one',
-                            convertToExpression(
-                                SQLWhereLike.escape(v.toLocaleLowerCase()),
-                            ),
-                        ),
-                        SQLWhereSign.NotEqual,
-                        new SQLNull(),
-                    );
-                }
-                // else
-                return new SQLJsonContains(
-                    sqlExpression,
-                    convertToExpression(v),
-                );
-            }
-
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, convertToExpression(norm(f.$eq)));
-        }
-
-        if ('$in' in f) {
-            if (!Array.isArray(f.$in)) {
-                throw new SimpleError({
-                    code: 'invalid_filter',
-                    message: 'Expected array at $in filter',
-                });
-            }
-
-            if (f.$in.length === 0) {
-                return new SQLWhereEqual(new SQLSafeValue(1), SQLWhereSign.Equal, new SQLSafeValue(0));
-            }
+export function createWildcardColumnFilter(getColumn: (key: string) => SQLCurrentColumn, childDefinitions?: (key: string) => SQLFilterDefinitions, options?: { checkPermission?: (key: string) => Promise<void> }): SQLFilterCompiler {
+    const wildcardCompiler = (filter: StamhoofdFilter, _, key: string) => {
+        const compiler = childDefinitions ? filterDefinitionsToCompiler(childDefinitions(key)) : filterDefinitionsToCompiler(baseSQLFilterCompilers);
+        const runner = $andSQLFilterCompiler(filter, compiler);
 
-            const v = f.$in.map(a => norm(a));
-            const nullIncluded = v.includes(null);
-
-            if (isJSONObject) {
-                if (nullIncluded) {
-                    // PROBLEM: The sql expression can either not exist (= resolve to mysql null), contains null in json (= JSON null), or contain a value.
-                    // that makes comparing more difficult, to combat this, we still need to use SQLJsonOverlaps with the JSON null value
-                    return new SQLWhereOr([
-                        new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, new SQLNull()), // checks path not exists (= mysql null)
-                        new SQLJsonOverlaps(
-                            sqlExpression,
-                            convertToExpression(JSON.stringify(v)), // contains json null
-                        ),
-                    ]);
-                }
-
-                // else
-                return new SQLJsonOverlaps(
-                    sqlExpression,
-                    convertToExpression(JSON.stringify(v)),
-                );
+        return async (_: SQLCurrentColumn) => {
+            if (options?.checkPermission) {
+                await options.checkPermission(key);
             }
-
-            const createSqlArray = (value: SQLScalarValue[]): SQLArray => {
-                if (isJSONValue) {
-                    const type = options.type;
-
-                    switch (type) {
-                        case SQLValueType.JSONBoolean: {
-                            // todo;
-                            break;
-                        }
-                        case SQLValueType.JSONString: {
-                            break;
-                        }
-                    }
-                }
-
-                return new SQLArray(value);
-            };
-
-            if (nullIncluded) {
-                const remaining = v.filter(v => v !== null);
-                if (remaining.length === 0) {
-                    return new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, new SQLNull());
-                }
-                return new SQLWhereOr([
-                    new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, new SQLNull()),
-                    new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, createSqlArray(remaining)),
-                ]);
+            const column = getColumn(key);
+            if (column.checkPermission) {
+                await column.checkPermission();
             }
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, createSqlArray(v as SQLScalarValue[]));
-        }
+            return await runner({
+                nullable: false,
+                ...column,
+            });
+        };
+    };
 
-        if ('$neq' in f) {
-            if (isJSONObject) {
-                const v = norm(f.$neq);
+    return (filter: StamhoofdFilter) => {
+        return $andSQLFilterCompiler(filter, wildcardCompiler);
+    };
+}
 
-                return new SQLWhereNot(
-                    new SQLJsonContains(
-                        sqlExpression,
-                        convertToExpression(JSON.stringify(v)),
-                    ),
-                );
-            }
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.NotEqual, convertToExpression(norm(f.$neq)));
+/**
+ * Filter with a subquery that should return at least one result.
+ */
+export function createExistsFilter(baseSelect: InstanceType<typeof SQLSelect> & SQLExpression, definitions: SQLFilterDefinitions): SQLFilterCompiler {
+    return (filter: StamhoofdFilter, _: SQLFilterCompiler) => {
+        if (filter !== null && typeof filter === 'object' && '$elemMatch' in filter) {
+            filter = filter['$elemMatch'] as StamhoofdFilter;
         }
 
-        if ('$gt' in f) {
-            if (isJSONObject) {
-                throw new Error('Greater than is not supported in this place');
-            }
-
-            if (f.$gt === null) {
-                // > null is same as not equal to null (everything is larger than null in mysql) - to be consistent with order by behaviour
-                return new SQLWhereEqual(sqlExpression, SQLWhereSign.NotEqual, convertToExpression(null));
-            }
-
-            // For MySQL null values are never included in greater than, but we need this for consistent sorting behaviour
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.Greater, convertToExpression(norm(f.$gt)));
-        }
+        const runner = compileToSQLRunner(filter, definitions);
 
-        if ('$gte' in f) {
-            if (isJSONObject) {
-                throw new Error('Greater than is not supported in this place');
-            }
+        return async (_: SQLCurrentColumn) => {
+            const w = await runner({
+                expression: SQLRootExpression,
+                type: SQLValueType.Table,
+                nullable: false,
+            });
+            const q = baseSelect.clone().andWhere(w);
+            return new SQLWhereExists(q);
+        };
+    };
+}
 
-            if (f.$gte === null) {
-                // >= null is always everything
-                return new SQLWhereEqual(new SQLSafeValue(1), SQLWhereSign.Equal, new SQLSafeValue(1));
-            }
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.GreaterEqual, convertToExpression(norm(f.$gte)));
+/**
+ * WARNING: only use this on one-to-one relations. Using it on one-to-many relations will result in duplicate results.
+ *
+ * By default doesRelationAlwaysExist is set to true, this means we expect the relation to always exist. This helps optimize the query (dropping the join if the where clause in the join is always true)
+ */
+export function createJoinedRelationFilter(join: SQLJoin, definitions: SQLFilterDefinitions, options: { doesRelationAlwaysExist: boolean } = { doesRelationAlwaysExist: true }): SQLFilterCompiler {
+    return (filter: StamhoofdFilter, _: SQLFilterCompiler) => {
+        if (filter !== null && typeof filter === 'object' && '$elemMatch' in filter) {
+            filter = filter['$elemMatch'] as StamhoofdFilter;
         }
 
-        if ('$lte' in f) {
-            if (isJSONObject) {
-                throw new Error('Greater than is not supported in this place');
-            }
+        return async (_: SQLCurrentColumn) => {
+            const w = await compileToSQLFilter(filter, definitions);
+            return new SQLWhereJoin(join, w, {
+                doesRelationAlwaysExist: options.doesRelationAlwaysExist,
+            });
+        };
+    };
+}
 
-            if (f.$lte === null) {
-                // <= null is same as equal to null
-                return new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, convertToExpression(norm(f.$lte)));
-            }
+export function $andSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const runners = compileSQLFilter(filter, filters);
 
-            const base = new SQLWhereEqual(sqlExpression, SQLWhereSign.LessEqual, convertToExpression(norm(f.$lte)));
+    return async (column: SQLCurrentColumn) => {
+        const wheres = (await Promise.all(
+            runners.map(runner => (runner(column))),
+        ));
 
-            if (nullable) {
-                return new SQLWhereOr([
-                    // Null values are also smaller than any value  - required for sorting
-                    new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, new SQLNull()),
-                    base,
-                ]);
-            }
+        return new SQLWhereAnd(wheres);
+    };
+}
 
-            return new SQLWhereEqual(sqlExpression, SQLWhereSign.LessEqual, convertToExpression(norm(f.$lte)));
-        }
+export function $orSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const runners = compileSQLFilter(filter, filters);
 
-        if ('$lt' in f) {
-            if (isJSONObject) {
-                throw new Error('Less than is not supported in this place');
-            }
+    return async (column: SQLCurrentColumn) => {
+        const wheres = (await Promise.all(
+            runners.map(runner => (runner(column))),
+        ));
 
-            if (f.$lt === null) {
-                // < null is always nothing, there is nothing smaller than null in MySQL - to be consistent with order by behaviour
-                return new SQLWhereEqual(new SQLSafeValue(1), SQLWhereSign.Equal, new SQLSafeValue(0));
-            }
+        return new SQLWhereOr(wheres);
+    };
+}
 
-            const base = new SQLWhereEqual(sqlExpression, SQLWhereSign.Less, convertToExpression(norm(f.$lt)));
+export function $notSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const andRunner = $andSQLFilterCompiler(filter, filters);
 
-            if (nullable) {
-                return new SQLWhereOr([
-                    // Null values are also smaller than any value  - required for sorting
-                    new SQLWhereEqual(sqlExpression, SQLWhereSign.Equal, new SQLNull()),
-                    base,
-                ]);
-            }
+    return async (column: SQLCurrentColumn) => {
+        return new SQLWhereNot(await andRunner(column));
+    };
+}
 
-            return base;
-        }
+function invertFilterCompiler(compiler: SQLRequiredFilterCompiler): SQLRequiredFilterCompiler {
+    return (filter: StamhoofdFilter, parentCompiler: SQLFilterCompiler) => {
+        const runner = compiler(filter, parentCompiler);
+        return async (column) => {
+            return new SQLWhereNot(await runner(column));
+        };
+    };
+}
 
-        if ('$contains' in f) {
-            const needle = norm(f.$contains);
+export const baseSQLFilterCompilers: SQLFilterDefinitions = {
+    $and: $andSQLFilterCompiler,
+    $or: $orSQLFilterCompiler,
+    $not: $notSQLFilterCompiler,
+    $eq: $equalsSQLFilterCompiler,
+    $neq: invertFilterCompiler($equalsSQLFilterCompiler),
 
-            if (typeof needle !== 'string') {
-                throw new Error('Invalid needle for contains filter');
-            }
+    $lt: $lessThanSQLFilterCompiler,
+    $gt: $greaterThanSQLFilterCompiler,
+    $lte: invertFilterCompiler($greaterThanSQLFilterCompiler),
+    $gte: invertFilterCompiler($lessThanSQLFilterCompiler),
 
-            if (isJSONObject) {
-                return new SQLWhereEqual(
-                    new SQLJsonSearch(
-                        new SQLLower(sqlExpression),
-                        'one',
-                        convertToExpression(
-                            '%' + SQLWhereLike.escape(needle) + '%',
-                        ),
-                    ),
-                    SQLWhereSign.NotEqual,
-                    new SQLNull(),
-                );
-            }
+    $in: $inSQLFilterCompiler,
 
-            if (isJSONValue) {
-                // We need to do case insensitive search, so need to convert the sqlExpression from utf8mb4 to varchar
-                return new SQLWhereLike(
-                    new SQLCast(new SQLJsonUnquote(sqlExpression), 'CHAR'),
-                    convertToExpression(
-                        '%' + SQLWhereLike.escape(needle) + '%',
-                    ),
-                );
-            }
+    $contains: $containsSQLFilterCompiler,
+};
 
-            return new SQLWhereLike(
-                sqlExpression,
-                convertToExpression(
-                    '%' + SQLWhereLike.escape(needle) + '%',
-                ),
-            );
-        }
+const compileSQLFilter = compileFilter<SQLFilterRunner>;
 
+export const SQLRootExpression: SQLExpression = {
+    getSQL(options?: SQLExpressionOptions): SQLQuery {
         throw new SimpleError({
             code: 'invalid_filter',
-            message: 'Invalid filter',
-            human: $t('a5c30846-b8ae-410d-8fcd-bfc3f127623d'),
+            message: 'Root level filters are not allowed to use $eq or $neq',
         });
-    };
-}
+    },
+};
 
-export function createSQLColumnFilterCompiler(name: string | SQLColumnExpression, options?: SQLExpressionFilterOptions): SQLFilterCompiler {
-    const column = name instanceof SQLColumnExpression ? name : SQL.column(name);
-    return createSQLExpressionFilterCompiler(column, options);
-}
+export function compileToSQLRunner(filter: StamhoofdFilter, definitions: SQLFilterDefinitions): SQLFilterRunner {
+    if (filter === null) {
+        return () => {
+            return new SQLWhereAnd([]); // No filter, return empty where
+        };
+    }
+    const compiler = filterDefinitionsToCompiler(definitions); // this compiler searches in the definition for the right compiler for the given key
+    const runner = $andSQLFilterCompiler(filter, compiler);
+    return runner;
+};
 
-export const baseSQLFilterCompilers: SQLFilterDefinitions = {
-    $and: andSQLFilterCompiler,
-    $or: orSQLFilterCompiler,
-    $not: notSQLFilterCompiler,
+export async function compileToSQLFilter(filter: StamhoofdFilter, filters: SQLFilterDefinitions): Promise<SQLWhere> {
+    const runner = compileToSQLRunner(filter, filters);
+    return await runner({
+        expression: SQLRootExpression,
+        type: SQLValueType.Table,
+        nullable: false,
+    });
 };
 
-function objectToArray(f: StamhoofdFilter & object): StamhoofdFilter[] {
-    const splitted: StamhoofdFilter[] = [];
-    for (const key of Object.keys(f)) {
-        splitted.push({ [key]: f[key] });
+/**
+ * Casts json strings, numbers and booleans to native MySQL types. This includes json null to mysql null.
+ */
+export function normalizeColumn(column: SQLCurrentColumn): SQLCurrentColumn {
+    if (column.type === SQLValueType.JSONString) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'CHAR'),
+            type: SQLValueType.String,
+            nullable: column.nullable,
+        };
     }
-    return splitted;
-}
 
-async function compileSQLFilter(filter: StamhoofdFilter, definitions: SQLFilterDefinitions): Promise<SQLWhere[]> {
-    if (filter === undefined) {
-        return [];
+    if (column.type === SQLValueType.JSONBoolean) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'UNSIGNED'),
+            type: SQLValueType.Boolean,
+            nullable: column.nullable,
+        };
     }
 
-    const runners: SQLWhere[] = [];
-
-    for (const f of (Array.isArray(filter) ? filter : (typeof filter === 'object' && filter !== null ? objectToArray(filter) : [filter]))) {
-        if (!f) {
-            continue;
-        }
-        if (!(typeof f === 'object' && f !== null)) {
-            throw new Error('Unsupported filter at this position: ' + f);
-        }
-
-        if (Object.keys(f).length > 1) {
-            // Multiple keys in the same object should always be combined with AND
-            runners.push(await andSQLFilterCompiler(objectToArray(f), definitions));
-            continue;
-        }
-
-        if (Array.isArray(f)) {
-            // Arrays in filters not direclty underneath $and or $or should be combined with AND
-            runners.push(await andSQLFilterCompiler(f, definitions));
-            continue;
-        }
-
-        for (const key of Object.keys(f)) {
-            let ff = definitions[key];
-            let value: StamhoofdFilter = f[key];
-
-            if (!ff) {
-                // Search with dot syntax shortcuts
-                if (key.includes('.')) {
-                    const parts = key.split('.');
-
-                    if (parts.length >= 2 && parts.every(p => p.length > 0)) {
-                        let subKey = parts.shift() ?? '';
-
-                        while (parts.length && !definitions[subKey]) {
-                            subKey = subKey + '.' + parts.shift();
-                        }
-
-                        if (subKey && definitions[subKey]) {
-                            const remaining = parts.join('.');
-
-                            const transformeInto = {
-                                [remaining]: value,
-                            };
-
-                            ff = definitions[subKey];
-                            value = transformeInto;
-                        }
-                    }
-                }
-            }
-
-            if (!ff) {
-                throw new SimpleError({
-                    code: 'invalid_filter',
-                    message: 'Invalid filter ' + key,
-                    human: $t('a5c30846-b8ae-410d-8fcd-bfc3f127623d'),
-                });
-            }
-
-            const s = await ff(value, definitions);
-            if (s === undefined || s === null) {
-                throw new SimpleError({
-                    code: 'invalid_filter',
-                    message: 'Invalid filter value for filter ' + key,
-                    human: $t('a5c30846-b8ae-410d-8fcd-bfc3f127623d'),
-                });
-            }
-            runners.push(s);
-        }
+    if (column.type === SQLValueType.JSONNumber) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'UNSIGNED'),
+            type: SQLValueType.Number,
+            nullable: column.nullable,
+        };
     }
-
-    return runners;
+    return column;
 }
-
-export const compileToSQLFilter = andSQLFilterCompiler;