@stamhoofd/sql 2.83.5 → 2.84.0

package/src/filters/modern/SQLModernFilter.ts

@@ -0,0 +1,256 @@
+import { SimpleError } from '@simonbackx/simple-errors';
+import { compileFilter, FilterCompiler, FilterDefinitions, filterDefinitionsToCompiler, RequiredFilterCompiler, StamhoofdFilter } from '@stamhoofd/structures';
+import { SQLExpression, SQLExpressionOptions, SQLQuery } from '../../SQLExpression';
+import { SQLJoin } from '../../SQLJoin';
+import { SQLJsonValue } from '../../SQLJsonExpressions';
+import { SQLSelect } from '../../SQLSelect';
+import { SQLWhere, SQLWhereAnd, SQLWhereExists, SQLWhereJoin, SQLWhereNot, SQLWhereOr } from '../../SQLWhere';
+import { $equalsSQLFilterCompiler, $greaterThanSQLFilterCompiler, $inSQLFilterCompiler, $lessThanSQLFilterCompiler } from './compilers';
+import { $containsSQLFilterCompiler } from './compilers/contains';
+
+export type SQLSyncFilterRunner = (column: SQLCurrentColumn) => SQLWhere;
+export type SQLFilterRunner = (column: SQLCurrentColumn) => Promise<SQLWhere> | SQLWhere;
+export type SQLFilterCompiler = FilterCompiler<SQLFilterRunner>;
+export type SQLRequiredFilterCompiler = RequiredFilterCompiler<SQLFilterRunner>;
+export type SQLModernFilterDefinitions = FilterDefinitions<SQLFilterRunner>;
+
+export enum SQLModernValueType {
+    /** At the root of a select */
+    Table = 'Table',
+
+    /** Column with type string */
+    String = 'String',
+
+    /** MySQL Datetime */
+    Datetime = 'Datetime',
+
+    /** Column with type number */
+    Number = 'Number',
+
+    /** Column with type boolean, meaning 1 or 0 */
+    Boolean = 'Boolean',
+
+    /** True or false in JSON */
+    JSONBoolean = 'JSONBoolean',
+    JSONString = 'JSONString',
+
+    JSONNumber = 'JSONNumber',
+
+    /** [...] */
+    JSONArray = 'JSONArray',
+
+    /** {...} */
+    JSONObject = 'JSONObject',
+}
+
+export type SQLCurrentColumn = {
+    expression: SQLExpression;
+
+    /**
+     * MySQL nullable. Please fill this in correctly! If a value can be null, can not exist (=mysql null), or can be JSONNull, set this to true.
+     *
+     * Mainly > and < operators will make sure the behaviour is consistent with MySQL sorting (normally comparing with null will always return false in MySQL)
+     */
+    nullable?: boolean;
+
+    /**
+     * JSON nullable
+     */
+
+    /**
+     * Type of this column, use to normalize values received from filters
+     */
+    type: SQLModernValueType;
+    checkPermission?: () => Promise<void>;
+};
+
+export function createColumnFilter(column: SQLCurrentColumn, childDefinitions?: SQLModernFilterDefinitions): SQLFilterCompiler {
+    return (filter: StamhoofdFilter) => {
+        const compiler = childDefinitions ? filterDefinitionsToCompiler(childDefinitions) : filterDefinitionsToCompiler(baseModernSQLFilterCompilers);
+        const runner = $andSQLFilterCompiler(filter, compiler);
+
+        return (_: SQLCurrentColumn) => {
+            return runner({
+                nullable: false,
+                ...column,
+            });
+        };
+    };
+}
+
+export function createWildcardColumnFilter(getColumn: (key: string) => SQLCurrentColumn, childDefinitions?: (key: string) => SQLModernFilterDefinitions): SQLFilterCompiler {
+    const wildcardCompiler = (filter: StamhoofdFilter, _, key: string) => {
+        const compiler = childDefinitions ? filterDefinitionsToCompiler(childDefinitions(key)) : filterDefinitionsToCompiler(baseModernSQLFilterCompilers);
+        const runner = $andSQLFilterCompiler(filter, compiler);
+
+        return (_: SQLCurrentColumn) => {
+            return runner({
+                nullable: false,
+                ...getColumn(key),
+            });
+        };
+    };
+
+    return (filter: StamhoofdFilter) => {
+        return $andSQLFilterCompiler(filter, wildcardCompiler);
+    };
+}
+
+/**
+ * Filter with a subquery that should return at least one result.
+ */
+export function createExistsFilter(baseSelect: InstanceType<typeof SQLSelect> & SQLExpression, definitions: SQLModernFilterDefinitions): SQLFilterCompiler {
+    return (filter: StamhoofdFilter, _: SQLFilterCompiler) => {
+        if (filter !== null && typeof filter === 'object' && '$elemMatch' in filter) {
+            filter = filter['$elemMatch'] as StamhoofdFilter;
+        }
+
+        const runner = compileToSQLRunner(filter, definitions);
+
+        return async (_: SQLCurrentColumn) => {
+            const w = await runner({
+                expression: SQLRootExpression,
+                type: SQLModernValueType.Table,
+                nullable: false,
+            });
+            const q = baseSelect.clone().andWhere(w);
+            return new SQLWhereExists(q);
+        };
+    };
+}
+
+/**
+ * WARNING: only use this on one-to-one relations. Using it on one-to-many relations will result in duplicate results.
+ *
+ * By default doesRelationAlwaysExist is set to true, this means we expect the relation to always exist. This helps optimize the query (dropping the join if the where clause in the join is always true)
+ */
+export function createJoinedRelationFilter(join: SQLJoin, definitions: SQLModernFilterDefinitions, options: { doesRelationAlwaysExist: boolean } = { doesRelationAlwaysExist: true }): SQLFilterCompiler {
+    return (filter: StamhoofdFilter, _: SQLFilterCompiler) => {
+        if (filter !== null && typeof filter === 'object' && '$elemMatch' in filter) {
+            filter = filter['$elemMatch'] as StamhoofdFilter;
+        }
+
+        return async (_: SQLCurrentColumn) => {
+            const w = await compileToModernSQLFilter(filter, definitions);
+            return new SQLWhereJoin(join, w, {
+                doesRelationAlwaysExist: options.doesRelationAlwaysExist,
+            });
+        };
+    };
+}
+
+export function $andSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const runners = compileSQLFilter(filter, filters);
+
+    return async (column: SQLCurrentColumn) => {
+        const wheres = (await Promise.all(
+            runners.map(runner => (runner(column))),
+        ));
+
+        return new SQLWhereAnd(wheres);
+    };
+}
+
+export function $orSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const runners = compileSQLFilter(filter, filters);
+
+    return async (column: SQLCurrentColumn) => {
+        const wheres = (await Promise.all(
+            runners.map(runner => (runner(column))),
+        ));
+
+        return new SQLWhereOr(wheres);
+    };
+}
+
+export function $notSQLFilterCompiler(filter: StamhoofdFilter, filters: SQLFilterCompiler): SQLFilterRunner {
+    const andRunner = $andSQLFilterCompiler(filter, filters);
+
+    return async (column: SQLCurrentColumn) => {
+        return new SQLWhereNot(await andRunner(column));
+    };
+}
+
+function invertFilterCompiler(compiler: SQLRequiredFilterCompiler): SQLRequiredFilterCompiler {
+    return (filter: StamhoofdFilter, parentCompiler: SQLFilterCompiler) => {
+        const runner = compiler(filter, parentCompiler);
+        return async (column) => {
+            return new SQLWhereNot(await runner(column));
+        };
+    };
+}
+
+export const baseModernSQLFilterCompilers: SQLModernFilterDefinitions = {
+    $and: $andSQLFilterCompiler,
+    $or: $orSQLFilterCompiler,
+    $not: $notSQLFilterCompiler,
+    $eq: $equalsSQLFilterCompiler,
+    $neq: invertFilterCompiler($equalsSQLFilterCompiler),
+
+    $lt: $lessThanSQLFilterCompiler,
+    $gt: $greaterThanSQLFilterCompiler,
+    $lte: invertFilterCompiler($greaterThanSQLFilterCompiler),
+    $gte: invertFilterCompiler($lessThanSQLFilterCompiler),
+
+    $in: $inSQLFilterCompiler,
+
+    $contains: $containsSQLFilterCompiler,
+};
+
+const compileSQLFilter = compileFilter<SQLFilterRunner>;
+
+export const SQLRootExpression: SQLExpression = {
+    getSQL(options?: SQLExpressionOptions): SQLQuery {
+        throw new SimpleError({
+            code: 'invalid_filter',
+            message: 'Root level filters are not allowed to use $eq or $neq',
+        });
+    },
+};
+
+export function compileToSQLRunner(filter: StamhoofdFilter, definitions: SQLModernFilterDefinitions): SQLFilterRunner {
+    if (filter === null) {
+        return () => {
+            return new SQLWhereAnd([]); // No filter, return empty where
+        };
+    }
+    const compiler = filterDefinitionsToCompiler(definitions); // this compiler searches in the definition for the right compiler for the given key
+    const runner = $andSQLFilterCompiler(filter, compiler);
+    return runner;
+};
+
+export async function compileToModernSQLFilter(filter: StamhoofdFilter, filters: SQLModernFilterDefinitions): Promise<SQLWhere> {
+    const runner = compileToSQLRunner(filter, filters);
+    return await runner({
+        expression: SQLRootExpression,
+        type: SQLModernValueType.Table,
+        nullable: false,
+    });
+};
+
+/**
+ * Casts json strings, numbers and booleans to native MySQL types. This includes json null to mysql null.
+ */
+export function normalizeColumn(column: SQLCurrentColumn): SQLCurrentColumn {
+    if (column.type === SQLModernValueType.JSONString) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'CHAR'),
+            type: SQLModernValueType.String,
+        };
+    }
+
+    if (column.type === SQLModernValueType.JSONBoolean) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'UNSIGNED'),
+            type: SQLModernValueType.Boolean,
+        };
+    }
+
+    if (column.type === SQLModernValueType.JSONNumber) {
+        return {
+            expression: new SQLJsonValue(column.expression, 'UNSIGNED'),
+            type: SQLModernValueType.Number,
+        };
+    }
+    return column;
+}

package/src/filters/modern/compilers/contains.ts

@@ -0,0 +1,43 @@
+import { SimpleError } from '@simonbackx/simple-errors';
+import { assertFilterCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { scalarToSQLExpression, SQLLower, SQLNull } from '../../../SQLExpressions';
+import { SQLJsonSearch } from '../../../SQLJsonExpressions';
+import { SQLWhereEqual, SQLWhereLike, SQLWhereSign } from '../../../SQLWhere';
+import { normalizeColumn, SQLCurrentColumn, SQLSyncFilterRunner, SQLModernValueType } from '../SQLModernFilter';
+import { normalizeCompareValue } from '../helpers/normalizeCompareValue';
+
+export function $containsSQLFilterCompiler(filter: StamhoofdFilter): SQLSyncFilterRunner {
+    return (originalColumn: SQLCurrentColumn) => {
+        const column = normalizeColumn(originalColumn);
+        const value = normalizeCompareValue(assertFilterCompareValue(filter), column.type);
+
+        if (typeof value !== 'string') {
+            throw new SimpleError({
+                code: 'invalid_filter',
+                message: 'Expected string at $contains filter',
+            });
+        }
+
+        if (column.type === SQLModernValueType.JSONArray || column.type === SQLModernValueType.JSONObject) {
+            // For JSON arrays and objects, we use JSON_CONTAINS
+            return new SQLWhereEqual(
+                new SQLJsonSearch(
+                    new SQLLower(column.expression),
+                    'one',
+                    scalarToSQLExpression(
+                        '%' + SQLWhereLike.escape(value) + '%',
+                    ),
+                ),
+                SQLWhereSign.NotEqual,
+                new SQLNull(),
+            );
+        }
+
+        return new SQLWhereLike(
+            column.expression,
+            scalarToSQLExpression(
+                '%' + SQLWhereLike.escape(value) + '%',
+            ),
+        );
+    };
+}

package/src/filters/modern/compilers/equals.ts

@@ -0,0 +1,72 @@
+import { assertFilterCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { scalarToSQLExpression, SQLLower, SQLNull } from '../../../SQLExpressions';
+import { SQLJsonContains, SQLJsonSearch, SQLJsonValue } from '../../../SQLJsonExpressions';
+import { SQLWhere, SQLWhereEqual, SQLWhereLike, SQLWhereOr, SQLWhereSign } from '../../../SQLWhere';
+import { normalizeColumn, SQLCurrentColumn, SQLSyncFilterRunner, SQLModernValueType } from '../SQLModernFilter';
+import { isJSONColumn } from '../helpers/isJSONColumn';
+import { normalizeCompareValue } from '../helpers/normalizeCompareValue';
+
+export function $equalsSQLFilterCompiler(filter: StamhoofdFilter): SQLSyncFilterRunner {
+    return (originalColumn: SQLCurrentColumn) => {
+        const column = normalizeColumn(originalColumn);
+        const value = normalizeCompareValue(assertFilterCompareValue(filter), column.type);
+        /**
+         * Special case, checking for equality with a JSON array.
+         * This should return true if the JSON array contains the value exactly.
+         *
+         * This differs from $contains, which will check for 'LIKE' inside the JSON array.
+         */
+        if (column.type === SQLModernValueType.JSONArray) {
+            let where: SQLWhere;
+
+            if (typeof value === 'string') {
+                where = new SQLWhereEqual(
+                    new SQLJsonSearch(
+                        new SQLLower(column.expression),
+                        'one',
+                        scalarToSQLExpression(
+                            SQLWhereLike.escape(value),
+                        ),
+                    ),
+                    SQLWhereSign.NotEqual,
+                    new SQLNull(),
+                );
+            }
+            else {
+                where = new SQLJsonContains(
+                    column.expression,
+                    scalarToSQLExpression(JSON.stringify(value)),
+                );
+            }
+
+            // If comparing against null, also check for native MySQL null (the column does not exist)
+            if (value === null) {
+                where = new SQLWhereOr([
+                    where,
+                    new SQLWhereEqual(
+                        column.expression,
+                        SQLWhereSign.Equal,
+                        new SQLNull(),
+                    ),
+                ]);
+            }
+
+            return where;
+        }
+
+        if (value === null && isJSONColumn(column)) {
+            // JSON values can either resolve to null or "null" in MySQL.
+            return new SQLWhereEqual(
+                new SQLJsonValue(column.expression), // casts json null to null, including invalid paths
+                SQLWhereSign.Equal,
+                new SQLNull(),
+            );
+        }
+
+        return new SQLWhereEqual(
+            column.expression,
+            SQLWhereSign.Equal,
+            scalarToSQLExpression(value),
+        );
+    };
+}

package/src/filters/modern/compilers/greater.ts

@@ -0,0 +1,20 @@
+import { assertFilterCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { scalarToSQLExpression } from '../../../SQLExpressions';
+import { SQLWhereEqual, SQLWhereSign } from '../../../SQLWhere';
+import { normalizeColumn, SQLCurrentColumn, SQLSyncFilterRunner } from '../SQLModernFilter';
+import { normalizeCompareValue } from '../helpers/normalizeCompareValue';
+
+export function $greaterThanSQLFilterCompiler(filter: StamhoofdFilter): SQLSyncFilterRunner {
+    return (originalColumn: SQLCurrentColumn) => {
+        const column = normalizeColumn(originalColumn);
+        const value = normalizeCompareValue(assertFilterCompareValue(filter), column.type);
+
+        const base = new SQLWhereEqual(
+            column.expression,
+            SQLWhereSign.Greater,
+            scalarToSQLExpression(value),
+        ).setNullable(column.nullable);
+
+        return base;
+    };
+}

package/src/filters/modern/compilers/in.ts

@@ -0,0 +1,62 @@
+import { SimpleError } from '@simonbackx/simple-errors';
+import { assertFilterCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { scalarToSQLExpression, SQLArray } from '../../../SQLExpressions';
+import { SQLJsonOverlaps } from '../../../SQLJsonExpressions';
+import { SQLWhereEqual, SQLWhereOr, SQLWhereSign } from '../../../SQLWhere';
+import { normalizeColumn, SQLCurrentColumn, SQLSyncFilterRunner, SQLModernValueType } from '../SQLModernFilter';
+import { normalizeCompareValue } from '../helpers/normalizeCompareValue';
+import { $equalsSQLFilterCompiler } from './equals';
+
+export function $inSQLFilterCompiler(filter: StamhoofdFilter): SQLSyncFilterRunner {
+    return (originalColumn: SQLCurrentColumn) => {
+        if (!Array.isArray(filter)) {
+            throw new SimpleError({
+                code: 'invalid_filter',
+                message: 'Expected array at $in filter',
+            });
+        }
+
+        if (filter.length > 100) {
+            throw new SimpleError({
+                code: 'invalid_filter',
+                message: 'Too many values in $in filter, maximum is 100',
+            });
+        }
+
+        const column = normalizeColumn(originalColumn);
+        const values = filter.map(val => normalizeCompareValue(assertFilterCompareValue(val), column.type));
+
+        if (values.length === 0) {
+            // Return always false
+            return new SQLWhereOr([]);
+        }
+
+        const valuesWithoutNulls = values.filter(val => val !== null);
+        const hasNull = values.length !== valuesWithoutNulls.length;
+
+        if (hasNull) {
+            // We cannot 'in' check with null. On top of that, null has some special behaviour when used in JSON values (e.g. JSON null vs MySQL null).
+            return new SQLWhereOr([
+                $equalsSQLFilterCompiler(null)(column),
+                $inSQLFilterCompiler(valuesWithoutNulls)(column),
+            ]);
+        }
+
+        if (column.type === SQLModernValueType.JSONArray) {
+            const jsonValues = JSON.stringify(valuesWithoutNulls);
+            const valuesExpression = scalarToSQLExpression(jsonValues);
+
+            return new SQLJsonOverlaps(
+                column.expression,
+                valuesExpression,
+            );
+        }
+        const valuesExpression = valuesWithoutNulls.length === 1 ? scalarToSQLExpression(valuesWithoutNulls[0]) : new SQLArray(valuesWithoutNulls);
+
+        return new SQLWhereEqual(
+            column.expression,
+            SQLWhereSign.Equal,
+            valuesExpression,
+        );
+    };
+}

package/src/filters/modern/compilers/index.ts

@@ -0,0 +1,4 @@
+export * from './less';
+export * from './greater';
+export * from './equals';
+export * from './in';

package/src/filters/modern/compilers/less.ts

@@ -0,0 +1,19 @@
+import { assertFilterCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { scalarToSQLExpression } from '../../../SQLExpressions';
+import { SQLWhereEqual, SQLWhereSign } from '../../../SQLWhere';
+import { normalizeColumn, SQLCurrentColumn, SQLSyncFilterRunner } from '../SQLModernFilter';
+import { normalizeCompareValue } from '../helpers/normalizeCompareValue';
+
+export function $lessThanSQLFilterCompiler(filter: StamhoofdFilter): SQLSyncFilterRunner {
+    return (originalColumn: SQLCurrentColumn) => {
+        const column = normalizeColumn(originalColumn);
+        const value = normalizeCompareValue(assertFilterCompareValue(filter), column.type);
+
+        const base = new SQLWhereEqual(
+            column.expression,
+            SQLWhereSign.Less,
+            scalarToSQLExpression(value),
+        ).setNullable(column.nullable);
+        return base;
+    };
+}

package/src/filters/modern/helpers/isJSONColumn.ts

@@ -0,0 +1,13 @@
+import { SQLCurrentColumn, SQLModernValueType } from '../SQLModernFilter';
+
+export function isJSONColumn({ type }: SQLCurrentColumn): boolean {
+    return isJSONType(type);
+}
+
+export function isJSONType(type: SQLModernValueType): boolean {
+    return type === SQLModernValueType.JSONString
+        || type === SQLModernValueType.JSONBoolean
+        || type === SQLModernValueType.JSONNumber
+        || type === SQLModernValueType.JSONArray
+        || type === SQLModernValueType.JSONObject;
+}

package/src/filters/modern/helpers/normalizeCompareValue.ts

@@ -0,0 +1,95 @@
+import { StamhoofdCompareValue } from '@stamhoofd/structures';
+import { SQLModernValueType } from '../SQLModernFilter';
+
+/**
+ * Prepares a compare value so we can compare it, given a certain column type.
+ *
+ * E.g. if you pass in true - and we are comparing against a mysql boolean column, convert it to 1.
+ */
+export function normalizeCompareValue(val: StamhoofdCompareValue, againstType: SQLModernValueType): string | number | Date | null | boolean {
+    if (againstType === SQLModernValueType.Table) {
+        throw new Error('Cannot compare at root level');
+    }
+
+    if (againstType === SQLModernValueType.JSONObject) {
+        if (val === null) {
+            return null;
+        }
+        throw new Error('Cannot compare with a JSON object');
+    }
+
+    if (val instanceof Date) {
+        if (againstType === SQLModernValueType.Datetime) {
+            return val;
+        }
+        throw new Error('Cannot compare a date with a non-datetime column');
+    }
+
+    if (typeof val === 'string') {
+        if (againstType === SQLModernValueType.String || againstType === SQLModernValueType.JSONString) {
+            return val.toLocaleLowerCase();
+        }
+
+        if (againstType === SQLModernValueType.JSONArray) {
+            // We'll search inside the array
+            return val.toLocaleLowerCase();
+        }
+
+        throw new Error('Cannot compare a string with a non-string column');
+    }
+
+    if (typeof val === 'boolean') {
+        if (againstType === SQLModernValueType.JSONBoolean) {
+            return val;
+        }
+        if (againstType === SQLModernValueType.Boolean || againstType === SQLModernValueType.Number) {
+            return val === true ? 1 : 0;
+        }
+        if (againstType === SQLModernValueType.JSONArray) {
+            // We'll search inside the array
+            return val;
+        }
+        throw new Error('Cannot compare a boolean with a non-boolean column');
+    }
+
+    if (typeof val === 'number') {
+        if (againstType === SQLModernValueType.JSONBoolean) {
+            return val === 1 ? true : false;
+        }
+
+        if (againstType === SQLModernValueType.Boolean) {
+            if (val !== 1 && val !== 0) {
+                throw new Error('Cannot compare a number with a boolean column');
+            }
+            return val;
+        }
+
+        if (againstType === SQLModernValueType.Number || againstType === SQLModernValueType.JSONNumber) {
+            return val;
+        }
+
+        if (againstType === SQLModernValueType.JSONArray) {
+            // We'll search inside the array
+            return val;
+        }
+
+        throw new Error('Cannot compare a number with a non-number column');
+    }
+
+    if (val === null) {
+        return null;
+    }
+
+    if (typeof val === 'object' && '$' in val) {
+        const specialValue = val['$'];
+
+        switch (specialValue) {
+            case '$now':
+                return normalizeCompareValue(new Date(), againstType);
+            default:
+                throw new Error('Unsupported magic value ' + specialValue);
+        }
+    }
+
+    return val;
+}