@abedmurrar/objection 0.0.0-stage → 4.0.0-alpha.0

package/lib/queryBuilder/operations/eager/WhereInEagerOperation.js

@@ -0,0 +1,285 @@
+'use strict';
+
+const promiseUtils = require('../../../utils/promiseUtils');
+
+const { EagerOperation } = require('./EagerOperation');
+const { isMsSql, isOracle, isSqlite } = require('../../../utils/knexUtils');
+const { isObject, asArray, flatten, chunk } = require('../../../utils/objectUtils');
+const { ValidationErrorType } = require('../../../model/ValidationError');
+const { createModifier } = require('../../../utils/createModifier');
+const { RelationDoesNotExistError } = require('../../../model/RelationDoesNotExistError');
+const { RelationOwner } = require('../../../relations/RelationOwner');
+
+class WhereInEagerOperation extends EagerOperation {
+  constructor(name, opt) {
+    super(name, opt);
+
+    this.relationsToFetch = [];
+    this.omitProps = [];
+  }
+
+  batchSize(knex) {
+    if (this.graphOptions.maxBatchSize) {
+      return this.graphOptions.maxBatchSize;
+    } else if (isMsSql(knex)) {
+      // On MSSQL the parameter limit is actually 2100, but since I couldn't figure out
+      // if the limit is for all parameters in a query or for individual clauses, we set
+      // the limit to 2000 to leave 100 parameters for where clauses etc.
+      return 2000;
+    } else if (isOracle(knex)) {
+      return 1000;
+    } else if (isSqlite(knex)) {
+      // SQLITE_MAX_VARIABLE_NUMBER is 999 by default
+      return 999;
+    } else {
+      // I'm sure there is some kind of limit for other databases too, but let's lower
+      // this if someone ever hits those limits.
+      return 10000;
+    }
+  }
+
+  onBuild(builder) {
+    const relationsToFetch = findRelationsToFetch(
+      builder.modelClass(),
+      this.buildFinalExpression(),
+    );
+
+    const { selectionsToAdd, selectedProps } = findRelationPropsToSelect(builder, relationsToFetch);
+
+    if (selectionsToAdd.length) {
+      builder.select(selectionsToAdd);
+    }
+
+    this.relationsToFetch = relationsToFetch;
+    this.omitProps = selectedProps;
+  }
+
+  async onAfter2(builder, result) {
+    const modelClass = builder.resultModelClass();
+
+    if (!result) {
+      return result;
+    }
+
+    const models = asArray(result);
+
+    // Check models to be actual objects, to filter out `count` results (#2397).
+    if (!models.length || !isObject(models[0])) {
+      return result;
+    }
+
+    await promiseUtils.map(
+      this.relationsToFetch,
+      (it) => this.fetchRelation(builder, models, it.relation, it.childExpression),
+      { concurrency: modelClass.getConcurrency(builder.unsafeKnex()) },
+    );
+
+    const intOpt = builder.internalOptions();
+
+    if (!this.omitProps.length || intOpt.keepImplicitJoinProps) {
+      return result;
+    }
+
+    // Now that relations have been fetched for `models` we can omit the
+    // columns that were implicitly selected by this class.
+    for (let i = 0, l = result.length; i < l; ++i) {
+      const model = result[i];
+
+      for (let c = 0, lc = this.omitProps.length; c < lc; ++c) {
+        modelClass.omitImpl(model, this.omitProps[c]);
+      }
+    }
+
+    return result;
+  }
+
+  async fetchRelation(builder, models, relation, expr) {
+    const modelClass = builder.resultModelClass();
+    const batchSize = this.batchSize(builder.knex());
+    const modelBatches = chunk(models, batchSize);
+
+    const result = await promiseUtils.map(
+      modelBatches,
+      (batch) => this.fetchRelationBatch(builder, batch, relation, expr),
+      {
+        concurrency: modelClass.getConcurrency(builder.unsafeKnex()),
+      },
+    );
+
+    return flatten(result);
+  }
+
+  fetchRelationBatch(builder, models, relation, expr) {
+    if (this.shouldSkipFetched(models, relation, expr)) {
+      return this.createSkippedQuery(builder, models, relation, expr);
+    }
+
+    const queryBuilder = this.createRelationQuery(builder, relation, expr);
+    const findOperation = relation.find(queryBuilder, RelationOwner.create(models));
+
+    findOperation.alwaysReturnArray = true;
+    findOperation.assignResultToOwner = true;
+    findOperation.relationProperty = expr.node.$name;
+
+    queryBuilder.addOperation(findOperation, []);
+
+    for (const modifierName of expr.node.$modify) {
+      const modifier = createModifier({
+        modifier: modifierName,
+        modelClass: relation.relatedModelClass,
+        modifiers: this.buildFinalModifiers(builder),
+      });
+
+      try {
+        modifier(queryBuilder);
+      } catch (err) {
+        const modelClass = builder.modelClass();
+
+        if (err instanceof modelClass.ModifierNotFoundError) {
+          throw modelClass.createValidationError({
+            type: ValidationErrorType.RelationExpression,
+            message: `could not find modifier "${modifierName}" for relation "${relation.name}"`,
+          });
+        } else {
+          throw err;
+        }
+      }
+    }
+
+    return queryBuilder;
+  }
+
+  shouldSkipFetched(models, relation, expr) {
+    if (!this.graphOptions.skipFetched) {
+      return false;
+    }
+
+    if (models.some((it) => it[expr.node.$name] === undefined)) {
+      return false;
+    }
+
+    const relationsToFetch = findRelationsToFetch(relation.relatedModelClass, expr);
+    const childModels = getRelatedModels(models, expr);
+
+    // We can only skip fetching a relation if all already fetched models
+    // have all needed relation properties so that we can fetch the next
+    // level of relations.
+    for (const { relation } of relationsToFetch) {
+      const { ownerProp } = relation;
+
+      for (let c = 0, lc = ownerProp.size; c < lc; ++c) {
+        const prop = ownerProp.props[c];
+
+        for (const model of childModels) {
+          if (model[prop] === undefined) {
+            return false;
+          }
+        }
+      }
+    }
+
+    return true;
+  }
+
+  createSkippedQuery(builder, models, relation, expr) {
+    const childModels = getRelatedModels(models, expr);
+
+    return relation.relatedModelClass
+      .query()
+      .childQueryOf(builder)
+      .findOptions({ dontCallFindHooks: true })
+      .withGraphFetched(expr, this.graphOptions)
+      .resolve(childModels);
+  }
+
+  createRelationQuery(builder, relation, expr) {
+    return relation.relatedModelClass
+      .query()
+      .childQueryOf(builder)
+      .withGraphFetched(expr, this.graphOptions)
+      .modifiers(this.buildFinalModifiers(builder));
+  }
+
+  clone() {
+    const clone = super.clone();
+
+    clone.relationsToFetch = this.relationsToFetch.slice();
+    clone.omitProps = this.omitProps.slice();
+
+    return clone;
+  }
+}
+
+function findRelationsToFetch(modelClass, eagerExpression) {
+  const relationsToFetch = [];
+
+  try {
+    eagerExpression.forEachChildExpression(modelClass, (childExpression, relation) => {
+      relationsToFetch.push({
+        childExpression,
+        relation,
+      });
+    });
+  } catch (err) {
+    if (err instanceof RelationDoesNotExistError) {
+      throw modelClass.createValidationError({
+        type: ValidationErrorType.RelationExpression,
+        message: `unknown relation "${err.relationName}" in an eager expression`,
+      });
+    }
+    throw err;
+  }
+
+  return relationsToFetch;
+}
+
+function findRelationPropsToSelect(builder, relationsToFetch) {
+  const selectionsToAdd = [];
+  const selectedProps = [];
+
+  // Collect columns that need to be selected for the eager fetch
+  // to work that are not currently selected.
+  for (const { relation } of relationsToFetch) {
+    const ownerProp = relation.ownerProp;
+
+    for (let c = 0, lc = ownerProp.size; c < lc; ++c) {
+      const fullCol = ownerProp.ref(builder, c).fullColumn(builder);
+      const prop = ownerProp.props[c];
+      const col = ownerProp.cols[c];
+
+      if (!builder.hasSelectionAs(fullCol, col) && selectionsToAdd.indexOf(fullCol) === -1) {
+        selectedProps.push(prop);
+        selectionsToAdd.push(fullCol);
+      }
+    }
+  }
+
+  return {
+    selectionsToAdd,
+    selectedProps,
+  };
+}
+
+function getRelatedModels(models, expr) {
+  const allRelated = [];
+
+  for (const model of models) {
+    const related = model[expr.node.$name];
+
+    if (related) {
+      if (Array.isArray(related)) {
+        for (const rel of related) {
+          allRelated.push(rel);
+        }
+      } else {
+        allRelated.push(related);
+      }
+    }
+  }
+
+  return allRelated;
+}
+
+module.exports = {
+  WhereInEagerOperation,
+};

package/lib/queryBuilder/operations/jsonApi/WhereJsonHasOperation.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const postgresJsonApi = require('./postgresJsonApi');
+const mysqlJsonApi = require('./mysqlJsonApi');
+const { getJsonDialect } = require('../../../utils/postgresJsonPath');
+const { ObjectionToKnexConvertingOperation } = require('../ObjectionToKnexConvertingOperation');
+
+// `whereJsonHasAny` and `whereJsonHasAll`.
+class WhereJsonHasOperation extends ObjectionToKnexConvertingOperation {
+  onBuildKnex(knexBuilder, builder) {
+    const dialect = getJsonDialect(builder.unsafeKnex(), this.name);
+    const args = this.getKnexArgs(builder);
+
+    const [sql, bindings] =
+      dialect === 'mysql'
+        ? mysqlJsonApi.whereJsonHas(args[0], this.opt.operator, args[1])
+        : postgresJsonApi.whereJsonFieldRightStringArrayOnLeftQuery(
+            builder.knex(),
+            args[0],
+            this.opt.operator,
+            args[1],
+          );
+
+    if (this.opt.bool === 'or') {
+      knexBuilder = knexBuilder.orWhereRaw(sql, bindings);
+    } else {
+      knexBuilder = knexBuilder.whereRaw(sql, bindings);
+    }
+
+    return knexBuilder;
+  }
+}
+
+module.exports = {
+  WhereJsonHasOperation,
+};

package/lib/queryBuilder/operations/jsonApi/WhereJsonNotObjectOperation.js

@@ -0,0 +1,53 @@
+'use strict';
+
+const jsonApi = require('./postgresJsonApi');
+const mysqlJsonApi = require('./mysqlJsonApi');
+const { getJsonDialect } = require('../../../utils/postgresJsonPath');
+const { ObjectionToKnexConvertingOperation } = require('../ObjectionToKnexConvertingOperation');
+
+// `whereJsonNotArray` and `whereJsonNotObject`.
+class WhereJsonNotObjectOperation extends ObjectionToKnexConvertingOperation {
+  onBuildKnex(knexBuilder, builder) {
+    const dialect = getJsonDialect(builder.unsafeKnex(), this.name);
+    const fieldExpression = this.getKnexArgs(builder)[0];
+
+    if (dialect === 'mysql') {
+      const [sql, bindings] = mysqlJsonApi.whereJsonNotContains(
+        fieldExpression,
+        this.opt.compareValue,
+      );
+
+      return this.opt.bool === 'or'
+        ? knexBuilder.orWhereRaw(sql, bindings)
+        : knexBuilder.whereRaw(sql, bindings);
+    }
+
+    return this.whereJsonNotObject(knexBuilder, builder.knex(), fieldExpression);
+  }
+
+  whereJsonNotObject(knexBuilder, knex, fieldExpression) {
+    const innerQuery = (innerQuery) => {
+      const builder = jsonApi.whereJsonbRefOnLeftJsonbValOrRefOnRight(
+        innerQuery,
+        fieldExpression,
+        '@>',
+        this.opt.compareValue,
+        'not',
+      );
+
+      builder.orWhereRaw(...jsonApi.whereJsonFieldQuery(knex, fieldExpression, 'IS', null));
+    };
+
+    if (this.opt.bool === 'or') {
+      knexBuilder = knexBuilder.orWhere(innerQuery);
+    } else {
+      knexBuilder = knexBuilder.where(innerQuery);
+    }
+
+    return knexBuilder;
+  }
+}
+
+module.exports = {
+  WhereJsonNotObjectOperation,
+};

package/lib/queryBuilder/operations/jsonApi/WhereJsonOperation.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const postgresJsonApi = require('./postgresJsonApi');
+const mysqlJsonApi = require('./mysqlJsonApi');
+const { getJsonDialect } = require('../../../utils/postgresJsonPath');
+const { ObjectionToKnexConvertingOperation } = require('../ObjectionToKnexConvertingOperation');
+
+// `whereJsonSupersetOf`, `whereJsonSubsetOf` and their variants.
+class WhereJsonOperation extends ObjectionToKnexConvertingOperation {
+  onBuildKnex(knexBuilder, builder) {
+    const dialect = getJsonDialect(builder.unsafeKnex(), this.name);
+    const args = this.getKnexArgs(builder);
+    const jsonApi =
+      dialect === 'mysql'
+        ? mysqlJsonApi.whereJsonContains
+        : postgresJsonApi.whereJsonbRefOnLeftJsonbValOrRefOnRightRawQueryParams;
+    const rawArgs = jsonApi(args[0], this.opt.operator, args[1], this.opt.prefix);
+
+    if (this.opt.bool === 'or') {
+      knexBuilder = knexBuilder.orWhereRaw.apply(knexBuilder, rawArgs);
+    } else {
+      knexBuilder = knexBuilder.whereRaw.apply(knexBuilder, rawArgs);
+    }
+
+    return knexBuilder;
+  }
+}
+
+module.exports = {
+  WhereJsonOperation,
+};

package/lib/queryBuilder/operations/jsonApi/WhereJsonOverlapsOperation.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const mysqlJsonApi = require('./mysqlJsonApi');
+const { getJsonDialect } = require('../../../utils/postgresJsonPath');
+const { ObjectionToKnexConvertingOperation } = require('../ObjectionToKnexConvertingOperation');
+
+// `whereJsonOverlaps` and its variants. Postgres has no operator for this.
+class WhereJsonOverlapsOperation extends ObjectionToKnexConvertingOperation {
+  onBuildKnex(knexBuilder, builder) {
+    const knex = builder.unsafeKnex();
+
+    if (knex && getJsonDialect(knex, this.name) !== 'mysql') {
+      throw new Error(`${this.name} is only supported on mysql.`);
+    }
+
+    const args = this.getKnexArgs(builder);
+    const [sql, bindings] = mysqlJsonApi.whereJsonOverlaps(args[0], args[1], this.opt.prefix);
+
+    if (this.opt.bool === 'or') {
+      knexBuilder = knexBuilder.orWhereRaw(sql, bindings);
+    } else {
+      knexBuilder = knexBuilder.whereRaw(sql, bindings);
+    }
+
+    return knexBuilder;
+  }
+}
+
+module.exports = {
+  WhereJsonOverlapsOperation,
+};

package/lib/queryBuilder/operations/jsonApi/mysqlJsonApi.js

@@ -0,0 +1,167 @@
+'use strict';
+
+const { parseFieldExpression } = require('../../../utils/parseFieldExpression');
+const { asArray, isObject, isPlainObject, isString } = require('../../../utils/objectUtils');
+const { toMySqlJsonPath } = require('../../../utils/mysqlJsonPath');
+
+// The MySQL versions of the `whereJson*` methods. They mirror the postgres
+// operators (`@>`, `<@`, `?|`, `?&`) so that the methods find the same rows on
+// both databases.
+//
+// The parts of the queries are `{ sql, bindings, type }` objects. `type` is
+// `'object'` or `'nonObject'` when the JSON type of the part is known.
+
+// Joins sql parts: sql`JSON_CONTAINS(${target}, ${candidate})`.
+function sql(strings, ...parts) {
+  let text = strings[0];
+  const bindings = [];
+
+  parts.forEach((part, i) => {
+    text += part.sql + strings[i + 1];
+    bindings.push(...part.bindings);
+  });
+
+  return { sql: text, bindings };
+}
+
+function literal(text) {
+  return { sql: text, bindings: [] };
+}
+
+// A field expression like `Table.column:a.b[0]`.
+function field(expression) {
+  const parsed = parseFieldExpression(expression);
+
+  if (parsed.access.length === 0) {
+    return { sql: '??', bindings: [parsed.columnName] };
+  } else {
+    return {
+      sql: 'JSON_EXTRACT(??, ?)',
+      bindings: [parsed.columnName, toMySqlJsonPath(parsed.access)],
+    };
+  }
+}
+
+function jsonValue(value) {
+  return {
+    sql: 'CAST(? AS JSON)',
+    bindings: [JSON.stringify(value)],
+    type: isPlainObject(value) ? 'object' : 'nonObject',
+  };
+}
+
+// The right hand side of the methods: a field expression or a JSON value.
+function fieldOrValue(value) {
+  if (isString(value)) {
+    return field(value);
+  } else if (isObject(value)) {
+    return jsonValue(value);
+  } else {
+    throw new Error('Invalid right hand expression.');
+  }
+}
+
+// `target @> candidate`. On postgres an object is only contained in an object, but
+// MySQL's `JSON_CONTAINS()` also finds objects inside arrays.
+function contains(target, candidate) {
+  const containsSql = sql`JSON_CONTAINS(${target}, ${candidate})`;
+  let guard = null;
+
+  if (candidate.type === 'nonObject' || target.type === 'object') {
+    guard = null;
+  } else if (candidate.type === 'object') {
+    guard = target.type === 'nonObject' ? literal('FALSE') : sql`JSON_TYPE(${target}) = 'OBJECT'`;
+  } else if (target.type === 'nonObject') {
+    guard = sql`JSON_TYPE(${candidate}) <> 'OBJECT'`;
+  } else {
+    guard = sql`(JSON_TYPE(${candidate}) <> 'OBJECT' OR JSON_TYPE(${target}) = 'OBJECT')`;
+  }
+
+  return guard ? sql`(${containsSql} AND ${guard})` : containsSql;
+}
+
+function not(part, prefix) {
+  return prefix === 'not' ? sql`NOT ${part}` : part;
+}
+
+function toRawArgs(part) {
+  return [part.sql, part.bindings];
+}
+
+// `whereJsonSupersetOf` (`@>`) and `whereJsonSubsetOf` (`<@`).
+function whereJsonContains(fieldExpression, operator, jsonObjectOrFieldExpression, prefix) {
+  const left = field(fieldExpression);
+  const right = fieldOrValue(jsonObjectOrFieldExpression);
+  const condition = operator === '@>' ? contains(left, right) : contains(right, left);
+
+  return toRawArgs(not(condition, prefix));
+}
+
+// `whereJsonNotArray` and `whereJsonNotObject`. Like on postgres, NULL is neither
+// an array nor an object.
+function whereJsonNotContains(fieldExpression, compareValue) {
+  const left = field(fieldExpression);
+  return toRawArgs(sql`(NOT ${contains(left, jsonValue(compareValue))} OR ${left} IS NULL)`);
+}
+
+// `whereJsonHasAny` (`?|`) and `whereJsonHasAll` (`?&`). Like on postgres, the keys
+// are looked for in the keys of objects, in the string items of arrays and in
+// strings.
+function whereJsonHas(fieldExpression, operator, keys) {
+  const left = field(fieldExpression);
+  const all = operator === '?&';
+
+  keys = asArray(keys);
+
+  for (const key of keys) {
+    if (!isString(key)) {
+      throw new Error('All keys to find must be strings.');
+    }
+  }
+
+  if (keys.length === 0) {
+    return toRawArgs(all ? sql`${left} IS NOT NULL` : literal('FALSE'));
+  }
+
+  const paths = {
+    sql: keys.map(() => '?').join(', '),
+    bindings: keys.map((key) => toMySqlJsonPath([{ type: 'object', ref: key }])),
+  };
+
+  const keysJson = jsonValue(keys);
+  const stringChecks = keys.map((key) => ({
+    sql: `JSON_UNQUOTE(${left.sql}) = ?`,
+    bindings: [...left.bindings, key],
+  }));
+
+  const stringCheck = {
+    sql: `(${stringChecks.map((it) => it.sql).join(all ? ' AND ' : ' OR ')})`,
+    bindings: stringChecks.flatMap((it) => it.bindings),
+  };
+
+  const objectCheck = sql`JSON_CONTAINS_PATH(${left}, ${literal(all ? "'all'" : "'one'")}, ${paths})`;
+  const arrayCheck = all
+    ? sql`JSON_CONTAINS(${left}, ${keysJson})`
+    : sql`JSON_OVERLAPS(${left}, ${keysJson})`;
+
+  return toRawArgs(
+    sql`(CASE JSON_TYPE(${left}) WHEN 'OBJECT' THEN ${objectCheck} WHEN 'ARRAY' THEN ${arrayCheck} WHEN 'STRING' THEN ${stringCheck} ELSE FALSE END)`,
+  );
+}
+
+// `whereJsonOverlaps`. A NULL or missing field doesn't overlap anything, so
+// `whereJsonNotOverlaps` finds it.
+function whereJsonOverlaps(fieldExpression, jsonObjectOrFieldExpression, prefix) {
+  const overlaps = sql`JSON_OVERLAPS(${field(fieldExpression)}, ${fieldOrValue(
+    jsonObjectOrFieldExpression,
+  )})`;
+
+  return toRawArgs(prefix === 'not' ? sql`NOT COALESCE(${overlaps}, FALSE)` : overlaps);
+}
+
+module.exports = {
+  whereJsonContains,
+  whereJsonNotContains,
+  whereJsonHas,
+  whereJsonOverlaps,
+};