objection 4.0.0-rc.2 → 4.0.0-rc.4

package/lib/queryBuilder/graph/GraphWriter.js

@@ -2,19 +2,21 @@ import { getModel } from '../../model/getModel.js';
 import { restoreOmitFromDatabaseJson } from '../../model/modelUtils.js';
 import { valuesKey } from '../../model/modelValues.js';
 import { UpdateOperation } from '../operations/UpdateOperation.js';
+import { removeFieldExpressions } from '../operations/fieldExpressions.js';
 import { getMaxRows, hasBatchInsert } from '../../utils/knexUtils.js';
 import { isInternalProp } from '../../utils/internalPropUtils.js';
-import { chunk, difference, groupBy, isObject, nonStrictEquals } from '../../utils/objectUtils.js';
-import { map as promiseMap } from '../../utils/promiseUtils/index.js';
 import {
-  clearKeysOfRemovedRows,
-  fetchGeneratedKeys,
-  getChangedKeysOfRelatedRows,
-  resolveDependencies,
-  resolveFilledDependencies,
-  resolveReference,
-  setKeyArrays,
-} from './graphKeys.js';
+  chunk,
+  difference,
+  get,
+  groupBy,
+  isObject,
+  jsonEquals,
+  nonStrictEquals,
+  set,
+} from '../../utils/objectUtils.js';
+import { map as promiseMap } from '../../utils/promiseUtils/index.js';
+import { hasOwnerKey, isOtherKey } from './graphKeys.js';
 
 const MAX_INSERT_BATCH_SIZE = 100;
 const MAX_CONCURRENCY = 100;
@@ -94,7 +96,9 @@ export class GraphWriter {
           // Rows that reference their owners are found through the relation.
           // BelongsToOne rows aren't referenced anymore once their owners are
           // patched.
-          const owners = [...new Set(groupNodes.map((node) => node.parent.obj))];
+          const owners = [...new Set(groupNodes.map((node) => node.parent))].map((parent) =>
+            getRelationOwner(this.schedule.keys, parent.node, parent.obj, relation),
+          );
           const query = (
             areReferenced
               ? relation.ownerModelClass.relatedQuery(relation.name).for(owners)
@@ -122,14 +126,15 @@ export class GraphWriter {
     await promiseMap(batches, (batch) => this._insertBatch(builder, batch), {
       concurrency: getConcurrency(builder, nodes),
     });
-    await this._patch(
-      builder,
-      nodes.filter((node) => this.schedule.slotMovedNodes.has(node)),
-    );
 
     for (const node of references) {
       resolveDependencies(this.schedule, node);
     }
+
+    await this._patch(
+      builder,
+      [...nodes, ...references].filter((node) => this.schedule.earlyNodes.has(node)),
+    );
   }
 
   async _insertBatch(builder, nodes) {
@@ -272,6 +277,8 @@ export class GraphWriter {
     } finally {
       for (const [node, props] of omittedProps) {
         restoreOmitFromDatabaseJson(node.obj, props);
+        // The keys of `setKeyPaths()` stay where a hook resolves the patch:
+        removeFieldExpressions(node.modelClass, node.obj);
       }
     }
   }
@@ -281,6 +288,12 @@ export class GraphWriter {
   _preparePatch(builder, node) {
     const { referencedNode } = node;
 
+    // A row that the graph relates by its key alone isn't looked up, its key is
+    // only written to its owner:
+    if (this.plan.isRelatedByKey(node)) {
+      return null;
+    }
+
     // The props that the graph fills into the model of a `#ref` model are only
     // final now:
     if (referencedNode) {
@@ -301,6 +314,8 @@ export class GraphWriter {
         ? node.relation.relatedProp.props
         : []),
     ];
+    // The keys in json paths that the write sets or clears:
+    let pathKeys = new Map();
     // Changing the id never makes sense with `upsertGraph()`. A many-to-many
     // `#ref` model whose model writes its row only writes its join row:
     const idProps = node.modelClass.getIdPropertyArray();
@@ -308,11 +323,15 @@ export class GraphWriter {
       !!referencedNode && node.relation.keySide === 'join' && this.plan.isCoveredByModel(node);
     const isWritable = (prop) =>
       !idProps.includes(prop) &&
+      !pathKeys.has(prop) &&
       !this.schedule.keys.isOnlyRead(node, prop) &&
       (!isJoinRowOnly || node.relation.joinTableExtras.some((extra) => extra.aliasProp === prop));
 
     const getPropsToUpdate = (changedProps) =>
-      (shouldPatch || shouldUpdate ? changedProps : keyProps).filter(isWritable);
+      (shouldPatch || shouldUpdate
+        ? changedProps
+        : keyProps.flatMap((prop) => pathKeys.get(prop) || prop)
+      ).filter(isWritable);
 
     // Leave out the props that don't change, and return true if any do.
     const handleUpdate = ({ changedProps, unchangedProps } = this._findChanges(node)) => {
@@ -332,21 +351,43 @@ export class GraphWriter {
     // what to update. Otherwise, the hook may still change the model, and we
     // check again after it runs, but only if the model has props to update, as
     // relates are no updates (#2233, #2605).
-    const changes = this._findChanges(node);
-    const propsToUpdate = getPropsToUpdate(changes.changedProps);
     const hasBeforeUpdate = hasHook(node.obj, 'beforeUpdate');
 
-    if (propsToUpdate.length === 0) {
+    let changes = this._findChanges(node);
+
+    if (getPropsToUpdate(changes.changedProps).length === 0) {
       return null;
-    } else if (!hasBeforeUpdate) {
-      handleUpdate(changes);
     }
 
     node.obj.$validate(null, {
       dataPath: node.dataPathKey,
-      patch: shouldPatch || !shouldUpdate,
+      patch: !shouldUpdate,
     });
 
+    // Write the keys in json columns as path updates where their column holds
+    // nothing else, which keeps its other values (#415). After the validation,
+    // as `update()` validates them as properties of their own:
+    pathKeys = setKeyPaths(node, [
+      ...this.schedule.keys.getSetCells(node),
+      ...this.plan
+        .getRemovedOwnerRelations(node)
+        .flatMap(({ ownerProp }) =>
+          ownerProp.props.map((_, index) => ({ prop: ownerProp, index })),
+        ),
+    ]);
+
+    if (pathKeys.size > 0) {
+      changes = this._findChanges(node);
+    }
+
+    const propsToUpdate = getPropsToUpdate(changes.changedProps);
+
+    if (propsToUpdate.length === 0) {
+      return null;
+    } else if (!hasBeforeUpdate) {
+      handleUpdate(changes);
+    }
+
     // Rows that their owners relate by their keys can't be found through the
     // relation until the owners are patched, so they're patched directly.
     const { relation } = node;
@@ -357,23 +398,35 @@ export class GraphWriter {
 
     return {
       node,
-      shouldPatch,
+      shouldUpdate,
       hasBeforeUpdate,
       handleUpdate,
       propsToUpdate,
+      fieldExpressionKeys: [...pathKeys.values()].flat(),
       isPatchedThroughRelation,
     };
   }
 
   _writePatch(
     builder,
-    { node, shouldPatch, hasBeforeUpdate, handleUpdate, propsToUpdate, isPatchedThroughRelation },
+    {
+      node,
+      shouldUpdate,
+      hasBeforeUpdate,
+      handleUpdate,
+      propsToUpdate,
+      fieldExpressionKeys,
+      isPatchedThroughRelation,
+    },
   ) {
     const query = (
       isPatchedThroughRelation
-        ? // Find the row through the keys of the owner's current row, as the
-          // graph may leave them out of the owner:
-          (node.parent.existing || node.parent).obj
+        ? getRelationOwner(
+            this.schedule.keys,
+            node.parent,
+            (node.parent.existing || node.parent).obj,
+            node.relation,
+          )
             .$relatedQuery(node.relation.name)
             .findById(node.obj.$id())
         : (node.current?.obj || node.obj).$query()
@@ -386,6 +439,8 @@ export class GraphWriter {
         // Join rows are patched with the extras among the props to update. The
         // extras of new join rows are written with them, see `_findChanges()`:
         joinTableExtras: node.relation?.keySide === 'join' && !node.current ? [] : propsToUpdate,
+        // The keys in json columns, also in updates:
+        fieldExpressionKeys,
       })
       .copyFrom(builder, isReturningAll);
 
@@ -399,7 +454,8 @@ export class GraphWriter {
       });
     }
 
-    return (shouldPatch ? query.patch(node.obj) : query.update(node.obj))
+    // Writes of only the keys under `noUpdate` are patches too:
+    return (shouldUpdate ? query.update(node.obj) : query.patch(node.obj))
       .runAfter(async (result, query) => {
         if (isObject(result) && result.$isObjectionModel) {
           // Handle `returning('*')`.
@@ -484,6 +540,26 @@ async function throwForMissingRow(builder, node) {
   }
 }
 
+// Return the owner to find the rows of `relation` through: the current row
+// `obj` of the owner, as the graph may leave its keys out, with the final keys
+// of `node` if they change, as the relation was read at them.
+function getRelationOwner(keys, node, obj, relation) {
+  const { ownerProp } = relation;
+  const values = keys.getReadProps(node, ownerProp);
+
+  if (isOtherKey(values, ownerProp.getProps(obj))) {
+    const owner = obj.$clone({ shallow: true });
+    values.forEach((value, i) => {
+      if (value !== undefined) {
+        ownerProp.setProp(owner, i, value);
+      }
+    });
+    return owner;
+  } else {
+    return obj;
+  }
+}
+
 function createPatchNotFoundError(builder, node) {
   return node.modelClass.createNotFoundError(builder.context(), {
     message: `model (id=${node.obj.$id()}) was not found when updating it. An earlier write of the graph or a concurrent query may have deleted it`,
@@ -506,3 +582,254 @@ function getConcurrency(builder, nodes) {
 function isReturningAll(operation) {
   return operation.name === 'returning' && operation.args.includes('*');
 }
+
+// Copy the keys and `#ref{}` values that `node` waits for into its model.
+function resolveDependencies(schedule, node) {
+  for (const dependency of schedule.dependencies.get(node) || []) {
+    if (!schedule.deferredDependencies.has(dependency)) {
+      if (dependency.relation) {
+        // Hooks see the props of the models that `#ref` models refer to:
+        resolveReference(dependency.related);
+        resolveReference(dependency.dependent);
+      } else if (dependency.reference) {
+        resolvePropertyReference(dependency.dependent, dependency.reference);
+      }
+    }
+  }
+
+  schedule.keys.resolvedNodes.add(node);
+  schedule.keys.fill(node);
+}
+
+// Copy the keys and `#ref{}` values that `node` waits for into its model,
+// after those that the graph fills into the models that it reads them from,
+// e.g. the new key of an owner whose BelongsToOne row is replaced. Rows that
+// move first only wait for rows that exist, see `GraphSchedule#_waitsForWrites()`.
+function resolveFilledDependencies(schedule, node, visitedNodes = new Set()) {
+  visitedNodes.add(node);
+
+  for (const dependency of schedule.dependencies.get(node) || []) {
+    const referencedNode = dependency.node.referencedNode || dependency.node;
+
+    if (
+      !visitedNodes.has(referencedNode) &&
+      (dependency.isKeyWait || schedule.readsFilledProps(dependency))
+    ) {
+      resolveFilledDependencies(schedule, referencedNode, visitedNodes);
+    }
+  }
+
+  resolveDependencies(schedule, node);
+}
+
+// Change the arrays of keys that `node` holds for the relations that the graph
+// gives it: Remove the keys of the current related rows that it deletes or
+// unrelates, and add the keys of the related models that it writes. Other
+// keys stay, e.g. of rows that the relation's filter leaves out. Relations
+// that the graph leaves out keep their arrays (#1202). Return the props that
+// change.
+function setKeyArrays(plan, node) {
+  const { actions, graphOptions } = plan;
+  const { existing } = node;
+  const props = [];
+
+  for (const relation of node.relations) {
+    const { name } = relation;
+
+    if (hasOwnerKeyArray(relation)) {
+      const { ownerProp, relatedProp } = relation;
+      const getKey = (related) => relatedProp.getProp(related.obj, 0);
+      const keys = relation.getKeys(existing ? existing.obj : node.obj);
+      const removed = (existing?.activeChildren || []).filter(
+        (child) =>
+          child.relation.name === name && ['delete', 'unrelate'].includes(actions.get(child)),
+      );
+      const added = node.activeChildren.filter(
+        (child) =>
+          child.relation.name === name &&
+          !child.toBeUnrelatedOrDeleted &&
+          actions.get(child) !== 'ignore',
+      );
+
+      for (const related of added) {
+        resolveReference(related);
+      }
+
+      const newKeys = relation.changeKeys(keys, {
+        add: added.map(getKey),
+        remove: removed.map(getKey),
+      });
+
+      if (!existing || graphOptions.shouldUpdate(node)) {
+        // Inserts and updates write the whole row:
+        ownerProp.setProp(node.obj, 0, newKeys);
+        props.push(ownerProp.props[0]);
+      } else if (!jsonEquals(keys, newKeys)) {
+        const patch = relation.createKeysPatch(existing.obj, newKeys);
+        Object.assign(node.obj, patch);
+        props.push(...Object.keys(patch));
+      }
+    }
+  }
+
+  return props;
+}
+
+// Return the keys of `node` that point to other BelongsToOne rows than before,
+// which the graph inserts or relates, and which `GraphKeys` filled in for their
+// places.
+function getChangedKeysOfRelatedRows(plan, keys, node) {
+  const { actions } = plan;
+  const props = [];
+
+  for (const child of node.activeChildren) {
+    const { relation } = child;
+
+    // Ignored models don't replace the current one, and models marked with
+    // `#unrelate` or `#delete` are handled by `clearKeysOfRemovedRows()`:
+    if (hasOwnerKey(relation) && actions.get(child) !== 'ignore') {
+      relation.ownerProp.forEach((i) => {
+        const currentValue = node.existing && relation.ownerProp.getProp(node.existing.obj, i);
+        const value = relation.ownerProp.getProp(node.obj, i);
+
+        if (
+          value !== undefined &&
+          currentValue != value &&
+          keys.isSet(node, relation.ownerProp, i)
+        ) {
+          props.push(relation.ownerProp.props[i]);
+        }
+      });
+    }
+  }
+
+  return props;
+}
+
+// Clear the keys of `node` that point to BelongsToOne rows that are removed,
+// and return the props that change.
+function clearKeysOfRemovedRows(plan, node) {
+  const props = [];
+
+  for (const relation of plan.getRemovedOwnerRelations(node)) {
+    relation.ownerProp.forEach((i) => {
+      if (relation.ownerProp.getProp(node.existing.obj, i) != null) {
+        relation.ownerProp.setProp(node.obj, i, null);
+        props.push(relation.ownerProp.props[i]);
+      }
+    });
+  }
+
+  return props;
+}
+
+// Give a `#ref` model the props of the model it refers to that it doesn't have,
+// and the `filledProps` again, as they may have changed since. Its own keys
+// stay its own, and the join table extras of the place of the model belong to
+// the join row there.
+function resolveReference(node, filledProps = []) {
+  const { referencedNode, relation } = node;
+
+  if (referencedNode) {
+    const ownKeys = relation?.keySide === 'related' ? relation.relatedProp.props : [];
+    const modelExtras =
+      referencedNode.relation?.keySide === 'join'
+        ? referencedNode.relation.joinTableExtras.map((extra) => extra.aliasProp)
+        : [];
+
+    for (const prop of difference(Object.keys(referencedNode.obj), modelExtras)) {
+      if (
+        !Object.hasOwn(node.obj, prop) ||
+        (filledProps.includes(prop) && !ownKeys.includes(prop))
+      ) {
+        node.obj[prop] = referencedNode.obj[prop];
+      }
+    }
+  }
+}
+
+// Replace a `#ref{}` in a prop of `node` with the value it refers to:
+function resolvePropertyReference(node, { path, match, refPath, isWholeValue, referencedNode }) {
+  const value = get(referencedNode.obj, refPath);
+  set(node.obj, path, isWholeValue ? value : get(node.obj, path).replace(match, value));
+}
+
+// Fetch the keys that the relations of the inserted nodes need, but that the
+// database generated and the insert didn't return, like a uuid that isn't the
+// id (Vincit#853).
+async function fetchGeneratedKeys(builder, modelClass, nodes) {
+  const keyProps = (node) => [
+    ...(node.relation && node.relation.keySide !== 'related' ? [node.relation.relatedProp] : []),
+    ...node.activeChildren
+      .filter((child) => child.relation.keySide !== 'owner')
+      .map((child) => child.relation.ownerProp),
+  ];
+  const missingNodes = nodes.filter((node) =>
+    keyProps(node).some((prop) => prop.getProps(node.obj).some((value) => value === undefined)),
+  );
+
+  if (missingNodes.length > 0 && missingNodes.every((node) => node.obj.$hasId())) {
+    // SQL Server returns columns selected twice as arrays, so the ids are
+    // part of the set:
+    const columns = new Set([
+      ...modelClass.getIdColumnArray(),
+      ...missingNodes.flatMap((node) => keyProps(node)).flatMap((prop) => prop.cols),
+    ]);
+    const rows = await modelClass
+      .query()
+      .childQueryOf(builder, { fork: true, isInternalQuery: true })
+      .findByIds(missingNodes.map((node) => node.obj.$id()))
+      .select([...columns]);
+    const rowsByKey = new Map(rows.map((row) => [row.$idKey(), row]));
+
+    for (const node of missingNodes) {
+      const row = rowsByKey.get(node.obj.$idKey());
+
+      for (const prop of keyProps(node)) {
+        prop.forEach((i) => {
+          if (prop.getProp(node.obj, i) === undefined) {
+            prop.setProp(node.obj, i, prop.getProp(row, i));
+          }
+        });
+      }
+    }
+  }
+}
+
+// Add the changed keys of `cells` at the top level of json columns to `node` as
+// field expressions where their column holds nothing but these keys, and return
+// them by column. PostgreSQL and MySQL can't write nested paths whose parents
+// are missing, so their columns are written whole.
+function setKeyPaths(node, cells) {
+  const keysByColumn = new Map();
+  const pathCells = cells.filter(({ prop, index }) => prop.isJsonKey(index));
+
+  for (const [column, columnCells] of groupBy(pathCells, ({ prop, index }) => prop.props[index])) {
+    const keys = {};
+
+    for (const { prop, index } of columnCells) {
+      prop.setProp(keys, index, prop.getProp(node.obj, index));
+    }
+
+    if (jsonEquals(node.obj[column], keys[column])) {
+      const patch = {};
+
+      for (const { prop, index } of columnCells) {
+        const value = prop.getProp(node.obj, index);
+
+        if (!node.current || !nonStrictEquals(prop.getProp(node.current.obj, index), value)) {
+          prop.patch(patch, index, value);
+        }
+      }
+
+      Object.assign(node.obj, patch);
+      keysByColumn.set(column, Object.keys(patch));
+    }
+  }
+
+  return keysByColumn;
+}
+
+function hasOwnerKeyArray(relation) {
+  return relation.keySide === 'owner' && !relation.isOneToOne();
+}