objection 4.0.0-rc.1 → 4.0.0-rc.3

package/lib/queryBuilder/graph/GraphLoader.js

@@ -1,57 +1,57 @@
-import { asArray, chunk, groupBy, uniqBy } from '../../utils/objectUtils.js';
+import { asArray, chunk, groupBy } from '../../utils/objectUtils.js';
 import { getMaxRows } from '../../utils/knexUtils.js';
 import { FetchStrategy } from './GraphOptions.js';
+import { isOtherKey } from './graphKeys.js';
 import { GraphNode } from './GraphNode.js';
 import { RelationExpression } from '../RelationExpression.js';
 import { getDeclaredColumns } from '../operations/fieldExpressions.js';
 import { getFieldExpressionColumn } from '../../utils/parseFieldExpression.js';
 
 // Fetch the current state of the graph from the database and pair its rows with
-// the models of the graph: First the roots of the pass with their relations,
-// then the rows of the models that the graph relates from elsewhere, with the
-// relations that the graph gives them, level by level, as the rows of related
-// models decide which models below them are related too. Models that
-// `insertMissing` inserts if they don't exist are looked up too. The rows that
-// a relate moves into a HasManyRelation or HasOneRelation are fetched without
-// their relations, which change with the move. Return the ones that the graph
-// gives relations, to be upserted in a pass of their own after the move.
-export async function loadCurrentGraph(builder, plan) {
-  const passNodes = new Set(plan.nodes);
+// the models of the graph, from scratch: First the roots, then the rows of the
+// models that the graph relates from elsewhere, each with the relations that
+// the graph gives them, level by level, as the rows of related models decide
+// which models below them are related too. Models that `insertMissing` inserts
+// if they don't exist are looked up too. The relations are read at their
+// current keys, or at the ones of `readKeys`, see `getReadKeys()`.
+export async function loadCurrentGraph(builder, plan, readKeys = null) {
+  const { graphOptions } = plan;
   const fetchedNodes = new Set();
-  const movedNodes = [];
+  plan.reset();
 
   const fetch = async (nodes, { areRelated }) => {
     for (const node of nodes) {
       fetchedNodes.add(node);
     }
 
-    for (const [isMovedRow, groupNodes] of groupBy(nodes, (node) => areRelated && isMoved(node))) {
-      for (const nodesOfClass of groupBy(groupNodes, (node) => node.modelClass).values()) {
-        const models = await fetchModels(builder, plan.graphOptions, nodesOfClass, {
-          withRelations: !isMovedRow,
-        });
-        const modelsByKey = new Map(models.map((model) => [model.$idKey(), model]));
-
-        for (const node of nodesOfClass) {
-          const model = modelsByKey.get(node.obj.$idKey());
-
-          if (model && isMovedRow && hasRelations(node.obj)) {
-            movedNodes.push(node);
-          } else if (model) {
-            const currentNodes = GraphNode.buildGraph(node.modelClass, model, node.relationPath);
-            const [currentNode] = currentNodes;
-            plan.currentNodes.push(...currentNodes);
-
-            if (areRelated) {
-              node.row = currentNode;
-              currentNode.node = node;
-              pairChildren(node, currentNode);
-            } else {
-              pair(node, currentNode);
-            }
-          } else if (areRelated) {
-            node.isMissing = true;
-          }
+    for (const nodesOfClass of groupBy(nodes, (node) => node.modelClass).values()) {
+      const models = await fetchModels(builder, graphOptions, nodesOfClass);
+      const modelsByKey = new Map(models.map((model) => [model.$idKey(), model]));
+      const entries = [];
+
+      for (const node of nodesOfClass) {
+        const model = modelsByKey.get(node.obj.$idKey());
+
+        if (model) {
+          entries.push([node, model]);
+        } else if (areRelated) {
+          node.isMissing = true;
+        }
+      }
+
+      await readRelations(builder, graphOptions, entries, readKeys);
+
+      for (const [node, model] of entries) {
+        const currentNodes = GraphNode.buildGraph(node.modelClass, model, node.relationPath);
+        const [currentNode] = currentNodes;
+        plan.currentNodes.push(...currentNodes);
+
+        if (areRelated) {
+          node.row = currentNode;
+          currentNode.node = node;
+          pairChildren(node, currentNode);
+        } else {
+          pair(node, currentNode);
         }
       }
     }
@@ -68,13 +68,14 @@ export async function loadCurrentGraph(builder, plan) {
         !fetchedNodes.has(node) &&
         !node.current &&
         node.obj.$hasId() &&
+        !hasIdReference(node) &&
         plan.actionFor(node) === 'relate' &&
         isFetched(node, rowKeys),
     );
     const nodeSet = new Set(nodes);
 
     return nodes.filter((node) => {
-      for (let parent = node.parent; passNodes.has(parent); parent = parent.parent) {
+      for (let parent = node.parent; parent; parent = parent.parent) {
         if (nodeSet.has(parent) || !plan.writesBranchOf(parent)) {
           return false;
         }
@@ -84,16 +85,17 @@ export async function loadCurrentGraph(builder, plan) {
     });
   };
 
+  // The roots are at their places:
   await fetch(
-    plan.nodes.filter((node) => !passNodes.has(node.parent) && node.hasId),
+    plan.nodes.filter((node) => !node.parent && node.hasId && !hasIdReference(node)),
     { areRelated: false },
   );
 
   // The rows of the related models that the graph gives relations, or inserts
   // with `insertMissing` if they don't exist:
   const hasRelatedRow = (node, rowKeys) =>
-    hasRelations(node.obj) ||
-    (plan.graphOptions.has(node, 'insertMissing') && !rowKeys.has(node.rowKey));
+    node.relations.length > 0 ||
+    (graphOptions.has(node, 'insertMissing') && !rowKeys.has(node.rowKey));
 
   for (
     let nodes = findRelatedNodes(hasRelatedRow);
@@ -109,17 +111,100 @@ export async function loadCurrentGraph(builder, plan) {
   const nodes = findRelatedNodes((node, rowKeys) => !node.isReference && !rowKeys.has(node.rowKey));
 
   for (const nodesOfClass of groupBy(nodes, (node) => node.modelClass).values()) {
-    const models = await fetchModels(builder, plan.graphOptions, nodesOfClass, {
-      onlyIds: true,
-    });
+    const models = await fetchModels(builder, graphOptions, nodesOfClass, { onlyIds: true });
     const idKeys = new Set(models.map((model) => model.$idKey()));
 
     for (const node of nodesOfClass) {
       node.isMissing = !idKeys.has(node.obj.$idKey());
     }
   }
+}
 
-  return movedNodes;
+// Return the keys at which the relations of the rows whose keys change are read
+// once the graph is planned, by node and relation, or `null` if all are read at
+// their current keys: where the rows that give the final keys are now, see
+// `GraphKeys.getReadProps()`. A BelongsToOne relation is read at its final key
+// only where another place moves the row on its column; otherwise its row is
+// replaced.
+export function getReadKeys(plan, keys) {
+  const readKeys = new Map();
+  let isChanged = false;
+
+  for (const node of plan.activeNodes) {
+    if (node.existing) {
+      for (const relation of node.relations) {
+        if (relation.keySide !== 'owner' || keys.isSetByOtherPlaces(node, relation)) {
+          const values = keys.getReadProps(node, relation.ownerProp);
+          const nodeKeys = readKeys.get(node) || new Map();
+          readKeys.set(node, nodeKeys.set(relation, values));
+          isChanged ||= isOtherKey(values, relation.ownerProp.getProps(node.existing.obj));
+        }
+      }
+    }
+  }
+
+  return isChanged ? readKeys : null;
+}
+
+// Read the relations that the graph gives the models of `entries`, `[node,
+// model]` pairs, into the models, level by level, at the keys of `readKeys` or
+// at their current ones. The rows of a model class are read together, with the
+// relations of all their places.
+async function readRelations(builder, graphOptions, entries, readKeys) {
+  for (let level = entries; level.length > 0;) {
+    const nextLevel = [];
+
+    for (const levelEntries of groupBy(level, ([node]) => node.modelClass).values()) {
+      const [[{ modelClass }]] = levelEntries;
+      const nodes = levelEntries.map(([node]) => node);
+      const relations = [...new Set(nodes.flatMap((node) => node.relations))];
+      const models = [...new Set(levelEntries.map(([, model]) => model))];
+      // Set the read keys, and keep the current ones to put back:
+      const currentKeys = new Map(models.map((model) => [model, {}]));
+
+      for (const [node, model] of levelEntries) {
+        for (const [relation, values] of readKeys?.get(node) || []) {
+          relation.ownerProp.forEach((i) => {
+            const keys = currentKeys.get(model);
+            const prop = relation.ownerProp.props[i];
+            // Relations on the same column keep its first, current value:
+            if (!(prop in keys)) {
+              keys[prop] = relation.ownerProp.getProp(model, i);
+            }
+
+            relation.ownerProp.setProp(model, i, values[i] ?? null);
+          });
+        }
+      }
+
+      if (relations.length > 0) {
+        const expression = RelationExpression.create(
+          `[${relations.map((relation) => relation.name).join(', ')}]`,
+        );
+        const fetched = await createFetchQuery(builder, graphOptions, modelClass, nodes, expression)
+          .resolve(models)
+          .findOptions({ dontCallFindHooks: true });
+
+        models.forEach((model, i) => Object.assign(model, fetched[i], currentKeys.get(model)));
+      }
+
+      // The related rows that the graph gives relations are read next:
+      for (const [node, model] of levelEntries) {
+        for (const child of node.children) {
+          if (child.obj.$hasId() && child.relations.length > 0) {
+            const key = child.obj.$idKey();
+            const row = asArray(model[child.relation.name]).find((row) => row?.$idKey() === key);
+
+            if (row) {
+              nextLevel.push([child, row]);
+            }
+          }
+        }
+      }
+    }
+
+    level = nextLevel;
+  }
 }
 
 function pair(node, currentNode) {
@@ -133,72 +218,77 @@ function pair(node, currentNode) {
 // them pair with the same model.
 function pairChildren(node, currentNode) {
   const childrenByRelation = groupBy(currentNode.children, (child) => child.relation.name);
+  // The current rows of a relation by id, once the graph pairs models in it:
+  const childrenByKey = new Map();
+  const getChildrenByKey = (name) => {
+    let children = childrenByKey.get(name);
+
+    if (!children) {
+      children = groupBy(childrenByRelation.get(name) || [], (child) => child.obj.$idKey());
+      childrenByKey.set(name, children);
+    }
+
+    return children;
+  };
 
   for (const child of node.children) {
     if (child.obj.$hasId()) {
-      const key = child.obj.$idKey();
+      const currentChildren = getChildrenByKey(child.relation.name).get(child.obj.$idKey());
 
-      for (const currentChild of childrenByRelation.get(child.relation.name) || []) {
-        if (currentChild.obj.$idKey() === key) {
-          pair(child, currentChild);
-        }
+      for (const currentChild of currentChildren || []) {
+        pair(child, currentChild);
       }
     }
   }
 }
 
-function isMoved(node) {
-  return node.relation.keySide === 'related';
+// Return true if an id of `node` is a `#ref{}`, which is only known once the
+// model that it refers to is written, so that its row can't be looked up.
+function hasIdReference(node) {
+  const idProps = node.modelClass.getIdPropertyArray();
+  return node.propertyReferences.some(({ path }) => idProps.includes(path[0]));
 }
 
-function hasRelations(obj) {
-  return obj.constructor.getRelationNames().some((name) => obj[name] !== undefined);
-}
-
-// Fetch the rows of `nodes`, with their relations unless they're moved, or only
-// their ids, in as many queries as the number of their ids requires.
-async function fetchModels(builder, graphOptions, nodes, options) {
+// Fetch the rows of `nodes`, or only their ids, in as many queries as the
+// number of their ids requires, each row once for all its places.
+async function fetchModels(builder, graphOptions, nodes, { onlyIds = false } = {}) {
   const [{ modelClass }] = nodes;
-  const uniqueNodes = uniqBy(nodes, (node) => node.obj.$idKey());
+  const nodesByRow = [...groupBy(nodes, (node) => node.obj.$idKey()).values()];
   const maxIds = getMaxRows(builder.unsafeKnex(), modelClass.getIdColumnArray().length);
+  const expression = RelationExpression.create('[]');
   const models = [];
 
-  for (const chunkNodes of chunk(uniqueNodes, maxIds)) {
-    models.push(...(await fetchModelsOfNodes(builder, graphOptions, chunkNodes, options)));
+  for (const chunkNodes of chunk(nodesByRow, maxIds)) {
+    models.push(
+      ...(await createFetchQuery(
+        builder,
+        graphOptions,
+        modelClass,
+        chunkNodes.flat(),
+        expression,
+        onlyIds,
+      ).findByIds(chunkNodes.map(([node]) => node.obj.$id()))),
+    );
   }
 
   return models;
 }
 
-function fetchModelsOfNodes(
-  builder,
-  graphOptions,
-  nodes,
-  { withRelations = false, onlyIds = false },
-) {
-  const [{ modelClass }] = nodes;
-  const ids = nodes.map((node) => node.obj.$id());
-  const expression = RelationExpression.fromModelGraph(
-    withRelations ? nodes.map((node) => node.obj) : [],
-  );
-
+// Create the query that fetches the rows of `nodes` with the relations of
+// `expression`, locked like the upsert query asks for, and with the columns
+// that the fetch strategy selects for them and their related models.
+function createFetchQuery(builder, graphOptions, modelClass, nodes, expression, onlyIds = false) {
   return modelClass
     .query()
     .childQueryOf(builder, { fork: true, isInternalQuery: true })
     .modify((fetchBuilder) => {
-      // Lock the fetched rows like the upsert query asks for.
       for (const method of ['forUpdate', 'forShare', 'forNoKeyUpdate', 'forKeyShare']) {
         if (builder.has(method)) {
           fetchBuilder[method]();
         }
       }
-    })
-    .modify((fetchBuilder) => {
-      const nodesByRelationPath = withRelations
-        ? getNodesByRelationPath(nodes, expression)
-        : new Map([['', nodes]]);
 
-      for (const [relationPath, pathNodes] of nodesByRelationPath) {
+      for (const [relationPath, pathNodes] of getNodesByRelationPath(nodes, expression)) {
         const select = onlyIds
           ? (builder) => builder.select(builder.fullIdColumn())
           : createSelect(pathNodes, graphOptions, !relationPath);
@@ -210,52 +300,39 @@ function fetchModelsOfNodes(
         }
       }
     })
-    .findByIds(ids)
     .withGraphFetched(expression)
     .internalOptions({ keepImplicitJoinProps: true });
 }
 
 // Group the fetched nodes and their descendants by their relation paths below
-// the fetched nodes. Relations with nulls or empty arrays have no nodes, but
-// are fetched too.
+// the fetched nodes, for the paths of `expression`. Relations with nulls or
+// empty arrays have no nodes, but are fetched too.
 function getNodesByRelationPath(nodes, expression) {
-  const nodesByRelationPath = new Map();
-
-  for (const node of nodes) {
-    const depth = node.relationPath.length;
-
-    for (const descendant of [node, ...node.descendants]) {
-      const relationPath = descendant.relationPath.slice(depth).join('.');
-      const pathNodes = nodesByRelationPath.get(relationPath);
-
-      if (pathNodes) {
-        pathNodes.push(descendant);
-      } else {
-        nodesByRelationPath.set(relationPath, [descendant]);
-      }
-    }
-  }
-
+  const nodesByRelationPath = new Map([['', []]]);
   const addPaths = (expressionNode, path) => {
     for (const name of expressionNode.$childNames) {
       const relationPath = [...path, name];
-      const key = relationPath.join('.');
-
-      if (!nodesByRelationPath.has(key)) {
-        nodesByRelationPath.set(key, []);
-      }
-
+      nodesByRelationPath.set(relationPath.join('.'), []);
       addPaths(expressionNode[name], relationPath);
     }
   };
 
   addPaths(expression.node, []);
+
+  for (const node of nodes) {
+    const depth = node.relationPath.length;
+
+    for (const descendant of [node, ...node.descendants]) {
+      nodesByRelationPath.get(descendant.relationPath.slice(depth).join('.'))?.push(descendant);
+    }
+  }
+
   return nodesByRelationPath;
 }
 
 function createSelect(nodes, graphOptions, isFetchRoot) {
   if (graphOptions.isFetchStrategy(FetchStrategy.OnlyIdentifiers)) {
-    return (builder) => builder.select(builder.fullIdColumn());
+    return (builder) => builder.select(getKeySelections(builder, nodes));
   } else if (graphOptions.isFetchStrategy(FetchStrategy.OnlyNeeded)) {
     return (builder) => builder.select(getInputColumnSelections(builder, nodes, isFetchRoot));
   } else {
@@ -263,6 +340,22 @@ function createSelect(nodes, graphOptions, isFetchRoot) {
   }
 }
 
+// Select the ids, and the keys of the relations that the graph gives, which
+// `readRelations()` reads the relations at.
+function getKeySelections(builder, nodes) {
+  const columns = new Set([
+    ...asArray(builder.fullIdColumn()),
+    ...nodes.flatMap((node) =>
+      node.relations.flatMap((relation) =>
+        relation.ownerProp.cols.map(
+          (column) => `${builder.tableRefFor(node.modelClass)}.${column}`,
+        ),
+      ),
+    ),
+  ]);
+  return [...columns];
+}
+
 // Select only the columns that the graph gives, and always the ids. Related
 // models are fetched without the join tables of their relations, so their join
 // table extras are left out at the root of the fetch.
@@ -292,13 +385,5 @@ function getInputColumnSelections(builder, nodes, isFetchRoot) {
     }
   }
 
-  const values = [...selections.values()];
-
-  for (const idColumn of asArray(builder.fullIdColumn())) {
-    if (!values.includes(idColumn)) {
-      values.push(idColumn);
-    }
-  }
-
-  return values;
+  return [...new Set([...selections.values(), ...getKeySelections(builder, nodes)])];
 }

package/lib/queryBuilder/graph/GraphNode.js

@@ -29,8 +29,7 @@ export class GraphNode {
     // The node of a `#ref` model, and the `#ref{}` properties of this model:
     this.referencedNode = null;
     this.propertyReferences = [];
-    // The nodes whose `#ref` or `#ref{}` point to this node:
-    this.referencingNodes = [];
+    // The nodes whose `#ref{}` point to this node:
     this.propertyReferencingNodes = [];
 
     // How the graph and the current graph pair up, see `GraphLoader`: The
@@ -61,6 +60,15 @@ export class GraphNode {
     return !this.parent;
   }
 
+  // The relations that the graph gives the model of this node, in its order.
+  get relations() {
+    const { modelClass, obj } = this;
+    const names = modelClass.getRelationNames();
+    return Object.keys(obj)
+      .filter((name) => names.includes(name) && obj[name] !== undefined)
+      .map((name) => modelClass.getRelation(name));
+  }
+
   // The current node of the row of this node, at its place or where a relate
   // moves it from.
   get existing() {
@@ -184,7 +192,6 @@ export class GraphNode {
     for (const node of nodes) {
       if (node.isReference) {
         node.referencedNode = getNode(node.reference);
-        node.referencedNode.referencingNodes.push(node);
       }
 
       forEachPropertyReference(node.obj, ({ path, match, uid, refPath, isWholeValue }) => {
@@ -233,3 +240,9 @@ export function createNotModelError(modelClass, value) {
     message: `expected value "${value}" to be an instance of ${modelClass.name}`,
   });
 }
+
+// Name the place of a model in error messages, by its data path.
+export function formatPath(dataPathKey) {
+  const path = dataPathKey.replace(/^\./, '');
+  return path ? `'${path}'` : 'the root';
+}

package/lib/queryBuilder/graph/GraphOptions.js

@@ -1,44 +1,30 @@
 import { RelationExpression } from '../RelationExpression.js';
 import { isString } from '../../utils/objectUtils.js';
 
-const NO_RELATE = 'noRelate';
-const NO_UNRELATE = 'noUnrelate';
-const NO_INSERT = 'noInsert';
-const NO_UPDATE = 'noUpdate';
-const NO_DELETE = 'noDelete';
-
-const UPDATE = 'update';
-const RELATE = 'relate';
-const UNRELATE = 'unrelate';
-const INSERT_MISSING = 'insertMissing';
-const FETCH_STRATEGY = 'fetchStrategy';
-const ALLOW_REFS = 'allowRefs';
-const ALLOW_MARKERS = 'allowMarkers';
-
 // The options that are matched against the relation path of each node, see
 // `has()`.
 const NODE_OPTION_NAMES = [
-  NO_RELATE,
-  NO_UNRELATE,
-  NO_INSERT,
-  NO_UPDATE,
-  NO_DELETE,
-  UPDATE,
-  RELATE,
-  UNRELATE,
-  INSERT_MISSING,
+  'noRelate',
+  'noUnrelate',
+  'noInsert',
+  'noUpdate',
+  'noDelete',
+  'update',
+  'relate',
+  'unrelate',
+  'insertMissing',
 ];
 
-const OPTION_NAMES = [...NODE_OPTION_NAMES, FETCH_STRATEGY, ALLOW_REFS, ALLOW_MARKERS];
+const OPTION_NAMES = [...NODE_OPTION_NAMES, 'fetchStrategy', 'allowRefs', 'allowMarkers'];
 
 // `insertGraph` never fetches, updates or removes existing rows, so only the
 // options about inserting and relating apply to it, and the others are set:
-const INSERT_OPTION_NAMES = [RELATE, NO_RELATE, NO_INSERT, ALLOW_REFS];
+const INSERT_OPTION_NAMES = ['relate', 'noRelate', 'noInsert', 'allowRefs'];
 const INSERT_ONLY_OPTIONS = {
-  [NO_UPDATE]: true,
-  [NO_DELETE]: true,
-  [NO_UNRELATE]: true,
-  [INSERT_MISSING]: true,
+  noUpdate: true,
+  noDelete: true,
+  noUnrelate: true,
+  insertMissing: true,
 };
 
 export const FetchStrategy = {
@@ -59,14 +45,10 @@ export class GraphOptions {
   }
 
   isFetchStrategy(strategy) {
-    if (!FetchStrategy[strategy]) {
-      throw new Error(`unknown strategy "${strategy}"`);
-    }
-
-    if (!this.options[FETCH_STRATEGY]) {
+    if (!this.options.fetchStrategy) {
       return strategy === FetchStrategy.OnlyNeeded;
     } else {
-      return this.options[FETCH_STRATEGY] === strategy;
+      return this.options.fetchStrategy === strategy;
     }
   }
 
@@ -78,19 +60,19 @@ export class GraphOptions {
   // or `update()`. With `noUpdate`, it's neither, and only the foreign keys of
   // its relates and BelongsToOneRelations are patched.
   shouldPatch(node) {
-    return !this.has(node, NO_UPDATE) && !this.has(node, UPDATE);
+    return !this.has(node, 'noUpdate') && !this.has(node, 'update');
   }
 
   shouldUpdate(node) {
-    return !this.has(node, NO_UPDATE) && this.has(node, UPDATE);
+    return !this.has(node, 'noUpdate') && this.has(node, 'update');
   }
 
   allowRefs() {
-    return !!this.options[ALLOW_REFS];
+    return !!this.options.allowRefs;
   }
 
   allowMarkers() {
-    return !!this.options[ALLOW_MARKERS];
+    return !!this.options.allowMarkers;
   }
 
   // Whether the option applies to the relation path of `node`.
@@ -125,16 +107,12 @@ function checkForUnknownOptions(options, insertOnly) {
 // Parses the string values of the node options into relation expressions,
 // once per operation, and checks the types of the other values.
 function parseRelationExpressions(options) {
-  let parsedOptions = options;
+  const parsedOptions = { ...options };
 
   for (const optionName of NODE_OPTION_NAMES) {
     const option = options[optionName];
 
     if (isString(option)) {
-      if (parsedOptions === options) {
-        parsedOptions = { ...options };
-      }
-
       parsedOptions[optionName] = parseRelationExpression(optionName, option);
     } else if (!(option === undefined || typeof option === 'boolean' || Array.isArray(option))) {
       throw new Error(
@@ -147,11 +125,8 @@ function parseRelationExpressions(options) {
 }
 
 function parseRelationExpression(optionName, expression) {
-  const createError = (message) => {
-    return new Error(
-      `invalid relation expression "${expression}" in ${optionName} option: ${message}`,
-    );
-  };
+  const createError = (message) =>
+    new Error(`invalid relation expression "${expression}" in ${optionName} option: ${message}`);
 
   let parsedExpression;