objection 4.0.0-rc.3 → 4.0.0

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/CheckViolationError/parser.js

@@ -1,19 +1,12 @@
 import { CheckViolationError } from '../../../../../errors/CheckViolationError.js';
-
-const REGEX = /SQLITE_CONSTRAINT: CHECK constraint failed/;
+import { getConstraintMessage } from '../constraintMessage.js';
 
 export const checkViolationErrorParser = {
   error: CheckViolationError,
 
   parse: (err) => {
-    const match = REGEX.exec(err.message);
-
-    if (!match) {
-      return null;
-    }
-
     // No way to extract anything reliably.
-    return {};
+    return getConstraintMessage(err, 'CHECK') ? {} : null;
   },
 
   subclassParsers: [],

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/ForeignKeyViolationError/parser.js

@@ -1,19 +1,12 @@
 import { ForeignKeyViolationError } from '../../../../../errors/ForeignKeyViolationError.js';
-
-const REGEX = /SQLITE_CONSTRAINT: FOREIGN KEY constraint failed/;
+import { getConstraintMessage } from '../constraintMessage.js';
 
 export const foreignKeyViolationErrorParser = {
   error: ForeignKeyViolationError,
 
   parse: (err) => {
-    const match = REGEX.exec(err.message);
-
-    if (!match) {
-      return null;
-    }
-
     // No way to extract anything reliably.
-    return {};
+    return getConstraintMessage(err, 'FOREIGN KEY') ? {} : null;
   },
 
   subclassParsers: [],

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/NotNullViolationError/parser.js

@@ -1,12 +1,14 @@
 import { NotNullViolationError } from '../../../../../errors/NotNullViolationError.js';
+import { getConstraintMessage } from '../constraintMessage.js';
 
-const NOT_COLUMN_REGEX = /SQLITE_CONSTRAINT: NOT NULL constraint failed: (.+)$/;
+const NOT_COLUMN_REGEX = /^NOT NULL constraint failed: (.+)$/s;
 
 export const notNullViolationErrorParser = {
   error: NotNullViolationError,
 
   parse: (err) => {
-    const colMatch = NOT_COLUMN_REGEX.exec(err.message);
+    const message = getConstraintMessage(err, 'NOT NULL');
+    const colMatch = message && NOT_COLUMN_REGEX.exec(message);
 
     if (!colMatch) {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/UniqueViolationError/parser.js

@@ -1,12 +1,14 @@
 import { UniqueViolationError } from '../../../../../errors/UniqueViolationError.js';
+import { getConstraintMessage } from '../constraintMessage.js';
 
-const UNIQUE_COLUMNS_REGEX = /SQLITE_CONSTRAINT: UNIQUE constraint failed: (.+)$/;
+const UNIQUE_COLUMNS_REGEX = /^UNIQUE constraint failed: (.+)$/s;
 
 export const uniqueViolationErrorParser = {
   error: UniqueViolationError,
 
   parse: (err) => {
-    const colsMatch = UNIQUE_COLUMNS_REGEX.exec(err.message);
+    const message = getConstraintMessage(err, 'UNIQUE');
+    const colsMatch = message && UNIQUE_COLUMNS_REGEX.exec(message);
 
     if (!colsMatch) {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/constraintMessage.js

@@ -0,0 +1,16 @@
+// Return the text of the SQLite error `err` from `<kind> constraint failed` on,
+// e.g. 'UNIQUE constraint failed: t.a', if it's a constraint error of that kind,
+// or else `null`. `better-sqlite3` names the kind in its extended code, e.g.
+// `SQLITE_CONSTRAINT_UNIQUE`, `sqlite3` only after `SQLITE_CONSTRAINT: `. knex
+// puts the SQL with its values in front, so the last match is SQLite's own.
+export function getConstraintMessage(err, kind) {
+  const text = `${kind} constraint failed`;
+  // A duplicate primary key is a unique violation, with its own code:
+  const codes = kind === 'UNIQUE' ? ['UNIQUE', 'PRIMARYKEY'] : [kind.replace(' ', '')];
+  const isKind =
+    codes.some((code) => err.code === `SQLITE_CONSTRAINT_${code}`) ||
+    (err.code === 'SQLITE_CONSTRAINT' &&
+      /.*SQLITE_CONSTRAINT: (.*)$/s.exec(err.message)?.[1].startsWith(text));
+
+  return isKind ? err.message.slice(err.message.lastIndexOf(text)) : null;
+}

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/parser.js

@@ -8,7 +8,8 @@ export const constraintViolationErrorParser = {
   error: ConstraintViolationError,
 
   parse: (err) => {
-    if (err.code === 'SQLITE_CONSTRAINT') {
+    // `better-sqlite3` uses the extended codes, e.g. `SQLITE_CONSTRAINT_UNIQUE`:
+    if (/^SQLITE_CONSTRAINT(_|$)/.test(err.code)) {
       return {};
     } else {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/parser.js

@@ -5,10 +5,11 @@ export const dbErrorParser = {
   error: DBError,
 
   parse: (err) => {
+    // `sqlite3` gives its errors an `errno`, `better-sqlite3` names them:
     if (
       typeof err.code === 'string' &&
       err.code.startsWith('SQLITE_') &&
-      typeof err.errno === 'number'
+      (typeof err.errno === 'number' || err.name === 'SqliteError')
     ) {
       return {
         nativeError: err,

package/lib/model/modelJsonAttributes.js

@@ -1,12 +1,12 @@
 import { asArray, isObject, flatten, isString } from '../utils/objectUtils.js';
 import { isQueryProp } from './modelQueryProps.js';
-import { isJsonParsedByDriver } from '../utils/knexUtils.js';
+import { isJsonParsedByDriver, isMsSql, isSqlite } from '../utils/knexUtils.js';
 
 export function parseJsonAttributes(json, modelClass, knex) {
   const jsonAttr = modelClass.getJsonAttributes();
   const format = getJsonColumnFormat(modelClass, knex);
 
-  if (jsonAttr.length && format !== 'parsed') {
+  if (jsonAttr.length && format !== 'json') {
     // JSON attributes may be returned as strings depending on the database and
     // the database client. Convert them to objects here.
     for (let i = 0, l = jsonAttr.length; i < l; ++i) {
@@ -85,14 +85,23 @@ function getPropertyTypes(prop) {
   return types;
 }
 
-// Resolve 'auto' to 'parsed' for drivers that return JSON columns parsed, or
-// else to `null`: JSON attributes may be text then, and are parsed leniently.
+// Resolve 'auto' by the driver: the PostgreSQL drivers return JSON columns
+// parsed, SQLite and SQL Server return JSON text, and MySQL's drivers and others
+// may return JSON or plain text, e.g. from MariaDB, which stores JSON as text,
+// and which knex can't tell apart from MySQL. Without a driver, rows passed by
+// hand may hold either.
 function getJsonColumnFormat(modelClass, knex) {
   const { jsonColumnFormat } = modelClass;
 
   if (jsonColumnFormat === 'auto') {
-    return knex && isJsonParsedByDriver(knex) ? 'parsed' : null;
-  } else if (jsonColumnFormat === 'text' || jsonColumnFormat === 'parsed') {
+    return !knex
+      ? 'mixed'
+      : isJsonParsedByDriver(knex)
+        ? 'json'
+        : isSqlite(knex) || isMsSql(knex)
+          ? 'text'
+          : 'mixed';
+  } else if (['json', 'text', 'mixed'].includes(jsonColumnFormat)) {
     return jsonColumnFormat;
   } else {
     throw new Error(

package/lib/objection.js

@@ -12,6 +12,7 @@ export {
   ConstraintViolationError,
   CheckViolationError,
   DataError,
+  wrapError,
 } from './dbErrors/index.js';
 export { QueryBuilderBase } from './queryBuilder/QueryBuilderBase.js';
 export { QueryBuilderOperation } from './queryBuilder/operations/QueryBuilderOperation.js';

package/lib/queryBuilder/InternalOptions.js

@@ -6,6 +6,9 @@ export class InternalOptions {
     this.skipRelationFilter = false;
     // The join table extras that related updates write, all of them if null:
     this.joinTableExtras = null;
+    // The path updates that `upsertGraph()` writes the keys in json columns of
+    // updated rows with, by column:
+    this.jsonKeyPatches = null;
     this.debug = false;
     this.schema = undefined;
   }
@@ -18,6 +21,7 @@ export class InternalOptions {
     copy.isInternalQuery = this.isInternalQuery;
     copy.skipRelationFilter = this.skipRelationFilter;
     copy.joinTableExtras = this.joinTableExtras;
+    copy.jsonKeyPatches = this.jsonKeyPatches;
     copy.debug = this.debug;
     copy.schema = this.schema;
 

package/lib/queryBuilder/StaticHookArguments.js

@@ -1,4 +1,5 @@
 import { asArray, isObject } from '../utils/objectUtils.js';
+import { AsArrayOperation } from './operations/AsArrayOperation.js';
 
 const BUILDER_SYMBOL = Symbol();
 
@@ -26,12 +27,13 @@ export class StaticHookArguments {
       // ones installed by `throwIfNotFound()` and `traverse()`) are meant for
       // its result, not for the result of this find query. The same goes for
       // the value it was resolved with, e.g. the 0 of an update that only
-      // changes the join rows of a many-to-many relation.
+      // changes the join rows of a many-to-many relation. The query resolves to
+      // an array also for single-result writes, unless the hook adds `first()`:
       return this[BUILDER_SYMBOL].toFindQuery()
         .clear('runAfter')
         .clearResolve()
         .clearWithGraphFetched()
-        .runAfter(asArray);
+        .addOperation(new AsArrayOperation('asArray'), []);
     };
   }
 

package/lib/queryBuilder/graph/GraphNode.js

@@ -191,6 +191,14 @@ export class GraphNode {
 
     for (const node of nodes) {
       if (node.isReference) {
+        // A `#ref` places its model in a relation, which a root doesn't have:
+        if (!node.parent) {
+          throw node.modelClass.createValidationError({
+            type: ValidationErrorType.InvalidGraph,
+            message: `the reference at ${formatPath(node.dataPathKey)} has no relation to place its model in`,
+          });
+        }
+
         node.referencedNode = getNode(node.reference);
       }
 

package/lib/queryBuilder/graph/GraphPlan.js

@@ -91,6 +91,19 @@ export class GraphPlan {
     );
   }
 
+  // Return true if the graph relates the row of `node` by the key of its
+  // BelongsToOneRelation alone, without an id or a reference.
+  isRelatedByKey(node) {
+    return (
+      this.actions.get(node) === 'relate' &&
+      node.relation.keySide === 'owner' &&
+      !node.hasId &&
+      !node.isReference &&
+      !node.isDbReference &&
+      node.propertyReferences.length === 0
+    );
+  }
+
   // Return true if the model of the `#ref` model `node` writes what `node`
   // would write to the row: The model is inserted, or it's written and its path
   // allows any write that the path of `node` allows.

package/lib/queryBuilder/graph/GraphSchedule.js

@@ -1,17 +1,23 @@
 import { hasOwnerKey, hasSameColumns, hasSameKey } from './graphKeys.js';
 import { groupBy } from '../../utils/objectUtils.js';
+import { ValidationErrorType } from '../../model/ValidationError.js';
+import { formatPath } from './GraphNode.js';
 
 // Order the writes of a planned graph in steps that keep keys and constraints
 // intact (#214), which `GraphWriter` writes:
 // 1. Move the rows that the graph relates to existing owners of HasMany and
 //    HasOne relations, as they may still sit below rows that the next step
-//    deletes. Rows that wait for inserts are related in step 3 or 4.
+//    deletes. Rows that wait for inserts or changed keys are related in step 3
+//    or 4.
 // 2. Remove rows that reference their owners, before the inserts that may need
 //    their unique keys.
 // 3. Insert the new rows in the order of their keys and references, then the
 //    keys deferred to break cycles and the join rows. Rows that move out of
 //    the slots of HasOneRelations that new rows go into are related in the
-//    same order, as the foreign key may be unique.
+//    same order, as the foreign key may be unique, and so are the patches and
+//    relates that change the keys that these rows read or the values that
+//    their `#ref{}` copies, as a foreign key may reference them (#428, #491).
+//    Copies of values that are no keys wait unless that closes a cycle.
 // 4. Patch the changed rows, the other relates, and the keys of owners whose
 //    BelongsToOne rows were inserted, related or removed.
 // 5. Unrelate the rows that their own to-many relation on their own key leaves
@@ -25,7 +31,9 @@ export class GraphSchedule {
     this.plan = plan;
     this.keys = keys;
     // The nodes that each node waits for, through its keys, its references and
-    // the rows that move out of the slots that it goes into:
+    // the rows that move out of the slots that it goes into. `isKeyChange`
+    // marks the patches and relates that change the keys it reads or the values
+    // that its `#ref{}` copies, and `isCopy` the latter:
     this.dependencies = createDependencies(plan, this.keys);
     // Dependencies that aren't waited for, to break cycles. Deferred keys are
     // written after the inserts:
@@ -36,9 +44,8 @@ export class GraphSchedule {
         ? []
         : plan.activeNodes.filter((node) => this._isMovedFirst(node)),
     );
-    // Relates that move the rows out of the slots of HasOneRelations that new
-    // rows go into, and that aren't moved first:
-    this.slotMovedNodes = this._getSlotMovedNodes();
+    // Relates and patches that aren't moved first but written in the inserts:
+    this.earlyNodes = this._getEarlyNodes();
     this.steps = this._planSteps();
   }
 
@@ -91,10 +98,9 @@ export class GraphSchedule {
           activeNodes.filter((node) => {
             const action = actions.get(node);
             return (
-              (action === 'patch' ||
-                (action === 'relate' &&
-                  !this.movedNodes.has(node) &&
-                  !this.slotMovedNodes.has(node))) &&
+              ['patch', 'relate'].includes(action) &&
+              !this.movedNodes.has(node) &&
+              !this.earlyNodes.has(node) &&
               // The row of a `#ref` model is written once, by its model:
               !this._isWrittenByModel(node)
             );
@@ -140,6 +146,7 @@ export class GraphSchedule {
           !handledNodes.has(dependency.node) &&
           (dependency.isSlotMove ||
             dependency.isKeyWait ||
+            dependency.isKeyChange ||
             !hasRow(dependency.node) ||
             this.readsFilledProps(dependency)),
       );
@@ -160,12 +167,14 @@ export class GraphSchedule {
 
     // Break a cycle by deferring one BelongsToOne key that points to a `#ref`
     // model: Its owner is inserted without the key, which is patched in once
-    // the referenced model is inserted. Without `#ref`, the graph is a tree,
-    // and only waits for the rows that move out of slots can close a cycle,
-    // like rows that swap slots. One of these isn't waited for then.
+    // the referenced model is inserted. Without `#ref` models, the graph is a
+    // tree, and only `#ref{}` copies of changed values and the waits for the
+    // rows that move out of slots can close a cycle, like rows that swap slots.
+    // One of these isn't waited for then, the copies first.
     const breakCycle = () => {
       for (const canDefer of [
         (dependency, node) => this._canDefer(dependency, node),
+        (dependency) => dependency.isCopy,
         (dependency) => dependency.isSlotMove,
       ]) {
         for (const node of nodes) {
@@ -212,7 +221,25 @@ export class GraphSchedule {
           }
         }
       } else if (!breakCycle()) {
-        throw new Error('the object graph contains cyclic references');
+        // A patch that changes a key that the rows it waits for read can be
+        // written neither before nor after them:
+        const { node: changingNode } =
+          nodes
+            .filter((node) => !handledNodes.has(node))
+            .flatMap(getPendingDependencies)
+            .find(
+              ({ node, dependent, isKeyChange }) => isKeyChange && isBlockedBy(node, dependent),
+            ) || {};
+
+        if (changingNode) {
+          const { modelClass, dataPathKey } = changingNode;
+          throw modelClass.createValidationError({
+            type: ValidationErrorType.InvalidGraph,
+            message: `the graph changes a key of the ${modelClass.name} at ${formatPath(dataPathKey)} that rows read which it also waits for; write the key in a separate query first`,
+          });
+        } else {
+          throw new Error('the object graph contains cyclic references');
+        }
       }
     }
 
@@ -245,9 +272,10 @@ export class GraphSchedule {
 
   // Return true if the HasMany or HasOne relate of `node` moves its row to an
   // existing owner before the removals, see the steps above. `#ref` models may
-  // get their ids from the inserts, rows that wait for the inserts are written
-  // once they're known, and a relate into the slot of a HasOne row waits for
-  // the removal of that row, or its move, as its key may be unique.
+  // get their ids from the inserts, rows that wait for the inserts or for
+  // changed keys are written after them, and a relate into the slot of a HasOne
+  // row waits for the removal of that row, or its move, as its key may be
+  // unique.
   _isMovedFirst(node, visitedNodes = new Set()) {
     const { actions } = this.plan;
     visitedNodes.add(node);
@@ -257,7 +285,7 @@ export class GraphSchedule {
       !node.isReference &&
       node.relation.keySide === 'related' &&
       node.parent.existing &&
-      !this._waitsForInserts(node)
+      !this._waitsForWrites(node)
     ) {
       const slotNode = getSlotNode(node);
       const moverNode = getSlotMover(this.plan, node);
@@ -271,24 +299,32 @@ export class GraphSchedule {
     }
   }
 
-  // Find the relates that move the rows out of the slots of HasOneRelations
-  // that new rows go into, and that aren't moved first. They're related in the
-  // inserts, along with the relates that move the rows out of their own slots,
-  // as the foreign key may be unique.
-  _getSlotMovedNodes() {
+  // Find the relates and patches that are written in the inserts, as the rows
+  // written there wait for them: The relates that move the rows out of the
+  // slots of HasOneRelations that new rows go into, along with the relates that
+  // move the rows out of their own slots, as the foreign key may be unique, and
+  // the patches and relates that change the keys that these rows or join rows
+  // read, as a foreign key may reference them (#428).
+  _getEarlyNodes() {
+    const { actions } = this.plan;
     const nodes = new Set();
-    const addMoverNodes = (node) => {
-      for (const { node: moverNode, isSlotMove } of this.dependencies.get(node) || []) {
-        if (isSlotMove && !this.movedNodes.has(moverNode) && !nodes.has(moverNode)) {
-          nodes.add(moverNode);
-          addMoverNodes(moverNode);
+    const addEarlyNodes = (node) => {
+      for (const dependency of this.dependencies.get(node) || []) {
+        const { node: earlyNode, isSlotMove, isKeyChange } = dependency;
+        if (
+          (isSlotMove || isKeyChange) &&
+          !this.movedNodes.has(earlyNode) &&
+          !nodes.has(earlyNode)
+        ) {
+          nodes.add(earlyNode);
+          addEarlyNodes(earlyNode);
         }
       }
     };
 
     for (const node of this.plan.activeNodes) {
-      if (this.plan.actions.get(node) === 'insert') {
-        addMoverNodes(node);
+      if (actions.get(node) === 'insert' || (node.relation?.keySide === 'join' && !node.current)) {
+        addEarlyNodes(node);
       }
     }
 
@@ -317,19 +353,21 @@ export class GraphSchedule {
   }
 
   // Return true if `node` waits for the keys or references of inserted models,
-  // also through the models that it references (#356), and through the models
-  // that it waits for to get its keys.
-  _waitsForInserts(node, visitedNodes = new Set()) {
+  // or for keys that patches change (#428), also through the models that it
+  // references (#356), and through the models that it waits for to get its
+  // keys.
+  _waitsForWrites(node, visitedNodes = new Set()) {
     const { actions } = this.plan;
     visitedNodes.add(node);
     return (this.dependencies.get(node) || []).some((dependency) => {
-      const { node: dependencyNode, relation } = dependency;
+      const { node: dependencyNode, relation, isKeyWait, isKeyChange } = dependency;
       const referencedNode = dependencyNode.referencedNode || dependencyNode;
       return (
         actions.get(referencedNode) === 'insert' ||
-        ((!relation || dependency.isKeyWait) &&
+        isKeyChange ||
+        ((!relation || isKeyWait) &&
           !visitedNodes.has(referencedNode) &&
-          this._waitsForInserts(referencedNode, visitedNodes))
+          this._waitsForWrites(referencedNode, visitedNodes))
       );
     });
   }
@@ -458,18 +496,32 @@ function createDependencies(plan, keys) {
 
     // A place waits for the inserts and `#ref{}` values that bind the slots of
     // its keys. A dependency on the same row through the same key becomes such
-    // a wait, and stays deferrable to break cycles. The other waits aren't:
-    for (const { node: waitedNode, prop } of keys.waitsFor(node)) {
+    // a wait, and stays deferrable to break cycles. The other waits aren't. A
+    // key change is waited for at the node that writes it, not at a `#ref`
+    // model of its row, which writes nothing:
+    for (const { node: waitedNode, prop, isKeyChange } of keys.waitsFor(node)) {
       const dependency = (dependencies.get(node) || []).find(
         (dependency) =>
-          getRow(dependency.node) === getRow(waitedNode) &&
+          (isKeyChange
+            ? dependency.node === waitedNode
+            : getRow(dependency.node) === getRow(waitedNode)) &&
           (!dependency.relation || getKeyProps(dependency, node).includes(prop)),
       );
 
       if (dependency) {
         dependency.isKeyWait = true;
+        dependency.isKeyChange ||= isKeyChange;
       } else {
-        addDependency(node, { node: waitedNode, isKeyWait: true });
+        addDependency(node, { node: waitedNode, isKeyWait: true, isKeyChange });
+      }
+    }
+
+    // A `#ref{}` that copies a value that the graph changes waits for the
+    // change, as a foreign key may reference it, like the keys (#491). As the
+    // copied value may be no key, the wait is deferred first to break cycles:
+    for (const reference of node.propertyReferences) {
+      for (const changingNode of keys.getChangingNodes(node, reference)) {
+        addDependency(node, { node: changingNode, isKeyChange: true, isCopy: true });
       }
     }
 

package/lib/queryBuilder/graph/GraphUpsert.js

@@ -9,6 +9,7 @@ import {
   checkChangedIds,
   checkGraph,
   checkJoinTables,
+  checkKeyRelates,
   checkMissingModels,
   checkPlacements,
   checkReadKeys,
@@ -100,6 +101,7 @@ export class GraphUpsert {
       }
 
       checkChangedIds(builder, plan);
+      checkKeyRelates(builder, plan);
       checkReferences(builder, plan);
       checkJoinTables(builder, plan);
       await new GraphWriter(new GraphSchedule(plan, keys)).write(builder);