objection 4.0.0-rc.4 → 4.0.0

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/CheckViolationError/parser.js

@@ -1,19 +1,12 @@
 import { CheckViolationError } from '../../../../../errors/CheckViolationError.js';
-
-const REGEX = /SQLITE_CONSTRAINT: CHECK constraint failed/;
+import { getConstraintMessage } from '../constraintMessage.js';
 
 export const checkViolationErrorParser = {
   error: CheckViolationError,
 
   parse: (err) => {
-    const match = REGEX.exec(err.message);
-
-    if (!match) {
-      return null;
-    }
-
     // No way to extract anything reliably.
-    return {};
+    return getConstraintMessage(err, 'CHECK') ? {} : null;
   },
 
   subclassParsers: [],

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/ForeignKeyViolationError/parser.js

@@ -1,19 +1,12 @@
 import { ForeignKeyViolationError } from '../../../../../errors/ForeignKeyViolationError.js';
-
-const REGEX = /SQLITE_CONSTRAINT: FOREIGN KEY constraint failed/;
+import { getConstraintMessage } from '../constraintMessage.js';
 
 export const foreignKeyViolationErrorParser = {
   error: ForeignKeyViolationError,
 
   parse: (err) => {
-    const match = REGEX.exec(err.message);
-
-    if (!match) {
-      return null;
-    }
-
     // No way to extract anything reliably.
-    return {};
+    return getConstraintMessage(err, 'FOREIGN KEY') ? {} : null;
   },
 
   subclassParsers: [],

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/NotNullViolationError/parser.js

@@ -1,12 +1,14 @@
 import { NotNullViolationError } from '../../../../../errors/NotNullViolationError.js';
+import { getConstraintMessage } from '../constraintMessage.js';
 
-const NOT_COLUMN_REGEX = /SQLITE_CONSTRAINT: NOT NULL constraint failed: (.+)$/;
+const NOT_COLUMN_REGEX = /^NOT NULL constraint failed: (.+)$/s;
 
 export const notNullViolationErrorParser = {
   error: NotNullViolationError,
 
   parse: (err) => {
-    const colMatch = NOT_COLUMN_REGEX.exec(err.message);
+    const message = getConstraintMessage(err, 'NOT NULL');
+    const colMatch = message && NOT_COLUMN_REGEX.exec(message);
 
     if (!colMatch) {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/UniqueViolationError/parser.js

@@ -1,12 +1,14 @@
 import { UniqueViolationError } from '../../../../../errors/UniqueViolationError.js';
+import { getConstraintMessage } from '../constraintMessage.js';
 
-const UNIQUE_COLUMNS_REGEX = /SQLITE_CONSTRAINT: UNIQUE constraint failed: (.+)$/;
+const UNIQUE_COLUMNS_REGEX = /^UNIQUE constraint failed: (.+)$/s;
 
 export const uniqueViolationErrorParser = {
   error: UniqueViolationError,
 
   parse: (err) => {
-    const colsMatch = UNIQUE_COLUMNS_REGEX.exec(err.message);
+    const message = getConstraintMessage(err, 'UNIQUE');
+    const colsMatch = message && UNIQUE_COLUMNS_REGEX.exec(message);
 
     if (!colsMatch) {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/constraintMessage.js

@@ -0,0 +1,16 @@
+// Return the text of the SQLite error `err` from `<kind> constraint failed` on,
+// e.g. 'UNIQUE constraint failed: t.a', if it's a constraint error of that kind,
+// or else `null`. `better-sqlite3` names the kind in its extended code, e.g.
+// `SQLITE_CONSTRAINT_UNIQUE`, `sqlite3` only after `SQLITE_CONSTRAINT: `. knex
+// puts the SQL with its values in front, so the last match is SQLite's own.
+export function getConstraintMessage(err, kind) {
+  const text = `${kind} constraint failed`;
+  // A duplicate primary key is a unique violation, with its own code:
+  const codes = kind === 'UNIQUE' ? ['UNIQUE', 'PRIMARYKEY'] : [kind.replace(' ', '')];
+  const isKind =
+    codes.some((code) => err.code === `SQLITE_CONSTRAINT_${code}`) ||
+    (err.code === 'SQLITE_CONSTRAINT' &&
+      /.*SQLITE_CONSTRAINT: (.*)$/s.exec(err.message)?.[1].startsWith(text));
+
+  return isKind ? err.message.slice(err.message.lastIndexOf(text)) : null;
+}

package/lib/dbErrors/parsers/sqlite/DBError/ConstraintViolationError/parser.js

@@ -8,7 +8,8 @@ export const constraintViolationErrorParser = {
   error: ConstraintViolationError,
 
   parse: (err) => {
-    if (err.code === 'SQLITE_CONSTRAINT') {
+    // `better-sqlite3` uses the extended codes, e.g. `SQLITE_CONSTRAINT_UNIQUE`:
+    if (/^SQLITE_CONSTRAINT(_|$)/.test(err.code)) {
       return {};
     } else {
       return null;

package/lib/dbErrors/parsers/sqlite/DBError/parser.js

@@ -5,10 +5,11 @@ export const dbErrorParser = {
   error: DBError,
 
   parse: (err) => {
+    // `sqlite3` gives its errors an `errno`, `better-sqlite3` names them:
     if (
       typeof err.code === 'string' &&
       err.code.startsWith('SQLITE_') &&
-      typeof err.errno === 'number'
+      (typeof err.errno === 'number' || err.name === 'SqliteError')
     ) {
       return {
         nativeError: err,

package/lib/model/modelJsonAttributes.js

@@ -1,12 +1,12 @@
 import { asArray, isObject, flatten, isString } from '../utils/objectUtils.js';
 import { isQueryProp } from './modelQueryProps.js';
-import { isJsonParsedByDriver } from '../utils/knexUtils.js';
+import { isJsonParsedByDriver, isMsSql, isSqlite } from '../utils/knexUtils.js';
 
 export function parseJsonAttributes(json, modelClass, knex) {
   const jsonAttr = modelClass.getJsonAttributes();
   const format = getJsonColumnFormat(modelClass, knex);
 
-  if (jsonAttr.length && format !== 'parsed') {
+  if (jsonAttr.length && format !== 'json') {
     // JSON attributes may be returned as strings depending on the database and
     // the database client. Convert them to objects here.
     for (let i = 0, l = jsonAttr.length; i < l; ++i) {
@@ -85,14 +85,23 @@ function getPropertyTypes(prop) {
   return types;
 }
 
-// Resolve 'auto' to 'parsed' for drivers that return JSON columns parsed, or
-// else to `null`: JSON attributes may be text then, and are parsed leniently.
+// Resolve 'auto' by the driver: the PostgreSQL drivers return JSON columns
+// parsed, SQLite and SQL Server return JSON text, and MySQL's drivers and others
+// may return JSON or plain text, e.g. from MariaDB, which stores JSON as text,
+// and which knex can't tell apart from MySQL. Without a driver, rows passed by
+// hand may hold either.
 function getJsonColumnFormat(modelClass, knex) {
   const { jsonColumnFormat } = modelClass;
 
   if (jsonColumnFormat === 'auto') {
-    return knex && isJsonParsedByDriver(knex) ? 'parsed' : null;
-  } else if (jsonColumnFormat === 'text' || jsonColumnFormat === 'parsed') {
+    return !knex
+      ? 'mixed'
+      : isJsonParsedByDriver(knex)
+        ? 'json'
+        : isSqlite(knex) || isMsSql(knex)
+          ? 'text'
+          : 'mixed';
+  } else if (['json', 'text', 'mixed'].includes(jsonColumnFormat)) {
     return jsonColumnFormat;
   } else {
     throw new Error(

package/lib/objection.js

@@ -12,6 +12,7 @@ export {
   ConstraintViolationError,
   CheckViolationError,
   DataError,
+  wrapError,
 } from './dbErrors/index.js';
 export { QueryBuilderBase } from './queryBuilder/QueryBuilderBase.js';
 export { QueryBuilderOperation } from './queryBuilder/operations/QueryBuilderOperation.js';

package/lib/queryBuilder/InternalOptions.js

@@ -6,9 +6,9 @@ export class InternalOptions {
     this.skipRelationFilter = false;
     // The join table extras that related updates write, all of them if null:
     this.joinTableExtras = null;
-    // The field expressions that `upsertGraph()` writes keys in json columns of
-    // updated rows with:
-    this.fieldExpressionKeys = [];
+    // The path updates that `upsertGraph()` writes the keys in json columns of
+    // updated rows with, by column:
+    this.jsonKeyPatches = null;
     this.debug = false;
     this.schema = undefined;
   }
@@ -21,7 +21,7 @@ export class InternalOptions {
     copy.isInternalQuery = this.isInternalQuery;
     copy.skipRelationFilter = this.skipRelationFilter;
     copy.joinTableExtras = this.joinTableExtras;
-    copy.fieldExpressionKeys = this.fieldExpressionKeys;
+    copy.jsonKeyPatches = this.jsonKeyPatches;
     copy.debug = this.debug;
     copy.schema = this.schema;
 

package/lib/queryBuilder/StaticHookArguments.js

@@ -1,4 +1,5 @@
 import { asArray, isObject } from '../utils/objectUtils.js';
+import { AsArrayOperation } from './operations/AsArrayOperation.js';
 
 const BUILDER_SYMBOL = Symbol();
 
@@ -26,12 +27,13 @@ export class StaticHookArguments {
       // ones installed by `throwIfNotFound()` and `traverse()`) are meant for
       // its result, not for the result of this find query. The same goes for
       // the value it was resolved with, e.g. the 0 of an update that only
-      // changes the join rows of a many-to-many relation.
+      // changes the join rows of a many-to-many relation. The query resolves to
+      // an array also for single-result writes, unless the hook adds `first()`:
       return this[BUILDER_SYMBOL].toFindQuery()
         .clear('runAfter')
         .clearResolve()
         .clearWithGraphFetched()
-        .runAfter(asArray);
+        .addOperation(new AsArrayOperation('asArray'), []);
     };
   }
 

package/lib/queryBuilder/graph/GraphSchedule.js

@@ -15,8 +15,9 @@ import { formatPath } from './GraphNode.js';
 //    keys deferred to break cycles and the join rows. Rows that move out of
 //    the slots of HasOneRelations that new rows go into are related in the
 //    same order, as the foreign key may be unique, and so are the patches and
-//    relates that change the keys that these rows read, as a foreign key may
-//    reference them (#428).
+//    relates that change the keys that these rows read or the values that
+//    their `#ref{}` copies, as a foreign key may reference them (#428, #491).
+//    Copies of values that are no keys wait unless that closes a cycle.
 // 4. Patch the changed rows, the other relates, and the keys of owners whose
 //    BelongsToOne rows were inserted, related or removed.
 // 5. Unrelate the rows that their own to-many relation on their own key leaves
@@ -31,7 +32,8 @@ export class GraphSchedule {
     this.keys = keys;
     // The nodes that each node waits for, through its keys, its references and
     // the rows that move out of the slots that it goes into. `isKeyChange`
-    // marks the patches and relates that change the keys it reads:
+    // marks the patches and relates that change the keys it reads or the values
+    // that its `#ref{}` copies, and `isCopy` the latter:
     this.dependencies = createDependencies(plan, this.keys);
     // Dependencies that aren't waited for, to break cycles. Deferred keys are
     // written after the inserts:
@@ -144,6 +146,7 @@ export class GraphSchedule {
           !handledNodes.has(dependency.node) &&
           (dependency.isSlotMove ||
             dependency.isKeyWait ||
+            dependency.isKeyChange ||
             !hasRow(dependency.node) ||
             this.readsFilledProps(dependency)),
       );
@@ -164,12 +167,14 @@ export class GraphSchedule {
 
     // Break a cycle by deferring one BelongsToOne key that points to a `#ref`
     // model: Its owner is inserted without the key, which is patched in once
-    // the referenced model is inserted. Without `#ref`, the graph is a tree,
-    // and only waits for the rows that move out of slots can close a cycle,
-    // like rows that swap slots. One of these isn't waited for then.
+    // the referenced model is inserted. Without `#ref` models, the graph is a
+    // tree, and only `#ref{}` copies of changed values and the waits for the
+    // rows that move out of slots can close a cycle, like rows that swap slots.
+    // One of these isn't waited for then, the copies first.
     const breakCycle = () => {
       for (const canDefer of [
         (dependency, node) => this._canDefer(dependency, node),
+        (dependency) => dependency.isCopy,
         (dependency) => dependency.isSlotMove,
       ]) {
         for (const node of nodes) {
@@ -511,6 +516,15 @@ function createDependencies(plan, keys) {
       }
     }
 
+    // A `#ref{}` that copies a value that the graph changes waits for the
+    // change, as a foreign key may reference it, like the keys (#491). As the
+    // copied value may be no key, the wait is deferred first to break cycles:
+    for (const reference of node.propertyReferences) {
+      for (const changingNode of keys.getChangingNodes(node, reference)) {
+        addDependency(node, { node: changingNode, isKeyChange: true, isCopy: true });
+      }
+    }
+
     // A row that goes into the slot of a HasOneRelation waits for the relate
     // that moves the current row out of it, as the foreign key may be unique.
     const moverNode = getSlotMover(plan, node);

package/lib/queryBuilder/graph/GraphWriter.js

@@ -2,7 +2,7 @@ import { getModel } from '../../model/getModel.js';
 import { restoreOmitFromDatabaseJson } from '../../model/modelUtils.js';
 import { valuesKey } from '../../model/modelValues.js';
 import { UpdateOperation } from '../operations/UpdateOperation.js';
-import { removeFieldExpressions } from '../operations/fieldExpressions.js';
+import { getJsonKeyPatch } from '../operations/fieldExpressions.js';
 import { getMaxRows, hasBatchInsert } from '../../utils/knexUtils.js';
 import { isInternalProp } from '../../utils/internalPropUtils.js';
 import {
@@ -10,6 +10,7 @@ import {
   difference,
   get,
   groupBy,
+  isEmpty,
   isObject,
   jsonEquals,
   nonStrictEquals,
@@ -26,6 +27,10 @@ export class GraphWriter {
   constructor(schedule) {
     this.schedule = schedule;
     this.plan = schedule.plan;
+    // The rows that are written, as the nodes of their models, and whether the
+    // key columns of relations may be null:
+    this.writtenRows = new Set();
+    this.nullableProps = new Map();
   }
 
   async write(builder) {
@@ -154,6 +159,7 @@ export class GraphWriter {
     }
 
     const insertNodes = nodes.filter((node) => actions.get(node) === 'insert');
+    await this._clearSlots(builder, insertNodes);
 
     if (insertNodes.length > 0) {
       const [{ modelClass }] = insertNodes;
@@ -179,6 +185,10 @@ export class GraphWriter {
       await fetchGeneratedKeys(builder, modelClass, insertNodes);
       this.schedule.keys.setInserted(insertNodes);
     }
+
+    for (const node of insertNodes) {
+      this.writtenRows.add(node);
+    }
   }
 
   _writeDeferredKeys(builder, dependencies) {
@@ -253,7 +263,60 @@ export class GraphWriter {
     });
   }
 
+  // Rows that swap slots in a cycle can't take them while the other rows still
+  // hold their keys, if they're unique. Under `unrelate`, the rows that leave
+  // the slots clear their keys first, as 3.x does, if they may be null, and
+  // take their new keys with their own writes, once the other rows took their
+  // slots (#489). Keys that can't be null are swapped directly, and so are keys
+  // in json columns, which hold other values:
+  async _clearSlots(builder, nodes) {
+    for (const { dependent, node, isSlotMove } of this.schedule.deferredDependencies) {
+      // The row that moves out of the slot, with its key:
+      const { modelClass } = node;
+      const relatedProp = isSlotMove && node.relation.relatedProp;
+      if (
+        relatedProp &&
+        nodes.includes(dependent) &&
+        !this.writtenRows.has(node.referencedNode || node) &&
+        relatedProp.props.every((_, i) => relatedProp.isColumn(i)) &&
+        this.plan.graphOptions.has(node, 'unrelate') &&
+        (await this._isNullable(builder, relatedProp))
+      ) {
+        // Without validation or hooks, as the row's own write follows:
+        const { schema } = builder.internalOptions();
+        const query = builder.unsafeKnex()(modelClass.getTableName());
+        const idColumns = modelClass.getIdColumnArray();
+        const ids = modelClass.getIdRelationProperty().getProps(node.obj);
+        await (schema ? query.withSchema(schema) : query)
+          .where(Object.fromEntries(idColumns.map((column, i) => [column, ids[i]])))
+          .update(Object.fromEntries(relatedProp.cols.map((column) => [column, null])));
+      }
+    }
+  }
+
+  // Return true if the database allows null in the columns of `prop`:
+  _isNullable(builder, prop) {
+    let isNullable = this.nullableProps.get(prop);
+
+    if (!isNullable) {
+      isNullable = prop.modelClass
+        .query()
+        .childQueryOf(builder)
+        .columnInfo()
+        .then((columnInfo) => prop.cols.every((column) => columnInfo[column]?.nullable));
+      this.nullableProps.set(prop, isNullable);
+    }
+
+    return isNullable;
+  }
+
   async _patch(builder, nodes) {
+    await this._clearSlots(builder, nodes);
+
+    for (const node of nodes) {
+      this.writtenRows.add(node.referencedNode || node);
+    }
+
     // The patches leave out the props that don't change, but the models that
     // the graph returns get them back once the step is written (#371):
     const omittedProps = new Map(nodes.map((node) => [node, node.obj.$omitFromDatabaseJson()]));
@@ -277,8 +340,6 @@ export class GraphWriter {
     } finally {
       for (const [node, props] of omittedProps) {
         restoreOmitFromDatabaseJson(node.obj, props);
-        // The keys of `setKeyPaths()` stay where a hook resolves the patch:
-        removeFieldExpressions(node.modelClass, node.obj);
       }
     }
   }
@@ -314,24 +375,27 @@ export class GraphWriter {
         ? node.relation.relatedProp.props
         : []),
     ];
-    // The keys in json paths that the write sets or clears:
-    let pathKeys = new Map();
+    // The path updates of the keys in json columns that the write sets or
+    // clears, by column:
+    let jsonKeyPatches = new Map();
     // Changing the id never makes sense with `upsertGraph()`. A many-to-many
     // `#ref` model whose model writes its row only writes its join row:
     const idProps = node.modelClass.getIdPropertyArray();
     const isJoinRowOnly =
       !!referencedNode && node.relation.keySide === 'join' && this.plan.isCoveredByModel(node);
-    const isWritable = (prop) =>
-      !idProps.includes(prop) &&
-      !pathKeys.has(prop) &&
-      !this.schedule.keys.isOnlyRead(node, prop) &&
-      (!isJoinRowOnly || node.relation.joinTableExtras.some((extra) => extra.aliasProp === prop));
+    const isWritable = (prop) => {
+      // A json column that its key paths write is unchanged if they are:
+      const jsonKeyPatch = getJsonKeyPatch(node.obj, prop, jsonKeyPatches);
+      return (
+        !idProps.includes(prop) &&
+        !(jsonKeyPatch && isEmpty(jsonKeyPatch)) &&
+        !this.schedule.keys.isOnlyRead(node, prop) &&
+        (!isJoinRowOnly || node.relation.joinTableExtras.some((extra) => extra.aliasProp === prop))
+      );
+    };
 
     const getPropsToUpdate = (changedProps) =>
-      (shouldPatch || shouldUpdate
-        ? changedProps
-        : keyProps.flatMap((prop) => pathKeys.get(prop) || prop)
-      ).filter(isWritable);
+      (shouldPatch || shouldUpdate ? changedProps : keyProps).filter(isWritable);
 
     // Leave out the props that don't change, and return true if any do.
     const handleUpdate = ({ changedProps, unchangedProps } = this._findChanges(node)) => {
@@ -353,7 +417,7 @@ export class GraphWriter {
     // relates are no updates (#2233, #2605).
     const hasBeforeUpdate = hasHook(node.obj, 'beforeUpdate');
 
-    let changes = this._findChanges(node);
+    const changes = this._findChanges(node);
 
     if (getPropsToUpdate(changes.changedProps).length === 0) {
       return null;
@@ -365,9 +429,8 @@ export class GraphWriter {
     });
 
     // Write the keys in json columns as path updates where their column holds
-    // nothing else, which keeps its other values (#415). After the validation,
-    // as `update()` validates them as properties of their own:
-    pathKeys = setKeyPaths(node, [
+    // nothing else, which keeps its other values (#415):
+    jsonKeyPatches = getJsonKeyPatches(node, [
       ...this.schedule.keys.getSetCells(node),
       ...this.plan
         .getRemovedOwnerRelations(node)
@@ -376,10 +439,6 @@ export class GraphWriter {
         ),
     ]);
 
-    if (pathKeys.size > 0) {
-      changes = this._findChanges(node);
-    }
-
     const propsToUpdate = getPropsToUpdate(changes.changedProps);
 
     if (propsToUpdate.length === 0) {
@@ -402,7 +461,7 @@ export class GraphWriter {
       hasBeforeUpdate,
       handleUpdate,
       propsToUpdate,
-      fieldExpressionKeys: [...pathKeys.values()].flat(),
+      jsonKeyPatches,
       isPatchedThroughRelation,
     };
   }
@@ -415,7 +474,7 @@ export class GraphWriter {
       hasBeforeUpdate,
       handleUpdate,
       propsToUpdate,
-      fieldExpressionKeys,
+      jsonKeyPatches,
       isPatchedThroughRelation,
     },
   ) {
@@ -439,8 +498,7 @@ export class GraphWriter {
         // Join rows are patched with the extras among the props to update. The
         // extras of new join rows are written with them, see `_findChanges()`:
         joinTableExtras: node.relation?.keySide === 'join' && !node.current ? [] : propsToUpdate,
-        // The keys in json columns, also in updates:
-        fieldExpressionKeys,
+        jsonKeyPatches,
       })
       .copyFrom(builder, isReturningAll);
 
@@ -796,12 +854,12 @@ async function fetchGeneratedKeys(builder, modelClass, nodes) {
   }
 }
 
-// Add the changed keys of `cells` at the top level of json columns to `node` as
-// field expressions where their column holds nothing but these keys, and return
-// them by column. PostgreSQL and MySQL can't write nested paths whose parents
-// are missing, so their columns are written whole.
-function setKeyPaths(node, cells) {
-  const keysByColumn = new Map();
+// Return the path updates of the changed keys of `cells` at the top level of
+// json columns, by the columns that hold nothing but these keys. PostgreSQL and
+// MySQL can't write nested paths whose parents are missing, so their columns
+// are written whole.
+function getJsonKeyPatches(node, cells) {
+  const patchesByColumn = new Map();
   const pathCells = cells.filter(({ prop, index }) => prop.isJsonKey(index));
 
   for (const [column, columnCells] of groupBy(pathCells, ({ prop, index }) => prop.props[index])) {
@@ -822,12 +880,11 @@ function setKeyPaths(node, cells) {
         }
       }
 
-      Object.assign(node.obj, patch);
-      keysByColumn.set(column, Object.keys(patch));
+      patchesByColumn.set(column, { keys: keys[column], patch });
     }
   }
 
-  return keysByColumn;
+  return patchesByColumn;
 }
 
 function hasOwnerKeyArray(relation) {