json-rest-api 3.0.0-rc.0 → 3.0.0

package/plugins/core/lib/storage/many-row-data-helpers.js

@@ -2,9 +2,11 @@
 // multi-row inserts, by-id CASE updates, by-id deletes and row locks. Every write
 // runs one statement per chunk; none of them runs through the filter hooks, so
 // MySQL never sees an UPDATE or DELETE whose subquery reads its own table.
-import { RestApiBulkInsertIdsError, RestApiValidationError } from '../../../../lib/rest-api-errors.js'
+import { RestApiBulkInsertIdsError, RestApiBulkWriteUnsupportedError, RestApiResourceError, RestApiValidationError } from '../../../../lib/rest-api-errors.js'
 import { withQueryFilteringContext } from '../querying/query-builder-utils.js'
-import { applyDatabaseReadOptions, databaseIdentityExpression } from '../querying-writing/database-value-normalizers.js'
+import { ANY_OWNER, applyWriteConditions, hasWriteConditions, writeConditions } from '../querying/query-constraint.js'
+import { applyDatabaseReadOptions, databaseIdentityExpression, normalizeStoredFieldValue } from '../querying-writing/database-value-normalizers.js'
+import { currentKnexMysqlSchema, listKnexMysqlIndexes, listKnexUniqueKeyColumns } from '../dbIntrospection.js'
 import { applyFieldSelectionToQuery, buildFieldSelection } from '../querying-writing/knex-field-helpers.js'
 import { applyInsertReturning } from '../querying-writing/database-capabilities.js'
 import { loadRelationshipIdentifiers } from '../querying/relationship-identifiers.js'
@@ -24,6 +26,38 @@ export function databaseFamily (db) {
   return 'mysql'
 }
 
+// The mysql and mysql2 drivers' client flag that makes an UPDATE count the rows it matched.
+const FOUND_ROWS = 0x2
+
+/**
+ * Refuses a selection write on MySQL or MariaDB whose scope joins a derived
+ * table, as a filter on another table's field does: InnoDB reads a derived
+ * table from the snapshot without locking it, so the write could not re-check
+ * that filter on a row it waited for.
+ * @param {{ db: any, scope: any, filters?: Record<string, unknown>, method: string, scopeName: string }} request
+ */
+function refuseUnlockedSelection ({ db, scope, filters, method, scopeName }) {
+  if (!filters || !Object.keys(filters).length || databaseFamily(db) !== 'mysql') return
+  if (!scope._statements.some(statement => statement.grouping === 'join' && typeof statement.table !== 'string')) return
+  const message = `${method} on '${scopeName}' filters on another table, which MySQL and MariaDB read from the transaction snapshot without locking it; filter on the row's own columns instead, for example on a link after locking its targets with lockByIds`
+  throw new RestApiValidationError(message, { fields: ['filters'], violations: [{ field: 'filters', rule: 'selection_cross_table', message }] })
+}
+
+/**
+ * Whether an UPDATE's count is the rows it matched. PostgreSQL and SQLite
+ * always count those; MySQL and MariaDB do so only on a connection with the
+ * FOUND_ROWS client flag, the drivers' default, and count the rows it changed
+ * otherwise. A connection this cannot see counts as not.
+ * @param {any} db
+ */
+export async function countsMatchedRows (db) {
+  if (databaseFamily(db) !== 'mysql') return true
+  if (!db.isTransaction) return false
+  // A transaction's client hands back its own connection.
+  const connection = await db.client.acquireConnection()
+  return ((connection?.config?.clientFlags ?? 0) & FOUND_ROWS) !== 0
+}
+
 /** @param {{ dbInfo?: { version?: string } } | undefined} capabilities */
 function supportsMariaDbReturning (capabilities) {
   const version = String(capabilities?.dbInfo?.version || '').toLowerCase()
@@ -53,15 +87,47 @@ export function chunk (items, size) {
   return chunks
 }
 
-/** @param {unknown} value @returns {boolean} */
-export const isRawValue = value => Boolean(value && typeof value === 'object' && /** @type {{ isRawInstance?: boolean }} */ (value).isRawInstance === true)
+/**
+ * A Knex raw expression. JSON input cannot produce one: the check needs its
+ * toSQL function, not only the isRawInstance flag.
+ * @param {unknown} value @returns {boolean}
+ */
+export function isRawValue (value) {
+  const raw = /** @type {{ isRawInstance?: boolean, toSQL?: unknown } | null} */ (value)
+  return Boolean(raw && typeof raw === 'object' && raw.isRawInstance === true && typeof raw.toSQL === 'function')
+}
 
-// Conflict keys compare by exact value; drivers return numbers for integer keys.
-const keyOf = (row, columns) => JSON.stringify(columns.map(column => {
-  const value = row[column]
-  if (value === null || value === undefined) return null
-  return typeof value === 'bigint' || typeof value === 'number' ? String(value) : value instanceof Date ? value.toISOString() : String(value)
-}))
+/**
+ * Conflict keys compare exactly, in their public form: drivers return booleans
+ * as 1 and 0, numbers as strings and dates in their own shapes, so each value
+ * goes through its field's read normalization first. Strings compare as they
+ * are, without the column's collation. A field with a storage serializer
+ * compares its stored form.
+ * @param {Record<string, any>} schemaStructure
+ * @param {string[]} keys
+ * @param {string[]} keyColumns
+ */
+function conflictKeyMatcher (schemaStructure, keys, keyColumns) {
+  const definitions = keys.map(field => schemaStructure[field] || {})
+  const canonical = (value, index) => {
+    if (value === null || value === undefined) return null
+    const definition = definitions[index]
+    const normal = definition.storage?.serialize ? value : normalizeStoredFieldValue(value, definition, { fieldName: keys[index] })
+    if (normal === null || normal === undefined) return null
+    if (normal instanceof Date) return normal.toISOString()
+    if (['number', 'integer'].includes(definition.type) && typeof normal === 'string') {
+      return /^-?\d+$/.test(normal) ? String(BigInt(normal)) : String(Number(normal))
+    }
+    return String(normal)
+  }
+  return {
+    /** A row's key from its public attributes, or its storage row for serialized fields. */
+    input: (attributes, storageRow) => JSON.stringify(keys.map((field, index) =>
+      canonical(definitions[index].storage?.serialize ? storageRow[keyColumns[index]] : attributes[field], index))),
+    /** A key read from the table. */
+    stored: row => JSON.stringify(keyColumns.map((column, index) => canonical(row[column], index)))
+  }
+}
 
 /**
  * @param {object} dependencies
@@ -77,9 +143,31 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     return adapter
   }
 
+  // Tables and key columns a unique index was found to cover. A missing index is
+  // looked for again on the next call, so adding one needs no restart.
+  const coveredKeys = new Set()
+  /**
+   * Refuses conflict keys that no unique index covers. Without one, the keys
+   * could match several rows, and a locking read on them would examine (and on
+   * MySQL lock) every row of the table.
+   */
+  const requireUniqueKeys = async ({ scopeName, db, tableName, keyColumns }) => {
+    const cacheKey = JSON.stringify([tableName, [...keyColumns].sort()])
+    if (coveredKeys.has(cacheKey)) return
+    const uniqueKeys = await listKnexUniqueKeyColumns(db, tableName)
+    if (!uniqueKeys.some(columns => columns.every(column => keyColumns.includes(column)))) {
+      throw new RestApiBulkWriteUnsupportedError({
+        resourceType: scopeName,
+        operation: 'postMany',
+        reason: `no unique index of '${tableName}' covers the onConflict keys (${keyColumns.join(', ')}); add one, so that the keys identify one row`
+      })
+    }
+    coveredKeys.add(cacheKey)
+  }
+
   /**
    * The resource's base query with ownership, row policy and `filters` applied,
-   * as `{ query }`: a builder is thenable, so an async function cannot return it bare.
+   * as `{ query, alias }`: a builder is thenable, so an async function cannot return it bare.
    * @param {{ scopeName: string, context: any, runHooks: Function, alias?: string, purpose: string, filters?: Record<string, unknown> }} request
    */
   const dataScopedQuery = async ({ scopeName, context, runHooks, alias, purpose, filters }) => {
@@ -99,7 +187,7 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
       adapter: storageAdapter,
       storageAdapter
     }, () => runHooks('knexQueryFiltering'))
-    return { query: state.query }
+    return { query: state.query, alias: tableAlias }
   }
 
   /**
@@ -114,6 +202,8 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     const tableName = storageAdapter.getTableName()
     const idColumn = `${tableName}.${storageAdapter.getIdColumn()}`
     const { query } = await dataScopedQuery({ scopeName, context, runHooks, purpose: queryPurpose, filters })
+    // A selection write reads its rows here first, so it is refused here whatever rows match.
+    if (queryPurpose === 'bulk-write') refuseUnlockedSelection({ db, scope: query, filters, method: context.method, scopeName })
     if (ids) query.whereIn(idColumn, ids.map(id => storageAdapter.translateFilterValue('id', id)))
     if (after !== undefined) query.where(idColumn, '>', storageAdapter.translateFilterValue('id', after))
     // Policy joins can repeat a row; the identity projection keeps big integers exact.
@@ -123,11 +213,49 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     return rows.map(row => row.id)
   }
 
+  /**
+   * The stored identifier of the row each input row matches by `keys`, or
+   * undefined, read with a row lock (SQLite's single writer needs none), one
+   * statement per chunk. Upserts decide from it which rows to insert and which
+   * to merge.
+   * @param {{ scopeName: string, context: any, rows: Record<string, unknown>[], keys: string[], chunkSize: number }} request
+   * @returns {Promise<Array<string | number | undefined>>}
+   */
+  const dataIdsByKeys = async ({ scopeName, context, rows, keys, chunkSize }) => {
+    const storageAdapter = adapterFor(scopeName)
+    const db = context.transaction
+    const tableName = storageAdapter.getTableName()
+    const keyColumns = keys.map(field => storageAdapter.translateColumn(field))
+    await requireUniqueKeys({ scopeName, db, tableName, keyColumns })
+    await takeSqliteWriteLock(db, tableName, storageAdapter.getIdColumn())
+    const storageRows = rows.map(attributes => storageAdapter.toStorageRow(attributes, { context, operation: 'post' }))
+    const matcher = conflictKeyMatcher(scopes[scopeName].vars.schemaInfo.schemaStructure, keys, keyColumns)
+    const inputKeys = rows.map((attributes, index) => matcher.input(attributes, storageRows[index]))
+    const wanted = new Set(inputKeys)
+    const found = new Map()
+    for (const part of chunk(storageRows, rowsPerStatement(db, chunkSize, keyColumns.length))) {
+      const query = db(tableName)
+        .whereIn(keyColumns, part.map(row => keyColumns.map(column => row[column] ?? null)))
+        .select({ id: databaseIdentityExpression(db, storageAdapter.getIdColumn()) }, ...keyColumns)
+      for (const row of await applyDatabaseReadOptions(lockRows(query, db))) {
+        const key = matcher.stored(row)
+        // The database matched by its own comparison (a collation, say); merging or skipping would guess.
+        if (!wanted.has(key)) {
+          throw new RestApiResourceError(`A stored ${scopeName} record matches the onConflict keys (${keys.join(', ')}) only as the database compares them, for example ignoring letter case; upsert keys must match exactly`, {
+            subtype: 'conflict', resourceType: scopeName
+          })
+        }
+        found.set(key, row.id)
+      }
+    }
+    return inputKeys.map(key => found.get(key))
+  }
+
   /**
    * Insert validated rows with one statement per chunk and column set.
    * Returns the stored identifier for each input row (undefined for rows an
    * ignore conflict skipped), or nothing when `needIds` is false.
-   * @param {{ scopeName: string, context: any, rows: Record<string, unknown>[], chunkSize: number, needIds: boolean, onConflict?: { keys: string[], action: 'ignore' | 'merge', merge: string[] } }} request
+   * @param {{ scopeName: string, context: any, rows: Record<string, unknown>[], chunkSize: number, needIds: boolean, onConflict?: { keys: string[], action: 'ignore' } }} request
    * @returns {Promise<Array<string | number | undefined> | undefined>}
    */
   const dataInsertMany = async ({ scopeName, context, rows, chunkSize, needIds, onConflict }) => {
@@ -136,12 +264,25 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     const tableName = storageAdapter.getTableName()
     const idColumn = storageAdapter.getIdColumn()
     const family = databaseFamily(db)
-    const storageRows = rows.map(attributes => storageAdapter.toStorageRow(attributes, { context, operation: 'post' }))
-    const keyColumns = onConflict?.keys.map(field => storageAdapter.translateColumn(field))
-    const lockVersionColumn = context.schemaInfo.lockVersionField ? storageAdapter.translateColumn(context.schemaInfo.lockVersionField) : null
     /** @type {Array<string | number | undefined>} */
     const ids = new Array(rows.length)
 
+    if (onConflict && family === 'mysql') {
+      // ON DUPLICATE KEY fires on every unique index, the primary key included:
+      // lock the rows that match by the keys, then insert only the others with a
+      // plain INSERT, so a clash on any other unique index fails the batch.
+      const existing = await dataIdsByKeys({ scopeName, context, rows, keys: onConflict.keys, chunkSize })
+      const missing = rows.flatMap((_row, index) => existing[index] === undefined ? [index] : [])
+      const inserted = missing.length
+        ? await dataInsertMany({ scopeName, context, rows: missing.map(index => rows[index]), chunkSize, needIds })
+        : []
+      missing.forEach((index, position) => { ids[index] = inserted?.[position] })
+      return needIds ? ids : undefined
+    }
+
+    const storageRows = rows.map(attributes => storageAdapter.toStorageRow(attributes, { context, operation: 'post' }))
+    const keyColumns = onConflict?.keys.map(field => storageAdapter.translateColumn(field))
+
     // Rows with the same columns share a statement; SQLite cannot insert DEFAULT for a missing value.
     const groups = new Map()
     storageRows.forEach((row, index) => {
@@ -155,22 +296,8 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
       for (const part of chunk(indexes, rowsPerStatement(db, chunkSize, columns.length + 1, { insert: true }))) {
         const values = part.map(index => storageRows[index])
         let query = db(tableName).insert(values)
-        if (onConflict) query = applyConflict(query, { db, family, tableName, idColumn, keyColumns, onConflict, storageAdapter, lockVersionColumn })
-
-        if (onConflict && family === 'mysql') {
-          // Neither MySQL nor MariaDB reports which rows an upsert inserted: read the keys around it.
-          const existing = onConflict.action === 'ignore' && needIds
-            ? new Set((await readByKeys(db, tableName, idColumn, keyColumns, values)).map(row => keyOf(row, keyColumns)))
-            : null
-          await query
-          const found = new Map((await readByKeys(db, tableName, idColumn, keyColumns, values)).map(row => [keyOf(row, keyColumns), row.id]))
-          part.forEach((index, position) => {
-            const key = keyOf(values[position], keyColumns)
-            if (!found.has(key)) throw conflictElsewhere(scopeName, onConflict)
-            if (!existing?.has(key)) ids[index] = found.get(key)
-          })
-          continue
-        }
+        // Only a clash on the conflict keys is skipped; any other unique index still fails.
+        if (onConflict) query = query.onConflict(keyColumns).ignore()
 
         if (!needIds) {
           await query
@@ -178,8 +305,7 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
         }
 
         if (family === 'mysql' && supportsMariaDbReturning(api.knex.capabilities)) {
-          const { sql, bindings } = query.toSQL()
-          const [returned] = await db.raw(`${sql} returning ${db.raw('??', [idColumn]).toQuery()}`, bindings)
+          const [returned] = await db.raw('? returning ??', [query, idColumn])
           returned.forEach((row, position) => { ids[part[position]] = row[idColumn] })
           continue
         }
@@ -203,8 +329,9 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
         let returned = await applyDatabaseReadOptions(applyInsertReturning(query, returnColumns))
         if (onConflict) {
           // Ignored rows return nothing, so returned rows are matched to inputs by their keys.
-          const byKey = new Map(returned.map(row => [keyOf(Object.fromEntries(keyColumns.map(column => [column, row[`__jra_key_${column}`]])), keyColumns), row[idColumn]]))
-          part.forEach((index, position) => { ids[index] = byKey.get(keyOf(values[position], keyColumns)) })
+          const matcher = conflictKeyMatcher(scopes[scopeName].vars.schemaInfo.schemaStructure, onConflict.keys, keyColumns)
+          const byKey = new Map(returned.map(row => [matcher.stored(Object.fromEntries(keyColumns.map(column => [column, row[`__jra_key_${column}`]]))), row[idColumn]]))
+          part.forEach((index, position) => { ids[index] = byKey.get(matcher.input(rows[index], values[position])) })
           continue
         }
         if (family === 'sqlite' && values.some(row => row[idColumn] === undefined || row[idColumn] === null)) {
@@ -217,14 +344,61 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     return needIds ? ids : undefined
   }
 
+  // The rows a by-id UPDATE or DELETE writes: the ids, onlyIf, the caller's
+  // ownership conditions and, with `scoped`, the caller's scope (with a
+  // selection's filters), built once (its filtering hooks record the ownership
+  // conditions).
+  const writeMatcher = async ({ scopeName, context, runHooks, scoped, onlyIf, filters }) => {
+    const storageAdapter = adapterFor(scopeName)
+    const db = context.transaction
+    const family = databaseFamily(db)
+    const tableName = storageAdapter.getTableName()
+    const idColumn = storageAdapter.getIdColumn()
+    const identity = { id: databaseIdentityExpression(db, `${tableName}.${idColumn}`) }
+    const scope = scoped ? (await dataScopedQuery({ scopeName, context, runHooks, purpose: 'bulk-write', filters })).query : null
+    const guard = query => applyWriteConditions({ query: applyOnlyIf(query, onlyIf || {}, tableName), context, scopeName, storageAdapter, tableName })
+    // The WHERE of the write, with the scope as a subquery.
+    const matching = ids => {
+      const query = guard(db(tableName).whereIn(idColumn, ids))
+      return scope ? query.whereIn(idColumn, scope.clone().whereIn(`${tableName}.${idColumn}`, ids).clearSelect().select(`${tableName}.${idColumn}`)) : query
+    }
+    // A selection's filters must hold for a row as it is once locked, after any
+    // wait for another writer, or two writers that both matched the old row
+    // would both write it. A subquery does not see that row: InnoDB reads it
+    // from the transaction snapshot, PostgreSQL as of the statement's start.
+    const selection = Boolean(scope && filters && Object.keys(filters).length)
+    if (selection) refuseUnlockedSelection({ db, scope, filters, method: context.method, scopeName })
+    return {
+      matching,
+      // PostgreSQL: lock the rows first, so the statement that writes them
+      // starts after any wait and reads what their holder committed.
+      settle: async ids => {
+        if (selection && family === 'pg') await lockRows(db(tableName).whereIn(idColumn, ids).orderBy(idColumn), db).select(idColumn)
+      },
+      // The ids that still match, locked. MySQL and MariaDB check a selection's
+      // scope and filters in the locking read's own query block, on the
+      // current row.
+      locked: async ids => {
+        const query = selection && family === 'mysql'
+          ? guard(scope.clone().whereIn(`${tableName}.${idColumn}`, ids)).clearSelect()
+          : matching(ids)
+        const rows = await applyDatabaseReadOptions(lockRows(query, db).select(identity))
+        // A join of the scope or the filters can repeat a row.
+        return [...new Map(rows.map(row => [String(row.id), row.id])).values()]
+      }
+    }
+  }
+
   /**
    * Update visible rows by id: one UPDATE per chunk, `CASE id WHEN … THEN …`
    * for values that differ between rows and a plain SET for shared ones.
+   * With `scoped`, each statement carries the caller's scope (row policy
+   * included) and `filters` as a subquery.
    * Returns the identifiers the statements changed.
-   * @param {{ scopeName: string, context: any, updates: Array<{ id: string | number, row: Record<string, unknown> }>, onlyIf?: Record<string, unknown>, chunkSize: number }} request
+   * @param {{ scopeName: string, context: any, runHooks?: Function, updates: Array<{ id: string | number, row: Record<string, unknown> }>, onlyIf?: Record<string, unknown>, chunkSize: number, scoped?: boolean, filters?: Record<string, unknown> }} request
    * @returns {Promise<Array<string | number>>}
    */
-  const dataUpdateByIds = async ({ scopeName, context, updates, onlyIf, chunkSize }) => {
+  const dataUpdateByIds = async ({ scopeName, context, runHooks, updates, onlyIf, chunkSize, scoped = false, filters }) => {
     const storageAdapter = adapterFor(scopeName)
     const db = context.transaction
     const tableName = storageAdapter.getTableName()
@@ -232,9 +406,13 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     const family = databaseFamily(db)
     const lockVersionColumn = context.schemaInfo.lockVersionField ? storageAdapter.translateColumn(context.schemaInfo.lockVersionField) : null
     const columnCount = new Set(updates.flatMap(update => Object.keys(update.row))).size
+    const { matching, settle, locked } = await writeMatcher({ scopeName, context, runHooks, scoped, onlyIf, filters })
+    // onlyIf, the scope and the caller's ownership narrow each statement; without them every listed id is written.
+    const guarded = scoped || Boolean(onlyIf) || hasWriteConditions(context, scopeName)
     const changed = []
 
-    for (const part of chunk(updates, rowsPerStatement(db, chunkSize, 2 * columnCount + 1))) {
+    // A scoped statement binds each id twice, once more in its subquery.
+    for (const part of chunk(updates, rowsPerStatement(db, chunkSize, 2 * columnCount + (scoped ? 2 : 1)))) {
       const set = {}
       const columns = [...new Set(part.flatMap(update => Object.keys(update.row)))]
       for (const column of columns) {
@@ -254,23 +432,27 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
         set[column] = db.raw(`${sql} else ?? end`, [...bindings, column])
       }
       if (lockVersionColumn) set[lockVersionColumn] = db.raw('?? + 1', [lockVersionColumn])
-      if (!Object.keys(set).length) {
-        changed.push(...part.map(update => update.id))
-        continue
-      }
+      const writes = Object.keys(set).length > 0
       const ids = part.map(update => storageAdapter.translateFilterValue('id', update.id))
-      const query = db(tableName).whereIn(idColumn, ids)
-      if (!onlyIf) {
-        await query.update(set)
+      if (!guarded) {
+        if (writes) await db(tableName).whereIn(idColumn, ids).update(set)
         changed.push(...part.map(update => update.id))
-      } else if (family === 'mysql') {
-        // MySQL cannot return updated rows: lock the rows that match, then update exactly those.
-        const matched = await applyDatabaseReadOptions(applyOnlyIf(db(tableName).whereIn(idColumn, ids), onlyIf)
-          .forUpdate().select({ id: databaseIdentityExpression(db, idColumn) }))
-        if (matched.length) await db(tableName).whereIn(idColumn, matched.map(row => row.id)).update(set)
-        changed.push(...matched.map(row => row.id))
+      } else if (family === 'mysql' || !writes) {
+        // MySQL cannot return updated rows. One row's UPDATE answers from its
+        // count when that counts matched rows; with a scope subquery it would
+        // name the updated table, which MySQL refuses.
+        if (writes && !scoped && part.length === 1 && await countsMatchedRows(db)) {
+          if (await matching(ids).update(set)) changed.push(part[0].id)
+          continue
+        }
+        // Otherwise lock the rows that match, then update exactly those.
+        await settle(ids)
+        const matched = await locked(ids)
+        if (matched.length && writes) await db(tableName).whereIn(idColumn, matched).update(set)
+        changed.push(...matched)
       } else {
-        const rows = await applyDatabaseReadOptions(applyOnlyIf(query, onlyIf).returning({ id: databaseIdentityExpression(db, idColumn) }).update(set))
+        await settle(ids)
+        const rows = await applyDatabaseReadOptions(matching(ids).returning({ id: databaseIdentityExpression(db, idColumn) }).update(set))
         changed.push(...rows.map(row => row.id))
       }
     }
@@ -279,15 +461,45 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
 
   /**
    * Delete visible rows by id, one DELETE per chunk, then their pivot rows.
-   * @param {{ scopeName: string, context: any, ids: Array<string | number>, chunkSize: number }} request
+   * With `scoped`, each DELETE carries the caller's scope and `filters` (see
+   * dataUpdateByIds).
+   * Returns the identifiers the statements deleted.
+   * @param {{ scopeName: string, context: any, runHooks?: Function, ids: Array<string | number>, chunkSize: number, scoped?: boolean, filters?: Record<string, unknown> }} request
+   * @returns {Promise<Array<string | number>>}
    */
-  const dataDeleteByIds = async ({ scopeName, context, ids, chunkSize }) => {
+  const dataDeleteByIds = async ({ scopeName, context, runHooks, ids, chunkSize, scoped = false, filters }) => {
     const storageAdapter = adapterFor(scopeName)
     const db = context.transaction
-    for (const part of chunk(ids, rowsPerStatement(db, chunkSize, 1))) {
-      await db(storageAdapter.getTableName()).whereIn(storageAdapter.getIdColumn(), part.map(id => storageAdapter.translateFilterValue('id', id))).delete()
-      await deletePivotReferences(api, scopeName, context, part)
+    const family = databaseFamily(db)
+    const tableName = storageAdapter.getTableName()
+    const idColumn = storageAdapter.getIdColumn()
+    const identity = { id: databaseIdentityExpression(db, idColumn) }
+    const { matching, settle, locked } = await writeMatcher({ scopeName, context, runHooks, scoped, filters })
+    const guarded = scoped || hasWriteConditions(context, scopeName)
+    const deleted = []
+    for (const part of chunk(ids, rowsPerStatement(db, chunkSize, scoped ? 2 : 1))) {
+      let values = part.map(id => storageAdapter.translateFilterValue('id', id))
+      if (guarded && family === 'pg') {
+        // The scope and the caller's ownership narrow the DELETE, which returns the rows it removed.
+        await settle(values)
+        values = (await applyDatabaseReadOptions(matching(values).returning(identity).delete())).map(row => row.id)
+      } else if (guarded && family === 'sqlite') {
+        // Knex reads no rows back from a SQLite DELETE, so its RETURNING runs as
+        // raw SQL around the builder, which keeps its own bindings: a '?' inside
+        // a string literal of a row policy stays text.
+        values = (await db.raw('? returning ? as ??', [matching(values).delete(), identity.id, 'id'])).map(row => row.id)
+      } else if (guarded) {
+        // MySQL cannot return deleted rows: lock the rows that still match, then delete exactly those.
+        values = await locked(values)
+        if (values.length) await db(tableName).whereIn(idColumn, values).delete()
+      } else {
+        await db(tableName).whereIn(idColumn, values).delete()
+      }
+      if (!values.length) continue
+      await deletePivotReferences(api, scopeName, context, values)
+      deleted.push(...values)
     }
+    return deleted
   }
 
   // Reads rows through the get field selection, so they leave as response records.
@@ -305,24 +517,133 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
     return buildJsonApiResponse(scope, rows, [], false, scopeName, context)
   }
 
+  // SQLite has one writer, but a deferred transaction's reads take only a shared
+  // lock: two connections could both read as if they held the rows, and the
+  // later write would fail with SQLITE_BUSY halfway through. A write statement
+  // takes the write lock even when it changes nothing, so a SQLite lock read
+  // starts with one, once per transaction; the lock read itself is then plain.
+  const writeLocked = new WeakSet()
+  const takeSqliteWriteLock = async (db, tableName, idColumn) => {
+    if (databaseFamily(db) !== 'sqlite' || writeLocked.has(db)) return
+    await db(tableName).whereRaw('0 = 1').update({ [idColumn]: db.ref(idColumn) })
+    writeLocked.add(db)
+  }
+
   const lockRows = (query, db, { skipLocked = false } = {}) => {
-    // SQLite has a single writer, so its transaction already excludes other writers.
     if (databaseFamily(db) === 'sqlite') return query
     query.forUpdate()
     return skipLocked ? query.skipLocked() : query
   }
 
+  // The scoped query of a lock or claim, refused unless it is still a SELECT on
+  // the resource's own table: FOR UPDATE on a derived table or a common table
+  // expression locks nothing and reads the transaction snapshot, and InnoDB
+  // would lock every joined row the read examines.
+  const lockingQuery = async ({ scopeName, context, runHooks, operation, purpose, filters }) => {
+    const storageAdapter = adapterFor(scopeName)
+    const tableName = storageAdapter.getTableName()
+    const { query } = await dataScopedQuery({ scopeName, context, runHooks, purpose, filters })
+    const added = query._statements.find(statement => ['join', 'with', 'union'].includes(statement.grouping))?.grouping
+    if (added || query._single.table !== tableName) {
+      const what = { join: 'joins', with: 'a common table expression', union: 'a set operation' }[added] ?? 'another source table'
+      const message = `${operation} on '${scopeName}' needs a single-table query, but its filters or row policy add ${what}`
+      throw new RestApiValidationError(message, { fields: ['filters'], violations: [{ field: 'filters', rule: `${purpose}_single_table`, message }] })
+    }
+    await takeSqliteWriteLock(context.db, tableName, storageAdapter.getIdColumn())
+    return { query, storageAdapter, tableName, idColumn: `${tableName}.${storageAdapter.getIdColumn()}` }
+  }
+
   /**
-   * Lock rows already known to be visible, in id order, on the bare table.
+   * Lock the visible rows among `ids` in id order and read them, in one locking
+   * read that carries the scope and row-policy predicates. No plain read comes
+   * first: under REPEATABLE READ (MySQL, MariaDB) it would fix the snapshot
+   * before the lock wait, hiding what the lock holder commits meanwhile.
    * Returns the rows the locking read saw as a JSON:API document.
-   * @param {{ scopeName: string, context: any, ids: Array<string | number> }} request
+   * @param {{ scopeName: string, context: any, runHooks: Function, ids: Array<string | number> }} request
    */
-  const dataLockByIds = async ({ scopeName, context, ids }) => {
-    const storageAdapter = adapterFor(scopeName)
-    const tableName = storageAdapter.getTableName()
-    const idColumn = `${tableName}.${storageAdapter.getIdColumn()}`
-    const query = lockRows(context.db(tableName).whereIn(idColumn, ids.map(id => storageAdapter.translateFilterValue('id', id))).orderBy(idColumn), context.db)
-    return readRecords({ scopeName, context, query, tableName })
+  const dataLockByIds = async ({ scopeName, context, runHooks, ids }) => {
+    const { query, storageAdapter, tableName, idColumn } = await lockingQuery({ scopeName, context, runHooks, operation: 'lockByIds', purpose: 'lock' })
+    query.whereIn(idColumn, ids.map(id => storageAdapter.translateFilterValue('id', id))).orderBy(idColumn)
+    return readRecords({ scopeName, context, query: lockRows(query, context.db), tableName })
+  }
+
+  // The index lockWhere forces on MySQL, by schema, table and the read's columns (see lockIndexFor).
+  const lockIndexes = new Map()
+  /**
+   * InnoDB locks every row a locking read examines, and holds those locks until
+   * commit. lockWhere on MySQL therefore forces an index that keeps its read
+   * inside the caller's rows: one whose leading columns are the whole
+   * ownership columns the read is limited to, or, without ownership
+   * conditions, one that starts with a column the filters compare. Otherwise
+   * the read could walk the primary key, locking other owners' rows and the
+   * gaps between them. Invisible and ignored indexes cannot be forced.
+   * Returns the index name.
+   * @param {{ scopeName: string, context: any, storageAdapter: any, tableName: string, filters?: Record<string, unknown> }} request
+   * @returns {Promise<string>}
+   */
+  const lockIndexFor = async ({ scopeName, context, storageAdapter, tableName, filters = {} }) => {
+    const owned = Object.entries(context[writeConditions]?.[scopeName] || {})
+      .filter(([, value]) => value !== null && value !== ANY_OWNER)
+      .map(([field]) => storageAdapter.translateColumn(field))
+    const search = context.schemaInfo.searchSchemaStructure || {}
+    const compared = Object.keys(filters).flatMap(key => {
+      const definition = search[key]
+      if (!definition || definition.oneOf || definition.applyFilter || definition.polymorphicField || definition.actualField?.includes('.')) return []
+      return [storageAdapter.translateColumn(definition.actualField || key)]
+    })
+    // One pool can serve several databases, so the cache is per schema.
+    const schema = await currentKnexMysqlSchema(context.db)
+    const cacheKey = JSON.stringify([schema, tableName, [...owned].sort(), [...compared].sort()])
+    if (lockIndexes.has(cacheKey)) return lockIndexes.get(cacheKey)
+    const indexes = await listKnexMysqlIndexes(context.db, schema, tableName)
+    const leading = owned.length
+      ? indexes.filter(({ columns }) => columns.length >= owned.length &&
+        columns.slice(0, owned.length).every(column => owned.includes(column.name) && column.prefix === null))
+      : indexes.filter(({ columns }) => compared.includes(columns[0]?.name))
+    // Among those, one whose next column the filters compare locks fewer rows.
+    const index = leading.find(({ columns }) => compared.includes(columns[owned.length]?.name)) ?? leading[0]
+    if (!index) {
+      throw new RestApiBulkWriteUnsupportedError({
+        resourceType: scopeName,
+        operation: 'lockWhere',
+        reason: `on MySQL and MariaDB a locking read locks every row it examines; add a visible index of '${tableName}' that starts with ${owned.length ? `the whole ownership column(s) ${owned.join(', ')}` : 'a column the filters compare'}`
+      })
+    }
+    lockIndexes.set(cacheKey, index.name)
+    return index.name
+  }
+
+  /**
+   * Lock up to `limit` visible rows the filters select, waiting for
+   * transactions that hold them, in one locking read like dataLockByIds: in id
+   * order, except on MySQL, where the read is forced onto the index that
+   * lockIndexFor picks and the rows come in its order, as ORDER BY id could
+   * walk the primary key.
+   * @param {{ scopeName: string, context: any, runHooks: Function, filters?: Record<string, unknown>, limit: number }} request
+   */
+  const dataLockWhere = async ({ scopeName, context, runHooks, filters, limit }) => {
+    const read = async () => {
+      const { query, storageAdapter, tableName, idColumn } = await lockingQuery({ scopeName, context, runHooks, operation: 'lockWhere', purpose: 'lock', filters })
+      if (databaseFamily(context.db) === 'mysql') {
+        // Without FORCE INDEX the optimizer could still pick another index or a full scan.
+        const index = await lockIndexFor({ scopeName, context, storageAdapter, tableName, filters })
+        query.from(context.db.raw('?? force index (??)', [tableName, index]))
+      } else query.orderBy(idColumn)
+      return readRecords({ scopeName, context, query: lockRows(query.limit(limit), context.db), tableName })
+    }
+    let document = await read()
+    // A PostgreSQL statement finds only the rows committed when it began; after a
+    // lock wait it rechecks those, but never sees rows that started to match
+    // meanwhile. Read again, under the locks already held, until none appears.
+    // (InnoDB's locking reads see the latest rows, and its next-key locks keep
+    // new rows out of the range the read has passed.)
+    if (databaseFamily(context.db) === 'pg') {
+      for (let previous = -1; document.data.length > previous && document.data.length < limit;) {
+        previous = document.data.length
+        document = await read()
+      }
+    }
+    return document
   }
 
   /**
@@ -330,23 +651,10 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
    * @param {{ scopeName: string, context: any, runHooks: Function, filters?: Record<string, unknown>, sort: Array<{ field: string, direction: 'asc' | 'desc' }>, limit: number }} request
    */
   const dataClaim = async ({ scopeName, context, runHooks, filters, sort, limit }) => {
-    const storageAdapter = adapterFor(scopeName)
-    const tableName = storageAdapter.getTableName()
-    const idColumn = `${tableName}.${storageAdapter.getIdColumn()}`
     const db = context.db
-    // Resolves to { query }: awaiting a bare builder would run it.
-    const scoped = async () => {
-      const { query } = await dataScopedQuery({ scopeName, context, runHooks, purpose: 'claim', filters })
-      if (query._statements.some(statement => statement.grouping === 'join')) {
-        throw new RestApiValidationError(`claim on '${scopeName}' needs a single-table query, but its filters or row policy add joins`, {
-          fields: ['filters'],
-          violations: [{ field: 'filters', rule: 'claim_single_table', message: 'A row policy or filter that joins tables cannot be combined with SKIP LOCKED' }]
-        })
-      }
-      for (const { field, direction } of sort) query.orderBy(`${tableName}.${storageAdapter.translateColumn(field)}`, direction)
-      return { query }
-    }
-    const { query } = await scoped()
+    const claimQuery = () => lockingQuery({ scopeName, context, runHooks, operation: 'claim', purpose: 'claim', filters })
+    const { query, storageAdapter, tableName, idColumn } = await claimQuery()
+    for (const { field, direction } of sort) query.orderBy(`${tableName}.${storageAdapter.translateColumn(field)}`, direction)
     if (databaseFamily(db) !== 'mysql') {
       // PostgreSQL locks rows as the LIMIT takes them; SQLite has one writer.
       return readRecords({ scopeName, context, query: lockRows(query.limit(limit), db, { skipLocked: true }), tableName })
@@ -354,26 +662,33 @@ export function createManyRowDataHelpers ({ api, getScopeStorageAdapter, scopes,
 
     // InnoDB locks every row a locking read examines, including rows a sort
     // reads and the LIMIT then drops, so one claim would hold every match.
-    // Instead: candidates in sort order by a plain read, then locking reads of
-    // windows of candidates by primary key, each no larger than what is still needed.
+    // Instead: candidates in sort order by a plain read; then, for windows of
+    // candidates no larger than what is still needed, a SKIP LOCKED lock by
+    // primary key on the bare table, and a locking read of the held rows that
+    // drops any a committed change made stop matching. SKIP LOCKED never waits,
+    // so the snapshot the plain read starts misses nothing a lock wait let commit.
     const claimed = { data: [] }
+    const identity = { id: databaseIdentityExpression(db, idColumn) }
     const pageSize = Math.max(limit * 2, 20)
     for (let offset = 0; claimed.data.length < limit; offset += pageSize) {
-      const candidates = (await applyDatabaseReadOptions(query.clone().offset(offset).limit(pageSize)
-        .select({ id: databaseIdentityExpression(db, idColumn) }))).map(row => row.id)
+      const candidates = (await applyDatabaseReadOptions(query.clone().offset(offset).limit(pageSize).select(identity))).map(row => row.id)
       for (let next = 0; next < candidates.length && claimed.data.length < limit;) {
         const window = candidates.slice(next, next + limit - claimed.data.length)
         next += window.length
-        const locked = (await scoped()).query.whereIn(idColumn, window).forUpdate().skipLocked()
-        const document = await readRecords({ scopeName, context, query: locked, tableName })
-        claimed.data.push(...document.data)
+        const byKey = applyWriteConditions({ query: db(tableName).whereIn(idColumn, window), context, scopeName, storageAdapter, tableName })
+        const held = (await applyDatabaseReadOptions(byKey.forUpdate().skipLocked().select(identity))).map(row => row.id)
+        if (!held.length) continue
+        const { query: current } = await claimQuery()
+        const document = await readRecords({ scopeName, context, query: current.whereIn(idColumn, held).forUpdate().skipLocked(), tableName })
+        const order = new Map(window.map((id, position) => [String(id), position]))
+        claimed.data.push(...document.data.sort((left, right) => order.get(String(left.id)) - order.get(String(right.id))))
       }
       if (candidates.length < pageSize) break
     }
     return claimed
   }
 
-  return { dataScopedQuery, dataVisibleIds, dataInsertMany, dataUpdateByIds, dataDeleteByIds, dataLockByIds, dataClaim }
+  return { dataScopedQuery, dataVisibleIds, dataIdsByKeys, dataInsertMany, dataUpdateByIds, dataDeleteByIds, dataLockByIds, dataLockWhere, dataClaim }
 }
 
 const sameValue = (left, right) => {
@@ -383,44 +698,14 @@ const sameValue = (left, right) => {
   return false
 }
 
-function applyOnlyIf (query, onlyIf) {
+function applyOnlyIf (query, onlyIf, tableName) {
   for (const [column, value] of Object.entries(onlyIf)) {
-    if (value === null) query.whereNull(column)
-    else query.where(column, value)
+    if (value === null) query.whereNull(`${tableName}.${column}`)
+    else query.where(`${tableName}.${column}`, value)
   }
   return query
 }
 
-function conflictElsewhere (scopeName, onConflict) {
-  return new RestApiValidationError(`A '${scopeName}' row conflicted on a unique index other than (${onConflict.keys.join(', ')})`, {
-    fields: onConflict.keys,
-    violations: [{ field: 'onConflict.keys', rule: 'conflict_keys', message: 'Every input row must insert or match a row by the conflict keys' }]
-  })
-}
-
-async function readByKeys (db, tableName, idColumn, keyColumns, values) {
-  const rows = await applyDatabaseReadOptions(db(tableName)
-    .whereIn(keyColumns, values.map(row => keyColumns.map(column => row[column] ?? null)))
-    .select({ id: databaseIdentityExpression(db, idColumn) }, ...keyColumns))
-  return rows
-}
-
-function applyConflict (query, { db, family, tableName, idColumn, keyColumns, onConflict, storageAdapter, lockVersionColumn }) {
-  if (onConflict.action === 'ignore') {
-    // INSERT IGNORE would also turn other errors into warnings; keep them errors.
-    return family === 'mysql'
-      ? query.onConflict(keyColumns).merge({ [idColumn]: db.raw('??', [idColumn]) })
-      : query.onConflict(keyColumns).ignore()
-  }
-  const updates = {}
-  for (const field of onConflict.merge) {
-    const column = storageAdapter.translateColumn(field)
-    updates[column] = family === 'mysql' ? db.raw('values(??)', [column]) : db.raw('excluded.??', [column])
-  }
-  if (lockVersionColumn) updates[lockVersionColumn] = db.raw('??.?? + 1', [tableName, lockVersionColumn])
-  return query.onConflict(keyColumns).merge(updates)
-}
-
 // MySQL reports only the first generated id. One multi-row INSERT is expected to
 // take consecutive values; prove it by reading the range back inside the
 // transaction and comparing the rows with what was inserted.