json-rest-api 3.0.0-rc.0 → 3.0.1

package/plugins/core/rest-api-plugin-methods/post-many.js

@@ -4,14 +4,16 @@
 import { lockRelationshipTargets, processRelationships } from '../lib/writing/relationship-processor.js'
 import { getRequestContracts, validateRequestContractOrThrow } from '../lib/querying-writing/request-contracts.js'
 import { requireDocumentResourceId } from '../lib/querying-writing/resource-id-normalization.js'
-import { rejectRemovedOptions, resolveFormat, resolveReturning } from '../lib/querying-writing/response-options.js'
+import { rejectRemovedOptions, resolveFormat } from '../lib/querying-writing/response-options.js'
 import { beginWriteTransaction } from '../../../lib/error-context.js'
-import { RestApiValidationError } from '../../../lib/rest-api-errors.js'
+import { RestApiResourceError, RestApiValidationError } from '../../../lib/rest-api-errors.js'
+import { chunk } from '../lib/storage/many-row-data-helpers.js'
 import {
   applyFieldSetters,
   collectRelationshipTargets,
   commitOwnedTransaction,
   handleWriteMethodError,
+  keepHiddenLinks,
   validateRelationshipTargets,
   validateResourceAttributesBeforeWrite
 } from './common.js'
@@ -22,32 +24,45 @@ import {
   refuseUnsupportedResource,
   rejectBulkOptions,
   resolveChunkSize,
+  resolveManyRowReturning,
   selectManyInput,
   toDocument,
   unsupported
 } from './many-row-common.js'
+import { VISIBLE_READ_CHUNK } from './patch-many.js'
 
 /**
  * Upserts may only match the caller's own rows: the conflict keys must hold
  * every ownership field, and merge cannot run under a row policy it cannot see.
+ * The id (by its own name or the idProperty's) may be the only key on a
+ * resource with neither, where no other owner's row can match.
  */
 function resolveOnConflict (onConflict, { scopeName, schemaInfo, scopes }) {
   if (onConflict === undefined) return undefined
   const fail = message => { throw new RestApiValidationError(message, { fields: ['onConflict'] }) }
   if (!onConflict || typeof onConflict !== 'object' || Array.isArray(onConflict)) fail('onConflict must be an object')
-  const { keys, action, merge, ...unknown } = onConflict
+  const { keys: requestedKeys, action, merge, ...unknown } = onConflict
+  let keys = requestedKeys
   if (Object.keys(unknown).length) fail(`Unknown onConflict options: ${Object.keys(unknown).join(', ')}`)
   if (action !== 'ignore' && action !== 'merge') fail("onConflict.action must be 'ignore' or 'merge'")
   const fields = schemaInfo.schemaStructure
   const stored = field => typeof field === 'string' && Object.hasOwn(fields, field) && !fields[field].virtual && !fields[field].computed
-  if (!Array.isArray(keys) || !keys.length || !keys.every(stored)) fail('onConflict.keys must list stored fields of the resource')
-
   const ownership = schemaInfo.ownershipFields || []
+  const rowPolicy = scopes[scopeName]?.vars?.rowPolicy
+  const idKey = Array.isArray(keys) && keys.some(field => field === 'id' || field === schemaInfo.idProperty)
+  if (idKey) {
+    if (keys.length !== 1) fail('onConflict on the id takes the id as its only key')
+    if (ownership.length || rowPolicy) {
+      throw unsupported(scopeName, 'postMany', 'onConflict on the id needs a resource without ownership fields or a row policy, or a conflict could match a row the caller cannot see; use keys that include the ownership field(s)')
+    }
+    keys = ['id']
+  } else if (!Array.isArray(keys) || !keys.length || !keys.every(stored)) fail('onConflict.keys must list stored fields of the resource')
+
   const missingOwners = ownership.filter(field => !keys.includes(field))
   if (missingOwners.length) {
     throw unsupported(scopeName, 'postMany', `onConflict.keys must include the ownership field(s) ${missingOwners.join(', ')}, or a conflict could match another owner's row`)
   }
-  if (action === 'merge' && scopes[scopeName]?.vars?.rowPolicy) {
+  if (action === 'merge' && rowPolicy) {
     throw unsupported(scopeName, 'postMany', 'onConflict merge cannot see the row policy; insert with ignore, then patchMany the rows you may change')
   }
   const protectedFields = new Set(['id', ...keys, ...ownership, schemaInfo.lockVersionField].filter(Boolean))
@@ -59,11 +74,79 @@ function resolveOnConflict (onConflict, { scopeName, schemaInfo, scopes }) {
     fail('onConflict.merge applies only to the merge action')
   }
   // Without a list, merge updates every stored field the rows supply.
-  const mergeFields = rows => merge ?? [...new Set(rows.flatMap(row => Object.keys(row)))]
+  const mergeFields = supplied => (merge ?? [...new Set(supplied.flatMap(fields => [...fields]))])
     .filter(field => stored(field) && !protectedFields.has(field))
   return { keys, action, mergeFields }
 }
 
+/**
+ * The fields a row supplies: those the caller sent (with belongs-to links),
+ * and those a setter added. Schema defaults are not supplied, even when a
+ * setter rewrites them, so a merge never resets a stored value to a default.
+ * @param {any} row
+ * @param {Record<string, unknown>} beforeSetters
+ * @param {Record<string, unknown>} afterSetters
+ */
+function suppliedFields (row, beforeSetters, afterSetters) {
+  const fields = new Set(Object.keys(row.originalInputAttributes || {}))
+  for (const field of Object.keys(afterSetters)) if (!Object.hasOwn(beforeSetters, field)) fields.add(field)
+  return fields
+}
+
+/**
+ * Upsert merge: lock the rows the conflict keys match, check that the caller
+ * may patch each of them, insert the other rows with a plain INSERT (a clash on
+ * any other unique index fails the batch), then update the matched rows by id
+ * with the merge fields each row supplies.
+ * Returns the identifier of every row, merged or inserted.
+ * @param {any} args
+ * @param {{ rows: any[], attributes: Record<string, unknown>[], supplied: Set<string>[], onConflict: { keys: string[], merge: string[] }, chunkSize: number, needIds: boolean }} batch
+ */
+async function mergeRows ({ context, helpers, scope, scopes, runHooks, scopeName }, { rows, attributes, supplied, onConflict, chunkSize, needIds }) {
+  const existing = await helpers.dataIdsByKeys({ scopeName, context, rows: attributes, keys: onConflict.keys, chunkSize })
+  const matched = attributes.flatMap((_row, index) => existing[index] === undefined ? [] : [index])
+  const missing = attributes.flatMap((_row, index) => existing[index] === undefined ? [index] : [])
+
+  const visible = new Map()
+  // dataIdsByKeys locked these rows, and may have found one that another
+  // transaction committed after this one's snapshot: read them as locked.
+  for (const part of chunk(matched.map(index => existing[index]), VISIBLE_READ_CHUNK)) {
+    const records = await helpers.dataGetMinimal({ scopeName, context, runHooks, ids: part, filters: {}, queryPurpose: 'bulk-write', locked: true })
+    for (const record of records) visible.set(String(record.id), record)
+  }
+  for (const index of matched) {
+    const id = String(existing[index])
+    const minimalRecord = visible.get(id)
+    try {
+      if (!minimalRecord) {
+        throw new RestApiResourceError(`The ${scopeName} record these conflict keys match cannot be changed by the caller`, { subtype: 'forbidden', resourceType: scopeName })
+      }
+      await scope.checkPermissions({ method: 'patch', originalContext: { ...rows[index], method: 'patch', id, minimalRecord, originalMinimalRecord: minimalRecord } })
+    } catch (error) { throw atIndex(error, index) }
+  }
+
+  const ids = [...existing]
+  if (missing.length) {
+    const inserted = await helpers.dataInsertMany({ scopeName, context, rows: missing.map(index => attributes[index]), chunkSize, needIds })
+    missing.forEach((index, position) => { ids[index] = inserted?.[position] })
+  }
+  if (matched.length) {
+    const storageAdapter = helpers.getStorageAdapter(scopeName)
+    // Only the merge fields a row supplies change; the lock version advances.
+    const merges = matched.map(index => Object.fromEntries(onConflict.merge
+      .filter(field => supplied[index].has(field))
+      .map(field => [field, attributes[index][field]])))
+    // A merge clears a link the way a patch does, so it keeps one to a hidden target.
+    await keepHiddenLinks({ context, scopes, schemaInfo: context.schemaInfo, rows: matched.map((index, position) => ({ updates: merges[position], stored: visible.get(String(existing[index])) })) })
+    const updates = matched.map((index, position) => ({
+      id: existing[index],
+      row: storageAdapter.toStorageRow(merges[position], { context: rows[index], operation: 'patch' })
+    }))
+    await helpers.dataUpdateByIds({ scopeName, context, updates, chunkSize })
+  }
+  return ids
+}
+
 /** @param {any} args */
 export default async function postManyMethod ({
   params, context, vars, helpers, scope, scopes, runHooks, hasHooks, perRecordHooks, scopeOptions, scopeName, api, log
@@ -76,7 +159,7 @@ export default async function postManyMethod ({
     context.schemaInfo = scopes[scopeName].vars.schemaInfo
     context.format = resolveFormat(params.format, vars.format)
     context.simplified = context.format === 'plain'
-    context.returning = resolveReturning(params.returning, 'none')
+    context.returning = resolveManyRowReturning(params.returning)
     context.params = params
     const chunkSize = resolveChunkSize(params.chunkSize)
     const { inputKey, records } = selectManyInput(params)
@@ -84,6 +167,12 @@ export default async function postManyMethod ({
       api, helpers, perRecordHooks, scopeName, schemaInfo: context.schemaInfo, operation: 'postMany', kind: 'post', storageHelper: 'dataInsertMany'
     })
     const conflict = resolveOnConflict(params.onConflict, { scopeName, schemaInfo: context.schemaInfo, scopes })
+    if (conflict?.action === 'merge') {
+      // Merged rows are updates: the resource must take patches as a batch too.
+      refuseUnsupportedResource({
+        api, helpers, perRecordHooks, scopeName, schemaInfo: context.schemaInfo, operation: 'postMany', kind: 'patch', storageHelper: 'dataUpdateByIds'
+      })
+    }
 
     await beginWriteTransaction(context, params.transaction, helpers.newTransaction, runHooks)
     context.db = context.transaction
@@ -135,16 +224,20 @@ export default async function postManyMethod ({
       } catch (error) { throw atIndex(error, index) }
     }
 
-    let onConflict
     if (conflict) {
-      const rowAttributes = rows.map(row => row.inputRecord.data.attributes || {})
-      rowAttributes.forEach((attributes, index) => {
-        const empty = conflict.keys.filter(field => attributes[field] === undefined || attributes[field] === null)
-        if (empty.length) throw atIndex(new RestApiValidationError(`Conflict keys need values: ${empty.join(', ')}`, { fields: empty.map(field => `data.attributes.${field}`) }), index)
+      const byId = conflict.keys[0] === 'id'
+      rows.forEach((row, index) => {
+        const id = row.inputRecord.data.id
+        if (byId && (id === undefined || id === null)) throw atIndex(new RestApiValidationError('An upsert on the id needs an id in every row', { fields: ['data.id'] }), index)
+        if (byId || id === undefined || id === null) return
+        throw atIndex(new RestApiValidationError('An upsert row cannot carry an id: its conflict keys identify it', { fields: ['data.id'] }), index)
       })
-      onConflict = { keys: conflict.keys, action: conflict.action, merge: conflict.action === 'merge' ? conflict.mergeFields(rowAttributes) : [] }
-      if (conflict.action === 'merge' && !onConflict.merge.length) {
-        throw new RestApiValidationError('onConflict merge has no fields to update; use ignore', { fields: ['onConflict'] })
+      if (!byId) {
+        rows.forEach((row, index) => {
+          const attributes = row.inputRecord.data.attributes || {}
+          const empty = conflict.keys.filter(field => attributes[field] === undefined || attributes[field] === null)
+          if (empty.length) throw atIndex(new RestApiValidationError(`Conflict keys need values: ${empty.join(', ')}`, { fields: empty.map(field => `data.attributes.${field}`) }), index)
+        })
       }
     }
 
@@ -153,22 +246,57 @@ export default async function postManyMethod ({
     await lockRelationshipTargets(api, context.transaction, parents)
 
     const attributes = []
+    const supplied = []
     for (const [index, row] of rows.entries()) {
+      const beforeSetters = { ...(row.inputRecord.data.attributes || {}) }
       try {
         await applyFieldSetters(row, api, helpers)
       } catch (error) { throw atIndex(error, index) }
       const data = row.inputRecord.data
+      supplied.push(suppliedFields(row, beforeSetters, data.attributes || {}))
       attributes.push({ ...(data.attributes || {}), ...(data.id !== undefined ? { id: data.id } : {}) })
     }
 
-    const needIds = context.returning !== 'none' || hasHooks('afterDataCallPostMany')
-    const ids = await helpers.dataInsertMany({ scopeName, context, rows: attributes, chunkSize, needIds, onConflict })
+    let onConflict
+    if (conflict) {
+      onConflict = { keys: conflict.keys, action: conflict.action, merge: conflict.action === 'merge' ? conflict.mergeFields(supplied) : [] }
+      if (conflict.action === 'merge' && !onConflict.merge.length) {
+        throw new RestApiValidationError('onConflict merge has no fields to update; use ignore', { fields: ['onConflict'] })
+      }
+      const seen = new Map()
+      attributes.forEach((row, index) => {
+        const key = JSON.stringify(onConflict.keys.map(field => row[field]))
+        if (seen.has(key)) {
+          throw atIndex(new RestApiValidationError(`Rows ${seen.get(key)} and ${index} have the same conflict keys`, {
+            fields: onConflict.keys.map(field => `data.attributes.${field}`)
+          }), index)
+        }
+        seen.set(key, index)
+      })
+    }
+
+    // Without a conflict every row is inserted or the batch fails, so a count needs no ids.
+    const countsRows = context.returning === 'count' && !onConflict
+    const needIds = (context.returning !== 'none' && !countsRows) || hasHooks('afterDataCallPostMany')
+    const ids = onConflict?.action === 'merge'
+      ? await mergeRows({ context, helpers, scope, scopes, runHooks, scopeName }, { rows, attributes, supplied, onConflict, chunkSize, needIds })
+      : await helpers.dataInsertMany({ scopeName, context, rows: attributes, chunkSize, needIds, onConflict })
     context.ids = ids?.map(id => id === undefined || id === null ? undefined : String(id))
     await runHooks('afterDataCallPostMany')
 
-    const response = await manyRowResponse({
-      api, scopeName, context, ids: context.ids || [], returning: context.returning, format: context.format, queryParams: params.queryParams
-    })
+    const response = countsRows
+      ? { meta: { count: rows.length } }
+      : await manyRowResponse({
+        api,
+        scopeName,
+        context,
+        ids: context.ids || [],
+        returning: context.returning,
+        format: context.format,
+        queryParams: params.queryParams,
+        // A merge may match a row committed after this transaction's snapshot and leave it unchanged.
+        locked: onConflict?.action === 'merge'
+      })
     await commitOwnedTransaction(context)
     return response
   } catch (error) {

package/plugins/core/rest-api-plugin-methods/put.js

@@ -19,7 +19,8 @@ import {
   validatePivotResource,
   handleRecordReturnAfterWrite,
   handleWriteMethodError,
-  commitOwnedTransaction
+  commitOwnedTransaction,
+  keepHiddenLinks
 } from './common.js'
 
 /**
@@ -130,6 +131,9 @@ export default async function putMethod ({
         }
       }
     }
+    if (context.isUpdate) {
+      await keepHiddenLinks({ context, scopes, schemaInfo: context.schemaInfo, rows: [{ updates: belongsToUpdates, stored: context.minimalRecord, inputRecord: context.inputRecord }] })
+    }
 
     // Merge belongsTo updates with attributes
     if (Object.keys(belongsToUpdates).length > 0) {

package/plugins/core/rest-api-plugin-methods/scoped-reads.js

@@ -1,6 +1,6 @@
 // Reads that keep the resource's scope: the read-only scoped builder, every-row
 // reads in cursor pages, and row locks inside a transaction.
-import { RestApiBulkWriteUnsupportedError, RestApiRowLimitError, RestApiValidationError } from '../../../lib/rest-api-errors.js'
+import { RestApiBulkWriteUnsupportedError, RestApiRecordsNotFoundError, RestApiRowLimitError, RestApiValidationError } from '../../../lib/rest-api-errors.js'
 import { readOnlyQueryBuilder } from '../lib/querying/read-only-query.js'
 import { normalizeRecordAttributes } from '../lib/querying-writing/database-value-normalizers.js'
 import { filterResponseFields, getResourceFieldset } from '../lib/querying-writing/field-utils.js'
@@ -9,21 +9,24 @@ import { resolveFormat } from '../lib/querying-writing/response-options.js'
 import { transformJsonApiToSimplified } from '../lib/querying-writing/simplified-helpers.js'
 import { requireExistingResourceId } from '../lib/querying-writing/resource-id-normalization.js'
 import { chunk } from '../lib/storage/many-row-data-helpers.js'
-import { keepVisible, refuseDuplicateIds, requireLockTransaction, resolveMissing, validateFilters } from './many-row-common.js'
+import { filterToOneResponseLinkage, snapshotReadRecord } from './common.js'
+import { refuseDuplicateIds, requireLockTransaction, resolveMissing, validateFilters } from './many-row-common.js'
 
 export const SCOPED_QUERY_PURPOSES = new Set(['report', 'bulk-write', 'lock', 'claim'])
 const DEFAULT_QUERY_ALL_PAGE = 1000
 const MAX_CLAIM = 1000
+const MAX_LOCK_WHERE = 1000
 
 const invalid = (field, message) => new RestApiValidationError(message, { fields: [field], violations: [{ field, rule: 'scoped_read', message }] })
 
 /**
  * The resource's table with ownership and row policy applied, as a read-only
- * Knex builder. Resolves to `{ query }`: a builder is thenable, so returning it
- * from an async method would run it.
+ * Knex builder over a derived table named after the alias (or the table), so
+ * nothing a caller adds, `orWhere` included, can widen the scope. Resolves to
+ * `{ query }`: a builder is thenable, so returning it from an async method would run it.
  * @param {any} args
  */
-export async function scopedQueryMethod ({ params, context, vars, helpers, scope, scopes, runHooks, scopeName }) {
+export async function scopedQueryMethod ({ params, context, vars, helpers, scope, scopes, runHooks, scopeName, api }) {
   const { transaction, alias, purpose = 'report', filters, ...unknown } = params
   if (Object.keys(unknown).length) throw invalid(Object.keys(unknown)[0], `Unknown scopedQuery options: ${Object.keys(unknown).join(', ')}`)
   if (!SCOPED_QUERY_PURPOSES.has(purpose)) throw invalid('purpose', `purpose must be one of ${[...SCOPED_QUERY_PURPOSES].join(', ')}`)
@@ -41,8 +44,9 @@ export async function scopedQueryMethod ({ params, context, vars, helpers, scope
   context.queryParams = { filters: validateFilters({ filters, scopeName, schemaInfo: context.schemaInfo, vars }) || {} }
   // A caller who may not query the resource may not build reports on it either.
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', scopedQuery: true } })
-  const { query } = await helpers.dataScopedQuery({ scopeName, context, runHooks, alias, purpose, filters: context.queryParams.filters })
-  return { query: readOnlyQueryBuilder(query, { resourceType: scopeName }) }
+  const { query, alias: name } = await helpers.dataScopedQuery({ scopeName, context, runHooks, alias, purpose, filters: context.queryParams.filters })
+  const derived = (transaction || api.knex.instance).from(query.select(`${name}.*`).as(name))
+  return { query: readOnlyQueryBuilder(derived, { resourceType: scopeName }) }
 }
 
 /**
@@ -76,13 +80,26 @@ export async function queryAllMethod ({ params, context, vars, scopeName, api })
   return resolvedFormat === 'jsonapi' && included.size ? { data, included: [...included.values()] } : { data }
 }
 
-// The read pipeline's last steps for rows a lock read: normalization, computed
-// fields, sparse fieldsets and the plain format.
-async function finishRecords ({ document, context, scope, scopes, scopeName }) {
-  let record = normalizeRecordAttributes(document, scopes)
+// The rows a lock read leave through the read pipeline `query` runs after its
+// data call, so masking and denials apply to them too: checkDataPermissions for
+// each row as `get` runs it, enrichRecord, computed fields, the finish hooks,
+// sparse fieldsets and the plain format. Hooks see method 'query' and lock: true.
+async function finishLockedRead ({ document, context, scope, scopes, scopeName, runHooks, hasHooks }) {
+  context.method = 'query'
+  context.lock = true
+  context.record = normalizeRecordAttributes(document, scopes)
+  if (hasHooks('checkDataPermissions') || hasHooks('checkDataPermissionsGet')) {
+    for (const entry of context.record.data) {
+      const recordContext = { ...context, method: 'get', id: entry.id, record: { data: entry } }
+      await runHooks('checkDataPermissions', recordContext)
+      await runHooks('checkDataPermissionsGet', recordContext)
+    }
+  }
+  snapshotReadRecord(context, scopes, hasHooks, 'query')
+  await runHooks('enrichRecord')
   const computedFields = scope.vars.schemaInfo?.computed || {}
   const requestedComputedFields = getRequestedComputedFields(scopeName, getResourceFieldset(context.queryParams.fields, scopeName), computedFields)
-  for (const entry of record.data) {
+  for (const entry of context.record.data) {
     entry.attributes = await scope.enrichAttributes({
       id: entry.id,
       attributes: entry.attributes,
@@ -92,7 +109,10 @@ async function finishRecords ({ document, context, scope, scopes, scopeName }) {
       computedDependencies: context.computedDependencies
     })
   }
-  record = normalizeRecordAttributes(record, scopes, { source: 'response' })
+  await runHooks('finish')
+  await runHooks('finishQuery')
+  await filterToOneResponseLinkage(context, scopes)
+  const record = normalizeRecordAttributes(context.record, scopes, { source: 'response' })
   filterResponseFields(record, context.queryParams.fields)
   if (context.format !== 'plain') return { data: record.data }
   const schemaInfo = scope.vars.schemaInfo
@@ -123,30 +143,55 @@ async function setupLockRead ({ params, context, vars, helpers, scopes, scopeNam
 
 /**
  * Locks the visible rows among `ids` until the transaction ends, in id order,
- * and returns them as the locking read saw them.
+ * and returns them as the locking read saw them. That read is the first
+ * statement on the rows and decides visibility itself, so the transaction's
+ * snapshot starts after any lock wait. On MySQL it locks every listed row,
+ * hidden ones too, before it checks the scope (see guide 34).
  * @param {any} args
  */
 export async function lockByIdsMethod (args) {
-  const { params, context, helpers, scope, scopes, runHooks, scopeName, scopeOptions, vars } = args
+  const { params, context, helpers, scope, scopes, runHooks, hasHooks, scopeName, scopeOptions, vars } = args
   await setupLockRead(args, 'lockByIds', 'dataLockByIds')
   if (!Array.isArray(params.ids) || !params.ids.length) throw invalid('ids', 'ids must be a non-empty array')
   const ids = params.ids.map(id => requireExistingResourceId(id, { scopeOptions, vars, scopeName }))
   const storageAdapter = helpers.getStorageAdapter(scopeName)
   refuseDuplicateIds(ids, storageAdapter, 'ids')
   const missing = resolveMissing(params.missing)
-
-  const visible = new Set()
-  for (const part of chunk(ids, 1000)) {
-    for (const id of await helpers.dataVisibleIds({ scopeName, context, runHooks, ids: part, queryPurpose: 'lock' })) visible.add(String(id))
-  }
-  const kept = keepVisible({ requested: ids, visible, missing, scopeName, storageAdapter })
-  if (!kept.length) return { data: [] }
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
+  // Sorted before chunking, so lists longer than a chunk still lock in one
+  // order across transactions: a list of integers numerically, any other list
+  // by code unit (the database's order for a binary collation).
+  const keys = ids.map(id => String(storageAdapter.translateFilterValue('id', id)))
+  const numeric = keys.every(key => /^-?\d+$/.test(key))
+  const sorted = ids
+    .map((id, index) => ({ id, key: numeric ? BigInt(keys[index]) : keys[index] }))
+    .sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0)
+    .map(entry => entry.id)
   const document = { data: [] }
-  for (const part of chunk(kept, 1000)) {
-    document.data.push(...(await helpers.dataLockByIds({ scopeName, context, ids: part })).data)
+  for (const part of chunk(sorted, 1000)) {
+    document.data.push(...(await helpers.dataLockByIds({ scopeName, context, runHooks, ids: part })).data)
+  }
+  if (document.data.length < ids.length && missing === 'throw') {
+    throw new RestApiRecordsNotFoundError({ resourceType: scopeName, missing: ids.length - document.data.length })
   }
-  return finishRecords({ document, context, scope, scopes, scopeName })
+  if (!document.data.length) return { data: [] }
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
+}
+
+/**
+ * Locks the visible rows the filters select until the transaction ends, in id
+ * order, waiting for transactions that hold them, so a row that starts to match
+ * while the lock waits is locked and returned too. At most 1,000 rows.
+ * @param {any} args
+ */
+export async function lockWhereMethod (args) {
+  const { params, context, vars, helpers, scope, scopes, runHooks, hasHooks, scopeName } = args
+  await setupLockRead(args, 'lockWhere', 'dataLockWhere')
+  const filters = validateFilters({ filters: params.filters, scopeName, schemaInfo: context.schemaInfo, vars })
+  await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
+  const document = await helpers.dataLockWhere({ scopeName, context, runHooks, filters, limit: MAX_LOCK_WHERE + 1 })
+  if (document.data.length > MAX_LOCK_WHERE) throw new RestApiRowLimitError({ resourceType: scopeName, rowLimit: MAX_LOCK_WHERE, operation: 'lockWhere' })
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
 }
 
 /**
@@ -155,7 +200,7 @@ export async function lockByIdsMethod (args) {
  * @param {any} args
  */
 export async function claimMethod (args) {
-  const { params, context, vars, helpers, scope, scopes, runHooks, scopeName } = args
+  const { params, context, vars, helpers, scope, scopes, runHooks, hasHooks, scopeName } = args
   await setupLockRead(args, 'claim', 'dataClaim')
   const limit = params.limit ?? 1
   if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_CLAIM) throw invalid('limit', `limit must be an integer from 1 to ${MAX_CLAIM}`)
@@ -172,5 +217,5 @@ export async function claimMethod (args) {
   })
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
   const document = await helpers.dataClaim({ scopeName, context, runHooks, filters, sort, limit })
-  return finishRecords({ document, context, scope, scopes, scopeName })
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
 }

package/plugins/core/rest-api-plugin.js

@@ -19,7 +19,7 @@ import deleteMethod from './rest-api-plugin-methods/delete.js'
 import postManyMethod from './rest-api-plugin-methods/post-many.js'
 import { patchManyMethod, patchWhereIdsMethod, patchWhereMethod } from './rest-api-plugin-methods/patch-many.js'
 import { deleteManyMethod, deleteWhereMethod } from './rest-api-plugin-methods/delete-many.js'
-import { claimMethod, lockByIdsMethod, queryAllMethod, scopedQueryMethod } from './rest-api-plugin-methods/scoped-reads.js'
+import { claimMethod, lockByIdsMethod, lockWhereMethod, queryAllMethod, scopedQueryMethod } from './rest-api-plugin-methods/scoped-reads.js'
 import enrichAttributesMethod from './rest-api-plugin-methods/enrich-attributes.js'
 import checkPermissionsMethod from './rest-api-plugin-methods/check-permissions.js'
 import applyQueryFiltersMethod from './rest-api-plugin-methods/apply-query-filters.js'
@@ -112,6 +112,7 @@ export const RestApiPlugin = {
     addResourceMethod('queryAll', withAvailableContext(queryAllMethod))
     addResourceMethod('scopedQuery', withAvailableContext(scopedQueryMethod))
     addResourceMethod('lockByIds', withAvailableContext(lockByIdsMethod))
+    addResourceMethod('lockWhere', withAvailableContext(lockWhereMethod))
     addResourceMethod('claim', withAvailableContext(claimMethod))
 
     // Relationship methods

package/plugins/core/rest-api-positioning-plugin.js

@@ -214,7 +214,7 @@ export const PositioningPlugin = {
     // create no positioned rows and change no positions.
     const refusePositions = ({ context, scopeName }) => {
       if (!enabled(scopeName)) return
-      const moves = context.inputRecords.some(record => {
+      const moves = [config.field, config.beforeIdField].some(field => context.rawFields?.includes(field)) || context.inputRecords.some(record => {
         const attributes = record?.data?.attributes || {}
         return attributes[config.field] !== undefined || attributes[config.beforeIdField] !== undefined
       })

package/types/data-access-methods.d.ts

@@ -3,11 +3,16 @@ import type { ManagedTransaction } from './transactions.js'
 import type { DirectResourceId, InputResourceId, JsonApiResource, PlainResource, ResourceFormat, ResourceIdentifier } from './representations.js'
 import type { JsonApiWriteDocument, QueryParams, RemovedResourceOptions, SelectionParams } from './resource-methods.js'
 
-// Many-row writes return nothing by default; 'minimal' returns identifiers, 'full' the records.
-export type ManyRowReturning = 'none' | 'minimal' | 'full'
+// Many-row writes return nothing by default; 'minimal' returns identifiers, 'full' the records,
+// 'count' the number of rows written.
+export type ManyRowReturning = 'none' | 'minimal' | 'full' | 'count'
+export type ManyRowCount = { meta: { count: number } }
 export type ManyRowResult<Format extends ResourceFormat, Returning extends ManyRowReturning, Fields extends object, Type extends string> =
   Returning extends 'none' ? void
+    : Returning extends 'count' ? ManyRowCount
     : { data: Array<Returning extends 'minimal' ? ResourceIdentifier<Type> : Format extends 'plain' ? PlainResource<Fields> : JsonApiResource<Fields, Type>> }
+type DeleteResult<Returning extends 'none' | 'minimal' | 'count', Type extends string> =
+  Returning extends 'none' ? void : Returning extends 'count' ? ManyRowCount : { data: ResourceIdentifier<Type>[] }
 export type RecordsResult<Format extends ResourceFormat, Fields extends object, Type extends string> =
   { data: Array<Format extends 'plain' ? PlainResource<Fields> : JsonApiResource<Fields, Type>>; included?: JsonApiResource[] }
 // knex.raw values: accepted by many-row patches for stored, non-relationship fields.
@@ -27,12 +32,17 @@ export type ManyRowCreateInput<Input extends object, Type extends string> =
   { data: Array<Partial<Input> & { id?: InputResourceId }>; document?: never } |
   { data?: never; document: { data: Array<JsonApiWriteDocument<Input, Type>['data']> } }
 export interface UpsertOptions<Input extends object> {
-  // Must include every ownership (autofilter) field.
-  keys: Array<keyof Input & string>
+  // Must include every ownership (autofilter) field. Rows may not carry an id or repeat a key.
+  // ['id'] (or the idProperty) alone is allowed on a resource without ownership fields or a
+  // row policy; every row then carries its id.
+  keys: Array<keyof Input & string> | ['id']
+  // merge also runs the patch permission checks on every row it updates.
   action: 'ignore' | 'merge'
   // merge only: fields to update; default every stored field the rows supply but the keys.
   merge?: Array<keyof Input & string>
 }
+// patchWhere and deleteWhere need filters, or all: true to act on every visible row.
+export type WhereScope = { filters: Record<string, unknown>; all?: never } | { filters?: never; all: true }
 type Patchable<Input extends object> = { [Field in keyof Input]?: Input[Field] | RawValue }
 export type ManyRowPatchInput<Input extends object, Type extends string> =
   { data: Array<Patchable<Input> & { id: DirectResourceId }>; document?: never } |
@@ -54,7 +64,8 @@ export interface ScopedQueryParams {
   purpose?: 'report' | 'bulk-write' | 'lock' | 'claim'
   filters?: Record<string, unknown>
 }
-// A Knex builder that refuses writes, locks, table changes and clearing its predicates.
+// A Knex builder over a derived table of the scoped rows. It refuses writes, locks,
+// table changes, set operations, common table expressions and access to the client.
 export type ReadOnlyQueryBuilder = Knex.QueryBuilder
 
 export interface DataAccessMethods<
@@ -88,17 +99,18 @@ export interface DataAccessMethods<
     context?: Context
   ): Promise<ManyRowResult<Format, Returning, Fields, Type>>
   patchWhere<Format extends ResourceFormat = DefaultFormat, Returning extends ManyRowReturning = 'none'>(
-    params: ManyRowWriteOptions<Format, Returning> & SharedPatchInput<Input, Type> & Omit<PatchConditionOptions<Input>, 'missing'> & { filters?: Record<string, unknown> },
+    params: ManyRowWriteOptions<Format, Returning> & SharedPatchInput<Input, Type> & Omit<PatchConditionOptions<Input>, 'missing'> & WhereScope,
     context?: Context
   ): Promise<ManyRowResult<Format, Returning, Fields, Type>>
-  deleteMany<Returning extends 'none' | 'minimal' = 'none'>(
+  deleteMany<Returning extends 'none' | 'minimal' | 'count' = 'none'>(
     params: RemovedResourceOptions & { ids: DirectResourceId[]; transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number; missing?: MissingRows },
     context?: Context
-  ): Promise<Returning extends 'none' ? void : { data: ResourceIdentifier<Type>[] }>
-  deleteWhere<Returning extends 'none' | 'minimal' = 'none'>(
-    params: RemovedResourceOptions & { filters?: Record<string, unknown>; transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number },
+  ): Promise<DeleteResult<Returning, Type>>
+  // Deletes the rows its selection finds and answers with those it deleted; it never throws for missing rows.
+  deleteWhere<Returning extends 'none' | 'minimal' | 'count' = 'none'>(
+    params: RemovedResourceOptions & WhereScope & { transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number },
     context?: Context
-  ): Promise<Returning extends 'none' ? void : { data: ResourceIdentifier<Type>[] }>
+  ): Promise<DeleteResult<Returning, Type>>
   // Locks the visible rows among ids (FOR UPDATE, id order) until the transaction ends.
   lockByIds<Format extends ResourceFormat = DefaultFormat>(params: {
     ids: DirectResourceId[]
@@ -107,6 +119,15 @@ export interface DataAccessMethods<
     format?: Format
     queryParams?: { fields?: Record<string, string> }
   }, context?: Context): Promise<RecordsResult<Format, Fields, Type>>
+  // Locks the visible rows the filters select (FOR UPDATE; id order, on MySQL the
+  // order of the forced index, which starts with the ownership or a filtered column),
+  // waiting for other transactions; throws RestApiRowLimitError past 1000 rows.
+  lockWhere<Format extends ResourceFormat = DefaultFormat>(params: {
+    transaction: ManagedTransaction | Knex.Transaction
+    filters?: Record<string, unknown>
+    format?: Format
+    queryParams?: { fields?: Record<string, string> }
+  }, context?: Context): Promise<RecordsResult<Format, Fields, Type>>
   // Takes up to limit (default 1, at most 1000) matching rows no other transaction holds.
   claim<Format extends ResourceFormat = DefaultFormat>(params: {
     transaction: ManagedTransaction | Knex.Transaction

package/types/errors.d.ts

@@ -118,11 +118,12 @@ export class RestApiTemporalDataError extends RestApiError {
 }
 
 export class RestApiBulkWriteUnsupportedError extends RestApiError {
-  constructor(options?: { resourceType?: string; operation?: string; reason?: string })
+  constructor(options?: { resourceType?: string; operation?: string; reason?: string; hooks?: string[] })
   code: 'REST_API_BULK_WRITE_UNSUPPORTED'
   type: 'rest_api_bulk_write_unsupported'
   statusCode: 422
-  details: { resourceType?: string; operation?: string; reason?: string }
+  /** `hooks` names the per-record hooks that caused the refusal; HTTP responses leave details out. */
+  details: { resourceType?: string; operation?: string; reason?: string; hooks?: string[] }
   toJSON(): SerializedError & Pick<RestApiBulkWriteUnsupportedError, 'code' | 'type' | 'statusCode' | 'details'>
 }
 
@@ -154,9 +155,9 @@ export class RestApiScopedQueryReadOnlyError extends RestApiError {
 }
 
 export class RestApiRowLimitError extends RestApiError {
-  constructor(options?: { resourceType?: string; rowLimit?: number })
+  constructor(options?: { resourceType?: string; rowLimit?: number; operation?: 'queryAll' | 'lockWhere' })
   code: 'REST_API_ROW_LIMIT'
   type: 'rest_api_row_limit'
-  details: { resourceType?: string; rowLimit?: number }
+  details: { resourceType?: string; rowLimit?: number; operation: 'queryAll' | 'lockWhere' }
   toJSON(): SerializedError & Pick<RestApiRowLimitError, 'code' | 'type' | 'details'>
 }

package/types/hook-context.d.ts

@@ -189,6 +189,8 @@ export type ManyRowHookContext<Context extends object = Record<string, unknown>,
   readonly transaction: ManagedTransaction
   // JSON:API documents of the rows, in input order (creates and patches).
   readonly inputRecords?: Array<{ data: { id?: InputResourceId; attributes?: Record<string, unknown>; relationships?: Record<string, unknown> } }>
+  // Patches: fields some row sets with a knex.raw value (those values are not in inputRecords).
+  readonly rawFields?: string[]
   readonly ids?: Array<string | undefined>
 }