json-rest-api 3.0.0-rc.0 → 3.0.0

package/plugins/core/rest-api-plugin-methods/scoped-reads.js

@@ -1,6 +1,6 @@
 // Reads that keep the resource's scope: the read-only scoped builder, every-row
 // reads in cursor pages, and row locks inside a transaction.
-import { RestApiBulkWriteUnsupportedError, RestApiRowLimitError, RestApiValidationError } from '../../../lib/rest-api-errors.js'
+import { RestApiBulkWriteUnsupportedError, RestApiRecordsNotFoundError, RestApiRowLimitError, RestApiValidationError } from '../../../lib/rest-api-errors.js'
 import { readOnlyQueryBuilder } from '../lib/querying/read-only-query.js'
 import { normalizeRecordAttributes } from '../lib/querying-writing/database-value-normalizers.js'
 import { filterResponseFields, getResourceFieldset } from '../lib/querying-writing/field-utils.js'
@@ -9,21 +9,24 @@ import { resolveFormat } from '../lib/querying-writing/response-options.js'
 import { transformJsonApiToSimplified } from '../lib/querying-writing/simplified-helpers.js'
 import { requireExistingResourceId } from '../lib/querying-writing/resource-id-normalization.js'
 import { chunk } from '../lib/storage/many-row-data-helpers.js'
-import { keepVisible, refuseDuplicateIds, requireLockTransaction, resolveMissing, validateFilters } from './many-row-common.js'
+import { filterToOneResponseLinkage, snapshotReadRecord } from './common.js'
+import { refuseDuplicateIds, requireLockTransaction, resolveMissing, validateFilters } from './many-row-common.js'
 
 export const SCOPED_QUERY_PURPOSES = new Set(['report', 'bulk-write', 'lock', 'claim'])
 const DEFAULT_QUERY_ALL_PAGE = 1000
 const MAX_CLAIM = 1000
+const MAX_LOCK_WHERE = 1000
 
 const invalid = (field, message) => new RestApiValidationError(message, { fields: [field], violations: [{ field, rule: 'scoped_read', message }] })
 
 /**
  * The resource's table with ownership and row policy applied, as a read-only
- * Knex builder. Resolves to `{ query }`: a builder is thenable, so returning it
- * from an async method would run it.
+ * Knex builder over a derived table named after the alias (or the table), so
+ * nothing a caller adds, `orWhere` included, can widen the scope. Resolves to
+ * `{ query }`: a builder is thenable, so returning it from an async method would run it.
  * @param {any} args
  */
-export async function scopedQueryMethod ({ params, context, vars, helpers, scope, scopes, runHooks, scopeName }) {
+export async function scopedQueryMethod ({ params, context, vars, helpers, scope, scopes, runHooks, scopeName, api }) {
   const { transaction, alias, purpose = 'report', filters, ...unknown } = params
   if (Object.keys(unknown).length) throw invalid(Object.keys(unknown)[0], `Unknown scopedQuery options: ${Object.keys(unknown).join(', ')}`)
   if (!SCOPED_QUERY_PURPOSES.has(purpose)) throw invalid('purpose', `purpose must be one of ${[...SCOPED_QUERY_PURPOSES].join(', ')}`)
@@ -41,8 +44,9 @@ export async function scopedQueryMethod ({ params, context, vars, helpers, scope
   context.queryParams = { filters: validateFilters({ filters, scopeName, schemaInfo: context.schemaInfo, vars }) || {} }
   // A caller who may not query the resource may not build reports on it either.
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', scopedQuery: true } })
-  const { query } = await helpers.dataScopedQuery({ scopeName, context, runHooks, alias, purpose, filters: context.queryParams.filters })
-  return { query: readOnlyQueryBuilder(query, { resourceType: scopeName }) }
+  const { query, alias: name } = await helpers.dataScopedQuery({ scopeName, context, runHooks, alias, purpose, filters: context.queryParams.filters })
+  const derived = (transaction || api.knex.instance).from(query.select(`${name}.*`).as(name))
+  return { query: readOnlyQueryBuilder(derived, { resourceType: scopeName }) }
 }
 
 /**
@@ -76,13 +80,26 @@ export async function queryAllMethod ({ params, context, vars, scopeName, api })
   return resolvedFormat === 'jsonapi' && included.size ? { data, included: [...included.values()] } : { data }
 }
 
-// The read pipeline's last steps for rows a lock read: normalization, computed
-// fields, sparse fieldsets and the plain format.
-async function finishRecords ({ document, context, scope, scopes, scopeName }) {
-  let record = normalizeRecordAttributes(document, scopes)
+// The rows a lock read leave through the read pipeline `query` runs after its
+// data call, so masking and denials apply to them too: checkDataPermissions for
+// each row as `get` runs it, enrichRecord, computed fields, the finish hooks,
+// sparse fieldsets and the plain format. Hooks see method 'query' and lock: true.
+async function finishLockedRead ({ document, context, scope, scopes, scopeName, runHooks, hasHooks }) {
+  context.method = 'query'
+  context.lock = true
+  context.record = normalizeRecordAttributes(document, scopes)
+  if (hasHooks('checkDataPermissions') || hasHooks('checkDataPermissionsGet')) {
+    for (const entry of context.record.data) {
+      const recordContext = { ...context, method: 'get', id: entry.id, record: { data: entry } }
+      await runHooks('checkDataPermissions', recordContext)
+      await runHooks('checkDataPermissionsGet', recordContext)
+    }
+  }
+  snapshotReadRecord(context, scopes, hasHooks, 'query')
+  await runHooks('enrichRecord')
   const computedFields = scope.vars.schemaInfo?.computed || {}
   const requestedComputedFields = getRequestedComputedFields(scopeName, getResourceFieldset(context.queryParams.fields, scopeName), computedFields)
-  for (const entry of record.data) {
+  for (const entry of context.record.data) {
     entry.attributes = await scope.enrichAttributes({
       id: entry.id,
       attributes: entry.attributes,
@@ -92,7 +109,10 @@ async function finishRecords ({ document, context, scope, scopes, scopeName }) {
       computedDependencies: context.computedDependencies
     })
   }
-  record = normalizeRecordAttributes(record, scopes, { source: 'response' })
+  await runHooks('finish')
+  await runHooks('finishQuery')
+  await filterToOneResponseLinkage(context, scopes)
+  const record = normalizeRecordAttributes(context.record, scopes, { source: 'response' })
   filterResponseFields(record, context.queryParams.fields)
   if (context.format !== 'plain') return { data: record.data }
   const schemaInfo = scope.vars.schemaInfo
@@ -123,30 +143,55 @@ async function setupLockRead ({ params, context, vars, helpers, scopes, scopeNam
 
 /**
  * Locks the visible rows among `ids` until the transaction ends, in id order,
- * and returns them as the locking read saw them.
+ * and returns them as the locking read saw them. That read is the first
+ * statement on the rows and decides visibility itself, so the transaction's
+ * snapshot starts after any lock wait. On MySQL it locks every listed row,
+ * hidden ones too, before it checks the scope (see guide 34).
  * @param {any} args
  */
 export async function lockByIdsMethod (args) {
-  const { params, context, helpers, scope, scopes, runHooks, scopeName, scopeOptions, vars } = args
+  const { params, context, helpers, scope, scopes, runHooks, hasHooks, scopeName, scopeOptions, vars } = args
   await setupLockRead(args, 'lockByIds', 'dataLockByIds')
   if (!Array.isArray(params.ids) || !params.ids.length) throw invalid('ids', 'ids must be a non-empty array')
   const ids = params.ids.map(id => requireExistingResourceId(id, { scopeOptions, vars, scopeName }))
   const storageAdapter = helpers.getStorageAdapter(scopeName)
   refuseDuplicateIds(ids, storageAdapter, 'ids')
   const missing = resolveMissing(params.missing)
-
-  const visible = new Set()
-  for (const part of chunk(ids, 1000)) {
-    for (const id of await helpers.dataVisibleIds({ scopeName, context, runHooks, ids: part, queryPurpose: 'lock' })) visible.add(String(id))
-  }
-  const kept = keepVisible({ requested: ids, visible, missing, scopeName, storageAdapter })
-  if (!kept.length) return { data: [] }
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
+  // Sorted before chunking, so lists longer than a chunk still lock in one
+  // order across transactions: a list of integers numerically, any other list
+  // by code unit (the database's order for a binary collation).
+  const keys = ids.map(id => String(storageAdapter.translateFilterValue('id', id)))
+  const numeric = keys.every(key => /^-?\d+$/.test(key))
+  const sorted = ids
+    .map((id, index) => ({ id, key: numeric ? BigInt(keys[index]) : keys[index] }))
+    .sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0)
+    .map(entry => entry.id)
   const document = { data: [] }
-  for (const part of chunk(kept, 1000)) {
-    document.data.push(...(await helpers.dataLockByIds({ scopeName, context, ids: part })).data)
+  for (const part of chunk(sorted, 1000)) {
+    document.data.push(...(await helpers.dataLockByIds({ scopeName, context, runHooks, ids: part })).data)
+  }
+  if (document.data.length < ids.length && missing === 'throw') {
+    throw new RestApiRecordsNotFoundError({ resourceType: scopeName, missing: ids.length - document.data.length })
   }
-  return finishRecords({ document, context, scope, scopes, scopeName })
+  if (!document.data.length) return { data: [] }
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
+}
+
+/**
+ * Locks the visible rows the filters select until the transaction ends, in id
+ * order, waiting for transactions that hold them, so a row that starts to match
+ * while the lock waits is locked and returned too. At most 1,000 rows.
+ * @param {any} args
+ */
+export async function lockWhereMethod (args) {
+  const { params, context, vars, helpers, scope, scopes, runHooks, hasHooks, scopeName } = args
+  await setupLockRead(args, 'lockWhere', 'dataLockWhere')
+  const filters = validateFilters({ filters: params.filters, scopeName, schemaInfo: context.schemaInfo, vars })
+  await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
+  const document = await helpers.dataLockWhere({ scopeName, context, runHooks, filters, limit: MAX_LOCK_WHERE + 1 })
+  if (document.data.length > MAX_LOCK_WHERE) throw new RestApiRowLimitError({ resourceType: scopeName, rowLimit: MAX_LOCK_WHERE, operation: 'lockWhere' })
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
 }
 
 /**
@@ -155,7 +200,7 @@ export async function lockByIdsMethod (args) {
  * @param {any} args
  */
 export async function claimMethod (args) {
-  const { params, context, vars, helpers, scope, scopes, runHooks, scopeName } = args
+  const { params, context, vars, helpers, scope, scopes, runHooks, hasHooks, scopeName } = args
   await setupLockRead(args, 'claim', 'dataClaim')
   const limit = params.limit ?? 1
   if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_CLAIM) throw invalid('limit', `limit must be an integer from 1 to ${MAX_CLAIM}`)
@@ -172,5 +217,5 @@ export async function claimMethod (args) {
   })
   await scope.checkPermissions({ method: 'query', originalContext: { ...context, method: 'query', lock: true } })
   const document = await helpers.dataClaim({ scopeName, context, runHooks, filters, sort, limit })
-  return finishRecords({ document, context, scope, scopes, scopeName })
+  return finishLockedRead({ document, context, scope, scopes, scopeName, runHooks, hasHooks })
 }

package/plugins/core/rest-api-plugin.js

@@ -19,7 +19,7 @@ import deleteMethod from './rest-api-plugin-methods/delete.js'
 import postManyMethod from './rest-api-plugin-methods/post-many.js'
 import { patchManyMethod, patchWhereIdsMethod, patchWhereMethod } from './rest-api-plugin-methods/patch-many.js'
 import { deleteManyMethod, deleteWhereMethod } from './rest-api-plugin-methods/delete-many.js'
-import { claimMethod, lockByIdsMethod, queryAllMethod, scopedQueryMethod } from './rest-api-plugin-methods/scoped-reads.js'
+import { claimMethod, lockByIdsMethod, lockWhereMethod, queryAllMethod, scopedQueryMethod } from './rest-api-plugin-methods/scoped-reads.js'
 import enrichAttributesMethod from './rest-api-plugin-methods/enrich-attributes.js'
 import checkPermissionsMethod from './rest-api-plugin-methods/check-permissions.js'
 import applyQueryFiltersMethod from './rest-api-plugin-methods/apply-query-filters.js'
@@ -112,6 +112,7 @@ export const RestApiPlugin = {
     addResourceMethod('queryAll', withAvailableContext(queryAllMethod))
     addResourceMethod('scopedQuery', withAvailableContext(scopedQueryMethod))
     addResourceMethod('lockByIds', withAvailableContext(lockByIdsMethod))
+    addResourceMethod('lockWhere', withAvailableContext(lockWhereMethod))
     addResourceMethod('claim', withAvailableContext(claimMethod))
 
     // Relationship methods

package/plugins/core/rest-api-positioning-plugin.js

@@ -214,7 +214,7 @@ export const PositioningPlugin = {
     // create no positioned rows and change no positions.
     const refusePositions = ({ context, scopeName }) => {
       if (!enabled(scopeName)) return
-      const moves = context.inputRecords.some(record => {
+      const moves = [config.field, config.beforeIdField].some(field => context.rawFields?.includes(field)) || context.inputRecords.some(record => {
         const attributes = record?.data?.attributes || {}
         return attributes[config.field] !== undefined || attributes[config.beforeIdField] !== undefined
       })

package/types/data-access-methods.d.ts

@@ -3,11 +3,16 @@ import type { ManagedTransaction } from './transactions.js'
 import type { DirectResourceId, InputResourceId, JsonApiResource, PlainResource, ResourceFormat, ResourceIdentifier } from './representations.js'
 import type { JsonApiWriteDocument, QueryParams, RemovedResourceOptions, SelectionParams } from './resource-methods.js'
 
-// Many-row writes return nothing by default; 'minimal' returns identifiers, 'full' the records.
-export type ManyRowReturning = 'none' | 'minimal' | 'full'
+// Many-row writes return nothing by default; 'minimal' returns identifiers, 'full' the records,
+// 'count' the number of rows written.
+export type ManyRowReturning = 'none' | 'minimal' | 'full' | 'count'
+export type ManyRowCount = { meta: { count: number } }
 export type ManyRowResult<Format extends ResourceFormat, Returning extends ManyRowReturning, Fields extends object, Type extends string> =
   Returning extends 'none' ? void
+    : Returning extends 'count' ? ManyRowCount
     : { data: Array<Returning extends 'minimal' ? ResourceIdentifier<Type> : Format extends 'plain' ? PlainResource<Fields> : JsonApiResource<Fields, Type>> }
+type DeleteResult<Returning extends 'none' | 'minimal' | 'count', Type extends string> =
+  Returning extends 'none' ? void : Returning extends 'count' ? ManyRowCount : { data: ResourceIdentifier<Type>[] }
 export type RecordsResult<Format extends ResourceFormat, Fields extends object, Type extends string> =
   { data: Array<Format extends 'plain' ? PlainResource<Fields> : JsonApiResource<Fields, Type>>; included?: JsonApiResource[] }
 // knex.raw values: accepted by many-row patches for stored, non-relationship fields.
@@ -27,12 +32,17 @@ export type ManyRowCreateInput<Input extends object, Type extends string> =
   { data: Array<Partial<Input> & { id?: InputResourceId }>; document?: never } |
   { data?: never; document: { data: Array<JsonApiWriteDocument<Input, Type>['data']> } }
 export interface UpsertOptions<Input extends object> {
-  // Must include every ownership (autofilter) field.
-  keys: Array<keyof Input & string>
+  // Must include every ownership (autofilter) field. Rows may not carry an id or repeat a key.
+  // ['id'] (or the idProperty) alone is allowed on a resource without ownership fields or a
+  // row policy; every row then carries its id.
+  keys: Array<keyof Input & string> | ['id']
+  // merge also runs the patch permission checks on every row it updates.
   action: 'ignore' | 'merge'
   // merge only: fields to update; default every stored field the rows supply but the keys.
   merge?: Array<keyof Input & string>
 }
+// patchWhere and deleteWhere need filters, or all: true to act on every visible row.
+export type WhereScope = { filters: Record<string, unknown>; all?: never } | { filters?: never; all: true }
 type Patchable<Input extends object> = { [Field in keyof Input]?: Input[Field] | RawValue }
 export type ManyRowPatchInput<Input extends object, Type extends string> =
   { data: Array<Patchable<Input> & { id: DirectResourceId }>; document?: never } |
@@ -54,7 +64,8 @@ export interface ScopedQueryParams {
   purpose?: 'report' | 'bulk-write' | 'lock' | 'claim'
   filters?: Record<string, unknown>
 }
-// A Knex builder that refuses writes, locks, table changes and clearing its predicates.
+// A Knex builder over a derived table of the scoped rows. It refuses writes, locks,
+// table changes, set operations, common table expressions and access to the client.
 export type ReadOnlyQueryBuilder = Knex.QueryBuilder
 
 export interface DataAccessMethods<
@@ -88,17 +99,18 @@ export interface DataAccessMethods<
     context?: Context
   ): Promise<ManyRowResult<Format, Returning, Fields, Type>>
   patchWhere<Format extends ResourceFormat = DefaultFormat, Returning extends ManyRowReturning = 'none'>(
-    params: ManyRowWriteOptions<Format, Returning> & SharedPatchInput<Input, Type> & Omit<PatchConditionOptions<Input>, 'missing'> & { filters?: Record<string, unknown> },
+    params: ManyRowWriteOptions<Format, Returning> & SharedPatchInput<Input, Type> & Omit<PatchConditionOptions<Input>, 'missing'> & WhereScope,
     context?: Context
   ): Promise<ManyRowResult<Format, Returning, Fields, Type>>
-  deleteMany<Returning extends 'none' | 'minimal' = 'none'>(
+  deleteMany<Returning extends 'none' | 'minimal' | 'count' = 'none'>(
     params: RemovedResourceOptions & { ids: DirectResourceId[]; transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number; missing?: MissingRows },
     context?: Context
-  ): Promise<Returning extends 'none' ? void : { data: ResourceIdentifier<Type>[] }>
-  deleteWhere<Returning extends 'none' | 'minimal' = 'none'>(
-    params: RemovedResourceOptions & { filters?: Record<string, unknown>; transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number },
+  ): Promise<DeleteResult<Returning, Type>>
+  // Deletes the rows its selection finds and answers with those it deleted; it never throws for missing rows.
+  deleteWhere<Returning extends 'none' | 'minimal' | 'count' = 'none'>(
+    params: RemovedResourceOptions & WhereScope & { transaction?: ManagedTransaction; returning?: Returning; chunkSize?: number },
     context?: Context
-  ): Promise<Returning extends 'none' ? void : { data: ResourceIdentifier<Type>[] }>
+  ): Promise<DeleteResult<Returning, Type>>
   // Locks the visible rows among ids (FOR UPDATE, id order) until the transaction ends.
   lockByIds<Format extends ResourceFormat = DefaultFormat>(params: {
     ids: DirectResourceId[]
@@ -107,6 +119,15 @@ export interface DataAccessMethods<
     format?: Format
     queryParams?: { fields?: Record<string, string> }
   }, context?: Context): Promise<RecordsResult<Format, Fields, Type>>
+  // Locks the visible rows the filters select (FOR UPDATE; id order, on MySQL the
+  // order of the forced index, which starts with the ownership or a filtered column),
+  // waiting for other transactions; throws RestApiRowLimitError past 1000 rows.
+  lockWhere<Format extends ResourceFormat = DefaultFormat>(params: {
+    transaction: ManagedTransaction | Knex.Transaction
+    filters?: Record<string, unknown>
+    format?: Format
+    queryParams?: { fields?: Record<string, string> }
+  }, context?: Context): Promise<RecordsResult<Format, Fields, Type>>
   // Takes up to limit (default 1, at most 1000) matching rows no other transaction holds.
   claim<Format extends ResourceFormat = DefaultFormat>(params: {
     transaction: ManagedTransaction | Knex.Transaction

package/types/errors.d.ts

@@ -118,11 +118,12 @@ export class RestApiTemporalDataError extends RestApiError {
 }
 
 export class RestApiBulkWriteUnsupportedError extends RestApiError {
-  constructor(options?: { resourceType?: string; operation?: string; reason?: string })
+  constructor(options?: { resourceType?: string; operation?: string; reason?: string; hooks?: string[] })
   code: 'REST_API_BULK_WRITE_UNSUPPORTED'
   type: 'rest_api_bulk_write_unsupported'
   statusCode: 422
-  details: { resourceType?: string; operation?: string; reason?: string }
+  /** `hooks` names the per-record hooks that caused the refusal; HTTP responses leave details out. */
+  details: { resourceType?: string; operation?: string; reason?: string; hooks?: string[] }
   toJSON(): SerializedError & Pick<RestApiBulkWriteUnsupportedError, 'code' | 'type' | 'statusCode' | 'details'>
 }
 
@@ -154,9 +155,9 @@ export class RestApiScopedQueryReadOnlyError extends RestApiError {
 }
 
 export class RestApiRowLimitError extends RestApiError {
-  constructor(options?: { resourceType?: string; rowLimit?: number })
+  constructor(options?: { resourceType?: string; rowLimit?: number; operation?: 'queryAll' | 'lockWhere' })
   code: 'REST_API_ROW_LIMIT'
   type: 'rest_api_row_limit'
-  details: { resourceType?: string; rowLimit?: number }
+  details: { resourceType?: string; rowLimit?: number; operation: 'queryAll' | 'lockWhere' }
   toJSON(): SerializedError & Pick<RestApiRowLimitError, 'code' | 'type' | 'details'>
 }

package/types/hook-context.d.ts

@@ -189,6 +189,8 @@ export type ManyRowHookContext<Context extends object = Record<string, unknown>,
   readonly transaction: ManagedTransaction
   // JSON:API documents of the rows, in input order (creates and patches).
   readonly inputRecords?: Array<{ data: { id?: InputResourceId; attributes?: Record<string, unknown>; relationships?: Record<string, unknown> } }>
+  // Patches: fields some row sets with a knex.raw value (those values are not in inputRecords).
+  readonly rawFields?: string[]
   readonly ids?: Array<string | undefined>
 }