json-rest-api 3.0.0-rc.0 → 3.0.1

package/plugins/core/lib/anyapi/canonical-relationship-reader.js

@@ -1,7 +1,7 @@
 import { whereInIdentifiers } from '../querying/identifier-query.js'
 import { loadIncludedParentMap, parseIncludeTree } from '../querying/include-query-helpers.js'
 import { getResourceFieldset, parseFieldset } from '../querying-writing/field-utils.js'
-import { COMPUTED_DEPENDENCIES_KEY, RELATIONSHIP_READ_BATCH_SIZE } from '../querying-writing/knex-constants.js'
+import { COMPUTED_DEPENDENCIES_KEY } from '../querying-writing/knex-constants.js'
 import { buildResourceUrl, buildRelationshipUrl } from '../querying/url-helpers.js'
 import { normalizeId } from './utils/descriptor-helpers.js'
 import { applyFieldSelectionToQuery, buildFieldSelection } from '../querying-writing/knex-field-helpers.js'
@@ -177,36 +177,34 @@ export function createCanonicalRelationshipReader ({ api, knex, getDescriptor, g
         const targetDescriptor = getDescriptor(belongsToInfo.target)
 
         const nestedResources = []
-        for (let offset = 0; offset < ids.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-          let query = db(targetDescriptor.canonical.tableName)
-            .where(targetDescriptor.canonical.tenantColumn, targetDescriptor.tenant)
-            .where(targetDescriptor.canonical.resourceColumn, targetDescriptor.resource)
-            .whereIn(getCanonicalResourceIdColumn(targetDescriptor), ids.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-          const selectionState = await applyIncludeFieldSelection({
-            query,
-            resourceName: targetDescriptor.resource,
-            tableName: targetDescriptor.canonical.tableName,
-            context
-          })
-          query = selectionState.query
-          const scopedQueryState = await applyTargetScopeFilters({
-            query,
-            resourceName: targetDescriptor.resource,
-            tableName: targetDescriptor.canonical.tableName,
-            context
-          })
-          query = scopedQueryState.query
-          const rows = await applyDatabaseReadOptions(query)
+        let query = db(targetDescriptor.canonical.tableName)
+          .where(targetDescriptor.canonical.tenantColumn, targetDescriptor.tenant)
+          .where(targetDescriptor.canonical.resourceColumn, targetDescriptor.resource)
+          .modify(whereInIdentifiers, getCanonicalResourceIdColumn(targetDescriptor), ids)
+        const selectionState = await applyIncludeFieldSelection({
+          query,
+          resourceName: targetDescriptor.resource,
+          tableName: targetDescriptor.canonical.tableName,
+          context
+        })
+        query = selectionState.query
+        const scopedQueryState = await applyTargetScopeFilters({
+          query,
+          resourceName: targetDescriptor.resource,
+          tableName: targetDescriptor.canonical.tableName,
+          context
+        })
+        query = scopedQueryState.query
+        const rows = await applyDatabaseReadOptions(query)
 
-          for (const row of rows) {
-            const includeResource = includeRow({
-              row,
-              descriptor: targetDescriptor,
-              context,
-              fieldSelectionInfo: selectionState.fieldSelectionInfo
-            }, includes)
-            nestedResources.push(includeResource)
-          }
+        for (const row of rows) {
+          const includeResource = includeRow({
+            row,
+            descriptor: targetDescriptor,
+            context,
+            fieldSelectionInfo: selectionState.fieldSelectionInfo
+          }, includes)
+          nestedResources.push(includeResource)
         }
 
         if (childKeys.length > 0 && nestedResources.length > 0) {
@@ -244,36 +242,34 @@ export function createCanonicalRelationshipReader ({ api, knex, getDescriptor, g
           const targetDescriptor = getDescriptor(targetType)
 
           const ids = [...idSet]
-          for (let offset = 0; offset < ids.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-            let query = db(targetDescriptor.canonical.tableName)
-              .where(targetDescriptor.canonical.tenantColumn, targetDescriptor.tenant)
-              .where(targetDescriptor.canonical.resourceColumn, targetDescriptor.resource)
-              .whereIn(getCanonicalResourceIdColumn(targetDescriptor), ids.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-            const selectionState = await applyIncludeFieldSelection({
-              query,
-              resourceName: targetDescriptor.resource,
-              tableName: targetDescriptor.canonical.tableName,
-              context
-            })
-            query = selectionState.query
-            const scopedQueryState = await applyTargetScopeFilters({
-              query,
-              resourceName: targetDescriptor.resource,
-              tableName: targetDescriptor.canonical.tableName,
-              context
-            })
-            query = scopedQueryState.query
-            const rows = await applyDatabaseReadOptions(query)
-
-            for (const row of rows) {
-              const includeResource = includeRow({
-                row,
-                descriptor: targetDescriptor,
-                context,
-                fieldSelectionInfo: selectionState.fieldSelectionInfo
-              }, includes)
-              nestedResources.push(includeResource)
-            }
+          let query = db(targetDescriptor.canonical.tableName)
+            .where(targetDescriptor.canonical.tenantColumn, targetDescriptor.tenant)
+            .where(targetDescriptor.canonical.resourceColumn, targetDescriptor.resource)
+            .modify(whereInIdentifiers, getCanonicalResourceIdColumn(targetDescriptor), ids)
+          const selectionState = await applyIncludeFieldSelection({
+            query,
+            resourceName: targetDescriptor.resource,
+            tableName: targetDescriptor.canonical.tableName,
+            context
+          })
+          query = selectionState.query
+          const scopedQueryState = await applyTargetScopeFilters({
+            query,
+            resourceName: targetDescriptor.resource,
+            tableName: targetDescriptor.canonical.tableName,
+            context
+          })
+          query = scopedQueryState.query
+          const rows = await applyDatabaseReadOptions(query)
+
+          for (const row of rows) {
+            const includeResource = includeRow({
+              row,
+              descriptor: targetDescriptor,
+              context,
+              fieldSelectionInfo: selectionState.fieldSelectionInfo
+            }, includes)
+            nestedResources.push(includeResource)
           }
         }
 

package/plugins/core/lib/dbIntrospection.js

@@ -117,8 +117,17 @@ function parseEnumLikeValues (columnType = '', typeName = 'enum', backslashEscap
   return values
 }
 
-function normalizeMysqlDefault (row) {
+// MariaDB (10.2.7+) writes COLUMN_DEFAULT as SQL: a literal is quoted, NULL is
+// the word NULL and an expression is bare, where MySQL gives the bare literal.
+function normalizeMariadbDefault (value) {
+  if (typeof value !== 'string') return value
+  if (value === 'NULL') return null
+  return /^'(?:''|\\.|[^'\\])*'$/s.test(value) ? parseEnumLikeValues(`enum(${value})`, 'enum', true)[0] : value
+}
+
+function normalizeMysqlDefault (row, mariadb) {
   const value = Object.hasOwn(row, 'columnDefault') ? row.columnDefault : row.column_default
+  if (mariadb) return normalizeMariadbDefault(value)
   if (row.dataType !== 'json' || typeof value !== 'string') return value
   let expression = value.trim()
   if (expression.startsWith('(') && expression.endsWith(')')) expression = expression.slice(1, -1).trim()
@@ -917,8 +926,9 @@ function normalizeDbSchemaName (rows = []) {
 }
 
 async function introspectMysqlTableSnapshot (knex, { tableName, idColumn }) {
-  const schemaRows = normalizeRows(await knex.raw('SELECT DATABASE() AS schemaName'))
+  const schemaRows = normalizeRows(await knex.raw('SELECT DATABASE() AS schemaName, VERSION() AS version'))
   const schemaName = normalizeDbSchemaName(schemaRows)
+  const mariadb = /mariadb/i.test(normalizeText(schemaRows[0]?.version))
 
   const tableRows = normalizeRows(
     await knex.raw(
@@ -1052,12 +1062,17 @@ async function introspectMysqlTableSnapshot (knex, { tableName, idColumn }) {
     )
   )
 
+  // MariaDB stores JSON as LONGTEXT with a column check json_valid(`column`):
+  // that pair is a JSON column, and its check is part of the type.
+  const jsonCheck = (/** @type {any} */ row) => mariadb && /^json_valid\(`((?:``|[^`])+)`\)$/i.exec(normalizeText(row.checkClause))?.[1].replace(/``/g, '`')
+  const jsonColumns = new Set(checkConstraintRows.map(jsonCheck).filter(Boolean))
   const columns = columnRows.map((row) => buildColumnSnapshot({
     name: row.columnName,
-    dataType: row.dataType,
-    columnType: row.columnType,
+    ...(jsonColumns.has(row.columnName) && normalizeText(row.dataType).toLowerCase() === 'longtext'
+      ? { dataType: 'json', columnType: 'json' }
+      : { dataType: row.dataType, columnType: row.columnType }),
     nullable: normalizeText(row.isNullable).toUpperCase() === 'YES',
-    defaultValue: normalizeMysqlDefault(row),
+    defaultValue: normalizeMysqlDefault(row, mariadb),
     defaultIsLiteral: true,
     extra: row.extra,
     autoIncrement: normalizeText(row.extra).toLowerCase().includes('auto_increment'),
@@ -1088,7 +1103,7 @@ async function introspectMysqlTableSnapshot (knex, { tableName, idColumn }) {
     indexes: normalizeIndexes(indexRows),
     foreignKeys: normalizeForeignKeys(foreignKeyRows, [], tableName),
     checkConstraints: normalizeCheckConstraints(
-      checkConstraintRows.map((row) => ({
+      checkConstraintRows.filter(row => !jsonColumns.has(jsonCheck(row))).map((row) => ({
         name: row.constraintName,
         clause: row.checkClause
       })),
@@ -1265,6 +1280,91 @@ export async function listKnexTableIndexes (knex, tableName) {
   return new Map(rows.map(row => [row.name, { valid: row.valid !== false, building: row.building === true }]))
 }
 
+/**
+ * The schema a MySQL or MariaDB connection is using, from `database()`, a
+ * statement that reads no table.
+ * @param {any} knex
+ * @returns {Promise<string>}
+ */
+export async function currentKnexMysqlSchema (knex) {
+  return normalizeRows(await knex.raw('select database() as name'))[0]?.name
+}
+
+/**
+ * The indexes of a MySQL or MariaDB table that the optimizer may use, unique
+ * or not, the primary key included: each with its columns in key order and
+ * the length of a column prefix (`prefix`, null for the whole column). A
+ * column list ends before its first functional key part, which names no
+ * column. Invisible (MySQL) and ignored (MariaDB) indexes are left out.
+ * @param {any} knex
+ * @param {string} schema
+ * @param {string} tableName
+ * @returns {Promise<Array<{ name: string, columns: Array<{ name: string, prefix: number | null }> }>>}
+ */
+export async function listKnexMysqlIndexes (knex, schema, tableName) {
+  const rows = normalizeRows(await knex.raw(`
+    select * from information_schema.statistics
+    where table_schema = ? and table_name = ?
+    order by index_name, seq_in_index`, [schema, tableName]))
+  const byIndex = new Map()
+  for (const raw of rows) {
+    const row = Object.fromEntries(Object.entries(raw).map(([key, value]) => [key.toLowerCase(), value]))
+    if (row.is_visible === 'NO' || row.ignored === 'YES') continue
+    if (!byIndex.has(row.index_name)) byIndex.set(row.index_name, [])
+    byIndex.get(row.index_name).push(row.column_name == null ? null : { name: row.column_name, prefix: row.sub_part == null ? null : Number(row.sub_part) })
+  }
+  return [...byIndex].map(([name, columns]) => ({ name, columns: columns.includes(null) ? columns.slice(0, columns.indexOf(null)) : columns }))
+}
+
+/**
+ * The column lists of a table's unique indexes and constraints, primary key
+ * included, that guarantee uniqueness for every row: partial, expression and
+ * invalid indexes are left out.
+ * @param {any} knex
+ * @param {string} tableName
+ * @returns {Promise<string[][]>}
+ */
+export async function listKnexUniqueKeyColumns (knex, tableName) {
+  const dialect = detectDialect(knex)
+  if (dialect === 'sqlite') {
+    const keys = []
+    for (const index of normalizeRows(await knex.raw('PRAGMA index_list(??)', [tableName]))) {
+      if (Number(index.unique) !== 1 || Number(index.partial) === 1) continue
+      const columns = normalizeRows(await knex.raw('PRAGMA index_info(??)', [index.name]))
+      if (columns.length && columns.every(column => column.name)) keys.push(columns.sort((a, b) => a.seqno - b.seqno).map(column => column.name))
+    }
+    // An INTEGER PRIMARY KEY is the rowid and has no index entry.
+    const primary = normalizeRows(await knex.raw('PRAGMA table_info(??)', [tableName])).filter(column => Number(column.pk) > 0)
+    if (primary.length) keys.push(primary.sort((a, b) => a.pk - b.pk).map(column => column.name))
+    return keys
+  }
+  const rows = dialect === 'mysql2'
+    ? normalizeRows(await knex.raw(`
+      select index_name as indexName, column_name as columnName, seq_in_index as position
+      from information_schema.statistics
+      where table_schema = database() and table_name = ? and non_unique = 0
+      order by index_name, seq_in_index`, [tableName]))
+    : normalizeRows(await knex.raw(`
+      select c.relname as "indexName", a.attname as "columnName", k.position
+      from pg_index i
+      join pg_class t on t.oid = i.indrelid
+      join pg_namespace n on n.oid = t.relnamespace
+      join pg_class c on c.oid = i.indexrelid
+      cross join lateral unnest(i.indkey) with ordinality as k(num, position)
+      left join pg_attribute a on a.attrelid = t.oid and a.attnum = k.num
+      where t.relname = ? and n.nspname = current_schema() and i.indisunique and i.indisvalid
+        and i.indpred is null and i.indexprs is null and k.position <= i.indnkeyatts
+      order by c.relname, k.position`, [tableName]))
+  const byIndex = new Map()
+  for (const row of rows) {
+    const name = row.indexName ?? row.INDEX_NAME
+    if (!byIndex.has(name)) byIndex.set(name, [])
+    byIndex.get(name).push(row.columnName ?? row.COLUMN_NAME ?? null)
+  }
+  // A functional index (MySQL) has no column name.
+  return [...byIndex.values()].filter(columns => columns.every(Boolean))
+}
+
 // Column-only ALTER must not impose the resource snapshot's ID or index restrictions.
 export async function introspectKnexColumnConstraints (knex, tableName) {
   requireKnexRaw(knex)

package/plugins/core/lib/querying/aggregate-query.js

@@ -44,9 +44,13 @@ const positiveInteger = (value, field) => {
  * Validate an aggregate request against a resource's compiled schema.
  * Only stored scalar fields whose stored values are their public values qualify:
  * getters and storage serializers would make SQL group or sum other values than clients see.
- * @param {{ params: Record<string, any>, schemaInfo: Record<string, any>, scopeName: string, defaultLimit: number, maxLimit: number }} request
+ * A system context may also group by, and count distinct, a hidden ownership
+ * field, so a job can find which owners have rows. A scoped caller would see
+ * only its own owner, and one whose optional owner filter did not resolve
+ * would see every owner's hidden value, so neither may.
+ * @param {{ params: Record<string, any>, schemaInfo: Record<string, any>, scopeName: string, defaultLimit: number, maxLimit: number, system?: boolean }} request
  */
-export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaultLimit, maxLimit }) {
+export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaultLimit, maxLimit, system = false }) {
   if (!isPlainObject(params)) invalid('params', 'aggregate requires a parameter object')
   for (const key of Object.keys(params)) {
     if (!PARAMS.has(key)) invalid(key, `Unknown aggregate parameter '${key}'`)
@@ -60,9 +64,11 @@ export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaul
   const fields = schemaInfo.schemaStructure || {}
   // A storage serializer changes what SQL sees; date and dateTime fields may still be
   // bucketed and take min/max, as SQL orders and truncates their stored temporal values.
-  const fieldDefinition = (name, parameter, { serializedTemporal = false } = {}) => {
+  const owners = new Set(system ? schemaInfo.ownershipFields || [] : [])
+  const fieldDefinition = (name, parameter, { serializedTemporal = false, owner = false } = {}) => {
     const definition = typeof name === 'string' && Object.hasOwn(fields, name) ? fields[name] : null
-    if (!definition || name === 'id' || name === schemaInfo.idProperty || definition.virtual === true || definition.hidden === true) {
+    const hidden = definition?.hidden === true && !(owner && owners.has(name))
+    if (!definition || name === 'id' || name === schemaInfo.idProperty || definition.virtual === true || hidden) {
       invalid(parameter, `'${name}' is not an attribute of '${scopeName}' that can be aggregated`)
     }
     const serialized = Boolean(definition.storage?.serialize) && !(serializedTemporal && TEMPORAL_TYPES.has(definition.type))
@@ -77,7 +83,7 @@ export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaul
   const groups = groupBy.map((entry, index) => {
     const parameter = `groupBy.${index}`
     if (isPlainObject(entry)) return compileBucket(entry, { parameter, alias: `g${index}`, fieldDefinition, names })
-    const definition = fieldDefinition(entry, parameter)
+    const definition = fieldDefinition(entry, parameter, { owner: true })
     if (!SCALAR_TYPES.has(definition.type)) invalid(parameter, `Field '${entry}' has type '${definition.type}' and cannot be grouped`)
     if (names.has(entry)) invalid('groupBy', 'groupBy lists a field twice')
     names.add(entry)
@@ -101,7 +107,7 @@ export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaul
       if (fn !== 'count') invalid(parameter, "Only count accepts '*'")
       return { name, fn, field, definition: null, alias: `m${index}`, where: condition }
     }
-    const definition = fieldDefinition(field, parameter, { serializedTemporal: fn === 'min' || fn === 'max' })
+    const definition = fieldDefinition(field, parameter, { serializedTemporal: fn === 'min' || fn === 'max', owner: fn === 'countDistinct' })
     if (!MEASURES[fn].has(definition.type)) invalid(parameter, `${fn} cannot be applied to field '${field}' of type '${definition.type}'`)
     return { name, fn, field, definition, alias: `m${index}`, where: condition }
   })

package/plugins/core/lib/querying/include-query-helpers.js

@@ -188,11 +188,9 @@ export const applyScopeFiltersToIncludeQuery = async ({
   }
 }
 
-// Linked ids checked per visibility query: a 1,000-row page costs one query and
-// one row-policy run per linked type, and any number of ids stays bounded SQL.
-const VISIBILITY_BATCH_SIZE = 1000
-
-export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, context }) => {
+// One query and one row-policy run per linked type, however many ids are linked.
+// With `scoped: false`, the identifiers whose records exist, visible or not.
+export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, context, scoped = true }) => {
   const idsByType = new Map()
   for (const { type, id } of identifiers) {
     if (!idsByType.has(type)) idsByType.set(type, new Set())
@@ -209,12 +207,12 @@ export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, cont
     const visibleIds = new Set()
     visible.set(type, visibleIds)
     const values = [...ids].map(id => adapter.translateFilterValue('id', id))
-    for (let start = 0; start < values.length; start += VISIBILITY_BATCH_SIZE) {
-      // whereInIdentifiers binds a long id list as one parameter where the database allows it.
-      const query = adapter.buildBaseQuery({ transaction: context.transaction })
-        .modify(whereInIdentifiers, column, values.slice(start, start + VISIBILITY_BATCH_SIZE))
-        .distinct({ [idColumn]: adapter.isCanonical() ? column : databaseIdentityExpression(knex, column) })
-      const state = await applyScopeFiltersToIncludeQuery({
+    // whereInIdentifiers binds a long id list as one parameter where the database limits parameters.
+    const query = adapter.buildBaseQuery({ transaction: context.transaction })
+      .modify(whereInIdentifiers, column, values)
+      .distinct({ [idColumn]: adapter.isCanonical() ? column : databaseIdentityExpression(knex, column) })
+    const state = scoped
+      ? await applyScopeFiltersToIncludeQuery({
         query,
         scopes,
         scopeName: type,
@@ -223,8 +221,8 @@ export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, cont
         tableName,
         queryPurpose: 'relationship-identifiers'
       })
-      for (const row of await state.query) visibleIds.add(String(row[idColumn]))
-    }
+      : { query: applyDatabaseReadOptions(query) }
+    for (const row of await state.query) visibleIds.add(String(row[idColumn]))
   }
   return identifiers.filter(({ type, id }) => visible.get(type)?.has(String(id)))
 }

package/plugins/core/lib/querying/include-to-one.js

@@ -6,7 +6,6 @@ import {
   RELATIONSHIPS_KEY,
   RELATIONSHIP_METADATA_KEY,
   COMPUTED_DEPENDENCIES_KEY,
-  RELATIONSHIP_READ_BATCH_SIZE,
 } from '../querying-writing/knex-constants.js'
 import { getOperationDiagnosticContext, wrapUnexpectedError } from '../../../../lib/error-context.js'
 import { getPolymorphicLinkage } from '../querying-writing/relationship-contracts.js'
@@ -109,36 +108,34 @@ export const loadBelongsTo = async (scope, deps) => {
 
     // Load the target records
     const targetRecords = []
-    for (let offset = 0; offset < uniqueIds.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-      let query = knex(targetTableName).whereIn(targetIdColumn, uniqueIds.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-      const selectionState = await applyFieldSelectionToQuery({
-        query,
-        scope: targetScopeObject,
-        fieldSelectionInfo,
-        tableName: targetTableName,
-        useTablePrefix: false,
-        storageAdapter,
-        db: knex,
-        context: {
-          ...(requestContext || {}),
-          scopeName: targetScope,
-          schemaInfo: targetScopeObject.vars.schemaInfo,
-          storageAdapter,
-        },
-        scopeName: targetScope
-      })
-      query = selectionState.query
-      const scopedQueryState = await applyScopeFiltersToIncludeQuery({
-        query,
-        scopes,
+    let query = knex(targetTableName).modify(whereInIdentifiers, targetIdColumn, uniqueIds)
+    const selectionState = await applyFieldSelectionToQuery({
+      query,
+      scope: targetScopeObject,
+      fieldSelectionInfo,
+      tableName: targetTableName,
+      useTablePrefix: false,
+      storageAdapter,
+      db: knex,
+      context: {
+        ...(requestContext || {}),
         scopeName: targetScope,
-        requestContext,
-        db: knex,
-        tableName: targetTableName
-      })
-      query = scopedQueryState.query
-      for (const record of await query) targetRecords.push(record)
-    }
+        schemaInfo: targetScopeObject.vars.schemaInfo,
+        storageAdapter,
+      },
+      scopeName: targetScope
+    })
+    query = selectionState.query
+    const scopedQueryState = await applyScopeFiltersToIncludeQuery({
+      query,
+      scopes,
+      scopeName: targetScope,
+      requestContext,
+      db: knex,
+      tableName: targetTableName
+    })
+    query = scopedQueryState.query
+    for (const record of await query) targetRecords.push(record)
 
     // Load relationship metadata for all target records
     await loadRelationshipMetadata(scopes, targetRecords, targetScope)
@@ -450,36 +447,34 @@ export const loadPolymorphicBelongsTo = async (scope, deps) => {
       // Query for this type
       const targetIdColumn = getIdColumnForScope(scopes, targetType)
       const targetRecords = []
-      for (let offset = 0; offset < targetIds.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-        let query = knex(targetTable).whereIn(targetIdColumn, targetIds.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-        const selectionState = await applyFieldSelectionToQuery({
-          query,
-          scope: targetScopeObject,
-          fieldSelectionInfo,
-          tableName: targetTable,
-          useTablePrefix: false,
-          storageAdapter,
-          db: knex,
-          context: {
-            ...(requestContext || {}),
-            scopeName: targetType,
-            schemaInfo: targetScopeObject.vars.schemaInfo,
-            storageAdapter,
-          },
-          scopeName: targetType
-        })
-        query = selectionState.query
-        const scopedQueryState = await applyScopeFiltersToIncludeQuery({
-          query,
-          scopes,
+      let query = knex(targetTable).modify(whereInIdentifiers, targetIdColumn, targetIds)
+      const selectionState = await applyFieldSelectionToQuery({
+        query,
+        scope: targetScopeObject,
+        fieldSelectionInfo,
+        tableName: targetTable,
+        useTablePrefix: false,
+        storageAdapter,
+        db: knex,
+        context: {
+          ...(requestContext || {}),
           scopeName: targetType,
-          requestContext,
-          db: knex,
-          tableName: targetTable
-        })
-        query = scopedQueryState.query
-        for (const record of await query) targetRecords.push(record)
-      }
+          schemaInfo: targetScopeObject.vars.schemaInfo,
+          storageAdapter,
+        },
+        scopeName: targetType
+      })
+      query = selectionState.query
+      const scopedQueryState = await applyScopeFiltersToIncludeQuery({
+        query,
+        scopes,
+        scopeName: targetType,
+        requestContext,
+        db: knex,
+        tableName: targetTable
+      })
+      query = scopedQueryState.query
+      for (const record of await query) targetRecords.push(record)
 
       // Load relationship metadata for all target records
       await loadRelationshipMetadata(scopes, targetRecords, targetType)

package/plugins/core/lib/querying/knex-pagination-helpers.js

@@ -41,7 +41,9 @@ export const applyPaginationToQuery = ({ query, page = {}, vars, pageSizeLimit,
     ))
   }
 
-  query.limit(pageSize + (mode === 'cursor' ? 1 : 0))
+  // One extra row tells whether another page follows: always for cursors, and
+  // for numbered pages when no count query answers it.
+  query.limit(pageSize + (mode === 'cursor' || (mode === 'offset' && !vars.enablePaginationCounts) ? 1 : 0))
   if (mode === 'offset') query.offset((pageNumber - 1) * pageSize)
   return { mode, page: pageNumber, pageSize, before }
 }
@@ -89,7 +91,7 @@ export const generatePaginationLinks = (urlPrefix, scopeName, queryParams, pagin
   // Allow empty urlPrefix to generate relative links
   // if (!urlPrefix) return null;
 
-  const { page, pageCount, pageSize } = paginationMeta
+  const { page, pageCount, pageSize, hasMore } = paginationMeta
   const links = {}
 
   const baseUrl = `${urlPrefix}/${scopeName}`
@@ -107,6 +109,11 @@ export const generatePaginationLinks = (urlPrefix, scopeName, queryParams, pagin
     if (page < pageCount) {
       links.next = buildJsonApiLink(baseUrl, queryParams, { number: page + 1, size: pageSize })
     }
+  } else if (hasMore !== undefined) {
+    // Without counts there is no last page, but the extra row tells about the next one.
+    links.first = buildJsonApiLink(baseUrl, queryParams, { number: 1, size: pageSize })
+    if (page > 1) links.prev = buildJsonApiLink(baseUrl, queryParams, { number: page - 1, size: pageSize })
+    if (hasMore) links.next = buildJsonApiLink(baseUrl, queryParams, { number: page + 1, size: pageSize })
   }
 
   return links

package/plugins/core/lib/querying/query-constraint.d.ts

@@ -10,3 +10,14 @@ export declare function applyQueryConstraint(options: {
   storageAdapter: StorageAdapter
   tableName: string
 }): void
+
+export declare const writeConditions: unique symbol
+
+/** Repeats the ownership conditions the scoped reads of `scopeName` recorded on `context`. */
+export declare function applyWriteConditions<Query extends StorageQuery>(options: {
+  query: Query
+  context: { [writeConditions]?: Record<string, Record<string, unknown>> }
+  scopeName: string
+  storageAdapter: StorageAdapter
+  tableName: string
+}): Query

package/plugins/core/lib/querying/query-constraint.js

@@ -25,3 +25,48 @@ export function applyQueryConstraint ({ query, context, scopeName, storageAdapte
     })
   }
 }
+
+// Ownership values a scope filter applied to a read, by resource and field: the
+// owner a write of the rows read must still match. The by-id UPDATE, DELETE and
+// lock statements, single or many-row, repeat them as plain column predicates,
+// so a row that moved to another owner after the read is neither written nor
+// locked.
+export const writeConditions = Symbol('writeConditions')
+// A condition value meaning "any owner but not the shared (NULL) rows".
+export const ANY_OWNER = Symbol('anyOwner')
+
+/**
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} context
+ * @param {string} scopeName
+ * @param {string} field
+ * @param {unknown} value
+ */
+export function recordWriteCondition (context, scopeName, field, value) {
+  const byScope = context[writeConditions] ||= {}
+  byScope[scopeName] ||= {}
+  byScope[scopeName][field] = value
+}
+
+/**
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} context
+ * @param {string} scopeName
+ */
+export const hasWriteConditions = (context, scopeName) => Object.keys(context[writeConditions]?.[scopeName] || {}).length > 0
+
+/**
+ * @param {object} options
+ * @param {StorageQuery} options.query
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} options.context
+ * @param {string} options.scopeName
+ * @param {StorageAdapter} options.storageAdapter
+ * @param {string} options.tableName
+ */
+export function applyWriteConditions ({ query, context, scopeName, storageAdapter, tableName }) {
+  for (const [field, value] of Object.entries(context[writeConditions]?.[scopeName] || {})) {
+    const column = `${tableName}.${storageAdapter.translateColumn(field)}`
+    if (value === ANY_OWNER) query.whereNotNull(column)
+    else if (value === null) query.whereNull(column)
+    else query.where(column, value)
+  }
+  return query
+}