json-rest-api 2.4.0 → 3.0.0

package/plugins/core/lib/querying/include-query-helpers.js

@@ -188,11 +188,9 @@ export const applyScopeFiltersToIncludeQuery = async ({
   }
 }
 
-// Linked ids checked per visibility query: a 1,000-row page costs one query and
-// one row-policy run per linked type, and any number of ids stays bounded SQL.
-const VISIBILITY_BATCH_SIZE = 1000
-
-export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, context }) => {
+// One query and one row-policy run per linked type, however many ids are linked.
+// With `scoped: false`, the identifiers whose records exist, visible or not.
+export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, context, scoped = true }) => {
   const idsByType = new Map()
   for (const { type, id } of identifiers) {
     if (!idsByType.has(type)) idsByType.set(type, new Set())
@@ -209,12 +207,12 @@ export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, cont
     const visibleIds = new Set()
     visible.set(type, visibleIds)
     const values = [...ids].map(id => adapter.translateFilterValue('id', id))
-    for (let start = 0; start < values.length; start += VISIBILITY_BATCH_SIZE) {
-      // whereInIdentifiers binds a long id list as one parameter where the database allows it.
-      const query = adapter.buildBaseQuery({ transaction: context.transaction })
-        .modify(whereInIdentifiers, column, values.slice(start, start + VISIBILITY_BATCH_SIZE))
-        .distinct({ [idColumn]: adapter.isCanonical() ? column : databaseIdentityExpression(knex, column) })
-      const state = await applyScopeFiltersToIncludeQuery({
+    // whereInIdentifiers binds a long id list as one parameter where the database limits parameters.
+    const query = adapter.buildBaseQuery({ transaction: context.transaction })
+      .modify(whereInIdentifiers, column, values)
+      .distinct({ [idColumn]: adapter.isCanonical() ? column : databaseIdentityExpression(knex, column) })
+    const state = scoped
+      ? await applyScopeFiltersToIncludeQuery({
         query,
         scopes,
         scopeName: type,
@@ -223,8 +221,8 @@ export const filterVisibleIdentifiers = async ({ identifiers, scopes, knex, cont
         tableName,
         queryPurpose: 'relationship-identifiers'
       })
-      for (const row of await state.query) visibleIds.add(String(row[idColumn]))
-    }
+      : { query: applyDatabaseReadOptions(query) }
+    for (const row of await state.query) visibleIds.add(String(row[idColumn]))
   }
   return identifiers.filter(({ type, id }) => visible.get(type)?.has(String(id)))
 }

package/plugins/core/lib/querying/include-to-one.js

@@ -6,7 +6,6 @@ import {
   RELATIONSHIPS_KEY,
   RELATIONSHIP_METADATA_KEY,
   COMPUTED_DEPENDENCIES_KEY,
-  RELATIONSHIP_READ_BATCH_SIZE,
 } from '../querying-writing/knex-constants.js'
 import { getOperationDiagnosticContext, wrapUnexpectedError } from '../../../../lib/error-context.js'
 import { getPolymorphicLinkage } from '../querying-writing/relationship-contracts.js'
@@ -109,36 +108,34 @@ export const loadBelongsTo = async (scope, deps) => {
 
     // Load the target records
     const targetRecords = []
-    for (let offset = 0; offset < uniqueIds.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-      let query = knex(targetTableName).whereIn(targetIdColumn, uniqueIds.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-      const selectionState = await applyFieldSelectionToQuery({
-        query,
-        scope: targetScopeObject,
-        fieldSelectionInfo,
-        tableName: targetTableName,
-        useTablePrefix: false,
-        storageAdapter,
-        db: knex,
-        context: {
-          ...(requestContext || {}),
-          scopeName: targetScope,
-          schemaInfo: targetScopeObject.vars.schemaInfo,
-          storageAdapter,
-        },
-        scopeName: targetScope
-      })
-      query = selectionState.query
-      const scopedQueryState = await applyScopeFiltersToIncludeQuery({
-        query,
-        scopes,
+    let query = knex(targetTableName).modify(whereInIdentifiers, targetIdColumn, uniqueIds)
+    const selectionState = await applyFieldSelectionToQuery({
+      query,
+      scope: targetScopeObject,
+      fieldSelectionInfo,
+      tableName: targetTableName,
+      useTablePrefix: false,
+      storageAdapter,
+      db: knex,
+      context: {
+        ...(requestContext || {}),
         scopeName: targetScope,
-        requestContext,
-        db: knex,
-        tableName: targetTableName
-      })
-      query = scopedQueryState.query
-      for (const record of await query) targetRecords.push(record)
-    }
+        schemaInfo: targetScopeObject.vars.schemaInfo,
+        storageAdapter,
+      },
+      scopeName: targetScope
+    })
+    query = selectionState.query
+    const scopedQueryState = await applyScopeFiltersToIncludeQuery({
+      query,
+      scopes,
+      scopeName: targetScope,
+      requestContext,
+      db: knex,
+      tableName: targetTableName
+    })
+    query = scopedQueryState.query
+    for (const record of await query) targetRecords.push(record)
 
     // Load relationship metadata for all target records
     await loadRelationshipMetadata(scopes, targetRecords, targetScope)
@@ -450,36 +447,34 @@ export const loadPolymorphicBelongsTo = async (scope, deps) => {
       // Query for this type
       const targetIdColumn = getIdColumnForScope(scopes, targetType)
       const targetRecords = []
-      for (let offset = 0; offset < targetIds.length; offset += RELATIONSHIP_READ_BATCH_SIZE) {
-        let query = knex(targetTable).whereIn(targetIdColumn, targetIds.slice(offset, offset + RELATIONSHIP_READ_BATCH_SIZE))
-        const selectionState = await applyFieldSelectionToQuery({
-          query,
-          scope: targetScopeObject,
-          fieldSelectionInfo,
-          tableName: targetTable,
-          useTablePrefix: false,
-          storageAdapter,
-          db: knex,
-          context: {
-            ...(requestContext || {}),
-            scopeName: targetType,
-            schemaInfo: targetScopeObject.vars.schemaInfo,
-            storageAdapter,
-          },
-          scopeName: targetType
-        })
-        query = selectionState.query
-        const scopedQueryState = await applyScopeFiltersToIncludeQuery({
-          query,
-          scopes,
+      let query = knex(targetTable).modify(whereInIdentifiers, targetIdColumn, targetIds)
+      const selectionState = await applyFieldSelectionToQuery({
+        query,
+        scope: targetScopeObject,
+        fieldSelectionInfo,
+        tableName: targetTable,
+        useTablePrefix: false,
+        storageAdapter,
+        db: knex,
+        context: {
+          ...(requestContext || {}),
           scopeName: targetType,
-          requestContext,
-          db: knex,
-          tableName: targetTable
-        })
-        query = scopedQueryState.query
-        for (const record of await query) targetRecords.push(record)
-      }
+          schemaInfo: targetScopeObject.vars.schemaInfo,
+          storageAdapter,
+        },
+        scopeName: targetType
+      })
+      query = selectionState.query
+      const scopedQueryState = await applyScopeFiltersToIncludeQuery({
+        query,
+        scopes,
+        scopeName: targetType,
+        requestContext,
+        db: knex,
+        tableName: targetTable
+      })
+      query = scopedQueryState.query
+      for (const record of await query) targetRecords.push(record)
 
       // Load relationship metadata for all target records
       await loadRelationshipMetadata(scopes, targetRecords, targetType)

package/plugins/core/lib/querying/knex-pagination-helpers.js

@@ -41,7 +41,9 @@ export const applyPaginationToQuery = ({ query, page = {}, vars, pageSizeLimit,
     ))
   }
 
-  query.limit(pageSize + (mode === 'cursor' ? 1 : 0))
+  // One extra row tells whether another page follows: always for cursors, and
+  // for numbered pages when no count query answers it.
+  query.limit(pageSize + (mode === 'cursor' || (mode === 'offset' && !vars.enablePaginationCounts) ? 1 : 0))
   if (mode === 'offset') query.offset((pageNumber - 1) * pageSize)
   return { mode, page: pageNumber, pageSize, before }
 }
@@ -89,7 +91,7 @@ export const generatePaginationLinks = (urlPrefix, scopeName, queryParams, pagin
   // Allow empty urlPrefix to generate relative links
   // if (!urlPrefix) return null;
 
-  const { page, pageCount, pageSize } = paginationMeta
+  const { page, pageCount, pageSize, hasMore } = paginationMeta
   const links = {}
 
   const baseUrl = `${urlPrefix}/${scopeName}`
@@ -107,6 +109,11 @@ export const generatePaginationLinks = (urlPrefix, scopeName, queryParams, pagin
     if (page < pageCount) {
       links.next = buildJsonApiLink(baseUrl, queryParams, { number: page + 1, size: pageSize })
     }
+  } else if (hasMore !== undefined) {
+    // Without counts there is no last page, but the extra row tells about the next one.
+    links.first = buildJsonApiLink(baseUrl, queryParams, { number: 1, size: pageSize })
+    if (page > 1) links.prev = buildJsonApiLink(baseUrl, queryParams, { number: page - 1, size: pageSize })
+    if (hasMore) links.next = buildJsonApiLink(baseUrl, queryParams, { number: page + 1, size: pageSize })
   }
 
   return links

package/plugins/core/lib/querying/query-constraint.d.ts

@@ -10,3 +10,14 @@ export declare function applyQueryConstraint(options: {
   storageAdapter: StorageAdapter
   tableName: string
 }): void
+
+export declare const writeConditions: unique symbol
+
+/** Repeats the ownership conditions the scoped reads of `scopeName` recorded on `context`. */
+export declare function applyWriteConditions<Query extends StorageQuery>(options: {
+  query: Query
+  context: { [writeConditions]?: Record<string, Record<string, unknown>> }
+  scopeName: string
+  storageAdapter: StorageAdapter
+  tableName: string
+}): Query

package/plugins/core/lib/querying/query-constraint.js

@@ -25,3 +25,48 @@ export function applyQueryConstraint ({ query, context, scopeName, storageAdapte
     })
   }
 }
+
+// Ownership values a scope filter applied to a read, by resource and field: the
+// owner a write of the rows read must still match. The by-id UPDATE, DELETE and
+// lock statements, single or many-row, repeat them as plain column predicates,
+// so a row that moved to another owner after the read is neither written nor
+// locked.
+export const writeConditions = Symbol('writeConditions')
+// A condition value meaning "any owner but not the shared (NULL) rows".
+export const ANY_OWNER = Symbol('anyOwner')
+
+/**
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} context
+ * @param {string} scopeName
+ * @param {string} field
+ * @param {unknown} value
+ */
+export function recordWriteCondition (context, scopeName, field, value) {
+  const byScope = context[writeConditions] ||= {}
+  byScope[scopeName] ||= {}
+  byScope[scopeName][field] = value
+}
+
+/**
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} context
+ * @param {string} scopeName
+ */
+export const hasWriteConditions = (context, scopeName) => Object.keys(context[writeConditions]?.[scopeName] || {}).length > 0
+
+/**
+ * @param {object} options
+ * @param {StorageQuery} options.query
+ * @param {{ [writeConditions]?: Record<string, Record<string, unknown>> }} options.context
+ * @param {string} options.scopeName
+ * @param {StorageAdapter} options.storageAdapter
+ * @param {string} options.tableName
+ */
+export function applyWriteConditions ({ query, context, scopeName, storageAdapter, tableName }) {
+  for (const [field, value] of Object.entries(context[writeConditions]?.[scopeName] || {})) {
+    const column = `${tableName}.${storageAdapter.translateColumn(field)}`
+    if (value === ANY_OWNER) query.whereNotNull(column)
+    else if (value === null) query.whereNull(column)
+    else query.where(column, value)
+  }
+  return query
+}

package/plugins/core/lib/querying/read-only-query.js

@@ -0,0 +1,92 @@
+// @ts-check
+import { RestApiScopedQueryReadOnlyError } from '../../../../lib/rest-api-errors.js'
+
+// Calls that write, lock, change the source table, clear its predicates, or
+// add rows from outside the scope (common table expressions and set operations).
+const FORBIDDEN_METHODS = new Set([
+  'insert', 'update', 'del', 'delete', 'truncate', 'upsert', 'increment', 'decrement',
+  'onConflict', 'merge', 'ignore', 'returning',
+  'forUpdate', 'forShare', 'forNoKeyUpdate', 'forKeyShare', 'skipLocked', 'noWait',
+  'from', 'fromRaw', 'table', 'into', 'withSchema', 'clear', 'clearWhere',
+  'with', 'withRecursive', 'withMaterialized', 'withNotMaterialized', 'withWrapped',
+  'union', 'unionAll', 'intersect', 'except'
+])
+// The statement kinds a SELECT-only builder may run.
+const READ_METHODS = new Set(['select', 'first', 'pluck'])
+
+/** @param {unknown} value */
+const isQueryBuilder = value => value !== null && typeof value === 'object' &&
+  typeof /** @type {{ toSQL?: unknown }} */ (value).toSQL === 'function' &&
+  typeof /** @type {{ where?: unknown }} */ (value).where === 'function'
+
+// Knex's own methods reach the client and replace the statement lists through
+// `this`; code outside them may not. One counter serves every builder because a
+// method of one builder can compile another (a subquery) through its client.
+let insideBuilderMethod = 0
+// The compiled query's state, which only Knex's own methods may replace.
+const BUILDER_STATE = new Set(['client', '_single', '_statements'])
+
+// Outside Knex's methods the statement lists read as frozen copies, so code
+// cannot push, splice or edit a predicate in place. The single values (table,
+// limit, offset and the like) read as a plain copy: Knex's SQLite compiler
+// rewrites the copy's limit when it compiles the builder as a subquery, and a
+// change to the copy reaches no query.
+/** @param {unknown} value */
+const stateCopy = value => {
+  if (Array.isArray(value)) return Object.freeze(value.map(entry => entry && typeof entry === 'object' ? Object.freeze({ ...entry }) : entry))
+  return value && typeof value === 'object' ? { ...value } : value
+}
+
+/**
+ * A SELECT-only view of a Knex query builder. Reads, joins, subqueries and
+ * execution work as usual; writes, row locks, set operations, common table
+ * expressions, calls that change the source table or clear its predicates, and
+ * access to the Knex client throw RestApiScopedQueryReadOnlyError. Builders it
+ * returns, including clones, are read-only too. It guards against mistakes, not
+ * hostile code: code that means to bypass it can use Knex directly.
+ * @template {object} Builder
+ * @param {Builder} builder
+ * @param {{ resourceType?: string }} [options]
+ * @returns {Builder}
+ */
+export function readOnlyQueryBuilder (builder, { resourceType = 'query' } = {}) {
+  if (!isQueryBuilder(builder)) throw new TypeError('readOnlyQueryBuilder needs a Knex query builder')
+  /** @param {string} method @returns {never} */
+  const refuse = method => { throw new RestApiScopedQueryReadOnlyError({ resourceType, method }) }
+  /** @type {Builder} */
+  const proxy = new Proxy(builder, {
+    get (target, property, receiver) {
+      if (typeof property === 'string' && FORBIDDEN_METHODS.has(property)) return () => refuse(property)
+      if (property === 'client' && !insideBuilderMethod) refuse('client')
+      const value = Reflect.get(target, property, receiver)
+      if ((property === '_single' || property === '_statements') && !insideBuilderMethod) return stateCopy(value)
+      if (typeof value !== 'function' || property === 'constructor') return value
+      return /** @this {unknown} */ function (/** @type {unknown[]} */ ...args) {
+        insideBuilderMethod++
+        let result
+        try {
+          // Knex methods return `this`, which is the proxy; new builders get wrapped.
+          result = value.apply(this === undefined ? receiver : this, args)
+        } finally {
+          insideBuilderMethod--
+        }
+        return result !== receiver && result !== target && isQueryBuilder(result)
+          ? readOnlyQueryBuilder(result, { resourceType })
+          : result
+      }
+    },
+    set (target, property, value) {
+      if (property === '_method' && !READ_METHODS.has(value)) refuse(String(value))
+      if (property === 'client' || (BUILDER_STATE.has(String(property)) && !insideBuilderMethod)) refuse(`set ${String(property)}`)
+      return Reflect.set(target, property, value)
+    },
+    defineProperty (_target, property) { return refuse(`defineProperty ${String(property)}`) },
+    // Deletes follow the rules of set: lodash's isPlainObject, which Knex's MySQL
+    // compiler runs on a subquery value, sets and then deletes Symbol.toStringTag.
+    deleteProperty (target, property) {
+      if (property === 'client' || property === '_method' || (BUILDER_STATE.has(String(property)) && !insideBuilderMethod)) refuse(`delete ${String(property)}`)
+      return Reflect.deleteProperty(target, property)
+    }
+  })
+  return proxy
+}

package/plugins/core/lib/querying-writing/compile-schemas.js

@@ -238,6 +238,19 @@ export async function compileSchemas (scope, deps) {
     }
   }
 
+  // An integer every library update advances by one, for compare-and-set writes.
+  const lockVersionField = scopeOptions.lockVersionField
+  if (lockVersionField !== undefined) {
+    const definition = typeof lockVersionField === 'string' && Object.hasOwn(schemaObject.structure, lockVersionField)
+      ? schemaObject.structure[lockVersionField]
+      : undefined
+    if (!lockVersionField || lockVersionField === idProperty || lockVersionField === versionField || !definition ||
+      !['integer', 'number'].includes(definition.type) || definition.nullable === true ||
+      definition.computed || definition.virtual || definition.belongsTo || definition.getter || definition.setter || definition.storage?.serialize) {
+      throw new Error(`lockVersionField in '${scopeName}' must name a stored, non-nullable integer attribute without getters, setters, serializers or relationships`)
+    }
+  }
+
   // Cache everything
   const schemaInfo = {
 
@@ -253,6 +266,7 @@ export async function compileSchemas (scope, deps) {
     outputFields,
     outputRelationships,
     ...(versionField === undefined ? {} : { versionField }),
+    ...(lockVersionField === undefined ? {} : { lockVersionField }),
     tableName: scopeOptions.tableName || scopeName,
     idProperty,
     storage,

package/plugins/core/lib/querying-writing/database-value-normalizers.js

@@ -207,6 +207,25 @@ export function normalizeDateValue (value, type, {
   })
 }
 
+/**
+ * Refuses an object or array as the value of a single column. Drivers do not
+ * store one as written, and MySQL drivers that format queries on the client
+ * (mysql2 before 3.17, mysql) expand a plain object into `key` = value pairs,
+ * which rewrites the statement: a value for one column could set others.
+ * Dates and byte arrays (Buffer, Uint8Array) are single column values.
+ * @param {unknown} value
+ * @param {{ fieldName?: string, parameter?: string }} [options] `parameter` is the input path the error names
+ * @returns {void}
+ */
+export function assertSingleColumnValue (value, { fieldName, parameter = `data.attributes.${fieldName}` } = {}) {
+  if (value === null || typeof value !== 'object' || value instanceof Date || value instanceof Uint8Array) return
+  const message = `Field '${fieldName}' takes a single value, not ${Array.isArray(value) ? 'an array' : 'an object'}. Declare it with type 'object' or 'array', or give it a storage serializer.`
+  throw new RestApiValidationError(message, {
+    fields: [parameter],
+    violations: [{ field: parameter, rule: 'single_value', message }]
+  })
+}
+
 /**
  * Converts structured JSON and temporal values to the values expected by
  * database drivers. JSON schema validation keeps date and dateTime values as
@@ -343,6 +362,19 @@ export function normalizeStructuredValue (value, type, { fieldName, resourceType
   }
 }
 
+// Drivers return BIGINT and some other numeric columns as strings. A value a
+// JavaScript number holds exactly, so that it prints back as the same digits
+// (trailing fraction zeros aside), becomes one; any other, and any integer past
+// 2^53 even when it prints back, keeps its digits as a string. Decimals
+// (precision and scale) keep theirs, to avoid rounding.
+const isDecimalField = definition => definition.precision !== undefined && definition.scale !== undefined
+/** @param {string} value */
+const storedNumber = value => {
+  const number = Number(value)
+  const digits = (value.includes('.') ? value.replace(/\.?0+$/, '') : value).replace(/^-0$/, '0')
+  return Number.isFinite(number) && (!Number.isInteger(number) || Number.isSafeInteger(number)) && String(number) === digits ? number : value
+}
+
 /**
  * Normalizes database values in an attributes object
  *
@@ -386,6 +418,13 @@ export function normalizeAttributes (attributes, schemaStructure, {
       }
     }
 
+    // Stored values only: an id stays a string, and a getter, computed or
+    // virtual field keeps the value its code made.
+    if (['number', 'integer'].includes(fieldDef.type) && typeof value === 'string' && fieldName !== 'id' && !isDecimalField(fieldDef) &&
+        typeof fieldDef.getter !== 'function' && fieldDef.computed !== true && fieldDef.virtual !== true) {
+      normalized[fieldName] = storedNumber(value)
+    }
+
     // Normalize boolean values
     if (fieldDef.type === 'boolean') {
       if (value === 1 || value === '1') {
@@ -426,9 +465,7 @@ export function normalizeAttributes (attributes, schemaStructure, {
 export function normalizeStoredFieldValue (value, definition, { fieldName, resourceType }) {
   if (value == null) return null
   if (definition.type === 'id') return String(value)
-  const decimal = definition.precision !== undefined && definition.scale !== undefined
-  // Drivers return some numeric columns as strings; decimals keep theirs to avoid rounding.
-  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !decimal) return Number(value)
+  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !isDecimalField(definition)) return storedNumber(value)
   return normalizeAttributes({ [fieldName]: value }, { [fieldName]: definition }, { resourceType })[fieldName]
 }