json-rest-api 3.0.0-rc.0 → 3.0.1

package/plugins/core/lib/querying/read-only-query.js

@@ -1,12 +1,15 @@
 // @ts-check
 import { RestApiScopedQueryReadOnlyError } from '../../../../lib/rest-api-errors.js'
 
-// Calls that write, lock, change the source table or drop the scope predicates.
+// Calls that write, lock, change the source table, clear its predicates, or
+// add rows from outside the scope (common table expressions and set operations).
 const FORBIDDEN_METHODS = new Set([
   'insert', 'update', 'del', 'delete', 'truncate', 'upsert', 'increment', 'decrement',
   'onConflict', 'merge', 'ignore', 'returning',
   'forUpdate', 'forShare', 'forNoKeyUpdate', 'forKeyShare', 'skipLocked', 'noWait',
-  'from', 'table', 'into', 'withSchema', 'clear', 'clearWhere'
+  'from', 'fromRaw', 'table', 'into', 'withSchema', 'clear', 'clearWhere',
+  'with', 'withRecursive', 'withMaterialized', 'withNotMaterialized', 'withWrapped',
+  'union', 'unionAll', 'intersect', 'except'
 ])
 // The statement kinds a SELECT-only builder may run.
 const READ_METHODS = new Set(['select', 'first', 'pluck'])
@@ -16,11 +19,31 @@ const isQueryBuilder = value => value !== null && typeof value === 'object' &&
   typeof /** @type {{ toSQL?: unknown }} */ (value).toSQL === 'function' &&
   typeof /** @type {{ where?: unknown }} */ (value).where === 'function'
 
+// Knex's own methods reach the client and replace the statement lists through
+// `this`; code outside them may not. One counter serves every builder because a
+// method of one builder can compile another (a subquery) through its client.
+let insideBuilderMethod = 0
+// The compiled query's state, which only Knex's own methods may replace.
+const BUILDER_STATE = new Set(['client', '_single', '_statements'])
+
+// Outside Knex's methods the statement lists read as frozen copies, so code
+// cannot push, splice or edit a predicate in place. The single values (table,
+// limit, offset and the like) read as a plain copy: Knex's SQLite compiler
+// rewrites the copy's limit when it compiles the builder as a subquery, and a
+// change to the copy reaches no query.
+/** @param {unknown} value */
+const stateCopy = value => {
+  if (Array.isArray(value)) return Object.freeze(value.map(entry => entry && typeof entry === 'object' ? Object.freeze({ ...entry }) : entry))
+  return value && typeof value === 'object' ? { ...value } : value
+}
+
 /**
  * A SELECT-only view of a Knex query builder. Reads, joins, subqueries and
- * execution work as usual; writes, row locks and calls that change the source
- * table or clear its predicates throw RestApiScopedQueryReadOnlyError. Builders
- * it returns, including clones, are read-only too.
+ * execution work as usual; writes, row locks, set operations, common table
+ * expressions, calls that change the source table or clear its predicates, and
+ * access to the Knex client throw RestApiScopedQueryReadOnlyError. Builders it
+ * returns, including clones, are read-only too. It guards against mistakes, not
+ * hostile code: code that means to bypass it can use Knex directly.
  * @template {object} Builder
  * @param {Builder} builder
  * @param {{ resourceType?: string }} [options]
@@ -28,27 +51,41 @@ const isQueryBuilder = value => value !== null && typeof value === 'object' &&
  */
 export function readOnlyQueryBuilder (builder, { resourceType = 'query' } = {}) {
   if (!isQueryBuilder(builder)) throw new TypeError('readOnlyQueryBuilder needs a Knex query builder')
+  /** @param {string} method @returns {never} */
+  const refuse = method => { throw new RestApiScopedQueryReadOnlyError({ resourceType, method }) }
   /** @type {Builder} */
   const proxy = new Proxy(builder, {
     get (target, property, receiver) {
-      if (typeof property === 'string' && FORBIDDEN_METHODS.has(property)) {
-        return () => { throw new RestApiScopedQueryReadOnlyError({ resourceType, method: property }) }
-      }
+      if (typeof property === 'string' && FORBIDDEN_METHODS.has(property)) return () => refuse(property)
+      if (property === 'client' && !insideBuilderMethod) refuse('client')
       const value = Reflect.get(target, property, receiver)
+      if ((property === '_single' || property === '_statements') && !insideBuilderMethod) return stateCopy(value)
       if (typeof value !== 'function' || property === 'constructor') return value
       return /** @this {unknown} */ function (/** @type {unknown[]} */ ...args) {
-        // Knex methods return `this`, which is the proxy; new builders get wrapped.
-        const result = value.apply(this === undefined ? receiver : this, args)
+        insideBuilderMethod++
+        let result
+        try {
+          // Knex methods return `this`, which is the proxy; new builders get wrapped.
+          result = value.apply(this === undefined ? receiver : this, args)
+        } finally {
+          insideBuilderMethod--
+        }
         return result !== receiver && result !== target && isQueryBuilder(result)
           ? readOnlyQueryBuilder(result, { resourceType })
           : result
       }
     },
     set (target, property, value) {
-      if (property === '_method' && !READ_METHODS.has(value)) {
-        throw new RestApiScopedQueryReadOnlyError({ resourceType, method: String(value) })
-      }
+      if (property === '_method' && !READ_METHODS.has(value)) refuse(String(value))
+      if (property === 'client' || (BUILDER_STATE.has(String(property)) && !insideBuilderMethod)) refuse(`set ${String(property)}`)
       return Reflect.set(target, property, value)
+    },
+    defineProperty (_target, property) { return refuse(`defineProperty ${String(property)}`) },
+    // Deletes follow the rules of set: lodash's isPlainObject, which Knex's MySQL
+    // compiler runs on a subquery value, sets and then deletes Symbol.toStringTag.
+    deleteProperty (target, property) {
+      if (property === 'client' || property === '_method' || (BUILDER_STATE.has(String(property)) && !insideBuilderMethod)) refuse(`delete ${String(property)}`)
+      return Reflect.deleteProperty(target, property)
     }
   })
   return proxy

package/plugins/core/lib/querying-writing/database-value-normalizers.js

@@ -207,6 +207,25 @@ export function normalizeDateValue (value, type, {
   })
 }
 
+/**
+ * Refuses an object or array as the value of a single column. Drivers do not
+ * store one as written, and MySQL drivers that format queries on the client
+ * (mysql2 before 3.17, mysql) expand a plain object into `key` = value pairs,
+ * which rewrites the statement: a value for one column could set others.
+ * Dates and byte arrays (Buffer, Uint8Array) are single column values.
+ * @param {unknown} value
+ * @param {{ fieldName?: string, parameter?: string }} [options] `parameter` is the input path the error names
+ * @returns {void}
+ */
+export function assertSingleColumnValue (value, { fieldName, parameter = `data.attributes.${fieldName}` } = {}) {
+  if (value === null || typeof value !== 'object' || value instanceof Date || value instanceof Uint8Array) return
+  const message = `Field '${fieldName}' takes a single value, not ${Array.isArray(value) ? 'an array' : 'an object'}. Declare it with type 'object' or 'array', or give it a storage serializer.`
+  throw new RestApiValidationError(message, {
+    fields: [parameter],
+    violations: [{ field: parameter, rule: 'single_value', message }]
+  })
+}
+
 /**
  * Converts structured JSON and temporal values to the values expected by
  * database drivers. JSON schema validation keeps date and dateTime values as
@@ -343,6 +362,19 @@ export function normalizeStructuredValue (value, type, { fieldName, resourceType
   }
 }
 
+// Drivers return BIGINT and some other numeric columns as strings. A value a
+// JavaScript number holds exactly, so that it prints back as the same digits
+// (trailing fraction zeros aside), becomes one; any other, and any integer past
+// 2^53 even when it prints back, keeps its digits as a string. Decimals
+// (precision and scale) keep theirs, to avoid rounding.
+const isDecimalField = definition => definition.precision !== undefined && definition.scale !== undefined
+/** @param {string} value */
+const storedNumber = value => {
+  const number = Number(value)
+  const digits = (value.includes('.') ? value.replace(/\.?0+$/, '') : value).replace(/^-0$/, '0')
+  return Number.isFinite(number) && (!Number.isInteger(number) || Number.isSafeInteger(number)) && String(number) === digits ? number : value
+}
+
 /**
  * Normalizes database values in an attributes object
  *
@@ -386,6 +418,13 @@ export function normalizeAttributes (attributes, schemaStructure, {
       }
     }
 
+    // Stored values only: an id stays a string, and a getter, computed or
+    // virtual field keeps the value its code made.
+    if (['number', 'integer'].includes(fieldDef.type) && typeof value === 'string' && fieldName !== 'id' && !isDecimalField(fieldDef) &&
+        typeof fieldDef.getter !== 'function' && fieldDef.computed !== true && fieldDef.virtual !== true) {
+      normalized[fieldName] = storedNumber(value)
+    }
+
     // Normalize boolean values
     if (fieldDef.type === 'boolean') {
       if (value === 1 || value === '1') {
@@ -426,9 +465,7 @@ export function normalizeAttributes (attributes, schemaStructure, {
 export function normalizeStoredFieldValue (value, definition, { fieldName, resourceType }) {
   if (value == null) return null
   if (definition.type === 'id') return String(value)
-  const decimal = definition.precision !== undefined && definition.scale !== undefined
-  // Drivers return some numeric columns as strings; decimals keep theirs to avoid rounding.
-  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !decimal) return Number(value)
+  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !isDecimalField(definition)) return storedNumber(value)
   return normalizeAttributes({ [fieldName]: value }, { [fieldName]: definition }, { resourceType })[fieldName]
 }
 

package/plugins/core/lib/storage/canonical-data-helpers.js

@@ -14,6 +14,8 @@ import { translateCanonicalAttributesForStorage, getCanonicalResourceId, getCano
 import { unwrapQueryBuilderState, withQueryFilteringContext } from '../querying/query-builder-utils.js'
 import { applyInsertReturning } from '../querying-writing/database-capabilities.js'
 import { serializeJsonApiQuery } from '../querying-writing/connectors-query-parser.js'
+import { deleteFoundRow, foundRow, lockFoundRow, updateFoundRow } from './single-row-writes.js'
+import { whereInIdentifiers } from '../querying/identifier-query.js'
 
 /** @param {StorageRow} row @param {CanonicalDescriptor} descriptor */
 function requireCanonicalResourceId (row, descriptor) {
@@ -91,6 +93,25 @@ export function createCanonicalDataHelpers ({
     return descriptors
   }
 
+  // The canonical row of the record a single write found with its scoped read.
+  /** @param {string} scopeName @param {{ db: import('./storage-types.js').StorageDatabase, id: string | number }} context */
+  const foundRecord = (scopeName, context) => {
+    const storageAdapter = getScopeStorageAdapter(scopeName)
+    if (!storageAdapter) throw new Error(`Storage adapter for resource '${scopeName}' not found`)
+    const descriptor = getDescriptor(scopeName)
+    const { canonical } = descriptor
+    return foundRow({
+      byId: () => context.db(canonical.tableName)
+        .where(getCanonicalResourceIdColumn(descriptor), context.id)
+        .where(canonical.resourceColumn, descriptor.resource)
+        .where(canonical.tenantColumn, descriptor.tenant),
+      context,
+      scopeName,
+      storageAdapter,
+      tableName: canonical.tableName
+    })
+  }
+
   /** @type {CanonicalDataWriteHelpers['dataExists']} */
   const dataExists = async ({ scopeName, context }) => {
     const storageAdapter = getScopeStorageAdapter(scopeName)
@@ -188,13 +209,7 @@ export function createCanonicalDataHelpers ({
       return 1
     }
 
-    const result = await context.db(canonical.tableName)
-      .where(logicalIdColumn, id)
-      .where(canonical.resourceColumn, descriptor.resource)
-      .where(canonical.tenantColumn, descriptor.tenant)
-      .update(writeRow)
-
-    return result
+    return updateFoundRow(foundRecord(scopeName, context), { db: context.db, scopeName, id, idColumn: logicalIdColumn, values: writeRow })
   }
 
   /** @type {CanonicalDataWriteHelpers['dataPatch']} */
@@ -204,7 +219,6 @@ export function createCanonicalDataHelpers ({
       context.storageAdapter = storageAdapter
     }
     const descriptor = getDescriptor(scopeName)
-    const { canonical } = descriptor
     const id = context.id
     const logicalIdColumn = getCanonicalResourceIdColumn(descriptor)
     const attributes = context.inputRecord?.data?.attributes || {}
@@ -216,17 +230,7 @@ export function createCanonicalDataHelpers ({
       Object.entries(row).filter(([, value]) => value !== undefined)
     )
 
-    if (Object.keys(updateRow).length === 0) {
-      return 0
-    }
-
-    const result = await context.db(canonical.tableName)
-      .where(logicalIdColumn, id)
-      .where(canonical.resourceColumn, descriptor.resource)
-      .where(canonical.tenantColumn, descriptor.tenant)
-      .update(updateRow)
-
-    return result
+    return updateFoundRow(foundRecord(scopeName, context), { db: context.db, scopeName, id, idColumn: logicalIdColumn, values: updateRow })
   }
 
   /** @type {CanonicalDataWriteHelpers['dataDelete']} */
@@ -236,23 +240,18 @@ export function createCanonicalDataHelpers ({
       context.storageAdapter = storageAdapter
     }
     const descriptor = getDescriptor(scopeName)
-    const { canonical } = descriptor
     const id = context.id
-    const logicalIdColumn = getCanonicalResourceIdColumn(descriptor)
-
-    await linkStore.invalidateDeletedLinkTargets(scopeName, context)
-    const result = await context.db(canonical.tableName)
-      .where(logicalIdColumn, id)
-      .where(canonical.resourceColumn, descriptor.resource)
-      .where(canonical.tenantColumn, descriptor.tenant)
-      .delete()
+    const row = foundRecord(scopeName, context)
 
+    const idColumn = getCanonicalResourceIdColumn(descriptor)
+    await linkStore.invalidateDeletedLinkTargets(scopeName, context, () => lockFoundRow(row, { scopeName, id, idColumn }))
+    const deleted = await deleteFoundRow(row, { scopeName, id })
     await linkStore.deleteResourceLinks({ descriptor, scopeName, id, db: context.db })
 
-    return result
+    return deleted
   }
 
-  // Supplying ids returns minimal records for the caller's bounded validation batch.
+  // Supplying ids returns minimal records for all of them, in one statement.
   /** @overload @param {DataMinimalRequest & { ids: Array<string | number> }} request @returns {Promise<DataResource[]>} */
   /** @overload @param {DataMinimalRequest & { ids?: undefined, context: DataReadContext & {id: string | number} }} request @returns {Promise<DataResource | null>} */
   /** @param {DataMinimalRequest & { ids?: Array<string | number> }} request */
@@ -279,8 +278,7 @@ export function createCanonicalDataHelpers ({
       .where(canonical.tenantColumn, descriptor.tenant)
     if (ids === undefined) query.where(logicalIdColumn, id)
     else {
-      // Knex types exclude unknown bindings; the configured driver still validates serializer results.
-      query.whereIn(logicalIdColumn, /** @type {import('knex').Knex.Value[]} */ (ids.map(value => storageAdapter.translateFilterValue('id', value))))
+      query.modify(whereInIdentifiers, logicalIdColumn, ids.map(value => storageAdapter.translateFilterValue('id', value)))
         .distinct(`${canonical.tableName}.${logicalIdColumn}`)
     }
 
@@ -332,7 +330,7 @@ export function createCanonicalDataHelpers ({
       return minimal
     }
     if (ids !== undefined) {
-      // Filtering owns predicates; the validation batch owns its identity projection.
+      // Filtering owns predicates; the id list owns its identity projection.
       query.clearSelect().distinct(`${canonical.tableName}.${logicalIdColumn}`)
       /** @type {StorageRow[]} */
       const rows = await applyDatabaseReadOptions(storageAdapter.buildBaseQuery({ transaction: context.db }).whereIn(logicalIdColumn, query))
@@ -343,7 +341,7 @@ export function createCanonicalDataHelpers ({
   }
 
   /** @type {CanonicalDataReadHelpers['dataGet']} */
-  const dataGet = async ({ scopeName, context }) => {
+  const dataGet = async ({ scopeName, context, runHooks, scoped = false }) => {
     const storageAdapter = getScopeStorageAdapter(scopeName)
     if (!storageAdapter) throw new Error(`Storage adapter for resource '${scopeName}' not found`)
     context.storageAdapter = storageAdapter
@@ -357,17 +355,40 @@ export function createCanonicalDataHelpers ({
     const fieldSelectionInfo = await buildFieldSelection(scope, { context })
     context.computedDependencies = fieldSelectionInfo.computedDependencies
 
+    // The preceding minimal read has already applied permission and row filters,
+    // unless the read is `scoped`: then it applies them itself, as the only read,
+    // and names its columns' table, since a row policy may join tables.
+    const column = (/** @type {string} */ name) => scoped ? `${canonical.tableName}.${name}` : name
     let query = context.db(canonical.tableName)
-      .where(logicalIdColumn, id)
-      .where(canonical.resourceColumn, descriptor.resource)
-      .where(canonical.tenantColumn, descriptor.tenant)
+      .where(column(logicalIdColumn), id)
+      .where(column(canonical.resourceColumn), descriptor.resource)
+      .where(column(canonical.tenantColumn), descriptor.tenant)
+    if (scoped) {
+      const filterHooks = runHooks
+      if (!filterHooks) throw new Error('A scoped read needs runHooks')
+      query = (await withQueryFilteringContext(context, {
+        query,
+        filters: context.queryParams?.filters,
+        schemaInfo: context.schemaInfo,
+        scopeName,
+        tableName: canonical.tableName,
+        db: context.db,
+        queryPurpose: 'single',
+        isAnyApi: true,
+        adapter: storageAdapter,
+        storageAdapter
+      }, async () => {
+        await applyBuiltInAnyApiQueryFilters(context)
+        await filterHooks('knexQueryFiltering')
+      })).query
+    }
 
     const selectionState = await applyFieldSelectionToQuery({
       query,
       scope,
       fieldSelectionInfo,
       tableName: canonical.tableName,
-      useTablePrefix: false,
+      useTablePrefix: scoped,
       storageAdapter,
       db: context.db,
       context,
@@ -512,6 +533,11 @@ export function createCanonicalDataHelpers ({
     let rows = await applyDatabaseReadOptions(queryBuilder)
     let cursorRecords = null
     let hasMore = false
+    // Without a count, a numbered page's one extra row answers hasMore.
+    if (paginationInfo.mode === 'offset' && !scope.vars.enablePaginationCounts) {
+      hasMore = rows.length > paginationInfo.pageSize
+      if (hasMore) rows = rows.slice(0, paginationInfo.pageSize)
+    }
 
     if (paginationInfo.mode === 'cursor') {
       const { pageSize } = paginationInfo
@@ -615,7 +641,7 @@ export function createCanonicalDataHelpers ({
         const total = Number(countResult?.count ?? countResult?.total ?? 0)
         paginationMeta = calculatePaginationMeta(total, page, pageSize)
       } else {
-        paginationMeta = { page, pageSize }
+        paginationMeta = { page, pageSize, hasMore }
       }
 
       context.returnMeta.paginationMeta = paginationMeta
@@ -702,7 +728,7 @@ export function createCanonicalDataHelpers ({
   /**
    * The canonical rows of one resource with ownership and row policy applied.
    * Many-row writes and row locks are table-storage features; AnyAPI resources
-   * get the read-only builder only.
+   * get the read-only builder only. Resolves to `{ query, alias }`.
    * @param {{ scopeName: string, context: DataReadContext, runHooks: (name: 'knexQueryFiltering') => unknown, alias?: string, purpose: string, filters?: Record<string, unknown> }} request
    */
   const dataScopedQuery = async ({ scopeName, context, runHooks, alias, purpose, filters }) => {
@@ -728,9 +754,13 @@ export function createCanonicalDataHelpers ({
       await runHooks('knexQueryFiltering')
     })
     // Wrapped: a builder is thenable, so an async function cannot return it bare.
-    return { query: filteredState.query }
+    return { query: filteredState.query, alias: tableAlias }
   }
 
+  // A get with no reader of the minimal record reads once, scoped.
+  /** @type {NonNullable<CanonicalDataReadHelpers['dataGetScoped']>} */
+  const dataGetScoped = ({ scopeName, context, runHooks }) => dataGet({ scopeName, context, runHooks, scoped: true })
+
   return {
     dataScopedQuery,
     dataExists,
@@ -739,6 +769,7 @@ export function createCanonicalDataHelpers ({
     dataPatch,
     dataDelete,
     dataGet,
+    dataGetScoped,
     dataQuery,
     dataAggregate,
     dataGetMinimal