json-rest-api 2.1.0 → 2.3.0

package/plugins/core/lib/anyapi/schema-utils.js

@@ -1,7 +1,7 @@
-import { hasKnexTableIndex } from '../dbIntrospection.js'
+import { hasKnexTableIndex, listKnexTableIndexes } from '../dbIntrospection.js'
 
-// Default layout of a new any_records table. The registry reads the actual
-// columns, so a migration that adds slots raises the limits for every server.
+// Default layout of any_records. The registry reads the actual columns, so a
+// migration that adds slots raises the limits for every server.
 export const SLOT_LIMITS = Object.freeze({
   string: 10,
   number: 10,
@@ -11,6 +11,18 @@ export const SLOT_LIMITS = Object.freeze({
   belongsTo: 5,
 })
 
+// Indexed slots carry a (tenant_id, resource, slot) index. Fields declared
+// `indexed: true` take them; every index costs each write to any_records.
+export const INDEXED_SLOT_LIMITS = Object.freeze({
+  string: 5,
+  number: 3,
+  boolean: 0,
+  date: 2,
+})
+
+// Indexed string slots are VARCHAR so MySQL can index them whole.
+export const INDEXED_STRING_LENGTH = 255
+
 const SLOT_COLUMNS = {
   string: index => [`string_${index}`],
   number: index => [`number_${index}`],
@@ -20,25 +32,37 @@ const SLOT_COLUMNS = {
   belongsTo: index => [`rel_${index}_id`, `rel_${index}_type`],
 }
 
+export const indexedSlotColumn = (pool, index) => `indexed_${pool}_${index}`
+
 const addSlotColumn = (table, column) => {
-  if (column.startsWith('number_')) table.double(column)
-  else if (column.startsWith('boolean_')) table.boolean(column)
-  else if (column.startsWith('date_')) table.dateTime(column, { precision: 6 })
-  else if (column.startsWith('rel_')) table.string(column)
+  const [, indexed, pool] = /^(indexed_)?([a-z]+)_/.exec(column)
+  if (pool === 'number') table.double(column)
+  else if (pool === 'boolean') table.boolean(column)
+  else if (pool === 'date') table.dateTime(column, { precision: 6 })
+  else if (pool === 'rel' || indexed) table.string(column, INDEXED_STRING_LENGTH)
   else table.text(column)
 }
 
-const resolveSlotCounts = (slots = {}) => {
+const resolveSlotCounts = (slots = {}, limits = SLOT_LIMITS, { label = 'slot', minimum = 1 } = {}) => {
   if (!slots || typeof slots !== 'object' || Array.isArray(slots)) {
-    throw new Error('AnyAPI slots must be an object of slot counts')
+    throw new Error(`AnyAPI ${label}s must be an object of slot counts`)
   }
   for (const [pool, count] of Object.entries(slots)) {
-    if (!Object.hasOwn(SLOT_LIMITS, pool)) throw new Error(`Unknown AnyAPI slot pool '${pool}'`)
-    if (!Number.isInteger(count) || count < 1) throw new Error(`AnyAPI ${pool} slot count must be a positive integer`)
+    if (!Object.hasOwn(limits, pool)) throw new Error(`Unknown AnyAPI ${label} pool '${pool}'`)
+    if (!Number.isInteger(count) || count < minimum) {
+      throw new Error(`AnyAPI ${pool} ${label} count must be ${minimum ? 'a positive' : 'a non-negative'} integer`)
+    }
   }
-  return { ...SLOT_LIMITS, ...slots }
+  return { ...limits, ...slots }
 }
 
+const layoutColumns = (counts, indexedCounts) => [
+  ...Object.entries(counts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => SLOT_COLUMNS[pool](index + 1)).flat()),
+  ...Object.entries(indexedCounts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => indexedSlotColumn(pool, index + 1)))
+]
+
+const slotIndexName = column => `any_records_${column}_idx`
+
 export const DEFAULT_CANONICAL_CONFIG = {
   scopeName: 'anyapi_records',
   tableName: 'any_records',
@@ -53,52 +77,121 @@ export const DEFAULT_CANONICAL_CONFIG = {
   rightResourceColumn: 'right_resource',
 }
 
-const ensureTable = async (knex, tableName, tableBuilder) => {
-  const exists = await knex.schema.hasTable(tableName)
-  if (!exists) {
-    await knex.schema.createTable(tableName, tableBuilder)
+// Several servers may start at once: DDL another one completed first is success.
+const tolerateConcurrentDdl = async (run, done) => {
+  try {
+    await run()
+  } catch (error) {
+    if (!await done().catch(() => false)) throw error
   }
 }
 
+// Returns whether the table was missing, so its indexes can be built offline.
+const ensureTable = async (knex, tableName, tableBuilder) => {
+  if (await knex.schema.hasTable(tableName)) return false
+  await tolerateConcurrentDdl(() => knex.schema.createTable(tableName, tableBuilder), () => knex.schema.hasTable(tableName))
+  return true
+}
+
 const ensureColumn = async (knex, tableName, columnName, columnBuilder) => {
   const exists = await knex.schema.hasTable(tableName)
   if (!exists) return
 
-  const hasColumn = await knex.schema.hasColumn(tableName, columnName)
-  if (!hasColumn) {
-    await knex.schema.table(tableName, columnBuilder)
+  const hasColumn = () => knex.schema.hasColumn(tableName, columnName)
+  if (!await hasColumn()) {
+    await tolerateConcurrentDdl(() => knex.schema.table(tableName, columnBuilder), hasColumn)
   }
 }
 
 const ensureUniqueIndex = async (knex, tableName, columns, indexName) => {
   const exists = await knex.schema.hasTable(tableName)
   if (!exists) return
-  const hasIndex = await hasKnexTableIndex(knex, tableName, indexName)
-  if (!hasIndex) {
-    await knex.schema.table(tableName, (table) => {
+  const hasIndex = () => hasKnexTableIndex(knex, tableName, indexName)
+  if (!await hasIndex()) {
+    await tolerateConcurrentDdl(() => knex.schema.table(tableName, (table) => {
       table.unique(columns, indexName)
-    })
+    }), hasIndex)
+  }
+}
+
+/**
+ * Create missing secondary indexes. On an existing table they are built without
+ * blocking writes: in place on MySQL, and concurrently on PostgreSQL unless the
+ * pool has a single connection. Either way the build reads the whole table, so
+ * on a big table run it from a migration before the deploy.
+ */
+const ensureIndexes = async (knex, tableName, indexes, { online }) => {
+  const client = knex.client.config.client
+  const pg = ['pg', 'postgresql'].includes(client)
+  const mysql = ['mysql', 'mysql2'].includes(client)
+  // The build lock needs a connection of its own beside the builds.
+  const concurrently = online && pg && !knex.isTransaction && (knex.client.pool?.max ?? 0) > 1
+  const missing = async () => {
+    const existing = await listKnexTableIndexes(knex, tableName)
+    return indexes.filter(({ name }) => !existing.get(name)?.valid && !existing.get(name)?.building)
+      .map(index => ({ ...index, invalid: existing.has(index.name) }))
+  }
+  const build = async () => {
+    for (const { name, columns, invalid } of await missing()) {
+      const list = columns.map(() => '??').join(', ')
+      if (invalid) await knex.raw(`drop index ${concurrently ? 'concurrently ' : ''}if exists ??`, [name])
+      if (mysql) {
+        await tolerateConcurrentDdl(
+          () => knex.raw(`alter table ?? add index ?? (${list})${online ? ', algorithm=inplace, lock=none' : ''}`, [tableName, name, ...columns]),
+          () => hasKnexTableIndex(knex, tableName, name)
+        )
+      } else {
+        await tolerateConcurrentDdl(
+          () => knex.raw(`create index ${concurrently ? 'concurrently ' : ''}if not exists ?? on ?? (${list})`, [name, tableName, ...columns]),
+          async () => (await listKnexTableIndexes(knex, tableName)).has(name)
+        )
+      }
+    }
+  }
+  if (!(await missing()).length) return
+  if (concurrently) await withPgIndexBuildLock(knex, tableName, build)
+  else await build()
+}
+
+// Concurrent builds on one table deadlock each other, so one server builds at a
+// time. The lock lives on an idle session, which concurrent builds never wait
+// for. A server that finds it taken carries on: the holder builds the same indexes.
+const withPgIndexBuildLock = async (knex, tableName, build) => {
+  const connection = await knex.client.acquireConnection()
+  const lock = name => knex.select(knex.raw(`${name}(hashtext(?)) as result`, [`json-rest-api:indexes:${tableName}`]))
+    .connection(connection).first()
+  try {
+    if (!(await lock('pg_try_advisory_lock')).result) return
+    try {
+      await build()
+    } finally {
+      await lock('pg_advisory_unlock')
+    }
+  } finally {
+    await knex.client.releaseConnection(connection)
   }
 }
 
 /**
  * Create or upgrade the shared canonical tables. Run it from a migration, or let
- * the plugin run it at start-up. `slots` only adds missing slot columns; lowering
- * a count never drops columns that may hold data.
+ * the plugin run it at start-up. Counts only add missing slot columns, never
+ * drop them; any_records always has at least the default layout.
  * @param {import('knex').Knex} knex
- * @param {{ slots?: Partial<Record<keyof typeof SLOT_LIMITS, number>> }} [options]
+ * @param {{
+ *   slots?: Partial<Record<keyof typeof SLOT_LIMITS, number>>,
+ *   indexedSlots?: Partial<Record<keyof typeof INDEXED_SLOT_LIMITS, number>>
+ * }} [options]
  */
-export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
+export const ensureAnyApiSchema = async (knex, { slots, indexedSlots } = {}) => {
   const counts = resolveSlotCounts(slots)
-  await ensureTable(knex, 'any_records', (table) => {
+  const indexedCounts = resolveSlotCounts(indexedSlots, INDEXED_SLOT_LIMITS, { label: 'indexed slot', minimum: 0 })
+  const created = await ensureTable(knex, 'any_records', (table) => {
     table.increments('id').primary()
     table.string('logical_id')
     table.string('tenant_id').notNullable()
     table.string('resource').notNullable()
 
-    for (const [pool, count] of Object.entries(counts)) {
-      for (let index = 1; index <= count; index += 1) SLOT_COLUMNS[pool](index).forEach(column => addSlotColumn(table, column))
-    }
+    layoutColumns(counts, indexedCounts).forEach(column => addSlotColumn(table, column))
 
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
@@ -109,14 +202,18 @@ export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
     table.index(['resource'])
   })
 
-  if (slots) {
-    const existing = new Set(Object.keys(await knex('any_records').columnInfo()))
-    const missing = Object.entries(counts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => SLOT_COLUMNS[pool](index + 1)))
-      .flat()
-      .filter(column => !existing.has(column))
-    if (missing.length) {
-      await knex.schema.alterTable('any_records', (table) => missing.forEach(column => addSlotColumn(table, column)))
-    }
+  const readColumns = async () => new Set(Object.keys(await knex('any_records').columnInfo()))
+  let recordColumns = await readColumns()
+  const missing = layoutColumns(counts, indexedCounts).filter(column => !recordColumns.has(column))
+  if (missing.length) {
+    await tolerateConcurrentDdl(
+      () => knex.schema.alterTable('any_records', (table) => missing.forEach(column => addSlotColumn(table, column))),
+      async () => {
+        const columns = await readColumns()
+        return missing.every(column => columns.has(column))
+      }
+    )
+    recordColumns = await readColumns()
   }
 
   const client = knex.client.config.client
@@ -125,7 +222,7 @@ export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
       .select({ column: 'column_name', precision: 'datetime_precision' })
       .where('table_schema', knex.raw(client === 'pg' || client === 'postgresql' ? 'current_schema()' : 'DATABASE()'))
       .where('table_name', 'any_records'))
-      .filter(column => /^date_\d+$/.test(column.column))
+      .filter(column => /^(indexed_)?date_\d+$/.test(column.column))
     if (!columns.length || columns.some(column => Number(column.precision) !== 6)) {
       throw new Error('AnyAPI temporal storage migration required: every any_records date slot must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
     }
@@ -151,7 +248,7 @@ export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
       .update({ logical_id: String(row.id) })
   }
 
-  await ensureTable(knex, 'any_links', (table) => {
+  const linksCreated = await ensureTable(knex, 'any_links', (table) => {
     table.increments('id').primary()
     table.string('tenant_id').notNullable()
     table.string('relationship').notNullable()
@@ -172,6 +269,17 @@ export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
     table.index(['inverse_relationship'])
   })
 
+  // Filters on indexed fields, child lookups by parent id and link lookups by
+  // record id all stay inside one tenant's rows.
+  const recordIndexes = [...recordColumns]
+    .filter(column => /^(indexed_[a-z]+_\d+|rel_\d+_id)$/.test(column))
+    .map(column => ({ name: slotIndexName(column), columns: ['tenant_id', 'resource', column] }))
+  await ensureIndexes(knex, 'any_records', recordIndexes, { online: !created })
+  await ensureIndexes(knex, 'any_links', [
+    { name: 'any_links_tenant_left_id_idx', columns: ['tenant_id', 'left_id'] },
+    { name: 'any_links_tenant_right_id_idx', columns: ['tenant_id', 'right_id'] }
+  ], { online: !linksCreated })
+
   await ensureTable(knex, 'any_resource_configs', (table) => {
     table.increments('id').primary()
     table.string('tenant_id').notNullable()
@@ -249,43 +357,50 @@ export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
   })
 }
 
-const buildSlotPools = (hasColumns) => Object.fromEntries(Object.keys(SLOT_LIMITS).map(pool => {
-  const indexes = []
-  for (let index = 1; hasColumns(pool, index); index += 1) indexes.push(index)
+// The default unindexed layout, for migrations written against it.
+export const SLOT_POOLS = Object.fromEntries(Object.entries(SLOT_LIMITS).map(([pool, count]) => {
+  const indexes = Array.from({ length: count }, (_, index) => index + 1)
   return [pool, pool === 'belongsTo' ? indexes : indexes.map(index => SLOT_COLUMNS[pool](index)[0])]
 }))
 
-export const SLOT_POOLS = buildSlotPools((pool, index) => index <= SLOT_LIMITS[pool])
-
 /**
  * Read the slot columns any_records actually has. Slots are numbered from 1
- * without gaps; a pool ends at its first missing column.
+ * without gaps; a pool ends at its first missing column. `indexed` lists the
+ * indexed columns of each pool that has them.
  * @param {import('knex').Knex | import('knex').Knex.Transaction} db
  */
 export const readSlotPools = async (db) => {
   const columns = new Set(Object.keys(await db('any_records').columnInfo()))
   if (!columns.size) throw new Error('AnyAPI storage tables are missing; run ensureAnyApiSchema in a migration')
-  return buildSlotPools((pool, index) => SLOT_COLUMNS[pool](index).every(column => columns.has(column)))
+  const run = columnsAt => {
+    const indexes = []
+    for (let index = 1; columnsAt(index).every(column => columns.has(column)); index += 1) indexes.push(index)
+    return indexes
+  }
+  return {
+    ...Object.fromEntries(Object.keys(SLOT_LIMITS).map(pool => {
+      const indexes = run(SLOT_COLUMNS[pool])
+      return [pool, pool === 'belongsTo' ? indexes : indexes.map(index => SLOT_COLUMNS[pool](index)[0])]
+    })),
+    indexed: Object.fromEntries(Object.keys(INDEXED_SLOT_LIMITS).map(pool => [
+      pool, run(index => [indexedSlotColumn(pool, index)]).map(index => indexedSlotColumn(pool, index))
+    ]))
+  }
 }
 
 export const TYPE_TO_POOL = new Map([
   ['string', 'string'],
-  ['text', 'string'],
   ['uuid', 'string'],
   ['email', 'string'],
   ['file', 'string'],
   ['number', 'number'],
   ['integer', 'number'],
-  ['float', 'number'],
-  ['decimal', 'number'],
   ['boolean', 'boolean'],
   ['date', 'string'],
-  ['datetime', 'date'],
   ['dateTime', 'date'],
   ['time', 'string'],
   ['epochMilliseconds', 'number'],
   ['epochSeconds', 'number'],
-  ['json', 'json'],
   ['object', 'json'],
   ['array', 'json'],
 ])

package/plugins/core/lib/anyapi/stored-definition.js

@@ -72,6 +72,9 @@ export const composeResourceOptions = ({ declaration = {}, stored = null, local
   merge('searchSchema', searchSchema, declaredSearch, local.searchSchema, hiddenFilters)
 
   for (const name of hidden) delete schema[name]
+  for (const name of stored?.retired || []) {
+    if (Object.hasOwn(schema, name)) schema[name] = { ...schema[name], retired: true }
+  }
   for (const [name, definition] of Object.entries(searchSchema)) {
     if (hiddenFilters.has(name) || targetsOf(definition).some(target => hidden.has(target))) {
       hiddenFilters.add(name)

package/plugins/core/lib/dbIntrospection.js

@@ -1237,6 +1237,34 @@ export async function hasKnexTableIndex (knex, tableName, indexName) {
   throw new Error(`Unsupported knex client '${client}' for hasKnexTableIndex`)
 }
 
+/**
+ * Index names of one table. On PostgreSQL an interrupted concurrent build leaves
+ * an invalid index; `building` tells it from one another session is building.
+ * @returns {Promise<Map<string, { valid: boolean, building: boolean }>>}
+ */
+export async function listKnexTableIndexes (knex, tableName) {
+  const client = knex.client.config.client
+  let rows
+  if (['sqlite3', 'better-sqlite3'].includes(client)) {
+    rows = normalizeRows(await knex.raw('PRAGMA index_list(??)', [tableName])).map(row => ({ name: row.name }))
+  } else if (['mysql', 'mysql2'].includes(client)) {
+    rows = (await knex('information_schema.statistics').distinct({ name: 'INDEX_NAME' })
+      .where({ TABLE_SCHEMA: knex.raw('DATABASE()'), TABLE_NAME: tableName }))
+  } else if (['pg', 'postgresql'].includes(client)) {
+    rows = normalizeRows(await knex.raw(`
+      select c.relname as name, i.indisvalid as valid,
+        exists (select 1 from pg_stat_progress_create_index p where p.index_relid = i.indexrelid) as building
+      from pg_index i
+      join pg_class c on c.oid = i.indexrelid
+      join pg_class t on t.oid = i.indrelid
+      join pg_namespace n on n.oid = t.relnamespace
+      where t.relname = ? and n.nspname = current_schema()`, [tableName]))
+  } else {
+    throw new Error(`Unsupported knex client '${client}' for listKnexTableIndexes`)
+  }
+  return new Map(rows.map(row => [row.name, { valid: row.valid !== false, building: row.building === true }]))
+}
+
 // Column-only ALTER must not impose the resource snapshot's ID or index restrictions.
 export async function introspectKnexColumnConstraints (knex, tableName) {
   requireKnexRaw(knex)

package/plugins/core/lib/querying/aggregate-query.js

@@ -0,0 +1,170 @@
+import { RestApiValidationError } from '../../../../lib/rest-api-errors.js'
+import { applyDatabaseReadOptions, normalizeStoredFieldValue } from '../querying-writing/database-value-normalizers.js'
+
+const NUMERIC_TYPES = new Set(['number', 'integer', 'epochMilliseconds', 'epochSeconds'])
+const ORDERED_TYPES = new Set([...NUMERIC_TYPES, 'string', 'email', 'uuid', 'date', 'dateTime', 'time'])
+const SCALAR_TYPES = new Set([...ORDERED_TYPES, 'boolean', 'id'])
+// Measure functions and the field types each accepts.
+const MEASURES = {
+  count: SCALAR_TYPES,
+  countDistinct: SCALAR_TYPES,
+  sum: NUMERIC_TYPES,
+  avg: NUMERIC_TYPES,
+  min: ORDERED_TYPES,
+  max: ORDERED_TYPES,
+}
+const PARAMS = new Set(['groupBy', 'measures', 'queryParams', 'transaction'])
+const QUERY_PARAMS = new Set(['filters', 'sort', 'page'])
+const OUTPUT_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
+const AGGREGATE_ALIAS = '__jra_aggregate'
+
+const invalid = (field, message) => {
+  throw new RestApiValidationError(message, { fields: [field], violations: [{ field, rule: 'aggregate', message }] })
+}
+
+const isPlainObject = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+
+const isDecimal = definition => definition.type === 'number' && definition.precision !== undefined && definition.scale !== undefined
+
+const positiveInteger = (value, field) => {
+  const number = typeof value === 'string' && value.trim() !== '' ? Number(value) : value
+  if (!Number.isSafeInteger(number) || number < 1) invalid(field, `${field} must be a positive integer`)
+  return number
+}
+
+/**
+ * Validate an aggregate request against a resource's compiled schema.
+ * Only stored scalar fields whose stored values are their public values qualify:
+ * getters and storage serializers would make SQL group or sum other values than clients see.
+ * @param {{ params: Record<string, any>, schemaInfo: Record<string, any>, scopeName: string, defaultLimit: number, maxLimit: number }} request
+ */
+export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaultLimit, maxLimit }) {
+  if (!isPlainObject(params)) invalid('params', 'aggregate requires a parameter object')
+  for (const key of Object.keys(params)) {
+    if (!PARAMS.has(key)) invalid(key, `Unknown aggregate parameter '${key}'`)
+  }
+  const { groupBy = [], measures, queryParams = {} } = params
+  if (!isPlainObject(queryParams)) invalid('queryParams', 'queryParams must be an object')
+  for (const key of Object.keys(queryParams)) {
+    if (!QUERY_PARAMS.has(key)) invalid(`queryParams.${key}`, `aggregate does not support queryParams.${key}`)
+  }
+
+  const fields = schemaInfo.schemaStructure || {}
+  const fieldDefinition = (name, parameter) => {
+    const definition = typeof name === 'string' && Object.hasOwn(fields, name) ? fields[name] : null
+    if (!definition || name === 'id' || name === schemaInfo.idProperty || definition.virtual === true || definition.hidden === true) {
+      invalid(parameter, `'${name}' is not an attribute of '${scopeName}' that can be aggregated`)
+    }
+    if (typeof definition.getter === 'function' || definition.storage?.serialize) {
+      invalid(parameter, `Field '${name}' transforms its stored values, so it cannot be aggregated`)
+    }
+    return definition
+  }
+
+  if (!Array.isArray(groupBy)) invalid('groupBy', 'groupBy must be an array of field names')
+  const groups = groupBy.map((field, index) => {
+    const definition = fieldDefinition(field, `groupBy.${index}`)
+    if (!SCALAR_TYPES.has(definition.type)) invalid(`groupBy.${index}`, `Field '${field}' has type '${definition.type}' and cannot be grouped`)
+    return { name: field, field, definition, alias: `g${index}` }
+  })
+  if (new Set(groupBy).size !== groupBy.length) invalid('groupBy', 'groupBy lists a field twice')
+
+  if (!isPlainObject(measures) || !Object.keys(measures).length) invalid('measures', 'measures must name at least one measure')
+  const names = new Set(groupBy)
+  const measureList = Object.entries(measures).map(([name, spec], index) => {
+    const parameter = `measures.${name}`
+    if (!OUTPUT_NAME.test(name)) invalid(parameter, `Measure name '${name}' must be a letter or underscore followed by letters, digits or underscores`)
+    if (names.has(name)) invalid(parameter, `Measure '${name}' has the name of a grouped field`)
+    names.add(name)
+    const entries = isPlainObject(spec) ? Object.entries(spec) : []
+    if (entries.length !== 1 || !Object.hasOwn(MEASURES, entries[0][0])) {
+      invalid(parameter, `Measure '${name}' must be one of { ${Object.keys(MEASURES).join(' | ')}: field }`)
+    }
+    const [fn, field] = entries[0]
+    if (field === '*') {
+      if (fn !== 'count') invalid(parameter, "Only count accepts '*'")
+      return { name, fn, field, definition: null, alias: `m${index}` }
+    }
+    const definition = fieldDefinition(field, parameter)
+    if (!MEASURES[fn].has(definition.type)) invalid(parameter, `${fn} cannot be applied to field '${field}' of type '${definition.type}'`)
+    return { name, fn, field, definition, alias: `m${index}` }
+  })
+
+  const outputs = new Map([...groups, ...measureList].map(output => [output.name, output.alias]))
+  const sortParam = queryParams.sort ?? []
+  if (!Array.isArray(sortParam)) invalid('queryParams.sort', 'queryParams.sort must be an array')
+  const sort = sortParam.map((entry, index) => {
+    const name = typeof entry === 'string' ? entry.replace(/^-/, '') : null
+    if (!outputs.has(name)) invalid(`queryParams.sort.${index}`, `Sort '${entry}' must name a grouped field or a measure`)
+    return { alias: outputs.get(name), direction: entry.startsWith('-') ? 'desc' : 'asc' }
+  })
+  // Grouped fields break ties, so pages never overlap.
+  for (const { alias } of groups) {
+    if (!sort.some(entry => entry.alias === alias)) sort.push({ alias, direction: 'asc' })
+  }
+
+  let page = null
+  if (groups.length) {
+    const pageParam = queryParams.page ?? {}
+    if (!isPlainObject(pageParam)) invalid('queryParams.page', 'queryParams.page must be an object')
+    for (const key of Object.keys(pageParam)) {
+      if (!['size', 'number'].includes(key)) invalid(`queryParams.page.${key}`, `aggregate pages support only size and number, not ${key}`)
+    }
+    const size = pageParam.size === undefined ? defaultLimit : positiveInteger(pageParam.size, 'queryParams.page.size')
+    if (size > maxLimit) invalid('queryParams.page.size', `queryParams.page.size cannot exceed ${maxLimit}`)
+    const number = pageParam.number === undefined ? 1 : positiveInteger(pageParam.number, 'queryParams.page.number')
+    page = { size, number }
+  } else if (queryParams.page !== undefined) {
+    invalid('queryParams.page', 'An aggregate without groupBy returns one row and takes no page')
+  }
+
+  return { groups, measures: measureList, sort, page, scopeName }
+}
+
+const measureValue = (value, measure, resourceType) => {
+  if (measure.fn === 'count' || measure.fn === 'countDistinct') return Number(value ?? 0)
+  if (value == null) return null
+  if (measure.fn === 'min' || measure.fn === 'max') return normalizeStoredFieldValue(value, measure.definition, { fieldName: measure.field, resourceType })
+  // Drivers return numeric aggregates as strings; decimals keep theirs to avoid rounding.
+  return isDecimal(measure.definition) ? value : Number(value)
+}
+
+/**
+ * Run a compiled aggregate over the rows a filtered query selects.
+ * The filtered query only supplies identifiers, so joins added by filters or
+ * policies cannot repeat a row inside a group.
+ * @param {{ source: import('knex').Knex.QueryBuilder, sourceTable: string, storageAdapter: Record<string, any>, db: import('knex').Knex | import('knex').Knex.Transaction, plan: ReturnType<typeof compileAggregateRequest> }} request
+ */
+export async function runAggregateQuery ({ source, sourceTable, storageAdapter, db, plan }) {
+  const idColumn = storageAdapter.getIdColumn()
+  const ids = source.clearSelect().clearOrder().select(`${sourceTable}.${idColumn}`)
+  const query = storageAdapter.buildBaseQuery({ transaction: db, tableAlias: AGGREGATE_ALIAS })
+    .whereIn(`${AGGREGATE_ALIAS}.${idColumn}`, ids)
+  const column = field => `${AGGREGATE_ALIAS}.${storageAdapter.translateColumn(field)}`
+
+  for (const group of plan.groups) {
+    query.select({ [group.alias]: column(group.field) }).groupBy(column(group.field))
+  }
+  for (const measure of plan.measures) {
+    query[measure.fn]({ [measure.alias]: measure.field === '*' ? '*' : column(measure.field) })
+  }
+  // NULL sorts as the lowest value, as MySQL and SQLite order it by default.
+  const pg = ['pg', 'postgresql'].includes(db.client.config.client)
+  for (const { alias, direction } of plan.sort) {
+    if (pg) query.orderBy(alias, direction, direction === 'asc' ? 'first' : 'last')
+    else query.orderBy(alias, direction)
+  }
+  if (plan.page) query.limit(plan.page.size + 1).offset((plan.page.number - 1) * plan.page.size)
+
+  const rows = await applyDatabaseReadOptions(query)
+  const hasMore = Boolean(plan.page && rows.length > plan.page.size)
+  if (hasMore) rows.pop()
+
+  const data = rows.map(row => {
+    const result = {}
+    for (const group of plan.groups) result[group.name] = normalizeStoredFieldValue(row[group.alias], group.definition, { fieldName: group.field, resourceType: plan.scopeName })
+    for (const measure of plan.measures) result[measure.name] = measureValue(row[measure.alias], measure, plan.scopeName)
+    return result
+  })
+  return plan.page ? { data, meta: { page: { ...plan.page, hasMore } } } : { data }
+}

package/plugins/core/lib/querying/knex-query-helpers.js

@@ -7,6 +7,8 @@ import { resolveSortField } from './query-field-sort-helpers.js'
 import { databaseIdentityExpression } from '../querying-writing/database-value-normalizers.js'
 
 const referenceVisibility = Symbol('referenceVisibility')
+// SQL never matches null against a non-null operand with these.
+const NULL_REJECTING_OPERATORS = new Set(['=', '==', '!=', '<>', '<', '<=', '>', '>=', 'in', 'between', 'like', 'contains', 'startswith', 'endswith'])
 
 async function buildSearchJoinQuery ({ scopeName, scopes, context, adapterUtils, db }) {
   const scope = scopes[scopeName]
@@ -135,12 +137,20 @@ export function applyWhereForOperator ({ builder, columnRef, operator, value, kn
       if (selections.length === 0) this.whereRaw('1 = 0')
       for (const selection of selections) this.orWhereExists(selection)
     }
+    const visibleMatch = function () {
+      this.where(visible)
+      applyWhereForOperator({ builder: this, columnRef: columnRef.column, operator, value, knex })
+    }
+    // A hidden reference reads as null. When null cannot match, leave that branch
+    // out: an OR with it keeps planners from using an index on the column.
+    const values = Array.isArray(value) ? value : [value]
+    if (NULL_REJECTING_OPERATORS.has(String(operator || '=').toLowerCase()) && values.length && values.every(item => item != null)) {
+      builder[method](visibleMatch)
+      return
+    }
     // Compare the column itself to preserve driver affinity and collation.
     builder[method](function () {
-      this.where(function () {
-        this.where(visible)
-        applyWhereForOperator({ builder: this, columnRef: columnRef.column, operator, value, knex })
-      }).orWhere(function () {
+      this.where(visibleMatch).orWhere(function () {
         this.whereNot(visible)
         applyWhereForOperator({ builder: this, columnRef: knex.raw('null'), operator, value, knex })
       })

package/plugins/core/lib/querying-writing/compile-schemas.js

@@ -5,6 +5,7 @@ import { compileQueryFields } from './query-field-helpers.js'
 import { buildEffectiveSortList } from '../querying/query-field-sort-helpers.js'
 import { assertScalarQueryField, assertFieldNameMap } from './field-utils.js'
 import { validateRelationshipFieldNames, validateSchemaFieldNames } from './scope-validations.js'
+import { assertKnownFieldTypes, installFieldTypes } from './field-types.js'
 
 /**
  * Compile one resource and publish its completed metadata in vars.schemaInfo.
@@ -74,7 +75,22 @@ export async function compileSchemas (scope, deps) {
   assertFieldNameMap(schemaContext.queryFields, `query fields in '${scopeName}'`)
 
   // Create schema object
+  // A retired field keeps its stored values readable and accepts no new ones.
+  for (const [fieldName, fieldDef] of Object.entries(schemaContext.fields)) {
+    if (fieldDef.retired === undefined) continue
+    if (typeof fieldDef.retired !== 'boolean') throw new Error(`Field '${fieldName}' in resource '${scopeName}' has a non-boolean retired option`)
+    if (fieldDef.retired && (fieldDef.belongsTo || fieldDef.virtual === true)) {
+      throw new Error(`Field '${fieldName}' in resource '${scopeName}' cannot be retired: only stored attributes can`)
+    }
+    if (fieldDef.retired) {
+      delete fieldDef.required
+      delete fieldDef.defaultTo
+    }
+  }
+
+  installFieldTypes()
   const schemaObject = createSchema(schemaContext.fields)
+  assertKnownFieldTypes(schemaObject, `resource '${scopeName}'`)
 
   // Generate searchSchema by merging explicit searchSchema with fields marked search:true.
   // This allows two ways to define searchable fields: either mark fields with search:true
@@ -98,6 +114,7 @@ export async function compileSchemas (scope, deps) {
   assertFieldNameMap(searchSchemaContext.fields, `search schema in '${scopeName}'`)
   ensureSearchFieldsAreIndexed(searchSchemaContext.fields)
   const searchSchemaObject = createSchema(searchSchemaContext.fields)
+  assertKnownFieldTypes(searchSchemaObject, `the search schema of '${scopeName}'`)
 
   // Derived fields must be rebuilt from the current candidate before publication.
   const idProperty = scopeOptions.idProperty || scope.vars.idProperty || 'id'

package/plugins/core/lib/querying-writing/database-value-normalizers.js

@@ -416,6 +416,22 @@ export function normalizeAttributes (attributes, schemaStructure, {
   return normalized
 }
 
+/**
+ * Convert one value read from a field's storage column to its public value,
+ * without the field's getter.
+ * @param {unknown} value
+ * @param {Record<string, any>} definition
+ * @param {{ fieldName: string, resourceType?: string }} options
+ */
+export function normalizeStoredFieldValue (value, definition, { fieldName, resourceType }) {
+  if (value == null) return null
+  if (definition.type === 'id') return String(value)
+  const decimal = definition.precision !== undefined && definition.scale !== undefined
+  // Drivers return some numeric columns as strings; decimals keep theirs to avoid rounding.
+  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !decimal) return Number(value)
+  return normalizeAttributes({ [fieldName]: value }, { [fieldName]: definition }, { resourceType })[fieldName]
+}
+
 /**
  * Normalize a JSON:API document or one plain resource using compiled output fields.
  * JSON:API primary/included attributes are replaced on the supplied document;