json-rest-api 2.0.0 → 2.2.0

package/plugins/core/lib/anyapi/schema-utils.js

@@ -1,19 +1,42 @@
 import { hasKnexTableIndex } from '../dbIntrospection.js'
 
-const STRING_SLOT_COUNT = 10
-const NUMBER_SLOT_COUNT = 10
-const BOOLEAN_SLOT_COUNT = 5
-const DATE_SLOT_COUNT = 5
-const JSON_SLOT_COUNT = 5
-const BELONGS_TO_SLOT_COUNT = 5
-
-export const SLOT_LIMITS = {
-  string: STRING_SLOT_COUNT,
-  number: NUMBER_SLOT_COUNT,
-  boolean: BOOLEAN_SLOT_COUNT,
-  date: DATE_SLOT_COUNT,
-  json: JSON_SLOT_COUNT,
-  belongsTo: BELONGS_TO_SLOT_COUNT,
+// Default layout of a new any_records table. The registry reads the actual
+// columns, so a migration that adds slots raises the limits for every server.
+export const SLOT_LIMITS = Object.freeze({
+  string: 10,
+  number: 10,
+  boolean: 5,
+  date: 5,
+  json: 5,
+  belongsTo: 5,
+})
+
+const SLOT_COLUMNS = {
+  string: index => [`string_${index}`],
+  number: index => [`number_${index}`],
+  boolean: index => [`boolean_${index}`],
+  date: index => [`date_${index}`],
+  json: index => [`json_${index}`],
+  belongsTo: index => [`rel_${index}_id`, `rel_${index}_type`],
+}
+
+const addSlotColumn = (table, column) => {
+  if (column.startsWith('number_')) table.double(column)
+  else if (column.startsWith('boolean_')) table.boolean(column)
+  else if (column.startsWith('date_')) table.dateTime(column, { precision: 6 })
+  else if (column.startsWith('rel_')) table.string(column)
+  else table.text(column)
+}
+
+const resolveSlotCounts = (slots = {}) => {
+  if (!slots || typeof slots !== 'object' || Array.isArray(slots)) {
+    throw new Error('AnyAPI slots must be an object of slot counts')
+  }
+  for (const [pool, count] of Object.entries(slots)) {
+    if (!Object.hasOwn(SLOT_LIMITS, pool)) throw new Error(`Unknown AnyAPI slot pool '${pool}'`)
+    if (!Number.isInteger(count) || count < 1) throw new Error(`AnyAPI ${pool} slot count must be a positive integer`)
+  }
+  return { ...SLOT_LIMITS, ...slots }
 }
 
 export const DEFAULT_CANONICAL_CONFIG = {
@@ -58,32 +81,23 @@ const ensureUniqueIndex = async (knex, tableName, columns, indexName) => {
   }
 }
 
-export const ensureAnyApiSchema = async (knex) => {
+/**
+ * Create or upgrade the shared canonical tables. Run it from a migration, or let
+ * the plugin run it at start-up. `slots` only adds missing slot columns; lowering
+ * a count never drops columns that may hold data.
+ * @param {import('knex').Knex} knex
+ * @param {{ slots?: Partial<Record<keyof typeof SLOT_LIMITS, number>> }} [options]
+ */
+export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
+  const counts = resolveSlotCounts(slots)
   await ensureTable(knex, 'any_records', (table) => {
     table.increments('id').primary()
     table.string('logical_id')
     table.string('tenant_id').notNullable()
     table.string('resource').notNullable()
 
-    for (let index = 1; index <= STRING_SLOT_COUNT; index += 1) {
-      table.text(`string_${index}`)
-    }
-    for (let index = 1; index <= NUMBER_SLOT_COUNT; index += 1) {
-      table.double(`number_${index}`)
-    }
-    for (let index = 1; index <= BOOLEAN_SLOT_COUNT; index += 1) {
-      table.boolean(`boolean_${index}`)
-    }
-    for (let index = 1; index <= DATE_SLOT_COUNT; index += 1) {
-      table.dateTime(`date_${index}`, { precision: 6 })
-    }
-    for (let index = 1; index <= JSON_SLOT_COUNT; index += 1) {
-      table.text(`json_${index}`)
-    }
-
-    for (let index = 1; index <= BELONGS_TO_SLOT_COUNT; index += 1) {
-      table.string(`rel_${index}_id`)
-      table.string(`rel_${index}_type`)
+    for (const [pool, count] of Object.entries(counts)) {
+      for (let index = 1; index <= count; index += 1) SLOT_COLUMNS[pool](index).forEach(column => addSlotColumn(table, column))
     }
 
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
@@ -95,15 +109,25 @@ export const ensureAnyApiSchema = async (knex) => {
     table.index(['resource'])
   })
 
+  if (slots) {
+    const existing = new Set(Object.keys(await knex('any_records').columnInfo()))
+    const missing = Object.entries(counts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => SLOT_COLUMNS[pool](index + 1)))
+      .flat()
+      .filter(column => !existing.has(column))
+    if (missing.length) {
+      await knex.schema.alterTable('any_records', (table) => missing.forEach(column => addSlotColumn(table, column)))
+    }
+  }
+
   const client = knex.client.config.client
   if (['pg', 'postgresql', 'mysql', 'mysql2'].includes(client)) {
-    const columns = await knex('information_schema.columns')
+    const columns = (await knex('information_schema.columns')
       .select({ column: 'column_name', precision: 'datetime_precision' })
       .where('table_schema', knex.raw(client === 'pg' || client === 'postgresql' ? 'current_schema()' : 'DATABASE()'))
-      .where('table_name', 'any_records')
-      .whereIn('column_name', Array.from({ length: DATE_SLOT_COUNT }, (_, index) => `date_${index + 1}`))
-    if (columns.length !== DATE_SLOT_COUNT || columns.some(column => Number(column.precision) !== 6)) {
-      throw new Error('AnyAPI temporal storage migration required: any_records.date_1 through date_5 must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
+      .where('table_name', 'any_records'))
+      .filter(column => /^date_\d+$/.test(column.column))
+    if (!columns.length || columns.some(column => Number(column.precision) !== 6)) {
+      throw new Error('AnyAPI temporal storage migration required: every any_records date slot must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
     }
   }
 
@@ -154,11 +178,20 @@ export const ensureAnyApiSchema = async (knex) => {
     table.string('resource').notNullable()
     table.text('schema_json').notNullable()
     table.text('relationships_json').notNullable()
+    // Holds the search schema and runtime additions too: MySQL TEXT stops at 64 KB.
+    table.text('definition_json', 'longtext')
+    table.integer('schema_version').notNullable().defaultTo(1)
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['tenant_id', 'resource'])
   })
+  await ensureColumn(knex, 'any_resource_configs', 'definition_json', (table) => {
+    table.text('definition_json', 'longtext')
+  })
+  await ensureColumn(knex, 'any_resource_configs', 'schema_version', (table) => {
+    table.integer('schema_version').notNullable().defaultTo(1)
+  })
 
   await ensureTable(knex, 'any_field_configs', (table) => {
     table.increments('id').primary()
@@ -177,8 +210,20 @@ export const ensureAnyApiSchema = async (knex) => {
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['resource_config_id', 'field_name'])
+    table.unique(['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
     table.index(['resource_config_id'])
   })
+  if (!await hasKnexTableIndex(knex, 'any_field_configs', 'any_field_configs_resource_slot_unique')) {
+    const shared = await knex('any_field_configs')
+      .select('resource_config_id', 'slot_column')
+      .groupBy('resource_config_id', 'slot_column')
+      .havingRaw('count(*) > 1')
+      .first()
+    if (shared) {
+      throw new Error(`AnyAPI slot metadata repair required: several fields of resource config ${shared.resource_config_id} share ${shared.slot_column}. Reassign them before upgrading.`)
+    }
+    await ensureUniqueIndex(knex, 'any_field_configs', ['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
+  }
 
   await ensureTable(knex, 'any_relationship_configs', (table) => {
     table.increments('id').primary()
@@ -204,33 +249,38 @@ export const ensureAnyApiSchema = async (knex) => {
   })
 }
 
-export const SLOT_POOLS = {
-  string: Array.from({ length: STRING_SLOT_COUNT }, (_, index) => `string_${index + 1}`),
-  number: Array.from({ length: NUMBER_SLOT_COUNT }, (_, index) => `number_${index + 1}`),
-  boolean: Array.from({ length: BOOLEAN_SLOT_COUNT }, (_, index) => `boolean_${index + 1}`),
-  date: Array.from({ length: DATE_SLOT_COUNT }, (_, index) => `date_${index + 1}`),
-  json: Array.from({ length: JSON_SLOT_COUNT }, (_, index) => `json_${index + 1}`),
-  belongsTo: Array.from({ length: BELONGS_TO_SLOT_COUNT }, (_, index) => index + 1),
+const buildSlotPools = (hasColumns) => Object.fromEntries(Object.keys(SLOT_LIMITS).map(pool => {
+  const indexes = []
+  for (let index = 1; hasColumns(pool, index); index += 1) indexes.push(index)
+  return [pool, pool === 'belongsTo' ? indexes : indexes.map(index => SLOT_COLUMNS[pool](index)[0])]
+}))
+
+export const SLOT_POOLS = buildSlotPools((pool, index) => index <= SLOT_LIMITS[pool])
+
+/**
+ * Read the slot columns any_records actually has. Slots are numbered from 1
+ * without gaps; a pool ends at its first missing column.
+ * @param {import('knex').Knex | import('knex').Knex.Transaction} db
+ */
+export const readSlotPools = async (db) => {
+  const columns = new Set(Object.keys(await db('any_records').columnInfo()))
+  if (!columns.size) throw new Error('AnyAPI storage tables are missing; run ensureAnyApiSchema in a migration')
+  return buildSlotPools((pool, index) => SLOT_COLUMNS[pool](index).every(column => columns.has(column)))
 }
 
 export const TYPE_TO_POOL = new Map([
   ['string', 'string'],
-  ['text', 'string'],
   ['uuid', 'string'],
   ['email', 'string'],
   ['file', 'string'],
   ['number', 'number'],
   ['integer', 'number'],
-  ['float', 'number'],
-  ['decimal', 'number'],
   ['boolean', 'boolean'],
   ['date', 'string'],
-  ['datetime', 'date'],
   ['dateTime', 'date'],
   ['time', 'string'],
   ['epochMilliseconds', 'number'],
   ['epochSeconds', 'number'],
-  ['json', 'json'],
   ['object', 'json'],
   ['array', 'json'],
 ])

package/plugins/core/lib/anyapi/stored-definition.js

@@ -0,0 +1,112 @@
+// Stored resource definitions keep JSON values only. The paths of values JSON
+// cannot hold (callbacks, class instances, RegExps) are recorded, so a server
+// loading the definition knows which fields need code it may not have.
+
+const isPlainObject = value => {
+  const prototype = Object.getPrototypeOf(value)
+  return prototype === Object.prototype || prototype === null
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ json: any, callbacks: Array<Array<string|number>> }}
+ */
+export const toStoredJson = (value) => {
+  const callbacks = []
+  const visit = (current, path) => {
+    if (current === undefined) return undefined
+    if (current === null || typeof current === 'string' || typeof current === 'boolean') return current
+    if (typeof current === 'number' && Number.isFinite(current)) return current
+    if (Array.isArray(current)) return current.map((item, index) => visit(item, [...path, index]) ?? null)
+    if (typeof current === 'object' && isPlainObject(current)) {
+      const result = {}
+      for (const [key, item] of Object.entries(current)) {
+        const stored = visit(item, [...path, key])
+        if (stored !== undefined) result[key] = stored
+      }
+      return result
+    }
+    callbacks.push(path)
+    return undefined
+  }
+  return { json: visit(value, []), callbacks }
+}
+
+const targetsOf = definition => definition?.oneOf || [definition?.actualField].filter(Boolean)
+
+/**
+ * Combine a resource's declaration with the fields other servers added at run
+ * time. Declared fields win; runtime fields this server added keep its local
+ * callbacks. A field whose callbacks exist in no available code is hidden
+ * rather than served without them, because a missing serializer or setter
+ * would read or write the wrong values.
+ * @param {{ declaration?: Record<string, any>, stored?: Record<string, any> | null, local?: { schema?: Record<string, any>, searchSchema?: Record<string, any> }, fromStorage?: boolean }} input
+ */
+export const composeResourceOptions = ({ declaration = {}, stored = null, local = {}, fromStorage = false }) => {
+  const hidden = new Set()
+  const hiddenFilters = new Set()
+  const declaredSchema = declaration.schema || {}
+  const declaredSearch = declaration.searchSchema || {}
+  const schema = { ...declaredSchema }
+  const searchSchema = { ...declaredSearch }
+
+  if (fromStorage) {
+    for (const path of stored?.callbacks || []) {
+      if (path[0] === 'schema' && path.length > 1) hidden.add(path[1])
+      else if (path[0] === 'searchSchema' && path.length > 1) hiddenFilters.add(path[1])
+      else throw new Error(`Resource option '${path.join('.')}' cannot be loaded from storage; declare the resource in code`)
+    }
+  }
+
+  const runtime = stored?.runtime || {}
+  const needsCode = new Set((runtime.callbacks || []).map(path => `${path[0]}.${path[1]}`))
+  const merge = (kind, target, declared, localDefinitions = {}, hide) => {
+    for (const [name, definition] of Object.entries(runtime[kind] || {})) {
+      if (Object.hasOwn(declared, name)) continue
+      if (Object.hasOwn(localDefinitions, name)) target[name] = localDefinitions[name]
+      else if (needsCode.has(`${kind}.${name}`)) hide.add(name)
+      else target[name] = definition
+    }
+  }
+  merge('schema', schema, declaredSchema, local.schema, hidden)
+  merge('searchSchema', searchSchema, declaredSearch, local.searchSchema, hiddenFilters)
+
+  for (const name of hidden) delete schema[name]
+  for (const name of stored?.retired || []) {
+    if (Object.hasOwn(schema, name)) schema[name] = { ...schema[name], retired: true }
+  }
+  for (const [name, definition] of Object.entries(searchSchema)) {
+    if (hiddenFilters.has(name) || targetsOf(definition).some(target => hidden.has(target))) {
+      hiddenFilters.add(name)
+      delete searchSchema[name]
+    }
+  }
+
+  const options = { ...declaration, schema }
+  if (Object.keys(searchSchema).length || declaration.searchSchema) options.searchSchema = searchSchema
+  if (hidden.size && Array.isArray(declaration.sortableFields)) {
+    options.sortableFields = declaration.sortableFields.filter(field => !hidden.has(String(field).replace(/^-/, '')))
+  }
+  return { options, hidden: [...hidden], hiddenFilters: [...hiddenFilters] }
+}
+
+/**
+ * Resources a stored declaration refers to. Polymorphic targets must be
+ * registered before the resource itself; the others can follow.
+ * @param {Record<string, any>} options
+ */
+export const relatedResources = (options = {}) => {
+  const before = new Set()
+  const after = new Set()
+  for (const definition of Object.values(options.schema || {})) {
+    if (definition?.belongsTo) after.add(definition.belongsTo)
+  }
+  for (const [name, definition] of Object.entries(options.relationships || {})) {
+    for (const type of definition?.belongsToPolymorphic?.types || []) before.add(type)
+    if (definition?.type === 'hasMany' || definition?.type === 'manyToMany' || definition?.hasMany) {
+      after.add(definition.target || definition.hasMany || name)
+    }
+    if (definition?.through) after.add(definition.through)
+  }
+  return { before: [...before], after: [...after].filter(name => !before.has(name)) }
+}

package/plugins/core/lib/querying/aggregate-query.js

@@ -0,0 +1,170 @@
+import { RestApiValidationError } from '../../../../lib/rest-api-errors.js'
+import { applyDatabaseReadOptions, normalizeStoredFieldValue } from '../querying-writing/database-value-normalizers.js'
+
+const NUMERIC_TYPES = new Set(['number', 'integer', 'epochMilliseconds', 'epochSeconds'])
+const ORDERED_TYPES = new Set([...NUMERIC_TYPES, 'string', 'email', 'uuid', 'date', 'dateTime', 'time'])
+const SCALAR_TYPES = new Set([...ORDERED_TYPES, 'boolean', 'id'])
+// Measure functions and the field types each accepts.
+const MEASURES = {
+  count: SCALAR_TYPES,
+  countDistinct: SCALAR_TYPES,
+  sum: NUMERIC_TYPES,
+  avg: NUMERIC_TYPES,
+  min: ORDERED_TYPES,
+  max: ORDERED_TYPES,
+}
+const PARAMS = new Set(['groupBy', 'measures', 'queryParams', 'transaction'])
+const QUERY_PARAMS = new Set(['filters', 'sort', 'page'])
+const OUTPUT_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
+const AGGREGATE_ALIAS = '__jra_aggregate'
+
+const invalid = (field, message) => {
+  throw new RestApiValidationError(message, { fields: [field], violations: [{ field, rule: 'aggregate', message }] })
+}
+
+const isPlainObject = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+
+const isDecimal = definition => definition.type === 'number' && definition.precision !== undefined && definition.scale !== undefined
+
+const positiveInteger = (value, field) => {
+  const number = typeof value === 'string' && value.trim() !== '' ? Number(value) : value
+  if (!Number.isSafeInteger(number) || number < 1) invalid(field, `${field} must be a positive integer`)
+  return number
+}
+
+/**
+ * Validate an aggregate request against a resource's compiled schema.
+ * Only stored scalar fields whose stored values are their public values qualify:
+ * getters and storage serializers would make SQL group or sum other values than clients see.
+ * @param {{ params: Record<string, any>, schemaInfo: Record<string, any>, scopeName: string, defaultLimit: number, maxLimit: number }} request
+ */
+export function compileAggregateRequest ({ params, schemaInfo, scopeName, defaultLimit, maxLimit }) {
+  if (!isPlainObject(params)) invalid('params', 'aggregate requires a parameter object')
+  for (const key of Object.keys(params)) {
+    if (!PARAMS.has(key)) invalid(key, `Unknown aggregate parameter '${key}'`)
+  }
+  const { groupBy = [], measures, queryParams = {} } = params
+  if (!isPlainObject(queryParams)) invalid('queryParams', 'queryParams must be an object')
+  for (const key of Object.keys(queryParams)) {
+    if (!QUERY_PARAMS.has(key)) invalid(`queryParams.${key}`, `aggregate does not support queryParams.${key}`)
+  }
+
+  const fields = schemaInfo.schemaStructure || {}
+  const fieldDefinition = (name, parameter) => {
+    const definition = typeof name === 'string' && Object.hasOwn(fields, name) ? fields[name] : null
+    if (!definition || name === 'id' || name === schemaInfo.idProperty || definition.virtual === true || definition.hidden === true) {
+      invalid(parameter, `'${name}' is not an attribute of '${scopeName}' that can be aggregated`)
+    }
+    if (typeof definition.getter === 'function' || definition.storage?.serialize) {
+      invalid(parameter, `Field '${name}' transforms its stored values, so it cannot be aggregated`)
+    }
+    return definition
+  }
+
+  if (!Array.isArray(groupBy)) invalid('groupBy', 'groupBy must be an array of field names')
+  const groups = groupBy.map((field, index) => {
+    const definition = fieldDefinition(field, `groupBy.${index}`)
+    if (!SCALAR_TYPES.has(definition.type)) invalid(`groupBy.${index}`, `Field '${field}' has type '${definition.type}' and cannot be grouped`)
+    return { name: field, field, definition, alias: `g${index}` }
+  })
+  if (new Set(groupBy).size !== groupBy.length) invalid('groupBy', 'groupBy lists a field twice')
+
+  if (!isPlainObject(measures) || !Object.keys(measures).length) invalid('measures', 'measures must name at least one measure')
+  const names = new Set(groupBy)
+  const measureList = Object.entries(measures).map(([name, spec], index) => {
+    const parameter = `measures.${name}`
+    if (!OUTPUT_NAME.test(name)) invalid(parameter, `Measure name '${name}' must be a letter or underscore followed by letters, digits or underscores`)
+    if (names.has(name)) invalid(parameter, `Measure '${name}' has the name of a grouped field`)
+    names.add(name)
+    const entries = isPlainObject(spec) ? Object.entries(spec) : []
+    if (entries.length !== 1 || !Object.hasOwn(MEASURES, entries[0][0])) {
+      invalid(parameter, `Measure '${name}' must be one of { ${Object.keys(MEASURES).join(' | ')}: field }`)
+    }
+    const [fn, field] = entries[0]
+    if (field === '*') {
+      if (fn !== 'count') invalid(parameter, "Only count accepts '*'")
+      return { name, fn, field, definition: null, alias: `m${index}` }
+    }
+    const definition = fieldDefinition(field, parameter)
+    if (!MEASURES[fn].has(definition.type)) invalid(parameter, `${fn} cannot be applied to field '${field}' of type '${definition.type}'`)
+    return { name, fn, field, definition, alias: `m${index}` }
+  })
+
+  const outputs = new Map([...groups, ...measureList].map(output => [output.name, output.alias]))
+  const sortParam = queryParams.sort ?? []
+  if (!Array.isArray(sortParam)) invalid('queryParams.sort', 'queryParams.sort must be an array')
+  const sort = sortParam.map((entry, index) => {
+    const name = typeof entry === 'string' ? entry.replace(/^-/, '') : null
+    if (!outputs.has(name)) invalid(`queryParams.sort.${index}`, `Sort '${entry}' must name a grouped field or a measure`)
+    return { alias: outputs.get(name), direction: entry.startsWith('-') ? 'desc' : 'asc' }
+  })
+  // Grouped fields break ties, so pages never overlap.
+  for (const { alias } of groups) {
+    if (!sort.some(entry => entry.alias === alias)) sort.push({ alias, direction: 'asc' })
+  }
+
+  let page = null
+  if (groups.length) {
+    const pageParam = queryParams.page ?? {}
+    if (!isPlainObject(pageParam)) invalid('queryParams.page', 'queryParams.page must be an object')
+    for (const key of Object.keys(pageParam)) {
+      if (!['size', 'number'].includes(key)) invalid(`queryParams.page.${key}`, `aggregate pages support only size and number, not ${key}`)
+    }
+    const size = pageParam.size === undefined ? defaultLimit : positiveInteger(pageParam.size, 'queryParams.page.size')
+    if (size > maxLimit) invalid('queryParams.page.size', `queryParams.page.size cannot exceed ${maxLimit}`)
+    const number = pageParam.number === undefined ? 1 : positiveInteger(pageParam.number, 'queryParams.page.number')
+    page = { size, number }
+  } else if (queryParams.page !== undefined) {
+    invalid('queryParams.page', 'An aggregate without groupBy returns one row and takes no page')
+  }
+
+  return { groups, measures: measureList, sort, page, scopeName }
+}
+
+const measureValue = (value, measure, resourceType) => {
+  if (measure.fn === 'count' || measure.fn === 'countDistinct') return Number(value ?? 0)
+  if (value == null) return null
+  if (measure.fn === 'min' || measure.fn === 'max') return normalizeStoredFieldValue(value, measure.definition, { fieldName: measure.field, resourceType })
+  // Drivers return numeric aggregates as strings; decimals keep theirs to avoid rounding.
+  return isDecimal(measure.definition) ? value : Number(value)
+}
+
+/**
+ * Run a compiled aggregate over the rows a filtered query selects.
+ * The filtered query only supplies identifiers, so joins added by filters or
+ * policies cannot repeat a row inside a group.
+ * @param {{ source: import('knex').Knex.QueryBuilder, sourceTable: string, storageAdapter: Record<string, any>, db: import('knex').Knex | import('knex').Knex.Transaction, plan: ReturnType<typeof compileAggregateRequest> }} request
+ */
+export async function runAggregateQuery ({ source, sourceTable, storageAdapter, db, plan }) {
+  const idColumn = storageAdapter.getIdColumn()
+  const ids = source.clearSelect().clearOrder().select(`${sourceTable}.${idColumn}`)
+  const query = storageAdapter.buildBaseQuery({ transaction: db, tableAlias: AGGREGATE_ALIAS })
+    .whereIn(`${AGGREGATE_ALIAS}.${idColumn}`, ids)
+  const column = field => `${AGGREGATE_ALIAS}.${storageAdapter.translateColumn(field)}`
+
+  for (const group of plan.groups) {
+    query.select({ [group.alias]: column(group.field) }).groupBy(column(group.field))
+  }
+  for (const measure of plan.measures) {
+    query[measure.fn]({ [measure.alias]: measure.field === '*' ? '*' : column(measure.field) })
+  }
+  // NULL sorts as the lowest value, as MySQL and SQLite order it by default.
+  const pg = ['pg', 'postgresql'].includes(db.client.config.client)
+  for (const { alias, direction } of plan.sort) {
+    if (pg) query.orderBy(alias, direction, direction === 'asc' ? 'first' : 'last')
+    else query.orderBy(alias, direction)
+  }
+  if (plan.page) query.limit(plan.page.size + 1).offset((plan.page.number - 1) * plan.page.size)
+
+  const rows = await applyDatabaseReadOptions(query)
+  const hasMore = Boolean(plan.page && rows.length > plan.page.size)
+  if (hasMore) rows.pop()
+
+  const data = rows.map(row => {
+    const result = {}
+    for (const group of plan.groups) result[group.name] = normalizeStoredFieldValue(row[group.alias], group.definition, { fieldName: group.field, resourceType: plan.scopeName })
+    for (const measure of plan.measures) result[measure.name] = measureValue(row[measure.alias], measure, plan.scopeName)
+    return result
+  })
+  return plan.page ? { data, meta: { page: { ...plan.page, hasMore } } } : { data }
+}

package/plugins/core/lib/querying-writing/compile-schemas.js

@@ -5,6 +5,7 @@ import { compileQueryFields } from './query-field-helpers.js'
 import { buildEffectiveSortList } from '../querying/query-field-sort-helpers.js'
 import { assertScalarQueryField, assertFieldNameMap } from './field-utils.js'
 import { validateRelationshipFieldNames, validateSchemaFieldNames } from './scope-validations.js'
+import { assertKnownFieldTypes, installFieldTypes } from './field-types.js'
 
 /**
  * Compile one resource and publish its completed metadata in vars.schemaInfo.
@@ -74,7 +75,22 @@ export async function compileSchemas (scope, deps) {
   assertFieldNameMap(schemaContext.queryFields, `query fields in '${scopeName}'`)
 
   // Create schema object
+  // A retired field keeps its stored values readable and accepts no new ones.
+  for (const [fieldName, fieldDef] of Object.entries(schemaContext.fields)) {
+    if (fieldDef.retired === undefined) continue
+    if (typeof fieldDef.retired !== 'boolean') throw new Error(`Field '${fieldName}' in resource '${scopeName}' has a non-boolean retired option`)
+    if (fieldDef.retired && (fieldDef.belongsTo || fieldDef.virtual === true)) {
+      throw new Error(`Field '${fieldName}' in resource '${scopeName}' cannot be retired: only stored attributes can`)
+    }
+    if (fieldDef.retired) {
+      delete fieldDef.required
+      delete fieldDef.defaultTo
+    }
+  }
+
+  installFieldTypes()
   const schemaObject = createSchema(schemaContext.fields)
+  assertKnownFieldTypes(schemaObject, `resource '${scopeName}'`)
 
   // Generate searchSchema by merging explicit searchSchema with fields marked search:true.
   // This allows two ways to define searchable fields: either mark fields with search:true
@@ -98,6 +114,7 @@ export async function compileSchemas (scope, deps) {
   assertFieldNameMap(searchSchemaContext.fields, `search schema in '${scopeName}'`)
   ensureSearchFieldsAreIndexed(searchSchemaContext.fields)
   const searchSchemaObject = createSchema(searchSchemaContext.fields)
+  assertKnownFieldTypes(searchSchemaObject, `the search schema of '${scopeName}'`)
 
   // Derived fields must be rebuilt from the current candidate before publication.
   const idProperty = scopeOptions.idProperty || scope.vars.idProperty || 'id'

package/plugins/core/lib/querying-writing/database-value-normalizers.js

@@ -416,6 +416,22 @@ export function normalizeAttributes (attributes, schemaStructure, {
   return normalized
 }
 
+/**
+ * Convert one value read from a field's storage column to its public value,
+ * without the field's getter.
+ * @param {unknown} value
+ * @param {Record<string, any>} definition
+ * @param {{ fieldName: string, resourceType?: string }} options
+ */
+export function normalizeStoredFieldValue (value, definition, { fieldName, resourceType }) {
+  if (value == null) return null
+  if (definition.type === 'id') return String(value)
+  const decimal = definition.precision !== undefined && definition.scale !== undefined
+  // Drivers return some numeric columns as strings; decimals keep theirs to avoid rounding.
+  if (['number', 'integer'].includes(definition.type) && typeof value === 'string' && !decimal) return Number(value)
+  return normalizeAttributes({ [fieldName]: value }, { [fieldName]: definition }, { resourceType })[fieldName]
+}
+
 /**
  * Normalize a JSON:API document or one plain resource using compiled output fields.
  * JSON:API primary/included attributes are replaced on the supplied document;

package/plugins/core/lib/querying-writing/default-data-helpers.js

@@ -24,6 +24,10 @@ export const defaultDataHelpers = {
     throw new Error('No storage implementation for query. Install a storage plugin.')
   },
 
+  dataAggregate: async function (request) {
+    throw new Error('No storage implementation for aggregate. Install a storage plugin.')
+  },
+
   dataPost: async function (request) {
     throw new Error('No storage implementation for post. Install a storage plugin.')
   },

package/plugins/core/lib/querying-writing/field-types.js

@@ -0,0 +1,43 @@
+import { addType } from 'json-rest-schema'
+
+const EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
+const UUID_PATTERN = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/
+const TEXT_SEARCH_OPERATORS = new Set(['like', 'contains', 'startswith', 'endswith'])
+
+// Partial-match filters search fragments of a value, not whole values.
+const isTextSearch = definition => TEXT_SEARCH_OPERATORS.has(String(definition?.filterOperator).toLowerCase())
+
+const formattedStringType = (pattern, format) => {
+  const type = (context) => {
+    if (typeof context.value !== 'string') context.throwTypeError()
+    const value = context.definition.noTrim ? context.value : context.value.trim()
+    if (!isTextSearch(context.definition) && !pattern.test(value)) context.throwTypeError()
+    return value
+  }
+  type.toJsonSchema = ({ definition }) => isTextSearch(definition) ? { type: 'string' } : { type: 'string', format, pattern: pattern.source }
+  return type
+}
+
+let installed = false
+
+/** Register the string types json-rest-schema lacks; both storages keep them in string columns. */
+export function installFieldTypes () {
+  if (installed) return
+  installed = true
+  addType('email', formattedStringType(EMAIL_PATTERN, 'email'))
+  addType('uuid', formattedStringType(UUID_PATTERN, 'uuid'))
+}
+
+/**
+ * Reject field types no casting function exists for, at registration rather than on the first write.
+ * @param {{ structure: Record<string, any>, types: Record<string, unknown> }} schema
+ * @param {string} label
+ */
+export function assertKnownFieldTypes (schema, label) {
+  for (const [fieldName, definition] of Object.entries(schema.structure)) {
+    const type = definition?.type
+    if (typeof type === 'string' && !Object.hasOwn(schema.types, type)) {
+      throw new Error(`Field '${fieldName}' in ${label} has unknown type '${type}'`)
+    }
+  }
+}

package/plugins/core/lib/querying-writing/request-contracts.js

@@ -354,7 +354,7 @@ function buildWritableAttributesStructure (schemaInfo = {}) {
     if (!fieldDef) continue
     if (fieldName === idProperty) continue
     if (excludedFieldNames.has(fieldName)) continue
-    if (fieldDef.computed === true) continue
+    if (fieldDef.computed === true || fieldDef.retired === true) continue
     if (fieldDef.type === undefined) continue
     attributesStructure[fieldName] = { ...fieldDef }
   }