json-rest-api 2.0.0 → 2.2.0

package/plugins/core/lib/storage/canonical-data-helpers.js

@@ -5,6 +5,7 @@ import { applySortDescriptorOrder, buildEffectiveSortList, parseSortEntry, resol
 import { normalizeId, resolveFieldInfo } from '../anyapi/utils/descriptor-helpers.js'
 import { decorateResourceLinks, getAllowedQueryFieldNames } from '../anyapi/canonical-relationship-reader.js'
 import { applyQueryConstraint } from '../querying/query-constraint.js'
+import { runAggregateQuery } from '../querying/aggregate-query.js'
 import { calculatePaginationMeta, generatePaginationLinks, generateCursorPaginationLinks, buildCursorMeta, applyPaginationToQuery } from '../querying/knex-pagination-helpers.js'
 import { getUrlPrefix, buildJsonApiLink } from '../querying/url-helpers.js'
 import { applyFieldSelectionToQuery, buildFieldSelection } from '../querying-writing/knex-field-helpers.js'
@@ -670,6 +671,33 @@ export function createCanonicalDataHelpers ({
     return response
   }
 
+  /** @type {CanonicalDataReadHelpers['dataAggregate']} */
+  const dataAggregate = async ({ scopeName, context, runHooks, plan }) => {
+    const storageAdapter = getScopeStorageAdapter(scopeName)
+    if (!storageAdapter) throw new Error(`Storage adapter for resource '${scopeName}' not found`)
+    context.storageAdapter = storageAdapter
+    const descriptor = getDescriptor(scopeName)
+    const db = context.db || context.transaction || api.knex.instance
+    const tableAlias = context.schemaInfo.tableName || descriptor.resource
+
+    const filteredState = await withQueryFilteringContext(context, {
+      query: storageAdapter.buildBaseQuery({ transaction: db, tableAlias }),
+      filters: context.queryParams.filters,
+      schemaInfo: context.schemaInfo,
+      scopeName,
+      tableName: tableAlias,
+      db,
+      queryPurpose: 'aggregate',
+      isAnyApi: true,
+      adapter: storageAdapter,
+      storageAdapter,
+    }, async () => {
+      await applyBuiltInAnyApiQueryFilters(context)
+      await runHooks('knexQueryFiltering')
+    })
+    return runAggregateQuery({ source: filteredState.query, sourceTable: tableAlias, storageAdapter, db, plan })
+  }
+
   return {
     dataExists,
     dataPost,
@@ -678,6 +706,7 @@ export function createCanonicalDataHelpers ({
     dataDelete,
     dataGet,
     dataQuery,
+    dataAggregate,
     dataGetMinimal
   }
 }

package/plugins/core/lib/storage/ordinary-data-helpers.js

@@ -9,6 +9,7 @@ import { toJsonApiRecordWithBelongsTo } from '../querying-writing/knex-json-api-
 import { processIncludes } from '../querying/knex-process-includes.js'
 import { loadRelationshipIdentifiers } from '../querying/relationship-identifiers.js'
 import { applyQueryConstraint } from '../querying/query-constraint.js'
+import { runAggregateQuery } from '../querying/aggregate-query.js'
 import { RestApiResourceError } from '../../../../lib/rest-api-errors.js'
 import { getOperationDiagnosticContext } from '../../../../lib/error-context.js'
 import { applyInsertReturning } from '../querying-writing/database-capabilities.js'
@@ -505,6 +506,29 @@ export function createOrdinaryDataHelpers ({ api, scopes, knex, log, getScopeSto
     return buildJsonApiResponse(scope, records, included, false, scopeName, context)
   }
 
+  /** @type {OrdinaryDataReadHelpers['dataAggregate']} */
+  const dataAggregate = async ({ scopeName, context, runHooks, plan }) => {
+    const storageAdapter = getScopeStorageAdapter(scopeName)
+    context.storageAdapter = storageAdapter
+    const tableName = storageAdapter.getTableName()
+    const db = context.db || api.knex.instance
+
+    const filteredState = await withQueryFilteringContext(context, {
+      query: db(tableName),
+      filters: context.queryParams.filters,
+      schemaInfo: context.schemaInfo,
+      scopeName,
+      tableName,
+      db,
+      queryPurpose: 'aggregate',
+      adapter: storageAdapter,
+      storageAdapter,
+    }, async () => {
+      await runHooks('knexQueryFiltering')
+    })
+    return runAggregateQuery({ source: filteredState.query, sourceTable: tableName, storageAdapter, db, plan })
+  }
+
   /** @type {OrdinaryDataWriteHelpers['dataPost']} */
   const dataPost = async ({ scopeName, context }) => {
     const storageAdapter = getScopeStorageAdapter(scopeName)
@@ -701,6 +725,7 @@ export function createOrdinaryDataHelpers ({ api, scopes, knex, log, getScopeSto
     dataExists,
     dataGet,
     dataQuery,
+    dataAggregate,
     dataPost,
     dataPut,
     dataPatch,

package/plugins/core/lib/storage/storage-types.d.ts

@@ -307,6 +307,27 @@ export interface DataReadHelpers<GetResult> {
       queryParams: NonNullable<DataReadContext['queryParams']>
     }
   }) => Promise<DataDocument<DataResource[]>>
+  // The plan comes from compileAggregateRequest; filters are in context.queryParams.
+  dataAggregate: (request: DataReadRequest & {
+    runHooks: NonNullable<DataReadRequest['runHooks']>
+    context: DataReadContext & {
+      queryParams: NonNullable<DataReadContext['queryParams']>
+    }
+    plan: AggregatePlan
+  }) => Promise<AggregateResult>
+}
+
+// Compiled by compileAggregateRequest: outputs carry SQL aliases.
+export interface AggregatePlan {
+  scopeName: string
+  groups: Array<{ name: string; field: string; definition: StorageFieldDefinition; alias: string }>
+  measures: Array<{ name: string; fn: 'count' | 'countDistinct' | 'sum' | 'avg' | 'min' | 'max'; field: string; definition: StorageFieldDefinition | null; alias: string }>
+  sort: Array<{ alias: string; direction: 'asc' | 'desc' }>
+  page: { size: number; number: number } | null
+}
+export interface AggregateResult {
+  data: Array<Record<string, unknown>>
+  meta?: { page: { size: number; number: number; hasMore: boolean } }
 }
 
 export type OrdinaryDataReadHelpers = DataReadHelpers<DataDocument<DataResource>>

package/plugins/core/rest-api-anyapi-knex-plugin.js

@@ -5,15 +5,30 @@ import { createKnexTransaction } from '../../lib/knex-transaction.js'
 import { createEnhancedLogger } from '../../lib/enhanced-logger.js'
 import { ensureAnyApiSchema } from './lib/anyapi/schema-utils.js'
 import { AnyapiRegistry } from './lib/anyapi/anyapi-registry.js'
-import { RestApiResourceError } from '../../lib/rest-api-errors.js'
+import { composeResourceOptions, relatedResources } from './lib/anyapi/stored-definition.js'
+import { RestApiResourceError, RestApiValidationError } from '../../lib/rest-api-errors.js'
 import { compileSchemas } from './lib/querying-writing/compile-schemas.js'
 import { snapshotResourceConfiguration } from './lib/querying-writing/schema-helpers.js'
 import { assertFieldNameMap } from './lib/querying-writing/field-utils.js'
 import { polymorphicFiltersHook, crossTableFiltersHook, basicFiltersHook } from './lib/querying/knex-query-helpers.js'
 import { createStorageAdapterLookup } from './lib/storage/storage-adapter.js'
-import { getDatabaseCapabilities } from './lib/querying-writing/database-capabilities.js'
+import { translateCanonicalAttributesForStorage } from './lib/storage/canonical-storage-mapping.js'
+import { applyDatabaseReadOptions, normalizeStoredFieldValue } from './lib/querying-writing/database-value-normalizers.js'
 
 const DEFAULT_TENANT = 'default'
+const MIGRATION_BATCH_SIZE = 500
+// Methods that read a published schema; schema checks wrap these.
+const REQUEST_METHODS = ['query', 'aggregate', 'get', 'post', 'put', 'patch', 'delete', 'getRelated', 'getRelationship', 'postRelationship', 'patchRelationship', 'deleteRelationship']
+
+const optionsSignature = options => JSON.stringify([
+  Object.keys(options.schema || {}).sort(),
+  Object.keys(options.searchSchema || {}).sort(),
+  Object.entries(options.schema || {}).filter(([, definition]) => definition?.retired === true).map(([name]) => name).sort()
+])
+
+// A validation failure may come from a schema another server already changed.
+// Writes that may have committed are never repeated.
+const isStaleSchemaRejection = error => error?.type === 'rest_api_validation' && !['committed', 'unknown'].includes(error.transactionOutcome)
 
 export const RestApiAnyapiKnexPlugin = {
   name: 'rest-api-anyapi-knex',
@@ -24,14 +39,23 @@ export const RestApiAnyapiKnexPlugin = {
     const options = pluginOptions || {}
     const knex = options.knex
     const tenantId = options.tenantId || DEFAULT_TENANT
+    const { ensureSchema = true, ownsKnex = true, onSchemaChanged, schemaCheckInterval, slots } = options
 
     if (!knex) {
       throw new Error('AnyapiKnexPlugin requires a knex instance')
     }
+    if (options.registry !== undefined && (!(options.registry instanceof AnyapiRegistry) || options.registry.knex !== knex)) {
+      throw new Error('AnyapiKnexPlugin registry must be an AnyapiRegistry for the same knex instance')
+    }
+    if (onSchemaChanged !== undefined && typeof onSchemaChanged !== 'function') {
+      throw new Error('AnyapiKnexPlugin onSchemaChanged must be a function')
+    }
+    if (schemaCheckInterval !== undefined && !(Number.isFinite(schemaCheckInterval) && schemaCheckInterval >= 0)) {
+      throw new Error('AnyapiKnexPlugin schemaCheckInterval must be a non-negative number of milliseconds')
+    }
 
-    await ensureAnyApiSchema(knex)
-
-    const registry = new AnyapiRegistry({ knex, log })
+    const registry = options.registry || new AnyapiRegistry({ knex, log })
+    if (ensureSchema) await registry.ensureSchema({ slots })
 
     api.anyapi = api.anyapi || {}
     api.anyapi.registry = registry
@@ -39,7 +63,7 @@ export const RestApiAnyapiKnexPlugin = {
     api.knex = {
       instance: knex,
       helpers: {},
-      capabilities: await getDatabaseCapabilities(knex, log),
+      capabilities: await registry.getCapabilities(log),
     }
 
     const getScopeStorageAdapter = createStorageAdapterLookup({
@@ -50,13 +74,21 @@ export const RestApiAnyapiKnexPlugin = {
     api.knex.helpers.getStorageAdapter = getScopeStorageAdapter
     helpers.getStorageAdapter = getScopeStorageAdapter
 
-    const scopeOptionsRegistry = new Map()
+    // Owned copies of what this server declared and added, callbacks included.
+    const declarations = new Map()
+    const runtimeDeclarations = new Map()
+    const storageLoaded = new Set()
+    const compiledSignatures = new Map()
+    const lastChecked = new Map()
+    const reloads = new Map()
+    const checks = new Map()
+    const loading = new Map()
 
     /** @type {import('../../lib/transaction-types.js').TransactionFactory} */
     helpers.newTransaction = context => createKnexTransaction(knex, context)
 
     addHook('release', 'anyapi-knex-release', {}, async ({ api }) => {
-      if (api.knex?.instance) {
+      if (ownsKnex && api.knex?.instance) {
         await api.knex.instance.destroy()
       }
     })
@@ -76,11 +108,38 @@ export const RestApiAnyapiKnexPlugin = {
       api, knex, getDescriptor, getScopeStorageAdapter, linkStore
     })
 
-    const refreshStorageDescriptor = async (scopeName, compiledSchemaInfo) => {
+    const publishedVersion = scopeName => api.resources[scopeName]?.vars?.schemaInfo?.descriptor?.schemaVersion ?? 0
+
+    const composeOptions = (scopeName, descriptor) => composeResourceOptions({
+      declaration: declarations.get(scopeName),
+      stored: descriptor.definition,
+      local: runtimeDeclarations.get(scopeName),
+      fromStorage: storageLoaded.has(scopeName)
+    })
+
+    const compileOptions = async (scopeName, scopeOptions) => {
+      const compiled = { scopeOptions, vars: { idProperty: api.resources[scopeName].vars.idProperty } }
+      await compileSchemas(compiled, { context: { scopeName }, runHooks: (name, context) => api.runHooks(name, context) })
+      return compiled.vars.schemaInfo
+    }
+
+    // Publish a descriptor with the schema it implies. Recompile only when the
+    // field set changed, so schema hooks do not rerun on every refresh.
+    const publish = async (scopeName, descriptor, { compiled } = {}) => {
       const scope = api.resources[scopeName]
-      const schemaInfo = compiledSchemaInfo || scope.vars.schemaInfo
-      const descriptor = await registry.getDescriptor(tenantId, scopeName)
-      if (!descriptor) throw new Error(`Descriptor not found for resource '${scopeName}'`)
+      if (publishedVersion(scopeName) > descriptor.schemaVersion) return
+      const { options: scopeOptions, hidden, hiddenFilters } = composeOptions(scopeName, descriptor)
+      if (hidden.length || hiddenFilters.length) {
+        await log.warn(`AnyAPI resource '${scopeName}' hides stored fields whose callbacks this server does not declare`, {
+          tenant: tenantId, scopeName, fields: hidden, filters: hiddenFilters
+        })
+      }
+      const signature = optionsSignature(scopeOptions)
+      let schemaInfo = scope.vars.schemaInfo
+      if (compiled?.signature === signature) schemaInfo = compiled.schemaInfo
+      else if (compiledSignatures.get(scopeName) !== signature) schemaInfo = await compileOptions(scopeName, scopeOptions)
+      compiledSignatures.set(scopeName, signature)
+
       descriptor.idProperty = schemaInfo.idProperty
       scope.vars.schemaInfo = {
         ...schemaInfo,
@@ -88,10 +147,124 @@ export const RestApiAnyapiKnexPlugin = {
         canonicalFieldMap: descriptor.canonicalFieldMap,
       }
       getScopeStorageAdapter(scopeName)
-      scopeOptionsRegistry.set(scopeName, {
-        ...scopeOptionsRegistry.get(scopeName),
-        canonicalFieldsMap: descriptor.canonicalFieldMap,
+      lastChecked.set(scopeName, Date.now())
+    }
+
+    // Reloads of one resource run in order, so an older read never wins.
+    const reload = (scopeName) => {
+      const run = (reloads.get(scopeName) || Promise.resolve()).catch(() => {}).then(async () => {
+        registry.invalidateDescriptor(tenantId, scopeName)
+        const descriptor = await registry.getDescriptor(tenantId, scopeName)
+        if (!descriptor) {
+          throw new RestApiResourceError(`Resource '${scopeName}' is not registered in AnyAPI storage`, {
+            subtype: 'not_found', resourceType: 'scope', resourceId: scopeName
+          })
+        }
+        await publish(scopeName, descriptor)
+        return publishedVersion(scopeName)
       })
+      reloads.set(scopeName, run)
+      run.finally(() => { if (reloads.get(scopeName) === run) reloads.delete(scopeName) }).catch(() => {})
+      return run
+    }
+
+    const notifySchemaChanged = async (scopeName, version) => {
+      if (!onSchemaChanged) return
+      try {
+        await onSchemaChanged({ tenant: tenantId, resource: scopeName, version })
+      } catch (error) {
+        // The change is committed; other servers still catch up through schema checks.
+        await log.logError('AnyAPI onSchemaChanged failed', error, { tenant: tenantId, scopeName, version })
+      }
+    }
+
+    const ensureFresh = (scopeName, maxAge) => {
+      if (maxAge > 0 && Date.now() - (lastChecked.get(scopeName) ?? 0) < maxAge) return false
+      if (!checks.has(scopeName)) {
+        const check = (async () => {
+          lastChecked.set(scopeName, Date.now())
+          const version = await registry.getSchemaVersion(tenantId, scopeName)
+          if (version === null || version <= publishedVersion(scopeName)) return false
+          await reload(scopeName)
+          return true
+        })()
+        checks.set(scopeName, check)
+        check.finally(() => checks.delete(scopeName)).catch(() => {})
+      }
+      return checks.get(scopeName)
+    }
+
+    const watchSchema = (scope) => {
+      for (const name of REQUEST_METHODS) {
+        const method = scope[name]
+        if (typeof method !== 'function') continue
+        scope[name] = async function (params = {}, context = {}) {
+          // A caller transaction may hold uncommitted metadata; never publish from it.
+          if (params?.transaction) return method.call(this, params, context)
+          await ensureFresh(scope.name, schemaCheckInterval)
+          try {
+            return await method.call(this, params, context)
+          } catch (error) {
+            if (!isStaleSchemaRejection(error)) throw error
+            let reloaded = false
+            try { reloaded = await ensureFresh(scope.name, 0) } catch { throw error }
+            if (!reloaded) throw error
+            return method.call(this, params, context)
+          }
+        }
+      }
+    }
+
+    const loadResource = async (name, chain = new Set()) => {
+      if (typeof name !== 'string' || !name) throw new TypeError('loadResource requires a resource name')
+      if (Object.hasOwn(api.resources, name)) return api.resources[name]
+      if (chain.has(name)) return null
+      if (!loading.has(name)) {
+        const load = (async () => {
+          const descriptor = await registry.getDescriptor(tenantId, name)
+          if (!descriptor) return null
+          if (!descriptor.definition?.options) {
+            throw new Error(`AnyAPI resource '${name}' has no stored definition; declare it in code or register it again with this version`)
+          }
+          const { options: declaration } = composeResourceOptions({
+            declaration: descriptor.definition.options,
+            stored: { callbacks: descriptor.definition.callbacks },
+            fromStorage: true
+          })
+          const { before, after } = relatedResources(declaration)
+          const nextChain = new Set([...chain, name])
+          for (const target of before) await loadResource(target, nextChain)
+          if (Object.hasOwn(api.resources, name)) return api.resources[name]
+          storageLoaded.add(name)
+          try {
+            await api.addResource(name, declaration)
+          } catch (error) {
+            // A failed registration must not leave a half-published resource behind.
+            storageLoaded.delete(name)
+            declarations.delete(name)
+            delete api.resources[name]
+            throw error
+          }
+          for (const target of after) await loadResource(target, nextChain)
+          return api.resources[name]
+        })()
+        loading.set(name, load)
+        load.finally(() => loading.delete(name)).catch(() => {})
+      }
+      return loading.get(name)
+    }
+
+    api.anyapi.loadResource = name => loadResource(name)
+    api.anyapi.loadResources = async () => {
+      for (const { resource } of await registry.listResources(tenantId)) await loadResource(resource)
+      return api.resources
+    }
+    api.anyapi.getSchemaVersion = resource => registry.getSchemaVersion(tenantId, resource)
+    api.anyapi.handleSchemaChange = async ({ tenant, resource, version } = {}) => {
+      if ((tenant !== undefined && tenant !== tenantId) || typeof resource !== 'string' || !Object.hasOwn(api.resources, resource)) return false
+      if (version !== undefined && publishedVersion(resource) >= version) return false
+      await reload(resource)
+      return true
     }
 
     const buildSearchMembershipQuery = ({ sourceScopeName, relationshipName, targetScopeName, foreignKey, otherKey, db }) => {
@@ -131,28 +304,77 @@ export const RestApiAnyapiKnexPlugin = {
     addHook('resource:added', 'anyapi-register-resource', {}, async ({ context }) => {
       const { scopeName, scopeOptions = {} } = context
       const scope = api.resources[scopeName]
-      const { schemaStructure, computed, schemaRelationships, idProperty } = scope.vars.schemaInfo
-      const descriptor = await registry.registerResource({
-        tenant: tenantId,
-        resource: scopeName,
-        schema: { ...schemaStructure, ...computed },
-        relationships: schemaRelationships,
-        canonicalFieldMap: scopeOptions.canonicalFieldsMap || null,
-        idProperty
-      })
-      scopeOptionsRegistry.set(scopeName, snapshotResourceConfiguration({
-        ...scopeOptions,
-        idProperty,
-        canonicalFieldsMap: descriptor.canonicalFieldMap
-      }))
-      await refreshStorageDescriptor(scopeName)
+      const declaration = snapshotResourceConfiguration(scopeOptions)
+      declarations.set(scopeName, declaration)
+      compiledSignatures.set(scopeName, optionsSignature(declaration))
+      let descriptor
+      let changed = false
+      if (storageLoaded.has(scopeName)) {
+        descriptor = await registry.getDescriptor(tenantId, scopeName)
+        if (!descriptor) throw new Error(`Descriptor not found for resource '${scopeName}'`)
+      } else {
+        const { schemaStructure, computed, schemaRelationships, idProperty } = scope.vars.schemaInfo
+        ;({ descriptor, changed } = await registry.register({
+          tenant: tenantId,
+          resource: scopeName,
+          schema: { ...schemaStructure, ...computed },
+          relationships: schemaRelationships,
+          canonicalFieldMap: scopeOptions.canonicalFieldsMap || null,
+          idProperty,
+          declaration
+        }))
+      }
+      await publish(scopeName, descriptor)
+      if (schemaCheckInterval !== undefined) watchSchema(scope)
+      if (changed) await notifySchemaChanged(scopeName, descriptor.schemaVersion)
     })
 
     addResourceMethod('createKnexTable', async ({ scopeName }) => {
-      await ensureAnyApiSchema(knex)
-      await refreshStorageDescriptor(scopeName)
+      if (ensureSchema) await ensureAnyApiSchema(knex, { slots })
+      const descriptor = await registry.getDescriptor(tenantId, scopeName)
+      if (!descriptor) throw new Error(`Descriptor not found for resource '${scopeName}'`)
+      await publish(scopeName, descriptor)
     })
 
+    addResourceMethod('reloadSchema', async ({ scopeName }) => reload(scopeName))
+
+    // Compile the resource with the additions, so allocation stores enriched definitions.
+    const prepareAdditions = async (scopeName, { fields, searchSchema = {}, canonicalFieldsMap }) => {
+      const current = composeOptions(scopeName, api.resources[scopeName].vars.schemaInfo.descriptor).options
+      const candidate = snapshotResourceConfiguration({
+        ...current,
+        schema: { ...current.schema, ...fields },
+        searchSchema: { ...current.searchSchema, ...searchSchema }
+      })
+      const schemaInfo = await compileOptions(scopeName, candidate)
+      const allocation = {
+        tenant: tenantId,
+        resource: scopeName,
+        fields: Object.fromEntries(Object.keys(fields).map(fieldName => [fieldName, {
+          definition: schemaInfo.schemaStructure[fieldName] || schemaInfo.computed[fieldName],
+          declaration: fields[fieldName],
+          canonicalField: canonicalFieldsMap?.[fieldName]
+        }])),
+        searchSchema
+      }
+      return { candidate, schemaInfo, allocation }
+    }
+
+    const publishChange = async (scopeName, { fields = {}, searchSchema = {}, compiled } = {}) => {
+      registry.invalidateDescriptor(tenantId, scopeName)
+      if (Object.keys(fields).length || Object.keys(searchSchema).length) {
+        const local = runtimeDeclarations.get(scopeName) || {}
+        runtimeDeclarations.set(scopeName, {
+          schema: { ...local.schema, ...fields },
+          searchSchema: { ...local.searchSchema, ...searchSchema }
+        })
+      }
+      // The fresh descriptor also carries changes other servers made meanwhile.
+      const descriptor = await registry.getDescriptor(tenantId, scopeName)
+      await publish(scopeName, descriptor, { compiled })
+      await notifySchemaChanged(scopeName, descriptor.schemaVersion)
+    }
+
     addResourceMethod('addKnexFields', async ({ scopeName, params }) => {
       if (!params?.fields || typeof params.fields !== 'object' || Array.isArray(params.fields)) {
         throw new Error('fields parameter is required for addKnexFields')
@@ -160,35 +382,90 @@ export const RestApiAnyapiKnexPlugin = {
       assertFieldNameMap(params.fields, `added fields in '${scopeName}'`)
       assertFieldNameMap(params.searchSchema, `added search schema in '${scopeName}'`)
       assertFieldNameMap(params.canonicalFieldsMap, `added canonical fields in '${scopeName}'`)
-      const scope = api.resources[scopeName]
-      const storedBefore = scopeOptionsRegistry.get(scopeName) || {}
       const unknown = Object.keys(params.canonicalFieldsMap || {}).filter(field => !Object.hasOwn(params.fields, field))
       if (unknown.length) {
         throw new Error(`canonicalFieldsMap contains unknown fields for addKnexFields: ${unknown}`)
       }
-      const updatedStored = snapshotResourceConfiguration({
-        ...storedBefore,
-        schema: { ...storedBefore.schema, ...params.fields },
-        searchSchema: { ...storedBefore.searchSchema, ...params.searchSchema }
-      })
-      const compiled = { scopeOptions: updatedStored, vars: { idProperty: scope.vars.idProperty } }
-      await compileSchemas(compiled, { context: { scopeName }, runHooks: (name, context) => api.runHooks(name, context) })
-      await api.transaction(async transaction => {
-        for (const fieldName of Object.keys(params.fields)) {
-          await registry.allocateField({
-            tenant: tenantId,
-            resource: scopeName,
-            fieldName,
-            definition: compiled.vars.schemaInfo.schemaStructure[fieldName] || compiled.vars.schemaInfo.computed[fieldName],
-            canonicalField: params.canonicalFieldsMap?.[fieldName]
-          }, { transaction })
-        }
+      const fields = snapshotResourceConfiguration(params.fields)
+      const searchSchema = snapshotResourceConfiguration(params.searchSchema || {})
+      const { candidate, schemaInfo, allocation } = await prepareAdditions(scopeName, { fields, searchSchema, canonicalFieldsMap: params.canonicalFieldsMap })
+      await api.transaction(transaction => registry.allocateFields(allocation, { transaction }))
+      await publishChange(scopeName, { fields, searchSchema, compiled: { signature: optionsSignature(candidate), schemaInfo } })
+    })
+
+    const retirable = (scopeName, fieldName) => {
+      const definition = api.resources[scopeName].vars.schemaInfo.schemaStructure[fieldName]
+      if (!definition || fieldName === 'id' || definition.virtual === true || definition.belongsTo) {
+        throw new Error(`'${scopeName}.${fieldName}' is not a stored attribute`)
+      }
+      return definition
+    }
+
+    addResourceMethod('retireKnexFields', async ({ scopeName, params }) => {
+      const fields = Array.isArray(params) ? params : params?.fields
+      if (!Array.isArray(fields) || !fields.length) throw new Error('retireKnexFields requires an array of field names')
+      fields.forEach(fieldName => retirable(scopeName, fieldName))
+      await registry.retireFields({ tenant: tenantId, resource: scopeName, fields })
+      await publishChange(scopeName)
+    })
+
+    // Change a field's type: copy its values into a new field, then retire it.
+    addResourceMethod('migrateField', async ({ scopeName, params }) => {
+      const { from, to, transform } = params || {}
+      const { name, ...declaration } = to || {}
+      if (typeof from !== 'string' || typeof name !== 'string' || !name) {
+        throw new Error('migrateField requires from and to: { name, ...definition }')
+      }
+      if (transform !== undefined && typeof transform !== 'function') throw new Error('migrateField transform must be a function')
+      const fromDefinition = retirable(scopeName, from)
+      if (fromDefinition.retired) throw new Error(`Field '${scopeName}.${from}' is already retired`)
+      if (declaration.computed || declaration.virtual || declaration.belongsTo || declaration.getter || declaration.setter || declaration.storage) {
+        throw new Error(`migrateField target '${name}' must be a plain stored attribute; add other fields with addKnexFields`)
+      }
+      assertFieldNameMap({ [name]: declaration }, `migrated field in '${scopeName}'`)
+      const fields = snapshotResourceConfiguration({ [name]: declaration })
+      const { schemaInfo, allocation } = await prepareAdditions(scopeName, { fields })
+
+      await api.transaction(async (transaction) => {
+        const descriptor = await registry.allocateFields(allocation, { transaction })
+        await copyFieldValues({ transaction, descriptor, schemaInfo, from, fromDefinition, to: name, transform })
+        await registry.retireFields({ tenant: tenantId, resource: scopeName, fields: [from] }, { transaction })
       })
-      registry.invalidateDescriptor(tenantId, scopeName)
-      scopeOptionsRegistry.set(scopeName, updatedStored)
-      await refreshStorageDescriptor(scopeName, compiled.vars.schemaInfo)
+      await publishChange(scopeName, { fields })
     })
 
+    const copyFieldValues = async ({ transaction, descriptor, schemaInfo, from, fromDefinition, to, transform }) => {
+      const { canonical } = descriptor
+      const source = descriptor.fields[from].slot
+      const target = descriptor.fields[to].slot
+      const idColumn = canonical.logicalIdColumn
+      for (let lastId = null; ;) {
+        const query = transaction(canonical.tableName)
+          .where(canonical.tenantColumn, tenantId)
+          .where(canonical.resourceColumn, descriptor.resource)
+          .orderBy('id')
+          .limit(MIGRATION_BATCH_SIZE)
+          .select('id', idColumn, source)
+        if (lastId !== null) query.where('id', '>', lastId)
+        const rows = await applyDatabaseReadOptions(query)
+        if (!rows.length) return
+        for (const row of rows) {
+          lastId = row.id
+          const value = normalizeStoredFieldValue(row[source], fromDefinition, { fieldName: from, resourceType: descriptor.resource })
+          const next = transform ? await transform(value, { id: String(row[idColumn]) }) : value
+          const { validatedObject, errors } = await schemaInfo.schemaInstance.patch({ [to]: next ?? null })
+          if (Object.keys(errors).length) {
+            const message = `Record '${row[idColumn]}' cannot migrate '${from}' to '${to}': ${Object.values(errors)[0].message}`
+            throw new RestApiValidationError(message, { fields: [to], violations: [{ field: to, rule: 'field_migration', message }] })
+          }
+          const stored = translateCanonicalAttributesForStorage({ [to]: validatedObject[to] ?? null }, descriptor, {
+            schemaInfo, databaseClient: knex.client.config.client
+          })
+          await transaction(canonical.tableName).where('id', row.id).update({ [target]: stored[target] ?? null })
+        }
+      }
+    }
+
     addResourceMethod('alterKnexFields', async () => {
       throw new Error('alterKnexFields is not supported by AnyAPI Knex plugin yet')
     })

package/plugins/core/rest-api-plugin-methods/aggregate.js

@@ -0,0 +1,41 @@
+import { getRequestContracts, validateRequestContractOrThrow } from '../lib/querying-writing/request-contracts.js'
+import { getEffectiveSortableFields } from '../lib/querying/query-field-sort-helpers.js'
+import { compileAggregateRequest } from '../lib/querying/aggregate-query.js'
+
+/**
+ * Group a filtered collection and compute measures for each group.
+ * Filters are validated like query filters; permissions are checked as a
+ * query, and row policies and autofilters see queryPurpose 'aggregate'.
+ * Returns plain rows in either format; grouped results are paged.
+ */
+export default async function aggregateMethod ({ params, context, vars, helpers, scope, runHooks, scopeName, api }) {
+  context.method = 'aggregate'
+  context.scopeName = scopeName
+  context.schemaInfo = scope.vars.schemaInfo
+
+  const plan = compileAggregateRequest({
+    params,
+    schemaInfo: context.schemaInfo,
+    scopeName,
+    defaultLimit: vars.queryDefaultLimit,
+    maxLimit: vars.queryMaxLimit
+  })
+  const requestContracts = getRequestContracts({
+    scopeName,
+    schemaInfo: context.schemaInfo,
+    includeDepthLimit: vars.includeDepthLimit,
+    sortableFields: getEffectiveSortableFields(vars)
+  })
+  const validated = validateRequestContractOrThrow(
+    requestContracts.query,
+    { queryParams: { filters: params.queryParams?.filters ?? {} } },
+    'Aggregate filters are invalid'
+  )
+  context.queryParams = { filters: validated.queryParams?.filters || {} }
+
+  context.transaction = params.transaction
+  context.db = context.transaction || api.knex.instance
+
+  await scope.checkPermissions({ method: 'query', originalContext: context })
+  return helpers.dataAggregate({ scopeName, context, runHooks, plan })
+}