json-rest-api 1.0.29 → 2.1.0

package/plugins/core/lib/anyapi/schema-utils.js

@@ -1,17 +1,42 @@
-const STRING_SLOT_COUNT = 10
-const NUMBER_SLOT_COUNT = 10
-const BOOLEAN_SLOT_COUNT = 5
-const DATE_SLOT_COUNT = 5
-const JSON_SLOT_COUNT = 5
-const BELONGS_TO_SLOT_COUNT = 5
-
-export const SLOT_LIMITS = {
-  string: STRING_SLOT_COUNT,
-  number: NUMBER_SLOT_COUNT,
-  boolean: BOOLEAN_SLOT_COUNT,
-  date: DATE_SLOT_COUNT,
-  json: JSON_SLOT_COUNT,
-  belongsTo: BELONGS_TO_SLOT_COUNT,
+import { hasKnexTableIndex } from '../dbIntrospection.js'
+
+// Default layout of a new any_records table. The registry reads the actual
+// columns, so a migration that adds slots raises the limits for every server.
+export const SLOT_LIMITS = Object.freeze({
+  string: 10,
+  number: 10,
+  boolean: 5,
+  date: 5,
+  json: 5,
+  belongsTo: 5,
+})
+
+const SLOT_COLUMNS = {
+  string: index => [`string_${index}`],
+  number: index => [`number_${index}`],
+  boolean: index => [`boolean_${index}`],
+  date: index => [`date_${index}`],
+  json: index => [`json_${index}`],
+  belongsTo: index => [`rel_${index}_id`, `rel_${index}_type`],
+}
+
+const addSlotColumn = (table, column) => {
+  if (column.startsWith('number_')) table.double(column)
+  else if (column.startsWith('boolean_')) table.boolean(column)
+  else if (column.startsWith('date_')) table.dateTime(column, { precision: 6 })
+  else if (column.startsWith('rel_')) table.string(column)
+  else table.text(column)
+}
+
+const resolveSlotCounts = (slots = {}) => {
+  if (!slots || typeof slots !== 'object' || Array.isArray(slots)) {
+    throw new Error('AnyAPI slots must be an object of slot counts')
+  }
+  for (const [pool, count] of Object.entries(slots)) {
+    if (!Object.hasOwn(SLOT_LIMITS, pool)) throw new Error(`Unknown AnyAPI slot pool '${pool}'`)
+    if (!Number.isInteger(count) || count < 1) throw new Error(`AnyAPI ${pool} slot count must be a positive integer`)
+  }
+  return { ...SLOT_LIMITS, ...slots }
 }
 
 export const DEFAULT_CANONICAL_CONFIG = {
@@ -48,46 +73,31 @@ const ensureColumn = async (knex, tableName, columnName, columnBuilder) => {
 const ensureUniqueIndex = async (knex, tableName, columns, indexName) => {
   const exists = await knex.schema.hasTable(tableName)
   if (!exists) return
-  const hasIndex = await knex.schema.withSchema('main').hasIndex?.(tableName, indexName)
+  const hasIndex = await hasKnexTableIndex(knex, tableName, indexName)
   if (!hasIndex) {
-    try {
-      await knex.schema.table(tableName, (table) => {
-        table.unique(columns, indexName)
-      })
-    } catch (error) {
-      if (!String(error?.message || '').includes('already exists')) {
-        throw error
-      }
-    }
+    await knex.schema.table(tableName, (table) => {
+      table.unique(columns, indexName)
+    })
   }
 }
 
-export const ensureAnyApiSchema = async (knex) => {
+/**
+ * Create or upgrade the shared canonical tables. Run it from a migration, or let
+ * the plugin run it at start-up. `slots` only adds missing slot columns; lowering
+ * a count never drops columns that may hold data.
+ * @param {import('knex').Knex} knex
+ * @param {{ slots?: Partial<Record<keyof typeof SLOT_LIMITS, number>> }} [options]
+ */
+export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
+  const counts = resolveSlotCounts(slots)
   await ensureTable(knex, 'any_records', (table) => {
     table.increments('id').primary()
     table.string('logical_id')
     table.string('tenant_id').notNullable()
     table.string('resource').notNullable()
 
-    for (let index = 1; index <= STRING_SLOT_COUNT; index += 1) {
-      table.text(`string_${index}`)
-    }
-    for (let index = 1; index <= NUMBER_SLOT_COUNT; index += 1) {
-      table.double(`number_${index}`)
-    }
-    for (let index = 1; index <= BOOLEAN_SLOT_COUNT; index += 1) {
-      table.boolean(`boolean_${index}`)
-    }
-    for (let index = 1; index <= DATE_SLOT_COUNT; index += 1) {
-      table.dateTime(`date_${index}`)
-    }
-    for (let index = 1; index <= JSON_SLOT_COUNT; index += 1) {
-      table.text(`json_${index}`)
-    }
-
-    for (let index = 1; index <= BELONGS_TO_SLOT_COUNT; index += 1) {
-      table.string(`rel_${index}_id`)
-      table.string(`rel_${index}_type`)
+    for (const [pool, count] of Object.entries(counts)) {
+      for (let index = 1; index <= count; index += 1) SLOT_COLUMNS[pool](index).forEach(column => addSlotColumn(table, column))
     }
 
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
@@ -99,6 +109,28 @@ export const ensureAnyApiSchema = async (knex) => {
     table.index(['resource'])
   })
 
+  if (slots) {
+    const existing = new Set(Object.keys(await knex('any_records').columnInfo()))
+    const missing = Object.entries(counts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => SLOT_COLUMNS[pool](index + 1)))
+      .flat()
+      .filter(column => !existing.has(column))
+    if (missing.length) {
+      await knex.schema.alterTable('any_records', (table) => missing.forEach(column => addSlotColumn(table, column)))
+    }
+  }
+
+  const client = knex.client.config.client
+  if (['pg', 'postgresql', 'mysql', 'mysql2'].includes(client)) {
+    const columns = (await knex('information_schema.columns')
+      .select({ column: 'column_name', precision: 'datetime_precision' })
+      .where('table_schema', knex.raw(client === 'pg' || client === 'postgresql' ? 'current_schema()' : 'DATABASE()'))
+      .where('table_name', 'any_records'))
+      .filter(column => /^date_\d+$/.test(column.column))
+    if (!columns.length || columns.some(column => Number(column.precision) !== 6)) {
+      throw new Error('AnyAPI temporal storage migration required: every any_records date slot must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
+    }
+  }
+
   await ensureColumn(knex, 'any_records', 'logical_id', (table) => {
     table.string('logical_id')
   })
@@ -146,15 +178,24 @@ export const ensureAnyApiSchema = async (knex) => {
     table.string('resource').notNullable()
     table.text('schema_json').notNullable()
     table.text('relationships_json').notNullable()
+    // Holds the search schema and runtime additions too: MySQL TEXT stops at 64 KB.
+    table.text('definition_json', 'longtext')
+    table.integer('schema_version').notNullable().defaultTo(1)
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['tenant_id', 'resource'])
   })
+  await ensureColumn(knex, 'any_resource_configs', 'definition_json', (table) => {
+    table.text('definition_json', 'longtext')
+  })
+  await ensureColumn(knex, 'any_resource_configs', 'schema_version', (table) => {
+    table.integer('schema_version').notNullable().defaultTo(1)
+  })
 
   await ensureTable(knex, 'any_field_configs', (table) => {
     table.increments('id').primary()
-    table.integer('resource_config_id').notNullable()
+    table.integer('resource_config_id').unsigned().notNullable()
       .references('id').inTable('any_resource_configs').onDelete('CASCADE')
     table.string('field_name').notNullable()
     table.string('slot_type').notNullable()
@@ -169,12 +210,24 @@ export const ensureAnyApiSchema = async (knex) => {
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['resource_config_id', 'field_name'])
+    table.unique(['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
     table.index(['resource_config_id'])
   })
+  if (!await hasKnexTableIndex(knex, 'any_field_configs', 'any_field_configs_resource_slot_unique')) {
+    const shared = await knex('any_field_configs')
+      .select('resource_config_id', 'slot_column')
+      .groupBy('resource_config_id', 'slot_column')
+      .havingRaw('count(*) > 1')
+      .first()
+    if (shared) {
+      throw new Error(`AnyAPI slot metadata repair required: several fields of resource config ${shared.resource_config_id} share ${shared.slot_column}. Reassign them before upgrading.`)
+    }
+    await ensureUniqueIndex(knex, 'any_field_configs', ['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
+  }
 
   await ensureTable(knex, 'any_relationship_configs', (table) => {
     table.increments('id').primary()
-    table.integer('resource_config_id').notNullable()
+    table.integer('resource_config_id').unsigned().notNullable()
       .references('id').inTable('any_resource_configs').onDelete('CASCADE')
     table.string('relationship_name').notNullable()
     table.string('relationship_type').notNullable()
@@ -191,18 +244,28 @@ export const ensureAnyApiSchema = async (knex) => {
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
-    table.unique(['resource_config_id', 'relationship_name'])
+    table.unique(['resource_config_id', 'relationship_name'], 'any_relationship_configs_resource_relationship_unique')
     table.index(['resource_config_id'])
   })
 }
 
-export const SLOT_POOLS = {
-  string: Array.from({ length: STRING_SLOT_COUNT }, (_, index) => `string_${index + 1}`),
-  number: Array.from({ length: NUMBER_SLOT_COUNT }, (_, index) => `number_${index + 1}`),
-  boolean: Array.from({ length: BOOLEAN_SLOT_COUNT }, (_, index) => `boolean_${index + 1}`),
-  date: Array.from({ length: DATE_SLOT_COUNT }, (_, index) => `date_${index + 1}`),
-  json: Array.from({ length: JSON_SLOT_COUNT }, (_, index) => `json_${index + 1}`),
-  belongsTo: Array.from({ length: BELONGS_TO_SLOT_COUNT }, (_, index) => index + 1),
+const buildSlotPools = (hasColumns) => Object.fromEntries(Object.keys(SLOT_LIMITS).map(pool => {
+  const indexes = []
+  for (let index = 1; hasColumns(pool, index); index += 1) indexes.push(index)
+  return [pool, pool === 'belongsTo' ? indexes : indexes.map(index => SLOT_COLUMNS[pool](index)[0])]
+}))
+
+export const SLOT_POOLS = buildSlotPools((pool, index) => index <= SLOT_LIMITS[pool])
+
+/**
+ * Read the slot columns any_records actually has. Slots are numbered from 1
+ * without gaps; a pool ends at its first missing column.
+ * @param {import('knex').Knex | import('knex').Knex.Transaction} db
+ */
+export const readSlotPools = async (db) => {
+  const columns = new Set(Object.keys(await db('any_records').columnInfo()))
+  if (!columns.size) throw new Error('AnyAPI storage tables are missing; run ensureAnyApiSchema in a migration')
+  return buildSlotPools((pool, index) => SLOT_COLUMNS[pool](index).every(column => columns.has(column)))
 }
 
 export const TYPE_TO_POOL = new Map([
@@ -216,12 +279,13 @@ export const TYPE_TO_POOL = new Map([
   ['float', 'number'],
   ['decimal', 'number'],
   ['boolean', 'boolean'],
-  ['date', 'date'],
+  ['date', 'string'],
   ['datetime', 'date'],
   ['dateTime', 'date'],
-  ['time', 'date'],
+  ['time', 'string'],
   ['epochMilliseconds', 'number'],
   ['epochSeconds', 'number'],
   ['json', 'json'],
   ['object', 'json'],
+  ['array', 'json'],
 ])

package/plugins/core/lib/anyapi/stored-definition.js

@@ -0,0 +1,109 @@
+// Stored resource definitions keep JSON values only. The paths of values JSON
+// cannot hold (callbacks, class instances, RegExps) are recorded, so a server
+// loading the definition knows which fields need code it may not have.
+
+const isPlainObject = value => {
+  const prototype = Object.getPrototypeOf(value)
+  return prototype === Object.prototype || prototype === null
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ json: any, callbacks: Array<Array<string|number>> }}
+ */
+export const toStoredJson = (value) => {
+  const callbacks = []
+  const visit = (current, path) => {
+    if (current === undefined) return undefined
+    if (current === null || typeof current === 'string' || typeof current === 'boolean') return current
+    if (typeof current === 'number' && Number.isFinite(current)) return current
+    if (Array.isArray(current)) return current.map((item, index) => visit(item, [...path, index]) ?? null)
+    if (typeof current === 'object' && isPlainObject(current)) {
+      const result = {}
+      for (const [key, item] of Object.entries(current)) {
+        const stored = visit(item, [...path, key])
+        if (stored !== undefined) result[key] = stored
+      }
+      return result
+    }
+    callbacks.push(path)
+    return undefined
+  }
+  return { json: visit(value, []), callbacks }
+}
+
+const targetsOf = definition => definition?.oneOf || [definition?.actualField].filter(Boolean)
+
+/**
+ * Combine a resource's declaration with the fields other servers added at run
+ * time. Declared fields win; runtime fields this server added keep its local
+ * callbacks. A field whose callbacks exist in no available code is hidden
+ * rather than served without them, because a missing serializer or setter
+ * would read or write the wrong values.
+ * @param {{ declaration?: Record<string, any>, stored?: Record<string, any> | null, local?: { schema?: Record<string, any>, searchSchema?: Record<string, any> }, fromStorage?: boolean }} input
+ */
+export const composeResourceOptions = ({ declaration = {}, stored = null, local = {}, fromStorage = false }) => {
+  const hidden = new Set()
+  const hiddenFilters = new Set()
+  const declaredSchema = declaration.schema || {}
+  const declaredSearch = declaration.searchSchema || {}
+  const schema = { ...declaredSchema }
+  const searchSchema = { ...declaredSearch }
+
+  if (fromStorage) {
+    for (const path of stored?.callbacks || []) {
+      if (path[0] === 'schema' && path.length > 1) hidden.add(path[1])
+      else if (path[0] === 'searchSchema' && path.length > 1) hiddenFilters.add(path[1])
+      else throw new Error(`Resource option '${path.join('.')}' cannot be loaded from storage; declare the resource in code`)
+    }
+  }
+
+  const runtime = stored?.runtime || {}
+  const needsCode = new Set((runtime.callbacks || []).map(path => `${path[0]}.${path[1]}`))
+  const merge = (kind, target, declared, localDefinitions = {}, hide) => {
+    for (const [name, definition] of Object.entries(runtime[kind] || {})) {
+      if (Object.hasOwn(declared, name)) continue
+      if (Object.hasOwn(localDefinitions, name)) target[name] = localDefinitions[name]
+      else if (needsCode.has(`${kind}.${name}`)) hide.add(name)
+      else target[name] = definition
+    }
+  }
+  merge('schema', schema, declaredSchema, local.schema, hidden)
+  merge('searchSchema', searchSchema, declaredSearch, local.searchSchema, hiddenFilters)
+
+  for (const name of hidden) delete schema[name]
+  for (const [name, definition] of Object.entries(searchSchema)) {
+    if (hiddenFilters.has(name) || targetsOf(definition).some(target => hidden.has(target))) {
+      hiddenFilters.add(name)
+      delete searchSchema[name]
+    }
+  }
+
+  const options = { ...declaration, schema }
+  if (Object.keys(searchSchema).length || declaration.searchSchema) options.searchSchema = searchSchema
+  if (hidden.size && Array.isArray(declaration.sortableFields)) {
+    options.sortableFields = declaration.sortableFields.filter(field => !hidden.has(String(field).replace(/^-/, '')))
+  }
+  return { options, hidden: [...hidden], hiddenFilters: [...hiddenFilters] }
+}
+
+/**
+ * Resources a stored declaration refers to. Polymorphic targets must be
+ * registered before the resource itself; the others can follow.
+ * @param {Record<string, any>} options
+ */
+export const relatedResources = (options = {}) => {
+  const before = new Set()
+  const after = new Set()
+  for (const definition of Object.values(options.schema || {})) {
+    if (definition?.belongsTo) after.add(definition.belongsTo)
+  }
+  for (const [name, definition] of Object.entries(options.relationships || {})) {
+    for (const type of definition?.belongsToPolymorphic?.types || []) before.add(type)
+    if (definition?.type === 'hasMany' || definition?.type === 'manyToMany' || definition?.hasMany) {
+      after.add(definition.target || definition.hasMany || name)
+    }
+    if (definition?.through) after.add(definition.through)
+  }
+  return { before: [...before], after: [...after].filter(name => !before.has(name)) }
+}

package/plugins/core/lib/anyapi/utils/descriptor-helpers.js

@@ -1,6 +1,3 @@
-import { RestApiValidationError } from '../../../../../lib/rest-api-errors.js'
-import { normalizeValueForDatabaseStorage } from '../../querying-writing/database-value-normalizers.js'
-
 export const normalizeId = (value) => (value === null || value === undefined ? null : String(value))
 
 export const findSchemaFieldByAlias = (descriptor, alias) => {
@@ -61,64 +58,3 @@ export const resolveFieldInfo = (descriptor, field) => {
 
   return null
 }
-
-export const coerceValueForDefinition = (value, definition, { isRelationship } = {}) => {
-  if (value === null || value === undefined) return null
-
-  if (isRelationship) {
-    return normalizeId(value)
-  }
-
-  const type = definition?.type || definition?.dataType
-  if (!type) {
-    return value
-  }
-
-  if (['number', 'integer', 'float', 'decimal'].includes(type)) {
-    const numeric = Number(value)
-    return Number.isNaN(numeric) ? value : numeric
-  }
-
-  if (type === 'boolean') {
-    if (typeof value === 'boolean') return value
-    if (typeof value === 'string') {
-      if (value.toLowerCase() === 'true') return true
-      if (value.toLowerCase() === 'false') return false
-    }
-    return Boolean(value)
-  }
-
-  if (['string', 'text', 'uuid', 'email'].includes(type)) {
-    return String(value)
-  }
-
-  if (['date', 'dateTime', 'time'].includes(type)) {
-    return normalizeValueForDatabaseStorage(value, type, {
-      temporalPrecision: definition?.temporalPrecision
-    })
-  }
-
-  return value
-}
-
-export const normalizeFilterValues = (rawValue, definition, options) => {
-  if (Array.isArray(rawValue)) {
-    return rawValue.map((item) => coerceValueForDefinition(item, definition, options))
-  }
-  return [coerceValueForDefinition(rawValue, definition, options)]
-}
-
-export const ensureFilterableField = (descriptor, field) => {
-  const fieldInfo = resolveFieldInfo(descriptor, field)
-  if (!fieldInfo?.column) {
-    throw new RestApiValidationError('Invalid filter field', {
-      fields: [`filters.${field}`],
-      violations: [{
-        field: `filters.${field}`,
-        rule: 'unknown_field',
-        message: `Filter on '${field}' is not supported in AnyAPI mode`,
-      }],
-    })
-  }
-  return fieldInfo
-}