json-rest-api 2.0.0 → 2.1.0

package/plugins/core/lib/anyapi/schema-utils.js

@@ -1,19 +1,42 @@
 import { hasKnexTableIndex } from '../dbIntrospection.js'
 
-const STRING_SLOT_COUNT = 10
-const NUMBER_SLOT_COUNT = 10
-const BOOLEAN_SLOT_COUNT = 5
-const DATE_SLOT_COUNT = 5
-const JSON_SLOT_COUNT = 5
-const BELONGS_TO_SLOT_COUNT = 5
-
-export const SLOT_LIMITS = {
-  string: STRING_SLOT_COUNT,
-  number: NUMBER_SLOT_COUNT,
-  boolean: BOOLEAN_SLOT_COUNT,
-  date: DATE_SLOT_COUNT,
-  json: JSON_SLOT_COUNT,
-  belongsTo: BELONGS_TO_SLOT_COUNT,
+// Default layout of a new any_records table. The registry reads the actual
+// columns, so a migration that adds slots raises the limits for every server.
+export const SLOT_LIMITS = Object.freeze({
+  string: 10,
+  number: 10,
+  boolean: 5,
+  date: 5,
+  json: 5,
+  belongsTo: 5,
+})
+
+const SLOT_COLUMNS = {
+  string: index => [`string_${index}`],
+  number: index => [`number_${index}`],
+  boolean: index => [`boolean_${index}`],
+  date: index => [`date_${index}`],
+  json: index => [`json_${index}`],
+  belongsTo: index => [`rel_${index}_id`, `rel_${index}_type`],
+}
+
+const addSlotColumn = (table, column) => {
+  if (column.startsWith('number_')) table.double(column)
+  else if (column.startsWith('boolean_')) table.boolean(column)
+  else if (column.startsWith('date_')) table.dateTime(column, { precision: 6 })
+  else if (column.startsWith('rel_')) table.string(column)
+  else table.text(column)
+}
+
+const resolveSlotCounts = (slots = {}) => {
+  if (!slots || typeof slots !== 'object' || Array.isArray(slots)) {
+    throw new Error('AnyAPI slots must be an object of slot counts')
+  }
+  for (const [pool, count] of Object.entries(slots)) {
+    if (!Object.hasOwn(SLOT_LIMITS, pool)) throw new Error(`Unknown AnyAPI slot pool '${pool}'`)
+    if (!Number.isInteger(count) || count < 1) throw new Error(`AnyAPI ${pool} slot count must be a positive integer`)
+  }
+  return { ...SLOT_LIMITS, ...slots }
 }
 
 export const DEFAULT_CANONICAL_CONFIG = {
@@ -58,32 +81,23 @@ const ensureUniqueIndex = async (knex, tableName, columns, indexName) => {
   }
 }
 
-export const ensureAnyApiSchema = async (knex) => {
+/**
+ * Create or upgrade the shared canonical tables. Run it from a migration, or let
+ * the plugin run it at start-up. `slots` only adds missing slot columns; lowering
+ * a count never drops columns that may hold data.
+ * @param {import('knex').Knex} knex
+ * @param {{ slots?: Partial<Record<keyof typeof SLOT_LIMITS, number>> }} [options]
+ */
+export const ensureAnyApiSchema = async (knex, { slots } = {}) => {
+  const counts = resolveSlotCounts(slots)
   await ensureTable(knex, 'any_records', (table) => {
     table.increments('id').primary()
     table.string('logical_id')
     table.string('tenant_id').notNullable()
     table.string('resource').notNullable()
 
-    for (let index = 1; index <= STRING_SLOT_COUNT; index += 1) {
-      table.text(`string_${index}`)
-    }
-    for (let index = 1; index <= NUMBER_SLOT_COUNT; index += 1) {
-      table.double(`number_${index}`)
-    }
-    for (let index = 1; index <= BOOLEAN_SLOT_COUNT; index += 1) {
-      table.boolean(`boolean_${index}`)
-    }
-    for (let index = 1; index <= DATE_SLOT_COUNT; index += 1) {
-      table.dateTime(`date_${index}`, { precision: 6 })
-    }
-    for (let index = 1; index <= JSON_SLOT_COUNT; index += 1) {
-      table.text(`json_${index}`)
-    }
-
-    for (let index = 1; index <= BELONGS_TO_SLOT_COUNT; index += 1) {
-      table.string(`rel_${index}_id`)
-      table.string(`rel_${index}_type`)
+    for (const [pool, count] of Object.entries(counts)) {
+      for (let index = 1; index <= count; index += 1) SLOT_COLUMNS[pool](index).forEach(column => addSlotColumn(table, column))
     }
 
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
@@ -95,15 +109,25 @@ export const ensureAnyApiSchema = async (knex) => {
     table.index(['resource'])
   })
 
+  if (slots) {
+    const existing = new Set(Object.keys(await knex('any_records').columnInfo()))
+    const missing = Object.entries(counts).flatMap(([pool, count]) => Array.from({ length: count }, (_, index) => SLOT_COLUMNS[pool](index + 1)))
+      .flat()
+      .filter(column => !existing.has(column))
+    if (missing.length) {
+      await knex.schema.alterTable('any_records', (table) => missing.forEach(column => addSlotColumn(table, column)))
+    }
+  }
+
   const client = knex.client.config.client
   if (['pg', 'postgresql', 'mysql', 'mysql2'].includes(client)) {
-    const columns = await knex('information_schema.columns')
+    const columns = (await knex('information_schema.columns')
       .select({ column: 'column_name', precision: 'datetime_precision' })
       .where('table_schema', knex.raw(client === 'pg' || client === 'postgresql' ? 'current_schema()' : 'DATABASE()'))
-      .where('table_name', 'any_records')
-      .whereIn('column_name', Array.from({ length: DATE_SLOT_COUNT }, (_, index) => `date_${index + 1}`))
-    if (columns.length !== DATE_SLOT_COUNT || columns.some(column => Number(column.precision) !== 6)) {
-      throw new Error('AnyAPI temporal storage migration required: any_records.date_1 through date_5 must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
+      .where('table_name', 'any_records'))
+      .filter(column => /^date_\d+$/.test(column.column))
+    if (!columns.length || columns.some(column => Number(column.precision) !== 6)) {
+      throw new Error('AnyAPI temporal storage migration required: every any_records date slot must use timestamp/datetime precision 6. See 33-migrating-to-v2.md.')
     }
   }
 
@@ -154,11 +178,20 @@ export const ensureAnyApiSchema = async (knex) => {
     table.string('resource').notNullable()
     table.text('schema_json').notNullable()
     table.text('relationships_json').notNullable()
+    // Holds the search schema and runtime additions too: MySQL TEXT stops at 64 KB.
+    table.text('definition_json', 'longtext')
+    table.integer('schema_version').notNullable().defaultTo(1)
     table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable()
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['tenant_id', 'resource'])
   })
+  await ensureColumn(knex, 'any_resource_configs', 'definition_json', (table) => {
+    table.text('definition_json', 'longtext')
+  })
+  await ensureColumn(knex, 'any_resource_configs', 'schema_version', (table) => {
+    table.integer('schema_version').notNullable().defaultTo(1)
+  })
 
   await ensureTable(knex, 'any_field_configs', (table) => {
     table.increments('id').primary()
@@ -177,8 +210,20 @@ export const ensureAnyApiSchema = async (knex) => {
     table.dateTime('updated_at').defaultTo(knex.fn.now()).notNullable()
 
     table.unique(['resource_config_id', 'field_name'])
+    table.unique(['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
     table.index(['resource_config_id'])
   })
+  if (!await hasKnexTableIndex(knex, 'any_field_configs', 'any_field_configs_resource_slot_unique')) {
+    const shared = await knex('any_field_configs')
+      .select('resource_config_id', 'slot_column')
+      .groupBy('resource_config_id', 'slot_column')
+      .havingRaw('count(*) > 1')
+      .first()
+    if (shared) {
+      throw new Error(`AnyAPI slot metadata repair required: several fields of resource config ${shared.resource_config_id} share ${shared.slot_column}. Reassign them before upgrading.`)
+    }
+    await ensureUniqueIndex(knex, 'any_field_configs', ['resource_config_id', 'slot_column'], 'any_field_configs_resource_slot_unique')
+  }
 
   await ensureTable(knex, 'any_relationship_configs', (table) => {
     table.increments('id').primary()
@@ -204,13 +249,23 @@ export const ensureAnyApiSchema = async (knex) => {
   })
 }
 
-export const SLOT_POOLS = {
-  string: Array.from({ length: STRING_SLOT_COUNT }, (_, index) => `string_${index + 1}`),
-  number: Array.from({ length: NUMBER_SLOT_COUNT }, (_, index) => `number_${index + 1}`),
-  boolean: Array.from({ length: BOOLEAN_SLOT_COUNT }, (_, index) => `boolean_${index + 1}`),
-  date: Array.from({ length: DATE_SLOT_COUNT }, (_, index) => `date_${index + 1}`),
-  json: Array.from({ length: JSON_SLOT_COUNT }, (_, index) => `json_${index + 1}`),
-  belongsTo: Array.from({ length: BELONGS_TO_SLOT_COUNT }, (_, index) => index + 1),
+const buildSlotPools = (hasColumns) => Object.fromEntries(Object.keys(SLOT_LIMITS).map(pool => {
+  const indexes = []
+  for (let index = 1; hasColumns(pool, index); index += 1) indexes.push(index)
+  return [pool, pool === 'belongsTo' ? indexes : indexes.map(index => SLOT_COLUMNS[pool](index)[0])]
+}))
+
+export const SLOT_POOLS = buildSlotPools((pool, index) => index <= SLOT_LIMITS[pool])
+
+/**
+ * Read the slot columns any_records actually has. Slots are numbered from 1
+ * without gaps; a pool ends at its first missing column.
+ * @param {import('knex').Knex | import('knex').Knex.Transaction} db
+ */
+export const readSlotPools = async (db) => {
+  const columns = new Set(Object.keys(await db('any_records').columnInfo()))
+  if (!columns.size) throw new Error('AnyAPI storage tables are missing; run ensureAnyApiSchema in a migration')
+  return buildSlotPools((pool, index) => SLOT_COLUMNS[pool](index).every(column => columns.has(column)))
 }
 
 export const TYPE_TO_POOL = new Map([

package/plugins/core/lib/anyapi/stored-definition.js

@@ -0,0 +1,109 @@
+// Stored resource definitions keep JSON values only. The paths of values JSON
+// cannot hold (callbacks, class instances, RegExps) are recorded, so a server
+// loading the definition knows which fields need code it may not have.
+
+const isPlainObject = value => {
+  const prototype = Object.getPrototypeOf(value)
+  return prototype === Object.prototype || prototype === null
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ json: any, callbacks: Array<Array<string|number>> }}
+ */
+export const toStoredJson = (value) => {
+  const callbacks = []
+  const visit = (current, path) => {
+    if (current === undefined) return undefined
+    if (current === null || typeof current === 'string' || typeof current === 'boolean') return current
+    if (typeof current === 'number' && Number.isFinite(current)) return current
+    if (Array.isArray(current)) return current.map((item, index) => visit(item, [...path, index]) ?? null)
+    if (typeof current === 'object' && isPlainObject(current)) {
+      const result = {}
+      for (const [key, item] of Object.entries(current)) {
+        const stored = visit(item, [...path, key])
+        if (stored !== undefined) result[key] = stored
+      }
+      return result
+    }
+    callbacks.push(path)
+    return undefined
+  }
+  return { json: visit(value, []), callbacks }
+}
+
+const targetsOf = definition => definition?.oneOf || [definition?.actualField].filter(Boolean)
+
+/**
+ * Combine a resource's declaration with the fields other servers added at run
+ * time. Declared fields win; runtime fields this server added keep its local
+ * callbacks. A field whose callbacks exist in no available code is hidden
+ * rather than served without them, because a missing serializer or setter
+ * would read or write the wrong values.
+ * @param {{ declaration?: Record<string, any>, stored?: Record<string, any> | null, local?: { schema?: Record<string, any>, searchSchema?: Record<string, any> }, fromStorage?: boolean }} input
+ */
+export const composeResourceOptions = ({ declaration = {}, stored = null, local = {}, fromStorage = false }) => {
+  const hidden = new Set()
+  const hiddenFilters = new Set()
+  const declaredSchema = declaration.schema || {}
+  const declaredSearch = declaration.searchSchema || {}
+  const schema = { ...declaredSchema }
+  const searchSchema = { ...declaredSearch }
+
+  if (fromStorage) {
+    for (const path of stored?.callbacks || []) {
+      if (path[0] === 'schema' && path.length > 1) hidden.add(path[1])
+      else if (path[0] === 'searchSchema' && path.length > 1) hiddenFilters.add(path[1])
+      else throw new Error(`Resource option '${path.join('.')}' cannot be loaded from storage; declare the resource in code`)
+    }
+  }
+
+  const runtime = stored?.runtime || {}
+  const needsCode = new Set((runtime.callbacks || []).map(path => `${path[0]}.${path[1]}`))
+  const merge = (kind, target, declared, localDefinitions = {}, hide) => {
+    for (const [name, definition] of Object.entries(runtime[kind] || {})) {
+      if (Object.hasOwn(declared, name)) continue
+      if (Object.hasOwn(localDefinitions, name)) target[name] = localDefinitions[name]
+      else if (needsCode.has(`${kind}.${name}`)) hide.add(name)
+      else target[name] = definition
+    }
+  }
+  merge('schema', schema, declaredSchema, local.schema, hidden)
+  merge('searchSchema', searchSchema, declaredSearch, local.searchSchema, hiddenFilters)
+
+  for (const name of hidden) delete schema[name]
+  for (const [name, definition] of Object.entries(searchSchema)) {
+    if (hiddenFilters.has(name) || targetsOf(definition).some(target => hidden.has(target))) {
+      hiddenFilters.add(name)
+      delete searchSchema[name]
+    }
+  }
+
+  const options = { ...declaration, schema }
+  if (Object.keys(searchSchema).length || declaration.searchSchema) options.searchSchema = searchSchema
+  if (hidden.size && Array.isArray(declaration.sortableFields)) {
+    options.sortableFields = declaration.sortableFields.filter(field => !hidden.has(String(field).replace(/^-/, '')))
+  }
+  return { options, hidden: [...hidden], hiddenFilters: [...hiddenFilters] }
+}
+
+/**
+ * Resources a stored declaration refers to. Polymorphic targets must be
+ * registered before the resource itself; the others can follow.
+ * @param {Record<string, any>} options
+ */
+export const relatedResources = (options = {}) => {
+  const before = new Set()
+  const after = new Set()
+  for (const definition of Object.values(options.schema || {})) {
+    if (definition?.belongsTo) after.add(definition.belongsTo)
+  }
+  for (const [name, definition] of Object.entries(options.relationships || {})) {
+    for (const type of definition?.belongsToPolymorphic?.types || []) before.add(type)
+    if (definition?.type === 'hasMany' || definition?.type === 'manyToMany' || definition?.hasMany) {
+      after.add(definition.target || definition.hasMany || name)
+    }
+    if (definition?.through) after.add(definition.through)
+  }
+  return { before: [...before], after: [...after].filter(name => !before.has(name)) }
+}