@toa.io/norm 1.0.0-alpha.284 → 1.0.0-alpha.286

package/src/.component/validate.js

@@ -4,7 +4,9 @@ import { readFileSync } from 'node:fs'
 import { yaml } from '@toa.io/generic'
 import * as schemas from '@toa.io/schemas'
 
-const object = yaml.load(readFileSync(path.resolve(import.meta.dirname, 'schema.yaml'), 'utf8'))
+const object = yaml.load(
+  readFileSync(path.resolve(import.meta.dirname, 'schema.yaml'), 'utf8')
+)
 const schema = schemas.schema(object)
 
 export const validate = async (manifest) => {
@@ -12,10 +14,24 @@ export const validate = async (manifest) => {
 
   if (error) throw error
 
+  if (manifest.entity !== undefined) entity(manifest)
   if (manifest.events !== undefined) await events(manifest)
   if (manifest.receivers !== undefined) receivers(manifest)
 }
 
+/** What only the entity's own declaration can answer: whether a name it uses is one it declares. */
+const entity = (manifest) => {
+  const { properties, required, blank } = manifest.entity
+
+  for (const name of required ?? [])
+    if (properties[name] === undefined)
+      throw new Error(`Entity requires property '${name}', which is not defined`)
+
+  for (const name of Object.keys(blank ?? {}))
+    if (properties[name] === undefined)
+      throw new Error(`Entity blank names property '${name}', which is not defined`)
+}
+
 const events = async (manifest) => {
   for (const [label, event] of Object.entries(manifest.events)) {
     const { properties } = await import(event.binding)
@@ -29,11 +45,15 @@ const events = async (manifest) => {
 const receivers = (manifest) => {
   for (const [locator, receiver] of Object.entries(manifest.receivers)) {
     if (manifest.operations?.[receiver.operation] === undefined) {
-      throw new Error(`Receiver '${locator}' refers to undefined operation '${receiver.operation}'`)
+      throw new Error(
+        `Receiver '${locator}' refers to undefined operation '${receiver.operation}'`
+      )
     }
 
     if (!TYPES.has(manifest.operations[receiver.operation].type)) {
-      throw new Error(`Receiver '${locator}' must refer to an operation of the allowed types: ${Array.from(TYPES).join(', ')}`)
+      throw new Error(
+        `Receiver '${locator}' must refer to an operation of the allowed types: ${Array.from(TYPES).join(', ')}`
+      )
     }
   }
 }

package/src/.context/.dependencies/connectors.js

@@ -3,10 +3,10 @@ import { resolve } from 'node:path'
 export const connectors = (context, extracted) => {
   const connectors = {}
 
-  const components = (context.components === undefined
+  const components =
+    (context.components === undefined
       ? extracted
-      : context.components.concat(extracted)
-  ) ?? []
+      : context.components.concat(extracted)) ?? []
 
   for (const component of components) {
     if (component.entity !== undefined) {

package/src/.context/.dependencies/describe.js

@@ -3,4 +3,7 @@
  * @param {Object} manifest
  * @returns {toa.norm.context.dependencies.Instance}
  */
-export const describe = (component, manifest = undefined) => ({ locator: component.locator, manifest })
+export const describe = (component, manifest = undefined) => ({
+  locator: component.locator,
+  manifest
+})

package/src/.context/.dependencies/extensions.js

@@ -10,7 +10,12 @@ export const extensions = async (context) => {
   const declared = await extractDeclaredServices(context, extensions)
 
   const extracted = declared.concat(
-    await extractExtensionComponents(components.concat(declared), extensions, context.annotations))
+    await extractExtensionComponents(
+      components.concat(declared),
+      extensions,
+      context.annotations
+    )
+  )
 
   components.push(...extracted)
 
@@ -27,7 +32,7 @@ export const extensions = async (context) => {
   return { extensions, components: extracted }
 }
 
-async function extractDeclaredServices (context, extensions) {
+async function extractDeclaredServices(context, extensions) {
   const extracted = []
 
   for (const composition of context.compositions ?? [])
@@ -35,18 +40,20 @@ async function extractDeclaredServices (context, extensions) {
       if (reference in extensions) continue
 
       try {
-        extracted.push(...await extract(reference, extensions, context.annotations))
+        extracted.push(...(await extract(reference, extensions, context.annotations)))
       } catch (e) {
-        throw new Error(`Composition '${composition.name}' lists '${reference}', ` +
-          `which cannot be resolved: ${e.message}`, { cause: e })
+        throw new Error(
+          `Composition '${composition.name}' lists '${reference}', ` +
+            `which cannot be resolved: ${e.message}`,
+          { cause: e }
+        )
       }
     }
 
   return extracted
 }
 
-async function extractExtensionComponents (components, extensions, annotations) {
-
+async function extractExtensionComponents(components, extensions, annotations) {
   const extracted = []
 
   for (const component of components) {
@@ -55,12 +62,11 @@ async function extractExtensionComponents (components, extensions, annotations)
     for (const reference of Object.keys(component.extensions)) {
       if (reference in extensions) continue
 
-      extracted.push(...await extract(reference, extensions, annotations))
+      extracted.push(...(await extract(reference, extensions, annotations)))
     }
   }
 
-  if (extracted.length === 0)
-    return extracted
+  if (extracted.length === 0) return extracted
 
   const deeper = await extractExtensionComponents(extracted, extensions, annotations)
 
@@ -75,7 +81,7 @@ async function extractExtensionComponents (components, extensions, annotations)
  * @param {object} [annotations]
  * @returns {Promise<Array<toa.norm.Component>>}
  */
-async function extract (reference, extensions, annotations) {
+async function extract(reference, extensions, annotations) {
   extensions[reference] = []
 
   const { metadata, module: mod } = await loadDependency(reference)
@@ -86,8 +92,7 @@ async function extract (reference, extensions, annotations) {
   const annotation = annotations?.[metadata?.name ?? reference]
   const extracted = []
 
-  for (const path of mod.components(annotation).paths)
-    extracted.push(await load(path))
+  for (const path of mod.components(annotation).paths) extracted.push(await load(path))
 
   return extracted
 }

package/src/.context/.dependencies/load.js

@@ -11,16 +11,18 @@ const require = createRequire(import.meta.url)
  * @param {string} path
  * @returns {Promise<{ metadata: object, module: object }>}
  */
-export async function load (path) {
+export async function load(path) {
   const metadata = loadMetadata(path)
   const module = await loadModule(path)
 
   return { metadata, module }
 }
 
-function loadMetadata (reference) {
+function loadMetadata(reference) {
   try {
-    return JSON.parse(readFileSync(require.resolve(join(reference, 'package.json')), 'utf8'))
+    return JSON.parse(
+      readFileSync(require.resolve(join(reference, 'package.json')), 'utf8')
+    )
   } catch {
     return null
   }
@@ -30,7 +32,7 @@ function loadMetadata (reference) {
  * @param {string} reference
  * @returns {Promise<object>}
  */
-async function loadModule (reference) {
+async function loadModule(reference) {
   const path = require.resolve(reference)
 
   return await import(pathToFileURL(path).href)

package/src/.context/.dependencies/resolve.js

@@ -38,7 +38,7 @@ export const resolve = async (references, annotations = {}) => {
  * @param {string} reference
  * @returns {object | null}
  */
-async function optional (reference) {
+async function optional(reference) {
   try {
     return (await load(reference)).module
   } catch {

package/src/.context/dereference.js

@@ -12,7 +12,9 @@ export const dereference = (context) => {
         const component = components[id]
 
         if (component === undefined)
-          throw new Error(`Composition '${composition.name}' lists an unknown component '${id}'.`)
+          throw new Error(
+            `Composition '${composition.name}' lists an unknown component '${id}'.`
+          )
 
         return component
       })

package/src/.context/normalize.js

@@ -12,4 +12,9 @@ export const normalize = (context) => {
   }
 
   if (typeof context.registry === 'string') context.registry = { base: context.registry }
+
+  // what an image is built for, where the Context does not say
+  if (context.registry !== undefined) context.registry.platforms ??= PLATFORMS
 }
+
+const PLATFORMS = ['linux/amd64', 'linux/arm/v7', 'linux/arm64']

package/src/.context/schema.yaml

@@ -40,10 +40,12 @@ properties:
         minItems: 1
         items:
           type: string
-        default:
-          - linux/amd64
-          - linux/arm/v7
-          - linux/arm64
+      services:
+        description: >
+          Where an extension service's image comes from. `build`, which absence means,
+          builds it into `registry.base` the way a composition is built. `published` uses
+          the image the extension ships, which is neither built nor pushed.
+        enum: [build, published]
     # required: [base]
   compositions:
     type: array

package/src/component.js

@@ -5,7 +5,17 @@ import { yaml as jsyaml } from '@toa.io/generic'
 import { find } from '@toa.io/generic'
 import { Locator } from '@toa.io/core'
 
-import { expand, merge, validate, collapse, dereference, defaults, normalize, extensions } from './.component/index.js'
+import {
+  expand,
+  merge,
+  migrations,
+  validate,
+  collapse,
+  dereference,
+  defaults,
+  normalize,
+  extensions
+} from './.component/index.js'
 
 export const component = async (path) => {
   const manifest = await load(path)
@@ -23,7 +33,7 @@ const load = async (path, base, proto = false) => {
   if (base !== undefined) path = find(path, base, MANIFEST)
 
   const file = join(path, MANIFEST)
-  const manifest = await read(file) ?? {}
+  const manifest = (await read(file)) ?? {}
 
   manifest.path = path
 
@@ -31,6 +41,7 @@ const load = async (path, base, proto = false) => {
   await expand(manifest)
 
   await merge(path, manifest)
+  await migrations(path, manifest)
 
   if (manifest.prototype !== null) {
     const prototype = await load(manifest.prototype, path, true)
@@ -46,8 +57,6 @@ const load = async (path, base, proto = false) => {
 
 const MANIFEST = 'manifest.toa.yaml'
 
-
-
 /**
  * Reads a YAML file, resolving anchors into distinct objects so that
  * mutating one node cannot reach another.
@@ -55,7 +64,7 @@ const MANIFEST = 'manifest.toa.yaml'
  * @param {string} path
  * @return {Promise<object>}
  */
-async function read (path) {
+async function read(path) {
   const object = jsyaml.load(await readFile(path, 'utf8'))
 
   return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))

package/src/context.js

@@ -6,7 +6,14 @@ import { yaml as jsyaml } from '@toa.io/generic'
 
 import { component } from './component.js'
 
-import { dependencies, normalize, complete, dereference, expand, validate } from './.context/index.js'
+import {
+  dependencies,
+  normalize,
+  complete,
+  dereference,
+  expand,
+  validate
+} from './.context/index.js'
 
 export const context = async (root, environment = process.env.TOA_ENV) => {
   const path = resolve(root, CONTEXT)
@@ -34,11 +41,8 @@ export const context = async (root, environment = process.env.TOA_ENV) => {
 const CONTEXT = 'context.toa.yaml'
 const COMPONENTS = 'components/*'
 
-
 const GLOB = { onlyDirectories: true, absolute: true }
 
-
-
 /**
  * Reads a YAML file, resolving anchors into distinct objects so that
  * mutating one node cannot reach another.
@@ -46,7 +50,7 @@ const GLOB = { onlyDirectories: true, absolute: true }
  * @param {string} path
  * @return {Promise<object>}
  */
-async function read (path) {
+async function read(path) {
   const object = jsyaml.load(await readFile(path, 'utf8'))
 
   return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))

package/src/entity.js

@@ -0,0 +1,15 @@
+/**
+ * What a component declares about its records is a property map and a list of the properties a
+ * whole one has. A validator wants a schema, so it is assembled here rather than declared: one
+ * that a stored record must fit, and one for a changeset, which is whatever subset of the same
+ * properties an assignment writes.
+ */
+export const schema = (entity) =>
+  entity.required === undefined
+    ? changeset(entity)
+    : { type: 'object', properties: entity.properties, required: entity.required }
+
+export const changeset = (entity) => ({
+  type: 'object',
+  properties: entity.properties
+})

package/src/index.js

@@ -1,3 +1,4 @@
 export { context } from './context.js'
 export { component } from './component.js'
 export * as shortcuts from './shortcuts.js'
+export * as entity from './entity.js'

package/src/shortcuts.js

@@ -41,7 +41,6 @@ export const SHORTCUTS = {
   node: '@toa.io/bridges.node',
   bash: '@toa.io/bridges.bash',
   mongodb: '@toa.io/storages.mongodb',
-  sql: '@toa.io/storages.sql',
   queues: '@toa.io/storages.queues',
   exposition: '@toa.io/extensions.exposition',
   realtime: '@toa.io/extensions.realtime',

package/test/component/collapse.fixtures.js

@@ -3,51 +3,45 @@ import { generate } from 'randomstring'
 const entity = {
   manifest: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            type: 'string'
-          },
-          bar: {
-            type: 'integer'
-          }
+      properties: {
+        foo: {
+          type: 'string'
         },
-        required: ['foo']
-      }
+        bar: {
+          type: 'integer'
+        }
+      },
+      required: ['foo']
     }
   },
   prototype: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            default: 'ok'
-          },
-          baz: {
-            type: 'boolean'
-          }
+      properties: {
+        foo: {
+          minLength: 1
         },
-        required: ['baz']
-      }
+        baz: {
+          type: 'boolean'
+        }
+      },
+      required: ['baz']
     }
   },
   result: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            type: 'string',
-            default: 'ok'
-          },
-          bar: {
-            type: 'integer'
-          },
-          baz: {
-            type: 'boolean'
-          }
+      properties: {
+        foo: {
+          type: 'string',
+          minLength: 1
         },
-        required: ['foo', 'baz']
-      }
+        bar: {
+          type: 'integer'
+        },
+        baz: {
+          type: 'boolean'
+        }
+      },
+      required: ['foo', 'baz']
     }
   }
 }

package/test/component/collapse.test.js

@@ -37,6 +37,27 @@ describe('entity', () => {
     collapse(manifest, samples.entity.prototype)
     assert.deepStrictEqual(manifest, samples.entity.result)
   })
+
+  it('should not inherit migrations', () => {
+    const manifest = { entity: { migrations: [{ id: '0002', steps: [] }] } }
+    const prototype = { entity: { migrations: [{ id: '0001', steps: [] }] } }
+
+    collapse(manifest, prototype)
+
+    assert.deepStrictEqual(
+      manifest.entity.migrations.map(({ id }) => id),
+      ['0002']
+    )
+  })
+
+  it('should leave a component with no migrations without any', () => {
+    const manifest = {}
+    const prototype = { entity: { migrations: [{ id: '0001', steps: [] }] } }
+
+    collapse(manifest, prototype)
+
+    assert.strictEqual(manifest.entity?.migrations, undefined)
+  })
 })
 
 it('should ignore bindings', () => {

package/test/component/dependencies.test.js

@@ -21,15 +21,15 @@ for (const [_, reference] of [
   ['package id', '@toa.io/norm'],
   ['relative path', '../../']
 ])
-   describe(`${_}`, () => {
-  it('should resolve storage', async () => {
-    component.entity = { storage: reference, schema: {} }
+  describe(`${_}`, () => {
+    it('should resolve storage', async () => {
+      component.entity = { storage: reference, schema: {} }
 
-    dependencies(component)
+      dependencies(component)
 
-    assert.deepStrictEqual(component.entity.storage, NORM)
+      assert.deepStrictEqual(component.entity.storage, NORM)
+    })
   })
-})
 
 it('should resolve toa packages', async () => {
   component.entity = { storage: '@toa.io/core', schema: {} }

package/test/component/dereference.fixtures.js

@@ -1,14 +1,12 @@
 export const source = {
   entity: {
-    schema: {
-      properties: {
-        foo: {
-          type: 'string'
-        },
-        bar: {
-          type: 'string',
-          default: '.foo'
-        }
+    properties: {
+      foo: {
+        type: 'string'
+      },
+      bar: {
+        type: 'string',
+        default: '.foo'
       }
     }
   },
@@ -66,14 +64,12 @@ export const source = {
 
 export const target = {
   entity: {
-    schema: {
-      properties: {
-        foo: {
-          type: 'string'
-        },
-        bar: {
-          type: 'string'
-        }
+    properties: {
+      foo: {
+        type: 'string'
+      },
+      bar: {
+        type: 'string'
       }
     }
   },

package/test/component/dereference.test.js

@@ -20,13 +20,22 @@ it('should dereference', () => {
 
 it('should throw on invalid schema reference', () => {
   source.operations.transit.output.properties.baz = { type: 'string', default: '.' }
-  assert.throws(() => dereference(source), (error) => /is not defined/.test(error.message))
+  assert.throws(
+    () => dereference(source),
+    (error) => /is not defined/.test(error.message)
+  )
 
   source.operations.transit.output.properties.baz = { type: 'string', default: '.baz' }
-  assert.throws(() => dereference(source), (error) => /is not defined/.test(error.message))
+  assert.throws(
+    () => dereference(source),
+    (error) => /is not defined/.test(error.message)
+  )
 })
 
 it('should throw on invalid forwarding', () => {
   source.operations.create.forward = 'foo'
-  assert.throws(() => dereference(source), (error) => /is not defined/.test(error.message))
+  assert.throws(
+    () => dereference(source),
+    (error) => /is not defined/.test(error.message)
+  )
 })

package/test/component/dummies/migrations/conflicting/migrations/0001-one.yaml

@@ -0,0 +1,2 @@
+- delete:
+    filter: {}

package/test/component/dummies/migrations/conflicting/migrations/0001-one.yml

@@ -0,0 +1,2 @@
+- delete:
+    filter: {}

package/test/component/dummies/migrations/malformed/migrations/0001-one.yaml

@@ -0,0 +1 @@
+index: not a list

package/test/component/dummies/migrations/ordered/migrations/0001-first.yml

@@ -0,0 +1,4 @@
+- index:
+    name: index_a
+    keys:
+      a: asc

package/test/component/dummies/migrations/ordered/migrations/0002-second.yaml

@@ -0,0 +1,4 @@
+- index:
+    name: index_b
+    keys:
+      b: asc

package/test/component/dummies/migrations/ordered/migrations/0010-third.json

@@ -0,0 +1 @@
+[{ "delete": { "filter": {} } }]

package/test/component/dummies/migrations/stateless/migrations/0001-one.yaml

@@ -0,0 +1,2 @@
+- delete:
+    filter: {}

package/test/component/migrations.test.js

@@ -0,0 +1,62 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { join } from 'node:path'
+
+import { migrations } from '../../src/.component/index.js'
+
+const DUMMIES = join(import.meta.dirname, 'dummies/migrations')
+
+const entity = () => ({ entity: { schema: { properties: {} } } })
+
+describe('migrations', () => {
+  it('should read a directory of migrations', async () => {
+    const manifest = entity()
+
+    await migrations(join(DUMMIES, 'ordered'), manifest)
+
+    assert.deepEqual(
+      manifest.entity.migrations.map(({ id }) => id),
+      ['0001-first', '0002-second', '0010-third']
+    )
+  })
+
+  it('should read whatever extension each was written with', async () => {
+    const manifest = entity()
+
+    await migrations(join(DUMMIES, 'ordered'), manifest)
+
+    assert.deepEqual(manifest.entity.migrations[0].steps, [
+      { index: { name: 'index_a', keys: { a: 'asc' } } }
+    ])
+    assert.deepEqual(manifest.entity.migrations[2].steps, [{ delete: { filter: {} } }])
+  })
+
+  it('should leave a component with no migrations alone', async () => {
+    const manifest = entity()
+
+    await migrations(join(DUMMIES, 'nothing-here'), manifest)
+
+    assert.equal(manifest.entity.migrations, undefined)
+  })
+
+  it('should refuse two files resolving to one id', async () => {
+    await assert.rejects(
+      migrations(join(DUMMIES, 'conflicting'), entity()),
+      /has more than one migrations\/0001-one/
+    )
+  })
+
+  it('should refuse migrations on a component that stores nothing', async () => {
+    await assert.rejects(
+      migrations(join(DUMMIES, 'stateless'), {}),
+      /declares migrations but stores nothing/
+    )
+  })
+
+  it('should refuse a migration that is not a list of steps', async () => {
+    await assert.rejects(
+      migrations(join(DUMMIES, 'malformed'), entity()),
+      /is not a list of steps/
+    )
+  })
+})