@toa.io/norm 1.0.0-alpha.29 → 1.0.0-alpha.291

package/src/.component/validate.js

@@ -1,25 +1,42 @@
-'use strict'
+import path from 'node:path'
 
-const path = require('node:path')
+import { readFileSync } from 'node:fs'
+import { yaml } from '@toa.io/generic'
+import * as schemas from '@toa.io/schemas'
 
-const { load } = require('@toa.io/yaml')
-const { Schema } = require('@toa.io/schema')
+const object = yaml.load(
+  readFileSync(path.resolve(import.meta.dirname, 'schema.yaml'), 'utf8')
+)
+const schema = schemas.schema(object)
 
-const object = load.sync(path.resolve(__dirname, 'schema.yaml'))
-const schema = new Schema(object)
-
-const validate = (manifest) => {
+export const validate = async (manifest) => {
   const error = schema.fit(manifest)
 
-  if (error) throw new Error(error.message)
+  if (error) throw error
 
-  if (manifest.events !== undefined) events(manifest)
+  if (manifest.entity !== undefined) entity(manifest)
+  if (manifest.events !== undefined) await events(manifest)
   if (manifest.receivers !== undefined) receivers(manifest)
 }
 
-const events = (manifest) => {
+/** What only the entity's own declaration can answer: whether a name it uses is one it declares. */
+const entity = (manifest) => {
+  const { properties, required, blank } = manifest.entity
+
+  for (const name of required ?? [])
+    if (properties[name] === undefined)
+      throw new Error(`Entity requires property '${name}', which is not defined`)
+
+  for (const name of Object.keys(blank ?? {}))
+    if (properties[name] === undefined)
+      throw new Error(`Entity blank names property '${name}', which is not defined`)
+}
+
+const events = async (manifest) => {
   for (const [label, event] of Object.entries(manifest.events)) {
-    if (require(event.binding).properties.async !== true) {
+    const { properties } = await import(event.binding)
+
+    if (properties.async !== true) {
       throw new Error(`Event '${label}' binding '${event.binding}' is not async`)
     }
   }
@@ -28,15 +45,17 @@ const events = (manifest) => {
 const receivers = (manifest) => {
   for (const [locator, receiver] of Object.entries(manifest.receivers)) {
     if (manifest.operations?.[receiver.operation] === undefined) {
-      throw new Error(`Receiver '${locator}' refers to undefined operation '${receiver.operation}'`)
+      throw new Error(
+        `Receiver '${locator}' refers to undefined operation '${receiver.operation}'`
+      )
     }
 
     if (!TYPES.has(manifest.operations[receiver.operation].type)) {
-      throw new Error(`Receiver '${locator}' must refer to an operation of one of the allowed types: ${Array.from(TYPES).join(', ')}`)
+      throw new Error(
+        `Receiver '${locator}' must refer to an operation of the allowed types: ${Array.from(TYPES).join(', ')}`
+      )
     }
   }
 }
 
-const TYPES = new Set(['transition', 'effect'])
-
-exports.validate = validate
+const TYPES = new Set(['transition', 'assignment', 'effect', 'unmanaged'])

package/src/.context/.dependencies/connectors.js

@@ -1,14 +1,12 @@
-'use strict'
+import { resolve } from 'node:path'
 
-const { resolve } = require('node:path')
-
-const connectors = (context, extracted) => {
+export const connectors = (context, extracted) => {
   const connectors = {}
 
-  const components = (context.components === undefined
+  const components =
+    (context.components === undefined
       ? extracted
-      : context.components.concat(extracted)
-  ) ?? []
+      : context.components.concat(extracted)) ?? []
 
   for (const component of components) {
     if (component.entity !== undefined) {
@@ -47,5 +45,3 @@ const connectors = (context, extracted) => {
 
   return connectors
 }
-
-exports.connectors = connectors

package/src/.context/.dependencies/describe.js

@@ -1,10 +1,9 @@
-'use strict'
-
 /**
  * @param {toa.norm.Component} component
  * @param {Object} manifest
  * @returns {toa.norm.context.dependencies.Instance}
  */
-const describe = (component, manifest = undefined) => ({ locator: component.locator, manifest })
-
-exports.describe = describe
+export const describe = (component, manifest = undefined) => ({
+  locator: component.locator,
+  manifest
+})

package/src/.context/.dependencies/extensions.js

@@ -1,9 +1,21 @@
-'use strict'
+import { component as load } from '../../component.js'
+import { load as loadDependency } from './load.js'
 
-const extensions = async (context) => {
+export const extensions = async (context) => {
   const extensions = {}
   const components = context.components?.slice() ?? []
-  const extracted = await extractExtensionComponents(components, extensions)
+
+  // an extension a composition runs may be referenced by no component of this context,
+  // and its own components must be extracted all the same
+  const declared = await extractDeclaredServices(context, extensions)
+
+  const extracted = declared.concat(
+    await extractExtensionComponents(
+      components.concat(declared),
+      extensions,
+      context.annotations
+    )
+  )
 
   components.push(...extracted)
 
@@ -20,9 +32,28 @@ const extensions = async (context) => {
   return { extensions, components: extracted }
 }
 
-async function extractExtensionComponents (components, extensions) {
-  const { component: load } = require('../../component')
+async function extractDeclaredServices(context, extensions) {
+  const extracted = []
+
+  for (const composition of context.compositions ?? [])
+    for (const reference of composition.services ?? []) {
+      if (reference in extensions) continue
 
+      try {
+        extracted.push(...(await extract(reference, extensions, context.annotations)))
+      } catch (e) {
+        throw new Error(
+          `Composition '${composition.name}' lists '${reference}', ` +
+            `which cannot be resolved: ${e.message}`,
+          { cause: e }
+        )
+      }
+    }
+
+  return extracted
+}
+
+async function extractExtensionComponents(components, extensions, annotations) {
   const extracted = []
 
   for (const component of components) {
@@ -31,26 +62,37 @@ async function extractExtensionComponents (components, extensions) {
     for (const reference of Object.keys(component.extensions)) {
       if (reference in extensions) continue
 
-      extensions[reference] = []
-
-      const mod = require(reference)
-
-      if (mod.components === undefined) continue
-
-      for (const path of mod.components().paths) {
-        const component = await load(path)
-
-        extracted.push(component)
-      }
+      extracted.push(...(await extract(reference, extensions, annotations)))
     }
   }
 
-  if (extracted.length === 0)
-    return extracted
+  if (extracted.length === 0) return extracted
 
-  const deeper = await extractExtensionComponents(extracted, extensions)
+  const deeper = await extractExtensionComponents(extracted, extensions, annotations)
 
   return extracted.concat(deeper)
 }
 
-exports.extensions = extensions
+/**
+ * The components an extension contributes, if it contributes any.
+ *
+ * @param {string} reference
+ * @param {object} extensions
+ * @param {object} [annotations]
+ * @returns {Promise<Array<toa.norm.Component>>}
+ */
+async function extract(reference, extensions, annotations) {
+  extensions[reference] = []
+
+  const { metadata, module: mod } = await loadDependency(reference)
+
+  if (mod.components === undefined) return []
+
+  // the annotation decides whether an extension contributes components at all
+  const annotation = annotations?.[metadata?.name ?? reference]
+  const extracted = []
+
+  for (const path of mod.components(annotation).paths) extracted.push(await load(path))
+
+  return extracted
+}

package/src/.context/.dependencies/index.js

@@ -1,9 +1,3 @@
-'use strict'
-
-const { connectors } = require('./connectors')
-const { extensions } = require('./extensions')
-const { resolve } = require('./resolve')
-
-exports.connectors = connectors
-exports.extensions = extensions
-exports.resolve = resolve
+export { connectors } from './connectors.js'
+export { extensions } from './extensions.js'
+export { resolve } from './resolve.js'

package/src/.context/.dependencies/load.js

@@ -1,23 +1,28 @@
-'use strict'
+import { createRequire } from 'node:module'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
 
-const { join } = require('node:path')
+// import.meta.resolve takes no paths, and a dependency is named rather than
+// written as a path
+const require = createRequire(import.meta.url)
 
 /**
  * @param {string} path
- * @returns {{ metadata: object, module: object }}
+ * @returns {Promise<{ metadata: object, module: object }>}
  */
-function load (path) {
+export async function load(path) {
   const metadata = loadMetadata(path)
-  const module = loadModule(path)
+  const module = await loadModule(path)
 
   return { metadata, module }
 }
 
-function loadMetadata (reference) {
-  reference = join(reference, 'package.json')
-
+function loadMetadata(reference) {
   try {
-    return require(reference)
+    return JSON.parse(
+      readFileSync(require.resolve(join(reference, 'package.json')), 'utf8')
+    )
   } catch {
     return null
   }
@@ -25,12 +30,10 @@ function loadMetadata (reference) {
 
 /**
  * @param {string} reference
- * @returns {object}
+ * @returns {Promise<object>}
  */
-function loadModule (reference) {
+async function loadModule(reference) {
   const path = require.resolve(reference)
 
-  return require(path)
+  return await import(pathToFileURL(path).href)
 }
-
-exports.load = load

package/src/.context/.dependencies/resolve.js

@@ -1,12 +1,10 @@
-'use strict'
+import { load } from './load.js'
 
-const { load } = require('./load')
-
-const resolve = (references, annotations) => {
+export const resolve = async (references, annotations = {}) => {
   const dependencies = {}
 
   for (const [dependency, components] of Object.entries(references)) {
-    const { metadata, module } = load(dependency)
+    const { metadata, module } = await load(dependency)
     const id = metadata.name
 
     const instances = components.map((component) => ({
@@ -24,7 +22,26 @@ const resolve = (references, annotations) => {
     }
   }
 
+  for (const dependency of Object.keys(annotations)) {
+    if (dependency in dependencies) continue
+
+    // an annotation may be keyed by a dependency id rather than by a module reference
+    const module = await optional(dependency)
+
+    if (module?.standalone === true) dependencies[dependency] = []
+  }
+
   return dependencies
 }
 
-exports.resolve = resolve
+/**
+ * @param {string} reference
+ * @returns {object | null}
+ */
+async function optional(reference) {
+  try {
+    return (await load(reference)).module
+  } catch {
+    return null
+  }
+}

package/src/.context/complete.js

@@ -1,11 +1,9 @@
-'use strict'
-
 /**
  * Completes missing compositions with unused components
  * @param {toa.norm.Context} context
  * @returns {void}
  */
-const complete = (context) => {
+export const complete = (context) => {
   /** @type {Set<string>} */
   const composed = new Set()
 
@@ -29,5 +27,3 @@ const complete = (context) => {
     context.compositions.push({ name, components: [component] })
   }
 }
-
-exports.complete = complete

package/src/.context/dependencies.js

@@ -1,13 +1,9 @@
-'use strict'
+import { connectors, extensions, resolve } from './.dependencies/index.js'
 
-const { connectors, extensions, resolve } = require('./.dependencies')
-
-const dependencies = async (context) => {
+export const dependencies = async (context) => {
   const { extensions: e, components } = await extensions(context)
   const c = connectors(context, components)
   const references = { ...c, ...e }
 
   return resolve(references, context.annotations)
 }
-
-exports.dependencies = dependencies

package/src/.context/dereference.js

@@ -1,16 +1,23 @@
-'use strict'
-
 /**
  * Resolves component IDs within compositions with Component objects
  * @param {toa.norm.Context} context
  * @returns {void}
  */
-const dereference = (context) => {
+export const dereference = (context) => {
   const components = map(context.components)
 
   if (context.compositions !== undefined) {
     for (const composition of context.compositions) {
-      composition.components = composition.components.map((id) => components[id])
+      composition.components = composition.components.map((id) => {
+        const component = components[id]
+
+        if (component === undefined)
+          throw new Error(
+            `Composition '${composition.name}' lists an unknown component '${id}'.`
+          )
+
+        return component
+      })
     }
   }
 }
@@ -27,5 +34,3 @@ const map = (components) => {
 
   return map
 }
-
-exports.dereference = dereference

package/src/.context/expand.js

@@ -1,13 +1,15 @@
-'use strict'
-
-const shortcuts = require('../shortcuts')
+import * as shortcuts from '../shortcuts.js'
 
 /**
  * @param {toa.norm.context.Declaration | object} context
  */
-const expand = (context) => {
+export const expand = (context) => {
   shortcuts.recognize(shortcuts.SHORTCUTS, context, 'annotations')
   shortcuts.recognize(shortcuts.SHORTCUTS, context.annotations)
-}
 
-exports.expand = expand
+  // a composition names a service the way a manifest names an extension, and everything
+  // downstream matches it against a dependency, which is keyed by package reference
+  for (const composition of context.compositions ?? [])
+    if (composition.services !== undefined)
+      composition.services = composition.services.map(shortcuts.resolve)
+}

package/src/.context/index.js

@@ -1,15 +1,6 @@
-'use strict'
-
-const { complete } = require('./complete')
-const { dependencies } = require('./dependencies')
-const { dereference } = require('./dereference')
-const { expand } = require('./expand')
-const { normalize } = require('./normalize')
-const { validate } = require('./validate')
-
-exports.complete = complete
-exports.dependencies = dependencies
-exports.dereference = dereference
-exports.expand = expand
-exports.normalize = normalize
-exports.validate = validate
+export { complete } from './complete.js'
+export { dependencies } from './dependencies.js'
+export { dereference } from './dereference.js'
+export { expand } from './expand.js'
+export { normalize } from './normalize.js'
+export { validate } from './validate.js'

package/src/.context/normalize.js

@@ -1,11 +1,9 @@
-'use strict'
+import * as runtime from '@toa.io/runtime'
 
 /**
  * @param {toa.norm.context.Declaration | Object} context
  */
-const normalize = (context) => {
-  const runtime = require('@toa.io/runtime')
-
+export const normalize = (context) => {
   if (context.runtime === undefined) context.runtime = { version: runtime.version }
   if (typeof context.runtime === 'string') context.runtime = { version: context.runtime }
 
@@ -14,6 +12,10 @@ const normalize = (context) => {
   }
 
   if (typeof context.registry === 'string') context.registry = { base: context.registry }
+
+  // what an image is built for, where the Context does not say; `~` says the runner's own
+  if (context.registry !== undefined && context.registry.platforms === undefined)
+    context.registry.platforms = PLATFORMS
 }
 
-exports.normalize = normalize
+const PLATFORMS = ['linux/amd64', 'linux/arm/v7', 'linux/arm64']

package/src/.context/schema.yaml

@@ -6,12 +6,10 @@ properties:
   version:
     type: string
   name:
-    $ref: 'definitions#/definitions/label'
-  description:
     type: string
-  packages:
+    pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(-([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))*$
+  description:
     type: string
-    default: components/*
   build:
     type: object
     properties:
@@ -42,28 +40,151 @@ properties:
         minItems: 1
         items:
           type: string
-        default:
-          - linux/amd64
-          - linux/arm/v7
-          - linux/arm64
-    required: [base, platforms]
+      services:
+        description: >
+          Where an extension service's image comes from. `build`, which absence means,
+          builds it into `registry.base` the way a composition is built. `published` uses
+          the image the extension ships, which is neither built nor pushed.
+        enum: [build, published]
+    # required: [base]
   compositions:
     type: array
     minItems: 1
     items:
       type: object
+      additionalProperties: false
       properties:
         name:
-          $ref: 'definitions#/definitions/token'
+          type: string
+          pattern: ^[a-zA-Z]([a-zA-Z0-9]{1,31})?$
         image:
           type: string
         components:
           type: array
           minItems: 1
           items:
-            $ref: 'definitions#/definitions/locator'
+            type: string
+            pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(\.([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))$
+        services:
+          description: >
+            Extension services this composition runs in its own pod, named by shortcut or
+            package reference, rather than each as its own deployment. A service listed here
+            keeps its own Service and Ingress; what changes is the pod they select.
+          type: array
+          minItems: 1
+          items:
+            type: string
+            minLength: 1
+        resources:
+          $ref: '#/definitions/resources'
       required: [name, components]
+  atomicity:
+    description: >
+      Where the replicas of a component decide together: who owns what, what they have spent
+      between them, and which of them holds a name while it works. Absent or unreachable,
+      nothing is owned and whoever asked stands down; metering and locking are refused.
+    type: object
+    additionalProperties: false
+    properties:
+      redis:
+        description: >
+          One address, or an odd number of them — independent servers, not the nodes of a
+          cluster. A lock is taken on a quorum of them; the registry and the meter use the
+          first, because each of them counts on a single key.
+        oneOf:
+          - type: string
+          - type: array
+            minItems: 1
+            items:
+              type: string
+      interval:
+        description: >
+          How often a replica registers, in milliseconds. It has to comfortably exceed a round
+          trip to Redis and the jitter around it, and nothing is owned until two consecutive
+          intervals have agreed — so this is also how long a replica owns nothing after it
+          joins, restarts or stalls.
+        type: integer
+        minimum: 100
+    required: [redis]
+  outbox:
+    type: object
+    additionalProperties: false
+    properties:
+      interval:
+        description: >
+          The pump's cycle, in milliseconds. One cycle reads what is unpublished, publishes it
+          and marks it. In steady state it reads nothing, so this is budgeted for how fast a rare
+          failure heals rather than for how fast events flow.
+        type: integer
+        minimum: 100
+      batch:
+        description: >
+          How many rows one read brings back. A page that comes back full is read on from,
+          so this bounds a read rather than a cycle.
+        type: integer
+        minimum: 1
+      retention:
+        description: >
+          Seconds a published row is kept as a change log before it expires. A row that never
+          made it out has no expiry and is never reaped.
+        type: integer
+        minimum: 0
+  events:
+    description: >
+      Events of this context that are consumed outside it. An event is otherwise published only
+      where something in the context receives it.
+    type: array
+    items:
+      type: string
+      pattern: ^[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*$
+  mono:
+    type: object
+    additionalProperties: false
+    properties:
+      replicas:
+        type: integer
+        minimum: 1
+      resources:
+        $ref: '#/definitions/resources'
+  ingress:
+    type: object
+    additionalProperties: false
+    properties:
+      hosts:
+        type: array
+        minItems: 1
+        items:
+          type: string
+      class:
+        type: string
+      annotations:
+        type: object
+      default:
+        type: boolean
   annotations:
     type: object
+  resources:
+    $ref: '#/definitions/resources'
 required: [runtime, name, registry]
 additionalProperties: false
+
+definitions:
+  # What a deployment may take. `null` deploys it without any, which is a decision
+  # rather than an omission — every deployment has to make one.
+  resources:
+    type: object
+    nullable: true
+    additionalProperties: false
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2