@toa.io/norm 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/.context/.dependencies/resolve.js

@@ -1,13 +1,10 @@
-'use strict'
+import { definition } from '../../definition.js'
 
-const { load } = require('./load')
-
-const resolve = (references, annotations) => {
+export const resolve = async (references, annotations = {}, context = {}) => {
   const dependencies = {}
 
   for (const [dependency, components] of Object.entries(references)) {
-    const { metadata, module } = load(dependency)
-    const id = metadata.name
+    const { name: id, module } = await definition(dependency)
 
     const instances = components.map((component) => ({
       locator: component.locator,
@@ -22,9 +19,49 @@ const resolve = (references, annotations) => {
     if (annotation !== undefined && module.annotation !== undefined) {
       annotations[id] = module.annotation(annotation, instances)
     }
+
+    // what a declaration costs to install, read from the definition rather than from the
+    // package: a deploy installs it, and nothing that declares nothing carries it. A component
+    // is asked for its own instance; a composition that runs the package's service is asked
+    // with none, because what a service brings is not a component's to declare
+    if (module.installs !== undefined) {
+      for (const instance of instances)
+        Object.assign(
+          (instance.component.packages ??= {}),
+          module.installs(instance, annotations[id])
+        )
+
+      const service = module.installs(undefined, annotations[id])
+
+      // mono runs every service in one process, so it takes every service's packages
+      Object.assign((context.packages ??= {}), service)
+
+      for (const composition of context.compositions ?? [])
+        if (composition.services?.includes(dependency) === true)
+          Object.assign((composition.packages ??= {}), service)
+    }
+  }
+
+  for (const dependency of Object.keys(annotations)) {
+    if (dependency in dependencies) continue
+
+    // an annotation may be keyed by a dependency id rather than by a module reference
+    const module = await optional(dependency)
+
+    if (module?.standalone === true) dependencies[dependency] = []
   }
 
   return dependencies
 }
 
-exports.resolve = resolve
+/**
+ * @param {string} reference
+ * @returns {object | null}
+ */
+async function optional(reference) {
+  try {
+    return (await definition(reference)).module
+  } catch {
+    return null
+  }
+}

package/src/.context/complete.js

@@ -1,11 +1,9 @@
-'use strict'
-
 /**
  * Completes missing compositions with unused components
  * @param {toa.norm.Context} context
  * @returns {void}
  */
-const complete = (context) => {
+export const complete = (context) => {
   /** @type {Set<string>} */
   const composed = new Set()
 
@@ -29,5 +27,3 @@ const complete = (context) => {
     context.compositions.push({ name, components: [component] })
   }
 }
-
-exports.complete = complete

package/src/.context/converge.js

@@ -0,0 +1,31 @@
+/**
+ * A context that declares convergence converges every component that stores anything, and no
+ * manifest says so. An extension is loaded, deployed and hooked because a component's
+ * `extensions` names it, so this is where they come to name it.
+ *
+ * **Including the components an extension ships**, which is why this runs where they are
+ * known rather than over what the context declares: an application deployed as two regions
+ * whose identity does not converge is one where a user registered in the first does not
+ * exist in the second.
+ *
+ * There is nothing per-component to normalize and nothing to opt out of: a region is
+ * infrastructure, and what a component stores is not its business. What is genuinely owned by
+ * one region — which delayed calls this one makes — is said where that is decided, in
+ * `cadence.regions`, and not by holding the records back.
+ *
+ * @param {toa.norm.Context} context
+ * @param {toa.norm.Component[]} components every component the context deploys
+ * @returns {void}
+ */
+export const converge = (context, components) => {
+  if (context.annotations?.[REFERENCE] === undefined) return
+
+  for (const component of components) {
+    if (component.entity === undefined) continue
+
+    component.extensions ??= {}
+    component.extensions[REFERENCE] ??= null
+  }
+}
+
+const REFERENCE = '@toa.io/extensions.convergence'

package/src/.context/dependencies.js

@@ -1,13 +1,50 @@
-'use strict'
+import { connectors, extensions, resolve } from './.dependencies/index.js'
 
-const { connectors, extensions, resolve } = require('./.dependencies')
+export const dependencies = async (context) => {
+  const { extensions: e, components, contributed } = await extensions(context)
+
+  unmanaged(context, contributed)
 
-const dependencies = async (context) => {
-  const { extensions: e, components } = await extensions(context)
   const c = connectors(context, components)
   const references = { ...c, ...e }
 
-  return resolve(references, context.annotations)
+  return resolve(references, context.annotations, context)
 }
 
-exports.dependencies = dependencies
+/**
+ * Marks as evicted what only evicted components require: the components an extension brings,
+ * where nothing a deployment carries references the extension and no composition runs it. A
+ * storage or a binding needs no mark of its own — its instances are components, and a
+ * deployment reads theirs.
+ *
+ * @param {toa.norm.Context} context
+ * @param {Map<string, toa.norm.Component[]>} contributed
+ * @returns {void}
+ */
+function unmanaged(context, contributed) {
+  const components = context.components ?? []
+
+  if (!components.some((component) => component.evicted === true)) return
+
+  const required = new Set()
+  const pending = components.filter((component) => component.evicted !== true)
+
+  // what a composition runs is deployed for being listed, whoever references it
+  for (const composition of context.compositions ?? [])
+    for (const reference of composition.services ?? [])
+      pending.push(...(contributed.get(reference) ?? []))
+
+  while (pending.length > 0) {
+    const component = pending.pop()
+
+    if (required.has(component)) continue
+
+    required.add(component)
+
+    for (const reference of Object.keys(component.extensions ?? {}))
+      pending.push(...(contributed.get(reference) ?? []))
+  }
+
+  for (const brought of contributed.values())
+    for (const component of brought) if (!required.has(component)) component.evicted = true
+}

package/src/.context/dereference.js

@@ -1,16 +1,23 @@
-'use strict'
-
 /**
  * Resolves component IDs within compositions with Component objects
  * @param {toa.norm.Context} context
  * @returns {void}
  */
-const dereference = (context) => {
+export const dereference = (context) => {
   const components = map(context.components)
 
   if (context.compositions !== undefined) {
     for (const composition of context.compositions) {
-      composition.components = composition.components.map((id) => components[id])
+      composition.components = composition.components.map((id) => {
+        const component = components[id]
+
+        if (component === undefined)
+          throw new Error(
+            `Composition '${composition.name}' lists an unknown component '${id}'.`
+          )
+
+        return component
+      })
     }
   }
 }
@@ -27,5 +34,3 @@ const map = (components) => {
 
   return map
 }
-
-exports.dereference = dereference

package/src/.context/evict.js

@@ -0,0 +1,38 @@
+/**
+ * Marks what the context evicts. An evicted component stays in the context like any other — it
+ * is called, its types are written, what it receives is published — and only a deployment
+ * leaves it out, because something else deploys it. What only evicted components require is
+ * marked once the extensions are extracted, by `dependencies`.
+ *
+ * An evicted service is taken off the compositions that list it, so that none of them runs it.
+ *
+ * @param {toa.norm.Context} context
+ * @returns {void}
+ */
+export const evict = (context) => {
+  if (context.evicted === undefined) return
+
+  const components = new Set(context.evicted.components ?? [])
+  const services = new Set(context.evicted.services ?? [])
+
+  if (components.size > 0) {
+    const present = new Set(context.components.map((component) => component.locator.id))
+
+    for (const id of components)
+      if (!present.has(id))
+        throw new Error(`'evicted' names an unknown component '${id}'.`)
+
+    for (const component of context.components)
+      if (components.has(component.locator.id)) component.evicted = true
+  }
+
+  if (context.compositions === undefined) return
+
+  for (const composition of context.compositions) {
+    if (composition.services === undefined) continue
+
+    composition.services = composition.services.filter((reference) => !services.has(reference))
+
+    if (composition.services.length === 0) delete composition.services
+  }
+}

package/src/.context/expand.js

@@ -1,13 +1,18 @@
-'use strict'
-
-const shortcuts = require('../shortcuts')
+import * as shortcuts from '../shortcuts.js'
 
 /**
  * @param {toa.norm.context.Declaration | object} context
  */
-const expand = (context) => {
+export const expand = (context) => {
   shortcuts.recognize(shortcuts.SHORTCUTS, context, 'annotations')
   shortcuts.recognize(shortcuts.SHORTCUTS, context.annotations)
-}
 
-exports.expand = expand
+  // a composition names a service the way a manifest names an extension, and everything
+  // downstream matches it against a dependency, which is keyed by package reference
+  for (const composition of context.compositions ?? [])
+    if (composition.services !== undefined)
+      composition.services = composition.services.map(shortcuts.resolve)
+
+  if (context.evicted?.services !== undefined)
+    context.evicted.services = context.evicted.services.map(shortcuts.resolve)
+}

package/src/.context/index.js

@@ -1,15 +1,8 @@
-'use strict'
-
-const { complete } = require('./complete')
-const { dependencies } = require('./dependencies')
-const { dereference } = require('./dereference')
-const { expand } = require('./expand')
-const { normalize } = require('./normalize')
-const { validate } = require('./validate')
-
-exports.complete = complete
-exports.dependencies = dependencies
-exports.dereference = dereference
-exports.expand = expand
-exports.normalize = normalize
-exports.validate = validate
+export { complete } from './complete.js'
+export { converge } from './converge.js'
+export { dependencies } from './dependencies.js'
+export { dereference } from './dereference.js'
+export { evict } from './evict.js'
+export { expand } from './expand.js'
+export { normalize } from './normalize.js'
+export { validate } from './validate.js'

package/src/.context/normalize.js

@@ -1,19 +1,21 @@
-'use strict'
+import * as definitions from '@toa.io/definitions'
 
 /**
  * @param {toa.norm.context.Declaration | Object} context
  */
-const normalize = (context) => {
-  const runtime = require('@toa.io/runtime')
-
-  if (context.runtime === undefined) context.runtime = { version: runtime.version }
+export const normalize = (context) => {
+  if (context.runtime === undefined) context.runtime = { version: definitions.version }
   if (typeof context.runtime === 'string') context.runtime = { version: context.runtime }
 
   if (context.runtime.version === undefined || context.runtime.version === '.') {
-    context.runtime.version = runtime.version
+    context.runtime.version = definitions.version
   }
 
   if (typeof context.registry === 'string') context.registry = { base: context.registry }
+
+  // what an image is built for, where the Context does not say; `~` says the runner's own
+  if (context.registry !== undefined && context.registry.platforms === undefined)
+    context.registry.platforms = PLATFORMS
 }
 
-exports.normalize = normalize
+const PLATFORMS = ['linux/amd64', 'linux/arm/v7', 'linux/arm64']

package/src/.context/schema.yaml

@@ -6,12 +6,10 @@ properties:
   version:
     type: string
   name:
-    $ref: 'definitions#/definitions/label'
-  description:
     type: string
-  packages:
+    pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(-([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))*$
+  description:
     type: string
-    default: components/*
   build:
     type: object
     properties:
@@ -42,28 +40,200 @@ properties:
         minItems: 1
         items:
           type: string
-        default:
-          - linux/amd64
-          - linux/arm/v7
-          - linux/arm64
-    required: [base, platforms]
+      services:
+        description: >
+          Where an extension service's image comes from. `build`, which absence means,
+          builds it into `registry.base` the way a composition is built. `published` uses
+          the image the extension ships, which is neither built nor pushed.
+        enum: [build, published]
+    # required: [base]
   compositions:
     type: array
     minItems: 1
     items:
       type: object
+      additionalProperties: false
       properties:
         name:
-          $ref: 'definitions#/definitions/token'
+          type: string
+          pattern: ^[a-zA-Z]([a-zA-Z0-9]{1,31})?$
         image:
           type: string
         components:
           type: array
           minItems: 1
           items:
-            $ref: 'definitions#/definitions/locator'
+            type: string
+            pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(\.([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))$
+        services:
+          description: >
+            Extension services this composition runs in its own pod, named by shortcut or
+            package reference, rather than each as its own deployment. A service listed here
+            keeps its own Service and Ingress; what changes is the pod they select.
+          type: array
+          minItems: 1
+          items:
+            type: string
+            minLength: 1
+        resources:
+          $ref: '#/definitions/resources'
+        replicas:
+          type: integer
+          minimum: 1
       required: [name, components]
+  evicted:
+    description: >
+      What Toa does not deploy for this context, even where a composition lists it: it stays
+      part of the context, and is deployed by other means. A component named
+      here is in no pod, and what only it declared — an extension, a storage, a binding — is
+      not deployed either. A service named here gets no Deployment, Service or Ingress, and
+      no composition runs it.
+    type: object
+    additionalProperties: false
+    minProperties: 1
+    properties:
+      components:
+        type: array
+        minItems: 1
+        items:
+          type: string
+          pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(\.([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))$
+      services:
+        type: array
+        minItems: 1
+        items:
+          type: string
+          minLength: 1
+  atomicity:
+    description: >
+      Where the replicas of a component decide together: who owns what, what they have spent
+      between them, and which of them holds a name while it works. Absent or unreachable,
+      nothing is owned and whoever asked stands down; metering and locking are refused.
+    type: object
+    additionalProperties: false
+    properties:
+      redis:
+        description: >
+          One address, or an odd number of them — independent servers, not the nodes of a
+          cluster. A lock is taken on a quorum of them; the registry and the meter use the
+          first, because each of them counts on a single key.
+        oneOf:
+          - type: string
+          - type: array
+            minItems: 1
+            items:
+              type: string
+      interval:
+        description: >
+          How often a replica registers, in milliseconds. It has to comfortably exceed a round
+          trip to Redis and the jitter around it, and nothing is owned until two consecutive
+          intervals have agreed — so this is also how long a replica owns nothing after it
+          joins, restarts or stalls.
+        type: integer
+        minimum: 100
+    required: [redis]
+  outbox:
+    type: object
+    additionalProperties: false
+    properties:
+      interval:
+        description: >
+          The pump's cycle, in milliseconds. One cycle reads what is unpublished, publishes it
+          and marks it. In steady state it reads nothing, so this is budgeted for how fast a rare
+          failure heals rather than for how fast events flow.
+        type: integer
+        minimum: 100
+      batch:
+        description: >
+          How many rows one read brings back. A page that comes back full is read on from,
+          so this bounds a read rather than a cycle.
+        type: integer
+        minimum: 1
+      retention:
+        description: >
+          Seconds a published row is kept as a change log before it expires. A row that never
+          made it out has no expiry and is never reaped.
+        type: integer
+        minimum: 0
+  inbox:
+    type: object
+    additionalProperties: false
+    properties:
+      retention:
+        description: >
+          Seconds a call is remembered, which is the window a duplicate of it is caught in. Budget
+          it for how late a duplicate can arrive — a redelivery, the attempts the broker library
+          makes of a message, whatever a client retries on — rather than for how long the change
+          matters. Every call of an operation that declares `once` is a record for this long.
+        type: integer
+        minimum: 1
+  addressed:
+    type: object
+    additionalProperties: false
+    properties:
+      timeout:
+        description: >
+          Milliseconds an addressed call waits for its reply where it names no timeout of its own.
+          Nothing else ends a call to a process that has gone, so every addressed call waits for a
+          set time.
+        type: integer
+        minimum: 1
+  events:
+    description: >
+      Events of this context that are consumed outside it. An event is otherwise published only
+      where something in the context receives it.
+    type: array
+    items:
+      type: string
+      pattern: ^[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*$
+  mono:
+    type: object
+    additionalProperties: false
+    properties:
+      replicas:
+        type: integer
+        minimum: 1
+      resources:
+        $ref: '#/definitions/resources'
+  ingress:
+    type: object
+    additionalProperties: false
+    properties:
+      hosts:
+        type: array
+        minItems: 1
+        items:
+          type: string
+      class:
+        type: string
+      annotations:
+        type: object
+      default:
+        type: boolean
   annotations:
     type: object
+  resources:
+    $ref: '#/definitions/resources'
 required: [runtime, name, registry]
 additionalProperties: false
+
+definitions:
+  # What a deployment may take. `null` deploys it without any, which is a decision
+  # rather than an omission — every deployment has to make one.
+  resources:
+    type: object
+    nullable: true
+    additionalProperties: false
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2

package/src/.context/validate.js

@@ -1,19 +1,13 @@
-'use strict'
+import { resolve } from 'node:path'
 
-const { resolve } = require('node:path')
+import { readFileSync } from 'node:fs'
+import { yaml } from '@toa.io/generic'
+import * as schemas from '@toa.io/schemas'
 
-const { load } = require('@toa.io/yaml')
-const { Schema } = require('@toa.io/schema')
+const path = resolve(import.meta.dirname, 'schema.yaml')
+const object = yaml.load(readFileSync(path, 'utf8'))
+const schema = schemas.schema(object)
 
-const path = resolve(__dirname, 'schema.yaml')
-const object = load.sync(path)
-const schema = new Schema(object)
-
-/**
- * @param {toa.norm.context.Declaration} context
- */
-const validate = (context) => {
+export const validate = (context) => {
   schema.validate(context)
 }
-
-exports.validate = validate

package/src/component.js

@@ -1,28 +1,28 @@
-'use strict'
+import { join } from 'node:path'
 
-const { join } = require('node:path')
+import { readFile } from 'node:fs/promises'
+import { yaml as jsyaml } from '@toa.io/generic'
+import { find } from '@toa.io/generic'
+import { Locator } from '@toa.io/core'
 
-const { load: yaml } = require('@toa.io/yaml')
-const { directory: { find } } = require('@toa.io/filesystem')
-const { Locator } = require('@toa.io/core')
-
-const {
+import {
   expand,
   merge,
+  migrations,
   validate,
   collapse,
   dereference,
   defaults,
   normalize,
   extensions
-} = require('./.component')
+} from './.component/index.js'
 
-const component = async (path) => {
+export const component = async (path) => {
   const manifest = await load(path)
 
-  normalize(manifest, path)
-  validate(manifest)
-  extensions(manifest)
+  await normalize(manifest, path)
+  await validate(manifest)
+  await extensions(manifest)
 
   manifest.locator = new Locator(manifest.name, manifest.namespace)
 
@@ -33,14 +33,22 @@ const load = async (path, base, proto = false) => {
   if (base !== undefined) path = find(path, base, MANIFEST)
 
   const file = join(path, MANIFEST)
-  const manifest = /** @type {toa.norm.Component} */ await yaml(file) ?? {}
+  const manifest = (await read(file)) ?? {}
 
   manifest.path = path
 
+  const anonymous = manifest.name === undefined
+
   defaults(manifest, proto)
   await expand(manifest)
 
   await merge(path, manifest)
+  await migrations(path, manifest)
+
+  // an inherited migration is recorded under the prototype's name, and a generated one is not
+  // the same on the next machine
+  if (proto && anonymous && manifest.entity?.migrations !== undefined)
+    throw new Error(`Prototype at '${path}' declares migrations, so it has to be named`)
 
   if (manifest.prototype !== null) {
     const prototype = await load(manifest.prototype, path, true)
@@ -56,4 +64,15 @@ const load = async (path, base, proto = false) => {
 
 const MANIFEST = 'manifest.toa.yaml'
 
-exports.component = component
+/**
+ * Reads a YAML file, resolving anchors into distinct objects so that
+ * mutating one node cannot reach another.
+ *
+ * @param {string} path
+ * @return {Promise<object>}
+ */
+async function read(path) {
+  const object = jsyaml.load(await readFile(path, 'utf8'))
+
+  return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))
+}