@toa.io/norm 1.0.0-alpha.32 → 1.0.0-alpha.321

package/src/.context/schema.yaml

@@ -6,12 +6,10 @@ properties:
   version:
     type: string
   name:
-    $ref: 'definitions#/definitions/label'
-  description:
     type: string
-  packages:
+    pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(-([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))*$
+  description:
     type: string
-    default: components/*
   build:
     type: object
     properties:
@@ -42,28 +40,202 @@ properties:
         minItems: 1
         items:
           type: string
-        default:
-          - linux/amd64
-          - linux/arm/v7
-          - linux/arm64
-    required: [base, platforms]
+      services:
+        description: >
+          Where an extension service's image comes from. `build`, which absence means,
+          builds it into `registry.base` the way a composition is built. `published` uses
+          the image the extension ships, which is neither built nor pushed.
+        enum: [build, published]
+    # required: [base]
   compositions:
     type: array
     minItems: 1
     items:
       type: object
+      additionalProperties: false
       properties:
         name:
-          $ref: 'definitions#/definitions/token'
+          type: string
+          pattern: ^[a-zA-Z]([a-zA-Z0-9]{1,31})?$
         image:
           type: string
         components:
           type: array
           minItems: 1
           items:
-            $ref: 'definitions#/definitions/locator'
+            type: string
+            pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(\.([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))$
+        services:
+          description: >
+            Extension services this composition runs in its own pod, named by shortcut or
+            package reference, rather than each as its own deployment. A service listed here
+            keeps its own Service and Ingress; what changes is the pod they select.
+          type: array
+          minItems: 1
+          items:
+            type: string
+            minLength: 1
+        resources:
+          $ref: '#/definitions/resources'
+        replicas:
+          type: integer
+          minimum: 1
       required: [name, components]
+  evicted:
+    description: >
+      What Toa does not deploy for this context, even where a composition lists it: it stays
+      part of the context, and is deployed by other means. A component named
+      here is in no pod, and what only it declared — an extension, a storage, a binding — is
+      not deployed either. A service named here gets no Deployment, Service or Ingress, and
+      no composition runs it.
+    type: object
+    additionalProperties: false
+    minProperties: 1
+    properties:
+      components:
+        type: array
+        minItems: 1
+        items:
+          type: string
+          pattern: ^([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?)(\.([a-zA-Z]+([_a-zA-Z0-9]*[a-zA-Z0-9]+)?))$
+      services:
+        type: array
+        minItems: 1
+        items:
+          type: string
+          minLength: 1
+  atomicity:
+    description: >
+      Where the replicas of a component decide together: who owns what, what they have spent
+      between them, and which of them holds a name while it works. Absent or unreachable,
+      nothing is owned and whoever asked stands down; metering and locking are refused.
+    type: object
+    additionalProperties: false
+    properties:
+      redis:
+        description: >
+          One address, or an odd number of them — independent servers, not the nodes of a
+          cluster. A lock is taken on a quorum of them; the registry and the meter use the
+          first, because each of them counts on a single key.
+        oneOf:
+          - type: string
+          - type: array
+            minItems: 1
+            items:
+              type: string
+      interval:
+        description: >
+          How often a replica registers, in milliseconds. It has to comfortably exceed a round
+          trip to Redis and the jitter around it, and nothing is owned until two consecutive
+          intervals have agreed — so this is also how long a replica owns nothing after it
+          joins, restarts or stalls.
+        type: integer
+        minimum: 100
+    required: [redis]
+  outbox:
+    type: object
+    additionalProperties: false
+    properties:
+      interval:
+        description: >
+          The pump's cycle, in milliseconds. One cycle reads what is unpublished, publishes it
+          and marks it. In steady state it reads nothing, so this is budgeted for how fast a rare
+          failure heals rather than for how fast events flow.
+        type: integer
+        minimum: 100
+      batch:
+        description: >
+          How many rows one read brings back. A page that comes back full is read on from,
+          so this bounds a read rather than a cycle.
+        type: integer
+        minimum: 1
+      retention:
+        description: >
+          Seconds a published row is kept as a change log before it expires. A row that never
+          made it out has no expiry and is never reaped.
+        type: integer
+        minimum: 0
+  inbox:
+    type: object
+    additionalProperties: false
+    properties:
+      retention:
+        description: >
+          Seconds a call is remembered, which is the window a duplicate of it is caught in. Budget
+          it for how late a duplicate can arrive — a redelivery, the attempts the broker library
+          makes of a message, whatever a client retries on — rather than for how long the change
+          matters. Every call of an operation that declares `once: true` is a record for this
+          long; an operation that states a number of its own is kept for that instead. Ten minutes
+          at least, because the broker alone repeats a call for about that long.
+        type: integer
+        minimum: 600
+  addressed:
+    type: object
+    additionalProperties: false
+    properties:
+      timeout:
+        description: >
+          Milliseconds an addressed call waits for its reply where it names no timeout of its own.
+          Nothing else ends a call to a process that has gone, so every addressed call waits for a
+          set time.
+        type: integer
+        minimum: 1
+  events:
+    description: >
+      Events of this context that are consumed outside it. An event is otherwise published only
+      where something in the context receives it.
+    type: array
+    items:
+      type: string
+      pattern: ^[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*\.[a-zA-Z][a-zA-Z0-9]*$
+  mono:
+    type: object
+    additionalProperties: false
+    properties:
+      replicas:
+        type: integer
+        minimum: 1
+      resources:
+        $ref: '#/definitions/resources'
+  ingress:
+    type: object
+    additionalProperties: false
+    properties:
+      hosts:
+        type: array
+        minItems: 1
+        items:
+          type: string
+      class:
+        type: string
+      annotations:
+        type: object
+      default:
+        type: boolean
   annotations:
     type: object
+  resources:
+    $ref: '#/definitions/resources'
 required: [runtime, name, registry]
 additionalProperties: false
+
+definitions:
+  # What a deployment may take. `null` deploys it without any, which is a decision
+  # rather than an omission — every deployment has to make one.
+  resources:
+    type: object
+    nullable: true
+    additionalProperties: false
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2

package/src/.context/validate.js

@@ -1,19 +1,16 @@
-'use strict'
+import { resolve } from 'node:path'
 
-const { resolve } = require('node:path')
+import { readFileSync } from 'node:fs'
+import { yaml } from '@toa.io/generic'
+import * as schemas from '@toa.io/schemas'
 
-const { load } = require('@toa.io/yaml')
-const { Schema } = require('@toa.io/schema')
+const path = resolve(import.meta.dirname, 'schema.yaml')
+const object = yaml.load(readFileSync(path, 'utf8'))
+const schema = schemas.schema(object)
 
-const path = resolve(__dirname, 'schema.yaml')
-const object = load.sync(path)
-const schema = new Schema(object)
-
-/**
- * @param {toa.norm.context.Declaration} context
- */
-const validate = (context) => {
-  schema.validate(context)
+export const validate = (context) => {
+  schema.validate(context, CONTEXT)
 }
 
-exports.validate = validate
+/** The file a reader has to edit, since nothing else in the message names it. */
+const CONTEXT = 'context.toa.yaml'

package/src/component.js

@@ -1,28 +1,28 @@
-'use strict'
+import { join } from 'node:path'
 
-const { join } = require('node:path')
+import { readFile } from 'node:fs/promises'
+import { yaml as jsyaml } from '@toa.io/generic'
+import { find } from '@toa.io/generic'
+import { Locator } from '@toa.io/core'
 
-const { load: yaml } = require('@toa.io/yaml')
-const { directory: { find } } = require('@toa.io/filesystem')
-const { Locator } = require('@toa.io/core')
-
-const {
+import {
   expand,
   merge,
+  migrations,
   validate,
   collapse,
   dereference,
   defaults,
   normalize,
   extensions
-} = require('./.component')
+} from './.component/index.js'
 
-const component = async (path) => {
+export const component = async (path) => {
   const manifest = await load(path)
 
-  normalize(manifest, path)
-  validate(manifest)
-  extensions(manifest)
+  await normalize(manifest, path)
+  await validate(manifest)
+  await extensions(manifest)
 
   manifest.locator = new Locator(manifest.name, manifest.namespace)
 
@@ -33,14 +33,22 @@ const load = async (path, base, proto = false) => {
   if (base !== undefined) path = find(path, base, MANIFEST)
 
   const file = join(path, MANIFEST)
-  const manifest = /** @type {toa.norm.Component} */ await yaml(file) ?? {}
+  const manifest = (await read(file)) ?? {}
 
   manifest.path = path
 
+  const anonymous = manifest.name === undefined
+
   defaults(manifest, proto)
   await expand(manifest)
 
   await merge(path, manifest)
+  await migrations(path, manifest)
+
+  // an inherited migration is recorded under the prototype's name, and a generated one is not
+  // the same on the next machine
+  if (proto && anonymous && manifest.entity?.migrations !== undefined)
+    throw new Error(`Prototype at '${path}' declares migrations, so it has to be named`)
 
   if (manifest.prototype !== null) {
     const prototype = await load(manifest.prototype, path, true)
@@ -56,4 +64,15 @@ const load = async (path, base, proto = false) => {
 
 const MANIFEST = 'manifest.toa.yaml'
 
-exports.component = component
+/**
+ * Reads a YAML file, resolving anchors into distinct objects so that
+ * mutating one node cannot reach another.
+ *
+ * @param {string} path
+ * @return {Promise<object>}
+ */
+async function read(path) {
+  const object = jsyaml.load(await readFile(path, 'utf8'))
+
+  return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))
+}

package/src/context.js

@@ -1,27 +1,35 @@
-'use strict'
+import { resolve } from 'node:path'
+import { convolve, environment as variables } from '@toa.io/generic'
+import glob from 'fast-glob'
+import { readFile } from 'node:fs/promises'
+import { yaml as jsyaml } from '@toa.io/generic'
 
-const { resolve } = require('node:path')
-const { convolve } = require('@toa.io/generic')
-const { directory: { glob } } = require('@toa.io/filesystem')
-const { load } = require('@toa.io/yaml')
+import { component } from './component.js'
 
-const { component } = require('./component')
-
-const {
+import {
   dependencies,
   normalize,
   complete,
   dereference,
+  evict,
   expand,
   validate
-} = require('./.context')
+} from './.context/index.js'
 
-const context = async (root, environment = process.env.TOA_ENV) => {
+export const context = async (
+  root,
+  environment = variables.get('TOA_ENV'),
+  options = {}
+) => {
   const path = resolve(root, CONTEXT)
-  const context = /** @type {toa.norm.Context} */ await load(path)
-  const pattern = resolve(root, context.packages)
+  const context = /** @type {toa.norm.Context} */ await read(path)
+
+  const names = environment?.split(':')
 
-  context.environment = environment
+  if (names !== undefined && names.length > 1 && names.some((name) => name.length === 0))
+    throw new Error(`Environment '${environment}' contains an empty name.`)
+
+  context.environment = names?.[0]
 
   convolve(context, environment)
   expand(context)
@@ -29,9 +37,16 @@ const context = async (root, environment = process.env.TOA_ENV) => {
 
   validate(context)
 
-  const paths = await glob(pattern)
+  const paths = await glob(resolve(root, COMPONENTS), GLOB)
 
   context.components = await Promise.all(paths.map(component))
+
+  // before the dependencies, which mark what only evicted components require.
+  // `evicted: false` leaves that off: a local run of a component still needs its variables
+  if (options.evicted !== false) evict(context)
+
+  // what a context declares of every component that stores anything, its own and the ones its
+  // extensions bring, is given to them where those are known: inside `dependencies`
   context.dependencies = await dependencies(context)
 
   dereference(context)
@@ -41,5 +56,19 @@ const context = async (root, environment = process.env.TOA_ENV) => {
 }
 
 const CONTEXT = 'context.toa.yaml'
+const COMPONENTS = 'components/*'
+
+const GLOB = { onlyDirectories: true, absolute: true }
 
-exports.context = context
+/**
+ * Reads a YAML file, resolving anchors into distinct objects so that
+ * mutating one node cannot reach another.
+ *
+ * @param {string} path
+ * @return {Promise<object>}
+ */
+async function read(path) {
+  const object = jsyaml.load(await readFile(path, 'utf8'))
+
+  return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))
+}

package/src/definition.js

@@ -0,0 +1,63 @@
+import { createRequire } from 'node:module'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import * as definitions from '@toa.io/definitions'
+
+// import.meta.resolve takes no paths, and a dependency is named rather than
+// written as a path
+const require = createRequire(import.meta.url)
+
+/**
+ * What a package declares, read without running it.
+ *
+ * A first-party package's definition is in `@toa.io/definitions`. Another package's is
+ * its `definition.js`, or its entry where it ships none — so an extension that exports
+ * its `manifest` and `deployment` beside its `Factory` is read as it always was.
+ *
+ * @param {string} reference a package name, or a directory
+ * @returns {Promise<toa.norm.Definition>}
+ */
+export function definition(reference) {
+  cache[reference] ??= load(reference)
+
+  return cache[reference]
+}
+
+const cache = {}
+
+async function load(reference) {
+  const known = await definitions.definition(reference)
+
+  if (known !== undefined) return { name: reference, module: known }
+
+  // `package#key` is a declaration of its own that a package claims beside its main one
+  const [path, key] = reference.split('#')
+  const name = (metadata(path)?.name ?? path) + (key === undefined ? '' : '#' + key)
+  const module = await import(pathToFileURL(resolve(path)).href)
+
+  if (key === undefined) return { name, module }
+
+  if (module.keys?.[key] === undefined)
+    throw new Error(`'${reference}' names a key that '${path}' does not claim`)
+
+  return { name, module: module.keys[key] }
+}
+
+function resolve(reference) {
+  try {
+    return require.resolve(join(reference, 'definition.js'))
+  } catch {
+    return require.resolve(reference)
+  }
+}
+
+function metadata(reference) {
+  try {
+    return JSON.parse(
+      readFileSync(require.resolve(join(reference, 'package.json')), 'utf8')
+    )
+  } catch {
+    return null
+  }
+}

package/src/entity.js

@@ -0,0 +1,15 @@
+/**
+ * What a component declares about its records is a property map and a list of the properties a
+ * whole one has. A validator wants a schema, so it is assembled here rather than declared: one
+ * that a stored record must fit, and one for a changeset, which is whatever subset of the same
+ * properties an assignment writes.
+ */
+export const schema = (entity) =>
+  entity.required === undefined
+    ? changeset(entity)
+    : { type: 'object', properties: entity.properties, required: entity.required }
+
+export const changeset = (entity) => ({
+  type: 'object',
+  properties: entity.properties
+})

package/src/index.js

@@ -1,9 +1,7 @@
-'use strict'
-
-const { context } = require('./context')
-const { component } = require('./component')
-
-exports.shortcuts = require('./shortcuts')
-
-exports.context = context
-exports.component = component
+export { context } from './context.js'
+export { component } from './component.js'
+export { map } from './map.js'
+export { plain, revive, NORMALIZED } from './manifest.js'
+export { definition } from './definition.js'
+export * as shortcuts from './shortcuts.js'
+export * as entity from './entity.js'

package/src/manifest.js

@@ -0,0 +1,154 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { dirname, isAbsolute, join, parse, relative, sep } from 'node:path'
+
+import { Locator } from '@toa.io/core'
+import { find } from '@toa.io/generic'
+
+/**
+ * A normalised manifest, as it is carried in a file beside the component it is of.
+ *
+ * A build normalises every component to tag its image; the process that runs it normalised them
+ * again at start, reading every operation module's source through its bridge to do it. What the
+ * build derived is written instead, and the two cannot disagree: the sources it was read from are
+ * what the image's tag digests, and the version of Toa that read them is in that tag as well.
+ *
+ * What cannot be carried is where things are. A workspace and an image put a component in one
+ * directory and Toa's own packages in another, so every path is written as what it is of rather
+ * than as where it was — the component itself, or the package that holds it — and is resolved
+ * again on the other side, the way norm resolved it.
+ *
+ * A key with no value is not carried: a file has no way to say `undefined`, and nothing reads
+ * one apart from a key that is not there.
+ */
+export function plain(manifest) {
+  const { locator, packages, path, ...rest } = manifest
+
+  // what is carried is written, not the manifest the caller goes on using, and it is written
+  // as a file holds it: a key with no value does not survive and is not meant to
+  const declared = JSON.parse(JSON.stringify(rest))
+
+  walk(declared, (item) => {
+    item.path = carried(item.path, path)
+  })
+
+  local(declared, path)
+
+  return declared
+}
+
+/** What `plain` wrote, read back beside the component it is of. */
+export function revive(declared, path) {
+  const manifest = { ...structuredClone(declared), path }
+
+  walk(manifest, (item) => {
+    item.path = resolved(item.path, path)
+  })
+
+  manifest.locator = new Locator(manifest.name, manifest.namespace)
+
+  return manifest
+}
+
+/**
+ * Everything in a manifest that says where a module of it is: what an inherited one declares is
+ * in the prototype it came from, and what the component declares is in the component.
+ */
+function walk(manifest, rewrite) {
+  for (const property of ROOTED)
+    for (const item of Object.values(manifest[property] ?? {}))
+      if (item.path !== undefined) rewrite(item)
+
+  for (let prototype = manifest.prototype; prototype != null; prototype = prototype.prototype)
+    if (prototype.path !== undefined) rewrite(prototype)
+}
+
+/** Where it is written. A workspace has none: it is a build's output, read by a process. */
+export const NORMALIZED = 'manifest.toa.json'
+
+/** Where norm says a module is, beside what it says about it. */
+const ROOTED = ['events', 'receivers', 'guards']
+
+/** The component's own directory, which is where it is read back. */
+const SELF = '.'
+
+function carried(path, root) {
+  if (path === root) return SELF
+  if (path.startsWith(root + sep)) return SELF + '/' + relative(root, path).split(sep).join('/')
+
+  return specifier(path)
+}
+
+function resolved(path, root) {
+  if (path === SELF) return root
+  if (path.startsWith(SELF + '/')) return join(root, path.slice(2))
+
+  return find(path, root, MANIFEST)
+}
+
+/**
+ * A directory in a package, named the way a manifest would name it, so that it is found again
+ * wherever the package is installed. A prototype that is a directory of the application's own is
+ * refused here rather than in the container: an image carries components and nothing between
+ * them, so nothing would be there to find, whether it is normalised here or there.
+ */
+function specifier(path) {
+  const root = above(path)
+
+  if (root !== undefined) {
+    const { name, private: hidden } = JSON.parse(
+      readFileSync(join(root, 'package.json'), 'utf8')
+    )
+
+    if (name !== undefined && hidden !== true)
+      return [name, ...relative(root, path).split(sep)].filter(Boolean).join('/')
+  }
+
+  throw new Error(
+    `'${path}' is in no package, so an image cannot carry what is in it: ` +
+      'what a component inherits is reached by the reference of the package it is in.'
+  )
+}
+
+/** The directory of the package a path is in. */
+function above(path) {
+  const { root } = parse(path)
+
+  let current = path
+
+  while (current !== root) {
+    if (existsSync(join(current, 'package.json'))) return current
+
+    current = dirname(current)
+  }
+
+  return undefined
+}
+
+/**
+ * Nothing that is carried may say where it was on the machine that read it. What is checked is
+ * what can be told apart from a value of the application's own — an HTTP route is `/accounts`
+ * and is a path to nobody — so: anything inside the component, and any directory a manifest
+ * declares a component or a prototype in. This is what catches a path norm starts writing
+ * somewhere this file does not know about, at the build rather than at the start of a container
+ * that cannot find it.
+ */
+function local(value, root, at = '') {
+  if (typeof value === 'string') {
+    if (value === root || value.startsWith(root + sep) || component(value))
+      throw new Error(`'${at}' carries a path of the machine that read it: ${value}`)
+
+    return
+  }
+
+  if (value === null || typeof value !== 'object') return
+
+  for (const [key, item] of Object.entries(value))
+    local(item, root, at === '' ? key : `${at}.${key}`)
+}
+
+/** Whether a value is a directory a component or a prototype is declared in. */
+function component(value) {
+  return isAbsolute(value) && existsSync(join(value, MANIFEST))
+}
+
+const MANIFEST = 'manifest.toa.yaml'