@toa.io/norm 1.0.0-alpha.30 → 1.0.0-alpha.301

package/src/context.js

@@ -1,27 +1,35 @@
-'use strict'
+import { resolve } from 'node:path'
+import { convolve, environment as variables } from '@toa.io/generic'
+import glob from 'fast-glob'
+import { readFile } from 'node:fs/promises'
+import { yaml as jsyaml } from '@toa.io/generic'
 
-const { resolve } = require('node:path')
-const { convolve } = require('@toa.io/generic')
-const { directory: { glob } } = require('@toa.io/filesystem')
-const { load } = require('@toa.io/yaml')
+import { component } from './component.js'
 
-const { component } = require('./component')
-
-const {
+import {
   dependencies,
   normalize,
   complete,
   dereference,
+  evict,
   expand,
   validate
-} = require('./.context')
+} from './.context/index.js'
 
-const context = async (root, environment = process.env.TOA_ENV) => {
+export const context = async (
+  root,
+  environment = variables.get('TOA_ENV'),
+  options = {}
+) => {
   const path = resolve(root, CONTEXT)
-  const context = /** @type {toa.norm.Context} */ await load(path)
-  const pattern = resolve(root, context.packages)
+  const context = /** @type {toa.norm.Context} */ await read(path)
+
+  const names = environment?.split(':')
 
-  context.environment = environment
+  if (names !== undefined && names.length > 1 && names.some((name) => name.length === 0))
+    throw new Error(`Environment '${environment}' contains an empty name.`)
+
+  context.environment = names?.[0]
 
   convolve(context, environment)
   expand(context)
@@ -29,9 +37,16 @@ const context = async (root, environment = process.env.TOA_ENV) => {
 
   validate(context)
 
-  const paths = await glob(pattern)
+  const paths = await glob(resolve(root, COMPONENTS), GLOB)
 
   context.components = await Promise.all(paths.map(component))
+
+  // before the dependencies, which mark what only evicted components require.
+  // `evicted: false` leaves that off: a local run of a component still needs its variables
+  if (options.evicted !== false) evict(context)
+
+  // what a context declares of every component that stores anything, its own and the ones its
+  // extensions bring, is given to them where those are known: inside `dependencies`
   context.dependencies = await dependencies(context)
 
   dereference(context)
@@ -41,5 +56,19 @@ const context = async (root, environment = process.env.TOA_ENV) => {
 }
 
 const CONTEXT = 'context.toa.yaml'
+const COMPONENTS = 'components/*'
+
+const GLOB = { onlyDirectories: true, absolute: true }
 
-exports.context = context
+/**
+ * Reads a YAML file, resolving anchors into distinct objects so that
+ * mutating one node cannot reach another.
+ *
+ * @param {string} path
+ * @return {Promise<object>}
+ */
+async function read(path) {
+  const object = jsyaml.load(await readFile(path, 'utf8'))
+
+  return jsyaml.load(jsyaml.dump(object, { noRefs: true, lineWidth: -1 }))
+}

package/src/definition.js

@@ -0,0 +1,56 @@
+import { createRequire } from 'node:module'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import * as definitions from '@toa.io/definitions'
+
+// import.meta.resolve takes no paths, and a dependency is named rather than
+// written as a path
+const require = createRequire(import.meta.url)
+
+/**
+ * What a package declares, read without running it.
+ *
+ * A first-party package's definition is in `@toa.io/definitions`. Another package's is
+ * its `definition.js`, or its entry where it ships none — so an extension that exports
+ * its `manifest` and `deployment` beside its `Factory` is read as it always was.
+ *
+ * @param {string} reference a package name, or a directory
+ * @returns {Promise<toa.norm.Definition>}
+ */
+export function definition(reference) {
+  cache[reference] ??= load(reference)
+
+  return cache[reference]
+}
+
+const cache = {}
+
+async function load(reference) {
+  const known = await definitions.definition(reference)
+
+  if (known !== undefined) return { name: reference, module: known }
+
+  const name = metadata(reference)?.name ?? reference
+  const module = await import(pathToFileURL(resolve(reference)).href)
+
+  return { name, module }
+}
+
+function resolve(reference) {
+  try {
+    return require.resolve(join(reference, 'definition.js'))
+  } catch {
+    return require.resolve(reference)
+  }
+}
+
+function metadata(reference) {
+  try {
+    return JSON.parse(
+      readFileSync(require.resolve(join(reference, 'package.json')), 'utf8')
+    )
+  } catch {
+    return null
+  }
+}

package/src/entity.js

@@ -0,0 +1,15 @@
+/**
+ * What a component declares about its records is a property map and a list of the properties a
+ * whole one has. A validator wants a schema, so it is assembled here rather than declared: one
+ * that a stored record must fit, and one for a changeset, which is whatever subset of the same
+ * properties an assignment writes.
+ */
+export const schema = (entity) =>
+  entity.required === undefined
+    ? changeset(entity)
+    : { type: 'object', properties: entity.properties, required: entity.required }
+
+export const changeset = (entity) => ({
+  type: 'object',
+  properties: entity.properties
+})

package/src/index.js

@@ -1,9 +1,6 @@
-'use strict'
-
-const { context } = require('./context')
-const { component } = require('./component')
-
-exports.shortcuts = require('./shortcuts')
-
-exports.context = context
-exports.component = component
+export { context } from './context.js'
+export { component } from './component.js'
+export { plain, revive, NORMALIZED } from './manifest.js'
+export { definition } from './definition.js'
+export * as shortcuts from './shortcuts.js'
+export * as entity from './entity.js'

package/src/manifest.js

@@ -0,0 +1,154 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { dirname, isAbsolute, join, parse, relative, sep } from 'node:path'
+
+import { Locator } from '@toa.io/core'
+import { find } from '@toa.io/generic'
+
+/**
+ * A normalised manifest, as it is carried in a file beside the component it is of.
+ *
+ * A build normalises every component to tag its image; the process that runs it normalised them
+ * again at start, reading every operation module's source through its bridge to do it. What the
+ * build derived is written instead, and the two cannot disagree: the sources it was read from are
+ * what the image's tag digests, and the version of Toa that read them is in that tag as well.
+ *
+ * What cannot be carried is where things are. A workspace and an image put a component in one
+ * directory and Toa's own packages in another, so every path is written as what it is of rather
+ * than as where it was — the component itself, or the package that holds it — and is resolved
+ * again on the other side, the way norm resolved it.
+ *
+ * A key with no value is not carried: a file has no way to say `undefined`, and nothing reads
+ * one apart from a key that is not there.
+ */
+export function plain(manifest) {
+  const { locator, packages, path, ...rest } = manifest
+
+  // what is carried is written, not the manifest the caller goes on using, and it is written
+  // as a file holds it: a key with no value does not survive and is not meant to
+  const declared = JSON.parse(JSON.stringify(rest))
+
+  walk(declared, (item) => {
+    item.path = carried(item.path, path)
+  })
+
+  local(declared, path)
+
+  return declared
+}
+
+/** What `plain` wrote, read back beside the component it is of. */
+export function revive(declared, path) {
+  const manifest = { ...structuredClone(declared), path }
+
+  walk(manifest, (item) => {
+    item.path = resolved(item.path, path)
+  })
+
+  manifest.locator = new Locator(manifest.name, manifest.namespace)
+
+  return manifest
+}
+
+/**
+ * Everything in a manifest that says where a module of it is: what an inherited one declares is
+ * in the prototype it came from, and what the component declares is in the component.
+ */
+function walk(manifest, rewrite) {
+  for (const property of ROOTED)
+    for (const item of Object.values(manifest[property] ?? {}))
+      if (item.path !== undefined) rewrite(item)
+
+  for (let prototype = manifest.prototype; prototype != null; prototype = prototype.prototype)
+    if (prototype.path !== undefined) rewrite(prototype)
+}
+
+/** Where it is written. A workspace has none: it is a build's output, read by a process. */
+export const NORMALIZED = 'manifest.toa.json'
+
+/** Where norm says a module is, beside what it says about it. */
+const ROOTED = ['events', 'receivers', 'guards']
+
+/** The component's own directory, which is where it is read back. */
+const SELF = '.'
+
+function carried(path, root) {
+  if (path === root) return SELF
+  if (path.startsWith(root + sep)) return SELF + '/' + relative(root, path).split(sep).join('/')
+
+  return specifier(path)
+}
+
+function resolved(path, root) {
+  if (path === SELF) return root
+  if (path.startsWith(SELF + '/')) return join(root, path.slice(2))
+
+  return find(path, root, MANIFEST)
+}
+
+/**
+ * A directory in a package, named the way a manifest would name it, so that it is found again
+ * wherever the package is installed. A prototype that is a directory of the application's own is
+ * refused here rather than in the container: an image carries components and nothing between
+ * them, so nothing would be there to find, whether it is normalised here or there.
+ */
+function specifier(path) {
+  const root = above(path)
+
+  if (root !== undefined) {
+    const { name, private: hidden } = JSON.parse(
+      readFileSync(join(root, 'package.json'), 'utf8')
+    )
+
+    if (name !== undefined && hidden !== true)
+      return [name, ...relative(root, path).split(sep)].filter(Boolean).join('/')
+  }
+
+  throw new Error(
+    `'${path}' is in no package, so an image cannot carry what is in it: ` +
+      'what a component inherits is reached by the reference of the package it is in.'
+  )
+}
+
+/** The directory of the package a path is in. */
+function above(path) {
+  const { root } = parse(path)
+
+  let current = path
+
+  while (current !== root) {
+    if (existsSync(join(current, 'package.json'))) return current
+
+    current = dirname(current)
+  }
+
+  return undefined
+}
+
+/**
+ * Nothing that is carried may say where it was on the machine that read it. What is checked is
+ * what can be told apart from a value of the application's own — an HTTP route is `/accounts`
+ * and is a path to nobody — so: anything inside the component, and any directory a manifest
+ * declares a component or a prototype in. This is what catches a path norm starts writing
+ * somewhere this file does not know about, at the build rather than at the start of a container
+ * that cannot find it.
+ */
+function local(value, root, at = '') {
+  if (typeof value === 'string') {
+    if (value === root || value.startsWith(root + sep) || component(value))
+      throw new Error(`'${at}' carries a path of the machine that read it: ${value}`)
+
+    return
+  }
+
+  if (value === null || typeof value !== 'object') return
+
+  for (const [key, item] of Object.entries(value))
+    local(item, root, at === '' ? key : `${at}.${key}`)
+}
+
+/** Whether a value is a directory a component or a prototype is declared in. */
+function component(value) {
+  return isAbsolute(value) && existsSync(join(value, MANIFEST))
+}
+
+const MANIFEST = 'manifest.toa.yaml'

package/src/shortcuts.js

@@ -1,6 +1,4 @@
-'use strict'
-
-const { empty, merge } = require('@toa.io/generic')
+import { empty, merge } from '@toa.io/generic'
 
 /**
  * Resolves shortcuts
@@ -8,7 +6,7 @@ const { empty, merge } = require('@toa.io/generic')
  * @param {string} token
  * @returns {string}
  */
-const resolve = (token) => {
+export const resolve = (token) => {
   if (token in SHORTCUTS) return SHORTCUTS[token]
   else return token
 }
@@ -20,7 +18,7 @@ const resolve = (token) => {
  * @param {Object} object
  * @param {string} [group]
  */
-const recognize = (shortcuts, object, group) => {
+export const recognize = (shortcuts, object, group) => {
   if (object === undefined) return
 
   const target = group === undefined ? object : {}
@@ -38,21 +36,18 @@ const recognize = (shortcuts, object, group) => {
   if (group !== undefined && !empty(target)) object[group] = merge(object[group], target)
 }
 
-const SHORTCUTS = {
+export const SHORTCUTS = {
   amqp: '@toa.io/bindings.amqp',
   node: '@toa.io/bridges.node',
   bash: '@toa.io/bridges.bash',
   mongodb: '@toa.io/storages.mongodb',
-  sql: '@toa.io/storages.sql',
-  queues: '@toa.io/storages.queues',
   exposition: '@toa.io/extensions.exposition',
   realtime: '@toa.io/extensions.realtime',
   configuration: '@toa.io/extensions.configuration',
-  origins: '@toa.io/extensions.origins',
   stash: '@toa.io/extensions.stash',
-  storages: '@toa.io/extensions.storages'
+  storages: '@toa.io/extensions.storages',
+  telemetry: '@toa.io/extensions.telemetry',
+  cadence: '@toa.io/extensions.cadence',
+  convergence: '@toa.io/extensions.convergence',
+  introspection: '@toa.io/extensions.introspection'
 }
-
-exports.recognize = recognize
-exports.resolve = resolve
-exports.SHORTCUTS = SHORTCUTS

package/test/component/collapse.fixtures.js

@@ -1,55 +1,47 @@
-'use strict'
-
-const { generate } = require('randomstring')
+import { generate } from 'randomstring'
 
 const entity = {
   manifest: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            type: 'string'
-          },
-          bar: {
-            type: 'integer'
-          }
+      properties: {
+        foo: {
+          type: 'string'
         },
-        required: ['foo']
-      }
+        bar: {
+          type: 'integer'
+        }
+      },
+      required: ['foo']
     }
   },
   prototype: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            default: 'ok'
-          },
-          baz: {
-            type: 'boolean'
-          }
+      properties: {
+        foo: {
+          minLength: 1
         },
-        required: ['baz']
-      }
+        baz: {
+          type: 'boolean'
+        }
+      },
+      required: ['baz']
     }
   },
   result: {
     entity: {
-      schema: {
-        properties: {
-          foo: {
-            type: 'string',
-            default: 'ok'
-          },
-          bar: {
-            type: 'integer'
-          },
-          baz: {
-            type: 'boolean'
-          }
+      properties: {
+        foo: {
+          type: 'string',
+          minLength: 1
         },
-        required: ['foo', 'baz']
-      }
+        bar: {
+          type: 'integer'
+        },
+        baz: {
+          type: 'boolean'
+        }
+      },
+      required: ['foo', 'baz']
     }
   }
 }
@@ -88,7 +80,6 @@ const operations = {
   result: {
     prototype: {
       prototype: null,
-      path: expect.any(String),
       operations: {
         add: {
           bridge: 'a'
@@ -131,4 +122,4 @@ const remotes = {
   }
 }
 
-exports.samples = { entity, operations, remotes }
+export const samples = { entity, operations, remotes }

package/test/component/collapse.test.js

@@ -1,9 +1,10 @@
-'use strict'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
 
-const clone = require('clone-deep')
+import clone from 'clone-deep'
 
-const fixtures = require('./collapse.fixtures')
-const { collapse } = require('../../src/.component')
+import * as fixtures from './collapse.fixtures.js'
+import { collapse } from '../../src/.component/index.js'
 
 let samples
 
@@ -18,7 +19,7 @@ it('should ignore locator', () => {
 
   collapse(manifest, prototype)
 
-  expect(manifest).toStrictEqual(source)
+  assert.deepStrictEqual(manifest, source)
 })
 
 it('should remove prototype property', () => {
@@ -26,7 +27,7 @@ it('should remove prototype property', () => {
 
   collapse(manifest, {})
 
-  expect(manifest).toStrictEqual({})
+  assert.deepStrictEqual(manifest, {})
 })
 
 describe('entity', () => {
@@ -34,7 +35,80 @@ describe('entity', () => {
     const manifest = clone(samples.entity.manifest)
 
     collapse(manifest, samples.entity.prototype)
-    expect(manifest).toStrictEqual(samples.entity.result)
+    assert.deepStrictEqual(manifest, samples.entity.result)
+  })
+
+  it('should let a component state its own id', () => {
+    const prototype = { entity: { properties: { id: { type: 'string' }, VERSION: {} } } }
+    const manifest = { name: 'pot', entity: { properties: { id: { type: 'number' } } } }
+
+    collapse(manifest, prototype)
+
+    assert.strictEqual(manifest.entity.custom, true)
+    assert.deepStrictEqual(manifest.entity.properties.id, { type: 'number' })
+  })
+
+  it('should refuse a component that states a property the runtime writes', () => {
+    const prototype = { entity: { properties: { DELETED: { type: 'integer' } } } }
+    const manifest = {
+      name: 'pot',
+      entity: { properties: { DELETED: { type: 'integer' } } }
+    }
+
+    assert.throws(
+      () => collapse(manifest, prototype),
+      /System property 'DELETED' cannot be overridden/
+    )
+  })
+
+  it('should inherit migrations ahead of its own, under the prototype name', () => {
+    const manifest = { entity: { migrations: [{ id: '0002', steps: [] }] } }
+    const prototype = {
+      name: 'base',
+      entity: { migrations: [{ id: '0001', steps: [] }] }
+    }
+
+    collapse(manifest, prototype)
+
+    assert.deepStrictEqual(manifest.entity.migrations, [
+      { id: 'base:0001', steps: [], prototype: 'base' },
+      { id: '0002', steps: [] }
+    ])
+  })
+
+  it('should pass on what the prototype inherited as it is', () => {
+    const manifest = { entity: { migrations: [{ id: '0002', steps: [] }] } }
+    const prototype = {
+      name: 'base',
+      entity: {
+        migrations: [
+          { id: 'system:0001', steps: [], prototype: 'system' },
+          { id: '0001', steps: [] }
+        ]
+      }
+    }
+
+    collapse(manifest, prototype)
+
+    assert.deepStrictEqual(
+      manifest.entity.migrations.map(({ id }) => id),
+      ['system:0001', 'base:0001', '0002']
+    )
+  })
+
+  it('should inherit migrations into a component that declares none', () => {
+    const manifest = {}
+    const prototype = {
+      name: 'base',
+      entity: { migrations: [{ id: '0001', steps: [] }] }
+    }
+
+    collapse(manifest, prototype)
+
+    assert.deepStrictEqual(
+      manifest.entity.migrations.map(({ id }) => id),
+      ['base:0001']
+    )
   })
 })
 
@@ -44,12 +118,12 @@ it('should ignore bindings', () => {
   const manifest = clone(source)
 
   collapse(manifest, prototype)
-  expect(manifest).toStrictEqual(source)
+  assert.deepStrictEqual(manifest, source)
 
   delete manifest.bindings
 
   collapse(manifest, prototype)
-  expect(manifest).toStrictEqual({})
+  assert.deepStrictEqual(manifest, {})
 })
 
 it('should merge operations', () => {
@@ -57,5 +131,11 @@ it('should merge operations', () => {
   const prototype = clone(samples.operations.prototype)
 
   collapse(manifest, prototype, '/somewhere')
-  expect(manifest).toStrictEqual(samples.operations.result)
+
+  // the prototype's path is generated, so it is checked apart from the shape
+  assert.strictEqual(typeof manifest.prototype.path, 'string')
+
+  const { path, ...rest } = manifest.prototype
+
+  assert.deepStrictEqual({ ...manifest, prototype: rest }, samples.operations.result)
 })

package/test/component/dependencies.test.js

@@ -1,38 +1,40 @@
-'use strict'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
 
-const { resolve } = require('node:path')
-const { dependencies } = require('../../src/.component')
+import { resolve } from 'node:path'
+import { dependencies } from '../../src/.component/index.js'
 
-const NORM = resolve(__dirname, '../../')
+const NORM = resolve(import.meta.dirname, '../../')
 
 it('should be', async () => {
-  expect(dependencies).toBeInstanceOf(Function)
+  assert.ok(dependencies instanceof Function)
 })
 
 /** @type {toa.norm.Component} */
 let component
 
 beforeEach(() => {
-  component = /** @type {toa.norm.Component} */ { path: __dirname }
+  component = /** @type {toa.norm.Component} */ { path: import.meta.dirname }
 })
 
-describe.each(/** @type {[string, string][]} */ [
+for (const [_, reference] of [
   ['package id', '@toa.io/norm'],
   ['relative path', '../../']
-])('%s', (_, reference) => {
-  it('should resolve storage', async () => {
-    component.entity = { storage: reference, schema: {} }
+])
+  describe(`${_}`, () => {
+    it('should resolve storage', async () => {
+      component.entity = { storage: reference, schema: {} }
 
-    dependencies(component)
+      dependencies(component)
 
-    expect(component.entity.storage).toStrictEqual(NORM)
+      assert.deepStrictEqual(component.entity.storage, NORM)
+    })
   })
-})
 
 it('should resolve toa packages', async () => {
   component.entity = { storage: '@toa.io/core', schema: {} }
 
   dependencies(component)
 
-  expect(component.entity.storage).toBeDefined()
+  assert.notStrictEqual(component.entity.storage, undefined)
 })