@toa.io/extensions.origins 0.7.1 → 0.8.0-dev.0

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@toa.io/extensions.origins",
-  "version": "0.7.1",
-  "description": "Toa operations context HTTP client",
+  "version": "0.8.0-dev.0",
+  "description": "Toa Origins",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "src/index.js",
+  "main": "source/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -19,14 +19,15 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/core": "0.7.1",
-    "@toa.io/generic": "0.7.0",
-    "@toa.io/schema": "0.7.1",
-    "@toa.io/yaml": "0.7.1",
+    "@toa.io/core": "0.8.0-dev.0",
+    "@toa.io/generic": "0.8.0-dev.0",
+    "@toa.io/schemas": "0.8.0-dev.0",
+    "@toa.io/yaml": "0.7.2-dev.1",
+    "comq": "0.6.0",
     "node-fetch": "2.6.7"
   },
   "devDependencies": {
     "@types/node-fetch": "2.6.2"
   },
-  "gitHead": "9989ea316ebf5698b1eee933e83a86e6a18b406b"
+  "gitHead": "5f1bfd167bb7919492d249f78bbcc6bd9fc645a4"
 }

package/readme.md

@@ -0,0 +1,57 @@
+# Toa Origins
+
+Origins extension enables external communications over supported protocols (`HTTP` and `AMQP`).
+
+## TL;DR
+
+```yaml
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
+
+origins:
+  website: http://www.domain.com/docs/
+  messages: amqps://amqp.amazon.com
+```
+
+```javascript
+// Node.js bridge 
+async function transition (input, object, context) {
+  await context.http.example.get() // GET http://www.domain.com/docs/example
+  await context.amqp.emit('something_happened', { really: true })
+}
+```
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    messages: amqps://amqp.azure.com
+    messages@staging: amqp://amqp.stage
+```
+
+## Declaration
+
+Origins extension declaration is a [Pointer](/libraries/pointer). Declarations can be overridden by
+the context annotations.
+
+## HTTP
+
+Uses [node-fetch](https://github.com/node-fetch/node-fetch) and returns its result.
+
+## AMQP
+
+Uses [ComQ](https://github.com/toa-io/comq), thus, provides interface of `comq.IO` restricted
+to `emit` and `request` methods.
+
+AMQP origins require credential secrets to be deployed. Secret's name must
+follow `toa-origins-{namespace}-{component}-{origin}` and it must have keys `username`
+and `password`.
+
+### Example
+
+```shell
+# deploy credentials to the current kubectl context
+$ toa conceal toa-origins-dummies-dummiy-messages username developer
+$ toa conceal toa-origins-dummies-dummiy-messages password secret
+```

package/source/.deployment/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { uris } = require('./uris')
+
+exports.uris = uris

package/source/.deployment/uris.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const { PREFIX } = require('../constants')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} instances
+ * @param {toa.origins.Annotations} annotations
+ * @returns {toa.deployment.dependency.Variables}
+ */
+function uris (instances, annotations) {
+  const variables = {}
+
+  for (const [id, annotation] of Object.entries(annotations)) {
+    const component = instances.find((instance) => instance.locator.id === id)
+
+    if (component === undefined) throw new Error(`Origins annotations error: component '${id}' is not found`)
+
+    for (const origin of Object.keys(annotation)) {
+      if (!(origin in component.manifest)) {
+        throw new Error(`Origins annotations error: component '${id}' doesn't have '${origin}' origin`)
+      }
+    }
+
+    const name = PREFIX + component.locator.uppercase
+    const json = JSON.stringify(annotation)
+    const value = btoa(json)
+    const variable = { name, value }
+
+    variables[component.locator.label] = [variable]
+  }
+
+  return variables
+}
+
+exports.uris = uris

package/source/.test/constants.js

@@ -0,0 +1,3 @@
+'use strict'
+
+exports.PROTOCOLS = ['http:', 'https:', 'amqp:', 'amqps:']

package/source/.test/deployment.fixtures.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { Locator } = require('@toa.io/core')
+const { random, sample } = require('@toa.io/generic')
+
+const { PROTOCOLS } = require('./constants')
+
+const component = () => ({
+  locator: new Locator(generate(), generate()),
+  manifest: { [generate()]: sample(PROTOCOLS) + '//' + generate() }
+})
+
+const components = () => {
+  const length = random(20) + 10
+
+  return Array.from({ length }, component)
+}
+
+exports.components = components

package/source/.test/factory.fixtures.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+/** @type {Record<string, string>} */
+const manifest = {
+  [generate()]: 'https://toa.io',
+  [generate()]: 'https://api.domain.com',
+  [generate()]: 'amqp://localhost',
+  [generate()]: 'amqps://localhost'
+}
+
+exports.manifest = manifest

package/source/constants.js

@@ -0,0 +1,3 @@
+'use strict'
+
+exports.PREFIX = 'TOA_ORIGINS_'

package/source/deployment.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { merge } = require('@toa.io/generic')
+const schemas = require('./schemas')
+const protocols = require('./protocols')
+const create = require('./.deployment')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} instances
+ * @param {toa.origins.Annotations} annotations
+ * @returns {toa.deployment.dependency.Declaration}
+ */
+function deployment (instances, annotations = {}) {
+  schemas.annotations.validate(annotations)
+
+  const uris = create.uris(instances, annotations)
+  const variables = { ...uris }
+
+  protocols.reduce((variables, provider) => {
+    const specifics = provider.deployment?.(instances)
+
+    return merge(variables, specifics)
+  }, variables)
+
+  return { variables }
+}
+
+exports.deployment = deployment

package/source/deployment.test.js

@@ -0,0 +1,159 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { sample, letters: { up } } = require('@toa.io/generic')
+
+const fixtures = require('./.test/deployment.fixtures')
+const { deployment } = require('../')
+
+it('should be', async () => {
+  expect(deployment).toBeInstanceOf(Function)
+})
+
+/** @type {toa.norm.context.dependencies.Instance[]} */
+let components
+
+/** @type {string} */
+let origin
+
+/** @type {toa.norm.context.dependencies.Instance} */
+let component
+
+beforeEach(() => {
+  components = fixtures.components()
+
+  component = sample(components)
+
+  const origins = Object.keys(component.manifest)
+
+  origin = sample(origins)
+})
+
+describe('validation', () => {
+  it('should throw on annotation component mismatch', async () => {
+    const id = generate()
+
+    const annotations = {
+      [id]: {
+        [origin]: 'dev://' + generate()
+      }
+    }
+
+    expect(() => deployment(components, annotations))
+      .toThrow(`Origins annotations error: component '${id}' is not found`)
+  })
+
+  it('should throw on annotation origin mismatch', async () => {
+    const id = component.locator.id
+    const origin = generate()
+
+    const annotations = {
+      [id]: {
+        [origin]: 'dev://' + generate()
+      }
+    }
+
+    expect(() => deployment(components, annotations))
+      .toThrow(`Origins annotations error: component '${id}' doesn't have '${origin}' origin`)
+  })
+
+  it('should throw if annotation is not valid', async () => {
+    const annotations = /** @type {toa.origins.Annotations} */ { [component.locator.id]: generate() }
+
+    expect(() => deployment(components, annotations)).toThrow('must be object')
+  })
+
+  it('should throw if annotation is URI is not valid', async () => {
+    const annotations = {
+      [component.locator.id]: {
+        [origin]: 'hello!'
+      }
+    }
+
+    expect(() => deployment(components, annotations)).toThrow('must match format')
+  })
+})
+
+it('should create variables', () => {
+  const value = 'dev://' + generate()
+
+  /** @type {toa.origins.Annotations} */
+  const annotations = {
+    [component.locator.id]: {
+      [origin]: value
+    }
+  }
+
+  const output = deployment(components, annotations)
+
+  expect(output.variables).not.toBeUndefined()
+
+  const variables = output.variables[component.locator.label]
+  const varName = 'TOA_ORIGINS_' + component.locator.uppercase
+  const variable = findVariable(variables, varName)
+
+  expect(variable).toBeDefined()
+
+  const json = JSON.stringify(annotations[component.locator.id])
+  const base64 = btoa(json)
+
+  expect(variable.value).toStrictEqual(base64)
+})
+
+describe('amqp', () => {
+  beforeEach(() => {
+    const amqpComponents = components.filter(
+      (component) => {
+        const origin = Object.keys(component.manifest)
+        const url = new URL(component.manifest[origin])
+
+        return url.protocol === 'amqp:' || url.protocol === 'amqps:'
+      }
+    )
+
+    component = sample(amqpComponents)
+
+    const origins = Object.keys(component.manifest)
+
+    origin = sample(origins)
+  })
+
+  it('should create credential secrets', () => {
+    /** @type {toa.origins.Annotations} */
+    const annotations = {
+      [component.locator.id]: {
+        [origin]: 'amqps://whatever'
+      }
+    }
+
+    const output = deployment(components, annotations)
+    const variables = output.variables[component.locator.label]
+
+    expect(variables).toBeDefined()
+
+    const envPrefix = `TOA_ORIGINS_${component.locator.uppercase}_`
+    const secPrefix = `toa-origins-${component.locator.label}-`
+
+    for (const property of ['username', 'password']) {
+      const variableName = envPrefix + up(property)
+      const secretName = secPrefix + property
+      const variable = findVariable(variables, variableName)
+
+      expect(variable).toBeDefined()
+
+      expect(variable.secret).toStrictEqual({
+        name: secretName,
+        key: property
+      })
+    }
+  })
+})
+
+/**
+ * @param {toa.deployment.dependency.Variable[]} variables
+ * @param {string} name
+ * @returns {toa.deployment.dependency.Variable}
+ */
+function findVariable (variables, name) {
+  return variables.find((variable) => variable.name === name)
+}

package/source/env.js

@@ -0,0 +1,50 @@
+'use strict'
+
+const { PREFIX } = require('./constants')
+const { overwrite, remap, letters: { up } } = require('@toa.io/generic')
+
+/**
+ * @param {toa.core.Locator} locator
+ * @param {toa.origins.Manifest} manifest
+ */
+function apply (locator, manifest) {
+  const variable = PREFIX + locator.uppercase
+  const envValue = readEnv(variable)
+
+  overwrite(manifest, envValue)
+  addCredentials(manifest, variable)
+}
+
+function readEnv (variable) {
+  if (!(variable in process.env)) return
+
+  const base64 = process.env[variable]
+  const json = atob(base64)
+
+  return JSON.parse(json)
+}
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @param {string} variable
+ */
+function addCredentials (manifest, variable) {
+  const prefix = variable + '_'
+
+  remap(manifest, (reference, origin) => {
+    const originPrefix = prefix + up(origin)
+    const username = process.env[originPrefix + '_USERNAME']
+    const password = process.env[originPrefix + '_PASSWORD']
+
+    if (username === undefined && password === undefined) return
+
+    const url = new URL(reference)
+
+    url.username = username
+    url.password = password
+
+    manifest[origin] = url.href
+  })
+}
+
+exports.apply = apply

package/source/factory.js

@@ -0,0 +1,34 @@
+'use strict'
+
+const protocols = require('./protocols')
+const env = require('./env')
+
+/**
+ * @implements {toa.core.extensions.Factory}
+ */
+class Factory {
+  aspect (locator, manifest) {
+    env.apply(locator, /** @type {toa.origins.Manifest} */ manifest)
+
+    return protocols.map((protocol) => this.#createAspect(protocol, manifest))
+  }
+
+  /**
+   * @param {object} protocol
+   * @param {toa.origins.Manifest} manifest
+   * @return {toa.core.extensions.Aspect}
+   */
+  #createAspect (protocol, manifest) {
+    const protocolManifest = {}
+
+    for (const [origin, reference] of Object.entries(manifest)) {
+      const url = new URL(reference)
+
+      if (protocol.protocols.includes(url.protocol)) protocolManifest[origin] = reference
+    }
+
+    return protocol.create(protocolManifest)
+  }
+}
+
+exports.Factory = Factory

package/source/factory.test.js

@@ -0,0 +1,122 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { Locator } = require('@toa.io/core')
+const { sample, overwrite, letters: { up } } = require('@toa.io/generic')
+
+jest.mock('./protocols/http/aspect')
+jest.mock('./protocols/amqp/aspect')
+
+const http = require('./protocols/http/aspect')
+const amqp = require('./protocols/amqp/aspect')
+
+const fixtures = require('./.test/factory.fixtures')
+const { Factory } = require('../')
+
+/** @type {toa.core.extensions.Factory} */
+let factory
+
+beforeEach(() => {
+  jest.clearAllMocks()
+
+  factory = new Factory()
+})
+
+it('should create aspects', () => {
+  factory.aspect(new Locator(generate(), generate()), fixtures.manifest)
+
+  const httpManifest = filterManifest(fixtures.manifest, 'http')
+  const amqpManifest = filterManifest(fixtures.manifest, 'amqp')
+
+  expect(http.create).toHaveBeenCalledWith(httpManifest)
+  expect(amqp.create).toHaveBeenCalledWith(amqpManifest)
+})
+
+describe('env', () => {
+  it('should overwrites URLs from environment', async () => {
+    const httpManifest = filterManifest(fixtures.manifest, 'http')
+    const key = sample(Object.keys(httpManifest))
+    const override = { [key]: 'http://' + generate() }
+    const json = JSON.stringify(override)
+    const base64 = btoa(json)
+    const locator = new Locator(generate(), generate())
+
+    process.env['TOA_ORIGINS_' + locator.uppercase] = base64
+
+    factory.aspect(locator, fixtures.manifest)
+
+    const expected = overwrite(httpManifest, override)
+
+    expect(http.create).toHaveBeenCalledWith(expected)
+  })
+
+  describe('amqp', () => {
+    /** @type {toa.origins.Manifest} */
+    let amqpManifest
+
+    beforeEach(() => {
+      amqpManifest = filterManifest(fixtures.manifest, 'amqp')
+    })
+
+    it('should add credentials from environment', async () => {
+      const key = sample(Object.keys(amqpManifest))
+      const locator = new Locator(generate(), generate())
+      const envPrefix = 'TOA_ORIGINS_' + locator.uppercase + '_' + up(key) + '_'
+      const username = generate()
+      const password = generate()
+
+      process.env[envPrefix + 'USERNAME'] = username
+      process.env[envPrefix + 'PASSWORD'] = password
+
+      factory.aspect(locator, amqpManifest)
+
+      const manifest = amqp.create.mock.calls[0][0]
+      const origin = manifest[key]
+      const url = new URL(origin)
+
+      expect(url.username).toStrictEqual(username)
+      expect(url.password).toStrictEqual(password)
+    })
+
+    it('should add credentials to URLs from environment', async () => {
+      const key = sample(Object.keys(amqpManifest))
+      const hostname = generate().toLowerCase()
+      const override = { [key]: 'amqp://' + hostname }
+      const json = JSON.stringify(override)
+      const base64 = btoa(json)
+      const locator = new Locator(generate(), generate())
+      const envPrefix = 'TOA_ORIGINS_' + locator.uppercase + '_' + up(key) + '_'
+      const username = generate()
+      const password = generate()
+
+      process.env['TOA_ORIGINS_' + locator.uppercase] = base64
+      process.env[envPrefix + 'USERNAME'] = username
+      process.env[envPrefix + 'PASSWORD'] = password
+
+      factory.aspect(locator, fixtures.manifest)
+
+      const manifest = amqp.create.mock.calls[0][0]
+      const origin = manifest[key]
+      const url = new URL(origin)
+
+      expect(url.hostname).toStrictEqual(hostname)
+      expect(url.username).toStrictEqual(username)
+      expect(url.password).toStrictEqual(password)
+    })
+  })
+})
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @param {string} protocol
+ * @return {toa.origins.Manifest}
+ */
+function filterManifest (manifest, protocol) {
+  const result = {}
+
+  for (const [origin, reference] of Object.entries(manifest)) {
+    if (reference.slice(0, protocol.length) === protocol) result[origin] = reference
+  }
+
+  return result
+}

package/{src → source}/index.js

@@ -1,7 +1,9 @@
 'use strict'
 
 const { manifest } = require('./manifest')
+const { deployment } = require('./deployment')
 const { Factory } = require('./factory')
 
 exports.manifest = manifest
+exports.deployment = deployment
 exports.Factory = Factory

package/source/manifest.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const schemas = require('./schemas')
+const protocols = require('./protocols')
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @returns {toa.origins.Manifest}
+ */
+function manifest (manifest) {
+  schemas.manifest.validate(manifest)
+
+  for (const uri of Object.values(manifest)) {
+    const protocol = new URL(uri).protocol
+    const supported = protocols.find((provider) => provider.protocols.includes(protocol))
+
+    if (supported === undefined) throw new Error(`'${protocol}' protocol is not supported`)
+  }
+
+  return manifest
+}
+
+exports.manifest = manifest

package/source/manifest.test.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { PROTOCOLS } = require('./.test/constants')
+
+const { manifest } = require('../')
+
+it('should be', async () => {
+  expect(manifest).toBeInstanceOf(Function)
+})
+
+it('should return manifest', async () => {
+  const input = { [generate()]: 'http://' + generate() }
+  const output = manifest(input)
+
+  expect(output).toStrictEqual(input)
+})
+
+it('should fail if not Record<string, string>', async () => {
+  const input = /** @type {toa.origins.Manifest} */ {
+    foo: {
+      bar: 'dev://null'
+    }
+  }
+
+  expect(() => manifest(input)).toThrow()
+})
+
+it('should pass if valid', async () => {
+  const input = { foo: 'amqp://' + generate() }
+
+  expect(() => manifest(input)).not.toThrow()
+})
+
+it('should fail if not uri', async () => {
+  const input = { [generate()]: generate() }
+
+  expect(() => manifest(input)).toThrow('must match format')
+})
+
+it('should throw if protocol is not supported', async () => {
+  const input = { foo: 'wat://' + generate() }
+
+  expect(() => manifest(input)).toThrow('is not supported')
+})
+
+it.each(PROTOCOLS)('should support %s protocol', async (protocol) => {
+  const input = { foo: protocol + '//' + generate() }
+
+  expect(() => manifest(input)).not.toThrow()
+})

package/source/protocols/amqp/.test/aspect.fixtures.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { random } = require('@toa.io/generic')
+
+const manifest = {}
+const originCount = random(5) + 2
+
+for (let j = 0; j < originCount; j++) {
+  const origin = generate()
+
+  manifest[origin] = 'amqp://' + generate()
+}
+
+exports.manifest = manifest