@toa.io/extensions.origins 0.7.2-dev.0 → 0.8.0-dev.1

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@toa.io/extensions.origins",
-  "version": "0.7.2-dev.0",
-  "description": "Toa operations context HTTP client",
+  "version": "0.8.0-dev.1",
+  "description": "Toa Origins",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "src/index.js",
+  "main": "source/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -19,14 +19,15 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/core": "0.7.2-dev.0",
-    "@toa.io/generic": "0.7.0",
-    "@toa.io/schema": "0.7.2-dev.0",
-    "@toa.io/yaml": "0.7.2-dev.0",
+    "@toa.io/core": "0.8.0-dev.1",
+    "@toa.io/generic": "0.8.0-dev.1",
+    "@toa.io/schemas": "0.8.0-dev.1",
+    "@toa.io/yaml": "0.7.2-dev.2",
+    "comq": "0.6.0",
     "node-fetch": "2.6.7"
   },
   "devDependencies": {
     "@types/node-fetch": "2.6.2"
   },
-  "gitHead": "e9a0781a797c3ec5e2dff2251fbc683ec8d08ecf"
+  "gitHead": "9bb2f372c556440ee3e2a42e10eb415220ba1cb0"
 }

package/readme.md

@@ -0,0 +1,113 @@
+# Toa Origins
+
+`origins` extension enables external communications over supported protocols (HTTP and AMQP).
+
+## TL;DR
+
+```yaml
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
+
+origins:
+  docs: http://www.domain.com/docs/
+  amazon: amqps://amqp.amazon.com
+```
+
+```javascript
+// Node.js bridge 
+async function transition (input, object, context) {
+  // direct Aspect invocation
+  await context.aspects.http('docs', './example', { method: 'GET' })
+
+  // shortcuts
+  await context.http.docs.example.get() // GET http://www.domain.com/docs/example
+  await context.amqp.amazon.emit('something_happened', { really: true })
+}
+```
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    amazon: amqps://amqp.azure.com
+    amazon@staging: amqp://amqp.stage
+```
+
+## Manifest
+
+`origins` manifest is an object conforming declaring origin names as keys an origin URLs as values.
+
+## HTTP Aspect
+
+Uses [node-fetch](https://github.com/node-fetch/node-fetch) and returns its result.
+
+Aspect invocation function
+signature: `async (origin: string, rel: string, reuest: fetch.Request): fetch.Response`
+
+- `origin`: name of the origin in the manifest
+- `rel`: relative reference to a resource
+- `request`: `Request` form `node-fetch`
+
+### Absolute URLs
+
+Requests to arbitrary URLs can be implemented with overloaded direct Aspect invocation.
+
+`async (url: string, request: fetch.Request): fetch.Response`
+
+By default, requests to arbitrary URLs are not allowed and must be explicitly permitted by setting
+permissions in the Origins Annotation.
+
+The Rules object is stored in the `.http` property of the corresponding component. Each key in the
+Rules object is a regular expression that URLs will be tested against, and each value is a
+permission — either `true` to allow the URL or `false` to deny it. In cases where a URL matches
+multiple rules, denial takes priority.
+
+> The `null` key is a special case that represents "any URL".
+
+#### Example
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    .http:
+      /^https?:\/\/api.domain.com/: true
+      /^http:\/\/sandbox.domain.com/@staging: true  # staging environment
+      /.*hackers.*/: false                          # deny rule
+      ~: true                                       # allow any URL
+```
+
+```javascript
+// Node.js bridge 
+async function transition (input, object, context) {
+  await context.aspects.http('https://api.domain.com/example', { method: 'POST' })
+}
+```
+
+#### `null` origins
+
+To enable the extension for a component that uses arbitrary URLs without any specific origins to
+declare, the Origins manifest should be set to `null`.
+
+```yaml
+# manifest.toa.yaml
+origins: ~
+```
+
+## AMQP Aspect
+
+Uses [ComQ](https://github.com/toa-io/comq), thus, provides interface of `comq.IO` restricted
+to `emit` and `request` methods.
+
+AMQP origins can have credential secrets deployed. Secret's name must
+follow `toa-origins-{namespace}-{component}-{origin}` and it must have keys `username`
+and `password`.
+
+### Example
+
+```shell
+# deploy credentials to the current kubectl context
+$ toa conceal toa-origins-dummies-dummiy-messages username developer
+$ toa conceal toa-origins-dummies-dummiy-messages password secret
+```

package/source/.deployment/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { uris } = require('./uris')
+
+exports.uris = uris

package/source/.deployment/uris.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const { PREFIX } = require('../constants')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} instances
+ * @param {toa.origins.Annotations} annotations
+ * @returns {toa.deployment.dependency.Variables}
+ */
+function uris (instances, annotations) {
+  const variables = {}
+
+  for (const [id, annotation] of Object.entries(annotations)) {
+    const component = instances.find((instance) => instance.locator.id === id)
+
+    if (component === undefined) throw new Error(`Origins annotations error: component '${id}' is not found`)
+
+    for (const origin of Object.keys(annotation)) {
+      if (!(origin in component.manifest)) {
+        throw new Error(`Origins annotations error: component '${id}' doesn't have '${origin}' origin`)
+      }
+    }
+
+    const name = PREFIX + component.locator.uppercase
+    const json = JSON.stringify(annotation)
+    const value = btoa(json)
+    const variable = { name, value }
+
+    variables[component.locator.label] = [variable]
+  }
+
+  return variables
+}
+
+exports.uris = uris

package/source/.test/constants.js

@@ -0,0 +1,3 @@
+'use strict'
+
+exports.PROTOCOLS = ['http:', 'https:', 'amqp:', 'amqps:']

package/source/.test/deployment.fixtures.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { Locator } = require('@toa.io/core')
+const { random, sample } = require('@toa.io/generic')
+
+const { PROTOCOLS } = require('./constants')
+
+const component = () => ({
+  locator: new Locator(generate(), generate()),
+  manifest: { [generate()]: sample(PROTOCOLS) + '//' + generate() }
+})
+
+const components = () => {
+  const length = random(20) + 10
+
+  return Array.from({ length }, component)
+}
+
+exports.components = components

package/source/.test/factory.fixtures.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+/** @type {Record<string, string>} */
+const manifest = {
+  [generate()]: 'https://toa.io',
+  [generate()]: 'https://api.domain.com',
+  [generate()]: 'amqp://localhost',
+  [generate()]: 'amqps://localhost'
+}
+
+exports.manifest = manifest

package/source/constants.js

@@ -0,0 +1,3 @@
+'use strict'
+
+exports.PREFIX = 'TOA_ORIGINS_'

package/source/deployment.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { merge } = require('@toa.io/generic')
+const schemas = require('./schemas')
+const protocols = require('./protocols')
+const create = require('./.deployment')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} instances
+ * @param {toa.origins.Annotations} annotations
+ * @returns {toa.deployment.dependency.Declaration}
+ */
+function deployment (instances, annotations = {}) {
+  schemas.annotations.validate(annotations)
+
+  const uris = create.uris(instances, annotations)
+  const variables = { ...uris }
+
+  protocols.reduce((variables, provider) => {
+    const specifics = provider.deployment?.(instances)
+
+    return merge(variables, specifics)
+  }, variables)
+
+  return { variables }
+}
+
+exports.deployment = deployment

package/source/deployment.test.js

@@ -0,0 +1,158 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { sample, letters: { up } } = require('@toa.io/generic')
+
+const fixtures = require('./.test/deployment.fixtures')
+const { deployment } = require('../')
+
+it('should be', async () => {
+  expect(deployment).toBeInstanceOf(Function)
+})
+
+/** @type {toa.norm.context.dependencies.Instance[]} */
+let components
+
+/** @type {string} */
+let origin
+
+/** @type {toa.norm.context.dependencies.Instance} */
+let component
+
+beforeEach(() => {
+  components = fixtures.components()
+
+  component = sample(components)
+
+  const origins = Object.keys(component.manifest)
+
+  origin = sample(origins)
+})
+
+describe('validation', () => {
+  it('should throw on annotation component mismatch', async () => {
+    const id = generate()
+
+    const annotations = {
+      [id]: {
+        [origin]: 'dev://' + generate()
+      }
+    }
+
+    expect(() => deployment(components, annotations))
+      .toThrow(`Origins annotations error: component '${id}' is not found`)
+  })
+
+  it('should throw on annotation origin mismatch', async () => {
+    const id = component.locator.id
+    const origin = generate()
+
+    const annotations = {
+      [id]: {
+        [origin]: 'dev://' + generate()
+      }
+    }
+
+    expect(() => deployment(components, annotations))
+      .toThrow(`Origins annotations error: component '${id}' doesn't have '${origin}' origin`)
+  })
+
+  it('should throw if annotation is not valid', async () => {
+    const annotations = /** @type {toa.origins.Annotations} */ { [component.locator.id]: generate() }
+
+    expect(() => deployment(components, annotations)).toThrow('must be object')
+  })
+
+  it('should throw if annotation is URI is not valid', async () => {
+    const annotations = {
+      [component.locator.id]: {
+        [origin]: 'hello!'
+      }
+    }
+
+    expect(() => deployment(components, annotations)).toThrow('must match format')
+  })
+})
+
+it('should create variables', () => {
+  const value = 'dev://' + generate()
+
+  /** @type {toa.origins.Annotations} */
+  const annotations = {
+    [component.locator.id]: {
+      [origin]: value
+    }
+  }
+
+  const output = deployment(components, annotations)
+
+  expect(output.variables).not.toBeUndefined()
+
+  const variables = output.variables[component.locator.label]
+  const varName = 'TOA_ORIGINS_' + component.locator.uppercase
+  const variable = findVariable(variables, varName)
+
+  expect(variable).toBeDefined()
+
+  const json = JSON.stringify(annotations[component.locator.id])
+  const base64 = btoa(json)
+
+  expect(variable.value).toStrictEqual(base64)
+})
+
+describe('amqp', () => {
+  beforeEach(() => {
+    const amqpComponents = components.filter(
+      (component) => {
+        const origin = Object.keys(component.manifest)
+        const url = new URL(component.manifest[origin])
+
+        return url.protocol === 'amqp:' || url.protocol === 'amqps:'
+      }
+    )
+
+    component = sample(amqpComponents)
+
+    const origins = Object.keys(component.manifest)
+
+    origin = sample(origins)
+  })
+
+  it('should create credential secrets', () => {
+    /** @type {toa.origins.Annotations} */
+    const annotations = {
+      [component.locator.id]: {
+        [origin]: 'amqps://whatever'
+      }
+    }
+
+    const output = deployment(components, annotations)
+    const variables = output.variables[component.locator.label]
+
+    expect(variables).toBeDefined()
+
+    const envPrefix = `TOA_ORIGINS_${component.locator.uppercase}_${up(origin)}_`
+    const secretName = `toa-origins-${component.locator.label}-${origin}`
+
+    for (const property of ['username', 'password']) {
+      const variableName = envPrefix + up(property)
+      const variable = findVariable(variables, variableName)
+
+      expect(variable).toBeDefined()
+
+      expect(variable.secret).toStrictEqual({
+        name: secretName,
+        key: property
+      })
+    }
+  })
+})
+
+/**
+ * @param {toa.deployment.dependency.Variable[]} variables
+ * @param {string} name
+ * @returns {toa.deployment.dependency.Variable}
+ */
+function findVariable (variables, name) {
+  return variables.find((variable) => variable.name === name)
+}

package/source/env.js

@@ -0,0 +1,50 @@
+'use strict'
+
+const { PREFIX } = require('./constants')
+const { overwrite, remap, letters: { up } } = require('@toa.io/generic')
+
+/**
+ * @param {toa.core.Locator} locator
+ * @param {toa.origins.Manifest} manifest
+ */
+function apply (locator, manifest) {
+  const variable = PREFIX + locator.uppercase
+  const envValue = readEnv(variable)
+
+  overwrite(manifest, envValue)
+  addCredentials(manifest, variable)
+}
+
+function readEnv (variable) {
+  if (!(variable in process.env)) return
+
+  const base64 = process.env[variable]
+  const json = atob(base64)
+
+  return JSON.parse(json)
+}
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @param {string} variable
+ */
+function addCredentials (manifest, variable) {
+  const prefix = variable + '_'
+
+  remap(manifest, (reference, origin) => {
+    const originPrefix = prefix + up(origin)
+    const username = process.env[originPrefix + '_USERNAME']
+    const password = process.env[originPrefix + '_PASSWORD']
+
+    if (username === undefined && password === undefined) return
+
+    const url = new URL(reference)
+
+    url.username = username
+    url.password = password
+
+    manifest[origin] = url.href
+  })
+}
+
+exports.apply = apply

package/source/factory.js

@@ -0,0 +1,46 @@
+'use strict'
+
+const protocols = require('./protocols')
+const env = require('./env')
+
+class Factory {
+  /**
+   * @param {toa.core.Locator} locator
+   * @param {toa.origins.Manifest} manifest
+   * @return {toa.core.extensions.Aspect[]}
+   */
+  aspect (locator, manifest) {
+    env.apply(locator, manifest)
+
+    return protocols.map((protocol) => this.#createAspect(protocol, manifest))
+  }
+
+  /**
+   * @param {object} protocol
+   * @param {toa.origins.Manifest} manifest
+   * @return {toa.core.extensions.Aspect}
+   */
+  #createAspect (protocol, manifest) {
+    const protocolManifest = {}
+
+    let properties
+
+    // let properties
+
+    for (const [origin, reference] of Object.entries(manifest)) {
+      if (origin[0] === '.') {
+        if (origin.substring(1) === protocol.id) properties = reference
+
+        continue
+      }
+
+      const url = new URL(reference)
+
+      if (protocol.protocols.includes(url.protocol)) protocolManifest[origin] = reference
+    }
+
+    return protocol.create(protocolManifest, properties)
+  }
+}
+
+exports.Factory = Factory

package/source/factory.test.js

@@ -0,0 +1,140 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const { Locator } = require('@toa.io/core')
+const { sample, overwrite, letters: { up } } = require('@toa.io/generic')
+
+jest.mock('./protocols/http/aspect')
+jest.mock('./protocols/amqp/aspect')
+
+const http = require('./protocols/http/aspect')
+const amqp = require('./protocols/amqp/aspect')
+
+const fixtures = require('./.test/factory.fixtures')
+const { Factory } = require('../')
+
+let factory
+
+beforeEach(() => {
+  jest.clearAllMocks()
+
+  factory = new Factory()
+})
+
+it('should create aspects', () => {
+  factory.aspect(new Locator(generate(), generate()), fixtures.manifest)
+
+  const httpManifest = filterManifest(fixtures.manifest, 'http')
+  const amqpManifest = filterManifest(fixtures.manifest, 'amqp')
+
+  expect(http.create).toHaveBeenCalledWith(httpManifest, undefined)
+  expect(amqp.create).toHaveBeenCalledWith(amqpManifest, undefined)
+})
+
+describe('env', () => {
+  it('should overwrites URLs from environment', async () => {
+    const httpManifest = filterManifest(fixtures.manifest, 'http')
+    const key = sample(Object.keys(httpManifest))
+    const override = { [key]: 'http://' + generate() }
+    const json = JSON.stringify(override)
+    const base64 = btoa(json)
+    const locator = new Locator(generate(), generate())
+
+    process.env['TOA_ORIGINS_' + locator.uppercase] = base64
+
+    factory.aspect(locator, fixtures.manifest)
+
+    const expected = overwrite(httpManifest, override)
+
+    expect(http.create.mock.calls[0][0]).toStrictEqual(expected)
+  })
+
+  describe('amqp', () => {
+    /** @type {toa.origins.Manifest} */
+    let amqpManifest
+
+    beforeEach(() => {
+      amqpManifest = filterManifest(fixtures.manifest, 'amqp')
+    })
+
+    it('should add credentials from environment', async () => {
+      const key = sample(Object.keys(amqpManifest))
+      const locator = new Locator(generate(), generate())
+      const envPrefix = 'TOA_ORIGINS_' + locator.uppercase + '_' + up(key) + '_'
+      const username = generate()
+      const password = generate()
+
+      process.env[envPrefix + 'USERNAME'] = username
+      process.env[envPrefix + 'PASSWORD'] = password
+
+      factory.aspect(locator, amqpManifest)
+
+      const manifest = amqp.create.mock.calls[0][0]
+      const origin = manifest[key]
+      const url = new URL(origin)
+
+      expect(url.username).toStrictEqual(username)
+      expect(url.password).toStrictEqual(password)
+    })
+
+    it('should add credentials to URLs from environment', async () => {
+      const key = sample(Object.keys(amqpManifest))
+      const hostname = generate().toLowerCase()
+      const override = { [key]: 'amqp://' + hostname }
+      const json = JSON.stringify(override)
+      const base64 = btoa(json)
+      const locator = new Locator(generate(), generate())
+      const envPrefix = 'TOA_ORIGINS_' + locator.uppercase + '_' + up(key) + '_'
+      const username = generate()
+      const password = generate()
+
+      process.env['TOA_ORIGINS_' + locator.uppercase] = base64
+      process.env[envPrefix + 'USERNAME'] = username
+      process.env[envPrefix + 'PASSWORD'] = password
+
+      factory.aspect(locator, fixtures.manifest)
+
+      const manifest = amqp.create.mock.calls[0][0]
+      const origin = manifest[key]
+      const url = new URL(origin)
+
+      expect(url.hostname).toStrictEqual(hostname)
+      expect(url.username).toStrictEqual(username)
+      expect(url.password).toStrictEqual(password)
+    })
+  })
+
+  describe('http', () => {
+    it('should read properties', async () => {
+      const properties = {
+        '.http': {
+          [generate()]: generate()
+        }
+      }
+
+      const locator = new Locator(generate(), generate())
+      const json = JSON.stringify(properties)
+
+      process.env['TOA_ORIGINS_' + locator.uppercase] = btoa(json)
+
+      factory.aspect(locator, fixtures.manifest)
+
+      expect(http.create).toHaveBeenCalledWith(expect.anything(), properties['.http'])
+    })
+  })
+})
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @param {string} protocol
+ * @return {toa.origins.Manifest}
+ */
+function filterManifest (manifest, protocol) {
+  const result = {}
+
+  for (const [origin, reference] of Object.entries(manifest)) {
+    if (reference.slice(0, protocol.length) === protocol) result[origin] = reference
+  }
+
+  return result
+}

package/{src → source}/index.js

@@ -1,7 +1,9 @@
 'use strict'
 
 const { manifest } = require('./manifest')
+const { deployment } = require('./deployment')
 const { Factory } = require('./factory')
 
 exports.manifest = manifest
+exports.deployment = deployment
 exports.Factory = Factory

package/source/manifest.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const schemas = require('./schemas')
+const protocols = require('./protocols')
+
+/**
+ * @param {toa.origins.Manifest} manifest
+ * @returns {toa.origins.Manifest}
+ */
+function manifest (manifest) {
+  if (manifest === null) return {}
+
+  schemas.manifest.validate(manifest)
+
+  for (const uri of Object.values(manifest)) {
+    const protocol = new URL(uri).protocol
+    const supported = protocols.find((provider) => provider.protocols.includes(protocol))
+
+    if (supported === undefined) throw new Error(`'${protocol}' protocol is not supported`)
+  }
+
+  return manifest
+}
+
+exports.manifest = manifest