@toa.io/extensions.origins 0.20.0-dev.9 → 0.20.1-alpha.0

package/package.json

@@ -1,10 +1,9 @@
 {
   "name": "@toa.io/extensions.origins",
-  "version": "0.20.0-dev.9",
+  "version": "0.20.1-alpha.0",
   "description": "Toa Origins",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "source/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -15,19 +14,26 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "test": "echo \"Error: run tests from root\" && exit 1"
-  },
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
   "dependencies": {
-    "@toa.io/core": "0.20.0-dev.9",
-    "@toa.io/generic": "0.20.0-dev.9",
-    "@toa.io/schemas": "0.20.0-dev.9",
-    "@toa.io/yaml": "0.20.0-dev.9",
-    "comq": "0.7.0",
+    "@toa.io/core": "0.21.0-alpha.0",
+    "@toa.io/generic": "0.20.1-alpha.0",
+    "@toa.io/pointer": "0.20.1-alpha.0",
+    "@toa.io/schemas": "0.20.1-alpha.0",
+    "comq": "0.10.1",
+    "msgpackr": "1.9.5",
     "node-fetch": "2.6.7"
   },
   "devDependencies": {
     "@types/node-fetch": "2.6.2"
   },
-  "gitHead": "4e503ffe4fe6dfab1165e795832d3d4fba711582"
+  "scripts": {
+    "transpile": "tsc"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
+  },
+  "gitHead": "ed28dc0d2823022fbb1188bb28b994bc827a4432"
 }

package/readme.md

@@ -1,6 +1,6 @@
 # Toa Origins
 
-Enables external communications over supported protocols (HTTP and AMQP).
+External communications with permissions over supported protocols (HTTP and AMQP).
 
 ## TL;DR
 
@@ -11,18 +11,17 @@ namespace: dummies
 
 origins:
   docs: http://www.domain.com/docs/
-  amazon: amqps://amqp.amazon.com
+  queues: ~
 ```
 
 ```javascript
-// Node.js bridge 
 async function transition (input, object, context) {
-  // direct Aspect invocation
-  await context.aspects.http('docs', './example', { method: 'GET' })
-
-  // shortcuts
   await context.http.docs.example.get() // GET http://www.domain.com/docs/example
-  await context.amqp.amazon.emit('something_happened', { really: true })
+  await context.amqp.queues.emit('something_happened', { really: true })
+
+  // direct Aspect invocation
+  await context.aspects.http('docs', 'example', { method: 'GET' })
+  await context.aspects.http('http://api.example.com', { method: 'GET' })
 }
 ```
 
@@ -30,31 +29,9 @@ async function transition (input, object, context) {
 # context.toa.yaml
 origins:
   dummies.dummy:
-    amazon: amqps://amqp.azure.com
-    amazon@staging: amqp://amqp.stage
-```
-
-## Manifest
-
-`origins` manifest is an object with origin names as keys an origin URLs as values.
-Component's `origins` manifest can be overridden by the Context `origins` annotation.
-
-### Sharded Connections
-
-Origin value may contain [shards](/libraries/generic/readme.md#shards) placeholders.
-
-### Environment Variables
-
-Origin value may contain environment variable placeholders.
-
-```yaml
-# manifest.toa.yaml
-origins:
-  foo@dev: stage${STAGE_NUMBER}.stages.com
+    queues: amqps://amqp.azure.com
 ```
 
-This is only usable in local development environment.
-
 ## HTTP Aspect
 
 Uses [node-fetch](https://github.com/node-fetch/node-fetch) and returns its result.
@@ -63,46 +40,37 @@ Aspect invocation function
 signature: `async (origin: string, rel: string, reuest: fetch.Request): fetch.Response`
 
 - `origin`: name of the origin in the manifest
-- `rel`: relative reference to a resource
+- `rel`: reference to a resource relative to the origin's value
 - `request`: `Request` form `node-fetch`
 
 ### Absolute URLs
 
-Requests to arbitrary URLs can be implemented with overloaded direct Aspect invocation.
-
 `async (url: string, request: fetch.Request): fetch.Response`
 
-By default, requests to arbitrary URLs are not allowed and must be explicitly permitted by setting
-permissions in the Origins Annotation.
-
-The Rules object is stored in the `.http` property of the corresponding component. Each key in the
-Rules object is a regular expression that URLs will be tested against, and each value is a
-permission — either `true` to allow the URL or `false` to deny it. In cases where a URL matches
-multiple rules, denial takes priority.
-
-> The `null` key is a special case that represents "any URL".
-
-#### Example
+Requests to arbitrary URLs can be implemented with overloaded direct Aspect invocation.
 
-```yaml
-# context.toa.yaml
-origins:
-  dummies.dummy:
-    .http:
-      /^https?:\/\/api.domain.com/: true
-      /^http:\/\/sandbox.domain.com/@staging: true  # staging environment
-      /.*hackers.*/: false                          # deny rule
-      ~: true                                       # allow any URL
-```
+By default, requests to arbitrary URLs are not allowed and must be explicitly permitted by setting
+permissions in the [Origins annotation](#context-annotation).
 
 ```javascript
-// Node.js bridge 
+// Node.js bridge
 async function transition (input, object, context) {
   await context.aspects.http('https://api.domain.com/example', { method: 'POST' })
 }
 ```
 
-#### `null` manifest
+## AMQP Aspect
+
+Uses [ComQ](https://github.com/toa-io/comq), thus, provides interface of `comq.IO` restricted
+to `emit` and `request` methods.
+
+## Manifest
+
+`origins` manifest is a [Pointer](/libraries/pointer) with origin names as keys.
+Its values can be overridden by the context [annotation](#context-annotation).
+If the value is `null`, then it _must_ be overriden.
+
+### `null` manifest
 
 To enable the extension for a component that uses arbitrary URLs without any specific origins to
 declare, the Origins manifest should be set to `null`.
@@ -112,19 +80,54 @@ declare, the Origins manifest should be set to `null`.
 origins: ~
 ```
 
-## AMQP Aspect
+## Context annotation
 
-Uses [ComQ](https://github.com/toa-io/comq), thus, provides interface of `comq.IO` restricted
-to `emit` and `request` methods.
+The `origins` annotation is a set of Pointers defined for the corresponding components.
+The values of each pointer override the values defined in the manifest.
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    queues: amqps://amqp.azure.com
+```
+
+### HTTP URL Permissions
+
+The rules for arbitrary HTTP requests are stored in the `http` property of the corresponding
+component as an object.
+Each key in the rules object is a regular expression that URLs will be tested against, and each
+value is a permission — either `true` to allow the URL or `false` to deny it.
+In cases where a URL matches multiple rules, denial takes priority.
 
-AMQP origins can have credential secrets deployed. Secret's name must
-follow `origins-{namespace}-{component}-{origin}` and it must have keys `username`
-and `password`.
+> The `null` is a special key that represents any URL.
 
-### Example
+#### Example
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    http:
+      /^https?:\/\/api.domain.com/: true
+      /^http:\/\/sandbox.domain.com/@staging: true  # `staging` environment
+      /.*hackers.*/: false                          # deny
+      ~: true                                       # allow any URL
+```
+
+## Deployment
+
+Each key of the annotation is deployed as a Pointer with ID
+following `origins-{component}-{origin}` with dots replaced by dashes.
+This means credentials for the declared origins must be deployed as follows:
+
+```yaml
+# context.toa.yaml
+origins:
+  dummies.dummy:
+    queues: amqp://rmq.example.com
+```
 
 ```shell
-# deploy credentials to the current kubectl context
-$ toa conceal origins-dummies-dummiy-messages username developer
-$ toa conceal origins-dummies-dummiy-messages password secret
+$ toa conceal origins-dummies-dummy-queues username=developer password=secret
 ```

package/schemas/annotation.cos.yaml

@@ -0,0 +1,3 @@
+/^[a-zA-Z0-9]{1,32}\.[a-zA-Z0-9]{1,32}$/:
+  /^\.http$/: <boolean>
+  /^[a-zA-Z0-9]{1,32}$/+: string

package/schemas/manifest.cos.yaml

@@ -0,0 +1,4 @@
+/^[a-zA-Z0-9]{1,32}$/:
+  - type: string
+    format: uri
+  - type: 'null'

package/source/Factory.ts

@@ -0,0 +1,88 @@
+import { decode } from 'msgpackr'
+import { resolve, type URIMap } from '@toa.io/pointer'
+import { memo } from '@toa.io/generic'
+import { type Protocol, protocols } from './protocols'
+import { ENV_PREFIX, ID_PREFIX, PROPERTIES_SUFFIX } from './extension'
+import type { Properties } from './annotation'
+import type { Locator, extensions } from '@toa.io/core'
+import type { Manifest } from './manifest'
+
+export class Factory implements extensions.Factory {
+  public aspect (locator: Locator, manifest: Manifest): extensions.Aspect[] {
+    return protocols.map((protocol) => this.createAspect(locator, manifest, protocol))
+  }
+
+  private createAspect (locator: Locator, manifest: Manifest, protocol: Protocol):
+  extensions.Aspect {
+    const resolver = this.resolver(locator, manifest, protocol)
+
+    return protocol.create(resolver)
+  }
+
+  private resolver (locator: Locator, manifest: Manifest, protocol: Protocol): Resolver {
+    return memo(async (): Promise<Configuration> => {
+      const uris = await this.getURIs(locator, manifest)
+      const allProperties = this.getProperties(locator)
+
+      const origins = this.filterOrigins(uris, protocol.protocols)
+      const properties = allProperties['.' + protocol.id as keyof Properties] ?? {}
+
+      return { origins, properties }
+    })
+  }
+
+  private async getURIs (locator: Locator, manifest: Manifest): Promise<URIMap> {
+    const map: URIMap = {}
+
+    if (manifest === null) return map
+
+    for (const [name, value] of Object.entries(manifest))
+      try {
+        map[name] = await this.readOrigin(locator, name)
+      } catch {
+        // eslint-disable-next-line max-depth
+        if (value === null) throw new Error(`Origin value ${name} is not defined`)
+
+        map[name] = [value]
+      }
+
+    return map
+  }
+
+  private filterOrigins (uris: URIMap, protocols: string[]): URIMap {
+    const filtered: URIMap = {}
+
+    for (const [name, references] of Object.entries(uris)) {
+      const url = new URL(references[0])
+
+      if (protocols.includes(url.protocol))
+        filtered[name] = references
+    }
+
+    return filtered
+  }
+
+  private async readOrigin (locator: Locator, name: string): Promise<string[]> {
+    const id = ID_PREFIX + locator.label
+
+    return await resolve(id, name)
+  }
+
+  private getProperties (locator: Locator): Properties {
+    const variable = ENV_PREFIX + locator.uppercase + PROPERTIES_SUFFIX
+    const value = process.env[variable]
+
+    if (value === undefined) return {}
+
+    const buffer = Buffer.from(value, 'base64')
+
+    return decode(buffer)
+  }
+}
+
+export interface Configuration {
+  origins: URIMap
+  properties: Record<string, boolean>
+}
+
+export type Resolver = () => Promise<Configuration>

package/source/annotation.test.ts

@@ -0,0 +1,150 @@
+import { generate } from 'randomstring'
+import { Locator } from '@toa.io/core'
+import { split, normalize, type Component, type Annotation } from './annotation'
+import { type Instance } from './extension'
+
+let annotation: Annotation
+let instances: Instance[]
+let component: Component
+
+const locator = new Locator(generate(), generate())
+
+beforeEach(() => {
+  annotation = {}
+  instances = []
+})
+
+describe('normalize', () => {
+  it('should throw if a key is not a component ID', async () => {
+    annotation = {
+      dummies: {
+        one: 'amqp://host{0,2}-' + generate()
+      }
+    }
+
+    expect(run)
+      .toThrow('not expected')
+  })
+
+  it('should not throw if valid', async () => {
+    annotation = {
+      'dummies.dummy': {
+        '.http': {
+          '/.*hackers.*/': false
+        },
+        one: 'http://host{0,2}-' + generate(),
+        two: [
+          'http://hostA-' + generate(),
+          'https://hostB-' + generate()
+        ],
+        three: [
+          'amqp://hostB-' + generate(),
+          'amqps://hostB-' + generate()
+        ]
+      }
+    }
+
+    expect(run).not.toThrow()
+  })
+
+  it('should merge defaults', async () => {
+    annotation = {}
+
+    const one = 'http://api.' + generate()
+
+    instances.push({
+      locator,
+      manifest: { one }
+    } as unknown as Instance)
+
+    run()
+
+    expect(annotation)
+      .toStrictEqual({
+        [locator.id]: { one }
+      })
+  })
+
+  it('should thow if null origin is not defined', async () => {
+    annotation = {}
+
+    instances.push({
+      locator,
+      manifest: {
+        one: null
+      }
+    } as unknown as Instance)
+
+    expect(run)
+      .toThrow('is not defined for')
+  })
+
+  it('should not thow if null origin is defined', async () => {
+    annotation = {
+      [locator.id]: {
+        one: 'http://api.' + generate()
+      }
+    }
+
+    instances.push({
+      locator,
+      manifest: {
+        one: null
+      }
+    } as unknown as Instance)
+
+    expect(run)
+      .not.toThrow()
+  })
+
+  it('should throw if protocol is not supported', async () => {
+    annotation = {
+      [locator.id]: {
+        one: 'mqtt://host-' + generate()
+      }
+    }
+
+    expect(run)
+      .toThrow('is not supported')
+  })
+
+  it('should throw if origin url protocols are inconsistent', async () => {
+    annotation = {
+      [locator.id]: {
+        one: ['http://host-' + generate(), 'amqp://host-' + generate()]
+      }
+    }
+
+    expect(run)
+      .toThrow('inconsistent')
+  })
+
+  it('should not throw if annotation is undefined', async () => {
+    expect(run).not.toThrow()
+  })
+})
+
+describe('split', () => {
+  it('should split', async () => {
+    const one = 'amqp://host{0,2}-' + generate()
+
+    component = {
+      '.http': {
+        '/.*hackers.*/': false
+      },
+      one
+    }
+
+    const { origins, properties } = split(component)
+
+    expect(origins)
+      .toStrictEqual({ one })
+
+    expect(properties)
+      .toStrictEqual({ '.http': component['.http'] })
+  })
+})
+
+function run (): void {
+  normalize(instances, annotation)
+}

package/source/annotation.ts

@@ -0,0 +1,83 @@
+import { resolve } from 'node:path'
+import * as schemas from '@toa.io/schemas'
+import { type Protocol, protocols } from './protocols'
+import type { Instance } from './extension'
+
+export function normalize (instances: Instance[], annotation: Annotation): void {
+  schema.validate(annotation)
+  mergeDefaults(annotation, instances)
+  checkProtocols(annotation)
+}
+
+export function split (component: Component): {
+  origins: Origins
+  properties: Properties
+} {
+  const origins: Origins = {}
+  const properties: Properties = {}
+
+  for (const [key, value] of Object.entries(component))
+    if (key[0] === '.') properties[key as keyof Properties] = value
+    else origins[key] = value
+
+  return { origins, properties }
+}
+
+function mergeDefaults (annotation: Annotation, instances: Instance[]): void {
+  for (const instance of instances) {
+    const component = annotation[instance.locator.id] as Origins ?? {}
+
+    annotation[instance.locator.id] = mergeInstance(component, instance)
+  }
+}
+
+function mergeInstance (origins: Origins, instance: Instance): Component {
+  if (instance.manifest === null) return origins
+
+  for (const [origin, value] of Object.entries(instance.manifest))
+    if (origins[origin] === undefined)
+      if (value === null)
+        throw new Error(`Origin '${origin}' is not defined for '${instance.locator.id}'`)
+      else origins[origin] = value
+
+  return origins
+}
+
+function checkProtocols (annotation: Annotation): void {
+  for (const component of Object.values(annotation)) {
+    const { origins } = split(component)
+    const urlSets = Object.values(origins)
+
+    for (const urls of urlSets)
+      checkURLs(Array.isArray(urls) ? urls : [urls])
+  }
+}
+
+function checkURLs (urls: string[]): void {
+  let id: string | null = null
+
+  for (const url of urls) {
+    const protocol = resolveProtocol(url)
+
+    if (id === null) id = protocol.id
+    else if (id !== protocol.id)
+      throw new Error(`Origin has inconsistent protocols: ${id}, ${protocol.id}`)
+  }
+}
+
+function resolveProtocol (reference: string): Protocol {
+  const url = new URL(reference)
+
+  for (const protocol of protocols)
+    if (protocol.protocols.includes(url.protocol)) return protocol
+
+  throw new Error(`Protocol '${url.protocol}' is not supported.`)
+}
+
+const path = resolve(__dirname, '../schemas/annotation.cos.yaml')
+const schema = schemas.schema(path)
+
+export type Component = Origins | Properties
+export type Annotation = Record<string, Component>
+export type Properties = Partial<Record<'.http', Record<string, boolean>>>
+export type Origins = Record<string, string | string[]>