@toa.io/pointer 0.9.0-canary.2 → 0.20.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/pointer",
-  "version": "0.9.0-canary.2",
+  "version": "0.20.0-alpha.1",
   "description": "Toa Pointer",
   "homepage": "https://toa.io",
   "author": {
@@ -21,12 +21,19 @@
   "publishConfig": {
     "access": "public"
   },
-  "main": "source/index.js",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
   "dependencies": {
-    "@toa.io/console": "0.6.0",
-    "@toa.io/generic": "0.11.0-canary.2",
-    "@toa.io/schemas": "0.9.0-canary.2",
-    "@toa.io/yaml": "0.8.0-canary.2"
+    "@toa.io/dns": "0.8.0-alpha.1",
+    "@toa.io/generic": "0.20.0-alpha.1",
+    "@toa.io/schemas": "0.20.0-alpha.1"
   },
-  "gitHead": "a80d81d4d9da9fec07606154c959dd6512de4c76"
+  "scripts": {
+    "transpile": "tsc"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
+  },
+  "gitHead": "d1a5369c0ee4ee423f792b83e9753fd2fd943cce"
 }

package/readme.md

@@ -1,126 +1,161 @@
-# Pointer
+# Toa Pointer
 
-Library to declare, deploy and resolve URL to connect to at the runtime. Used by connectors.
+The Pointer is a mechanism to deploy and resolve at runtime a set of URLs.
+Used by [connectors](/connectors) and [extensions](/extensions).
 
-## Pointer class
+## Annotation syntax
 
-Pointer builds URL for a given locator, protocol and a package prefix. Prefix must be unique value
-for a given package conforming [label format](#) (ex.: `bindings-ampq`).
+The Pointer annotation is an object whose keys are arbitrary dot-separated strings.
+The meaning of these keys are defined in the corresponding module that unilizes the Poiner.
 
-### Constructor signature
+The values of the annotation object must be a URL or a list of URLs without credentials.
 
-<dl>
-<dt><code>prefix</code></dt>
-<dd><code>string</code> unique for each package</dd>
-<dt><code>locator</code></dt>
-<dd><code>toa.core.Locator</code> component for which Pointer is intended</dd>
-<dt><code>options</code></dt>
-<dd><code><a href="./types/pointer.d.ts">toa.pointer.Options</a></code> Pointer options</dd>
-</dl>
-
-### Local Environment
-
-If `TOA_ENV` is `local` then these values are used:
-
-| Property | Value       |
-|----------|-------------|
-| hostname | `localhost` |
-| username | `developer` |
-| password | `secret`    |
+```yaml
+something:
+  one: amqp://all-dummies.rmq.example.com
+  two:
+    - amqp://rmq0.example.com
+    - amqp://rmq1.example.com
+```
 
-> In the local environment there is no way to provide values for `protocol` or `port`.
+If the Pointer annotation is a `string`, then its value is considered as default (`.`).
 
-See [types](types/pointer.d.ts) and [tests](test/pointer.test.js) for details.
+```yaml
+mongodb: mongo://default.db.example.com
+```
 
-## Annotation
+### Shards
 
-Declaration of a set of hosts, matching exact components, namespaces with a default value for
-non-matched ones.
+Pointer values can contain [shards](/libraries/generic/readme.md#shards) syntax.
 
 ```yaml
 something:
-  default: protocol://host1
-  namespace1: protocol://host2
-  namespace1.component1: protocol://host3
-  namespace2.component2: protocol://host4
+  one: redis://redis{0-2}.example.com
 ```
 
-See the [schema](source/uris/.construct/schema.yaml).
+### Deduplication
 
-### Concise Declaration
+Pointer values are [DNS-deduplicated](/libraries/dns/readme.md#deduplication) except `http:` schema.
 
-Next two declarations are equivalent.
+### `default` namespace
 
-```yaml
-something: host1
-```
+When the Pointer is used for component-specific URLs and the component is in the `default`
+namespace (that is, no `namepsace` is specified in its manifest), the annotation value must
+explicitly contain the `default` namespace.
 
 ```yaml
-something:
-  default: host1
+amqp:
+  # value for the `default` namespace
+  default: amqp://default-ns.rmq.example.com
+  # value for `teapots` component of the `default` namespace
+  default.teapots: amqp://default-dummy.rmq.example.com
 ```
 
-### Usage
+## Resolution
 
-```javascript
-const { uris } = require('@toa.io/pointer')
+Although the most common use case of the Pointer is component-specific connection strings, where
+the keys are namespaces or component IDs, it is not mandatory.
+The Pointer considers keys as nested dot-separated alphanumeric strings.
 
-const annotation = uris.construct(declaration)
-const url = uris.resolve(annotation, locator)
-```
+A module that uses the Pointer, resolves a set of URLs by providing annotation and a set of
+_selectors_ during deployment.
+The resolution finds _the deepest match for a given dot-separated string_ within the annotation,
+falling back to the default URL (`.` key) if no matches are found.
 
-See [types](types/uris.d.ts) and [tests](test/uris.test.js) for details.
+### Example
 
-### Custom Extensions
+```yaml
+# context.toa.yaml
 
-Packages using Pointer Annotation may use or require additional properties.
-See [Deployment Extensions](#extensions) below.
+amqp:
+  .: amqp://default.rmq.example.com # default URL
+  dummies: amqp://all-dummies.rmq.example.com # default URL for the `dummies` namespace
+  dummies.dummy: amqp://dummies1.rmq.example.com # URL for the `dummies.dummy` component
+```
 
-## Deployment
+URL resolution for the example above is as follows:
 
-Deployment function builds global[^1] variables: containing Annotation and a set of variables for
-pointer credentials.
+| Selector          | Result                               | Description        |
+|-------------------|--------------------------------------|--------------------|
+| `dummies.dummy`   | `amqp://d1.rmq.example.com`          | Exact match        |
+| `dummies.another` | `amqp://all-dummies.rmq.example.com` | Partial match      |
+| `what.ever`       | `amqp://default.rmq.example.com`     | No match (default) |
 
-See [types](types/deployment.d.ts) and [tests](test/deployment.test.js) for details.
+## Credentials
 
-### Credentials
+For security reasons, URL credentials cannot be specified in the annotation;
+they must be deployed secretly.
 
-Each entry of the Annotation requires values for `username` and `password`. These values are being
-deployed as global[^1] secret variables. Secret names are following the
-convention: `toa-package-prefix-entry-label` and `key` names match corresponding URL
-properties (`username` and `password`).
+Secret names must match the annotation keys with dots replaced by dashes and prefixed by an
+identifier provided by the corresponding connector or extension using the Pointer.
 
-> Note that declaration is made of `Locator.id` while secret names contain `Locator.label`.
-> See [Locator](#).
+### Example
 
-#### Example
+Given the [AMQP binding](/connectors/bindings.amqp)
+providing `amqp-context` Pointer identifier and the follwing annotation:
 
 ```yaml
 # context.toa.yaml
 amqp:
-  default: amqp://host0
-  dummies.dummy: amqps://host1:5671
+  context:
+    .: amqp://default.rmq.example.com
+    dummies: amqp://dummies.rmq.example.com
+    dummies.dummy: amqp://dummy.rmq.example.com
 ```
 
+Secret names for the specified keys are as follows:
+
+- `amqp-context.default`
+- `amqp-context-dummies`
+- `amqp-context-dummies-dummy`
+
+> If secrets are not deployed with the [`toa conceal`](/runtime/cli/readme.md#conceal), then their
+> names must be prefixed with `toa-`.
+
+Only one secret can be deployed per annotation key. Therefore, when using a URL set, all URLs will
+utilize the same credentials.
+
+### Plain
+
+Secret's value for plain authentication must contain `username` and `password` keys.
+
 ```shell
-$ toa conceal bindings-amqp-default username admin
-$ toa conceal bindings-amqp-default password ibreakthings
-$ toa conceal bindings-amqp-dummies-dummy username developer
-$ toa conceal bindings-amqp-dummies-dummy password iluvtests
+$ toa conceal amqp-context-dummies-dummy username=developer password=secret
 ```
 
-See [`toa conceal`](../../runtime/cli/readme.md#conceal).
+### TLS
+
+Not implemented. [#367](https://github.com/toa-io/toa/issues/367)
 
-### Extensions
+## API
 
-For each custom extension additional proxy is being deployed.
+### Deployment
 
-#### Example
+```typescript
+import { createVariables } from '@toa.io/pointer'
 
-AMQP binding is a systematic binding used by the runtime. That is a system proxy must be deployed
-and compositions require variables to establish connection through that proxy.
+// ...
 
-Therefore, AMQP binding declares a `system` extension and deployment function declares additional
-proxy `bindings-amqp-system`.
+const variables = createVariables(id, annotation, requests)
+```
+
+`id` is a Pointer identifier, `annotation` the Pointer annotation value, and `requests` is an array
+of objects conforming to the following:
+
+```yaml
+# label of the component
+label: string
+# list of the annotation keys to be deployed for the component
+selectors: string[]
+```
+
+### Resolution
 
-[^1]: [#174](https://github.com/toa-io/toa/issues/174)
+```typescript
+// runtime
+import { resolve } from '@toa.io/pointer'
+
+// resolve a set of URLs with credentials
+// for a given pointer identifier and a selector
+const urls = resolve(id, locator.id)
+```

package/schemas/urimap.cos.yaml

@@ -0,0 +1,2 @@
+.: [uri]
+/^[a-zA-Z0-9]{1,32}(\.[a-zA-Z0-9]{1,32}){0,3}$/: [uri]

package/source/Deployment.ts

@@ -0,0 +1,90 @@
+import { nameVariable, nameSecret } from './naming'
+import { resolveRecord } from './resolve'
+import type { Variables, Variable } from '@toa.io/operations'
+
+export class Deployment {
+  private readonly id: string
+  private readonly annotation: URIMap
+
+  public constructor (id: string, annotation: URIMap) {
+    this.id = id
+    this.annotation = annotation
+  }
+
+  public export (requests: Request[]): Variables {
+    const variables: Variables = {}
+
+    for (const request of requests)
+      variables[request.group] = this.createVariables(request.selectors)
+
+    return variables
+  }
+
+  private createVariables (selectors: string[]): Variable[] {
+    const variables: Variable[] = []
+
+    for (const selector of selectors) {
+      const variable = this.createVariable(selector)
+      const secrets = this.createSecrets(selector)
+
+      variables.push(variable, ...secrets)
+    }
+
+    return variables
+  }
+
+  private createVariable (selector: string): Variable {
+    const name = nameVariable(this.id, selector)
+    const { references } = this.resolveRecord(selector)
+    const value = references.join(' ')
+
+    return { name, value }
+  }
+
+  private createSecrets (selector: string): Variable[] {
+    const varialbes: Variable[] = []
+    const { key, references } = this.resolveRecord(selector)
+
+    if (this.insecure(references)) return []
+
+    for (const token of ['username', 'password']) {
+      const varName = nameVariable(this.id, selector, token)
+      const secretName = nameSecret(this.id, key)
+
+      varialbes.push({
+        name: varName,
+        secret: {
+          name: secretName,
+          key: token
+        }
+      })
+    }
+
+    return varialbes
+  }
+
+  private resolveRecord (selector: string): AnnotationRecord {
+    return resolveRecord(this.annotation, selector)
+  }
+
+  private insecure (references: string[]): boolean {
+    const reference = references[0]
+    const url = new URL(reference)
+
+    return insecureProtocols.includes(url.protocol)
+  }
+}
+
+export interface Request {
+  group: string
+  selectors: string[]
+}
+
+export interface AnnotationRecord {
+  key: string
+  references: string[]
+}
+
+const insecureProtocols = ['http:', 'https:', 'redis:']
+
+export type URIMap = Record<string, string[]>

package/source/annotation.test.ts

@@ -0,0 +1,66 @@
+import { normalize } from './annotation'
+
+describe('normalize', () => {
+  it('should expand default shards', async () => {
+    const declaration = 'amqp://rmq{0-2}.example.com'
+    const annotation = normalize(declaration)
+
+    expect(annotation)
+      .toStrictEqual({
+        '.': [
+          'amqp://rmq0.example.com',
+          'amqp://rmq1.example.com',
+          'amqp://rmq2.example.com'
+        ]
+      })
+  })
+
+  it('should expand shards within array', async () => {
+    const declaration = {
+      foo: ['amqp://rmq.example.com', 'amqp://rmq{0-2}.example.com']
+    }
+
+    const annotation = normalize(declaration)
+
+    expect(annotation)
+      .toStrictEqual({
+        foo: [
+          'amqp://rmq.example.com',
+          'amqp://rmq0.example.com',
+          'amqp://rmq1.example.com',
+          'amqp://rmq2.example.com'
+        ]
+      })
+  })
+})
+
+describe('validation', () => {
+  it('should pass', async () => {
+    const declaration = 'http://localhost'
+
+    expect(() => normalize(declaration))
+      .not.toThrow()
+  })
+
+  it('should throw if non-uri', async () => {
+    const declaration = 'non uri'
+
+    expect(() => normalize(declaration))
+      .toThrow('must match format')
+  })
+
+  it.each(['user:pass', 'user', ':pass'])('should throw if uri has credentials',
+    async (credentials) => {
+      const declaration = `http://${credentials}@localhost`
+
+      expect(() => normalize(declaration))
+        .toThrow('must not contain credentials')
+    })
+
+  it('should throw if key is not deployable', async () => {
+    const declaration = { 'foo bar': 'http://localhost' }
+
+    expect(() => normalize(declaration))
+      .toThrow('not expected')
+  })
+})

package/source/annotation.ts

@@ -0,0 +1,46 @@
+import { resolve } from 'node:path'
+import * as schemas from '@toa.io/schemas'
+import { shards } from '@toa.io/generic'
+import type { URIMap } from './Deployment'
+
+export function normalize (declaration: Declaration): URIMap {
+  const map: URIMap = {}
+
+  if (declaration === undefined) return map
+
+  if (typeof declaration === 'string' || Array.isArray(declaration))
+    declaration = { '.': declaration }
+
+  for (const [key, value] of Object.entries(declaration))
+    if (typeof value === 'string') map[key] = format([value])
+    else map[key] = format(value)
+
+  validate(map)
+
+  return map
+}
+
+function format (values: string[]): string[] {
+  return values.map(shards).flat()
+}
+
+function validate (map: URIMap): void {
+  schema.validate(map)
+
+  for (const uris of Object.values(map))
+    for (const uri of uris)
+      checkCredentials(uri)
+}
+
+function checkCredentials (uri: string): void {
+  const url = new URL(uri)
+
+  if (url.username !== '' || url.password !== '')
+    throw new Error(`Pointer URI '${uri}' must not contain credentials. ` +
+      'Please refer to the "Credentials" section in the documentation for more information.')
+}
+
+const path = resolve(__dirname, '../schemas/urimap.cos.yaml')
+const schema = schemas.schema(path)
+
+export type Declaration = string | string[] | Record<string, string | string[]> | undefined

package/source/createVariables.test.ts

@@ -0,0 +1,186 @@
+import { generate } from 'randomstring'
+import { createVariables } from './createVariables'
+import type { Variable, Variables } from '@toa.io/operations'
+import type { Request } from './Deployment'
+
+it('should create from selector', async () => {
+  const id = generate()
+  const selector = generate()
+  const annotation = { [selector]: uri() }
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  const expectation: Variables = {
+    [request.group]: [{
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}`,
+      value: annotation[selector]
+    }]
+  }
+
+  expect(variables)
+    .toStrictEqual(expectation)
+})
+
+it('should create from partial match', async () => {
+  const id = generate()
+  const key = generate()
+  const selector = `${key}.${generate()}`
+  const annotation = { [key]: uri() }
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  const expectation: Variables = {
+    [request.group]: [{
+      name: `TOA_${id.toUpperCase()}_${
+        selector
+          .replace('.', '_')
+          .toUpperCase()
+      }`,
+      value: annotation[key]
+    }]
+  }
+
+  expect(variables)
+    .toStrictEqual(expectation)
+})
+
+it('should create from default', async () => {
+  const id = generate()
+  const annotation = uri()
+  const selector = generate()
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  const expectation: Variables = {
+    [request.group]: [{
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}`,
+      value: annotation
+    }]
+  }
+
+  expect(variables)
+    .toStrictEqual(expectation)
+})
+
+it('should create from array', async () => {
+  const id = generate()
+  const selector = generate()
+  const values = [uri(), uri()]
+  const annotation = { [selector]: values }
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  const expectation: Variables = {
+    [request.group]: [{
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}`,
+      value: values.join(' ')
+    }]
+  }
+
+  expect(variables)
+    .toStrictEqual(expectation)
+})
+
+it('should create from default array', async () => {
+  const id = generate()
+  const selector = generate()
+  const values = [uri(), uri()]
+  const annotation = values
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  const expectation: Variables = {
+    [request.group]: [{
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}`,
+      value: values.join(' ')
+    }]
+  }
+
+  expect(variables)
+    .toStrictEqual(expectation)
+})
+
+it('should throw if selector cannot be resolved', async () => {
+  const id = generate()
+  const selector = generate()
+  const annotation = {}
+  const request: Request = { group: generate(), selectors: [selector] }
+
+  expect(() => createVariables(id, annotation, [request]))
+    .toThrow('cannot be resolved.')
+})
+
+it('should create credential secrets for annotation keys', async () => {
+  const id = generate()
+
+  const group = generate()
+  const namespace = generate()
+  const name1 = 'one'
+  const name2 = 'two'
+  const name3 = 'three'
+  const selector1 = `${namespace}.${name1}`
+  const selector2 = `${namespace}.${name2}`
+  const selector3 = `${generate()}.${name3}`
+
+  const annotation = {
+    '.': uri('amqp:'),
+    [namespace]: uri('amqp:'),
+    [selector1]: uri('amqp:')
+  }
+
+  const request: Request = { group, selectors: [selector1, selector2, selector3] }
+  const variables = createVariables(id, annotation, [request])
+
+  const secrets1 = createExpectedSecrets(id, selector1, selector1)
+  const secrets2 = createExpectedSecrets(id, namespace, selector2)
+  const secrets3 = createExpectedSecrets(id, '.', selector3)
+
+  const expectation: Variables = {
+    [group]: expect.arrayContaining([
+      ...secrets1,
+      ...secrets2,
+      ...secrets3
+    ])
+  }
+
+  expect(variables).toStrictEqual(expectation)
+})
+
+it.each([
+  'http:', 'redis:'
+])('should not create credetial secrets for %s', async (protocol) => {
+  const id = generate()
+  const selector = generate()
+  const annotation = { [selector]: uri(protocol) }
+  const request: Request = { group: generate(), selectors: [selector] }
+  const variables = createVariables(id, annotation, [request])
+
+  for (const variable of variables[request.group])
+    expect(variable.secret).toBeUndefined()
+})
+
+function uri (protocol = 'http:'): string {
+  return protocol + '//host-' + generate()
+}
+
+function createExpectedSecrets (id: string, key: string, selector: string): Variable[] {
+  key = key === '.' ? '.default' : '-' + key.replaceAll('.', '-')
+  selector = selector.replaceAll('.', '_').toUpperCase()
+
+  return [
+    {
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}_USERNAME`,
+      secret: {
+        name: `toa-${id}${key}`,
+        key: 'username'
+      }
+    },
+    {
+      name: `TOA_${id.toUpperCase()}_${selector.toUpperCase()}_PASSWORD`,
+      secret: {
+        name: `toa-${id}${key}`,
+        key: 'password'
+      }
+    }
+  ]
+}

package/source/createVariables.ts

@@ -0,0 +1,12 @@
+import { Deployment, type Request } from './Deployment'
+import { normalize } from './annotation'
+import type { Variables } from '@toa.io/operations'
+import type { Declaration } from './annotation'
+
+export function createVariables
+(id: string, declaration: Declaration, requests: Request[]): Variables {
+  const annotation = normalize(declaration)
+  const deployment = new Deployment(id, annotation)
+
+  return deployment.export(requests)
+}