@toa.io/extensions.storages 0.20.0-alpha.3

package/source/deployment.ts

@@ -0,0 +1,61 @@
+import { encode } from 'msgpackr'
+import { providers } from './providers'
+import type { Dependency, Variable } from '@toa.io/operations'
+import type { context } from '@toa.io/norm'
+
+export function deployment (instances: Instance[], annotation: Annotation): Dependency {
+  validate(instances, annotation)
+
+  const value = encode(annotation).toString('base64')
+  const pointer: Variable = { name: 'TOA_STORAGES', value }
+  const secrets = getSecrets(annotation)
+
+  return {
+    variables: { global: [pointer, ...secrets] }
+  }
+}
+
+function validate (instances: Instance[], annotation: Annotation): void {
+  for (const instance of instances)
+    contains(instance, annotation)
+
+  for (const ref of Object.values(annotation)) {
+    const url = new URL(ref)
+
+    if (!(url.protocol in providers))
+      throw new Error(`Unknown storage provider '${url.protocol}'`)
+  }
+}
+
+function contains (instance: Instance, annotation: Annotation): void {
+  for (const name of instance.manifest)
+    if (!(name in annotation))
+      throw new Error(`Missing '${name}' storage annotation ` +
+        `declared in '${instance.component.locator.id}'`)
+}
+
+function getSecrets (annotation: Annotation): Variable[] {
+  const secrets: Variable[] = []
+
+  for (const [storage, ref] of Object.entries(annotation)) {
+    const url = new URL(ref)
+    const Provider = providers[url.protocol]
+
+    if (Provider.SECRETS === undefined)
+      continue
+
+    for (const secret of Provider.SECRETS)
+      secrets.push({
+        name: `TOA_STORAGES_${storage.toUpperCase()}_${secret.toUpperCase()}`,
+        secret: {
+          name: `toa-storages-${storage}`,
+          key: secret
+        }
+      } satisfies Variable)
+  }
+
+  return secrets
+}
+
+type Annotation = Record<string, string>
+export type Instance = context.Dependency<string[]>

package/source/index.ts

@@ -0,0 +1,3 @@
+export { Factory } from './Factory'
+export { deployment } from './deployment'
+export { manifest } from './manifest'

package/source/manifest.ts

@@ -0,0 +1,8 @@
+import { match } from '@toa.io/match'
+
+export function manifest (manifest: string | string[] | null): string[] {
+  return match(manifest,
+    () => typeof manifest === 'string', (name: string) => [name],
+    Array, manifest,
+    null, [])
+}

package/source/providers/FileSystem.test.ts

@@ -0,0 +1,13 @@
+import { FileSystem } from './FileSystem'
+
+it('should be ok', async () => {
+  const url = new URL('file:///path')
+
+  expect(() => new FileSystem(url)).not.toThrow()
+})
+
+it('should throw if url contains host', async () => {
+  const url = new URL('file://host/path')
+
+  expect(() => new FileSystem(url)).toThrow()
+})

package/source/providers/FileSystem.ts

@@ -0,0 +1,46 @@
+import { type Readable } from 'node:stream'
+import { dirname, join } from 'node:path'
+import { createReadStream } from 'node:fs'
+import fs from 'node:fs/promises'
+import fse from 'fs-extra'
+import { type Provider } from '../Provider'
+
+export class FileSystem implements Provider {
+  protected readonly path: string
+
+  public constructor (url: URL) {
+    if (url.host !== '')
+      throw new Error('File system URL must not contain host')
+
+    this.path = url.pathname
+  }
+
+  public async get (path: string): Promise<Readable | null> {
+    path = join(this.path, path)
+
+    if (!await fse.exists(path))
+      return null
+
+    return createReadStream(path)
+  }
+
+  public async put (rel: string, filename: string, stream: Readable): Promise<void> {
+    const dir = join(this.path, rel)
+    const path = join(dir, filename)
+
+    await fse.ensureDir(dir)
+    await fs.writeFile(path, stream)
+  }
+
+  public async delete (path: string): Promise<void> {
+    await fse.remove(join(this.path, path))
+  }
+
+  public async move (from: string, to: string): Promise<void> {
+    from = join(this.path, from)
+    to = join(this.path, to)
+
+    await fse.ensureDir(dirname(to))
+    await fs.rename(from, to)
+  }
+}

package/source/providers/Temporary.ts

@@ -0,0 +1,14 @@
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { FileSystem } from './FileSystem'
+
+export class Temporary extends FileSystem {
+  protected override readonly path: string
+
+  public constructor (url: URL) {
+    super(url)
+
+    this.path = join(tmpdir(), url.pathname)
+  }
+}

package/source/providers/Test.ts

@@ -0,0 +1,13 @@
+import assert from 'node:assert'
+import { Temporary } from './Temporary'
+
+export class Test extends Temporary {
+  public static readonly SECRETS = ['USERNAME', 'PASSWORD']
+
+  public constructor (url: URL, secrets: Record<string, string>) {
+    super(url)
+
+    assert(secrets.USERNAME !== undefined, 'Missing USERNAME')
+    assert(secrets.PASSWORD !== undefined, 'Missing PASSWORD')
+  }
+}

package/source/providers/index.test.ts

@@ -0,0 +1,116 @@
+import { type Readable } from 'node:stream'
+import { buffer } from '@toa.io/streams'
+import { cases, open, rnd, read } from '../.test/util'
+import { providers } from './index'
+
+describe.each(cases)('%s', (protocol, url, secrets) => {
+  const Provider = providers[protocol]
+  const provider = new Provider(url, secrets)
+
+  let dir: string
+
+  beforeEach(() => {
+    dir = '/' + rnd()
+  })
+
+  it('should be', async () => {
+    expect(provider).toBeInstanceOf(Provider)
+  })
+
+  it('should return null if file not found', async () => {
+    const result = await provider.get(rnd())
+
+    expect(result).toBeNull()
+  })
+
+  it('should create entry', async () => {
+    const stream = await open('lenna.png')
+
+    await provider.put(dir, 'lenna.png', stream)
+
+    const readable = await provider.get(dir + '/lenna.png') as Readable
+    const output = await buffer(readable)
+    const lenna = await read('lenna.png')
+
+    expect(output.compare(lenna)).toBe(0)
+  })
+
+  it('should overwrite existing entry', async () => {
+    const stream0 = await open('lenna.png')
+    const stream1 = await open('albert.jpg')
+
+    await provider.put(dir, 'lenna.png', stream0)
+    await provider.put(dir, 'lenna.png', stream1)
+
+    const readable = await provider.get(dir + '/lenna.png') as Readable
+    const output = await buffer(readable)
+    const albert = await read('albert.jpg')
+
+    expect(output.compare(albert)).toBe(0)
+  })
+
+  it('should get by path', async () => {
+    const stream = await open('lenna.png')
+
+    await provider.put(dir, 'lenna.png', stream)
+
+    const result = await provider.get('/bar/lenna.png')
+
+    expect(result).toBeNull()
+  })
+
+  describe('danger', () => {
+    /*
+
+    WHEN MAKING CHANGES TO DELETION,
+    ALWAYS RUN TESTS IN STEP-BY-STEP DEBUGGING MODE
+
+    YOU MAY EVENTUALLY DELETE YOUR ENTIRE FILE SYSTEM
+
+                              Sincerely yours, Murphy
+
+     */
+
+    it('should delete entry', async () => {
+      const stream = await open('lenna.png')
+
+      await provider.put(dir, 'lenna.png', stream)
+      await provider.delete(dir + '/lenna.png')
+
+      const result = await provider.get(dir + '/lenna.png')
+
+      expect(result).toBeNull()
+    })
+
+    it('should not throw if path does not exists', async () => {
+      await expect(provider.delete(dir + '/whatever')).resolves.not.toThrow()
+    })
+
+    it('should delete directory', async () => {
+      const stream = await open('lenna.png')
+
+      await provider.put(dir, 'lenna.png', stream)
+      await provider.delete(dir)
+
+      const result = await provider.get(dir + '/lenna.png')
+
+      expect(result).toBeNull()
+    })
+
+    it('should move an entry', async () => {
+      const stream = await open('lenna.png')
+      const dir2 = '/' + rnd()
+
+      await provider.put(dir, 'lenna.png', stream)
+      await provider.move(dir + '/lenna.png', dir2 + '/lenna2.png')
+
+      const result = await provider.get(dir2 + '/lenna2.png') as Readable
+
+      expect(result).not.toBeNull()
+
+      const nope = await provider.get(dir + '/lenna.png')
+
+      expect(nope).toBeNull()
+    })
+  })
+})

package/source/providers/index.ts

@@ -0,0 +1,13 @@
+import { type Provider } from '../Provider'
+import { FileSystem } from './FileSystem'
+import { Temporary } from './Temporary'
+import { Test } from './Test'
+
+export const providers: Record<string, ProviderClass> = {
+  'file:': FileSystem,
+  'tmp:': Temporary,
+  'test:': Test
+}
+
+export type ProviderClass =
+  (new (url: URL, secrets: Record<string, string>) => Provider) & { SECRETS?: string[] }

package/source/providers/readme.md

@@ -0,0 +1,20 @@
+# Developing a provider
+
+1. Add an implementaion of the [Provider](../Provider.ts) interface under this directory.
+2. Add an entry to the provider map in [`index.ts`](./index.ts).
+3. Add a suite to the `suites` object in [`util.ts`](../.test/util.ts).
+4. Run `$ npm test` in the [`storages` directory](../..).
+
+Provider's constructor must have the following signature:
+
+`constructor(url: URL)`
+
+## Secrets
+
+Provider class may have static `SECRETS` property of type `string[]` that lists the names of the secrets that it
+requires.
+The secrets are passed to the constructor as the second argument.
+
+`constructor(url: URL, secrets: Record<string, string>)`
+
+See [`Test` provider](./Test.ts) for an example.

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./transpiled"
+  },
+  "include": [
+    "source"
+  ],
+  "exclude": [
+    "**/*.test.ts"
+  ]
+}