@toa.io/extensions.storages 1.0.0-alpha.31 → 1.0.0-alpha.310

package/source/deployment.ts

@@ -1,57 +0,0 @@
-import * as assert from 'node:assert'
-import { encode } from '@toa.io/generic'
-import { providers } from './providers'
-import { validateAnnotation } from './Annotation'
-import type { Annotation } from './Annotation'
-import type { Dependency, Variable } from '@toa.io/operations'
-import type { context } from '@toa.io/norm'
-
-export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
-
-export function deployment (instances: Instance[], annotation: unknown): Dependency {
-  validate(instances, annotation)
-
-  const value = encode(annotation)
-  const pointer: Variable = { name: SERIALIZATION_PREFIX, value }
-  const secrets = getSecrets(annotation)
-
-  return {
-    variables: { global: [pointer, ...secrets] }
-  }
-}
-
-function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
-  validateAnnotation(annotation)
-
-  for (const instance of instances)
-    contains(instance, annotation)
-}
-
-function contains (instance: Instance, annotation: Annotation): void {
-  for (const name of instance.manifest)
-    assert.ok(name in annotation,
-      `Missing '${name}' storage annotation ` +
-      `declared in '${instance.component.locator.id}'`)
-}
-
-function getSecrets (annotation: Annotation): Variable[] {
-  const secrets: Variable[] = []
-
-  for (const [name, declaration] of Object.entries(annotation)) {
-    const Provider = providers[declaration.provider]
-
-    for (const secret of Provider.SECRETS)
-      secrets.push({
-        name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
-        secret: {
-          name: `toa-storages-${name}`,
-          key: secret.name,
-          optional: secret.optional
-        }
-      })
-  }
-
-  return secrets
-}
-
-export type Instance = context.Dependency<string[]>

package/source/index.ts

@@ -1,5 +0,0 @@
-export { Factory } from './Factory'
-export { deployment } from './deployment'
-export { manifest } from './manifest'
-
-export type { Entry } from './Entry'

package/source/manifest.ts

@@ -1,8 +0,0 @@
-import { match } from 'matchacho'
-
-export function manifest (manifest: string | string[] | null): string[] {
-  return match(manifest,
-    String, (name: string) => [name],
-    Array, manifest,
-    null, [])
-}

package/source/providers/Declaration.ts

@@ -1,10 +0,0 @@
-import type { S3Options } from './S3'
-import type { FileSystemOptions } from './FileSystem'
-import type { TemporaryOptions } from './Temporary'
-
-export type Declaration =
-  ({ provider: 's3' } & S3Options)
-  | ({ provider: 'fs' } & FileSystemOptions)
-  | ({ provider: 'tmp' } & TemporaryOptions)
-  | ({ provider: 'mem' })
-  | ({ provider: 'test' } & TemporaryOptions)

package/source/providers/FileSystem.test.ts

@@ -1,5 +0,0 @@
-import { FileSystem } from './FileSystem'
-
-it('should be ok', async () => {
-  expect(() => new FileSystem({ path: 'path' })).not.toThrow()
-})

package/source/providers/FileSystem.ts

@@ -1,63 +0,0 @@
-import { type Readable } from 'node:stream'
-import { dirname, join } from 'node:path'
-import fs from 'node:fs/promises'
-import { Provider } from '../Provider'
-import type { ProviderSecrets } from '../Provider'
-
-export interface FileSystemOptions {
-  path: string
-}
-
-export class FileSystem extends Provider<FileSystemOptions> {
-  protected readonly path: string
-
-  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
-    super(options, secrets)
-
-    this.path = options.path
-  }
-
-  public async get (path: string): Promise<Readable | null> {
-    try {
-      const fd = await fs.open(join(this.path, path))
-
-      return fd.createReadStream()
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return null
-
-      throw err
-    }
-  }
-
-  public async list (path: string): Promise<string[]> {
-    const dir = join(this.path, path)
-
-    return (await fs.readdir(dir, { withFileTypes: true }))
-      .filter((dirent) => dirent.isFile())
-      .map((dirent) => dirent.name)
-  }
-
-  public async put (rel: string, filename: string, stream: Readable): Promise<void> {
-    const dir = join(this.path, rel)
-    const path = join(dir, filename)
-
-    await fs.mkdir(dir, { recursive: true })
-    await fs.writeFile(path, stream)
-  }
-
-  public async delete (path: string): Promise<void> {
-    await fs.rm(join(this.path, path), { recursive: true, force: true })
-  }
-
-  public async move (from: string, to: string): Promise<void> {
-    from = join(this.path, from)
-    to = join(this.path, to)
-
-    await fs.mkdir(dirname(to), { recursive: true })
-    await fs.rename(from, to)
-  }
-
-  public async moveDir (from: string, to: string): Promise<void> {
-    await this.move(from, to).catch(() => null)
-  }
-}

package/source/providers/Memory.ts

@@ -1,55 +0,0 @@
-import { Readable } from 'node:stream'
-import { join, posix } from 'node:path'
-import { buffer } from 'node:stream/consumers'
-import * as assert from 'node:assert'
-
-import { Provider } from '../Provider'
-
-/**
- * In-memory provider
- */
-export class InMemory extends Provider {
-  private readonly storage = new Map<string, Buffer>()
-
-  public async get (path: string): Promise<Readable | null> {
-    const data = this.storage.get(path)
-
-    if (data === undefined) return null
-
-    return Readable.from(data)
-  }
-
-  public async list (path: string): Promise<string[]> {
-    return Array.from(this.storage.keys())
-      .filter((f) => posix.dirname(f) === path)
-      .map((f) => posix.basename(f))
-  }
-
-  public async put (path: string, filename: string, stream: Readable): Promise<void> {
-    this.storage.set(join(path, filename), await buffer(stream))
-  }
-
-  public async delete (path: string): Promise<void> {
-    for (const f of this.storage.keys())
-      if (f.startsWith(path)) this.storage.delete(f)
-  }
-
-  public async move (from: string, to: string): Promise<void> {
-    const buf = this.storage.get(from)
-
-    assert.ok(buf !== undefined, `File not found: ${from}`)
-
-    this.storage.set(to, buf)
-    this.storage.delete(from)
-  }
-
-  public async moveDir (from: string, to: string): Promise<void> {
-    for (const f of this.storage.keys())
-      if (f.startsWith(from)) {
-        const toPath = to + f.slice(from.length)
-
-        this.storage.set(toPath, this.storage.get(f)!)
-        this.storage.delete(f)
-      }
-  }
-}

package/source/providers/S3.test.ts

@@ -1,134 +0,0 @@
-import { Readable } from 'node:stream'
-import { randomUUID } from 'node:crypto'
-import streamConsumers from 'node:stream/consumers'
-import { http, HttpResponse } from 'msw'
-import { setupServer } from 'msw/node'
-
-import { S3 } from './S3'
-
-/* eslint-disable @typescript-eslint/no-non-null-assertion -- jest expect is not type guard */
-
-describe('S3 storage provider', () => {
-  let provider: S3
-  let s3endpoint: ReturnType<typeof setupServer>
-
-  const AWS_REGION = 'us-east-1'
-
-  const TEST_BUCKET_NAME = 'test-bucket'
-  const S3_ENDPOINT = `https://${TEST_BUCKET_NAME}.s3.${AWS_REGION}.amazonaws.com`
-
-  beforeAll(async () => {
-    jest.useRealTimers()
-    s3endpoint = setupServer()
-    s3endpoint.listen({ onUnhandledRequest: 'error' })
-
-    provider = new S3({
-      bucket: TEST_BUCKET_NAME,
-      region: AWS_REGION
-    }, {
-      ACCESS_KEY_ID: 'test-key',
-      SECRET_ACCESS_KEY: 'test-key-secret'
-    })
-  })
-
-  afterEach(() => {
-    s3endpoint.resetHandlers()
-  })
-
-  afterAll(() => {
-    s3endpoint.close()
-  })
-
-  test('should be able to get file from S3 handling leading slash', async () => {
-    const testBody = randomUUID().repeat(10)
-
-    s3endpoint.use(http.get(`${S3_ENDPOINT}/some/absolute/path/filename`,
-      () => HttpResponse.text(testBody)))
-
-    const body = await provider.get('/some/absolute/path/filename')
-
-    expect(body).toBeInstanceOf(Readable)
-    expect((await streamConsumers.text(body!))).toBe(testBody)
-  })
-
-  test('should remove folder with all files', async () => {
-    const headHandler = jest.fn().mockReturnValue(HttpResponse.xml('', { status: 404 }))
-
-    const listHandler = jest.fn().mockReturnValueOnce(HttpResponse.xml(`
-    <ListBucketResult>
-      <NextContinuationToken>someToken</NextContinuationToken>
-      <KeyCount>1001</KeyCount>
-      <MaxKeys>1000</MaxKeys>
-      <IsTruncated>true</IsTruncated>
-      <Contents>
-        <Key>happy_face1.jpg</Key>
-      </Contents>
-    </ListBucketResult>
-    `, {
-      status: 200
-    })).mockReturnValueOnce(HttpResponse.xml(`
-    <ListBucketResult>
-      <KeyCount>1001</KeyCount>
-      <MaxKeys>1000</MaxKeys>
-      <Contents>
-        <Key>happy_face2.jpg</Key>
-      </Contents>
-      <Contents>
-        <Key>happy_face3.jpg</Key>
-      </Contents>
-    </ListBucketResult>
-    `, {
-      status: 200
-    }))
-
-    const deleteHandler = jest.fn().mockReturnValue(HttpResponse.xml(`
-    <DeleteResult>
-      <Deleted>
-        <Key>happy_face1.jpg</Key>
-      </Deleted>
-      <Deleted>
-        <Key>happy_face2.jpg</Key>
-      </Deleted>
-      <Deleted>
-        <Key>happy_face3.jpg</Key>
-      </Deleted>
-    </DeleteResult>
-    `, {
-      status: 200
-    }))
-
-    s3endpoint.use(http.head(`${S3_ENDPOINT}/some/absolute/path_to_remove`,
-      headHandler),
-    http.get(`${S3_ENDPOINT}/`,
-      listHandler),
-    http.post(`${S3_ENDPOINT}/`,
-      deleteHandler))
-
-    await provider.delete('/some/absolute/path_to_remove')
-
-    expect(headHandler).toHaveBeenCalledTimes(1)
-    expect(listHandler).toHaveBeenCalledTimes(2)
-    expect(deleteHandler).toHaveBeenCalledTimes(1)
-  })
-
-  test('should be able to upload a file to S3', async () => {
-    const body = Readable.from('test content')
-
-    const handler = jest.fn().mockReturnValue(HttpResponse.xml('', {
-      status: 200,
-      headers: {
-        ETag: 'test-etag',
-        'x-amz-id-2': 'test_id_2',
-        'x-amz-request-id': 'test_request_id',
-        Server: 'AmazonS3'
-      }
-    }))
-
-    s3endpoint.use(http.put(`${S3_ENDPOINT}/some/absolute/path/filename`,
-      handler))
-
-    await provider.put('/some/absolute/path', 'filename', body)
-    expect(handler).toHaveBeenCalledTimes(1)
-    expect(body.readableEnded).toBe(true)
-  })
-})

package/source/providers/S3.ts

@@ -1,196 +0,0 @@
-import { Readable } from 'node:stream'
-import { Blob } from 'node:buffer'
-import { join } from 'node:path/posix'
-import assert from 'node:assert'
-import { posix } from 'node:path'
-import { Upload } from '@aws-sdk/lib-storage'
-import {
-  S3Client,
-  GetObjectCommand,
-  CopyObjectCommand,
-  DeleteObjectsCommand,
-  paginateListObjectsV2,
-  HeadObjectCommand,
-  NotFound,
-  DeleteObjectCommand,
-  NoSuchKey,
-  type S3ClientConfigType,
-  type ObjectIdentifier
-} from '@aws-sdk/client-s3'
-import * as nodeNativeFetch from 'smithy-node-native-fetch'
-import { Provider, type ProviderSecret, type ProviderSecrets } from '../Provider'
-import type { ReadableStream } from 'node:stream/web'
-
-export interface S3Options {
-  bucket: string
-  region?: string
-  prefix?: string
-  endpoint?: string
-}
-
-type S3Secrets = ProviderSecrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
-
-export class S3 extends Provider<S3Options> {
-  public static override readonly SECRETS: readonly ProviderSecret[] = [
-    { name: 'ACCESS_KEY_ID', optional: true },
-    { name: 'SECRET_ACCESS_KEY', optional: true }
-  ]
-
-  protected readonly bucket: string
-  protected readonly client: S3Client
-
-  public constructor (options: S3Options, secrets?: S3Secrets) {
-    super(options)
-
-    this.bucket = options.bucket
-
-    const s3Config: S3ClientConfigType = {
-      retryMode: 'adaptive',
-      ...nodeNativeFetch
-    }
-
-    if (options.endpoint !== undefined) {
-      s3Config.forcePathStyle = options.endpoint.startsWith('http://')
-      s3Config.endpoint = options.endpoint
-    }
-
-    if (options.region !== undefined) s3Config.region = options.region
-
-    if (typeof secrets?.ACCESS_KEY_ID === 'string') {
-      assert.ok(secrets.SECRET_ACCESS_KEY !== undefined,
-        'SECRET_ACCESS_KEY is required if ACCESS_KEY_ID is provided')
-
-      s3Config.credentials = {
-        accessKeyId: secrets.ACCESS_KEY_ID,
-        secretAccessKey: secrets.SECRET_ACCESS_KEY
-      }
-    }
-
-    this.client = new S3Client(s3Config)
-
-    this.client.middlewareStack.add((next, _context) => async (args) => {
-      if ('Key' in args.input && typeof args.input.Key === 'string')
-      // removes leading slash
-
-        args.input.Key = args.input.Key.replace(/^\//, '')
-
-      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
-      // removes leading slash and ensures finishing slash
-
-        args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
-
-      return next(args)
-    },
-    {
-      step: 'initialize',
-      priority: 'high',
-      name: 'normalizesSlashesInPath'
-    })
-  }
-
-  public async get (Key: string): Promise<Readable | null> {
-    try {
-      const fileResponse = await this.client.send(new GetObjectCommand({
-        Bucket: this.bucket,
-        Key
-      }))
-
-      if (fileResponse.Body === undefined) return null // should never happen
-
-      if (fileResponse.Body instanceof Readable) return fileResponse.Body
-
-      return Readable.fromWeb((fileResponse.Body instanceof Blob
-        ? fileResponse.Body.stream()
-        : fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
-    } catch (err) {
-      if (err instanceof NotFound || err instanceof NoSuchKey)
-        return null
-      else
-        throw err
-    }
-  }
-
-  public async list (prefix: string): Promise<string[]> {
-    return (await this.listObjects(prefix))
-      .map(({ Key }) => posix.basename(Key!))
-  }
-
-  public async put (path: string, filename: string, stream: Readable): Promise<void> {
-    await new Upload({
-      client: this.client,
-      params: {
-        Bucket: this.bucket,
-        Key: join(path, filename),
-        Body: stream
-      }
-    }).done()
-  }
-
-  /**
-   * Deletes either a single object or "directory" - all objects
-   * with given prefix (prefix will be enforced to finish with `/`)
-   * @param Key - key name or path prefix
-   */
-  public async delete (Key: string): Promise<void> {
-    const { client, bucket: Bucket } = this
-
-    // checking if given key is a single file
-    if (!Key.endsWith('/'))
-      try {
-        // DeleteObject on S3 returns no error if object does not exist
-        await client.send(new HeadObjectCommand({ Bucket, Key }))
-        await client.send(new DeleteObjectCommand({ Bucket, Key }))
-
-        return
-      } catch (err) {
-        assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
-      }
-
-    const objectsToRemove: ObjectIdentifier[] = await this.listObjects(Key)
-
-    // Removing all objects in parallel in batches
-    await Promise.all((function * () {
-      while (objectsToRemove.length > 0)
-        yield client.send(new DeleteObjectsCommand({
-          Bucket,
-          Delete: {
-            Objects: objectsToRemove.splice(0,
-              1000 /* max batch size for DeleteObjects */)
-          }
-        }))
-    })())
-  }
-
-  public async move (from: string, keyTo: string): Promise<void> {
-    await this.client.send(new CopyObjectCommand({
-      Bucket: this.bucket,
-      Key: keyTo,
-      CopySource: join(this.bucket, from)
-    }))
-
-    await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
-  }
-
-  public async moveDir (from: string, to: string): Promise<void> {
-    const objects: ObjectIdentifier[] = await this.listObjects(from)
-
-    await Promise.all(objects.map(async ({ Key }) => {
-      const ent = posix.basename(Key!)
-      const source = posix.join(from, ent)
-      const target = posix.join(to, ent)
-
-      return this.move(source, target)
-    }))
-  }
-
-  private async listObjects (Prefix: string): Promise<ObjectIdentifier[]> {
-    const { client, bucket: Bucket } = this
-    const objects: ObjectIdentifier[] = []
-
-    for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix }))
-      for (const { Key } of page.Contents ?? [])
-        objects.push({ Key })
-
-    return objects
-  }
-}

package/source/providers/Temporary.ts

@@ -1,16 +0,0 @@
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { FileSystem } from './FileSystem'
-import type { ProviderSecrets } from '../Provider'
-
-export interface TemporaryOptions {
-  directory: string
-}
-
-export class Temporary extends FileSystem {
-  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
-    const path = join(tmpdir(), options.directory)
-
-    super({ path }, secrets)
-  }
-}

package/source/providers/Test.ts

@@ -1,13 +0,0 @@
-import { Temporary, type TemporaryOptions } from './Temporary'
-import type { ProviderSecret, ProviderSecrets } from '../Provider'
-
-export class Test extends Temporary {
-  public static override readonly SECRETS: readonly ProviderSecret[] = [
-    { name: 'USERNAME' },
-    { name: 'PASSWORD' }
-  ]
-
-  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
-    super(options, secrets)
-  }
-}