@toa.io/extensions.storages 1.0.0-alpha.0 → 1.0.0-alpha.2

package/source/providers/FileSystem.test.ts

@@ -1,13 +1,5 @@
 import { FileSystem } from './FileSystem'
 
 it('should be ok', async () => {
-  const url = new URL('file:///path')
-
-  expect(() => new FileSystem(url)).not.toThrow()
-})
-
-it('should throw if url contains host', async () => {
-  const url = new URL('file://host/path')
-
-  expect(() => new FileSystem(url)).toThrow()
+  expect(() => new FileSystem({ path: 'path' })).not.toThrow()
 })

package/source/providers/FileSystem.ts

@@ -1,46 +1,51 @@
 import { type Readable } from 'node:stream'
 import { dirname, join } from 'node:path'
-import { createReadStream } from 'node:fs'
 import fs from 'node:fs/promises'
-import fse from 'fs-extra'
-import { type Provider } from '../Provider'
+import { Provider } from '../Provider'
+import type { ProviderSecrets } from '../Provider'
 
-export class FileSystem implements Provider {
+export interface FileSystemOptions {
+  path: string
+}
+
+export class FileSystem extends Provider<FileSystemOptions> {
   protected readonly path: string
 
-  public constructor (url: URL) {
-    if (url.host !== '')
-      throw new Error('File system URL must not contain host')
+  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
+    super(options, secrets)
 
-    this.path = url.pathname
+    this.path = options.path
   }
 
   public async get (path: string): Promise<Readable | null> {
-    path = join(this.path, path)
+    try {
+      const fd = await fs.open(join(this.path, path))
 
-    if (!await fse.exists(path))
-      return null
+      return fd.createReadStream()
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return null
 
-    return createReadStream(path)
+      throw err
+    }
   }
 
   public async put (rel: string, filename: string, stream: Readable): Promise<void> {
     const dir = join(this.path, rel)
     const path = join(dir, filename)
 
-    await fse.ensureDir(dir)
+    await fs.mkdir(dirname(path), { recursive: true })
     await fs.writeFile(path, stream)
   }
 
   public async delete (path: string): Promise<void> {
-    await fse.remove(join(this.path, path))
+    await fs.rm(join(this.path, path), { recursive: true, force: true })
   }
 
   public async move (from: string, to: string): Promise<void> {
     from = join(this.path, from)
     to = join(this.path, to)
 
-    await fse.ensureDir(dirname(to))
+    await fs.mkdir(dirname(to), { recursive: true })
     await fs.rename(from, to)
   }
 }

package/source/providers/Memory.ts

@@ -0,0 +1,41 @@
+import { Readable } from 'node:stream'
+import { join } from 'node:path'
+import { buffer } from 'node:stream/consumers'
+import * as assert from 'node:assert'
+
+import { Provider } from '../Provider'
+
+/**
+ * In-memory provider
+ */
+export class InMemory extends Provider {
+  private readonly storage = new Map<string, Buffer>()
+
+  public override async get (path: string): Promise<Readable | null> {
+    const data = this.storage.get(path)
+
+    if (data === undefined) return null
+
+    return Readable.from(data)
+  }
+
+  public override async put (path: string, filename: string, stream: Readable): Promise<void> {
+    this.storage.set(join(path, filename), await buffer(stream))
+  }
+
+  public override async delete (path: string): Promise<void> {
+    for (const f of this.storage.keys())
+      if (f.startsWith(path)) this.storage.delete(f)
+  }
+
+  public override async move (from: string, to: string): Promise<void> {
+    assert.notEqual(from, to, 'Source and destination are the same')
+
+    const buf = this.storage.get(from)
+
+    assert.ok(buf !== undefined, `File not found: ${from}`)
+
+    this.storage.set(to, buf)
+    this.storage.delete(from)
+  }
+}

package/source/providers/S3.test.ts

@@ -0,0 +1,133 @@
+import { Readable } from 'node:stream'
+import { randomUUID } from 'node:crypto'
+import streamConsumers from 'node:stream/consumers'
+import { http, HttpResponse } from 'msw'
+import { setupServer } from 'msw/node'
+
+import { S3 } from './S3'
+
+/* eslint-disable @typescript-eslint/no-non-null-assertion -- jest expect is not type guard */
+
+describe('S3 storage provider', () => {
+  let provider: S3
+  let s3endpoint: ReturnType<typeof setupServer>
+
+  const AWS_REGION = 'us-east-1'
+
+  const TEST_BUCKET_NAME = 'test-bucket'
+  const S3_ENDPOINT = `https://${TEST_BUCKET_NAME}.s3.${AWS_REGION}.amazonaws.com`
+
+  beforeAll(async () => {
+    jest.useRealTimers()
+    s3endpoint = setupServer()
+    s3endpoint.listen({ onUnhandledRequest: 'error' })
+
+    provider = new S3({
+      bucket: TEST_BUCKET_NAME,
+      region: AWS_REGION
+    }, {
+      ACCESS_KEY_ID: 'test-key',
+      SECRET_ACCESS_KEY: 'test-key-secret'
+    })
+  })
+
+  afterEach(() => {
+    s3endpoint.resetHandlers()
+  })
+
+  afterAll(() => {
+    s3endpoint.close()
+  })
+
+  test('should be able to get file from S3 handling leading slash', async () => {
+    const testBody = randomUUID().repeat(10)
+
+    s3endpoint.use(http.get(`${S3_ENDPOINT}/some/absolute/path/filename`,
+      () => HttpResponse.text(testBody)))
+
+    const body = await provider.get('/some/absolute/path/filename')
+
+    expect(body).toBeInstanceOf(Readable)
+    expect((await streamConsumers.text(body!))).toBe(testBody)
+  })
+
+  test('should remove folder with all files', async () => {
+    const headHandler = jest.fn().mockReturnValue(HttpResponse.xml('', { status: 404 }))
+
+    const listHandler = jest.fn().mockReturnValueOnce(HttpResponse.xml(`
+    <ListBucketResult>
+      <NextContinuationToken>someToken</NextContinuationToken>
+      <KeyCount>1001</KeyCount>
+      <MaxKeys>1000</MaxKeys>
+      <IsTruncated>true</IsTruncated>
+      <Contents>
+        <Key>happy_face1.jpg</Key>
+      </Contents>
+    </ListBucketResult>
+    `, {
+      status: 200
+    })).mockReturnValueOnce(HttpResponse.xml(`
+    <ListBucketResult>
+      <KeyCount>1001</KeyCount>
+      <MaxKeys>1000</MaxKeys>
+      <Contents>
+        <Key>happy_face2.jpg</Key>
+      </Contents>
+      <Contents>
+        <Key>happy_face3.jpg</Key>
+      </Contents>
+    </ListBucketResult>
+    `, {
+      status: 200
+    }))
+
+    const deleteHandler = jest.fn().mockReturnValue(HttpResponse.xml(`
+    <DeleteResult>
+      <Deleted>
+        <Key>happy_face1.jpg</Key>
+      </Deleted>
+      <Deleted>
+        <Key>happy_face2.jpg</Key>
+      </Deleted>
+      <Deleted>
+        <Key>happy_face3.jpg</Key>
+      </Deleted>
+    </DeleteResult>
+    `, {
+      status: 200
+    }))
+
+    s3endpoint.use(http.head(`${S3_ENDPOINT}/some/absolute/path_to_remove`,
+      headHandler),
+    http.get(`${S3_ENDPOINT}/`,
+      listHandler),
+    http.post(`${S3_ENDPOINT}/`,
+      deleteHandler))
+
+    await provider.delete('/some/absolute/path_to_remove')
+    expect(headHandler).toHaveBeenCalledTimes(1)
+    expect(listHandler).toHaveBeenCalledTimes(2)
+    expect(deleteHandler).toHaveBeenCalledTimes(1)
+  })
+
+  test('should be able to upload a file to S3', async () => {
+    const body = Readable.from('test content')
+
+    const handler = jest.fn().mockReturnValue(HttpResponse.xml('', {
+      status: 200,
+      headers: {
+        ETag: 'test-etag',
+        'x-amz-id-2': 'test_id_2',
+        'x-amz-request-id': 'test_request_id',
+        Server: 'AmazonS3'
+      }
+    }))
+
+    s3endpoint.use(http.put(`${S3_ENDPOINT}/some/absolute/path/filename`,
+      handler))
+
+    await provider.put('/some/absolute/path', 'filename', body)
+    expect(handler).toHaveBeenCalledTimes(1)
+    expect(body.readableEnded).toBe(true)
+  })
+})

package/source/providers/S3.ts

@@ -1,91 +1,166 @@
-import { type Readable } from 'node:stream'
+import { Readable } from 'node:stream'
+import { Blob } from 'node:buffer'
 import { join } from 'node:path/posix'
+import assert from 'node:assert'
 import { Upload } from '@aws-sdk/lib-storage'
 import {
   S3Client,
   GetObjectCommand,
   CopyObjectCommand,
-  ListObjectsV2Command,
   DeleteObjectsCommand,
-  type S3ClientConfigType
+  paginateListObjectsV2,
+  HeadObjectCommand,
+  NotFound,
+  DeleteObjectCommand,
+  NoSuchKey,
+  type S3ClientConfigType,
+  type ObjectIdentifier
 } from '@aws-sdk/client-s3'
-import { type Provider } from '../Provider'
+import * as nodeNativeFetch from 'smithy-node-native-fetch'
+import { Provider, type ProviderSecret, type ProviderSecrets } from '../Provider'
+import type { ReadableStream } from 'node:stream/web'
+
+export interface S3Options {
+  bucket: string
+  region?: string
+  prefix?: string
+  endpoint?: string
+}
+
+type S3Secrets = ProviderSecrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
+
+export class S3 extends Provider<S3Options> {
+  public static override readonly SECRETS: readonly ProviderSecret[] = [
+    { name: 'ACCESS_KEY_ID', optional: true },
+    { name: 'SECRET_ACCESS_KEY', optional: true }
+  ]
 
-export class S3 implements Provider {
   protected readonly bucket: string
   protected readonly client: S3Client
 
-  public constructor (url: URL, secrets: Record<string, string>) {
-    this.bucket = url.pathname.split('/')[1]
+  public constructor (options: S3Options, secrets?: S3Secrets) {
+    super(options)
 
-    if (this.bucket === undefined || this.bucket === '')
-      throw new Error('S3 bucket not specified')
+    this.bucket = options.bucket
 
     const s3Config: S3ClientConfigType = {
-      credentials: {
+      retryMode: 'adaptive',
+      ...nodeNativeFetch
+    }
+
+    if (options.endpoint !== undefined) {
+      s3Config.forcePathStyle = options.endpoint.startsWith('http://')
+      s3Config.endpoint = options.endpoint
+    }
+
+    if (options.region !== undefined) s3Config.region = options.region
+
+    if (typeof secrets?.ACCESS_KEY_ID === 'string') {
+      assert.ok(secrets.SECRET_ACCESS_KEY !== undefined,
+        'SECRET_ACCESS_KEY is required if ACCESS_KEY_ID is provided')
+
+      s3Config.credentials = {
         accessKeyId: secrets.ACCESS_KEY_ID,
         secretAccessKey: secrets.SECRET_ACCESS_KEY
-      },
-      region: url.host,
-      endpoint: url.searchParams.get('endpoint') ?? undefined
+      }
     }
 
     this.client = new S3Client(s3Config)
+
+    this.client.middlewareStack.add((next, _context) => async (args) => {
+      if ('Key' in args.input && typeof args.input.Key === 'string')
+      // removes leading slash
+        args.input.Key = args.input.Key.replace(/^\//, '')
+
+      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
+      // removes leading slash and ensures finishing slash
+        args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
+
+      return next(args)
+    },
+    {
+      step: 'initialize',
+      priority: 'high',
+      name: 'normalizesSlashesInPath'
+    })
   }
 
-  public async get (key: string): Promise<Readable | null> {
+  public async get (Key: string): Promise<Readable | null> {
     try {
       const fileResponse = await this.client.send(new GetObjectCommand({
         Bucket: this.bucket,
-        Key: key
+        Key
       }))
 
-      return fileResponse.Body as Readable
-    } catch (err: any) {
-      if (err?.Code === 'NoSuchKey') return null
+      if (fileResponse.Body === undefined) return null // should never happen
+
+      if (fileResponse.Body instanceof Readable) return fileResponse.Body
+
+      return Readable.fromWeb((fileResponse.Body instanceof Blob
+        ? fileResponse.Body.stream()
+        : fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
+    } catch (err) {
+      if (err instanceof NotFound || err instanceof NoSuchKey) return null
       else throw err
     }
   }
 
   public async put (path: string, filename: string, stream: Readable): Promise<void> {
-    const key = join(path, filename)
-
     await new Upload({
       client: this.client,
       params: {
         Bucket: this.bucket,
-        Key: key,
+        Key: join(path, filename),
         Body: stream
       }
     }).done()
   }
 
-  public async delete (key: string): Promise<void> {
-    const listResponse = await this.client.send(new ListObjectsV2Command({
-      Bucket: this.bucket,
-      Prefix: key.substring(1)
-    }))
+  /**
+   * Deletes either a single object or "directory" - all objects
+   * with given prefix (prefix will be enforced to finish with `/`)
+   * @param Key - key name or path prefix
+   */
+  public async delete (Key: string): Promise<void> {
+    const { client, bucket: Bucket } = this
 
-    if (listResponse.Contents !== undefined && listResponse.Contents.length > 0)
-      await this.client.send(new DeleteObjectsCommand({
-        Bucket: this.bucket,
-        Delete: {
-          Objects: listResponse.Contents.map(({ Key }) => ({ Key }))
-        }
-      }))
-  }
+    // checking if given key is a single file
+    if (!Key.endsWith('/'))
+      try {
+        // DeleteObject on S3 returns no error if object does not exist
+        await client.send(new HeadObjectCommand({ Bucket, Key }))
+        await client.send(new DeleteObjectCommand({ Bucket, Key }))
 
-  public async move (from: string, keyTo: string): Promise<void> {
-    const keyFrom = join(this.bucket, from)
+        return
+      } catch (err) {
+        assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
+      }
+
+    const objectsToRemove: ObjectIdentifier[] = []
 
-    await new Promise((resolve) => setTimeout(resolve, 500))
+    for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix: Key }))
+      for (const { Key } of page.Contents ?? []) objectsToRemove.push({ Key })
 
+    // Removing all objects in parallel in batches
+    await Promise.all((function * () {
+      while (objectsToRemove.length > 0)
+        yield client.send(new DeleteObjectsCommand({
+          Bucket,
+          Delete: {
+            Objects: objectsToRemove.splice(0,
+              1000 /* max batch size for DeleteObjects */)
+          }
+        }))
+    })())
+  }
+
+  public async move (from: string, keyTo: string): Promise<void> {
     await this.client.send(new CopyObjectCommand({
       Bucket: this.bucket,
       Key: keyTo,
-      CopySource: keyFrom
+      CopySource: join(this.bucket, from)
     }))
 
-    await this.delete(from)
+    await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
   }
 }

package/source/providers/Temporary.ts

@@ -1,14 +1,16 @@
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-
 import { FileSystem } from './FileSystem'
+import type { ProviderSecrets } from '../Provider'
 
-export class Temporary extends FileSystem {
-  protected override readonly path: string
+export interface TemporaryOptions {
+  directory: string
+}
 
-  public constructor (url: URL) {
-    super(url)
+export class Temporary extends FileSystem {
+  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
+    const path = join(tmpdir(), options.directory)
 
-    this.path = join(tmpdir(), url.pathname)
+    super({ path }, secrets)
   }
 }

package/source/providers/Test.ts

@@ -1,13 +1,13 @@
-import assert from 'node:assert'
-import { Temporary } from './Temporary'
+import { Temporary, type TemporaryOptions } from './Temporary'
+import type { ProviderSecret, ProviderSecrets } from '../Provider'
 
 export class Test extends Temporary {
-  public static readonly SECRETS = ['USERNAME', 'PASSWORD']
+  public static override readonly SECRETS: readonly ProviderSecret[] = [
+    { name: 'USERNAME' },
+    { name: 'PASSWORD' }
+  ]
 
-  public constructor (url: URL, secrets: Record<string, string>) {
-    super(url)
-
-    assert(secrets.USERNAME !== undefined, 'Missing USERNAME')
-    assert(secrets.PASSWORD !== undefined, 'Missing PASSWORD')
+  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
+    super(options, secrets)
   }
 }

package/source/providers/index.test.ts

@@ -1,21 +1,26 @@
 import { type Readable } from 'node:stream'
-import { buffer } from '@toa.io/streams'
-import { cases, open, rnd, read } from '../test/util'
+import { buffer } from 'node:stream/consumers'
+import { readFile } from 'node:fs/promises'
+import { createReadStream } from 'node:fs'
+import path from 'node:path'
+import { randomUUID } from 'node:crypto'
+import { suites } from '../test/util'
 import { providers } from './index'
+import type { ProviderConstructor } from '../Provider'
 
-describe.each(cases)('%s', (...suite) => {
-  const [protocol, url, secrets, init] = suite
-  const Provider = providers[protocol]
-  const provider = new Provider(url, secrets)
+describe.each(suites)('$provider', (suite) => {
+  const it = suite.run ? global.it : global.it.skip
+  const Provider: ProviderConstructor = providers[suite.provider]
+  const provider = new Provider(suite.options, suite.secrets)
 
   let dir: string
 
   beforeAll(async () => {
-    await init?.(url, secrets)
+    process.chdir(path.resolve(__dirname, '../test/'))
   })
 
   beforeEach(() => {
-    dir = '/' + rnd()
+    dir = '/' + randomUUID()
   })
 
   it('should be', async () => {
@@ -23,39 +28,39 @@ describe.each(cases)('%s', (...suite) => {
   })
 
   it('should return null if file not found', async () => {
-    const result = await provider.get(rnd())
+    const result = await provider.get(randomUUID())
 
     expect(result).toBeNull()
   })
 
   it('should create entry', async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
     await provider.put(dir, 'lenna.png', stream)
 
     const readable = await provider.get(dir + '/lenna.png') as Readable
     const output = await buffer(readable)
-    const lenna = await read('lenna.png')
+    const lenna = await readFile('lenna.png')
 
     expect(output.compare(lenna)).toBe(0)
   })
 
   it('should overwrite existing entry', async () => {
-    const stream0 = open('lenna.png')
-    const stream1 = open('albert.jpg')
+    const stream0 = createReadStream('lenna.png')
+    const stream1 = createReadStream('albert.jpg')
 
     await provider.put(dir, 'lenna.png', stream0)
     await provider.put(dir, 'lenna.png', stream1)
 
     const readable = await provider.get(dir + '/lenna.png') as Readable
     const output = await buffer(readable)
-    const albert = await read('albert.jpg')
+    const albert = await readFile('albert.jpg')
 
     expect(output.compare(albert)).toBe(0)
   })
 
   it('should get by path', async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
     await provider.put(dir, 'lenna.png', stream)
 
@@ -77,7 +82,7 @@ describe.each(cases)('%s', (...suite) => {
      */
 
     it('should delete entry', async () => {
-      const stream = open('lenna.png')
+      const stream = createReadStream('lenna.png')
 
       await provider.put(dir, 'lenna.png', stream)
       await provider.delete(dir + '/lenna.png')
@@ -92,7 +97,7 @@ describe.each(cases)('%s', (...suite) => {
     })
 
     it('should delete directory', async () => {
-      const stream = open('lenna.png')
+      const stream = createReadStream('lenna.png')
 
       await provider.put(dir, 'lenna.png', stream)
       await provider.delete(dir)
@@ -103,8 +108,8 @@ describe.each(cases)('%s', (...suite) => {
     })
 
     it('should move an entry', async () => {
-      const stream = open('lenna.png')
-      const dir2 = '/' + rnd()
+      const stream = createReadStream('lenna.png')
+      const dir2 = '/' + randomUUID()
 
       await provider.put(dir, 'lenna.png', stream)
       await provider.move(dir + '/lenna.png', dir2 + '/lenna2.png')

package/source/providers/index.ts

@@ -1,15 +1,16 @@
-import { type Provider } from '../Provider'
 import { FileSystem } from './FileSystem'
 import { S3 } from './S3'
 import { Temporary } from './Temporary'
 import { Test } from './Test'
+import { InMemory } from './Memory'
+import type { ProviderConstructor } from '../Provider'
 
-export const providers: Record<string, ProviderClass> = {
-  'file:': FileSystem,
-  'tmp:': Temporary,
-  'test:': Test,
-  's3:': S3
-}
+export const providers = {
+  s3: S3,
+  fs: FileSystem,
+  tmp: Temporary,
+  mem: InMemory,
+  test: Test
+} as const satisfies Record<string, ProviderConstructor>
 
-export type ProviderClass =
-  (new (url: URL, secrets: Record<string, string>) => Provider) & { SECRETS?: string[] }
+export type { Declaration } from './Declaration'

package/source/providers/readme.md

@@ -1,6 +1,6 @@
 # Developing a provider
 
-1. Add an implementaion of the [Provider](../Provider.ts) interface under this directory.
+1. Add an implementation of the [Provider](../Provider.ts) interface under this directory.
 2. Add an entry to the provider map in [`index.ts`](./index.ts).
 3. Add a suite to the `suites` object in [`util.ts`](../.test/util.ts).
 4. Run `$ npm test` in the [`storages` directory](../..).