@toa.io/extensions.storages 1.0.0-alpha.13 → 1.0.0-alpha.143

package/source/providers/Declaration.ts

@@ -1,10 +1,11 @@
 import type { S3Options } from './S3'
+import type { CloudinaryOptions } from './Cloudinary'
 import type { FileSystemOptions } from './FileSystem'
 import type { TemporaryOptions } from './Temporary'
 
 export type Declaration =
   ({ provider: 's3' } & S3Options)
+  | ({ provider: 'cloudinary' } & CloudinaryOptions)
   | ({ provider: 'fs' } & FileSystemOptions)
   | ({ provider: 'tmp' } & TemporaryOptions)
-  | ({ provider: 'mem' })
   | ({ provider: 'test' } & TemporaryOptions)

package/source/providers/FileSystem.ts

@@ -1,51 +1,105 @@
-import { type Readable } from 'node:stream'
-import { dirname, join } from 'node:path'
 import fs from 'node:fs/promises'
+import { createReadStream } from 'node:fs'
+import { dirname, join } from 'node:path'
 import { Provider } from '../Provider'
-import type { ProviderSecrets } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
+import type { Readable } from 'node:stream'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
 
 export interface FileSystemOptions {
   path: string
+  claim?: string
 }
 
 export class FileSystem extends Provider<FileSystemOptions> {
-  protected readonly path: string
+  public override readonly root: string
 
-  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
-    super(options, secrets)
+  public constructor (options: FileSystemOptions) {
+    super(options)
 
-    this.path = options.path
+    this.root = options.path
   }
 
-  public async get (path: string): Promise<Readable | null> {
-    try {
-      const fd = await fs.open(join(this.path, path))
+  public async get (rel: string): Promise<Maybe<Stream>> {
+    const path = this.blob(rel)
+    const metadata = await this.head(rel)
 
-      return fd.createReadStream()
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return null
+    if (metadata instanceof Error)
+      return metadata
 
-      throw err
-    }
+    const stream = createReadStream(path)
+
+    return { stream, ...metadata }
+  }
+
+  public async head (rel: string): Promise<Maybe<Metadata>> {
+    const path = this.meta(rel)
+
+    return this.try(async () => {
+      const contents = await fs.readFile(path, 'utf8')
+
+      return JSON.parse(contents)
+    })
   }
 
-  public async put (rel: string, filename: string, stream: Readable): Promise<void> {
-    const dir = join(this.path, rel)
-    const path = join(dir, filename)
+  public async put (rel: string, stream: Readable): Promise<void> {
+    const path = this.blob(rel)
+    const dir = dirname(path)
 
-    await fs.mkdir(dirname(path), { recursive: true })
+    await fs.mkdir(dir, { recursive: true })
     await fs.writeFile(path, stream)
   }
 
+  public async commit (rel: string, metadata: Metadata): Promise<void> {
+    const path = this.meta(rel)
+
+    await fs.writeFile(path, JSON.stringify(metadata), 'utf8')
+  }
+
   public async delete (path: string): Promise<void> {
-    await fs.rm(join(this.path, path), { recursive: true, force: true })
+    await Promise.all([
+      fs.rm(this.blob(path), { force: true }),
+      fs.rm(this.meta(path), { force: true })
+    ])
   }
 
-  public async move (from: string, to: string): Promise<void> {
-    from = join(this.path, from)
-    to = join(this.path, to)
+  public async move (from: string, to: string): Promise<Maybe<void>> {
+    const bf = this.blob(from)
+    const bt = this.blob(to)
+    const mf = this.meta(from)
+    const mt = this.meta(to)
 
-    await fs.mkdir(dirname(to), { recursive: true })
-    await fs.rename(from, to)
+    await fs.mkdir(dirname(bt), { recursive: true })
+
+    return await this.try(async () => {
+      await Promise.all([
+        fs.rename(bf, bt),
+        fs.rename(mf, mt)
+      ])
+    })
+  }
+
+  private blob (rel: string): string {
+    return this.join(rel, '.blob')
+  }
+
+  private meta (rel: string): string {
+    return this.join(rel, '.meta')
+  }
+
+  private join (rel: string, ext: string): string {
+    return join(this.root, rel) + ext
+  }
+
+  private async try<T = void> (action: () => Promise<T>): Promise<Maybe<T>> {
+    try {
+      return await action()
+    } catch (err: NodeJS.ErrnoException | any) {
+      if (err?.code === 'ENOENT')
+        return ERR_NOT_FOUND
+      else
+        throw err
+    }
   }
 }

package/source/providers/S3.ts

@@ -3,22 +3,15 @@ import { Blob } from 'node:buffer'
 import { join } from 'node:path/posix'
 import assert from 'node:assert'
 import { Upload } from '@aws-sdk/lib-storage'
-import {
-  S3Client,
-  GetObjectCommand,
-  CopyObjectCommand,
-  DeleteObjectsCommand,
-  paginateListObjectsV2,
-  HeadObjectCommand,
-  NotFound,
-  DeleteObjectCommand,
-  NoSuchKey,
-  type S3ClientConfigType,
-  type ObjectIdentifier
-} from '@aws-sdk/client-s3'
+import * as s3 from '@aws-sdk/client-s3'
 import * as nodeNativeFetch from 'smithy-node-native-fetch'
-import { Provider, type ProviderSecret, type ProviderSecrets } from '../Provider'
+import { console } from 'openspan'
+import { Provider } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
 import type { ReadableStream } from 'node:stream/web'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
+import type { Secret, Secrets } from '../Secrets'
 
 export interface S3Options {
   bucket: string
@@ -27,23 +20,23 @@ export interface S3Options {
   endpoint?: string
 }
 
-type S3Secrets = ProviderSecrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
+type S3Secrets = Secrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
 
 export class S3 extends Provider<S3Options> {
-  public static override readonly SECRETS: readonly ProviderSecret[] = [
+  public static override readonly SECRETS: readonly Secret[] = [
     { name: 'ACCESS_KEY_ID', optional: true },
     { name: 'SECRET_ACCESS_KEY', optional: true }
   ]
 
-  protected readonly bucket: string
-  protected readonly client: S3Client
+  private readonly bucket: string
+  private readonly client: s3.S3Client
 
   public constructor (options: S3Options, secrets?: S3Secrets) {
-    super(options)
+    super(options, secrets)
 
     this.bucket = options.bucket
 
-    const s3Config: S3ClientConfigType = {
+    const s3Config: s3.S3ClientConfigType = {
       retryMode: 'adaptive',
       ...nodeNativeFetch
     }
@@ -53,7 +46,8 @@ export class S3 extends Provider<S3Options> {
       s3Config.endpoint = options.endpoint
     }
 
-    if (options.region !== undefined) s3Config.region = options.region
+    if (options.region !== undefined)
+      s3Config.region = options.region
 
     if (typeof secrets?.ACCESS_KEY_ID === 'string') {
       assert.ok(secrets.SECRET_ACCESS_KEY !== undefined,
@@ -65,15 +59,15 @@ export class S3 extends Provider<S3Options> {
       }
     }
 
-    this.client = new S3Client(s3Config)
+    this.client = new s3.S3Client(s3Config)
 
     this.client.middlewareStack.add((next, _context) => async (args) => {
-      if ('Key' in args.input && typeof args.input.Key === 'string')
       // removes leading slash
+      if ('Key' in args.input && typeof args.input.Key === 'string')
         args.input.Key = args.input.Key.replace(/^\//, '')
 
-      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
       // removes leading slash and ensures finishing slash
+      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
         args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
 
       return next(args)
@@ -85,82 +79,89 @@ export class S3 extends Provider<S3Options> {
     })
   }
 
-  public async get (Key: string): Promise<Readable | null> {
-    try {
-      const fileResponse = await this.client.send(new GetObjectCommand({
+  public async get (Key: string): Promise<Maybe<Stream>> {
+    return await this.try<Stream>(async () => {
+      const entry = await this.client.send(new s3.GetObjectCommand({
         Bucket: this.bucket,
         Key
       }))
 
-      if (fileResponse.Body === undefined) return null // should never happen
+      const stream = entry.Body instanceof Readable
+        ? entry.Body
+        : Readable.fromWeb((entry.Body instanceof Blob
+          ? entry.Body.stream()
+          : entry.Body) as ReadableStream)
 
-      if (fileResponse.Body instanceof Readable) return fileResponse.Body
+      if (entry.Metadata?.value === undefined)
+        return ERR_NOT_FOUND
 
-      return Readable.fromWeb((fileResponse.Body instanceof Blob
-        ? fileResponse.Body.stream()
-        : fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
-    } catch (err) {
-      if (err instanceof NotFound || err instanceof NoSuchKey) return null
-      else throw err
-    }
+      const metadata = JSON.parse(entry.Metadata.value)
+
+      return { stream, ...metadata }
+    })
+  }
+
+  public async head (Key: string): Promise<Maybe<Metadata>> {
+    return await this.try<Metadata>(async () => {
+      const entry = await this.client.send(new s3.HeadObjectCommand({
+        Bucket: this.bucket,
+        Key
+      }))
+
+      if (entry.Metadata?.value === undefined)
+        return ERR_NOT_FOUND
+
+      return JSON.parse(entry.Metadata.value)
+    })
   }
 
-  public async put (path: string, filename: string, stream: Readable): Promise<void> {
+  public async put (Key: string, stream: Readable): Promise<void> {
     await new Upload({
       client: this.client,
       params: {
         Bucket: this.bucket,
-        Key: join(path, filename),
+        Key,
         Body: stream
       }
     }).done()
   }
 
-  /**
-   * Deletes either a single object or "directory" - all objects
-   * with given prefix (prefix will be enforced to finish with `/`)
-   * @param Key - key name or path prefix
-   */
-  public async delete (Key: string): Promise<void> {
-    const { client, bucket: Bucket } = this
-
-    // checking if given key is a single file
-    if (!Key.endsWith('/'))
-      try {
-        // DeleteObject on S3 returns no error if object does not exist
-        await client.send(new HeadObjectCommand({ Bucket, Key }))
-        await client.send(new DeleteObjectCommand({ Bucket, Key }))
-
-        return
-      } catch (err) {
-        assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
-      }
+  public async commit (Key: string, metadata: object): Promise<void> {
+    await this.client.send(new s3.CopyObjectCommand({
+      Bucket: this.bucket,
+      Key,
+      CopySource: join(this.bucket, Key),
+      Metadata: { value: JSON.stringify(metadata) },
+      MetadataDirective: 'REPLACE'
+    }))
 
-    const objectsToRemove: ObjectIdentifier[] = []
-
-    for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix: Key }))
-      for (const { Key } of page.Contents ?? []) objectsToRemove.push({ Key })
-
-    // Removing all objects in parallel in batches
-    await Promise.all((function * () {
-      while (objectsToRemove.length > 0)
-        yield client.send(new DeleteObjectsCommand({
-          Bucket,
-          Delete: {
-            Objects: objectsToRemove.splice(0,
-              1000 /* max batch size for DeleteObjects */)
-          }
-        }))
-    })())
+    console.debug('Uploaded to S3', { bucket: this.bucket, path: Key, metadata })
   }
 
-  public async move (from: string, keyTo: string): Promise<void> {
-    await this.client.send(new CopyObjectCommand({
-      Bucket: this.bucket,
-      Key: keyTo,
-      CopySource: join(this.bucket, from)
-    }))
+  public async delete (Key: string): Promise<void> {
+    await this.client.send(new s3.DeleteObjectCommand({ Bucket: this.bucket, Key }))
+  }
 
-    await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
+  public async move (from: string, keyTo: string): Promise<Maybe<void>> {
+    return await this.try(async () => {
+      await this.client.send(new s3.CopyObjectCommand({
+        Bucket: this.bucket,
+        Key: keyTo,
+        CopySource: join(this.bucket, from)
+      }))
+
+      await this.client.send(new s3.DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
+    })
+  }
+
+  private async try<T = void> (action: () => Promise<T>): Promise<Maybe<T>> {
+    try {
+      return await action()
+    } catch (err: any) {
+      if (err?.name === 'NotFound' || err?.name === 'NoSuchKey')
+        return ERR_NOT_FOUND
+      else
+        throw err
+    }
   }
 }

package/source/providers/Temporary.ts

@@ -1,16 +1,15 @@
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { FileSystem } from './FileSystem'
-import type { ProviderSecrets } from '../Provider'
 
 export interface TemporaryOptions {
   directory: string
 }
 
 export class Temporary extends FileSystem {
-  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
+  public constructor (options: TemporaryOptions) {
     const path = join(tmpdir(), options.directory)
 
-    super({ path }, secrets)
+    super({ path })
   }
 }

package/source/providers/Test.ts

@@ -1,13 +1,13 @@
 import { Temporary, type TemporaryOptions } from './Temporary'
-import type { ProviderSecret, ProviderSecrets } from '../Provider'
+import type { Secret } from '../Secrets'
 
 export class Test extends Temporary {
-  public static override readonly SECRETS: readonly ProviderSecret[] = [
+  public static override readonly SECRETS: readonly Secret[] = [
     { name: 'USERNAME' },
     { name: 'PASSWORD' }
   ]
 
-  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
-    super(options, secrets)
+  public constructor (options: TemporaryOptions) {
+    super(options)
   }
 }

package/source/providers/index.test.ts

@@ -1,126 +1,152 @@
-import { type Readable } from 'node:stream'
-import { buffer } from 'node:stream/consumers'
-import { readFile } from 'node:fs/promises'
+import assert from 'node:assert'
 import { createReadStream } from 'node:fs'
-import path from 'node:path'
-import { randomUUID } from 'node:crypto'
+import { resolve } from 'node:path'
 import { suites } from '../test/util'
 import { providers } from './index'
-import type { ProviderConstructor } from '../Provider'
+import type { Constructor } from '../Provider'
+import type { Metadata, Stream } from '../Entry'
+
+const sample = resolve(__dirname, '../test/sample.jpeg')
+const lenna = resolve(__dirname, '../test/lenna.png')
+
+const metadata: Metadata = {
+  type: 'image/jpeg',
+  size: 73444,
+  checksum: 'd41d8cd98f00b204e9800998ecf8427e',
+  created: new Date().toISOString(),
+  attributes: {}
+}
 
 describe.each(suites)('$provider', (suite) => {
+  const id = Math.random().toString(36).substring(7)
   const it = suite.run ? global.it : global.it.skip
-  const Provider: ProviderConstructor = providers[suite.provider]
+  const Provider: Constructor = providers[suite.provider]
   const provider = new Provider(suite.options, suite.secrets)
 
-  let dir: string
+  describe('put, get, head', () => {
+    let entry: Stream
 
-  beforeAll(async () => {
-    process.chdir(path.resolve(__dirname, '../test/'))
-  })
+    beforeAll(async () => {
+      await provider.put(id, createReadStream(sample))
+      await provider.commit(id, metadata)
 
-  beforeEach(() => {
-    dir = '/' + randomUUID()
-  })
+      entry = await provider.get(id) as Stream
+    })
 
-  it('should be', async () => {
-    expect(provider).toBeInstanceOf(Provider)
-  })
+    afterAll(() => entry.stream.destroy())
 
-  it('should return null if file not found', async () => {
-    const result = await provider.get(randomUUID())
+    it('should create metadata', async () => {
+      expect(entry.size).toEqual(metadata.size)
+    })
 
-    expect(result).toBeNull()
-  })
+    if (suite.provider !== 'cloudinary')
+      it('should store attributes', async () => {
+        const path = '/path/to/file'
 
-  it('should create entry', async () => {
-    const stream = createReadStream('lenna.png')
+        await provider.put(path, createReadStream(sample))
+        await provider.commit(path, { ...metadata, attributes: { foo: 'bar' } })
 
-    await provider.put(dir, 'lenna.png', stream)
+        const entry = await provider.get(path) as Stream
 
-    const readable = await provider.get(dir + '/lenna.png') as Readable
-    const output = await buffer(readable)
-    const lenna = await readFile('lenna.png')
+        expect(entry.attributes).toEqual({ foo: 'bar' })
 
-    expect(output.compare(lenna)).toBe(0)
-  })
+        entry.stream.destroy()
+      })
 
-  it('should overwrite existing entry', async () => {
-    const stream0 = createReadStream('lenna.png')
-    const stream1 = createReadStream('albert.jpg')
+    it('should overwrite', async () => {
+      const path = Math.random().toString(36).substring(7)
 
-    await provider.put(dir, 'lenna.png', stream0)
-    await provider.put(dir, 'lenna.png', stream1)
+      await provider.put(path, createReadStream(sample))
+      await provider.commit(path, metadata)
 
-    const readable = await provider.get(dir + '/lenna.png') as Readable
-    const output = await buffer(readable)
-    const albert = await readFile('albert.jpg')
+      if (suite.provider === 'cloudinary')
+        await new Promise((resolve) => setTimeout(resolve, 5_000))
 
-    expect(output.compare(albert)).toBe(0)
-  })
+      await expect(provider.put(path, createReadStream(lenna)))
+        .resolves.not.toThrow()
 
-  it('should get by path', async () => {
-    const stream = createReadStream('lenna.png')
+      const meta = { ...metadata, size: 473831 } // lenna size
 
-    await provider.put(dir, 'lenna.png', stream)
+      await provider.commit(path, meta)
 
-    const result = await provider.get('/bar/lenna.png')
+      if (suite.provider === 'cloudinary')
+        await new Promise((resolve) => setTimeout(resolve, 10_000))
 
-    expect(result).toBeNull()
-  })
+      const overwritten = await provider.get(path) as Stream
 
-  describe('danger', () => {
-    /*
+      expect(overwritten.size).toEqual(meta.size)
 
-    WHEN MAKING CHANGES TO DELETION,
-    ALWAYS RUN TESTS IN STEP-BY-STEP DEBUGGING MODE
+      overwritten.stream.destroy()
+    })
 
-    YOU MAY EVENTUALLY DELETE YOUR ENTIRE FILE SYSTEM
+    it('should return error if not found', async () => {
+      const error = await provider.get(Math.random().toString(36).substring(7)) as any
 
-                              Sincerely yours, Murphy
+      expect(error).toBeInstanceOf(Error)
+      expect(error.code).toBe('NOT_FOUND')
+    })
+
+    it('should return entry', async () => {
+      const entry = await provider.head(id)
+
+      assert.ok(!(entry instanceof Error))
 
-     */
+      expect(entry.size).toEqual(metadata.size)
+    })
+  })
 
-    it('should delete entry', async () => {
-      const stream = createReadStream('lenna.png')
+  describe('delete', () => {
+    const path = '/path/to/' + Math.random().toString(36).substring(7)
 
-      await provider.put(dir, 'lenna.png', stream)
-      await provider.delete(dir + '/lenna.png')
+    it('should remove file', async () => {
+      await provider.put(path, createReadStream(sample))
+      await provider.commit(path, metadata)
+      await provider.delete(path)
 
-      const result = await provider.get(dir + '/lenna.png')
+      const error = await provider.get(path) as any
 
-      expect(result).toBeNull()
+      expect(error).toBeInstanceOf(Error)
+      expect(error.code).toBe('NOT_FOUND')
     })
 
-    it('should not throw if path does not exists', async () => {
-      await expect(provider.delete(dir + '/whatever')).resolves.not.toThrow()
+    it('should not return error if not found', async () => {
+      const empty = await provider.delete(Math.random().toString(36).substring(7))
+
+      expect(empty).toBeUndefined()
     })
+  })
 
-    it('should delete directory', async () => {
-      const stream = createReadStream('lenna.png')
+  describe('move', () => {
+    it.each([true, false])('should move (commit: %s)', async (committed) => {
+      const from = Math.random().toString(36).substring(7)
+      const to = Math.random().toString(36).substring(7)
 
-      await provider.put(dir, 'lenna.png', stream)
-      await provider.delete(dir)
+      await provider.put(from, createReadStream(sample))
 
-      const result = await provider.get(dir + '/lenna.png')
+      if (committed)
+        await provider.commit(from, metadata)
 
-      expect(result).toBeNull()
-    })
+      await provider.move(from, to)
 
-    it('should move an entry', async () => {
-      const stream = createReadStream('lenna.png')
-      const dir2 = '/' + randomUUID()
+      if (!committed)
+        await provider.commit(to, metadata)
 
-      await provider.put(dir, 'lenna.png', stream)
-      await provider.move(dir + '/lenna.png', dir2 + '/lenna2.png')
+      const error = await provider.get(from) as any
 
-      const result = await provider.get(dir2 + '/lenna2.png') as Readable
+      expect(error).toBeInstanceOf(Error)
 
-      expect(result).not.toBeNull()
+      const entry = await provider.get(to) as Stream
 
-      const nope = await provider.get(dir + '/lenna.png')
+      expect(entry.stream).toBeDefined()
+    })
 
-      expect(nope).toBeNull()
+    it('should return error if not found', async () => {
+      const error = await provider.move(Math.random().toString(36).substring(7), 'whatever') as any
+
+      expect(error).toBeInstanceOf(Error)
+      expect(error.code).toBe('NOT_FOUND')
     })
   })
 })
+
+jest.setTimeout(30_000)