@toa.io/extensions.storages 1.0.0-alpha.21 → 1.0.0-alpha.219

package/source/providers/Cloudinary.ts

@@ -0,0 +1,320 @@
+import { basename, dirname, join } from 'node:path'
+import { Readable } from 'node:stream'
+import { v2 as cloudinary } from 'cloudinary'
+import { console } from 'openspan'
+import { Provider } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
+import type { Secret, Secrets } from '../Secrets'
+import type { ReadableStream } from 'node:stream/web'
+import type {
+  ConfigOptions,
+  TransformationOptions,
+  UploadApiOptions
+} from 'cloudinary'
+
+export type CloudinarySecrets = Secrets<'API_KEY' | 'API_SECRET'>
+
+export class Cloudinary extends Provider<CloudinaryOptions> {
+  public static override readonly SECRETS: readonly Secret[] = [
+    { name: 'API_KEY' },
+    { name: 'API_SECRET' }
+  ]
+
+  private readonly type: StorageType
+  private readonly eager: TransformationOptions[] = []
+  private readonly transformations: Transformation[] = []
+  private readonly config: ConfigOptions
+  private readonly prefix: string
+
+  public constructor (options: CloudinaryOptions, secrets?: CloudinarySecrets) {
+    super(options, secrets)
+
+    this.type = options.type
+
+    if (options.eager !== undefined)
+      this.eager = options.eager
+
+    if (options.transformations !== undefined)
+      this.transformations = options.transformations.map((transformation) => {
+        const { extension, ...rest } = transformation
+
+        let expression = extension
+
+        if (!extension.startsWith('^'))
+          expression = '^' + extension
+
+        if (!extension.endsWith('$'))
+          expression = extension + '$'
+
+        return {
+          extension: new RegExp(expression),
+          ...rest
+        }
+      })
+
+    this.config = {
+      cloud_name: options.environment,
+      api_key: secrets!.API_KEY,
+      api_secret: secrets!.API_SECRET
+    }
+
+    this.prefix = options.prefix ?? '/'
+  }
+
+  public async get (path: string, options?: GetOptions): Promise<Maybe<Stream>> {
+    const headers: Record<string, string> = {}
+
+    if (options?.range !== undefined)
+      headers.range = options.range
+
+    if (options?.agent !== undefined)
+      headers['user-agent'] = options.agent
+
+    const response = await this.fetch(path, { method: 'GET', headers })
+
+    if (response instanceof Error)
+      return ERR_NOT_FOUND
+
+    const metadata = this.metadata(response)
+
+    return {
+      stream: Readable.fromWeb(response.body as ReadableStream),
+      ...metadata
+    }
+  }
+
+  public async head (path: string): Promise<Maybe<Metadata>> {
+    const response = await this.fetch(path, { method: 'HEAD' })
+
+    if (response instanceof Error)
+      return ERR_NOT_FOUND
+
+    return this.metadata(response)
+  }
+
+  public async put (path: string, stream: Readable): Promise<void> {
+    const id = basename(path)
+    const folder = join(this.prefix, dirname(path))
+
+    await new Promise((resolve, reject) => {
+      const options: UploadApiOptions = {
+        public_id: id,
+        folder,
+        eager: this.eager,
+        resource_type: this.type
+      }
+
+      console.debug('Uploading to Cloudinary', { path, options })
+
+      stream.pipe(this.cloudinary().uploader.upload_stream(options,
+        (error, result) => {
+          if (error !== undefined) reject(error)
+          else resolve(result)
+        }))
+
+      stream.on('error', (e) => {
+        console.error('Cloudinary stream error', { path, error: e })
+        stream.destroy()
+
+        reject(e)
+      })
+    })
+  }
+
+  public async commit (): Promise<void> {
+    // metadata is read-only
+  }
+
+  public async delete (path: string): Promise<void> {
+    const id = join(this.prefix, path)
+
+    console.debug('Deleting from Cloudinary', { path: id })
+
+    await this.cloudinary().uploader.destroy(id,
+      { resource_type: this.type, invalidate: true })
+  }
+
+  public async move (from: string, to: string): Promise<void | Error> {
+    const source = join(this.prefix, from)
+    const target = join(this.prefix, to)
+
+    try {
+      await this.cloudinary().uploader.rename(source, target,
+        { resource_type: this.type, overwrite: true })
+    } catch (error: any) {
+      if (error.http_code === 404)
+        return ERR_NOT_FOUND
+      else
+        throw error
+    }
+  }
+
+  private async fetch (path: string, options: RequestInit = { method: 'GET' }): Promise<Maybe<Response>> {
+    const url = this.url(path)
+
+    if (url === null)
+      return ERR_NOT_FOUND
+
+    console.debug('Fetching from Cloudinary', {
+      method: options.method,
+      path,
+      url
+    })
+
+    const response = await fetch(url, options).catch((e) => e)
+
+    if (response instanceof Error || response.ok === false) {
+      console.debug('Failed to fetch from Cloudinary', {
+        url,
+        status: response.status,
+        message: response.message
+      })
+
+      return ERR_NOT_FOUND
+    } else
+      console.debug('Received response from Cloudinary', {
+        url,
+        status: response.status,
+        headers: response.headers
+      })
+
+    return response
+  }
+
+  private url (path: string): string | null {
+    const [base, transformation] = this.toTransformation(path)
+
+    if (base === null)
+      return null
+
+    const id = join(this.prefix, base)
+
+    return this.cloudinary().url(id, {
+      resource_type: this.type,
+      transformation,
+      version: 2
+    })
+  }
+
+  private toTransformation (path: string): [string | null, TransformationOptions[] | undefined] {
+    if (this.transformations.length === 0)
+      return [path, undefined]
+
+    const [base, ...extensions] = path.split('.')
+    const transformations: TransformationOptions[] = []
+
+    let t = 0
+
+    for (const extension of extensions) {
+      let found = false
+
+      for (t; t < this.transformations.length && !found; t++) {
+        const { extension: regex, condition, transformation, optional } = this.transformations[t]
+
+        const match = regex.exec(extension)
+
+        // eslint-disable-next-line max-depth
+        if (match === null)
+          if (optional === true)
+            continue
+          else
+            return [null, undefined]
+
+        const options = Array.isArray(transformation) ? transformation : [transformation]
+        const stages = options.map((stage) => this.mapTransformation(stage, match.groups!))
+
+        found = true
+
+        if (condition === undefined)
+          transformations.push(...stages)
+        else {
+          transformations.push({ if: condition })
+          transformations.push(...stages)
+          transformations.push({ if: 'end' })
+        }
+      }
+
+      if (!found)
+        return [null, undefined]
+    }
+
+    for (t; t < this.transformations.length; t++)
+      if (this.transformations[t].optional !== true)
+        return [null, undefined]
+
+    return [base, transformations]
+  }
+
+  private mapTransformation (options: Record<string, unknown>, groups: Record<string, string>): TransformationOptions {
+    return Object.fromEntries(Object.entries(options).map(([key, value]) => {
+      if (typeof value !== 'string')
+        return [key, value]
+
+      if (value.startsWith('<') && value.endsWith('>'))
+        value = groups[value.slice(1, -1)]
+
+      if (key === 'zoom' && value !== undefined)
+        value = Number.parseInt(value as string) / 100
+
+      if (key === 'fetch_format' && value === 'jpeg')
+        value = 'jpg'
+
+      return [key, value]
+    }))
+  }
+
+  private metadata (response: Response): Metadata {
+    const size = response.headers.get('content-length') === null
+      ? null
+      : Number.parseInt(response.headers.get('content-length')!)
+
+    const created = response.headers.get('date') ?? new Date().toISOString()
+    const etag = response.headers.get('etag')
+    const checksum = etag === null ? basename(response.url) : etag.slice(1, -1)
+    const range = response.headers.get('content-range')
+
+    return {
+      type: response.headers.get('content-type')!,
+      size,
+      checksum,
+      created,
+      range: range ?? undefined,
+      partial: response.status === 206,
+      attributes: {
+        url: response.url
+      }
+    }
+  }
+
+  private cloudinary (): typeof cloudinary {
+    cloudinary.config(this.config)
+
+    return cloudinary
+  }
+}
+
+interface GetOptions {
+  agent?: string
+  range?: string
+}
+
+export interface CloudinaryOptions {
+  environment: string
+  type: StorageType
+  prefix?: string
+  eager?: TransformationOptions[]
+  transformations?: TransformationDeclaration[]
+}
+
+type TransformationDeclaration = Omit<Transformation, 'extension'> & { extension: string }
+
+interface Transformation {
+  extension: RegExp
+  condition?: string
+  transformation: Record<string, unknown> | Array<Record<string, unknown>>
+  optional?: boolean
+}
+
+type StorageType = 'image' | 'video'

package/source/providers/Declaration.ts

@@ -1,10 +1,11 @@
 import type { S3Options } from './S3'
+import type { CloudinaryOptions } from './Cloudinary'
 import type { FileSystemOptions } from './FileSystem'
 import type { TemporaryOptions } from './Temporary'
 
 export type Declaration =
   ({ provider: 's3' } & S3Options)
+  | ({ provider: 'cloudinary' } & CloudinaryOptions)
   | ({ provider: 'fs' } & FileSystemOptions)
   | ({ provider: 'tmp' } & TemporaryOptions)
-  | ({ provider: 'mem' })
   | ({ provider: 'test' } & TemporaryOptions)

package/source/providers/FileSystem.ts

@@ -1,51 +1,105 @@
-import { type Readable } from 'node:stream'
-import { dirname, join } from 'node:path'
 import fs from 'node:fs/promises'
+import { createReadStream } from 'node:fs'
+import { dirname, join } from 'node:path'
 import { Provider } from '../Provider'
-import type { ProviderSecrets } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
+import type { Readable } from 'node:stream'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
 
 export interface FileSystemOptions {
   path: string
+  claim?: string
 }
 
 export class FileSystem extends Provider<FileSystemOptions> {
-  protected readonly path: string
+  public override readonly root: string
 
-  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
-    super(options, secrets)
+  public constructor (options: FileSystemOptions) {
+    super(options)
 
-    this.path = options.path
+    this.root = options.path
   }
 
-  public async get (path: string): Promise<Readable | null> {
-    try {
-      const fd = await fs.open(join(this.path, path))
+  public async get (rel: string): Promise<Maybe<Stream>> {
+    const path = this.blob(rel)
+    const metadata = await this.head(rel)
 
-      return fd.createReadStream()
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return null
+    if (metadata instanceof Error)
+      return metadata
 
-      throw err
-    }
+    const stream = createReadStream(path)
+
+    return { stream, ...metadata }
+  }
+
+  public async head (rel: string): Promise<Maybe<Metadata>> {
+    const path = this.meta(rel)
+
+    return this.try(async () => {
+      const contents = await fs.readFile(path, 'utf8')
+
+      return JSON.parse(contents)
+    })
   }
 
-  public async put (rel: string, filename: string, stream: Readable): Promise<void> {
-    const dir = join(this.path, rel)
-    const path = join(dir, filename)
+  public async put (rel: string, stream: Readable): Promise<void> {
+    const path = this.blob(rel)
+    const dir = dirname(path)
 
-    await fs.mkdir(dirname(path), { recursive: true })
+    await fs.mkdir(dir, { recursive: true })
     await fs.writeFile(path, stream)
   }
 
+  public async commit (rel: string, metadata: Metadata): Promise<void> {
+    const path = this.meta(rel)
+
+    await fs.writeFile(path, JSON.stringify(metadata), 'utf8')
+  }
+
   public async delete (path: string): Promise<void> {
-    await fs.rm(join(this.path, path), { recursive: true, force: true })
+    await Promise.all([
+      fs.rm(this.blob(path), { force: true }),
+      fs.rm(this.meta(path), { force: true })
+    ])
   }
 
-  public async move (from: string, to: string): Promise<void> {
-    from = join(this.path, from)
-    to = join(this.path, to)
+  public async move (from: string, to: string): Promise<Maybe<void>> {
+    const bf = this.blob(from)
+    const bt = this.blob(to)
+    const mf = this.meta(from)
+    const mt = this.meta(to)
 
-    await fs.mkdir(dirname(to), { recursive: true })
-    await fs.rename(from, to)
+    await fs.mkdir(dirname(bt), { recursive: true })
+
+    return await this.try(async () => {
+      await Promise.all([
+        fs.rename(bf, bt),
+        fs.rename(mf, mt)
+      ])
+    })
+  }
+
+  private blob (rel: string): string {
+    return this.join(rel, '.blob')
+  }
+
+  private meta (rel: string): string {
+    return this.join(rel, '.meta')
+  }
+
+  private join (rel: string, ext: string): string {
+    return join(this.root, rel) + ext
+  }
+
+  private async try<T = void>(action: () => Promise<T>): Promise<Maybe<T>> {
+    try {
+      return await action()
+    } catch (err: NodeJS.ErrnoException | any) {
+      if (err?.code === 'ENOENT')
+        return ERR_NOT_FOUND
+      else
+        throw err
+    }
   }
 }

package/source/providers/S3.ts

@@ -3,22 +3,15 @@ import { Blob } from 'node:buffer'
 import { join } from 'node:path/posix'
 import assert from 'node:assert'
 import { Upload } from '@aws-sdk/lib-storage'
-import {
-  S3Client,
-  GetObjectCommand,
-  CopyObjectCommand,
-  DeleteObjectsCommand,
-  paginateListObjectsV2,
-  HeadObjectCommand,
-  NotFound,
-  DeleteObjectCommand,
-  NoSuchKey,
-  type S3ClientConfigType,
-  type ObjectIdentifier
-} from '@aws-sdk/client-s3'
+import * as s3 from '@aws-sdk/client-s3'
 import * as nodeNativeFetch from 'smithy-node-native-fetch'
-import { Provider, type ProviderSecret, type ProviderSecrets } from '../Provider'
+import { console } from 'openspan'
+import { Provider } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
 import type { ReadableStream } from 'node:stream/web'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
+import type { Secret, Secrets } from '../Secrets'
 
 export interface S3Options {
   bucket: string
@@ -27,23 +20,23 @@ export interface S3Options {
   endpoint?: string
 }
 
-type S3Secrets = ProviderSecrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
+type S3Secrets = Secrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
 
 export class S3 extends Provider<S3Options> {
-  public static override readonly SECRETS: readonly ProviderSecret[] = [
+  public static override readonly SECRETS: readonly Secret[] = [
     { name: 'ACCESS_KEY_ID', optional: true },
     { name: 'SECRET_ACCESS_KEY', optional: true }
   ]
 
-  protected readonly bucket: string
-  protected readonly client: S3Client
+  private readonly bucket: string
+  private readonly client: s3.S3Client
 
   public constructor (options: S3Options, secrets?: S3Secrets) {
-    super(options)
+    super(options, secrets)
 
     this.bucket = options.bucket
 
-    const s3Config: S3ClientConfigType = {
+    const s3Config: s3.S3ClientConfigType = {
       retryMode: 'adaptive',
       ...nodeNativeFetch
     }
@@ -53,7 +46,8 @@ export class S3 extends Provider<S3Options> {
       s3Config.endpoint = options.endpoint
     }
 
-    if (options.region !== undefined) s3Config.region = options.region
+    if (options.region !== undefined)
+      s3Config.region = options.region
 
     if (typeof secrets?.ACCESS_KEY_ID === 'string') {
       assert.ok(secrets.SECRET_ACCESS_KEY !== undefined,
@@ -65,15 +59,15 @@ export class S3 extends Provider<S3Options> {
       }
     }
 
-    this.client = new S3Client(s3Config)
+    this.client = new s3.S3Client(s3Config)
 
     this.client.middlewareStack.add((next, _context) => async (args) => {
-      if ('Key' in args.input && typeof args.input.Key === 'string')
       // removes leading slash
+      if ('Key' in args.input && typeof args.input.Key === 'string')
         args.input.Key = args.input.Key.replace(/^\//, '')
 
-      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
       // removes leading slash and ensures finishing slash
+      if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
         args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
 
       return next(args)
@@ -85,82 +79,89 @@ export class S3 extends Provider<S3Options> {
     })
   }
 
-  public async get (Key: string): Promise<Readable | null> {
-    try {
-      const fileResponse = await this.client.send(new GetObjectCommand({
+  public async get (Key: string): Promise<Maybe<Stream>> {
+    return await this.try<Stream>(async () => {
+      const entry = await this.client.send(new s3.GetObjectCommand({
         Bucket: this.bucket,
         Key
       }))
 
-      if (fileResponse.Body === undefined) return null // should never happen
+      const stream = entry.Body instanceof Readable
+        ? entry.Body
+        : Readable.fromWeb((entry.Body instanceof Blob
+          ? entry.Body.stream()
+          : entry.Body) as ReadableStream)
 
-      if (fileResponse.Body instanceof Readable) return fileResponse.Body
+      if (entry.Metadata?.value === undefined)
+        return ERR_NOT_FOUND
 
-      return Readable.fromWeb((fileResponse.Body instanceof Blob
-        ? fileResponse.Body.stream()
-        : fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
-    } catch (err) {
-      if (err instanceof NotFound || err instanceof NoSuchKey) return null
-      else throw err
-    }
+      const metadata = JSON.parse(entry.Metadata.value)
+
+      return { stream, ...metadata }
+    })
+  }
+
+  public async head (Key: string): Promise<Maybe<Metadata>> {
+    return await this.try<Metadata>(async () => {
+      const entry = await this.client.send(new s3.HeadObjectCommand({
+        Bucket: this.bucket,
+        Key
+      }))
+
+      if (entry.Metadata?.value === undefined)
+        return ERR_NOT_FOUND
+
+      return JSON.parse(entry.Metadata.value)
+    })
   }
 
-  public async put (path: string, filename: string, stream: Readable): Promise<void> {
+  public async put (Key: string, stream: Readable): Promise<void> {
     await new Upload({
       client: this.client,
       params: {
         Bucket: this.bucket,
-        Key: join(path, filename),
+        Key,
         Body: stream
       }
     }).done()
   }
 
-  /**
-   * Deletes either a single object or "directory" - all objects
-   * with given prefix (prefix will be enforced to finish with `/`)
-   * @param Key - key name or path prefix
-   */
-  public async delete (Key: string): Promise<void> {
-    const { client, bucket: Bucket } = this
-
-    // checking if given key is a single file
-    if (!Key.endsWith('/'))
-      try {
-        // DeleteObject on S3 returns no error if object does not exist
-        await client.send(new HeadObjectCommand({ Bucket, Key }))
-        await client.send(new DeleteObjectCommand({ Bucket, Key }))
-
-        return
-      } catch (err) {
-        assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
-      }
+  public async commit (Key: string, metadata: object): Promise<void> {
+    await this.client.send(new s3.CopyObjectCommand({
+      Bucket: this.bucket,
+      Key,
+      CopySource: join(this.bucket, Key),
+      Metadata: { value: JSON.stringify(metadata) },
+      MetadataDirective: 'REPLACE'
+    }))
 
-    const objectsToRemove: ObjectIdentifier[] = []
-
-    for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix: Key }))
-      for (const { Key } of page.Contents ?? []) objectsToRemove.push({ Key })
-
-    // Removing all objects in parallel in batches
-    await Promise.all((function * () {
-      while (objectsToRemove.length > 0)
-        yield client.send(new DeleteObjectsCommand({
-          Bucket,
-          Delete: {
-            Objects: objectsToRemove.splice(0,
-              1000 /* max batch size for DeleteObjects */)
-          }
-        }))
-    })())
+    console.debug('Uploaded to S3', { bucket: this.bucket, path: Key, metadata })
   }
 
-  public async move (from: string, keyTo: string): Promise<void> {
-    await this.client.send(new CopyObjectCommand({
-      Bucket: this.bucket,
-      Key: keyTo,
-      CopySource: join(this.bucket, from)
-    }))
+  public async delete (Key: string): Promise<void> {
+    await this.client.send(new s3.DeleteObjectCommand({ Bucket: this.bucket, Key }))
+  }
 
-    await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
+  public async move (from: string, keyTo: string): Promise<Maybe<void>> {
+    return await this.try(async () => {
+      await this.client.send(new s3.CopyObjectCommand({
+        Bucket: this.bucket,
+        Key: keyTo,
+        CopySource: join(this.bucket, from)
+      }))
+
+      await this.client.send(new s3.DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
+    })
+  }
+
+  private async try<T = void>(action: () => Promise<T>): Promise<Maybe<T>> {
+    try {
+      return await action()
+    } catch (err: any) {
+      if (err?.name === 'NotFound' || err?.name === 'NoSuchKey')
+        return ERR_NOT_FOUND
+      else
+        throw err
+    }
   }
 }