@toa.io/extensions.storages 1.0.0-alpha.13 → 1.0.0-alpha.143

package/source/Storage.ts

@@ -1,12 +1,10 @@
-import { Readable } from 'node:stream'
-import { posix } from 'node:path'
-import { decode, encode } from 'msgpackr'
-import { buffer, newid } from '@toa.io/generic'
-import { Err } from 'error-value'
+import { basename, join } from 'node:path'
+import { randomUUID } from 'node:crypto'
 import { Scanner } from './Scanner'
+import type { Readable } from 'node:stream'
+import type { Attributes, Entry, Stream } from './Entry'
 import type { ScanOptions } from './Scanner'
 import type { Provider } from './Provider'
-import type { Entry } from './Entry'
 
 export class Storage {
   private readonly provider: Provider
@@ -17,229 +15,82 @@ export class Storage {
 
   public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
     const scanner = new Scanner(options)
-    const pipe = stream.pipe(scanner)
-    const tempname = await this.transit(pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const id = scanner.digest()
-
-    await this.persist(tempname, id)
-
-    return await this.create(path, id, scanner.size, scanner.type, options?.meta)
-  }
-
-  public async get (path: string): Maybe<Entry> {
-    const metapath = posix.join(ENTRIES, path, META)
-    const result = await this.provider.get(metapath)
-
-    if (result === null) return ERR_NOT_FOUND
-    else return decode(await buffer(result))
-  }
-
-  public async fetch (path: string): Maybe<Readable> {
-    const { rel, id, variant } = this.parse(path)
-
-    if (variant === null && rel !== '') {
-      const entry = await this.get(path)
-
-      if (entry instanceof Error)
-        return entry
+    const pipe = stream.pipe(scanner).on('error', () => undefined)
+    const id = options?.id ?? randomUUID().replace(/-/g, '')
+    const location = this.locate(path, id)
+
+    /**
+     * Provider can return or throw an error.
+     * If thrown error is TYPE_MISMATCH from the Scanner, it should be returned.
+     */
+    const error: Error | undefined = await this.provider.put(location, pipe).catch((error: any) => {
+      if (error === scanner.error) return error
+      else throw error
+    })
+
+    if (error instanceof Error)
+      return error
+
+    const metadata: Entry = {
+      id,
+      size: scanner.size,
+      type: scanner.type,
+      checksum: scanner.digest(),
+      created: new Date().toISOString(),
+      attributes: options?.attributes ?? {}
     }
 
-    const blob = variant === null
-      ? posix.join(BLOBs, id)
-      : posix.join(ENTRIES, rel, id, variant)
-
-    const stream = await this.provider.get(blob)
-
-    if (stream === null) return ERR_NOT_FOUND
-    else return stream
-  }
-
-  public async delete (path: string): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    await this.conceal(path)
-    await this.provider.delete(posix.join(ENTRIES, path))
-  }
+    if (options?.origin !== undefined)
+      metadata.attributes.origin = options.origin
 
-  public async list (path: string): Promise<string[]> {
-    const listfile = posix.join(ENTRIES, path, LIST)
-    const stream = await this.provider.get(listfile)
+    await this.provider.commit(location, metadata)
 
-    return stream === null ? [] : decode(await buffer(stream))
+    return metadata
   }
 
-  public async permute (path: string, ids: string[]): Maybe<void> {
-    const unique = new Set(ids)
-    const dir = posix.join(ENTRIES, path)
-    const list = await this.list(path)
-
-    if (list.length !== ids.length || unique.size !== ids.length)
-      return ERR_PERMUTATION_MISMATCH
-
-    for (const id of ids)
-      if (!list.includes(id))
-        return ERR_PERMUTATION_MISMATCH
+  public async get (path: string): Maybe<Stream> {
+    const location = this.locate(path)
 
-    await this.provider.put(dir, LIST, Readable.from(encode(ids)))
+    return await this.provider.get(location)
   }
 
-  public async conceal (path: string): Maybe<void> {
-    const { id, rel } = this.parse(path)
-    const dir = posix.join(ENTRIES, rel)
-    const list = await this.list(rel)
-    const index = list.indexOf(id)
-
-    if (index === -1)
-      return ERR_NOT_FOUND
-
-    list.splice(index, 1)
-
-    await this.provider.put(dir, LIST, Readable.from(encode(list)))
-  }
-
-  public async reveal (path: string): Maybe<void> {
-    const { id, rel } = this.parse(path)
-
-    return await this.enroll(rel, id)
-  }
-
-  public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
-    const scanner = new Scanner()
-    const pipe = stream.pipe(scanner)
-
-    await this.provider.put(posix.join(ENTRIES, path), name, pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const { size, type } = scanner
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
+  public async head (path: string): Promise<Maybe<Entry>> {
+    const id = basename(path)
+    const location = this.locate(path)
+    const metadata = await this.provider.head(location)
 
-    entry.variants = entry.variants.filter((variant) => variant.name !== name)
-    entry.variants.push({ name, size, type })
+    if (metadata instanceof Error)
+      return metadata
 
-    await this.save(path, entry)
-  }
-
-  public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-    if (value === undefined) delete entry.meta[key]
-    else entry.meta[key] = value
-
-    await this.save(path, entry)
-  }
-
-  private async transit (stream: Readable): Promise<string> {
-    const tempname = newid()
-
-    await this.provider.put(TEMP, tempname, stream)
-
-    return tempname
-  }
-
-  private async persist (tempname: string, id: string): Promise<void> {
-    const temp = posix.join(TEMP, tempname)
-    const blob = posix.join(BLOBs, id)
-
-    await this.provider.move(temp, blob)
-  }
-
-  // eslint-disable-next-line max-params
-  private async create
-  (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
-    const entry: Entry = {
+    return {
       id,
-      size,
-      type,
-      created: Date.now(),
-      variants: [],
-      meta
+      ...metadata
     }
-
-    const metafile = posix.join(path, entry.id)
-    const existing = await this.get(metafile)
-
-    if (existing instanceof Error)
-      await this.save(metafile, entry)
-
-    await this.enroll(path, id, true)
-
-    return entry
   }
 
-  private async save (rel: string, entry: Entry): Promise<void> {
-    const buffer = encode(entry)
-    const stream = Readable.from(buffer)
+  public async delete (path: string): Maybe<void> {
+    const location = this.locate(path)
 
-    await this.provider.put(posix.join(ENTRIES, rel), META, stream)
+    return this.provider.delete(location)
   }
 
-  private async enroll (path: string, id: string, addition: boolean = false): Maybe<void> {
-    const dir = posix.join(ENTRIES, path)
-    const list = await this.list(path)
-    const index = list.indexOf(id)
-
-    if (index !== -1)
-      if (addition) list.splice(index, 1)
-      else return
-    else if (!addition) {
-      const entry = await this.get(posix.join(path, id))
-
-      if (entry instanceof Error)
-        return entry
-    }
-
-    list.push(id)
-
-    await this.provider.put(dir, LIST, Readable.from(encode(list)))
+  public path (): string | null {
+    return this.provider.root ?? null
   }
 
-  private parse (path: string): Path {
-    const [last, ...segments] = path.split('/').reverse()
-    const [id, ...rest] = last.split('.')
-    const variant = rest.length > 0 ? rest.join('.') : null
-    const rel = segments.reverse().join('/')
-
-    return { rel, id, variant }
+  private locate (...rel: string[]): string {
+    return join(ENTRIES, ...rel)
   }
 }
 
-const ERR_NOT_FOUND = Err('NOT_FOUND')
-const ERR_PERMUTATION_MISMATCH = Err('PERMUTATION_MISMATCH')
-
-const TEMP = '/temp'
-const BLOBs = '/blobs'
-const ENTRIES = '/entries'
-const LIST = '.list'
-const META = '.meta'
-
-type Maybe<T> = Promise<T | Error>
-
-interface Path {
-  rel: string
-  id: string
-  variant: string | null
-}
-
-type Meta = Record<string, string>
+const ENTRIES = '/'
 
 interface Options extends ScanOptions {
-  meta?: Meta
+  id?: string
+  origin?: string
+  attributes?: Attributes
 }
 
+type Maybe<T> = Promise<T | Error>
+
 export type Storages = Record<string, Storage>

package/source/deployment.ts

@@ -3,21 +3,25 @@ import { encode } from '@toa.io/generic'
 import { providers } from './providers'
 import { validateAnnotation } from './Annotation'
 import type { Annotation } from './Annotation'
-import type { Dependency, Variable } from '@toa.io/operations'
+import type { Dependency, Variable, Mounts } from '@toa.io/operations'
 import type { context } from '@toa.io/norm'
 
-export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
+export const ENV_PREFIX = 'TOA_STORAGES'
 
 export function deployment (instances: Instance[], annotation: unknown): Dependency {
   validate(instances, annotation)
 
   const value = encode(annotation)
-  const pointer: Variable = { name: SERIALIZATION_PREFIX, value }
+  const pointer: Variable = { name: ENV_PREFIX, value }
   const secrets = getSecrets(annotation)
+  const mounts = getMounts(instances, annotation)
 
-  return {
-    variables: { global: [pointer, ...secrets] }
-  }
+  const dependency: Dependency = { variables: { global: [pointer, ...secrets] } }
+
+  if (mounts !== null)
+    dependency.mounts = mounts
+
+  return dependency
 }
 
 function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
@@ -40,18 +44,47 @@ function getSecrets (annotation: Annotation): Variable[] {
   for (const [name, declaration] of Object.entries(annotation)) {
     const Provider = providers[declaration.provider]
 
-    for (const secret of Provider.SECRETS)
-      secrets.push({
-        name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
-        secret: {
-          name: `toa-storages-${name}`,
-          key: secret.name,
-          optional: secret.optional
-        }
-      })
+    if (Provider.SECRETS !== undefined)
+      // eslint-disable-next-line max-depth
+      for (const secret of Provider.SECRETS)
+        secrets.push({
+          name: `${ENV_PREFIX}_${name}_${secret.name}`.toUpperCase(),
+          secret: {
+            name: `toa-storages-${name}`,
+            key: secret.name,
+            optional: secret.optional
+          }
+        })
   }
 
   return secrets
 }
 
+function getMounts (instances: Instance[], annotation: Annotation): Mounts | null {
+  let mounts: Mounts | null = null
+
+  for (const { locator, manifest } of instances)
+    for (const name of manifest) {
+      const declaration = annotation[name]
+
+      // eslint-disable-next-line max-depth
+      if (declaration.provider !== 'fs')
+        continue
+
+      // eslint-disable-next-line max-depth
+      if (declaration.claim !== undefined) {
+        mounts ??= {}
+        mounts[locator.label] ??= []
+
+        mounts[locator.label].push({
+          name,
+          path: declaration.path,
+          claim: declaration.claim
+        })
+      }
+    }
+
+  return mounts
+}
+
 export type Instance = context.Dependency<string[]>

package/source/errors.ts

@@ -0,0 +1,3 @@
+import { Err } from 'error-value'
+
+export const ERR_NOT_FOUND = new Err('NOT_FOUND')

package/source/index.ts

@@ -2,4 +2,4 @@ export { Factory } from './Factory'
 export { deployment } from './deployment'
 export { manifest } from './manifest'
 
-export type { Entry } from './Entry'
+export type { Entry, Stream } from './Entry'

package/source/manifest.ts

@@ -1,8 +1,9 @@
-import { match } from 'matchacho'
-
 export function manifest (manifest: string | string[] | null): string[] {
-  return match(manifest,
-    String, (name: string) => [name],
-    Array, manifest,
-    null, [])
+  if (manifest === null)
+    return []
+
+  if (typeof manifest === 'string')
+    return [manifest]
+
+  return manifest
 }

package/source/providers/Cloudinary.ts

@@ -0,0 +1,274 @@
+import { basename, dirname, join } from 'node:path'
+import { Readable } from 'node:stream'
+import { v2 as cloudinary } from 'cloudinary'
+import { console } from 'openspan'
+import { Provider } from '../Provider'
+import { ERR_NOT_FOUND } from '../errors'
+import type { Maybe } from '@toa.io/types'
+import type { Metadata, Stream } from '../Entry'
+import type { Secret, Secrets } from '../Secrets'
+import type { ReadableStream } from 'node:stream/web'
+import type {
+  ConfigOptions,
+  ImageTransformationOptions,
+  VideoTransformationOptions
+} from 'cloudinary'
+
+export type CloudinarySecrets = Secrets<'API_KEY' | 'API_SECRET'>
+
+export class Cloudinary extends Provider<CloudinaryOptions> {
+  public static override readonly SECRETS: readonly Secret[] = [
+    { name: 'API_KEY' },
+    { name: 'API_SECRET' }
+  ]
+
+  private readonly type: StorageType
+  private readonly transformations: Transformation[] = []
+  private readonly config: ConfigOptions
+  private readonly prefix: string
+
+  public constructor (options: CloudinaryOptions, secrets?: CloudinarySecrets) {
+    super(options, secrets)
+
+    this.type = options.type
+
+    if (options.transformations !== undefined)
+      this.transformations = options.transformations.map((transformation) => {
+        const { extension, ...rest } = transformation
+
+        let expression = extension
+
+        if (!extension.startsWith('^'))
+          expression = '^' + extension
+
+        if (!extension.endsWith('$'))
+          expression = extension + '$'
+
+        return {
+          extension: new RegExp(expression),
+          ...rest
+        }
+      })
+
+    this.config = {
+      cloud_name: options.environment,
+      api_key: secrets!.API_KEY,
+      api_secret: secrets!.API_SECRET
+    }
+
+    this.prefix = options.prefix ?? '/'
+  }
+
+  public async get (path: string): Promise<Maybe<Stream>> {
+    const response = await this.fetch(path)
+
+    if (response instanceof Error)
+      return ERR_NOT_FOUND
+
+    const metadata = this.metadata(response)
+
+    return {
+      stream: Readable.fromWeb(response.body as ReadableStream),
+      ...metadata
+    }
+  }
+
+  public async head (path: string): Promise<Maybe<Metadata>> {
+    const response = await this.fetch(path, 'HEAD')
+
+    if (response instanceof Error)
+      return ERR_NOT_FOUND
+
+    return this.metadata(response)
+  }
+
+  public async put (path: string, stream: Readable): Promise<void> {
+    const id = basename(path)
+    const folder = join(this.prefix, dirname(path))
+
+    console.debug('Uploading to Cloudinary', { path })
+
+    await new Promise((resolve, reject) => {
+      stream.pipe(this.cloudinary().uploader.upload_stream({
+        public_id: id,
+        folder,
+        resource_type: 'auto',
+        overwrite: true,
+        invalidate: true
+      },
+      (error, result) => {
+        if (error !== undefined) reject(error)
+        else resolve(result)
+      }))
+
+      stream.on('error', reject)
+    })
+  }
+
+  public async commit (): Promise<void> {
+    // metadata is read-only
+  }
+
+  public async delete (path: string): Promise<void> {
+    const id = join(this.prefix, path)
+
+    console.debug('Deleting from Cloudinary', { path: id })
+
+    await this.cloudinary().uploader.destroy(id,
+      { resource_type: this.type, invalidate: true })
+  }
+
+  public async move (from: string, to: string): Promise<void | Error> {
+    const source = join(this.prefix, from)
+    const target = join(this.prefix, to)
+
+    try {
+      await this.cloudinary().uploader.rename(source, target,
+        { resource_type: this.type, overwrite: true })
+    } catch (error: any) {
+      if (error.http_code === 404)
+        return ERR_NOT_FOUND
+      else
+        throw error
+    }
+  }
+
+  private async fetch (path: string, method = 'GET'): Promise<Maybe<Response>> {
+    const url = this.url(path)
+
+    if (url === null)
+      return ERR_NOT_FOUND
+
+    console.debug('Fetching from Cloudinary', {
+      method,
+      path,
+      url
+    })
+
+    const response = await fetch(url, { method }).catch((e) => e)
+
+    if (response instanceof Error || response.ok === false) {
+      console.debug('Failed to fetch from Cloudinary', {
+        url,
+        message: response.message
+      })
+
+      return ERR_NOT_FOUND
+    }
+
+    return response
+  }
+
+  private url (path: string): string | null {
+    const [base, transformation] = this.toTransformation(path)
+
+    if (base === null)
+      return null
+
+    const id = join(this.prefix, base)
+
+    return this.cloudinary().url(id, {
+      resource_type: this.type,
+      transformation,
+      version: 2
+    })
+  }
+
+  private toTransformation (path: string): [string | null, TransformationOptions[] | undefined] {
+    if (this.transformations.length === 0)
+      return [path, undefined]
+
+    const [base, ...extensions] = path.split('.')
+    const transformations: TransformationOptions[] = []
+
+    let t = 0
+
+    for (const extension of extensions) {
+      let found = false
+
+      for (t; t < this.transformations.length && !found; t++) {
+        const { extension: regex, transformation: options, optional } = this.transformations[t]
+
+        const match = regex.exec(extension)
+
+        // eslint-disable-next-line max-depth
+        if (match === null)
+          if (optional === true)
+            continue
+          else
+            return [null, undefined]
+
+        const transformation = Object.fromEntries(Object.entries(options).map(([key, value]) => {
+          if (typeof value !== 'string')
+            return [key, value]
+
+          if (value.startsWith('<') && value.endsWith('>'))
+            value = match.groups![value.slice(1, -1)]
+
+          if (key === 'zoom' && value !== undefined)
+            value = Number.parseInt(value as string) / 100
+
+          if (key === 'fetch_format' && value === 'jpeg')
+            value = 'jpg'
+
+          return [key, value]
+        }))
+
+        transformations.push(transformation)
+        found = true
+      }
+
+      if (!found)
+        return [null, undefined]
+    }
+
+    for (t; t < this.transformations.length; t++)
+      if (this.transformations[t].optional !== true)
+        return [null, undefined]
+
+    return [base, transformations]
+  }
+
+  private metadata (response: Response): Metadata {
+    const size = response.headers.get('content-length') === null
+      ? 0
+      : Number.parseInt(response.headers.get('content-length')!)
+
+    const created = response.headers.get('date') ?? new Date().toISOString()
+    const etag = response.headers.get('etag')
+    const checksum = etag === null ? basename(response.url) : etag.slice(1, -1)
+
+    return {
+      type: response.headers.get('content-type')!,
+      size,
+      checksum,
+      created,
+      attributes: {}
+    }
+  }
+
+  private cloudinary (): typeof cloudinary {
+    cloudinary.config(this.config)
+
+    return cloudinary
+  }
+}
+
+export interface CloudinaryOptions {
+  environment: string
+  type: StorageType
+  prefix?: string
+  transformations?: TransformationDeclaration[]
+}
+
+type TransformationDeclaration = Omit<Transformation, 'extension'> & { extension: string }
+
+interface Transformation {
+  extension: RegExp
+  transformation: object
+  optional?: boolean
+}
+
+type TransformationOptions = ImageTransformationOptions | VideoTransformationOptions
+
+type StorageType = 'image' | 'video'