@toa.io/extensions.storages 1.0.0-alpha.21 → 1.0.0-alpha.219

package/source/Storage.ts

@@ -1,245 +1,100 @@
-import { Readable } from 'node:stream'
-import { posix } from 'node:path'
-import { decode, encode } from 'msgpackr'
-import { buffer, newid } from '@toa.io/generic'
-import { Err } from 'error-value'
+import { basename, join } from 'node:path'
+import { randomUUID } from 'node:crypto'
 import { Scanner } from './Scanner'
+import type { Readable } from 'node:stream'
+import type { Attributes, Entry, Stream } from './Entry'
 import type { ScanOptions } from './Scanner'
 import type { Provider } from './Provider'
-import type { Entry } from './Entry'
 
-export class Storage {
-  private readonly provider: Provider
+export class Storage<T extends Provider = Provider> {
+  private readonly provider: T
 
-  public constructor (provider: Provider) {
+  public constructor (provider: T) {
     this.provider = provider
   }
 
-  public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
-    const scanner = new Scanner(options)
-    const pipe = stream.pipe(scanner)
-    const tempname = await this.transit(pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const id = scanner.digest()
-
-    await this.persist(tempname, id)
-
-    return await this.create(path, id, scanner.size, scanner.type, options?.meta)
-  }
-
-  public async get (path: string): Maybe<Entry> {
-    const metapath = posix.join(ENTRIES, path, META)
-    const result = await this.provider.get(metapath)
-
-    if (result === null) return ERR_NOT_FOUND
-    else return decode(await buffer(result))
+  public options (): T['options'] {
+    return this.provider.options
   }
 
-  public async fetch (path: string): Maybe<Readable> {
-    const { rel, id, variant } = this.parse(path)
-
-    if (variant === null && rel !== '') {
-      const entry = await this.get(path)
-
-      if (entry instanceof Error)
-        return entry
+  public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
+    const scanner = new Scanner(options)
+    const pipe = stream.pipe(scanner).on('error', () => undefined)
+    const id = options?.id ?? randomUUID().replace(/-/g, '')
+    const location = this.locate(path, id)
+
+    /**
+     * Provider can return or throw an error.
+     * If thrown error is TYPE_MISMATCH from the Scanner, it should be returned.
+     */
+    const error: Error | undefined = await this.provider.put(location, pipe).catch((error: any) => {
+      if (error === scanner.error) return error
+      else throw error
+    })
+
+    if (error instanceof Error)
+      return error
+
+    const metadata: Entry = {
+      id,
+      size: scanner.size,
+      type: scanner.type,
+      checksum: scanner.digest(),
+      created: new Date().toISOString(),
+      attributes: options?.attributes ?? {}
     }
 
-    const blob = variant === null
-      ? posix.join(BLOBs, id)
-      : posix.join(ENTRIES, rel, id, variant)
+    if (options?.origin !== undefined)
+      metadata.attributes.origin = options.origin
 
-    const stream = await this.provider.get(blob)
+    await this.provider.commit(location, metadata)
 
-    if (stream === null) return ERR_NOT_FOUND
-    else return stream
+    return metadata
   }
 
-  public async delete (path: string): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
+  public async get (path: string, options?: unknown): Maybe<Stream> {
+    const location = this.locate(path)
 
-    await this.conceal(path)
-    await this.provider.delete(posix.join(ENTRIES, path))
+    return await this.provider.get(location, options)
   }
 
-  public async list (path: string): Promise<string[]> {
-    const listfile = posix.join(ENTRIES, path, LIST)
-    const stream = await this.provider.get(listfile)
-
-    return stream === null ? [] : decode(await buffer(stream))
-  }
-
-  public async permute (path: string, ids: string[]): Maybe<void> {
-    const unique = new Set(ids)
-    const dir = posix.join(ENTRIES, path)
-    const list = await this.list(path)
-
-    if (list.length !== ids.length || unique.size !== ids.length)
-      return ERR_PERMUTATION_MISMATCH
-
-    for (const id of ids)
-      if (!list.includes(id))
-        return ERR_PERMUTATION_MISMATCH
-
-    await this.provider.put(dir, LIST, Readable.from(encode(ids)))
-  }
-
-  public async conceal (path: string): Maybe<void> {
-    const { id, rel } = this.parse(path)
-    const dir = posix.join(ENTRIES, rel)
-    const list = await this.list(rel)
-    const index = list.indexOf(id)
-
-    if (index === -1)
-      return ERR_NOT_FOUND
-
-    list.splice(index, 1)
-
-    await this.provider.put(dir, LIST, Readable.from(encode(list)))
-  }
-
-  public async reveal (path: string): Maybe<void> {
-    const { id, rel } = this.parse(path)
-
-    return await this.enroll(rel, id)
-  }
-
-  public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
-    const scanner = new Scanner()
-    const pipe = stream.pipe(scanner)
-
-    await this.provider.put(posix.join(ENTRIES, path), name, pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const { size, type } = scanner
-    const entry = await this.get(path)
+  public async head (path: string): Promise<Maybe<Entry>> {
+    const id = basename(path).split('.')[0]
+    const location = this.locate(path)
+    const metadata = await this.provider.head(location)
 
-    if (entry instanceof Error)
-      return entry
+    if (metadata instanceof Error)
+      return metadata
 
-    entry.variants = entry.variants.filter((variant) => variant.name !== name)
-    entry.variants.push({ name, size, type })
-
-    await this.save(path, entry)
-  }
-
-  public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-    if (value === undefined) delete entry.meta[key]
-    else entry.meta[key] = value
-
-    await this.save(path, entry)
-  }
-
-  private async transit (stream: Readable): Promise<string> {
-    const tempname = newid()
-
-    await this.provider.put(TEMP, tempname, stream)
-
-    return tempname
-  }
-
-  private async persist (tempname: string, id: string): Promise<void> {
-    const temp = posix.join(TEMP, tempname)
-    const blob = posix.join(BLOBs, id)
-
-    await this.provider.move(temp, blob)
-  }
-
-  // eslint-disable-next-line max-params
-  private async create
-  (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
-    const entry: Entry = {
+    return {
       id,
-      size,
-      type,
-      created: Date.now(),
-      variants: [],
-      meta
+      ...metadata
     }
-
-    const metafile = posix.join(path, entry.id)
-    const existing = await this.get(metafile)
-
-    if (existing instanceof Error)
-      await this.save(metafile, entry)
-
-    await this.enroll(path, id, true)
-
-    return entry
   }
 
-  private async save (rel: string, entry: Entry): Promise<void> {
-    const buffer = encode(entry)
-    const stream = Readable.from(buffer)
+  public async delete (path: string): Maybe<void> {
+    const location = this.locate(path)
 
-    await this.provider.put(posix.join(ENTRIES, rel), META, stream)
+    return this.provider.delete(location)
   }
 
-  private async enroll (path: string, id: string, addition: boolean = false): Maybe<void> {
-    const dir = posix.join(ENTRIES, path)
-    const list = await this.list(path)
-    const index = list.indexOf(id)
-
-    if (index !== -1)
-      if (addition) list.splice(index, 1)
-      else return
-    else if (!addition) {
-      const entry = await this.get(posix.join(path, id))
-
-      if (entry instanceof Error)
-        return entry
-    }
-
-    list.push(id)
-
-    await this.provider.put(dir, LIST, Readable.from(encode(list)))
+  public path (): string | null {
+    return this.provider.root ?? null
   }
 
-  private parse (path: string): Path {
-    const [last, ...segments] = path.split('/').reverse()
-    const [id, ...rest] = last.split('.')
-    const variant = rest.length > 0 ? rest.join('.') : null
-    const rel = segments.reverse().join('/')
-
-    return { rel, id, variant }
+  private locate (...rel: string[]): string {
+    return join(ENTRIES, ...rel)
   }
 }
 
-const ERR_NOT_FOUND = Err('NOT_FOUND')
-const ERR_PERMUTATION_MISMATCH = Err('PERMUTATION_MISMATCH')
-
-const TEMP = '/temp'
-const BLOBs = '/blobs'
-const ENTRIES = '/entries'
-const LIST = '.list'
-const META = '.meta'
-
-type Maybe<T> = Promise<T | Error>
-
-interface Path {
-  rel: string
-  id: string
-  variant: string | null
-}
-
-type Meta = Record<string, string>
+const ENTRIES = '/'
 
 interface Options extends ScanOptions {
-  meta?: Meta
+  id?: string
+  origin?: string
+  attributes?: Attributes
 }
 
+type Maybe<T> = Promise<T | Error>
+
 export type Storages = Record<string, Storage>

package/source/deployment.ts

@@ -3,28 +3,35 @@ import { encode } from '@toa.io/generic'
 import { providers } from './providers'
 import { validateAnnotation } from './Annotation'
 import type { Annotation } from './Annotation'
-import type { Dependency, Variable } from '@toa.io/operations'
+import type { Dependency, Variable, Mounts } from '@toa.io/operations'
 import type { context } from '@toa.io/norm'
 
-export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
+export const ENV_PREFIX = 'TOA_STORAGES'
 
 export function deployment (instances: Instance[], annotation: unknown): Dependency {
   validate(instances, annotation)
 
   const value = encode(annotation)
-  const pointer: Variable = { name: SERIALIZATION_PREFIX, value }
+  const pointer: Variable = { name: ENV_PREFIX, value }
   const secrets = getSecrets(annotation)
+  const mounts = getMounts(instances, annotation)
 
-  return {
-    variables: { global: [pointer, ...secrets] }
-  }
+  const dependency: Dependency = { variables: { global: [pointer, ...secrets] } }
+
+  if (mounts !== null)
+    dependency.mounts = mounts
+
+  return dependency
 }
 
 function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
   validateAnnotation(annotation)
 
-  for (const instance of instances)
+  for (const instance of instances) {
+    instance.manifest ??= []
+
     contains(instance, annotation)
+  }
 }
 
 function contains (instance: Instance, annotation: Annotation): void {
@@ -40,18 +47,47 @@ function getSecrets (annotation: Annotation): Variable[] {
   for (const [name, declaration] of Object.entries(annotation)) {
     const Provider = providers[declaration.provider]
 
-    for (const secret of Provider.SECRETS)
-      secrets.push({
-        name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
-        secret: {
-          name: `toa-storages-${name}`,
-          key: secret.name,
-          optional: secret.optional
-        }
-      })
+    if (Provider.SECRETS !== undefined)
+      // eslint-disable-next-line max-depth
+      for (const secret of Provider.SECRETS)
+        secrets.push({
+          name: `${ENV_PREFIX}_${name}_${secret.name}`.toUpperCase(),
+          secret: {
+            name: `toa-storages-${name}`,
+            key: secret.name,
+            optional: secret.optional
+          }
+        })
   }
 
   return secrets
 }
 
+function getMounts (instances: Instance[], annotation: Annotation): Mounts | null {
+  let mounts: Mounts | null = null
+
+  for (const { locator, manifest } of instances)
+    for (const name of manifest) {
+      const declaration = annotation[name]
+
+      // eslint-disable-next-line max-depth
+      if (declaration.provider !== 'fs')
+        continue
+
+      // eslint-disable-next-line max-depth
+      if (declaration.claim !== undefined) {
+        mounts ??= {}
+        mounts[locator.label] ??= []
+
+        mounts[locator.label].push({
+          name,
+          path: declaration.path,
+          claim: declaration.claim
+        })
+      }
+    }
+
+  return mounts
+}
+
 export type Instance = context.Dependency<string[]>

package/source/errors.ts

@@ -0,0 +1,3 @@
+import { Err } from 'error-value'
+
+export const ERR_NOT_FOUND = new Err('NOT_FOUND')

package/source/index.ts

@@ -2,4 +2,6 @@ export { Factory } from './Factory'
 export { deployment } from './deployment'
 export { manifest } from './manifest'
 
-export type { Entry } from './Entry'
+export type { Entry, Stream } from './Entry'
+export type { Storage } from './Storage'
+export type * from './providers'

package/source/manifest.ts

@@ -1,8 +1,9 @@
-import { match } from 'matchacho'
-
 export function manifest (manifest: string | string[] | null): string[] {
-  return match(manifest,
-    String, (name: string) => [name],
-    Array, manifest,
-    null, [])
+  if (manifest === null)
+    return []
+
+  if (typeof manifest === 'string')
+    return [manifest]
+
+  return manifest
 }