@toa.io/extensions.storages 1.0.0-alpha.0 → 1.0.0-alpha.2

package/source/Storage.test.ts

@@ -1,22 +1,29 @@
 import { Readable } from 'node:stream'
-import { match } from 'matchacho'
-import { buffer } from '@toa.io/generic'
+import { randomUUID } from 'node:crypto'
+import { buffer } from 'node:stream/consumers'
+import { createReadStream } from 'node:fs'
+import path from 'node:path'
+import assert from 'node:assert'
 import { Storage } from './Storage'
-import { cases, open, rnd } from './test/util'
+import { suites } from './test/util'
 import { type Entry } from './Entry'
 import { providers } from './providers'
-import type { ErrorType } from 'error-value'
+import type { ProviderConstructor } from './Provider'
 
 let storage: Storage
 let dir: string
 
-const [, url, secrets] = cases[0]
+const suite = suites[0]
+
+beforeAll(async () => {
+  process.chdir(path.join(__dirname, 'test'))
+})
 
 beforeEach(() => {
-  dir = '/' + rnd()
+  dir = '/' + randomUUID()
 
-  const Provider = providers[url.protocol]
-  const provider = new Provider(url, secrets)
+  const Provider: ProviderConstructor = providers[suite.provider]
+  const provider = new Provider(suite.options)
 
   storage = new Storage(provider)
 })
@@ -28,9 +35,7 @@ it('should be', async () => {
 it('should return error if entry is not found', async () => {
   const result = await storage.get('not-found')
 
-  if (!(result instanceof Error))
-    throw new Error('Expected error')
-
+  expect(result).toBeInstanceOf(Error)
   expect(result).toMatchObject({ code: 'NOT_FOUND' })
 })
 
@@ -39,10 +44,10 @@ describe('put', () => {
   let startCreation: number
 
   beforeEach(async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
     startCreation = Date.now()
-    lenna = await storage.put(dir, stream) as Entry
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
   it('should not return error', async () => {
@@ -54,19 +59,19 @@ describe('put', () => {
   })
 
   it('should return id as checksum', async () => {
-    const stream = open('lenna.png')
-    const dir2 = '/' + rnd()
-    const copy = await storage.put(dir2, stream) as Entry
+    const stream = createReadStream('lenna.png')
+    const dir2 = '/' + randomUUID()
+    const copy = (await storage.put(dir2, stream)) as Entry
 
-    expect(copy.id).toBe(lenna.id)
+    expect(copy).toHaveProperty('id', lenna.id)
   })
 
   it('should detect file type', async () => {
-    expect(lenna.type).toBe('image/png')
+    expect(lenna).toHaveProperty('type', 'image/png')
   })
 
   it('should count size', async () => {
-    expect(lenna.size).toBe(473831)
+    expect(lenna).toHaveProperty('size', 473831)
   })
 
   it('should return entry', async () => {
@@ -81,18 +86,17 @@ describe('put', () => {
   it('should create entry', async () => {
     const entry = await storage.get(`${dir}/${lenna.id}`)
 
-    match(entry,
-      {
-        id: lenna.id,
-        type: 'image/png',
-        variants: [],
-        meta: {}
-      }, undefined)
+    expect(entry).toMatchObject({
+      id: lenna.id,
+      type: 'image/png',
+      variants: [],
+      meta: {}
+    })
   })
 
   it('should set timestamp', async () => {
     const now = Date.now()
-    const entry = await storage.get(`${dir}/${lenna.id}`) as Entry
+    const entry = (await storage.get(`${dir}/${lenna.id}`)) as Entry
 
     expect(entry.created).toBeLessThanOrEqual(now)
     expect(entry.created).toBeGreaterThanOrEqual(startCreation)
@@ -100,7 +104,7 @@ describe('put', () => {
 
   describe('existing entry', () => {
     it('should unhide existing', async () => {
-      const stream = open('lenna.png')
+      const stream = createReadStream('lenna.png')
       const path = `${dir}/${lenna.id}`
 
       await storage.conceal(path)
@@ -113,14 +117,14 @@ describe('put', () => {
 
     it('should preserve meta', async () => {
       const path = `${dir}/${lenna.id}`
-      const stream = open('lenna.png')
+      const stream = createReadStream('lenna.png')
 
       await storage.annotate(path, 'foo', 'bar')
       await storage.put(dir, stream)
 
-      const entry = await storage.get(path) as Entry
+      const entry = await storage.get(path)
 
-      expect(entry.meta).toMatchObject({ foo: 'bar' })
+      expect(entry).toHaveProperty('meta', expect.objectContaining({ foo: 'bar' }))
     })
   })
 })
@@ -130,17 +134,17 @@ describe('list', () => {
   let lenna: Entry
 
   beforeEach(async () => {
-    const stream0 = open('albert.jpg')
-    const stream1 = open('lenna.png')
+    const stream0 = createReadStream('albert.jpg')
+    const stream1 = createReadStream('lenna.png')
 
-    albert = await storage.put(dir, stream0) as Entry
-    lenna = await storage.put(dir, stream1) as Entry
+    albert = (await storage.put(dir, stream0)) as Entry
+    lenna = (await storage.put(dir, stream1)) as Entry
   })
 
   it('should list entries', async () => {
     const list = await storage.list(dir)
 
-    expect(list).toMatchObject([albert.id, lenna.id])
+    expect(list).toEqual([albert.id, lenna.id])
   })
 
   it('should permutate', async () => {
@@ -150,7 +154,7 @@ describe('list', () => {
 
     const list = await storage.list(dir)
 
-    expect(list).toMatchObject([lenna.id, albert.id])
+    expect(list).toEqual([lenna.id, albert.id])
   })
 
   it('should return PERMUTATION_MISMATCH', async () => {
@@ -165,7 +169,7 @@ describe('list', () => {
       const error = await storage.permute(dir, permutation)
 
       expect(error).toBeInstanceOf(Error)
-      expect(error).toMatchObject({ code: 'PERMUTATION_MISMATCH' })
+      expect(error).toHaveProperty('code', 'PERMUTATION_MISMATCH')
     }
   })
 
@@ -176,7 +180,7 @@ describe('list', () => {
 
     const entries = await storage.list(dir)
 
-    expect(entries).toMatchObject([albert.id])
+    expect(entries).toEqual([albert.id])
   })
 
   it('should reveal', async () => {
@@ -188,7 +192,7 @@ describe('list', () => {
 
     const entries = await storage.list(dir)
 
-    expect(entries).toMatchObject([albert.id, lenna.id])
+    expect(entries).toEqual([albert.id, lenna.id])
   })
 
   it('should return ERR_NOT_FOOUD if entry doesnt exist', async () => {
@@ -200,7 +204,7 @@ describe('list', () => {
       const error = await storage[method](path)
 
       expect(error).toBeInstanceOf(Error)
-      expect(error).toMatchObject({ code: 'NOT_FOUND' })
+      expect(error).toHaveProperty('code', 'NOT_FOUND')
     }
   })
 })
@@ -209,9 +213,9 @@ describe('annotate', () => {
   let lenna: Entry
 
   beforeEach(async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
-    lenna = await storage.put(dir, stream) as Entry
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
   it('should set meta', async () => {
@@ -219,15 +223,15 @@ describe('annotate', () => {
 
     await storage.annotate(path, 'foo', 'bar')
 
-    const state0 = await storage.get(path) as Entry
+    const state0 = (await storage.get(path)) as Entry
 
-    expect(state0.meta).toMatchObject({ foo: 'bar' })
+    expect(state0).toHaveProperty('meta.foo', 'bar')
 
     await storage.annotate(path, 'foo')
 
-    const state1 = await storage.get(path) as Entry
+    const state1 = (await storage.get(path)) as Entry
 
-    expect('foo' in state1.meta).toBe(false)
+    expect(state1.meta).not.toHaveProperty('foo')
   })
 })
 
@@ -235,34 +239,36 @@ describe('variants', () => {
   let lenna: Entry
 
   beforeEach(async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
-    lenna = await storage.put(dir, stream) as Entry
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
   it('should add variant', async () => {
-    const stream = open('sample.jpeg')
+    const stream = createReadStream('sample.jpeg')
 
     const path = `${dir}/${lenna.id}`
 
     await storage.diversify(path, 'foo', stream)
 
-    const state = await storage.get(path) as Entry
+    const state = (await storage.get(path)) as Entry
 
-    expect(state.variants).toMatchObject([{ name: 'foo', size: 73444, type: 'image/jpeg' }])
+    expect(state).toHaveProperty('variants',
+      expect.arrayContaining([{ name: 'foo', size: 73444, type: 'image/jpeg' }]))
   })
 
   it('should replace variant', async () => {
-    const stream0 = open('sample.jpeg')
-    const stream1 = open('sample.webp')
+    const stream0 = createReadStream('sample.jpeg')
+    const stream1 = createReadStream('sample.webp')
     const path = `${dir}/${lenna.id}`
 
     await storage.diversify(path, 'foo', stream0)
     await storage.diversify(path, 'foo', stream1)
 
-    const state = await storage.get(path) as Entry
+    const state = (await storage.get(path)) as Entry
 
-    expect(state.variants).toMatchObject([{ name: 'foo', type: 'image/webp' }])
+    expect(state).toHaveProperty('variants',
+      expect.arrayContaining([expect.objectContaining({ name: 'foo', type: 'image/webp' })]))
   })
 })
 
@@ -270,39 +276,38 @@ describe('fetch', () => {
   let lenna: Entry
 
   beforeEach(async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
-    lenna = await storage.put(dir, stream) as Entry
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
   it('should fetch', async () => {
     const path = `${dir}/${lenna.id}`
     const stream = await storage.fetch(path)
 
-    const stored: Buffer = await match(stream,
-      Readable, async (stream: Readable) => await buffer(stream))
+    assert.ok(stream instanceof Readable)
 
-    const buf = await buffer(open('lenna.png'))
+    const stored = await buffer(stream)
+    const buf = await buffer(createReadStream('lenna.png'))
 
     expect(stored.compare(buf)).toBe(0)
   })
 
   it('should fetch blob by id', async () => {
-    const stream = open('lenna.ascii')
-    const entry = await storage.put(dir, stream) as Entry
+    const stream = createReadStream('lenna.ascii')
+    const entry = (await storage.put(dir, stream)) as Entry
     const stored = await storage.fetch(entry.id)
 
-    if (stored instanceof Error)
-      throw stored
+    if (stored instanceof Error) throw stored
 
     const buf = await buffer(stored)
-    const expected = await buffer(open('lenna.ascii'))
+    const expected = await buffer(createReadStream('lenna.ascii'))
 
     expect(buf.compare(expected)).toBe(0)
   })
 
   it('should fetch variant', async () => {
-    const stream = open('sample.jpeg')
+    const stream = createReadStream('sample.jpeg')
 
     const buf = await buffer(stream)
     const path = `${dir}/${lenna.id}`
@@ -311,8 +316,9 @@ describe('fetch', () => {
 
     const variant = await storage.fetch(`${path}.100x100.jpeg`)
 
-    const stored = await match<Promise<Buffer>>(variant,
-      Readable, async (stream: Readable) => await buffer(stream))
+    assert.ok(variant instanceof Readable)
+
+    const stored = await buffer(variant)
 
     expect(stored.compare(buf)).toBe(0)
   })
@@ -320,8 +326,8 @@ describe('fetch', () => {
   it('should not fetch blob by id and fake path', async () => {
     const stored = await storage.fetch(`fake/${lenna.id}`)
 
-    match(stored,
-      Error, (error: ErrorType) => expect(error.code).toBe('NOT_FOUND'))
+    expect(stored).toBeInstanceOf(Error)
+    expect(stored).toHaveProperty('code', 'NOT_FOUND')
   })
 })
 
@@ -329,9 +335,9 @@ describe('delete', () => {
   let lenna: Entry
 
   beforeEach(async () => {
-    const stream = open('lenna.png')
+    const stream = createReadStream('lenna.png')
 
-    lenna = await storage.put(dir, stream) as Entry
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
   it('should remove from the list', async () => {
@@ -347,24 +353,23 @@ describe('delete', () => {
 
     const result = await storage.get(`${dir}/${lenna.id}`)
 
-    match(result,
-      Error, (error: ErrorType) => expect(error.code).toBe('NOT_FOUND'))
+    expect(result).toBeInstanceOf(Error)
+    expect(result).toHaveProperty('code', 'NOT_FOUND')
   })
 
   it('should delete variants', async () => {
-    const stream = open('sample.jpeg')
+    const stream = createReadStream('sample.jpeg')
 
     const path = `${dir}/${lenna.id}`
 
     await storage.diversify(path, 'foo', stream)
     await storage.delete(`${dir}/${lenna.id}`)
+    stream.destroy()
 
     const variant = await storage.fetch(`${path}.foo`)
 
-    match(variant,
-      Error, (error: ErrorType) => expect(error.code).toBe('NOT_FOUND'))
-
-    stream.destroy()
+    expect(variant).toBeInstanceOf(Error)
+    expect(variant).toHaveProperty('code', 'NOT_FOUND')
   })
 
   it('should throw if path is not an entry', async () => {
@@ -378,72 +383,72 @@ describe('delete', () => {
 describe('signatures', () => {
   it.each(['jpeg', 'gif', 'webp', 'heic', 'jxl', 'avif'])('should detect image/%s',
     async (type) => {
-      const stream = open('sample.' + type)
+      const stream = createReadStream('sample.' + type)
 
-      const entry = await storage.put(dir, stream) as Entry
+      const entry = (await storage.put(dir, stream)) as Entry
 
-      expect(entry.type).toBe('image/' + type)
+      expect(entry).toHaveProperty('type', 'image/' + type)
     })
 })
 
-it('should return error if type doesnt match', async () => {
-  const stream = open('sample.jpeg')
+it("should return error if type doesn't match", async () => {
+  const stream = createReadStream('sample.jpeg')
 
   const result = await storage.put(dir, stream, { claim: 'image/png' })
 
-  match(result,
-    Error, (error: ErrorType) => expect(error.code).toBe('TYPE_MISMATCH'))
+  expect(result).toBeInstanceOf(Error)
+  expect(result).toHaveProperty('code', 'TYPE_MISMATCH')
 })
 
 it('should trust unknown types', async () => {
-  const stream = open('lenna.ascii')
+  const stream = createReadStream('lenna.ascii')
 
   const result = await storage.put(dir, stream, { claim: 'text/plain' })
 
   expect(result).not.toBeInstanceOf(Error)
-  expect(result).toMatchObject({ type: 'text/plain' })
+  expect(result).toHaveProperty('type', 'text/plain')
 })
 
 it('should return error if type is identifiable', async () => {
-  const stream = open('lenna.ascii')
+  const stream = createReadStream('lenna.ascii')
 
   const result = await storage.put(dir, stream, { claim: 'image/jpeg' })
 
   expect(result).toBeInstanceOf(Error)
-  expect(result).toMatchObject({ code: 'TYPE_MISMATCH' })
+  expect(result).toHaveProperty('code', 'TYPE_MISMATCH')
 })
 
 it('should not return error if type application/octet-stream', async () => {
-  const stream = open('sample.jpeg')
+  const stream = createReadStream('sample.jpeg')
 
   const result = await storage.put(dir, stream, { claim: 'application/octet-stream' })
 
   expect(result).not.toBeInstanceOf(Error)
-  expect(result).toMatchObject({ type: 'image/jpeg' })
+  expect(result).toHaveProperty('type', 'image/jpeg')
 })
 
 it('should return error if type is not acceptable', async () => {
-  const stream = open('sample.jpeg')
+  const stream = createReadStream('sample.jpeg')
 
   const result = await storage.put(dir, stream, { accept: 'image/png' })
 
-  match(result,
-    Error, (error: ErrorType) => expect(error.code).toBe('NOT_ACCEPTABLE'))
+  expect(result).toBeInstanceOf(Error)
+  expect(result).toHaveProperty('code', 'NOT_ACCEPTABLE')
 })
 
 it('should accept wildcard types', async () => {
-  const stream = open('sample.jpeg')
+  const stream = createReadStream('sample.jpeg')
 
   const result = await storage.put(dir, stream, { accept: 'image/*' })
 
   expect(result).not.toBeInstanceOf(Error)
-  expect(result).toMatchObject({ type: 'image/jpeg' })
+  expect(result).toHaveProperty('type', 'image/jpeg')
 })
 
 it('should handle root entries', async () => {
-  const stream = open('sample.jpeg')
+  const stream = createReadStream('sample.jpeg')
 
-  const result = await storage.put('hello', stream) as Entry
+  const result = (await storage.put('hello', stream)) as Entry
 
   expect(result).not.toBeInstanceOf(Error)
 
@@ -453,16 +458,16 @@ it('should handle root entries', async () => {
 })
 
 it('should store empty file', async () => {
-  const stream = open('empty.txt')
-  const result = await storage.put('empty', stream) as Entry
+  const stream = createReadStream('empty.txt')
+  const result = (await storage.put('empty', stream)) as Entry
 
-  expect(result.size).toBe(0)
+  expect(result).toHaveProperty('size', 0)
 
-  const stored = await storage.fetch(result.id) as Readable
+  const stored = (await storage.fetch(result.id)) as Readable
 
   expect(stored).not.toBeInstanceOf(Error)
 
   const buf = await buffer(stored)
 
-  expect(buf.length).toBe(0)
+  expect(buf).toHaveLength(0)
 })

package/source/Storage.ts

@@ -4,7 +4,7 @@ import { decode, encode } from 'msgpackr'
 import { buffer, newid } from '@toa.io/generic'
 import { Err } from 'error-value'
 import { Scanner } from './Scanner'
-import type { TypeControl } from './Scanner'
+import type { ScanOptions } from './Scanner'
 import type { Provider } from './Provider'
 import type { Entry } from './Entry'
 
@@ -15,8 +15,8 @@ export class Storage {
     this.provider = provider
   }
 
-  public async put (path: string, stream: Readable, type?: TypeControl): Maybe<Entry> {
-    const scanner = new Scanner(type)
+  public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
+    const scanner = new Scanner(options)
     const pipe = stream.pipe(scanner)
     const tempname = await this.transit(pipe)
 
@@ -27,7 +27,7 @@ export class Storage {
 
     await this.persist(tempname, id)
 
-    return await this.create(path, id, scanner.size, scanner.type)
+    return await this.create(path, id, scanner.size, scanner.type, options?.meta)
   }
 
   public async get (path: string): Maybe<Entry> {
@@ -160,14 +160,15 @@ export class Storage {
   }
 
   // eslint-disable-next-line max-params
-  private async create (path: string, id: string, size: number, type: string): Promise<Entry> {
+  private async create
+  (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
     const entry: Entry = {
       id,
       size,
       type,
       created: Date.now(),
       variants: [],
-      meta: {}
+      meta
     }
 
     const metafile = posix.join(path, entry.id)
@@ -235,4 +236,10 @@ interface Path {
   variant: string | null
 }
 
+type Meta = Record<string, string>
+
+interface Options extends ScanOptions {
+  meta?: Meta
+}
+
 export type Storages = Record<string, Storage>

package/source/deployment.ts

@@ -1,13 +1,18 @@
-import { encode } from 'msgpackr'
+import * as assert from 'node:assert'
+import { encode } from '@toa.io/generic'
 import { providers } from './providers'
+import { validateAnnotation } from './Annotation'
+import type { Annotation } from './Annotation'
 import type { Dependency, Variable } from '@toa.io/operations'
 import type { context } from '@toa.io/norm'
 
-export function deployment (instances: Instance[], annotation: Annotation): Dependency {
+export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
+
+export function deployment (instances: Instance[], annotation: unknown): Dependency {
   validate(instances, annotation)
 
-  const value = encode(annotation).toString('base64')
-  const pointer: Variable = { name: 'TOA_STORAGES', value }
+  const value = encode(annotation)
+  const pointer: Variable = { name: SERIALIZATION_PREFIX, value }
   const secrets = getSecrets(annotation)
 
   return {
@@ -15,51 +20,38 @@ export function deployment (instances: Instance[], annotation: Annotation): Depe
   }
 }
 
-function validate (instances: Instance[], annotation: Annotation): void {
-  if (annotation === undefined)
-    throw new Error('Storages annotation is required by: ' +
-      `'${instances.map((i) => i.component.locator.id).join("', '")}'`)
+function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
+  validateAnnotation(annotation)
 
   for (const instance of instances)
     contains(instance, annotation)
-
-  for (const ref of Object.values(annotation)) {
-    const url = new URL(ref)
-
-    if (!(url.protocol in providers))
-      throw new Error(`Unknown storage provider '${url.protocol}'`)
-  }
 }
 
 function contains (instance: Instance, annotation: Annotation): void {
   for (const name of instance.manifest)
-    if (!(name in annotation))
-      throw new Error(`Missing '${name}' storage annotation ` +
-        `declared in '${instance.component.locator.id}'`)
+    assert.ok(name in annotation,
+      `Missing '${name}' storage annotation ` +
+      `declared in '${instance.component.locator.id}'`)
 }
 
 function getSecrets (annotation: Annotation): Variable[] {
   const secrets: Variable[] = []
 
-  for (const [storage, ref] of Object.entries(annotation)) {
-    const url = new URL(ref)
-    const Provider = providers[url.protocol]
-
-    if (Provider.SECRETS === undefined)
-      continue
+  for (const [name, declaration] of Object.entries(annotation)) {
+    const Provider = providers[declaration.provider]
 
     for (const secret of Provider.SECRETS)
       secrets.push({
-        name: `TOA_STORAGES_${storage.toUpperCase()}_${secret.toUpperCase()}`,
+        name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
         secret: {
-          name: `toa-storages-${storage}`,
-          key: secret
+          name: `toa-storages-${name}`,
+          key: secret.name,
+          optional: secret.optional
         }
-      } satisfies Variable)
+      })
   }
 
   return secrets
 }
 
-type Annotation = Record<string, string>
 export type Instance = context.Dependency<string[]>

package/source/providers/Declaration.ts

@@ -0,0 +1,10 @@
+import type { S3Options } from './S3'
+import type { FileSystemOptions } from './FileSystem'
+import type { TemporaryOptions } from './Temporary'
+
+export type Declaration =
+  ({ provider: 's3' } & S3Options)
+  | ({ provider: 'fs' } & FileSystemOptions)
+  | ({ provider: 'tmp' } & TemporaryOptions)
+  | ({ provider: 'mem' })
+  | ({ provider: 'test' } & TemporaryOptions)