@toa.io/extensions.storages 0.20.0-alpha.3

package/source/Storage.test.ts

@@ -0,0 +1,445 @@
+import { Readable } from 'node:stream'
+import { match } from '@toa.io/match'
+import { buffer } from '@toa.io/generic'
+import { Storage } from './Storage'
+import { cases, open, rnd } from './.test/util'
+import { type Entry } from './Entry'
+import { providers } from './providers'
+
+let storage: Storage
+let dir: string
+
+describe.each(cases)('%s', (_, url, secrets) => {
+  beforeEach(() => {
+    dir = '/' + rnd()
+
+    const Provider = providers[url.protocol]
+    const provider = new Provider(url, secrets)
+
+    storage = new Storage(provider)
+  })
+
+  it('should be', async () => {
+    expect(storage).toBeInstanceOf(Storage)
+  })
+
+  it('should return error if entry is not found', async () => {
+    const result = await storage.get('not-found')
+
+    match(result,
+      Error, (error: Error) => expect(error.message).toBe('NOT_FOUND'))
+  })
+
+  describe('put', () => {
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream = await open('lenna.png')
+
+      lenna = await storage.put(dir, stream) as Entry
+    })
+
+    it('should not return error', async () => {
+      expect(lenna).not.toBeInstanceOf(Error)
+    })
+
+    it('should return entry id', async () => {
+      expect(lenna.id).toBeDefined()
+    })
+
+    it('should return id as checksum', async () => {
+      const stream = await open('lenna.png')
+      const dir2 = '/' + rnd()
+      const copy = await storage.put(dir2, stream) as Entry
+
+      expect(copy.id).toBe(lenna.id)
+    })
+
+    it('should detect file type', async () => {
+      expect(lenna.type).toBe('image/png')
+    })
+
+    it('should count size', async () => {
+      expect(lenna.size).toBe(473831)
+    })
+
+    it('should return entry', async () => {
+      expect(lenna).toMatchObject({
+        id: lenna.id,
+        type: 'image/png',
+        variants: [],
+        meta: {}
+      })
+    })
+
+    it('should create entry', async () => {
+      const entry = await storage.get(`${dir}/${lenna.id}`)
+
+      match(entry,
+        {
+          id: lenna.id,
+          type: 'image/png',
+          variants: [],
+          meta: {}
+        }, undefined)
+    })
+
+    it('should set timestamp', async () => {
+      const now = Date.now()
+      const entry = await storage.get(`${dir}/${lenna.id}`) as Entry
+
+      expect(entry.created).toBeLessThanOrEqual(now)
+      expect(entry.created).toBeGreaterThan(now - 100)
+    })
+
+    describe('existing entry', () => {
+      it('should unhide existing', async () => {
+        const stream = await open('lenna.png')
+        const path = `${dir}/${lenna.id}`
+
+        await storage.conceal(path)
+        await storage.put(dir, stream)
+
+        const list = await storage.list(dir)
+
+        expect(list).toContainEqual(lenna.id)
+      })
+
+      it('should preserve meta', async () => {
+        const path = `${dir}/${lenna.id}`
+        const stream = await open('lenna.png')
+
+        await storage.annotate(path, 'foo', 'bar')
+        await storage.put(dir, stream)
+
+        const entry = await storage.get(path) as Entry
+
+        expect(entry.meta).toMatchObject({ foo: 'bar' })
+      })
+    })
+  })
+
+  describe('list', () => {
+    let albert: Entry
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream0 = await open('albert.jpg')
+      const stream1 = await open('lenna.png')
+
+      albert = await storage.put(dir, stream0) as Entry
+      lenna = await storage.put(dir, stream1) as Entry
+    })
+
+    it('should list entries', async () => {
+      const list = await storage.list(dir)
+
+      expect(list).toMatchObject([albert.id, lenna.id])
+    })
+
+    it('should permutate', async () => {
+      const error = await storage.reorder(dir, [lenna.id, albert.id])
+
+      expect(error).toBeUndefined()
+
+      const list = await storage.list(dir)
+
+      expect(list).toMatchObject([lenna.id, albert.id])
+    })
+
+    it('should return PERMUTATION_MISMATCH', async () => {
+      const cases = [
+        [lenna.id],
+        [albert.id, lenna.id, 'unknown'],
+        [lenna.id, lenna.id],
+        [lenna.id, lenna.id, albert.id]
+      ]
+
+      for (const permutation of cases) {
+        const error = await storage.reorder(dir, permutation)
+
+        expect(error).toBeInstanceOf(Error)
+        expect(error).toMatchObject({ message: 'PERMUTATION_MISMATCH' })
+      }
+    })
+
+    it('should exclude concealed', async () => {
+      const path = `${dir}/${lenna.id}`
+
+      await storage.conceal(path)
+
+      const entries = await storage.list(dir)
+
+      expect(entries).toMatchObject([albert.id])
+    })
+
+    it('should reveal', async () => {
+      const path = `${dir}/${lenna.id}`
+
+      await storage.conceal(path)
+      await storage.reveal(path)
+      await storage.reveal(path) // test that no duplicates are created
+
+      const entries = await storage.list(dir)
+
+      expect(entries).toMatchObject([albert.id, lenna.id])
+    })
+
+    it('should return ERR_NOT_FOOUD if entry doesnt exist', async () => {
+      const path = `${dir}/oopsie`
+
+      const methods: Array<'reveal' | 'conceal'> = ['reveal', 'conceal']
+
+      for (const method of methods) {
+        const error = await storage[method](path)
+
+        expect(error).toBeInstanceOf(Error)
+        expect(error).toMatchObject({ message: 'NOT_FOUND' })
+      }
+    })
+  })
+
+  describe('annotate', () => {
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream = await open('lenna.png')
+
+      lenna = await storage.put(dir, stream) as Entry
+    })
+
+    it('should set meta', async () => {
+      const path = `${dir}/${lenna.id}`
+
+      await storage.annotate(path, 'foo', 'bar')
+
+      const state0 = await storage.get(path) as Entry
+
+      expect(state0.meta).toMatchObject({ foo: 'bar' })
+
+      await storage.annotate(path, 'foo')
+
+      const state1 = await storage.get(path) as Entry
+
+      expect('foo' in state1.meta).toBe(false)
+    })
+  })
+
+  describe('variants', () => {
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream = await open('lenna.png')
+
+      lenna = await storage.put(dir, stream) as Entry
+    })
+
+    it('should add variant', async () => {
+      const stream = await open('sample.jpeg')
+
+      const path = `${dir}/${lenna.id}`
+
+      await storage.diversify(path, 'foo', stream)
+
+      const state = await storage.get(path) as Entry
+
+      expect(state.variants).toMatchObject([{ name: 'foo', size: 73444, type: 'image/jpeg' }])
+    })
+
+    it('should replace variant', async () => {
+      const stream0 = await open('sample.jpeg')
+      const stream1 = await open('sample.webp')
+      const path = `${dir}/${lenna.id}`
+
+      await storage.diversify(path, 'foo', stream0)
+      await storage.diversify(path, 'foo', stream1)
+
+      const state = await storage.get(path) as Entry
+
+      expect(state.variants).toMatchObject([{ name: 'foo', type: 'image/webp' }])
+    })
+  })
+
+  describe('fetch', () => {
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream = await open('lenna.png')
+
+      lenna = await storage.put(dir, stream) as Entry
+    })
+
+    it('should fetch', async () => {
+      const path = `${dir}/${lenna.id}`
+      const stream = await storage.fetch(path)
+
+      const stored: Buffer = await match(stream,
+        Readable, async (stream: Readable) => await buffer(stream))
+
+      const buf = await buffer(await open('lenna.png'))
+
+      expect(stored.compare(buf)).toBe(0)
+    })
+
+    it('should fetch blob by id', async () => {
+      const stream = await open('lenna.ascii')
+      const entry = await storage.put(dir, stream) as Entry
+      const stored = await storage.fetch(entry.id)
+
+      if (stored instanceof Error)
+        throw stored
+
+      const buf = await buffer(stored)
+      const expected = await buffer(await open('lenna.ascii'))
+
+      expect(buf.compare(expected)).toBe(0)
+    })
+
+    it('should fetch variant', async () => {
+      const stream = await open('sample.jpeg')
+
+      const buf = await buffer(stream)
+      const path = `${dir}/${lenna.id}`
+
+      await storage.diversify(path, '100x100.jpeg', Readable.from(buf))
+
+      const variant = await storage.fetch(`${path}.100x100.jpeg`)
+
+      const stored = await match(variant,
+        Readable, async (stream: Readable) => await buffer(stream))
+
+      expect(stored.compare(buf)).toBe(0)
+    })
+
+    it('should not fetch blob by id and fake path', async () => {
+      const stored = await storage.fetch(`fake/${lenna.id}`)
+
+      match(stored,
+        Error, (error: Error) => expect(error.message).toBe('NOT_FOUND'))
+    })
+  })
+
+  describe('delete', () => {
+    let lenna: Entry
+
+    beforeEach(async () => {
+      const stream = await open('lenna.png')
+
+      lenna = await storage.put(dir, stream) as Entry
+    })
+
+    it('should remove from the list', async () => {
+      await storage.delete(`${dir}/${lenna.id}`)
+
+      const list = await storage.list(dir)
+
+      expect(list).not.toContain(lenna.id)
+    })
+
+    it('should delete entry', async () => {
+      await storage.delete(`${dir}/${lenna.id}`)
+
+      const result = await storage.get(`${dir}/${lenna.id}`)
+
+      match(result,
+        Error, (error: Error) => expect(error.message).toBe('NOT_FOUND'))
+    })
+
+    it('should delete variants', async () => {
+      const stream = await open('sample.jpeg')
+
+      const path = `${dir}/${lenna.id}`
+
+      await storage.diversify(path, 'foo', stream)
+      await storage.delete(`${dir}/${lenna.id}`)
+
+      const variant = await storage.fetch(`${path}.foo`)
+
+      match(variant,
+        Error, (error: Error) => expect(error.message).toBe('NOT_FOUND'))
+
+      stream.destroy()
+    })
+
+    it('should throw if path is not an entry', async () => {
+      const result = await storage.delete(dir)
+
+      expect(result).toBeInstanceOf(Error)
+      expect(result).toMatchObject({ message: 'NOT_FOUND' })
+    })
+  })
+
+  describe('signatures', () => {
+    it.each(['jpeg', 'gif', 'webp', 'heic', 'jxl', 'avif'])('should detect image/%s',
+      async (type) => {
+        const stream = await open('sample.' + type)
+
+        const entry = await storage.put(dir, stream) as Entry
+
+        expect(entry.type).toBe('image/' + type)
+      })
+  })
+
+  it('should return error if type doesnt match', async () => {
+    const stream = await open('sample.jpeg')
+
+    const result = await storage.put(dir, stream, 'image/png')
+
+    match(result,
+      Error, (error: Error) => expect(error.message).toBe('TYPE_MISMATCH'))
+  })
+
+  it('should trust unknown types', async () => {
+    const stream = await open('lenna.ascii')
+
+    const result = await storage.put(dir, stream, 'text/plain')
+
+    expect(result).not.toBeInstanceOf(Error)
+    expect(result).toMatchObject({ type: 'text/plain' })
+  })
+
+  it('should return error if type is identifiable', async () => {
+    const stream = await open('lenna.ascii')
+
+    const result = await storage.put(dir, stream, 'image/jpeg')
+
+    expect(result).toBeInstanceOf(Error)
+    expect(result).toMatchObject({ message: 'TYPE_MISMATCH' })
+  })
+
+  it('should not return error if type application/octet-stream', async () => {
+    const stream = await open('sample.jpeg')
+
+    const result = await storage.put(dir, stream, 'application/octet-stream')
+
+    expect(result).not.toBeInstanceOf(Error)
+    expect(result).toMatchObject({ type: 'image/jpeg' })
+  })
+
+  it('should handle root entries', async () => {
+    const stream = await open('sample.jpeg')
+
+    const result = await storage.put('hello', stream) as Entry
+
+    expect(result).not.toBeInstanceOf(Error)
+
+    const stored = await storage.fetch(result.id)
+
+    expect(stored).not.toBeInstanceOf(Error)
+  })
+
+  it('should store empty file', async () => {
+    const stream = await open('empty.txt')
+    const result = await storage.put('empty', stream) as Entry
+
+    expect(result.size).toBe(0)
+
+    const stored = await storage.fetch(result.id) as Readable
+
+    expect(stored).not.toBeInstanceOf(Error)
+
+    const buf = await buffer(stored)
+
+    expect(buf.length).toBe(0)
+  })
+})

package/source/Storage.ts

@@ -0,0 +1,236 @@
+import { Readable } from 'node:stream'
+import { posix } from 'node:path'
+import { decode, encode } from 'msgpackr'
+import { buffer, newid } from '@toa.io/generic'
+import { Scanner } from './Scanner'
+import type { Provider } from './Provider'
+import type { Entry } from './Entry'
+
+export class Storage {
+  private readonly provider: Provider
+
+  public constructor (provider: Provider) {
+    this.provider = provider
+  }
+
+  public async put (path: string, stream: Readable, type?: string): Maybe<Entry> {
+    const scanner = new Scanner(type)
+    const pipe = stream.pipe(scanner)
+    const tempname = await this.transit(pipe)
+
+    if (scanner.error !== null)
+      return scanner.error
+
+    const id = scanner.digest()
+
+    await this.persist(tempname, id)
+
+    return await this.create(path, id, scanner.size, scanner.type)
+  }
+
+  public async get (path: string): Maybe<Entry> {
+    const metapath = posix.join(ENTRIES, path, META)
+    const result = await this.provider.get(metapath)
+
+    if (result === null) return ERR_NOT_FOUND
+    else return decode(await buffer(result))
+  }
+
+  public async fetch (path: string): Maybe<Readable> {
+    const { rel, id, variant } = this.parse(path)
+
+    if (variant === null && rel !== '') {
+      const entry = await this.get(path)
+
+      if (entry instanceof Error)
+        return entry
+    }
+
+    const blob = variant === null
+      ? posix.join(BLOBs, id)
+      : posix.join(ENTRIES, rel, id, variant)
+
+    const stream = await this.provider.get(blob)
+
+    if (stream === null) return ERR_NOT_FOUND
+    else return stream
+  }
+
+  public async delete (path: string): Maybe<void> {
+    const entry = await this.get(path)
+
+    if (entry instanceof Error)
+      return entry
+
+    await this.conceal(path)
+    await this.provider.delete(posix.join(ENTRIES, path))
+  }
+
+  public async list (path: string): Promise<string[]> {
+    const listfile = posix.join(ENTRIES, path, LIST)
+    const stream = await this.provider.get(listfile)
+
+    return stream === null ? [] : decode(await buffer(stream))
+  }
+
+  public async reorder (path: string, ids: string[]): Maybe<void> {
+    const unique = new Set(ids)
+    const dir = posix.join(ENTRIES, path)
+    const list = await this.list(path)
+
+    if (list.length !== ids.length || unique.size !== ids.length)
+      return ERR_PERMUTATION_MISMATCH
+
+    for (const id of ids)
+      if (!list.includes(id))
+        return ERR_PERMUTATION_MISMATCH
+
+    await this.provider.put(dir, LIST, Readable.from(encode(ids)))
+  }
+
+  public async conceal (path: string): Maybe<void> {
+    const { id, rel } = this.parse(path)
+    const dir = posix.join(ENTRIES, rel)
+    const list = await this.list(rel)
+    const index = list.indexOf(id)
+
+    if (index === -1)
+      return ERR_NOT_FOUND
+
+    list.splice(index, 1)
+
+    await this.provider.put(dir, LIST, Readable.from(encode(list)))
+  }
+
+  public async reveal (path: string): Maybe<void> {
+    const { id, rel } = this.parse(path)
+
+    return await this.enroll(rel, id)
+  }
+
+  public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
+    const scanner = new Scanner()
+    const pipe = stream.pipe(scanner)
+
+    await this.provider.put(posix.join(ENTRIES, path), name, pipe)
+
+    if (scanner.error !== null)
+      return scanner.error
+
+    const { size, type } = scanner
+    const entry = await this.get(path)
+
+    if (entry instanceof Error)
+      return entry
+
+    entry.variants = entry.variants.filter((variant) => variant.name !== name)
+    entry.variants.push({ name, size, type })
+
+    await this.save(path, entry)
+  }
+
+  public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
+    const entry = await this.get(path)
+
+    if (entry instanceof Error)
+      return entry
+
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+    if (value === undefined) delete entry.meta[key]
+    else entry.meta[key] = value
+
+    await this.save(path, entry)
+  }
+
+  private async transit (stream: Readable): Promise<string> {
+    const tempname = newid()
+
+    await this.provider.put(TEMP, tempname, stream)
+
+    return tempname
+  }
+
+  private async persist (tempname: string, id: string): Promise<void> {
+    const temp = posix.join(TEMP, tempname)
+    const blob = posix.join(BLOBs, id)
+
+    await this.provider.move(temp, blob)
+  }
+
+  // eslint-disable-next-line max-params
+  private async create (path: string, id: string, size: number, type: string): Promise<Entry> {
+    const entry: Entry = {
+      id,
+      size,
+      type,
+      created: Date.now(),
+      variants: [],
+      meta: {}
+    }
+
+    const metafile = posix.join(path, entry.id)
+    const existing = await this.get(metafile)
+
+    if (existing instanceof Error)
+      await this.save(metafile, entry)
+
+    await this.enroll(path, id, true)
+
+    return entry
+  }
+
+  private async save (rel: string, entry: Entry): Promise<void> {
+    const buffer = encode(entry)
+    const stream = Readable.from(buffer)
+
+    await this.provider.put(posix.join(ENTRIES, rel), META, stream)
+  }
+
+  private async enroll (path: string, id: string, addition: boolean = false): Maybe<void> {
+    const dir = posix.join(ENTRIES, path)
+    const list = await this.list(path)
+    const index = list.indexOf(id)
+
+    if (index !== -1)
+      if (addition) list.splice(index, 1)
+      else return
+    else if (!addition) {
+      const entry = await this.get(posix.join(path, id))
+
+      if (entry instanceof Error)
+        return entry
+    }
+
+    list.push(id)
+
+    await this.provider.put(dir, LIST, Readable.from(encode(list)))
+  }
+
+  private parse (path: string): Path {
+    const [last, ...segments] = path.split('/').reverse()
+    const [id, ...rest] = last.split('.')
+    const variant = rest.length > 0 ? rest.join('.') : null
+    const rel = segments.reverse().join('/')
+
+    return { rel, id, variant }
+  }
+}
+
+const ERR_NOT_FOUND = new Error('NOT_FOUND')
+const ERR_PERMUTATION_MISMATCH = new Error('PERMUTATION_MISMATCH')
+
+const TEMP = '/temp'
+const BLOBs = '/blobs'
+const ENTRIES = '/entries'
+const LIST = '.list'
+const META = '.meta'
+
+type Maybe<T> = Promise<T | Error>
+
+interface Path {
+  rel: string
+  id: string
+  variant: string | null
+}
+
+export type Storages = Record<string, Storage>