@toa.io/extensions.storages 1.0.0-alpha.31 → 1.0.0-alpha.310

package/source/Storage.test.ts

@@ -1,472 +0,0 @@
-import { Readable } from 'node:stream'
-import { randomUUID } from 'node:crypto'
-import { buffer } from 'node:stream/consumers'
-import { createReadStream } from 'node:fs'
-import path from 'node:path'
-import assert from 'node:assert'
-import { Storage } from './Storage'
-import { suites } from './test/util'
-import { type Entry } from './Entry'
-import { providers } from './providers'
-import type { ProviderConstructor } from './Provider'
-
-let storage: Storage
-let dir: string
-
-const suite = suites[0] // replace with 2 to run tests for S3
-
-beforeAll(async () => {
-  process.chdir(path.join(__dirname, 'test'))
-})
-
-beforeEach(() => {
-  dir = '/' + randomUUID()
-
-  const Provider: ProviderConstructor = providers[suite.provider]
-  const provider = new Provider(suite.options)
-
-  storage = new Storage(provider)
-})
-
-it('should be', async () => {
-  expect(storage).toBeInstanceOf(Storage)
-})
-
-it('should return error if entry is not found', async () => {
-  const result = await storage.get('not-found')
-
-  expect(result).toBeInstanceOf(Error)
-  expect(result).toMatchObject({ code: 'NOT_FOUND' })
-})
-
-describe('put', () => {
-  let lenna: Entry
-  let startCreation: number
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    startCreation = Date.now()
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should not return error', async () => {
-    expect(lenna).not.toBeInstanceOf(Error)
-  })
-
-  it('should return entry id', async () => {
-    expect(lenna.id).toBeDefined()
-  })
-
-  it('should return id as checksum', async () => {
-    const stream = createReadStream('lenna.png')
-    const dir2 = '/' + randomUUID()
-    const copy = (await storage.put(dir2, stream)) as Entry
-
-    expect(copy).toHaveProperty('id', lenna.id)
-  })
-
-  it('should detect file type', async () => {
-    expect(lenna).toHaveProperty('type', 'image/png')
-  })
-
-  it('should count size', async () => {
-    expect(lenna).toHaveProperty('size', 473831)
-  })
-
-  it('should return entry', async () => {
-    expect(lenna).toMatchObject({
-      id: lenna.id,
-      type: 'image/png',
-      variants: [],
-      meta: {}
-    })
-  })
-
-  it('should create entry', async () => {
-    const entry = await storage.get(`${dir}/${lenna.id}`)
-
-    expect(entry).toMatchObject({
-      id: lenna.id,
-      type: 'image/png',
-      variants: [],
-      meta: {}
-    })
-  })
-
-  it('should set timestamp', async () => {
-    const now = Date.now()
-    const entry = (await storage.get(`${dir}/${lenna.id}`)) as Entry
-
-    expect(entry.created).toBeLessThanOrEqual(now)
-    expect(entry.created).toBeGreaterThanOrEqual(startCreation)
-  })
-
-  describe('existing entry', () => {
-    it('should preserve meta', async () => {
-      const path = `${dir}/${lenna.id}`
-      const stream = createReadStream('lenna.png')
-
-      await storage.annotate(path, 'foo', 'bar')
-      await storage.put(dir, stream)
-
-      const entry = await storage.get(path)
-
-      expect(entry).toHaveProperty('meta', expect.objectContaining({ foo: 'bar' }))
-    })
-  })
-})
-
-describe('list', () => {
-  let albert: Entry
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream0 = createReadStream('albert.jpg')
-    const stream1 = createReadStream('lenna.png')
-
-    albert = (await storage.put(dir, stream0)) as Entry
-    lenna = (await storage.put(dir, stream1)) as Entry
-  })
-
-  it('should list entries', async () => {
-    const list = await storage.list(dir)
-
-    expect(list).toEqual([albert.id, lenna.id])
-  })
-})
-
-describe('annotate', () => {
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should set meta', async () => {
-    const path = `${dir}/${lenna.id}`
-
-    await storage.annotate(path, 'foo', 'bar')
-
-    const state0 = (await storage.get(path)) as Entry
-
-    expect(state0).toHaveProperty('meta.foo', 'bar')
-
-    await storage.annotate(path, 'foo')
-
-    const state1 = (await storage.get(path)) as Entry
-
-    expect(state1.meta).not.toHaveProperty('foo')
-  })
-})
-
-describe('variants', () => {
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should add variant', async () => {
-    const stream = createReadStream('sample.jpeg')
-
-    const path = `${dir}/${lenna.id}`
-
-    await storage.diversify(path, 'foo', stream)
-
-    const state = (await storage.get(path)) as Entry
-
-    expect(state).toHaveProperty('variants',
-      expect.arrayContaining([{ name: 'foo', size: 73444, type: 'image/jpeg' }]))
-  })
-
-  it('should replace variant', async () => {
-    const stream0 = createReadStream('sample.jpeg')
-    const stream1 = createReadStream('sample.webp')
-    const path = `${dir}/${lenna.id}`
-
-    await storage.diversify(path, 'foo', stream0)
-    await storage.diversify(path, 'foo', stream1)
-
-    const state = (await storage.get(path)) as Entry
-
-    expect(state).toHaveProperty('variants',
-      expect.arrayContaining([expect.objectContaining({ name: 'foo', type: 'image/webp' })]))
-  })
-})
-
-describe('fetch', () => {
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should fetch', async () => {
-    const path = `${dir}/${lenna.id}`
-    const stream = await storage.fetch(path)
-
-    assert.ok(stream instanceof Readable)
-
-    const stored = await buffer(stream)
-    const buf = await buffer(createReadStream('lenna.png'))
-
-    expect(stored.compare(buf)).toBe(0)
-  })
-
-  it('should fetch blob by id', async () => {
-    const stream = createReadStream('lenna.ascii')
-    const entry = (await storage.put(dir, stream)) as Entry
-    const stored = await storage.fetch(entry.id)
-
-    if (stored instanceof Error) throw stored
-
-    const buf = await buffer(stored)
-    const expected = await buffer(createReadStream('lenna.ascii'))
-
-    expect(buf.compare(expected)).toBe(0)
-  })
-
-  it('should fetch variant', async () => {
-    const stream = createReadStream('sample.jpeg')
-
-    const buf = await buffer(stream)
-    const path = `${dir}/${lenna.id}`
-
-    await storage.diversify(path, '100x100.jpeg', Readable.from(buf))
-
-    const variant = await storage.fetch(`${path}.100x100.jpeg`)
-
-    assert.ok(variant instanceof Readable)
-
-    const stored = await buffer(variant)
-
-    expect(stored.compare(buf)).toBe(0)
-  })
-
-  it('should not fetch blob by id and fake path', async () => {
-    const stored = await storage.fetch(`fake/${lenna.id}`)
-
-    expect(stored).toBeInstanceOf(Error)
-    expect(stored).toHaveProperty('code', 'NOT_FOUND')
-  })
-})
-
-describe('delete', () => {
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should remove from the list', async () => {
-    await storage.delete(`${dir}/${lenna.id}`)
-
-    const list = await storage.list(dir)
-
-    expect(list).not.toContain(lenna.id)
-  })
-
-  it('should delete entry', async () => {
-    await storage.delete(`${dir}/${lenna.id}`)
-
-    const result = await storage.get(`${dir}/${lenna.id}`)
-
-    expect(result).toBeInstanceOf(Error)
-    expect(result).toHaveProperty('code', 'NOT_FOUND')
-  })
-
-  it('should delete variants', async () => {
-    const stream = createReadStream('sample.jpeg')
-
-    const path = `${dir}/${lenna.id}`
-
-    await storage.diversify(path, 'foo', stream)
-    await storage.delete(`${dir}/${lenna.id}`)
-    stream.destroy()
-
-    const variant = await storage.fetch(`${path}.foo`)
-
-    expect(variant).toBeInstanceOf(Error)
-    expect(variant).toHaveProperty('code', 'NOT_FOUND')
-  })
-
-  it('should throw if path is not an entry', async () => {
-    const result = await storage.delete(dir)
-
-    expect(result).toBeInstanceOf(Error)
-    expect(result).toMatchObject({ code: 'NOT_FOUND' })
-  })
-})
-
-describe('move', () => {
-  let lenna: Entry
-
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
-
-    lenna = (await storage.put(dir, stream)) as Entry
-  })
-
-  it('should move entry', async () => {
-    const path = `${dir}/${lenna.id}`
-    const to = `${dir}/lenna`
-
-    await storage.move(path, to)
-
-    const entry = await storage.get(to)
-
-    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
-  })
-
-  it('should move to subdirectory', async () => {
-    const path = `${dir}/${lenna.id}`
-    const to = `${dir}/sub/`
-
-    await storage.move(path, to)
-
-    const entry = await storage.get(to + lenna.id)
-
-    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
-  })
-
-  it('should move to relative path', async () => {
-    const path = `${dir}/${lenna.id}`
-    const to = './sub/'
-
-    await storage.move(path, to)
-
-    const entry = await storage.get(`${dir}/sub/${lenna.id}`)
-
-    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
-  })
-
-  it('should move variants', async () => {
-    const stream = createReadStream('sample.jpeg')
-
-    const path = `${dir}/${lenna.id}`
-
-    await storage.diversify(path, 'foo', stream)
-    await storage.move(path, `${dir}/lenna`)
-
-    const variant = await storage.fetch(`${dir}/lenna.foo`)
-
-    assert.ok(variant instanceof Readable)
-  })
-})
-
-describe('signatures', () => {
-  it.each(['jpeg', 'gif', 'webp', 'heic', 'jxl', 'avif'])('should detect image/%s',
-    async (type) => {
-      const stream = createReadStream('sample.' + type)
-
-      const entry = (await storage.put(dir, stream)) as Entry
-
-      expect(entry).toHaveProperty('type', 'image/' + type)
-    })
-  it.each(['avi'])('should detect video/%s',
-    async (type) => {
-      const stream = createReadStream('sample.' + type)
-
-      const entry = (await storage.put(dir, stream)) as Entry
-
-      expect(entry).toHaveProperty('type', 'video/' + type)
-    })
-  it.each(['wav'])('should detect audio/%s',
-    async (type) => {
-      const stream = createReadStream('sample.' + type)
-
-      const entry = (await storage.put(dir, stream)) as Entry
-
-      expect(entry).toHaveProperty('type', 'audio/' + type)
-    })
-})
-
-it('should return error if type doesn\'t match', async () => {
-  const stream = createReadStream('sample.jpeg')
-
-  const result = await storage.put(dir, stream, { claim: 'image/png' })
-
-  expect(result).toBeInstanceOf(Error)
-  expect(result).toHaveProperty('code', 'TYPE_MISMATCH')
-})
-
-it('should trust unknown types', async () => {
-  const stream = createReadStream('lenna.ascii')
-
-  const result = await storage.put(dir, stream, { claim: 'text/plain' })
-
-  expect(result).not.toBeInstanceOf(Error)
-  expect(result).toHaveProperty('type', 'text/plain')
-})
-
-it('should return error if type is identifiable', async () => {
-  const stream = createReadStream('lenna.ascii')
-
-  const result = await storage.put(dir, stream, { claim: 'image/jpeg' })
-
-  expect(result).toBeInstanceOf(Error)
-  expect(result).toHaveProperty('code', 'TYPE_MISMATCH')
-})
-
-it('should not return error if type application/octet-stream', async () => {
-  const stream = createReadStream('sample.jpeg')
-
-  const result = await storage.put(dir, stream, { claim: 'application/octet-stream' })
-
-  expect(result).not.toBeInstanceOf(Error)
-  expect(result).toHaveProperty('type', 'image/jpeg')
-})
-
-it('should return error if type is not acceptable', async () => {
-  const stream = createReadStream('sample.jpeg')
-
-  const result = await storage.put(dir, stream, { accept: 'image/png' })
-
-  expect(result).toBeInstanceOf(Error)
-  expect(result).toHaveProperty('code', 'NOT_ACCEPTABLE')
-})
-
-it('should accept wildcard types', async () => {
-  const stream = createReadStream('sample.jpeg')
-
-  const result = await storage.put(dir, stream, { accept: 'image/*' })
-
-  expect(result).not.toBeInstanceOf(Error)
-  expect(result).toHaveProperty('type', 'image/jpeg')
-})
-
-it('should handle root entries', async () => {
-  const stream = createReadStream('sample.jpeg')
-
-  const result = (await storage.put('hello', stream)) as Entry
-
-  expect(result).not.toBeInstanceOf(Error)
-
-  const stored = await storage.fetch(result.id)
-
-  expect(stored).not.toBeInstanceOf(Error)
-})
-
-it('should store empty file', async () => {
-  const stream = createReadStream('empty.txt')
-  const result = (await storage.put('empty', stream)) as Entry
-
-  expect(result).toHaveProperty('size', 0)
-
-  const stored = (await storage.fetch(result.id)) as Readable
-
-  expect(stored).not.toBeInstanceOf(Error)
-
-  const buf = await buffer(stored)
-
-  expect(buf).toHaveLength(0)
-})

package/source/Storage.ts

@@ -1,235 +0,0 @@
-import { Readable } from 'node:stream'
-import { posix } from 'node:path'
-import { buffer } from 'node:stream/consumers'
-import { decode, encode } from 'msgpackr'
-import { newid } from '@toa.io/generic'
-import { Err } from 'error-value'
-import { Scanner } from './Scanner'
-import type { ScanOptions } from './Scanner'
-import type { Provider } from './Provider'
-import type { Entry } from './Entry'
-
-export class Storage {
-  private readonly provider: Provider
-
-  public constructor (provider: Provider) {
-    this.provider = provider
-  }
-
-  public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
-    const scanner = new Scanner(options)
-    const pipe = stream.pipe(scanner)
-    const tempname = await this.transit(pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const id = scanner.digest()
-
-    await this.persist(tempname, id)
-
-    return await this.create(path, id, scanner.size, scanner.type, options?.meta)
-  }
-
-  public async get (path: string): Maybe<Entry> {
-    const paths = this.destruct(path)
-    const result = await this.provider.get(paths.metafile)
-
-    if (result === null)
-      return ERR_NOT_FOUND
-    else
-      return decode(await buffer(result))
-  }
-
-  public async list (path: string): Promise<string[]> {
-    const dir = posix.join(ENTRIES_ROOT, path, ENTRIES_DIR)
-
-    return await this.provider.list(dir)
-  }
-
-  public async fetch (path: string): Maybe<Readable> {
-    const { rel, id, variant } = this.parse(path)
-
-    if (variant === null && rel !== '') {
-      const entry = await this.get(path)
-
-      if (entry instanceof Error)
-        return entry
-    }
-
-    const blob = variant === null
-      ? posix.join(BLOBs, id)
-      : posix.join(ENTRIES_ROOT, rel, id, variant)
-
-    const stream = await this.provider.get(blob)
-
-    if (stream === null)
-      return ERR_NOT_FOUND
-    else
-      return stream
-  }
-
-  public async delete (path: string): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    const paths = this.destruct(path)
-
-    await Promise.all([
-      this.provider.delete(paths.metafile),
-      this.provider.delete(paths.vardir)
-    ])
-  }
-
-  public async move (path: string, to: string): Maybe<void> {
-    const source = this.destruct(path)
-    const rel = to.startsWith('.')
-    const dir = to.endsWith('/')
-
-    if (rel)
-      to = posix.resolve(source.rel + '/', to)
-
-    if (dir)
-      to = posix.join(to, source.ent)
-
-    const target = this.destruct(to)
-
-    await Promise.all([
-      this.provider.move(source.metafile, target.metafile),
-      this.provider.moveDir(source.vardir, target.vardir)
-    ])
-  }
-
-  public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
-    const scanner = new Scanner()
-    const pipe = stream.pipe(scanner)
-
-    await this.provider.put(posix.join(ENTRIES_ROOT, path), name, pipe)
-
-    if (scanner.error !== null)
-      return scanner.error
-
-    const { size, type } = scanner
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    entry.variants = entry.variants.filter((variant) => variant.name !== name)
-    entry.variants.push({ name, size, type })
-
-    await this.save(path, entry)
-  }
-
-  public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
-    const entry = await this.get(path)
-
-    if (entry instanceof Error)
-      return entry
-
-    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-    if (value === undefined)
-      delete entry.meta[key]
-    else
-      entry.meta[key] = value
-
-    await this.save(path, entry)
-  }
-
-  private async transit (stream: Readable): Promise<string> {
-    const tempname = newid()
-
-    await this.provider.put(TEMP, tempname, stream)
-
-    return tempname
-  }
-
-  private async persist (tempname: string, id: string): Promise<void> {
-    const temp = posix.join(TEMP, tempname)
-    const blob = posix.join(BLOBs, id)
-
-    await this.provider.move(temp, blob)
-  }
-
-  // eslint-disable-next-line max-params
-  private async create
-  (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
-    const entry: Entry = {
-      id,
-      size,
-      type,
-      created: Date.now(),
-      variants: [],
-      meta
-    }
-
-    const metafile = posix.join(path, entry.id)
-    const existing = await this.get(metafile)
-
-    if (existing instanceof Error)
-      await this.save(metafile, entry)
-
-    return entry
-  }
-
-  private async save (path: string, entry: Entry): Promise<void> {
-    const paths = this.destruct(path)
-    const stream = Readable.from(encode(entry))
-
-    await this.provider.put(paths.metadir, paths.ent, stream)
-  }
-
-  private parse (path: string): Path {
-    const [last, ...segments] = path.split('/').reverse()
-    const [id, ...rest] = last.split('.')
-    const variant = rest.length > 0 ? rest.join('.') : null
-    const rel = segments.reverse().join('/')
-
-    return { rel, id, variant }
-  }
-
-  private destruct (path: string): Paths {
-    const rel = posix.dirname(path)
-    const dir = posix.join(ENTRIES_ROOT, rel)
-    const ent = posix.basename(path)
-    const metadir = posix.join(dir, ENTRIES_DIR)
-    const metafile = posix.join(metadir, ent)
-    const vardir = posix.join(dir, ent)
-
-    return { rel, dir, ent, metadir, metafile, vardir }
-  }
-}
-
-const ERR_NOT_FOUND = Err('NOT_FOUND')
-
-const TEMP = '/temp'
-const BLOBs = '/blobs'
-const ENTRIES_ROOT = '/entries'
-const ENTRIES_DIR = '.meta'
-
-type Maybe<T> = Promise<T | Error>
-
-interface Path {
-  rel: string
-  id: string
-  variant: string | null
-}
-
-interface Paths {
-  rel: string
-  dir: string
-  ent: string
-  metadir: string
-  metafile: string
-  vardir: string
-}
-
-type Meta = Record<string, string>
-
-interface Options extends ScanOptions {
-  meta?: Meta
-}
-
-export type Storages = Record<string, Storage>