@toa.io/extensions.storages 1.0.0-alpha.7 → 1.0.0-alpha.78

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "1.0.0-alpha.7",
+  "version": "1.0.0-alpha.78",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -24,20 +24,20 @@
     "test": "jest",
     "transpile": "npx tsc"
   },
-  "devDependencies": {
-    "@toa.io/streams": "1.0.0-alpha.7",
-    "dotenv": "16.3.1"
+  "peerDependencies": {
+    "dotenv": "*"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.481.0",
     "@aws-sdk/lib-storage": "3.481.0",
-    "@toa.io/agent": "1.0.0-alpha.7",
-    "@toa.io/generic": "1.0.0-alpha.7",
-    "@toa.io/schemas": "1.0.0-alpha.7",
+    "@toa.io/agent": "1.0.0-alpha.71",
+    "@toa.io/generic": "1.0.0-alpha.63",
+    "@toa.io/schemas": "1.0.0-alpha.63",
     "error-value": "0.3.0",
     "matchacho": "0.6.0",
     "msgpackr": "1.10.1",
+    "openspan": "1.0.0-alpha.73",
     "smithy-node-native-fetch": "1.0.6"
   },
-  "gitHead": "4f5ac0bc342d4b7bd469fbe5c74266f050b55c9f"
+  "gitHead": "8936ef7a76c95d83532ee7115ad2d698c3e2eac4"
 }

package/readme.md

@@ -9,6 +9,7 @@ BLOBs are stored with the meta-information object (Entry) having the following p
 - `id` - checksum
 - `size` - size in bytes
 - `type` - MIME type
+- `origin` - URL of the original BLOB (optional)
 - `created` - creation timestamp (UNIX time, ms)
 - `variants` - array of:
   - `name` - unique name
@@ -75,8 +76,6 @@ are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/
 
 See [source](source/Scanner.ts).
 
-If the entry already exists, it is returned and [revealed](#async-revealpath-string-maybevoid).
-
 #### `async get(path: string): Maybe<Entry>`
 
 Get an entry.
@@ -97,28 +96,28 @@ Fetch the BLOB specified by `path`. If the path does not exist, a `NOT_FOUND` er
 
 Delete the entry specified by `path`.
 
-#### `async list(path: string): string[]`
-
-Get ordered list of `id`s of entries in under the `path`.
+#### `async move(path: string, to: string): Maybe<void>`
 
-#### `async permute(path: string, ids: string[]): Maybe<void>`
+Moves the entry specified by `path` to the new path.
 
-Reorder entries under the `path`.
+`to` may be an absolute or relative path (starting with `.`), if it ends with `/`, the entry is
+moved to the `to` directory, otherwise, the entry is moved to the `to` path.
 
-Given list must be a permutation of the current list, otherwise a `PERMUTATION_MISMATCH` error is
-returned.
-
-#### `async diversify(path: string, name: string, stream: Readable): Maybe<void>`
+The following examples are equivalent:
 
-Add or replace a `name` variant of the entry specified by `path`.
+```javascript
+await storage.move('/path/to/eecd837c', '/path/to/sub/eecd837c')
+await storage.move('/path/to/eecd837c', './sub/eecd837c')
+await storage.move('/path/to/eecd837c', './sub/')
+```
 
-#### `async conceal(path: string): Maybe<void>`
+#### `async entries(path: string): Entry[]`
 
-Remove the entry from the list.
+Get a list of entries under the `path`.
 
-#### `async reveal(path: string): Maybe<void>`
+#### `async diversify(path: string, name: string, stream: Readable): Maybe<void>`
 
-Restore the entry to the list.
+Add or replace a `name` variant of the entry specified by `path`.
 
 #### `async annotate(path: string, key: string, value: any): Maybe<void>`
 
@@ -132,7 +131,7 @@ Custom providers are not supported.
 
 ### Amazon S3
 
-Annotation formats is like:
+Annotation formats are like:
 
 ```yaml
 storages:
@@ -160,9 +159,20 @@ Annotation format is:
 
 ```yaml
 storages:
-  photos@dev:
+  photos:
     provider: fs
-    path: /var/my-storage
+    path: /var/my-photos
+```
+
+[Kubernetes PVC](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) can be mounted to
+the storage:
+
+```yaml
+storages:
+  photos:
+    provider: fs
+    path: /var/my-photos
+    claim: photos-pvc
 ```
 
 ### Temporary
@@ -205,10 +215,10 @@ Underlying directory structure:
   b4f577e0            # checksum
 /storage
   /path/to
-    .list             # list of entries
+    /.entries/
+      b4f577e0        # entry
     /b4f577e0
-      .meta           # entry
-      thumbnail.jpeg  # variant BLOBs
+      thumbnail.jpeg  # variant
       thumbnail.webp
 ```
 

package/schemas/fs.cos.yaml

@@ -5,5 +5,7 @@ properties:
   path:
     type: string
     maxLength: 4096
+  claim:
+    type: string
 required: [provider, path]
 additionalProperties: false

package/schemas/s3.cos.yaml

@@ -14,3 +14,4 @@ properties:
     type: string
     format: uri
 required: [provider, bucket]
+additionalProperties: false

package/schemas/tmp.cos.yaml

@@ -1,7 +1,6 @@
 type: object
 properties:
   provider:
-    type: string
     const: tmp
   directory:
     type: string

package/source/Entry.ts

@@ -3,6 +3,7 @@ export interface Entry {
   size: number
   type: string
   created: number
+  origin?: string
   variants: Variant[]
   meta: Record<string, unknown>
 }

package/source/Factory.ts

@@ -1,9 +1,10 @@
 import assert from 'node:assert'
+import { console } from 'openspan'
 import { decode } from '@toa.io/generic'
 import { providers } from './providers'
 import { Storage, type Storages } from './Storage'
 import { Aspect } from './Aspect'
-import { SERIALIZATION_PREFIX } from './deployment'
+import { ENV_PREFIX } from './deployment'
 import { validateAnnotation } from './Annotation'
 import type { Declaration } from './providers'
 import type { Annotation } from './Annotation'
@@ -13,9 +14,9 @@ export class Factory {
   private readonly annotation: Annotation
 
   public constructor () {
-    const env = process.env.TOA_STORAGES
+    const env = process.env[ENV_PREFIX]
 
-    assert.ok(env !== undefined, 'TOA_STORAGES is not defined')
+    assert.ok(env !== undefined, `${ENV_PREFIX} is not defined`)
 
     this.annotation = decode(env)
 
@@ -43,6 +44,8 @@ export class Factory {
     const secrets = this.resolveSecrets(name, Provider)
     const provider = new Provider(options, secrets)
 
+    console.debug('Storage created', { name, provider: providerId, options, path: provider.path })
+
     return new Storage(provider)
   }
 
@@ -54,7 +57,7 @@ export class Factory {
     const secrets: Record<string, string | undefined> = {}
 
     for (const secret of Class.SECRETS) {
-      const variable = `${SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
+      const variable = `${ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
       const value = process.env[variable]
 
       assert.ok(secret.optional === true || value !== undefined,

package/source/Provider.ts

@@ -10,6 +10,7 @@ export interface ProviderSecret {
 
 export abstract class Provider<Options = void> {
   public static readonly SECRETS: readonly ProviderSecret[] = []
+  public readonly path: string | null = null
 
   public constructor (_: Options, secrets?: ProviderSecrets) {
     for (const { name, optional = false } of new.target.SECRETS)
@@ -18,11 +19,15 @@ export abstract class Provider<Options = void> {
 
   public abstract get (path: string): Promise<Readable | null>
 
+  public abstract list (path: string): Promise<string[]>
+
   public abstract put (path: string, filename: string, stream: Readable): Promise<void>
 
   public abstract delete (path: string): Promise<void>
 
   public abstract move (from: string, to: string): Promise<void>
+
+  public abstract moveDir (from: string, to: string): Promise<void>
 }
 
 export interface ProviderConstructor {

package/source/Scanner.ts

@@ -11,8 +11,9 @@ export class Scanner extends PassThrough {
   private readonly hash = createHash('md5')
   private readonly claim?: string
   private readonly accept?: string
+  private readonly limit?: number
   private position = 0
-  private detected = false
+  private completed = false
   private readonly chunks: Buffer[] = []
 
   public constructor (control?: ScanOptions) {
@@ -20,6 +21,7 @@ export class Scanner extends PassThrough {
 
     this.claim = control?.claim
     this.accept = control?.accept
+    this.limit = control?.limit
   }
 
   public digest (): string {
@@ -37,7 +39,7 @@ export class Scanner extends PassThrough {
     this.size += buffer.length
     this.hash.update(buffer)
 
-    if (this.detected)
+    if (this.completed)
       return
 
     if (this.position + buffer.length > HEADER_SIZE)
@@ -48,13 +50,22 @@ export class Scanner extends PassThrough {
 
     if (this.position === HEADER_SIZE)
       this.complete()
+
+    if (this.limit !== undefined && this.size > this.limit)
+      this.interrupt(ERR_LIMIT_EXCEEDED)
   }
 
   private complete (): void {
     const header = Buffer.concat(this.chunks).toString('hex')
 
     const signature = SIGNATURES
-      .find(({ hex, off }) => header.slice(off, off + hex.length) === hex)
+      .find(({ off, hex, expression }) => {
+        const sig = header.slice(off, off + hex.length)
+
+        return expression === undefined
+          ? sig === hex
+          : expression.test(sig)
+      })
 
     const type = signature?.type ?? this.claim
 
@@ -64,7 +75,7 @@ export class Scanner extends PassThrough {
     }
 
     this.verify(signature)
-    this.detected = true
+    this.completed = true
   }
 
   private verify (signature: Signature | undefined): void {
@@ -90,6 +101,7 @@ export class Scanner extends PassThrough {
   }
 
   private interrupt (error: Error): void {
+    this.completed = true
     this.error = error
     this.end()
   }
@@ -97,21 +109,35 @@ export class Scanner extends PassThrough {
 
 // https://en.wikipedia.org/wiki/List_of_file_signatures
 const SIGNATURES: Signature[] = [
-  { hex: 'ffd8ffe0', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffe1', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffee', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffdb', off: 0, type: 'image/jpeg' },
-  { hex: '89504e47', off: 0, type: 'image/png' },
-  { hex: '47494638', off: 0, type: 'image/gif' },
-  { hex: '52494646', off: 0, type: 'image/webp' },
-  { hex: '4a584c200d0a870a', off: 8, type: 'image/jxl' },
-  { hex: '6674797068656963', off: 8, type: 'image/heic' },
-  { hex: '6674797061766966', off: 8, type: 'image/avif' }
+  { hex: 'ff d8 ff e0', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff e1', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff e2', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff ee', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff db', off: 0, type: 'image/jpeg' },
+  { hex: '89 50 4e 47', off: 0, type: 'image/png' },
+  { hex: '47 49 46 38', off: 0, type: 'image/gif' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 57 45 42 50', off: 0, type: 'image/webp' },
+  { hex: '4a 58 4c 20 0d 0a 87 0a', off: 8, type: 'image/jxl' },
+  { hex: '66 74 79 70 68 65 69 63', off: 8, type: 'image/heic' },
+  { hex: '66 74 79 70 61 76 69 66', off: 8, type: 'image/avif' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 41 56 49 20', off: 0, type: 'video/avi' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 57 41 56 45', off: 0, type: 'audio/wav' }
   /*
   When adding a new signature, include a copyright-free sample file in the `.tests` directory
   and update the 'signatures' test group in `Storage.test.ts`.
    */
-]
+].map((signature: Signature) => {
+  signature.hex = signature.hex.replaceAll(' ', '')
+
+  if (signature.hex.includes('??')) {
+    const expression = signature.hex.replaceAll(/(?<wildcards>\?{1,24})/g,
+      (_, wildcards) => `[0-9a-f]{${wildcards.length}}`)
+
+    signature.expression = new RegExp(expression, 'i')
+  }
+
+  return signature
+})
 
 const HEADER_SIZE = SIGNATURES
   .reduce((max, { off, hex }) => Math.max(max, off + hex.length), 0) / 2
@@ -120,14 +146,17 @@ const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
 
 const ERR_TYPE_MISMATCH = Err('TYPE_MISMATCH')
 const ERR_NOT_ACCEPTABLE = Err('NOT_ACCEPTABLE')
+const ERR_LIMIT_EXCEEDED = Err('LIMIT_EXCEEDED')
 
 export interface ScanOptions {
   claim?: string
   accept?: string
+  limit?: number
 }
 
 interface Signature {
-  hex: string
-  off: number
   type: string
+  off: number
+  hex: string
+  expression?: RegExp
 }

package/source/Storage.test.ts

@@ -13,7 +13,7 @@ import type { ProviderConstructor } from './Provider'
 let storage: Storage
 let dir: string
 
-const suite = suites[0]
+const suite = suites[0] // replace with 2 to run tests for S3
 
 beforeAll(async () => {
   process.chdir(path.join(__dirname, 'test'))
@@ -103,18 +103,6 @@ describe('put', () => {
   })
 
   describe('existing entry', () => {
-    it('should unhide existing', async () => {
-      const stream = createReadStream('lenna.png')
-      const path = `${dir}/${lenna.id}`
-
-      await storage.conceal(path)
-      await storage.put(dir, stream)
-
-      const list = await storage.list(dir)
-
-      expect(list).toContainEqual(lenna.id)
-    })
-
     it('should preserve meta', async () => {
       const path = `${dir}/${lenna.id}`
       const stream = createReadStream('lenna.png')
@@ -146,86 +134,41 @@ describe('list', () => {
 
     expect(list).toEqual([albert.id, lenna.id])
   })
+})
 
-  it('should permutate', async () => {
-    const error = await storage.permute(dir, [lenna.id, albert.id])
-
-    expect(error).toBeUndefined()
-
-    const list = await storage.list(dir)
-
-    expect(list).toEqual([lenna.id, albert.id])
-  })
-
-  it('should return PERMUTATION_MISMATCH', async () => {
-    const cases = [
-      [lenna.id],
-      [albert.id, lenna.id, 'unknown'],
-      [lenna.id, lenna.id],
-      [lenna.id, lenna.id, albert.id]
-    ]
-
-    for (const permutation of cases) {
-      const error = await storage.permute(dir, permutation)
-
-      expect(error).toBeInstanceOf(Error)
-      expect(error).toHaveProperty('code', 'PERMUTATION_MISMATCH')
-    }
-  })
-
-  it('should exclude concealed', async () => {
-    const path = `${dir}/${lenna.id}`
-
-    await storage.conceal(path)
+describe('annotate', () => {
+  let lenna: Entry
 
-    const entries = await storage.list(dir)
+  beforeEach(async () => {
+    const stream = createReadStream('lenna.png')
 
-    expect(entries).toEqual([albert.id])
+    lenna = (await storage.put(dir, stream)) as Entry
   })
 
-  it('should reveal', async () => {
+  it('should set meta property', async () => {
     const path = `${dir}/${lenna.id}`
 
-    await storage.conceal(path)
-    await storage.reveal(path)
-    await storage.reveal(path) // test that no duplicates are created
-
-    const entries = await storage.list(dir)
-
-    expect(entries).toEqual([albert.id, lenna.id])
-  })
-
-  it('should return ERR_NOT_FOOUD if entry doesnt exist', async () => {
-    const path = `${dir}/oopsie`
-
-    const methods: Array<'reveal' | 'conceal'> = ['reveal', 'conceal']
+    await storage.annotate(path, 'foo', 'bar')
 
-    for (const method of methods) {
-      const error = await storage[method](path)
+    const state0 = (await storage.get(path)) as Entry
 
-      expect(error).toBeInstanceOf(Error)
-      expect(error).toHaveProperty('code', 'NOT_FOUND')
-    }
-  })
-})
+    expect(state0).toHaveProperty('meta.foo', 'bar')
 
-describe('annotate', () => {
-  let lenna: Entry
+    await storage.annotate(path, 'foo')
 
-  beforeEach(async () => {
-    const stream = createReadStream('lenna.png')
+    const state1 = (await storage.get(path)) as Entry
 
-    lenna = (await storage.put(dir, stream)) as Entry
+    expect(state1.meta).not.toHaveProperty('foo')
   })
 
-  it('should set meta', async () => {
+  it('should set meta with object', async () => {
     const path = `${dir}/${lenna.id}`
 
-    await storage.annotate(path, 'foo', 'bar')
+    await storage.annotate(path, { foo: 1, bar: 'baz' })
 
     const state0 = (await storage.get(path)) as Entry
 
-    expect(state0).toHaveProperty('meta.foo', 'bar')
+    expect(state0.meta).toStrictEqual({ foo: 1, bar: 'baz' })
 
     await storage.annotate(path, 'foo')
 
@@ -380,6 +323,62 @@ describe('delete', () => {
   })
 })
 
+describe('move', () => {
+  let lenna: Entry
+
+  beforeEach(async () => {
+    const stream = createReadStream('lenna.png')
+
+    lenna = (await storage.put(dir, stream)) as Entry
+  })
+
+  it('should move entry', async () => {
+    const path = `${dir}/${lenna.id}`
+    const to = `${dir}/lenna`
+
+    await storage.move(path, to)
+
+    const entry = await storage.get(to)
+
+    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
+  })
+
+  it('should move to subdirectory', async () => {
+    const path = `${dir}/${lenna.id}`
+    const to = `${dir}/sub/`
+
+    await storage.move(path, to)
+
+    const entry = await storage.get(to + lenna.id)
+
+    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
+  })
+
+  it('should move to relative path', async () => {
+    const path = `${dir}/${lenna.id}`
+    const to = './sub/'
+
+    await storage.move(path, to)
+
+    const entry = await storage.get(`${dir}/sub/${lenna.id}`)
+
+    expect(entry).toMatchObject({ id: lenna.id, type: 'image/png' })
+  })
+
+  it('should move variants', async () => {
+    const stream = createReadStream('sample.jpeg')
+
+    const path = `${dir}/${lenna.id}`
+
+    await storage.diversify(path, 'foo', stream)
+    await storage.move(path, `${dir}/lenna`)
+
+    const variant = await storage.fetch(`${dir}/lenna.foo`)
+
+    assert.ok(variant instanceof Readable)
+  })
+})
+
 describe('signatures', () => {
   it.each(['jpeg', 'gif', 'webp', 'heic', 'jxl', 'avif'])('should detect image/%s',
     async (type) => {
@@ -389,9 +388,33 @@ describe('signatures', () => {
 
       expect(entry).toHaveProperty('type', 'image/' + type)
     })
+
+  it.each(['avi'])('should detect video/%s',
+    async (type) => {
+      const stream = createReadStream('sample.' + type)
+
+      const entry = (await storage.put(dir, stream)) as Entry
+
+      expect(entry).toHaveProperty('type', 'video/' + type)
+    })
+
+  it.each(['wav'])('should detect audio/%s',
+    async (type) => {
+      const stream = createReadStream('sample.' + type)
+      const entry = (await storage.put(dir, stream)) as Entry
+
+      expect(entry).toHaveProperty('type', 'audio/' + type)
+    })
+
+  it('should be ok with Arny', async () => {
+    const stream = createReadStream('arny.jpg')
+    const entry = (await storage.put(dir, stream)) as Entry
+
+    expect(entry).toHaveProperty('type', 'image/jpeg')
+  })
 })
 
-it("should return error if type doesn't match", async () => {
+it('should return error if type doesn\'t match', async () => {
   const stream = createReadStream('sample.jpeg')
 
   const result = await storage.put(dir, stream, { claim: 'image/png' })
@@ -447,7 +470,6 @@ it('should accept wildcard types', async () => {
 
 it('should handle root entries', async () => {
   const stream = createReadStream('sample.jpeg')
-
   const result = (await storage.put('hello', stream)) as Entry
 
   expect(result).not.toBeInstanceOf(Error)
@@ -471,3 +493,31 @@ it('should store empty file', async () => {
 
   expect(buf).toHaveLength(0)
 })
+
+it('should return error of stream size limit exceeded', async () => {
+  const stream = createReadStream('sample.jpeg')
+  const result = await storage.put(dir, stream, { limit: 1024 })
+
+  expect(result).toBeInstanceOf(Error)
+  expect(result).toHaveProperty('code', 'LIMIT_EXCEEDED')
+})
+
+it('should set origin', async () => {
+  const origin = 'https://example.com/image.jpeg'
+  const stream = createReadStream('sample.jpeg')
+  const result = (await storage.put('/origins', stream, { origin })) as Entry
+
+  assert.ok(!(result instanceof Error))
+
+  const stored = await storage.get('/origins/' + result.id)
+
+  assert.ok(!(stored instanceof Error))
+
+  expect(stored.origin).toBe(origin)
+})
+
+it('should expose path', () => {
+  const path = storage.path()
+
+  expect(typeof path === 'string' || path === null)
+})