@toa.io/extensions.storages 0.20.0-alpha.3

package/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020-present Artem Gurtovoi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@toa.io/extensions.storages",
+  "version": "0.20.0-alpha.3",
+  "description": "Toa Storages",
+  "author": "temich <tema.gurtovoy@gmail.com>",
+  "homepage": "https://github.com/toa-io/toa#readme",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toa-io/toa.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toa-io/toa/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
+  },
+  "scripts": {
+    "test": "jest",
+    "transpile": "rm -rf transpiled && npx tsc"
+  },
+  "devDependencies": {
+    "@toa.io/match": "0.3.0",
+    "@toa.io/streams": "0.1.0-alpha.2",
+    "@types/fs-extra": "11.0.3"
+  },
+  "dependencies": {
+    "@toa.io/generic": "0.20.0-alpha.2",
+    "fs-extra": "11.1.1",
+    "msgpackr": "1.9.9"
+  },
+  "gitHead": "ed28dc0d2823022fbb1188bb28b994bc827a4432"
+}

package/readme.md

@@ -0,0 +1,195 @@
+# Toa Storages
+
+Shared BLOB storage.
+
+## Entry
+
+BLOBs are stored with the meta-information object (Entry) having the following properties:
+
+- `id` - checksum
+- `size` - size in bytes
+- `type` - MIME type
+- `created` - creation timestamp (UNIX time, ms)
+- `variants` - array of:
+  - `name` - unique name
+  - `size` - size in bytes
+  - `type` - variant MIME type
+- `meta` - object with application-specific information, empty by default
+
+### Example
+
+```yaml
+id: eecd837c
+type: image/jpeg
+created: 1698004822358
+variants:
+  - name: thumbnail.jpeg
+    type: image/jpeg
+  - name: thumbnail.webp
+    type: image/webp
+meta:
+  face: true
+  nudity: false
+```
+
+## Aspect
+
+The Storages extension provides `storages` aspect,
+containing named Storage instances, according to the annotation.
+
+```javascript
+async function effect (_, context) {
+  await context.storages.photos.fetch('/path/to/b4f577e0.thumbnail.jpeg')
+}
+```
+
+### Storage interface
+
+> `Maybe<T> = T | Error`
+
+#### `async put(path: string, stream: Readable, type?: string): Maybe<Entry>`
+
+Add a BLOB to the storage and create an entry under specified `path`.
+
+BLOB type is identified
+using [magick numbers](https://github.com/sindresorhus/file-type).
+If the `type` argument is specified and does not match the BLOB type, then a `TYPE_MISMATCH` error
+is returned.
+If the BLOB type cannot be identified
+and the value of `type` is not in the list of known types, then the given value is used.
+
+Known types
+are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/jxl`, `image/avif`.
+
+See [source](source/Scanner.ts).
+
+If the entry already exists, it is returned and [revealed](#async-revealpath-string-maybevoid).
+
+#### `async get(path: string): Maybe<Entry>`
+
+Get an entry.
+
+If the entry does not exist, a `NOT_FOUND` error is returned.
+
+#### `async fetch(path: string): Maybe<Readable>`
+
+Fetch the BLOB specified by `path`. If the path does not exist, a `NOT_FOUND` error is returned.
+
+`path` can be an entry id, or a path to the entry, or a path to a variant of the entry.
+
+- `eecd837c` - fetch the BLOB by `id`
+- `/path/to/eecd837c` - fetch the BLOB by path
+- `/path/to/eecd837c.thumbnail.jpeg` - fetch the `thumbnail.jpeg` variant of the BLOB
+
+#### `async delete(path: string): Maybe<void>`
+
+Delete the entry specified by `path`.
+
+#### `async list(path: string): string[]`
+
+Get ordered list of `id`s of entries in under the `path`.
+
+#### `async reorder(path: string, ids: string[]): Maybe<void>`
+
+Reorder entries under the `path`.
+
+Given list must be a permutation of the current list, otherwise a `PERMUTATION_MISMATCH` error is
+returned.
+
+#### `async diversify(path: string, name: string, stream: Readable): Maybe<void>`
+
+Add or replace a `name` variant of the entry specified by `path`.
+
+#### `async conceal(path: string): Maybe<void>`
+
+Remove the entry from the list.
+
+#### `async reveal(path: string): Maybe<void>`
+
+Restore the entry to the list.
+
+#### `async annotate(path: string, key: string, value: any): Maybe<void>`
+
+Set a `key` property in the `meta` of the entry specified by `path`.
+
+## Providers
+
+Storage uses underlying providers to store BLOBs and entries.
+
+Custom providers are not supported yet.
+
+### Amazon S3
+
+Annotation value format is `s3://{region}/{bucket}`.
+
+Requires secrets for the access key and secret key.
+
+`s3://us-east-1/my-bucket`
+
+### Filesystem
+
+Annotation value format is `file:///{path}`.
+
+`file:///var/my-storage`
+
+### Temporary
+
+Filesystem using OS temporary directory.
+
+Annotation value format is `tmp:///{path}`.
+
+`tmp:///my-storage`
+
+## Deduplication
+
+BLOBs are stored in the underlying storage with their checksum as the key, ensuring that identical BLOBs
+are stored only once.
+Variants, on the other hand, are not deduplicated across different entries.
+
+Underlying directory structure:
+
+```
+/temp
+  c28f4dfd            # random id
+/blobs
+  b4f577e0            # checksum
+/storage
+  /path/to
+    .list             # list of entries
+    /b4f577e0
+      .meta           # entry
+      thumbnail.jpeg  # variant BLOBs
+      thumbnail.webp
+```
+
+## Manifest
+
+Storage extension can be enabled by adding `storages` key to the component manifest.
+
+```yaml
+storages: [photos, videos]
+```
+
+Value of the `storages` key is an array of storage names, that should be declared in the context.
+
+It the names are unknown, `null` declaration can be used:
+
+```yaml
+storages: ~
+```
+
+## Annotation
+
+The `storages` context annotation is an object with keys that reference the storage name and
+provider-specific URLs as values.
+
+```yaml
+storages:
+  photos: s3://us-east-1/my-bucket
+  photos@dev: file:///var/my-storage
+```
+
+## Secrets
+
+Secrets declared by storage providers can be deployed by [`toa conceal`](/runtime/cli/readme.md#conceal),
+or set locally by [`toa env`](/runtime/cli/readme.md#env).

package/source/.test/lenna.ascii

@@ -0,0 +1,64 @@
+***************...............******.*********************************.***.*.....********************@@@@*....................**
+***************..................**********************************.****.*.*......********************@@@@*................**.
+***************..................**************************************.****.......********************@@@@*..........****..
+***************.................*.***********.****.....*...*..****.**..............**********************@@@@.........**.
+***************..................*.**..**.**.****.......*....********...............**********************@@@@*.....**.
+***************......................***..*******..*...........*..**.*.***..........***********************@@@@*.....
+***************.....................*****..***.******************....**.*...........************************@@@@*..
+*****..********..........................***...*****************@@@*................*************************@@@@*
+****...********........................*.......*.****************@@@@@*.............**************************@@.
+***....********.......................*........*****************@@@@@**@@*..........*******..*****************.             .  *
+**.....********.....................*...........*************@****@@@@@@@@@.........*******...***************              . .**
+*......********...............................*.**************@@@@@@@@@@@@@@@.......*******.  ..***********.             ...****
+.......********..............................***************@@@@@@@@@@@@@@@@@@@.....*******.   ..*********.              .******
+.......********.................**..........*********..***@@@*@**@@@@@@@@@@@@@@@*...*******.     .*******               .*******
+.......********.................@.........********...*****@***@@@@@@@@@@@@@@@@@@@*..*******..    .@@@@@@              .*********
+.......********................@*........**..**...************@@@@**@@@@@@@@@@@@@@*.*******.   @@*@@@@@@*            .**********
+.......********................@*...........***.****************@@*@@@@@@@**@@@@@@@@******..*@*@@@@@@@@@*           ************
+.......********...............***.............***********************@@@***@@@@@@@**@****@@**@@@@@@@@@@@.         .*************
+.......********...............@**...........****.*************************************@@@@@@@@@@@@@@@@@@         .**************
+.......********...............@****.......****.***********************************@**@@@@@@@@@@@@@****@         .***************
+.......********...............@*****....****.**********************************@**@@@@@@@@@@@@@@*******       ..****************
+.......********...............@@*****.****..***********...****.*************@**@@***@@@@@@@@@@@***..*@        .*****************
+.......********...............@@@****.....***.***.*****.....***.....     @**@@@@@@@@@@@@@@@@@*....*@*        .******************
+.......********...............@@@*.**...***..**.**..*....   .*... ..  .**@**@*@****@@@@@@*.....**@*         .*******************
+.......********................@*****..**...*....*.....    .. ...   ***@*********@@@@* ..********          .********************
+.......********.................@@**.**..*...*.......  .  . .  .. *.*@@*************....*.***@.           .*********************
+.......********..................@@**...*..*.*......  ...      ...*************@@@@@*....***.            .**********************
+.......********..................@@......**..      .. . *.    *.****@********@@@@@@@*.. ..**             .**********************
+.......********............*.....*@.....*..   . .   . ..    *.***************@@@@@@@@*.  .**      .     .***********************
+.......********...................*..***.  .    .      . ...****************@@@@@@@@@*..  .*.   .      .************************
+.......********.................*..*.*.... .  ..   . ..  *******@*..*********@@@@@@@@@*   .*.   .     .*************************
+........*******...............***  ...... .. .  .   .. ..*****@*.*****...*******@**.....   ** ...     .*************************
+........********...........******..  ..... ...   .    ****@***.     .  .******@@*.         .*.       .**************************
+........*******............****..  . . ..*    .      ****@@*..  ... @@....***@@.....*.    ..*...     ***************************
+........********......* .***@  .... .....*.         .***@*..***...*******.***@@*****. .   ..*..     .***************************
+...... .********.......**.... .....  .......       .**@@...******************@@*****...   . *..    .***********************@@***
+...... .********.....****..  .....    ......     ..*@@.....****************.*@@******..   . **     *********************@@@@**@@
+.....  .********............. ..        . .*.   ..*@@  ...********************@@*****..     **    .*****************.**@@@@@@@@@
+.....  .********..*.*.. ... ..*** .    ...**..  .*@*   ...****************..**@@*****..  .. **.   *******************@@@@@@@@@@@
+....   .********........ .... .*..    ..*...  ..**.   .....************..**.**@@*****.   .  .**  .******************@@@@@@@@@@@@
+.....  .********.......... *. ..*.  .  ...   ****     .....************.*. ...******.        **  ******************@@@@@@@@@@@@@
+.......*********.....* .. . ....... .* . ..**.**      ......*****************@@****.    ..   *...******************@@@@@@@@@@@@@
+.......*********......... .  ... **..  ..**..*     .  .....****************@*@@****    ...   ...******************@@@@@@@@@@@@@@
+.  . ..*********..*... ....  ..........@@*..*  .   .  .....*******..*****..**.*.**     ... . .*.****************.*@@@@@@@@@@@@@@
+   . ..*********........  .. *.... ..**@*.*.        . ........******...****@@****       .. . .******************.*@@@@@@@@@@@@@@
+     ..*********..*.. .  . ...*....*.*.***          . ........*********...*.***.       .....  ******************.*@@@@@@@@@@@@@@
+    ...*********.......  .  *..*...*..*@                 .......**************..   ..  .....  *****************..@@@@@@@@@@@@@@@
+    ...*********....  .  .  *. ....**..      .          . .  ....************.     .   ...... *****************.*@@@@@@@@@@@@@@*
+     ...********.*.* . ..   .*...*..****.   .. .          .......*.**********.     .  ........**...***********..*@@@@@@@@@*.
+..   ...*******@....  .   .  ..*.**.....**  .. .           ......************@@@@*.    .......****..............@@@@@@@@**   ...
+**.... .********..    ..  ..  .........*. ...  .    .   .. .....*.************@@@@@@@.  ......*********.........*@@@@@*.  ......
+.****. .*********.    . .      * .....**..*..  .    .  ...  ....****************@@@@@@@.. .. *************...*.*@@@@@*. ........
+ .***. .*********.   .    .   . .......*** *@  .    .  .... ..*******************@@@@@@@*    ******************@@@@@*...........
+  ****  ********..        . .   .......**...*  ..   .   ... ..********************@@@@@@@*   *************..*@@@@@@@............
+  .***  *********        ..  .    . * .**..**.    .    .... ..********************@@@@@@@@. .************..*@@@@@@@*............
+  .***. .********.       ..       .  .**. ...*....    ........*********************@@@@@@@@ .***********..*@@@@@@@*.............
+  .***. .*******.     ..      . .  ......*..   .     ..... ..**********************@@@@@@@@*.***************@@@@@*..............
+   ***. .********.         .       ..  ....*        ........************************@@@@@@@@*.....************@@*...............
+   **** .*******..     . . .     . ..* ...         ...... .**************************@@@@@@@@...........*****@@@*..............
+...****..*******..                .  .*...        ..... ..***************************@@@@@@@@........    *@**@@.. ............
+...****..******...         . ..  . . ....       ..... ....****************************@@@@@@@@...........*@@**.  ............. .
+ ...***..*******.       .    ... .  . ...         ......********************************@@@@@@*..........**.................. .
+   .*@**.******.        .  .  .* .. .. ..     ........**********************************@@@@@@@.........................*...
+   .@@@*.******.       .   . ...* .. .  ... ...........**********************************@@@@@@*......*....*..........*...    ..

package/source/.test/util.ts

@@ -0,0 +1,50 @@
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import fs from 'node:fs/promises'
+import { createReadStream } from 'node:fs'
+import { Readable } from 'node:stream'
+
+const suites: Suite[] = [
+  {
+    run: true,
+    ref: `file:///${join(tmpdir(), 'toa-storages-file')}`,
+  },
+  {
+    run: true,
+    ref: 'tmp:///toa-storages-temp',
+  },
+  // add more providers here, use `run` as a condition to run the test
+  // e.g.: `run: process.env.ACCESS_KEY_ID !== undefined`
+]
+
+function map (suite: Suite): Case{
+  const url = new URL(suite.ref)
+
+  return [url.protocol, url, suite.secrets ?? {}]
+}
+
+export const cases = suites.filter(({ run }) => run).map(map)
+
+export function rnd (): string{
+  return Math.random().toString(36).slice(2)
+}
+
+export async function open (rel: string): Promise<Readable>{
+  const path = join(__dirname, rel)
+
+  return createReadStream(path)
+}
+
+export async function read (rel: string): Promise<Buffer>{
+  const path = join(__dirname, rel)
+
+  return fs.readFile(path)
+}
+
+interface Suite{
+  run: boolean
+  ref: string
+  secrets?: Record<string, string>
+}
+
+type Case = [string, URL, Record<string, string>]

package/source/Aspect.ts

@@ -0,0 +1,23 @@
+import { Connector, type extensions } from '@toa.io/core'
+import { type Storage, type Storages } from './Storage'
+
+export class Aspect extends Connector implements extensions.Aspect {
+  public readonly name = 'storages'
+
+  private readonly storages: Storages
+
+  public constructor (storages: Storages) {
+    super()
+
+    this.storages = storages
+  }
+
+  public invoke (name: string, method: keyof Storage, ...args: any[]): any {
+    if (!(name in this.storages))
+      throw new Error(`Storage '${name}' is not defined`)
+
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    return this.storages[name][method](...args)
+  }
+}

package/source/Entry.ts

@@ -0,0 +1,14 @@
+export interface Entry {
+  id: string
+  size: number
+  type: string
+  created: number
+  variants: Variant[]
+  meta: Record<string, unknown>
+}
+
+interface Variant {
+  name: string
+  size: number
+  type: string
+}

package/source/Factory.ts

@@ -0,0 +1,65 @@
+import { decode } from 'msgpackr'
+import { type ProviderClass, providers } from './providers'
+import { Storage, type Storages } from './Storage'
+import { Aspect } from './Aspect'
+
+export class Factory {
+  private readonly declaration: Record<string, string>
+
+  public constructor () {
+    const env = process.env.TOA_STORAGES
+
+    if (env === undefined)
+      throw new Error('TOA_STORAGES is not defined')
+
+    this.declaration = decode(Buffer.from(env, 'base64')) as Record<string, string>
+  }
+
+  public aspect (): Aspect {
+    const storages = this.createStorages()
+
+    return new Aspect(storages)
+  }
+
+  public createStorage (name: string, ref: string): Storage {
+    const url = new URL(ref)
+    const Provider = providers[url.protocol]
+
+    if (Provider === undefined)
+      throw new Error(`No provider found for '${url.protocol}'`)
+
+    const secrets = this.resolveSecrets(name, Provider)
+
+    const provider = new Provider(url, secrets)
+
+    return new Storage(provider)
+  }
+
+  private createStorages (): Storages {
+    const storages: Storages = {}
+
+    for (const [name, ref] of Object.entries(this.declaration))
+      storages[name] = this.createStorage(name, ref)
+
+    return storages
+  }
+
+  private resolveSecrets (name: string, Class: ProviderClass): Record<string, string> {
+    if (Class.SECRETS === undefined)
+      return {}
+
+    const secrets: Record<string, string> = {}
+
+    for (const secret of Class.SECRETS) {
+      const variable = `TOA_STORAGES_${name.toUpperCase()}_${secret.toUpperCase()}`
+      const value = process.env[variable]
+
+      if (value === undefined)
+        throw new Error(`${variable} is not defined`)
+
+      secrets[secret] = value
+    }
+
+    return secrets
+  }
+}

package/source/Provider.ts

@@ -0,0 +1,8 @@
+import { type Readable } from 'node:stream'
+
+export interface Provider {
+  get: (path: string) => Promise<Readable | null>
+  put: (path: string, filename: string, stream: Readable) => Promise<void>
+  delete: (path: string) => Promise<void>
+  move: (from: string, to: string) => Promise<void>
+}

package/source/Scanner.ts

@@ -0,0 +1,109 @@
+import { PassThrough, type TransformCallback } from 'node:stream'
+import { createHash } from 'node:crypto'
+
+export class Scanner extends PassThrough {
+  public size = 0
+  public type = 'application/octet-stream'
+  public error: Error | null = null
+
+  private readonly hash = createHash('md5')
+  private readonly assertion: string | undefined
+  private position = 0
+  private detected = false
+  private readonly chunks: Buffer[] = []
+
+  public constructor (assertion?: string) {
+    super()
+
+    this.assertion = assertion
+  }
+
+  public digest (): string {
+    return this.hash.digest('hex')
+  }
+
+  public override _transform
+  (buffer: Buffer, encoding: BufferEncoding, callback: TransformCallback): void {
+    super._transform(buffer, encoding, callback)
+
+    this.process(buffer)
+  }
+
+  private readonly process = (buffer: Buffer): void => {
+    this.size += buffer.length
+    this.hash.update(buffer)
+
+    if (this.detected)
+      return
+
+    if (this.position + buffer.length > HEADER_SIZE)
+      buffer = buffer.subarray(0, HEADER_SIZE - this.position)
+
+    this.chunks.push(buffer)
+    this.position += buffer.length
+
+    if (this.position === HEADER_SIZE)
+      this.complete()
+  }
+
+  private complete (): void {
+    this.detected = true
+
+    const header = Buffer.concat(this.chunks).toString('hex')
+
+    const signature = SIGNATURES
+      .find(({ hex, off }) => header.slice(off, off + hex.length) === hex)
+
+    this.verify(signature)
+
+    const value = signature?.type ?? this.assertion
+
+    if (value !== undefined)
+      this.type = value
+  }
+
+  private verify (signature: Signature | undefined): void {
+    if (this.assertion === undefined || this.assertion === 'application/octet-stream')
+      return
+
+    const mismatch = signature === undefined
+      ? KNOWN_TYPES.has(this.assertion)
+      : this.assertion !== signature.type
+
+    if (mismatch) {
+      this.error = ERR_TYPE_MISMATCH
+      this.end()
+    }
+  }
+}
+
+// https://en.wikipedia.org/wiki/List_of_file_signatures
+const SIGNATURES: Signature[] = [
+  { hex: 'ffd8ffe0', off: 0, type: 'image/jpeg' },
+  { hex: 'ffd8ffe1', off: 0, type: 'image/jpeg' },
+  { hex: 'ffd8ffee', off: 0, type: 'image/jpeg' },
+  { hex: 'ffd8ffdb', off: 0, type: 'image/jpeg' },
+  { hex: '89504e47', off: 0, type: 'image/png' },
+  { hex: '47494638', off: 0, type: 'image/gif' },
+  { hex: '52494646', off: 0, type: 'image/webp' },
+  { hex: '4a584c200d0a870a', off: 8, type: 'image/jxl' },
+  { hex: '6674797068656963', off: 8, type: 'image/heic' },
+  { hex: '6674797061766966', off: 8, type: 'image/avif' }
+  /*
+  When adding a new signature, include a copyright-free sample file in the `.tests` directory
+  and update the 'signatures' test group in `Storage.test.ts`.
+   */
+]
+
+const HEADER_SIZE = SIGNATURES
+  .reduce((max, { off, hex }) => Math.max(max, off + hex.length), 0) / 2
+
+const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
+
+const ERR_TYPE_MISMATCH = new Error('TYPE_MISMATCH')
+
+interface Signature {
+  hex: string
+  off: number
+  type: string
+}