@toa.io/extensions.storages 1.0.0-alpha.20 → 1.0.0-alpha.202

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "1.0.0-alpha.20",
+  "version": "1.0.0-alpha.202",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -24,20 +24,20 @@
     "test": "jest",
     "transpile": "npx tsc"
   },
-  "devDependencies": {
-    "@toa.io/streams": "1.0.0-alpha.20",
-    "dotenv": "16.3.1"
+  "peerDependencies": {
+    "dotenv": "*"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.481.0",
-    "@aws-sdk/lib-storage": "3.481.0",
-    "@toa.io/agent": "1.0.0-alpha.20",
-    "@toa.io/generic": "1.0.0-alpha.20",
-    "@toa.io/schemas": "1.0.0-alpha.20",
-    "error-value": "0.3.0",
-    "matchacho": "0.6.0",
-    "msgpackr": "1.10.1",
-    "smithy-node-native-fetch": "1.0.6"
+    "@aws-sdk/client-s3": "3.989.0",
+    "@aws-sdk/lib-storage": "3.989.0",
+    "@aws-sdk/protocol-http": "3.374.0",
+    "@toa.io/generic": "1.0.0-alpha.173",
+    "@toa.io/schemas": "1.0.0-alpha.200",
+    "cloudinary": "2.9.0",
+    "error-value": "0.4.1",
+    "negotiator": "0.6.3",
+    "openspan": "1.0.0-alpha.173",
+    "smithy-node-native-fetch": "1.2.1"
   },
-  "gitHead": "dcde3389a86ab38e377ebc5ab18e5d5a512b85bc"
+  "gitHead": "4d230bb899fee0cd49b099db09749a01d01cda7e"
 }

package/readme.md

@@ -6,15 +6,11 @@ Shared BLOB storage.
 
 BLOBs are stored with the meta-information object (Entry) having the following properties:
 
-- `id` - checksum
-- `size` - size in bytes
-- `type` - MIME type
-- `created` - creation timestamp (UNIX time, ms)
-- `variants` - array of:
-  - `name` - unique name
-  - `size` - size in bytes
-  - `type` - variant MIME type
-- `meta` - object with application-specific information, empty by default
+- `size`: size in bytes
+- `type`: MIME type
+- `checksum`: content checksum
+- `created`: creation timestamp (ISO 8601)
+- `attributes`: `Record<string, string>` with application-specific information, empty by default
 
 ### Example
 
@@ -22,12 +18,7 @@ BLOBs are stored with the meta-information object (Entry) having the following p
 id: eecd837c
 type: image/jpeg
 created: 1698004822358
-variants:
-  - name: thumbnail.jpeg
-    type: image/jpeg
-  - name: thumbnail.webp
-    type: image/webp
-meta:
+attributes:
   face: true
   nudity: false
 ```
@@ -39,7 +30,7 @@ containing named Storage instances, according to the annotation.
 
 ```javascript
 async function effect (_, context) {
-  await context.storages.photos.fetch('/path/to/b4f577e0.thumbnail.jpeg')
+  await context.storages.photos.get('/path/to/b4f577e0.thumbnail.jpeg')
 }
 ```
 
@@ -49,11 +40,11 @@ async function effect (_, context) {
 
 #### `async put(path: string, stream: Readable, options?: Options): Maybe<Entry>`
 
-```
+```ts
 interface Options {
   claim?: string
   accept?: string
-  meta?: Record<string, string>
+  attributes?: Record<string, string>
 }
 ```
 
@@ -75,15 +66,13 @@ are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/
 
 See [source](source/Scanner.ts).
 
-If the entry already exists, it is returned and [revealed](#async-revealpath-string-maybevoid).
-
-#### `async get(path: string): Maybe<Entry>`
+#### `async head(path: string): Maybe<Entry>`
 
 Get an entry.
 
 If the entry does not exist, a `NOT_FOUND` error is returned.
 
-#### `async fetch(path: string): Maybe<Readable>`
+#### `async get(path: string): Maybe<Readable>`
 
 Fetch the BLOB specified by `path`. If the path does not exist, a `NOT_FOUND` error is returned.
 
@@ -97,32 +86,20 @@ Fetch the BLOB specified by `path`. If the path does not exist, a `NOT_FOUND` er
 
 Delete the entry specified by `path`.
 
-#### `async list(path: string): string[]`
-
-Get ordered list of `id`s of entries in under the `path`.
-
-#### `async permute(path: string, ids: string[]): Maybe<void>`
-
-Reorder entries under the `path`.
-
-Given list must be a permutation of the current list, otherwise a `PERMUTATION_MISMATCH` error is
-returned.
-
-#### `async diversify(path: string, name: string, stream: Readable): Maybe<void>`
-
-Add or replace a `name` variant of the entry specified by `path`.
-
-#### `async conceal(path: string): Maybe<void>`
-
-Remove the entry from the list.
+#### `async move(path: string, to: string): Maybe<void>`
 
-#### `async reveal(path: string): Maybe<void>`
+Moves the entry specified by `path` to the new path.
 
-Restore the entry to the list.
+`to` may be an absolute or relative path (starting with `.`), if it ends with `/`, the entry is
+moved to the `to` directory, otherwise, the entry is moved to the `to` path.
 
-#### `async annotate(path: string, key: string, value: any): Maybe<void>`
+The following examples are equivalent:
 
-Set a `key` property in the `meta` of the entry specified by `path`.
+```javascript
+await storage.move('/path/to/eecd837c', '/path/to/sub/eecd837c')
+await storage.move('/path/to/eecd837c', './sub/eecd837c')
+await storage.move('/path/to/eecd837c', './sub/')
+```
 
 ## Providers
 
@@ -132,7 +109,7 @@ Custom providers are not supported.
 
 ### Amazon S3
 
-Annotation formats is like:
+Annotation formats are like:
 
 ```yaml
 storages:
@@ -154,15 +131,83 @@ and [`toa env`](/runtime/cli/readme.md#env)
 for local environment.
 `endpoint` parameter is optional.
 
+### Cloudinary
+
+[Cloudinary](https://cloudinary.com) provider is used to store and transform media files.
+
+Objects stored in the Cloudinary storage can be requested with transformations,
+using dot-separated path extensions: `/path/to/eecd837c.{extension}.{extension}`.
+
+Transformation is defined by the `extension` property, which is a regular expression with named
+groups.
+The named groups are used to replace the placeholders in the `transform` object.
+
+`transform` object is an element of
+the [Cloudinary `transformation` array](https://cloudinary.com/documentation/node_image_manipulation#apply_common_image_transformations).
+
+> - `zoom` should be specified as integer [0–100].
+> - `format` value `jpeg` is converted to `jpg`.
+> - `^` and `$` are added to the regular expression automatically, unless they are already present.
+
+Annotation format is:
+
+```yaml
+storages:
+  media:
+    provider: cloudinary
+    environment: my-cloud
+    type: image # image or video
+    prefix: my-app
+    transformations:
+      - extension: (?<width>\d*)x(?<height>\d*)(z(?<zoom>\d*))?
+        transformation:
+          width: <width>
+          height: <height>
+          zoom: <zoom>
+          crop: thumb
+          gravity: face
+        optional: true
+      - extension: (?<format>jpeg|webp)
+        transformation:
+          fetch_format: <format>
+```
+
+Path extensions must be specified in the same order as in the annotation.
+
+For the example above:
+
+- `/path/to/eecd837c.100x100.jpeg`: matches
+- `/path/to/eecd837c.jpeg.100x100`: does not match
+
+If a path extension is not matched,
+or if at least one of the transformations is not matched (unless it is `optional`),
+then an error is returned.
+
+Secrets:
+
+- `API_KEY`
+- `API_SECRET`
+
 ### Filesystem
 
 Annotation format is:
 
 ```yaml
 storages:
-  photos@dev:
+  photos:
     provider: fs
-    path: /var/my-storage
+    path: /var/my-photos
+```
+
+[Kubernetes PVC](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) can be mounted to
+the storage:
+
+```yaml
+storages:
+  photos:
+    provider: fs
+    path: /var/my-photos
+    claim: photos-pvc
 ```
 
 ### Temporary
@@ -198,17 +243,17 @@ Variants, on the other hand, are not deduplicated across different entries.
 
 Underlying directory structure:
 
-```
+```text
 /temp
   c28f4dfd            # random id
 /blobs
   b4f577e0            # checksum
 /storage
   /path/to
-    .list             # list of entries
+    /.entries/
+      b4f577e0        # entry
     /b4f577e0
-      .meta           # entry
-      thumbnail.jpeg  # variant BLOBs
+      thumbnail.jpeg  # variant
       thumbnail.webp
 ```
 

package/schemas/annotation.cos.yaml

@@ -3,6 +3,7 @@ patternProperties:
   '^[a-z0-9_]{1,32}$':
     oneOf:
       - $ref: s3
+      - $ref: cloudinary
       - $ref: fs
       - $ref: tmp
       - $ref: mem

package/schemas/cloudinary.cos.yaml

@@ -0,0 +1,37 @@
+type: object
+properties:
+  provider:
+    const: cloudinary
+  environment:
+    type: string
+  type:
+    enum:
+      - image
+      - video
+  prefix:
+    type: string
+  eager:
+    type: array
+    items: &transformation
+      type: object
+  transformations:
+    type: array
+    items:
+      type: object
+      properties:
+        extension:
+          type: string
+        transformation:
+          anyOf:
+            - *transformation
+            - type: array
+              items: *transformation
+        optional:
+          type: boolean
+      required:
+        - extension
+        - transformation
+required:
+  - environment
+  - type
+additionalProperties: false

package/schemas/fs.cos.yaml

@@ -5,5 +5,7 @@ properties:
   path:
     type: string
     maxLength: 4096
+  claim:
+    type: string
 required: [provider, path]
 additionalProperties: false

package/schemas/s3.cos.yaml

@@ -14,3 +14,4 @@ properties:
     type: string
     format: uri
 required: [provider, bucket]
+additionalProperties: false

package/schemas/tmp.cos.yaml

@@ -1,7 +1,6 @@
 type: object
 properties:
   provider:
-    type: string
     const: tmp
   directory:
     type: string

package/source/Entry.ts

@@ -1,14 +1,16 @@
-export interface Entry {
-  id: string
-  size: number
-  type: string
-  created: number
-  variants: Variant[]
-  meta: Record<string, unknown>
-}
+import type { Readable } from 'node:stream'
 
-interface Variant {
-  name: string
-  size: number
+export type Entry = { id: string } & Metadata
+export type Stream = { stream: Readable } & Metadata
+
+export interface Metadata {
   type: string
+  size: number | null
+  checksum: string
+  created: string
+  attributes: Attributes
+  range?: string
+  partial?: boolean
 }
+
+export type Attributes = Record<string, string>

package/source/Factory.ts

@@ -1,21 +1,23 @@
 import assert from 'node:assert'
+import { console } from 'openspan'
 import { decode } from '@toa.io/generic'
 import { providers } from './providers'
 import { Storage, type Storages } from './Storage'
 import { Aspect } from './Aspect'
-import { SERIALIZATION_PREFIX } from './deployment'
+import { ENV_PREFIX } from './deployment'
 import { validateAnnotation } from './Annotation'
+import type { Constructor } from './Provider'
 import type { Declaration } from './providers'
 import type { Annotation } from './Annotation'
-import type { ProviderConstructor, ProviderSecrets } from './Provider'
+import type { Secrets } from './Secrets'
 
 export class Factory {
   private readonly annotation: Annotation
 
   public constructor () {
-    const env = process.env.TOA_STORAGES
+    const env = process.env[ENV_PREFIX]
 
-    assert.ok(env !== undefined, 'TOA_STORAGES is not defined')
+    assert.ok(env !== undefined, `${ENV_PREFIX} is not defined`)
 
     this.annotation = decode(env)
 
@@ -38,23 +40,28 @@ export class Factory {
   }
 
   private createStorage (name: string, declaration: Declaration): Storage {
-    const { provider: providerId, ...options } = declaration
-    const Provider: ProviderConstructor = providers[providerId]
+    const { provider: id, ...options } = declaration
+    const Provider: Constructor = providers[id]
     const secrets = this.resolveSecrets(name, Provider)
     const provider = new Provider(options, secrets)
 
+    console.debug('Storage created', {
+      name,
+      provider: id,
+      ...(provider.root === undefined ? undefined : { root: provider.root })
+    })
+
     return new Storage(provider)
   }
 
-  private resolveSecrets (storageName: string,
-    Class: ProviderConstructor): ProviderSecrets {
+  private resolveSecrets (storageName: string, Class: Constructor): Secrets {
     if (Class.SECRETS === undefined)
       return {}
 
     const secrets: Record<string, string | undefined> = {}
 
     for (const secret of Class.SECRETS) {
-      const variable = `${SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
+      const variable = `${ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
       const value = process.env[variable]
 
       assert.ok(secret.optional === true || value !== undefined,

package/source/Provider.ts

@@ -1,33 +1,36 @@
-import { type Readable } from 'node:stream'
 import * as assert from 'node:assert'
+import type { Metadata, Stream } from './Entry'
+import type { Readable } from 'node:stream'
+import type { Maybe } from '@toa.io/types'
+import type { Secret, Secrets } from './Secrets'
 
-export type ProviderSecrets<K extends string = string> = Record<K | string, string | undefined>
+export abstract class Provider<Options = unknown> {
+  public static readonly SECRETS?: readonly Secret[]
+  public readonly root?: string
+  public readonly options: Options
 
-export interface ProviderSecret {
-  readonly name: string
-  readonly optional?: boolean
-}
-
-export abstract class Provider<Options = void> {
-  public static readonly SECRETS: readonly ProviderSecret[] = []
+  protected constructor (options: Options, secrets?: Secrets) {
+    this.options = options
 
-  public constructor (_: Options, secrets?: ProviderSecrets) {
-    for (const { name, optional = false } of new.target.SECRETS)
-      assert.ok(optional || secrets?.[name] !== undefined, `Missing secret '${name}'`)
+    new.target.SECRETS?.forEach(({ name, optional }) =>
+      assert.ok(optional === true || secrets?.[name] !== undefined, `Missing secret '${name}'`))
   }
 
-  public abstract get (path: string): Promise<Readable | null>
+  public abstract get (path: string, options?: unknown): Promise<Maybe<Stream>>
+
+  public abstract head (path: string): Promise<Maybe<Metadata>>
+
+  public abstract put (path: string, stream: Readable): Promise<void>
 
-  public abstract put (path: string, filename: string, stream: Readable): Promise<void>
+  public abstract commit (path: string, metadata: Metadata): Promise<void>
 
   public abstract delete (path: string): Promise<void>
 
-  public abstract move (from: string, to: string): Promise<void>
+  public abstract move (from: string, to: string): Promise<Maybe<void>>
 }
 
-export interface ProviderConstructor {
-  readonly SECRETS: readonly ProviderSecret[]
-  prototype: Provider
+export interface Constructor<Options = any> {
+  SECRETS?: readonly Secret[]
 
-  new (options: any, secrets?: ProviderSecrets): Provider
+  new (options: Options, secrets?: Secrets): Provider<Options>
 }

package/source/Scanner.ts

@@ -1,18 +1,19 @@
 import { PassThrough, type TransformCallback } from 'node:stream'
 import { createHash } from 'node:crypto'
-import { negotiate } from '@toa.io/agent'
 import { Err } from 'error-value'
+import Negotiator from 'negotiator'
 
 export class Scanner extends PassThrough {
   public size = 0
   public type = 'application/octet-stream'
-  public error: Error | null = null
+  public error?: Error
 
   private readonly hash = createHash('md5')
   private readonly claim?: string
   private readonly accept?: string
+  private readonly limit?: number
   private position = 0
-  private detected = false
+  private completed = false
   private readonly chunks: Buffer[] = []
 
   public constructor (control?: ScanOptions) {
@@ -20,14 +21,14 @@ export class Scanner extends PassThrough {
 
     this.claim = control?.claim
     this.accept = control?.accept
+    this.limit = control?.limit
   }
 
   public digest (): string {
     return this.hash.digest('hex')
   }
 
-  public override _transform
-  (buffer: Buffer, encoding: BufferEncoding, callback: TransformCallback): void {
+  public override _transform (buffer: Buffer, encoding: BufferEncoding, callback: TransformCallback): void {
     super._transform(buffer, encoding, callback)
 
     this.process(buffer)
@@ -37,7 +38,7 @@ export class Scanner extends PassThrough {
     this.size += buffer.length
     this.hash.update(buffer)
 
-    if (this.detected)
+    if (this.completed)
       return
 
     if (this.position + buffer.length > HEADER_SIZE)
@@ -48,13 +49,22 @@ export class Scanner extends PassThrough {
 
     if (this.position === HEADER_SIZE)
       this.complete()
+
+    if (this.limit !== undefined && this.size > this.limit)
+      this.interrupt(ERR_LIMIT_EXCEEDED)
   }
 
   private complete (): void {
     const header = Buffer.concat(this.chunks).toString('hex')
 
     const signature = SIGNATURES
-      .find(({ hex, off }) => header.slice(off, off + hex.length) === hex)
+      .find(({ off, hex, expression }) => {
+        const sig = header.slice(off, off + hex.length)
+
+        return expression === undefined
+          ? sig === hex
+          : expression.test(sig)
+      })
 
     const type = signature?.type ?? this.claim
 
@@ -64,7 +74,7 @@ export class Scanner extends PassThrough {
     }
 
     this.verify(signature)
-    this.detected = true
+    this.completed = true
   }
 
   private verify (signature: Signature | undefined): void {
@@ -83,51 +93,73 @@ export class Scanner extends PassThrough {
     if (this.accept === undefined)
       return
 
-    const unacceptable = negotiate(this.accept, [type]) === null
+    const unacceptable = this.negotiate(this.accept, [type]) === null
 
     if (unacceptable)
       this.interrupt(ERR_NOT_ACCEPTABLE)
   }
 
+  private negotiate (accept: string, type: string[]): string | null {
+    return new Negotiator({ headers: { accept } }).mediaType(type) ?? null
+  }
+
   private interrupt (error: Error): void {
+    this.completed = true
     this.error = error
-    this.end()
+    this.destroy(error)
   }
 }
 
 // https://en.wikipedia.org/wiki/List_of_file_signatures
 const SIGNATURES: Signature[] = [
-  { hex: 'ffd8ffe0', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffe1', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffee', off: 0, type: 'image/jpeg' },
-  { hex: 'ffd8ffdb', off: 0, type: 'image/jpeg' },
-  { hex: '89504e47', off: 0, type: 'image/png' },
-  { hex: '47494638', off: 0, type: 'image/gif' },
-  { hex: '52494646', off: 0, type: 'image/webp' },
-  { hex: '4a584c200d0a870a', off: 8, type: 'image/jxl' },
-  { hex: '6674797068656963', off: 8, type: 'image/heic' },
-  { hex: '6674797061766966', off: 8, type: 'image/avif' }
+  { hex: 'ff d8 ff e0', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff e1', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff e2', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff ee', off: 0, type: 'image/jpeg' },
+  { hex: 'ff d8 ff db', off: 0, type: 'image/jpeg' },
+  { hex: '89 50 4e 47', off: 0, type: 'image/png' },
+  { hex: '47 49 46 38', off: 0, type: 'image/gif' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 57 45 42 50', off: 0, type: 'image/webp' },
+  { hex: '4a 58 4c 20 0d 0a 87 0a', off: 8, type: 'image/jxl' },
+  { hex: '66 74 79 70 68 65 69 63', off: 8, type: 'image/heic' },
+  { hex: '66 74 79 70 61 76 69 66', off: 8, type: 'image/avif' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 41 56 49 20', off: 0, type: 'video/avi' },
+  { hex: '52 49 46 46 ?? ?? ?? ?? 57 41 56 45', off: 0, type: 'audio/wav' }
   /*
   When adding a new signature, include a copyright-free sample file in the `.tests` directory
-  and update the 'signatures' test group in `Storage.test.ts`.
+  and update the `signatures` test group in `Storage.test.ts`.
    */
-]
+].map((signature: Signature) => {
+  signature.hex = signature.hex.replaceAll(' ', '')
+
+  if (signature.hex.includes('??')) {
+    const expression = signature.hex.replaceAll(/(?<wildcards>\?{1,24})/g,
+      (_, wildcards) => `[0-9a-f]{${wildcards.length}}`)
+
+    signature.expression = new RegExp(expression, 'i')
+  }
+
+  return signature
+})
 
 const HEADER_SIZE = SIGNATURES
   .reduce((max, { off, hex }) => Math.max(max, off + hex.length), 0) / 2
 
 const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
 
-const ERR_TYPE_MISMATCH = Err('TYPE_MISMATCH')
-const ERR_NOT_ACCEPTABLE = Err('NOT_ACCEPTABLE')
+const ERR_TYPE_MISMATCH = new Err('TYPE_MISMATCH')
+const ERR_NOT_ACCEPTABLE = new Err('NOT_ACCEPTABLE')
+const ERR_LIMIT_EXCEEDED = new Err('LIMIT_EXCEEDED')
 
 export interface ScanOptions {
   claim?: string
   accept?: string
+  limit?: number
 }
 
 interface Signature {
-  hex: string
-  off: number
   type: string
+  off: number
+  hex: string
+  expression?: RegExp
 }

package/source/Secrets.ts

@@ -0,0 +1,6 @@
+export type Secrets<K extends string = string> = Record<K | string, string | undefined>
+
+export interface Secret {
+  readonly name: string
+  readonly optional?: boolean
+}