@toa.io/extensions.storages 1.0.0-alpha.0 → 1.0.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "1.0.0-alpha.0",
+  "version": "1.0.0-alpha.2",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -22,21 +22,22 @@
   },
   "scripts": {
     "test": "jest",
-    "transpile": "rm -rf transpiled && npx tsc"
+    "transpile": "npx tsc"
   },
   "devDependencies": {
-    "@toa.io/streams": "1.0.0-alpha.0",
-    "@types/fs-extra": "11.0.3",
+    "@toa.io/streams": "1.0.0-alpha.2",
     "dotenv": "16.3.1"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.435.0",
-    "@aws-sdk/lib-storage": "3.437.0",
-    "@toa.io/generic": "0.20.0-alpha.2",
+    "@aws-sdk/client-s3": "3.481.0",
+    "@aws-sdk/lib-storage": "3.481.0",
+    "@toa.io/agent": "1.0.0-alpha.2",
+    "@toa.io/generic": "1.0.0-alpha.2",
+    "@toa.io/schemas": "1.0.0-alpha.2",
     "error-value": "0.3.0",
-    "fs-extra": "11.1.1",
-    "matchacho": "0.3.5",
-    "msgpackr": "1.9.9"
+    "matchacho": "0.6.0",
+    "msgpackr": "1.10.1",
+    "smithy-node-native-fetch": "1.0.6"
   },
-  "gitHead": "06c64546f6292cc07c52f74b31415101037f7616"
+  "gitHead": "7688e6e980a65c82ac2e459be4e355eebf406cd0"
 }

package/readme.md

@@ -47,26 +47,28 @@ async function effect (_, context) {
 
 > `Maybe<T> = T | Error`
 
-#### `async put(path: string, stream: Readable, type?: TypeControl): Maybe<Entry>`
+#### `async put(path: string, stream: Readable, options?: Options): Maybe<Entry>`
 
 ```
-interface TypeControl {
+interface Options {
   claim?: string
   accept?: string
+  meta?: Record<string, string>
 }
 ```
 
-Add a BLOB to the storage and create an entry under specified `path`.
+Add a BLOB to the storage and create an entry under specified `path`, with given meta-information.
 
 BLOB type is identified
 using [magick numbers](https://github.com/sindresorhus/file-type).
 
-If the `type` argument is specified and the value of the `claim` does not match the detected BLOB type, then
-a `TYPE_MISMATCH` error is returned.
-If the BLOB type cannot be identified and the value of the `claim` is not in the list of known types, then the given
-value is used.
-If the list of [acceptable types](https://datatracker.ietf.org/doc/html/rfc2616#section-14.1) is passed and the type of
-the BLOB does not match any of its values, then a `NOT_ACCEPTABLE` error is returned.
+If the `type` argument is specified and the value of the `claim` does not match the detected BLOB
+type, then a `TYPE_MISMATCH` error is returned.
+If the BLOB type cannot be identified and the value of the `claim` is not in the list of known
+types, then the given value is used.
+If the list of [acceptable types](https://datatracker.ietf.org/doc/html/rfc2616#section-14.1) is
+passed and the type of the BLOB does not match any of its values, then a `NOT_ACCEPTABLE` error is
+returned.
 
 Known types
 are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/jxl`, `image/avif`.
@@ -130,29 +132,63 @@ Custom providers are not supported.
 
 ### Amazon S3
 
-Annotation value formats is `s3://{region}/{bucket}?endpoint={endpoint}`.
+Annotation formats is like:
+
+```yaml
+storages:
+  photos:
+    provider: s3
+    bucket: my-bucket
+    region: eu-west-1
+  tmp:
+    provider: s3
+    endpoint: http://localhost:9000
+    bucket: my-bucket
+```
 
-Requires secrets for the access key and secret key.
+Secrets for the AWS access key and secret key can be provided via SECRETS constructs property. If
+missed standard AWS SDK credentials resolve chain will be used (that means environment variable,
+shared config file, EC2 metadata service, etc.).
 See [`toa conceal`](/runtime/cli/readme.md#conceal) for deployment
 and [`toa env`](/runtime/cli/readme.md#env)
 for local environment.
 `endpoint` parameter is optional.
 
-`s3://us-east-1/my-bucket?endpoint=http://s3.my-instance.com:4566`
-
 ### Filesystem
 
-Annotation value format is `file:///{path}`.
+Annotation format is:
 
-`file:///var/my-storage`
+```yaml
+storages:
+  photos@dev:
+    provider: fs
+    path: /var/my-storage
+```
 
 ### Temporary
 
 Filesystem using OS temporary directory.
 
-Annotation value format is `tmp:///{path}`.
+Annotation format is:
 
-`tmp:///my-storage`
+```yaml
+storages:
+  photos@dev:
+    provider: tmp
+    directory: my-app-tmp
+```
+
+### Memory
+
+In-memory non-persistent storage.
+
+Annotation value format is:
+
+```yaml
+storages:
+  photos@dev:
+    provider: mem
+```
 
 ## Deduplication
 
@@ -199,8 +235,16 @@ provider-specific URLs as values.
 
 ```yaml
 storages:
-  photos: s3://us-east-1/my-bucket
-  photos@dev: file:///var/my-storage
+  photos:
+    provider: s3
+    bucket: my-bucket
+  photos@dev:
+    provider: fs
+    path: /var/my-storage
+  tmp:
+    provider: s3
+    endpoint: http://localhost:9000
+    bucket: my-bucket
 ```
 
 ## Secrets

package/schemas/annotation.cos.yaml

@@ -0,0 +1,10 @@
+type: object
+patternProperties:
+  '^[a-z0-9_]{1,32}$':
+    oneOf:
+      - $ref: s3
+      - $ref: fs
+      - $ref: tmp
+      - $ref: mem
+      - $ref: test
+additionalProperties: false

package/schemas/fs.cos.yaml

@@ -0,0 +1,9 @@
+type: object
+properties:
+  provider:
+    const: fs
+  path:
+    type: string
+    maxLength: 4096
+required: [provider, path]
+additionalProperties: false

package/schemas/mem.cos.yaml

@@ -0,0 +1,6 @@
+type: object
+properties:
+  provider:
+    const: mem
+required: [provider]
+additionalProperties: false

package/schemas/s3.cos.yaml

@@ -0,0 +1,16 @@
+type: object
+properties:
+  provider:
+    const: s3
+  bucket:
+    type: string
+    minLength: 3
+    maxLength: 63
+  region:
+    type: string
+  prefix:
+    type: string
+  endpoint:
+    type: string
+    format: uri
+required: [provider, bucket]

package/schemas/test.cos.yaml

@@ -0,0 +1,8 @@
+type: object
+properties:
+  provider:
+    const: test
+  directory:
+    type: string
+required: [provider, directory]
+additionalProperties: false

package/schemas/tmp.cos.yaml

@@ -0,0 +1,9 @@
+type: object
+properties:
+  provider:
+    type: string
+    const: tmp
+  directory:
+    type: string
+required: [provider, directory]
+additionalProperties: false

package/source/Annotation.ts

@@ -0,0 +1,39 @@
+import assert from 'node:assert'
+import { providers } from './providers'
+import * as schemas from './schemas'
+import type { Declaration } from './providers'
+import type { Schema } from '@toa.io/schemas'
+
+export type Annotation = Record<string, Declaration>
+
+export function validateAnnotation (annotation: unknown): asserts annotation is Annotation {
+  try {
+    schemas.annotation.validate(annotation)
+  } catch (error) {
+    explain(annotation)
+
+    // if all declarations are valid, re-throw the error
+    throw error
+  }
+}
+
+/*
+It is required because `oneOf` schema is used for the annotation validation.
+ */
+function explain (annotation: unknown): void {
+  assert.ok(typeof annotation === 'object' && annotation !== null,
+    'TOA_STORAGES is not an object')
+
+  for (const declaration of Object.values(annotation)) {
+    assert.ok(typeof declaration === 'object' && declaration !== null &&
+      declaration.provider in providers,
+      `Unknown provider '${declaration.provider}'`)
+
+    assert.ok(declaration.provider in schemas,
+      `No schema for provider '${declaration.provider}'`)
+
+    const schema: Schema<Declaration> = schemas[declaration.provider as keyof typeof providers]
+
+    schema.validate(declaration, `Storage '${declaration.provider}' annotation`)
+  }
+}

package/source/Aspect.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import { Connector, type extensions } from '@toa.io/core'
 import { type Storage, type Storages } from './Storage'
 
@@ -12,12 +13,13 @@ export class Aspect extends Connector implements extensions.Aspect {
     this.storages = storages
   }
 
-  public invoke (name: string, method: keyof Storage, ...args: any[]): any {
-    if (!(name in this.storages))
-      throw new Error(`Storage '${name}' is not defined`)
+  public invoke (name: string, method: keyof Storage, ...args: unknown[]): unknown {
+    const storage = this.storages[name]
+
+    assert.ok(storage !== undefined, `Storage '${name}' is not defined`)
 
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-expect-error
-    return this.storages[name][method](...args)
+    return storage[method](...args)
   }
 }

package/source/Factory.ts

@@ -1,18 +1,25 @@
-import { decode } from 'msgpackr'
-import { type ProviderClass, providers } from './providers'
+import assert from 'node:assert'
+import { decode } from '@toa.io/generic'
+import { providers } from './providers'
 import { Storage, type Storages } from './Storage'
 import { Aspect } from './Aspect'
+import { SERIALIZATION_PREFIX } from './deployment'
+import { validateAnnotation } from './Annotation'
+import type { Declaration } from './providers'
+import type { Annotation } from './Annotation'
+import type { ProviderConstructor, ProviderSecrets } from './Provider'
 
 export class Factory {
-  private readonly declaration: Record<string, string>
+  private readonly annotation: Annotation
 
   public constructor () {
     const env = process.env.TOA_STORAGES
 
-    if (env === undefined)
-      throw new Error('TOA_STORAGES is not defined')
+    assert.ok(env !== undefined, 'TOA_STORAGES is not defined')
 
-    this.declaration = decode(Buffer.from(env, 'base64')) as Record<string, string>
+    this.annotation = decode(env)
+
+    validateAnnotation(this.annotation)
   }
 
   public aspect (): Aspect {
@@ -21,43 +28,39 @@ export class Factory {
     return new Aspect(storages)
   }
 
-  public createStorage (name: string, ref: string): Storage {
-    const url = new URL(ref)
-    const Provider = providers[url.protocol]
-
-    if (Provider === undefined)
-      throw new Error(`No provider found for '${url.protocol}'`)
-
-    const secrets = this.resolveSecrets(name, Provider)
-
-    const provider = new Provider(url, secrets)
-
-    return new Storage(provider)
-  }
-
   private createStorages (): Storages {
     const storages: Storages = {}
 
-    for (const [name, ref] of Object.entries(this.declaration))
-      storages[name] = this.createStorage(name, ref)
+    for (const [name, declaration] of Object.entries(this.annotation))
+      storages[name] = this.createStorage(name, declaration)
 
     return storages
   }
 
-  private resolveSecrets (name: string, Class: ProviderClass): Record<string, string> {
+  private createStorage (name: string, declaration: Declaration): Storage {
+    const { provider: providerId, ...options } = declaration
+    const Provider: ProviderConstructor = providers[providerId]
+    const secrets = this.resolveSecrets(name, Provider)
+    const provider = new Provider(options, secrets)
+
+    return new Storage(provider)
+  }
+
+  private resolveSecrets (storageName: string,
+    Class: ProviderConstructor): ProviderSecrets {
     if (Class.SECRETS === undefined)
       return {}
 
-    const secrets: Record<string, string> = {}
+    const secrets: Record<string, string | undefined> = {}
 
     for (const secret of Class.SECRETS) {
-      const variable = `TOA_STORAGES_${name.toUpperCase()}_${secret.toUpperCase()}`
+      const variable = `${SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
       const value = process.env[variable]
 
-      if (value === undefined)
-        throw new Error(`${variable} is not defined`)
+      assert.ok(secret.optional === true || value !== undefined,
+        `'${variable}' is not defined`)
 
-      secrets[secret] = value
+      secrets[secret.name] = value
     }
 
     return secrets

package/source/Provider.ts

@@ -1,8 +1,33 @@
 import { type Readable } from 'node:stream'
+import * as assert from 'node:assert'
 
-export interface Provider {
-  get: (path: string) => Promise<Readable | null>
-  put: (path: string, filename: string, stream: Readable) => Promise<void>
-  delete: (path: string) => Promise<void>
-  move: (from: string, to: string) => Promise<void>
+export type ProviderSecrets<K extends string = string> = Record<K | string, string | undefined>
+
+export interface ProviderSecret {
+  readonly name: string
+  readonly optional?: boolean
+}
+
+export abstract class Provider<Options = void> {
+  public static readonly SECRETS: readonly ProviderSecret[] = []
+
+  public constructor (_: Options, secrets?: ProviderSecrets) {
+    for (const { name, optional = false } of new.target.SECRETS)
+      assert.ok(optional || secrets?.[name] !== undefined, `Missing secret '${name}'`)
+  }
+
+  public abstract get (path: string): Promise<Readable | null>
+
+  public abstract put (path: string, filename: string, stream: Readable): Promise<void>
+
+  public abstract delete (path: string): Promise<void>
+
+  public abstract move (from: string, to: string): Promise<void>
+}
+
+export interface ProviderConstructor {
+  readonly SECRETS: readonly ProviderSecret[]
+  prototype: Provider
+
+  new (options: any, secrets?: ProviderSecrets): Provider
 }

package/source/Scanner.ts

@@ -1,6 +1,6 @@
 import { PassThrough, type TransformCallback } from 'node:stream'
 import { createHash } from 'node:crypto'
-import { negotiate } from '@toa.io/http'
+import { negotiate } from '@toa.io/agent'
 import { Err } from 'error-value'
 
 export class Scanner extends PassThrough {
@@ -15,7 +15,7 @@ export class Scanner extends PassThrough {
   private detected = false
   private readonly chunks: Buffer[] = []
 
-  public constructor (control?: TypeControl) {
+  public constructor (control?: ScanOptions) {
     super()
 
     this.claim = control?.claim
@@ -121,7 +121,7 @@ const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
 const ERR_TYPE_MISMATCH = Err('TYPE_MISMATCH')
 const ERR_NOT_ACCEPTABLE = Err('NOT_ACCEPTABLE')
 
-export interface TypeControl {
+export interface ScanOptions {
   claim?: string
   accept?: string
 }