@toa.io/extensions.storages 0.24.0-alpha.21 → 0.24.0-alpha.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "0.24.0-alpha.21",
+  "version": "0.24.0-alpha.23",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -25,18 +25,19 @@
     "transpile": "npx tsc"
   },
   "devDependencies": {
-    "@toa.io/streams": "0.24.0-alpha.21",
+    "@toa.io/streams": "0.24.0-alpha.23",
     "dotenv": "16.3.1"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.481.0",
     "@aws-sdk/lib-storage": "3.481.0",
-    "@toa.io/generic": "0.20.0-alpha.2",
-    "@toa.io/http": "0.24.0-alpha.21",
+    "@toa.io/agent": "0.24.0-alpha.23",
+    "@toa.io/generic": "0.24.0-alpha.23",
+    "@toa.io/schemas": "0.24.0-alpha.23",
     "error-value": "0.3.0",
     "matchacho": "0.6.0",
     "msgpackr": "1.10.1",
     "smithy-node-native-fetch": "1.0.6"
   },
-  "gitHead": "b30bbcf178be9339914e2e5203c61c0da2c3daf2"
+  "gitHead": "df9de3cbb530e8f985a660bca0bf65bd027dbb01"
 }

package/readme.md

@@ -47,29 +47,28 @@ async function effect (_, context) {
 
 > `Maybe<T> = T | Error`
 
-#### `async put(path: string, stream: Readable, type?: TypeControl): Maybe<Entry>`
+#### `async put(path: string, stream: Readable, options?: Options): Maybe<Entry>`
 
 ```
-interface TypeControl {
+interface Options {
   claim?: string
   accept?: string
+  meta?: Record<string, string>
 }
 ```
 
-Add a BLOB to the storage and create an entry under specified `path`.
+Add a BLOB to the storage and create an entry under specified `path`, with given meta-information.
 
 BLOB type is identified
 using [magick numbers](https://github.com/sindresorhus/file-type).
 
 If the `type` argument is specified and the value of the `claim` does not match the detected BLOB
-type, then
-a `TYPE_MISMATCH` error is returned.
+type, then a `TYPE_MISMATCH` error is returned.
 If the BLOB type cannot be identified and the value of the `claim` is not in the list of known
-types, then the given
-value is used.
+types, then the given value is used.
 If the list of [acceptable types](https://datatracker.ietf.org/doc/html/rfc2616#section-14.1) is
-passed and the type of
-the BLOB does not match any of its values, then a `NOT_ACCEPTABLE` error is returned.
+passed and the type of the BLOB does not match any of its values, then a `NOT_ACCEPTABLE` error is
+returned.
 
 Known types
 are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/jxl`, `image/avif`.
@@ -155,7 +154,6 @@ and [`toa env`](/runtime/cli/readme.md#env)
 for local environment.
 `endpoint` parameter is optional.
 
-
 ### Filesystem
 
 Annotation format is:
@@ -177,7 +175,7 @@ Annotation format is:
 storages:
   photos@dev:
     provider: tmp
-    prefix: my-app-tmp
+    directory: my-app-tmp
 ```
 
 ### Memory
@@ -189,7 +187,7 @@ Annotation value format is:
 ```yaml
 storages:
   photos@dev:
-    provider: memory
+    provider: mem
 ```
 
 ## Deduplication

package/schemas/annotation.cos.yaml

@@ -0,0 +1,10 @@
+type: object
+patternProperties:
+  '^[a-z0-9_]{1,32}$':
+    oneOf:
+      - $ref: s3
+      - $ref: fs
+      - $ref: tmp
+      - $ref: mem
+      - $ref: test
+additionalProperties: false

package/schemas/fs.cos.yaml

@@ -0,0 +1,9 @@
+type: object
+properties:
+  provider:
+    const: fs
+  path:
+    type: string
+    maxLength: 4096
+required: [provider, path]
+additionalProperties: false

package/schemas/mem.cos.yaml

@@ -0,0 +1,6 @@
+type: object
+properties:
+  provider:
+    const: mem
+required: [provider]
+additionalProperties: false

package/schemas/s3.cos.yaml

@@ -0,0 +1,16 @@
+type: object
+properties:
+  provider:
+    const: s3
+  bucket:
+    type: string
+    minLength: 3
+    maxLength: 63
+  region:
+    type: string
+  prefix:
+    type: string
+  endpoint:
+    type: string
+    format: uri
+required: [provider, bucket]

package/schemas/test.cos.yaml

@@ -0,0 +1,8 @@
+type: object
+properties:
+  provider:
+    const: test
+  directory:
+    type: string
+required: [provider, directory]
+additionalProperties: false

package/schemas/tmp.cos.yaml

@@ -0,0 +1,9 @@
+type: object
+properties:
+  provider:
+    type: string
+    const: tmp
+  directory:
+    type: string
+required: [provider, directory]
+additionalProperties: false

package/source/Annotation.ts

@@ -0,0 +1,39 @@
+import assert from 'node:assert'
+import { providers } from './providers'
+import * as schemas from './schemas'
+import type { Declaration } from './providers'
+import type { Schema } from '@toa.io/schemas'
+
+export type Annotation = Record<string, Declaration>
+
+export function validateAnnotation (annotation: unknown): asserts annotation is Annotation {
+  try {
+    schemas.annotation.validate(annotation)
+  } catch (error) {
+    explain(annotation)
+
+    // if all declarations are valid, re-throw the error
+    throw error
+  }
+}
+
+/*
+It is required because `oneOf` schema is used for the annotation validation.
+ */
+function explain (annotation: unknown): void {
+  assert.ok(typeof annotation === 'object' && annotation !== null,
+    'TOA_STORAGES is not an object')
+
+  for (const declaration of Object.values(annotation)) {
+    assert.ok(typeof declaration === 'object' && declaration !== null &&
+      declaration.provider in providers,
+      `Unknown provider '${declaration.provider}'`)
+
+    assert.ok(declaration.provider in schemas,
+      `No schema for provider '${declaration.provider}'`)
+
+    const schema: Schema<Declaration> = schemas[declaration.provider as keyof typeof providers]
+
+    schema.validate(declaration, `Storage '${declaration.provider}' annotation`)
+  }
+}

package/source/Factory.ts

@@ -3,18 +3,23 @@ import { decode } from '@toa.io/generic'
 import { providers } from './providers'
 import { Storage, type Storages } from './Storage'
 import { Aspect } from './Aspect'
-import { validateProviderId, SERIALIZATION_PREFIX } from './deployment'
-import type { ProviderConstructor } from './Provider'
+import { SERIALIZATION_PREFIX } from './deployment'
+import { validateAnnotation } from './Annotation'
+import type { Declaration } from './providers'
+import type { Annotation } from './Annotation'
+import type { ProviderConstructor, ProviderSecrets } from './Provider'
 
 export class Factory {
-  private readonly declaration: Record<string, Record<string, unknown>>
+  private readonly annotation: Annotation
 
   public constructor () {
     const env = process.env.TOA_STORAGES
 
-    if (env === undefined) throw new Error('TOA_STORAGES is not defined')
+    assert.ok(env !== undefined, 'TOA_STORAGES is not defined')
 
-    this.declaration = decode(env)
+    this.annotation = decode(env)
+
+    validateAnnotation(this.annotation)
   }
 
   public aspect (): Aspect {
@@ -23,35 +28,33 @@ export class Factory {
     return new Aspect(storages)
   }
 
-  public createStorage (componentName: string, { provider: providerId, ...props }: any): Storage {
-    validateProviderId(providerId)
-
-    const Provider = providers[providerId]
-
-    const secrets = this.resolveSecrets(componentName, Provider)
-
-    const provider = new Provider({ ...props, secrets })
-
-    return new Storage(provider)
-  }
-
   private createStorages (): Storages {
     const storages: Storages = {}
 
-    for (const [componentName, props] of Object.entries(this.declaration))
-      storages[componentName] = this.createStorage(componentName, props)
+    for (const [name, declaration] of Object.entries(this.annotation))
+      storages[name] = this.createStorage(name, declaration)
 
     return storages
   }
 
-  private resolveSecrets (componentName: string,
-    Class: ProviderConstructor): Record<string, string | undefined> {
-    if (Class.SECRETS === undefined) return {}
+  private createStorage (name: string, declaration: Declaration): Storage {
+    const { provider: providerId, ...options } = declaration
+    const Provider: ProviderConstructor = providers[providerId]
+    const secrets = this.resolveSecrets(name, Provider)
+    const provider = new Provider(options, secrets)
+
+    return new Storage(provider)
+  }
+
+  private resolveSecrets (storageName: string,
+    Class: ProviderConstructor): ProviderSecrets {
+    if (Class.SECRETS === undefined)
+      return {}
 
     const secrets: Record<string, string | undefined> = {}
 
     for (const secret of Class.SECRETS) {
-      const variable = `${SERIALIZATION_PREFIX}_${componentName}_${secret.name}`.toUpperCase()
+      const variable = `${SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
       const value = process.env[variable]
 
       assert.ok(secret.optional === true || value !== undefined,

package/source/Provider.ts

@@ -1,9 +1,7 @@
 import { type Readable } from 'node:stream'
 import * as assert from 'node:assert'
 
-export interface ProviderSecrets {
-  secrets?: Record<string, string>
-}
+export type ProviderSecrets<K extends string = string> = Record<K | string, string | undefined>
 
 export interface ProviderSecret {
   readonly name: string
@@ -13,19 +11,23 @@ export interface ProviderSecret {
 export abstract class Provider<Options = void> {
   public static readonly SECRETS: readonly ProviderSecret[] = []
 
-  public constructor (props: Options & ProviderSecrets) {
+  public constructor (_: Options, secrets?: ProviderSecrets) {
     for (const { name, optional = false } of new.target.SECRETS)
-      assert.ok(optional || props.secrets?.[name] !== undefined, `Missing secret '${name}'`)
+      assert.ok(optional || secrets?.[name] !== undefined, `Missing secret '${name}'`)
   }
 
   public abstract get (path: string): Promise<Readable | null>
+
   public abstract put (path: string, filename: string, stream: Readable): Promise<void>
+
   public abstract delete (path: string): Promise<void>
+
   public abstract move (from: string, to: string): Promise<void>
 }
 
 export interface ProviderConstructor {
   readonly SECRETS: readonly ProviderSecret[]
   prototype: Provider
-  new(props: any): Provider
+
+  new (options: any, secrets?: ProviderSecrets): Provider
 }

package/source/Scanner.ts

@@ -1,6 +1,6 @@
 import { PassThrough, type TransformCallback } from 'node:stream'
 import { createHash } from 'node:crypto'
-import { negotiate } from '@toa.io/http'
+import { negotiate } from '@toa.io/agent'
 import { Err } from 'error-value'
 
 export class Scanner extends PassThrough {
@@ -15,7 +15,7 @@ export class Scanner extends PassThrough {
   private detected = false
   private readonly chunks: Buffer[] = []
 
-  public constructor (control?: TypeControl) {
+  public constructor (control?: ScanOptions) {
     super()
 
     this.claim = control?.claim
@@ -121,7 +121,7 @@ const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
 const ERR_TYPE_MISMATCH = Err('TYPE_MISMATCH')
 const ERR_NOT_ACCEPTABLE = Err('NOT_ACCEPTABLE')
 
-export interface TypeControl {
+export interface ScanOptions {
   claim?: string
   accept?: string
 }

package/source/Storage.test.ts

@@ -8,11 +8,12 @@ import { Storage } from './Storage'
 import { suites } from './test/util'
 import { type Entry } from './Entry'
 import { providers } from './providers'
+import type { ProviderConstructor } from './Provider'
 
 let storage: Storage
 let dir: string
 
-const { run, provider: providerId, ...props } = suites[0]
+const suite = suites[0]
 
 beforeAll(async () => {
   process.chdir(path.join(__dirname, 'test'))
@@ -21,8 +22,8 @@ beforeAll(async () => {
 beforeEach(() => {
   dir = '/' + randomUUID()
 
-  const Provider = providers[providerId]
-  const provider = new Provider(props)
+  const Provider: ProviderConstructor = providers[suite.provider]
+  const provider = new Provider(suite.options)
 
   storage = new Storage(provider)
 })

package/source/Storage.ts

@@ -4,7 +4,7 @@ import { decode, encode } from 'msgpackr'
 import { buffer, newid } from '@toa.io/generic'
 import { Err } from 'error-value'
 import { Scanner } from './Scanner'
-import type { TypeControl } from './Scanner'
+import type { ScanOptions } from './Scanner'
 import type { Provider } from './Provider'
 import type { Entry } from './Entry'
 
@@ -15,8 +15,8 @@ export class Storage {
     this.provider = provider
   }
 
-  public async put (path: string, stream: Readable, type?: TypeControl): Maybe<Entry> {
-    const scanner = new Scanner(type)
+  public async put (path: string, stream: Readable, options?: Options): Maybe<Entry> {
+    const scanner = new Scanner(options)
     const pipe = stream.pipe(scanner)
     const tempname = await this.transit(pipe)
 
@@ -27,7 +27,7 @@ export class Storage {
 
     await this.persist(tempname, id)
 
-    return await this.create(path, id, scanner.size, scanner.type)
+    return await this.create(path, id, scanner.size, scanner.type, options?.meta)
   }
 
   public async get (path: string): Maybe<Entry> {
@@ -160,14 +160,15 @@ export class Storage {
   }
 
   // eslint-disable-next-line max-params
-  private async create (path: string, id: string, size: number, type: string): Promise<Entry> {
+  private async create
+  (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
     const entry: Entry = {
       id,
       size,
       type,
       created: Date.now(),
       variants: [],
-      meta: {}
+      meta
     }
 
     const metafile = posix.join(path, entry.id)
@@ -235,4 +236,10 @@ interface Path {
   variant: string | null
 }
 
+type Meta = Record<string, string>
+
+interface Options extends ScanOptions {
+  meta?: Meta
+}
+
 export type Storages = Record<string, Storage>

package/source/deployment.ts

@@ -1,12 +1,14 @@
 import * as assert from 'node:assert'
 import { encode } from '@toa.io/generic'
 import { providers } from './providers'
+import { validateAnnotation } from './Annotation'
+import type { Annotation } from './Annotation'
 import type { Dependency, Variable } from '@toa.io/operations'
 import type { context } from '@toa.io/norm'
 
 export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
 
-export function deployment (instances: Instance[], annotation: Annotation): Dependency {
+export function deployment (instances: Instance[], annotation: unknown): Dependency {
   validate(instances, annotation)
 
   const value = encode(annotation)
@@ -18,16 +20,11 @@ export function deployment (instances: Instance[], annotation: Annotation): Depe
   }
 }
 
-function validate (instances: Instance[],
-  annotation: Annotation): asserts annotation is ValidatedAnnotation {
-  assert.ok(annotation !== undefined,
-    `Storages annotation is required by: '${instances
-    .map((i) => i.component.locator.id)
-    .join("', '")}'`)
+function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
+  validateAnnotation(annotation)
 
-  for (const instance of instances) contains(instance, annotation)
-
-  for (const { provider } of Object.values(annotation)) validateProviderId(provider)
+  for (const instance of instances)
+    contains(instance, annotation)
 }
 
 function contains (instance: Instance, annotation: Annotation): void {
@@ -37,21 +34,17 @@ function contains (instance: Instance, annotation: Annotation): void {
       `declared in '${instance.component.locator.id}'`)
 }
 
-export function validateProviderId (id: string | undefined): asserts id is keyof typeof providers {
-  assert.ok(typeof id === 'string' && id in providers, `Unknown storage provider '${id}'`)
-}
-
-function getSecrets (annotation: ValidatedAnnotation): Variable[] {
+function getSecrets (annotation: Annotation): Variable[] {
   const secrets: Variable[] = []
 
-  for (const [componentName, props] of Object.entries(annotation)) {
-    const Provider = providers[props.provider]
+  for (const [name, declaration] of Object.entries(annotation)) {
+    const Provider = providers[declaration.provider]
 
     for (const secret of Provider.SECRETS)
       secrets.push({
-        name: `${SERIALIZATION_PREFIX}_${componentName}_${secret.name}`.toUpperCase(),
+        name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
         secret: {
-          name: `toa-storages-${componentName}`,
+          name: `toa-storages-${name}`,
           key: secret.name,
           optional: secret.optional
         }
@@ -61,8 +54,4 @@ function getSecrets (annotation: ValidatedAnnotation): Variable[] {
   return secrets
 }
 
-type Annotation = Record<string, { [k: string]: unknown, provider: string }>
-type ValidatedAnnotation = Readonly<
-Record<string, { [k: string]: unknown, provider: keyof typeof providers }>
->
 export type Instance = context.Dependency<string[]>

package/source/providers/Declaration.ts

@@ -0,0 +1,10 @@
+import type { S3Options } from './S3'
+import type { FileSystemOptions } from './FileSystem'
+import type { TemporaryOptions } from './Temporary'
+
+export type Declaration =
+  ({ provider: 's3' } & S3Options)
+  | ({ provider: 'fs' } & FileSystemOptions)
+  | ({ provider: 'tmp' } & TemporaryOptions)
+  | ({ provider: 'mem' })
+  | ({ provider: 'test' } & TemporaryOptions)

package/source/providers/FileSystem.ts

@@ -1,8 +1,8 @@
 import { type Readable } from 'node:stream'
 import { dirname, join } from 'node:path'
 import fs from 'node:fs/promises'
-import assert from 'node:assert'
 import { Provider } from '../Provider'
+import type { ProviderSecrets } from '../Provider'
 
 export interface FileSystemOptions {
   path: string
@@ -11,10 +11,10 @@ export interface FileSystemOptions {
 export class FileSystem extends Provider<FileSystemOptions> {
   protected readonly path: string
 
-  public constructor (props: FileSystemOptions) {
-    super(props)
-    assert.ok(props.path, 'Missing path')
-    this.path = props.path
+  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
+    super(options, secrets)
+
+    this.path = options.path
   }
 
   public async get (path: string): Promise<Readable | null> {

package/source/providers/S3.test.ts

@@ -24,11 +24,10 @@ describe('S3 storage provider', () => {
 
     provider = new S3({
       bucket: TEST_BUCKET_NAME,
-      region: AWS_REGION,
-      secrets: {
-        ACCESS_KEY_ID: 'test-key',
-        SECRET_ACCESS_KEY: 'test-key-secret'
-      }
+      region: AWS_REGION
+    }, {
+      ACCESS_KEY_ID: 'test-key',
+      SECRET_ACCESS_KEY: 'test-key-secret'
     })
   })
 

package/source/providers/S3.ts

@@ -22,11 +22,13 @@ import type { ReadableStream } from 'node:stream/web'
 
 export interface S3Options {
   bucket: string
-  prefix?: string
   region?: string
+  prefix?: string
   endpoint?: string
 }
 
+type S3Secrets = ProviderSecrets<'ACCESS_KEY_ID' | 'SECRET_ACCESS_KEY'>
+
 export class S3 extends Provider<S3Options> {
   public static override readonly SECRETS: readonly ProviderSecret[] = [
     { name: 'ACCESS_KEY_ID', optional: true },
@@ -36,29 +38,32 @@ export class S3 extends Provider<S3Options> {
   protected readonly bucket: string
   protected readonly client: S3Client
 
-  public constructor (props: S3Options & ProviderSecrets) {
-    super(props)
+  public constructor (options: S3Options, secrets?: S3Secrets) {
+    super(options)
 
-    assert.ok(props.bucket, 'Missing bucket name')
-    this.bucket = props.bucket
+    this.bucket = options.bucket
 
     const s3Config: S3ClientConfigType = {
       retryMode: 'adaptive',
       ...nodeNativeFetch
     }
 
-    if (props.endpoint !== undefined) {
-      s3Config.forcePathStyle = props.endpoint.startsWith('http://')
-      s3Config.endpoint = props.endpoint
+    if (options.endpoint !== undefined) {
+      s3Config.forcePathStyle = options.endpoint.startsWith('http://')
+      s3Config.endpoint = options.endpoint
     }
 
-    if (props.region !== undefined) s3Config.region = props.region
+    if (options.region !== undefined) s3Config.region = options.region
+
+    if (typeof secrets?.ACCESS_KEY_ID === 'string') {
+      assert.ok(secrets.SECRET_ACCESS_KEY !== undefined,
+        'SECRET_ACCESS_KEY is required if ACCESS_KEY_ID is provided')
 
-    if (typeof props.secrets?.ACCESS_KEY_ID === 'string')
       s3Config.credentials = {
-        accessKeyId: props.secrets.ACCESS_KEY_ID,
-        secretAccessKey: props.secrets.SECRET_ACCESS_KEY
+        accessKeyId: secrets.ACCESS_KEY_ID,
+        secretAccessKey: secrets.SECRET_ACCESS_KEY
       }
+    }
 
     this.client = new S3Client(s3Config)
 

package/source/providers/Temporary.ts

@@ -1,14 +1,16 @@
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-
 import { FileSystem } from './FileSystem'
+import type { ProviderSecrets } from '../Provider'
 
 export interface TemporaryOptions {
-  prefix?: string
+  directory: string
 }
 
 export class Temporary extends FileSystem {
-  public constructor (props: TemporaryOptions) {
-    super({ path: join(tmpdir(), props.prefix ?? '') })
+  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
+    const path = join(tmpdir(), options.directory)
+
+    super({ path }, secrets)
   }
 }

package/source/providers/Test.ts

@@ -1,5 +1,5 @@
 import { Temporary, type TemporaryOptions } from './Temporary'
-import type { ProviderSecret } from '../Provider'
+import type { ProviderSecret, ProviderSecrets } from '../Provider'
 
 export class Test extends Temporary {
   public static override readonly SECRETS: readonly ProviderSecret[] = [
@@ -7,7 +7,7 @@ export class Test extends Temporary {
     { name: 'PASSWORD' }
   ]
 
-  public constructor (props: TemporaryOptions) {
-    super(props)
+  public constructor (options: TemporaryOptions, secrets?: ProviderSecrets) {
+    super(options, secrets)
   }
 }