@toa.io/extensions.storages 1.0.0-alpha.29 → 1.0.0-alpha.291

package/CHANGELOG.md

@@ -0,0 +1,27 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+# [1.0.0-alpha.291](https://github.com/toa-io/toa/compare/v1.0.0-alpha.290...v1.0.0-alpha.291) (2026-09-07)
+
+**Note:** Version bump only for package @toa.io/extensions.storages
+
+
+
+
+
+# [1.0.0-alpha.289](https://github.com/toa-io/toa/compare/v1.0.0-alpha.288...v1.0.0-alpha.289) (2026-09-07)
+
+* A deploy moves only what changed, and a component sees none of the runtime's environment (#1073) ([f38e3db](https://github.com/toa-io/toa/commit/f38e3db533f24db866c57bfa1ef294eff1eaf499)), closes [#1073](https://github.com/toa-io/toa/issues/1073) [#1064](https://github.com/toa-io/toa/issues/1064) [#1066](https://github.com/toa-io/toa/issues/1066) [#1067](https://github.com/toa-io/toa/issues/1067) [#1068](https://github.com/toa-io/toa/issues/1068) [#1069](https://github.com/toa-io/toa/issues/1069) [#1071](https://github.com/toa-io/toa/issues/1071) [#1070](https://github.com/toa-io/toa/issues/1070) [#1072](https://github.com/toa-io/toa/issues/1072)
+
+### BREAKING CHANGES
+
+* a component that read `process.env.TOA_*` reads `context` instead;
+  `echo(input)` no longer substitutes from the environment; a bash operation sees no
+  `TOA_*`; images no longer set `USER node` — see migrations/289.md.
+
+
+# [1.0.0-alpha.287](https://github.com/toa-io/toa/compare/v1.0.0-alpha.286...v1.0.0-alpha.287) (2026-09-06)
+
+**Note:** Version bump only for package @toa.io/extensions.storages

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "1.0.0-alpha.29",
+  "version": "1.0.0-alpha.291",
+  "type": "module",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -16,27 +17,21 @@
   "publishConfig": {
     "access": "public"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node"
-  },
   "scripts": {
-    "test": "jest",
+    "test": "echo \"Error: run tests from root\" && exit 1",
     "transpile": "npx tsc"
   },
-  "peerDependencies": {
-    "dotenv": "*"
+  "devDependencies": {
+    "dotenv": "17.4.2"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.481.0",
-    "@aws-sdk/lib-storage": "3.481.0",
-    "@toa.io/agent": "1.0.0-alpha.29",
-    "@toa.io/generic": "1.0.0-alpha.29",
-    "@toa.io/schemas": "1.0.0-alpha.29",
-    "error-value": "0.3.0",
-    "matchacho": "0.6.0",
-    "msgpackr": "1.10.1",
-    "smithy-node-native-fetch": "1.0.6"
+    "@aws-sdk/client-s3": "3.1125.0",
+    "@aws-sdk/lib-storage": "3.1125.0",
+    "@toa.io/generic": "1.0.0-alpha.291",
+    "@toa.io/schemas": "1.0.0-alpha.291",
+    "cloudinary": "2.11.0",
+    "negotiator": "1.1.0",
+    "openspan": "1.0.0-alpha.286"
   },
-  "gitHead": "186d0e17c12a9016b8076edf925963cbb32b9219"
+  "gitHead": "bcd8d8d07bd3e64fc826af8719472404662cc37f"
 }

package/readme.md

@@ -6,15 +6,11 @@ Shared BLOB storage.
 
 BLOBs are stored with the meta-information object (Entry) having the following properties:
 
-- `id` - checksum
-- `size` - size in bytes
-- `type` - MIME type
-- `created` - creation timestamp (UNIX time, ms)
-- `variants` - array of:
-  - `name` - unique name
-  - `size` - size in bytes
-  - `type` - variant MIME type
-- `meta` - object with application-specific information, empty by default
+- `size`: size in bytes
+- `type`: MIME type
+- `checksum`: content checksum
+- `created`: creation timestamp (ISO 8601)
+- `attributes`: `Record<string, string>` with application-specific information, empty by default
 
 ### Example
 
@@ -22,12 +18,7 @@ BLOBs are stored with the meta-information object (Entry) having the following p
 id: eecd837c
 type: image/jpeg
 created: 1698004822358
-variants:
-  - name: thumbnail.jpeg
-    type: image/jpeg
-  - name: thumbnail.webp
-    type: image/webp
-meta:
+attributes:
   face: true
   nudity: false
 ```
@@ -38,8 +29,8 @@ The Storages extension provides `storages` aspect,
 containing named Storage instances, according to the annotation.
 
 ```javascript
-async function effect (_, context) {
-  await context.storages.photos.fetch('/path/to/b4f577e0.thumbnail.jpeg')
+async function effect(_, context) {
+  await context.storages.photos.get('/path/to/b4f577e0.thumbnail.jpeg')
 }
 ```
 
@@ -49,11 +40,11 @@ async function effect (_, context) {
 
 #### `async put(path: string, stream: Readable, options?: Options): Maybe<Entry>`
 
-```
+```ts
 interface Options {
   claim?: string
   accept?: string
-  meta?: Record<string, string>
+  attributes?: Record<string, string>
 }
 ```
 
@@ -75,13 +66,13 @@ are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/
 
 See [source](source/Scanner.ts).
 
-#### `async get(path: string): Maybe<Entry>`
+#### `async head(path: string): Maybe<Entry>`
 
 Get an entry.
 
 If the entry does not exist, a `NOT_FOUND` error is returned.
 
-#### `async fetch(path: string): Maybe<Readable>`
+#### `async get(path: string): Maybe<Readable>`
 
 Fetch the BLOB specified by `path`. If the path does not exist, a `NOT_FOUND` error is returned.
 
@@ -110,18 +101,6 @@ await storage.move('/path/to/eecd837c', './sub/eecd837c')
 await storage.move('/path/to/eecd837c', './sub/')
 ```
 
-#### `async entries(path: string): Entry[]`
-
-Get a list of entries under the `path`.
-
-#### `async diversify(path: string, name: string, stream: Readable): Maybe<void>`
-
-Add or replace a `name` variant of the entry specified by `path`.
-
-#### `async annotate(path: string, key: string, value: any): Maybe<void>`
-
-Set a `key` property in the `meta` of the entry specified by `path`.
-
 ## Providers
 
 Storage uses underlying providers to store BLOBs and entries.
@@ -152,40 +131,114 @@ and [`toa env`](/runtime/cli/readme.md#env)
 for local environment.
 `endpoint` parameter is optional.
 
-### Filesystem
+### DigitalOcean Spaces
 
 Annotation format is:
 
 ```yaml
 storages:
-  photos@dev:
-    provider: fs
-    path: /var/my-storage
+  photos:
+    provider: spaces
+    space: my-space
+    region: fra1
 ```
 
-### Temporary
+`region` is the datacenter slug, the endpoint is derived from it.
 
-Filesystem using OS temporary directory.
+Secrets `ACCESS_KEY_ID` and `SECRET_ACCESS_KEY` are required.
+Keys are created in the DigitalOcean control panel,
+under Spaces Object Storage, on the Access Keys tab.
+
+### Cloudinary
+
+[Cloudinary](https://cloudinary.com) provider is used to store and transform media files.
+
+Objects stored in the Cloudinary storage can be requested with transformations,
+using dot-separated path extensions: `/path/to/eecd837c.{extension}.{extension}`.
+
+Transformation is defined by the `extension` property, which is a regular expression with named
+groups.
+The named groups are used to replace the placeholders in the `transform` object.
+
+`transform` object is an element of
+the [Cloudinary `transformation` array](https://cloudinary.com/documentation/node_image_manipulation#apply_common_image_transformations).
+
+> - `zoom` should be specified as integer [0–100].
+> - `format` value `jpeg` is converted to `jpg`.
+> - `^` and `$` are added to the regular expression automatically, unless they are already present.
 
 Annotation format is:
 
 ```yaml
 storages:
-  photos@dev:
-    provider: tmp
-    directory: my-app-tmp
+  media:
+    provider: cloudinary
+    environment: my-cloud
+    type: image # image or video
+    prefix: my-app
+    transformations:
+      - extension: (?<width>\d*)x(?<height>\d*)(z(?<zoom>\d*))?
+        transformation:
+          width: <width>
+          height: <height>
+          zoom: <zoom>
+          crop: thumb
+          gravity: face
+        optional: true
+      - extension: (?<format>jpeg|webp)
+        transformation:
+          fetch_format: <format>
+```
+
+Path extensions must be specified in the same order as in the annotation.
+
+For the example above:
+
+- `/path/to/eecd837c.100x100.jpeg`: matches
+- `/path/to/eecd837c.jpeg.100x100`: does not match
+
+If a path extension is not matched,
+or if at least one of the transformations is not matched (unless it is `optional`),
+then an error is returned.
+
+Secrets:
+
+- `API_KEY`
+- `API_SECRET`
+
+### Filesystem
+
+Annotation format is:
+
+```yaml
+storages:
+  photos:
+    provider: fs
+    path: /var/my-photos
 ```
 
-### Memory
+[Kubernetes PVC](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) can be mounted to
+the storage:
 
-In-memory non-persistent storage.
+```yaml
+storages:
+  photos:
+    provider: fs
+    path: /var/my-photos
+    claim: photos-pvc
+```
 
-Annotation value format is:
+### Temporary
+
+Filesystem using OS temporary directory.
+
+Annotation format is:
 
 ```yaml
 storages:
   photos@dev:
-    provider: mem
+    provider: tmp
+    directory: my-app-tmp
 ```
 
 ## Deduplication
@@ -196,7 +249,7 @@ Variants, on the other hand, are not deduplicated across different entries.
 
 Underlying directory structure:
 
-```
+```text
 /temp
   c28f4dfd            # random id
 /blobs

package/schemas/annotation.cos.yaml

@@ -1,10 +1,11 @@
 type: object
 patternProperties:
-  '^[a-z0-9_]{1,32}$':
+  ^[a-z0-9_]{1,32}$:
     oneOf:
       - $ref: s3
+      - $ref: spaces
+      - $ref: cloudinary
       - $ref: fs
       - $ref: tmp
-      - $ref: mem
       - $ref: test
 additionalProperties: false

package/schemas/cloudinary.cos.yaml

@@ -0,0 +1,38 @@
+type: object
+properties:
+  provider:
+    const: cloudinary
+  environment:
+    type: string
+  type:
+    enum:
+      - image
+      - video
+  prefix:
+    type: string
+  eager:
+    type: array
+    items:
+      type: object
+  transformations:
+    type: array
+    items:
+      type: object
+      properties:
+        extension:
+          type: string
+        transformation:
+          anyOf:
+            - type: object
+            - type: array
+              items:
+                type: object
+        optional:
+          type: boolean
+      required:
+        - extension
+        - transformation
+required:
+  - environment
+  - type
+additionalProperties: false

package/schemas/fs.cos.yaml

@@ -5,5 +5,9 @@ properties:
   path:
     type: string
     maxLength: 4096
-required: [provider, path]
+  claim:
+    type: string
+required:
+  - provider
+  - path
 additionalProperties: false

package/schemas/s3.cos.yaml

@@ -8,9 +8,10 @@ properties:
     maxLength: 63
   region:
     type: string
-  prefix:
-    type: string
   endpoint:
     type: string
     format: uri
-required: [provider, bucket]
+required:
+  - provider
+  - bucket
+additionalProperties: false

package/schemas/spaces.cos.yaml

@@ -0,0 +1,16 @@
+type: object
+properties:
+  provider:
+    const: spaces
+  space:
+    type: string
+    minLength: 3
+    maxLength: 63
+  region:
+    type: string
+    minLength: 1
+required:
+  - provider
+  - space
+  - region
+additionalProperties: false

package/schemas/test.cos.yaml

@@ -4,5 +4,7 @@ properties:
     const: test
   directory:
     type: string
-required: [provider, directory]
+required:
+  - provider
+  - directory
 additionalProperties: false

package/schemas/tmp.cos.yaml

@@ -1,9 +1,10 @@
 type: object
 properties:
   provider:
-    type: string
     const: tmp
   directory:
     type: string
-required: [provider, directory]
+required:
+  - provider
+  - directory
 additionalProperties: false

package/source/Annotation.ts

@@ -1,12 +1,14 @@
 import assert from 'node:assert'
-import { providers } from './providers'
-import * as schemas from './schemas'
-import type { Declaration } from './providers'
+import { providers } from './providers/index.js'
+import * as schemas from './schemas.js'
+import type { Declaration } from './providers/index.js'
 import type { Schema } from '@toa.io/schemas'
 
 export type Annotation = Record<string, Declaration>
 
-export function validateAnnotation (annotation: unknown): asserts annotation is Annotation {
+export function validateAnnotation(
+  annotation: unknown
+): asserts annotation is Annotation {
   try {
     schemas.annotation.validate(annotation)
   } catch (error) {
@@ -20,19 +22,30 @@ export function validateAnnotation (annotation: unknown): asserts annotation is
 /*
 It is required because `oneOf` schema is used for the annotation validation.
  */
-function explain (annotation: unknown): void {
-  assert.ok(typeof annotation === 'object' && annotation !== null,
-    'TOA_STORAGES is not an object')
+function explain(annotation: unknown): void {
+  assert.ok(
+    typeof annotation === 'object' && annotation !== null,
+    'TOA_STORAGES is not an object'
+  )
 
   for (const declaration of Object.values(annotation)) {
-    assert.ok(typeof declaration === 'object' && declaration !== null &&
-      declaration.provider in providers,
-      `Unknown provider '${declaration.provider}'`)
-
-    assert.ok(declaration.provider in schemas,
-      `No schema for provider '${declaration.provider}'`)
-
-    const schema: Schema<Declaration> = schemas[declaration.provider as keyof typeof providers]
+    assert.ok(
+      typeof declaration === 'object' &&
+        declaration !== null &&
+        declaration.provider in providers,
+      `Unknown provider '${declaration.provider}'`
+    )
+
+    assert.ok(
+      declaration.provider in schemas,
+      `No schema for provider '${declaration.provider}'`
+    )
+
+    const provider = declaration.provider as keyof typeof providers
+
+    // the declaration names the provider
+    // oxlint-disable-next-line import/namespace
+    const schema: Schema<Declaration> = schemas[provider]
 
     schema.validate(declaration, `Storage '${declaration.provider}' annotation`)
   }

package/source/Aspect.ts

@@ -1,19 +1,26 @@
 import assert from 'node:assert'
-import { Connector, type extensions } from '@toa.io/core'
-import { type Storage, type Storages } from './Storage'
+import { Connector } from '@toa.io/core'
+import type { extensions } from '@toa.io/core/types'
+import { type Storage, type Storages } from './Storage.js'
 
 export class Aspect extends Connector implements extensions.Aspect {
   public readonly name = 'storages'
 
-  private readonly storages: Storages
+  private readonly create: () => Promise<Storages>
+  private storages: Storages = {}
 
-  public constructor (storages: Storages) {
+  /** The storages are made as the aspect connects: their providers load then, not before. */
+  public constructor(create: () => Promise<Storages>) {
     super()
 
-    this.storages = storages
+    this.create = create
   }
 
-  public invoke (name: string, method: keyof Storage, ...args: unknown[]): unknown {
+  protected override async open(): Promise<void> {
+    this.storages = await this.create()
+  }
+
+  public invoke(name: string, method: keyof Storage, ...args: unknown[]): unknown {
     const storage = this.storages[name]
 
     assert.ok(storage !== undefined, `Storage '${name}' is not defined`)

package/source/Entry.ts

@@ -1,14 +1,16 @@
-export interface Entry {
-  id: string
-  size: number
-  type: string
-  created: number
-  variants: Variant[]
-  meta: Record<string, unknown>
-}
+import type { Readable } from 'node:stream'
 
-interface Variant {
-  name: string
-  size: number
+export type Entry = { id: string } & Metadata
+export type Stream = { stream: Readable } & Metadata
+
+export interface Metadata {
   type: string
+  size: number | null
+  checksum: string
+  created: string
+  attributes: Attributes
+  range?: string
+  partial?: boolean
 }
+
+export type Attributes = Record<string, string>

package/source/Factory.ts

@@ -1,64 +1,70 @@
 import assert from 'node:assert'
-import { decode } from '@toa.io/generic'
-import { providers } from './providers'
-import { Storage, type Storages } from './Storage'
-import { Aspect } from './Aspect'
-import { SERIALIZATION_PREFIX } from './deployment'
-import { validateAnnotation } from './Annotation'
-import type { Declaration } from './providers'
-import type { Annotation } from './Annotation'
-import type { ProviderConstructor, ProviderSecrets } from './Provider'
+import { console } from 'openspan'
+import { providers } from './providers/index.js'
+import { Storage, type Storages } from './Storage.js'
+import { Aspect } from './Aspect.js'
+import { ENV_PREFIX } from './deployment.js'
+import { validateAnnotation } from './Annotation.js'
+import { environment } from '@toa.io/generic'
+import type { Constructor } from './Provider.js'
+import type { Declaration } from './providers/index.js'
+import type { Annotation } from './Annotation.js'
+import type { Secrets } from './Secrets.js'
 
 export class Factory {
   private readonly annotation: Annotation
 
-  public constructor () {
-    const env = process.env.TOA_STORAGES
+  public constructor() {
+    const env = environment.get(ENV_PREFIX)
 
-    assert.ok(env !== undefined, 'TOA_STORAGES is not defined')
+    assert.ok(env !== undefined, `${ENV_PREFIX} is not defined`)
 
-    this.annotation = decode(env)
+    this.annotation = JSON.parse(env)
 
     validateAnnotation(this.annotation)
   }
 
-  public aspect (): Aspect {
-    const storages = this.createStorages()
-
-    return new Aspect(storages)
+  public aspect(): Aspect {
+    return new Aspect(() => this.createStorages())
   }
 
-  private createStorages (): Storages {
+  private async createStorages(): Promise<Storages> {
     const storages: Storages = {}
 
     for (const [name, declaration] of Object.entries(this.annotation))
-      storages[name] = this.createStorage(name, declaration)
+      storages[name] = await this.createStorage(name, declaration)
 
     return storages
   }
 
-  private createStorage (name: string, declaration: Declaration): Storage {
-    const { provider: providerId, ...options } = declaration
-    const Provider: ProviderConstructor = providers[providerId]
+  private async createStorage(name: string, declaration: Declaration): Promise<Storage> {
+    const { provider: id, ...options } = declaration
+    const Provider: Constructor = await providers[id]()
     const secrets = this.resolveSecrets(name, Provider)
     const provider = new Provider(options, secrets)
 
-    return new Storage(provider)
+    console.debug('Storage created', {
+      name,
+      provider: id,
+      ...(provider.root === undefined ? undefined : { root: provider.root })
+    })
+
+    return new Storage(provider, { name, provider: id })
   }
 
-  private resolveSecrets (storageName: string,
-    Class: ProviderConstructor): ProviderSecrets {
-    if (Class.SECRETS === undefined)
-      return {}
+  private resolveSecrets(storageName: string, Class: Constructor): Secrets {
+    if (Class.SECRETS === undefined) return {}
 
     const secrets: Record<string, string | undefined> = {}
 
     for (const secret of Class.SECRETS) {
-      const variable = `${SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
-      const value = process.env[variable]
+      const variable = `${ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
+      const value = environment.get(variable)
 
-      assert.ok(secret.optional === true || value !== undefined,
-        `'${variable}' is not defined`)
+      assert.ok(
+        secret.optional === true || value !== undefined,
+        `'${variable}' is not defined`
+      )
 
       secrets[secret.name] = value
     }