@toa.io/extensions.storages 0.22.0 → 0.23.0-dev.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.storages",
-  "version": "0.22.0",
+  "version": "0.23.0-dev.0",
   "description": "Toa Storages",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -25,14 +25,18 @@
     "transpile": "rm -rf transpiled && npx tsc"
   },
   "devDependencies": {
-    "@toa.io/match": "0.3.0",
-    "@toa.io/streams": "0.1.0-alpha.2",
-    "@types/fs-extra": "11.0.3"
+    "@toa.io/streams": "0.23.0-dev.0",
+    "@types/fs-extra": "11.0.3",
+    "dotenv": "16.3.1"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "3.435.0",
+    "@aws-sdk/lib-storage": "3.437.0",
     "@toa.io/generic": "0.20.0-alpha.2",
+    "error-value": "0.3.0",
     "fs-extra": "11.1.1",
+    "matchacho": "0.3.5",
     "msgpackr": "1.9.9"
   },
-  "gitHead": "c463348c8eb54a43f7755ec8aeba9acafb56d53b"
+  "gitHead": "19d4af8ff3b8a1a8f191644f44113c88de4bb404"
 }

package/readme.md

@@ -47,16 +47,26 @@ async function effect (_, context) {
 
 > `Maybe<T> = T | Error`
 
-#### `async put(path: string, stream: Readable, type?: string): Maybe<Entry>`
+#### `async put(path: string, stream: Readable, type?: TypeControl): Maybe<Entry>`
+
+```
+interface TypeControl {
+  claim?: string
+  accept?: string
+}
+```
 
 Add a BLOB to the storage and create an entry under specified `path`.
 
 BLOB type is identified
 using [magick numbers](https://github.com/sindresorhus/file-type).
-If the `type` argument is specified and does not match the BLOB type, then a `TYPE_MISMATCH` error
-is returned.
-If the BLOB type cannot be identified
-and the value of `type` is not in the list of known types, then the given value is used.
+
+If the `type` argument is specified and the value of the `claim` does not match the detected BLOB type, then
+a `TYPE_MISMATCH` error is returned.
+If the BLOB type cannot be identified and the value of the `claim` is not in the list of known types, then the given
+value is used.
+If the list of [acceptable types](https://datatracker.ietf.org/doc/html/rfc2616#section-14.1) is passed and the type of
+the BLOB does not match any of its values, then a `NOT_ACCEPTABLE` error is returned.
 
 Known types
 are: `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/heic`, `image/jxl`, `image/avif`.
@@ -89,7 +99,7 @@ Delete the entry specified by `path`.
 
 Get ordered list of `id`s of entries in under the `path`.
 
-#### `async reorder(path: string, ids: string[]): Maybe<void>`
+#### `async permute(path: string, ids: string[]): Maybe<void>`
 
 Reorder entries under the `path`.
 
@@ -116,15 +126,19 @@ Set a `key` property in the `meta` of the entry specified by `path`.
 
 Storage uses underlying providers to store BLOBs and entries.
 
-Custom providers are not supported yet.
+Custom providers are not supported.
 
 ### Amazon S3
 
-Annotation value format is `s3://{region}/{bucket}`.
+Annotation value formats is `s3://{region}/{bucket}?endpoint={endpoint}`.
 
 Requires secrets for the access key and secret key.
+See [`toa conceal`](/runtime/cli/readme.md#conceal) for deployment
+and [`toa env`](/runtime/cli/readme.md#env)
+for local environment.
+`endpoint` parameter is optional.
 
-`s3://us-east-1/my-bucket`
+`s3://us-east-1/my-bucket?endpoint=http://s3.my-instance.com:4566`
 
 ### Filesystem
 
@@ -142,8 +156,8 @@ Annotation value format is `tmp:///{path}`.
 
 ## Deduplication
 
-BLOBs are stored in the underlying storage with their checksum as the key, ensuring that identical BLOBs
-are stored only once.
+BLOBs are stored in the underlying storage with their checksum as the key, ensuring that identical
+BLOBs are stored only once.
 Variants, on the other hand, are not deduplicated across different entries.
 
 Underlying directory structure:
@@ -191,5 +205,6 @@ storages:
 
 ## Secrets
 
-Secrets declared by storage providers can be deployed by [`toa conceal`](/runtime/cli/readme.md#conceal),
+Secrets declared by storage providers can be deployed
+by [`toa conceal`](/runtime/cli/readme.md#conceal),
 or set locally by [`toa env`](/runtime/cli/readme.md#env).

package/source/Scanner.ts

@@ -1,5 +1,7 @@
 import { PassThrough, type TransformCallback } from 'node:stream'
 import { createHash } from 'node:crypto'
+import { negotiate } from '@toa.io/http'
+import { Err } from 'error-value'
 
 export class Scanner extends PassThrough {
   public size = 0
@@ -7,15 +9,17 @@ export class Scanner extends PassThrough {
   public error: Error | null = null
 
   private readonly hash = createHash('md5')
-  private readonly assertion: string | undefined
+  private readonly claim?: string
+  private readonly accept?: string
   private position = 0
   private detected = false
   private readonly chunks: Buffer[] = []
 
-  public constructor (assertion?: string) {
+  public constructor (control?: TypeControl) {
     super()
 
-    this.assertion = assertion
+    this.claim = control?.claim
+    this.accept = control?.accept
   }
 
   public digest (): string {
@@ -47,33 +51,47 @@ export class Scanner extends PassThrough {
   }
 
   private complete (): void {
-    this.detected = true
-
     const header = Buffer.concat(this.chunks).toString('hex')
 
     const signature = SIGNATURES
       .find(({ hex, off }) => header.slice(off, off + hex.length) === hex)
 
-    this.verify(signature)
+    const type = signature?.type ?? this.claim
 
-    const value = signature?.type ?? this.assertion
+    if (type !== undefined) {
+      this.match(type)
+      this.type = type
+    }
 
-    if (value !== undefined)
-      this.type = value
+    this.verify(signature)
+    this.detected = true
   }
 
   private verify (signature: Signature | undefined): void {
-    if (this.assertion === undefined || this.assertion === 'application/octet-stream')
+    if (this.claim === undefined || this.claim === 'application/octet-stream')
       return
 
     const mismatch = signature === undefined
-      ? KNOWN_TYPES.has(this.assertion)
-      : this.assertion !== signature.type
+      ? KNOWN_TYPES.has(this.claim)
+      : this.claim !== signature.type
 
-    if (mismatch) {
-      this.error = ERR_TYPE_MISMATCH
-      this.end()
-    }
+    if (mismatch)
+      this.interrupt(ERR_TYPE_MISMATCH)
+  }
+
+  private match (type: string): void {
+    if (this.accept === undefined)
+      return
+
+    const unacceptable = negotiate(this.accept, [type]) === null
+
+    if (unacceptable)
+      this.interrupt(ERR_NOT_ACCEPTABLE)
+  }
+
+  private interrupt (error: Error): void {
+    this.error = error
+    this.end()
   }
 }
 
@@ -100,7 +118,13 @@ const HEADER_SIZE = SIGNATURES
 
 const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
 
-const ERR_TYPE_MISMATCH = new Error('TYPE_MISMATCH')
+const ERR_TYPE_MISMATCH = Err('TYPE_MISMATCH')
+const ERR_NOT_ACCEPTABLE = Err('NOT_ACCEPTABLE')
+
+export interface TypeControl {
+  claim?: string
+  accept?: string
+}
 
 interface Signature {
   hex: string