@toa.io/extensions.exposition 1.0.0-alpha.60 → 1.0.0-alpha.61

package/components/octets.storage/manifest.toa.yaml

@@ -10,6 +10,7 @@ operations:
       storage*: string
       request*: ~
       accept: string
+      limit: number
       meta: <string>
       trust: ~ # array of strings or regular expressions
     errors:

package/components/octets.storage/operations/get.js

@@ -1,7 +1,7 @@
 'use strict'
 
-function get (input, context) {
-  return context.storages[input.storage].get(input.path)
+async function get (input, context) {
+  return await context.storages[input.storage].get(input.path)
 }
 
 exports.computation = get

package/components/octets.storage/operations/store.js

@@ -5,16 +5,38 @@ const { Err } = require('error-value')
 const { match } = require('matchacho')
 
 async function store (input, context) {
-  const { storage, request, accept, trust } = input
+  const { storage, request, accept, limit, trust } = input
   const path = request.url
   const claim = request.headers['content-type']
   const meta = parseMeta(request.headers['content-meta'])
-  const body = await download(request, trust)
+  const location = request.headers['content-location']
+
+  /** @type {Readable} */
+  let body = request
+
+  const options = { claim, accept, meta }
+
+  if (location !== undefined) {
+    const length = Number.parseInt(request.headers['content-length'])
+
+    if (length !== 0)
+      return ERR_LENGTH
+
+    if (!trusted(location, trust))
+      return ERR_UNTRUSTED
+
+    body = await download(location)
+
+    if (body instanceof Error)
+      return body
+
+    options.origin = location
+  }
 
-  if (body instanceof Error)
-    return body
+  if (limit !== undefined)
+    options.limit = limit
 
-  return context.storages[storage].put(path, body, { claim, accept, meta })
+  return context.storages[storage].put(path, body, options)
 }
 
 /**
@@ -41,25 +63,10 @@ function parseMeta (values) {
 }
 
 /**
- * @param {Request} request
- * @param {Trust | undefined} trust
- * @return {import('node:stream').Readable | Error}
+ * @param {string} location
+ * @return {Readable | Error}
  */
-async function download (request, trust) {
-  /** @type {string | undefined} */
-  const location = request.headers['content-location']
-
-  if (location === undefined)
-    return request
-
-  const length = Number.parseInt(request.headers['content-length'])
-
-  if (length !== 0)
-    return ERR_LENGTH
-
-  if (!trusted(location, trust))
-    return ERR_UNTRUSTED
-
+async function download (location) {
   const response = await fetch(location)
 
   if (!response.ok)
@@ -111,3 +118,4 @@ const ERR_UNAVAILABLE = Err('LOCATION_UNAVAILABLE', 'Location is not available')
 exports.effect = store
 
 /** @typedef {Array<string | RegExp>} Trust */
+/** @typedef {import('node:stream').Readable} Readable */

package/documentation/octets.md

@@ -25,10 +25,7 @@ the request is rejected with a `415 Unsupported Media Type` response.
 
 The value of the directive is `null` or an object with the following properties:
 
-- `limit`: a number of bytes (or
-  a [string with units](https://www.npmjs.com/package/bytes#bytesparsestringnumber-value-numbernull))
-  to limit the size of the uploaded content
-  (default is 64MB, which should be enough for everyone ©).
+- `limit`: [maximum size](#stream-size-limit) of the incoming stream.
 - `accept`: a media type or an array of media types that are acceptable.
   If the `accept` property is not specified, any media type is acceptable (which is the default).
 - `workflow`: [workflow](#workflows) to be executed once the content is successfully stored.
@@ -70,6 +67,23 @@ meta:
 
 If the Entry already exists, the `content-meta` header is ignored.
 
+### Stream size limit
+
+The `limit` property can be used to set the maximum size of the incoming stream in bytes.
+
+The property value can be specified as a number
+(representing bytes) or a string that combines a number with a unit (e.g., `1MB`).
+Both [binary and decimal prefixes](https://en.wikipedia.org/wiki/Binary_prefix) are supported.
+If the prefix or unit is specified _incorrectly_ (e.g., `1mb`),
+it will default to a binary prefix interpretation.
+
+- `1b`, `1B`: 1 byte
+- `1KB`: 1000 bytes
+- `1KiB`: 1024 bytes
+- `1kb`: 1024 bytes
+
+The default value is `64MiB`.
+
 ### Downloading external content
 
 The `octets:store` directive can be used to download external content:
@@ -85,6 +99,9 @@ Requests with `content-location` header must have an empty body (`content-length
 Target origin must be allowed by the `trust` property,
 which can contain a list of trusted origins or regular expressions to match the full URL.
 
+URL of the downloaded content is stored in the `origin` property of
+the [Entry](/extensions/storages/readme.md#entry).
+
 ```yaml
 /images:
   octets:context: images

package/features/cors.feature

@@ -20,7 +20,7 @@ Feature: CORS Support
       """
       204 No Content
       access-control-allow-origin: https://hello.world
-      access-control-allow-methods: GET, POST, PUT, PATCH, DELETE
+      access-control-allow-methods: GET, POST, PUT, PATCH, DELETE, LOCK, UNLOCK
       access-control-allow-headers: accept, authorization, content-type, etag, if-match, if-none-match
       access-control-allow-credentials: true
       access-control-max-age: 3600

package/features/methods.feature

@@ -0,0 +1,47 @@
+Feature: Supported methods
+
+  Scenario Outline: <method> is supported
+    Given the annotation:
+      """yaml
+      /:
+        <method>:
+          dev:stub:
+            hello: world
+      """
+    And the `greeter` is running with the following manifest:
+      """yaml
+      exposition:
+        /:
+          <method>:
+            dev:stub:
+              hello: world
+      """
+    Examples:
+      | method |
+      | GET    |
+      | POST   |
+      | PUT    |
+      | DELETE |
+      | PATCH  |
+      | LOCK   |
+      | UNLOCK |
+
+  Scenario: CORS allowed methods
+    Given the annotation:
+      """yaml
+      /:
+        GET:
+          dev:stub:
+            hello: world
+      """
+    When the following request is received:
+      """
+      OPTIONS / HTTP/1.1
+      host: nex.toa.io
+      origin: https://hello.world
+      """
+    Then the following reply is sent:
+      """
+      204 No Content
+      access-control-allow-methods: GET, POST, PUT, PATCH, DELETE, LOCK, UNLOCK
+      """

package/features/octets.download.feature

@@ -14,7 +14,8 @@ Feature: Download and store
               - https://github.com
         /*:
           GET:
-            octets:fetch: ~
+            octets:fetch:
+              meta: true
       """
 
     When the following request is received:
@@ -43,6 +44,24 @@ Feature: Download and store
       200 OK
       content-type: image/png
       content-length: 1288
+      etag: "${{ id }}"
+      """
+
+    # origin is stored in the entry
+    When the following request is received:
+      """
+      GET /${{ id }} HTTP/1.1
+      host: nex.toa.io
+      accept: application/vnd.toa.octets.entry+yaml
+      """
+    Then the following reply is sent:
+      """
+      200 OK
+      content-type: application/yaml
+
+      id: ${{ id }}
+      type: image/png
+      origin: https://avatars.githubusercontent.com/u/92763022?s=48&v=4
       """
 
     # untrusted location
@@ -115,3 +134,56 @@ Feature: Download and store
       | type       | location                                   |
       | origin     | https://avatars.githubusercontent.com      |
       | expression | /^https://avatars\.githubusercontent\.com/ |
+
+  Scenario: Download size limit
+    Given the annotation:
+      """yaml
+      /:
+        io:output: true
+        auth:anonymous: true
+        octets:context: octets
+        POST:
+          octets:store:
+            limit: 1kb
+            trust:
+              - https://avatars.githubusercontent.com
+      """
+    When the following request is received:
+      """
+      POST / HTTP/1.1
+      host: nex.toa.io
+      content-location: https://avatars.githubusercontent.com/u/92763022?s=48&v=4
+      content-length: 0
+      accept: text/plain
+      """
+    Then the following reply is sent:
+      """
+      413 Request Entity Too Large
+
+      Size limit is 1kb
+      """
+
+  Scenario: Allow `content-location` request header
+    Given the annotation:
+      """yaml
+      /:
+        io:output: true
+        auth:anonymous: true
+        octets:context: octets
+        POST:
+          octets:store:
+            limit: 1kb
+            trust:
+              - https://avatars.githubusercontent.com
+      """
+    When the following request is received:
+      """
+      OPTIONS / HTTP/1.1
+      host: nex.toa.io
+      origin: https://hello.world
+      """
+    Then the following reply is sent:
+      """
+      204 No Content
+      access-control-allow-headers: accept, authorization, content-type, etag, if-match, if-none-match, content-meta, content-location
+      """

package/features/octets.feature

@@ -34,6 +34,14 @@ Feature: Octets directive family
             /*:
               GET:
                 octets:fetch: ~
+        /limit-1kb:
+          POST:
+            octets:store:
+              limit: 1kb
+        /limit-100kb:
+          POST:
+            octets:store:
+              limit: 100kb
       """
 
   Scenario: Basic storage operations
@@ -146,6 +154,32 @@ Feature: Octets directive family
       201 Created
       """
 
+  Scenario: Size limit
+    When the stream of `albert.jpg` is received with the following headers:
+      """
+      POST /limit-1kb/ HTTP/1.1
+      host: nex.toa.io
+      content-type: image/jpeg
+      accept: text/plain
+      """
+    Then the following reply is sent:
+      """
+      413 Request Entity Too Large
+
+      Size limit is 1kb
+      """
+
+    When the stream of `albert.jpg` is received with the following headers:
+      """
+      POST /limit-100kb/ HTTP/1.1
+      host: nex.toa.io
+      content-type: image/jpeg
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      """
+
   Scenario Outline: Detecting `<type>`
     When the stream of `sample.<ext>` is received with the following headers:
       """

package/features/steps/Parameters.ts

@@ -16,6 +16,6 @@ process.env.TOA_DEV = '1'
 process.env.TOA_STORAGES = encode({
   octets: {
     provider: 'tmp',
-    directory: 'exposition'
+    directory: Math.random().toString(36).substring(2)
   }
 })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.exposition",
-  "version": "1.0.0-alpha.60",
+  "version": "1.0.0-alpha.61",
   "description": "Toa Exposition",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,7 +19,7 @@
   "dependencies": {
     "@toa.io/core": "1.0.0-alpha.59",
     "@toa.io/generic": "1.0.0-alpha.59",
-    "@toa.io/schemas": "1.0.0-alpha.59",
+    "@toa.io/schemas": "1.0.0-alpha.61",
     "bcryptjs": "2.4.3",
     "error-value": "0.3.0",
     "js-yaml": "4.1.0",
@@ -51,11 +51,11 @@
     "@swc/core": "1.6.6",
     "@swc/helpers": "0.5.11",
     "@toa.io/agent": "1.0.0-alpha.59",
-    "@toa.io/extensions.storages": "1.0.0-alpha.59",
+    "@toa.io/extensions.storages": "1.0.0-alpha.61",
     "@types/bcryptjs": "2.4.3",
     "@types/cors": "2.8.13",
     "@types/negotiator": "0.6.1",
     "jest-esbuild": "0.3.0"
   },
-  "gitHead": "81bf1dbf5b3f8dcb19097aca19947ddb60cb96be"
+  "gitHead": "25e32ccba495a1f7197e378a5208cfc1a6b3b8a1"
 }

package/schemas/method.cos.yaml

@@ -1,4 +1,4 @@
-verb: /^(GET|POST|PUT|PATCH|DELETE)$/
+verb: /^(GET|POST|PUT|PATCH|DELETE|LOCK|UNLOCK)$/
 mapping:
   namespace: string
   component: string

package/schemas/octets/store.cos.yaml

@@ -1,4 +1,25 @@
-accept+: string
-workflow+: <string>
-trust: [string]
-_: true
+type: object
+nullable: true
+properties:
+  accept:
+    anyOf:
+      - type: string
+      - type: array
+        items:
+          type: string
+  limit:
+    type: string
+    pattern: ^(\d+(\.\d+)?)([kmgtKMBGT]i?)?[bB]?$
+  trust:
+    type: array
+    items:
+      type: string
+  workflow:
+    anyOf:
+      - &unit
+        type: object
+        patternProperties:
+          ^.+$:
+            type: string
+      - type: array
+        items: *unit

package/source/HTTP/Server.ts

@@ -142,7 +142,7 @@ export const PORT = 8000
 export const DELAY = 3 // seconds
 
 const DEFAULTS: Omit<Properties, 'authorities'> = {
-  methods: new Set<string>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']),
+  methods: new Set<string>(['OPTIONS', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'LOCK', 'UNLOCK']),
   debug: false,
   trace: false,
   port: PORT,

package/source/HTTP/exceptions.ts

@@ -71,3 +71,9 @@ export class PreconditionFailed extends ClientError {
     super(412)
   }
 }
+
+export class RequestEntityTooLarge extends ClientError {
+  public constructor (body?: any) {
+    super(413, body)
+  }
+}

package/source/RTD/syntax/types.ts

@@ -47,4 +47,4 @@ export interface Range {
   range: [number, number]
 }
 
-export const verbs = new Set<string>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])
+export const verbs = new Set<string>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'LOCK', 'UNLOCK'])

package/source/directives/cors/CORS.ts

@@ -14,7 +14,7 @@ export class CORS implements Interceptor {
   ])
 
   private readonly headers = new Headers({
-    'access-control-allow-methods': 'GET, POST, PUT, PATCH, DELETE',
+    'access-control-allow-methods': 'GET, POST, PUT, PATCH, DELETE, LOCK, UNLOCK',
     'access-control-allow-credentials': 'true',
     'access-control-allow-headers': Array.from(this.requestHeaders).join(', '),
     'access-control-max-age': '3600',

package/source/directives/io/Input.ts

@@ -12,7 +12,7 @@ export class Input implements Directive {
   }
 
   public static validate (permissions: unknown): asserts permissions is Permissions {
-    schemas.input.validate(permissions, 'Incorrect \'io:input\' format')
+    schemas.input.validate<Permissions>(permissions, 'Incorrect \'io:input\' format')
   }
 
   public attach (context: Context): void {
@@ -21,7 +21,7 @@ export class Input implements Directive {
 
   private check (body: unknown): unknown {
     try {
-      schemas.message.validate(body)
+      schemas.message.validate<Message | Message[]>(body)
     } catch {
       throw new BadRequest('Invalid request body')
     }

package/source/directives/io/Output.ts

@@ -48,7 +48,7 @@ export class Output implements Directive {
         return
       }
 
-      schemas.message.validate(message.body,
+      schemas.message.validate<Message>(message.body,
         '\'io:output\' expects response to be an object or array of objects')
 
       if (Array.isArray(message.body))

package/source/directives/octets/Context.ts

@@ -1,4 +1,4 @@
-import * as schemas from './schemas'
+import assert from 'node:assert'
 import { Directive } from './Directive'
 import type { Output } from '../../io'
 
@@ -8,7 +8,8 @@ export class Context extends Directive {
 
   public constructor (value: unknown) {
     super()
-    schemas.context.validate(value)
+
+    assert.ok(typeof value === 'string', 'Directive \'octets:context\' must must be a string')
 
     this.storage = value
   }

package/source/directives/octets/Fetch.ts

@@ -64,7 +64,7 @@ export class Fetch extends Directive {
     const headers = new Headers({
       'content-type': result.type,
       'content-length': result.size.toString(),
-      etag: result.checksum
+      etag: `"${result.checksum}"`
     })
 
     return {

package/source/directives/octets/Store.ts

@@ -5,6 +5,7 @@ import { cors } from '../cors'
 import * as schemas from './schemas'
 import { Workflow } from './workflows'
 import { Directive } from './Directive'
+import { toBytes } from './bytes'
 import type { Readable } from 'stream'
 import type { Parameter } from '../../RTD'
 import type { Unit } from './workflows'
@@ -19,6 +20,8 @@ export class Store extends Directive {
   public readonly targeted = false
 
   private readonly accept?: string
+  private readonly limit: number
+  private readonly limitString: string
   private readonly trust?: Array<string | RegExp>
   private readonly workflow?: Workflow
   private readonly discovery: Record<string, Promise<Component>> = {}
@@ -27,7 +30,8 @@ export class Store extends Directive {
   public constructor
   (options: Options | null, discovery: Promise<Component>, remotes: Remotes) {
     super()
-    schemas.store.validate(options)
+
+    schemas.store.validate<Options>(options)
 
     this.accept = match(options?.accept,
       String, (value: string) => value,
@@ -41,9 +45,12 @@ export class Store extends Directive {
       this.trust = options.trust.map((value: string) =>
         value.startsWith('/') ? new RegExp(value.slice(1, -1)) : value)
 
+    this.limitString = options?.limit ?? '64MiB'
+    this.limit = toBytes(this.limitString)
     this.discovery.storage = discovery
 
     cors.allow('content-meta')
+    cors.allow('content-location')
   }
 
   public async apply (storage: string, input: Input, parameters: Parameter[]): Promise<Output> {
@@ -53,15 +60,14 @@ export class Store extends Directive {
       input: {
         storage,
         request: input.request,
+        accept: this.accept,
+        limit: this.limit,
         trust: this.trust
       }
     }
 
     const meta = input.request.headers['content-meta']
 
-    if (this.accept !== undefined)
-      request.input.accept = this.accept
-
     if (meta !== undefined)
       request.input.meta = this.meta(meta)
 
@@ -97,6 +103,7 @@ export class Store extends Directive {
     throw match(error.code,
       'NOT_ACCEPTABLE', () => new http.UnsupportedMediaType(),
       'TYPE_MISMATCH', () => new http.BadRequest(),
+      'LIMIT_EXCEEDED', () => new http.RequestEntityTooLarge(`Size limit is ${this.limitString}`),
       'LOCATION_UNTRUSTED', () => new http.Forbidden(error.message),
       'LOCATION_LENGTH', () => new http.BadRequest(error.message),
       'LOCATION_UNAVAILABLE', () => new http.NotFound(error.message),
@@ -122,6 +129,7 @@ export class Store extends Directive {
 
 export interface Options {
   accept?: string | string[]
+  limit?: string
   workflow?: Unit[] | Unit
   trust?: string[]
 }
@@ -131,6 +139,7 @@ interface StoreRequest {
     storage: string
     request: Input['request']
     accept?: string
+    limit?: number
     trust?: Array<string | RegExp>
     meta?: Record<string, string>
   }

package/source/directives/octets/bytes.test.ts

@@ -0,0 +1,30 @@
+import { toBytes } from './bytes'
+
+it('should parse bytes', async () => {
+  expect(toBytes('10')).toBe(10)
+  expect(toBytes('10B')).toBe(10)
+})
+
+it('should parse binary prefix', async () => {
+  expect(toBytes('10KiB')).toBe(10240)
+  expect(toBytes('10MiB')).toBe(10485760)
+  expect(toBytes('10GiB')).toBe(10737418240)
+  expect(toBytes('10TiB')).toBe(10995116277760)
+})
+
+it('should parse decimal prefix', async () => {
+  expect(toBytes('10kB')).toBe(10000)
+  expect(toBytes('10MB')).toBe(10000000)
+  expect(toBytes('10GB')).toBe(10000000000)
+  expect(toBytes('10TB')).toBe(10000000000000)
+})
+
+it('should parse incorrect value as binary', async () => {
+  expect(toBytes('10b')).toBe(10)
+  expect(toBytes('10kb')).toBe(10240)
+  expect(toBytes('10kib')).toBe(10240)
+  expect(toBytes('10mb')).toBe(10485760)
+  expect(toBytes('10gb')).toBe(10737418240)
+  expect(toBytes('10tib')).toBe(10995116277760)
+  expect(toBytes('10Mb')).toBe(10485760)
+})

package/source/directives/octets/bytes.ts

@@ -0,0 +1,18 @@
+import assert from 'node:assert'
+
+export function toBytes (input: string): number {
+  const match = RX.exec(input)
+
+  assert.ok(match !== null, `Invalid bytes format: ${input}`)
+
+  const value = parseFloat(match.groups!.value)
+  const prefix = match.groups!.prefix?.[0].toLowerCase() ?? ''
+  const binary = match.groups!.binary !== undefined || match.groups!.unit === 'b'
+  const base = binary ? 1024 : 1000
+  const power = POWERS.indexOf(prefix)
+
+  return value * Math.pow(base, power)
+}
+
+const POWERS = ['', 'k', 'm', 'g', 't']
+const RX = /^(?<value>(\d+)(\.\d+)?)(?<prefix>[kmgt](?<binary>i)?)?(?<unit>b)?$/i

package/source/directives/octets/schemas.ts

@@ -10,10 +10,8 @@ import type { Unit } from './workflows'
 const path = resolve(__dirname, '../../../schemas/octets')
 const namespace = schemas.namespace(path)
 
-export const context: Schema<string> = namespace.schema('context')
 export const store: Schema<StoreOptions | null> = namespace.schema('store')
 export const fetch: Schema<FetchOptions | null> = namespace.schema('fetch')
 export const remove: Schema<DeleteOptions | null> = namespace.schema('delete')
 export const list: Schema<ListOptions | null> = namespace.schema('list')
-export const permute: Schema<null> = namespace.schema('permute')
 export const workflow: Schema<Unit[] | Unit> = namespace.schema('workflow')

package/transpiled/HTTP/Server.js

@@ -134,7 +134,7 @@ async function adam(request) {
 exports.PORT = 8000;
 exports.DELAY = 3; // seconds
 const DEFAULTS = {
-    methods: new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']),
+    methods: new Set(['OPTIONS', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'LOCK', 'UNLOCK']),
     debug: false,
     trace: false,
     port: exports.PORT,