@toa.io/extensions.exposition 1.0.0-alpha.28 → 1.0.0-alpha.283

package/source/directives/octets/Octets.ts

@@ -1,16 +1,15 @@
-import { NotFound } from '../../HTTP'
-import { Context } from './Context'
-import { Store } from './Store'
-import { Fetch } from './Fetch'
-import { List } from './List'
-import { Delete } from './Delete'
-import { WorkflowDirective } from './Workflow'
-import type { Directive } from './Directive'
-import type { Output } from '../../io'
+import { NotFound } from '../../HTTP/index.js'
+import { Context } from './Context.js'
+import { Put } from './Put.js'
+import { Get } from './Get.js'
+import { Delete } from './Delete.js'
+import { WorkflowDirective } from './Workflow.js'
+import type { Directive } from './Directive.js'
+import type { Output } from '../../io.js'
 import type { Component } from '@toa.io/core'
-import type { Remotes } from '../../Remotes'
-import type { Parameter, DirectiveFamily } from '../../RTD'
-import type { Input } from './types'
+import type { Remotes } from '../../Remotes.js'
+import type { Parameter, DirectiveFamily } from '../../RTD/index.js'
+import type { Input } from './types.js'
 
 export class Octets implements DirectiveFamily<Directive> {
   public readonly name: string = 'octets'
@@ -22,15 +21,14 @@ export class Octets implements DirectiveFamily<Directive> {
     const Class = DIRECTIVES[name]
 
     if (Class === undefined)
-      throw new Error(`Directive 'octets:${name}' is not implemented.`)
+      throw new Error(`Directive 'octets:${name}' is not implemented`)
 
-    this.discovery ??= remotes.discover('octets', 'storage')
+    this.discovery ??= remotes.discover('exposition', 'octets')
 
     return new Class(value, this.discovery, remotes)
   }
 
-  public async preflight
-  (directives: Directive[], input: Input, parameters: Parameter[]): Promise<Output> {
+  public async preflight (directives: Directive[], input: Input, parameters: Parameter[]): Promise<Output> {
     let context: Context | null = null
     let action: Directive | null = null
 
@@ -40,19 +38,19 @@ export class Octets implements DirectiveFamily<Directive> {
       else if (action === null)
         action = directive
       else
-        throw new Error('Octets action is umbiguous.')
+        throw new Error('Octets action is ambiguous')
 
     if (action === null)
       return null
 
     // noinspection PointlessBooleanExpressionJS
     if (context === null)
-      throw new Error('Octets context is not defined.')
+      throw new Error('Octets context is not defined')
 
     const targeted = input.request.url[input.request.url.length - 1] !== '/'
 
     if (targeted !== action.targeted)
-      throw new NotFound(`Trailing slash is ${action.targeted ? 'redundant' : 'required'}.`)
+      throw new NotFound(`Trailing slash is ${action.targeted ? 'redundant' : 'required'}`)
 
     // noinspection JSObjectNullOrUndefined
     return await input.timing.capture(action.name, action.apply(context.storage, input, parameters))
@@ -61,9 +59,9 @@ export class Octets implements DirectiveFamily<Directive> {
 
 const DIRECTIVES: Record<string, Constructor> = {
   context: Context,
-  store: Store,
-  fetch: Fetch,
-  list: List,
+  put: Put,
+  get: Get,
+  head: Get,
   delete: Delete,
   workflow: WorkflowDirective
 }

package/source/directives/octets/Put.ts

@@ -0,0 +1,141 @@
+import { PassThrough } from 'node:stream'
+import { match } from 'matchacho'
+import * as http from '../../HTTP/index.js'
+import { cors } from '../cors/index.js'
+import * as schemas from './schemas.js'
+import { Workflow } from './workflows/index.js'
+import { Directive } from './Directive.js'
+import { toBytes } from './bytes.js'
+import type { Readable } from 'stream'
+import type { Parameter } from '../../RTD/index.js'
+import type { Unit, Location } from './workflows/index.js'
+import type { Entry } from '@toa.io/extensions.storages'
+import type { Remotes } from '../../Remotes.js'
+import type { Component } from '@toa.io/core'
+import type { Output } from '../../io.js'
+import type { Input } from './types.js'
+
+export class Put extends Directive {
+  public readonly targeted = false
+
+  private readonly location?: string
+  private readonly accept?: string
+  private readonly limit: number
+  private readonly limitString: string
+  private readonly trust?: Array<string | RegExp>
+  private readonly workflow?: Workflow
+  private readonly discovery: Record<string, Promise<Component>> = {}
+  private storage: Component | null = null
+
+  public constructor (options: Options | null, discovery: Promise<Component>, remotes: Remotes) {
+    super()
+
+    schemas.put.validate<Options>(options)
+
+    this.accept = match(options?.accept,
+      String, (value: string) => value,
+      Array, (types: string[]) => types.join(','),
+      undefined)
+
+    if (options?.workflow !== undefined)
+      this.workflow = new Workflow(options.workflow, remotes)
+
+    if (options?.trust !== undefined)
+      this.trust = options.trust.map((value: string) =>
+        value.startsWith('/') ? new RegExp(value.slice(1, -1)) : value)
+
+    if (options?.location !== undefined)
+      this.location = options.location
+
+    this.limitString = options?.limit ?? '64MiB'
+    this.limit = toBytes(this.limitString)
+    this.discovery.storage = discovery
+
+    cors.allow('content-attributes')
+    cors.allow('content-location')
+  }
+
+  public async apply (storage: string, input: Input, parameters: Parameter[]): Promise<Output> {
+    this.storage ??= await this.discovery.storage
+
+    const request: StoreRequest = {
+      input: {
+        storage,
+        request: input.request,
+        location: this.location,
+        accept: this.accept,
+        limit: this.limit,
+        trust: this.trust
+      }
+    }
+
+    const entry = await this.storage.invoke<Entry>('put', request)
+
+    return match<Output>(entry,
+      Error, (error: CodedError) => this.throw(error),
+      () => this.reply(input, storage, entry, parameters))
+  }
+
+  // eslint-disable-next-line max-params
+  private reply (input: Input, storage: string, entry: Entry, parameters: Parameter[]): Output {
+    const body = this.workflow === undefined
+      ? entry
+      : this.execute(input, storage, entry, parameters)
+
+    return { body }
+  }
+
+  // eslint-disable-next-line max-params
+  private execute (input: Input, storage: string, entry: Entry, parameters: Parameter[]): Readable {
+    const stream = new PassThrough({ objectMode: true })
+
+    stream.push(entry)
+
+    const location: Location = {
+      storage,
+      authority: input.authority,
+      identity: input.identity?.id,
+      path: this.location ?? input.request.url
+    }
+
+    this.workflow!.execute(location, entry, parameters).pipe(stream)
+
+    return stream
+  }
+
+  private throw (error: CodedError): never {
+    throw match(error.code,
+      'NOT_ACCEPTABLE', () => new http.UnsupportedMediaType(),
+      'TYPE_MISMATCH', () => new http.BadRequest(),
+      'LIMIT_EXCEEDED', () => new http.RequestEntityTooLarge(`Size limit is ${this.limitString}`),
+      'LOCATION_UNTRUSTED', () => new http.Forbidden(error.message),
+      'LOCATION_LENGTH', () => new http.BadRequest(error.message),
+      'LOCATION_UNAVAILABLE', () => new http.NotFound(error.message),
+      'INVALID_ID', () => new http.BadRequest(error.message),
+      error)
+  }
+}
+
+export interface Options {
+  location?: string
+  accept?: string | string[]
+  limit?: string
+  workflow?: Unit[] | Unit
+  trust?: string[]
+}
+
+interface StoreRequest {
+  input: {
+    storage: string
+    request: Input['request']
+    location?: string
+    accept?: string
+    limit?: number
+    trust?: Array<string | RegExp>
+    attributes?: Record<string, string>
+  }
+}
+
+interface CodedError extends Error {
+  code: string
+}

package/source/directives/octets/Workflow.ts

@@ -1,15 +1,15 @@
-import { NotFound } from '../../HTTP'
-import * as schemas from './schemas'
-import { Workflow } from './workflows'
-import { Directive } from './Directive'
-import type { Unit } from './workflows'
-import type { Input } from './types'
+import { NotFound } from '../../HTTP/index.js'
+import * as schemas from './schemas.js'
+import { Workflow } from './workflows/index.js'
+import { Directive } from './Directive.js'
+import type { Unit, Location } from './workflows/index.js'
+import type { Input } from './types.js'
 import type { Component } from '@toa.io/core'
-import type { Output } from '../../io'
-import type { Remotes } from '../../Remotes'
-import type { Maybe } from '@toa.io/types'
+import type { Output } from '../../io.js'
+import type { Remotes } from '../../Remotes.js'
+import type { Maybe } from '@toa.io/core'
 import type { Entry } from '@toa.io/extensions.storages'
-import type { Parameter } from '../../RTD'
+import type { Parameter } from '../../RTD/index.js'
 
 export class WorkflowDirective extends Directive {
   public readonly targeted = true
@@ -29,7 +29,7 @@ export class WorkflowDirective extends Directive {
   public async apply (storage: string, input: Input, parameters: Parameter[]): Promise<Output> {
     this.storage ??= await this.discovery
 
-    const entry = await this.storage.invoke<Maybe<Entry>>('get',
+    const entry = await this.storage.invoke<Maybe<Entry>>('head',
       {
         input: {
           storage,
@@ -40,9 +40,15 @@ export class WorkflowDirective extends Directive {
     if (entry instanceof Error)
       throw new NotFound()
 
+    const location: Location = {
+      storage,
+      authority: input.authority,
+      path: input.request.url
+    }
+
     return {
       status: 202,
-      body: this.workflow.execute(input, storage, entry, parameters)
+      body: this.workflow.execute(location, entry, parameters)
     }
   }
 }

package/source/directives/octets/bytes.test.ts

@@ -0,0 +1,33 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { toBytes } from './bytes.js'
+
+it('should parse bytes', async () => {
+  assert.strictEqual(toBytes('10'), 10)
+  assert.strictEqual(toBytes('10B'), 10)
+})
+
+it('should parse binary prefix', async () => {
+  assert.strictEqual(toBytes('10KiB'), 10240)
+  assert.strictEqual(toBytes('10MiB'), 10485760)
+  assert.strictEqual(toBytes('10GiB'), 10737418240)
+  assert.strictEqual(toBytes('10TiB'), 10995116277760)
+})
+
+it('should parse decimal prefix', async () => {
+  assert.strictEqual(toBytes('10kB'), 10000)
+  assert.strictEqual(toBytes('10MB'), 10000000)
+  assert.strictEqual(toBytes('10GB'), 10000000000)
+  assert.strictEqual(toBytes('10TB'), 10000000000000)
+})
+
+it('should parse incorrect value as binary', async () => {
+  assert.strictEqual(toBytes('10b'), 10)
+  assert.strictEqual(toBytes('10kb'), 10240)
+  assert.strictEqual(toBytes('10kib'), 10240)
+  assert.strictEqual(toBytes('10mb'), 10485760)
+  assert.strictEqual(toBytes('10gb'), 10737418240)
+  assert.strictEqual(toBytes('10tib'), 10995116277760)
+  assert.strictEqual(toBytes('10Mb'), 10485760)
+})

package/source/directives/octets/bytes.ts

@@ -0,0 +1,18 @@
+import assert from 'node:assert'
+
+export function toBytes (input: string): number {
+  const match = RX.exec(input)
+
+  assert.ok(match !== null, `Invalid bytes format: ${input}`)
+
+  const value = parseFloat(match.groups!.value)
+  const prefix = match.groups!.prefix?.[0].toLowerCase() ?? ''
+  const binary = match.groups!.binary !== undefined || match.groups!.unit === 'b'
+  const base = binary ? 1024 : 1000
+  const power = POWERS.indexOf(prefix)
+
+  return value * Math.pow(base, power)
+}
+
+const POWERS = ['', 'k', 'm', 'g', 't']
+const RX = /^(?<value>(\d+)(\.\d+)?)(?<prefix>[kmgt](?<binary>i)?)?(?<unit>b)?$/i

package/source/directives/octets/index.ts

@@ -1,3 +1,3 @@
-import { Octets } from './Octets'
+import { Octets } from './Octets.js'
 
 export const octets = new Octets()

package/source/directives/octets/schemas.test.ts

@@ -1,4 +1,7 @@
-import * as schemas from './schemas'
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import * as schemas from './schemas.js'
 
 describe('workflow', () => {
   const ok = [
@@ -11,11 +14,13 @@ describe('workflow', () => {
     { echo: 'hello world', ok: { not: 'ok' } }
   ]
 
-  it.each(ok)('should be valid', (workflow) => {
-    expect(() => schemas.workflow.validate(workflow)).not.toThrow()
+  for (const workflow of ok)
+     it('should be valid', () => {
+    assert.doesNotThrow(() => schemas.workflow.validate(workflow))
   })
 
-  it.each(oh)('should not be valid', (workflow) => {
-    expect(() => schemas.workflow.validate(workflow)).toThrow()
+  for (const workflow of oh)
+     it('should not be valid', () => {
+    assert.throws(() => schemas.workflow.validate(workflow))
   })
 })

package/source/directives/octets/schemas.ts

@@ -1,19 +1,15 @@
 import { resolve } from 'node:path'
-import schemas from '@toa.io/schemas'
-import type { Permissions as FetchPermissions } from './Fetch'
-import type { Permissions as ListPermissions } from './List'
-import type { Options as StoreOptions } from './Store'
-import type { Options as DeleteOptions } from './Delete'
+import * as schemas from '@toa.io/schemas'
+import type { Options as GetOptions } from './Get.js'
+import type { Options as PutOptions } from './Put.js'
+import type { Options as DeleteOptions } from './Delete.js'
 import type { Schema } from '@toa.io/schemas'
-import type { Unit } from './workflows'
+import type { Unit } from './workflows/index.js'
 
-const path = resolve(__dirname, '../../../schemas/octets')
+const path = resolve(import.meta.dirname, '../../../schemas/octets')
 const namespace = schemas.namespace(path)
 
-export const context: Schema<string> = namespace.schema('context')
-export const store: Schema<StoreOptions | null> = namespace.schema('store')
-export const fetch: Schema<FetchPermissions | null> = namespace.schema('fetch')
+export const put: Schema<PutOptions | null> = namespace.schema('put')
+export const get: Schema<GetOptions | null> = namespace.schema('get')
 export const remove: Schema<DeleteOptions | null> = namespace.schema('delete')
-export const list: Schema<ListPermissions | null> = namespace.schema('list')
-export const permute: Schema<null> = namespace.schema('permute')
 export const workflow: Schema<Unit[] | Unit> = namespace.schema('workflow')

package/source/directives/octets/types.ts

@@ -1,6 +1,8 @@
-import type * as io from '../../io'
+import type * as io from '../../io.js'
+import type { Identity } from '../auth/types.js' // meh
 
 export interface Extension {
+  identity?: Identity
   octets?: string
 }
 

package/source/directives/octets/workflows/Execution.ts

@@ -1,8 +1,9 @@
-import { Readable } from 'stream'
-import type { Unit } from './Workflow'
-import type { Remotes } from '../../../Remotes'
+import { Readable } from 'node:stream'
+import { console } from 'openspan'
+import type { Unit } from './Workflow.js'
+import type { Remotes } from '../../../Remotes.js'
 import type { Component } from '@toa.io/core'
-import type { Maybe } from '@toa.io/types'
+import type { Maybe } from '@toa.io/core'
 import type { Entry } from '@toa.io/extensions.storages'
 
 export class Execution extends Readable {
@@ -39,25 +40,70 @@ export class Execution extends Readable {
 
   private async execute (unit: Unit): Promise<void> {
     const promises = Object.entries(unit).map(async ([step, endpoint]) => {
-      const result = await this.call(endpoint).catch((error: Error) => console.error(error))
+      try {
+        const result = await this.call(endpoint)
 
-      if (result instanceof Error) {
-        this.push({ error: { step, ...result } })
-        this.interrupted = true
-      } else
-        this.push({ [step]: result ?? null })
+        if (result instanceof Readable)
+          return await this.stream(step, result)
+
+        this.report(step, result)
+      } catch (e: unknown) {
+        this.exception(step, e)
+      }
     })
 
     await Promise.all(promises)
   }
 
+  private async stream (step: string, stream: Readable): Promise<void> {
+    try {
+      for await (const result of stream)
+        this.report(step, result, false)
+
+      this.report(step, undefined, true)
+    } catch (e: unknown) {
+      this.exception(step, e)
+    }
+  }
+
+  private report (step: string, result?: Maybe<unknown>, completed = true): void {
+    const report: Report = { step }
+
+    if (completed)
+      report.status = 'completed'
+
+    if (result instanceof Error) {
+      // an Error cannot be serialized where it sits, and the encoders only unwrap
+      // one they are handed directly — this one travels nested inside the report
+      report.error = { ...result }
+      this.interrupted = true
+    } else if (result !== undefined) {
+      report.output = result
+      this.context.steps[step] = structuredClone(result)
+    }
+
+    this.push(report)
+  }
+
+  private exception (step: string, error: unknown): void {
+    console.error('Workflow exception', error as Error)
+
+    this.push({ step, status: 'exception' } satisfies Report)
+    this.interrupted = true
+  }
+
   private async call (endpoint: string): Promise<Maybe<unknown>> {
+    const task = endpoint.startsWith('task:')
+
+    if (task)
+      endpoint = endpoint.slice(5)
+
     const [operation, component, namespace = 'default'] = endpoint.split('.').reverse()
     const key = `${namespace}.${component}`
 
     this.components[key] ??= await this.discover(key, namespace, component)
 
-    return await this.components[key].invoke(operation, { input: this.context })
+    return this.components[key].invoke(operation, { input: this.context, task })
   }
 
   private async discover (key: string, namespace: string, component: string): Promise<Component> {
@@ -69,8 +115,18 @@ export class Execution extends Readable {
 }
 
 export interface Context {
+  authority: string
+  identity?: string
   storage: string
   path: string
   entry: Entry
   parameters: Record<string, string>
+  steps: Record<string, unknown>
+}
+
+interface Report {
+  step: string
+  status?: 'completed' | 'exception'
+  output?: unknown
+  error?: Record<string, unknown>
 }

package/source/directives/octets/workflows/Workflow.ts

@@ -1,11 +1,10 @@
 import { posix } from 'node:path'
 import { match } from 'matchacho'
-import { Execution } from './Execution'
-import type { Context } from './Execution'
-import type { Parameter } from '../../../RTD'
-import type { Input } from '../types'
+import { Execution } from './Execution.js'
 import type { Entry } from '@toa.io/extensions.storages'
-import type { Remotes } from '../../../Remotes'
+import type { Context } from './Execution.js'
+import type { Parameter } from '../../../RTD/index.js'
+import type { Remotes } from '../../../Remotes.js'
 
 export class Workflow {
   private readonly units: Unit[]
@@ -19,19 +18,31 @@ export class Workflow {
     this.remotes = remotes
   }
 
-  // eslint-disable-next-line max-params
-  public execute
-  (input: Input, storage: string, entry: Entry, params: Parameter[]): Execution {
-    const path = posix.join(input.request.url, entry.id)
+  public execute (location: Location, entry: Entry, params: Parameter[]): Execution {
     const parameters: Record<string, string> = {}
 
     for (const { name, value } of params)
       parameters[name] = value
 
-    const context: Context = { storage, path, entry, parameters }
+    const context: Context = {
+      authority: location.authority,
+      identity: location.identity,
+      storage: location.storage,
+      path: posix.join(location.path, entry.id),
+      entry,
+      parameters,
+      steps: {}
+    }
 
     return new Execution(context, this.units, this.remotes)
   }
 }
 
+export interface Location {
+  authority: string
+  identity?: string
+  storage: string
+  path: string
+}
+
 export type Unit = Record<string, string>

package/source/directives/octets/workflows/index.ts

@@ -1 +1 @@
-export { Workflow, type Unit } from './Workflow'
+export { Workflow, type Unit, type Location } from './Workflow.js'

package/source/directives/require/Directive.ts

@@ -1,4 +1,4 @@
-import type { Input } from '../../io'
+import type { Input } from '../../io.js'
 
 export interface Directive {
   preflight: (context: Input) => void

package/source/directives/require/Headers.ts

@@ -1,6 +1,6 @@
-import { BadRequest } from '../../HTTP'
-import type { Input } from '../../io'
-import type { Directive } from './Directive'
+import { BadRequest } from '../../HTTP/index.js'
+import type { Input } from '../../io.js'
+import type { Directive } from './Directive.js'
 
 export class Headers implements Directive {
   private readonly headers: string[]

package/source/directives/require/Require.ts

@@ -1,8 +1,8 @@
 import assert from 'node:assert'
-import { Headers } from './Headers'
-import type { Input } from '../../io'
-import type { Directive } from './Directive'
-import type { DirectiveFamily } from '../../RTD'
+import { Headers } from './Headers.js'
+import type { Input } from '../../io.js'
+import type { Directive } from './Directive.js'
+import type { DirectiveFamily } from '../../RTD/index.js'
 
 export class Require implements DirectiveFamily {
   public readonly name = 'require'

package/source/directives/require/index.ts

@@ -1,3 +1,3 @@
-import { Require } from './Require'
+import { Require } from './Require.js'
 
 export const req = new Require()

package/source/exceptions.ts

@@ -1,6 +1,7 @@
 import { match } from 'matchacho'
-import * as http from './HTTP'
-import { Exception as HTTPException } from './HTTP'
+import { console } from 'openspan'
+import * as http from './HTTP/index.js'
+import { Exception as HTTPException } from './HTTP/index.js'
 import type { Exception } from '@toa.io/core'
 
 export function rethrow (exception: Exception | HTTPException): void {
@@ -10,11 +11,13 @@ export function rethrow (exception: Exception | HTTPException): void {
   // see /runtime/core/src/exceptions.js
   throw match(exception.code,
     badRequest, () => new http.BadRequest(exception.message),
-    302, NOT_FOUND,
-    303, PRECONDITION_FAILED,
-    306, () => new http.Conflict(),
+    CORE_EXCEPTIONS.StateNotFound, NOT_FOUND,
+    CORE_EXCEPTIONS.StatePrecondition, PRECONDITION_FAILED,
+    CORE_EXCEPTIONS.Duplicate, CONFLICT,
+    CORE_EXCEPTIONS.StateConcurrency, CONFLICT,
+    CORE_EXCEPTIONS.EntityGuard, CONFLICT,
     () => {
-      console.error(exception)
+      console.error('Request processing exception', exception)
 
       return exception
     })
@@ -26,3 +29,12 @@ function badRequest (code: number): boolean {
 
 const NOT_FOUND = new http.NotFound()
 const PRECONDITION_FAILED = new http.PreconditionFailed()
+const CONFLICT = new http.Conflict()
+
+const CORE_EXCEPTIONS = {
+  StateNotFound: 302,
+  StatePrecondition: 303,
+  StateConcurrency: 304,
+  EntityGuard: 213,
+  Duplicate: 306
+}

package/source/index.ts

@@ -1,6 +1,12 @@
-export { manifest } from './manifest'
-export { deployment } from './deployment'
-export { components } from './Composition'
-export { Factory } from './Factory'
+export { manifest } from './manifest.js'
+export { deployment } from './deployment.js'
+export { components } from './Composition.js'
+export { Factory } from './Factory.js'
 
-export type { Remotes } from './Remotes'
+export type { Remotes } from './Remotes.js'
+
+// what an operation is written against, rather than the gateway's own
+export type { Claims, Identity } from './userland.js'
+
+// what an octets workflow is given
+export type * as octets from './octets.js'