@toa.io/extensions.exposition 1.0.0-alpha.286 → 1.0.0-alpha.288

package/source/Discovery/Site.ts

@@ -0,0 +1,202 @@
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import { Readable } from 'node:stream'
+import { console } from 'openspan'
+import * as http from '../HTTP/index.js'
+import { DISCOVERY } from '../const.js'
+import type { Input, Output } from '../io.js'
+import type { Interceptor } from '../Interception.js'
+
+/**
+ * The page that reads the tree: the directory `ui` builds, and nothing else.
+ *
+ * An interceptor rather than an endpoint, for the reason the OAuth documents are one — it
+ * runs before a credential is resolved. A page is public, and one served after `auth` would
+ * refuse the client holding a stale token, who is the client most likely to have opened it.
+ * It is also what keeps `depart` off these replies: a re-issued credential sets `no-store`,
+ * which would take the build hash off every asset that carries one.
+ *
+ * It claims the whole prefix, so that nothing under it reaches the router — a crawler
+ * asking for assets that are not there would otherwise have every one of them counted as a
+ * route that has gone missing, and ping discovery about it.
+ *
+ * `OPTIONS` on the prefix itself is the exception it lets through: the tree is what answers
+ * there, and that needs the identity this stage does not have yet.
+ */
+export class Site implements Interceptor {
+  public readonly name = 'discovery'
+
+  /** What a test points at a fixture; the build otherwise. */
+  private readonly override: string | undefined
+  private root: string
+
+  public constructor(root?: string) {
+    this.override = root
+    this.root = root ?? site()
+  }
+
+  /**
+   * Resolved here rather than at construction because this is a module singleton, built
+   * when the module loads and before anything has said where the page is. Said once, too,
+   * because a page that is not built is otherwise a `503` per request and nothing else.
+   */
+  public mount(): void {
+    this.root = this.override ?? site()
+
+    if (!isFile(path.join(this.root, 'index.html')))
+      console.warn('Discovery UI is not built', { root: this.root })
+  }
+
+  public intercept(input: Input): Output {
+    const { pathname } = input.url
+
+    if (pathname !== DISCOVERY && !pathname.startsWith(DISCOVERY + '/')) return null
+
+    // the prefix itself, as opposed to something under it
+    const bare = pathname === DISCOVERY || pathname === DISCOVERY + '/'
+    const method = input.request.method
+
+    if (method !== 'GET' && method !== 'HEAD') {
+      // the tree answers here, and it needs the identity this stage does not have yet
+      if (bare && method === 'OPTIONS') return null
+
+      throw new http.MethodNotAllowed(
+        new Headers({ allow: bare ? 'GET, HEAD, OPTIONS' : 'GET, HEAD' })
+      )
+    }
+
+    // the page is a directory, and every asset it names is relative to it
+    if (pathname === DISCOVERY)
+      return {
+        status: 302,
+        headers: new Headers({ location: DISCOVERY + '/' + input.url.search })
+      }
+
+    const file = this.resolve(pathname)
+
+    if (file === null) throw new http.NotFound()
+
+    return this.send(file, method)
+  }
+
+  /**
+   * The file a request lands on, or `null` when nothing does. A path that exists is served
+   * as it is; anything else that could be a route falls back to the page, because the
+   * client router — not this server — knows what routes there are.
+   */
+  private resolve(pathname: string): string | null {
+    /*
+     * Decoded after the prefix is off, so that an encoded separator cannot smuggle a
+     * segment past the test that put us here. A dot segment never arrives — `..` and
+     * `%2e%2e` alike are collapsed while the URL is parsed — but `%2f` survives that and
+     * decodes to a separator here, which is what the guard below is for.
+     */
+    const relative = decode(pathname.slice(DISCOVERY.length))
+
+    if (relative === null || relative.includes('\0')) return null
+
+    const file = path.join(this.root, relative)
+
+    if (file !== this.root && !file.startsWith(this.root + path.sep)) return null
+
+    if (isFile(file)) return file
+
+    /*
+     * A missing asset is missing, but a route can look like one. What this server would
+     * have served is what it knows how to serve, so anything else is a route and falls
+     * back to the page — as does anything ending in a slash, which is no name for a file.
+     */
+    const asset = !relative.endsWith('/') && path.extname(relative) in TYPES
+
+    return asset ? null : path.join(this.root, 'index.html')
+  }
+
+  /**
+   * Always with a `content-type` of its own: a stream without one is framed as
+   * `multipart/*` and the page arrives as an envelope of JSON-encoded buffers.
+   */
+  private send(file: string, method: string): Output {
+    const stats = fs.statSync(file, { throwIfNoEntry: false })
+
+    if (stats === undefined)
+      return {
+        status: 503,
+        headers: new Headers({ 'content-type': 'text/plain; charset=utf-8' }),
+        body: Readable.from([Buffer.from(NOT_BUILT)])
+      }
+
+    const headers = new Headers({
+      'content-type': TYPES[path.extname(file)] ?? 'application/octet-stream',
+      'content-length': String(stats.size),
+      'cache-control': caching(path.relative(this.root, file))
+    })
+
+    // a HEAD reply carries no body but reports the length a GET would have returned; the
+    // HTTP/2 layer does not drop one written to it, so none is opened
+    if (method === 'HEAD') return { status: 200, headers }
+
+    return { status: 200, headers, body: fs.createReadStream(file) }
+  }
+}
+
+const NOT_BUILT = 'The discovery UI is not built. Run `npm run build:ui`.\n'
+
+function decode(pathname: string): string | null {
+  try {
+    return decodeURIComponent(pathname)
+  } catch {
+    return null
+  }
+}
+
+function isFile(file: string): boolean {
+  return fs.statSync(file, { throwIfNoEntry: false })?.isFile() === true
+}
+
+/**
+ * Where `npm run build:ui` puts the page. Two levels up because this module is a directory
+ * deeper than `source` itself, and `transpiled` mirrors that — so it resolves the same from
+ * either. The variable is the features', which run against a fixture rather than a build.
+ */
+function site(): string {
+  return (
+    process.env.__TESTING_EXPOSITION_DISCOVERY_ROOT ??
+    path.resolve(import.meta.dirname, '..', '..', 'ui', 'dist')
+  )
+}
+
+/** Assets under this prefix carry their build hash in the name. */
+const IMMUTABLE = path.join('_app', 'immutable')
+const FOREVER = 'public, max-age=31536000, immutable'
+
+/** The icons: named without a hash, so they are re-read, but rarely. */
+const ICONS = new Set(['favicon.ico', 'favicon-96x96.png', 'apple-touch-icon.png'])
+const DAY = 'public, max-age=86400'
+
+/**
+ * How long what is served may be held. The page is never cached — it names the assets, and
+ * their names carry the build. An icon is named for what it is rather than for its content,
+ * so it is asked about again, but not on every page load.
+ */
+function caching(relative: string): string {
+  if (relative.startsWith(IMMUTABLE)) return FOREVER
+
+  return ICONS.has(relative) ? DAY : 'no-cache'
+}
+
+const TYPES: Record<string, string> = {
+  '.css': 'text/css; charset=utf-8',
+  '.html': 'text/html; charset=utf-8',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.map': 'application/json; charset=utf-8',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml',
+  '.txt': 'text/plain; charset=utf-8',
+  '.webmanifest': 'application/manifest+json',
+  '.webp': 'image/webp',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2'
+}

package/source/Discovery/index.ts

@@ -0,0 +1,3 @@
+export { Explorer } from './Explorer.js'
+export { Site } from './Site.js'
+export { looking } from './trunk.js'

package/source/Discovery/tree.test.ts

@@ -0,0 +1,175 @@
+import { describe as suite, it, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { describe } from './tree.js'
+import { Tree } from '../RTD/Tree.js'
+import type * as http from '../HTTP/index.js'
+import type { Introspection } from '../Introspection.js'
+import type { EndpointsFactory } from '../Endpoint.js'
+import type { DirectiveFactory } from '../RTD/Directives.js'
+import type * as syntax from '../RTD/syntax/index.js'
+
+/** A method with no mapping has no endpoint, so what it says is its directives' alone. */
+const endpoints = {} as unknown as EndpointsFactory
+
+/** What a method marked with this describes as, standing for a directive that refuses. */
+const REFUSE = 'refuse'
+
+const directives = {
+  create: (stack: syntax.Directive[]) => ({
+    declared: () => undefined,
+    precall: async () => null,
+    settle: async () => undefined,
+    dispose: () => undefined,
+    explain: async (_: unknown, introspection: Introspection) => {
+      const mark = stack.find((directive) => directive.family === 'test')?.value
+
+      if (mark === REFUSE) return null
+
+      return mark === undefined ? introspection : { ...introspection, description: mark }
+    }
+  }),
+  preflight: async () => undefined,
+  depart: async () => undefined,
+  dispose: () => undefined
+} as unknown as DirectiveFactory
+
+const method = (verb: string, mark?: string): syntax.Method => ({
+  verb,
+  directives: mark === undefined ? [] : [{ family: 'test', name: 'mark', value: mark }]
+})
+
+const node = (
+  path: string,
+  methods: syntax.Method[] = [],
+  routes: syntax.Route[] = []
+): syntax.Route => ({ path, node: { routes, methods, directives: [] } })
+
+const trunk = (routes: syntax.Route[], methods: syntax.Method[] = []): syntax.Node => ({
+  routes,
+  methods,
+  directives: []
+})
+
+const request = {} as unknown as http.Context
+
+afterEach(() => {
+  delete process.env.__TESTING_EXPOSITION_BRANCH_TTL
+})
+
+suite('discovery tree', () => {
+  it('should key the trunk by the root', async () => {
+    const tree = new Tree(trunk([], [method('GET')]), endpoints, directives)
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), ['/'])
+  })
+
+  it('should key every route by the template it answers at', async () => {
+    const tree = new Tree(
+      trunk([
+        node('/pots', [method('GET'), method('POST')], [node('/:id', [method('GET')])])
+      ]),
+      endpoints,
+      directives
+    )
+
+    const { routes: resources } = await describe(tree, request)
+
+    assert.deepEqual(Object.keys(resources), ['/pots', '/pots/:id'])
+    assert.deepEqual(Object.keys(resources['/pots']), ['GET', 'POST'])
+  })
+
+  it('should include a route no name can spell', async () => {
+    // `/.rpc` and `/.mcp` leave these out; HTTP addresses a route by its path
+    const tree = new Tree(
+      trunk([
+        node('/v1.0', [method('GET')]),
+        node('/files', [], [node('/**', [method('GET')])]),
+        node('/any', [], [node('/*', [method('GET')])])
+      ]),
+      endpoints,
+      directives
+    )
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), [
+      '/any/*',
+      '/files/**',
+      '/v1.0'
+    ])
+  })
+
+  it('should skip an intermediate node, whose route answers in its place', async () => {
+    const tree = new Tree(
+      trunk([node('/posts', [method('PATCH')], [node('/', [method('PUT')])])]),
+      endpoints,
+      directives
+    )
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), ['/posts'])
+  })
+
+  it('should omit a resource this caller may reach no method of', async () => {
+    const tree = new Tree(
+      trunk([
+        node('/open', [method('GET')]),
+        node('/closed', [method('GET', REFUSE), method('POST', REFUSE)])
+      ]),
+      endpoints,
+      directives
+    )
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), ['/open'])
+  })
+
+  it('should carry only the verbs this caller may reach', async () => {
+    const tree = new Tree(
+      trunk([node('/pots', [method('GET'), method('POST', REFUSE)])]),
+      endpoints,
+      directives
+    )
+
+    const { routes: resources } = await describe(tree, request)
+
+    assert.deepEqual(Object.keys(resources['/pots']), ['GET'])
+  })
+
+  it('should answer the declaration a request would reach where two share a template', async () => {
+    // `/a/b` and `/a: { /b: }` are two routes and one template; the more specific matches
+    const tree = new Tree(
+      trunk([
+        node('/a/b', [method('GET', 'nested')]),
+        node('/a', [], [node('/b', [method('GET', 'adjacent')])])
+      ]),
+      endpoints,
+      directives
+    )
+
+    const { routes: resources } = await describe(tree, request)
+
+    assert.deepEqual(Object.keys(resources), ['/a/b'])
+    assert.equal(resources['/a/b'].GET.description, 'nested')
+  })
+
+  it('should drop a branch that has expired', async () => {
+    process.env.__TESTING_EXPOSITION_BRANCH_TTL = '-1'
+
+    const tree = new Tree(trunk([]), endpoints, directives)
+
+    tree.merge(trunk([node('/pots', [method('GET')])]), {
+      namespace: 'default',
+      component: 'pots'
+    })
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), [])
+  })
+
+  it('should sort, because branches merge in the order their tenants answer', async () => {
+    const tree = new Tree(
+      trunk([node('/zed', [method('GET')]), node('/alpha', [method('GET')])]),
+      endpoints,
+      directives
+    )
+
+    assert.deepEqual(Object.keys((await describe(tree, request)).routes), ['/alpha', '/zed'])
+  })
+})

package/source/Discovery/tree.ts

@@ -0,0 +1,77 @@
+import { describing } from '../Introspection.js'
+import { template } from '../RPC/names.js'
+import { variables } from '../RTD/segment.js'
+import { guarded, resource, type Described } from '../directives/help/index.js'
+import type * as http from '../HTTP/index.js'
+import type { Tree } from '../RTD/index.js'
+import type { Introspection } from '../Introspection.js'
+
+/**
+ * Every route in the tree this caller may reach, keyed by the template it answers at, and
+ * each described exactly as `OPTIONS` on that path describes it — so a key can be taken
+ * from the answer and sent back as a request.
+ *
+ * A route a name cannot spell is here, unlike at `/.rpc` and `/.mcp`: HTTP addresses a
+ * route by its path, so leaving one out would make this disagree with the router. A
+ * resource no method of which this caller may reach is not here at all — `OPTIONS` answers
+ * `403` to one, and an empty entry would still say it exists.
+ *
+ * Sorted, because a branch is merged whenever its tenant answers and two replicas would
+ * otherwise order the same tree differently. Verbs keep the order they were declared in,
+ * which is the order `OPTIONS` answers them in.
+ */
+export async function describe(tree: Tree, request: http.Context): Promise<Discovered> {
+  const context = describing(request)
+  const routes = new Map<string, Mounted>()
+
+  /*
+   * Sequentially: there is no I/O to overlap — an endpoint's own description is read once
+   * and memoized — so awaiting the whole tree at once would only allocate every clone of
+   * it at the same moment.
+   */
+  for (const { segments, verb, method } of tree.walk()) {
+    const introspection = await method.explain(context, variables(segments))
+
+    if (introspection === null) continue
+
+    const route = template(segments)
+    let mounted = routes.get(route)
+
+    if (mounted === undefined) {
+      mounted = { described: resource(method.directives), methods: {} }
+      routes.set(route, mounted)
+    }
+
+    // two declarations can walk to one template — `/a/b` and `/a: { /b: }` — and the walk
+    // is in the order `match` tries them, so the first is the one a request would reach
+    mounted.methods[verb] ??= introspection
+  }
+
+  const described: Record<string, Resource> = {}
+
+  for (const route of Array.from(routes.keys()).sort()) {
+    const { described: stated, methods } = routes.get(route)!
+
+    // what the resource is, beside the methods it serves; a verb is upper case and cannot
+    // collide with either key
+    described[route] = { ...stated, ...guarded(methods), ...methods }
+  }
+
+  return { routes: described }
+}
+
+/** What the tree answers. An object, so that what is said of the whole of it has somewhere to go. */
+export interface Discovered {
+  routes: Record<string, Resource>
+}
+
+/** What one resource is, and what it serves; a verb is upper case, and nothing else here is. */
+export interface Resource extends Described {
+  [verb: string]: unknown
+}
+
+/** One route template as the walk found it, before the two are answered as one. */
+interface Mounted {
+  described: Described | null
+  methods: Record<string, Introspection>
+}

package/source/Discovery/trunk.ts

@@ -0,0 +1,23 @@
+import Negotiator from 'negotiator'
+import { types } from '../HTTP/formats/index.js'
+import { DISCOVERY } from '../const.js'
+import type { Context, OutgoingMessage } from '../HTTP/index.js'
+
+/** What a browser asks for, and what nothing else here answers with. */
+const HTML = 'text/html'
+
+/**
+ * Where a browser that asked for the trunk is sent. An application serves what it declares,
+ * and `/` is usually not one of those — a person who typed the address into a browser is
+ * looking for something to look at, and the page is the only thing here that is one.
+ *
+ * A client that takes anything is not one: `accept` has to prefer a page over what the
+ * gateway answers with, which is what a browser sends and an API client does not.
+ */
+export function looking(context: Context): OutgoingMessage | null {
+  if (context.request.method !== 'GET' || context.procedural) return null
+
+  if (new Negotiator(context.request).mediaType([...types, HTML]) !== HTML) return null
+
+  return { status: 302, headers: new Headers({ location: DISCOVERY + '/' }) }
+}

package/source/Endpoint.ts

@@ -52,6 +52,10 @@ export class Endpoint implements RTD.Endpoint {
     return message
   }
 
+  public selection(): Record<string, Schema> | null {
+    return this.mapping.selection()
+  }
+
   public async explain(parameters: RTD.Parameter[]): Promise<Introspection> {
     this.introspection ??= await this.introspect(parameters)
 

package/source/Gateway.ts

@@ -4,9 +4,11 @@ import { console } from 'openspan'
 import { Connector } from '@toa.io/core'
 import type { bindings } from '@toa.io/core/types'
 import * as http from './HTTP/index.js'
-import { MCP, RPC } from './const.js'
+import { DISCOVERY, MCP, RPC } from './const.js'
 import { rethrow } from './exceptions.js'
 import { decide } from './Branch.js'
+import { describing } from './Introspection.js'
+import { Explorer, looking } from './Discovery/index.js'
 import type { Interception } from './Interception.js'
 import type { Dispatcher } from './RPC/index.js'
 import type { Server } from './MCP/index.js'
@@ -32,6 +34,9 @@ export class Gateway extends Connector {
 
   /** And the same of MCP. */
   private readonly mcp: Server | null
+
+  /** The tree, for every route at once. Served always, and so not nullable. */
+  private readonly explorer: Explorer
   private readonly branches = new Map<string, Exposed>()
   private lastMerge = 0
   private widestGap = 0
@@ -56,6 +61,7 @@ export class Gateway extends Connector {
     this.directives = directives
     this.dispatcher = dispatcher
     this.mcp = mcp
+    this.explorer = new Explorer(tree)
 
     this.depends(broadcast)
   }
@@ -89,9 +95,28 @@ export class Gateway extends Connector {
     if (this.mcp !== null && context.url.pathname === MCP)
       return await this.mcp.process(context, route)
 
+    // the page under this prefix is an interceptor's, and has already answered
+    if (context.url.pathname === DISCOVERY || context.url.pathname === DISCOVERY + '/')
+      return await this.explorer.process(context)
+
+    // a browser that asked for the trunk of an API is looking for the page, and where the
+    // application answers there itself, that is what answers
+    if (context.url.pathname === '/' && !this.serves(context)) {
+      const page = looking(context)
+
+      if (page !== null) return page
+    }
+
     return await this.route(context)
   }
 
+  /** Whether the application serves the trunk with the verb the request was made with. */
+  private serves(context: http.Context): boolean {
+    const match = this.tree.match('/')
+
+    return match !== null && context.request.method in match.node.methods
+  }
+
   /**
    * One call: everything that needs a node. A request makes one of these, and a request
    * that carries several calls makes one per call.
@@ -200,14 +225,19 @@ export class Gateway extends Connector {
     node: Node,
     parameters: Parameter[]
   ): Promise<http.OutgoingMessage> {
-    const body = await node.explain(context, parameters)
-    const verbs = Object.keys(body)
+    const { described, methods } = await node.explain(describing(context), parameters)
+    const verbs = Object.keys(methods)
 
     // what a caller may not use is not a resource to them, and describing it would be the leak
     if (verbs.length === 0 && Object.keys(node.methods).length > 0)
       throw new http.Forbidden()
 
-    return { body, headers: new Headers({ allow: verbs.join(', ') }) }
+    // what the resource is, beside the methods it serves; a verb is upper case and cannot
+    // collide with either key
+    return {
+      body: { ...described, ...methods },
+      headers: new Headers({ allow: verbs.join(', ') })
+    }
   }
 
   private async discover(): Promise<void> {

package/source/HTTP/Context.ts

@@ -26,6 +26,13 @@ export class Context {
    */
   public readonly procedural: boolean = false
 
+  /**
+   * Whether this describes a resource rather than calling one. What builds a describing
+   * context says so, and a directive that answers differently to a description than to a
+   * call reads it here.
+   */
+  public readonly exploratory: boolean = false
+
   public readonly pipelines: Pipelines = {
     body: [],
     response: []

package/source/Introspection.ts

@@ -1,4 +1,5 @@
 import type { Remote } from '@toa.io/core'
+import type { Context } from './HTTP/index.js'
 
 export interface Introspection {
   /**
@@ -11,24 +12,42 @@ export interface Introspection {
 
   /** what a person is shown where a client lists this method, which a name is not */
   title?: string
+
+  /** whether reaching it takes being someone, whoever — `auth:anyone` and its like */
+  authenticated?: boolean
+
+  /** whether reaching it is being the identity it is about — `auth:id` */
+  private?: boolean
+
+  /** whether reaching it takes a role — `auth:role` */
+  protected?: boolean
+
+  /** and whether that role is one of the `system` scope */
+  system?: boolean
+
+  /** whether the route publishes this method to a model — [`mcp:tool`](mcp.md) */
+  mcp?: boolean
   route?: Record<string, Schema>
   query?: Record<string, Schema>
-
-  /** what a request header carries, which is therefore not the body's to send */
-  headers?: Record<string, Sourced>
   input?: Schema
   output?: Schema
   errors?: string[]
 }
 
-/** A property the gateway reads from somewhere else, and the schema it was declared with. */
-export interface Sourced {
-  header: string
-  [keyword: string]: unknown
-}
-
 export type Schema = Awaited<ReturnType<Remote['explain']>>['input']
 
+/**
+ * The same request, as a description of a resource rather than a call to one. A directive
+ * that answers differently to the two reads `exploratory` — `Anonymous` is the one that
+ * does, because the rule refusing a credential at an `anonymous` route is about a reply a
+ * cache would hold, and a description is not one.
+ */
+export function describing(context: Context): Context {
+  return Object.create(context, {
+    exploratory: { value: true, enumerable: true }
+  }) as Context
+}
+
 /**
  * The same, in the order the shape states — whatever order the families that filled it ran
  * in. What a resource says about itself is read, so it is written the way it is documented.
@@ -45,9 +64,13 @@ export function order(introspection: Introspection): Introspection {
 const KEYS = [
   'title',
   'description',
+  'authenticated',
+  'private',
+  'protected',
+  'system',
+  'mcp',
   'route',
   'query',
-  'headers',
   'input',
   'output',
   'errors'