@c9up/ream 0.1.34 → 0.2.0

package/src/http/Response.ts

@@ -7,11 +7,20 @@
  * @implements FR21
  */
 
-import { readFileSync } from 'node:fs'
+import { randomUUID } from 'node:crypto'
+import { createReadStream } from 'node:fs'
+import { readFile, stat } from 'node:fs/promises'
 import { basename } from 'node:path'
 import etag from 'etag'
 import { contentType } from 'mime-types'
 import { durationToSeconds } from '../helpers/duration.js'
+
+/**
+ * Default ceiling on a buffered response body — the same 100MB the Rust layer
+ * caps an incoming body at (`MAX_BODY_BYTES` in crates/ream-http/src/server.rs).
+ */
+const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024
+
 import type { CookieSigner } from '../security/CookieSigner.js'
 import { Macroable } from '../utils/Macroable.js'
 import { E_HTTP_REQUEST_ABORTED } from './Exception.js'
@@ -133,13 +142,32 @@ export interface CookieOptions {
 
 export class Response extends Macroable {
   #status = 200
-  /** A body still being drained by {@link stream}; awaited by {@link settle}. */
+  /** A body still being drained by {@link stream}; awaited by {@link finish}. */
   #pendingStream: Promise<void> | undefined
+  /** The detached chunk pump, when one is running. See {@link streamed}. */
+  #pumping: Promise<void> | undefined
+  /** Whether {@link finish} has run — its idempotence guard. */
+  #finalised = false
   #headers: Record<string, string> = {}
   #cookies: string[] = []
   #body = ''
   readonly #finishCallbacks: Array<(err: Error | null) => void> = []
   #finished = false
+  /**
+   * The context this response belongs to (AdonisJS `response.ctx`).
+   *
+   * Wired by HttpContext. Optional because a Response built by hand — a test,
+   * a fixture — has no context around it.
+   */
+  ctx?: object
+
+  /** Default JSONP callback name (AdonisJS `http.jsonpCallbackName`). */
+  #jsonpCallbackName = 'callback'
+  /**
+   * Ceiling on a buffered response body. Mirrors the 100MB the Rust layer caps
+   * an incoming body at, so the two directions are explained the same way.
+   */
+  #maxBodyBytes = DEFAULT_MAX_RESPONSE_BYTES
   #redirectBuilderFactory?: () => RedirectBuilder
   #streamBackend?: StreamBackend
   #streamId?: string
@@ -268,7 +296,16 @@ export class Response extends Macroable {
    * (valid in JSON, but break JS), so neither the callback nor the payload can
    * inject script.
    */
-  jsonp(body: unknown, callbackName = 'callback'): void {
+  /**
+   * Default callback name when `jsonp()` is called without one (AdonisJS reads
+   * it from `http.jsonpCallbackName`). Injected by HttpKernel, the same way the
+   * cookie signer and signed-URL services are.
+   */
+  setJsonpCallbackName(name: string): void {
+    this.#jsonpCallbackName = name
+  }
+
+  jsonp(body: unknown, callbackName: string = this.#jsonpCallbackName): void {
     // Sanitise the callback name to identifier-safe characters — the JSONP XSS
     // guard (an attacker-controlled callback must not inject script).
     const safeCallback = callbackName.replace(/[^\w$.]/g, '')
@@ -323,6 +360,7 @@ export class Response extends Macroable {
    * unambiguous.
    */
   sendBuffer(buffer: Buffer): void {
+    this.#assertBodyFits(buffer.length)
     if (!this.#headers['content-type']) {
       this.#headers['content-type'] = 'application/octet-stream'
     }
@@ -331,6 +369,34 @@ export class Response extends Macroable {
     this.#finished = true
   }
 
+  /**
+   * Refuse a body too large to hold in memory.
+   *
+   * A response body is serialised whole across the NAPI boundary — there is no
+   * chunked write yet — and a binary one is base64'd on the way, so it costs
+   * roughly 2.3x its size in transient memory: the Buffer, the encoded string,
+   * and the Rust-side decode. Without a ceiling a large file did not fail, it
+   * grew until the process died, with no message naming the cause.
+   *
+   * Raise it with `maxResponseBytes` in the kernel config when you know the
+   * memory is there. For anything genuinely large, hand out a signed URL from
+   * `@c9up/archive` instead and let the client fetch it from storage — the
+   * bytes then never pass through the server at all.
+   */
+  #assertBodyFits(bytes: number): void {
+    if (bytes <= this.#maxBodyBytes) return
+    throw new Error(
+      `[E_RESPONSE_TOO_LARGE] Response body is ${Math.round(bytes / 1_048_576)}MB, over the ${Math.round(this.#maxBodyBytes / 1_048_576)}MB ceiling. ` +
+        'The body is held whole in memory (and base64-encoded, ~2.3x) because responses are not streamed yet. ' +
+        'Raise `maxResponseBytes` in the kernel config, or serve large files with a signed URL from @c9up/archive.',
+    )
+  }
+
+  /** @internal Set the ceiling — injected by HttpKernel from its config. */
+  setMaxBodyBytes(bytes: number): void {
+    this.#maxBodyBytes = bytes
+  }
+
   /** Send 204 No Content. */
   noContent(): void {
     this.#status = 204
@@ -588,14 +654,31 @@ export class Response extends Macroable {
 
   // ─── Redirect ─────────────────────────────────────────────
 
-  /** Get a redirect builder. */
-  redirect(): RedirectBuilder {
-    if (this.#redirectBuilderFactory) {
-      return this.#redirectBuilderFactory()
+  /**
+   * Redirect, or get the builder.
+   *
+   * `redirect()` with no argument hands back the builder to chain on. With a
+   * PATH it redirects immediately, as AdonisJS does
+   * (`redirect(path, forwardQueryString?, statusCode?)`); `'back'` goes to the
+   * referrer. Ream always returned the builder, so `response.redirect('/login')`
+   * — the shortest and most obvious spelling — silently did nothing.
+   */
+  redirect(): RedirectBuilder
+  redirect(path: string, forwardQueryString?: boolean, statusCode?: number): void
+  redirect(path?: string, forwardQueryString = false, statusCode = 302): RedirectBuilder | void {
+    if (!this.#redirectBuilderFactory) {
+      throw new Error(
+        'redirect() requires an HttpContext. Response was created outside a request handler.',
+      )
     }
-    throw new Error(
-      'redirect() requires an HttpContext. Response was created outside a request handler.',
-    )
+    const builder = this.#redirectBuilderFactory()
+    if (path === undefined) return builder
+    if (forwardQueryString) builder.withQs()
+    if (path === 'back') {
+      builder.status(statusCode).back()
+      return
+    }
+    builder.status(statusCode).toPath(path)
   }
 
   /** @internal Set the redirect builder factory (injected by HttpContext). */
@@ -670,7 +753,40 @@ export class Response extends Macroable {
    * @internal Wait for a body still being produced (see {@link stream}). The
    * kernel calls this before serializing; everything else resolves at once.
    */
-  async settle(): Promise<void> {
+  /**
+   * Resolves when a streamed body has finished feeding the socket.
+   *
+   * Deliberately NOT part of {@link finish}: the response has to reach Rust
+   * with its stream id before the body can be attached, so waiting for the
+   * pump there would deadlock — and closing the stream first is what made the
+   * client see `E_STREAM_UNKNOWN` instead of the file. This is for a test that
+   * wants the finished body, and for a shutdown that would rather let an
+   * in-flight download end than cut it.
+   */
+  streamed(): Promise<void> {
+    return this.#pumping ?? Promise.resolve()
+  }
+
+  /**
+   * Finalise the response — the terminal step (AdonisJS `response.finish()`).
+   *
+   * Same role and same guarantees as upstream: idempotent, it stamps the
+   * request id, and afterwards nothing more is added to the response. The
+   * kernel calls it before reading the response out; an app rarely needs to.
+   *
+   * NAMED DEVIATION — it returns a promise where upstream returns void, and it
+   * writes nothing. Ream never owns the socket: the whole response crosses the
+   * NAPI boundary and Rust writes it, so "finish" here means *sealed and ready
+   * to hand over* rather than *flushed*. The await is for a body still being
+   * produced — `download()` and a buffered `stream()` fill it asynchronously.
+   *
+   * A STREAMED body is deliberately not awaited: it feeds the socket after the
+   * response has been handed over. See {@link streamed}.
+   */
+  async finish(): Promise<void> {
+    if (this.#finalised) return
+    this.#finalised = true
+    this.setRequestId()
     const pending = this.#pendingStream
     if (pending === undefined) return
     this.#pendingStream = undefined
@@ -687,9 +803,25 @@ export class Response extends Macroable {
     generateEtag = false,
     errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
   ): void {
-    let content: Buffer
+    // Read asynchronously, parked on the same pending-body slot `stream()`
+    // uses and `finish()` already awaits before serialisation. It used to be
+    // `readFileSync`, which stalled the event loop for EVERY concurrent
+    // request while one client downloaded — the file's size became every
+    // other request's latency.
+    this.#pendingStream = this.#readForDownload(filePath, generateEtag, errorCallback)
+  }
+
+  async #readForDownload(
+    filePath: string,
+    generateEtag: boolean,
+    errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
+  ): Promise<void> {
+    // Stat first: a missing or unreadable file must answer 404 BEFORE any
+    // header goes out, and once a stream starts there is no status left to
+    // change.
+    let size: number
     try {
-      content = readFileSync(filePath)
+      size = (await stat(filePath)).size
     } catch (error) {
       if (errorCallback) {
         const [message, status] = errorCallback(error as NodeJS.ErrnoException)
@@ -699,8 +831,20 @@ export class Response extends Macroable {
       this.status(404).send('Not Found')
       return
     }
+
     const ct = contentType(basename(filePath))
     if (ct) this.#headers['content-type'] = ct
+
+    // An ETag is a hash of the whole file, so asking for one asks for the file
+    // in memory. Without one, stream it: the body goes out as it is read and
+    // nothing bigger than a chunk is ever held.
+    if (!generateEtag && this.#streamBackend?.writeStreamBytes !== undefined) {
+      this.#headers['content-length'] = String(size)
+      await this.#pipe(createReadStream(filePath), errorCallback)
+      return
+    }
+
+    const content = await readFile(filePath)
     if (generateEtag) this.setEtag(content)
     this.sendBuffer(content)
   }
@@ -771,11 +915,78 @@ export class Response extends Macroable {
     body: NodeJS.ReadableStream,
     errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
   ): Promise<void> {
-    const draining = this.#drain(body, errorCallback)
+    const draining = this.#pipe(body, errorCallback)
     this.#pendingStream = draining
     return draining
   }
 
+  /**
+   * Send a readable as the body.
+   *
+   * Chunks go to the socket as they arrive when the host can carry them —
+   * nothing bigger than one chunk is ever held, so the file's size stops being
+   * the process's memory ceiling. A host with no binary stream backend (a unit
+   * test, a mock server) falls back to draining into memory, which is what
+   * this always used to do.
+   */
+  async #pipe(
+    body: NodeJS.ReadableStream,
+    errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
+  ): Promise<void> {
+    const backend = this.#streamBackend
+    const writeBytes = backend?.writeStreamBytes
+    if (backend === undefined || writeBytes === undefined) {
+      return this.#drain(body, errorCallback)
+    }
+
+    const streamId = randomUUID()
+    if (!(await backend.registerStream(streamId))) {
+      // The id collided — vanishingly unlikely, but answering with a body
+      // nobody feeds would hang the client forever.
+      return this.#drain(body, errorCallback)
+    }
+    if (!this.#headers['content-type']) {
+      this.#headers['content-type'] = 'application/octet-stream'
+    }
+    this.#streamId = streamId
+    this.#finished = true
+
+    // Pump DETACHED, exactly as `sse()` does. What is awaited here is only the
+    // registration: the kernel awaits this before handing the response back to
+    // Rust, and Rust then looks the id up to attach the body. Awaiting the pump
+    // instead would close the stream before that lookup ever happened, and the
+    // client would get `E_STREAM_UNKNOWN` in place of the file.
+    this.#pumping = this.#pump(backend, writeBytes.bind(backend), streamId, body, errorCallback)
+  }
+
+  /** Feed a registered stream until the source ends or the client leaves. */
+  async #pump(
+    backend: StreamBackend,
+    writeBytes: (id: string, chunk: Uint8Array) => Promise<boolean>,
+    streamId: string,
+    body: NodeJS.ReadableStream,
+    errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
+  ): Promise<void> {
+    try {
+      for await (const chunk of body) {
+        const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk
+        // `false` means the client is gone. Stop reading the source rather
+        // than pumping a whole file into a closed socket.
+        if (!(await writeBytes(streamId, bytes))) break
+      }
+    } catch (err) {
+      // The headers are already out, so there is no status left to change: end
+      // the body and let the client see a short read. `errorCallback` is still
+      // consulted for whatever logging it does.
+      errorCallback?.(err as NodeJS.ErrnoException)
+    } finally {
+      // Never rethrows: this runs detached, and an unhandled rejection here
+      // would take the process down over one client's download.
+      await backend.closeStream(streamId).catch(() => {})
+    }
+  }
+
+  /** Read the whole body into memory. The fallback when nothing can stream. */
   async #drain(
     body: NodeJS.ReadableStream,
     errorCallback?: (error: NodeJS.ErrnoException) => [string, number?],
@@ -991,6 +1202,64 @@ export class Response extends Macroable {
     return this.#finished
   }
 
+  // ─── State (AdonisJS getters) ──────────────────────────────
+  //
+  // Same questions AdonisJS answers on its response, under the same names. Ours
+  // only had `isFinished()`, so a migrated `if (response.finished)` read
+  // `undefined` and took the wrong branch without a word.
+
+  /** Whether the response has been written out. */
+  get finished(): boolean {
+    return this.#finished
+  }
+
+  /** Whether a body has been set. */
+  get hasContent(): boolean {
+    return this.#body.length > 0
+  }
+
+  /** Whether the body is a stream still being drained. */
+  get hasStream(): boolean {
+    return this.#pendingStream !== undefined
+  }
+
+  /** Whether a body of any kind is waiting to go out. */
+  get hasLazyBody(): boolean {
+    return this.hasContent || this.hasStream
+  }
+
+  /**
+   * Whether the headers have gone out.
+   *
+   * Ream serialises the whole response across the NAPI boundary in one step,
+   * so headers and body leave together: this is true exactly when the response
+   * is finished, rather than tracking a separate Node `headersSent`.
+   */
+  get headersSent(): boolean {
+    return this.#finished
+  }
+
+  /** Whether nothing has been sent yet — the window a middleware can still write in. */
+  get isPending(): boolean {
+    return !this.headersSent && !this.finished
+  }
+
+  /**
+   * Echo the caller's `x-request-id` back on the response (AdonisJS
+   * `setRequestId`).
+   *
+   * Ream already READS that header into `ctx.id` — validating its shape and
+   * generating one when it is missing — but never sent it back, so a caller
+   * could not tie a response to the request id it issued. Nothing is echoed
+   * when the client sent none: inventing one here would hand back an id the
+   * caller never used.
+   */
+  setRequestId(): this {
+    const incoming = this.#request?.header('x-request-id')
+    if (incoming) this.header('x-request-id', incoming)
+    return this
+  }
+
   /** @internal Set body directly (used by redirect, exception handler). */
   setBody(body: string): void {
     this.#body = body

package/src/http/SseStream.ts

@@ -37,6 +37,16 @@
 export interface StreamBackend {
   registerStream(streamId: string): Promise<boolean>
   writeStream(streamId: string, chunk: string): Promise<boolean>
+  /**
+   * Push a BINARY chunk, waiting for room rather than dropping it.
+   *
+   * `writeStream` is shaped for events: text, and a full buffer discards the
+   * frame — right when a missed event is a missed event, wrong when the client
+   * is reassembling a file and a missing frame is a corrupt download. Optional
+   * so a host that only serves SSE need not implement it; `response.stream()`
+   * falls back to buffering when it is absent.
+   */
+  writeStreamBytes?(streamId: string, chunk: Uint8Array): Promise<boolean>
   closeStream(streamId: string): Promise<boolean>
   onStreamDisconnect(streamId: string, callback: () => void): void
 }

package/src/index.ts

@@ -14,6 +14,7 @@ export { Container } from './container/Container.js'
 export { ContainerResolver } from './container/ContainerResolver.js'
 export { ContextualBindingsBuilder } from './container/ContextualBindingsBuilder.js'
 export type { Binding, ServiceFactory, ServiceScope, ServiceToken } from './container/types.js'
+export { type DirectoriesNode, directories } from './directories.js'
 export { Env, EnvValidationException } from './env/Env.js'
 export { loadEnvFiles } from './env/loadEnvFiles.js'
 export { Secret } from './env/Secret.js'
@@ -25,6 +26,13 @@ export {
   type SchemaNode,
   type StringOptions,
 } from './env/schema.js'
+export { MigrationRegistry } from './migrations/MigrationRegistry.js'
+export type {
+  MigrationRunnerContract,
+  MigrationState,
+  MigrationStatusNode,
+  RegisteredMigrationSource,
+} from './migrations/types.js'
 
 // ─── HTTP (new — AdonisJS-compatible) ───────────────────────
 
@@ -164,8 +172,16 @@ export type {
   RouteDefinition,
   RouteHandler,
   RouteHandlerFunction,
+  SignedUrlOptions,
+} from './router/Router.js'
+export {
+  GroupBuilder,
+  matchers,
+  OnRouteBuilder,
+  RouteBuilder,
+  Router,
+  UrlBuilder,
 } from './router/Router.js'
-export { GroupBuilder, matchers, OnRouteBuilder, RouteBuilder, Router } from './router/Router.js'
 
 // ─── Middleware ──────────────────────────────────────────────
 
@@ -290,12 +306,22 @@ export { setServer } from './services/server.js'
 // `@c9up/ream/events` subpath so importing the main barrel stays lazy.
 export type { EventsConfig } from './events/config.js'
 export { defineConfig as defineEventsConfig } from './events/config.js'
+export type {
+  AsyncUnsubscribeFunction,
+  UnsubscribeFunction,
+} from './events/Emitter.js'
 export {
   BaseEvent,
   Emitter,
   type EmitterResolver,
   type ListenerClass,
 } from './events/Emitter.js'
+export {
+  type BufferedEvent,
+  type BufferedEventName,
+  type EventFinder,
+  EventsBuffer,
+} from './events/EventsBuffer.js'
 export type {
   ErrorReporter,
   LockBackend,

package/src/migrations/MigrationRegistry.ts

@@ -0,0 +1,49 @@
+import type { RegisteredMigrationSource } from './types.js'
+
+/**
+ * Where every store's migration runner registers itself.
+ *
+ * Bound in the container as `migrations`, always — an app with no data package
+ * gets an EMPTY registry, never a missing binding. That distinction is what
+ * lets the CLI tell "this app has no migrations" from "this app predates the
+ * registry", and say something useful in each case instead of failing the same
+ * way twice.
+ */
+export class MigrationRegistry {
+  readonly #sources = new Map<string, RegisteredMigrationSource>()
+
+  /**
+   * Register a store's runner. Registering the same name twice is refused:
+   * silently replacing it would mean one of two providers migrates nothing,
+   * and the run would still report success.
+   */
+  register(source: RegisteredMigrationSource): this {
+    if (this.#sources.has(source.name)) {
+      throw new Error(
+        `A migration source named '${source.name}' is already registered. Two providers cannot claim one name — one of them would migrate nothing while the run still reported success.`,
+      )
+    }
+    this.#sources.set(source.name, source)
+    return this
+  }
+
+  /** Every registered store, in registration order. */
+  all(): RegisteredMigrationSource[] {
+    return [...this.#sources.values()]
+  }
+
+  /** One store by name, or `undefined`. */
+  get(name: string): RegisteredMigrationSource | undefined {
+    return this.#sources.get(name)
+  }
+
+  /** The registered names, for an error message that lists what IS available. */
+  names(): string[] {
+    return [...this.#sources.keys()]
+  }
+
+  /** Whether anything registered. */
+  get isEmpty(): boolean {
+    return this.#sources.size === 0
+  }
+}

package/src/migrations/types.ts

@@ -0,0 +1,68 @@
+/**
+ * The contract a data package implements so `ream migrate` can drive it.
+ *
+ * REAM PARTICULARITY, deliberate — AdonisJS has no equivalent, because Lucid is
+ * its only migration source: `node ace migration:run` can name it directly.
+ * Ream expects several stores in one app (a relational one, a time-series one,
+ * whatever comes next), so the CLI must not name any of them. A store registers
+ * itself and the CLI stays generic for good.
+ *
+ * The vocabulary is Adonis' where Adonis has one: a status node is
+ * `{ name, status, batch? }`, spelled `status` as Lucid's `MigrationListNode`
+ * spells it.
+ */
+
+/** Whether a migration has run. */
+export type MigrationState = 'applied' | 'pending'
+
+/** One migration's position, as `ream migrate:status` prints it. */
+export interface MigrationStatusNode {
+  /** The migration's name, without extension. */
+  name: string
+  status: MigrationState
+  /** The batch it was applied in, when it has been. */
+  batch?: number
+}
+
+/**
+ * What a store's migration runner must provide.
+ *
+ * Only three methods are REQUIRED — the three `ream migrate`,
+ * `migrate:rollback` and `migrate:status` call. Everything else is optional on
+ * purpose: a new store must be able to register with a runner that only knows
+ * how to go forward, rather than stub methods it cannot honour. A caller that
+ * needs an optional one checks for it and says so, instead of crashing.
+ */
+export interface MigrationRunnerContract {
+  /** Apply every pending migration. Resolves to the names applied, in order. */
+  migrate(): Promise<string[]>
+  /** Undo the last batch. Resolves to the names rolled back. */
+  rollback(): Promise<string[]>
+  /** Every known migration and whether it has run. */
+  status(): Promise<MigrationStatusNode[]>
+
+  /** Undo every migration. */
+  reset?(): Promise<string[]>
+  /** Reset, then migrate. */
+  refresh?(): Promise<unknown>
+  /** Drop everything, then migrate. */
+  fresh?(): Promise<unknown>
+  /** What `migrate()` would do, without doing it. */
+  dryRun?(): Promise<unknown>
+  /** Release a lock a crashed run left behind. */
+  forceUnlock?(): Promise<boolean>
+}
+
+/** A registered store, as the CLI sees it. */
+export interface RegisteredMigrationSource {
+  /**
+   * How the store is named on the command line — `atlas`, `eon`, …
+   *
+   * It is what `ream migrate --only <name>` matches and what prefixes the
+   * output, so it belongs to the store, not to the app.
+   */
+  name: string
+  /** Where its migration files live, for the message when there are none. */
+  directory?: string
+  runner: MigrationRunnerContract
+}