@statewalker/webrun-streams 0.1.1 → 0.2.1

package/README.md

@@ -15,7 +15,17 @@ Every higher-level package in the `webrun-*` family (and its consumers — scann
 5. **Error (de)serialisation** for passing exceptions across structured-clone / JSON boundaries without losing stacks or extra fields.
 6. **Line / JSONL / text codecs** so stream-processing code doesn't re-invent split/join/encode/decode in every consumer.
 
-The MessagePack codec that previously rode along here is split out to [`@statewalker/webrun-msgpack`](../webrun-msgpack) so consumers that don't need framing don't pull in `@ygoe/msgpack`.
+The MessagePack codec that previously rode along here is split out to [`@statewalker/webrun-msgpack`](../webrun-msgpack) so consumers that don't need framing don't pull in a MessagePack implementation.
+
+## Install
+
+```sh
+npm install @statewalker/webrun-streams
+```
+
+Zero runtime dependencies, zero peer dependencies. ESM only
+(`"type": "module"`); runs in browsers, Node, Deno, Bun and Workers. This is the
+foundation package — everything else in the workspace depends on it.
 
 ## How to use
 
@@ -239,14 +249,18 @@ const stop = serve(async function* handler(input) {
 | `side` | `"initiator"` | Id allocation: initiator uses even ids, responder odd. Pick one per peer so they cannot collide. |
 | `maxStreams` | `256` | Concurrent streams before new calls are refused. |
 | `mtu` | `65536` | Largest payload per DATA frame; bigger chunks are split. |
-| `maxStreamBuffer` | `8388608` | Inbound bytes one stream may hold for a consumer that has not drained them. Exceeding it tears down that stream alone. |
+| `maxStreamBuffer` | `8388608` | The credit this side advertises to the peer, in bytes, and the hard cap on inbound bytes one stream may hold undrained. A peer that honours credit never reaches the cap; one that ignores it has that stream torn down. |
 
 ### Flow control
 
-One in-flight DATA frame per stream. The sender waits for an `ACK`, and the ACK
-is sent only once the consumer has pulled *past* that chunk — so a fast producer
-cannot run ahead of a slow consumer, and outbound is bounded to one `mtu` per
-stream even against a peer that never acknowledges.
+Receiver-advertised credit. Each side puts its `maxStreamBuffer` in the frame it
+opens with — `OPEN` for the caller, the `ACK` answering it for the responder —
+and the sender may only send what it has been granted. Both sides start at zero,
+so a caller pays one round trip per stream before its first DATA frame and none
+thereafter. The receiver grants more once its consumer has actually drained,
+batched at half the window and flushed as soon as its queue empties. A sender
+therefore cannot overrun the receiver's buffer, and `maxStreamBuffer` bounds only
+peers that ignore the protocol.
 
 Backpressure is **per-stream**, so a stalled stream does not block the others,
 and it applies symmetrically in both directions.
@@ -254,6 +268,10 @@ and it applies symmetrically in both directions.
 There is deliberately **no stall timeout**: a peer that never acknowledges
 blocks that producer indefinitely, exactly as a TCP receiver that never reads
 blocks its sender. `maxStreams` and `maxStreamBuffer` bound what that can cost.
+The bound is per stream and there is no mux-wide budget, so the worst case is
+`maxStreams × maxStreamBuffer` — 2 GiB at the defaults, against 16 MiB under the
+one-frame-in-flight rule this replaced. Lower `maxStreamBuffer` if that matters
+more than throughput.
 
 ### Behaviour on hostile input
 
@@ -263,6 +281,67 @@ rather than failing the connection — otherwise one malformed frame would tear
 down every stream sharing the mux. A stream that exceeds `maxStreamBuffer` is
 torn down on its own, with an error frame sent to the peer.
 
+## Exports
+
+Everything is exported from the package root.
+
+### Seam types
+
+| Export | Kind | Purpose |
+| --- | --- | --- |
+| `Duplex` | type | `(input) => AsyncGenerator<Uint8Array>` — one logical call. |
+| `Connect<P>` | type | `(params) => Promise<{ call: Duplex; close() }>`. |
+| `Serve<P>` | type | `(params, handler) => Promise<() => Promise<void>>`. |
+| `ByteChannel` | type | `{ send, recv, closed, close }` — the minimum a transport must expose. |
+| `ByteLike` | type | Accepted byte inputs before normalisation. |
+| `emulateMux(channel, opts?)` | function | Multi-stream over a single `ByteChannel`. |
+| `EmulateMuxOptions` | type | `maxStreams` (256), `mtu` (64 KiB), `maxStreamBuffer` (8 MiB), `side`. |
+| `TransportClosedError` | class | Thrown when the transport closes with calls in flight. Catch via `instanceof` or `error.name`. |
+
+The port multiplexing layer — `multiplexPort`, `PortMux`, `structuredCodec`,
+`PortEnvelope` and the `MessageTarget` family of types — lives in
+[`@statewalker/webrun-rpc`](../webrun-rpc#port-multiplexing), not here.
+
+### Credit
+
+| Export | Kind | Purpose |
+| --- | --- | --- |
+| `newCreditLedger(initial?)` | function | Sender-side credit: `reserve(upTo)` waits for any credit and returns how much it got (`upTo` must be >= 1; less rejects with a `RangeError`), `grant(units)` releases waiters in order, `fail(err)` unwinds them. Starts at zero unless told otherwise. |
+| `CreditLedger` | type | `{ available, reserve, grant, fail }`. |
+| `newCreditGrantor(window, threshold?)` | function | Receiver-side: `consumed(units, queueEmpty)` returns the credit to hand back, batched at `threshold` (default half the window) and flushed once the queue empties. |
+| `CreditGrantor` | type | `{ consumed(units, queueEmpty): number }`. |
+
+The unit is whatever the caller counts. `emulateMux` counts bytes; a value-
+oriented caller would count values. Nothing in this module interprets it.
+
+### Collectors and codecs
+
+| Export | Purpose |
+| --- | --- |
+| `collect` / `collectBytes` / `collectString` | Drain an async iterable to an array / `Uint8Array` / `string`. |
+| `encodeText` / `decodeText` | UTF-8 `string` ↔ `Uint8Array` streams. |
+| `splitLines` / `joinLines` | Cross-chunk-safe line splitting and rejoining. |
+| `encodeJsonl` / `decodeJsonl` | JSON values ↔ `\n`-delimited string stream. |
+| `map` | Stream-map over an `AsyncIterable<T>`. |
+| `toChunks` / `normalizeToUint8Array` | Coerce assorted byte-ish inputs into `Uint8Array` chunks. |
+
+### Iterator plumbing
+
+| Export | Purpose |
+| --- | --- |
+| `newAsyncGenerator` | Backpressure-aware queue turning `next`/`done` callbacks into an async generator. |
+| `sendIterator` / `recieveIterator` | Ship an async iterator across any transport. |
+| `IteratorChunk` | type — the `{ done, value, error }` chunk envelope they exchange. |
+| `ChunkSender` / `ChunkReceiver` / `ReceiverInstaller` | types — the transport-side callbacks those two are wired to. |
+| `toReadableStream` / `fromReadableStream` | `AsyncIterator<Uint8Array>` ↔ WHATWG `ReadableStream<Uint8Array>`. |
+
+### Errors
+
+| Export | Purpose |
+| --- | --- |
+| `serializeError` / `deserializeError` | Preserve `message`, `stack` and custom fields across JSON / structured-clone boundaries. |
+| `SerializedError` | type — the wire shape those two produce and consume. |
+
 ## Internals
 
 ### `newAsyncGenerator` — backpressure queue
@@ -332,17 +411,17 @@ strict one-way converters: no queuing strategy tricks, no transform.
 
 **Zero runtime dependencies.**
 
-Dev: TypeScript, vitest, tsdown, rimraf, `@types/node`
+Dev: TypeScript, vitest, rolldown, rimraf, `@types/node`
 (catalog versions from the monorepo root).
 
 ## Scripts
 
 ```sh
 pnpm test        # vitest run
-pnpm run build   # tsdown (publishes src + compiled dist)
+pnpm run build   # rolldown + tsc --emitDeclarationOnly (ships src + dist)
 pnpm lint        # biome check
 ```
 
 ## License
 
-MIT © statewalker
+MIT © statewalker — see [LICENSE](../../LICENSE).

package/dist/collect.d.ts

@@ -0,0 +1,7 @@
+/** Collect all items from an async iterable into an array. */
+export declare function collect<T>(input: AsyncIterable<T>): Promise<T[]>;
+/** Concatenate all Uint8Array chunks into a single Uint8Array. */
+export declare function collectBytes(input: AsyncIterable<Uint8Array>): Promise<Uint8Array>;
+/** Concatenate all string chunks into a single string. */
+export declare function collectString(input: AsyncIterable<string>): Promise<string>;
+//# sourceMappingURL=collect.d.ts.map

package/dist/collect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../src/collect.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,wBAAsB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAItE;AAED,kEAAkE;AAClE,wBAAsB,YAAY,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAexF;AAED,0DAA0D;AAC1D,wBAAsB,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAIjF"}

package/dist/duplex.d.ts

@@ -0,0 +1,52 @@
+/**
+ * The transport seam every `webrun-streams-*` adapter and the conformance suite
+ * are written against.
+ *
+ * These types live here rather than beside an implementation on purpose: the
+ * emulated multiplexer that once declared them is scheduled for deletion, and a
+ * seam that disappears with its first implementation is not a seam.
+ */
+/**
+ * Canonical seam for the webrun-streams transport family. A `Duplex` carries
+ * one logical call: caller emits an iterable of bytes as input, peer yields an
+ * async generator of bytes as output. Same shape on both sides — an in-process
+ * test can wire `const caller = handler` and run without any transport.
+ *
+ * Iterator semantics carry every signal:
+ *  - Consumer `.return()` on the output → producer's `finally` runs.
+ *  - Producer `throw` → consumer's `for await` throws.
+ *  - Normal exhaustion on either side → matching end on the other side.
+ */
+export type Duplex = (input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>) => AsyncGenerator<Uint8Array>;
+/**
+ * Adapter-side factory that stands up a transport connection and yields a
+ * caller `Duplex`. One `Connect` invocation owns one transport; each call
+ * to the resolved `call` opens a new sub-stream on it.
+ *
+ * A caller must either drain the returned generator or `.return()` it.
+ * Dropping the reference without doing either emits no observable signal:
+ * the abandoned consumer never acknowledges inbound data, so the peer's
+ * outbound pump blocks awaiting that acknowledgement, no end-of-stream is
+ * ever exchanged, and both peers hold the stream's slot open. There is no
+ * way for the transport to detect this — an unreferenced generator is not
+ * observable — so it is the consumer's obligation.
+ */
+export type Connect<P> = (params: P) => Promise<{
+    call: Duplex;
+    close: () => Promise<void>;
+}>;
+/**
+ * Adapter-side factory that registers a handler `Duplex` against a transport.
+ * Returns an idempotent teardown.
+ */
+export type Serve<P> = (params: P, handler: Duplex) => Promise<() => Promise<void>>;
+/**
+ * Thrown by `emulateMux` and adapters when the underlying transport closes
+ * while one or more `Duplex` calls are in flight. Consumers can catch by
+ * `instanceof TransportClosedError` or by checking `error.name`.
+ */
+export declare class TransportClosedError extends Error {
+    readonly name = "TransportClosedError";
+    constructor(message?: string);
+}
+//# sourceMappingURL=duplex.d.ts.map

package/dist/duplex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duplex.d.ts","sourceRoot":"","sources":["../src/duplex.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,MAAM,MAAM,GAAG,CACnB,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,KACpD,cAAc,CAAC,UAAU,CAAC,CAAC;AAEhC;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,CAAC;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAEpF;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,SAAkB,IAAI,0BAA0B;IAChD,YAAY,OAAO,SAAqB,EAEvC;CACF"}

package/dist/emulate-mux.d.ts

@@ -0,0 +1,39 @@
+import { type Duplex } from "./duplex.js";
+/**
+ * The minimum a transport must expose to be wrapped by `emulateMux`. Inbound
+ * bytes are surfaced as an async iterable; outbound is imperative `send`. All
+ * five message-oriented transports (WebSocket, LiveKit data channel, PeerJS
+ * DataConnection, MessagePort, in-process pipe) match this shape after a thin
+ * wrapper.
+ */
+export type ByteChannel = {
+    send(bytes: Uint8Array): void;
+    recv: AsyncIterable<Uint8Array>;
+    closed: Promise<void>;
+    close(): void;
+};
+export interface EmulateMuxOptions {
+    maxStreams?: number;
+    mtu?: number;
+    /**
+     * Cap on inbound bytes one stream may hold for a consumer that has not
+     * drained them. Exceeding it tears down that stream alone.
+     *
+     * It is also the credit this side advertises to the peer, so it must be at
+     * least 1 — a window of 0 authorises nothing and hangs the peer forever —
+     * and it travels in a uint32, so a value above `MAX_UINT32` (4 GiB - 1) is
+     * advertised as `MAX_UINT32`.
+     */
+    maxStreamBuffer?: number;
+    /**
+     * Stream-id allocation side. Initiator uses even ids (2, 4, …); responder
+     * uses odd ids (1, 3, …). Pick one per peer so allocations don't collide.
+     */
+    side?: "initiator" | "responder";
+}
+export declare function emulateMux(channel: ByteChannel, opts?: EmulateMuxOptions): {
+    call: Duplex;
+    serve: (handler: Duplex) => () => Promise<void>;
+    close: () => Promise<void>;
+};
+//# sourceMappingURL=emulate-mux.d.ts.map

package/dist/emulate-mux.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"emulate-mux.d.ts","sourceRoot":"","sources":["../src/emulate-mux.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAwB,MAAM,aAAa,CAAC;AAUhE;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAChC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AA4CF,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,IAAI,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC;CAClC;AAED,wBAAgB,UAAU,CACxB,OAAO,EAAE,WAAW,EACpB,IAAI,GAAE,iBAAsB,GAC3B;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CAuVA"}

package/dist/errors.d.ts

@@ -0,0 +1,8 @@
+export interface SerializedError {
+    message: string;
+    stack?: string;
+    [key: string]: unknown;
+}
+export declare function serializeError(error: unknown): SerializedError;
+export declare function deserializeError(error: SerializedError | string): Error;
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CAY9D;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,GAAG,KAAK,CAGvE"}

package/dist/flow-control.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Credit-based flow control, as a pair of pure state machines with no I/O.
+ *
+ * The sender holds a {@link CreditLedger}: it reserves credit before it puts
+ * anything on the wire, and stalls at zero. The receiver holds a
+ * {@link CreditGrantor}: it counts what its consumer has actually drained and
+ * says when to hand the sender more.
+ *
+ * A sender therefore cannot overrun a receiver's buffer, because it was never
+ * granted permission to. That is the property a sender-side window cannot
+ * offer: the receiver's capacity is not knowable to the sender unless the
+ * receiver states it.
+ *
+ * **The unit is opaque.** This module never interprets the numbers it counts.
+ * `emulateMux` passes byte counts and advertises `maxStreamBuffer`; the RPC
+ * tier passes 1 per value and advertises a maximum in-flight value count.
+ * Nothing here depends on which.
+ */
+export interface CreditLedger {
+    /** Units authorised by the peer and not yet reserved. */
+    readonly available: number;
+    /**
+     * Reserve up to `upTo` units, resolving with how many were actually
+     * granted — at least 1, never more than `upTo`. The caller sends exactly
+     * that much and calls `reserve` again for the rest.
+     *
+     * `upTo` must itself be at least 1; anything less rejects with a
+     * `RangeError` rather than resolving with 0, which would be a silent no-op
+     * that also consumed a waiter slot.
+     *
+     * Returning a partial amount rather than waiting for the full request is
+     * what makes the ledger deadlock-free: a peer that advertises less than
+     * one `upTo` still makes progress, one short piece at a time.
+     *
+     * Rejects if {@link fail} is called.
+     */
+    reserve(upTo: number): Promise<number>;
+    /** The peer authorised `units` more. */
+    grant(units: number): void;
+    /** Reject every pending and future reservation — transport or stream is gone. */
+    fail(err: Error): void;
+}
+export declare function newCreditLedger(initial?: number): CreditLedger;
+export interface CreditGrantor {
+    /**
+     * Record that the consumer drained `units`, and whether the receive queue
+     * is now empty. Returns the credit to hand back to the peer, or `0` to stay
+     * silent and keep accumulating.
+     */
+    consumed(units: number, queueEmpty: boolean): number;
+}
+/**
+ * Grants are batched: replenishing on every chunk while the receiver is
+ * behind would reinvent the per-frame ACK this change exists to remove.
+ * `threshold` is the fraction of the window that must drain before a grant is
+ * emitted.
+ *
+ * The batch is flushed unconditionally once the receive queue is empty, even
+ * below the threshold, so the receiver never sits on credit it owes. Note what
+ * this is and is not: paired with a {@link CreditLedger}, a sender blocks only
+ * at *exactly* zero credit, and at that point the receiver holds the entire
+ * window as `pending` — above any threshold at or below the whole window — so
+ * the threshold alone cannot deadlock that pairing. The flush is what keeps
+ * that from being an argument about a global accounting identity: it is
+ * locally decidable from one boolean, and it returns owed credit now rather
+ * than at the next threshold crossing. It costs an extra frame only when the
+ * consumer is keeping pace, which is exactly when the sender is not blocked
+ * and the frame is cheap.
+ */
+export declare function newCreditGrantor(window: number, threshold?: number): CreditGrantor;
+//# sourceMappingURL=flow-control.d.ts.map

package/dist/flow-control.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-control.d.ts","sourceRoot":"","sources":["../src/flow-control.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,YAAY;IAC3B,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC,wCAAwC;IACxC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iFAAiF;IACjF,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG,IAAI,CAAC;CACxB;AAQD,wBAAgB,eAAe,CAAC,OAAO,SAAI,GAAG,YAAY,CAqDzD;AAED,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,aAAa,CAa/E"}

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+export * from "./collect.js";
+export * from "./duplex.js";
+export * from "./emulate-mux.js";
+export * from "./errors.js";
+export * from "./flow-control.js";
+export * from "./jsonl.js";
+export * from "./lines.js";
+export * from "./map.js";
+export * from "./new-async-generator.js";
+export * from "./normalize.js";
+export * from "./readable-streams.js";
+export * from "./recieve-iterator.js";
+export * from "./send-iterator.js";
+export * from "./text.js";
+export * from "./to-chunks.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC"}