@statewalker/webrun-streams 0.1.0

package/src/new-async-generator.ts

@@ -0,0 +1,215 @@
+/**
+ * The newAsyncGenerator function creates async generators from callback-based initialization
+ * functions, providing a bridge between imperative event handling and declarative async
+ * iteration patterns.
+ *
+ * The initialization function receives two callback functions that control the generator's
+ * behavior. The `next` function yields values to consumers, while the `done` function
+ * signals completion or error conditions. The initialization function can return a cleanup
+ * function that will be called when the generator is closed or an error occurs.
+ *
+ * The generator manages an internal queue of values and completion signals, ensuring that
+ * producers can yield values without overwhelming consumers. Proper backpressure is implemented
+ * by having the `next` and `done` functions return promises that resolve only after the
+ * consumer has processed the values. This prevents memory leaks and ensures that producers
+ * are aware of whether their values were successfully handled.
+ * The generator also handles cleanup by draining any remaining items in the queue and notifying
+ * producers that their values were not processed if the generator is closed early. This ensures
+ * that resources are properly managed and that no memory leaks occur.
+ *
+ * Example usage:
+ * ```typescript
+ * const asyncGen = newAsyncGenerator<number>((next, done) => {
+ *   let count = 0;
+ *   const interval = setInterval(() => {
+ *     if (count < 5) {
+ *       next(count++);
+ *     } else {
+ *       done();
+ *       clearInterval(interval);
+ *     }
+ *   }, 1000);
+ *   return () => clearInterval(interval); // Cleanup function
+ * });
+ * (async () => {
+ *   for await (const num of asyncGen) {
+ *     console.log(num); // Logs numbers 0 to 4 at 1 second intervals
+ *   }
+ *   console.log("Completed");
+ * })();
+ * ```
+ * @template T The type of values yielded by the generator
+ * @template E The type of errors that can be thrown; defaults to Error
+ * @param init Initialization function that sets up the generator behavior with next/done callbacks;
+ *  The first parameter - `next` function - is used to yield values to consumers;
+ *  It returns a Promise<boolean> indicating whether the value was successfully handled;
+ *  The second parameter - `done` function - is used to signal completion or error;
+ *  It returns a Promise<boolean> indicating whether the completion was successfully handled;
+ *  The initialization function can optionally return a cleanup function that will be called
+ *  when the generator is closed or an error occurs;
+ * @param skipValues If true, only the most recent value is kept in the queue,
+ * skipping intermediate values (not consumed values are considered skipped);
+ * This is useful for scenarios where only the latest value matters, such as UI updates.
+ * Defaults to false, meaning all values are queued and processed in order.
+ * @returns AsyncGenerator that properly manages backpressure and resource cleanup
+ */
+export async function* newAsyncGenerator<T, E = Error>(
+  /**
+   * Initialization function that sets up the async generator behavior.
+   * @param next - Function to yield values to consumers. Returns a Promise<boolean>
+   *               indicating whether the value was successfully handled.
+   * @param done - Function to signal completion or error. Optional error parameter
+   *               will cause the generator to throw that error.
+   * @returns Optional cleanup function that will be called when the generator terminates.
+   */
+  init: (
+    next: (value: T) => Promise<boolean>,
+    done: (err?: E) => Promise<boolean>,
+  ) => void | (() => void | Promise<void>),
+
+  /**
+   * Skipping queue implementation that maintains only the most recent value.
+   */
+  skipValues = false,
+): AsyncGenerator<T> {
+  /**
+   * Internal queue slot type that wraps values and completion signals with resolution callbacks.
+   * This enables the async generator to communicate back to producers whether their values
+   * were successfully processed, enabling backpressure management.
+   */
+  type IterationSlot<T, E> =
+    | { done: false; value: T } // Regular value slot
+    | { done: true; error?: E }; // Completion/error slot
+  type QueueSlot<T, E> = IterationSlot<T, E> & {
+    next: QueueSlot<T, E> | undefined; // Pointer to the next slot in the queue
+    resolve: (handled: boolean) => void; // Callback to signal if the value was handled
+  };
+
+  let head: QueueSlot<T, E> | undefined; // Head of the queue
+  let tail: QueueSlot<T, E> | undefined; // Tail of the queue
+
+  /** Flag to prevent new values from being queued after generator closes */
+  let closed = false;
+
+  /** Wake-up function to notify the generator loop of new items */
+  let wakeUp: undefined | (() => void);
+
+  /**
+   * Drains the internal queue, notifying all pending producers that their values
+   * were not processed due to generator closure. This prevents memory leaks and
+   * ensures proper backpressure signaling.
+   */
+  const drainQueue = () => {
+    for (; head; head = head.next) {
+      closed = closed || head.done;
+      head.resolve(false); // Notify producer that value was not handled
+    }
+    tail = undefined;
+  };
+
+  /**
+   * Enqueues a value or completion signal with a promise that resolves when the item
+   * is processed. This enables backpressure by allowing producers to know when their
+   * values have been consumed.
+   */
+  const enqueue = (params: IterationSlot<T, E>): Promise<boolean> => {
+    if (skipValues) {
+      // Remove any previous value slots
+      drainQueue();
+    }
+    return !closed
+      ? new Promise<boolean>((resolve) => {
+          // Add the new slot to the queue
+          const next = { ...params, next: undefined, resolve };
+          if (tail) {
+            tail.next = next;
+          }
+          tail = next;
+          if (!head) {
+            head = tail;
+          }
+          // Notify the generator loop that a new item is available
+          wakeUp?.();
+        })
+      : // If the generator is closed, immediately resolve as not handled
+        Promise.resolve(false);
+  };
+
+  /**
+   * Producer function to yield a value to consumers. Returns a promise that resolves
+   * to true if the value was successfully processed, false if the generator is closed
+   * or the value was skipped due to backpressure.
+   */
+  const next = (value: T): Promise<boolean> => enqueue({ done: false, value });
+
+  /**
+   * Producer function to signal completion or error. Optional error parameter will
+   * cause the generator to throw that error to consumers.
+   */
+  const done = (error?: E): Promise<boolean> => enqueue({ done: true, error });
+
+  // Initialize the producer by calling the init function with our control functions
+  const unsubscribe = init(next, done);
+
+  try {
+    // Main async generator loop - processes queued items and yields values
+    while (!closed) {
+      // Try to get the next item from the queue
+      if (!head) {
+        // If no items available, wait for producers to add something
+        await new Promise<void>((resolve) => {
+          wakeUp = resolve;
+        }).then(() => {
+          wakeUp = undefined;
+        });
+        continue;
+      }
+
+      // Process the next item in the queue
+      const slot = head;
+      // Move head to the next item
+      head = head.next;
+      // If we removed the tail, clear it as well
+      if (tail === slot) {
+        tail = head;
+      }
+      try {
+        // Handle completion/error signals
+        if (slot.done) {
+          closed = true;
+          if (slot.error !== undefined) {
+            throw slot.error;
+          }
+          break;
+        }
+        // Yield the value to the consumer
+        yield slot.value;
+      } finally {
+        // Signal successful processing
+        slot.resolve(true);
+      }
+    }
+  } finally {
+    /**
+     * Cleanup phase - ensures proper resource management and notification of any
+     * remaining producers. This runs whether the generator completes normally,
+     * encounters an error, or is closed early by the consumer.
+     */
+    closed = true;
+
+    // Wake up any pending operations to allow them to exit
+    wakeUp?.();
+
+    // Call the cleanup function returned by the initialization function
+    if (typeof unsubscribe === "function") {
+      await unsubscribe();
+    }
+
+    /**
+     * Drain any remaining items in the queue and notify their producers that
+     * the values were not processed. This prevents memory leaks and ensures
+     * proper backpressure signaling.
+     */
+    drainQueue();
+  }
+}

package/src/normalize.ts

@@ -0,0 +1,31 @@
+const utf8 = new TextEncoder();
+
+export type ByteLike = Uint8Array | ArrayBuffer | ArrayBufferView | Blob | string;
+
+/**
+ * Coerce common byte-like inputs into a `Uint8Array`. Strings encode as UTF-8.
+ * Blobs return a `Promise<Uint8Array>`; every other input returns synchronously.
+ * Throws `TypeError` for anything else.
+ */
+export function normalizeToUint8Array(data: ByteLike): Uint8Array | Promise<Uint8Array> {
+  if (data instanceof Uint8Array) return data;
+  if (data instanceof ArrayBuffer) return new Uint8Array(data);
+  if (ArrayBuffer.isView(data)) {
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  }
+  if (typeof Blob !== "undefined" && data instanceof Blob) {
+    return data.arrayBuffer().then((buf) => new Uint8Array(buf));
+  }
+  if (typeof data === "string") return utf8.encode(data);
+  throw new TypeError(
+    `normalizeToUint8Array: unsupported input ${describe(data)}; expected Uint8Array, ArrayBuffer, ArrayBufferView, Blob, or string`,
+  );
+}
+
+function describe(value: unknown): string {
+  if (value === null) return "null";
+  const t = typeof value;
+  if (t !== "object") return t;
+  const ctor = (value as { constructor?: { name?: string } }).constructor;
+  return ctor?.name ?? "object";
+}

package/src/readable-streams.ts

@@ -0,0 +1,31 @@
+export function toReadableStream(it: AsyncIterator<Uint8Array>): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      let handled = false;
+      try {
+        while (true) {
+          const slot = await it.next();
+          if (!slot || slot.done) break;
+          const value = (await slot.value) as Uint8Array;
+          controller.enqueue(value);
+        }
+      } catch (error) {
+        handled = true;
+        controller.error(error);
+      } finally {
+        if (!handled) controller.close();
+      }
+    },
+  });
+}
+
+export async function* fromReadableStream(
+  stream: ReadableStream<Uint8Array>,
+): AsyncGenerator<Uint8Array, void, unknown> {
+  const reader = stream.getReader();
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (value !== undefined) yield value;
+  }
+}

package/src/recieve-iterator.ts

@@ -0,0 +1,33 @@
+import { newAsyncGenerator } from "./new-async-generator.js";
+import type { IteratorChunk } from "./send-iterator.js";
+
+export type ChunkReceiver<T> = (chunk?: IteratorChunk<T>) => Promise<boolean>;
+export type ReceiverInstaller<T> = (
+  deliver: ChunkReceiver<T>,
+) => (() => void | Promise<void>) | undefined | void;
+
+/**
+ * Inverse of {@link sendIterator}: turns a sequence of `{done, value, error}`
+ * chunks (delivered to the supplied callback by `installer`) into an async
+ * generator.
+ *
+ * The `installer` is given a `deliver` function. It should call `deliver`
+ * once for each incoming chunk and may return a cleanup callback that
+ * runs when the consumer stops iterating.
+ */
+export function recieveIterator<T>(installer: ReceiverInstaller<T>): AsyncGenerator<T> {
+  return newAsyncGenerator<T>((next, done) => {
+    const cleanup = installer(async (chunk = { done: true }) => {
+      const { done: isDone = true, value, error } = chunk;
+      if (error) return await done(error as Error);
+      if (isDone) return await done();
+      return await next(value as T);
+    });
+    if (cleanup) {
+      return async () => {
+        await cleanup();
+      };
+    }
+    return undefined;
+  });
+}

package/src/send-iterator.ts

@@ -0,0 +1,30 @@
+export interface IteratorChunk<T> {
+  done: boolean;
+  value?: T;
+  error?: unknown;
+}
+
+export type ChunkSender<T> = (chunk: IteratorChunk<T>) => void | Promise<void>;
+
+/**
+ * Drain an async iterator into a sink that consumes one chunk at a time.
+ *
+ * Each yielded value becomes `{ done: false, value }`. Completion emits
+ * `{ done: true }`. If the iterator throws, the error is caught and the
+ * final `done` chunk carries it so the peer can rethrow on its side.
+ */
+export async function sendIterator<T>(
+  send: ChunkSender<T>,
+  it: AsyncIterable<T> | Iterable<T>,
+): Promise<void> {
+  let error: unknown;
+  try {
+    for await (const value of it as AsyncIterable<T>) {
+      await send({ done: false, value });
+    }
+  } catch (err) {
+    error = err;
+  } finally {
+    await send({ done: true, error });
+  }
+}

package/src/text.ts

@@ -0,0 +1,18 @@
+/** Decode Uint8Array chunks to string chunks via TextDecoder, handling split multi-byte characters. */
+export async function* decodeText(input: AsyncIterable<Uint8Array>): AsyncGenerator<string> {
+  const decoder = new TextDecoder("utf-8", { fatal: false });
+  for await (const chunk of input) {
+    const text = decoder.decode(chunk, { stream: true });
+    if (text) yield text;
+  }
+  const tail = decoder.decode();
+  if (tail) yield tail;
+}
+
+/** Encode string chunks to Uint8Array chunks via TextEncoder. */
+export async function* encodeText(input: AsyncIterable<string>): AsyncGenerator<Uint8Array> {
+  const encoder = new TextEncoder();
+  for await (const str of input) {
+    yield encoder.encode(str);
+  }
+}

package/src/to-chunks.ts

@@ -0,0 +1,33 @@
+const DEFAULT_CHUNK_SIZE = 16 * 1024;
+
+/**
+ * Curried `Duplex`-shaped transformer that splits incoming `Uint8Array`s into
+ * chunks no larger than `size` bytes. Empty input chunks are skipped; small
+ * input chunks pass through unchanged (zero-copy `subarray` views are used
+ * when splitting, so no allocation per chunk).
+ *
+ * Use to respect transport MTUs before reaching `channel.send`. Reassembly on
+ * the receive side is not needed — consumers iterate a continuous byte stream.
+ */
+export function toChunks(
+  size: number = DEFAULT_CHUNK_SIZE,
+): (input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>) => AsyncGenerator<Uint8Array> {
+  if (!(Number.isInteger(size) && size > 0)) {
+    throw new RangeError(`toChunks: size must be a positive integer, got ${size}`);
+  }
+  return async function* (input) {
+    for await (const block of input) {
+      if (block.byteLength === 0) continue;
+      if (block.byteLength <= size) {
+        yield block;
+        continue;
+      }
+      let offset = 0;
+      while (offset < block.byteLength) {
+        const end = Math.min(offset + size, block.byteLength);
+        yield block.subarray(offset, end);
+        offset = end;
+      }
+    }
+  };
+}