@m4trix/core 0.5.0

package/dist/stream/index.js

@@ -0,0 +1,712 @@
+// src/stream/Pump.ts
+var Pump = class _Pump {
+  constructor(src) {
+    this.src = src;
+  }
+  /**
+   * Wrap an existing AsyncIterable or Readable stream into a Pump
+   *
+   * @template U The type of data in the source stream
+   * @param source The source stream to convert to a Pump (AsyncIterable, ReadableStream, or NodeJS.ReadableStream)
+   * @returns A new Pump instance that wraps the source
+   */
+  static from(source) {
+    async function* gen() {
+      let seq = 0;
+      function isAsyncIterable(obj) {
+        return Symbol.asyncIterator in obj;
+      }
+      function isWebReadableStream(obj) {
+        return "getReader" in obj && typeof obj.getReader === "function";
+      }
+      function isNodeReadableStream(obj) {
+        return "pipe" in obj && "on" in obj && typeof obj.pipe === "function" && typeof obj.on === "function";
+      }
+      if (isAsyncIterable(source)) {
+        const iterator = source[Symbol.asyncIterator]();
+        try {
+          while (true) {
+            const result = await iterator.next();
+            if (result.done)
+              break;
+            yield {
+              sequence: seq++,
+              data: result.value,
+              done: false
+            };
+          }
+        } finally {
+        }
+      } else if (isWebReadableStream(source)) {
+        const reader = source.getReader();
+        try {
+          while (true) {
+            const result = await reader.read();
+            if (result.done)
+              break;
+            yield {
+              sequence: seq++,
+              data: result.value,
+              done: false
+            };
+          }
+        } finally {
+          reader.releaseLock();
+        }
+      } else if (isNodeReadableStream(source)) {
+        try {
+          for await (const chunk of source) {
+            yield {
+              sequence: seq++,
+              data: chunk,
+              done: false
+            };
+          }
+        } catch (error) {
+          console.error("Error reading from Node.js stream:", error);
+          throw error;
+        }
+      }
+      yield { sequence: seq, data: void 0, done: true };
+    }
+    return new _Pump(gen());
+  }
+  /**
+   * Sync or async map over the data portion of each chunk
+   *
+   * @template U The output type after transformation
+   * @param fn The mapping function that transforms each chunk
+   * @returns A new Pump instance with the transformed data
+   */
+  map(fn) {
+    async function* gen() {
+      for await (const { sequence, data, done } of this.src) {
+        if (done) {
+          const out2 = data !== void 0 ? await fn(data) : void 0;
+          yield { sequence, data: out2, done };
+          break;
+        }
+        const out = await fn(data);
+        yield { sequence, data: out, done };
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Stateful map allows processing stream chunks with a persistent context object.
+   *
+   * The context is initialized when the first chunk arrives and can be updated with each chunk.
+   * This is useful for maintaining state across the stream processing.
+   *
+   * If you plan to use sockets you should rather opt for asyncStatefulMap.
+   *
+   * The pipe closes only after all processing is complete, including any final operations in onClose.
+   *
+   * TODO: Un-tested
+   *
+   * @param handlers Object containing callback functions for stream processing
+   * @param handlers.onFirstChunk Function called when the first chunk arrives, initializes the context
+   * @param handlers.onChunk Function called for each subsequent chunk, updates the context
+   * @param handlers.onClose Optional function called when the stream closes, allows final processing
+   * @returns A new Pump instance with transformed data
+   */
+  statefulMap(handlers) {
+    const { src } = this;
+    const gen = async function* () {
+      let context;
+      let initialized = false;
+      let lastChunk;
+      let seq = 0;
+      const queue = [];
+      const yieldData = (data) => {
+        queue.push(data);
+      };
+      for await (const { data, done } of src) {
+        if (done) {
+          if (context && handlers.onClose) {
+            await handlers.onClose(lastChunk, context, yieldData);
+          }
+          while (queue.length > 0) {
+            yield { sequence: seq++, data: queue.shift(), done: false };
+          }
+          yield {
+            sequence: seq++,
+            data: void 0,
+            done: true
+          };
+          break;
+        }
+        if (!initialized) {
+          context = await handlers.onFirstChunk(data, yieldData);
+          initialized = true;
+        } else if (context) {
+          context = await handlers.onChunk(data, context, yieldData);
+        }
+        lastChunk = data;
+        while (queue.length > 0) {
+          yield { sequence: seq++, data: queue.shift(), done: false };
+        }
+      }
+    };
+    return new _Pump(gen());
+  }
+  /**
+   * Async map means that each incoming chunk is causing an async operation that when it completes
+   * should yield a new chunk.
+   * The pipe closes only after you unlock the pipe by using the unlockCloseEvent callback.
+   *
+   * Stateful refers to the fact that you can create your own small context object that is passed in the subsequent callbacks.
+   * This allows you to keep track of things like a socket connection.
+   *
+   * Why is this nice? Well if you use things like a socket the pipe might have received the close event,
+   * before you got any or all of your socket responses. Sockets don't fit into the standard promise pattern,
+   * which makes it harder to wait for them.
+   *
+   * TODO: Un-tested
+   *
+   * @param handlers Object containing callback functions for stream processing
+   * @param handlers.onFirstChunk Function called when the first chunk arrives, initializes the context
+   * @param handlers.onChunk Function called for each subsequent chunk, updates the context
+   * @param handlers.onClose Optional function called when the stream closes, allows final processing
+   * @returns A new Pump instance with transformed data
+   */
+  asyncStatefulMap(handlers) {
+    const { src } = this;
+    const gen = async function* () {
+      let context;
+      let initialized = false;
+      let lastChunk;
+      let seq = 0;
+      let lockedCloseEvent = true;
+      const queue = [];
+      const yieldData = (data) => {
+        queue.push(data);
+      };
+      const unlockCloseEvent = () => {
+        lockedCloseEvent = false;
+      };
+      for await (const { data, done } of src) {
+        if (done) {
+          if (context && handlers.onClose) {
+            await handlers.onClose(
+              lastChunk,
+              context,
+              yieldData,
+              unlockCloseEvent
+            );
+          }
+          const timestamp = Date.now();
+          while (lockedCloseEvent && Date.now() - timestamp < 1e4) {
+            while (queue.length > 0) {
+              yield { sequence: seq++, data: queue.shift(), done: false };
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5));
+          }
+          while (queue.length > 0) {
+            yield { sequence: seq++, data: queue.shift(), done: false };
+          }
+          yield {
+            sequence: seq++,
+            data: void 0,
+            done: true
+          };
+          break;
+        }
+        if (!initialized) {
+          context = await handlers.onFirstChunk(
+            data,
+            yieldData,
+            unlockCloseEvent
+          );
+          initialized = true;
+        } else if (context) {
+          context = await handlers.onChunk(
+            data,
+            context,
+            yieldData,
+            unlockCloseEvent
+          );
+        }
+        lastChunk = data;
+        while (queue.length > 0) {
+          yield { sequence: seq++, data: queue.shift(), done: false };
+        }
+      }
+    };
+    return new _Pump(gen());
+  }
+  /**
+   * Filter items based on a predicate
+   *
+   * @param predicate A function that determines whether to keep each chunk
+   * @returns A new Pump instance containing only chunks that passed the predicate
+   */
+  filter(predicate) {
+    async function* gen() {
+      for await (const { sequence, data, done } of this.src) {
+        if (done) {
+          yield { sequence, data, done: true };
+          break;
+        }
+        const keep = await predicate(data);
+        if (keep) {
+          yield { sequence, data, done: false };
+        }
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Bundles (accumulates) chunks together based on a condition rather than a fixed size.
+   *
+   * This is useful when you need to group chunks dynamically based on their content or other criteria.
+   *
+   * Example: Bundling text chunks with a maximum character limit
+   *
+   * Input chunks: ["Hello", " this", " is", " a few", " chunks", " of text"]
+   * With max size of 10 characters:
+   * - First bundle: ["Hello", " this"] (10 chars)
+   * - Second bundle: [" is", " a few"] (8 chars)
+   * - Third bundle: [" chunks", " of text"] (13 chars)
+   *
+   * @param closeBundleCondition - Function that determines when to close the current bundle
+   *                              Returns true when the current bundle should be emitted
+   *                              Parameters:
+   *                              - chunk: The current chunk being processed
+   *                              - accumulatedChunks: Array of chunks in the current bundle
+   *
+   * @returns A pump that emits arrays of bundled items
+   */
+  bundle(closeBundleCondition) {
+    async function* gen() {
+      let buffer = [];
+      let lastSequence = 0;
+      for await (const { sequence, data, done } of this.src) {
+        lastSequence = sequence;
+        if (done) {
+          if (buffer.length > 0) {
+            yield { sequence, data: [...buffer], done: false };
+          }
+          yield {
+            sequence: lastSequence,
+            data: void 0,
+            done: true
+          };
+          break;
+        }
+        const shouldClose = await closeBundleCondition(data, buffer);
+        buffer.push(data);
+        if (shouldClose) {
+          yield {
+            sequence: lastSequence,
+            data: [...buffer],
+            done: false
+          };
+          buffer = [];
+        }
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Tap into each chunk without altering it
+   *
+   * @param fn A function that receives each chunk but doesn't affect the stream
+   * @returns The same pump instance with unmodified data
+   */
+  onChunk(fn) {
+    async function* gen() {
+      for await (const chunk of this.src) {
+        if (chunk.data === void 0 && chunk.done) {
+          yield chunk;
+        }
+        await fn(chunk.data);
+        yield chunk;
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Collect all chunks in the stream and run a callback when the stream is done.
+   * The callback receives an array of all chunks that passed through.
+   *
+   * This is useful for analytics, logging, or processing the complete stream history
+   * after all chunks have been received.
+   *
+   * @param fn - Callback function that receives the array of all chunks when the stream is complete
+   * @returns The same pump, for chaining
+   */
+  onClose(fn) {
+    async function* gen() {
+      const history = [];
+      for await (const chunk of this.src) {
+        if (chunk.data !== void 0) {
+          history.push(chunk.data);
+        }
+        if (chunk.done) {
+          await fn(history);
+        }
+        yield chunk;
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Batch `n` chunks into arrays before emitting
+   *
+   * @param n The number of chunks to batch together
+   * @returns A new Pump instance that emits arrays of batched chunks
+   */
+  batch(n) {
+    async function* gen() {
+      let buffer = [];
+      for await (const chunk of this.src) {
+        if (chunk.done) {
+          if (chunk.data === void 0) {
+            yield {
+              sequence: buffer[0].sequence,
+              data: buffer.map((c) => c.data),
+              done: false
+            };
+            yield {
+              sequence: chunk.sequence,
+              data: void 0,
+              done: true
+            };
+            buffer = [];
+          } else {
+            buffer.push(chunk);
+            yield {
+              sequence: buffer[0].sequence,
+              data: buffer.map((c) => c.data),
+              done: true
+            };
+          }
+          break;
+        }
+        buffer.push(chunk);
+        if (buffer.length === n) {
+          yield {
+            sequence: buffer[0].sequence,
+            data: buffer.map((c) => c.data),
+            done: chunk.done
+          };
+          buffer = [];
+        }
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * If you want to prevent chunk starvation, you can buffer the chunks.
+   * Chunks will not be bundled into arrays or object but kept as is,
+   * but the pipeline will not progress at that segment until the buffer is filled up.
+   * Once a buffer is filled up it will drain and never buffer again.
+   *
+   * @param n The number of chunks to buffer before processing continues
+   * @returns A new Pump instance with buffering behavior
+   */
+  buffer(n) {
+    async function* gen() {
+      let buffer = [];
+      let bufferFilled = false;
+      for await (const chunk of this.src) {
+        if (!bufferFilled) {
+          if (!chunk.done) {
+            buffer.push(chunk);
+          }
+          if (buffer.length >= n || chunk.done) {
+            bufferFilled = true;
+            for (const bufferedChunk of buffer) {
+              yield bufferedChunk;
+            }
+            if (chunk.done) {
+              yield {
+                sequence: chunk.sequence,
+                data: void 0,
+                done: true
+              };
+              break;
+            }
+            buffer = [];
+          }
+        } else {
+          yield chunk;
+        }
+      }
+      for (const bufferedChunk of buffer) {
+        yield bufferedChunk;
+      }
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Rechunk the stream: transform one chunk into zero, one, or many output chunks.
+   * The handler function receives the current buffer of chunks, a push function to emit new chunks,
+   * and a flag indicating if this is the last chunk in the stream.
+   *
+   * @param handler Function that transforms chunks and pushes new ones
+   * @returns A new Pump instance with rechunked data
+   */
+  rechunk(handler) {
+    async function* gen() {
+      let buffer = [];
+      let seq = 0;
+      const pending = [];
+      const push = (chunk) => {
+        pending.push(chunk);
+      };
+      for await (const { data, done } of this.src) {
+        if (!done) {
+          if (data !== void 0) {
+            buffer.push(data);
+          }
+          await handler({
+            buffer,
+            push,
+            lastChunk: false,
+            setBuffer: (b) => {
+              buffer = b;
+            }
+          });
+        } else {
+          await handler({
+            buffer,
+            push,
+            lastChunk: true,
+            setBuffer: (b) => {
+              buffer = b;
+            }
+          });
+        }
+        while (pending.length > 0) {
+          const out = pending.shift();
+          yield { sequence: seq++, data: out, done: false };
+        }
+        if (done) {
+          break;
+        }
+      }
+      yield { sequence: seq, data: void 0, done: true };
+    }
+    return new _Pump(gen.call(this));
+  }
+  slidingWindow(size, step, fn) {
+    async function* gen() {
+      const history = [];
+      let offset = 0;
+      let lastSeq = 0;
+      function buildWindow(_offset, _size, _history) {
+        const window = Array(_size).fill(void 0);
+        let windowIndex = 0;
+        for (let i = _offset; i > _offset - _size; i -= step) {
+          if (i >= history.length) {
+            windowIndex++;
+            continue;
+          }
+          if (i < 0) {
+            break;
+          }
+          window[windowIndex] = _history[i];
+          windowIndex++;
+        }
+        return window;
+      }
+      for await (const { sequence, data, done } of this.src) {
+        if (done) {
+          for (let i = 0; i < size - 1; i++) {
+            const window2 = buildWindow(offset + i, size, history);
+            yield { sequence: lastSeq, data: window2, done: false };
+          }
+          if (data === void 0) {
+            yield {
+              sequence: lastSeq,
+              data: void 0,
+              done: true
+            };
+          } else {
+            yield {
+              sequence: lastSeq,
+              data: [
+                history[history.length - 2] ?? void 0,
+                history[history.length - 3] ?? void 0,
+                history[history.length - 1]
+              ],
+              done: true
+            };
+          }
+          break;
+        }
+        lastSeq = sequence;
+        history.push(data);
+        const window = buildWindow(offset, size, history);
+        yield { sequence, data: window, done: false };
+        offset++;
+      }
+    }
+    const base = new _Pump(gen.call(this));
+    return fn ? base.map(fn) : base;
+  }
+  /**
+   * Sequentially flatten inner stream sources emitted by the pipeline.
+   * Works with any Source type (AsyncIterable or ReadableStream).
+   * This method is only available when the current Pump contains Source elements.
+   *
+   * @template U The type of data in the inner streams
+   * @template F The type of inner stream source (extends Source<U>)
+   * @returns A Pump instance with flattened stream data
+   */
+  sequenceStreams() {
+    async function* gen() {
+      let seq = 0;
+      for await (const { data: innerSource, done: outerDone } of this.src) {
+        if (outerDone)
+          break;
+        const innerPump = _Pump.from(innerSource);
+        for await (const { data, done } of innerPump.src) {
+          if (done)
+            break;
+          yield { sequence: seq++, data, done: false };
+        }
+      }
+      yield { sequence: seq, data: void 0, done: true };
+    }
+    return new _Pump(gen.call(this));
+  }
+  /**
+   * Fork the stream: two independent Pump<T> consumers
+   * Both resulting Pumps will receive the same data, allowing for divergent processing paths.
+   *
+   * @returns An array containing two independent Pump instances with the same source data
+   */
+  fork() {
+    const buffers = [[], []];
+    let done = false;
+    const srcIter = this.src[Symbol.asyncIterator]();
+    async function fill() {
+      const { value, done: streamDone } = await srcIter.next();
+      if (streamDone) {
+        done = true;
+        return;
+      }
+      buffers.forEach((q) => q.push(value));
+      if (value.done)
+        done = true;
+    }
+    function makeStream(buf) {
+      return {
+        [Symbol.asyncIterator]() {
+          return {
+            async next() {
+              while (buf.length === 0 && !done) {
+                await fill();
+              }
+              if (buf.length === 0)
+                return {
+                  done: true,
+                  value: void 0
+                };
+              return { done: false, value: buf.shift() };
+            }
+          };
+        }
+      };
+    }
+    return [new _Pump(makeStream(buffers[0])), new _Pump(makeStream(buffers[1]))];
+  }
+  /**
+   * Drain the pipeline, consuming all chunks.
+   * Returns a Promise that resolves when all chunks have been consumed.
+   *
+   * @returns A Promise that resolves when all chunks have been consumed
+   */
+  drain() {
+    return (async () => {
+      for await (const { done } of this.src) {
+        if (done)
+          break;
+      }
+    })();
+  }
+  /**
+   * Drain the pipeline to a StreamTransformer.
+   * Applies transform() to each data chunk, then closes the transformer,
+   * and returns its response (which can be of any type defined by the transformer).
+   *
+   * Example with httpStreamResponse:
+   * ```
+   * const { transform, response, close } = httpStreamResponse(options);
+   * return Pump.from(messageStream).drainTo({ transform, close, response });
+   * ```
+   *
+   * @template U The type of data expected by the transformer (extends T)
+   * @template R The response type produced by the transformer
+   * @param transformer The StreamTransformer to drain to
+   * @returns The response from the transformer
+   */
+  drainTo(transformer) {
+    (async () => {
+      for await (const { data, done } of this.src) {
+        if (done)
+          break;
+        transformer.transform(data);
+      }
+      transformer.close();
+    })();
+    return transformer.response;
+  }
+};
+
+// src/stream/utility/pipe-transformers/response.ts
+function httpStreamResponse(options = {}) {
+  const { init, encoder } = options;
+  const encodeFn = encoder ?? ((d) => {
+    if (d instanceof Uint8Array)
+      return d;
+    if (typeof d === "string")
+      return d;
+    return JSON.stringify(d);
+  });
+  const { readable, writable } = new TransformStream();
+  const writer = writable.getWriter();
+  const response = new Response(readable, init);
+  const transform = (chunk) => {
+    const encoded = encodeFn(chunk);
+    const bytes = typeof encoded === "string" ? new TextEncoder().encode(encoded) : encoded;
+    writer.write(bytes);
+    return chunk;
+  };
+  const close = () => {
+    writer.close();
+  };
+  return { transform, response, close };
+}
+
+// src/stream/utility/rechunker/ensure-full-words.ts
+async function ensureFullWords({
+  buffer,
+  push,
+  lastChunk
+}) {
+  const combined = buffer.join("");
+  const lastBoundary = Math.max(
+    combined.lastIndexOf(" "),
+    combined.lastIndexOf("\n"),
+    combined.lastIndexOf("	")
+  );
+  if (lastBoundary !== -1 || lastChunk) {
+    const emitPart = lastBoundary !== -1 ? combined.slice(0, lastBoundary + 1) : combined;
+    const leftoverPart = lastBoundary !== -1 ? combined.slice(lastBoundary + 1) : "";
+    if (emitPart.trim().length > 0) {
+      push(emitPart);
+    }
+    buffer.length = 0;
+    if (leftoverPart.length > 0) {
+      buffer.push(leftoverPart);
+    }
+  }
+}
+
+export { Pump, ensureFullWords, httpStreamResponse };
+//# sourceMappingURL=out.js.map
+//# sourceMappingURL=index.js.map