@effectionx/stream-helpers 0.1.0

package/README.md

@@ -0,0 +1,260 @@
+# Stream Helpers
+
+A collection of type-safe stream helpers built on top of
+[Effection](https://github.com/thefrontside/effection) for efficient and
+controlled stream processing.
+
+## Included Helpers
+
+### Filter
+
+The `filter` helper narrows a stream by only passing through items that match a
+predicate.
+
+```typescript
+import { filter } from "@effectionx/stream-helpers";
+import { each } from "effection";
+
+// Example: Synchronous filtering
+function* syncExample(source: Stream<number, unknown>) {	
+
+  const gt5 = filter<number>(function* (x) { return x > 5 });
+
+  for (const value of yield* each(gt5(stream))) {
+    console.log(value); // Only values > 5
+    yield* each.next();
+  }
+};
+
+// Example: Asynchronous filtering
+function* asyncExample(source: Stream<number, unknown>) {
+
+  const evensOf = filter<number>(function* (x) {
+    yield* sleep(100); // Simulate async operation
+    return x % 2 === 0; // Keep only even numbers
+  });
+
+  for (const value of yield* each(evensOf(stream))) {
+    console.log(value); // Only even numbers
+    yield* each.next();
+  }
+});
+```
+
+### Map
+
+The `map` helper transforms each item in a stream using a provided function.
+This is useful for data transformation operations where you need to process each
+item individually.
+
+```typescript
+import { map } from "@effectionx/stream-helpers";
+import { each } from "effection";
+
+function* example(stream: Stream<number, unknown>) {
+  const double = map<number>(function* (x) {
+    return x * 2;
+  });
+
+  for (const value of yield* each(double(stream))) {
+    console.log(value); // Each value is doubled
+    yield* each.next();
+  }
+}
+```
+
+### Batch
+
+The `batch` helper is useful when you want to convert individual items passing
+through the stream into arrays of items. The batches can be created either by
+specifying a maximum time or a maximum size. If both are specified, the batch
+will be created when either condition is met.
+
+```typescript
+import { batch } from "@effectionx/stream-helpers";
+import { each } from "effection";
+
+// Example: Batch by size
+function* exampleBySize(stream: Stream<number, unknown>) {
+  const byThree = batch({ maxSize: 3});
+
+  for (const items of yield* each(byThree(stream))) {
+    console.log(batch); // [1, 2, 3], [4, 5, 6], ...
+    yield* each.next();
+  }
+};
+
+// Example: Batch by time
+function* exampleByTime(stream: Stream<number, unknown>) {
+  const stream = batch({ maxTime: 1000 })(sourceStream);
+
+  for (const batch of yield* each(stream)) {
+    console.log(batch); // Items received within 1 second
+    yield* each.next();
+  }
+});
+
+// Example: Combined batching
+function* exampleCombined(stream: Stream<number, unknown>) {
+
+  const batched = batch({
+    maxSize: 5,
+    maxTime: 1000,
+  });
+
+  for (const batch of yield* each(batched(stream))) {
+    console.log(batch); // Up to 5 items within 1 second
+    yield* each.next();
+  }
+});
+```
+
+### Valve
+
+Allows to apply backpressure to the source stream to prevent overwhelming the
+downstream consumer. This is useful with any stream that generates items faster
+than the consumer can consume them. It was originally designed for use with
+Kafka where the producer can cause the service to run out of memory when the
+producer produces many faster than the consumer to process the messages. It can
+be used as a buffer for any infinite stream.
+
+```typescript
+import { valve } from "@effectionx/stream-helpers";
+import { each } from "effection";
+
+function* example() {
+  const regulated = valve({
+    // buffer size threshold when close operation will invoked
+    closeAt: 1000,
+    *close() {
+      // pause the source stream
+    },
+
+    // buffer size threshold when open operation will be invoked
+    openAt: 100,
+    *open() {
+      // resume the source stream
+    },
+  })(stream);
+
+  for (const value of yield* each(regulated)) {
+    console.log(value);
+    yield* each.next();
+  }
+}
+```
+
+### Passthrough Tracker
+
+Passthrough Tracker stream helper provides a way to know if all items that
+passed through the stream have been handled. This is especially helpful when you
+want to ensure that all items were processed before completing an operation.
+
+It's different from other stream helpers because you must first call
+`createTracker` function which retuns an object. The actual helper is on the
+`passthrough` method which you can call and chain as you would with other
+helpers.
+
+```typescript
+import { each, signal } from "effection";
+import { createTracker } from "@ffectionx/stream-helpers"
+
+const source = signal(0);
+
+// create the tracker
+const tracker = yield* createTracker();
+
+// create  passthrough stream helper
+const track = tracker.passthrough();
+
+for (const value of yield* each(track(source))) {
+  // mark items 
+  tracker.markOne(value);
+  yield* each.next();
+}
+
+// will resolve when all items that passed through the stream were seen
+yield* tracker;
+```
+
+### Composing stream helpers
+
+You can use a simple `pipe()` to compose a series of stream helpers together. In
+this example, we use one from [remeda](https://remedajs.com/docs/#pipe),
+
+```typescript
+import { batch, filter, map, valve } from "@effectionx/stream-helpers";
+import { each } from "effection";
+// any standard pipe function should work
+import { pipe } from "remeda";
+
+function* example(source: Stream<number, unknown>) {
+  // Compose stream helpers using pipe
+  const stream = pipe(
+    source,
+    valve({ open, close, openAt: 100, closeAt: 100 }),
+    filter(function* (x) {
+      return x > 0;
+    }),
+    map(function* (x) {
+      return x * 20;
+    }),
+    batch({ maxSize: 50 }),
+  );
+
+  for (const value of yield* each(stream)) {
+    console.log(value);
+    yield* each.next();
+  }
+}
+```
+
+## Testing Streams
+
+The library includes testing utilities to help you test your stream processing
+code. These are available in `@effectionx/stream-helpers/test-helpers` export.
+
+### Faucet
+
+The `useFaucet` function creates a stream that can be used to test the behavior
+of streams that use backpressure. It's particularly useful in tests where you
+need a controllable source stream.
+
+```typescript
+import { useFaucet } from "@effectionx/stream-helpers/test-helpers";
+import { each, run, spawn } from "effection";
+
+await run(function* () {
+  const faucet = yield* useFaucet<number>({ open: true });
+
+  // Remember to spawn the stream subscription before sending items to the stream
+  yield* spawn(function* () {
+    for (let i of yield* each(faucet)) {
+      console.log(i);
+      yield* each.next();
+    }
+  });
+
+  // Pass an array of items to send items to the stream one at a time synchronously
+  yield* faucet.pour([1, 2, 3]);
+
+  // Pass an operation to control the rate at which items are sent to the stream
+  yield* faucet.pour(function* (send) {
+    yield* sleep(10);
+    send(5);
+    yield* sleep(30);
+    send(6);
+    yield* sleep(10);
+    send(7);
+  });
+
+  // You can close the faucet to stop items from being sent
+  faucet.close();
+
+  // And open it again when needed
+  faucet.open();
+});
+```
+
+Items sent to the faucet stream while it's closed are not buffered, in other
+words, they'll be dropped.

package/esm/batch.d.ts

@@ -0,0 +1,21 @@
+import { type Stream } from "effection";
+type RequireAtLeastOne<T, Keys extends keyof T = keyof T> = Pick<T, Exclude<keyof T, Keys>> & {
+    [K in Keys]-?: Required<Pick<T, K>> & Partial<Pick<T, Exclude<Keys, K>>>;
+}[Keys];
+export interface BatchOptions {
+    readonly maxTime: number;
+    readonly maxSize: number;
+}
+/**
+ * Creates batches of items from the source stream. The batches can be created either by
+ * specifying a maximum time or a maximum size. If both are specified, the batch will be
+ * created when either condition is met.
+ *
+ * @param options - The options for the batch.
+ * @param options.maxTime - The maximum time to wait for a batch.
+ * @param options.maxSize - The maximum size of a batch.
+ * @returns A stream of arrays of items from the source stream.
+ */
+export declare function batch(options: RequireAtLeastOne<BatchOptions>): <T>(stream: Stream<T, never>) => Stream<T[], never>;
+export {};
+//# sourceMappingURL=batch.d.ts.map

package/esm/batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../src/batch.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,MAAM,EAAE,MAAM,WAAW,CAAC;AAGlE,KAAK,iBAAiB,CAAC,CAAC,EAAE,IAAI,SAAS,MAAM,CAAC,GAAG,MAAM,CAAC,IACpD,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,GAC/B;KACC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;CACzE,CAAC,IAAI,CAAC,CAAC;AAEV,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,KAAK,CACnB,OAAO,EAAE,iBAAiB,CAAC,YAAY,CAAC,GACvC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CA4CrD"}

package/esm/batch.js

@@ -0,0 +1,54 @@
+import { each, race, sleep, spawn } from "effection";
+import { createArraySignal, is } from "./signals.js";
+/**
+ * Creates batches of items from the source stream. The batches can be created either by
+ * specifying a maximum time or a maximum size. If both are specified, the batch will be
+ * created when either condition is met.
+ *
+ * @param options - The options for the batch.
+ * @param options.maxTime - The maximum time to wait for a batch.
+ * @param options.maxSize - The maximum size of a batch.
+ * @returns A stream of arrays of items from the source stream.
+ */
+export function batch(options) {
+    return function (stream) {
+        return {
+            *[Symbol.iterator]() {
+                let batch = yield* createArraySignal([]);
+                yield* spawn(function* () {
+                    for (let item of yield* each(stream)) {
+                        batch.push(item);
+                        if (options.maxSize && batch.length >= options.maxSize) {
+                            // wait until it's drained
+                            yield* is(batch, (batch) => batch.length === 0);
+                        }
+                        yield* each.next();
+                    }
+                });
+                function drain() {
+                    let value = batch.valueOf();
+                    batch.set([]);
+                    return value;
+                }
+                return {
+                    *next() {
+                        yield* is(batch, (batch) => batch.length >= 1);
+                        if (options.maxTime && options.maxSize) {
+                            yield* race([
+                                is(batch, (batch) => batch.length === options.maxSize),
+                                sleep(options.maxTime),
+                            ]);
+                        }
+                        else if (options.maxTime) {
+                            yield* sleep(options.maxTime);
+                        }
+                        else if (options.maxSize) {
+                            yield* is(batch, (batch) => batch.length === options.maxSize);
+                        }
+                        return { done: false, value: drain() };
+                    },
+                };
+            },
+        };
+    };
+}

package/esm/filter.d.ts

@@ -0,0 +1,23 @@
+import type { Operation, Stream } from "effection";
+/**
+ * Filters items from the stream based on a predicate function.
+ *
+ * @param predicate - The function to test each item
+ * @returns A stream transformer that only emits items that pass the predicate
+ *
+ * @example
+ * ```typescript
+ * import { filter } from "@effectionx/stream-helpers";
+ * import { run, each } from "effection";
+ *
+ * await run(function* () {
+ *   const stream = filter((x: number) => x > 5)(sourceStream);
+ *
+ *   for (const value of yield* each(stream)) {
+ *     console.log(value); // Only values > 5
+ *   }
+ * });
+ * ```
+ */
+export declare function filter<T>(predicate: (value: T) => Operation<boolean>): <TDone>(stream: Stream<T, TDone>) => Stream<T, TDone>;
+//# sourceMappingURL=filter.d.ts.map

package/esm/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../src/filter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,MAAM,CAAC,CAAC,EACtB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,SAAS,CAAC,OAAO,CAAC,GAC1C,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAyBvD"}

package/esm/filter.js

@@ -0,0 +1,45 @@
+/**
+ * Filters items from the stream based on a predicate function.
+ *
+ * @param predicate - The function to test each item
+ * @returns A stream transformer that only emits items that pass the predicate
+ *
+ * @example
+ * ```typescript
+ * import { filter } from "@effectionx/stream-helpers";
+ * import { run, each } from "effection";
+ *
+ * await run(function* () {
+ *   const stream = filter((x: number) => x > 5)(sourceStream);
+ *
+ *   for (const value of yield* each(stream)) {
+ *     console.log(value); // Only values > 5
+ *   }
+ * });
+ * ```
+ */
+export function filter(predicate) {
+    return function (stream) {
+        return {
+            *[Symbol.iterator]() {
+                const subscription = yield* stream;
+                return {
+                    *next() {
+                        while (true) {
+                            const next = yield* subscription.next();
+                            if (next.done) {
+                                return next;
+                            }
+                            if (yield* predicate(next.value)) {
+                                return {
+                                    done: false,
+                                    value: next.value,
+                                };
+                            }
+                        }
+                    },
+                };
+            },
+        };
+    };
+}

package/esm/map.d.ts

@@ -0,0 +1,9 @@
+import type { Operation, Stream } from "effection";
+/**
+ * Transforms each item in the stream using the provided function.
+ *
+ * @param fn - The function to transform each item
+ * @returns A stream transformer that applies the function to each item
+ */
+export declare function map<A, B>(fn: (value: A) => Operation<B>): <TClose>(stream: Stream<A, TClose>) => Stream<B, TClose>;
+//# sourceMappingURL=map.d.ts.map

package/esm/map.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../src/map.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC,EACtB,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,GAC7B,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAsB1D"}

package/esm/map.js

@@ -0,0 +1,27 @@
+/**
+ * Transforms each item in the stream using the provided function.
+ *
+ * @param fn - The function to transform each item
+ * @returns A stream transformer that applies the function to each item
+ */
+export function map(fn) {
+    return function (stream) {
+        return {
+            *[Symbol.iterator]() {
+                const subscription = yield* stream;
+                return {
+                    *next() {
+                        const next = yield* subscription.next();
+                        if (next.done) {
+                            return next;
+                        }
+                        return {
+                            done: false,
+                            value: yield* fn(next.value),
+                        };
+                    },
+                };
+            },
+        };
+    };
+}

package/esm/mod.d.ts

@@ -0,0 +1,6 @@
+export * from "./batch.js";
+export * from "./valve.js";
+export * from "./map.js";
+export * from "./filter.js";
+export * from "./tracker.js";
+//# sourceMappingURL=mod.d.ts.map

package/esm/mod.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC"}

package/esm/mod.js

@@ -0,0 +1,5 @@
+export * from "./batch.js";
+export * from "./valve.js";
+export * from "./map.js";
+export * from "./filter.js";
+export * from "./tracker.js";

package/esm/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/esm/signals.d.ts

@@ -0,0 +1,23 @@
+import { type Operation, type Stream } from "effection";
+interface ValueStream<T> extends Stream<T, void> {
+    valueOf(): T;
+}
+interface Settable<T> {
+    set(value: T): T;
+}
+export interface SettableValue<T> extends Settable<T>, ValueStream<T> {
+}
+interface ArraySignal<T> extends SettableValue<T[]> {
+    push(item: T): number;
+    shift(): Operation<T>;
+    valueOf(): T[];
+    get length(): number;
+}
+export declare function createArraySignal<T>(initial: Iterable<T>): Operation<ArraySignal<T>>;
+export declare function is<T>(array: ValueStream<T>, predicate: (item: T) => boolean): Generator<any, void, any>;
+export interface BooleanSignal extends SettableValue<boolean> {
+}
+export declare function createBoolean(initial?: boolean): any;
+export declare function createSetSignal<T>(initial?: Array<T>): any;
+export {};
+//# sourceMappingURL=signals.d.ts.map

package/esm/signals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"signals.d.ts","sourceRoot":"","sources":["../src/signals.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,SAAS,EAEd,KAAK,MAAM,EACZ,MAAM,WAAW,CAAC;AAGnB,UAAU,WAAW,CAAC,CAAC,CAAE,SAAQ,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC;IAC9C,OAAO,IAAI,CAAC,CAAC;CACd;AAED,UAAU,QAAQ,CAAC,CAAC;IAClB,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;CAClB;AAED,MAAM,WAAW,aAAa,CAAC,CAAC,CAAE,SAAQ,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;CAAG;AAExE,UAAU,WAAW,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,EAAE,CAAC;IACjD,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACtB,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;IACtB,OAAO,IAAI,CAAC,EAAE,CAAC;IACf,IAAI,MAAM,IAAI,MAAM,CAAC;CACtB;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GACnB,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAwC3B;AAED,wBAAiB,EAAE,CAAC,CAAC,EACnB,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EACrB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,6BAahC;AAED,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,OAAO,CAAC;CAC5D;AAED,wBAAgB,aAAa,CAAC,OAAO,GAAE,OAAe,OA0BrD;AASD,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,GAAE,KAAK,CAAC,CAAC,CAAM,OAwCxD"}

package/esm/signals.js

@@ -0,0 +1,119 @@
+import { createSignal, each, resource, } from "effection";
+import { List, Set } from "immutable";
+export function createArraySignal(initial) {
+    return resource(function* (provide) {
+        const signal = createSignal();
+        const ref = {
+            current: List.of(...initial),
+        };
+        const array = {
+            [Symbol.iterator]: signal[Symbol.iterator],
+            set(value) {
+                ref.current = List.of(...value);
+                signal.send(ref.current.toArray());
+                return ref.current.toArray();
+            },
+            push(item) {
+                ref.current = ref.current.push(item);
+                signal.send(ref.current.toArray());
+                return ref.current.size;
+            },
+            *shift() {
+                yield* is(array, (array) => array.length > 0);
+                let value = ref.current.first();
+                ref.current = ref.current.shift();
+                signal.send(ref.current.toArray());
+                return value;
+            },
+            valueOf() {
+                return ref.current.toArray();
+            },
+            get length() {
+                return ref.current.size;
+            },
+        };
+        try {
+            yield* provide(array);
+        }
+        finally {
+            signal.close();
+        }
+    });
+}
+export function* is(array, predicate) {
+    const result = predicate(array.valueOf());
+    if (result) {
+        return;
+    }
+    for (const value of yield* each(array)) {
+        const result = predicate(value);
+        if (result) {
+            return;
+        }
+        yield* each.next();
+    }
+}
+export function createBoolean(initial = false) {
+    return resource(function* (provide) {
+        const signal = createSignal();
+        const ref = { current: initial };
+        try {
+            yield* provide({
+                [Symbol.iterator]: signal[Symbol.iterator],
+                set(value) {
+                    if (value !== ref.current) {
+                        ref.current = value;
+                        signal.send(ref.current);
+                    }
+                    return ref.current;
+                },
+                valueOf() {
+                    return ref.current;
+                },
+            });
+        }
+        finally {
+            signal.close();
+        }
+    });
+}
+export function createSetSignal(initial = []) {
+    return resource(function* (provide) {
+        const signal = createSignal();
+        const ref = { current: Set.of(...initial) };
+        try {
+            yield* provide({
+                [Symbol.iterator]: signal[Symbol.iterator],
+                set(value) {
+                    ref.current = Set.of(...value);
+                    signal.send(ref.current.toSet());
+                    return ref.current;
+                },
+                add(item) {
+                    ref.current = ref.current.add(item);
+                    signal.send(ref.current.toSet());
+                    return ref.current.toSet();
+                },
+                difference(items) {
+                    ref.current = ref.current.subtract(items);
+                    signal.send(ref.current.toSet());
+                    return ref.current.toSet();
+                },
+                delete(item) {
+                    if (ref.current.has(item)) {
+                        ref.current = ref.current.delete(item);
+                        signal.send(ref.current.toSet());
+                        return true;
+                    }
+                    return false;
+                },
+                valueOf() {
+                    return ref.current.toSet();
+                },
+            });
+        }
+        finally {
+            signal.close();
+        }
+    });
+}