awaitly 2.0.0 → 3.0.0

package/dist/streaming.d.cts

@@ -0,0 +1,594 @@
+import { S as StreamStore, a as StreamReader } from './types-d8q8iQlk.cjs';
+export { b as STREAM_BACKPRESSURE_ERROR, c as STREAM_CLOSE_ERROR, d as STREAM_ENDED, e as STREAM_READ_ERROR, f as STREAM_STORE_ERROR, g as STREAM_WRITE_ERROR, h as StreamBackpressureError, i as StreamCloseError, j as StreamEndedMarker, k as StreamError, l as StreamForEachOptions, m as StreamForEachResult, n as StreamItem, o as StreamMetadata, p as StreamOptions, q as StreamReadError, r as StreamReadOptions, s as StreamStoreError, t as StreamWriteError, u as StreamWriter, U as Unsubscribe, v as isStreamBackpressureError, w as isStreamEnded, x as isStreamReadError, y as isStreamStoreError, z as isStreamWriteError, A as streamBackpressureError, B as streamCloseError, C as streamEnded, D as streamReadError, E as streamStoreError, F as streamWriteError } from './types-d8q8iQlk.cjs';
+import { A as AsyncResult, R as Result } from './index-ICmfQi08.cjs';
+import './errors-sDSeaZBO.cjs';
+
+/**
+ * awaitly/streaming - Backpressure Controller
+ *
+ * Implements flow control for streams using high-water mark.
+ * When buffered items exceed the threshold, writers are paused
+ * until consumers catch up.
+ */
+/**
+ * State of the backpressure controller.
+ */
+type BackpressureState = "flowing" | "paused";
+/**
+ * Callback invoked when backpressure state changes.
+ */
+type BackpressureCallback = (state: BackpressureState) => void;
+/**
+ * Options for creating a BackpressureController.
+ */
+interface BackpressureOptions {
+    /** High-water mark threshold (default: 16) */
+    highWaterMark?: number;
+    /** Low-water mark to resume (default: highWaterMark / 2) */
+    lowWaterMark?: number;
+    /** Callback when state changes */
+    onStateChange?: BackpressureCallback;
+}
+/**
+ * Controller for managing stream backpressure.
+ *
+ * When the number of buffered items exceeds the high-water mark,
+ * the controller enters "paused" state. It returns to "flowing"
+ * when items are consumed and the buffer drops below the low-water mark.
+ *
+ * @example
+ * ```typescript
+ * const controller = createBackpressureController({ highWaterMark: 16 });
+ *
+ * // Track writes
+ * controller.increment();
+ * if (controller.state === 'paused') {
+ *   await controller.waitForDrain();
+ * }
+ *
+ * // Track reads (consumer)
+ * controller.decrement();
+ * ```
+ */
+interface BackpressureController {
+    /** Current state */
+    readonly state: BackpressureState;
+    /** Current number of buffered items */
+    readonly bufferedCount: number;
+    /** High-water mark threshold */
+    readonly highWaterMark: number;
+    /** Low-water mark threshold */
+    readonly lowWaterMark: number;
+    /** Increment buffered count (called on write) */
+    increment(): void;
+    /** Decrement buffered count (called on read/consume) */
+    decrement(): void;
+    /** Set buffered count directly (for resuming) */
+    setCount(count: number): void;
+    /**
+     * Wait for the buffer to drain below low-water mark.
+     * Resolves immediately if already flowing.
+     */
+    waitForDrain(): Promise<void>;
+    /** Reset the controller to initial state */
+    reset(): void;
+}
+/**
+ * Create a backpressure controller.
+ *
+ * @param options - Configuration options
+ * @returns BackpressureController instance
+ */
+declare function createBackpressureController(options?: BackpressureOptions): BackpressureController;
+/**
+ * Check if backpressure should be applied.
+ */
+declare function shouldApplyBackpressure(controller: BackpressureController): boolean;
+
+/**
+ * awaitly/streaming - Memory Stream Store
+ *
+ * In-memory implementation of StreamStore for development and testing.
+ * Data is not persisted across process restarts.
+ */
+
+/**
+ * Options for creating a memory stream store.
+ */
+interface MemoryStreamStoreOptions {
+    /** Maximum items per stream (default: Infinity) */
+    maxItemsPerStream?: number;
+}
+/**
+ * Create an in-memory StreamStore.
+ *
+ * @param options - Configuration options
+ * @returns StreamStore implementation
+ *
+ * @example
+ * ```typescript
+ * const store = createMemoryStreamStore();
+ *
+ * // Use with workflow
+ * const workflow = createWorkflow(deps, { streamStore: store });
+ * ```
+ */
+declare function createMemoryStreamStore(options?: MemoryStreamStoreOptions): StreamStore;
+/**
+ * Create a memory stream store with a Map-like interface for testing.
+ * Exposes additional methods for inspection.
+ */
+interface TestableMemoryStreamStore extends StreamStore {
+    /** Clear all streams */
+    clear(): void;
+    /** Get all stream keys */
+    keys(): string[];
+    /** Check if a stream exists */
+    has(workflowId: string, namespace: string): boolean;
+    /** Delete a stream */
+    delete(workflowId: string, namespace: string): boolean;
+}
+/**
+ * Create a testable memory stream store with additional inspection methods.
+ */
+declare function createTestableMemoryStreamStore(options?: MemoryStreamStoreOptions): TestableMemoryStreamStore;
+
+/**
+ * awaitly/streaming - File Stream Store
+ *
+ * File-based implementation of StreamStore for persistent storage.
+ * Follows the patterns from persistence.ts.
+ */
+
+/**
+ * Minimal file system interface for stream operations.
+ * Same as FileSystemInterface in persistence.ts.
+ */
+interface FileSystemInterface {
+    readFile(path: string): Promise<string>;
+    writeFile(path: string, data: string): Promise<void>;
+    unlink(path: string): Promise<void>;
+    exists(path: string): Promise<boolean>;
+    readdir(path: string): Promise<string[]>;
+    mkdir(path: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+}
+/**
+ * Options for creating a file stream store.
+ */
+interface FileStreamStoreOptions {
+    /** Directory to store stream files */
+    directory: string;
+    /** File system implementation */
+    fs: FileSystemInterface;
+}
+/**
+ * Create a file-based StreamStore.
+ *
+ * Each stream is stored in a directory structure:
+ * - `{directory}/{workflowId}/{namespace}/metadata.json` - stream metadata
+ * - `{directory}/{workflowId}/{namespace}/items.jsonl` - items in JSON lines format
+ *
+ * @param options - Configuration options
+ * @returns StreamStore implementation
+ *
+ * @example
+ * ```typescript
+ * import * as fs from 'fs/promises';
+ *
+ * const store = createFileStreamStore({
+ *   directory: './streams',
+ *   fs: {
+ *     readFile: (path) => fs.readFile(path, 'utf-8'),
+ *     writeFile: (path, data) => fs.writeFile(path, data, 'utf-8'),
+ *     unlink: (path) => fs.unlink(path),
+ *     exists: async (path) => {
+ *       try { await fs.access(path); return true; }
+ *       catch { return false; }
+ *     },
+ *     readdir: (path) => fs.readdir(path),
+ *     mkdir: (path, options) => fs.mkdir(path, options),
+ *   },
+ * });
+ * ```
+ */
+declare function createFileStreamStore(options: FileStreamStoreOptions): StreamStore;
+
+/**
+ * awaitly/streaming - Stream Transformers
+ *
+ * Utilities for transforming streams: map, filter, chunk, flatMapAsync.
+ * All transformers work with both StreamReader and AsyncIterable sources.
+ */
+
+/**
+ * A transform function that can be applied to stream items.
+ */
+type TransformFn<T, U> = (item: T, index: number) => U | Promise<U>;
+/**
+ * A filter predicate for stream items.
+ */
+type FilterFn<T> = (item: T, index: number) => boolean | Promise<boolean>;
+/**
+ * An async transform function that returns a Result.
+ */
+type AsyncTransformFn<T, U, E> = (item: T, index: number) => AsyncResult<U, E>;
+/**
+ * Convert a StreamReader to an AsyncIterable.
+ *
+ * This allows using for-await-of with StreamReaders.
+ *
+ * @param reader - StreamReader to convert
+ * @returns AsyncIterable that yields stream values
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'tokens' });
+ *
+ * for await (const token of toAsyncIterable(reader)) {
+ *   process.stdout.write(token);
+ * }
+ * ```
+ */
+declare function toAsyncIterable<T>(reader: StreamReader<T>): AsyncIterable<T>;
+/**
+ * Transform each item in a stream.
+ *
+ * @param source - StreamReader or AsyncIterable to transform
+ * @param fn - Transform function applied to each item
+ * @returns AsyncIterable of transformed values
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * for await (const doubled of map(reader, (n) => n * 2)) {
+ *   console.log(doubled);
+ * }
+ * ```
+ */
+declare function map<T, U>(source: StreamReader<T> | AsyncIterable<T>, fn: TransformFn<T, U>): AsyncIterable<U>;
+/**
+ * Filter items in a stream.
+ *
+ * @param source - StreamReader or AsyncIterable to filter
+ * @param predicate - Filter function that returns true to keep item
+ * @returns AsyncIterable of filtered values
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * for await (const even of filter(reader, (n) => n % 2 === 0)) {
+ *   console.log(even);
+ * }
+ * ```
+ */
+declare function filter<T>(source: StreamReader<T> | AsyncIterable<T>, predicate: FilterFn<T>): AsyncIterable<T>;
+/**
+ * Group stream items into fixed-size chunks.
+ *
+ * @param source - StreamReader or AsyncIterable to chunk
+ * @param size - Maximum number of items per chunk
+ * @returns AsyncIterable of item arrays
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'messages' });
+ *
+ * for await (const batch of chunk(reader, 10)) {
+ *   await processBatch(batch);
+ * }
+ * ```
+ */
+declare function chunk<T>(source: StreamReader<T> | AsyncIterable<T>, size: number): AsyncIterable<T[]>;
+/**
+ * Transform each item to multiple items and flatten.
+ *
+ * @param source - StreamReader or AsyncIterable to transform
+ * @param fn - Transform function that returns an iterable
+ * @returns AsyncIterable of flattened values
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'lines' });
+ *
+ * for await (const word of flatMap(reader, (line) => line.split(' '))) {
+ *   console.log(word);
+ * }
+ * ```
+ */
+declare function flatMap<T, U>(source: StreamReader<T> | AsyncIterable<T>, fn: (item: T, index: number) => Iterable<U> | AsyncIterable<U>): AsyncIterable<U>;
+/**
+ * Transform each item with a Result-returning async function.
+ *
+ * This is the Result-aware version of flatMap. If the transform function
+ * returns an error Result, the stream is terminated with that error.
+ *
+ * @param source - StreamReader or AsyncIterable to transform
+ * @param fn - Transform function returning AsyncResult
+ * @returns AsyncIterable of transformed values wrapped in Results
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<Message>({ namespace: 'messages' });
+ *
+ * for await (const result of flatMapAsync(reader, async (msg) => {
+ *   const processed = await processMessage(msg);
+ *   return ok(processed);
+ * })) {
+ *   if (!result.ok) {
+ *     console.error('Processing failed:', result.error);
+ *     break;
+ *   }
+ *   console.log('Processed:', result.value);
+ * }
+ * ```
+ */
+declare function flatMapAsync<T, U, E>(source: StreamReader<T> | AsyncIterable<T>, fn: AsyncTransformFn<T, Iterable<U> | AsyncIterable<U>, E>): AsyncIterable<Result<U, E>>;
+/**
+ * Transform each item with a Result-returning async function.
+ *
+ * @param source - StreamReader or AsyncIterable to transform
+ * @param fn - Transform function returning AsyncResult
+ * @returns AsyncIterable of Results
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * for await (const result of mapAsync(reader, async (n) => {
+ *   if (n < 0) return err('NEGATIVE_NUMBER');
+ *   return ok(Math.sqrt(n));
+ * })) {
+ *   if (result.ok) {
+ *     console.log('Sqrt:', result.value);
+ *   }
+ * }
+ * ```
+ */
+declare function mapAsync<T, U, E>(source: StreamReader<T> | AsyncIterable<T>, fn: AsyncTransformFn<T, U, E>): AsyncIterable<Result<U, E>>;
+/**
+ * Take the first N items from a stream.
+ *
+ * @param source - StreamReader or AsyncIterable to take from
+ * @param count - Maximum number of items to take
+ * @returns AsyncIterable of up to N items
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'messages' });
+ *
+ * for await (const message of take(reader, 10)) {
+ *   console.log(message);
+ * }
+ * ```
+ */
+declare function take<T>(source: StreamReader<T> | AsyncIterable<T>, count: number): AsyncIterable<T>;
+/**
+ * Skip the first N items from a stream.
+ *
+ * @param source - StreamReader or AsyncIterable to skip from
+ * @param count - Number of items to skip
+ * @returns AsyncIterable starting after N items
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'messages' });
+ *
+ * for await (const message of skip(reader, 100)) {
+ *   console.log(message); // Messages 101+
+ * }
+ * ```
+ */
+declare function skip<T>(source: StreamReader<T> | AsyncIterable<T>, count: number): AsyncIterable<T>;
+/**
+ * Take items while predicate returns true.
+ *
+ * @param source - StreamReader or AsyncIterable
+ * @param predicate - Predicate function
+ * @returns AsyncIterable of items while predicate is true
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * for await (const n of takeWhile(reader, (n) => n < 100)) {
+ *   console.log(n);
+ * }
+ * ```
+ */
+declare function takeWhile<T>(source: StreamReader<T> | AsyncIterable<T>, predicate: FilterFn<T>): AsyncIterable<T>;
+/**
+ * Skip items while predicate returns true.
+ *
+ * @param source - StreamReader or AsyncIterable
+ * @param predicate - Predicate function
+ * @returns AsyncIterable of items after predicate becomes false
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * for await (const n of skipWhile(reader, (n) => n < 100)) {
+ *   console.log(n); // First n >= 100 and all after
+ * }
+ * ```
+ */
+declare function skipWhile<T>(source: StreamReader<T> | AsyncIterable<T>, predicate: FilterFn<T>): AsyncIterable<T>;
+/**
+ * Collect all stream items into an array.
+ *
+ * Warning: This loads all items into memory. Only use when you know
+ * the stream is bounded.
+ *
+ * @param source - StreamReader or AsyncIterable
+ * @returns Promise resolving to array of all items
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<string>({ namespace: 'small-data' });
+ * const items = await collect(reader);
+ * console.log('Got', items.length, 'items');
+ * ```
+ */
+declare function collect<T>(source: StreamReader<T> | AsyncIterable<T>): Promise<T[]>;
+/**
+ * Reduce stream items to a single value.
+ *
+ * @param source - StreamReader or AsyncIterable
+ * @param reducer - Reducer function
+ * @param initial - Initial accumulator value
+ * @returns Promise resolving to final accumulated value
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ * const sum = await reduce(reader, (acc, n) => acc + n, 0);
+ * console.log('Sum:', sum);
+ * ```
+ */
+declare function reduce<T, U>(source: StreamReader<T> | AsyncIterable<T>, reducer: (accumulator: U, item: T, index: number) => U | Promise<U>, initial: U): Promise<U>;
+/**
+ * Pipe a source through multiple transformers.
+ *
+ * @param source - Initial source
+ * @param transformers - Array of transformer functions
+ * @returns Final transformed AsyncIterable
+ *
+ * @example
+ * ```typescript
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * const result = pipe(
+ *   reader,
+ *   (s) => filter(s, (n) => n > 0),
+ *   (s) => map(s, (n) => n * 2),
+ *   (s) => take(s, 10)
+ * );
+ *
+ * for await (const n of result) {
+ *   console.log(n);
+ * }
+ * ```
+ */
+declare function pipe<T>(source: StreamReader<T> | AsyncIterable<T>): AsyncIterable<T>;
+declare function pipe<T, A>(source: StreamReader<T> | AsyncIterable<T>, t1: (s: AsyncIterable<T>) => AsyncIterable<A>): AsyncIterable<A>;
+declare function pipe<T, A, B>(source: StreamReader<T> | AsyncIterable<T>, t1: (s: AsyncIterable<T>) => AsyncIterable<A>, t2: (s: AsyncIterable<A>) => AsyncIterable<B>): AsyncIterable<B>;
+declare function pipe<T, A, B, C>(source: StreamReader<T> | AsyncIterable<T>, t1: (s: AsyncIterable<T>) => AsyncIterable<A>, t2: (s: AsyncIterable<A>) => AsyncIterable<B>, t3: (s: AsyncIterable<B>) => AsyncIterable<C>): AsyncIterable<C>;
+declare function pipe<T, A, B, C, D>(source: StreamReader<T> | AsyncIterable<T>, t1: (s: AsyncIterable<T>) => AsyncIterable<A>, t2: (s: AsyncIterable<A>) => AsyncIterable<B>, t3: (s: AsyncIterable<B>) => AsyncIterable<C>, t4: (s: AsyncIterable<C>) => AsyncIterable<D>): AsyncIterable<D>;
+
+/**
+ * awaitly/streaming
+ *
+ * Result-aware streaming for workflows.
+ * All stream operations return Result types, enabling typed error handling
+ * throughout the streaming pipeline.
+ *
+ * @example Basic usage
+ * ```typescript
+ * import { createWorkflow } from 'awaitly/workflow';
+ * import { createMemoryStreamStore } from 'awaitly/streaming';
+ *
+ * const streamStore = createMemoryStreamStore();
+ * const workflow = createWorkflow(deps, { streamStore });
+ *
+ * const result = await workflow(async ({ step }) => {
+ *   const writer = step.getWritable<string>({ namespace: 'tokens' });
+ *
+ *   await step(() => generateAI({
+ *     prompt: 'Hello',
+ *     onToken: async (token) => { await writer.write(token); }
+ *   }), { key: 'generate' });
+ *
+ *   await writer.close();
+ * });
+ * ```
+ *
+ * @example Consuming a stream
+ * ```typescript
+ * import { toAsyncIterable } from 'awaitly/streaming';
+ *
+ * const reader = step.getReadable<string>({ namespace: 'tokens' });
+ *
+ * for await (const token of toAsyncIterable(reader)) {
+ *   process.stdout.write(token);
+ * }
+ * ```
+ *
+ * @example Stream transformations
+ * ```typescript
+ * import { map, filter, chunk, collect } from 'awaitly/streaming';
+ *
+ * const reader = step.getReadable<number>({ namespace: 'numbers' });
+ *
+ * // Transform pipeline
+ * const evens = filter(reader, n => n % 2 === 0);
+ * const doubled = map(evens, n => n * 2);
+ * const batches = chunk(doubled, 10);
+ *
+ * for await (const batch of batches) {
+ *   await processBatch(batch);
+ * }
+ * ```
+ */
+
+/**
+ * Options for creating an external stream reader.
+ */
+interface ExternalReaderOptions {
+    /** Stream store instance */
+    store: StreamStore;
+    /** Workflow ID that owns the stream */
+    workflowId: string;
+    /** Stream namespace (default: 'default') */
+    namespace?: string;
+    /** Start reading from this position (default: 0) */
+    startIndex?: number;
+    /** Poll interval in ms when waiting for new items (default: 100) */
+    pollInterval?: number;
+    /** Stop polling after this many ms with no new items (default: 30000) */
+    pollTimeout?: number;
+}
+/**
+ * Create a stream reader for external consumption (outside workflows).
+ *
+ * Use this in HTTP handlers, WebSocket handlers, or other contexts
+ * where you need to consume a stream created by a workflow.
+ *
+ * @param options - Reader configuration
+ * @returns StreamReader that can be used with toAsyncIterable()
+ *
+ * @example HTTP streaming response
+ * ```typescript
+ * app.get('/stream/:workflowId', async (req, res) => {
+ *   const reader = getStreamReader({
+ *     store: streamStore,
+ *     workflowId: req.params.workflowId,
+ *     namespace: 'ai-response',
+ *   });
+ *
+ *   res.setHeader('Content-Type', 'text/event-stream');
+ *
+ *   for await (const chunk of toAsyncIterable(reader)) {
+ *     res.write(`data: ${chunk}\n\n`);
+ *   }
+ *
+ *   res.end();
+ * });
+ * ```
+ *
+ * @example Resume from last position
+ * ```typescript
+ * const reader = getStreamReader({
+ *   store: streamStore,
+ *   workflowId: runId,
+ *   namespace: 'tokens',
+ *   startIndex: lastReceivedPosition + 1,
+ * });
+ * ```
+ */
+declare function getStreamReader<T>(options: ExternalReaderOptions): StreamReader<T>;
+
+export { type AsyncTransformFn, type BackpressureCallback, type BackpressureController, type BackpressureOptions, type BackpressureState, type ExternalReaderOptions, type FileStreamStoreOptions, type FileSystemInterface, type FilterFn, type MemoryStreamStoreOptions, StreamReader, StreamStore, type TestableMemoryStreamStore, type TransformFn, chunk, collect, createBackpressureController, createFileStreamStore, createMemoryStreamStore, createTestableMemoryStreamStore, filter, flatMap, flatMapAsync, getStreamReader, map, mapAsync, pipe, reduce, shouldApplyBackpressure, skip, skipWhile, take, takeWhile, toAsyncIterable };