@od-oneapp/config 2026.2.2001-canary.1

package/typescript/dist/src/server/streaming-utilities.js

@@ -0,0 +1,822 @@
+import { pipeline } from 'node:stream/promises';
+import { scheduler } from 'node:timers/promises';
+import { safeThrowIfAborted } from './abort-support';
+import { ioQueue } from './concurrency';
+import { withRetry } from './retryable';
+import { yieldToEventLoop } from './scheduler';
+import { createSafeFunction } from './security';
+import { asyncGeneratorToReadableStream, createBatchingTransformStream, createCollectorWritableStream, createThrottlingTransformStream, createTransformStream, mergeReadableStreams, pipelineStreams, streamWithBackpressure, } from './streams';
+import { safeStringifyAdvanced } from './stringify-advanced.js';
+import { ok, runTool } from './tool-helpers';
+export const streamingUtilitiesTool = {
+    name: 'streaming_utilities',
+    description: 'Node.js 22+ streaming utilities for efficient data processing',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            action: {
+                type: 'string',
+                enum: [
+                    'streamArray',
+                    'streamTransform',
+                    'streamFilter',
+                    'streamReduce',
+                    'streamMerge',
+                    'streamBatch',
+                    'streamThrottle',
+                    'streamBuffer',
+                    'streamCompress',
+                    'streamParallel',
+                    'createAsyncIterator',
+                    'createWebReadableStream',
+                    'createWebTransformStream',
+                    'pipelineWebStreams',
+                    'webStreamBackpressure',
+                    'mergeWebStreams',
+                ],
+                description: 'Streaming operation to perform',
+            },
+            data: {
+                type: 'array',
+                description: 'Input data array for streaming operations',
+            },
+            streams: {
+                type: 'array',
+                items: { type: 'array' },
+                description: 'Multiple data streams for merge operations',
+            },
+            chunkSize: {
+                type: 'number',
+                description: 'Size of each streamed chunk',
+                default: 10,
+            },
+            batchSize: {
+                type: 'number',
+                description: 'Size of batches for batch operations',
+                default: 5,
+            },
+            transformFunction: {
+                type: 'string',
+                description: 'JavaScript function to transform data (e.g., "x => x * 2")',
+            },
+            filterFunction: {
+                type: 'string',
+                description: 'JavaScript function to filter data (e.g., "x => x > 10")',
+            },
+            reduceFunction: {
+                type: 'string',
+                description: 'JavaScript function to reduce data (e.g., "(acc, x) => acc + x")',
+            },
+            initialValue: {
+                description: 'Initial value for reduce operations',
+            },
+            throttleMs: {
+                type: 'number',
+                description: 'Throttling delay in milliseconds',
+                default: 100,
+            },
+            bufferSize: {
+                type: 'number',
+                description: 'Buffer size for backpressure management',
+                default: 1000,
+            },
+            parallelism: {
+                type: 'number',
+                description: 'Number of parallel workers',
+                default: 4,
+            },
+            compressionLevel: {
+                type: 'number',
+                description: 'Compression level (1-9)',
+                minimum: 1,
+                maximum: 9,
+                default: 6,
+            },
+            iteratorName: {
+                type: 'string',
+                description: 'Name for custom async iterator',
+            },
+            iteratorCode: {
+                type: 'string',
+                description: 'Custom async iterator implementation code',
+            },
+            signal: {
+                description: 'AbortSignal for cancelling the operation',
+            },
+        },
+        required: ['action'],
+    },
+    async execute(args) {
+        return runTool('StreamingUtilities', args.action, async () => {
+            const { action, data = [], streams = [], chunkSize = 10, batchSize = 5, transformFunction, filterFunction, reduceFunction, initialValue, throttleMs = 100, bufferSize = 1000, parallelism = 4, compressionLevel: _compressionLevel = 6, iteratorName, iteratorCode, signal, } = args;
+            safeThrowIfAborted(signal);
+            switch (action) {
+                case 'streamArray': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamArray operation');
+                    }
+                    const chunks = [];
+                    for await (const chunk of streamArray(data, { chunkSize, signal })) {
+                        safeThrowIfAborted(signal);
+                        chunks.push(chunk);
+                    }
+                    return ok({
+                        success: true,
+                        operation: 'streamArray',
+                        chunks: chunks.length,
+                        results: chunks,
+                        totalItems: data.length,
+                    });
+                }
+                case 'streamTransform': {
+                    if (!transformFunction) {
+                        throw new Error('Transform function required for streamTransform operation');
+                    }
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamTransform operation');
+                    }
+                    const transform = createSafeFunction(transformFunction, 'x');
+                    const chunks = [];
+                    for await (const chunk of streamTransform(data, transform, { chunkSize, signal })) {
+                        safeThrowIfAborted(signal);
+                        chunks.push(chunk);
+                    }
+                    return ok({
+                        success: true,
+                        operation: 'streamTransform',
+                        chunks: chunks.length,
+                        results: chunks,
+                        transformFunction,
+                    });
+                }
+                case 'streamFilter': {
+                    if (!filterFunction) {
+                        throw new Error('Filter function required for streamFilter operation');
+                    }
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamFilter operation');
+                    }
+                    const filterFn = createSafeFunction(filterFunction, 'x');
+                    const filter = (x) => Boolean(filterFn(x));
+                    const chunks = [];
+                    for await (const chunk of streamFilter(data, filter, { chunkSize, signal })) {
+                        chunks.push(chunk);
+                        if (chunks.length % 5 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamFilter',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    filterFunction,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamReduce': {
+                    if (!reduceFunction) {
+                        throw new Error('Reduce function required for streamReduce operation');
+                    }
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamReduce operation');
+                    }
+                    const reducer = createSafeFunction(reduceFunction, ['acc', 'x']);
+                    const result = await streamReduce(data, reducer, initialValue, { signal });
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamReduce',
+                                    result,
+                                    reduceFunction,
+                                    initialValue,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamMerge': {
+                    if (streams.length === 0) {
+                        throw new Error('Multiple streams required for streamMerge operation');
+                    }
+                    const chunks = [];
+                    for await (const chunk of streamMerge(streams, { chunkSize, signal })) {
+                        chunks.push(chunk);
+                        if (chunks.length % 5 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamMerge',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    streamCount: streams.length,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamBatch': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamBatch operation');
+                    }
+                    const chunks = [];
+                    for await (const chunk of streamBatch(data, { batchSize, signal })) {
+                        chunks.push(chunk);
+                        if (chunks.length % 3 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamBatch',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    batchSize,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamThrottle': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamThrottle operation');
+                    }
+                    const chunks = [];
+                    for await (const chunk of streamThrottle(data, { throttleMs, chunkSize, signal })) {
+                        chunks.push(chunk);
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamThrottle',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    throttleMs,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamBuffer': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamBuffer operation');
+                    }
+                    const chunks = [];
+                    for await (const chunk of streamBuffer(data, { bufferSize, chunkSize, signal })) {
+                        chunks.push(chunk);
+                        if (chunks.length % 5 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamBuffer',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    bufferSize,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'streamParallel': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for streamParallel operation');
+                    }
+                    if (!transformFunction) {
+                        throw new Error('Transform function required for parallel processing');
+                    }
+                    const transform = createSafeFunction(transformFunction, 'x');
+                    const chunks = [];
+                    for await (const chunk of streamParallel(data, transform, {
+                        parallelism,
+                        chunkSize,
+                        signal,
+                    })) {
+                        chunks.push(chunk);
+                        if (chunks.length % 3 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'streamParallel',
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    parallelism,
+                                    transformFunction,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'createAsyncIterator': {
+                    if (!iteratorName || !iteratorCode) {
+                        throw new Error('Iterator name and code required for createAsyncIterator');
+                    }
+                    const iteratorFactory = createSafeFunction(iteratorCode, 'signal');
+                    const iterator = iteratorFactory(signal);
+                    if (!iterator || typeof iterator[Symbol.asyncIterator] !== 'function') {
+                        throw new Error('Iterator code must return an async iterable');
+                    }
+                    const chunks = [];
+                    let index = 0;
+                    for await (const value of iterator) {
+                        safeThrowIfAborted(signal);
+                        chunks.push({
+                            data: value,
+                            index: index++,
+                            timestamp: new Date().toISOString(),
+                        });
+                        if (index > 100)
+                            break;
+                        if (index % 5 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'createAsyncIterator',
+                                    iteratorName,
+                                    iterations: chunks.length,
+                                    results: chunks,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'createWebReadableStream': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for createWebReadableStream operation');
+                    }
+                    const chunks = [];
+                    const generator = streamArray(data, { chunkSize, signal });
+                    const readableStream = asyncGeneratorToReadableStream(generator, { signal });
+                    const { stream: collectorStream, getCollectedData } = createCollectorWritableStream({
+                        signal,
+                    });
+                    try {
+                        await readableStream.pipeTo(collectorStream, { signal });
+                        const results = getCollectedData();
+                        return {
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: safeStringifyAdvanced({
+                                        success: true,
+                                        operation: 'createWebReadableStream',
+                                        streamType: 'ReadableStream',
+                                        chunks: results.length,
+                                        results,
+                                        totalItems: data.length,
+                                    }).result,
+                                },
+                            ],
+                        };
+                    }
+                    catch (error) {
+                        if (error instanceof Error && error.message.includes('aborted')) {
+                            return {
+                                content: [
+                                    {
+                                        type: 'text',
+                                        text: safeStringifyAdvanced({
+                                            success: false,
+                                            aborted: true,
+                                            chunks: chunks.length,
+                                        }).result,
+                                    },
+                                ],
+                                isError: true,
+                            };
+                        }
+                        throw error;
+                    }
+                }
+                case 'createWebTransformStream': {
+                    if (!transformFunction) {
+                        throw new Error('Transform function required for createWebTransformStream operation');
+                    }
+                    if (data.length === 0) {
+                        throw new Error('Data array required for createWebTransformStream operation');
+                    }
+                    const transform = createSafeFunction(transformFunction, 'x');
+                    const _results = [];
+                    const transformStream = createTransformStream(async (item) => transform(item), { signal });
+                    const sourceGenerator = streamArray(data, { chunkSize, signal });
+                    const sourceStream = asyncGeneratorToReadableStream(sourceGenerator, { signal });
+                    const { stream: collectorStream, getCollectedData } = createCollectorWritableStream({
+                        signal,
+                    });
+                    try {
+                        await pipelineStreams(sourceStream, [transformStream], collectorStream, signal);
+                        const transformedResults = getCollectedData();
+                        return {
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: safeStringifyAdvanced({
+                                        success: true,
+                                        operation: 'createWebTransformStream',
+                                        streamType: 'TransformStream',
+                                        chunks: transformedResults.length,
+                                        results: transformedResults,
+                                        transformFunction,
+                                    }).result,
+                                },
+                            ],
+                        };
+                    }
+                    catch (error) {
+                        if (error instanceof Error && error.message.includes('aborted')) {
+                            return {
+                                content: [
+                                    {
+                                        type: 'text',
+                                        text: safeStringifyAdvanced({ success: false, aborted: true }).result,
+                                    },
+                                ],
+                                isError: true,
+                            };
+                        }
+                        throw error;
+                    }
+                }
+                case 'webStreamBackpressure': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for webStreamBackpressure operation');
+                    }
+                    const chunks = [];
+                    for await (const item of streamWithBackpressure(data, async (x) => {
+                        await scheduler.yield();
+                        return x;
+                    }, { chunkSize, signal, backpressure: true })) {
+                        chunks.push({
+                            data: item,
+                            timestamp: new Date().toISOString(),
+                            index: chunks.length,
+                        });
+                        if (chunks.length % 10 === 0) {
+                            await yieldToEventLoop();
+                        }
+                    }
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: safeStringifyAdvanced({
+                                    success: true,
+                                    operation: 'webStreamBackpressure',
+                                    backpressureEnabled: true,
+                                    chunks: chunks.length,
+                                    results: chunks,
+                                    totalItems: data.length,
+                                }).result,
+                            },
+                        ],
+                    };
+                }
+                case 'mergeWebStreams': {
+                    if (streams.length === 0) {
+                        throw new Error('Multiple streams required for mergeWebStreams operation');
+                    }
+                    const readableStreams = streams.map(streamData => {
+                        const generator = streamArray(streamData, { chunkSize, signal });
+                        return asyncGeneratorToReadableStream(generator, { signal });
+                    });
+                    const mergedStream = mergeReadableStreams(readableStreams, { signal });
+                    const { stream: collectorStream, getCollectedData } = createCollectorWritableStream({
+                        signal,
+                    });
+                    try {
+                        await mergedStream.pipeTo(collectorStream, { signal });
+                        const results = getCollectedData();
+                        return {
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: safeStringifyAdvanced({
+                                        success: true,
+                                        operation: 'mergeWebStreams',
+                                        streamType: 'MergedReadableStream',
+                                        streamCount: streams.length,
+                                        chunks: results.length,
+                                        results,
+                                    }).result,
+                                },
+                            ],
+                        };
+                    }
+                    catch (error) {
+                        if (error instanceof Error && error.message.includes('aborted')) {
+                            return {
+                                content: [
+                                    {
+                                        type: 'text',
+                                        text: safeStringifyAdvanced({ success: false, aborted: true }).result,
+                                    },
+                                ],
+                                isError: true,
+                            };
+                        }
+                        throw error;
+                    }
+                }
+                case 'pipelineWebStreams': {
+                    if (data.length === 0) {
+                        throw new Error('Data array required for pipelineWebStreams operation');
+                    }
+                    if (!transformFunction) {
+                        throw new Error('Transform function required for pipeline demonstration');
+                    }
+                    const transform = createSafeFunction(transformFunction, 'x');
+                    const sourceGenerator = streamArray(data, { chunkSize, signal });
+                    const sourceStream = asyncGeneratorToReadableStream(sourceGenerator, { signal });
+                    const transforms = [
+                        createBatchingTransformStream(batchSize, { signal }),
+                        createTransformStream(async (batch) => batch.map(transform), { signal }),
+                        createThrottlingTransformStream(throttleMs, { signal }),
+                    ];
+                    const { stream: collectorStream, getCollectedData } = createCollectorWritableStream({
+                        signal,
+                    });
+                    try {
+                        await pipeline(sourceStream, ...transforms.map(t => t), collectorStream, { signal });
+                        const results = getCollectedData();
+                        return {
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: safeStringifyAdvanced({
+                                        success: true,
+                                        operation: 'pipelineWebStreams',
+                                        pipelineStages: transforms.length,
+                                        transformFunction,
+                                        chunks: results.length,
+                                        results: results.flat(),
+                                    }).result,
+                                },
+                            ],
+                        };
+                    }
+                    catch (error) {
+                        if (error instanceof Error && error.message.includes('aborted')) {
+                            return {
+                                content: [
+                                    {
+                                        type: 'text',
+                                        text: safeStringifyAdvanced({ success: false, aborted: true }).result,
+                                    },
+                                ],
+                                isError: true,
+                            };
+                        }
+                        throw error;
+                    }
+                }
+                default:
+                    throw new Error(`Unknown streaming utilities action: ${action}`);
+            }
+        });
+    },
+};
+async function* streamArray(data, options = {}) {
+    const { chunkSize = 10, signal } = options;
+    let bytesProcessed = 0;
+    for (let i = 0; i < data.length; i += chunkSize) {
+        safeThrowIfAborted(signal);
+        const chunk = data.slice(i, i + chunkSize);
+        const chunkBytes = Buffer.byteLength(JSON.stringify(chunk), 'utf8');
+        bytesProcessed += chunkBytes;
+        yield {
+            data: chunk,
+            index: Math.floor(i / chunkSize),
+            isComplete: i + chunkSize >= data.length,
+            timestamp: new Date().toISOString(),
+            bytesProcessed,
+            metadata: {
+                chunkSize: chunk.length,
+                totalItems: data.length,
+                progress: Math.round(((i + chunk.length) / data.length) * 100),
+            },
+        };
+        await yieldToEventLoop();
+    }
+}
+async function* streamTransform(data, transformFn, options = {}) {
+    const { chunkSize = 10, signal } = options;
+    let bytesProcessed = 0;
+    for (let i = 0; i < data.length; i += chunkSize) {
+        safeThrowIfAborted(signal);
+        const chunk = data.slice(i, i + chunkSize);
+        const transformedChunk = chunk.map(transformFn);
+        const chunkBytes = Buffer.byteLength(JSON.stringify(transformedChunk), 'utf8');
+        bytesProcessed += chunkBytes;
+        yield {
+            data: transformedChunk,
+            index: Math.floor(i / chunkSize),
+            isComplete: i + chunkSize >= data.length,
+            timestamp: new Date().toISOString(),
+            bytesProcessed,
+            metadata: {
+                originalSize: chunk.length,
+                transformedSize: transformedChunk.length,
+            },
+        };
+        await yieldToEventLoop();
+    }
+}
+async function* streamFilter(data, filterFn, options = {}) {
+    const { chunkSize = 10, signal } = options;
+    let bytesProcessed = 0;
+    for (let i = 0; i < data.length; i += chunkSize) {
+        safeThrowIfAborted(signal);
+        const chunk = data.slice(i, i + chunkSize);
+        const filteredChunk = chunk.filter(filterFn);
+        const chunkBytes = Buffer.byteLength(JSON.stringify(filteredChunk), 'utf8');
+        bytesProcessed += chunkBytes;
+        yield {
+            data: filteredChunk,
+            index: Math.floor(i / chunkSize),
+            isComplete: i + chunkSize >= data.length,
+            timestamp: new Date().toISOString(),
+            bytesProcessed,
+            metadata: {
+                originalSize: chunk.length,
+                filteredSize: filteredChunk.length,
+                filterRatio: chunk.length > 0 ? filteredChunk.length / chunk.length : 0,
+            },
+        };
+        await yieldToEventLoop();
+    }
+}
+async function streamReduce(data, reduceFn, initialValue, options = {}) {
+    const { signal } = options;
+    let accumulator = initialValue;
+    for (let i = 0; i < data.length; i++) {
+        safeThrowIfAborted(signal);
+        const item = data[i];
+        if (item === undefined && !(i in data)) {
+            continue;
+        }
+        accumulator = reduceFn(accumulator, item);
+        if (i % 100 === 0) {
+            await yieldToEventLoop();
+        }
+    }
+    return accumulator;
+}
+async function* streamMerge(streams, options = {}) {
+    const { chunkSize = 10, signal } = options;
+    const iterators = streams.map(stream => streamArray(stream, { chunkSize, signal }));
+    let chunkIndex = 0;
+    let bytesProcessed = 0;
+    while (iterators.length > 0) {
+        safeThrowIfAborted(signal);
+        for (let i = iterators.length - 1; i >= 0; i--) {
+            const iterator = iterators[i];
+            if (!iterator) {
+                continue;
+            }
+            const { value, done } = await iterator.next();
+            if (done) {
+                iterators.splice(i, 1);
+                continue;
+            }
+            const chunk = value.data;
+            const chunkBytes = Buffer.byteLength(JSON.stringify(chunk), 'utf8');
+            bytesProcessed += chunkBytes;
+            yield {
+                data: chunk,
+                index: chunkIndex++,
+                isComplete: iterators.length === 1 && i === iterators.length - 1,
+                timestamp: new Date().toISOString(),
+                bytesProcessed,
+                metadata: {
+                    sourceStream: streams.length - iterators.length,
+                    remainingStreams: iterators.length,
+                },
+            };
+            await yieldToEventLoop();
+        }
+    }
+}
+async function* streamBatch(data, options = {}) {
+    const { batchSize = 5, signal } = options;
+    let bytesProcessed = 0;
+    for (let i = 0; i < data.length; i += batchSize) {
+        safeThrowIfAborted(signal);
+        const batch = data.slice(i, i + batchSize);
+        const groupedBatch = [batch];
+        const chunkBytes = Buffer.byteLength(JSON.stringify(groupedBatch), 'utf8');
+        bytesProcessed += chunkBytes;
+        yield {
+            data: groupedBatch,
+            index: Math.floor(i / batchSize),
+            isComplete: i + batchSize >= data.length,
+            timestamp: new Date().toISOString(),
+            bytesProcessed,
+            metadata: {
+                batchSize: batch.length,
+                totalBatches: Math.ceil(data.length / batchSize),
+            },
+        };
+        await yieldToEventLoop();
+    }
+}
+async function* streamThrottle(data, options = {}) {
+    const { throttleMs = 100, chunkSize = 10, signal } = options;
+    for await (const chunk of streamArray(data, { chunkSize, signal })) {
+        safeThrowIfAborted(signal);
+        yield chunk;
+        if (!chunk.isComplete && throttleMs > 0) {
+            await scheduler.wait(throttleMs);
+        }
+    }
+}
+async function* streamBuffer(data, options = {}) {
+    const { bufferSize = 1000, chunkSize = 10, signal } = options;
+    const buffer = [];
+    for await (const chunk of streamArray(data, { chunkSize, signal })) {
+        safeThrowIfAborted(signal);
+        buffer.push(...chunk.data);
+        if (buffer.length >= bufferSize || chunk.isComplete) {
+            const bufferedData = buffer.splice(0, buffer.length);
+            yield {
+                ...chunk,
+                data: bufferedData,
+                metadata: {
+                    ...chunk.metadata,
+                    bufferSize: bufferedData.length,
+                    bufferFull: bufferedData.length >= bufferSize,
+                },
+            };
+        }
+    }
+}
+async function* streamParallel(data, transformFn, options = {}) {
+    const { parallelism = 4, chunkSize = 10, signal } = options;
+    let bytesProcessed = 0;
+    for (let i = 0; i < data.length; i += chunkSize * parallelism) {
+        safeThrowIfAborted(signal);
+        const batchFunctions = [];
+        for (let j = 0; j < parallelism && i + j * chunkSize < data.length; j++) {
+            const batchStart = i + j * chunkSize;
+            const batchEnd = Math.min(batchStart + chunkSize, data.length);
+            const batch = data.slice(batchStart, batchEnd);
+            batchFunctions.push(async () => withRetry(async () => batch.map(transformFn)));
+        }
+        const results = await ioQueue.addAll(batchFunctions);
+        const flatResults = results.flat();
+        const chunkBytes = Buffer.byteLength(JSON.stringify(flatResults), 'utf8');
+        bytesProcessed += chunkBytes;
+        yield {
+            data: flatResults,
+            index: Math.floor(i / (chunkSize * parallelism)),
+            isComplete: i + chunkSize * parallelism >= data.length,
+            timestamp: new Date().toISOString(),
+            bytesProcessed,
+            metadata: {
+                parallelBatches: batchFunctions.length,
+                parallelism,
+                processedItems: flatResults.length,
+            },
+        };
+        await yieldToEventLoop();
+    }
+}
+//# sourceMappingURL=streaming-utilities.js.map