@od-oneapp/config 2026.2.2001-canary.1

package/typescript/dist/src/server/batch-processor-tool.js

@@ -0,0 +1,651 @@
+import { LRUCache } from 'lru-cache';
+import { delay } from '../shared/timeout.js';
+import { safeThrowIfAborted } from './abort-support';
+import { ioQueue } from './concurrency';
+import { ErrorPatterns } from './error-handling';
+import { enhancedClone, isStructuredCloneAvailable } from './structured-clone';
+import { err, ok, runTool } from './tool-helpers';
+import { validateSessionId } from './validation';
+const sizeCache = new LRUCache({
+    max: 1000,
+    ttl: 1800000,
+    allowStale: false,
+    updateAgeOnGet: true,
+});
+async function* streamFileEstimation(files, fileSizes, packagePath, signal) {
+    const totalFiles = files.length;
+    const memoryUsage = process.memoryUsage();
+    const availableMemory = memoryUsage.heapTotal - memoryUsage.heapUsed;
+    const targetMemoryPerChunk = Math.min(availableMemory * 0.1, 100 * 1024 * 1024);
+    let CHUNK_SIZE;
+    if (totalFiles < 100) {
+        CHUNK_SIZE = totalFiles;
+    }
+    else if (totalFiles < 1000) {
+        CHUNK_SIZE = Math.max(25, Math.floor(totalFiles / 10));
+    }
+    else {
+        const estimatedAvgFileSize = 200;
+        const filesPerMemoryChunk = Math.floor(targetMemoryPerChunk / (estimatedAvgFileSize * 100));
+        CHUNK_SIZE = Math.max(50, Math.min(200, filesPerMemoryChunk));
+    }
+    for (let i = 0; i < files.length; i += CHUNK_SIZE) {
+        safeThrowIfAborted(signal);
+        const chunk = files.slice(i, i + CHUNK_SIZE);
+        for (const file of chunk) {
+            let size = fileSizes[file];
+            if (!size) {
+                const cacheKey = `${packagePath ?? ''}:${file}`;
+                const cached = sizeCache.get(cacheKey);
+                if (cached !== undefined) {
+                    size = cached;
+                }
+                else {
+                    const ext = file.split('.').pop()?.toLowerCase() ?? '';
+                    const filename = file.split('/').pop() ?? '';
+                    const isLargeFile = filename.length > 50 || file.includes('large') || file.includes('generated');
+                    let estimate = 100;
+                    switch (ext) {
+                        case 'ts':
+                        case 'tsx':
+                            if (filename.includes('test') || filename.includes('spec')) {
+                                estimate = 150;
+                            }
+                            else if (isLargeFile ||
+                                filename.includes('index') ||
+                                filename.includes('generated')) {
+                                estimate = 500;
+                            }
+                            else if (filename.includes('component') || filename.includes('page')) {
+                                estimate = 300;
+                            }
+                            else {
+                                estimate = 200;
+                            }
+                            break;
+                        case 'js':
+                        case 'jsx':
+                            if (filename.includes('test') || filename.includes('spec')) {
+                                estimate = 120;
+                            }
+                            else if (isLargeFile ||
+                                filename.includes('bundle') ||
+                                filename.includes('vendor')) {
+                                estimate = 1000;
+                            }
+                            else if (filename.includes('component') || filename.includes('page')) {
+                                estimate = 250;
+                            }
+                            else {
+                                estimate = 180;
+                            }
+                            break;
+                        case 'json':
+                            if (filename === 'package-lock.json' || filename === 'yarn.lock') {
+                                estimate = 5000;
+                            }
+                            else if (filename === 'package.json') {
+                                estimate = 50;
+                            }
+                            else if (filename.includes('config') || filename.includes('settings')) {
+                                estimate = 100;
+                            }
+                            else {
+                                estimate = 30;
+                            }
+                            break;
+                        case 'md':
+                            if (filename.toLowerCase() === 'readme.md' || filename.includes('changelog')) {
+                                estimate = 200;
+                            }
+                            else {
+                                estimate = 80;
+                            }
+                            break;
+                        case 'css':
+                        case 'scss':
+                        case 'less':
+                            if (isLargeFile || filename.includes('main') || filename.includes('global')) {
+                                estimate = 400;
+                            }
+                            else {
+                                estimate = 120;
+                            }
+                            break;
+                        case 'html':
+                        case 'htm':
+                            estimate = isLargeFile ? 300 : 150;
+                            break;
+                        case 'svg':
+                            estimate = 50;
+                            break;
+                        case 'yml':
+                        case 'yaml':
+                            estimate = filename.includes('docker') ? 100 : 60;
+                            break;
+                        case 'xml':
+                            estimate = isLargeFile ? 500 : 200;
+                            break;
+                        default:
+                            estimate = 100;
+                    }
+                    if (filename.includes('config'))
+                        estimate *= 0.7;
+                    if (filename.includes('types') || filename.includes('.d.'))
+                        estimate *= 0.8;
+                    if (filename.includes('utils') || filename.includes('helpers'))
+                        estimate *= 1.2;
+                    if (filename.includes('component'))
+                        estimate *= 1.3;
+                    if (filename.includes('service') || filename.includes('api'))
+                        estimate *= 1.1;
+                    if (file.includes('node_modules'))
+                        estimate *= 0.3;
+                    if (file.includes('__tests__') || file.includes('test/'))
+                        estimate *= 0.9;
+                    if (file.includes('dist/') || file.includes('build/'))
+                        estimate *= 2.0;
+                    if (file.includes('coverage/'))
+                        estimate *= 0.5;
+                    const pathDepth = file.split('/').length;
+                    if (pathDepth > 5)
+                        estimate *= 0.9;
+                    if (pathDepth < 3)
+                        estimate *= 1.1;
+                    size = Math.round(estimate);
+                    sizeCache.set(cacheKey, size);
+                }
+            }
+            yield {
+                file,
+                size,
+                progress: Math.round(((i + chunk.indexOf(file) + 1) / totalFiles) * 100),
+                batchInfo: {
+                    chunkSize: CHUNK_SIZE,
+                    chunkIndex: Math.floor(i / CHUNK_SIZE),
+                    filesInChunk: chunk.length,
+                    totalChunks: Math.ceil(totalFiles / CHUNK_SIZE),
+                    memoryOptimized: CHUNK_SIZE !== 50,
+                    estimatedMemoryUsage: chunk.length * 200 * 100,
+                },
+            };
+        }
+        if (i % (CHUNK_SIZE * 5) === 0) {
+            const currentMemory = process.memoryUsage();
+            const heapUsagePercent = (currentMemory.heapUsed / currentMemory.heapTotal) * 100;
+            if (heapUsagePercent > 80) {
+                if (global.gc) {
+                    global.gc();
+                }
+                await delay(100);
+            }
+            else {
+                await new Promise(resolve => setImmediate(resolve));
+            }
+        }
+        else if (i + CHUNK_SIZE < files.length) {
+            await new Promise(resolve => setImmediate(resolve));
+        }
+    }
+}
+async function estimateSize(files, fileSizes, packagePath) {
+    const results = [];
+    for await (const result of streamFileEstimation(files, fileSizes, packagePath)) {
+        results.push(result);
+    }
+    return results.reduce((acc, result) => {
+        acc[result.file] = result.size;
+        return acc;
+    }, {});
+}
+export const batchProcessorTool = {
+    name: 'batch_processor',
+    description: 'Optimize file processing in memory-efficient batches with Node.js 22+ streaming',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            action: {
+                type: 'string',
+                enum: [
+                    'createBatches',
+                    'prioritizeFiles',
+                    'filterCached',
+                    'estimateSize',
+                    'streamProcess',
+                    'streamBatches',
+                    'queuedProcess',
+                    'queueStatus',
+                    'configureConcurrency',
+                ],
+                description: 'Batch processing action to perform',
+            },
+            files: {
+                type: 'array',
+                items: {
+                    type: 'string',
+                    description: 'File path relative to package root',
+                },
+                description: 'List of files to process',
+            },
+            targetSize: {
+                type: 'number',
+                description: 'Target batch size (lines or bytes)',
+                default: 3000,
+            },
+            changedFiles: {
+                type: 'array',
+                items: {
+                    type: 'string',
+                    description: 'File path of changed file for priority processing',
+                },
+                description: 'List of changed files for prioritization',
+            },
+            sessionId: {
+                type: 'string',
+                description: 'Session ID for caching',
+            },
+            packagePath: {
+                type: 'string',
+                description: 'Base package path',
+            },
+            fileSizes: {
+                type: 'object',
+                description: 'Pre-computed file sizes (filename -> lines)',
+            },
+            streaming: {
+                type: 'boolean',
+                description: 'Enable streaming mode for large datasets',
+                default: false,
+            },
+        },
+        required: ['action', 'files'],
+    },
+    async execute(args) {
+        return runTool('batch_processor', args.action, async () => {
+            const { action, files, targetSize = 3000, changedFiles = [], sessionId, packagePath, fileSizes = {}, streaming = false, } = args;
+            if (sessionId) {
+                const sessionValidation = validateSessionId(sessionId);
+                if (!sessionValidation.isValid) {
+                    throw new Error(`Invalid session ID: ${sessionValidation.error}`);
+                }
+            }
+            switch (action) {
+                case 'estimateSize': {
+                    const estimates = {};
+                    for (const file of files) {
+                        let size = fileSizes[file];
+                        if (!size) {
+                            const cacheKey = `${packagePath ?? ''}:${file}`;
+                            const cached = sizeCache.get(cacheKey);
+                            if (cached !== undefined) {
+                                size = cached;
+                            }
+                            else {
+                                const ext = file.split('.').pop()?.toLowerCase() ?? '';
+                                const filename = file.split('/').pop() ?? '';
+                                let estimate = 100;
+                                switch (ext) {
+                                    case 'ts':
+                                    case 'tsx':
+                                        estimate = filename.includes('test') ? 150 : 200;
+                                        break;
+                                    case 'js':
+                                    case 'jsx':
+                                        estimate = filename.includes('test') ? 120 : 180;
+                                        break;
+                                    case 'json':
+                                        estimate = filename === 'package.json' ? 50 : 30;
+                                        break;
+                                    case 'md':
+                                        estimate = 80;
+                                        break;
+                                    case 'css':
+                                    case 'scss':
+                                        estimate = 120;
+                                        break;
+                                    default:
+                                        estimate = 100;
+                                }
+                                if (filename.includes('config'))
+                                    estimate *= 0.7;
+                                if (filename.includes('types') || filename.includes('.d.'))
+                                    estimate *= 0.8;
+                                if (filename.includes('utils') || filename.includes('helpers'))
+                                    estimate *= 1.2;
+                                if (filename.includes('component'))
+                                    estimate *= 1.3;
+                                if (file.includes('node_modules'))
+                                    estimate *= 0.5;
+                                size = Math.round(estimate);
+                                sizeCache.set(cacheKey, size);
+                            }
+                        }
+                        estimates[file] = size;
+                    }
+                    const result = {
+                        estimates,
+                        totalEstimatedLines: Object.values(estimates).reduce((sum, size) => sum + size, 0),
+                        averageSize: Object.values(estimates).reduce((sum, size) => sum + size, 0) / files.length,
+                        cacheHits: Object.keys(estimates).filter(f => {
+                            const cacheKey = `${packagePath ?? ''}:${f}`;
+                            return sizeCache.has(cacheKey);
+                        }).length,
+                    };
+                    return ok(result);
+                }
+                case 'prioritizeFiles': {
+                    if (changedFiles.length === 0) {
+                        return ok({
+                            priorityFiles: [],
+                            normalFiles: files,
+                            totalFiles: files.length,
+                        });
+                    }
+                    const changedSet = new Set(changedFiles);
+                    const priorityFiles = [];
+                    const normalFiles = [];
+                    for (const file of files) {
+                        if (changedSet.has(file)) {
+                            priorityFiles.push(file);
+                        }
+                        else {
+                            normalFiles.push(file);
+                        }
+                    }
+                    const result = {
+                        priorityFiles,
+                        normalFiles,
+                        totalFiles: files.length,
+                        priorityCount: priorityFiles.length,
+                        normalCount: normalFiles.length,
+                        prioritizedOrder: [...priorityFiles, ...normalFiles],
+                    };
+                    return ok(result);
+                }
+                case 'filterCached': {
+                    if (!sessionId) {
+                        return ok({
+                            toAnalyze: files,
+                            cachedFiles: [],
+                            totalFiles: files.length,
+                            cacheHitRate: 0,
+                        });
+                    }
+                    const cachedFiles = [];
+                    const toAnalyze = [];
+                    for (const file of files) {
+                        safeThrowIfAborted(args.signal);
+                        const _cacheKey = `${sessionId}:${file}`;
+                        const shouldCache = !file.includes('test') && !file.includes('spec') && file.length < 50;
+                        if (shouldCache) {
+                            cachedFiles.push(file);
+                        }
+                        else {
+                            toAnalyze.push(file);
+                        }
+                    }
+                    const cacheHitRate = Math.round((cachedFiles.length / files.length) * 100);
+                    const result = {
+                        toAnalyze,
+                        cachedFiles,
+                        totalFiles: files.length,
+                        cacheHitRate,
+                        sessionId,
+                    };
+                    return ok(result);
+                }
+                case 'createBatches': {
+                    const estimates = await estimateSize(files, fileSizes ?? {}, packagePath);
+                    const filesWithSizes = files.map(file => ({
+                        file,
+                        lines: estimates[file] ?? 100,
+                    }));
+                    filesWithSizes.sort((a, b) => b.lines - a.lines);
+                    const batches = [];
+                    let currentBatch = [];
+                    let currentSize = 0;
+                    for (const { file, lines } of filesWithSizes) {
+                        if (currentSize + lines > targetSize && currentBatch?.length) {
+                            batches.push([...currentBatch]);
+                            currentBatch = [file];
+                            currentSize = lines;
+                        }
+                        else {
+                            currentBatch.push(file);
+                            currentSize += lines;
+                        }
+                    }
+                    if (currentBatch?.length) {
+                        batches.push(currentBatch);
+                    }
+                    const result = {
+                        batches,
+                        batchCount: batches.length,
+                        targetSize,
+                        actualSizes: batches.map(batch => batch.reduce((sum, file) => sum + (estimates[file] ?? 100), 0)),
+                        totalFiles: files.length,
+                        averageBatchSize: batches?.length
+                            ? Math.round(batches.reduce((sum, batch) => sum + batch.length, 0) / batches.length)
+                            : 0,
+                    };
+                    return ok(result);
+                }
+                case 'streamProcess': {
+                    const { signal } = args;
+                    const results = [];
+                    let processedCount = 0;
+                    let totalMemoryUsed = 0;
+                    try {
+                        for await (const result of streamFileEstimation(files, fileSizes ?? {}, packagePath, signal)) {
+                            results.push(result);
+                            processedCount++;
+                            totalMemoryUsed += result.batchInfo?.estimatedMemoryUsage ?? 0;
+                            if (streaming && processedCount % 1000 === 0) {
+                                await new Promise(resolve => queueMicrotask(() => resolve(undefined)));
+                            }
+                        }
+                        const totalSize = results.reduce((sum, r) => sum + r.size, 0);
+                        const avgSize = totalSize / results.length;
+                        const memoryEfficiency = totalMemoryUsed > 0 ? (totalSize / totalMemoryUsed) * 100 : 100;
+                        return ok({
+                            streaming: true,
+                            processed: results.length,
+                            totalEstimatedLines: totalSize,
+                            averageSize: avgSize,
+                            completed: true,
+                            performance: {
+                                memoryUsed: totalMemoryUsed,
+                                memoryEfficiency: Math.round(memoryEfficiency),
+                                dynamicBatching: results.some(r => r.batchInfo?.memoryOptimized),
+                                avgChunkSize: results.length > 0
+                                    ? results.reduce((sum, r) => sum + (r.batchInfo?.chunkSize || 50), 0) /
+                                        results.length
+                                    : 50,
+                            },
+                            results: streaming ? results.slice(-5) : results,
+                        });
+                    }
+                    catch (error) {
+                        if (error instanceof Error && (error.name === 'AbortError' || signal?.aborted)) {
+                            return ok({
+                                streaming: true,
+                                processed: results.length,
+                                aborted: true,
+                                partialResults: results,
+                                partialStats: {
+                                    totalProcessed: processedCount,
+                                    partialSize: results.reduce((sum, r) => sum + r.size, 0),
+                                    memoryUsedBeforeAbort: totalMemoryUsed,
+                                },
+                            });
+                        }
+                        throw error;
+                    }
+                }
+                case 'streamBatches': {
+                    const estimates = await estimateSize(files, fileSizes ?? {}, packagePath);
+                    const batches = [];
+                    let currentBatch = [];
+                    let currentSize = 0;
+                    let batchIndex = 0;
+                    for (const file of files) {
+                        const fileSize = estimates[file] ?? 100;
+                        if (currentSize + fileSize > targetSize && currentBatch.length > 0) {
+                            let clonedFiles;
+                            let cloneMethod = 'spread';
+                            if (isStructuredCloneAvailable()) {
+                                const cloneResult = enhancedClone(currentBatch, {
+                                    fallbackToJson: false,
+                                    trackPerformance: true,
+                                    performanceMarker: `batch-clone-${batchIndex}`,
+                                });
+                                clonedFiles = cloneResult.data;
+                                cloneMethod = cloneResult.method;
+                            }
+                            else {
+                                clonedFiles = [...currentBatch];
+                            }
+                            batches.push({
+                                files: clonedFiles,
+                                estimatedSize: currentSize,
+                                batchIndex: batchIndex++,
+                                cloneMethod,
+                                batchOptimizations: {
+                                    usedStructuredClone: cloneMethod === 'structuredClone',
+                                    fileCount: currentBatch.length,
+                                    avgFileSize: currentBatch.length > 0 ? currentSize / currentBatch.length : 0,
+                                },
+                            });
+                            currentBatch = [file];
+                            currentSize = fileSize;
+                        }
+                        else {
+                            currentBatch.push(file);
+                            currentSize += fileSize;
+                        }
+                        if (batches.length % 50 === 0) {
+                            await new Promise(resolve => setImmediate(resolve));
+                        }
+                    }
+                    if (currentBatch.length > 0) {
+                        batches.push({
+                            files: isStructuredCloneAvailable()
+                                ? enhancedClone(currentBatch, { fallbackToJson: false }).data
+                                : [...currentBatch],
+                            estimatedSize: currentSize,
+                            batchIndex,
+                        });
+                    }
+                    return ok({
+                        streaming: true,
+                        totalBatches: batches.length,
+                        totalFiles: files.length,
+                        targetSize,
+                        batches,
+                        efficiency: {
+                            averageBatchSize: batches.reduce((sum, b) => sum + b.estimatedSize, 0) / batches.length,
+                            utilizationRate: batches.reduce((sum, b) => sum + b.estimatedSize, 0) /
+                                (batches.length * targetSize),
+                        },
+                    });
+                }
+                case 'queuedProcess': {
+                    if (!files || files.length === 0) {
+                        return err('No files provided for queued processing', 'queuedProcess', 'Validation');
+                    }
+                    const results = [];
+                    const startTime = performance.now();
+                    try {
+                        const queuedResults = await ioQueue.addAll(files.map(file => () => {
+                            safeThrowIfAborted(args.signal);
+                            return new Promise((resolve, reject) => {
+                                setImmediate(() => {
+                                    try {
+                                        const fileSize = fileSizes?.[file] || file.length * 10 + 50;
+                                        const processingTime = Math.min(file.length / 10, 15) + 1;
+                                        resolve({
+                                            file,
+                                            size: fileSize,
+                                            processingTime,
+                                            queuedAt: Date.now(),
+                                            concurrency: ioQueue.concurrency,
+                                        });
+                                    }
+                                    catch (error) {
+                                        reject(error);
+                                    }
+                                });
+                            });
+                        }));
+                        results.push(...queuedResults);
+                        const totalTime = performance.now() - startTime;
+                        const avgSize = results.length > 0 ? results.reduce((sum, r) => sum + r.size, 0) / results.length : 0;
+                        return ok({
+                            processed: results.length,
+                            totalTime: Math.round(totalTime),
+                            averageSize: Math.round(avgSize),
+                            concurrency: ioQueue.concurrency,
+                            queueStatus: {
+                                pending: ioQueue.pending,
+                                size: ioQueue.size,
+                            },
+                            results: streaming ? results.slice(-5) : results,
+                        });
+                    }
+                    catch (error) {
+                        if (error instanceof Error && (error.name === 'AbortError' || args.signal?.aborted)) {
+                            return ok({
+                                aborted: true,
+                                processed: results.length,
+                                partialResults: results,
+                            });
+                        }
+                        throw error;
+                    }
+                }
+                case 'queueStatus': {
+                    return ok({
+                        concurrency: ioQueue.concurrency,
+                        pendingOperations: ioQueue.pending,
+                        queueSize: ioQueue.size,
+                        isPaused: ioQueue.isPaused,
+                        memoryUsage: process.memoryUsage(),
+                        uptime: process.uptime(),
+                    }, {
+                        concurrency: ioQueue.concurrency,
+                        pending: ioQueue.pending,
+                        size: ioQueue.size,
+                        isPaused: ioQueue.isPaused,
+                    });
+                }
+                case 'configureConcurrency': {
+                    const { concurrency } = args;
+                    if (concurrency && typeof concurrency === 'number') {
+                        ioQueue.concurrency = concurrency;
+                    }
+                    return ok({
+                        configured: true,
+                        concurrency: ioQueue.concurrency,
+                        message: 'Concurrency configuration updated',
+                    }, {
+                        configured: true,
+                        concurrency: ioQueue.concurrency,
+                    });
+                }
+                default:
+                    ErrorPatterns.unknownAction(action, [
+                        'createBatches',
+                        'prioritizeFiles',
+                        'filterCached',
+                        'estimateSize',
+                        'streamProcess',
+                        'streamBatches',
+                        'queueStatus',
+                        'configureConcurrency',
+                    ]);
+            }
+            throw new Error('Unreachable code');
+        });
+    },
+};
+//# sourceMappingURL=batch-processor-tool.js.map