@danielsimonjr/memoryjs 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +385 -113
- package/README.md.backup-1768084780988 +266 -0
- package/dist/index.cjs +24156 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +16967 -0
- package/dist/index.d.ts +16963 -11
- package/dist/index.js +23887 -19
- package/dist/index.js.map +1 -1
- package/dist/workers/levenshteinWorker.cjs +102 -0
- package/dist/workers/levenshteinWorker.cjs.map +1 -0
- package/dist/workers/levenshteinWorker.js +57 -91
- package/dist/workers/levenshteinWorker.js.map +1 -1
- package/package.json +75 -69
- package/dist/core/EntityManager.d.ts +0 -268
- package/dist/core/EntityManager.d.ts.map +0 -1
- package/dist/core/EntityManager.js +0 -512
- package/dist/core/EntityManager.js.map +0 -1
- package/dist/core/GraphEventEmitter.d.ts +0 -202
- package/dist/core/GraphEventEmitter.d.ts.map +0 -1
- package/dist/core/GraphEventEmitter.js +0 -347
- package/dist/core/GraphEventEmitter.js.map +0 -1
- package/dist/core/GraphStorage.d.ts +0 -395
- package/dist/core/GraphStorage.d.ts.map +0 -1
- package/dist/core/GraphStorage.js +0 -786
- package/dist/core/GraphStorage.js.map +0 -1
- package/dist/core/GraphTraversal.d.ts +0 -141
- package/dist/core/GraphTraversal.d.ts.map +0 -1
- package/dist/core/GraphTraversal.js +0 -574
- package/dist/core/GraphTraversal.js.map +0 -1
- package/dist/core/HierarchyManager.d.ts +0 -111
- package/dist/core/HierarchyManager.d.ts.map +0 -1
- package/dist/core/HierarchyManager.js +0 -225
- package/dist/core/HierarchyManager.js.map +0 -1
- package/dist/core/ManagerContext.d.ts +0 -76
- package/dist/core/ManagerContext.d.ts.map +0 -1
- package/dist/core/ManagerContext.js +0 -129
- package/dist/core/ManagerContext.js.map +0 -1
- package/dist/core/ObservationManager.d.ts +0 -85
- package/dist/core/ObservationManager.d.ts.map +0 -1
- package/dist/core/ObservationManager.js +0 -124
- package/dist/core/ObservationManager.js.map +0 -1
- package/dist/core/RelationManager.d.ts +0 -131
- package/dist/core/RelationManager.d.ts.map +0 -1
- package/dist/core/RelationManager.js +0 -212
- package/dist/core/RelationManager.js.map +0 -1
- package/dist/core/SQLiteStorage.d.ts +0 -354
- package/dist/core/SQLiteStorage.d.ts.map +0 -1
- package/dist/core/SQLiteStorage.js +0 -919
- package/dist/core/SQLiteStorage.js.map +0 -1
- package/dist/core/StorageFactory.d.ts +0 -45
- package/dist/core/StorageFactory.d.ts.map +0 -1
- package/dist/core/StorageFactory.js +0 -65
- package/dist/core/StorageFactory.js.map +0 -1
- package/dist/core/TransactionManager.d.ts +0 -464
- package/dist/core/TransactionManager.d.ts.map +0 -1
- package/dist/core/TransactionManager.js +0 -869
- package/dist/core/TransactionManager.js.map +0 -1
- package/dist/core/index.d.ts +0 -17
- package/dist/core/index.d.ts.map +0 -1
- package/dist/core/index.js +0 -20
- package/dist/core/index.js.map +0 -1
- package/dist/features/AnalyticsManager.d.ts +0 -44
- package/dist/features/AnalyticsManager.d.ts.map +0 -1
- package/dist/features/AnalyticsManager.js +0 -224
- package/dist/features/AnalyticsManager.js.map +0 -1
- package/dist/features/ArchiveManager.d.ts +0 -133
- package/dist/features/ArchiveManager.d.ts.map +0 -1
- package/dist/features/ArchiveManager.js +0 -282
- package/dist/features/ArchiveManager.js.map +0 -1
- package/dist/features/CompressionManager.d.ts +0 -119
- package/dist/features/CompressionManager.d.ts.map +0 -1
- package/dist/features/CompressionManager.js +0 -470
- package/dist/features/CompressionManager.js.map +0 -1
- package/dist/features/IOManager.d.ts +0 -225
- package/dist/features/IOManager.d.ts.map +0 -1
- package/dist/features/IOManager.js +0 -1093
- package/dist/features/IOManager.js.map +0 -1
- package/dist/features/KeywordExtractor.d.ts +0 -61
- package/dist/features/KeywordExtractor.d.ts.map +0 -1
- package/dist/features/KeywordExtractor.js +0 -127
- package/dist/features/KeywordExtractor.js.map +0 -1
- package/dist/features/ObservationNormalizer.d.ts +0 -90
- package/dist/features/ObservationNormalizer.d.ts.map +0 -1
- package/dist/features/ObservationNormalizer.js +0 -194
- package/dist/features/ObservationNormalizer.js.map +0 -1
- package/dist/features/StreamingExporter.d.ts +0 -128
- package/dist/features/StreamingExporter.d.ts.map +0 -1
- package/dist/features/StreamingExporter.js +0 -212
- package/dist/features/StreamingExporter.js.map +0 -1
- package/dist/features/TagManager.d.ts +0 -147
- package/dist/features/TagManager.d.ts.map +0 -1
- package/dist/features/TagManager.js +0 -211
- package/dist/features/TagManager.js.map +0 -1
- package/dist/features/index.d.ts +0 -14
- package/dist/features/index.d.ts.map +0 -1
- package/dist/features/index.js +0 -15
- package/dist/features/index.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/search/BM25Search.d.ts +0 -148
- package/dist/search/BM25Search.d.ts.map +0 -1
- package/dist/search/BM25Search.js +0 -340
- package/dist/search/BM25Search.js.map +0 -1
- package/dist/search/BasicSearch.d.ts +0 -51
- package/dist/search/BasicSearch.d.ts.map +0 -1
- package/dist/search/BasicSearch.js +0 -138
- package/dist/search/BasicSearch.js.map +0 -1
- package/dist/search/BooleanSearch.d.ts +0 -98
- package/dist/search/BooleanSearch.d.ts.map +0 -1
- package/dist/search/BooleanSearch.js +0 -431
- package/dist/search/BooleanSearch.js.map +0 -1
- package/dist/search/EarlyTerminationManager.d.ts +0 -140
- package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
- package/dist/search/EarlyTerminationManager.js +0 -280
- package/dist/search/EarlyTerminationManager.js.map +0 -1
- package/dist/search/EmbeddingCache.d.ts +0 -175
- package/dist/search/EmbeddingCache.d.ts.map +0 -1
- package/dist/search/EmbeddingCache.js +0 -247
- package/dist/search/EmbeddingCache.js.map +0 -1
- package/dist/search/EmbeddingService.d.ts +0 -277
- package/dist/search/EmbeddingService.d.ts.map +0 -1
- package/dist/search/EmbeddingService.js +0 -531
- package/dist/search/EmbeddingService.js.map +0 -1
- package/dist/search/FuzzySearch.d.ts +0 -118
- package/dist/search/FuzzySearch.d.ts.map +0 -1
- package/dist/search/FuzzySearch.js +0 -313
- package/dist/search/FuzzySearch.js.map +0 -1
- package/dist/search/HybridScorer.d.ts +0 -181
- package/dist/search/HybridScorer.d.ts.map +0 -1
- package/dist/search/HybridScorer.js +0 -258
- package/dist/search/HybridScorer.js.map +0 -1
- package/dist/search/HybridSearchManager.d.ts +0 -80
- package/dist/search/HybridSearchManager.d.ts.map +0 -1
- package/dist/search/HybridSearchManager.js +0 -188
- package/dist/search/HybridSearchManager.js.map +0 -1
- package/dist/search/IncrementalIndexer.d.ts +0 -201
- package/dist/search/IncrementalIndexer.d.ts.map +0 -1
- package/dist/search/IncrementalIndexer.js +0 -343
- package/dist/search/IncrementalIndexer.js.map +0 -1
- package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
- package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
- package/dist/search/OptimizedInvertedIndex.js +0 -359
- package/dist/search/OptimizedInvertedIndex.js.map +0 -1
- package/dist/search/ParallelSearchExecutor.d.ts +0 -172
- package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
- package/dist/search/ParallelSearchExecutor.js +0 -310
- package/dist/search/ParallelSearchExecutor.js.map +0 -1
- package/dist/search/QuantizedVectorStore.d.ts +0 -171
- package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
- package/dist/search/QuantizedVectorStore.js +0 -308
- package/dist/search/QuantizedVectorStore.js.map +0 -1
- package/dist/search/QueryAnalyzer.d.ts +0 -76
- package/dist/search/QueryAnalyzer.d.ts.map +0 -1
- package/dist/search/QueryAnalyzer.js +0 -228
- package/dist/search/QueryAnalyzer.js.map +0 -1
- package/dist/search/QueryCostEstimator.d.ts +0 -244
- package/dist/search/QueryCostEstimator.d.ts.map +0 -1
- package/dist/search/QueryCostEstimator.js +0 -653
- package/dist/search/QueryCostEstimator.js.map +0 -1
- package/dist/search/QueryPlanCache.d.ts +0 -220
- package/dist/search/QueryPlanCache.d.ts.map +0 -1
- package/dist/search/QueryPlanCache.js +0 -380
- package/dist/search/QueryPlanCache.js.map +0 -1
- package/dist/search/QueryPlanner.d.ts +0 -58
- package/dist/search/QueryPlanner.d.ts.map +0 -1
- package/dist/search/QueryPlanner.js +0 -138
- package/dist/search/QueryPlanner.js.map +0 -1
- package/dist/search/RankedSearch.d.ts +0 -71
- package/dist/search/RankedSearch.d.ts.map +0 -1
- package/dist/search/RankedSearch.js +0 -239
- package/dist/search/RankedSearch.js.map +0 -1
- package/dist/search/ReflectionManager.d.ts +0 -120
- package/dist/search/ReflectionManager.d.ts.map +0 -1
- package/dist/search/ReflectionManager.js +0 -232
- package/dist/search/ReflectionManager.js.map +0 -1
- package/dist/search/SavedSearchManager.d.ts +0 -79
- package/dist/search/SavedSearchManager.d.ts.map +0 -1
- package/dist/search/SavedSearchManager.js +0 -147
- package/dist/search/SavedSearchManager.js.map +0 -1
- package/dist/search/SearchFilterChain.d.ts +0 -120
- package/dist/search/SearchFilterChain.d.ts.map +0 -1
- package/dist/search/SearchFilterChain.js +0 -186
- package/dist/search/SearchFilterChain.js.map +0 -1
- package/dist/search/SearchManager.d.ts +0 -326
- package/dist/search/SearchManager.d.ts.map +0 -1
- package/dist/search/SearchManager.js +0 -454
- package/dist/search/SearchManager.js.map +0 -1
- package/dist/search/SearchSuggestions.d.ts +0 -27
- package/dist/search/SearchSuggestions.d.ts.map +0 -1
- package/dist/search/SearchSuggestions.js +0 -58
- package/dist/search/SearchSuggestions.js.map +0 -1
- package/dist/search/SemanticSearch.d.ts +0 -149
- package/dist/search/SemanticSearch.d.ts.map +0 -1
- package/dist/search/SemanticSearch.js +0 -324
- package/dist/search/SemanticSearch.js.map +0 -1
- package/dist/search/SymbolicSearch.d.ts +0 -61
- package/dist/search/SymbolicSearch.d.ts.map +0 -1
- package/dist/search/SymbolicSearch.js +0 -164
- package/dist/search/SymbolicSearch.js.map +0 -1
- package/dist/search/TFIDFEventSync.d.ts +0 -85
- package/dist/search/TFIDFEventSync.d.ts.map +0 -1
- package/dist/search/TFIDFEventSync.js +0 -134
- package/dist/search/TFIDFEventSync.js.map +0 -1
- package/dist/search/TFIDFIndexManager.d.ts +0 -151
- package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
- package/dist/search/TFIDFIndexManager.js +0 -433
- package/dist/search/TFIDFIndexManager.js.map +0 -1
- package/dist/search/VectorStore.d.ts +0 -235
- package/dist/search/VectorStore.d.ts.map +0 -1
- package/dist/search/VectorStore.js +0 -312
- package/dist/search/VectorStore.js.map +0 -1
- package/dist/search/index.d.ts +0 -35
- package/dist/search/index.d.ts.map +0 -1
- package/dist/search/index.js +0 -53
- package/dist/search/index.js.map +0 -1
- package/dist/types/index.d.ts +0 -13
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -13
- package/dist/types/index.js.map +0 -1
- package/dist/types/types.d.ts +0 -1811
- package/dist/types/types.d.ts.map +0 -1
- package/dist/types/types.js +0 -10
- package/dist/types/types.js.map +0 -1
- package/dist/utils/BatchProcessor.d.ts +0 -271
- package/dist/utils/BatchProcessor.d.ts.map +0 -1
- package/dist/utils/BatchProcessor.js +0 -377
- package/dist/utils/BatchProcessor.js.map +0 -1
- package/dist/utils/MemoryMonitor.d.ts +0 -176
- package/dist/utils/MemoryMonitor.d.ts.map +0 -1
- package/dist/utils/MemoryMonitor.js +0 -306
- package/dist/utils/MemoryMonitor.js.map +0 -1
- package/dist/utils/WorkerPoolManager.d.ts +0 -233
- package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
- package/dist/utils/WorkerPoolManager.js +0 -421
- package/dist/utils/WorkerPoolManager.js.map +0 -1
- package/dist/utils/compressedCache.d.ts +0 -221
- package/dist/utils/compressedCache.d.ts.map +0 -1
- package/dist/utils/compressedCache.js +0 -349
- package/dist/utils/compressedCache.js.map +0 -1
- package/dist/utils/compressionUtil.d.ts +0 -214
- package/dist/utils/compressionUtil.d.ts.map +0 -1
- package/dist/utils/compressionUtil.js +0 -248
- package/dist/utils/compressionUtil.js.map +0 -1
- package/dist/utils/constants.d.ts +0 -245
- package/dist/utils/constants.d.ts.map +0 -1
- package/dist/utils/constants.js +0 -253
- package/dist/utils/constants.js.map +0 -1
- package/dist/utils/entityUtils.d.ts +0 -379
- package/dist/utils/entityUtils.d.ts.map +0 -1
- package/dist/utils/entityUtils.js +0 -649
- package/dist/utils/entityUtils.js.map +0 -1
- package/dist/utils/errors.d.ts +0 -95
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -146
- package/dist/utils/errors.js.map +0 -1
- package/dist/utils/formatters.d.ts +0 -145
- package/dist/utils/formatters.d.ts.map +0 -1
- package/dist/utils/formatters.js +0 -133
- package/dist/utils/formatters.js.map +0 -1
- package/dist/utils/index.d.ts +0 -26
- package/dist/utils/index.d.ts.map +0 -1
- package/dist/utils/index.js +0 -88
- package/dist/utils/index.js.map +0 -1
- package/dist/utils/indexes.d.ts +0 -270
- package/dist/utils/indexes.d.ts.map +0 -1
- package/dist/utils/indexes.js +0 -527
- package/dist/utils/indexes.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -31
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -41
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/operationUtils.d.ts +0 -124
- package/dist/utils/operationUtils.d.ts.map +0 -1
- package/dist/utils/operationUtils.js +0 -176
- package/dist/utils/operationUtils.js.map +0 -1
- package/dist/utils/parallelUtils.d.ts +0 -76
- package/dist/utils/parallelUtils.d.ts.map +0 -1
- package/dist/utils/parallelUtils.js +0 -192
- package/dist/utils/parallelUtils.js.map +0 -1
- package/dist/utils/schemas.d.ts +0 -556
- package/dist/utils/schemas.d.ts.map +0 -1
- package/dist/utils/schemas.js +0 -485
- package/dist/utils/schemas.js.map +0 -1
- package/dist/utils/searchAlgorithms.d.ts +0 -99
- package/dist/utils/searchAlgorithms.d.ts.map +0 -1
- package/dist/utils/searchAlgorithms.js +0 -168
- package/dist/utils/searchAlgorithms.js.map +0 -1
- package/dist/utils/searchCache.d.ts +0 -108
- package/dist/utils/searchCache.d.ts.map +0 -1
- package/dist/utils/searchCache.js +0 -210
- package/dist/utils/searchCache.js.map +0 -1
- package/dist/utils/taskScheduler.d.ts +0 -294
- package/dist/utils/taskScheduler.d.ts.map +0 -1
- package/dist/utils/taskScheduler.js +0 -487
- package/dist/utils/taskScheduler.js.map +0 -1
- package/dist/workers/index.d.ts +0 -12
- package/dist/workers/index.d.ts.map +0 -1
- package/dist/workers/index.js +0 -10
- package/dist/workers/index.js.map +0 -1
- package/dist/workers/levenshteinWorker.d.ts +0 -60
- package/dist/workers/levenshteinWorker.d.ts.map +0 -1
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Operation Utilities
|
|
3
|
-
*
|
|
4
|
-
* Phase 9B: Utilities for long-running operations with progress tracking
|
|
5
|
-
* and cancellation support.
|
|
6
|
-
*
|
|
7
|
-
* @module utils/operationUtils
|
|
8
|
-
*/
|
|
9
|
-
import type { ProgressCallback } from './taskScheduler.js';
|
|
10
|
-
/**
|
|
11
|
-
* Check if an operation has been cancelled via AbortSignal.
|
|
12
|
-
* Throws OperationCancelledError if the signal is aborted.
|
|
13
|
-
*
|
|
14
|
-
* @param signal - Optional AbortSignal to check
|
|
15
|
-
* @param operation - Optional operation name for error message
|
|
16
|
-
* @throws OperationCancelledError if signal is aborted
|
|
17
|
-
*
|
|
18
|
-
* @example
|
|
19
|
-
* ```typescript
|
|
20
|
-
* for (const item of items) {
|
|
21
|
-
* checkCancellation(options?.signal, 'batch processing');
|
|
22
|
-
* await processItem(item);
|
|
23
|
-
* }
|
|
24
|
-
* ```
|
|
25
|
-
*/
|
|
26
|
-
export declare function checkCancellation(signal?: AbortSignal, operation?: string): void;
|
|
27
|
-
/**
|
|
28
|
-
* Create a throttled progress reporter to avoid excessive callback invocations.
|
|
29
|
-
* Returns undefined if no callback is provided.
|
|
30
|
-
*
|
|
31
|
-
* @param callback - Optional progress callback to throttle
|
|
32
|
-
* @param throttleMs - Minimum time between callbacks (default: 100ms)
|
|
33
|
-
* @returns Throttled callback or undefined
|
|
34
|
-
*
|
|
35
|
-
* @example
|
|
36
|
-
* ```typescript
|
|
37
|
-
* const reportProgress = createProgressReporter(options?.onProgress, 50);
|
|
38
|
-
* for (let i = 0; i < total; i++) {
|
|
39
|
-
* reportProgress?.({ completed: i, total, percentage: (i / total) * 100 });
|
|
40
|
-
* }
|
|
41
|
-
* ```
|
|
42
|
-
*/
|
|
43
|
-
export declare function createProgressReporter(callback?: ProgressCallback, throttleMs?: number): ProgressCallback | undefined;
|
|
44
|
-
/**
|
|
45
|
-
* Create a progress object for reporting.
|
|
46
|
-
*
|
|
47
|
-
* @param completed - Number of completed items
|
|
48
|
-
* @param total - Total number of items
|
|
49
|
-
* @param currentTaskId - Optional current task identifier
|
|
50
|
-
* @returns Progress object suitable for ProgressCallback
|
|
51
|
-
*
|
|
52
|
-
* @example
|
|
53
|
-
* ```typescript
|
|
54
|
-
* reportProgress?.(createProgress(50, 100, 'processing entities'));
|
|
55
|
-
* // { completed: 50, total: 100, percentage: 50, currentTaskId: 'processing entities' }
|
|
56
|
-
* ```
|
|
57
|
-
*/
|
|
58
|
-
export declare function createProgress(completed: number, total: number, currentTaskId?: string): {
|
|
59
|
-
completed: number;
|
|
60
|
-
total: number;
|
|
61
|
-
percentage: number;
|
|
62
|
-
currentTaskId?: string;
|
|
63
|
-
};
|
|
64
|
-
/**
|
|
65
|
-
* Phase definition for executeWithPhases.
|
|
66
|
-
*/
|
|
67
|
-
export interface PhaseDefinition<T> {
|
|
68
|
-
/** Phase name (used for progress reporting and cancellation error messages) */
|
|
69
|
-
name: string;
|
|
70
|
-
/** Weight of this phase relative to others (higher = more of total progress) */
|
|
71
|
-
weight: number;
|
|
72
|
-
/** Executor function that performs the phase work */
|
|
73
|
-
execute: (phaseProgress: (pct: number) => void) => Promise<T>;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Execute an operation with multiple distinct phases.
|
|
77
|
-
* Useful when an operation has multiple distinct phases with different weights.
|
|
78
|
-
*
|
|
79
|
-
* @param phases - Array of phase definitions with weight and executor
|
|
80
|
-
* @param onProgress - Optional progress callback
|
|
81
|
-
* @param signal - Optional abort signal
|
|
82
|
-
* @returns Array of results from each phase
|
|
83
|
-
* @throws OperationCancelledError if cancelled during any phase
|
|
84
|
-
*
|
|
85
|
-
* @example
|
|
86
|
-
* ```typescript
|
|
87
|
-
* const [parseResult, processResult, saveResult] = await executeWithPhases([
|
|
88
|
-
* { name: 'parsing', weight: 20, execute: () => parseData() },
|
|
89
|
-
* { name: 'processing', weight: 60, execute: () => processEntities() },
|
|
90
|
-
* { name: 'saving', weight: 20, execute: () => saveResults() },
|
|
91
|
-
* ], options?.onProgress, options?.signal);
|
|
92
|
-
* ```
|
|
93
|
-
*/
|
|
94
|
-
export declare function executeWithPhases<T>(phases: PhaseDefinition<T>[], onProgress?: ProgressCallback, signal?: AbortSignal): Promise<T[]>;
|
|
95
|
-
/**
|
|
96
|
-
* Execute an operation in batches with progress tracking and cancellation support.
|
|
97
|
-
*
|
|
98
|
-
* @param items - Array of items to process
|
|
99
|
-
* @param batchSize - Size of each batch
|
|
100
|
-
* @param processBatch - Function to process each batch
|
|
101
|
-
* @param onProgress - Optional progress callback
|
|
102
|
-
* @param signal - Optional abort signal
|
|
103
|
-
* @param operationName - Optional operation name for cancellation error
|
|
104
|
-
* @returns Array of results from all batches
|
|
105
|
-
*
|
|
106
|
-
* @example
|
|
107
|
-
* ```typescript
|
|
108
|
-
* const results = await processBatchesWithProgress(
|
|
109
|
-
* entities,
|
|
110
|
-
* 100,
|
|
111
|
-
* async (batch) => {
|
|
112
|
-
* for (const entity of batch) {
|
|
113
|
-
* await saveEntity(entity);
|
|
114
|
-
* }
|
|
115
|
-
* return batch.length;
|
|
116
|
-
* },
|
|
117
|
-
* options?.onProgress,
|
|
118
|
-
* options?.signal,
|
|
119
|
-
* 'createEntities'
|
|
120
|
-
* );
|
|
121
|
-
* ```
|
|
122
|
-
*/
|
|
123
|
-
export declare function processBatchesWithProgress<T, R>(items: T[], batchSize: number, processBatch: (batch: T[], batchIndex: number) => Promise<R>, onProgress?: ProgressCallback, signal?: AbortSignal, operationName?: string): Promise<R[]>;
|
|
124
|
-
//# sourceMappingURL=operationUtils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"operationUtils.d.ts","sourceRoot":"","sources":["../../src/utils/operationUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAIhF;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,CAAC,EAAE,gBAAgB,EAC3B,UAAU,GAAE,MAAY,GACvB,gBAAgB,GAAG,SAAS,CAa9B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,aAAa,CAAC,EAAE,MAAM,GACrB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,CAOlF;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,OAAO,EAAE,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;CAC/D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,EAC5B,UAAU,CAAC,EAAE,gBAAgB,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,EAAE,CAAC,CAkCd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,0BAA0B,CAAC,CAAC,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EAC5D,UAAU,CAAC,EAAE,gBAAgB,EAC7B,MAAM,CAAC,EAAE,WAAW,EACpB,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,CAAC,EAAE,CAAC,CAqBd"}
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Operation Utilities
|
|
3
|
-
*
|
|
4
|
-
* Phase 9B: Utilities for long-running operations with progress tracking
|
|
5
|
-
* and cancellation support.
|
|
6
|
-
*
|
|
7
|
-
* @module utils/operationUtils
|
|
8
|
-
*/
|
|
9
|
-
import { OperationCancelledError } from './errors.js';
|
|
10
|
-
/**
|
|
11
|
-
* Check if an operation has been cancelled via AbortSignal.
|
|
12
|
-
* Throws OperationCancelledError if the signal is aborted.
|
|
13
|
-
*
|
|
14
|
-
* @param signal - Optional AbortSignal to check
|
|
15
|
-
* @param operation - Optional operation name for error message
|
|
16
|
-
* @throws OperationCancelledError if signal is aborted
|
|
17
|
-
*
|
|
18
|
-
* @example
|
|
19
|
-
* ```typescript
|
|
20
|
-
* for (const item of items) {
|
|
21
|
-
* checkCancellation(options?.signal, 'batch processing');
|
|
22
|
-
* await processItem(item);
|
|
23
|
-
* }
|
|
24
|
-
* ```
|
|
25
|
-
*/
|
|
26
|
-
export function checkCancellation(signal, operation) {
|
|
27
|
-
if (signal?.aborted) {
|
|
28
|
-
throw new OperationCancelledError(operation);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Create a throttled progress reporter to avoid excessive callback invocations.
|
|
33
|
-
* Returns undefined if no callback is provided.
|
|
34
|
-
*
|
|
35
|
-
* @param callback - Optional progress callback to throttle
|
|
36
|
-
* @param throttleMs - Minimum time between callbacks (default: 100ms)
|
|
37
|
-
* @returns Throttled callback or undefined
|
|
38
|
-
*
|
|
39
|
-
* @example
|
|
40
|
-
* ```typescript
|
|
41
|
-
* const reportProgress = createProgressReporter(options?.onProgress, 50);
|
|
42
|
-
* for (let i = 0; i < total; i++) {
|
|
43
|
-
* reportProgress?.({ completed: i, total, percentage: (i / total) * 100 });
|
|
44
|
-
* }
|
|
45
|
-
* ```
|
|
46
|
-
*/
|
|
47
|
-
export function createProgressReporter(callback, throttleMs = 100) {
|
|
48
|
-
if (!callback)
|
|
49
|
-
return undefined;
|
|
50
|
-
let lastCallTime = 0;
|
|
51
|
-
return (progress) => {
|
|
52
|
-
const now = Date.now();
|
|
53
|
-
// Always report 0% and 100%
|
|
54
|
-
if (progress.percentage === 0 || progress.percentage >= 100 || now - lastCallTime >= throttleMs) {
|
|
55
|
-
lastCallTime = now;
|
|
56
|
-
callback(progress);
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Create a progress object for reporting.
|
|
62
|
-
*
|
|
63
|
-
* @param completed - Number of completed items
|
|
64
|
-
* @param total - Total number of items
|
|
65
|
-
* @param currentTaskId - Optional current task identifier
|
|
66
|
-
* @returns Progress object suitable for ProgressCallback
|
|
67
|
-
*
|
|
68
|
-
* @example
|
|
69
|
-
* ```typescript
|
|
70
|
-
* reportProgress?.(createProgress(50, 100, 'processing entities'));
|
|
71
|
-
* // { completed: 50, total: 100, percentage: 50, currentTaskId: 'processing entities' }
|
|
72
|
-
* ```
|
|
73
|
-
*/
|
|
74
|
-
export function createProgress(completed, total, currentTaskId) {
|
|
75
|
-
return {
|
|
76
|
-
completed,
|
|
77
|
-
total,
|
|
78
|
-
percentage: total > 0 ? Math.round((completed / total) * 100) : 0,
|
|
79
|
-
currentTaskId,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Execute an operation with multiple distinct phases.
|
|
84
|
-
* Useful when an operation has multiple distinct phases with different weights.
|
|
85
|
-
*
|
|
86
|
-
* @param phases - Array of phase definitions with weight and executor
|
|
87
|
-
* @param onProgress - Optional progress callback
|
|
88
|
-
* @param signal - Optional abort signal
|
|
89
|
-
* @returns Array of results from each phase
|
|
90
|
-
* @throws OperationCancelledError if cancelled during any phase
|
|
91
|
-
*
|
|
92
|
-
* @example
|
|
93
|
-
* ```typescript
|
|
94
|
-
* const [parseResult, processResult, saveResult] = await executeWithPhases([
|
|
95
|
-
* { name: 'parsing', weight: 20, execute: () => parseData() },
|
|
96
|
-
* { name: 'processing', weight: 60, execute: () => processEntities() },
|
|
97
|
-
* { name: 'saving', weight: 20, execute: () => saveResults() },
|
|
98
|
-
* ], options?.onProgress, options?.signal);
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
export async function executeWithPhases(phases, onProgress, signal) {
|
|
102
|
-
const totalWeight = phases.reduce((sum, p) => sum + p.weight, 0);
|
|
103
|
-
let completedWeight = 0;
|
|
104
|
-
const results = [];
|
|
105
|
-
for (const phase of phases) {
|
|
106
|
-
checkCancellation(signal, phase.name);
|
|
107
|
-
const phaseStartWeight = completedWeight;
|
|
108
|
-
const phaseProgress = (phasePct) => {
|
|
109
|
-
if (onProgress) {
|
|
110
|
-
const overallPct = ((phaseStartWeight + (phase.weight * phasePct / 100)) / totalWeight) * 100;
|
|
111
|
-
onProgress({
|
|
112
|
-
completed: Math.round(overallPct),
|
|
113
|
-
total: 100,
|
|
114
|
-
percentage: Math.round(overallPct),
|
|
115
|
-
currentTaskId: phase.name,
|
|
116
|
-
});
|
|
117
|
-
}
|
|
118
|
-
};
|
|
119
|
-
const result = await phase.execute(phaseProgress);
|
|
120
|
-
results.push(result);
|
|
121
|
-
completedWeight += phase.weight;
|
|
122
|
-
}
|
|
123
|
-
// Report 100% completion
|
|
124
|
-
onProgress?.({
|
|
125
|
-
completed: 100,
|
|
126
|
-
total: 100,
|
|
127
|
-
percentage: 100,
|
|
128
|
-
});
|
|
129
|
-
return results;
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Execute an operation in batches with progress tracking and cancellation support.
|
|
133
|
-
*
|
|
134
|
-
* @param items - Array of items to process
|
|
135
|
-
* @param batchSize - Size of each batch
|
|
136
|
-
* @param processBatch - Function to process each batch
|
|
137
|
-
* @param onProgress - Optional progress callback
|
|
138
|
-
* @param signal - Optional abort signal
|
|
139
|
-
* @param operationName - Optional operation name for cancellation error
|
|
140
|
-
* @returns Array of results from all batches
|
|
141
|
-
*
|
|
142
|
-
* @example
|
|
143
|
-
* ```typescript
|
|
144
|
-
* const results = await processBatchesWithProgress(
|
|
145
|
-
* entities,
|
|
146
|
-
* 100,
|
|
147
|
-
* async (batch) => {
|
|
148
|
-
* for (const entity of batch) {
|
|
149
|
-
* await saveEntity(entity);
|
|
150
|
-
* }
|
|
151
|
-
* return batch.length;
|
|
152
|
-
* },
|
|
153
|
-
* options?.onProgress,
|
|
154
|
-
* options?.signal,
|
|
155
|
-
* 'createEntities'
|
|
156
|
-
* );
|
|
157
|
-
* ```
|
|
158
|
-
*/
|
|
159
|
-
export async function processBatchesWithProgress(items, batchSize, processBatch, onProgress, signal, operationName) {
|
|
160
|
-
const results = [];
|
|
161
|
-
const total = items.length;
|
|
162
|
-
let processed = 0;
|
|
163
|
-
const reportProgress = createProgressReporter(onProgress);
|
|
164
|
-
reportProgress?.(createProgress(0, total, operationName));
|
|
165
|
-
for (let i = 0; i < items.length; i += batchSize) {
|
|
166
|
-
checkCancellation(signal, operationName);
|
|
167
|
-
const batch = items.slice(i, i + batchSize);
|
|
168
|
-
const result = await processBatch(batch, Math.floor(i / batchSize));
|
|
169
|
-
results.push(result);
|
|
170
|
-
processed += batch.length;
|
|
171
|
-
reportProgress?.(createProgress(processed, total, operationName));
|
|
172
|
-
}
|
|
173
|
-
reportProgress?.(createProgress(total, total, operationName));
|
|
174
|
-
return results;
|
|
175
|
-
}
|
|
176
|
-
//# sourceMappingURL=operationUtils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"operationUtils.js","sourceRoot":"","sources":["../../src/utils/operationUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAGtD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAoB,EAAE,SAAkB;IACxE,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAA2B,EAC3B,aAAqB,GAAG;IAExB,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,4BAA4B;QAC5B,IAAI,QAAQ,CAAC,UAAU,KAAK,CAAC,IAAI,QAAQ,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,GAAG,YAAY,IAAI,UAAU,EAAE,CAAC;YAChG,YAAY,GAAG,GAAG,CAAC;YACnB,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,cAAc,CAC5B,SAAiB,EACjB,KAAa,EACb,aAAsB;IAEtB,OAAO;QACL,SAAS;QACT,KAAK;QACL,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,aAAa;KACd,CAAC;AACJ,CAAC;AAcD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAA4B,EAC5B,UAA6B,EAC7B,MAAoB;IAEpB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,OAAO,GAAQ,EAAE,CAAC;IAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,gBAAgB,GAAG,eAAe,CAAC;QACzC,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAE,EAAE;YACzC,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,UAAU,GAAG,CAAC,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,GAAG,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC;gBAC9F,UAAU,CAAC;oBACT,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;oBACjC,KAAK,EAAE,GAAG;oBACV,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;oBAClC,aAAa,EAAE,KAAK,CAAC,IAAI;iBAC1B,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;IAClC,CAAC;IAED,yBAAyB;IACzB,UAAU,EAAE,CAAC;QACX,SAAS,EAAE,GAAG;QACd,KAAK,EAAE,GAAG;QACV,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,KAAU,EACV,SAAiB,EACjB,YAA4D,EAC5D,UAA6B,EAC7B,MAAoB,EACpB,aAAsB;IAEtB,MAAM,OAAO,GAAQ,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,cAAc,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC1D,cAAc,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,iBAAiB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEzC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;QAC1B,cAAc,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,cAAc,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IAC9D,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Parallel Utilities
|
|
3
|
-
*
|
|
4
|
-
* Utilities for parallel array operations using workerpool.
|
|
5
|
-
* Phase 8 Sprint 3: Parallel array operations for improved performance.
|
|
6
|
-
*
|
|
7
|
-
* **SECURITY WARNING:** These functions use `new Function()` internally for worker serialization.
|
|
8
|
-
* The `fn` parameter MUST be a real function object, never a user-provided string.
|
|
9
|
-
* Runtime validation ensures only function objects are accepted.
|
|
10
|
-
*
|
|
11
|
-
* @module utils/parallelUtils
|
|
12
|
-
*/
|
|
13
|
-
import workerpool from '@danielsimonjr/workerpool';
|
|
14
|
-
/**
|
|
15
|
-
* Shutdown the shared worker pool and clean up resources.
|
|
16
|
-
* Should be called when parallel utilities are no longer needed.
|
|
17
|
-
*/
|
|
18
|
-
export declare function shutdownParallelUtils(): Promise<void>;
|
|
19
|
-
/**
|
|
20
|
-
* Map items in parallel using workerpool.
|
|
21
|
-
*
|
|
22
|
-
* Splits the array into chunks and processes each chunk in a worker thread.
|
|
23
|
-
* Falls back to single-threaded for small arrays (< MIN_PARALLEL_SIZE).
|
|
24
|
-
*
|
|
25
|
-
* **Note:** The mapping function must be serializable (no closures, external variables).
|
|
26
|
-
* Due to ESM/worker thread compatibility issues, this may fall back to single-threaded
|
|
27
|
-
* execution in some environments (e.g., vitest test runner).
|
|
28
|
-
*
|
|
29
|
-
* @template T - Input item type
|
|
30
|
-
* @template R - Output item type
|
|
31
|
-
* @param items - Array of items to map
|
|
32
|
-
* @param fn - Mapping function (must be serializable)
|
|
33
|
-
* @param chunkSize - Optional chunk size (default: DEFAULT_CHUNK_SIZE)
|
|
34
|
-
* @returns Promise resolving to array of mapped results
|
|
35
|
-
*
|
|
36
|
-
* @example
|
|
37
|
-
* ```typescript
|
|
38
|
-
* // Map numbers to their squares
|
|
39
|
-
* const numbers = [1, 2, 3, 4, 5];
|
|
40
|
-
* const squared = await parallelMap(numbers, (n: number) => n * n);
|
|
41
|
-
* // Result: [1, 4, 9, 16, 25]
|
|
42
|
-
* ```
|
|
43
|
-
*/
|
|
44
|
-
export declare function parallelMap<T, R>(items: T[], fn: (item: T) => R, chunkSize?: number): Promise<R[]>;
|
|
45
|
-
/**
|
|
46
|
-
* Filter items in parallel using workerpool.
|
|
47
|
-
*
|
|
48
|
-
* Splits the array into chunks and processes each chunk in a worker thread.
|
|
49
|
-
* Falls back to single-threaded for small arrays (< MIN_PARALLEL_SIZE).
|
|
50
|
-
*
|
|
51
|
-
* **Note:** The predicate function must be serializable (no closures, external variables).
|
|
52
|
-
* Due to ESM/worker thread compatibility issues, this may fall back to single-threaded
|
|
53
|
-
* execution in some environments (e.g., vitest test runner).
|
|
54
|
-
*
|
|
55
|
-
* @template T - Item type
|
|
56
|
-
* @param items - Array of items to filter
|
|
57
|
-
* @param predicate - Filter predicate (must be serializable)
|
|
58
|
-
* @param chunkSize - Optional chunk size (default: DEFAULT_CHUNK_SIZE)
|
|
59
|
-
* @returns Promise resolving to filtered array
|
|
60
|
-
*
|
|
61
|
-
* @example
|
|
62
|
-
* ```typescript
|
|
63
|
-
* // Filter even numbers
|
|
64
|
-
* const numbers = [1, 2, 3, 4, 5, 6];
|
|
65
|
-
* const evens = await parallelFilter(numbers, (n: number) => n % 2 === 0);
|
|
66
|
-
* // Result: [2, 4, 6]
|
|
67
|
-
* ```
|
|
68
|
-
*/
|
|
69
|
-
export declare function parallelFilter<T>(items: T[], predicate: (item: T) => boolean, chunkSize?: number): Promise<T[]>;
|
|
70
|
-
/**
|
|
71
|
-
* Get statistics about the worker pool.
|
|
72
|
-
*
|
|
73
|
-
* @returns Pool statistics or null if pool is not initialized
|
|
74
|
-
*/
|
|
75
|
-
export declare function getPoolStats(): workerpool.PoolStats | null;
|
|
76
|
-
//# sourceMappingURL=parallelUtils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parallelUtils.d.ts","sourceRoot":"","sources":["../../src/utils/parallelUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,UAAU,MAAM,2BAA2B,CAAC;AAmDnD;;;GAGG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAAE,CAAC,EACpC,KAAK,EAAE,CAAC,EAAE,EACV,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,EAClB,SAAS,GAAE,MAA2B,GACrC,OAAO,CAAC,CAAC,EAAE,CAAC,CA2Cd;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,cAAc,CAAC,CAAC,EACpC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,EAC/B,SAAS,GAAE,MAA2B,GACrC,OAAO,CAAC,CAAC,EAAE,CAAC,CA2Cd;AAED;;;;GAIG;AACH,wBAAgB,YAAY,IAAI,UAAU,CAAC,SAAS,GAAG,IAAI,CAK1D"}
|
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Parallel Utilities
|
|
3
|
-
*
|
|
4
|
-
* Utilities for parallel array operations using workerpool.
|
|
5
|
-
* Phase 8 Sprint 3: Parallel array operations for improved performance.
|
|
6
|
-
*
|
|
7
|
-
* **SECURITY WARNING:** These functions use `new Function()` internally for worker serialization.
|
|
8
|
-
* The `fn` parameter MUST be a real function object, never a user-provided string.
|
|
9
|
-
* Runtime validation ensures only function objects are accepted.
|
|
10
|
-
*
|
|
11
|
-
* @module utils/parallelUtils
|
|
12
|
-
*/
|
|
13
|
-
import workerpool from '@danielsimonjr/workerpool';
|
|
14
|
-
/**
|
|
15
|
-
* Validates that the input is a real function object.
|
|
16
|
-
* Prevents code injection through string masquerading as functions.
|
|
17
|
-
*
|
|
18
|
-
* @param fn - Function to validate
|
|
19
|
-
* @param paramName - Parameter name for error message
|
|
20
|
-
* @throws {TypeError} If fn is not a function
|
|
21
|
-
* @internal
|
|
22
|
-
*/
|
|
23
|
-
function validateFunction(fn, paramName) {
|
|
24
|
-
if (typeof fn !== 'function') {
|
|
25
|
-
throw new TypeError(`${paramName} must be a function, got ${typeof fn}`);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Default chunk size for parallel operations.
|
|
30
|
-
* Can be overridden per operation.
|
|
31
|
-
*/
|
|
32
|
-
const DEFAULT_CHUNK_SIZE = 100;
|
|
33
|
-
/**
|
|
34
|
-
* Minimum array size to activate parallel processing.
|
|
35
|
-
* For smaller arrays, single-threaded is more efficient due to worker overhead.
|
|
36
|
-
*/
|
|
37
|
-
const MIN_PARALLEL_SIZE = 200;
|
|
38
|
-
/**
|
|
39
|
-
* Shared worker pool instance for all parallel utilities.
|
|
40
|
-
* Initialized lazily on first use.
|
|
41
|
-
*/
|
|
42
|
-
let sharedPool = null;
|
|
43
|
-
/**
|
|
44
|
-
* Get or create the shared worker pool.
|
|
45
|
-
* Uses inline worker execution (no separate worker file needed).
|
|
46
|
-
*
|
|
47
|
-
* @returns Worker pool instance
|
|
48
|
-
*/
|
|
49
|
-
function getPool() {
|
|
50
|
-
if (!sharedPool) {
|
|
51
|
-
sharedPool = workerpool.pool({
|
|
52
|
-
maxWorkers: Math.max(1, workerpool.cpus - 1),
|
|
53
|
-
workerType: 'thread',
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
return sharedPool;
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Shutdown the shared worker pool and clean up resources.
|
|
60
|
-
* Should be called when parallel utilities are no longer needed.
|
|
61
|
-
*/
|
|
62
|
-
export async function shutdownParallelUtils() {
|
|
63
|
-
if (sharedPool) {
|
|
64
|
-
await sharedPool.terminate();
|
|
65
|
-
sharedPool = null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Map items in parallel using workerpool.
|
|
70
|
-
*
|
|
71
|
-
* Splits the array into chunks and processes each chunk in a worker thread.
|
|
72
|
-
* Falls back to single-threaded for small arrays (< MIN_PARALLEL_SIZE).
|
|
73
|
-
*
|
|
74
|
-
* **Note:** The mapping function must be serializable (no closures, external variables).
|
|
75
|
-
* Due to ESM/worker thread compatibility issues, this may fall back to single-threaded
|
|
76
|
-
* execution in some environments (e.g., vitest test runner).
|
|
77
|
-
*
|
|
78
|
-
* @template T - Input item type
|
|
79
|
-
* @template R - Output item type
|
|
80
|
-
* @param items - Array of items to map
|
|
81
|
-
* @param fn - Mapping function (must be serializable)
|
|
82
|
-
* @param chunkSize - Optional chunk size (default: DEFAULT_CHUNK_SIZE)
|
|
83
|
-
* @returns Promise resolving to array of mapped results
|
|
84
|
-
*
|
|
85
|
-
* @example
|
|
86
|
-
* ```typescript
|
|
87
|
-
* // Map numbers to their squares
|
|
88
|
-
* const numbers = [1, 2, 3, 4, 5];
|
|
89
|
-
* const squared = await parallelMap(numbers, (n: number) => n * n);
|
|
90
|
-
* // Result: [1, 4, 9, 16, 25]
|
|
91
|
-
* ```
|
|
92
|
-
*/
|
|
93
|
-
export async function parallelMap(items, fn, chunkSize = DEFAULT_CHUNK_SIZE) {
|
|
94
|
-
// Security: Validate that fn is a real function, not a user-provided string
|
|
95
|
-
validateFunction(fn, 'fn');
|
|
96
|
-
// Fall back to single-threaded for small arrays
|
|
97
|
-
if (items.length < MIN_PARALLEL_SIZE) {
|
|
98
|
-
return items.map(fn);
|
|
99
|
-
}
|
|
100
|
-
try {
|
|
101
|
-
const pool = getPool();
|
|
102
|
-
// Split items into chunks
|
|
103
|
-
const chunks = [];
|
|
104
|
-
for (let i = 0; i < items.length; i += chunkSize) {
|
|
105
|
-
chunks.push(items.slice(i, i + chunkSize));
|
|
106
|
-
}
|
|
107
|
-
// Convert function to string for serialization
|
|
108
|
-
const fnString = fn.toString();
|
|
109
|
-
// Process chunks in parallel using inline function execution
|
|
110
|
-
const results = await Promise.all(chunks.map(chunk => pool.exec((chunkData, fnStr) => {
|
|
111
|
-
// Reconstruct function from string
|
|
112
|
-
// eslint-disable-next-line no-new-func
|
|
113
|
-
const mapFn = new Function('return ' + fnStr)();
|
|
114
|
-
return chunkData.map(mapFn);
|
|
115
|
-
}, [chunk, fnString])));
|
|
116
|
-
// Flatten results
|
|
117
|
-
return results.flat();
|
|
118
|
-
}
|
|
119
|
-
catch (error) {
|
|
120
|
-
// Fall back to single-threaded if worker execution fails
|
|
121
|
-
// (e.g., in test environments with ESM/worker compatibility issues)
|
|
122
|
-
return items.map(fn);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* Filter items in parallel using workerpool.
|
|
127
|
-
*
|
|
128
|
-
* Splits the array into chunks and processes each chunk in a worker thread.
|
|
129
|
-
* Falls back to single-threaded for small arrays (< MIN_PARALLEL_SIZE).
|
|
130
|
-
*
|
|
131
|
-
* **Note:** The predicate function must be serializable (no closures, external variables).
|
|
132
|
-
* Due to ESM/worker thread compatibility issues, this may fall back to single-threaded
|
|
133
|
-
* execution in some environments (e.g., vitest test runner).
|
|
134
|
-
*
|
|
135
|
-
* @template T - Item type
|
|
136
|
-
* @param items - Array of items to filter
|
|
137
|
-
* @param predicate - Filter predicate (must be serializable)
|
|
138
|
-
* @param chunkSize - Optional chunk size (default: DEFAULT_CHUNK_SIZE)
|
|
139
|
-
* @returns Promise resolving to filtered array
|
|
140
|
-
*
|
|
141
|
-
* @example
|
|
142
|
-
* ```typescript
|
|
143
|
-
* // Filter even numbers
|
|
144
|
-
* const numbers = [1, 2, 3, 4, 5, 6];
|
|
145
|
-
* const evens = await parallelFilter(numbers, (n: number) => n % 2 === 0);
|
|
146
|
-
* // Result: [2, 4, 6]
|
|
147
|
-
* ```
|
|
148
|
-
*/
|
|
149
|
-
export async function parallelFilter(items, predicate, chunkSize = DEFAULT_CHUNK_SIZE) {
|
|
150
|
-
// Security: Validate that predicate is a real function, not a user-provided string
|
|
151
|
-
validateFunction(predicate, 'predicate');
|
|
152
|
-
// Fall back to single-threaded for small arrays
|
|
153
|
-
if (items.length < MIN_PARALLEL_SIZE) {
|
|
154
|
-
return items.filter(predicate);
|
|
155
|
-
}
|
|
156
|
-
try {
|
|
157
|
-
const pool = getPool();
|
|
158
|
-
// Split items into chunks
|
|
159
|
-
const chunks = [];
|
|
160
|
-
for (let i = 0; i < items.length; i += chunkSize) {
|
|
161
|
-
chunks.push(items.slice(i, i + chunkSize));
|
|
162
|
-
}
|
|
163
|
-
// Convert function to string for serialization
|
|
164
|
-
const predicateString = predicate.toString();
|
|
165
|
-
// Process chunks in parallel using inline function execution
|
|
166
|
-
const results = await Promise.all(chunks.map(chunk => pool.exec((chunkData, predicateStr) => {
|
|
167
|
-
// Reconstruct function from string
|
|
168
|
-
// eslint-disable-next-line no-new-func
|
|
169
|
-
const filterFn = new Function('return ' + predicateStr)();
|
|
170
|
-
return chunkData.filter(filterFn);
|
|
171
|
-
}, [chunk, predicateString])));
|
|
172
|
-
// Flatten results
|
|
173
|
-
return results.flat();
|
|
174
|
-
}
|
|
175
|
-
catch (error) {
|
|
176
|
-
// Fall back to single-threaded if worker execution fails
|
|
177
|
-
// (e.g., in test environments with ESM/worker compatibility issues)
|
|
178
|
-
return items.filter(predicate);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Get statistics about the worker pool.
|
|
183
|
-
*
|
|
184
|
-
* @returns Pool statistics or null if pool is not initialized
|
|
185
|
-
*/
|
|
186
|
-
export function getPoolStats() {
|
|
187
|
-
if (!sharedPool) {
|
|
188
|
-
return null;
|
|
189
|
-
}
|
|
190
|
-
return sharedPool.stats();
|
|
191
|
-
}
|
|
192
|
-
//# sourceMappingURL=parallelUtils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parallelUtils.js","sourceRoot":"","sources":["../../src/utils/parallelUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,UAAU,MAAM,2BAA2B,CAAC;AAEnD;;;;;;;;GAQG;AACH,SAAS,gBAAgB,CAAC,EAAW,EAAE,SAAiB;IACtD,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,SAAS,CAAC,GAAG,SAAS,4BAA4B,OAAO,EAAE,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B;;;GAGG;AACH,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;GAGG;AACH,IAAI,UAAU,GAA2B,IAAI,CAAC;AAE9C;;;;;GAKG;AACH,SAAS,OAAO;IACd,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC;YAC3B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC;YAC5C,UAAU,EAAE,QAAQ;SACrB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAC7B,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAU,EACV,EAAkB,EAClB,YAAoB,kBAAkB;IAEtC,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAE3B,gDAAgD;IAChD,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACrC,OAAO,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QAEvB,0BAA0B;QAC1B,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;QAE/B,6DAA6D;QAC7D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,IAAI,CAAC,IAAI,CACP,CAAC,SAAc,EAAE,KAAa,EAAE,EAAE;YAChC,mCAAmC;YACnC,uCAAuC;YACvC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,EAAoB,CAAC;YAClE,OAAO,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC,EACD,CAAC,KAAK,EAAE,QAAQ,CAAC,CACF,CAClB,CACF,CAAC;QAEF,kBAAkB;QAClB,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yDAAyD;QACzD,oEAAoE;QACpE,OAAO,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAU,EACV,SAA+B,EAC/B,YAAoB,kBAAkB;IAEtC,mFAAmF;IACnF,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAEzC,gDAAgD;IAChD,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QAEvB,0BAA0B;QAC1B,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,MAAM,eAAe,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;QAE7C,6DAA6D;QAC7D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,IAAI,CAAC,IAAI,CACP,CAAC,SAAc,EAAE,YAAoB,EAAE,EAAE;YACvC,mCAAmC;YACnC,uCAAuC;YACvC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,SAAS,GAAG,YAAY,CAAC,EAA0B,CAAC;YAClF,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,EACD,CAAC,KAAK,EAAE,eAAe,CAAC,CACT,CAClB,CACF,CAAC;QAEF,kBAAkB;QAClB,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yDAAyD;QACzD,oEAAoE;QACpE,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY;IAC1B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,EAAE,CAAC;AAC5B,CAAC"}
|