@alanse/clickup-multi-mcp-server 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +38 -0
- package/LICENSE +21 -0
- package/README.md +470 -0
- package/build/config.js +237 -0
- package/build/index.js +87 -0
- package/build/logger.js +163 -0
- package/build/middleware/security.js +231 -0
- package/build/server.js +288 -0
- package/build/services/clickup/base.js +432 -0
- package/build/services/clickup/bulk.js +180 -0
- package/build/services/clickup/document.js +159 -0
- package/build/services/clickup/folder.js +136 -0
- package/build/services/clickup/index.js +76 -0
- package/build/services/clickup/list.js +191 -0
- package/build/services/clickup/tag.js +239 -0
- package/build/services/clickup/task/index.js +32 -0
- package/build/services/clickup/task/task-attachments.js +105 -0
- package/build/services/clickup/task/task-comments.js +114 -0
- package/build/services/clickup/task/task-core.js +604 -0
- package/build/services/clickup/task/task-custom-fields.js +107 -0
- package/build/services/clickup/task/task-search.js +986 -0
- package/build/services/clickup/task/task-service.js +104 -0
- package/build/services/clickup/task/task-tags.js +113 -0
- package/build/services/clickup/time.js +244 -0
- package/build/services/clickup/types.js +33 -0
- package/build/services/clickup/workspace.js +397 -0
- package/build/services/shared.js +61 -0
- package/build/sse_server.js +277 -0
- package/build/tools/documents.js +489 -0
- package/build/tools/folder.js +331 -0
- package/build/tools/index.js +16 -0
- package/build/tools/list.js +428 -0
- package/build/tools/member.js +106 -0
- package/build/tools/tag.js +833 -0
- package/build/tools/task/attachments.js +357 -0
- package/build/tools/task/attachments.types.js +9 -0
- package/build/tools/task/bulk-operations.js +338 -0
- package/build/tools/task/handlers.js +919 -0
- package/build/tools/task/index.js +30 -0
- package/build/tools/task/main.js +233 -0
- package/build/tools/task/single-operations.js +469 -0
- package/build/tools/task/time-tracking.js +575 -0
- package/build/tools/task/utilities.js +310 -0
- package/build/tools/task/workspace-operations.js +258 -0
- package/build/tools/tool-enhancer.js +37 -0
- package/build/tools/utils.js +12 -0
- package/build/tools/workspace-helper.js +44 -0
- package/build/tools/workspace.js +73 -0
- package/build/utils/color-processor.js +183 -0
- package/build/utils/concurrency-utils.js +248 -0
- package/build/utils/date-utils.js +542 -0
- package/build/utils/resolver-utils.js +135 -0
- package/build/utils/sponsor-service.js +93 -0
- package/build/utils/token-utils.js +49 -0
- package/package.json +77 -0
- package/smithery.yaml +23 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SPDX-FileCopyrightText: © 2025 Talib Kareem <taazkareem@icloud.com>
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*
|
|
5
|
+
* ClickUp MCP Workspace Tools
|
|
6
|
+
*
|
|
7
|
+
* This module defines workspace-related tools like retrieving workspace hierarchy.
|
|
8
|
+
* It handles the workspace tool definitions and the implementation of their handlers.
|
|
9
|
+
*/
|
|
10
|
+
import { Logger } from '../logger.js';
|
|
11
|
+
import { sponsorService } from '../utils/sponsor-service.js';
|
|
12
|
+
import { getServicesForWorkspace, workspaceParameter } from './workspace-helper.js';
|
|
13
|
+
// Create a logger for workspace tools
|
|
14
|
+
const logger = new Logger('WorkspaceTool');
|
|
15
|
+
/**
|
|
16
|
+
* Tool definition for retrieving the complete workspace hierarchy
|
|
17
|
+
*/
|
|
18
|
+
export const workspaceHierarchyTool = {
|
|
19
|
+
name: 'get_workspace_hierarchy',
|
|
20
|
+
description: `Gets complete workspace hierarchy (spaces, folders, lists) for a specific workspace. Returns tree structure with names and IDs for navigation.`,
|
|
21
|
+
inputSchema: {
|
|
22
|
+
type: 'object',
|
|
23
|
+
properties: {
|
|
24
|
+
...workspaceParameter
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Handler for the get_workspace_hierarchy tool
|
|
30
|
+
*/
|
|
31
|
+
export async function handleGetWorkspaceHierarchy(params = {}) {
|
|
32
|
+
try {
|
|
33
|
+
// Get services for the specified workspace
|
|
34
|
+
const services = getServicesForWorkspace(params);
|
|
35
|
+
const { workspace: workspaceService } = services;
|
|
36
|
+
// Get workspace hierarchy from the workspace service
|
|
37
|
+
const hierarchy = await workspaceService.getWorkspaceHierarchy();
|
|
38
|
+
// Generate tree representation
|
|
39
|
+
const treeOutput = formatTreeOutput(hierarchy);
|
|
40
|
+
// Use sponsor service to create the response with optional sponsor message
|
|
41
|
+
return sponsorService.createResponse({ hierarchy: treeOutput }, true);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
return sponsorService.createErrorResponse(`Error getting workspace hierarchy: ${error.message}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Format the hierarchy as a tree string
|
|
49
|
+
*/
|
|
50
|
+
function formatTreeOutput(hierarchy) {
|
|
51
|
+
// Helper function to format a node and its children as a tree
|
|
52
|
+
const formatNodeAsTree = (node, level = 0, isLast = true, parentPrefix = '') => {
|
|
53
|
+
const lines = [];
|
|
54
|
+
// Calculate the current line's prefix
|
|
55
|
+
const currentPrefix = level === 0 ? '' : parentPrefix + (isLast ? '└── ' : '├── ');
|
|
56
|
+
// Format current node with descriptive ID type
|
|
57
|
+
const idType = 'type' in node ? `${node.type.charAt(0).toUpperCase() + node.type.slice(1)} ID` : 'Workspace ID';
|
|
58
|
+
lines.push(`${currentPrefix}${node.name} (${idType}: ${node.id})`);
|
|
59
|
+
// Calculate the prefix for children
|
|
60
|
+
const childPrefix = level === 0 ? '' : parentPrefix + (isLast ? ' ' : '│ ');
|
|
61
|
+
// Process children
|
|
62
|
+
const children = node.children || [];
|
|
63
|
+
children.forEach((child, index) => {
|
|
64
|
+
const childLines = formatNodeAsTree(child, level + 1, index === children.length - 1, childPrefix);
|
|
65
|
+
lines.push(...childLines);
|
|
66
|
+
});
|
|
67
|
+
return lines;
|
|
68
|
+
};
|
|
69
|
+
// Generate tree representation
|
|
70
|
+
const treeLines = formatNodeAsTree(hierarchy.root);
|
|
71
|
+
// Return plain text instead of adding code block markers
|
|
72
|
+
return treeLines.join('\n');
|
|
73
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SPDX-FileCopyrightText: © 2025 Talib Kareem <taazkareem@icloud.com>
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*
|
|
5
|
+
* Color Processor Utility
|
|
6
|
+
*
|
|
7
|
+
* Processes natural language color commands and converts them to HEX color values.
|
|
8
|
+
* Also generates appropriate foreground (text) colors for optimal contrast.
|
|
9
|
+
*/
|
|
10
|
+
// Basic color mapping with common color names to their HEX values
|
|
11
|
+
const COLOR_MAP = {
|
|
12
|
+
// Primary colors
|
|
13
|
+
red: '#FF0000',
|
|
14
|
+
green: '#00FF00',
|
|
15
|
+
blue: '#0000FF',
|
|
16
|
+
// Secondary colors
|
|
17
|
+
yellow: '#FFFF00',
|
|
18
|
+
purple: '#800080',
|
|
19
|
+
orange: '#FFA500',
|
|
20
|
+
pink: '#FFC0CB',
|
|
21
|
+
brown: '#A52A2A',
|
|
22
|
+
// Neutrals
|
|
23
|
+
black: '#000000',
|
|
24
|
+
white: '#FFFFFF',
|
|
25
|
+
gray: '#808080',
|
|
26
|
+
grey: '#808080',
|
|
27
|
+
// Extended colors
|
|
28
|
+
navy: '#000080',
|
|
29
|
+
teal: '#008080',
|
|
30
|
+
olive: '#808000',
|
|
31
|
+
maroon: '#800000',
|
|
32
|
+
aqua: '#00FFFF',
|
|
33
|
+
cyan: '#00FFFF',
|
|
34
|
+
magenta: '#FF00FF',
|
|
35
|
+
fuchsia: '#FF00FF',
|
|
36
|
+
lime: '#00FF00',
|
|
37
|
+
indigo: '#4B0082',
|
|
38
|
+
violet: '#EE82EE',
|
|
39
|
+
gold: '#FFD700',
|
|
40
|
+
silver: '#C0C0C0',
|
|
41
|
+
beige: '#F5F5DC',
|
|
42
|
+
tan: '#D2B48C',
|
|
43
|
+
coral: '#FF7F50',
|
|
44
|
+
crimson: '#DC143C',
|
|
45
|
+
khaki: '#F0E68C',
|
|
46
|
+
lavender: '#E6E6FA',
|
|
47
|
+
plum: '#DDA0DD',
|
|
48
|
+
salmon: '#FA8072',
|
|
49
|
+
turquoise: '#40E0D0',
|
|
50
|
+
};
|
|
51
|
+
// Extended color variations
|
|
52
|
+
const COLOR_VARIATIONS = {
|
|
53
|
+
red: {
|
|
54
|
+
light: '#FF6666',
|
|
55
|
+
dark: '#8B0000',
|
|
56
|
+
bright: '#FF0000',
|
|
57
|
+
deep: '#8B0000',
|
|
58
|
+
},
|
|
59
|
+
blue: {
|
|
60
|
+
light: '#ADD8E6',
|
|
61
|
+
dark: '#00008B',
|
|
62
|
+
sky: '#87CEEB',
|
|
63
|
+
navy: '#000080',
|
|
64
|
+
royal: '#4169E1',
|
|
65
|
+
deep: '#00008B',
|
|
66
|
+
},
|
|
67
|
+
green: {
|
|
68
|
+
light: '#90EE90',
|
|
69
|
+
dark: '#006400',
|
|
70
|
+
forest: '#228B22',
|
|
71
|
+
lime: '#32CD32',
|
|
72
|
+
mint: '#98FB98',
|
|
73
|
+
olive: '#808000',
|
|
74
|
+
},
|
|
75
|
+
yellow: {
|
|
76
|
+
light: '#FFFFE0',
|
|
77
|
+
dark: '#BDB76B',
|
|
78
|
+
pale: '#FFF9C4',
|
|
79
|
+
gold: '#FFD700',
|
|
80
|
+
lemon: '#FFFACD',
|
|
81
|
+
},
|
|
82
|
+
// Add more variations for other colors as needed
|
|
83
|
+
};
|
|
84
|
+
/**
|
|
85
|
+
* Extracts a color name from natural language text
|
|
86
|
+
* @param text - Natural language text that contains a color reference
|
|
87
|
+
* @returns The extracted color name or null if no color is found
|
|
88
|
+
*/
|
|
89
|
+
function extractColorFromText(text) {
|
|
90
|
+
if (!text)
|
|
91
|
+
return null;
|
|
92
|
+
// Convert to lowercase for case-insensitive matching
|
|
93
|
+
const lowercaseText = text.toLowerCase();
|
|
94
|
+
// First check for color variations (e.g., "dark blue", "light green")
|
|
95
|
+
for (const [baseColor, variations] of Object.entries(COLOR_VARIATIONS)) {
|
|
96
|
+
for (const [variation, _] of Object.entries(variations)) {
|
|
97
|
+
const colorPhrase = `${variation} ${baseColor}`;
|
|
98
|
+
if (lowercaseText.includes(colorPhrase)) {
|
|
99
|
+
return colorPhrase;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Then check for base colors
|
|
104
|
+
for (const color of Object.keys(COLOR_MAP)) {
|
|
105
|
+
// Use word boundary to make sure we're matching whole words
|
|
106
|
+
const regex = new RegExp(`\\b${color}\\b`, 'i');
|
|
107
|
+
if (regex.test(lowercaseText)) {
|
|
108
|
+
return color;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Converts a color name to its HEX value
|
|
115
|
+
* @param colorName - Name of the color to convert (e.g., "blue", "dark red")
|
|
116
|
+
* @returns HEX color code or null if color name is not recognized
|
|
117
|
+
*/
|
|
118
|
+
function colorNameToHex(colorName) {
|
|
119
|
+
if (!colorName)
|
|
120
|
+
return null;
|
|
121
|
+
const lowercaseColor = colorName.toLowerCase();
|
|
122
|
+
// Check if it's a color variation (e.g., "dark blue")
|
|
123
|
+
const parts = lowercaseColor.split(' ');
|
|
124
|
+
if (parts.length === 2) {
|
|
125
|
+
const variation = parts[0];
|
|
126
|
+
const baseColor = parts[1];
|
|
127
|
+
if (COLOR_VARIATIONS[baseColor] && COLOR_VARIATIONS[baseColor][variation]) {
|
|
128
|
+
return COLOR_VARIATIONS[baseColor][variation];
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// Check if it's a base color
|
|
132
|
+
return COLOR_MAP[lowercaseColor] || null;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Calculates the relative luminance of a color for WCAG contrast calculations
|
|
136
|
+
* @param hex - HEX color code
|
|
137
|
+
* @returns Relative luminance value
|
|
138
|
+
*/
|
|
139
|
+
function calculateLuminance(hex) {
|
|
140
|
+
// Remove # if present
|
|
141
|
+
const color = hex.startsWith('#') ? hex.slice(1) : hex;
|
|
142
|
+
// Convert HEX to RGB
|
|
143
|
+
const r = parseInt(color.substr(0, 2), 16) / 255;
|
|
144
|
+
const g = parseInt(color.substr(2, 2), 16) / 255;
|
|
145
|
+
const b = parseInt(color.substr(4, 2), 16) / 255;
|
|
146
|
+
// Calculate luminance using the formula from WCAG 2.0
|
|
147
|
+
const R = r <= 0.03928 ? r / 12.92 : Math.pow((r + 0.055) / 1.055, 2.4);
|
|
148
|
+
const G = g <= 0.03928 ? g / 12.92 : Math.pow((g + 0.055) / 1.055, 2.4);
|
|
149
|
+
const B = b <= 0.03928 ? b / 12.92 : Math.pow((b + 0.055) / 1.055, 2.4);
|
|
150
|
+
return 0.2126 * R + 0.7152 * G + 0.0722 * B;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Generates a contrasting foreground color for optimal readability
|
|
154
|
+
* @param backgroundColor - HEX code of the background color
|
|
155
|
+
* @returns HEX code of the foreground color (either black or white)
|
|
156
|
+
*/
|
|
157
|
+
function generateContrastingForeground(backgroundColor) {
|
|
158
|
+
const luminance = calculateLuminance(backgroundColor);
|
|
159
|
+
// Use white text on dark backgrounds and black text on light backgrounds
|
|
160
|
+
// The threshold 0.5 is based on WCAG guidelines for contrast
|
|
161
|
+
return luminance > 0.5 ? '#000000' : '#FFFFFF';
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Processes a natural language command to extract a color and convert it to HEX values
|
|
165
|
+
* @param command - Natural language command (e.g., "Create a blue tag")
|
|
166
|
+
* @returns Object containing background and foreground HEX colors, or null if color not recognized
|
|
167
|
+
*/
|
|
168
|
+
export function processColorCommand(command) {
|
|
169
|
+
// Extract color name from command
|
|
170
|
+
const colorName = extractColorFromText(command);
|
|
171
|
+
if (!colorName)
|
|
172
|
+
return null;
|
|
173
|
+
// Convert color name to HEX background color
|
|
174
|
+
const backgroundColor = colorNameToHex(colorName);
|
|
175
|
+
if (!backgroundColor)
|
|
176
|
+
return null;
|
|
177
|
+
// Generate appropriate foreground color
|
|
178
|
+
const foregroundColor = generateContrastingForeground(backgroundColor);
|
|
179
|
+
return {
|
|
180
|
+
background: backgroundColor,
|
|
181
|
+
foreground: foregroundColor
|
|
182
|
+
};
|
|
183
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SPDX-FileCopyrightText: © 2025 Talib Kareem <taazkareem@icloud.com>
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*
|
|
5
|
+
* Concurrency Utilities
|
|
6
|
+
*
|
|
7
|
+
* This module provides utilities for handling concurrent operations,
|
|
8
|
+
* batch processing, rate limiting, and retry logic.
|
|
9
|
+
*/
|
|
10
|
+
import { Logger } from '../logger.js';
|
|
11
|
+
// Create logger instance for this module
|
|
12
|
+
const logger = new Logger('ConcurrencyUtils');
|
|
13
|
+
/**
|
|
14
|
+
* Process a collection of items in batches with configurable concurrency
|
|
15
|
+
*
|
|
16
|
+
* This utility handles:
|
|
17
|
+
* - Breaking items into manageable batches
|
|
18
|
+
* - Processing multiple items concurrently
|
|
19
|
+
* - Retrying failed operations with backoff
|
|
20
|
+
* - Tracking progress and aggregating results
|
|
21
|
+
* - Graceful error handling
|
|
22
|
+
*
|
|
23
|
+
* @param items Array of items to process
|
|
24
|
+
* @param processor Function that processes a single item
|
|
25
|
+
* @param options Configuration options for batch processing
|
|
26
|
+
* @returns Results of the processing with success and failure information
|
|
27
|
+
*/
|
|
28
|
+
export async function processBatch(items, processor, options) {
|
|
29
|
+
// Apply default options
|
|
30
|
+
const opts = {
|
|
31
|
+
batchSize: options?.batchSize ?? 10,
|
|
32
|
+
concurrency: options?.concurrency ?? 3,
|
|
33
|
+
continueOnError: options?.continueOnError ?? true,
|
|
34
|
+
retryCount: options?.retryCount ?? 3,
|
|
35
|
+
retryDelay: options?.retryDelay ?? 1000,
|
|
36
|
+
exponentialBackoff: options?.exponentialBackoff ?? true,
|
|
37
|
+
progressCallback: options?.progressCallback ?? (() => { })
|
|
38
|
+
};
|
|
39
|
+
// Initialize results
|
|
40
|
+
const result = {
|
|
41
|
+
successful: [],
|
|
42
|
+
failed: [],
|
|
43
|
+
totals: {
|
|
44
|
+
success: 0,
|
|
45
|
+
failure: 0,
|
|
46
|
+
total: items.length
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
// Handle empty input array
|
|
50
|
+
if (items.length === 0) {
|
|
51
|
+
logger.info('processBatch called with empty items array');
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
try {
|
|
55
|
+
const totalBatches = Math.ceil(items.length / opts.batchSize);
|
|
56
|
+
let processedItems = 0;
|
|
57
|
+
logger.info(`Starting batch processing of ${items.length} items`, {
|
|
58
|
+
totalBatches,
|
|
59
|
+
batchSize: opts.batchSize,
|
|
60
|
+
concurrency: opts.concurrency
|
|
61
|
+
});
|
|
62
|
+
// Process items in batches
|
|
63
|
+
for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
|
|
64
|
+
const startIdx = batchIndex * opts.batchSize;
|
|
65
|
+
const endIdx = Math.min(startIdx + opts.batchSize, items.length);
|
|
66
|
+
const batch = items.slice(startIdx, endIdx);
|
|
67
|
+
logger.debug(`Processing batch ${batchIndex + 1}/${totalBatches}`, {
|
|
68
|
+
batchSize: batch.length,
|
|
69
|
+
startIdx,
|
|
70
|
+
endIdx
|
|
71
|
+
});
|
|
72
|
+
// Process the current batch
|
|
73
|
+
const batchResults = await processSingleBatch(batch, processor, startIdx, opts);
|
|
74
|
+
// Aggregate results
|
|
75
|
+
result.successful.push(...batchResults.successful);
|
|
76
|
+
result.failed.push(...batchResults.failed);
|
|
77
|
+
result.totals.success += batchResults.totals.success;
|
|
78
|
+
result.totals.failure += batchResults.totals.failure;
|
|
79
|
+
// Stop processing if an error occurred and continueOnError is false
|
|
80
|
+
if (batchResults.totals.failure > 0 && !opts.continueOnError) {
|
|
81
|
+
logger.warn(`Stopping batch processing due to failure and continueOnError=false`, {
|
|
82
|
+
failedItems: batchResults.totals.failure
|
|
83
|
+
});
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
// Update progress
|
|
87
|
+
processedItems += batch.length;
|
|
88
|
+
opts.progressCallback(processedItems, items.length, result.totals.success, result.totals.failure);
|
|
89
|
+
}
|
|
90
|
+
logger.info(`Batch processing completed`, {
|
|
91
|
+
totalItems: items.length,
|
|
92
|
+
successful: result.totals.success,
|
|
93
|
+
failed: result.totals.failure
|
|
94
|
+
});
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
logger.error(`Unexpected error in batch processing`, {
|
|
99
|
+
error: error.message || String(error)
|
|
100
|
+
});
|
|
101
|
+
// Add any unprocessed items as failures
|
|
102
|
+
const processedCount = result.totals.success + result.totals.failure;
|
|
103
|
+
if (processedCount < items.length) {
|
|
104
|
+
const remainingItems = items.slice(processedCount);
|
|
105
|
+
for (let i = 0; i < remainingItems.length; i++) {
|
|
106
|
+
const index = processedCount + i;
|
|
107
|
+
result.failed.push({
|
|
108
|
+
item: remainingItems[i],
|
|
109
|
+
error: new Error('Batch processing failed: ' + (error.message || 'Unknown error')),
|
|
110
|
+
index
|
|
111
|
+
});
|
|
112
|
+
result.totals.failure++;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return result;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Process a single batch of items with concurrency
|
|
120
|
+
*
|
|
121
|
+
* @param batch The batch of items to process
|
|
122
|
+
* @param processor The function to process each item
|
|
123
|
+
* @param startIndex The starting index of the batch in the original array
|
|
124
|
+
* @param opts Processing options
|
|
125
|
+
* @returns Results for this batch
|
|
126
|
+
*/
|
|
127
|
+
async function processSingleBatch(batch, processor, startIndex, opts) {
|
|
128
|
+
const result = {
|
|
129
|
+
successful: [],
|
|
130
|
+
failed: [],
|
|
131
|
+
totals: {
|
|
132
|
+
success: 0,
|
|
133
|
+
failure: 0,
|
|
134
|
+
total: batch.length
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
try {
|
|
138
|
+
// Process items in concurrent chunks
|
|
139
|
+
for (let i = 0; i < batch.length; i += opts.concurrency) {
|
|
140
|
+
const concurrentBatch = batch.slice(i, Math.min(i + opts.concurrency, batch.length));
|
|
141
|
+
// Create a promise for each item in the concurrent batch
|
|
142
|
+
const promises = concurrentBatch.map((item, idx) => {
|
|
143
|
+
const index = startIndex + i + idx;
|
|
144
|
+
return processWithRetry(() => processor(item, index), item, index, opts);
|
|
145
|
+
});
|
|
146
|
+
// Wait for all promises to settle (either resolve or reject)
|
|
147
|
+
const results = await Promise.allSettled(promises);
|
|
148
|
+
// Process the results
|
|
149
|
+
results.forEach((promiseResult, idx) => {
|
|
150
|
+
const index = startIndex + i + idx;
|
|
151
|
+
if (promiseResult.status === 'fulfilled') {
|
|
152
|
+
// Operation succeeded
|
|
153
|
+
result.successful.push(promiseResult.value);
|
|
154
|
+
result.totals.success++;
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
// Operation failed
|
|
158
|
+
const error = promiseResult.reason;
|
|
159
|
+
result.failed.push({
|
|
160
|
+
item: batch[i + idx],
|
|
161
|
+
error,
|
|
162
|
+
index
|
|
163
|
+
});
|
|
164
|
+
result.totals.failure++;
|
|
165
|
+
// If continueOnError is false, stop processing
|
|
166
|
+
if (!opts.continueOnError) {
|
|
167
|
+
throw new Error(`Operation failed at index ${index}: ${error.message || String(error)}`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
logger.error(`Error in batch processing`, {
|
|
176
|
+
batchSize: batch.length,
|
|
177
|
+
startIndex,
|
|
178
|
+
error: error instanceof Error ? error.message : String(error)
|
|
179
|
+
});
|
|
180
|
+
// If we've hit an error that stopped the whole batch (continueOnError=false),
|
|
181
|
+
// we need to record any unprocessed items as failures
|
|
182
|
+
const processedCount = result.totals.success + result.totals.failure;
|
|
183
|
+
if (processedCount < batch.length) {
|
|
184
|
+
const remainingItems = batch.slice(processedCount);
|
|
185
|
+
for (let i = 0; i < remainingItems.length; i++) {
|
|
186
|
+
const index = startIndex + processedCount + i;
|
|
187
|
+
result.failed.push({
|
|
188
|
+
item: remainingItems[i],
|
|
189
|
+
error: new Error('Batch processing aborted: ' +
|
|
190
|
+
(error instanceof Error ? error.message : String(error))),
|
|
191
|
+
index
|
|
192
|
+
});
|
|
193
|
+
result.totals.failure++;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return result;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Process a single item with retry logic
|
|
201
|
+
*
|
|
202
|
+
* @param operation The operation to perform
|
|
203
|
+
* @param item The item being processed (for context)
|
|
204
|
+
* @param index The index of the item (for logging)
|
|
205
|
+
* @param options Processing options
|
|
206
|
+
* @returns The result of the operation if successful
|
|
207
|
+
* @throws Error if all retry attempts fail
|
|
208
|
+
*/
|
|
209
|
+
async function processWithRetry(operation, item, index, options) {
|
|
210
|
+
let attempts = 0;
|
|
211
|
+
let lastError = null;
|
|
212
|
+
while (attempts <= options.retryCount) {
|
|
213
|
+
try {
|
|
214
|
+
// Attempt the operation
|
|
215
|
+
attempts++;
|
|
216
|
+
return await operation();
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
220
|
+
lastError = err;
|
|
221
|
+
logger.warn(`Operation failed for item at index ${index}`, {
|
|
222
|
+
attempt: attempts,
|
|
223
|
+
maxAttempts: options.retryCount + 1,
|
|
224
|
+
error: err.message
|
|
225
|
+
});
|
|
226
|
+
// If this was our last attempt, don't delay, just throw
|
|
227
|
+
if (attempts > options.retryCount) {
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
230
|
+
// Calculate delay for next retry
|
|
231
|
+
let delay = options.retryDelay;
|
|
232
|
+
if (options.exponentialBackoff) {
|
|
233
|
+
// Use exponential backoff with jitter
|
|
234
|
+
delay = options.retryDelay * Math.pow(2, attempts - 1) + Math.random() * 1000;
|
|
235
|
+
}
|
|
236
|
+
logger.debug(`Retrying operation after delay`, {
|
|
237
|
+
index,
|
|
238
|
+
attempt: attempts,
|
|
239
|
+
delayMs: delay
|
|
240
|
+
});
|
|
241
|
+
// Wait before next attempt
|
|
242
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
// If we get here, all retry attempts failed
|
|
246
|
+
throw new Error(`Operation failed after ${attempts} attempts for item at index ${index}: ` +
|
|
247
|
+
(lastError?.message || 'Unknown error'));
|
|
248
|
+
}
|