@onivoro/server-aws-ecs 24.30.12 → 24.30.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -585
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @onivoro/server-aws-ecs
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
AWS ECS integration for NestJS applications with task execution capabilities.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -8,677 +8,226 @@ A NestJS module for integrating with AWS ECS (Elastic Container Service), provid
|
|
|
8
8
|
npm install @onivoro/server-aws-ecs
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
##
|
|
11
|
+
## Overview
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
- **Network Configuration**: Configure VPC, subnets, and security groups
|
|
15
|
-
- **Environment Variable Mapping**: Convert objects to ECS environment variable format
|
|
16
|
-
- **Batch Task Running**: Execute multiple tasks concurrently
|
|
17
|
-
- **Task Overrides**: Dynamic container and task definition overrides
|
|
18
|
-
- **CSV String Parsing**: Utility for parsing comma-separated configuration values
|
|
19
|
-
- **Error Handling**: Comprehensive error handling for ECS operations
|
|
13
|
+
This library provides a simple ECS (Elastic Container Service) integration for NestJS applications, allowing you to run ECS tasks programmatically.
|
|
20
14
|
|
|
21
|
-
##
|
|
22
|
-
|
|
23
|
-
### 1. Module Configuration
|
|
15
|
+
## Module Setup
|
|
24
16
|
|
|
25
17
|
```typescript
|
|
18
|
+
import { Module } from '@nestjs/common';
|
|
26
19
|
import { ServerAwsEcsModule } from '@onivoro/server-aws-ecs';
|
|
27
20
|
|
|
28
21
|
@Module({
|
|
29
22
|
imports: [
|
|
30
|
-
ServerAwsEcsModule.configure(
|
|
31
|
-
|
|
32
|
-
CLUSTER_NAME: process.env.ECS_CLUSTER_NAME,
|
|
33
|
-
TASK_DEFINITION: process.env.ECS_TASK_DEFINITION,
|
|
34
|
-
SUBNETS: process.env.ECS_SUBNETS,
|
|
35
|
-
SECURITY_GROUPS: process.env.ECS_SECURITY_GROUPS,
|
|
36
|
-
AWS_PROFILE: process.env.AWS_PROFILE || 'default',
|
|
37
|
-
}),
|
|
38
|
-
],
|
|
23
|
+
ServerAwsEcsModule.configure()
|
|
24
|
+
]
|
|
39
25
|
})
|
|
40
26
|
export class AppModule {}
|
|
41
27
|
```
|
|
42
28
|
|
|
43
|
-
### 2. Basic Usage
|
|
44
|
-
|
|
45
|
-
```typescript
|
|
46
|
-
import { EcsService } from '@onivoro/server-aws-ecs';
|
|
47
|
-
|
|
48
|
-
@Injectable()
|
|
49
|
-
export class TaskRunnerService {
|
|
50
|
-
constructor(private ecsService: EcsService) {}
|
|
51
|
-
|
|
52
|
-
async runDataProcessingTask(jobData: any) {
|
|
53
|
-
const taskParams = {
|
|
54
|
-
taskDefinition: 'data-processor:1',
|
|
55
|
-
cluster: 'my-cluster',
|
|
56
|
-
subnets: 'subnet-12345,subnet-67890',
|
|
57
|
-
securityGroups: 'sg-12345',
|
|
58
|
-
taskCount: 1,
|
|
59
|
-
overrides: {
|
|
60
|
-
containerOverrides: [{
|
|
61
|
-
name: 'data-processor',
|
|
62
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
63
|
-
JOB_ID: jobData.id,
|
|
64
|
-
JOB_TYPE: jobData.type,
|
|
65
|
-
DATA_SOURCE: jobData.source
|
|
66
|
-
})
|
|
67
|
-
}]
|
|
68
|
-
}
|
|
69
|
-
};
|
|
70
|
-
|
|
71
|
-
return this.ecsService.runTasks(taskParams);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
```
|
|
75
|
-
|
|
76
29
|
## Configuration
|
|
77
30
|
|
|
78
|
-
|
|
31
|
+
The module uses environment-based configuration:
|
|
79
32
|
|
|
80
33
|
```typescript
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
AWS_REGION = process.env.AWS_REGION || 'us-east-1';
|
|
85
|
-
CLUSTER_NAME = process.env.ECS_CLUSTER_NAME || 'default-cluster';
|
|
86
|
-
TASK_DEFINITION = process.env.ECS_TASK_DEFINITION || 'my-task-definition';
|
|
87
|
-
SUBNETS = process.env.ECS_SUBNETS || 'subnet-12345,subnet-67890';
|
|
88
|
-
SECURITY_GROUPS = process.env.ECS_SECURITY_GROUPS || 'sg-12345';
|
|
89
|
-
AWS_PROFILE = process.env.AWS_PROFILE || 'default';
|
|
90
|
-
ASSIGN_PUBLIC_IP = process.env.ECS_ASSIGN_PUBLIC_IP === 'true' ? 'ENABLED' : 'DISABLED';
|
|
34
|
+
export class ServerAwsEcsConfig {
|
|
35
|
+
AWS_REGION: string;
|
|
36
|
+
AWS_PROFILE?: string; // Optional AWS profile
|
|
91
37
|
}
|
|
92
38
|
```
|
|
93
39
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
```bash
|
|
97
|
-
# AWS Configuration
|
|
98
|
-
AWS_REGION=us-east-1
|
|
99
|
-
AWS_PROFILE=default
|
|
100
|
-
|
|
101
|
-
# ECS Configuration
|
|
102
|
-
ECS_CLUSTER_NAME=my-application-cluster
|
|
103
|
-
ECS_TASK_DEFINITION=my-task-definition:1
|
|
104
|
-
ECS_SUBNETS=subnet-12345,subnet-67890,subnet-abcde
|
|
105
|
-
ECS_SECURITY_GROUPS=sg-12345,sg-67890
|
|
106
|
-
ECS_ASSIGN_PUBLIC_IP=false
|
|
107
|
-
```
|
|
108
|
-
|
|
109
|
-
## Services
|
|
40
|
+
## Service
|
|
110
41
|
|
|
111
42
|
### EcsService
|
|
112
43
|
|
|
113
|
-
The main service for ECS
|
|
44
|
+
The main service for running ECS tasks.
|
|
114
45
|
|
|
115
46
|
```typescript
|
|
47
|
+
import { Injectable } from '@nestjs/common';
|
|
116
48
|
import { EcsService } from '@onivoro/server-aws-ecs';
|
|
49
|
+
import { RunTaskCommandInput } from '@aws-sdk/client-ecs';
|
|
117
50
|
|
|
118
51
|
@Injectable()
|
|
119
|
-
export class
|
|
120
|
-
constructor(private ecsService: EcsService) {}
|
|
121
|
-
|
|
122
|
-
async runBatchJob(jobConfig: BatchJobConfig) {
|
|
123
|
-
const environmentVars = EcsService.mapObjectToEcsEnvironmentArray({
|
|
124
|
-
BATCH_ID: jobConfig.batchId,
|
|
125
|
-
INPUT_BUCKET: jobConfig.inputBucket,
|
|
126
|
-
OUTPUT_BUCKET: jobConfig.outputBucket,
|
|
127
|
-
PROCESSING_MODE: jobConfig.mode,
|
|
128
|
-
WORKER_COUNT: jobConfig.workerCount.toString()
|
|
129
|
-
});
|
|
52
|
+
export class TaskRunnerService {
|
|
53
|
+
constructor(private readonly ecsService: EcsService) {}
|
|
130
54
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
cluster:
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
55
|
+
async runDataProcessingTask() {
|
|
56
|
+
const params: RunTaskCommandInput = {
|
|
57
|
+
cluster: 'my-cluster',
|
|
58
|
+
taskDefinition: 'my-task-definition:1',
|
|
59
|
+
launchType: 'FARGATE',
|
|
60
|
+
networkConfiguration: {
|
|
61
|
+
awsvpcConfiguration: {
|
|
62
|
+
subnets: ['subnet-12345'],
|
|
63
|
+
securityGroups: ['sg-12345'],
|
|
64
|
+
assignPublicIp: 'ENABLED'
|
|
65
|
+
}
|
|
66
|
+
},
|
|
137
67
|
overrides: {
|
|
138
68
|
containerOverrides: [{
|
|
139
|
-
name:
|
|
140
|
-
environment:
|
|
141
|
-
|
|
142
|
-
|
|
69
|
+
name: 'my-container',
|
|
70
|
+
environment: [
|
|
71
|
+
{ name: 'ENV_VAR', value: 'value' }
|
|
72
|
+
]
|
|
143
73
|
}]
|
|
144
74
|
}
|
|
145
|
-
}
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const result = await this.ecsService.runTasks(params);
|
|
78
|
+
return result;
|
|
146
79
|
}
|
|
147
80
|
}
|
|
148
81
|
```
|
|
149
82
|
|
|
150
|
-
##
|
|
83
|
+
## Utility Function
|
|
151
84
|
|
|
152
|
-
###
|
|
85
|
+
### mapObjectToEcsEnvironmentArray
|
|
86
|
+
|
|
87
|
+
A static utility function to convert a plain object to ECS environment variable format:
|
|
153
88
|
|
|
154
89
|
```typescript
|
|
155
90
|
import { EcsService } from '@onivoro/server-aws-ecs';
|
|
156
91
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
// Step 1: Data validation task
|
|
163
|
-
const validationTasks = await this.runValidationTasks(dataset);
|
|
164
|
-
console.log(`Started ${validationTasks.length} validation tasks`);
|
|
165
|
-
|
|
166
|
-
// Step 2: Data transformation tasks (run in parallel)
|
|
167
|
-
const transformationTasks = await this.runTransformationTasks(dataset);
|
|
168
|
-
console.log(`Started ${transformationTasks.length} transformation tasks`);
|
|
169
|
-
|
|
170
|
-
// Step 3: Data aggregation task
|
|
171
|
-
const aggregationTasks = await this.runAggregationTasks(dataset);
|
|
172
|
-
console.log(`Started ${aggregationTasks.length} aggregation tasks`);
|
|
173
|
-
|
|
174
|
-
return {
|
|
175
|
-
validationTasks: validationTasks.length,
|
|
176
|
-
transformationTasks: transformationTasks.length,
|
|
177
|
-
aggregationTasks: aggregationTasks.length
|
|
178
|
-
};
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
private async runValidationTasks(dataset: DatasetConfig) {
|
|
182
|
-
return this.ecsService.runTasks({
|
|
183
|
-
taskDefinition: 'data-validator:1',
|
|
184
|
-
cluster: 'data-processing-cluster',
|
|
185
|
-
subnets: process.env.ECS_SUBNETS!,
|
|
186
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
187
|
-
taskCount: 1,
|
|
188
|
-
overrides: {
|
|
189
|
-
containerOverrides: [{
|
|
190
|
-
name: 'validator',
|
|
191
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
192
|
-
DATASET_ID: dataset.id,
|
|
193
|
-
VALIDATION_RULES: JSON.stringify(dataset.validationRules),
|
|
194
|
-
INPUT_LOCATION: dataset.inputLocation,
|
|
195
|
-
VALIDATION_OUTPUT: dataset.validationOutput
|
|
196
|
-
})
|
|
197
|
-
}]
|
|
198
|
-
}
|
|
199
|
-
});
|
|
200
|
-
}
|
|
92
|
+
const envVars = {
|
|
93
|
+
NODE_ENV: 'production',
|
|
94
|
+
API_KEY: 'secret-key',
|
|
95
|
+
PORT: '3000'
|
|
96
|
+
};
|
|
201
97
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
210
|
-
taskCount,
|
|
211
|
-
overrides: {
|
|
212
|
-
containerOverrides: [{
|
|
213
|
-
name: 'transformer',
|
|
214
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
215
|
-
DATASET_ID: dataset.id,
|
|
216
|
-
CHUNK_SIZE: dataset.chunkSize.toString(),
|
|
217
|
-
TRANSFORMATION_CONFIG: JSON.stringify(dataset.transformationConfig),
|
|
218
|
-
INPUT_LOCATION: dataset.inputLocation,
|
|
219
|
-
OUTPUT_LOCATION: dataset.outputLocation
|
|
220
|
-
}),
|
|
221
|
-
memory: 2048,
|
|
222
|
-
cpu: 1024
|
|
223
|
-
}]
|
|
224
|
-
}
|
|
225
|
-
});
|
|
226
|
-
}
|
|
98
|
+
const ecsEnvironment = EcsService.mapObjectToEcsEnvironmentArray(envVars);
|
|
99
|
+
// Returns:
|
|
100
|
+
// [
|
|
101
|
+
// { name: 'NODE_ENV', value: 'production' },
|
|
102
|
+
// { name: 'API_KEY', value: 'secret-key' },
|
|
103
|
+
// { name: 'PORT', value: '3000' }
|
|
104
|
+
// ]
|
|
227
105
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
name: 'aggregator',
|
|
238
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
239
|
-
DATASET_ID: dataset.id,
|
|
240
|
-
AGGREGATION_RULES: JSON.stringify(dataset.aggregationRules),
|
|
241
|
-
INPUT_LOCATION: dataset.outputLocation,
|
|
242
|
-
FINAL_OUTPUT: dataset.finalOutput
|
|
243
|
-
}),
|
|
244
|
-
memory: 4096,
|
|
245
|
-
cpu: 2048
|
|
246
|
-
}]
|
|
247
|
-
}
|
|
248
|
-
});
|
|
106
|
+
// Use in task configuration
|
|
107
|
+
const taskParams: RunTaskCommandInput = {
|
|
108
|
+
cluster: 'my-cluster',
|
|
109
|
+
taskDefinition: 'my-task',
|
|
110
|
+
overrides: {
|
|
111
|
+
containerOverrides: [{
|
|
112
|
+
name: 'container',
|
|
113
|
+
environment: ecsEnvironment
|
|
114
|
+
}]
|
|
249
115
|
}
|
|
250
|
-
}
|
|
116
|
+
};
|
|
251
117
|
```
|
|
252
118
|
|
|
253
|
-
|
|
119
|
+
## Direct Client Access
|
|
254
120
|
|
|
255
|
-
|
|
256
|
-
import { EcsService } from '@onivoro/server-aws-ecs';
|
|
257
|
-
import { Cron, CronExpression } from '@nestjs/schedule';
|
|
121
|
+
The service exposes the underlying ECS client for advanced operations:
|
|
258
122
|
|
|
123
|
+
```typescript
|
|
259
124
|
@Injectable()
|
|
260
|
-
export class
|
|
261
|
-
constructor(private ecsService: EcsService) {}
|
|
125
|
+
export class AdvancedEcsService {
|
|
126
|
+
constructor(private readonly ecsService: EcsService) {}
|
|
262
127
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const reportTasks = await this.ecsService.runTasks({
|
|
268
|
-
taskDefinition: 'report-generator:1',
|
|
269
|
-
cluster: 'reporting-cluster',
|
|
270
|
-
subnets: process.env.ECS_SUBNETS!,
|
|
271
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
272
|
-
taskCount: 1,
|
|
273
|
-
overrides: {
|
|
274
|
-
containerOverrides: [{
|
|
275
|
-
name: 'report-generator',
|
|
276
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
277
|
-
REPORT_TYPE: 'HOURLY',
|
|
278
|
-
REPORT_TIME: new Date().toISOString(),
|
|
279
|
-
OUTPUT_BUCKET: 'my-reports-bucket',
|
|
280
|
-
NOTIFICATION_TOPIC: 'arn:aws:sns:us-east-1:123456789012:reports'
|
|
281
|
-
})
|
|
282
|
-
}]
|
|
283
|
-
}
|
|
128
|
+
async describeCluster(clusterName: string) {
|
|
129
|
+
const command = new DescribeClustersCommand({
|
|
130
|
+
clusters: [clusterName]
|
|
284
131
|
});
|
|
285
|
-
|
|
286
|
-
console.log(`Started ${reportTasks.length} report generation tasks`);
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
|
290
|
-
async runDailyMaintenance() {
|
|
291
|
-
console.log('Starting daily maintenance tasks...');
|
|
292
|
-
|
|
293
|
-
// Run multiple maintenance tasks in parallel
|
|
294
|
-
const maintenanceTasks = [
|
|
295
|
-
this.runDataCleanupTask(),
|
|
296
|
-
this.runBackupTask(),
|
|
297
|
-
this.runHealthCheckTask()
|
|
298
|
-
];
|
|
299
|
-
|
|
300
|
-
const results = await Promise.allSettled(maintenanceTasks);
|
|
301
132
|
|
|
302
|
-
|
|
303
|
-
if (result.status === 'fulfilled') {
|
|
304
|
-
console.log(`Maintenance task ${index + 1} completed successfully`);
|
|
305
|
-
} else {
|
|
306
|
-
console.error(`Maintenance task ${index + 1} failed:`, result.reason);
|
|
307
|
-
}
|
|
308
|
-
});
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
private async runDataCleanupTask() {
|
|
312
|
-
return this.ecsService.runTasks({
|
|
313
|
-
taskDefinition: 'data-cleanup:1',
|
|
314
|
-
cluster: 'maintenance-cluster',
|
|
315
|
-
subnets: process.env.ECS_SUBNETS!,
|
|
316
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
317
|
-
taskCount: 1,
|
|
318
|
-
overrides: {
|
|
319
|
-
containerOverrides: [{
|
|
320
|
-
name: 'cleanup',
|
|
321
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
322
|
-
CLEANUP_MODE: 'DAILY',
|
|
323
|
-
RETENTION_DAYS: '30',
|
|
324
|
-
TARGET_TABLES: 'logs,temp_data,session_data'
|
|
325
|
-
})
|
|
326
|
-
}]
|
|
327
|
-
}
|
|
328
|
-
});
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
private async runBackupTask() {
|
|
332
|
-
return this.ecsService.runTasks({
|
|
333
|
-
taskDefinition: 'backup-service:1',
|
|
334
|
-
cluster: 'maintenance-cluster',
|
|
335
|
-
subnets: process.env.ECS_SUBNETS!,
|
|
336
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
337
|
-
taskCount: 1,
|
|
338
|
-
overrides: {
|
|
339
|
-
containerOverrides: [{
|
|
340
|
-
name: 'backup',
|
|
341
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
342
|
-
BACKUP_TYPE: 'DAILY',
|
|
343
|
-
BACKUP_TARGET: 's3://my-backup-bucket',
|
|
344
|
-
DATABASE_URL: process.env.DATABASE_URL!
|
|
345
|
-
})
|
|
346
|
-
}]
|
|
347
|
-
}
|
|
348
|
-
});
|
|
133
|
+
return await this.ecsService.ecsClient.send(command);
|
|
349
134
|
}
|
|
350
135
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
subnets: process.env.ECS_SUBNETS!,
|
|
356
|
-
securityGroups: process.env.ECS_SECURITY_GROUPS!,
|
|
357
|
-
taskCount: 1,
|
|
358
|
-
overrides: {
|
|
359
|
-
containerOverrides: [{
|
|
360
|
-
name: 'health-checker',
|
|
361
|
-
environment: EcsService.mapObjectToEcsEnvironmentArray({
|
|
362
|
-
CHECK_TYPE: 'COMPREHENSIVE',
|
|
363
|
-
NOTIFICATION_WEBHOOK: process.env.HEALTH_WEBHOOK_URL!,
|
|
364
|
-
SERVICES_TO_CHECK: 'api,database,cache,storage'
|
|
365
|
-
})
|
|
366
|
-
}]
|
|
367
|
-
}
|
|
136
|
+
async listTasks(cluster: string) {
|
|
137
|
+
const command = new ListTasksCommand({
|
|
138
|
+
cluster,
|
|
139
|
+
desiredStatus: 'RUNNING'
|
|
368
140
|
});
|
|
141
|
+
|
|
142
|
+
return await this.ecsService.ecsClient.send(command);
|
|
369
143
|
}
|
|
370
144
|
}
|
|
371
145
|
```
|
|
372
146
|
|
|
373
|
-
|
|
147
|
+
## Complete Example
|
|
374
148
|
|
|
375
149
|
```typescript
|
|
376
|
-
import {
|
|
150
|
+
import { Module, Injectable } from '@nestjs/common';
|
|
151
|
+
import { ServerAwsEcsModule, EcsService } from '@onivoro/server-aws-ecs';
|
|
152
|
+
import { RunTaskCommandInput } from '@aws-sdk/client-ecs';
|
|
377
153
|
|
|
378
|
-
@
|
|
379
|
-
|
|
380
|
-
|
|
154
|
+
@Module({
|
|
155
|
+
imports: [ServerAwsEcsModule.configure()],
|
|
156
|
+
providers: [BatchProcessorService],
|
|
157
|
+
exports: [BatchProcessorService]
|
|
158
|
+
})
|
|
159
|
+
export class BatchModule {}
|
|
381
160
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
161
|
+
@Injectable()
|
|
162
|
+
export class BatchProcessorService {
|
|
163
|
+
constructor(private readonly ecsService: EcsService) {}
|
|
385
164
|
|
|
386
|
-
|
|
165
|
+
async processBatch(batchId: string, items: string[]) {
|
|
166
|
+
// Convert environment variables
|
|
387
167
|
const environment = EcsService.mapObjectToEcsEnvironmentArray({
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
CONFIGURATION: JSON.stringify(taskConfig.configuration)
|
|
168
|
+
BATCH_ID: batchId,
|
|
169
|
+
ITEMS: items.join(','),
|
|
170
|
+
PROCESS_DATE: new Date().toISOString()
|
|
392
171
|
});
|
|
393
172
|
|
|
394
|
-
// Configure
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
173
|
+
// Configure task
|
|
174
|
+
const params: RunTaskCommandInput = {
|
|
175
|
+
cluster: 'batch-processing-cluster',
|
|
176
|
+
taskDefinition: 'batch-processor:latest',
|
|
177
|
+
launchType: 'FARGATE',
|
|
178
|
+
count: 1,
|
|
179
|
+
networkConfiguration: {
|
|
180
|
+
awsvpcConfiguration: {
|
|
181
|
+
subnets: [process.env.SUBNET_ID],
|
|
182
|
+
securityGroups: [process.env.SECURITY_GROUP_ID],
|
|
183
|
+
assignPublicIp: 'ENABLED'
|
|
184
|
+
}
|
|
185
|
+
},
|
|
403
186
|
overrides: {
|
|
404
187
|
containerOverrides: [{
|
|
405
|
-
name:
|
|
406
|
-
environment
|
|
407
|
-
|
|
408
|
-
cpu: resourceConfig.cpu,
|
|
409
|
-
...(taskConfig.command && { command: taskConfig.command })
|
|
410
|
-
}],
|
|
411
|
-
...(taskConfig.taskRoleArn && {
|
|
412
|
-
taskRoleArn: taskConfig.taskRoleArn
|
|
413
|
-
})
|
|
188
|
+
name: 'processor',
|
|
189
|
+
environment
|
|
190
|
+
}]
|
|
414
191
|
}
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
private validateTaskConfig(config: CustomTaskConfig) {
|
|
419
|
-
if (!config.taskDefinition) {
|
|
420
|
-
throw new Error('Task definition is required');
|
|
421
|
-
}
|
|
422
|
-
if (!config.containerName) {
|
|
423
|
-
throw new Error('Container name is required');
|
|
424
|
-
}
|
|
425
|
-
if (config.taskCount && (config.taskCount < 1 || config.taskCount > 100)) {
|
|
426
|
-
throw new Error('Task count must be between 1 and 100');
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
private generateTaskId(): string {
|
|
431
|
-
return `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
private getResourceConfiguration(taskType: string) {
|
|
435
|
-
const configurations = {
|
|
436
|
-
'cpu-intensive': { memory: 4096, cpu: 2048 },
|
|
437
|
-
'memory-intensive': { memory: 8192, cpu: 1024 },
|
|
438
|
-
'balanced': { memory: 2048, cpu: 1024 },
|
|
439
|
-
'lightweight': { memory: 512, cpu: 256 }
|
|
440
192
|
};
|
|
441
193
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
}
|
|
445
|
-
```
|
|
446
|
-
|
|
447
|
-
## Advanced Usage
|
|
448
|
-
|
|
449
|
-
### Task Monitoring and Logging
|
|
450
|
-
|
|
451
|
-
```typescript
|
|
452
|
-
import { ECS, DescribeTasksCommand } from '@aws-sdk/client-ecs';
|
|
453
|
-
|
|
454
|
-
@Injectable()
|
|
455
|
-
export class EcsTaskMonitoringService {
|
|
456
|
-
constructor(
|
|
457
|
-
private ecsService: EcsService,
|
|
458
|
-
private ecsClient: ECS
|
|
459
|
-
) {}
|
|
460
|
-
|
|
461
|
-
async runTaskWithMonitoring(taskParams: any, monitoringOptions?: TaskMonitoringOptions) {
|
|
462
|
-
// Start the task
|
|
463
|
-
const taskResults = await this.ecsService.runTasks(taskParams);
|
|
464
|
-
|
|
465
|
-
if (!taskResults.length || !taskResults[0].tasks?.length) {
|
|
466
|
-
throw new Error('Failed to start tasks');
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
const taskArns = taskResults[0].tasks.map(task => task.taskArn!);
|
|
194
|
+
// Run task
|
|
195
|
+
const result = await this.ecsService.runTasks(params);
|
|
470
196
|
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
this.monitorTasks(taskArns, taskParams.cluster, monitoringOptions);
|
|
197
|
+
if (result.failures && result.failures.length > 0) {
|
|
198
|
+
throw new Error(`Failed to start task: ${result.failures[0].reason}`);
|
|
474
199
|
}
|
|
475
200
|
|
|
476
|
-
return
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
private async monitorTasks(taskArns: string[], cluster: string, options: TaskMonitoringOptions) {
|
|
480
|
-
const checkInterval = options.checkInterval || 30000; // 30 seconds
|
|
481
|
-
const timeout = options.timeout || 3600000; // 1 hour
|
|
482
|
-
const startTime = Date.now();
|
|
483
|
-
|
|
484
|
-
const monitor = setInterval(async () => {
|
|
485
|
-
try {
|
|
486
|
-
const response = await this.ecsClient.send(new DescribeTasksCommand({
|
|
487
|
-
cluster,
|
|
488
|
-
tasks: taskArns
|
|
489
|
-
}));
|
|
490
|
-
|
|
491
|
-
const tasks = response.tasks || [];
|
|
492
|
-
const runningTasks = tasks.filter(task => task.lastStatus === 'RUNNING');
|
|
493
|
-
const stoppedTasks = tasks.filter(task => task.lastStatus === 'STOPPED');
|
|
494
|
-
|
|
495
|
-
console.log(`Task Status - Running: ${runningTasks.length}, Stopped: ${stoppedTasks.length}`);
|
|
496
|
-
|
|
497
|
-
// Check for failed tasks
|
|
498
|
-
const failedTasks = stoppedTasks.filter(task => task.stopCode !== 'EssentialContainerExited' || task.containers?.some(c => c.exitCode !== 0));
|
|
499
|
-
|
|
500
|
-
if (failedTasks.length > 0) {
|
|
501
|
-
console.error(`${failedTasks.length} tasks failed:`);
|
|
502
|
-
failedTasks.forEach(task => {
|
|
503
|
-
console.error(`Task ${task.taskArn} failed: ${task.stoppedReason}`);
|
|
504
|
-
});
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
// Stop monitoring if all tasks are complete
|
|
508
|
-
if (stoppedTasks.length === taskArns.length) {
|
|
509
|
-
clearInterval(monitor);
|
|
510
|
-
console.log('All tasks completed');
|
|
511
|
-
|
|
512
|
-
if (options.onComplete) {
|
|
513
|
-
options.onComplete(tasks);
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
// Check timeout
|
|
518
|
-
if (Date.now() - startTime > timeout) {
|
|
519
|
-
clearInterval(monitor);
|
|
520
|
-
console.warn('Task monitoring timeout reached');
|
|
521
|
-
|
|
522
|
-
if (options.onTimeout) {
|
|
523
|
-
options.onTimeout(tasks);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
} catch (error) {
|
|
527
|
-
console.error('Error monitoring tasks:', error);
|
|
528
|
-
}
|
|
529
|
-
}, checkInterval);
|
|
201
|
+
return result.tasks?.[0]?.taskArn;
|
|
530
202
|
}
|
|
531
203
|
}
|
|
532
204
|
```
|
|
533
205
|
|
|
534
|
-
|
|
206
|
+
## Environment Variables
|
|
535
207
|
|
|
536
|
-
|
|
208
|
+
Configure the module using these environment variables:
|
|
537
209
|
|
|
538
|
-
```
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
// Convert object to ECS environment variable format
|
|
542
|
-
const envVars = EcsService.mapObjectToEcsEnvironmentArray({
|
|
543
|
-
DATABASE_URL: 'postgresql://localhost:5432/mydb',
|
|
544
|
-
API_KEY: 'secret-key',
|
|
545
|
-
LOG_LEVEL: 'info',
|
|
546
|
-
FEATURE_FLAGS: JSON.stringify({ newFeature: true })
|
|
547
|
-
});
|
|
548
|
-
|
|
549
|
-
console.log(envVars);
|
|
550
|
-
// Output: [
|
|
551
|
-
// { Name: 'DATABASE_URL', Value: 'postgresql://localhost:5432/mydb' },
|
|
552
|
-
// { Name: 'API_KEY', Value: 'secret-key' },
|
|
553
|
-
// { Name: 'LOG_LEVEL', Value: 'info' },
|
|
554
|
-
// { Name: 'FEATURE_FLAGS', Value: '{"newFeature":true}' }
|
|
555
|
-
// ]
|
|
556
|
-
```
|
|
557
|
-
|
|
558
|
-
### Error Handling and Retry Logic
|
|
559
|
-
|
|
560
|
-
```typescript
|
|
561
|
-
@Injectable()
|
|
562
|
-
export class ResilientEcsService {
|
|
563
|
-
constructor(private ecsService: EcsService) {}
|
|
564
|
-
|
|
565
|
-
async runTaskWithRetry(taskParams: any, maxRetries: number = 3, backoffMs: number = 1000) {
|
|
566
|
-
let lastError: Error;
|
|
567
|
-
|
|
568
|
-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
569
|
-
try {
|
|
570
|
-
console.log(`Attempt ${attempt}/${maxRetries} to run ECS task`);
|
|
571
|
-
return await this.ecsService.runTasks(taskParams);
|
|
572
|
-
} catch (error: any) {
|
|
573
|
-
lastError = error;
|
|
574
|
-
console.error(`Attempt ${attempt} failed:`, error.message);
|
|
575
|
-
|
|
576
|
-
if (attempt < maxRetries) {
|
|
577
|
-
const delay = backoffMs * Math.pow(2, attempt - 1); // Exponential backoff
|
|
578
|
-
console.log(`Waiting ${delay}ms before retry...`);
|
|
579
|
-
await this.delay(delay);
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
throw new Error(`Failed to run ECS task after ${maxRetries} attempts. Last error: ${lastError!.message}`);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
private delay(ms: number): Promise<void> {
|
|
588
|
-
return new Promise(resolve => setTimeout(resolve, ms));
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
```
|
|
592
|
-
|
|
593
|
-
## Best Practices
|
|
594
|
-
|
|
595
|
-
### 1. Resource Management
|
|
596
|
-
|
|
597
|
-
```typescript
|
|
598
|
-
// Configure appropriate CPU and memory based on task requirements
|
|
599
|
-
const getResourceConfig = (taskType: 'cpu-intensive' | 'memory-intensive' | 'balanced') => {
|
|
600
|
-
switch (taskType) {
|
|
601
|
-
case 'cpu-intensive':
|
|
602
|
-
return { memory: 2048, cpu: 1024 };
|
|
603
|
-
case 'memory-intensive':
|
|
604
|
-
return { memory: 4096, cpu: 512 };
|
|
605
|
-
case 'balanced':
|
|
606
|
-
default:
|
|
607
|
-
return { memory: 1024, cpu: 512 };
|
|
608
|
-
}
|
|
609
|
-
};
|
|
610
|
-
```
|
|
611
|
-
|
|
612
|
-
### 2. Environment Variable Security
|
|
613
|
-
|
|
614
|
-
```typescript
|
|
615
|
-
// Use AWS Systems Manager Parameter Store for sensitive data
|
|
616
|
-
const secureEnvVars = EcsService.mapObjectToEcsEnvironmentArray({
|
|
617
|
-
DATABASE_URL: '${ssm:/app/database/url}',
|
|
618
|
-
API_KEY: '${ssm-secure:/app/api/key}',
|
|
619
|
-
PUBLIC_CONFIG: 'actual-value'
|
|
620
|
-
});
|
|
621
|
-
```
|
|
622
|
-
|
|
623
|
-
### 3. Task Definition Versioning
|
|
624
|
-
|
|
625
|
-
```typescript
|
|
626
|
-
// Always specify task definition versions
|
|
627
|
-
const taskDefinition = `${baseTaskDefinition}:${version}`;
|
|
628
|
-
```
|
|
629
|
-
|
|
630
|
-
## Testing
|
|
631
|
-
|
|
632
|
-
```typescript
|
|
633
|
-
import { Test, TestingModule } from '@nestjs/testing';
|
|
634
|
-
import { ServerAwsEcsModule, EcsService } from '@onivoro/server-aws-ecs';
|
|
210
|
+
```bash
|
|
211
|
+
# Required: AWS region
|
|
212
|
+
AWS_REGION=us-east-1
|
|
635
213
|
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
beforeEach(async () => {
|
|
640
|
-
const module: TestingModule = await Test.createTestingModule({
|
|
641
|
-
imports: [ServerAwsEcsModule.configure({
|
|
642
|
-
AWS_REGION: 'us-east-1',
|
|
643
|
-
CLUSTER_NAME: 'test-cluster',
|
|
644
|
-
TASK_DEFINITION: 'test-task:1',
|
|
645
|
-
SUBNETS: 'subnet-12345',
|
|
646
|
-
SECURITY_GROUPS: 'sg-12345',
|
|
647
|
-
AWS_PROFILE: 'test'
|
|
648
|
-
})],
|
|
649
|
-
}).compile();
|
|
650
|
-
|
|
651
|
-
service = module.get<EcsService>(EcsService);
|
|
652
|
-
});
|
|
653
|
-
|
|
654
|
-
it('should be defined', () => {
|
|
655
|
-
expect(service).toBeDefined();
|
|
656
|
-
});
|
|
657
|
-
|
|
658
|
-
it('should map object to ECS environment array', () => {
|
|
659
|
-
const envObject = { KEY1: 'value1', KEY2: 'value2' };
|
|
660
|
-
const result = EcsService.mapObjectToEcsEnvironmentArray(envObject);
|
|
661
|
-
|
|
662
|
-
expect(result).toEqual([
|
|
663
|
-
{ Name: 'KEY1', Value: 'value1' },
|
|
664
|
-
{ Name: 'KEY2', Value: 'value2' }
|
|
665
|
-
]);
|
|
666
|
-
});
|
|
667
|
-
});
|
|
214
|
+
# Optional: AWS profile
|
|
215
|
+
AWS_PROFILE=my-profile
|
|
668
216
|
```
|
|
669
217
|
|
|
670
|
-
##
|
|
218
|
+
## AWS Credentials
|
|
671
219
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
220
|
+
The module uses the standard AWS SDK credential chain:
|
|
221
|
+
1. Environment variables
|
|
222
|
+
2. Shared credentials file
|
|
223
|
+
3. IAM roles (for EC2/ECS/Lambda)
|
|
675
224
|
|
|
676
|
-
|
|
677
|
-
- `EcsService`: Main ECS service with task execution capabilities
|
|
225
|
+
## Limitations
|
|
678
226
|
|
|
679
|
-
|
|
680
|
-
-
|
|
227
|
+
- This is a thin wrapper around the AWS ECS SDK
|
|
228
|
+
- Only provides the `runTasks` method for task execution
|
|
229
|
+
- For more advanced ECS operations, use the exposed `ecsClient` directly
|
|
681
230
|
|
|
682
231
|
## License
|
|
683
232
|
|
|
684
|
-
|
|
233
|
+
MIT
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@onivoro/server-aws-ecs",
|
|
3
|
-
"version": "24.30.
|
|
3
|
+
"version": "24.30.14",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./src/index.js",
|
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
"url": "https://github.com/onivoro/monorepo.git"
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"@onivoro/server-aws-credential-providers": "24.30.
|
|
14
|
-
"@onivoro/server-common": "24.30.
|
|
13
|
+
"@onivoro/server-aws-credential-providers": "24.30.14",
|
|
14
|
+
"@onivoro/server-common": "24.30.14",
|
|
15
15
|
"tslib": "^2.3.0"
|
|
16
16
|
},
|
|
17
17
|
"peerDependencies": {
|