@onivoro/server-aws-kinesis 22.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +667 -0
- package/{dist/cjs/index.d.ts → index.ts} +2 -0
- package/jest.config.ts +11 -0
- package/package.json +10 -45
- package/project.json +23 -0
- package/{dist/esm/index.d.ts → src/index.ts} +2 -0
- package/src/lib/classes/server-aws-kinesis-config.class.ts +5 -0
- package/src/lib/server-aws-kinesis.module.ts +33 -0
- package/src/lib/services/kinesis.service.ts +25 -0
- package/tsconfig.json +16 -0
- package/tsconfig.lib.json +8 -0
- package/tsconfig.spec.json +21 -0
- package/dist/cjs/index.js +0 -19
- package/dist/cjs/lib/classes/server-aws-kinesis-config.class.d.ts +0 -7
- package/dist/cjs/lib/classes/server-aws-kinesis-config.class.js +0 -11
- package/dist/cjs/lib/server-aws-kinesis.module.d.ts +0 -4
- package/dist/cjs/lib/server-aws-kinesis.module.js +0 -46
- package/dist/cjs/lib/services/kinesis.service.d.ts +0 -8
- package/dist/cjs/lib/services/kinesis.service.js +0 -42
- package/dist/esm/index.js +0 -19
- package/dist/esm/lib/classes/server-aws-kinesis-config.class.d.ts +0 -7
- package/dist/esm/lib/classes/server-aws-kinesis-config.class.js +0 -11
- package/dist/esm/lib/server-aws-kinesis.module.d.ts +0 -4
- package/dist/esm/lib/server-aws-kinesis.module.js +0 -46
- package/dist/esm/lib/services/kinesis.service.d.ts +0 -8
- package/dist/esm/lib/services/kinesis.service.js +0 -42
- package/dist/types/index.d.ts +0 -3
- package/dist/types/lib/classes/server-aws-kinesis-config.class.d.ts +0 -7
- package/dist/types/lib/server-aws-kinesis.module.d.ts +0 -4
- package/dist/types/lib/services/kinesis.service.d.ts +0 -8
package/README.md
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
1
|
+
# @onivoro/server-aws-kinesis
|
|
2
|
+
|
|
3
|
+
A NestJS module for integrating with AWS Kinesis Data Streams, providing real-time data streaming, event publishing, and stream processing capabilities for your server applications.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @onivoro/server-aws-kinesis
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
- **Real-time Data Streaming**: Publish data to Kinesis streams in real-time
|
|
14
|
+
- **Event Publishing**: Send structured events with partition keys
|
|
15
|
+
- **Stream Management**: Create and manage Kinesis data streams
|
|
16
|
+
- **Partition Key Strategy**: Intelligent partition key generation for data distribution
|
|
17
|
+
- **Error Handling**: Robust error handling for stream operations
|
|
18
|
+
- **Batch Publishing**: Support for batch data publishing
|
|
19
|
+
- **Consumer Support**: Tools for building Kinesis stream consumers
|
|
20
|
+
- **Environment-Based Configuration**: Configurable stream settings per environment
|
|
21
|
+
|
|
22
|
+
## Quick Start
|
|
23
|
+
|
|
24
|
+
### 1. Module Configuration
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import { ServerAwsKinesisModule } from '@onivoro/server-aws-kinesis';
|
|
28
|
+
|
|
29
|
+
@Module({
|
|
30
|
+
imports: [
|
|
31
|
+
ServerAwsKinesisModule.configure({
|
|
32
|
+
AWS_REGION: 'us-east-1',
|
|
33
|
+
AWS_KINESIS_NAME: process.env.KINESIS_STREAM_NAME,
|
|
34
|
+
AWS_PROFILE: process.env.AWS_PROFILE || 'default',
|
|
35
|
+
}),
|
|
36
|
+
],
|
|
37
|
+
})
|
|
38
|
+
export class AppModule {}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### 2. Basic Usage
|
|
42
|
+
|
|
43
|
+
```typescript
|
|
44
|
+
import { KinesisService } from '@onivoro/server-aws-kinesis';
|
|
45
|
+
|
|
46
|
+
@Injectable()
|
|
47
|
+
export class EventStreamingService {
|
|
48
|
+
constructor(private kinesisService: KinesisService) {}
|
|
49
|
+
|
|
50
|
+
async publishUserEvent(userId: string, eventData: any) {
|
|
51
|
+
const event = {
|
|
52
|
+
eventType: 'USER_ACTION',
|
|
53
|
+
userId,
|
|
54
|
+
timestamp: new Date().toISOString(),
|
|
55
|
+
data: eventData
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
await this.kinesisService.publish(event, userId);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async publishOrderEvent(orderId: string, orderData: any) {
|
|
62
|
+
const event = {
|
|
63
|
+
eventType: 'ORDER_CREATED',
|
|
64
|
+
orderId,
|
|
65
|
+
timestamp: new Date().toISOString(),
|
|
66
|
+
data: orderData
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
// Use orderId as partition key to ensure order events are processed in sequence
|
|
70
|
+
await this.kinesisService.publish(event, orderId);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Configuration
|
|
76
|
+
|
|
77
|
+
### ServerAwsKinesisConfig
|
|
78
|
+
|
|
79
|
+
```typescript
|
|
80
|
+
import { ServerAwsKinesisConfig } from '@onivoro/server-aws-kinesis';
|
|
81
|
+
|
|
82
|
+
export class AppKinesisConfig extends ServerAwsKinesisConfig {
|
|
83
|
+
AWS_REGION = process.env.AWS_REGION || 'us-east-1';
|
|
84
|
+
AWS_KINESIS_NAME = process.env.KINESIS_STREAM_NAME || 'my-data-stream';
|
|
85
|
+
AWS_PROFILE = process.env.AWS_PROFILE || 'default';
|
|
86
|
+
KINESIS_SHARD_COUNT = parseInt(process.env.KINESIS_SHARD_COUNT) || 1;
|
|
87
|
+
KINESIS_RETENTION_PERIOD = parseInt(process.env.KINESIS_RETENTION_PERIOD) || 24; // hours
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Environment Variables
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
# AWS Configuration
|
|
95
|
+
AWS_REGION=us-east-1
|
|
96
|
+
AWS_PROFILE=default
|
|
97
|
+
|
|
98
|
+
# Kinesis Configuration
|
|
99
|
+
KINESIS_STREAM_NAME=my-application-stream
|
|
100
|
+
KINESIS_SHARD_COUNT=4
|
|
101
|
+
KINESIS_RETENTION_PERIOD=168 # 7 days in hours
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Services
|
|
105
|
+
|
|
106
|
+
### KinesisService
|
|
107
|
+
|
|
108
|
+
The main service for Kinesis operations:
|
|
109
|
+
|
|
110
|
+
```typescript
|
|
111
|
+
import { KinesisService } from '@onivoro/server-aws-kinesis';
|
|
112
|
+
|
|
113
|
+
@Injectable()
|
|
114
|
+
export class RealTimeDataService {
|
|
115
|
+
constructor(private kinesisService: KinesisService) {}
|
|
116
|
+
|
|
117
|
+
async publishMetrics(metrics: ApplicationMetrics) {
|
|
118
|
+
const event = {
|
|
119
|
+
type: 'METRICS',
|
|
120
|
+
timestamp: new Date().toISOString(),
|
|
121
|
+
metrics,
|
|
122
|
+
source: 'application-server'
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
// Use timestamp-based partition key for even distribution
|
|
126
|
+
const partitionKey = `metrics-${Date.now() % 1000}`;
|
|
127
|
+
await this.kinesisService.publish(event, partitionKey);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async publishLogEvent(logLevel: string, message: string, context: any) {
|
|
131
|
+
const logEvent = {
|
|
132
|
+
level: logLevel,
|
|
133
|
+
message,
|
|
134
|
+
context,
|
|
135
|
+
timestamp: new Date().toISOString(),
|
|
136
|
+
service: 'my-service'
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
// Use log level as partition key to group similar logs
|
|
140
|
+
await this.kinesisService.publish(logEvent, logLevel);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Usage Examples
|
|
146
|
+
|
|
147
|
+
### Event Publisher Service
|
|
148
|
+
|
|
149
|
+
```typescript
|
|
150
|
+
import { KinesisService } from '@onivoro/server-aws-kinesis';
|
|
151
|
+
|
|
152
|
+
@Injectable()
|
|
153
|
+
export class EventPublisherService {
|
|
154
|
+
constructor(private kinesisService: KinesisService) {}
|
|
155
|
+
|
|
156
|
+
async publishBusinessEvent<T>(eventType: string, entityId: string, eventData: T) {
|
|
157
|
+
const event = {
|
|
158
|
+
eventId: this.generateEventId(),
|
|
159
|
+
eventType,
|
|
160
|
+
entityId,
|
|
161
|
+
entityType: this.getEntityType(eventType),
|
|
162
|
+
timestamp: new Date().toISOString(),
|
|
163
|
+
version: '1.0',
|
|
164
|
+
data: eventData,
|
|
165
|
+
metadata: {
|
|
166
|
+
source: 'business-service',
|
|
167
|
+
correlationId: this.generateCorrelationId()
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
// Use entity ID as partition key to maintain order for the same entity
|
|
172
|
+
await this.kinesisService.publish(event, entityId);
|
|
173
|
+
|
|
174
|
+
console.log(`Published ${eventType} event for entity ${entityId}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async publishBulkEvents<T>(events: Array<{ eventType: string; entityId: string; data: T }>) {
|
|
178
|
+
const publishPromises = events.map(event =>
|
|
179
|
+
this.publishBusinessEvent(event.eventType, event.entityId, event.data)
|
|
180
|
+
);
|
|
181
|
+
|
|
182
|
+
await Promise.all(publishPromises);
|
|
183
|
+
console.log(`Published ${events.length} events to Kinesis stream`);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
private generateEventId(): string {
|
|
187
|
+
return `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private generateCorrelationId(): string {
|
|
191
|
+
return `corr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private getEntityType(eventType: string): string {
|
|
195
|
+
const entityMappings = {
|
|
196
|
+
'USER_CREATED': 'user',
|
|
197
|
+
'USER_UPDATED': 'user',
|
|
198
|
+
'ORDER_CREATED': 'order',
|
|
199
|
+
'ORDER_UPDATED': 'order',
|
|
200
|
+
'PAYMENT_PROCESSED': 'payment'
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
return entityMappings[eventType] || 'unknown';
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### Stream Analytics Service
|
|
209
|
+
|
|
210
|
+
```typescript
|
|
211
|
+
import { KinesisService } from '@onivoro/server-aws-kinesis';
|
|
212
|
+
|
|
213
|
+
@Injectable()
|
|
214
|
+
export class StreamAnalyticsService {
|
|
215
|
+
constructor(private kinesisService: KinesisService) {}
|
|
216
|
+
|
|
217
|
+
async publishUserBehavior(userId: string, action: string, context: any) {
|
|
218
|
+
const behaviorEvent = {
|
|
219
|
+
userId,
|
|
220
|
+
action,
|
|
221
|
+
context,
|
|
222
|
+
timestamp: new Date().toISOString(),
|
|
223
|
+
sessionId: context.sessionId,
|
|
224
|
+
deviceInfo: {
|
|
225
|
+
userAgent: context.userAgent,
|
|
226
|
+
ipAddress: context.ipAddress,
|
|
227
|
+
platform: context.platform
|
|
228
|
+
},
|
|
229
|
+
pageInfo: {
|
|
230
|
+
url: context.url,
|
|
231
|
+
referrer: context.referrer,
|
|
232
|
+
title: context.title
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
// Use user ID as partition key for user-specific analytics
|
|
237
|
+
await this.kinesisService.publish(behaviorEvent, userId);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async publishPerformanceMetrics(metrics: PerformanceMetrics) {
|
|
241
|
+
const performanceEvent = {
|
|
242
|
+
type: 'PERFORMANCE_METRICS',
|
|
243
|
+
metrics: {
|
|
244
|
+
responseTime: metrics.responseTime,
|
|
245
|
+
throughput: metrics.throughput,
|
|
246
|
+
errorRate: metrics.errorRate,
|
|
247
|
+
cpuUsage: metrics.cpuUsage,
|
|
248
|
+
memoryUsage: metrics.memoryUsage
|
|
249
|
+
},
|
|
250
|
+
timestamp: new Date().toISOString(),
|
|
251
|
+
service: metrics.serviceName,
|
|
252
|
+
environment: process.env.NODE_ENV || 'development'
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
// Use service name as partition key
|
|
256
|
+
await this.kinesisService.publish(performanceEvent, metrics.serviceName);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async publishBusinessInsights(insight: BusinessInsight) {
|
|
260
|
+
const insightEvent = {
|
|
261
|
+
type: 'BUSINESS_INSIGHT',
|
|
262
|
+
category: insight.category,
|
|
263
|
+
metric: insight.metric,
|
|
264
|
+
value: insight.value,
|
|
265
|
+
dimensions: insight.dimensions,
|
|
266
|
+
timestamp: new Date().toISOString(),
|
|
267
|
+
period: insight.period,
|
|
268
|
+
metadata: insight.metadata
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
// Use category as partition key for business insights
|
|
272
|
+
await this.kinesisService.publish(insightEvent, insight.category);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
### Partition Strategy Service
|
|
278
|
+
|
|
279
|
+
```typescript
|
|
280
|
+
import { KinesisService } from '@onivoro/server-aws-kinesis';
|
|
281
|
+
|
|
282
|
+
@Injectable()
|
|
283
|
+
export class PartitionStrategyService {
|
|
284
|
+
constructor(private kinesisService: KinesisService) {}
|
|
285
|
+
|
|
286
|
+
async publishWithHashPartitioning<T>(data: T, partitionField: string) {
|
|
287
|
+
const partitionKey = this.generateHashPartition(data[partitionField]);
|
|
288
|
+
await this.kinesisService.publish(data, partitionKey);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async publishWithTimeBasedPartitioning<T>(data: T, timeWindow: number = 60000) {
|
|
292
|
+
// Group events by time windows (default 1 minute)
|
|
293
|
+
const timeSlot = Math.floor(Date.now() / timeWindow);
|
|
294
|
+
const partitionKey = `time_${timeSlot}`;
|
|
295
|
+
await this.kinesisService.publish(data, partitionKey);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
async publishWithCustomPartitioning<T>(data: T, partitionStrategy: PartitionStrategy) {
|
|
299
|
+
let partitionKey: string;
|
|
300
|
+
|
|
301
|
+
switch (partitionStrategy.type) {
|
|
302
|
+
case 'random':
|
|
303
|
+
partitionKey = this.generateRandomPartition(partitionStrategy.shardCount);
|
|
304
|
+
break;
|
|
305
|
+
case 'round-robin':
|
|
306
|
+
partitionKey = this.generateRoundRobinPartition(partitionStrategy.shardCount);
|
|
307
|
+
break;
|
|
308
|
+
case 'field-based':
|
|
309
|
+
partitionKey = data[partitionStrategy.field];
|
|
310
|
+
break;
|
|
311
|
+
case 'composite':
|
|
312
|
+
partitionKey = this.generateCompositePartition(data, partitionStrategy.fields);
|
|
313
|
+
break;
|
|
314
|
+
default:
|
|
315
|
+
partitionKey = 'default';
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
await this.kinesisService.publish(data, partitionKey);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private generateHashPartition(value: string): string {
|
|
322
|
+
let hash = 0;
|
|
323
|
+
for (let i = 0; i < value.length; i++) {
|
|
324
|
+
const char = value.charCodeAt(i);
|
|
325
|
+
hash = ((hash << 5) - hash) + char;
|
|
326
|
+
hash = hash & hash; // Convert to 32bit integer
|
|
327
|
+
}
|
|
328
|
+
return `hash_${Math.abs(hash)}`;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
private generateRandomPartition(shardCount: number): string {
|
|
332
|
+
const randomNum = Math.floor(Math.random() * shardCount);
|
|
333
|
+
return `random_${randomNum}`;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
private generateRoundRobinPartition(shardCount: number): string {
|
|
337
|
+
// This would need to track state across calls
|
|
338
|
+
const counter = this.getRoundRobinCounter();
|
|
339
|
+
const partition = counter % shardCount;
|
|
340
|
+
this.incrementRoundRobinCounter();
|
|
341
|
+
return `rr_${partition}`;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
private generateCompositePartition(data: any, fields: string[]): string {
|
|
345
|
+
const values = fields.map(field => data[field]).join('_');
|
|
346
|
+
return `composite_${values}`;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
private getRoundRobinCounter(): number {
|
|
350
|
+
// Implementation would store counter state
|
|
351
|
+
return 0;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
private incrementRoundRobinCounter(): void {
|
|
355
|
+
// Implementation would increment counter state
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
### Stream Management Service
|
|
361
|
+
|
|
362
|
+
```typescript
|
|
363
|
+
import { KinesisClient, CreateStreamCommand, DescribeStreamCommand, DeleteStreamCommand } from '@aws-sdk/client-kinesis';
|
|
364
|
+
|
|
365
|
+
@Injectable()
|
|
366
|
+
export class KinesisStreamManagementService {
|
|
367
|
+
constructor(private kinesisClient: KinesisClient) {}
|
|
368
|
+
|
|
369
|
+
async createStream(streamName: string, shardCount: number = 1) {
|
|
370
|
+
const createStreamCommand = new CreateStreamCommand({
|
|
371
|
+
StreamName: streamName,
|
|
372
|
+
ShardCount: shardCount
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
return this.kinesisClient.send(createStreamCommand);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async getStreamStatus(streamName: string) {
|
|
379
|
+
const describeStreamCommand = new DescribeStreamCommand({
|
|
380
|
+
StreamName: streamName
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
return this.kinesisClient.send(describeStreamCommand);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
async waitForStreamActive(streamName: string, maxAttempts: number = 30) {
|
|
387
|
+
let attempts = 0;
|
|
388
|
+
|
|
389
|
+
while (attempts < maxAttempts) {
|
|
390
|
+
const response = await this.getStreamStatus(streamName);
|
|
391
|
+
const status = response.StreamDescription?.StreamStatus;
|
|
392
|
+
|
|
393
|
+
if (status === 'ACTIVE') {
|
|
394
|
+
console.log(`Stream ${streamName} is active`);
|
|
395
|
+
return response;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
if (status === 'DELETING') {
|
|
399
|
+
throw new Error(`Stream ${streamName} is being deleted`);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
console.log(`Stream ${streamName} status: ${status}, waiting...`);
|
|
403
|
+
await this.delay(10000); // Wait 10 seconds
|
|
404
|
+
attempts++;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
throw new Error(`Stream ${streamName} did not become active within timeout`);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
async deleteStream(streamName: string) {
|
|
411
|
+
const deleteStreamCommand = new DeleteStreamCommand({
|
|
412
|
+
StreamName: streamName
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
return this.kinesisClient.send(deleteStreamCommand);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async ensureStreamExists(streamName: string, shardCount: number = 1) {
|
|
419
|
+
try {
|
|
420
|
+
const response = await this.getStreamStatus(streamName);
|
|
421
|
+
console.log(`Stream ${streamName} already exists with status: ${response.StreamDescription?.StreamStatus}`);
|
|
422
|
+
return response;
|
|
423
|
+
} catch (error: any) {
|
|
424
|
+
if (error.name === 'ResourceNotFoundException') {
|
|
425
|
+
console.log(`Creating stream ${streamName} with ${shardCount} shards`);
|
|
426
|
+
await this.createStream(streamName, shardCount);
|
|
427
|
+
return this.waitForStreamActive(streamName);
|
|
428
|
+
}
|
|
429
|
+
throw error;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
private delay(ms: number): Promise<void> {
|
|
434
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
## Advanced Usage
|
|
440
|
+
|
|
441
|
+
### Stream Consumer Base Class
|
|
442
|
+
|
|
443
|
+
```typescript
|
|
444
|
+
import { KinesisClient, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
|
|
445
|
+
|
|
446
|
+
export abstract class KinesisConsumerBase {
|
|
447
|
+
protected abstract processRecord(record: any): Promise<void>;
|
|
448
|
+
|
|
449
|
+
constructor(
|
|
450
|
+
protected kinesisClient: KinesisClient,
|
|
451
|
+
protected streamName: string
|
|
452
|
+
) {}
|
|
453
|
+
|
|
454
|
+
async startConsuming(shardId: string, iteratorType: string = 'LATEST') {
|
|
455
|
+
try {
|
|
456
|
+
// Get shard iterator
|
|
457
|
+
const shardIteratorResponse = await this.kinesisClient.send(
|
|
458
|
+
new GetShardIteratorCommand({
|
|
459
|
+
StreamName: this.streamName,
|
|
460
|
+
ShardId: shardId,
|
|
461
|
+
ShardIteratorType: iteratorType
|
|
462
|
+
})
|
|
463
|
+
);
|
|
464
|
+
|
|
465
|
+
let shardIterator = shardIteratorResponse.ShardIterator;
|
|
466
|
+
|
|
467
|
+
// Start consuming records
|
|
468
|
+
while (shardIterator) {
|
|
469
|
+
const recordsResponse = await this.kinesisClient.send(
|
|
470
|
+
new GetRecordsCommand({
|
|
471
|
+
ShardIterator: shardIterator
|
|
472
|
+
})
|
|
473
|
+
);
|
|
474
|
+
|
|
475
|
+
const records = recordsResponse.Records || [];
|
|
476
|
+
|
|
477
|
+
if (records.length > 0) {
|
|
478
|
+
console.log(`Processing ${records.length} records`);
|
|
479
|
+
|
|
480
|
+
for (const record of records) {
|
|
481
|
+
try {
|
|
482
|
+
await this.processRecord(record);
|
|
483
|
+
} catch (error) {
|
|
484
|
+
console.error('Error processing record:', error);
|
|
485
|
+
// Implement your error handling strategy here
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
shardIterator = recordsResponse.NextShardIterator;
|
|
491
|
+
|
|
492
|
+
// Add delay to avoid hitting API limits
|
|
493
|
+
await this.delay(1000);
|
|
494
|
+
}
|
|
495
|
+
} catch (error) {
|
|
496
|
+
console.error('Error in consumer loop:', error);
|
|
497
|
+
throw error;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
private delay(ms: number): Promise<void> {
|
|
502
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
### Monitoring and Metrics
|
|
508
|
+
|
|
509
|
+
```typescript
|
|
510
|
+
@Injectable()
|
|
511
|
+
export class KinesisMonitoringService {
|
|
512
|
+
constructor(private kinesisService: KinesisService) {}
|
|
513
|
+
|
|
514
|
+
private metricsBuffer: Array<{ eventType: string; timestamp: number; size: number }> = [];
|
|
515
|
+
|
|
516
|
+
async publishWithMetrics<T>(data: T, partitionKey: string, eventType: string) {
|
|
517
|
+
const startTime = Date.now();
|
|
518
|
+
|
|
519
|
+
try {
|
|
520
|
+
await this.kinesisService.publish(data, partitionKey);
|
|
521
|
+
|
|
522
|
+
const endTime = Date.now();
|
|
523
|
+
const duration = endTime - startTime;
|
|
524
|
+
const dataSize = JSON.stringify(data).length;
|
|
525
|
+
|
|
526
|
+
this.recordMetrics(eventType, duration, dataSize, 'success');
|
|
527
|
+
} catch (error) {
|
|
528
|
+
const endTime = Date.now();
|
|
529
|
+
const duration = endTime - startTime;
|
|
530
|
+
|
|
531
|
+
this.recordMetrics(eventType, duration, 0, 'error');
|
|
532
|
+
throw error;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
private recordMetrics(eventType: string, duration: number, size: number, status: string) {
|
|
537
|
+
this.metricsBuffer.push({
|
|
538
|
+
eventType,
|
|
539
|
+
timestamp: Date.now(),
|
|
540
|
+
size
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
// Log metrics
|
|
544
|
+
console.log(`Kinesis publish - Type: ${eventType}, Duration: ${duration}ms, Size: ${size} bytes, Status: ${status}`);
|
|
545
|
+
|
|
546
|
+
// Publish metrics to monitoring system (e.g., CloudWatch)
|
|
547
|
+
this.publishMetricsToCloudWatch(eventType, duration, size, status);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
private async publishMetricsToCloudWatch(eventType: string, duration: number, size: number, status: string) {
|
|
551
|
+
// Implementation would send metrics to CloudWatch
|
|
552
|
+
// This is a placeholder for the actual implementation
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
getMetricsSummary() {
|
|
556
|
+
const now = Date.now();
|
|
557
|
+
const oneHourAgo = now - (60 * 60 * 1000);
|
|
558
|
+
|
|
559
|
+
const recentMetrics = this.metricsBuffer.filter(m => m.timestamp > oneHourAgo);
|
|
560
|
+
|
|
561
|
+
return {
|
|
562
|
+
totalEvents: recentMetrics.length,
|
|
563
|
+
totalSize: recentMetrics.reduce((sum, m) => sum + m.size, 0),
|
|
564
|
+
eventsByType: recentMetrics.reduce((acc, m) => {
|
|
565
|
+
acc[m.eventType] = (acc[m.eventType] || 0) + 1;
|
|
566
|
+
return acc;
|
|
567
|
+
}, {} as Record<string, number>)
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
## Best Practices
|
|
574
|
+
|
|
575
|
+
### 1. Partition Key Strategy
|
|
576
|
+
|
|
577
|
+
```typescript
|
|
578
|
+
// Good: Use entity ID for ordered processing
|
|
579
|
+
await kinesisService.publish(orderEvent, orderId);
|
|
580
|
+
|
|
581
|
+
// Good: Use hash for even distribution
|
|
582
|
+
const partitionKey = hashFunction(userId) % shardCount;
|
|
583
|
+
|
|
584
|
+
// Avoid: Using timestamp (creates hot shards)
|
|
585
|
+
// await kinesisService.publish(event, Date.now().toString());
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
### 2. Error Handling
|
|
589
|
+
|
|
590
|
+
```typescript
|
|
591
|
+
async safePublish<T>(data: T, partitionKey: string, retries: number = 3): Promise<boolean> {
|
|
592
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
593
|
+
try {
|
|
594
|
+
await this.kinesisService.publish(data, partitionKey);
|
|
595
|
+
return true;
|
|
596
|
+
} catch (error: any) {
|
|
597
|
+
console.error(`Publish attempt ${attempt} failed:`, error);
|
|
598
|
+
|
|
599
|
+
if (attempt === retries) {
|
|
600
|
+
console.error('Max retries reached, publish failed');
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// Exponential backoff
|
|
605
|
+
await this.delay(Math.pow(2, attempt) * 1000);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
return false;
|
|
609
|
+
}
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
### 3. Data Validation
|
|
613
|
+
|
|
614
|
+
```typescript
|
|
615
|
+
validateEventData<T>(data: T): boolean {
|
|
616
|
+
return data &&
|
|
617
|
+
typeof data === 'object' &&
|
|
618
|
+
JSON.stringify(data).length <= 1000000; // 1MB limit
|
|
619
|
+
}
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
## Testing
|
|
623
|
+
|
|
624
|
+
```typescript
|
|
625
|
+
import { Test, TestingModule } from '@nestjs/testing';
|
|
626
|
+
import { ServerAwsKinesisModule, KinesisService } from '@onivoro/server-aws-kinesis';
|
|
627
|
+
|
|
628
|
+
describe('KinesisService', () => {
|
|
629
|
+
let service: KinesisService;
|
|
630
|
+
|
|
631
|
+
beforeEach(async () => {
|
|
632
|
+
const module: TestingModule = await Test.createTestingModule({
|
|
633
|
+
imports: [ServerAwsKinesisModule.configure({
|
|
634
|
+
AWS_REGION: 'us-east-1',
|
|
635
|
+
AWS_KINESIS_NAME: 'test-stream',
|
|
636
|
+
AWS_PROFILE: 'test'
|
|
637
|
+
})],
|
|
638
|
+
}).compile();
|
|
639
|
+
|
|
640
|
+
service = module.get<KinesisService>(KinesisService);
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
it('should be defined', () => {
|
|
644
|
+
expect(service).toBeDefined();
|
|
645
|
+
});
|
|
646
|
+
|
|
647
|
+
it('should publish event', async () => {
|
|
648
|
+
const testData = { test: 'data' };
|
|
649
|
+
const partitionKey = 'test-key';
|
|
650
|
+
|
|
651
|
+
await expect(service.publish(testData, partitionKey)).resolves.not.toThrow();
|
|
652
|
+
});
|
|
653
|
+
});
|
|
654
|
+
```
|
|
655
|
+
|
|
656
|
+
## API Reference
|
|
657
|
+
|
|
658
|
+
### Exported Classes
|
|
659
|
+
- `ServerAwsKinesisConfig`: Configuration class for Kinesis settings
|
|
660
|
+
- `ServerAwsKinesisModule`: NestJS module for Kinesis integration
|
|
661
|
+
|
|
662
|
+
### Exported Services
|
|
663
|
+
- `KinesisService`: Main Kinesis service with data publishing capabilities
|
|
664
|
+
|
|
665
|
+
## License
|
|
666
|
+
|
|
667
|
+
This package is part of the Onivoro monorepo and follows the same licensing terms.
|
package/jest.config.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/* eslint-disable */
|
|
2
|
+
export default {
|
|
3
|
+
displayName: 'lib-server-aws-kinesis',
|
|
4
|
+
preset: '../../../jest.preset.js',
|
|
5
|
+
testEnvironment: 'node',
|
|
6
|
+
transform: {
|
|
7
|
+
'^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
|
|
8
|
+
},
|
|
9
|
+
moduleFileExtensions: ['ts', 'js', 'html'],
|
|
10
|
+
coverageDirectory: '../../../coverage/libs/server/aws-kinesis',
|
|
11
|
+
};
|
package/package.json
CHANGED
|
@@ -1,47 +1,12 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
|
|
12
|
-
],
|
|
13
|
-
"scripts": {
|
|
14
|
-
"onx": "onx",
|
|
15
|
-
"build": "onx Build",
|
|
16
|
-
"deploy": "onx Publish",
|
|
17
|
-
"test": "onx Test",
|
|
18
|
-
"update": "onx Update"
|
|
19
|
-
},
|
|
20
|
-
"exports": {
|
|
21
|
-
".": {
|
|
22
|
-
"types": "./dist/types/index.d.ts",
|
|
23
|
-
"require": "./dist/cjs/index.js",
|
|
24
|
-
"import": "./dist/esm/index.js",
|
|
25
|
-
"default": "./dist/esm/lib.js"
|
|
26
|
-
}
|
|
27
|
-
},
|
|
28
|
-
"onx": {
|
|
29
|
-
"platform": "server",
|
|
30
|
-
"module": "commonjs"
|
|
31
|
-
},
|
|
32
|
-
"devDependencies": {
|
|
33
|
-
"@onivoro/cli": "^22.0.8",
|
|
34
|
-
"@types/jest": "*",
|
|
35
|
-
"@types/node": "22.8.1",
|
|
36
|
-
"typescript": "*"
|
|
37
|
-
},
|
|
38
|
-
"engines": {
|
|
39
|
-
"node": "22.10.0",
|
|
40
|
-
"npm": "10.9.0"
|
|
41
|
-
},
|
|
42
|
-
"dependencies": {
|
|
43
|
-
"@aws-sdk/client-kinesis": "^3.699.0",
|
|
44
|
-
"@nestjs/common": "^10.4.6",
|
|
45
|
-
"@onivoro/server-common": "^22.0.3"
|
|
46
|
-
}
|
|
2
|
+
"name": "@onivoro/server-aws-kinesis",
|
|
3
|
+
"version": "24.0.0",
|
|
4
|
+
"type": "commonjs",
|
|
5
|
+
"main": "./src/index.js",
|
|
6
|
+
"types": "./src/index.d.ts",
|
|
7
|
+
"dependencies": {
|
|
8
|
+
"@onivoro/server-aws-credential-providers": "24.0.0",
|
|
9
|
+
"@onivoro/server-common": "24.0.0",
|
|
10
|
+
"tslib": "^2.3.0"
|
|
11
|
+
}
|
|
47
12
|
}
|
package/project.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "lib-server-aws-kinesis",
|
|
3
|
+
"$schema": "../../../node_modules/nx/schemas/project-schema.json",
|
|
4
|
+
"sourceRoot": "libs/server/aws-kinesis/src",
|
|
5
|
+
"projectType": "library",
|
|
6
|
+
"targets": {
|
|
7
|
+
"build": {
|
|
8
|
+
"executor": "@nx/js:tsc",
|
|
9
|
+
"outputs": ["{options.outputPath}"],
|
|
10
|
+
"options": {
|
|
11
|
+
"outputPath": "dist/libs/server/aws-kinesis",
|
|
12
|
+
"main": "libs/server/aws-kinesis/src/index.ts",
|
|
13
|
+
"tsConfig": "libs/server/aws-kinesis/tsconfig.lib.json",
|
|
14
|
+
"assets": [
|
|
15
|
+
"libs/server/aws-kinesis/README.md",
|
|
16
|
+
"libs/server/aws-kinesis/package.json"
|
|
17
|
+
],
|
|
18
|
+
"declaration": true
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"tags": []
|
|
23
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Module } from '@nestjs/common';
|
|
2
|
+
import { moduleFactory } from '@onivoro/server-common';
|
|
3
|
+
import { KinesisClient } from '@aws-sdk/client-kinesis';
|
|
4
|
+
import { KinesisService } from './services/kinesis.service';
|
|
5
|
+
import { ServerAwsKinesisConfig } from './classes/server-aws-kinesis-config.class';
|
|
6
|
+
import { AwsCredentials, ServerAwsCredentialProvidersModule } from '@onivoro/server-aws-credential-providers';
|
|
7
|
+
|
|
8
|
+
let kinesisClient: KinesisClient | null = null;
|
|
9
|
+
|
|
10
|
+
@Module({})
|
|
11
|
+
export class ServerAwsKinesisModule {
|
|
12
|
+
static configure(config: ServerAwsKinesisConfig) {
|
|
13
|
+
return moduleFactory({
|
|
14
|
+
module: ServerAwsKinesisModule,
|
|
15
|
+
imports: [ServerAwsCredentialProvidersModule.configure(config)],
|
|
16
|
+
providers: [
|
|
17
|
+
{
|
|
18
|
+
provide: KinesisClient,
|
|
19
|
+
useFactory: (credentials: AwsCredentials) => kinesisClient
|
|
20
|
+
? kinesisClient
|
|
21
|
+
: kinesisClient = new KinesisClient({
|
|
22
|
+
region: config.AWS_REGION,
|
|
23
|
+
logger: console,
|
|
24
|
+
credentials
|
|
25
|
+
}),
|
|
26
|
+
inject: [AwsCredentials]
|
|
27
|
+
},
|
|
28
|
+
{ provide: ServerAwsKinesisConfig, useValue: config },
|
|
29
|
+
KinesisService
|
|
30
|
+
]
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import { KinesisClient, PutRecordCommand, PutRecordCommandInput } from '@aws-sdk/client-kinesis';
|
|
3
|
+
import { ServerAwsKinesisConfig } from '../classes/server-aws-kinesis-config.class';
|
|
4
|
+
|
|
5
|
+
@Injectable()
|
|
6
|
+
export class KinesisService {
|
|
7
|
+
constructor(private kinesis: KinesisClient, private config: ServerAwsKinesisConfig) { }
|
|
8
|
+
|
|
9
|
+
async publish<TData>(event: TData, PartitionKey: string) {
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const input: PutRecordCommandInput = {
|
|
13
|
+
Data: Buffer.from(JSON.stringify(event)),
|
|
14
|
+
PartitionKey,
|
|
15
|
+
StreamName: this.config.AWS_KINESIS_NAME,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const command = new PutRecordCommand(input);
|
|
19
|
+
|
|
20
|
+
await this.kinesis.send(command);
|
|
21
|
+
} catch (err) {
|
|
22
|
+
console.error(`Error sending data to Kinesis:`, event, err);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "../../../tsconfig.server.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"outDir": "../../../dist/out-tsc"
|
|
5
|
+
},
|
|
6
|
+
"files": [],
|
|
7
|
+
"include": [],
|
|
8
|
+
"references": [
|
|
9
|
+
{
|
|
10
|
+
"path": "./tsconfig.lib.json"
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"path": "./tsconfig.spec.json"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "./tsconfig.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"types": [
|
|
5
|
+
"jest",
|
|
6
|
+
"node"
|
|
7
|
+
]
|
|
8
|
+
},
|
|
9
|
+
"include": [
|
|
10
|
+
"jest.config.ts",
|
|
11
|
+
"**/*.test.ts",
|
|
12
|
+
"**/*.spec.ts",
|
|
13
|
+
"**/*.test.tsx",
|
|
14
|
+
"**/*.spec.tsx",
|
|
15
|
+
"**/*.test.js",
|
|
16
|
+
"**/*.spec.js",
|
|
17
|
+
"**/*.test.jsx",
|
|
18
|
+
"**/*.spec.jsx",
|
|
19
|
+
"**/*.d.ts"
|
|
20
|
+
]
|
|
21
|
+
}
|
package/dist/cjs/index.js
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./lib/classes/server-aws-kinesis-config.class"), exports);
|
|
18
|
-
__exportStar(require("./lib/services/kinesis.service"), exports);
|
|
19
|
-
__exportStar(require("./lib/server-aws-kinesis.module"), exports);
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ServerAwsKinesisConfig = void 0;
|
|
4
|
-
class ServerAwsKinesisConfig {
|
|
5
|
-
AWS_ACCESS_KEY_ID;
|
|
6
|
-
AWS_KINESIS_NAME;
|
|
7
|
-
AWS_REGION;
|
|
8
|
-
AWS_SECRET_ACCESS_KEY;
|
|
9
|
-
NODE_ENV;
|
|
10
|
-
}
|
|
11
|
-
exports.ServerAwsKinesisConfig = ServerAwsKinesisConfig;
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var ServerAwsKinesisModule_1;
|
|
9
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.ServerAwsKinesisModule = void 0;
|
|
11
|
-
const common_1 = require("@nestjs/common");
|
|
12
|
-
const server_common_1 = require("@onivoro/server-common");
|
|
13
|
-
const client_kinesis_1 = require("@aws-sdk/client-kinesis");
|
|
14
|
-
const kinesis_service_1 = require("./services/kinesis.service");
|
|
15
|
-
const server_aws_kinesis_config_class_1 = require("./classes/server-aws-kinesis-config.class");
|
|
16
|
-
let kinesisClient = null;
|
|
17
|
-
let ServerAwsKinesisModule = ServerAwsKinesisModule_1 = class ServerAwsKinesisModule {
|
|
18
|
-
static configure(config) {
|
|
19
|
-
return (0, server_common_1.moduleFactory)({
|
|
20
|
-
module: ServerAwsKinesisModule_1,
|
|
21
|
-
providers: [
|
|
22
|
-
{
|
|
23
|
-
provide: client_kinesis_1.KinesisClient,
|
|
24
|
-
useFactory: () => kinesisClient
|
|
25
|
-
? kinesisClient
|
|
26
|
-
: kinesisClient = new client_kinesis_1.KinesisClient({
|
|
27
|
-
region: config.AWS_REGION,
|
|
28
|
-
logger: console,
|
|
29
|
-
credentials: config.NODE_ENV === 'production'
|
|
30
|
-
? undefined
|
|
31
|
-
: {
|
|
32
|
-
accessKeyId: config.AWS_ACCESS_KEY_ID,
|
|
33
|
-
secretAccessKey: config.AWS_SECRET_ACCESS_KEY
|
|
34
|
-
}
|
|
35
|
-
})
|
|
36
|
-
},
|
|
37
|
-
{ provide: server_aws_kinesis_config_class_1.ServerAwsKinesisConfig, useValue: config },
|
|
38
|
-
kinesis_service_1.KinesisService
|
|
39
|
-
]
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
};
|
|
43
|
-
exports.ServerAwsKinesisModule = ServerAwsKinesisModule;
|
|
44
|
-
exports.ServerAwsKinesisModule = ServerAwsKinesisModule = ServerAwsKinesisModule_1 = __decorate([
|
|
45
|
-
(0, common_1.Module)({})
|
|
46
|
-
], ServerAwsKinesisModule);
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { KinesisClient } from '@aws-sdk/client-kinesis';
|
|
2
|
-
import { ServerAwsKinesisConfig } from '../classes/server-aws-kinesis-config.class';
|
|
3
|
-
export declare class KinesisService {
|
|
4
|
-
private kinesis;
|
|
5
|
-
private config;
|
|
6
|
-
constructor(kinesis: KinesisClient, config: ServerAwsKinesisConfig);
|
|
7
|
-
publish<TData>(event: TData, PartitionKey: string): Promise<void>;
|
|
8
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.KinesisService = void 0;
|
|
13
|
-
const common_1 = require("@nestjs/common");
|
|
14
|
-
const client_kinesis_1 = require("@aws-sdk/client-kinesis");
|
|
15
|
-
const server_aws_kinesis_config_class_1 = require("../classes/server-aws-kinesis-config.class");
|
|
16
|
-
let KinesisService = class KinesisService {
|
|
17
|
-
kinesis;
|
|
18
|
-
config;
|
|
19
|
-
constructor(kinesis, config) {
|
|
20
|
-
this.kinesis = kinesis;
|
|
21
|
-
this.config = config;
|
|
22
|
-
}
|
|
23
|
-
async publish(event, PartitionKey) {
|
|
24
|
-
try {
|
|
25
|
-
const input = {
|
|
26
|
-
Data: Buffer.from(JSON.stringify(event)),
|
|
27
|
-
PartitionKey,
|
|
28
|
-
StreamName: this.config.AWS_KINESIS_NAME,
|
|
29
|
-
};
|
|
30
|
-
const command = new client_kinesis_1.PutRecordCommand(input);
|
|
31
|
-
await this.kinesis.send(command);
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
console.error(`Error sending data to Kinesis:`, event, err);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
exports.KinesisService = KinesisService;
|
|
39
|
-
exports.KinesisService = KinesisService = __decorate([
|
|
40
|
-
(0, common_1.Injectable)(),
|
|
41
|
-
__metadata("design:paramtypes", [client_kinesis_1.KinesisClient, server_aws_kinesis_config_class_1.ServerAwsKinesisConfig])
|
|
42
|
-
], KinesisService);
|
package/dist/esm/index.js
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./lib/classes/server-aws-kinesis-config.class"), exports);
|
|
18
|
-
__exportStar(require("./lib/services/kinesis.service"), exports);
|
|
19
|
-
__exportStar(require("./lib/server-aws-kinesis.module"), exports);
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ServerAwsKinesisConfig = void 0;
|
|
4
|
-
class ServerAwsKinesisConfig {
|
|
5
|
-
AWS_ACCESS_KEY_ID;
|
|
6
|
-
AWS_KINESIS_NAME;
|
|
7
|
-
AWS_REGION;
|
|
8
|
-
AWS_SECRET_ACCESS_KEY;
|
|
9
|
-
NODE_ENV;
|
|
10
|
-
}
|
|
11
|
-
exports.ServerAwsKinesisConfig = ServerAwsKinesisConfig;
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var ServerAwsKinesisModule_1;
|
|
9
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.ServerAwsKinesisModule = void 0;
|
|
11
|
-
const common_1 = require("@nestjs/common");
|
|
12
|
-
const server_common_1 = require("@onivoro/server-common");
|
|
13
|
-
const client_kinesis_1 = require("@aws-sdk/client-kinesis");
|
|
14
|
-
const kinesis_service_1 = require("./services/kinesis.service");
|
|
15
|
-
const server_aws_kinesis_config_class_1 = require("./classes/server-aws-kinesis-config.class");
|
|
16
|
-
let kinesisClient = null;
|
|
17
|
-
let ServerAwsKinesisModule = ServerAwsKinesisModule_1 = class ServerAwsKinesisModule {
|
|
18
|
-
static configure(config) {
|
|
19
|
-
return (0, server_common_1.moduleFactory)({
|
|
20
|
-
module: ServerAwsKinesisModule_1,
|
|
21
|
-
providers: [
|
|
22
|
-
{
|
|
23
|
-
provide: client_kinesis_1.KinesisClient,
|
|
24
|
-
useFactory: () => kinesisClient
|
|
25
|
-
? kinesisClient
|
|
26
|
-
: kinesisClient = new client_kinesis_1.KinesisClient({
|
|
27
|
-
region: config.AWS_REGION,
|
|
28
|
-
logger: console,
|
|
29
|
-
credentials: config.NODE_ENV === 'production'
|
|
30
|
-
? undefined
|
|
31
|
-
: {
|
|
32
|
-
accessKeyId: config.AWS_ACCESS_KEY_ID,
|
|
33
|
-
secretAccessKey: config.AWS_SECRET_ACCESS_KEY
|
|
34
|
-
}
|
|
35
|
-
})
|
|
36
|
-
},
|
|
37
|
-
{ provide: server_aws_kinesis_config_class_1.ServerAwsKinesisConfig, useValue: config },
|
|
38
|
-
kinesis_service_1.KinesisService
|
|
39
|
-
]
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
};
|
|
43
|
-
exports.ServerAwsKinesisModule = ServerAwsKinesisModule;
|
|
44
|
-
exports.ServerAwsKinesisModule = ServerAwsKinesisModule = ServerAwsKinesisModule_1 = __decorate([
|
|
45
|
-
(0, common_1.Module)({})
|
|
46
|
-
], ServerAwsKinesisModule);
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { KinesisClient } from '@aws-sdk/client-kinesis';
|
|
2
|
-
import { ServerAwsKinesisConfig } from '../classes/server-aws-kinesis-config.class';
|
|
3
|
-
export declare class KinesisService {
|
|
4
|
-
private kinesis;
|
|
5
|
-
private config;
|
|
6
|
-
constructor(kinesis: KinesisClient, config: ServerAwsKinesisConfig);
|
|
7
|
-
publish<TData>(event: TData, PartitionKey: string): Promise<void>;
|
|
8
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.KinesisService = void 0;
|
|
13
|
-
const common_1 = require("@nestjs/common");
|
|
14
|
-
const client_kinesis_1 = require("@aws-sdk/client-kinesis");
|
|
15
|
-
const server_aws_kinesis_config_class_1 = require("../classes/server-aws-kinesis-config.class");
|
|
16
|
-
let KinesisService = class KinesisService {
|
|
17
|
-
kinesis;
|
|
18
|
-
config;
|
|
19
|
-
constructor(kinesis, config) {
|
|
20
|
-
this.kinesis = kinesis;
|
|
21
|
-
this.config = config;
|
|
22
|
-
}
|
|
23
|
-
async publish(event, PartitionKey) {
|
|
24
|
-
try {
|
|
25
|
-
const input = {
|
|
26
|
-
Data: Buffer.from(JSON.stringify(event)),
|
|
27
|
-
PartitionKey,
|
|
28
|
-
StreamName: this.config.AWS_KINESIS_NAME,
|
|
29
|
-
};
|
|
30
|
-
const command = new client_kinesis_1.PutRecordCommand(input);
|
|
31
|
-
await this.kinesis.send(command);
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
console.error(`Error sending data to Kinesis:`, event, err);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
exports.KinesisService = KinesisService;
|
|
39
|
-
exports.KinesisService = KinesisService = __decorate([
|
|
40
|
-
(0, common_1.Injectable)(),
|
|
41
|
-
__metadata("design:paramtypes", [client_kinesis_1.KinesisClient, server_aws_kinesis_config_class_1.ServerAwsKinesisConfig])
|
|
42
|
-
], KinesisService);
|
package/dist/types/index.d.ts
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { KinesisClient } from '@aws-sdk/client-kinesis';
|
|
2
|
-
import { ServerAwsKinesisConfig } from '../classes/server-aws-kinesis-config.class';
|
|
3
|
-
export declare class KinesisService {
|
|
4
|
-
private kinesis;
|
|
5
|
-
private config;
|
|
6
|
-
constructor(kinesis: KinesisClient, config: ServerAwsKinesisConfig);
|
|
7
|
-
publish<TData>(event: TData, PartitionKey: string): Promise<void>;
|
|
8
|
-
}
|