@cumulus/ingest 21.1.0 → 21.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nycrc.json +1 -1
- package/consumerRateLimited.d.ts +43 -0
- package/consumerRateLimited.d.ts.map +1 -0
- package/consumerRateLimited.js +148 -0
- package/consumerRateLimited.js.map +1 -0
- package/package.json +14 -14
- package/src/consumerRateLimited.ts +186 -0
- package/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
package/.nycrc.json
CHANGED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { SQSMessage } from '@cumulus/aws-client/SQS';
|
|
2
|
+
export declare type MessageConsumerFunction = (queueUrl: string, message: SQSMessage) => Promise<void>;
|
|
3
|
+
/**
|
|
4
|
+
* Configuration parameters for the rate-limited consumer.
|
|
5
|
+
*/
|
|
6
|
+
export interface ConsumerConstructorParams {
|
|
7
|
+
/**
|
|
8
|
+
* URLs of the SQS queues to poll.
|
|
9
|
+
*/
|
|
10
|
+
queueUrls: string[];
|
|
11
|
+
/**
|
|
12
|
+
* Function that returns the remaining time in milliseconds before Lambda timeout.
|
|
13
|
+
*/
|
|
14
|
+
timeRemainingFunc: () => number;
|
|
15
|
+
/**
|
|
16
|
+
* The visibility timeout in milliseconds used when fetching messages from the SQS queues.
|
|
17
|
+
*/
|
|
18
|
+
visibilityTimeout: number;
|
|
19
|
+
/**
|
|
20
|
+
* Whether to delete messages after successful processing. Defaults to true.
|
|
21
|
+
*/
|
|
22
|
+
deleteProcessedMessage?: boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Maximum number of messages to process per second.
|
|
25
|
+
*/
|
|
26
|
+
rateLimitPerSecond: number;
|
|
27
|
+
}
|
|
28
|
+
export declare class ConsumerRateLimited {
|
|
29
|
+
private readonly deleteProcessedMessage;
|
|
30
|
+
private readonly queueUrls;
|
|
31
|
+
private readonly timeRemainingFunc;
|
|
32
|
+
private readonly visibilityTimeout;
|
|
33
|
+
private readonly rateLimitPerSecond;
|
|
34
|
+
private readonly messageLimitPerFetch;
|
|
35
|
+
private readonly waitTime;
|
|
36
|
+
constructor({ queueUrls, timeRemainingFunc, visibilityTimeout, rateLimitPerSecond, deleteProcessedMessage, }: ConsumerConstructorParams);
|
|
37
|
+
private processMessage;
|
|
38
|
+
private processMessages;
|
|
39
|
+
private fetchMessages;
|
|
40
|
+
private fetchMessagesFromAllQueues;
|
|
41
|
+
consume(fn: MessageConsumerFunction): Promise<number>;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=consumerRateLimited.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumerRateLimited.d.ts","sourceRoot":"","sources":["src/consumerRateLimited.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAMzE,oBAAY,uBAAuB,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAG/F;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB;;OAEG;IACH,iBAAiB,EAAE,MAAM,MAAM,CAAC;IAChC;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IACjD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoC;IACtE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAEtB,EACV,SAAS,EACT,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAA6B,GAC9B,EAAE,yBAAyB;YAYd,cAAc;YAuBd,eAAe;YAqBf,aAAa;YAWb,0BAA0B;IAOlC,OAAO,CAAC,EAAE,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC;CA6D5D"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.ConsumerRateLimited = void 0;
|
|
30
|
+
const SQS_1 = require("@cumulus/aws-client/SQS");
|
|
31
|
+
const sqs = __importStar(require("@cumulus/aws-client/SQS"));
|
|
32
|
+
const StepFunctions_1 = require("@cumulus/aws-client/StepFunctions");
|
|
33
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
34
|
+
const common_1 = require("@cumulus/common");
|
|
35
|
+
const log = new logger_1.default({ sender: '@cumulus/ingest/consumer' });
|
|
36
|
+
class ConsumerRateLimited {
|
|
37
|
+
constructor({ queueUrls, timeRemainingFunc, visibilityTimeout, rateLimitPerSecond, deleteProcessedMessage = true, }) {
|
|
38
|
+
this.queueUrls = queueUrls;
|
|
39
|
+
this.visibilityTimeout = visibilityTimeout;
|
|
40
|
+
this.timeRemainingFunc = timeRemainingFunc;
|
|
41
|
+
this.deleteProcessedMessage = deleteProcessedMessage;
|
|
42
|
+
this.rateLimitPerSecond = rateLimitPerSecond;
|
|
43
|
+
// The maximum number of messages to fetch in one request per queue
|
|
44
|
+
this.messageLimitPerFetch = 10;
|
|
45
|
+
// The amount of time to wait before retrying to fetch messages when none are found
|
|
46
|
+
this.waitTime = 5000;
|
|
47
|
+
}
|
|
48
|
+
async processMessage(message, fn, queueUrl) {
|
|
49
|
+
try {
|
|
50
|
+
await fn(queueUrl, message);
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
if (error instanceof StepFunctions_1.ExecutionAlreadyExists) {
|
|
54
|
+
log.debug('Deleting message for execution that already exists...');
|
|
55
|
+
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
|
|
56
|
+
log.debug('Completed deleting message.');
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
log.error(error);
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
if (this.deleteProcessedMessage) {
|
|
63
|
+
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
|
|
64
|
+
}
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
async processMessages(fn, messagesWithQueueUrls) {
|
|
68
|
+
let counter = 0;
|
|
69
|
+
for (const [message, queueUrl] of messagesWithQueueUrls) {
|
|
70
|
+
const waitTime = 1000 / this.rateLimitPerSecond;
|
|
71
|
+
log.debug(`Waiting for ${waitTime} ms`);
|
|
72
|
+
// We normally don't want to await in a loop due to decreased performance
|
|
73
|
+
// from running sequentially, but here we want to enforce rate limiting
|
|
74
|
+
// by specifically adding a delay to each loop iteration.
|
|
75
|
+
// eslint-disable-next-line no-await-in-loop
|
|
76
|
+
await (0, common_1.sleep)(waitTime);
|
|
77
|
+
// eslint-disable-next-line no-await-in-loop
|
|
78
|
+
if (await this.processMessage(message, fn, queueUrl)) {
|
|
79
|
+
counter += 1;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return counter;
|
|
83
|
+
}
|
|
84
|
+
async fetchMessages(queueUrl, messageLimit) {
|
|
85
|
+
const messages = await (0, SQS_1.receiveSQSMessages)(queueUrl, {
|
|
86
|
+
numOfMessages: messageLimit,
|
|
87
|
+
visibilityTimeout: this.visibilityTimeout,
|
|
88
|
+
});
|
|
89
|
+
return messages.map((message) => [message, queueUrl]);
|
|
90
|
+
}
|
|
91
|
+
async fetchMessagesFromAllQueues() {
|
|
92
|
+
return Promise.all(this.queueUrls.map((queueUrl) => this.fetchMessages(queueUrl, this.messageLimitPerFetch))).then((messageArrays) => messageArrays.flat());
|
|
93
|
+
}
|
|
94
|
+
async consume(fn) {
|
|
95
|
+
let messageCounter = 0;
|
|
96
|
+
let processingPromise;
|
|
97
|
+
let fetchPromise;
|
|
98
|
+
let messages;
|
|
99
|
+
let processTimeMilliseconds = 0;
|
|
100
|
+
let startTime;
|
|
101
|
+
// The below block of code attempts to always have a batch of messages
|
|
102
|
+
// available for `processMessages` to process, so, after the initial fetch,
|
|
103
|
+
// we'll immediately start fetching the next batch while processing the
|
|
104
|
+
// current one
|
|
105
|
+
// There are several await-in-loop instances below all required for flow control to assure
|
|
106
|
+
// we're submitting at a specified rate.
|
|
107
|
+
while (this.timeRemainingFunc(processTimeMilliseconds) > 0) {
|
|
108
|
+
if (messages === undefined) {
|
|
109
|
+
// This will be run in the first iteration, included in the loop in case of a small
|
|
110
|
+
// timeRemainingFunc value
|
|
111
|
+
// eslint-disable-next-line no-await-in-loop
|
|
112
|
+
messages = await this.fetchMessagesFromAllQueues();
|
|
113
|
+
}
|
|
114
|
+
if (messages.length === 0) {
|
|
115
|
+
log.info(`No messages fetched, waiting ${this.waitTime} ms before retrying`);
|
|
116
|
+
// eslint-disable-next-line no-await-in-loop
|
|
117
|
+
await (0, common_1.sleep)(this.waitTime);
|
|
118
|
+
// eslint-disable-next-line no-await-in-loop
|
|
119
|
+
messages = await this.fetchMessagesFromAllQueues();
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
// Start processing current batch and immediately fetch next batch
|
|
123
|
+
processingPromise = this.processMessages(fn, messages);
|
|
124
|
+
fetchPromise = this.fetchMessagesFromAllQueues();
|
|
125
|
+
startTime = Date.now();
|
|
126
|
+
// Wait for processing to complete and increment counter
|
|
127
|
+
// eslint-disable-next-line no-await-in-loop
|
|
128
|
+
messageCounter += await processingPromise;
|
|
129
|
+
if (processTimeMilliseconds === 0) {
|
|
130
|
+
// First processing time measurement, add 50% buffer to account for possible longer
|
|
131
|
+
// processing time on the last iteration
|
|
132
|
+
processTimeMilliseconds = (Date.now() - startTime) + (Date.now() - startTime) * 0.5;
|
|
133
|
+
}
|
|
134
|
+
// Get the next batch that was fetched concurrently
|
|
135
|
+
// eslint-disable-next-line no-await-in-loop
|
|
136
|
+
messages = await fetchPromise;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// Process any remaining messages after time has expired
|
|
140
|
+
if (messages !== undefined && messages.length > 0) {
|
|
141
|
+
messageCounter += await this.processMessages(fn, messages);
|
|
142
|
+
}
|
|
143
|
+
log.info(`${messageCounter} messages successfully processed from ${this.queueUrls}`);
|
|
144
|
+
return messageCounter;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
exports.ConsumerRateLimited = ConsumerRateLimited;
|
|
148
|
+
//# sourceMappingURL=consumerRateLimited.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumerRateLimited.js","sourceRoot":"","sources":["src/consumerRateLimited.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAyE;AACzE,6DAA+C;AAC/C,qEAA2E;AAC3E,6DAAqC;AACrC,4CAAwC;AAIxC,MAAM,GAAG,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AA2B/D,MAAa,mBAAmB;IAS9B,YAAY,EACV,SAAS,EACT,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,GAAG,IAAI,GACH;QAC1B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;QACrD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,mEAAmE;QACnE,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,mFAAmF;QACnF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAAmB,EACnB,EAA2B,EAC3B,QAAgB;QAEhB,IAAI;YACF,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;SAC7B;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,sCAAsB,EAAE;gBAC3C,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBACnE,MAAM,GAAG,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC5D,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBACzC,OAAO,IAAI,CAAC;aACb;YACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO,KAAK,CAAC;SACd;QACD,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC/B,MAAM,GAAG,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;SAC7D;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,EAA2B,EAC3B,qBAAkD;QAElD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,qBAAqB,EAAE;YACvD,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC;YAChD,GAAG,CAAC,KAAK,CAAC,eAAe,QAAQ,KAAK,CAAC,CAAC;YACxC,yEAAyE;YACzE,uEAAuE;YACvE,yDAAyD;YACzD,4CAA4C;YAC5C,MAAM,IAAA,cAAK,EAAC,QAAQ,CAAC,CAAC;YACtB,4CAA4C;YAC5C,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACpD,OAAO,IAAI,CAAC,CAAC;aACd;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,QAAgB,EAChB,YAAoB;QAEpB,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAkB,EAAC,QAAQ,EAAE;YAClD,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;SAC1C,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,0BAA0B;QACtC,OAAO,OAAO,CAAC,GAAG,CAChB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC9B,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAC3D,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAA2B;QACvC,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,iBAA8C,CAAC;QACnD,IAAI,YAA8D,CAAC;QACnE,IAAI,QAAiD,CAAC;QACtD,IAAI,uBAAuB,GAAW,CAAC,CAAC;QACxC,IAAI,SAAiB,CAAC;QAEtB,sEAAsE;QACtE,2EAA2E;QAC3E,uEAAuE;QACvE,cAAc;QAEd,0FAA0F;QAC1F,wCAAwC;QACxC,OAAO,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE;YAC1D,IAAI,QAAQ,KAAK,SAAS,EAAE;gBAC1B,mFAAmF;gBACnF,0BAA0B;gBAC1B,4CAA4C;gBAC5C,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;aACpD;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;gBACzB,GAAG,CAAC,IAAI,CACN,gCAAgC,IAAI,CAAC,QAAQ,qBAAqB,CACnE,CAAC;gBACF,4CAA4C;gBAC5C,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,4CAA4C;gBAC5C,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;aACpD;iBAAM;gBACL,kEAAkE;gBAClE,iBAAiB,GAAG,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBACvD,YAAY,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAEjD,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,wDAAwD;gBACxD,4CAA4C;gBAC5C,cAAc,IAAI,MAAM,iBAAiB,CAAC;gBAC1C,IAAI,uBAAuB,KAAK,CAAC,EAAE;oBACjC,mFAAmF;oBACnF,wCAAwC;oBACxC,uBAAuB,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC;iBACrF;gBAED,mDAAmD;gBACnD,4CAA4C;gBAC5C,QAAQ,GAAG,MAAM,YAAY,CAAC;aAC/B;SACF;QAED,wDAAwD;QACxD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACjD,cAAc,IAAI,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;SAC5D;QAED,GAAG,CAAC,IAAI,CACN,GAAG,cAAc,yCAAyC,IAAI,CAAC,SAAS,EAAE,CAC3E,CAAC;QACF,OAAO,cAAc,CAAC;IACxB,CAAC;CACF;AAtJD,kDAsJC"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "21.
|
|
3
|
+
"version": "21.3.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=
|
|
6
|
+
"node": ">=22.21.1"
|
|
7
7
|
},
|
|
8
8
|
"scripts": {
|
|
9
9
|
"build": "rm -rf dist && mkdir dist && npm run prepare",
|
|
@@ -41,13 +41,13 @@
|
|
|
41
41
|
"author": "Cumulus Authors",
|
|
42
42
|
"license": "Apache-2.0",
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@cumulus/aws-client": "21.
|
|
45
|
-
"@cumulus/common": "21.
|
|
46
|
-
"@cumulus/db": "21.
|
|
47
|
-
"@cumulus/errors": "21.
|
|
48
|
-
"@cumulus/logger": "21.
|
|
49
|
-
"@cumulus/message": "21.
|
|
50
|
-
"@cumulus/sftp-client": "21.
|
|
44
|
+
"@cumulus/aws-client": "21.3.0",
|
|
45
|
+
"@cumulus/common": "21.3.0",
|
|
46
|
+
"@cumulus/db": "21.3.0",
|
|
47
|
+
"@cumulus/errors": "21.3.0",
|
|
48
|
+
"@cumulus/logger": "21.3.0",
|
|
49
|
+
"@cumulus/message": "21.3.0",
|
|
50
|
+
"@cumulus/sftp-client": "21.3.0",
|
|
51
51
|
"cksum": "^1.3.0",
|
|
52
52
|
"encodeurl": "^1.0.2",
|
|
53
53
|
"fs-extra": "^5.0.0",
|
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "21.
|
|
65
|
-
"@cumulus/cmrjs": "21.
|
|
66
|
-
"@cumulus/test-data": "21.
|
|
67
|
-
"@cumulus/types": "21.
|
|
64
|
+
"@cumulus/checksum": "21.3.0",
|
|
65
|
+
"@cumulus/cmrjs": "21.3.0",
|
|
66
|
+
"@cumulus/test-data": "21.3.0",
|
|
67
|
+
"@cumulus/types": "21.3.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "9dd4a7d4b888ea713efa87d6086a762262d46dd2"
|
|
70
70
|
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { receiveSQSMessages, SQSMessage } from '@cumulus/aws-client/SQS';
|
|
2
|
+
import * as sqs from '@cumulus/aws-client/SQS';
|
|
3
|
+
import { ExecutionAlreadyExists } from '@cumulus/aws-client/StepFunctions';
|
|
4
|
+
import Logger from '@cumulus/logger';
|
|
5
|
+
import { sleep } from '@cumulus/common';
|
|
6
|
+
|
|
7
|
+
export type MessageConsumerFunction = (queueUrl: string, message: SQSMessage) => Promise<void>;
|
|
8
|
+
|
|
9
|
+
const log = new Logger({ sender: '@cumulus/ingest/consumer' });
|
|
10
|
+
/**
|
|
11
|
+
* Configuration parameters for the rate-limited consumer.
|
|
12
|
+
*/
|
|
13
|
+
export interface ConsumerConstructorParams {
|
|
14
|
+
/**
|
|
15
|
+
* URLs of the SQS queues to poll.
|
|
16
|
+
*/
|
|
17
|
+
queueUrls: string[];
|
|
18
|
+
/**
|
|
19
|
+
* Function that returns the remaining time in milliseconds before Lambda timeout.
|
|
20
|
+
*/
|
|
21
|
+
timeRemainingFunc: () => number;
|
|
22
|
+
/**
|
|
23
|
+
* The visibility timeout in milliseconds used when fetching messages from the SQS queues.
|
|
24
|
+
*/
|
|
25
|
+
visibilityTimeout: number;
|
|
26
|
+
/**
|
|
27
|
+
* Whether to delete messages after successful processing. Defaults to true.
|
|
28
|
+
*/
|
|
29
|
+
deleteProcessedMessage?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Maximum number of messages to process per second.
|
|
32
|
+
*/
|
|
33
|
+
rateLimitPerSecond: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export class ConsumerRateLimited {
|
|
37
|
+
private readonly deleteProcessedMessage: boolean;
|
|
38
|
+
private readonly queueUrls: string[];
|
|
39
|
+
private readonly timeRemainingFunc: (bufferSeconds: number) => number;
|
|
40
|
+
private readonly visibilityTimeout: number;
|
|
41
|
+
private readonly rateLimitPerSecond: number;
|
|
42
|
+
private readonly messageLimitPerFetch: number;
|
|
43
|
+
private readonly waitTime: number;
|
|
44
|
+
|
|
45
|
+
constructor({
|
|
46
|
+
queueUrls,
|
|
47
|
+
timeRemainingFunc,
|
|
48
|
+
visibilityTimeout,
|
|
49
|
+
rateLimitPerSecond,
|
|
50
|
+
deleteProcessedMessage = true,
|
|
51
|
+
}: ConsumerConstructorParams) {
|
|
52
|
+
this.queueUrls = queueUrls;
|
|
53
|
+
this.visibilityTimeout = visibilityTimeout;
|
|
54
|
+
this.timeRemainingFunc = timeRemainingFunc;
|
|
55
|
+
this.deleteProcessedMessage = deleteProcessedMessage;
|
|
56
|
+
this.rateLimitPerSecond = rateLimitPerSecond;
|
|
57
|
+
// The maximum number of messages to fetch in one request per queue
|
|
58
|
+
this.messageLimitPerFetch = 10;
|
|
59
|
+
// The amount of time to wait before retrying to fetch messages when none are found
|
|
60
|
+
this.waitTime = 5000;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private async processMessage(
|
|
64
|
+
message: SQSMessage,
|
|
65
|
+
fn: MessageConsumerFunction,
|
|
66
|
+
queueUrl: string
|
|
67
|
+
): Promise<boolean> {
|
|
68
|
+
try {
|
|
69
|
+
await fn(queueUrl, message);
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error instanceof ExecutionAlreadyExists) {
|
|
72
|
+
log.debug('Deleting message for execution that already exists...');
|
|
73
|
+
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
|
|
74
|
+
log.debug('Completed deleting message.');
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
log.error(error);
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
if (this.deleteProcessedMessage) {
|
|
81
|
+
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
|
|
82
|
+
}
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private async processMessages(
|
|
87
|
+
fn: MessageConsumerFunction,
|
|
88
|
+
messagesWithQueueUrls: Array<[SQSMessage, string]>
|
|
89
|
+
): Promise<number> {
|
|
90
|
+
let counter = 0;
|
|
91
|
+
for (const [message, queueUrl] of messagesWithQueueUrls) {
|
|
92
|
+
const waitTime = 1000 / this.rateLimitPerSecond;
|
|
93
|
+
log.debug(`Waiting for ${waitTime} ms`);
|
|
94
|
+
// We normally don't want to await in a loop due to decreased performance
|
|
95
|
+
// from running sequentially, but here we want to enforce rate limiting
|
|
96
|
+
// by specifically adding a delay to each loop iteration.
|
|
97
|
+
// eslint-disable-next-line no-await-in-loop
|
|
98
|
+
await sleep(waitTime);
|
|
99
|
+
// eslint-disable-next-line no-await-in-loop
|
|
100
|
+
if (await this.processMessage(message, fn, queueUrl)) {
|
|
101
|
+
counter += 1;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return counter;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async fetchMessages(
|
|
108
|
+
queueUrl: string,
|
|
109
|
+
messageLimit: number
|
|
110
|
+
): Promise<Array<[SQSMessage, string]>> {
|
|
111
|
+
const messages = await receiveSQSMessages(queueUrl, {
|
|
112
|
+
numOfMessages: messageLimit,
|
|
113
|
+
visibilityTimeout: this.visibilityTimeout,
|
|
114
|
+
});
|
|
115
|
+
return messages.map((message) => [message, queueUrl]);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private async fetchMessagesFromAllQueues(): Promise<Array<[SQSMessage, string]>> {
|
|
119
|
+
return Promise.all(
|
|
120
|
+
this.queueUrls.map((queueUrl) =>
|
|
121
|
+
this.fetchMessages(queueUrl, this.messageLimitPerFetch))
|
|
122
|
+
).then((messageArrays) => messageArrays.flat());
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async consume(fn: MessageConsumerFunction): Promise<number> {
|
|
126
|
+
let messageCounter = 0;
|
|
127
|
+
let processingPromise: Promise<number> | undefined;
|
|
128
|
+
let fetchPromise: Promise<Array<[SQSMessage, string]>> | undefined;
|
|
129
|
+
let messages: Array<[SQSMessage, string]> | undefined;
|
|
130
|
+
let processTimeMilliseconds: number = 0;
|
|
131
|
+
let startTime: number;
|
|
132
|
+
|
|
133
|
+
// The below block of code attempts to always have a batch of messages
|
|
134
|
+
// available for `processMessages` to process, so, after the initial fetch,
|
|
135
|
+
// we'll immediately start fetching the next batch while processing the
|
|
136
|
+
// current one
|
|
137
|
+
|
|
138
|
+
// There are several await-in-loop instances below all required for flow control to assure
|
|
139
|
+
// we're submitting at a specified rate.
|
|
140
|
+
while (this.timeRemainingFunc(processTimeMilliseconds) > 0) {
|
|
141
|
+
if (messages === undefined) {
|
|
142
|
+
// This will be run in the first iteration, included in the loop in case of a small
|
|
143
|
+
// timeRemainingFunc value
|
|
144
|
+
// eslint-disable-next-line no-await-in-loop
|
|
145
|
+
messages = await this.fetchMessagesFromAllQueues();
|
|
146
|
+
}
|
|
147
|
+
if (messages.length === 0) {
|
|
148
|
+
log.info(
|
|
149
|
+
`No messages fetched, waiting ${this.waitTime} ms before retrying`
|
|
150
|
+
);
|
|
151
|
+
// eslint-disable-next-line no-await-in-loop
|
|
152
|
+
await sleep(this.waitTime);
|
|
153
|
+
// eslint-disable-next-line no-await-in-loop
|
|
154
|
+
messages = await this.fetchMessagesFromAllQueues();
|
|
155
|
+
} else {
|
|
156
|
+
// Start processing current batch and immediately fetch next batch
|
|
157
|
+
processingPromise = this.processMessages(fn, messages);
|
|
158
|
+
fetchPromise = this.fetchMessagesFromAllQueues();
|
|
159
|
+
|
|
160
|
+
startTime = Date.now();
|
|
161
|
+
// Wait for processing to complete and increment counter
|
|
162
|
+
// eslint-disable-next-line no-await-in-loop
|
|
163
|
+
messageCounter += await processingPromise;
|
|
164
|
+
if (processTimeMilliseconds === 0) {
|
|
165
|
+
// First processing time measurement, add 50% buffer to account for possible longer
|
|
166
|
+
// processing time on the last iteration
|
|
167
|
+
processTimeMilliseconds = (Date.now() - startTime) + (Date.now() - startTime) * 0.5;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Get the next batch that was fetched concurrently
|
|
171
|
+
// eslint-disable-next-line no-await-in-loop
|
|
172
|
+
messages = await fetchPromise;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Process any remaining messages after time has expired
|
|
177
|
+
if (messages !== undefined && messages.length > 0) {
|
|
178
|
+
messageCounter += await this.processMessages(fn, messages);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
log.info(
|
|
182
|
+
`${messageCounter} messages successfully processed from ${this.queueUrls}`
|
|
183
|
+
);
|
|
184
|
+
return messageCounter;
|
|
185
|
+
}
|
|
186
|
+
}
|
package/tsconfig.json
CHANGED