@creative-realities/cutie 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -0
- package/index.js +591 -0
- package/package.json +30 -0
- package/uuid.js +52 -0
package/README.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# An AWS SQS package
|
|
2
|
+
## Setting Up SQS
|
|
3
|
+
First configure your SQS queue on AWs. Cutie is a minimalist package, so configure your queue's default values to what you want in the AWS interface (cutie does not provide an interface for this).
|
|
4
|
+
#### Please note:
|
|
5
|
+
- Cutie only processes one message at a time, so your queue should be set up for one message at a time.
|
|
6
|
+
- Cutie does not have an interface for configuring credentials. You must set this up through AWS roles.
|
|
7
|
+
- Cutie expects your queue to be configured with a Default Visibility Timeout of 2 minutes.
|
|
8
|
+
|
|
9
|
+
## Setting up cutie
|
|
10
|
+
```
|
|
11
|
+
var Cutie = require("cutie");
|
|
12
|
+
var options = {
|
|
13
|
+
verbose: true, // defaults to false
|
|
14
|
+
};
|
|
15
|
+
var cutie = new Cutie("http://queue.url.whatever"); // URL from AWS
|
|
16
|
+
```
|
|
17
|
+
## Adding tasks
|
|
18
|
+
Messages sent to cutie require a task property. This tells cutie which previously added task to perform. The task callback receives the message object and a done callback.
|
|
19
|
+
Task callbacks must always call the done method within 2 minutes, which it expects to be the Default Visibility Timeout of your queue. See the Changing the Timeout section if you need to extend the timeout.
|
|
20
|
+
It is very important that every code path in your task always completes by calling the done callback. The done callback takes an error parameter. If you pass an error parameter to the done callback,
|
|
21
|
+
cutie will not remove the message from the SQS queue; it will remain in the queue to be reprocessed.
|
|
22
|
+
```
|
|
23
|
+
cutie.addTask("userRegistrationNotification", function(message, done) {
|
|
24
|
+
// message is the object passed to send.
|
|
25
|
+
var user = message.user;
|
|
26
|
+
|
|
27
|
+
// Do stuff in some async method...
|
|
28
|
+
asyncMethod(user, function(err) {
|
|
29
|
+
if (err)
|
|
30
|
+
return done(err); // message will NOT be removed from queue
|
|
31
|
+
|
|
32
|
+
done(); // message will be removed from queue
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
```
|
|
36
|
+
## Start polling
|
|
37
|
+
```
|
|
38
|
+
cutie.start();
|
|
39
|
+
```
|
|
40
|
+
## Sending messages to queue
|
|
41
|
+
```
|
|
42
|
+
// The send method will continue to retry up to 100 times if send fails when no callback is provided.
|
|
43
|
+
cutie.send({
|
|
44
|
+
task: "userRegistrationNotification",
|
|
45
|
+
user: user
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
// If you provide a callback and optional context, cutie will not retry to send.
|
|
49
|
+
cutie.send(message, function(error, data) { /* Do stuff */ }, this);
|
|
50
|
+
```
|
|
51
|
+
## Sending message batch to queue
|
|
52
|
+
```
|
|
53
|
+
cutie.send([
|
|
54
|
+
{
|
|
55
|
+
task: "userRegistrationNotification",
|
|
56
|
+
user: user1,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
task: "userRegistrationNotification",
|
|
60
|
+
user: user2,
|
|
61
|
+
}
|
|
62
|
+
// ... etc.
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
// When an array parameter is passed in, cutie acutally calls sendBatch(batch [, callback, context])
|
|
66
|
+
```
|
|
67
|
+
## Changing the Timeout
|
|
68
|
+
Task callbacks will recieve a third object parameter that includes the changeTimeout method. Calling changeTimeout will extend the time that cuite and SQS allow for the task to complete.
|
|
69
|
+
SQS will not make the message available again, until the new timeout interval has elapsed. This should be done at the very beginning of your callback.
|
|
70
|
+
In the example, we'll give the task 10 minutes to complete which we pass in as seconds.
|
|
71
|
+
```
|
|
72
|
+
cutie.addTask("userRegistrationNotification", function(message, done, options) {
|
|
73
|
+
var taskTimeoutSeconds = 10 * 60;
|
|
74
|
+
options.changeTimeout(taskTimeoutSeconds);
|
|
75
|
+
// Do other stuff...
|
|
76
|
+
});
|
|
77
|
+
```
|
|
78
|
+
## Multiple queues
|
|
79
|
+
As of 3.0 cutie can be configured with multiple queues in order of priority. When ready to process a new message, cutie checks the queues in order of priority.
|
|
80
|
+
### Configuring
|
|
81
|
+
To configure cutie to check multiple queues in order of priority, pass an array of queue URLs sorted by priority where the highest priority is at index zero.
|
|
82
|
+
```
|
|
83
|
+
var cutie = new Cutie(["http://queue.url.priority", "http://queue.url.normal"]); // URL from AWS
|
|
84
|
+
```
|
|
85
|
+
### Sending
|
|
86
|
+
The send method takes an optional priority parameter, which should match the index into the array of priority sorted queues. Cutie will limit the range of the priority argument to valid array indices instead of giving an error, so if you pass in a priority `2` and the array only has one queue at index `0`, it will use index `0`. If you omit the priority argument index `0` will always be used.
|
|
87
|
+
```
|
|
88
|
+
var priorityQueueIndex = 0;
|
|
89
|
+
var normalQueueIndex = 1;
|
|
90
|
+
|
|
91
|
+
cutie.send(normalQueueIndex, message);
|
|
92
|
+
```
|
|
93
|
+
## Task locking
|
|
94
|
+
```
|
|
95
|
+
// Assuming your callback parameter is named done, you may indicate that a task cannot be completed,
|
|
96
|
+
// but needs to be tried later and is not a real error to be logged like this:
|
|
97
|
+
|
|
98
|
+
if (taskIsLocked)
|
|
99
|
+
return done({cutieMessage: "Task Locked"});
|
|
100
|
+
|
|
101
|
+
```
|
package/index.js
ADDED
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
var aws = require("aws-sdk");
|
|
4
|
+
var log = require("logger");
|
|
5
|
+
var fs = require("fs");
|
|
6
|
+
var path = require("path");
|
|
7
|
+
var util = require("util");
|
|
8
|
+
|
|
9
|
+
var initializing = true;
|
|
10
|
+
|
|
11
|
+
class Cutie {
|
|
12
|
+
constructor(url, options) {
|
|
13
|
+
this.tasks = {};
|
|
14
|
+
this.verbose = !!(options || (options = {})).verbose; // @doc
|
|
15
|
+
this.logTasks = !!options.logTasks; // @doc
|
|
16
|
+
this.excludeLogTasks = options.excludeLogTasks || []; // @doc
|
|
17
|
+
|
|
18
|
+
if (options.serverId) {
|
|
19
|
+
this._serverId = options.serverId;
|
|
20
|
+
}
|
|
21
|
+
else if (!initializing && process.env.NODE_APP_INSTANCE === "0") {
|
|
22
|
+
// The first instance in a pm2 cluster is responsible for writing the serverId.
|
|
23
|
+
var serverId = this.getServerId();
|
|
24
|
+
if (!serverId) {
|
|
25
|
+
// Create serverId file
|
|
26
|
+
try {
|
|
27
|
+
var uuid = require(path.resolve(__dirname, "uuid")).generate();
|
|
28
|
+
fs.writeFileSync(path.resolve(__dirname, "server-id.txt"), uuid, {encoding: "utf8"});
|
|
29
|
+
}
|
|
30
|
+
catch(error) {
|
|
31
|
+
log.e("Cutie error writing serverId file!!!", error);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (!url) {
|
|
37
|
+
if (!initializing)
|
|
38
|
+
log.w("Warning: Cutie url not configured: " + url);
|
|
39
|
+
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
var firstUrl;
|
|
45
|
+
var urlArray;
|
|
46
|
+
if (Object.prototype.toString.call(url) === "[object String]") {
|
|
47
|
+
urlArray = [url];
|
|
48
|
+
}
|
|
49
|
+
else if (Array.isArray(url)) {
|
|
50
|
+
urlArray = url;
|
|
51
|
+
|
|
52
|
+
for (let urlString of urlArray) {
|
|
53
|
+
if (Object.prototype.toString.call(urlString) !== "[object String]") {
|
|
54
|
+
return log.e("Cutie error url array must only contain String. Received:", urlString);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return log.e("Cutie error url parameter must be a String or an Array of String. Received:", url);
|
|
60
|
+
}
|
|
61
|
+
this._queueCount = urlArray.length;
|
|
62
|
+
|
|
63
|
+
var regex = /sqs\.([^.]+)\.amazonaws/;
|
|
64
|
+
var testUrl = urlArray[0];
|
|
65
|
+
var matches = testUrl.match(regex);
|
|
66
|
+
if (!matches || matches.length < 2) {
|
|
67
|
+
log.w("Warning: Cutie could not parse region from url: ", testUrl, url);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
var region = matches[1];
|
|
71
|
+
this.sqs = new aws.SQS({region: region});
|
|
72
|
+
this.queueUrls = urlArray;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
getServerId() {
|
|
76
|
+
if (!this._serverId) {
|
|
77
|
+
try {
|
|
78
|
+
this._serverId = fs.readFileSync(path.resolve(__dirname, "server-id.txt"), {encoding: "utf8"});
|
|
79
|
+
}
|
|
80
|
+
catch(error) {
|
|
81
|
+
// Do nothing
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return this._serverId;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
send(/* [ priority, ] message, callback, context */) {
|
|
88
|
+
var priority;
|
|
89
|
+
var message;
|
|
90
|
+
var callback;
|
|
91
|
+
var context;
|
|
92
|
+
var sendUrl;
|
|
93
|
+
|
|
94
|
+
[priority, message, callback, context] = _parseSendParameters.call(this, arguments);
|
|
95
|
+
|
|
96
|
+
if (Object.prototype.toString.call(message) === "[object Array]")
|
|
97
|
+
return this.sendBatch.apply(this, arguments);
|
|
98
|
+
|
|
99
|
+
sendUrl = this.queueUrls[priority || 0];
|
|
100
|
+
|
|
101
|
+
if (!this.sqs) {
|
|
102
|
+
if (this.tasks[message.task]) {
|
|
103
|
+
var options = {changeTimeout: function() {}};
|
|
104
|
+
try {
|
|
105
|
+
message = JSON.parse(JSON.stringify(message));
|
|
106
|
+
}
|
|
107
|
+
catch(error) {
|
|
108
|
+
return callback(error);
|
|
109
|
+
}
|
|
110
|
+
if (this.logTasks && this.excludeLogTasks.indexOf(message.task) === -1)
|
|
111
|
+
log.i(message.task);
|
|
112
|
+
|
|
113
|
+
this.tasks[message.task](message, (function(err, data) {
|
|
114
|
+
if (err)
|
|
115
|
+
log.e(err);
|
|
116
|
+
|
|
117
|
+
if (this.verbose)
|
|
118
|
+
log.v(data);
|
|
119
|
+
|
|
120
|
+
if (typeof callback === "function")
|
|
121
|
+
callback.apply(context || this, arguments); // no arrow function so we can have arguments here.
|
|
122
|
+
|
|
123
|
+
}).bind(this), options);
|
|
124
|
+
}
|
|
125
|
+
else
|
|
126
|
+
log.d("Warning: SQS was not set up and task", message.task, "was not registered as a callback! Doing nothing.");
|
|
127
|
+
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (typeof callback !== "function")
|
|
132
|
+
return _sendOrRetry.call(this, sendUrl, message);
|
|
133
|
+
|
|
134
|
+
var sqsParameters = {
|
|
135
|
+
MessageBody: JSON.stringify(message),
|
|
136
|
+
QueueUrl: sendUrl
|
|
137
|
+
};
|
|
138
|
+
this.sqs.sendMessage(sqsParameters, function(err, data) {
|
|
139
|
+
if (err) {
|
|
140
|
+
err.sqsParameters = JSON.stringify(sqsParameters);
|
|
141
|
+
log.e("Cutie send message error.", err);
|
|
142
|
+
}
|
|
143
|
+
else if (this.verbose)
|
|
144
|
+
log.v("Cutie data:", data); // @todo: probably remove.
|
|
145
|
+
|
|
146
|
+
callback(err, data);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
sendServer(/* [ priority, ] message, callback, context */) {
|
|
151
|
+
var priority;
|
|
152
|
+
var message;
|
|
153
|
+
var callback;
|
|
154
|
+
var context;
|
|
155
|
+
|
|
156
|
+
[priority, message, callback, context] = _parseSendParameters.call(this, arguments);
|
|
157
|
+
|
|
158
|
+
if (Object.prototype.toString.call(message) === "[object Array]")
|
|
159
|
+
return this.sendServerBatch.apply(this, arguments);
|
|
160
|
+
|
|
161
|
+
message.__serverId = this.getServerId();
|
|
162
|
+
this.send.apply(this, arguments);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
sendBatch(/* [ priority, ] batch, callback, context */) {
|
|
166
|
+
var priority;
|
|
167
|
+
var batch;
|
|
168
|
+
var callback;
|
|
169
|
+
var context;
|
|
170
|
+
var sendUrl;
|
|
171
|
+
|
|
172
|
+
[priority, batch, callback, context] = _parseSendParameters.call(this, arguments);
|
|
173
|
+
|
|
174
|
+
if (Object.prototype.toString.call(batch) !== "[object Array]") {
|
|
175
|
+
return log.w("Warning: Cutie batch did not receive an array. Doing nothing.");
|
|
176
|
+
}
|
|
177
|
+
if (batch.length === 0) {
|
|
178
|
+
return log.w("Warning: Cutie batch was emtpy. Doing nothing.");
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
sendUrl = this.queueUrls[priority || 0];
|
|
182
|
+
|
|
183
|
+
if (!this.sqs) {
|
|
184
|
+
var length = batch.length;
|
|
185
|
+
var didError = false;
|
|
186
|
+
var dataArray = new Array(length);
|
|
187
|
+
var numberOfCallbacks = 0;
|
|
188
|
+
var callOnce = typeof callback !== "function" ? null : function(err, data, index) {
|
|
189
|
+
if (didError)
|
|
190
|
+
return;
|
|
191
|
+
|
|
192
|
+
dataArray[index] = data;
|
|
193
|
+
if (err) {
|
|
194
|
+
didError = true;
|
|
195
|
+
return callback({index: index, error: err}, data);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if (++numberOfCallbacks === length) {
|
|
199
|
+
callback(null, dataArray);
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
for (let i = 0; i < batch.length; i++) {
|
|
203
|
+
this.send(batch[i], callOnce, i);
|
|
204
|
+
}
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// sendBatch is limited to 10, so split them up into chunks of 10.
|
|
209
|
+
var allEntries = [[]];
|
|
210
|
+
var message;
|
|
211
|
+
var btyeSum = 0;
|
|
212
|
+
var textEncoder = new util.TextEncoder();
|
|
213
|
+
for (var i = 0, j = 0, k = 0; i < batch.length; i++) {
|
|
214
|
+
message = batch[i];
|
|
215
|
+
let messageEntry = {
|
|
216
|
+
Id: message.task + "-" + i,
|
|
217
|
+
MessageBody: JSON.stringify(message)
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
btyeSum += (textEncoder.encode(JSON.stringify(messageEntry))).length;
|
|
221
|
+
|
|
222
|
+
if (k === 9 || (btyeSum > 260000 && k > 0) ) {
|
|
223
|
+
// we can only send 10 and max message size is 262144
|
|
224
|
+
k = 0;
|
|
225
|
+
btyeSum = 0;
|
|
226
|
+
allEntries[++j] = [];
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
allEntries[j][k++] = messageEntry;
|
|
230
|
+
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (typeof callback !== "function")
|
|
234
|
+
return allEntries.forEach((entries) => { _sendBatchOrRetry.call(this, sendUrl, entries) });
|
|
235
|
+
|
|
236
|
+
var errors = [];
|
|
237
|
+
var requests = 0;
|
|
238
|
+
allEntries.forEach((entries) => {
|
|
239
|
+
requests++;
|
|
240
|
+
var sqsParameters = {
|
|
241
|
+
Entries: entries,
|
|
242
|
+
QueueUrl: sendUrl
|
|
243
|
+
};
|
|
244
|
+
this.sqs.sendMessageBatch(sqsParameters, (err, data) => {
|
|
245
|
+
if (err) {
|
|
246
|
+
err.sqsParameters = JSON.stringify(sqsParameters);
|
|
247
|
+
errors.push(err);
|
|
248
|
+
log.e("Cutie send message error.", err);
|
|
249
|
+
}
|
|
250
|
+
else if (this.verbose)
|
|
251
|
+
log.v("Cutie data:", data); // @todo: probably remove.
|
|
252
|
+
|
|
253
|
+
if (--requests === 0)
|
|
254
|
+
callback.call(context || this, errors.length ? (errors.length > 1 ? errors : errors[0]) : null);
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
sendServerBatch(/* [ priority, ] batch, callback, context */) {
|
|
260
|
+
var priority;
|
|
261
|
+
var batch;
|
|
262
|
+
var callback;
|
|
263
|
+
var context;
|
|
264
|
+
var sendUrl;
|
|
265
|
+
|
|
266
|
+
[priority, batch, callback, context] = _parseSendParameters.call(this, arguments);
|
|
267
|
+
|
|
268
|
+
if (Object.prototype.toString.call(batch) !== "[object Array]") {
|
|
269
|
+
return log.w("Warning: Cutie server batch did not receive an array. Doing nothing.");
|
|
270
|
+
}
|
|
271
|
+
if (batch.length === 0) {
|
|
272
|
+
return log.w("Warning: Cutie server batch was emtpy. Doing nothing.");
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
var serverId = this.getServerId();
|
|
276
|
+
batch.forEach(function(message) {
|
|
277
|
+
message.__serverId = serverId;
|
|
278
|
+
});
|
|
279
|
+
this.sendBatch.apply(this, arguments);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
addTask(name, callback) {
|
|
283
|
+
this.tasks[name] = callback;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
start() {
|
|
287
|
+
if (this._started)
|
|
288
|
+
return;
|
|
289
|
+
|
|
290
|
+
_receiveMessage.call(this, true);
|
|
291
|
+
this._started = true;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
//! Private instance methods
|
|
296
|
+
function _receiveMessage(emptyReceive) {
|
|
297
|
+
if (!this.sqs) {
|
|
298
|
+
return log.d("Warning: SQS was not set up. Tasks will execute immediately and not be queued.");
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
if (!emptyReceive) {
|
|
302
|
+
// we received a message so start from highest priority queue again
|
|
303
|
+
this._receiveQueueIndex = 0;
|
|
304
|
+
}
|
|
305
|
+
else if (this._receiveQueueIndex === undefined) {
|
|
306
|
+
// receiveing for the first time
|
|
307
|
+
this._receiveQueueIndex = 0;
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
// we did not receive a message, so try the next priority level queue
|
|
311
|
+
this._receiveQueueIndex++;
|
|
312
|
+
|
|
313
|
+
if (this._receiveQueueIndex > this.queueUrls.length - 1)
|
|
314
|
+
this._receiveQueueIndex = 0;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
this.currentQueueUrl = this.queueUrls[this._receiveQueueIndex];
|
|
318
|
+
|
|
319
|
+
if (this.verbose)
|
|
320
|
+
log.v("Trying queue", this._receiveQueueIndex, this.currentQueueUrl);
|
|
321
|
+
|
|
322
|
+
this.sqs.receiveMessage({QueueUrl: this.currentQueueUrl}, (err, data) => {
|
|
323
|
+
if (err) {
|
|
324
|
+
log.e("Cutie receiveMessage error. Will retry.", err);
|
|
325
|
+
_receiveMessage.call(this);
|
|
326
|
+
}
|
|
327
|
+
else if (data && data.Messages) {
|
|
328
|
+
var message = data.Messages[0];
|
|
329
|
+
try {
|
|
330
|
+
var body = JSON.parse(message.Body);
|
|
331
|
+
|
|
332
|
+
// If this message came from SNS the message body will be wrapped in another message.
|
|
333
|
+
if (!body.task && body.MessageId && body.Message) {
|
|
334
|
+
body = JSON.parse(body.Message);
|
|
335
|
+
}
|
|
336
|
+
} catch(ex) {
|
|
337
|
+
log.e("Cutie JOSN parse error", ex);
|
|
338
|
+
return _removeMessage.call(this, message);
|
|
339
|
+
}
|
|
340
|
+
var messageServerId = body.__serverId;
|
|
341
|
+
if (messageServerId && messageServerId !== this.getServerId()) {
|
|
342
|
+
// This task is not for this server.
|
|
343
|
+
this.sqs.changeMessageVisibility({QueueUrl: this.currentQueueUrl, ReceiptHandle: message.ReceiptHandle, VisibilityTimeout: 0}, (err, data) => {
|
|
344
|
+
if (err)
|
|
345
|
+
log.e("cutie changeMessageVisibility", err);
|
|
346
|
+
|
|
347
|
+
if (this.verbose)
|
|
348
|
+
log.v(data);
|
|
349
|
+
|
|
350
|
+
_receiveMessage.call(this);
|
|
351
|
+
});
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
var task;
|
|
356
|
+
if (body.task && (task = this.tasks[body.task])) {
|
|
357
|
+
if (this.logTasks && this.excludeLogTasks.indexOf(body.task) === -1)
|
|
358
|
+
log.i(body.task);
|
|
359
|
+
|
|
360
|
+
var timeoutCallback = (function() {
|
|
361
|
+
log.e("Error Cutie task timed out. ", body);
|
|
362
|
+
timeout = null;
|
|
363
|
+
_receiveMessage.call(this);
|
|
364
|
+
}).bind(this);
|
|
365
|
+
|
|
366
|
+
var startTime = Date.now();
|
|
367
|
+
var timeout = setTimeout(timeoutCallback, 120000);
|
|
368
|
+
var changedTimeoutInterval = null;
|
|
369
|
+
|
|
370
|
+
var changeTimeout = (function(newInterval, changeCallback) {
|
|
371
|
+
if (!newInterval || isNaN(parseInt(newInterval))) {
|
|
372
|
+
log.e("cutie changeTimeout called with invalid newInterval parameter", newInterval);
|
|
373
|
+
if (typeof changeCallback === "function")
|
|
374
|
+
changeCallback();
|
|
375
|
+
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
newInterval = parseInt(newInterval);
|
|
379
|
+
if (newInterval > 43200) {
|
|
380
|
+
log.e("cutie changeTimeout called with newInterval parameter out of range", newInterval);
|
|
381
|
+
if (typeof changeCallback === "function")
|
|
382
|
+
changeCallback();
|
|
383
|
+
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
clearTimeout(timeout); // Do not set to null, done(error) can still happen before response comes back.
|
|
388
|
+
var changeMessageVisibilityParameters = {
|
|
389
|
+
QueueUrl: this.currentQueueUrl,
|
|
390
|
+
ReceiptHandle: message.ReceiptHandle,
|
|
391
|
+
VisibilityTimeout: newInterval
|
|
392
|
+
};
|
|
393
|
+
this.sqs.changeMessageVisibility(changeMessageVisibilityParameters, (err) => {
|
|
394
|
+
if (err) {
|
|
395
|
+
log.e("Failed to change visibility timeout.", this.verbose ? err : "");
|
|
396
|
+
|
|
397
|
+
if (timeout !== null)
|
|
398
|
+
timeout = setTimeout(timeoutCallback, 120000 - (Date.now() - startTime));
|
|
399
|
+
|
|
400
|
+
if (typeof changeCallback === "function")
|
|
401
|
+
changeCallback();
|
|
402
|
+
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
if (timeout !== null) {
|
|
407
|
+
timeout = setTimeout(timeoutCallback, newInterval * 1000);
|
|
408
|
+
changedTimeoutInterval = newInterval;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
if (typeof changeCallback === "function")
|
|
412
|
+
changeCallback();
|
|
413
|
+
});
|
|
414
|
+
}).bind(this);
|
|
415
|
+
|
|
416
|
+
task(body, (err) => {
|
|
417
|
+
// @doc: if err.cutieMessage is set to "Task Locked", let the task time out and SQS will naturally requeue.
|
|
418
|
+
var taskLocked = Object.prototype.toString.call(err) === "[object Object]" && err.cutieMessage === "Task Locked";
|
|
419
|
+
if (timeout !== null) {
|
|
420
|
+
clearTimeout(timeout);
|
|
421
|
+
timeout = null;
|
|
422
|
+
if (err) {
|
|
423
|
+
if (!taskLocked)
|
|
424
|
+
log.e(err);
|
|
425
|
+
|
|
426
|
+
if (changedTimeoutInterval) {
|
|
427
|
+
var newTimeoutOnError = parseInt(((changedTimeoutInterval * 1000) - (Date.now() - startTime) + 10000) / 1000); // 10 secands from now
|
|
428
|
+
changeTimeout(newTimeoutOnError, () => {
|
|
429
|
+
_receiveMessage.call(this);
|
|
430
|
+
});
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
else
|
|
434
|
+
return _receiveMessage.call(this);
|
|
435
|
+
}
|
|
436
|
+
_removeMessage.call(this, message);
|
|
437
|
+
}
|
|
438
|
+
else if (err) {
|
|
439
|
+
// @doc: we timed out, but if there was an error we still log it.
|
|
440
|
+
log.e(err);
|
|
441
|
+
}
|
|
442
|
+
}, {changeTimeout: changeTimeout});
|
|
443
|
+
}
|
|
444
|
+
else {
|
|
445
|
+
log.e("Cutie doesn't know what to do with message:", message);
|
|
446
|
+
_removeMessage.call(this, message);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
else {
|
|
450
|
+
if (this.verbose)
|
|
451
|
+
log.v("empty receive for ", this._receiveQueueIndex, this.currentQueueUrl);
|
|
452
|
+
|
|
453
|
+
_receiveMessage.call(this, true);
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function _removeMessage(message) {
|
|
459
|
+
this.sqs.deleteMessage({QueueUrl: this.currentQueueUrl, ReceiptHandle: message.ReceiptHandle}, (err, data) => {
|
|
460
|
+
if (err)
|
|
461
|
+
log.e(err);
|
|
462
|
+
|
|
463
|
+
if (this.verbose)
|
|
464
|
+
log.v(data);
|
|
465
|
+
|
|
466
|
+
_receiveMessage.call(this);
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
// For tasks that need to be queued and can't be allowed to fail if the send fails, retry.
|
|
472
|
+
// We will do this for any send that does not have a callback.
|
|
473
|
+
const sendImmediateLimit = 10;
|
|
474
|
+
const sendDelayedLimit = 1000;
|
|
475
|
+
const sendDelay = 5000;
|
|
476
|
+
|
|
477
|
+
function _sendOrRetry(sendUrl, message, depth) {
|
|
478
|
+
var sqsParameters = {
|
|
479
|
+
MessageBody: JSON.stringify(message),
|
|
480
|
+
QueueUrl: sendUrl
|
|
481
|
+
};
|
|
482
|
+
this.sqs.sendMessage(sqsParameters, (err, data) => {
|
|
483
|
+
if (err) {
|
|
484
|
+
err.sqsParameters = JSON.stringify(sqsParameters);
|
|
485
|
+
if (err.retryable == false) {
|
|
486
|
+
return log.e("Cutie send message error, not retryable:", err);
|
|
487
|
+
}
|
|
488
|
+
log.e("Cutie send message error, but will retry:", err);
|
|
489
|
+
depth = depth || 0;
|
|
490
|
+
if (depth < sendImmediateLimit) {
|
|
491
|
+
_sendOrRetry.call(this, sendUrl, message, depth + 1);
|
|
492
|
+
}
|
|
493
|
+
else if (depth < sendDelayedLimit) {
|
|
494
|
+
setTimeout(() => { _sendOrRetry.call(this, sendUrl, message, depth + 1) }, sendDelay);
|
|
495
|
+
}
|
|
496
|
+
else {
|
|
497
|
+
var errorMessage = [
|
|
498
|
+
"Cuite failed to queue message. Retried", sendDelayedLimit, "times, taking over",
|
|
499
|
+
(sendDelay * (sendDelayedLimit - sendImmediateLimit) / 60000).toFixed(1), "minutes.",
|
|
500
|
+
"There is likely a problem with SQS. Skipping message:",
|
|
501
|
+
].join(" ");
|
|
502
|
+
log.e(errorMessage, message);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
else if (this.verbose)
|
|
506
|
+
log.v("Cutie data:", data); // @todo: probably remove.
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function _sendBatchOrRetry(sendUrl, entries, depth) {
|
|
511
|
+
var sqsParameters = {
|
|
512
|
+
Entries: entries,
|
|
513
|
+
QueueUrl: sendUrl
|
|
514
|
+
};
|
|
515
|
+
this.sqs.sendMessageBatch(sqsParameters, (err, data) => {
|
|
516
|
+
if (err) {
|
|
517
|
+
err.sqsParameters = JSON.stringify(sqsParameters);
|
|
518
|
+
if (err.retryable == false) {
|
|
519
|
+
return log.e("Cutie send message batch error, not retryable:", err);
|
|
520
|
+
}
|
|
521
|
+
log.e("Cutie send message batch error, but will retry:", err);
|
|
522
|
+
depth = depth || 0;
|
|
523
|
+
if (depth < sendImmediateLimit) {
|
|
524
|
+
_sendBatchOrRetry.call(this, sendUrl, entries, depth + 1);
|
|
525
|
+
}
|
|
526
|
+
else if (depth < sendDelayedLimit) {
|
|
527
|
+
setTimeout(() => { _sendBatchOrRetry.call(this, sendUrl, entries, depth + 1) }, sendDelay);
|
|
528
|
+
}
|
|
529
|
+
else {
|
|
530
|
+
var errorMessage = [
|
|
531
|
+
"Cuite failed to queue batch of messages. Retried", sendDelayedLimit, "times, taking over",
|
|
532
|
+
(sendDelay * (sendDelayedLimit - sendImmediateLimit) / 60000).toFixed(1), "minutes.",
|
|
533
|
+
"There is likely a problem with SQS. Skipping message batch:",
|
|
534
|
+
].join(" ");
|
|
535
|
+
log.e(errorMessage, entries);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
else if (this.verbose)
|
|
539
|
+
log.v("Cutie data:", data); // @todo: probably remove.
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function _parseSendParameters(parameters) {
|
|
544
|
+
var priority = null;
|
|
545
|
+
var message;
|
|
546
|
+
var callback;
|
|
547
|
+
var context;
|
|
548
|
+
var sendUrl;
|
|
549
|
+
|
|
550
|
+
if (Object.prototype.toString.call(parameters[0]) === "[object Number]") {
|
|
551
|
+
priority = Math.min(this._queueCount - 1, Math.max(0, parameters[0]));
|
|
552
|
+
message = parameters[1];
|
|
553
|
+
callback = parameters[2];
|
|
554
|
+
context = parameters[3];
|
|
555
|
+
}
|
|
556
|
+
else {
|
|
557
|
+
message = parameters[0];
|
|
558
|
+
callback = parameters[1];
|
|
559
|
+
context = parameters[2];
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
return [priority, message, callback, context];
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Support older global instance methods
|
|
566
|
+
// Initialize without URL, in case setUrl is not called.
|
|
567
|
+
var instance = new Cutie();
|
|
568
|
+
|
|
569
|
+
Cutie.setUrl = function() {
|
|
570
|
+
instance.setUrl.apply(instance, arguments);
|
|
571
|
+
};
|
|
572
|
+
|
|
573
|
+
Cutie.send = function() {
|
|
574
|
+
instance.send.apply(instance, arguments);
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
Cutie.sendBatch = function() {
|
|
578
|
+
instance.sendBatch.apply(instance, arguments);
|
|
579
|
+
};
|
|
580
|
+
|
|
581
|
+
Cutie.addTask = function() {
|
|
582
|
+
instance.addTask.apply(instance, arguments);
|
|
583
|
+
};
|
|
584
|
+
|
|
585
|
+
Cutie.start = function() {
|
|
586
|
+
instance.start();
|
|
587
|
+
};
|
|
588
|
+
|
|
589
|
+
module.exports = Cutie;
|
|
590
|
+
|
|
591
|
+
initializing = false;
|
package/package.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@creative-realities/cutie",
|
|
3
|
+
"version": "3.1.1",
|
|
4
|
+
"description": "Package for AWS SQS.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"publishConfig": {
|
|
7
|
+
"access": "public"
|
|
8
|
+
},
|
|
9
|
+
"dependencies": {
|
|
10
|
+
"aws-sdk": "2.2.x",
|
|
11
|
+
"logger": "0.2.x"
|
|
12
|
+
},
|
|
13
|
+
"scripts": {
|
|
14
|
+
"test": "echo \"Error: no test specified\" && exit 1"
|
|
15
|
+
},
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git@gitlab.reflect.systems:Reflect/cutie.git"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"AWS",
|
|
22
|
+
"SQS",
|
|
23
|
+
"cutie"
|
|
24
|
+
],
|
|
25
|
+
"author": "Adam Lockhart <alockhart@reflectsystems.com> (http://ec2-54-145-234-173.compute-1.amazonaws.com/u/alockhart)",
|
|
26
|
+
"maintainers":[
|
|
27
|
+
{"name":"alockhart","email":"adam.lockhart@cri.com"}
|
|
28
|
+
],
|
|
29
|
+
"license": "MIT"
|
|
30
|
+
}
|
package/uuid.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Loose interpretation of the specification DCE 1.1: Remote Procedure Call
|
|
3
|
+
// described at http://www.opengroup.org/onlinepubs/009629399/apdxa.htm#tagtcjh_37
|
|
4
|
+
// since JavaScript doesn't allow access to internal systems, the last 48 bits
|
|
5
|
+
// of the node section is made up using a series of random numbers (6 octets long).
|
|
6
|
+
//
|
|
7
|
+
|
|
8
|
+
var UUID = function() {};
|
|
9
|
+
|
|
10
|
+
UUID.rand = function(max) { return Math.floor(Math.random() * (max + 1)) };
|
|
11
|
+
UUID.paddedHex = function(number, padding) {
|
|
12
|
+
var hex = number.toString(16).toUpperCase();
|
|
13
|
+
return padding.substr(0, padding.length - hex.length) + hex;
|
|
14
|
+
};
|
|
15
|
+
UUID.generateNode = function() {
|
|
16
|
+
return UUID.paddedHex(UUID.rand(0xFFFF), "0000") + UUID.paddedHex(UUID.rand(0xFFFF), "0000") + UUID.paddedHex(UUID.rand(0xFFFF), "0000");
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
UUID.generate = function(node) {
|
|
20
|
+
function rand(max) { return Math.floor(Math.random() * (max + 1)) }
|
|
21
|
+
function paddedHex(number, padding) {
|
|
22
|
+
var hex = number.toString(16).toUpperCase();
|
|
23
|
+
return padding.substr(0, padding.length - hex.length) + hex;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
var dg = new Date(1582, 10, 15, 0, 0, 0, 0);
|
|
27
|
+
var dc = new Date();
|
|
28
|
+
var t = (dc.getTime() - dg.getTime()) * 10000; // convert milliseconds to 100-nanosecond units
|
|
29
|
+
var h = "-";
|
|
30
|
+
var hex = t.toString(16).toUpperCase();
|
|
31
|
+
hex = "1" + paddedHex(t, "000000000000000"); // pad the hex to 60 bits and set top quad to "1" for version.
|
|
32
|
+
var tl = hex.substr(8,8);
|
|
33
|
+
var tm = hex.substr(4,4);
|
|
34
|
+
var thv = hex.substr(0,4);
|
|
35
|
+
var cs = rand(0xFFFF) | 0x008000 & 0x00BFFF; // mask first two bits as 10b for variant
|
|
36
|
+
var cshex = cs.toString(16).toUpperCase();
|
|
37
|
+
|
|
38
|
+
// since detection of anything about the machine/browser is far to buggy,
|
|
39
|
+
// include some more random numbers here
|
|
40
|
+
// if NIC or an IP can be obtained reliably, that should be put in
|
|
41
|
+
// here instead.
|
|
42
|
+
var n = node;
|
|
43
|
+
if (!n) {
|
|
44
|
+
n = UUID.generateNode();
|
|
45
|
+
}
|
|
46
|
+
else
|
|
47
|
+
n = n.toUpperCase();
|
|
48
|
+
return tl + h + tm + h + thv + h + cshex + h + n;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports.generateNode = UUID.generateNode;
|
|
52
|
+
exports.generate = UUID.generate;
|