keuss 2.0.7 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mocharc.yaml +3 -0
- package/Pipeline/BaseLink.js +1 -1
- package/Pipeline/Queue.js +114 -89
- package/Queue.js +15 -2
- package/backends/bucket-mongo-safe.js +166 -145
- package/backends/intraorder.js +94 -83
- package/backends/mongo.js +147 -95
- package/backends/pl-mongo.js +20 -31
- package/backends/postgres.js +16 -3
- package/backends/ps-mongo.js +131 -110
- package/backends/redis-list.js +2 -1
- package/backends/redis-oq.js +2 -3
- package/backends/stream-mongo.js +83 -64
- package/lib/mubsub/channel.js +313 -0
- package/lib/mubsub/connection.js +79 -0
- package/lib/mubsub/index.js +10 -0
- package/package.json +5 -7
- package/signal/mongo-capped.js +1 -1
- package/stats/mongo.js +91 -54
- package/utils/RedisOrderedQueue.js +23 -3
package/backends/intraorder.js
CHANGED
|
@@ -33,7 +33,7 @@ class IntraOrderedQueue extends Queue {
|
|
|
33
33
|
|
|
34
34
|
/////////////////////////////////////////
|
|
35
35
|
// add element to queue
|
|
36
|
-
insert (entry,
|
|
36
|
+
insert (entry, cb) {
|
|
37
37
|
const q = { _id: entry.payload.iid || uuid.v4 () };
|
|
38
38
|
const upd = {
|
|
39
39
|
$push: { q: entry },
|
|
@@ -42,28 +42,29 @@ class IntraOrderedQueue extends Queue {
|
|
|
42
42
|
};
|
|
43
43
|
const opts = { upsert: true };
|
|
44
44
|
|
|
45
|
-
this._col.updateOne (q, upd, opts
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
})
|
|
45
|
+
this._col.updateOne (q, upd, opts)
|
|
46
|
+
.then (res => {
|
|
47
|
+
debug ('insert: updateOne (%j, %j, %j) => (%j)', q, upd, opts, res);
|
|
48
|
+
cb (null, res.upsertedId || q._id);
|
|
49
|
+
})
|
|
50
|
+
.catch (cb);
|
|
50
51
|
}
|
|
51
52
|
|
|
52
53
|
|
|
53
54
|
/////////////////////////////////////////
|
|
54
55
|
// get element from queue
|
|
55
|
-
get (
|
|
56
|
+
get (cb) {
|
|
56
57
|
// actually, a reserve followed by a commit
|
|
57
58
|
this.reserve ((err, elem) => {
|
|
58
|
-
if (err) return
|
|
59
|
-
if (!elem) return
|
|
59
|
+
if (err) return cb (err);
|
|
60
|
+
if (!elem) return cb ();
|
|
60
61
|
|
|
61
62
|
this.commit (elem._id, (err, res) => {
|
|
62
|
-
if (err) return
|
|
63
|
+
if (err) return cb (err);
|
|
63
64
|
|
|
64
65
|
// clear _env: not needed here
|
|
65
66
|
delete elem._env;
|
|
66
|
-
|
|
67
|
+
cb (null, elem);
|
|
67
68
|
}, elem);
|
|
68
69
|
})
|
|
69
70
|
}
|
|
@@ -71,7 +72,7 @@ class IntraOrderedQueue extends Queue {
|
|
|
71
72
|
|
|
72
73
|
//////////////////////////////////
|
|
73
74
|
// reserve element: call cb (err, pl) where pl has an id
|
|
74
|
-
reserve (
|
|
75
|
+
reserve (cb) {
|
|
75
76
|
const delay = this._opts.reserve_delay || 120;
|
|
76
77
|
|
|
77
78
|
const q = {
|
|
@@ -89,14 +90,15 @@ class IntraOrderedQueue extends Queue {
|
|
|
89
90
|
|
|
90
91
|
const opts = {
|
|
91
92
|
sort: {mature : 1},
|
|
92
|
-
returnDocument: 'before'
|
|
93
|
+
returnDocument: 'before',
|
|
94
|
+
includeResultMetadata: true
|
|
93
95
|
};
|
|
94
96
|
|
|
95
|
-
this._col.findOneAndUpdate (q, upd, opts
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const v =
|
|
99
|
-
if (!v) return
|
|
97
|
+
this._col.findOneAndUpdate (q, upd, opts)
|
|
98
|
+
.then (res => {
|
|
99
|
+
debug ('reserve: findOneAndUpdate (%j, %j, %j) => (%j)', q, upd, opts, res);
|
|
100
|
+
const v = res && res.value;
|
|
101
|
+
if (!v) return cb ();
|
|
100
102
|
|
|
101
103
|
// construct the real object to return
|
|
102
104
|
const vq = v.q[0];
|
|
@@ -104,17 +106,18 @@ class IntraOrderedQueue extends Queue {
|
|
|
104
106
|
vq.tries = v.tries
|
|
105
107
|
vq._env = v; // pass along the whole obj too
|
|
106
108
|
if (vq.payload._bsontype == 'Binary') vq.payload = vq.payload.buffer;
|
|
107
|
-
|
|
108
|
-
})
|
|
109
|
+
cb (null, vq);
|
|
110
|
+
})
|
|
111
|
+
.catch(cb);
|
|
109
112
|
}
|
|
110
113
|
|
|
111
114
|
|
|
112
115
|
//////////////////////////////////
|
|
113
116
|
// commit previous reserve, by p.id
|
|
114
|
-
commit (id,
|
|
117
|
+
commit (id, cb, obj) {
|
|
115
118
|
if (!(obj || (obj && obj._id))) {
|
|
116
119
|
// full obj must be passed to commit
|
|
117
|
-
return
|
|
120
|
+
return cb ('full obj must be passed to commit');
|
|
118
121
|
}
|
|
119
122
|
|
|
120
123
|
const q = {
|
|
@@ -147,19 +150,20 @@ class IntraOrderedQueue extends Queue {
|
|
|
147
150
|
|
|
148
151
|
const opts = {};
|
|
149
152
|
|
|
150
|
-
this._col.updateOne (q, upd, opts
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
})
|
|
153
|
+
this._col.updateOne (q, upd, opts)
|
|
154
|
+
.then (res => {
|
|
155
|
+
debug ('commit: updateOne (%j, %j, %j) => (%j)', q, upd, opts, res);
|
|
156
|
+
cb (null, res && (res.modifiedCount == 1));
|
|
157
|
+
})
|
|
158
|
+
.catch (cb);
|
|
155
159
|
}
|
|
156
160
|
|
|
157
161
|
|
|
158
162
|
//////////////////////////////////
|
|
159
163
|
// rollback previous reserve, by p.id
|
|
160
|
-
rollback (id, next_t,
|
|
164
|
+
rollback (id, next_t, cb) {
|
|
161
165
|
if (_.isFunction (next_t)) {
|
|
162
|
-
|
|
166
|
+
cb = next_t;
|
|
163
167
|
next_t = null;
|
|
164
168
|
}
|
|
165
169
|
|
|
@@ -175,17 +179,18 @@ class IntraOrderedQueue extends Queue {
|
|
|
175
179
|
|
|
176
180
|
const opts = {};
|
|
177
181
|
|
|
178
|
-
this._col.updateOne (q, upd, opts
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
})
|
|
182
|
+
this._col.updateOne (q, upd, opts)
|
|
183
|
+
.then (res => {
|
|
184
|
+
debug ('rollback: updateOne (%j, %j, %j) => (%j)', q, upd, opts, res);
|
|
185
|
+
cb (null, res && (res.modifiedCount == 1));
|
|
186
|
+
})
|
|
187
|
+
.catch (cb);
|
|
183
188
|
}
|
|
184
189
|
|
|
185
190
|
|
|
186
191
|
//////////////////////////////////
|
|
187
192
|
// queue size of non-mature elements only
|
|
188
|
-
totalSize (
|
|
193
|
+
totalSize (cb) {
|
|
189
194
|
const cursor = this._col.aggregate ([
|
|
190
195
|
{$match: {
|
|
191
196
|
qcnt: {$gt: 0}
|
|
@@ -193,18 +198,19 @@ class IntraOrderedQueue extends Queue {
|
|
|
193
198
|
{$group:{_id:'t', v: {$sum: '$qcnt'}}}
|
|
194
199
|
]);
|
|
195
200
|
|
|
196
|
-
cursor.toArray (
|
|
201
|
+
cursor.toArray ()
|
|
202
|
+
.then (res => {
|
|
197
203
|
debug ('calculating schedSize: aggregation pipeline returns %o', res);
|
|
198
|
-
if (
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
204
|
+
if (res.length == 0) return cb (null, 0);
|
|
205
|
+
cb (null, res[0].v);
|
|
206
|
+
})
|
|
207
|
+
.catch (cb);
|
|
202
208
|
}
|
|
203
209
|
|
|
204
210
|
|
|
205
211
|
//////////////////////////////////
|
|
206
212
|
// queue size NOT including non-mature elements
|
|
207
|
-
size (
|
|
213
|
+
size (cb) {
|
|
208
214
|
const cursor = this._col.aggregate ([
|
|
209
215
|
{$match: {
|
|
210
216
|
mature: {$lte: Queue.now ()},
|
|
@@ -213,18 +219,19 @@ class IntraOrderedQueue extends Queue {
|
|
|
213
219
|
{$group:{_id:'t', v: {$sum: '$qcnt'}}}
|
|
214
220
|
]);
|
|
215
221
|
|
|
216
|
-
cursor.toArray (
|
|
222
|
+
cursor.toArray ()
|
|
223
|
+
.then (res => {
|
|
217
224
|
debug ('calculating schedSize: aggregation pipeline returns %o', res);
|
|
218
|
-
if (
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
225
|
+
if (res.length == 0) return cb (null, 0);
|
|
226
|
+
cb (null, res[0].v);
|
|
227
|
+
})
|
|
228
|
+
.catch (cb);
|
|
222
229
|
}
|
|
223
230
|
|
|
224
231
|
|
|
225
232
|
//////////////////////////////////
|
|
226
233
|
// queue size of non-mature elements only
|
|
227
|
-
schedSize (
|
|
234
|
+
schedSize (cb) {
|
|
228
235
|
const cursor = this._col.aggregate ([
|
|
229
236
|
{$match: {
|
|
230
237
|
mature: {$gt: Queue.now ()},
|
|
@@ -234,18 +241,19 @@ class IntraOrderedQueue extends Queue {
|
|
|
234
241
|
{$group:{_id:'t', v: {$sum: '$qcnt'}}}
|
|
235
242
|
]);
|
|
236
243
|
|
|
237
|
-
cursor.toArray (
|
|
244
|
+
cursor.toArray ()
|
|
245
|
+
.then (res => {
|
|
238
246
|
debug ('calculating schedSize: aggregation pipeline returns %o', res);
|
|
239
|
-
if (
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
247
|
+
if (res.length == 0) return cb (null, 0);
|
|
248
|
+
cb (null, res[0].v);
|
|
249
|
+
})
|
|
250
|
+
.catch (cb);
|
|
243
251
|
}
|
|
244
252
|
|
|
245
253
|
|
|
246
254
|
//////////////////////////////////
|
|
247
255
|
// queue size of reserved elements only
|
|
248
|
-
resvSize (
|
|
256
|
+
resvSize (cb) {
|
|
249
257
|
const cursor = this._col.aggregate ([
|
|
250
258
|
{$match: {
|
|
251
259
|
mature: {$gt: Queue.now ()},
|
|
@@ -255,18 +263,19 @@ class IntraOrderedQueue extends Queue {
|
|
|
255
263
|
{$group:{_id:'t', v: {$sum: '$qcnt'}}}
|
|
256
264
|
]);
|
|
257
265
|
|
|
258
|
-
cursor.toArray (
|
|
266
|
+
cursor.toArray ()
|
|
267
|
+
.then (res => {
|
|
259
268
|
debug ('calculating schedSize: aggregation pipeline returns %o', res);
|
|
260
|
-
if (
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
269
|
+
if (res.length == 0) return cb (null, 0);
|
|
270
|
+
cb (null, res[0].v);
|
|
271
|
+
})
|
|
272
|
+
.catch (cb);
|
|
264
273
|
}
|
|
265
274
|
|
|
266
275
|
|
|
267
276
|
//////////////////////////////////////////////
|
|
268
277
|
// remove by id
|
|
269
|
-
remove (id,
|
|
278
|
+
remove (id, cb) {
|
|
270
279
|
const q = {
|
|
271
280
|
_id: id,
|
|
272
281
|
qcnt: { $eq: 1 }, // allow deletion ONLY if it has just one element in the intraqueue
|
|
@@ -275,26 +284,26 @@ class IntraOrderedQueue extends Queue {
|
|
|
275
284
|
|
|
276
285
|
const opts = {};
|
|
277
286
|
|
|
278
|
-
this._col.deleteOne (q, opts
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
})
|
|
287
|
+
this._col.deleteOne (q, opts)
|
|
288
|
+
.then (res => {
|
|
289
|
+
debug ('remove: deleteOne (%j, %j) => (%j)', q, opts, res);
|
|
290
|
+
cb (null, res && (res.deletedCount == 1));
|
|
291
|
+
})
|
|
292
|
+
.catch (cb);
|
|
283
293
|
}
|
|
284
294
|
|
|
285
295
|
|
|
286
296
|
/////////////////////////////////////////
|
|
287
297
|
// get element from queue
|
|
288
|
-
next_t (
|
|
298
|
+
next_t (cb) {
|
|
289
299
|
this._col
|
|
290
300
|
.find ()
|
|
291
301
|
.limit(1)
|
|
292
302
|
.sort ({mature:1})
|
|
293
303
|
.project ({mature:1})
|
|
294
|
-
.next (
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
});
|
|
304
|
+
.next ()
|
|
305
|
+
.then (res => cb (null, res && res.mature))
|
|
306
|
+
.catch (cb);
|
|
298
307
|
}
|
|
299
308
|
|
|
300
309
|
|
|
@@ -305,12 +314,15 @@ class IntraOrderedQueue extends Queue {
|
|
|
305
314
|
// create needed indexes for O(1) functioning
|
|
306
315
|
_ensureIndexes (cb) {
|
|
307
316
|
async.series ([
|
|
308
|
-
cb => this._col.createIndex ({mature : 1, qcnt: 1}
|
|
309
|
-
]
|
|
317
|
+
cb => this._col.createIndex ({mature : 1, qcnt: 1}).then (res => cb(null, res)).catch (cb),
|
|
318
|
+
])
|
|
319
|
+
.then (() => cb (null, this))
|
|
320
|
+
.catch (cb);
|
|
310
321
|
}
|
|
311
322
|
}
|
|
312
323
|
|
|
313
324
|
|
|
325
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
314
326
|
class Factory extends QFactory_MongoDB_defaults {
|
|
315
327
|
constructor (opts, mongo_conn) {
|
|
316
328
|
super (opts);
|
|
@@ -352,25 +364,24 @@ class Factory extends QFactory_MongoDB_defaults {
|
|
|
352
364
|
reserve: true,
|
|
353
365
|
pipeline: false,
|
|
354
366
|
tape: false,
|
|
355
|
-
remove: false
|
|
367
|
+
remove: false,
|
|
368
|
+
id: false,
|
|
356
369
|
};
|
|
357
370
|
}
|
|
358
371
|
}
|
|
359
372
|
|
|
373
|
+
|
|
374
|
+
//////////////////////////////////////////////////////////////////
|
|
360
375
|
function creator (opts, cb) {
|
|
361
|
-
|
|
362
|
-
|
|
376
|
+
const _opts = opts || {};
|
|
377
|
+
const m_url = _opts.url || 'mongodb://localhost:27017/keuss';
|
|
363
378
|
|
|
364
|
-
MongoClient.connect (m_url
|
|
365
|
-
|
|
366
|
-
|
|
379
|
+
MongoClient.connect (m_url)
|
|
380
|
+
.then (cl => {
|
|
381
|
+
const F = new Factory (_opts, cl);
|
|
367
382
|
F.async_init (err => cb (null, F));
|
|
368
|
-
})
|
|
383
|
+
})
|
|
384
|
+
.catch (cb);
|
|
369
385
|
}
|
|
370
386
|
|
|
371
387
|
module.exports = creator;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|