keuss 1.7.4 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Pipeline/Queue.js +0 -1
- package/QFactory.js +49 -17
- package/Queue.js +1 -4
- package/README.md +2 -1
- package/TODO +0 -9
- package/backends/bucket-mongo-safe.js +11 -11
- package/backends/intraorder.js +13 -11
- package/backends/mongo.js +12 -9
- package/backends/pl-mongo.js +8 -5
- package/backends/postgres.js +380 -0
- package/backends/ps-mongo.js +11 -8
- package/backends/redis-list.js +8 -8
- package/backends/redis-oq.js +8 -3
- package/backends/stream-mongo.js +10 -15
- package/package.json +12 -8
- package/stats/mem.js +0 -1
- package/.github/workflows/codeql-analysis.yml +0 -72
- package/bench/all-mongo.js +0 -108
- package/bench/multi-q.js +0 -85
- package/bench/redis-oq-consumer-producer.js +0 -52
- package/bench/redis-oq-consumer.js +0 -40
- package/bench/redis-oq-producer.js +0 -43
- package/docker-compose/docker-compose.yaml +0 -18
- package/docker-compose.yaml +0 -18
- package/examples/pipelines/builder/index.js +0 -99
- package/examples/pipelines/fromRecipe/index.js +0 -127
- package/examples/pipelines/simplest/index.js +0 -38
- package/examples/pipelines/simulation-fork/index.js +0 -115
- package/examples/snippets/01-simplest-pop-push.js +0 -49
- package/examples/snippets/02-simplest-reserve-rollback-commit.js +0 -44
- package/examples/snippets/03-simplest-producer-consumer-loops.js +0 -77
- package/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js +0 -78
- package/examples/snippets/05-insert-reserve-rollback-deadletter.js +0 -105
- package/examples/snippets/06-random-consumer-producer.js +0 -270
- package/examples/snippets/07-stream-simple.js +0 -53
- package/examples/snippets/redislabs-consumer-producer.js +0 -44
- package/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js +0 -52
- package/examples/webhooks/README.md +0 -36
- package/examples/webhooks/app.js +0 -70
- package/examples/webhooks/consumer.js +0 -98
- package/examples/webhooks/index.js +0 -55
- package/examples/webhooks/package-lock.json +0 -500
- package/examples/webhooks/package.json +0 -23
- package/examples/webhooks/wh-payload.json +0 -38
- package/playground/irc.js +0 -53
- package/playground/pl-rollback.js +0 -55
- package/playground/pl1.js +0 -74
- package/playground/q1.js +0 -34
- package/playground/simple-pl.js +0 -42
- package/playground/stream-loops.js +0 -114
- package/test/backends_bucket-at-least-once.js +0 -302
- package/test/backends_deadletter.js +0 -227
- package/test/backends_payload.js +0 -542
- package/test/backends_push-pop.js +0 -170
- package/test/backends_remove.js +0 -320
- package/test/backends_reserve-commit-rollback.js +0 -1033
- package/test/intraorder.js +0 -325
- package/test/pause.js +0 -220
- package/test/pipeline-Builder.js +0 -285
- package/test/pipeline-ChoiceLink.js +0 -241
- package/test/pipeline-DirectLink.js +0 -376
- package/test/pipeline-Sink.js +0 -175
- package/test/signal.js +0 -196
- package/test/stats.js +0 -296
- package/test/stream-mongo.js +0 -166
package/test/backends_remove.js
DELETED
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
var async = require ('async');
|
|
2
|
-
var should = require ('should');
|
|
3
|
-
var _ = require ('lodash');
|
|
4
|
-
|
|
5
|
-
var LocalSignal = require ('../signal/local');
|
|
6
|
-
var MemStats = require ('../stats/mem');
|
|
7
|
-
|
|
8
|
-
const MongoClient = require ('mongodb').MongoClient;
|
|
9
|
-
const Redis = require("ioredis");
|
|
10
|
-
|
|
11
|
-
function stats (q, cb) {
|
|
12
|
-
async.series ({
|
|
13
|
-
stats: cb => q.stats(cb),
|
|
14
|
-
tsize: cb => q.totalSize(cb),
|
|
15
|
-
rsize: cb => q.resvSize(cb),
|
|
16
|
-
}, (err, res) => {
|
|
17
|
-
// console.log ('stats:', res);
|
|
18
|
-
cb (err, res);
|
|
19
|
-
});
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
function get_mq_factory (MQ, opts, cb) {
|
|
24
|
-
const common_opts = {
|
|
25
|
-
url: 'mongodb://localhost/keuss_test_backends_remove',
|
|
26
|
-
signaller: { provider: LocalSignal},
|
|
27
|
-
stats: {provider: MemStats},
|
|
28
|
-
deadletter: {
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
// initialize factory
|
|
33
|
-
MQ (_.merge ({}, common_opts, opts), cb);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function release_mq_factory (q, factory, cb) {
|
|
37
|
-
// console.log ('releasing mq factory');
|
|
38
|
-
|
|
39
|
-
setTimeout (() => {
|
|
40
|
-
q.cancel ();
|
|
41
|
-
factory.close (cb);
|
|
42
|
-
}, 1000);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
47
|
-
[
|
|
48
|
-
{label: 'Simple MongoDB', mq: require ('../backends/mongo')},
|
|
49
|
-
{label: 'Pipelined MongoDB', mq: require ('../backends/pl-mongo')},
|
|
50
|
-
{label: 'Tape MongoDB', mq: require ('../backends/ps-mongo')},
|
|
51
|
-
{label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
|
|
52
|
-
{label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
|
|
53
|
-
].forEach(function (MQ_item) {
|
|
54
|
-
var is_redis = (MQ_item.label == 'Redis OrderedQueue');
|
|
55
|
-
|
|
56
|
-
describe('remove operations on ' + MQ_item.label + ' queue backend', () => {
|
|
57
|
-
const MQ = MQ_item.mq;
|
|
58
|
-
|
|
59
|
-
beforeEach (done => {
|
|
60
|
-
done();
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
afterEach (done => async.series ([
|
|
64
|
-
cb => setTimeout (cb, 1000),
|
|
65
|
-
cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_remove', (err, cl) => {
|
|
66
|
-
if (err) return done (err);
|
|
67
|
-
cl.db().dropDatabase (() => cl.close (cb))
|
|
68
|
-
}),
|
|
69
|
-
cb => {
|
|
70
|
-
const redis = new Redis();
|
|
71
|
-
redis.keys ('*', (err, keys) => {
|
|
72
|
-
const tasks = [];
|
|
73
|
-
_.each (keys, k => tasks.push (cb => redis.del (k, cb)));
|
|
74
|
-
async.series (tasks, (err, res) => {
|
|
75
|
-
redis.disconnect();
|
|
76
|
-
cb (err, res);
|
|
77
|
-
});
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
], done));
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
it('fails on invalid id', done => {
|
|
84
|
-
// skip if redis
|
|
85
|
-
if (is_redis) return done();
|
|
86
|
-
|
|
87
|
-
async.waterfall ([
|
|
88
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
89
|
-
(factory, cb) => {
|
|
90
|
-
const q = factory.queue('test_queue_remove', {});
|
|
91
|
-
q.remove ('invalid-id', err => cb (null, err, q, factory))
|
|
92
|
-
},
|
|
93
|
-
(err, q, factory, cb) => {
|
|
94
|
-
err.should.match (/id \[invalid-id\] can not be used as remove id/);
|
|
95
|
-
release_mq_factory (q, factory, cb);
|
|
96
|
-
}
|
|
97
|
-
], done);
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
it('fails on nonexistent id', done => {
|
|
102
|
-
async.waterfall ([
|
|
103
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
104
|
-
(factory, cb) => {
|
|
105
|
-
const q = factory.queue('test_queue_remove', {});
|
|
106
|
-
q.remove ('00112233445566778899aabb', (err, res) => cb (null, err, res, q, factory))
|
|
107
|
-
},
|
|
108
|
-
(err, res, q, factory, cb) => {
|
|
109
|
-
should (err).be.null();
|
|
110
|
-
res.should.be.false();
|
|
111
|
-
release_mq_factory (q, factory, cb);
|
|
112
|
-
}
|
|
113
|
-
], done);
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
it('deletes regular element ok', done => {
|
|
118
|
-
const ctx = {};
|
|
119
|
-
async.waterfall ([
|
|
120
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
121
|
-
(factory, cb) => {
|
|
122
|
-
const q = factory.queue('test_queue_remove', {});
|
|
123
|
-
cb (null, q, factory);
|
|
124
|
-
},
|
|
125
|
-
(q, factory, cb) => async.series ([
|
|
126
|
-
cb => q.push ({a: 1, b: 'oo'}, cb),
|
|
127
|
-
cb => setTimeout (cb, 1000),
|
|
128
|
-
cb => stats (q, cb)
|
|
129
|
-
], (err, res) => {
|
|
130
|
-
ctx.id = res[0];
|
|
131
|
-
ctx.id.should.not.be.null();
|
|
132
|
-
res[2].should.eql ({
|
|
133
|
-
stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
|
|
134
|
-
tsize: 1,
|
|
135
|
-
rsize: is_redis ? null : 0
|
|
136
|
-
})
|
|
137
|
-
cb (err, q, factory);
|
|
138
|
-
}),
|
|
139
|
-
(q, factory, cb) => {
|
|
140
|
-
q.remove (ctx.id, (err, res) => {
|
|
141
|
-
should (err).be.null();
|
|
142
|
-
res.should.be.true();
|
|
143
|
-
cb (err, q, factory);
|
|
144
|
-
});
|
|
145
|
-
},
|
|
146
|
-
(q, factory, cb) => async.series ([
|
|
147
|
-
cb => stats (q, cb),
|
|
148
|
-
cb => setTimeout (cb, 1000),
|
|
149
|
-
], (err, res) => {
|
|
150
|
-
res[0].should.eql ({
|
|
151
|
-
stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
|
|
152
|
-
tsize: 0,
|
|
153
|
-
rsize: is_redis ? null : 0
|
|
154
|
-
});
|
|
155
|
-
cb (err, q, factory);
|
|
156
|
-
}),
|
|
157
|
-
(q, factory, cb) => {
|
|
158
|
-
release_mq_factory (q, factory, cb);
|
|
159
|
-
}
|
|
160
|
-
], done);
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
it('does not delete reserved element', done => {
|
|
165
|
-
const ctx = {};
|
|
166
|
-
async.waterfall ([
|
|
167
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
168
|
-
(factory, cb) => {
|
|
169
|
-
const q = factory.queue('test_queue_remove', {});
|
|
170
|
-
cb (null, q, factory);
|
|
171
|
-
},
|
|
172
|
-
(q, factory, cb) => async.series ([
|
|
173
|
-
cb => q.push ({a: 1, b: 'oo'}, cb),
|
|
174
|
-
cb => setTimeout (cb, 1000),
|
|
175
|
-
], (err, res) => {
|
|
176
|
-
ctx.id = res[0];
|
|
177
|
-
ctx.id.should.not.be.null();
|
|
178
|
-
cb (err, q, factory);
|
|
179
|
-
}),
|
|
180
|
-
(q, factory, cb) => {
|
|
181
|
-
q.pop('c1', { reserve: true }, (err, res) => {
|
|
182
|
-
// res._id.should.eql (ctx.id);
|
|
183
|
-
cb (err, q, factory);
|
|
184
|
-
});
|
|
185
|
-
},
|
|
186
|
-
(q, factory, cb) => {
|
|
187
|
-
q.remove (ctx.id, (err, res) => {
|
|
188
|
-
should (err).be.null();
|
|
189
|
-
res.should.be.false();
|
|
190
|
-
cb (err, q, factory);
|
|
191
|
-
});
|
|
192
|
-
},
|
|
193
|
-
(q, factory, cb) => async.series ([
|
|
194
|
-
cb => stats (q, cb),
|
|
195
|
-
cb => setTimeout (cb, 1000),
|
|
196
|
-
], (err, res) => {
|
|
197
|
-
res[0].should.eql ({
|
|
198
|
-
stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0, deadletter: 0 },
|
|
199
|
-
tsize: 1,
|
|
200
|
-
rsize: is_redis ? null : 1
|
|
201
|
-
});
|
|
202
|
-
cb (err, q, factory);
|
|
203
|
-
}),
|
|
204
|
-
(q, factory, cb) => {
|
|
205
|
-
release_mq_factory (q, factory, cb);
|
|
206
|
-
}
|
|
207
|
-
], done);
|
|
208
|
-
});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
it('does delete reserved+rolledback element', done => {
|
|
212
|
-
const ctx = {};
|
|
213
|
-
async.waterfall ([
|
|
214
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
215
|
-
(factory, cb) => {
|
|
216
|
-
const q = factory.queue('test_queue_remove', {});
|
|
217
|
-
cb (null, q, factory);
|
|
218
|
-
},
|
|
219
|
-
(q, factory, cb) => async.series ([
|
|
220
|
-
cb => q.push ({a: 1, b: 'oo'}, cb),
|
|
221
|
-
cb => setTimeout (cb, 1000),
|
|
222
|
-
], (err, res) => {
|
|
223
|
-
ctx.id = res[0];
|
|
224
|
-
ctx.id.should.not.be.null();
|
|
225
|
-
cb (err, q, factory);
|
|
226
|
-
}),
|
|
227
|
-
(q, factory, cb) => {
|
|
228
|
-
q.pop('c1', { reserve: true }, (err, res) => {
|
|
229
|
-
res._id.should.eql (ctx.id);
|
|
230
|
-
cb (err, q, factory);
|
|
231
|
-
});
|
|
232
|
-
},
|
|
233
|
-
(q, factory, cb) => async.series ([
|
|
234
|
-
cb => q.ko(ctx.id, new Date().getTime() + 2000, cb),
|
|
235
|
-
cb => setTimeout (cb, 1000),
|
|
236
|
-
], (err, res) => {
|
|
237
|
-
res[0].should.be.true();
|
|
238
|
-
cb (err, q, factory);
|
|
239
|
-
}),
|
|
240
|
-
(q, factory, cb) => {
|
|
241
|
-
q.remove (ctx.id, (err, res) => {
|
|
242
|
-
should (err).be.null();
|
|
243
|
-
res.should.be.true();
|
|
244
|
-
cb (err, q, factory);
|
|
245
|
-
});
|
|
246
|
-
},
|
|
247
|
-
(q, factory, cb) => async.series ([
|
|
248
|
-
cb => stats (q, cb),
|
|
249
|
-
cb => setTimeout (cb, 1000),
|
|
250
|
-
], (err, res) => {
|
|
251
|
-
res[0].should.eql ({
|
|
252
|
-
stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1, deadletter: 0 },
|
|
253
|
-
tsize: 0,
|
|
254
|
-
rsize: is_redis ? null : 0
|
|
255
|
-
});
|
|
256
|
-
cb (err, q, factory);
|
|
257
|
-
}),
|
|
258
|
-
(q, factory, cb) => {
|
|
259
|
-
release_mq_factory (q, factory, cb);
|
|
260
|
-
}
|
|
261
|
-
], done);
|
|
262
|
-
});
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
it('does fail trying to delete an already-deleted element', done => {
|
|
266
|
-
const ctx = {};
|
|
267
|
-
async.waterfall ([
|
|
268
|
-
cb => get_mq_factory (MQ, {}, cb),
|
|
269
|
-
(factory, cb) => {
|
|
270
|
-
const q = factory.queue('test_queue_remove', {});
|
|
271
|
-
cb (null, q, factory);
|
|
272
|
-
},
|
|
273
|
-
(q, factory, cb) => async.series ([
|
|
274
|
-
cb => q.push ({a: 1, b: 'oo'}, cb),
|
|
275
|
-
cb => setTimeout (cb, 1000),
|
|
276
|
-
cb => stats (q, cb)
|
|
277
|
-
], (err, res) => {
|
|
278
|
-
ctx.id = res[0];
|
|
279
|
-
ctx.id.should.not.be.null();
|
|
280
|
-
res[2].should.eql ({
|
|
281
|
-
stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
|
|
282
|
-
tsize: 1,
|
|
283
|
-
rsize: is_redis ? null : 0
|
|
284
|
-
})
|
|
285
|
-
cb (err, q, factory);
|
|
286
|
-
}),
|
|
287
|
-
(q, factory, cb) => {
|
|
288
|
-
q.remove (ctx.id, (err, res) => {
|
|
289
|
-
should (err).be.null();
|
|
290
|
-
res.should.be.true();
|
|
291
|
-
cb (err, q, factory);
|
|
292
|
-
});
|
|
293
|
-
},
|
|
294
|
-
(q, factory, cb) => {
|
|
295
|
-
q.remove (ctx.id, (err, res) => {
|
|
296
|
-
should (err).be.null();
|
|
297
|
-
res.should.be.false();
|
|
298
|
-
cb (err, q, factory);
|
|
299
|
-
});
|
|
300
|
-
},
|
|
301
|
-
(q, factory, cb) => async.series ([
|
|
302
|
-
cb => stats (q, cb),
|
|
303
|
-
cb => setTimeout (cb, 1000),
|
|
304
|
-
], (err, res) => {
|
|
305
|
-
res[0].should.eql ({
|
|
306
|
-
stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
|
|
307
|
-
tsize: 0,
|
|
308
|
-
rsize: is_redis ? null : 0
|
|
309
|
-
});
|
|
310
|
-
cb (err, q, factory);
|
|
311
|
-
}),
|
|
312
|
-
(q, factory, cb) => {
|
|
313
|
-
release_mq_factory (q, factory, cb);
|
|
314
|
-
}
|
|
315
|
-
], done);
|
|
316
|
-
});
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
});
|
|
320
|
-
});
|