keuss 1.7.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/Pipeline/Queue.js +0 -1
  2. package/QFactory.js +11 -6
  3. package/Queue.js +1 -4
  4. package/README.md +2 -1
  5. package/TODO +0 -9
  6. package/backends/bucket-mongo-safe.js +11 -11
  7. package/backends/intraorder.js +13 -11
  8. package/backends/mongo.js +12 -9
  9. package/backends/pl-mongo.js +8 -5
  10. package/backends/postgres.js +380 -0
  11. package/backends/ps-mongo.js +11 -8
  12. package/backends/redis-list.js +8 -8
  13. package/backends/redis-oq.js +8 -3
  14. package/backends/stream-mongo.js +10 -15
  15. package/package.json +12 -8
  16. package/stats/mem.js +0 -1
  17. package/.github/workflows/codeql-analysis.yml +0 -72
  18. package/bench/all-mongo.js +0 -108
  19. package/bench/multi-q.js +0 -85
  20. package/bench/redis-oq-consumer-producer.js +0 -52
  21. package/bench/redis-oq-consumer.js +0 -40
  22. package/bench/redis-oq-producer.js +0 -43
  23. package/docker-compose/docker-compose.yaml +0 -18
  24. package/docker-compose.yaml +0 -18
  25. package/examples/pipelines/builder/index.js +0 -99
  26. package/examples/pipelines/fromRecipe/index.js +0 -127
  27. package/examples/pipelines/simplest/index.js +0 -38
  28. package/examples/pipelines/simulation-fork/index.js +0 -115
  29. package/examples/snippets/01-simplest-pop-push.js +0 -49
  30. package/examples/snippets/02-simplest-reserve-rollback-commit.js +0 -44
  31. package/examples/snippets/03-simplest-producer-consumer-loops.js +0 -77
  32. package/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js +0 -78
  33. package/examples/snippets/05-insert-reserve-rollback-deadletter.js +0 -105
  34. package/examples/snippets/06-random-consumer-producer.js +0 -270
  35. package/examples/snippets/07-stream-simple.js +0 -53
  36. package/examples/snippets/redislabs-consumer-producer.js +0 -44
  37. package/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js +0 -52
  38. package/examples/webhooks/README.md +0 -36
  39. package/examples/webhooks/app.js +0 -70
  40. package/examples/webhooks/consumer.js +0 -98
  41. package/examples/webhooks/index.js +0 -55
  42. package/examples/webhooks/package-lock.json +0 -500
  43. package/examples/webhooks/package.json +0 -23
  44. package/examples/webhooks/wh-payload.json +0 -38
  45. package/playground/irc.js +0 -53
  46. package/playground/pl-rollback.js +0 -55
  47. package/playground/pl1.js +0 -74
  48. package/playground/q1.js +0 -34
  49. package/playground/simple-pl.js +0 -42
  50. package/playground/stream-loops.js +0 -114
  51. package/test/backends_bucket-at-least-once.js +0 -302
  52. package/test/backends_deadletter.js +0 -227
  53. package/test/backends_payload.js +0 -542
  54. package/test/backends_push-pop.js +0 -170
  55. package/test/backends_remove.js +0 -320
  56. package/test/backends_reserve-commit-rollback.js +0 -1033
  57. package/test/intraorder.js +0 -325
  58. package/test/pause.js +0 -220
  59. package/test/pipeline-Builder.js +0 -285
  60. package/test/pipeline-ChoiceLink.js +0 -241
  61. package/test/pipeline-DirectLink.js +0 -376
  62. package/test/pipeline-Sink.js +0 -175
  63. package/test/signal.js +0 -196
  64. package/test/stats.js +0 -296
  65. package/test/stream-mongo.js +0 -166
@@ -1,320 +0,0 @@
1
- var async = require ('async');
2
- var should = require ('should');
3
- var _ = require ('lodash');
4
-
5
- var LocalSignal = require ('../signal/local');
6
- var MemStats = require ('../stats/mem');
7
-
8
- const MongoClient = require ('mongodb').MongoClient;
9
- const Redis = require("ioredis");
10
-
11
- function stats (q, cb) {
12
- async.series ({
13
- stats: cb => q.stats(cb),
14
- tsize: cb => q.totalSize(cb),
15
- rsize: cb => q.resvSize(cb),
16
- }, (err, res) => {
17
- // console.log ('stats:', res);
18
- cb (err, res);
19
- });
20
- }
21
-
22
-
23
- function get_mq_factory (MQ, opts, cb) {
24
- const common_opts = {
25
- url: 'mongodb://localhost/keuss_test_backends_remove',
26
- signaller: { provider: LocalSignal},
27
- stats: {provider: MemStats},
28
- deadletter: {
29
- }
30
- };
31
-
32
- // initialize factory
33
- MQ (_.merge ({}, common_opts, opts), cb);
34
- }
35
-
36
- function release_mq_factory (q, factory, cb) {
37
- // console.log ('releasing mq factory');
38
-
39
- setTimeout (() => {
40
- q.cancel ();
41
- factory.close (cb);
42
- }, 1000);
43
- }
44
-
45
-
46
- /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
47
- [
48
- {label: 'Simple MongoDB', mq: require ('../backends/mongo')},
49
- {label: 'Pipelined MongoDB', mq: require ('../backends/pl-mongo')},
50
- {label: 'Tape MongoDB', mq: require ('../backends/ps-mongo')},
51
- {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
52
- {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
53
- ].forEach(function (MQ_item) {
54
- var is_redis = (MQ_item.label == 'Redis OrderedQueue');
55
-
56
- describe('remove operations on ' + MQ_item.label + ' queue backend', () => {
57
- const MQ = MQ_item.mq;
58
-
59
- beforeEach (done => {
60
- done();
61
- });
62
-
63
- afterEach (done => async.series ([
64
- cb => setTimeout (cb, 1000),
65
- cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_remove', (err, cl) => {
66
- if (err) return done (err);
67
- cl.db().dropDatabase (() => cl.close (cb))
68
- }),
69
- cb => {
70
- const redis = new Redis();
71
- redis.keys ('*', (err, keys) => {
72
- const tasks = [];
73
- _.each (keys, k => tasks.push (cb => redis.del (k, cb)));
74
- async.series (tasks, (err, res) => {
75
- redis.disconnect();
76
- cb (err, res);
77
- });
78
- });
79
- }
80
- ], done));
81
-
82
-
83
- it('fails on invalid id', done => {
84
- // skip if redis
85
- if (is_redis) return done();
86
-
87
- async.waterfall ([
88
- cb => get_mq_factory (MQ, {}, cb),
89
- (factory, cb) => {
90
- const q = factory.queue('test_queue_remove', {});
91
- q.remove ('invalid-id', err => cb (null, err, q, factory))
92
- },
93
- (err, q, factory, cb) => {
94
- err.should.match (/id \[invalid-id\] can not be used as remove id/);
95
- release_mq_factory (q, factory, cb);
96
- }
97
- ], done);
98
- });
99
-
100
-
101
- it('fails on nonexistent id', done => {
102
- async.waterfall ([
103
- cb => get_mq_factory (MQ, {}, cb),
104
- (factory, cb) => {
105
- const q = factory.queue('test_queue_remove', {});
106
- q.remove ('00112233445566778899aabb', (err, res) => cb (null, err, res, q, factory))
107
- },
108
- (err, res, q, factory, cb) => {
109
- should (err).be.null();
110
- res.should.be.false();
111
- release_mq_factory (q, factory, cb);
112
- }
113
- ], done);
114
- });
115
-
116
-
117
- it('deletes regular element ok', done => {
118
- const ctx = {};
119
- async.waterfall ([
120
- cb => get_mq_factory (MQ, {}, cb),
121
- (factory, cb) => {
122
- const q = factory.queue('test_queue_remove', {});
123
- cb (null, q, factory);
124
- },
125
- (q, factory, cb) => async.series ([
126
- cb => q.push ({a: 1, b: 'oo'}, cb),
127
- cb => setTimeout (cb, 1000),
128
- cb => stats (q, cb)
129
- ], (err, res) => {
130
- ctx.id = res[0];
131
- ctx.id.should.not.be.null();
132
- res[2].should.eql ({
133
- stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
134
- tsize: 1,
135
- rsize: is_redis ? null : 0
136
- })
137
- cb (err, q, factory);
138
- }),
139
- (q, factory, cb) => {
140
- q.remove (ctx.id, (err, res) => {
141
- should (err).be.null();
142
- res.should.be.true();
143
- cb (err, q, factory);
144
- });
145
- },
146
- (q, factory, cb) => async.series ([
147
- cb => stats (q, cb),
148
- cb => setTimeout (cb, 1000),
149
- ], (err, res) => {
150
- res[0].should.eql ({
151
- stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
152
- tsize: 0,
153
- rsize: is_redis ? null : 0
154
- });
155
- cb (err, q, factory);
156
- }),
157
- (q, factory, cb) => {
158
- release_mq_factory (q, factory, cb);
159
- }
160
- ], done);
161
- });
162
-
163
-
164
- it('does not delete reserved element', done => {
165
- const ctx = {};
166
- async.waterfall ([
167
- cb => get_mq_factory (MQ, {}, cb),
168
- (factory, cb) => {
169
- const q = factory.queue('test_queue_remove', {});
170
- cb (null, q, factory);
171
- },
172
- (q, factory, cb) => async.series ([
173
- cb => q.push ({a: 1, b: 'oo'}, cb),
174
- cb => setTimeout (cb, 1000),
175
- ], (err, res) => {
176
- ctx.id = res[0];
177
- ctx.id.should.not.be.null();
178
- cb (err, q, factory);
179
- }),
180
- (q, factory, cb) => {
181
- q.pop('c1', { reserve: true }, (err, res) => {
182
- // res._id.should.eql (ctx.id);
183
- cb (err, q, factory);
184
- });
185
- },
186
- (q, factory, cb) => {
187
- q.remove (ctx.id, (err, res) => {
188
- should (err).be.null();
189
- res.should.be.false();
190
- cb (err, q, factory);
191
- });
192
- },
193
- (q, factory, cb) => async.series ([
194
- cb => stats (q, cb),
195
- cb => setTimeout (cb, 1000),
196
- ], (err, res) => {
197
- res[0].should.eql ({
198
- stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0, deadletter: 0 },
199
- tsize: 1,
200
- rsize: is_redis ? null : 1
201
- });
202
- cb (err, q, factory);
203
- }),
204
- (q, factory, cb) => {
205
- release_mq_factory (q, factory, cb);
206
- }
207
- ], done);
208
- });
209
-
210
-
211
- it('does delete reserved+rolledback element', done => {
212
- const ctx = {};
213
- async.waterfall ([
214
- cb => get_mq_factory (MQ, {}, cb),
215
- (factory, cb) => {
216
- const q = factory.queue('test_queue_remove', {});
217
- cb (null, q, factory);
218
- },
219
- (q, factory, cb) => async.series ([
220
- cb => q.push ({a: 1, b: 'oo'}, cb),
221
- cb => setTimeout (cb, 1000),
222
- ], (err, res) => {
223
- ctx.id = res[0];
224
- ctx.id.should.not.be.null();
225
- cb (err, q, factory);
226
- }),
227
- (q, factory, cb) => {
228
- q.pop('c1', { reserve: true }, (err, res) => {
229
- res._id.should.eql (ctx.id);
230
- cb (err, q, factory);
231
- });
232
- },
233
- (q, factory, cb) => async.series ([
234
- cb => q.ko(ctx.id, new Date().getTime() + 2000, cb),
235
- cb => setTimeout (cb, 1000),
236
- ], (err, res) => {
237
- res[0].should.be.true();
238
- cb (err, q, factory);
239
- }),
240
- (q, factory, cb) => {
241
- q.remove (ctx.id, (err, res) => {
242
- should (err).be.null();
243
- res.should.be.true();
244
- cb (err, q, factory);
245
- });
246
- },
247
- (q, factory, cb) => async.series ([
248
- cb => stats (q, cb),
249
- cb => setTimeout (cb, 1000),
250
- ], (err, res) => {
251
- res[0].should.eql ({
252
- stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1, deadletter: 0 },
253
- tsize: 0,
254
- rsize: is_redis ? null : 0
255
- });
256
- cb (err, q, factory);
257
- }),
258
- (q, factory, cb) => {
259
- release_mq_factory (q, factory, cb);
260
- }
261
- ], done);
262
- });
263
-
264
-
265
- it('does fail trying to delete an already-deleted element', done => {
266
- const ctx = {};
267
- async.waterfall ([
268
- cb => get_mq_factory (MQ, {}, cb),
269
- (factory, cb) => {
270
- const q = factory.queue('test_queue_remove', {});
271
- cb (null, q, factory);
272
- },
273
- (q, factory, cb) => async.series ([
274
- cb => q.push ({a: 1, b: 'oo'}, cb),
275
- cb => setTimeout (cb, 1000),
276
- cb => stats (q, cb)
277
- ], (err, res) => {
278
- ctx.id = res[0];
279
- ctx.id.should.not.be.null();
280
- res[2].should.eql ({
281
- stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
282
- tsize: 1,
283
- rsize: is_redis ? null : 0
284
- })
285
- cb (err, q, factory);
286
- }),
287
- (q, factory, cb) => {
288
- q.remove (ctx.id, (err, res) => {
289
- should (err).be.null();
290
- res.should.be.true();
291
- cb (err, q, factory);
292
- });
293
- },
294
- (q, factory, cb) => {
295
- q.remove (ctx.id, (err, res) => {
296
- should (err).be.null();
297
- res.should.be.false();
298
- cb (err, q, factory);
299
- });
300
- },
301
- (q, factory, cb) => async.series ([
302
- cb => stats (q, cb),
303
- cb => setTimeout (cb, 1000),
304
- ], (err, res) => {
305
- res[0].should.eql ({
306
- stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
307
- tsize: 0,
308
- rsize: is_redis ? null : 0
309
- });
310
- cb (err, q, factory);
311
- }),
312
- (q, factory, cb) => {
313
- release_mq_factory (q, factory, cb);
314
- }
315
- ], done);
316
- });
317
-
318
-
319
- });
320
- });