keuss 1.7.4 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/Pipeline/Queue.js +0 -1
  2. package/QFactory.js +49 -17
  3. package/Queue.js +1 -4
  4. package/README.md +2 -1
  5. package/TODO +0 -9
  6. package/backends/bucket-mongo-safe.js +11 -11
  7. package/backends/intraorder.js +13 -11
  8. package/backends/mongo.js +12 -9
  9. package/backends/pl-mongo.js +8 -5
  10. package/backends/postgres.js +380 -0
  11. package/backends/ps-mongo.js +11 -8
  12. package/backends/redis-list.js +8 -8
  13. package/backends/redis-oq.js +8 -3
  14. package/backends/stream-mongo.js +10 -15
  15. package/package.json +12 -8
  16. package/stats/mem.js +0 -1
  17. package/.github/workflows/codeql-analysis.yml +0 -72
  18. package/bench/all-mongo.js +0 -108
  19. package/bench/multi-q.js +0 -85
  20. package/bench/redis-oq-consumer-producer.js +0 -52
  21. package/bench/redis-oq-consumer.js +0 -40
  22. package/bench/redis-oq-producer.js +0 -43
  23. package/docker-compose/docker-compose.yaml +0 -18
  24. package/docker-compose.yaml +0 -18
  25. package/examples/pipelines/builder/index.js +0 -99
  26. package/examples/pipelines/fromRecipe/index.js +0 -127
  27. package/examples/pipelines/simplest/index.js +0 -38
  28. package/examples/pipelines/simulation-fork/index.js +0 -115
  29. package/examples/snippets/01-simplest-pop-push.js +0 -49
  30. package/examples/snippets/02-simplest-reserve-rollback-commit.js +0 -44
  31. package/examples/snippets/03-simplest-producer-consumer-loops.js +0 -77
  32. package/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js +0 -78
  33. package/examples/snippets/05-insert-reserve-rollback-deadletter.js +0 -105
  34. package/examples/snippets/06-random-consumer-producer.js +0 -270
  35. package/examples/snippets/07-stream-simple.js +0 -53
  36. package/examples/snippets/redislabs-consumer-producer.js +0 -44
  37. package/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js +0 -52
  38. package/examples/webhooks/README.md +0 -36
  39. package/examples/webhooks/app.js +0 -70
  40. package/examples/webhooks/consumer.js +0 -98
  41. package/examples/webhooks/index.js +0 -55
  42. package/examples/webhooks/package-lock.json +0 -500
  43. package/examples/webhooks/package.json +0 -23
  44. package/examples/webhooks/wh-payload.json +0 -38
  45. package/playground/irc.js +0 -53
  46. package/playground/pl-rollback.js +0 -55
  47. package/playground/pl1.js +0 -74
  48. package/playground/q1.js +0 -34
  49. package/playground/simple-pl.js +0 -42
  50. package/playground/stream-loops.js +0 -114
  51. package/test/backends_bucket-at-least-once.js +0 -302
  52. package/test/backends_deadletter.js +0 -227
  53. package/test/backends_payload.js +0 -542
  54. package/test/backends_push-pop.js +0 -170
  55. package/test/backends_remove.js +0 -320
  56. package/test/backends_reserve-commit-rollback.js +0 -1033
  57. package/test/intraorder.js +0 -325
  58. package/test/pause.js +0 -220
  59. package/test/pipeline-Builder.js +0 -285
  60. package/test/pipeline-ChoiceLink.js +0 -241
  61. package/test/pipeline-DirectLink.js +0 -376
  62. package/test/pipeline-Sink.js +0 -175
  63. package/test/signal.js +0 -196
  64. package/test/stats.js +0 -296
  65. package/test/stream-mongo.js +0 -166
@@ -1,227 +0,0 @@
1
- var async = require ('async');
2
- var should = require ('should');
3
- var _ = require ('lodash');
4
-
5
- var LocalSignal = require ('../signal/local');
6
- var MemStats = require ('../stats/mem');
7
-
8
- const MongoClient = require ('mongodb').MongoClient;
9
-
10
-
11
- function stats (q, cb) {
12
- async.series ({
13
- stats: cb => q.stats(cb),
14
- tsize: cb => q.totalSize(cb)
15
- }, (err, res) => {
16
- // console.log ('stats:', res);
17
- cb (err, res);
18
- });
19
- }
20
-
21
- function pop (q, stage, cb) {
22
- q.pop('c1', { reserve: true }, (err, res) => {
23
- stage.obj = res;
24
- // console.log('reserved element %o', res);
25
- cb(err);
26
- });
27
- }
28
-
29
- function reject (q, stage, cb) {
30
- var next_t = new Date().getTime() + 2000;
31
-
32
- q.ko (stage.obj, next_t, (err, res) => {
33
- if (err) {
34
- // console.error ('error in rollback of %s: %j', stage.obj._id, err);
35
- return cb (err);
36
- }
37
-
38
- // console.log('rolled back element %s: %o', stage.obj._id, res);
39
- cb();
40
- });
41
- }
42
-
43
- function accept (q, stage, cb) {
44
- q.ok (stage.obj, (err, res) => {
45
- if (err) {
46
- // console.error ('error in rollback of %s: %j', stage.obj._id, err);
47
- return cb (err);
48
- }
49
-
50
- // console.log('commited element %s: %j', stage.obj._id, res);
51
- cb();
52
- });
53
- }
54
-
55
- function get_mq_factory (MQ, opts, cb) {
56
- const common_opts = {
57
- url: 'mongodb://localhost/keuss_test_backends_deadletter',
58
- signaller: { provider: LocalSignal},
59
- stats: {provider: MemStats},
60
- deadletter: {
61
- }
62
- };
63
-
64
- // initialize factory
65
- MQ (_.merge ({}, common_opts, opts), cb);
66
- }
67
-
68
- function release_mq_factory (q, factory, cb) {
69
- // console.log ('releasing mq factory');
70
-
71
- setTimeout (() => {
72
- q.cancel ();
73
- factory.close (cb);
74
- }, 1000);
75
- }
76
-
77
-
78
- /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
79
- [
80
- // {label: 'Simple MongoDB', mq: require ('../backends/mongo')},
81
- // {label: 'Pipelined MongoDB', mq: require ('../backends/pl-mongo')},
82
- // {label: 'Tape MongoDB', mq: require ('../backends/ps-mongo')},
83
- // {label: 'Stream MongoDB', mq: require ('../backends/stream-mongo')},
84
- // {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
85
- // {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')},
86
- {label: 'Mongo IntraOrder', mq: require ('../backends/intraorder')},
87
- ].forEach(function (MQ_item) {
88
- describe('rollback and deadletters with ' + MQ_item.label + ' queue backend', function () {
89
- const MQ = MQ_item.mq;
90
-
91
- beforeEach (done => {
92
- done();
93
- });
94
-
95
- afterEach (done => async.series ([
96
- cb => setTimeout (cb, 1000),
97
- cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_deadletter', (err, cl) => {
98
- if (err) return done (err);
99
- cl.db().dropDatabase (() => cl.close (cb))
100
- })
101
- ], done));
102
-
103
-
104
- it('repeated ko causes item to be moved to deadletter if deadletter is set', done => {
105
- var factory_opts = {
106
- deadletter: {
107
- max_ko: 3
108
- }
109
- };
110
-
111
- const pl = {
112
- elem: 1,
113
- pl: 'twetrwte'
114
- };
115
-
116
- const hdrs = {aaa: 'qw', bbb: '666'};
117
-
118
- async.waterfall ([
119
- cb => get_mq_factory (MQ, factory_opts, cb),
120
- (factory, cb) => {
121
- const q = factory.queue('test_queue_deadletter', {});
122
- const stage = {};
123
- let tries= 0;
124
-
125
- async.race ([
126
- cb => async.series([
127
- cb => q.push (pl, {hdrs}, cb),
128
- cb => pop (q, stage, cb),
129
- cb => reject (q, stage, (err) => {tries++;cb()}),
130
- cb => pop (q, stage, cb),
131
- cb => reject (q, stage, (err) => {tries++;cb()}),
132
- cb => pop (q, stage, cb),
133
- cb => reject (q, stage, (err) => {tries++;cb()}),
134
- cb => pop (q, stage, cb),
135
- cb => reject (q, stage, (err) => {tries++;cb()}),
136
- cb => pop (q, stage, cb),
137
- cb => reject (q, stage, (err) => {tries++;cb()}),
138
- cb => pop (q, stage, cb),
139
- cb => reject (q, stage, (err) => {tries++;cb()}),
140
- ], err => {
141
- err.should.equal ('cancel');
142
- cb ();
143
- }),
144
- cb => factory.deadletter_queue().pop('c2', (err, res) => {
145
- res.payload.should.eql (pl);
146
- res.hdrs.should.match ({
147
- aaa: "qw",
148
- bbb: "666",
149
- 'x-dl-from-queue': "test_queue_deadletter",
150
- 'x-dl-t': /.+/,
151
- 'x-dl-tries': 4
152
- });
153
- tries.should.equal (5);
154
- cb (err);
155
- })
156
- ], err => cb (err, q, factory));
157
- },
158
- (q, factory, cb) => {
159
- async.series ([
160
- cb => setTimeout (cb, 1000),
161
- cb => stats (q, cb),
162
- cb => stats (factory.deadletter_queue(), cb),
163
- ], (err, res) => {
164
- res[1].tsize.should.equal (0);
165
- res[2].tsize.should.equal (0);
166
-
167
- cb (err, q, factory);
168
- });
169
- },
170
- (q, factory, cb) => release_mq_factory (q, factory, cb)
171
- ], done);
172
- });
173
-
174
-
175
- it('repeated ko causes item NOT to be moved to deadletter if deadletter is not set', done => {
176
- var factory_opts = {
177
- };
178
-
179
- const pl = {
180
- elem: 1,
181
- pl: 'twetrwte'
182
- };
183
-
184
- async.waterfall ([
185
- cb => get_mq_factory (MQ, factory_opts, cb),
186
- (factory, cb) => {
187
- const q = factory.queue('test_queue_deadletter', {});
188
- const stage = {};
189
-
190
- async.series([
191
- cb => q.push (pl, cb),
192
- cb => pop (q, stage, cb),
193
- cb => reject (q, stage, cb),
194
- cb => pop (q, stage, cb),
195
- cb => reject (q, stage, cb),
196
- cb => pop (q, stage, cb),
197
- cb => reject (q, stage, cb),
198
- cb => pop (q, stage, cb),
199
- cb => reject (q, stage, cb),
200
- cb => pop (q, stage, cb),
201
- cb => reject (q, stage, cb),
202
- cb => pop (q, stage, cb),
203
- cb => accept (q, stage, cb)
204
- ], err => {
205
- cb (err, q, factory)
206
- });
207
- },
208
- (q, factory, cb) => {
209
- async.series ([
210
- cb => setTimeout (cb, 1000),
211
- cb => stats (q, cb),
212
- cb => stats (factory.deadletter_queue(), cb),
213
- ], (err, res) => {
214
- res[1].tsize.should.equal (0);
215
- res[2].tsize.should.equal (0);
216
-
217
- cb (err, q, factory);
218
- });
219
- },
220
- (q, factory, cb) => release_mq_factory (q, factory, cb)
221
- ], done);
222
- });
223
-
224
-
225
-
226
- });
227
- });