keuss 1.7.4 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Pipeline/Queue.js +0 -1
- package/QFactory.js +49 -17
- package/Queue.js +1 -4
- package/README.md +2 -1
- package/TODO +0 -9
- package/backends/bucket-mongo-safe.js +11 -11
- package/backends/intraorder.js +13 -11
- package/backends/mongo.js +12 -9
- package/backends/pl-mongo.js +8 -5
- package/backends/postgres.js +380 -0
- package/backends/ps-mongo.js +11 -8
- package/backends/redis-list.js +8 -8
- package/backends/redis-oq.js +8 -3
- package/backends/stream-mongo.js +10 -15
- package/package.json +12 -8
- package/stats/mem.js +0 -1
- package/.github/workflows/codeql-analysis.yml +0 -72
- package/bench/all-mongo.js +0 -108
- package/bench/multi-q.js +0 -85
- package/bench/redis-oq-consumer-producer.js +0 -52
- package/bench/redis-oq-consumer.js +0 -40
- package/bench/redis-oq-producer.js +0 -43
- package/docker-compose/docker-compose.yaml +0 -18
- package/docker-compose.yaml +0 -18
- package/examples/pipelines/builder/index.js +0 -99
- package/examples/pipelines/fromRecipe/index.js +0 -127
- package/examples/pipelines/simplest/index.js +0 -38
- package/examples/pipelines/simulation-fork/index.js +0 -115
- package/examples/snippets/01-simplest-pop-push.js +0 -49
- package/examples/snippets/02-simplest-reserve-rollback-commit.js +0 -44
- package/examples/snippets/03-simplest-producer-consumer-loops.js +0 -77
- package/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js +0 -78
- package/examples/snippets/05-insert-reserve-rollback-deadletter.js +0 -105
- package/examples/snippets/06-random-consumer-producer.js +0 -270
- package/examples/snippets/07-stream-simple.js +0 -53
- package/examples/snippets/redislabs-consumer-producer.js +0 -44
- package/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js +0 -52
- package/examples/webhooks/README.md +0 -36
- package/examples/webhooks/app.js +0 -70
- package/examples/webhooks/consumer.js +0 -98
- package/examples/webhooks/index.js +0 -55
- package/examples/webhooks/package-lock.json +0 -500
- package/examples/webhooks/package.json +0 -23
- package/examples/webhooks/wh-payload.json +0 -38
- package/playground/irc.js +0 -53
- package/playground/pl-rollback.js +0 -55
- package/playground/pl1.js +0 -74
- package/playground/q1.js +0 -34
- package/playground/simple-pl.js +0 -42
- package/playground/stream-loops.js +0 -114
- package/test/backends_bucket-at-least-once.js +0 -302
- package/test/backends_deadletter.js +0 -227
- package/test/backends_payload.js +0 -542
- package/test/backends_push-pop.js +0 -170
- package/test/backends_remove.js +0 -320
- package/test/backends_reserve-commit-rollback.js +0 -1033
- package/test/intraorder.js +0 -325
- package/test/pause.js +0 -220
- package/test/pipeline-Builder.js +0 -285
- package/test/pipeline-ChoiceLink.js +0 -241
- package/test/pipeline-DirectLink.js +0 -376
- package/test/pipeline-Sink.js +0 -175
- package/test/signal.js +0 -196
- package/test/stats.js +0 -296
- package/test/stream-mongo.js +0 -166
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
var async = require ('async');
|
|
2
|
-
var should = require ('should');
|
|
3
|
-
var _ = require ('lodash');
|
|
4
|
-
|
|
5
|
-
var LocalSignal = require ('../signal/local');
|
|
6
|
-
var MemStats = require ('../stats/mem');
|
|
7
|
-
|
|
8
|
-
const MongoClient = require ('mongodb').MongoClient;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
function stats (q, cb) {
|
|
12
|
-
async.series ({
|
|
13
|
-
stats: cb => q.stats(cb),
|
|
14
|
-
tsize: cb => q.totalSize(cb)
|
|
15
|
-
}, (err, res) => {
|
|
16
|
-
// console.log ('stats:', res);
|
|
17
|
-
cb (err, res);
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
function pop (q, stage, cb) {
|
|
22
|
-
q.pop('c1', { reserve: true }, (err, res) => {
|
|
23
|
-
stage.obj = res;
|
|
24
|
-
// console.log('reserved element %o', res);
|
|
25
|
-
cb(err);
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function reject (q, stage, cb) {
|
|
30
|
-
var next_t = new Date().getTime() + 2000;
|
|
31
|
-
|
|
32
|
-
q.ko (stage.obj, next_t, (err, res) => {
|
|
33
|
-
if (err) {
|
|
34
|
-
// console.error ('error in rollback of %s: %j', stage.obj._id, err);
|
|
35
|
-
return cb (err);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
// console.log('rolled back element %s: %o', stage.obj._id, res);
|
|
39
|
-
cb();
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function accept (q, stage, cb) {
|
|
44
|
-
q.ok (stage.obj, (err, res) => {
|
|
45
|
-
if (err) {
|
|
46
|
-
// console.error ('error in rollback of %s: %j', stage.obj._id, err);
|
|
47
|
-
return cb (err);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// console.log('commited element %s: %j', stage.obj._id, res);
|
|
51
|
-
cb();
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function get_mq_factory (MQ, opts, cb) {
|
|
56
|
-
const common_opts = {
|
|
57
|
-
url: 'mongodb://localhost/keuss_test_backends_deadletter',
|
|
58
|
-
signaller: { provider: LocalSignal},
|
|
59
|
-
stats: {provider: MemStats},
|
|
60
|
-
deadletter: {
|
|
61
|
-
}
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
// initialize factory
|
|
65
|
-
MQ (_.merge ({}, common_opts, opts), cb);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function release_mq_factory (q, factory, cb) {
|
|
69
|
-
// console.log ('releasing mq factory');
|
|
70
|
-
|
|
71
|
-
setTimeout (() => {
|
|
72
|
-
q.cancel ();
|
|
73
|
-
factory.close (cb);
|
|
74
|
-
}, 1000);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
79
|
-
[
|
|
80
|
-
// {label: 'Simple MongoDB', mq: require ('../backends/mongo')},
|
|
81
|
-
// {label: 'Pipelined MongoDB', mq: require ('../backends/pl-mongo')},
|
|
82
|
-
// {label: 'Tape MongoDB', mq: require ('../backends/ps-mongo')},
|
|
83
|
-
// {label: 'Stream MongoDB', mq: require ('../backends/stream-mongo')},
|
|
84
|
-
// {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
|
|
85
|
-
// {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')},
|
|
86
|
-
{label: 'Mongo IntraOrder', mq: require ('../backends/intraorder')},
|
|
87
|
-
].forEach(function (MQ_item) {
|
|
88
|
-
describe('rollback and deadletters with ' + MQ_item.label + ' queue backend', function () {
|
|
89
|
-
const MQ = MQ_item.mq;
|
|
90
|
-
|
|
91
|
-
beforeEach (done => {
|
|
92
|
-
done();
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
afterEach (done => async.series ([
|
|
96
|
-
cb => setTimeout (cb, 1000),
|
|
97
|
-
cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_deadletter', (err, cl) => {
|
|
98
|
-
if (err) return done (err);
|
|
99
|
-
cl.db().dropDatabase (() => cl.close (cb))
|
|
100
|
-
})
|
|
101
|
-
], done));
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
it('repeated ko causes item to be moved to deadletter if deadletter is set', done => {
|
|
105
|
-
var factory_opts = {
|
|
106
|
-
deadletter: {
|
|
107
|
-
max_ko: 3
|
|
108
|
-
}
|
|
109
|
-
};
|
|
110
|
-
|
|
111
|
-
const pl = {
|
|
112
|
-
elem: 1,
|
|
113
|
-
pl: 'twetrwte'
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
const hdrs = {aaa: 'qw', bbb: '666'};
|
|
117
|
-
|
|
118
|
-
async.waterfall ([
|
|
119
|
-
cb => get_mq_factory (MQ, factory_opts, cb),
|
|
120
|
-
(factory, cb) => {
|
|
121
|
-
const q = factory.queue('test_queue_deadletter', {});
|
|
122
|
-
const stage = {};
|
|
123
|
-
let tries= 0;
|
|
124
|
-
|
|
125
|
-
async.race ([
|
|
126
|
-
cb => async.series([
|
|
127
|
-
cb => q.push (pl, {hdrs}, cb),
|
|
128
|
-
cb => pop (q, stage, cb),
|
|
129
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
130
|
-
cb => pop (q, stage, cb),
|
|
131
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
132
|
-
cb => pop (q, stage, cb),
|
|
133
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
134
|
-
cb => pop (q, stage, cb),
|
|
135
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
136
|
-
cb => pop (q, stage, cb),
|
|
137
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
138
|
-
cb => pop (q, stage, cb),
|
|
139
|
-
cb => reject (q, stage, (err) => {tries++;cb()}),
|
|
140
|
-
], err => {
|
|
141
|
-
err.should.equal ('cancel');
|
|
142
|
-
cb ();
|
|
143
|
-
}),
|
|
144
|
-
cb => factory.deadletter_queue().pop('c2', (err, res) => {
|
|
145
|
-
res.payload.should.eql (pl);
|
|
146
|
-
res.hdrs.should.match ({
|
|
147
|
-
aaa: "qw",
|
|
148
|
-
bbb: "666",
|
|
149
|
-
'x-dl-from-queue': "test_queue_deadletter",
|
|
150
|
-
'x-dl-t': /.+/,
|
|
151
|
-
'x-dl-tries': 4
|
|
152
|
-
});
|
|
153
|
-
tries.should.equal (5);
|
|
154
|
-
cb (err);
|
|
155
|
-
})
|
|
156
|
-
], err => cb (err, q, factory));
|
|
157
|
-
},
|
|
158
|
-
(q, factory, cb) => {
|
|
159
|
-
async.series ([
|
|
160
|
-
cb => setTimeout (cb, 1000),
|
|
161
|
-
cb => stats (q, cb),
|
|
162
|
-
cb => stats (factory.deadletter_queue(), cb),
|
|
163
|
-
], (err, res) => {
|
|
164
|
-
res[1].tsize.should.equal (0);
|
|
165
|
-
res[2].tsize.should.equal (0);
|
|
166
|
-
|
|
167
|
-
cb (err, q, factory);
|
|
168
|
-
});
|
|
169
|
-
},
|
|
170
|
-
(q, factory, cb) => release_mq_factory (q, factory, cb)
|
|
171
|
-
], done);
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
it('repeated ko causes item NOT to be moved to deadletter if deadletter is not set', done => {
|
|
176
|
-
var factory_opts = {
|
|
177
|
-
};
|
|
178
|
-
|
|
179
|
-
const pl = {
|
|
180
|
-
elem: 1,
|
|
181
|
-
pl: 'twetrwte'
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
async.waterfall ([
|
|
185
|
-
cb => get_mq_factory (MQ, factory_opts, cb),
|
|
186
|
-
(factory, cb) => {
|
|
187
|
-
const q = factory.queue('test_queue_deadletter', {});
|
|
188
|
-
const stage = {};
|
|
189
|
-
|
|
190
|
-
async.series([
|
|
191
|
-
cb => q.push (pl, cb),
|
|
192
|
-
cb => pop (q, stage, cb),
|
|
193
|
-
cb => reject (q, stage, cb),
|
|
194
|
-
cb => pop (q, stage, cb),
|
|
195
|
-
cb => reject (q, stage, cb),
|
|
196
|
-
cb => pop (q, stage, cb),
|
|
197
|
-
cb => reject (q, stage, cb),
|
|
198
|
-
cb => pop (q, stage, cb),
|
|
199
|
-
cb => reject (q, stage, cb),
|
|
200
|
-
cb => pop (q, stage, cb),
|
|
201
|
-
cb => reject (q, stage, cb),
|
|
202
|
-
cb => pop (q, stage, cb),
|
|
203
|
-
cb => accept (q, stage, cb)
|
|
204
|
-
], err => {
|
|
205
|
-
cb (err, q, factory)
|
|
206
|
-
});
|
|
207
|
-
},
|
|
208
|
-
(q, factory, cb) => {
|
|
209
|
-
async.series ([
|
|
210
|
-
cb => setTimeout (cb, 1000),
|
|
211
|
-
cb => stats (q, cb),
|
|
212
|
-
cb => stats (factory.deadletter_queue(), cb),
|
|
213
|
-
], (err, res) => {
|
|
214
|
-
res[1].tsize.should.equal (0);
|
|
215
|
-
res[2].tsize.should.equal (0);
|
|
216
|
-
|
|
217
|
-
cb (err, q, factory);
|
|
218
|
-
});
|
|
219
|
-
},
|
|
220
|
-
(q, factory, cb) => release_mq_factory (q, factory, cb)
|
|
221
|
-
], done);
|
|
222
|
-
});
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
});
|
|
227
|
-
});
|