keuss 1.6.10 → 1.6.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Pipeline/Queue.js +12 -2
- package/Queue.js +18 -8
- package/TODO +0 -1
- package/backends/bucket-mongo-safe.js +42 -44
- package/backends/bucket-mongo.js +6 -8
- package/examples/snippets/06-random-consumer-producer.js +5 -5
- package/package.json +6 -6
- package/test/backends_bucket-at-least-once.js +43 -35
- package/test/backends_deadletter.js +10 -1
- package/.nyc_output/ad035de4-a3d3-451b-b2ad-402e8ed517b8.json +0 -1
- package/.nyc_output/c8683d6c-e7e6-496b-ab0d-1ff730837d08.json +0 -1
- package/.nyc_output/processinfo/ad035de4-a3d3-451b-b2ad-402e8ed517b8.json +0 -1
- package/.nyc_output/processinfo/c8683d6c-e7e6-496b-ab0d-1ff730837d08.json +0 -1
- package/.nyc_output/processinfo/index.json +0 -1
- package/coverage/base.css +0 -224
- package/coverage/block-navigation.js +0 -79
- package/coverage/index.html +0 -162
- package/coverage/keuss/Pipeline/BaseLink.js.html +0 -636
- package/coverage/keuss/Pipeline/Builder.js.html +0 -615
- package/coverage/keuss/Pipeline/ChoiceLink.js.html +0 -294
- package/coverage/keuss/Pipeline/DirectLink.js.html +0 -231
- package/coverage/keuss/Pipeline/Pipeline.js.html +0 -258
- package/coverage/keuss/Pipeline/Queue.js.html +0 -939
- package/coverage/keuss/Pipeline/Sink.js.html +0 -147
- package/coverage/keuss/Pipeline/index.html +0 -175
- package/coverage/keuss/QFactory-MongoDB-defaults.js.html +0 -219
- package/coverage/keuss/QFactory.js.html +0 -459
- package/coverage/keuss/Queue.js.html +0 -2274
- package/coverage/keuss/Signal.js.html +0 -312
- package/coverage/keuss/Stats.js.html +0 -174
- package/coverage/keuss/backends/bucket-mongo-safe.js.html +0 -2952
- package/coverage/keuss/backends/bucket-mongo.js.html +0 -1167
- package/coverage/keuss/backends/index.html +0 -175
- package/coverage/keuss/backends/mongo.js.html +0 -897
- package/coverage/keuss/backends/pl-mongo.js.html +0 -723
- package/coverage/keuss/backends/ps-mongo.js.html +0 -1038
- package/coverage/keuss/backends/redis-list.js.html +0 -513
- package/coverage/keuss/backends/redis-oq.js.html +0 -543
- package/coverage/keuss/bench/all-mongo.js.html +0 -393
- package/coverage/keuss/bench/index.html +0 -149
- package/coverage/keuss/bench/multi-q.js.html +0 -324
- package/coverage/keuss/bench/redis-oq-consumer-producer.js.html +0 -225
- package/coverage/keuss/bench/redis-oq-consumer.js.html +0 -189
- package/coverage/keuss/bench/redis-oq-producer.js.html +0 -198
- package/coverage/keuss/docusaurus/.docusaurus/client-modules.js.html +0 -93
- package/coverage/keuss/docusaurus/.docusaurus/docusaurus.config.js.html +0 -402
- package/coverage/keuss/docusaurus/.docusaurus/index.html +0 -136
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/01a85c17.4e57aa0a.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/09d5ad39.3009117f.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/1.12cf3eb8.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/12559443.a244228c.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/17896441.5efb3558.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/1be78505.f9c04817.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/2.4e2dc02c.js.html +0 -72
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/3.d549a05d.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/36.db0e809a.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/37637b3c.28c0900b.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/3d8d21df.2228b429.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/45a5cd1f.951da354.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/52c38f7a.785707bc.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/57ffbe3d.04d3e53e.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/5f7c35e9.2fa77599.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6299bd84.ec197a3e.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6875c492.cdf3e939.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6beec1d2.6873e708.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/7b786ad4.bbce08e6.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/8070e160.400a4e57.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/88c93c54.e868a433.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/8d3e6f9e.aadf6d32.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/9bc74468.516633c5.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/9beb87c2.71e3744d.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/a3732fac.b0f98dbf.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/a6aa9e1f.6ac55e6c.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/baf0d4a2.b50efd2d.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/c4f5d8e4.e9dfd999.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/ccc49370.1831890b.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/d4942345.7d23b349.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/e2244795.4ea3d168.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/e64fe266.308265e8.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/f78725cc.5f3e115d.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/fa8d8063.2312a332.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/index.html +0 -565
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/main.66f1008c.js.html +0 -72
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/runtime~main.7d73b048.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/styles.b5ee3016.js.html +0 -69
- package/coverage/keuss/docusaurus/.docusaurus/registry.js.html +0 -165
- package/coverage/keuss/docusaurus/.docusaurus/routes.js.html +0 -447
- package/coverage/keuss/docusaurus/build/01a85c17.4e57aa0a.js.html +0 -69
- package/coverage/keuss/docusaurus/build/09d5ad39.3009117f.js.html +0 -69
- package/coverage/keuss/docusaurus/build/1.12cf3eb8.js.html +0 -69
- package/coverage/keuss/docusaurus/build/12559443.a244228c.js.html +0 -69
- package/coverage/keuss/docusaurus/build/17896441.5efb3558.js.html +0 -69
- package/coverage/keuss/docusaurus/build/1be78505.f9c04817.js.html +0 -69
- package/coverage/keuss/docusaurus/build/2.4e2dc02c.js.html +0 -72
- package/coverage/keuss/docusaurus/build/3.d549a05d.js.html +0 -69
- package/coverage/keuss/docusaurus/build/36.db0e809a.js.html +0 -69
- package/coverage/keuss/docusaurus/build/37637b3c.28c0900b.js.html +0 -69
- package/coverage/keuss/docusaurus/build/3d8d21df.2228b429.js.html +0 -69
- package/coverage/keuss/docusaurus/build/45a5cd1f.951da354.js.html +0 -69
- package/coverage/keuss/docusaurus/build/52c38f7a.785707bc.js.html +0 -69
- package/coverage/keuss/docusaurus/build/57ffbe3d.04d3e53e.js.html +0 -69
- package/coverage/keuss/docusaurus/build/5f7c35e9.2fa77599.js.html +0 -69
- package/coverage/keuss/docusaurus/build/6299bd84.ec197a3e.js.html +0 -69
- package/coverage/keuss/docusaurus/build/6875c492.cdf3e939.js.html +0 -69
- package/coverage/keuss/docusaurus/build/6beec1d2.6873e708.js.html +0 -69
- package/coverage/keuss/docusaurus/build/7b786ad4.bbce08e6.js.html +0 -69
- package/coverage/keuss/docusaurus/build/8070e160.400a4e57.js.html +0 -69
- package/coverage/keuss/docusaurus/build/88c93c54.e868a433.js.html +0 -69
- package/coverage/keuss/docusaurus/build/8d3e6f9e.aadf6d32.js.html +0 -69
- package/coverage/keuss/docusaurus/build/9bc74468.516633c5.js.html +0 -69
- package/coverage/keuss/docusaurus/build/9beb87c2.71e3744d.js.html +0 -69
- package/coverage/keuss/docusaurus/build/a3732fac.b0f98dbf.js.html +0 -69
- package/coverage/keuss/docusaurus/build/a6aa9e1f.6ac55e6c.js.html +0 -69
- package/coverage/keuss/docusaurus/build/baf0d4a2.b50efd2d.js.html +0 -69
- package/coverage/keuss/docusaurus/build/c4f5d8e4.e9dfd999.js.html +0 -69
- package/coverage/keuss/docusaurus/build/ccc49370.1831890b.js.html +0 -69
- package/coverage/keuss/docusaurus/build/d4942345.7d23b349.js.html +0 -69
- package/coverage/keuss/docusaurus/build/e2244795.4ea3d168.js.html +0 -69
- package/coverage/keuss/docusaurus/build/e64fe266.308265e8.js.html +0 -69
- package/coverage/keuss/docusaurus/build/f78725cc.5f3e115d.js.html +0 -69
- package/coverage/keuss/docusaurus/build/fa8d8063.2312a332.js.html +0 -69
- package/coverage/keuss/docusaurus/build/index.html +0 -565
- package/coverage/keuss/docusaurus/build/main.66f1008c.js.html +0 -72
- package/coverage/keuss/docusaurus/build/runtime~main.7d73b048.js.html +0 -69
- package/coverage/keuss/docusaurus/build/styles.b5ee3016.js.html +0 -69
- package/coverage/keuss/docusaurus/docusaurus.config.js.html +0 -360
- package/coverage/keuss/docusaurus/index.html +0 -110
- package/coverage/keuss/docusaurus/sidebars.js.html +0 -96
- package/coverage/keuss/docusaurus/src/pages/index.html +0 -97
- package/coverage/keuss/docusaurus/src/pages/index.js.html +0 -450
- package/coverage/keuss/examples/pipelines/builder/index.html +0 -97
- package/coverage/keuss/examples/pipelines/builder/index.js.html +0 -450
- package/coverage/keuss/examples/pipelines/simplest/index.html +0 -97
- package/coverage/keuss/examples/pipelines/simplest/index.js.html +0 -183
- package/coverage/keuss/examples/pipelines/simulation-fork/index.html +0 -97
- package/coverage/keuss/examples/pipelines/simulation-fork/index.js.html +0 -414
- package/coverage/keuss/examples/snippets/01-simplest-pop-push.js.html +0 -195
- package/coverage/keuss/examples/snippets/02-simplest-reserve-rollback-commit.js.html +0 -201
- package/coverage/keuss/examples/snippets/03-simplest-producer-consumer-loops.js.html +0 -300
- package/coverage/keuss/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js.html +0 -303
- package/coverage/keuss/examples/snippets/05-insert-reserve-rollback-deadletter.js.html +0 -384
- package/coverage/keuss/examples/snippets/06-random-consumer-producer.js.html +0 -879
- package/coverage/keuss/examples/snippets/index.html +0 -188
- package/coverage/keuss/examples/snippets/redislabs-consumer-producer.js.html +0 -201
- package/coverage/keuss/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js.html +0 -222
- package/coverage/keuss/examples/webhooks/app.js.html +0 -279
- package/coverage/keuss/examples/webhooks/consumer.js.html +0 -363
- package/coverage/keuss/examples/webhooks/index.html +0 -123
- package/coverage/keuss/examples/webhooks/index.js.html +0 -234
- package/coverage/keuss/index.html +0 -149
- package/coverage/keuss/playground/index.html +0 -136
- package/coverage/keuss/playground/pl-rollback.js.html +0 -234
- package/coverage/keuss/playground/pl1.js.html +0 -291
- package/coverage/keuss/playground/q1.js.html +0 -168
- package/coverage/keuss/playground/simple-pl.js.html +0 -195
- package/coverage/keuss/signal/index.html +0 -123
- package/coverage/keuss/signal/local.js.html +0 -306
- package/coverage/keuss/signal/mongo-capped.js.html +0 -378
- package/coverage/keuss/signal/redis-pubsub.js.html +0 -399
- package/coverage/keuss/stats/index.html +0 -123
- package/coverage/keuss/stats/mem.js.html +0 -519
- package/coverage/keuss/stats/mongo.js.html +0 -930
- package/coverage/keuss/stats/redis.js.html +0 -894
- package/coverage/keuss/utils/RedisConn.js.html +0 -150
- package/coverage/keuss/utils/RedisOrderedQueue.js.html +0 -864
- package/coverage/keuss/utils/index.html +0 -110
- package/coverage/prettify.css +0 -1
- package/coverage/prettify.js +0 -2
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +0 -170
- package/docusaurus/README.md +0 -33
- package/docusaurus/babel.config.js +0 -3
- package/docusaurus/blog/2020-08-04-welcome.md +0 -10
- package/docusaurus/docs/about.md +0 -14
- package/docusaurus/docs/api/factory.md +0 -102
- package/docusaurus/docs/api/queue.md +0 -247
- package/docusaurus/docs/api/signal.md +0 -38
- package/docusaurus/docs/api/stats.md +0 -37
- package/docusaurus/docs/changelog.md +0 -51
- package/docusaurus/docs/concepts.md +0 -116
- package/docusaurus/docs/examples.md +0 -10
- package/docusaurus/docs/quickstart.md +0 -195
- package/docusaurus/docs/style-guide.md +0 -202
- package/docusaurus/docs/usage/buckets.md +0 -43
- package/docusaurus/docs/usage/no-signaller.md +0 -16
- package/docusaurus/docs/usage/pipelines/about.md +0 -128
- package/docusaurus/docs/usage/pipelines/building.md +0 -158
- package/docusaurus/docs/usage/pipelines/examples.md +0 -10
- package/docusaurus/docs/usage/pipelines/processors.md +0 -187
- package/docusaurus/docs/usage/putting-all-together.md +0 -153
- package/docusaurus/docs/usage/redis-conns.md +0 -70
- package/docusaurus/docs/usage/shutdown.md +0 -11
- package/docusaurus/docusaurus.config.js +0 -96
- package/docusaurus/package.json +0 -33
- package/docusaurus/sidebars.js +0 -44
- package/docusaurus/src/css/custom.css +0 -25
- package/docusaurus/src/pages/index.js +0 -127
- package/docusaurus/src/pages/styles.module.css +0 -37
- package/docusaurus/static/.nojekyll +0 -0
- package/docusaurus/static/img/favicon.ico +0 -0
- package/docusaurus/static/img/logo.svg +0 -1
- package/docusaurus/yarn.lock +0 -8384
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: building
|
|
3
|
-
title: Building Pipelines
|
|
4
|
-
sidebar_label: Building
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
Pipelines can be built in 3 ways:
|
|
8
|
-
|
|
9
|
-
* By directly creating queues and processors, and bonding them together. This is rather low-level and is not the recommended way
|
|
10
|
-
* By using a `PipelineBuilder`. This object provides a fluent API that's convenient and very simple. This is the recommended way to created pipelines in code
|
|
11
|
-
* By using the method `pipelineFromRecipe` offered by the Queues Factories supporting pipelining. This allows a whole pipeline to be defined in a set of strings and therefore in external files; this makes pipelies portable, reproductible and totally cluster-ready
|
|
12
|
-
|
|
13
|
-
## Direct Pipeline Creation
|
|
14
|
-
|
|
15
|
-
This is a quite simple approach: you create the queues, then you create the Processors that would glue them. Processors take i theit constructors the queues they use, so it's rather straightforward:
|
|
16
|
-
|
|
17
|
-
```javascript
|
|
18
|
-
const MQ = require ('../../../backends/pl-mongo');
|
|
19
|
-
const DCT = require ('../../../Pipeline/DirectLink');
|
|
20
|
-
const SNK = require ('../../../Pipeline/Sink');
|
|
21
|
-
const CHC = require ('../../../Pipeline/ChoiceLink');
|
|
22
|
-
|
|
23
|
-
function sink_process (elem, done) {
|
|
24
|
-
// define processing for Sinks
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const factory_opts = {
|
|
28
|
-
// ...
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
// initialize factory
|
|
32
|
-
MQ (factory_opts, (err, factory) => {
|
|
33
|
-
if (err) return console.error (err);
|
|
34
|
-
|
|
35
|
-
// factory ready, create queues on default pipeline
|
|
36
|
-
const q_opts = {aaa: 666, b: 'yy'};
|
|
37
|
-
const q1 = factory.queue ('pl_many_q_1', q_opts);
|
|
38
|
-
const q2 = factory.queue ('pl_many_q_2', q_opts);
|
|
39
|
-
const q3 = factory.queue ('pl_many_q_3', q_opts);
|
|
40
|
-
const q4 = factory.queue ('pl_many_q_4', q_opts);
|
|
41
|
-
const q5 = factory.queue ('pl_many_q_5', q_opts);
|
|
42
|
-
|
|
43
|
-
// tie them up:
|
|
44
|
-
const dl1 = new DCT (q1, q2);
|
|
45
|
-
const cl1 = new CHC (q2, [q3, q4, q5]);
|
|
46
|
-
const sk1 = new SNK (q3);
|
|
47
|
-
const sk2 = new SNK (q4);
|
|
48
|
-
const sk3 = new SNK (q5);
|
|
49
|
-
|
|
50
|
-
sk1.on_data (sink_process);
|
|
51
|
-
sk2.on_data (sink_process);
|
|
52
|
-
sk3.on_data (sink_process);
|
|
53
|
-
|
|
54
|
-
cl1.on_data (function (elem, done) {
|
|
55
|
-
// define processing for the ChoiceLink
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
dl1.on_data (function (elem, done) {
|
|
59
|
-
// define processing for the DirectLink
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
// start the whole lot
|
|
63
|
-
sk1.start ();
|
|
64
|
-
sk2.start ();
|
|
65
|
-
sk3.start ();
|
|
66
|
-
cl1.start ();
|
|
67
|
-
dl1.start ();
|
|
68
|
-
|
|
69
|
-
// pipeline is ready now. Push stuff to queues, see it work
|
|
70
|
-
});
|
|
71
|
-
|
|
72
|
-
```
|
|
73
|
-
|
|
74
|
-
See [Processors](processors.md) for all the available options and features (such as processing functions and error management)
|
|
75
|
-
|
|
76
|
-
## Creation with a `PipelineBuilder`
|
|
77
|
-
|
|
78
|
-
`PipelineBuilder` provides a simpler way to create pipelines using a fluent api. Builders are obtained through `factory.builder()` and offers the following methods:
|
|
79
|
-
|
|
80
|
-
* `pipeline(name)`: initializes a pipeline, passing a name to it. Must be called before any other method, and can be called only once
|
|
81
|
-
* `queue(name, opts)`: creates a queue and adds it to the pipeline
|
|
82
|
-
* `directLink (name_src_q, name_dst_q, process_fn)`: creates a DirectLink linking queues src_q and dst_q (specified by name), using the process function `process_fn`
|
|
83
|
-
* `choiceLink(name_src_q, [name_dst_q1, name_dst_q2, ...name_dst_qn], process_fn)`: creates a ChoiceLink linking src_q and the array of dst_q (specified by name), using the process function `process_fn`
|
|
84
|
-
* `sink(name_src_q, process_fn)`: creates a Sink on queue src_q (specified by name), using the process function `process_fn`
|
|
85
|
-
* `onError(fn)`: sets the `error` event handler for all processirs created in the pipeline. As with the error handler for Processors, `fn` will receive a single param with the error; in this case the error will be augmented by adding an extra field `processor`, which will be areference to the `Processor` object originating the error
|
|
86
|
-
* `done(err, pipeline)`: finished the pipeline creation. No other calls can be done to the builder afterwards. In case of error, the error will be passed in `err`; if all went well `err` will be `null` and the newly created pipeline, an object of type `Pipeline`, will be passed in the `pipeline`; all further interactions with the pipeline will happen through this object
|
|
87
|
-
|
|
88
|
-
### Pipepine object
|
|
89
|
-
|
|
90
|
-
The new Pipeline object exports the following methods:
|
|
91
|
-
|
|
92
|
-
* `start()`: starts the pipeline (simply calls `start()` on all processors)
|
|
93
|
-
* `stop()`: stops the pipeline (simply calls `stop()` on all processors)
|
|
94
|
-
|
|
95
|
-
Here's a simplified example (for a complete, working example see [here](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/builder)):
|
|
96
|
-
|
|
97
|
-
```javascript
|
|
98
|
-
MQ (factory_opts, (err, factory) => {
|
|
99
|
-
if (err) return console.error (err);
|
|
100
|
-
const q_opts = {};
|
|
101
|
-
|
|
102
|
-
factory
|
|
103
|
-
.builder ()
|
|
104
|
-
.pipeline ('the-pipeline')
|
|
105
|
-
.queue ('test_pl_1', q_opts)
|
|
106
|
-
.queue ('test_pl_2', q_opts)
|
|
107
|
-
.queue ('test_pl_3', q_opts)
|
|
108
|
-
.queue ('test_pl_4', q_opts)
|
|
109
|
-
.queue ('test_pl_5', q_opts)
|
|
110
|
-
.directLink ('test_pl_1', 'test_pl_2', dl_process)
|
|
111
|
-
.choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
|
|
112
|
-
.sink ('test_pl_3', sink_process)
|
|
113
|
-
.sink ('test_pl_4', sink_process)
|
|
114
|
-
.sink ('test_pl_5', sink_process)
|
|
115
|
-
.onError (console.log)
|
|
116
|
-
.done ((err, pl) => {
|
|
117
|
-
if (err) return console.error (err);
|
|
118
|
-
// pipeline pl is ready
|
|
119
|
-
pl.start ();
|
|
120
|
-
// pipeline pl is running
|
|
121
|
-
});
|
|
122
|
-
});
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
## Creation with `Factory.pipelineFromRecipe`
|
|
126
|
-
|
|
127
|
-
`Factory.pipelineFromRecipe` provides a way to define pipelines entirely from strings, including queue, processors, the functions
|
|
128
|
-
to be used as process functions and all the code used on those functions. In this way a full, self-contained pipeline can be specified
|
|
129
|
-
in a file or set of files
|
|
130
|
-
|
|
131
|
-
Under the hood it uses [node.js VM module](https://nodejs.org/dist/latest-v12.x/docs/api/vm.html) to create the `Pipeline` object: once created it can be used normally outside of the creation VM
|
|
132
|
-
|
|
133
|
-
`Factory.pipelineFromRecipe` is provided only on factories created from backends with pipelining support. This single method take the following parameters:
|
|
134
|
-
|
|
135
|
-
```javascript
|
|
136
|
-
Factory.pipelineFromRecipe (
|
|
137
|
-
pipeline_name,
|
|
138
|
-
array_of_bootstrap_code,
|
|
139
|
-
array_of_setup_code,
|
|
140
|
-
extra_context,
|
|
141
|
-
done
|
|
142
|
-
)
|
|
143
|
-
```
|
|
144
|
-
|
|
145
|
-
1. A new VM is created using the merge of the default context and the parameter `extra_context`. The default context contains the following:
|
|
146
|
-
* `Buffer`
|
|
147
|
-
* `require`
|
|
148
|
-
* `clearImmediate`, `clearInterval`, `clearTimeout`, `setImmediate`, `setTimeout`, `setInterval`
|
|
149
|
-
* `TextEncoder`, `TextDecoder`
|
|
150
|
-
* `URL`, `URLSearchParams`
|
|
151
|
-
* `builder`: an already initialized builder object, as in `factory.builder ().pipeline (name)`
|
|
152
|
-
* `done`: a function to call when the pipeline is ready, or an error arises. Expects to be `fn (err, pipeline)`
|
|
153
|
-
2. Each of the strings in the `array_of_bootstrap_code` is executed in the VM
|
|
154
|
-
3. Each of the strings in the `array_of_setup_code` is executed in the VM. It is expected to eventually call `done` with the error or the finished pipeline (`done`is accesible in the context)
|
|
155
|
-
|
|
156
|
-
The whole idea is to prepare all the needed code for processors' functions in the `array_of_bootstrap_code`, then create the pipeline in the `array_of_setup_code`, calling the `done` function whith the created pipeline
|
|
157
|
-
|
|
158
|
-
You can find a full example [here](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/fromRecipe)
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: examples
|
|
3
|
-
title: Examples
|
|
4
|
-
sidebar_label: Examples
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
* [simplest](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/simplest): a very simple pipeline with just 2 queues connected with a DirectLink
|
|
8
|
-
* [simulation-fork](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/simulation-fork): a somewhat complete example with `DirectLink`, `ChoiceLink` and `Sink` instances connecting 5 queues in a fork-like flow. Each process function adds some basic simulated processing with payload updates, random failures and random delays. It also uses deadletter
|
|
9
|
-
* [builder](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/builder): variant of `simulation-fork` done with a pipeline builder
|
|
10
|
-
* [fromRecipe](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/fromRecipe): variant of `simulation-fork`, almost indentical to `builder` but using a `factory.pipelineFromRecipe`
|
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: processors
|
|
3
|
-
title: Processors
|
|
4
|
-
sidebar_label: Processors
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
A small hierarchy of processors is provided with Pipelines:
|
|
8
|
-
|
|
9
|
-
## BaseLink
|
|
10
|
-
|
|
11
|
-
Common base for all Processors, provides all the functionality common to all. It can not be used directly
|
|
12
|
-
|
|
13
|
-
### Creation
|
|
14
|
-
|
|
15
|
-
```javascript
|
|
16
|
-
const bl = new BaseLink (src_q, opts)
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
Although not intended to be instantiated, this serves as common initialization to all Processors
|
|
20
|
-
|
|
21
|
-
* `src_q` must be a pipelined queue
|
|
22
|
-
* `opts` can contain:
|
|
23
|
-
* `retry_factor_t, retry_base_t`: they control the delay imposed to an element when it is rolled back. The formula is
|
|
24
|
-
|
|
25
|
-
`delay-in-seconds = item.tries * retry_factor_t + retry_base_t`
|
|
26
|
-
|
|
27
|
-
They default to `2` and `1` respectively
|
|
28
|
-
* `mature`: Date instance or unix timestamp (in milliseconds, as integer) expressing the not-before timestamp for the item, to be used when calling `pl_step()` in the src queue
|
|
29
|
-
* `delay`: delay in seconds to calculate `mature`, if `mature` is not specified
|
|
30
|
-
|
|
31
|
-
### Methods
|
|
32
|
-
|
|
33
|
-
* `src()`: returns src queue
|
|
34
|
-
* `name()`: returns Processor name
|
|
35
|
-
* `on_data(fn)`: specifies the process function to be applied to each element
|
|
36
|
-
* `start(fn)`: starts the processor. Optionally, a process function can be passed; if not passed the process function must have been previously specified using `on_data()`
|
|
37
|
-
* `stop()`: stops the Processor
|
|
38
|
-
|
|
39
|
-
### Process Function
|
|
40
|
-
|
|
41
|
-
The function passed into `on_data()` or `start()` provides the processor logic; this function is referred to as *processor function*. This function is called on each step of the Processor loop with the reserved item, and it is expected to calls its callback once done with the item. The way the function calls the callback determines what happens with the item afterwards
|
|
42
|
-
|
|
43
|
-
The function looks like this:
|
|
44
|
-
|
|
45
|
-
```javascript
|
|
46
|
-
function (item, cb) {
|
|
47
|
-
...
|
|
48
|
-
})
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
The `item` is received exactly as it comes as result of a (successful) `reserve()` call on the source queue; after processing the item `cb` should be called once to finish the processing of `item` and proceed with the next loop cycle. The callback has the following signature:
|
|
52
|
-
|
|
53
|
-
```javascript
|
|
54
|
-
cb (err, res);
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
where:
|
|
58
|
-
|
|
59
|
-
* if `err` is not nil
|
|
60
|
-
* if `err.drop` is exactly `true` the item is committed in the src queue and therefore dropped from the pipeline
|
|
61
|
-
* else the item is rolled back in the src queue, using the processor's `retry_factor_t` and `retry_base_t` to calculate the retry delay. If the queue was created with deadletter support, the item would be moved to the deadletter queue if it has reached its maximum number of rollbacks; in such case, the movement into deadletter is also atomic
|
|
62
|
-
* else (if `err` is nill)
|
|
63
|
-
* if (`res.drop` is exactly `true` the item is committed in the src queue and therefore dropped from the pipeline
|
|
64
|
-
* else the item is passed to the text queue in the pipeline (by means of `pl_next()`)
|
|
65
|
-
* if `res.mature` or `res.delay` exist (or they were specified at the processor's creation) they are used to calculate the delay/mature of the element in the destination queue
|
|
66
|
-
* if `res.payload` exists it is used to replace the item's payload entirely
|
|
67
|
-
* else if `res.update` exists it is used as mongodb-update operations on the item's payload
|
|
68
|
-
|
|
69
|
-
All those operations happen in an atomic way
|
|
70
|
-
|
|
71
|
-
#### Semantic `this` in process function
|
|
72
|
-
|
|
73
|
-
The function is bound to the processor, so the function can access and use processor's primitives. For example, it can insert copies of the item, or new items, in any of the source or destination queues
|
|
74
|
-
|
|
75
|
-
In order to use this functionality the process function can not be declared as an 'arrow' function, since those can not be bound. Use the classic `function xxxx (item, cb) {...}` if you intend to access the underlying Processor
|
|
76
|
-
|
|
77
|
-
### Events
|
|
78
|
-
|
|
79
|
-
BaseLink inherits from `EventEmitter` and publishes the following events:
|
|
80
|
-
|
|
81
|
-
* `error`: an error happened in the internal loop. It comes with one parameter, an object with the following fields:
|
|
82
|
-
* `on`: exact type of error:
|
|
83
|
-
* `src-queue-pop`: error while reserving an element on the src queue
|
|
84
|
-
* `src-queue-commit-on-error`: error while committing an element on the src queue when an error was passed and `err.drop==true`
|
|
85
|
-
* `src-queue-rollback-on-error`: error while rolling back an element on the src queue when an error was passed
|
|
86
|
-
* `src-queue-commit-on-drop`: error while committing an element an element on the src queue when processed ok and `res.drop==true`
|
|
87
|
-
* `next-queue`: error while atomically moving the element to the next queue
|
|
88
|
-
* `elem`: element that caused the error. Not present in `src-queue-pop`
|
|
89
|
-
* `error`: original error object
|
|
90
|
-
* `opts`: (only present in `next-queue`) options passed internally to `pl_step()`
|
|
91
|
-
|
|
92
|
-
## DirectLink
|
|
93
|
-
|
|
94
|
-
Processor that connects the source queue to exactly one destination queue:
|
|
95
|
-
|
|
96
|
-
```
|
|
97
|
-
src-queue --> DirectLink --> dst-queue
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
### Creation
|
|
101
|
-
|
|
102
|
-
```javascript
|
|
103
|
-
const PDL = require ('keuss/Pipeline/DirectLink');
|
|
104
|
-
const bl = new PDL (src_q, dst_q, opts);
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
In addition to `BaseLink`:
|
|
108
|
-
|
|
109
|
-
* `dst_q` must be a pipelined queue; also, both `src_q` and `dst_q` must be of the same type and must belong to the same pipeline
|
|
110
|
-
|
|
111
|
-
### Methods
|
|
112
|
-
|
|
113
|
-
In addition to those of `BaseLink`
|
|
114
|
-
|
|
115
|
-
* `dst()`: returns destination queue
|
|
116
|
-
|
|
117
|
-
### Process Function
|
|
118
|
-
|
|
119
|
-
In the case of successful processing (i.e.: no `err` in the callback invocation) the item is atomically moved to the `dst` queue.
|
|
120
|
-
|
|
121
|
-
No other semantics are added to the process function.
|
|
122
|
-
|
|
123
|
-
## ChoiceLink
|
|
124
|
-
|
|
125
|
-
Processor that connects the source queue to an array of queues; after processing, each item would be moved to exactly one of those queues:
|
|
126
|
-
|
|
127
|
-
```
|
|
128
|
-
|--> dst-queue-0
|
|
129
|
-
|--> dst-queue-1
|
|
130
|
-
src-queue --> ChoiceLink --|--> dst-queue-2
|
|
131
|
-
| ...
|
|
132
|
-
|--> dst-queue-n
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
### Creation
|
|
136
|
-
|
|
137
|
-
```javascript
|
|
138
|
-
const PCL = require ('keuss/Pipeline/ChoiceLink');
|
|
139
|
-
const cl = new PCL (src_q, array_of_dst_queues, opts);
|
|
140
|
-
```
|
|
141
|
-
|
|
142
|
-
In addition to `BaseLink`:
|
|
143
|
-
|
|
144
|
-
* `array_of_dst_queues` must be an array of pipelined queues; each one must be of the same type and must belong to the same pipeline than `src_q`
|
|
145
|
-
|
|
146
|
-
### Methods
|
|
147
|
-
|
|
148
|
-
In addition to those of `BaseLink`:
|
|
149
|
-
|
|
150
|
-
* `dst_by_idx(idx)`: returns destination queue from the array, selected by array index (integer)
|
|
151
|
-
* `dst_by_name(name)`: returns destination queue from the array, selected by queue name (string)
|
|
152
|
-
* `dst_dimension ()`: returns number of possible destination queues
|
|
153
|
-
* `dst_names ()`: returns an array with the names of all dst queues
|
|
154
|
-
|
|
155
|
-
### Process Function
|
|
156
|
-
|
|
157
|
-
ChoiceLink expects an `res.dst` in the callback invocation, which must fullfill one of those conditions:
|
|
158
|
-
|
|
159
|
-
* be an integer and resolve to a valid element when applied as index to the array of destination queues
|
|
160
|
-
* be a string and correspond to the name of one of the destination queues
|
|
161
|
-
|
|
162
|
-
The element will be moved atomically to the specified destination queue upon successful processing (i.e.: no `err`in the callback invocation)
|
|
163
|
-
|
|
164
|
-
## Sink
|
|
165
|
-
|
|
166
|
-
Processor that connects the source queue to exactly zero destination queue. That is, a termination point: successfully processed elements are always removed from the pipeline
|
|
167
|
-
|
|
168
|
-
```
|
|
169
|
-
src-queue --> Sink
|
|
170
|
-
```
|
|
171
|
-
|
|
172
|
-
### Creation
|
|
173
|
-
|
|
174
|
-
```javascript
|
|
175
|
-
const PS = require ('keuss/Pipeline/Sink');
|
|
176
|
-
const bl = new PS (src_q, opts);
|
|
177
|
-
```
|
|
178
|
-
|
|
179
|
-
No extra parameters are expected in addition to those of `BaseLink`
|
|
180
|
-
|
|
181
|
-
### Methods
|
|
182
|
-
|
|
183
|
-
No extra methods are provided in addition to those of `BaseLink`
|
|
184
|
-
|
|
185
|
-
### Process Function
|
|
186
|
-
|
|
187
|
-
In the case of successful processing (i.e.: no `err` in the callback invocation) the item is removed from the pipeline, exactly as if `res.drop` were specified. Actually, `res` is totally ignored
|
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: putting-all-together
|
|
3
|
-
title: Putting all together
|
|
4
|
-
sidebar_label: Putting all together
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Factory initialization
|
|
8
|
-
|
|
9
|
-
First, choose a factory, also known as backend:
|
|
10
|
-
|
|
11
|
-
```javascript
|
|
12
|
-
const MQ = require ('../../backends/mongo');
|
|
13
|
-
```
|
|
14
|
-
|
|
15
|
-
Then, simply execute the backend, passing the config, to obtain a working factory:
|
|
16
|
-
|
|
17
|
-
```javascript
|
|
18
|
-
MQ ({
|
|
19
|
-
url: 'mongodb://localhost/keuss_test'
|
|
20
|
-
}, (err, factory) => {
|
|
21
|
-
if (err) return console.error(err);
|
|
22
|
-
|
|
23
|
-
// factory is ready to be used
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
You can create and use as many factories as desided, from the same or many backends
|
|
29
|
-
|
|
30
|
-
## Queue creation
|
|
31
|
-
|
|
32
|
-
You use the factory to create queues:
|
|
33
|
-
|
|
34
|
-
```javascript
|
|
35
|
-
const q1 = factory.queue ('test_queue_1', {});
|
|
36
|
-
const q2 = factory.queue ('test_queue_2', {});
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
A queue can be created more than once with the same name, inside the same factory (this is a common procedure when consumer and producer are separated). The effect would be virtually the same as sharing the queue:
|
|
40
|
-
|
|
41
|
-
```javascript
|
|
42
|
-
const q_consumer = factory.queue ('test_queue', {});
|
|
43
|
-
const q_producer = factory.queue ('test_queue', {});
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
## Put elements in queue (push)
|
|
47
|
-
|
|
48
|
-
putting elements in a queue is simple enough:
|
|
49
|
-
|
|
50
|
-
```javascript
|
|
51
|
-
const elem = {
|
|
52
|
-
elem: 1,
|
|
53
|
-
headline: 'something something',
|
|
54
|
-
tags: {
|
|
55
|
-
a: 1,
|
|
56
|
-
b: 2
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
|
-
|
|
60
|
-
q1.push (elem, (err, res) => {
|
|
61
|
-
// push finished, either with error or success...
|
|
62
|
-
}),
|
|
63
|
-
```
|
|
64
|
-
|
|
65
|
-
Or, push also some headers:
|
|
66
|
-
|
|
67
|
-
```javascript
|
|
68
|
-
const elem = {
|
|
69
|
-
elem: 1,
|
|
70
|
-
headline: 'something something',
|
|
71
|
-
tags: {
|
|
72
|
-
a: 1,
|
|
73
|
-
b: 2
|
|
74
|
-
}
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
const headers = {
|
|
78
|
-
h1: 'a string',
|
|
79
|
-
h2: false,
|
|
80
|
-
h3: 666
|
|
81
|
-
};
|
|
82
|
-
|
|
83
|
-
q1.push (elem, {hdrs: headers}, (err, res) => {
|
|
84
|
-
// push finished, either with error or success...
|
|
85
|
-
}),
|
|
86
|
-
```
|
|
87
|
-
|
|
88
|
-
## Get elements from queue (pop)
|
|
89
|
-
|
|
90
|
-
The easiest way to get elements from a queue is with a simple pop(). This would block until an element is ready, it would remove it from the queue and return it.
|
|
91
|
-
|
|
92
|
-
This way of working is often referred to as *at-most-once* since it guarantees that each element in the queue will be processed no more than one time (it would be zero times, if something happens after `pop()` ands but before the element is actually managed)
|
|
93
|
-
|
|
94
|
-
```javascript
|
|
95
|
-
const consumer_label = 'consumer-1';
|
|
96
|
-
q1.pop (consumer_label, (err, res) => {
|
|
97
|
-
if (err) return console.error (err);
|
|
98
|
-
|
|
99
|
-
console.log (res);
|
|
100
|
-
// this should print something like:
|
|
101
|
-
// {
|
|
102
|
-
// _id: <some id>,
|
|
103
|
-
// mature: <some date>,
|
|
104
|
-
// payload: { elem: 1, headline: 'something something', tags: { a: 1, b: 2 } },
|
|
105
|
-
// tries: 0
|
|
106
|
-
// hdrs: {}
|
|
107
|
-
// }
|
|
108
|
-
// that is, the actual element is at res.payload
|
|
109
|
-
//
|
|
110
|
-
// if the element was pushed with headers, they will be placed inside hdrs:
|
|
111
|
-
// hdrs: {h1: 'a string', h2: false, h3: 666}
|
|
112
|
-
}
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
## Reserve-commit-rollback
|
|
116
|
-
|
|
117
|
-
A safer way to consume from a queue is using reserve: elements are reserved, processed and only then committed (and removed from the queue). A reserved element can also be rolled back (returned to queue) if the processing failed and the element needs to be reprocessed in the future; also, any reserved element will auto-rollback after some tiem elapsed, if neither commit nor rollback is done. This is known as *at-least-once* cause it guarantees all elements wold be processed at least once
|
|
118
|
-
|
|
119
|
-
```javascript
|
|
120
|
-
const consumer_label = 'consumer-1';
|
|
121
|
-
q1.pop (consumer_label, {reserve: true}, (err, elem) => {
|
|
122
|
-
if (err) return console.error (err);
|
|
123
|
-
|
|
124
|
-
// res is ready to be processed
|
|
125
|
-
do_some_processing (elem.payload, err => {
|
|
126
|
-
if (err) {
|
|
127
|
-
// error, rollback so it gets retried, adding a delay
|
|
128
|
-
const next_t = new Date().getTime () + 15000;
|
|
129
|
-
q1.ko (item, next_t, () = >{
|
|
130
|
-
// the element is returned to queue, but it won't be available until 15 secs have passed
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
else {
|
|
134
|
-
// processing went fine, commit element
|
|
135
|
-
q1.ok (item, () => {
|
|
136
|
-
// the element is removed from the queue
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
});
|
|
140
|
-
}
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
## Termination
|
|
144
|
-
|
|
145
|
-
Once all is done, you can free all the resources associated to the factory by closing it:
|
|
146
|
-
|
|
147
|
-
```javascript
|
|
148
|
-
factory.close (err => {
|
|
149
|
-
// factory is now closed and cannot be used anymore
|
|
150
|
-
});
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
Once a factory is closed it cannot be used, *and all the queues created through it will becomes unusable too*
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: redis-conns
|
|
3
|
-
title: Redis Connections
|
|
4
|
-
sidebar_label: Redis Connections
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
Keuss relies on [ioredis](https://www.npmjs.com/package/ioredis) for connecting to redis. Anytime a redis connection is needed, keuss will create it from the opts object passed:
|
|
8
|
-
|
|
9
|
-
* If `opts` is a function, it is executed. It is expected to return a redis connection
|
|
10
|
-
* If it's an object and contains a `Redis` field, this field is used to create a new [ioredis Redis object](https://github.com/luin/ioredis), as in *return new Redis (opts.Redis)*
|
|
11
|
-
* if it's an object and contains a `Cluster` field, this field is used to create a new [ioredis Redis.Cluster](https://redis.io/topics/cluster-spec) object, as in *return new Redis.Cluster (opts.Cluster)*
|
|
12
|
-
* Else, a ioredis Redis object is created with `opts` as param, as in *return new Redis (opts)*
|
|
13
|
-
|
|
14
|
-
This apparent complexity is required since redis connections are inherently created in a different way for standalone, sentinel and cluster servers
|
|
15
|
-
|
|
16
|
-
## Examples:
|
|
17
|
-
|
|
18
|
-
* Default options:
|
|
19
|
-
|
|
20
|
-
```javascript
|
|
21
|
-
var MQ = require ('keuss/backends/redis-list');
|
|
22
|
-
var factory_opts = {};
|
|
23
|
-
|
|
24
|
-
MQ (factory_opts, (err, factory) => {
|
|
25
|
-
...
|
|
26
|
-
});
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
* Specific redis params for ioredis Redis client:
|
|
30
|
-
|
|
31
|
-
```javascript
|
|
32
|
-
var MQ = require ('keuss/backends/redis-list');
|
|
33
|
-
var factory_opts = {
|
|
34
|
-
redis: {
|
|
35
|
-
Redis: {
|
|
36
|
-
port: 12293,
|
|
37
|
-
host: 'some-redis-instance.somewhere.com',
|
|
38
|
-
family: 4,
|
|
39
|
-
password: 'xxxx',
|
|
40
|
-
db: 0
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
};
|
|
44
|
-
|
|
45
|
-
MQ (factory_opts, (err, factory) => {
|
|
46
|
-
...
|
|
47
|
-
});
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
* Using a factory function:
|
|
51
|
-
|
|
52
|
-
```javascript
|
|
53
|
-
var MQ = require ('keuss/backends/redis-list');
|
|
54
|
-
var Redis = require ('ioredis');
|
|
55
|
-
var factory_opts = {
|
|
56
|
-
redis: function () {
|
|
57
|
-
return new Redis ({
|
|
58
|
-
port: 12293,
|
|
59
|
-
host: 'some-redis-instance.somewhere.com',
|
|
60
|
-
family: 4,
|
|
61
|
-
password: 'xxxx',
|
|
62
|
-
db: 0
|
|
63
|
-
})
|
|
64
|
-
}
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
MQ (factory_opts, (err, factory) => {
|
|
68
|
-
...
|
|
69
|
-
});
|
|
70
|
-
```
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: shutdown
|
|
3
|
-
title: Shutdown
|
|
4
|
-
sidebar_label: Shutdown
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
It is a good practice to call [`close(cb)`](../api/factory#close-factory-close) on the factories to release all resources once you're done, or at shutdown if you want your shutdowns clean and graceful (the log-lived redis or mongodb connections are terminated here, for example); also, you should loop over your queues and perform a [`drain()`](../api/queue#drain-drain-queue) on them before calling `close()` on their factories: this will ensure any un-consumed data is popped, and any unwritten data is written. Also, it'll ensure all your (local) waiting consumers will end (on 'cancel' error).
|
|
8
|
-
|
|
9
|
-
:::note
|
|
10
|
-
Factories do not keep track of the created Queues, so this can't be done internally as part of the `close()`; this may change in the future.
|
|
11
|
-
:::
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
module.exports = {
|
|
2
|
-
title: 'Keuss Job Queues',
|
|
3
|
-
tagline: 'Job Queues for node.js, backed by redis and/or MongoDB',
|
|
4
|
-
url: 'https://pepmartinez.github.io',
|
|
5
|
-
baseUrl: '/keuss/',
|
|
6
|
-
onBrokenLinks: 'throw',
|
|
7
|
-
favicon: 'img/favicon.ico',
|
|
8
|
-
organizationName: 'pepmartinez',
|
|
9
|
-
projectName: 'keuss',
|
|
10
|
-
themeConfig: {
|
|
11
|
-
navbar: {
|
|
12
|
-
title: 'Keuss job queues',
|
|
13
|
-
logo: {
|
|
14
|
-
alt: 'Site Logo',
|
|
15
|
-
src: 'img/logo.svg',
|
|
16
|
-
},
|
|
17
|
-
items: [
|
|
18
|
-
{
|
|
19
|
-
to: 'docs/',
|
|
20
|
-
activeBasePath: 'docs',
|
|
21
|
-
label: 'Docs',
|
|
22
|
-
position: 'left',
|
|
23
|
-
},
|
|
24
|
-
{to: 'blog', label: 'Blog', position: 'left'},
|
|
25
|
-
{
|
|
26
|
-
href: 'https://github.com/pepmartinez/keuss',
|
|
27
|
-
label: 'GitHub',
|
|
28
|
-
position: 'right',
|
|
29
|
-
},
|
|
30
|
-
],
|
|
31
|
-
},
|
|
32
|
-
footer: {
|
|
33
|
-
style: 'dark',
|
|
34
|
-
links: [
|
|
35
|
-
{
|
|
36
|
-
title: 'Start Here',
|
|
37
|
-
items: [
|
|
38
|
-
{
|
|
39
|
-
label: 'Quickstart',
|
|
40
|
-
to: 'docs/quickstart',
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
label: 'Documentation',
|
|
44
|
-
to: 'docs/',
|
|
45
|
-
},
|
|
46
|
-
{
|
|
47
|
-
label: 'Examples',
|
|
48
|
-
to: 'docs/examples',
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
label: 'ChangeLog',
|
|
52
|
-
to: 'docs/changelog',
|
|
53
|
-
},
|
|
54
|
-
],
|
|
55
|
-
},
|
|
56
|
-
{
|
|
57
|
-
title: 'More',
|
|
58
|
-
items: [
|
|
59
|
-
{
|
|
60
|
-
label: 'Blog',
|
|
61
|
-
to: 'blog',
|
|
62
|
-
},
|
|
63
|
-
{
|
|
64
|
-
label: 'GitHub',
|
|
65
|
-
href: 'https://github.com/pepmartinez/keuss',
|
|
66
|
-
},
|
|
67
|
-
],
|
|
68
|
-
},
|
|
69
|
-
],
|
|
70
|
-
copyright: `Copyright © ${new Date().getFullYear()}. Built with Docusaurus.`,
|
|
71
|
-
},
|
|
72
|
-
},
|
|
73
|
-
presets: [
|
|
74
|
-
[
|
|
75
|
-
'@docusaurus/preset-classic',
|
|
76
|
-
{
|
|
77
|
-
docs: {
|
|
78
|
-
// It is recommended to set document id as docs home page (`docs/` path).
|
|
79
|
-
sidebarPath: require.resolve('./sidebars.js'),
|
|
80
|
-
// Please change this to your repo.
|
|
81
|
-
editUrl:
|
|
82
|
-
'https://github.com/pepmartinez/keuss/edit/master/website/',
|
|
83
|
-
},
|
|
84
|
-
blog: {
|
|
85
|
-
showReadingTime: true,
|
|
86
|
-
// Please change this to your repo.
|
|
87
|
-
editUrl:
|
|
88
|
-
'https://github.com/pepmartinez/keuss/edit/master/website/blog/',
|
|
89
|
-
},
|
|
90
|
-
theme: {
|
|
91
|
-
customCss: require.resolve('./src/css/custom.css'),
|
|
92
|
-
},
|
|
93
|
-
},
|
|
94
|
-
],
|
|
95
|
-
],
|
|
96
|
-
};
|