keuss 1.6.10 → 1.6.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/Pipeline/Queue.js +12 -2
  2. package/Queue.js +18 -8
  3. package/TODO +0 -1
  4. package/backends/bucket-mongo-safe.js +42 -44
  5. package/backends/bucket-mongo.js +6 -8
  6. package/examples/snippets/06-random-consumer-producer.js +5 -5
  7. package/package.json +6 -6
  8. package/test/backends_bucket-at-least-once.js +43 -35
  9. package/test/backends_deadletter.js +10 -1
  10. package/.nyc_output/ad035de4-a3d3-451b-b2ad-402e8ed517b8.json +0 -1
  11. package/.nyc_output/c8683d6c-e7e6-496b-ab0d-1ff730837d08.json +0 -1
  12. package/.nyc_output/processinfo/ad035de4-a3d3-451b-b2ad-402e8ed517b8.json +0 -1
  13. package/.nyc_output/processinfo/c8683d6c-e7e6-496b-ab0d-1ff730837d08.json +0 -1
  14. package/.nyc_output/processinfo/index.json +0 -1
  15. package/coverage/base.css +0 -224
  16. package/coverage/block-navigation.js +0 -79
  17. package/coverage/index.html +0 -162
  18. package/coverage/keuss/Pipeline/BaseLink.js.html +0 -636
  19. package/coverage/keuss/Pipeline/Builder.js.html +0 -615
  20. package/coverage/keuss/Pipeline/ChoiceLink.js.html +0 -294
  21. package/coverage/keuss/Pipeline/DirectLink.js.html +0 -231
  22. package/coverage/keuss/Pipeline/Pipeline.js.html +0 -258
  23. package/coverage/keuss/Pipeline/Queue.js.html +0 -939
  24. package/coverage/keuss/Pipeline/Sink.js.html +0 -147
  25. package/coverage/keuss/Pipeline/index.html +0 -175
  26. package/coverage/keuss/QFactory-MongoDB-defaults.js.html +0 -219
  27. package/coverage/keuss/QFactory.js.html +0 -459
  28. package/coverage/keuss/Queue.js.html +0 -2274
  29. package/coverage/keuss/Signal.js.html +0 -312
  30. package/coverage/keuss/Stats.js.html +0 -174
  31. package/coverage/keuss/backends/bucket-mongo-safe.js.html +0 -2952
  32. package/coverage/keuss/backends/bucket-mongo.js.html +0 -1167
  33. package/coverage/keuss/backends/index.html +0 -175
  34. package/coverage/keuss/backends/mongo.js.html +0 -897
  35. package/coverage/keuss/backends/pl-mongo.js.html +0 -723
  36. package/coverage/keuss/backends/ps-mongo.js.html +0 -1038
  37. package/coverage/keuss/backends/redis-list.js.html +0 -513
  38. package/coverage/keuss/backends/redis-oq.js.html +0 -543
  39. package/coverage/keuss/bench/all-mongo.js.html +0 -393
  40. package/coverage/keuss/bench/index.html +0 -149
  41. package/coverage/keuss/bench/multi-q.js.html +0 -324
  42. package/coverage/keuss/bench/redis-oq-consumer-producer.js.html +0 -225
  43. package/coverage/keuss/bench/redis-oq-consumer.js.html +0 -189
  44. package/coverage/keuss/bench/redis-oq-producer.js.html +0 -198
  45. package/coverage/keuss/docusaurus/.docusaurus/client-modules.js.html +0 -93
  46. package/coverage/keuss/docusaurus/.docusaurus/docusaurus.config.js.html +0 -402
  47. package/coverage/keuss/docusaurus/.docusaurus/index.html +0 -136
  48. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/01a85c17.4e57aa0a.js.html +0 -69
  49. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/09d5ad39.3009117f.js.html +0 -69
  50. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/1.12cf3eb8.js.html +0 -69
  51. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/12559443.a244228c.js.html +0 -69
  52. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/17896441.5efb3558.js.html +0 -69
  53. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/1be78505.f9c04817.js.html +0 -69
  54. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/2.4e2dc02c.js.html +0 -72
  55. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/3.d549a05d.js.html +0 -69
  56. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/36.db0e809a.js.html +0 -69
  57. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/37637b3c.28c0900b.js.html +0 -69
  58. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/3d8d21df.2228b429.js.html +0 -69
  59. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/45a5cd1f.951da354.js.html +0 -69
  60. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/52c38f7a.785707bc.js.html +0 -69
  61. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/57ffbe3d.04d3e53e.js.html +0 -69
  62. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/5f7c35e9.2fa77599.js.html +0 -69
  63. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6299bd84.ec197a3e.js.html +0 -69
  64. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6875c492.cdf3e939.js.html +0 -69
  65. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/6beec1d2.6873e708.js.html +0 -69
  66. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/7b786ad4.bbce08e6.js.html +0 -69
  67. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/8070e160.400a4e57.js.html +0 -69
  68. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/88c93c54.e868a433.js.html +0 -69
  69. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/8d3e6f9e.aadf6d32.js.html +0 -69
  70. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/9bc74468.516633c5.js.html +0 -69
  71. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/9beb87c2.71e3744d.js.html +0 -69
  72. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/a3732fac.b0f98dbf.js.html +0 -69
  73. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/a6aa9e1f.6ac55e6c.js.html +0 -69
  74. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/baf0d4a2.b50efd2d.js.html +0 -69
  75. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/c4f5d8e4.e9dfd999.js.html +0 -69
  76. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/ccc49370.1831890b.js.html +0 -69
  77. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/d4942345.7d23b349.js.html +0 -69
  78. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/e2244795.4ea3d168.js.html +0 -69
  79. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/e64fe266.308265e8.js.html +0 -69
  80. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/f78725cc.5f3e115d.js.html +0 -69
  81. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/fa8d8063.2312a332.js.html +0 -69
  82. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/index.html +0 -565
  83. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/main.66f1008c.js.html +0 -72
  84. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/runtime~main.7d73b048.js.html +0 -69
  85. package/coverage/keuss/docusaurus/.docusaurus/keuss-gh-pages/styles.b5ee3016.js.html +0 -69
  86. package/coverage/keuss/docusaurus/.docusaurus/registry.js.html +0 -165
  87. package/coverage/keuss/docusaurus/.docusaurus/routes.js.html +0 -447
  88. package/coverage/keuss/docusaurus/build/01a85c17.4e57aa0a.js.html +0 -69
  89. package/coverage/keuss/docusaurus/build/09d5ad39.3009117f.js.html +0 -69
  90. package/coverage/keuss/docusaurus/build/1.12cf3eb8.js.html +0 -69
  91. package/coverage/keuss/docusaurus/build/12559443.a244228c.js.html +0 -69
  92. package/coverage/keuss/docusaurus/build/17896441.5efb3558.js.html +0 -69
  93. package/coverage/keuss/docusaurus/build/1be78505.f9c04817.js.html +0 -69
  94. package/coverage/keuss/docusaurus/build/2.4e2dc02c.js.html +0 -72
  95. package/coverage/keuss/docusaurus/build/3.d549a05d.js.html +0 -69
  96. package/coverage/keuss/docusaurus/build/36.db0e809a.js.html +0 -69
  97. package/coverage/keuss/docusaurus/build/37637b3c.28c0900b.js.html +0 -69
  98. package/coverage/keuss/docusaurus/build/3d8d21df.2228b429.js.html +0 -69
  99. package/coverage/keuss/docusaurus/build/45a5cd1f.951da354.js.html +0 -69
  100. package/coverage/keuss/docusaurus/build/52c38f7a.785707bc.js.html +0 -69
  101. package/coverage/keuss/docusaurus/build/57ffbe3d.04d3e53e.js.html +0 -69
  102. package/coverage/keuss/docusaurus/build/5f7c35e9.2fa77599.js.html +0 -69
  103. package/coverage/keuss/docusaurus/build/6299bd84.ec197a3e.js.html +0 -69
  104. package/coverage/keuss/docusaurus/build/6875c492.cdf3e939.js.html +0 -69
  105. package/coverage/keuss/docusaurus/build/6beec1d2.6873e708.js.html +0 -69
  106. package/coverage/keuss/docusaurus/build/7b786ad4.bbce08e6.js.html +0 -69
  107. package/coverage/keuss/docusaurus/build/8070e160.400a4e57.js.html +0 -69
  108. package/coverage/keuss/docusaurus/build/88c93c54.e868a433.js.html +0 -69
  109. package/coverage/keuss/docusaurus/build/8d3e6f9e.aadf6d32.js.html +0 -69
  110. package/coverage/keuss/docusaurus/build/9bc74468.516633c5.js.html +0 -69
  111. package/coverage/keuss/docusaurus/build/9beb87c2.71e3744d.js.html +0 -69
  112. package/coverage/keuss/docusaurus/build/a3732fac.b0f98dbf.js.html +0 -69
  113. package/coverage/keuss/docusaurus/build/a6aa9e1f.6ac55e6c.js.html +0 -69
  114. package/coverage/keuss/docusaurus/build/baf0d4a2.b50efd2d.js.html +0 -69
  115. package/coverage/keuss/docusaurus/build/c4f5d8e4.e9dfd999.js.html +0 -69
  116. package/coverage/keuss/docusaurus/build/ccc49370.1831890b.js.html +0 -69
  117. package/coverage/keuss/docusaurus/build/d4942345.7d23b349.js.html +0 -69
  118. package/coverage/keuss/docusaurus/build/e2244795.4ea3d168.js.html +0 -69
  119. package/coverage/keuss/docusaurus/build/e64fe266.308265e8.js.html +0 -69
  120. package/coverage/keuss/docusaurus/build/f78725cc.5f3e115d.js.html +0 -69
  121. package/coverage/keuss/docusaurus/build/fa8d8063.2312a332.js.html +0 -69
  122. package/coverage/keuss/docusaurus/build/index.html +0 -565
  123. package/coverage/keuss/docusaurus/build/main.66f1008c.js.html +0 -72
  124. package/coverage/keuss/docusaurus/build/runtime~main.7d73b048.js.html +0 -69
  125. package/coverage/keuss/docusaurus/build/styles.b5ee3016.js.html +0 -69
  126. package/coverage/keuss/docusaurus/docusaurus.config.js.html +0 -360
  127. package/coverage/keuss/docusaurus/index.html +0 -110
  128. package/coverage/keuss/docusaurus/sidebars.js.html +0 -96
  129. package/coverage/keuss/docusaurus/src/pages/index.html +0 -97
  130. package/coverage/keuss/docusaurus/src/pages/index.js.html +0 -450
  131. package/coverage/keuss/examples/pipelines/builder/index.html +0 -97
  132. package/coverage/keuss/examples/pipelines/builder/index.js.html +0 -450
  133. package/coverage/keuss/examples/pipelines/simplest/index.html +0 -97
  134. package/coverage/keuss/examples/pipelines/simplest/index.js.html +0 -183
  135. package/coverage/keuss/examples/pipelines/simulation-fork/index.html +0 -97
  136. package/coverage/keuss/examples/pipelines/simulation-fork/index.js.html +0 -414
  137. package/coverage/keuss/examples/snippets/01-simplest-pop-push.js.html +0 -195
  138. package/coverage/keuss/examples/snippets/02-simplest-reserve-rollback-commit.js.html +0 -201
  139. package/coverage/keuss/examples/snippets/03-simplest-producer-consumer-loops.js.html +0 -300
  140. package/coverage/keuss/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js.html +0 -303
  141. package/coverage/keuss/examples/snippets/05-insert-reserve-rollback-deadletter.js.html +0 -384
  142. package/coverage/keuss/examples/snippets/06-random-consumer-producer.js.html +0 -879
  143. package/coverage/keuss/examples/snippets/index.html +0 -188
  144. package/coverage/keuss/examples/snippets/redislabs-consumer-producer.js.html +0 -201
  145. package/coverage/keuss/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js.html +0 -222
  146. package/coverage/keuss/examples/webhooks/app.js.html +0 -279
  147. package/coverage/keuss/examples/webhooks/consumer.js.html +0 -363
  148. package/coverage/keuss/examples/webhooks/index.html +0 -123
  149. package/coverage/keuss/examples/webhooks/index.js.html +0 -234
  150. package/coverage/keuss/index.html +0 -149
  151. package/coverage/keuss/playground/index.html +0 -136
  152. package/coverage/keuss/playground/pl-rollback.js.html +0 -234
  153. package/coverage/keuss/playground/pl1.js.html +0 -291
  154. package/coverage/keuss/playground/q1.js.html +0 -168
  155. package/coverage/keuss/playground/simple-pl.js.html +0 -195
  156. package/coverage/keuss/signal/index.html +0 -123
  157. package/coverage/keuss/signal/local.js.html +0 -306
  158. package/coverage/keuss/signal/mongo-capped.js.html +0 -378
  159. package/coverage/keuss/signal/redis-pubsub.js.html +0 -399
  160. package/coverage/keuss/stats/index.html +0 -123
  161. package/coverage/keuss/stats/mem.js.html +0 -519
  162. package/coverage/keuss/stats/mongo.js.html +0 -930
  163. package/coverage/keuss/stats/redis.js.html +0 -894
  164. package/coverage/keuss/utils/RedisConn.js.html +0 -150
  165. package/coverage/keuss/utils/RedisOrderedQueue.js.html +0 -864
  166. package/coverage/keuss/utils/index.html +0 -110
  167. package/coverage/prettify.css +0 -1
  168. package/coverage/prettify.js +0 -2
  169. package/coverage/sort-arrow-sprite.png +0 -0
  170. package/coverage/sorter.js +0 -170
  171. package/docusaurus/README.md +0 -33
  172. package/docusaurus/babel.config.js +0 -3
  173. package/docusaurus/blog/2020-08-04-welcome.md +0 -10
  174. package/docusaurus/docs/about.md +0 -14
  175. package/docusaurus/docs/api/factory.md +0 -102
  176. package/docusaurus/docs/api/queue.md +0 -247
  177. package/docusaurus/docs/api/signal.md +0 -38
  178. package/docusaurus/docs/api/stats.md +0 -37
  179. package/docusaurus/docs/changelog.md +0 -51
  180. package/docusaurus/docs/concepts.md +0 -116
  181. package/docusaurus/docs/examples.md +0 -10
  182. package/docusaurus/docs/quickstart.md +0 -195
  183. package/docusaurus/docs/style-guide.md +0 -202
  184. package/docusaurus/docs/usage/buckets.md +0 -43
  185. package/docusaurus/docs/usage/no-signaller.md +0 -16
  186. package/docusaurus/docs/usage/pipelines/about.md +0 -128
  187. package/docusaurus/docs/usage/pipelines/building.md +0 -158
  188. package/docusaurus/docs/usage/pipelines/examples.md +0 -10
  189. package/docusaurus/docs/usage/pipelines/processors.md +0 -187
  190. package/docusaurus/docs/usage/putting-all-together.md +0 -153
  191. package/docusaurus/docs/usage/redis-conns.md +0 -70
  192. package/docusaurus/docs/usage/shutdown.md +0 -11
  193. package/docusaurus/docusaurus.config.js +0 -96
  194. package/docusaurus/package.json +0 -33
  195. package/docusaurus/sidebars.js +0 -44
  196. package/docusaurus/src/css/custom.css +0 -25
  197. package/docusaurus/src/pages/index.js +0 -127
  198. package/docusaurus/src/pages/styles.module.css +0 -37
  199. package/docusaurus/static/.nojekyll +0 -0
  200. package/docusaurus/static/img/favicon.ico +0 -0
  201. package/docusaurus/static/img/logo.svg +0 -1
  202. package/docusaurus/yarn.lock +0 -8384
@@ -1,158 +0,0 @@
1
- ---
2
- id: building
3
- title: Building Pipelines
4
- sidebar_label: Building
5
- ---
6
-
7
- Pipelines can be built in 3 ways:
8
-
9
- * By directly creating queues and processors, and bonding them together. This is rather low-level and is not the recommended way
10
- * By using a `PipelineBuilder`. This object provides a fluent API that's convenient and very simple. This is the recommended way to created pipelines in code
11
- * By using the method `pipelineFromRecipe` offered by the Queues Factories supporting pipelining. This allows a whole pipeline to be defined in a set of strings and therefore in external files; this makes pipelies portable, reproductible and totally cluster-ready
12
-
13
- ## Direct Pipeline Creation
14
-
15
- This is a quite simple approach: you create the queues, then you create the Processors that would glue them. Processors take i theit constructors the queues they use, so it's rather straightforward:
16
-
17
- ```javascript
18
- const MQ = require ('../../../backends/pl-mongo');
19
- const DCT = require ('../../../Pipeline/DirectLink');
20
- const SNK = require ('../../../Pipeline/Sink');
21
- const CHC = require ('../../../Pipeline/ChoiceLink');
22
-
23
- function sink_process (elem, done) {
24
- // define processing for Sinks
25
- }
26
-
27
- const factory_opts = {
28
- // ...
29
- };
30
-
31
- // initialize factory
32
- MQ (factory_opts, (err, factory) => {
33
- if (err) return console.error (err);
34
-
35
- // factory ready, create queues on default pipeline
36
- const q_opts = {aaa: 666, b: 'yy'};
37
- const q1 = factory.queue ('pl_many_q_1', q_opts);
38
- const q2 = factory.queue ('pl_many_q_2', q_opts);
39
- const q3 = factory.queue ('pl_many_q_3', q_opts);
40
- const q4 = factory.queue ('pl_many_q_4', q_opts);
41
- const q5 = factory.queue ('pl_many_q_5', q_opts);
42
-
43
- // tie them up:
44
- const dl1 = new DCT (q1, q2);
45
- const cl1 = new CHC (q2, [q3, q4, q5]);
46
- const sk1 = new SNK (q3);
47
- const sk2 = new SNK (q4);
48
- const sk3 = new SNK (q5);
49
-
50
- sk1.on_data (sink_process);
51
- sk2.on_data (sink_process);
52
- sk3.on_data (sink_process);
53
-
54
- cl1.on_data (function (elem, done) {
55
- // define processing for the ChoiceLink
56
- });
57
-
58
- dl1.on_data (function (elem, done) {
59
- // define processing for the DirectLink
60
- });
61
-
62
- // start the whole lot
63
- sk1.start ();
64
- sk2.start ();
65
- sk3.start ();
66
- cl1.start ();
67
- dl1.start ();
68
-
69
- // pipeline is ready now. Push stuff to queues, see it work
70
- });
71
-
72
- ```
73
-
74
- See [Processors](processors.md) for all the available options and features (such as processing functions and error management)
75
-
76
- ## Creation with a `PipelineBuilder`
77
-
78
- `PipelineBuilder` provides a simpler way to create pipelines using a fluent api. Builders are obtained through `factory.builder()` and offers the following methods:
79
-
80
- * `pipeline(name)`: initializes a pipeline, passing a name to it. Must be called before any other method, and can be called only once
81
- * `queue(name, opts)`: creates a queue and adds it to the pipeline
82
- * `directLink (name_src_q, name_dst_q, process_fn)`: creates a DirectLink linking queues src_q and dst_q (specified by name), using the process function `process_fn`
83
- * `choiceLink(name_src_q, [name_dst_q1, name_dst_q2, ...name_dst_qn], process_fn)`: creates a ChoiceLink linking src_q and the array of dst_q (specified by name), using the process function `process_fn`
84
- * `sink(name_src_q, process_fn)`: creates a Sink on queue src_q (specified by name), using the process function `process_fn`
85
- * `onError(fn)`: sets the `error` event handler for all processirs created in the pipeline. As with the error handler for Processors, `fn` will receive a single param with the error; in this case the error will be augmented by adding an extra field `processor`, which will be areference to the `Processor` object originating the error
86
- * `done(err, pipeline)`: finished the pipeline creation. No other calls can be done to the builder afterwards. In case of error, the error will be passed in `err`; if all went well `err` will be `null` and the newly created pipeline, an object of type `Pipeline`, will be passed in the `pipeline`; all further interactions with the pipeline will happen through this object
87
-
88
- ### Pipepine object
89
-
90
- The new Pipeline object exports the following methods:
91
-
92
- * `start()`: starts the pipeline (simply calls `start()` on all processors)
93
- * `stop()`: stops the pipeline (simply calls `stop()` on all processors)
94
-
95
- Here's a simplified example (for a complete, working example see [here](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/builder)):
96
-
97
- ```javascript
98
- MQ (factory_opts, (err, factory) => {
99
- if (err) return console.error (err);
100
- const q_opts = {};
101
-
102
- factory
103
- .builder ()
104
- .pipeline ('the-pipeline')
105
- .queue ('test_pl_1', q_opts)
106
- .queue ('test_pl_2', q_opts)
107
- .queue ('test_pl_3', q_opts)
108
- .queue ('test_pl_4', q_opts)
109
- .queue ('test_pl_5', q_opts)
110
- .directLink ('test_pl_1', 'test_pl_2', dl_process)
111
- .choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
112
- .sink ('test_pl_3', sink_process)
113
- .sink ('test_pl_4', sink_process)
114
- .sink ('test_pl_5', sink_process)
115
- .onError (console.log)
116
- .done ((err, pl) => {
117
- if (err) return console.error (err);
118
- // pipeline pl is ready
119
- pl.start ();
120
- // pipeline pl is running
121
- });
122
- });
123
- ```
124
-
125
- ## Creation with `Factory.pipelineFromRecipe`
126
-
127
- `Factory.pipelineFromRecipe` provides a way to define pipelines entirely from strings, including queue, processors, the functions
128
- to be used as process functions and all the code used on those functions. In this way a full, self-contained pipeline can be specified
129
- in a file or set of files
130
-
131
- Under the hood it uses [node.js VM module](https://nodejs.org/dist/latest-v12.x/docs/api/vm.html) to create the `Pipeline` object: once created it can be used normally outside of the creation VM
132
-
133
- `Factory.pipelineFromRecipe` is provided only on factories created from backends with pipelining support. This single method take the following parameters:
134
-
135
- ```javascript
136
- Factory.pipelineFromRecipe (
137
- pipeline_name,
138
- array_of_bootstrap_code,
139
- array_of_setup_code,
140
- extra_context,
141
- done
142
- )
143
- ```
144
-
145
- 1. A new VM is created using the merge of the default context and the parameter `extra_context`. The default context contains the following:
146
- * `Buffer`
147
- * `require`
148
- * `clearImmediate`, `clearInterval`, `clearTimeout`, `setImmediate`, `setTimeout`, `setInterval`
149
- * `TextEncoder`, `TextDecoder`
150
- * `URL`, `URLSearchParams`
151
- * `builder`: an already initialized builder object, as in `factory.builder ().pipeline (name)`
152
- * `done`: a function to call when the pipeline is ready, or an error arises. Expects to be `fn (err, pipeline)`
153
- 2. Each of the strings in the `array_of_bootstrap_code` is executed in the VM
154
- 3. Each of the strings in the `array_of_setup_code` is executed in the VM. It is expected to eventually call `done` with the error or the finished pipeline (`done`is accesible in the context)
155
-
156
- The whole idea is to prepare all the needed code for processors' functions in the `array_of_bootstrap_code`, then create the pipeline in the `array_of_setup_code`, calling the `done` function whith the created pipeline
157
-
158
- You can find a full example [here](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/fromRecipe)
@@ -1,10 +0,0 @@
1
- ---
2
- id: examples
3
- title: Examples
4
- sidebar_label: Examples
5
- ---
6
-
7
- * [simplest](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/simplest): a very simple pipeline with just 2 queues connected with a DirectLink
8
- * [simulation-fork](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/simulation-fork): a somewhat complete example with `DirectLink`, `ChoiceLink` and `Sink` instances connecting 5 queues in a fork-like flow. Each process function adds some basic simulated processing with payload updates, random failures and random delays. It also uses deadletter
9
- * [builder](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/builder): variant of `simulation-fork` done with a pipeline builder
10
- * [fromRecipe](https://github.com/pepmartinez/keuss/tree/master/examples/pipelines/fromRecipe): variant of `simulation-fork`, almost indentical to `builder` but using a `factory.pipelineFromRecipe`
@@ -1,187 +0,0 @@
1
- ---
2
- id: processors
3
- title: Processors
4
- sidebar_label: Processors
5
- ---
6
-
7
- A small hierarchy of processors is provided with Pipelines:
8
-
9
- ## BaseLink
10
-
11
- Common base for all Processors, provides all the functionality common to all. It can not be used directly
12
-
13
- ### Creation
14
-
15
- ```javascript
16
- const bl = new BaseLink (src_q, opts)
17
- ```
18
-
19
- Although not intended to be instantiated, this serves as common initialization to all Processors
20
-
21
- * `src_q` must be a pipelined queue
22
- * `opts` can contain:
23
- * `retry_factor_t, retry_base_t`: they control the delay imposed to an element when it is rolled back. The formula is
24
-
25
- `delay-in-seconds = item.tries * retry_factor_t + retry_base_t`
26
-
27
- They default to `2` and `1` respectively
28
- * `mature`: Date instance or unix timestamp (in milliseconds, as integer) expressing the not-before timestamp for the item, to be used when calling `pl_step()` in the src queue
29
- * `delay`: delay in seconds to calculate `mature`, if `mature` is not specified
30
-
31
- ### Methods
32
-
33
- * `src()`: returns src queue
34
- * `name()`: returns Processor name
35
- * `on_data(fn)`: specifies the process function to be applied to each element
36
- * `start(fn)`: starts the processor. Optionally, a process function can be passed; if not passed the process function must have been previously specified using `on_data()`
37
- * `stop()`: stops the Processor
38
-
39
- ### Process Function
40
-
41
- The function passed into `on_data()` or `start()` provides the processor logic; this function is referred to as *processor function*. This function is called on each step of the Processor loop with the reserved item, and it is expected to calls its callback once done with the item. The way the function calls the callback determines what happens with the item afterwards
42
-
43
- The function looks like this:
44
-
45
- ```javascript
46
- function (item, cb) {
47
- ...
48
- })
49
- ```
50
-
51
- The `item` is received exactly as it comes as result of a (successful) `reserve()` call on the source queue; after processing the item `cb` should be called once to finish the processing of `item` and proceed with the next loop cycle. The callback has the following signature:
52
-
53
- ```javascript
54
- cb (err, res);
55
- ```
56
-
57
- where:
58
-
59
- * if `err` is not nil
60
- * if `err.drop` is exactly `true` the item is committed in the src queue and therefore dropped from the pipeline
61
- * else the item is rolled back in the src queue, using the processor's `retry_factor_t` and `retry_base_t` to calculate the retry delay. If the queue was created with deadletter support, the item would be moved to the deadletter queue if it has reached its maximum number of rollbacks; in such case, the movement into deadletter is also atomic
62
- * else (if `err` is nill)
63
- * if (`res.drop` is exactly `true` the item is committed in the src queue and therefore dropped from the pipeline
64
- * else the item is passed to the text queue in the pipeline (by means of `pl_next()`)
65
- * if `res.mature` or `res.delay` exist (or they were specified at the processor's creation) they are used to calculate the delay/mature of the element in the destination queue
66
- * if `res.payload` exists it is used to replace the item's payload entirely
67
- * else if `res.update` exists it is used as mongodb-update operations on the item's payload
68
-
69
- All those operations happen in an atomic way
70
-
71
- #### Semantic `this` in process function
72
-
73
- The function is bound to the processor, so the function can access and use processor's primitives. For example, it can insert copies of the item, or new items, in any of the source or destination queues
74
-
75
- In order to use this functionality the process function can not be declared as an 'arrow' function, since those can not be bound. Use the classic `function xxxx (item, cb) {...}` if you intend to access the underlying Processor
76
-
77
- ### Events
78
-
79
- BaseLink inherits from `EventEmitter` and publishes the following events:
80
-
81
- * `error`: an error happened in the internal loop. It comes with one parameter, an object with the following fields:
82
- * `on`: exact type of error:
83
- * `src-queue-pop`: error while reserving an element on the src queue
84
- * `src-queue-commit-on-error`: error while committing an element on the src queue when an error was passed and `err.drop==true`
85
- * `src-queue-rollback-on-error`: error while rolling back an element on the src queue when an error was passed
86
- * `src-queue-commit-on-drop`: error while committing an element an element on the src queue when processed ok and `res.drop==true`
87
- * `next-queue`: error while atomically moving the element to the next queue
88
- * `elem`: element that caused the error. Not present in `src-queue-pop`
89
- * `error`: original error object
90
- * `opts`: (only present in `next-queue`) options passed internally to `pl_step()`
91
-
92
- ## DirectLink
93
-
94
- Processor that connects the source queue to exactly one destination queue:
95
-
96
- ```
97
- src-queue --> DirectLink --> dst-queue
98
- ```
99
-
100
- ### Creation
101
-
102
- ```javascript
103
- const PDL = require ('keuss/Pipeline/DirectLink');
104
- const bl = new PDL (src_q, dst_q, opts);
105
- ```
106
-
107
- In addition to `BaseLink`:
108
-
109
- * `dst_q` must be a pipelined queue; also, both `src_q` and `dst_q` must be of the same type and must belong to the same pipeline
110
-
111
- ### Methods
112
-
113
- In addition to those of `BaseLink`
114
-
115
- * `dst()`: returns destination queue
116
-
117
- ### Process Function
118
-
119
- In the case of successful processing (i.e.: no `err` in the callback invocation) the item is atomically moved to the `dst` queue.
120
-
121
- No other semantics are added to the process function.
122
-
123
- ## ChoiceLink
124
-
125
- Processor that connects the source queue to an array of queues; after processing, each item would be moved to exactly one of those queues:
126
-
127
- ```
128
- |--> dst-queue-0
129
- |--> dst-queue-1
130
- src-queue --> ChoiceLink --|--> dst-queue-2
131
- | ...
132
- |--> dst-queue-n
133
- ```
134
-
135
- ### Creation
136
-
137
- ```javascript
138
- const PCL = require ('keuss/Pipeline/ChoiceLink');
139
- const cl = new PCL (src_q, array_of_dst_queues, opts);
140
- ```
141
-
142
- In addition to `BaseLink`:
143
-
144
- * `array_of_dst_queues` must be an array of pipelined queues; each one must be of the same type and must belong to the same pipeline than `src_q`
145
-
146
- ### Methods
147
-
148
- In addition to those of `BaseLink`:
149
-
150
- * `dst_by_idx(idx)`: returns destination queue from the array, selected by array index (integer)
151
- * `dst_by_name(name)`: returns destination queue from the array, selected by queue name (string)
152
- * `dst_dimension ()`: returns number of possible destination queues
153
- * `dst_names ()`: returns an array with the names of all dst queues
154
-
155
- ### Process Function
156
-
157
- ChoiceLink expects an `res.dst` in the callback invocation, which must fullfill one of those conditions:
158
-
159
- * be an integer and resolve to a valid element when applied as index to the array of destination queues
160
- * be a string and correspond to the name of one of the destination queues
161
-
162
- The element will be moved atomically to the specified destination queue upon successful processing (i.e.: no `err`in the callback invocation)
163
-
164
- ## Sink
165
-
166
- Processor that connects the source queue to exactly zero destination queue. That is, a termination point: successfully processed elements are always removed from the pipeline
167
-
168
- ```
169
- src-queue --> Sink
170
- ```
171
-
172
- ### Creation
173
-
174
- ```javascript
175
- const PS = require ('keuss/Pipeline/Sink');
176
- const bl = new PS (src_q, opts);
177
- ```
178
-
179
- No extra parameters are expected in addition to those of `BaseLink`
180
-
181
- ### Methods
182
-
183
- No extra methods are provided in addition to those of `BaseLink`
184
-
185
- ### Process Function
186
-
187
- In the case of successful processing (i.e.: no `err` in the callback invocation) the item is removed from the pipeline, exactly as if `res.drop` were specified. Actually, `res` is totally ignored
@@ -1,153 +0,0 @@
1
- ---
2
- id: putting-all-together
3
- title: Putting all together
4
- sidebar_label: Putting all together
5
- ---
6
-
7
- ## Factory initialization
8
-
9
- First, choose a factory, also known as backend:
10
-
11
- ```javascript
12
- const MQ = require ('../../backends/mongo');
13
- ```
14
-
15
- Then, simply execute the backend, passing the config, to obtain a working factory:
16
-
17
- ```javascript
18
- MQ ({
19
- url: 'mongodb://localhost/keuss_test'
20
- }, (err, factory) => {
21
- if (err) return console.error(err);
22
-
23
- // factory is ready to be used
24
-
25
- }
26
- ```
27
-
28
- You can create and use as many factories as desided, from the same or many backends
29
-
30
- ## Queue creation
31
-
32
- You use the factory to create queues:
33
-
34
- ```javascript
35
- const q1 = factory.queue ('test_queue_1', {});
36
- const q2 = factory.queue ('test_queue_2', {});
37
- ```
38
-
39
- A queue can be created more than once with the same name, inside the same factory (this is a common procedure when consumer and producer are separated). The effect would be virtually the same as sharing the queue:
40
-
41
- ```javascript
42
- const q_consumer = factory.queue ('test_queue', {});
43
- const q_producer = factory.queue ('test_queue', {});
44
- ```
45
-
46
- ## Put elements in queue (push)
47
-
48
- putting elements in a queue is simple enough:
49
-
50
- ```javascript
51
- const elem = {
52
- elem: 1,
53
- headline: 'something something',
54
- tags: {
55
- a: 1,
56
- b: 2
57
- }
58
- };
59
-
60
- q1.push (elem, (err, res) => {
61
- // push finished, either with error or success...
62
- }),
63
- ```
64
-
65
- Or, push also some headers:
66
-
67
- ```javascript
68
- const elem = {
69
- elem: 1,
70
- headline: 'something something',
71
- tags: {
72
- a: 1,
73
- b: 2
74
- }
75
- };
76
-
77
- const headers = {
78
- h1: 'a string',
79
- h2: false,
80
- h3: 666
81
- };
82
-
83
- q1.push (elem, {hdrs: headers}, (err, res) => {
84
- // push finished, either with error or success...
85
- }),
86
- ```
87
-
88
- ## Get elements from queue (pop)
89
-
90
- The easiest way to get elements from a queue is with a simple pop(). This would block until an element is ready, it would remove it from the queue and return it.
91
-
92
- This way of working is often referred to as *at-most-once* since it guarantees that each element in the queue will be processed no more than one time (it would be zero times, if something happens after `pop()` ands but before the element is actually managed)
93
-
94
- ```javascript
95
- const consumer_label = 'consumer-1';
96
- q1.pop (consumer_label, (err, res) => {
97
- if (err) return console.error (err);
98
-
99
- console.log (res);
100
- // this should print something like:
101
- // {
102
- // _id: <some id>,
103
- // mature: <some date>,
104
- // payload: { elem: 1, headline: 'something something', tags: { a: 1, b: 2 } },
105
- // tries: 0
106
- // hdrs: {}
107
- // }
108
- // that is, the actual element is at res.payload
109
- //
110
- // if the element was pushed with headers, they will be placed inside hdrs:
111
- // hdrs: {h1: 'a string', h2: false, h3: 666}
112
- }
113
- ```
114
-
115
- ## Reserve-commit-rollback
116
-
117
- A safer way to consume from a queue is using reserve: elements are reserved, processed and only then committed (and removed from the queue). A reserved element can also be rolled back (returned to queue) if the processing failed and the element needs to be reprocessed in the future; also, any reserved element will auto-rollback after some tiem elapsed, if neither commit nor rollback is done. This is known as *at-least-once* cause it guarantees all elements wold be processed at least once
118
-
119
- ```javascript
120
- const consumer_label = 'consumer-1';
121
- q1.pop (consumer_label, {reserve: true}, (err, elem) => {
122
- if (err) return console.error (err);
123
-
124
- // res is ready to be processed
125
- do_some_processing (elem.payload, err => {
126
- if (err) {
127
- // error, rollback so it gets retried, adding a delay
128
- const next_t = new Date().getTime () + 15000;
129
- q1.ko (item, next_t, () = >{
130
- // the element is returned to queue, but it won't be available until 15 secs have passed
131
- });
132
- }
133
- else {
134
- // processing went fine, commit element
135
- q1.ok (item, () => {
136
- // the element is removed from the queue
137
- });
138
- }
139
- });
140
- }
141
- ```
142
-
143
- ## Termination
144
-
145
- Once all is done, you can free all the resources associated to the factory by closing it:
146
-
147
- ```javascript
148
- factory.close (err => {
149
- // factory is now closed and cannot be used anymore
150
- });
151
- ```
152
-
153
- Once a factory is closed it cannot be used, *and all the queues created through it will becomes unusable too*
@@ -1,70 +0,0 @@
1
- ---
2
- id: redis-conns
3
- title: Redis Connections
4
- sidebar_label: Redis Connections
5
- ---
6
-
7
- Keuss relies on [ioredis](https://www.npmjs.com/package/ioredis) for connecting to redis. Anytime a redis connection is needed, keuss will create it from the opts object passed:
8
-
9
- * If `opts` is a function, it is executed. It is expected to return a redis connection
10
- * If it's an object and contains a `Redis` field, this field is used to create a new [ioredis Redis object](https://github.com/luin/ioredis), as in *return new Redis (opts.Redis)*
11
- * if it's an object and contains a `Cluster` field, this field is used to create a new [ioredis Redis.Cluster](https://redis.io/topics/cluster-spec) object, as in *return new Redis.Cluster (opts.Cluster)*
12
- * Else, a ioredis Redis object is created with `opts` as param, as in *return new Redis (opts)*
13
-
14
- This apparent complexity is required since redis connections are inherently created in a different way for standalone, sentinel and cluster servers
15
-
16
- ## Examples:
17
-
18
- * Default options:
19
-
20
- ```javascript
21
- var MQ = require ('keuss/backends/redis-list');
22
- var factory_opts = {};
23
-
24
- MQ (factory_opts, (err, factory) => {
25
- ...
26
- });
27
- ```
28
-
29
- * Specific redis params for ioredis Redis client:
30
-
31
- ```javascript
32
- var MQ = require ('keuss/backends/redis-list');
33
- var factory_opts = {
34
- redis: {
35
- Redis: {
36
- port: 12293,
37
- host: 'some-redis-instance.somewhere.com',
38
- family: 4,
39
- password: 'xxxx',
40
- db: 0
41
- }
42
- }
43
- };
44
-
45
- MQ (factory_opts, (err, factory) => {
46
- ...
47
- });
48
- ```
49
-
50
- * Using a factory function:
51
-
52
- ```javascript
53
- var MQ = require ('keuss/backends/redis-list');
54
- var Redis = require ('ioredis');
55
- var factory_opts = {
56
- redis: function () {
57
- return new Redis ({
58
- port: 12293,
59
- host: 'some-redis-instance.somewhere.com',
60
- family: 4,
61
- password: 'xxxx',
62
- db: 0
63
- })
64
- }
65
- };
66
-
67
- MQ (factory_opts, (err, factory) => {
68
- ...
69
- });
70
- ```
@@ -1,11 +0,0 @@
1
- ---
2
- id: shutdown
3
- title: Shutdown
4
- sidebar_label: Shutdown
5
- ---
6
-
7
- It is a good practice to call [`close(cb)`](../api/factory#close-factory-close) on the factories to release all resources once you're done, or at shutdown if you want your shutdowns clean and graceful (the log-lived redis or mongodb connections are terminated here, for example); also, you should loop over your queues and perform a [`drain()`](../api/queue#drain-drain-queue) on them before calling `close()` on their factories: this will ensure any un-consumed data is popped, and any unwritten data is written. Also, it'll ensure all your (local) waiting consumers will end (on 'cancel' error).
8
-
9
- :::note
10
- Factories do not keep track of the created Queues, so this can't be done internally as part of the `close()`; this may change in the future.
11
- :::
@@ -1,96 +0,0 @@
1
- module.exports = {
2
- title: 'Keuss Job Queues',
3
- tagline: 'Job Queues for node.js, backed by redis and/or MongoDB',
4
- url: 'https://pepmartinez.github.io',
5
- baseUrl: '/keuss/',
6
- onBrokenLinks: 'throw',
7
- favicon: 'img/favicon.ico',
8
- organizationName: 'pepmartinez',
9
- projectName: 'keuss',
10
- themeConfig: {
11
- navbar: {
12
- title: 'Keuss job queues',
13
- logo: {
14
- alt: 'Site Logo',
15
- src: 'img/logo.svg',
16
- },
17
- items: [
18
- {
19
- to: 'docs/',
20
- activeBasePath: 'docs',
21
- label: 'Docs',
22
- position: 'left',
23
- },
24
- {to: 'blog', label: 'Blog', position: 'left'},
25
- {
26
- href: 'https://github.com/pepmartinez/keuss',
27
- label: 'GitHub',
28
- position: 'right',
29
- },
30
- ],
31
- },
32
- footer: {
33
- style: 'dark',
34
- links: [
35
- {
36
- title: 'Start Here',
37
- items: [
38
- {
39
- label: 'Quickstart',
40
- to: 'docs/quickstart',
41
- },
42
- {
43
- label: 'Documentation',
44
- to: 'docs/',
45
- },
46
- {
47
- label: 'Examples',
48
- to: 'docs/examples',
49
- },
50
- {
51
- label: 'ChangeLog',
52
- to: 'docs/changelog',
53
- },
54
- ],
55
- },
56
- {
57
- title: 'More',
58
- items: [
59
- {
60
- label: 'Blog',
61
- to: 'blog',
62
- },
63
- {
64
- label: 'GitHub',
65
- href: 'https://github.com/pepmartinez/keuss',
66
- },
67
- ],
68
- },
69
- ],
70
- copyright: `Copyright © ${new Date().getFullYear()}. Built with Docusaurus.`,
71
- },
72
- },
73
- presets: [
74
- [
75
- '@docusaurus/preset-classic',
76
- {
77
- docs: {
78
- // It is recommended to set document id as docs home page (`docs/` path).
79
- sidebarPath: require.resolve('./sidebars.js'),
80
- // Please change this to your repo.
81
- editUrl:
82
- 'https://github.com/pepmartinez/keuss/edit/master/website/',
83
- },
84
- blog: {
85
- showReadingTime: true,
86
- // Please change this to your repo.
87
- editUrl:
88
- 'https://github.com/pepmartinez/keuss/edit/master/website/blog/',
89
- },
90
- theme: {
91
- customCss: require.resolve('./src/css/custom.css'),
92
- },
93
- },
94
- ],
95
- ],
96
- };