jiggler 0.1.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +6 -0
  3. data/LICENSE +4 -0
  4. data/README.md +423 -0
  5. data/bin/jiggler +31 -0
  6. data/lib/jiggler/cleaner.rb +130 -0
  7. data/lib/jiggler/cli.rb +263 -0
  8. data/lib/jiggler/config.rb +165 -0
  9. data/lib/jiggler/core.rb +22 -0
  10. data/lib/jiggler/errors.rb +5 -0
  11. data/lib/jiggler/job.rb +116 -0
  12. data/lib/jiggler/launcher.rb +69 -0
  13. data/lib/jiggler/manager.rb +73 -0
  14. data/lib/jiggler/redis_store.rb +55 -0
  15. data/lib/jiggler/retrier.rb +122 -0
  16. data/lib/jiggler/scheduled/enqueuer.rb +78 -0
  17. data/lib/jiggler/scheduled/poller.rb +97 -0
  18. data/lib/jiggler/stats/collection.rb +26 -0
  19. data/lib/jiggler/stats/monitor.rb +103 -0
  20. data/lib/jiggler/summary.rb +101 -0
  21. data/lib/jiggler/support/helper.rb +35 -0
  22. data/lib/jiggler/version.rb +5 -0
  23. data/lib/jiggler/web/assets/stylesheets/application.css +64 -0
  24. data/lib/jiggler/web/views/application.erb +329 -0
  25. data/lib/jiggler/web.rb +80 -0
  26. data/lib/jiggler/worker.rb +179 -0
  27. data/lib/jiggler.rb +10 -0
  28. data/spec/examples.txt +79 -0
  29. data/spec/fixtures/config/jiggler.yml +4 -0
  30. data/spec/fixtures/jobs.rb +5 -0
  31. data/spec/fixtures/my_failed_job.rb +10 -0
  32. data/spec/fixtures/my_job.rb +9 -0
  33. data/spec/fixtures/my_job_with_args.rb +18 -0
  34. data/spec/jiggler/cleaner_spec.rb +171 -0
  35. data/spec/jiggler/cli_spec.rb +87 -0
  36. data/spec/jiggler/config_spec.rb +56 -0
  37. data/spec/jiggler/core_spec.rb +34 -0
  38. data/spec/jiggler/job_spec.rb +99 -0
  39. data/spec/jiggler/launcher_spec.rb +66 -0
  40. data/spec/jiggler/manager_spec.rb +52 -0
  41. data/spec/jiggler/redis_store_spec.rb +20 -0
  42. data/spec/jiggler/retrier_spec.rb +55 -0
  43. data/spec/jiggler/scheduled/enqueuer_spec.rb +81 -0
  44. data/spec/jiggler/scheduled/poller_spec.rb +40 -0
  45. data/spec/jiggler/stats/monitor_spec.rb +40 -0
  46. data/spec/jiggler/summary_spec.rb +168 -0
  47. data/spec/jiggler/web_spec.rb +37 -0
  48. data/spec/jiggler/worker_spec.rb +110 -0
  49. data/spec/spec_helper.rb +54 -0
  50. metadata +230 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: dfa0041250ddfa5ea28b9b11ffca6133be350722b0eda97b1df7ee35cc58ff39
4
+ data.tar.gz: 4b0bdd71886dbcc4b7b0733456f6481275e3401676d5f938758b87b93a2ca309
5
+ SHA512:
6
+ metadata.gz: 2f3192170564b7351fef54f8a3c897ae5c0757c5d8715732320145e2a95b8f24e23f8f60ff65a81acde9b0c2d1c54c3485fb29b45ed0e95c55bb3184bc68a937
7
+ data.tar.gz: 9a5f31bf304232fd4282ba37e6bd7b5afec8d7d24c20ff3c12f894a951810ae62e37efabdd735adb1ef7f406199d5ed7a07324130aa66250fc5b2a30cd916a88
data/CHANGELOG.md ADDED
@@ -0,0 +1,6 @@
1
+ # Jiggler Changelog
2
+
3
+ 0.1.0.rc.2
4
+ ----------
5
+
6
+ - Initial Release
data/LICENSE ADDED
@@ -0,0 +1,4 @@
1
+ Copyright (c) 2023 Julija Alieckaja
2
+
3
+ Jiggler is an Open Source project licensed under the terms of the LGPLv3 license.
4
+ Please see <http://www.gnu.org/licenses/lgpl-3.0.html> for license text.
data/README.md ADDED
@@ -0,0 +1,423 @@
1
+ # jiggler
2
+ [![Gem Version](https://badge.fury.io/rb/jiggler.svg)](https://badge.fury.io/rb/jiggler)
3
+
4
+ Background job processor based on Socketry Async
5
+
6
+ Jiggler is a [Sidekiq](https://github.com/mperham/sidekiq)-inspired background job processor using [Socketry Async](https://github.com/socketry/async) and [Optimized JSON](https://github.com/ohler55/oj). It uses fibers to processes jobs, making context switching lightweight. Requires Ruby 3+, Redis 6+.
7
+
8
+ *Jiggler is based on Sidekiq implementation, and re-uses most of its concepts and ideas.*
9
+
10
+ **NOTE**: Altrough some performance results may look interesting, it's absolutly not recommended to switch to it from well-tested stable solutions. Jiggler has a meager set of features and a very basic monitoring. It's a small indie gem made purely for fun and to gain some hand-on experience with async and fibers. It isn't tested with production projects and might have not-yet-discovered issues. \
11
+ However, it's good to play around and/or to try it in the name of science.
12
+
13
+ ### Installation
14
+
15
+ Install the gem:
16
+ ```
17
+ gem install jiggler
18
+ ```
19
+
20
+ For release candidate versions (currently only 0.1.0.rc1 is available):
21
+ ```
22
+ gem install jiggler --pre
23
+ ```
24
+
25
+ Start Jiggler server as a separate process with bin command:
26
+ ```
27
+ jiggler -r <FILE_PATH>
28
+ ```
29
+ `-r` specifies a file with loading instructions. \
30
+ For Rails apps the command'll be `jiggler -r ./config/environment.rb`
31
+
32
+ Run `jiggler --help` to see the list of command line arguments.
33
+
34
+ ### Performance
35
+
36
+ The tests were run on local (Ubuntu 22.04, Intel(R) Core(TM) i7 6700HQ 2.60GHz). \
37
+ On the other configurations the results may differ significantly, f.e. with Apple M1 Max chip it treats some IO operations as blocking and shows a poor performance ಠ_ಥ.
38
+
39
+ Ruby `3.2.0` \
40
+ Redis `7.0.7` \
41
+ Poller interval `5s` \
42
+ Monitoring interval `10s` \
43
+ Logging level `WARN`
44
+
45
+ #### Noop task measures
46
+
47
+ ```ruby
48
+ def perform
49
+ # just an empty job doing nothing
50
+ end
51
+ ```
52
+
53
+ The parent process enqueues the jobs, starts the monitoring, and then forks the child job-processor-process. Thus, `RSS` value is affected by the number of jobs uploaded in the parent process. See `bin/jigglerload` to see the load test structure and measuring. \
54
+ 1_000_000 jobs were enqueued in 100k batches x10.
55
+
56
+ | Job Processor | Concurrency | Jobs | Time | Start RSS | Finish RSS |
57
+ |------------------|-------------|-----------|-----------|------------|---------------|
58
+ | Sidekiq 7.0.2 | 5 | 100_000 | 20.01 sec | 132_080 kb | 103_168 kb (GC) |
59
+ | Jiggler 0.1.0rc1 | 5 | 100_000 | 14.25 sec | 87_532 kb | 91_464 kb |
60
+ | - | | | | | |
61
+ | Sidekiq 7.0.2 | 10 | 100_000 | 20.49 sec | 132_164 kb | 125_768 kb (GC) |
62
+ | Jiggler 0.1.0rc1 | 10 | 100_000 | 13.25 sec | 87_688 kb | 91_625 kb |
63
+ | - | | | | | |
64
+ | Sidekiq 7.0.2 | 5 | 1_000_000 | 186.90 sec | 159_712 kb | 186_224 kb |
65
+ | Jiggler 0.1.0rc1 | 5 | 1_000_000 | 123.13 sec | 113_212 kb | 116_336 kb |
66
+ | - | | | | | |
67
+ | Sidekiq 7.0.2 | 10 | 1_000_000 | 186.94 sec | 159_000 kb | 192_780 kb |
68
+ | Jiggler 0.1.0rc1 | 10 | 1_000_000 | 115.56 sec | 113_656 kb | 116_896 kb |
69
+
70
+
71
+ #### IO tests
72
+
73
+ The idea of the next tests is to simulate jobs with different kinds of IO tasks. \
74
+ Ruby 3 has introduced fiber scheduler interface, which allows to implement hooks related to IO/blocking operations.\
75
+ The context switching won't work well in case IO is performed by C-extentions which are not aware of Ruby scheduler.
76
+
77
+ ##### NET/HTTP requests
78
+
79
+ Spin-up a local sinatra app to exclude network issues while testing HTTP requests (it uses `falcon` web server).
80
+
81
+ ```ruby
82
+ require "sinatra"
83
+
84
+ class MyApp < Sinatra::Base
85
+ get "/hello" do
86
+ sleep(0.2)
87
+ "Hello World!"
88
+ end
89
+ end
90
+ ```
91
+
92
+ Then, the code which is going to be performed within the workers should make a `net/http` request to the local endpoint.
93
+
94
+ ```ruby
95
+ # a single job takes ~0.21s to perform
96
+ def perform
97
+ uri = URI("http://127.0.0.1:9292/hello")
98
+ res = Net::HTTP.get_response(uri)
99
+ puts "Request Error!!!" unless res.is_a?(Net::HTTPSuccess)
100
+ end
101
+ ```
102
+
103
+ It's not recommended to run sidekiq with high concurrency values, setting it for the sake of test. \
104
+ The time difference for these samples is small-ish, however the memory consumption is less with the fibers. \
105
+ Since fibers have relatively small memory foot-print and context switching is also relatively cheap, it's possible to set concurrency to higher values within Jiggler without too much trade-offs.
106
+
107
+ | Job Processor | Concurrency | Jobs | Time to complete | Start RSS | Finish RSS | %CPU |
108
+ |------------------|-------------|-------|-------------------|-----------|------------|------|
109
+ | Sidekiq 7.0.2 | 5 | 1_000 | 43.74 sec | 30_444 kb | 45_124 kb | 5.9 |
110
+ | Jiggler 0.1.0rc1 | 5 | 1_000 | 43.65 sec | 33_476 kb | 34_144 kb | 2.9 |
111
+ | - | | | | | | |
112
+ | Sidekiq 7.0.2 | 10 | 1_000 | 23.05 sec | 30_604 kb | 50_292 kb | 10.93 |
113
+ | Jiggler 0.1.0rc1 | 10 | 1_000 | 22.86 sec | 32_416 kb | 34_128 kb | 5.69 |
114
+ | - | | | | | | |
115
+ | Sidekiq 7.0.2 | 15 | 1_000 | 16.17 sec | 30_636 kb | 55_144 kb | 16.47 |
116
+ | Jiggler 0.1.0rc1 | 15 | 1_000 | 15.87 sec | 33_328 kb | 34_548 kb | 8.25 |
117
+
118
+ **NOTE**: Jiggler has more dependencies, so with small load `start RSS` takes more space.
119
+
120
+ ##### PostgreSQL connection/queries
121
+
122
+ `pg` gem supports Ruby's `Fiber.scheduler` starting from 1.3.0 version. Make sure yours DB-adapter supports it.
123
+
124
+ ```ruby
125
+ ### global namespace
126
+ require "pg"
127
+
128
+ $pg_pool = ConnectionPool.new(size: CONCURRENCY) do
129
+ PG.connect({ dbname: "test", password: "test", user: "test" })
130
+ end
131
+
132
+ ### worker context
133
+ # a single job takes ~0.102s to perform
134
+ def perform
135
+ $pg_pool.with do |conn|
136
+ conn.exec("SELECT pg_sleep(0.1)")
137
+ end
138
+ end
139
+ ```
140
+
141
+ | Job Processor | Concurrency | Jobs | Time | Start RSS | Finish RSS | %CPU |
142
+ |------------------|-------------|-------|-----------|-----------|------------|------|
143
+ | Sidekiq 7.0.2 | 5 | 1_000 | 23.44 sec | 31_436 kb | 48_856 kb | 7.56 |
144
+ | Jiggler 0.1.0rc1 | 5 | 1_000 | 23.20 sec | 35_312 kb | 38_592 kb | 2.91 |
145
+ | - | | | | | | |
146
+ | Sidekiq 7.0.2 | 10 | 1_000 | 13.15 sec | 31_272 kb | 52_808 kb | 13.76 |
147
+ | Jiggler 0.1.0rc1 | 10 | 1_000 | 12.65 sec | 35_296 kb | 38_784 kb | 6.11 |
148
+ | - | | | | | | |
149
+ | Sidekiq 7.0.2 | 15 | 1_000 | 9.63 sec | 31_016 kb | 59_868 kb | 20.32 |
150
+ | Jiggler 0.1.0rc1 | 15 | 1_000 | 9.17 sec | 35_188 kb | 38_948 kb | 9.26 |
151
+
152
+ ##### File IO
153
+
154
+ ```ruby
155
+ def perform(file_name, id)
156
+ File.open(file_name, "a") { |f| f.write("#{id}\n") }
157
+ end
158
+ ```
159
+
160
+ | Job Processor | Concurrency | Jobs | Time | Start RSS | Finish RSS | %CPU |
161
+ |------------------|-------------|--------|-----------|--------------|------------|-------|
162
+ | Sidekiq 7.0.2 | 5 | 30_000 | 11.94 sec | 61_944 kb | 71_948 kb | 94.34 |
163
+ | Jiggler 0.1.0rc1 | 5 | 30_000 | 7.87 sec | 50_140 kb | 51_272 kb | 61.7 |
164
+ | - | | | | | | |
165
+ | Sidekiq 7.0.2 | 10 | 30_000 | 11.6 sec | 62_020 kb | 78_952 kb | 94.44 |
166
+ | Jiggler 0.1.0rc1 | 10 | 30_000 | 7.17 sec | 50_060 kb | 51_464 kb | 69.25 |
167
+ | - | | | | | | |
168
+ | Sidekiq 7.0.2 | 15 | 30_000 | 11.24 sec | 62_016 kb | 83_808 kb | 94.16 |
169
+ | Jiggler 0.1.0rc1 | 15 | 30_000 | 7.02 sec | 49_988 kb | 51_428 kb | 70.3 |
170
+
171
+
172
+ Jiggler is effective only for tasks with a lot of IO. You must test the concurrency setting with your jobs to find out what configuration works best for your payload.
173
+
174
+ #### Simulate CPU-only job
175
+
176
+ With CPU-heavy jobs Jiggler has poor performance. Just to make sure it's generally able to work with CPU-only payloads:
177
+
178
+ ```ruby
179
+ def fib(n)
180
+ if n <= 1
181
+ 1
182
+ else
183
+ (fib(n-1) + fib(n-2))
184
+ end
185
+ end
186
+
187
+ # a single job takes ~0.035s to perform
188
+ def perform(_idx)
189
+ fib(25)
190
+ end
191
+ ```
192
+
193
+ | Job Processor | Concurrency | Jobs | Time | Start RSS | Finish RSS |
194
+ |------------------|-------------|------|----------|------------|------------|
195
+ | Sidekiq 7.0.2 | 5 | 100 | 5.81 sec | 27_792 kb | 42_464 kb |
196
+ | Jiggler 0.1.0rc1 | 5 | 100 | 5.29 sec | 31_304 kb | 32_320 kb |
197
+ | - | | | | | |
198
+ | Sidekiq 7.0.2 | 10 | 100 | 5.63 sec | 28_044 kb | 47_640 kb |
199
+ | Jiggler 0.1.0rc1 | 10 | 100 | 5.43 sec | 32_316 kb | 32_548 kb |
200
+
201
+ #### IO Event selector
202
+
203
+ `IO_EVENT_SELECTOR` is an env variable which allows to specify the event selector used by the Ruby scheduler. \
204
+ On default it uses `Epoll` (`IO_EVENT_SELECTOR=EPoll`). \
205
+ Another available option is `URing` (`IO_EVENT_SELECTOR=URing`). Underneath it uses `io_uring` library. It is a Linux kernel library that provides a high-performance interface for asynchronous I/O operations. It was introduced in Linux kernel version 5.1 and aims to address some of the limitations and scalability issues of the existing AIO (Asynchronous I/O) interface.
206
+ In the future it might bring a lot of performance boost into Ruby fibers world (once `async` project fully adopts it), but at the moment in the most cases its performance is similar to `EPoll`, yet it could give some boost with File IO.
207
+
208
+ #### Socketry stack
209
+
210
+ The gem allows to use libs from `socketry` stack (https://github.com/socketry) within workers. \
211
+ F.e. when making HTTP requests using `async/http/internet` to the Sinatra app described above:
212
+
213
+ ```ruby
214
+ ### global namespace
215
+ require "async/http/internet"
216
+ $internet = Async::HTTP::Internet.new
217
+
218
+ ### worker context
219
+ def perform
220
+ uri = "https://127.0.0.1/hello"
221
+ res = $internet.get(uri)
222
+ res.finish
223
+ puts "Request Error!!!" unless res.status == 200
224
+ end
225
+ ```
226
+
227
+ | Job Processor | Concurrency | Jobs | Time | Start RSS | Finish RSS | %CPU |
228
+ |------------------|-------------|-------|-----------|-----------|------------|------|
229
+ | Jiggler 0.1.0rc1 | 5 | 1_000 | 43.23 sec | 34_340 kb | 38_488 kb | 1.51 |
230
+ | - | | | | | | |
231
+ | Jiggler 0.1.0rc1 | 10 | 1_000 | 22.67 sec | 34_552 kb | 38_600 kb | 2.75 |
232
+ | - | | | | | | |
233
+ | Jiggler 0.1.0rc1 | 15 | 1_000 | 15.88 sec | 34_332 kb | 38_544 kb | 4.06 |
234
+
235
+ ### Getting Started
236
+
237
+ Conceptually Jiggler consists of two parts: the `client` and the `server`. \
238
+ The `client` is responsible for pushing jobs to `Redis` and allows to read stats, while the `server` reads jobs from `Redis`, processes them, and writes stats.
239
+
240
+ The `server` uses async `Redis` connections. \
241
+ The `client` on default is `sync`. It's possible to configure the client to be async as well via setting `client_async` to `true`. More info below. \
242
+ Client settings are:
243
+ - `client_concurrency`
244
+ - `async_client`
245
+ - `redis_url` (this one is shared with the `server`)
246
+
247
+ The rest of the settings are `server` specific.
248
+
249
+ **NOTE**: `require "jiggler"` loads only client classes. It doesn't include `async` lib, this dependency is being required only within the `server` part.
250
+
251
+ ```ruby
252
+ require "jiggler"
253
+
254
+ Jiggler.configure do |config|
255
+ config[:client_concurrency] = 12 # Should equal to the number of threads/fibers in the client app. Defaults to 10
256
+ config[:concurrency] = 12 # The number of running fibers on the server. Defaults to 10
257
+ config[:timeout] = 12 # Seconds Jiggler wait for jobs to finish before shotdown. Defaults to 25
258
+ config[:environment] = "myenv" # On default fetches the value ENV["APP_ENV"] and fallbacks to "development"
259
+ config[:require] = "./jobs.rb" # Path to file with jobs/app initializer
260
+ config[:redis_url] = ENV["REDIS_URL"] # On default fetches the value from ENV["REDIS_URL"]
261
+ config[:queues] = ["shippers"] # An array of queue names the server is going to listen to. On default uses ["default"]
262
+ config[:config_file] = "./jiggler.yml" # .yml file with Jiggler settings
263
+ end
264
+ ```
265
+
266
+ Internally Jiggler server consists of 3 parts: `Manager`, `Poller`, `Monitor`. \
267
+ `Manager` is responsible for workers. \
268
+ `Poller` fetches data for retries and scheduled jobs. \
269
+ `Monitor` periodically loads stats data into redis. \
270
+ `Manager` and `Monitor` are mandatory, while `Poller` can be disabled in case there's no need for retries/scheduled jobs.
271
+
272
+ ```ruby
273
+ Jiggler.configure do |config|
274
+ config[:stats_interval] = 12 # Defaults to 10
275
+ config[:poller_enabled] = true # Defaults to true
276
+ config[:poll_interval] = 12 # Defaults to 5
277
+ end
278
+ ```
279
+
280
+ `Jiggler::Web.new` is a rack application. It can be run on its own or be mounted in app routes, f.e. with Rails:
281
+
282
+ ```ruby
283
+ require "jiggler/web"
284
+
285
+ Rails.application.routes.draw do
286
+ mount Jiggler::Web.new => "/jiggler"
287
+
288
+ # ...
289
+ end
290
+ ```
291
+
292
+ To get the available stats run:
293
+ ```ruby
294
+ irb(main)> Jiggler.summary
295
+ =>
296
+ {"retry_jobs_count"=>0,
297
+ "dead_jobs_count"=>0,
298
+ "scheduled_jobs_count"=>0,
299
+ "failures_count"=>6,
300
+ "processed_count"=>0,
301
+ "processes"=>
302
+ {"jiggler:svr:3513d56f7ed2:10:25:default:1:1673875240:83568:JulijaA-MBP.local"=>
303
+ {"heartbeat"=>1673875270.551845,
304
+ "rss"=>32928,
305
+ "current_jobs"=>{},
306
+ "name"=>"jiggler:svr:3513d56f7ed2",
307
+ "concurrency"=>"10",
308
+ "timeout"=>"25",
309
+ "queues"=>"default",
310
+ "poller_enabled"=>true,
311
+ "started_at"=>"1673875240",
312
+ "pid"=>"83568",
313
+ "hostname"=>"JulijaA-MBP.local"}},
314
+ "queues"=>{"mine"=>1, "unknown"=>1, "test"=>1}}
315
+ ```
316
+ Note: Jiggler summary shows only queues which have enqueued jobs.
317
+
318
+ Job classes should include `Jiggler::Job` and implement `perform` method.
319
+
320
+ ```ruby
321
+ class MyJob
322
+ include Jiggler::Job
323
+
324
+ def perform
325
+ puts "Performing..."
326
+ end
327
+ end
328
+ ```
329
+
330
+ The job can be enqued with:
331
+ ```ruby
332
+ MyJob.enqueue
333
+ ```
334
+
335
+ Specify custom job options:
336
+ ```ruby
337
+ class AnotherJob
338
+ include Jiggler::Job
339
+ job_options queue: "custom", retries: 10, retry_queue: "custom_retries"
340
+
341
+ def perform(num1, num2)
342
+ puts num1 + num2
343
+ end
344
+ end
345
+ ```
346
+
347
+ To override the options for a specific job:
348
+ ```ruby
349
+ AnotherJob.with_options(queue: "default").enqueue(num1, num2)
350
+ ```
351
+
352
+ It's possible to enqueue multiple jobs at once with:
353
+ ```ruby
354
+ arr = [[num1, num2], [num3, num4], [num5, num6]]
355
+ AnotherJob.enqueue_bulk(arr)
356
+ ```
357
+
358
+ For the cases when you want to enqueue jobs with a delay or at a specific time run:
359
+ ```ruby
360
+ seconds = 100
361
+ AnotherJob.enqueue_in(seconds, num1, num2)
362
+ ```
363
+
364
+ To cleanup the data from Redis you can run one of these:
365
+ ```ruby
366
+ # prune data for a specific queue
367
+ Jiggler.config.cleaner.prune_queue(queue_name)
368
+
369
+ # prune all queues data
370
+ Jiggler.config.cleaner.prune_all_queues
371
+
372
+ # prune all Jiggler data from Redis including all enqued jobs, stats, etc.
373
+ Jiggler.config.cleaner.prune_all
374
+ ```
375
+
376
+ On default `client` uses synchronous `Redis` connections. \
377
+ In case the client is being used in async app (f.e. with [Falcon](https://github.com/socketry/falcon) web server, etc.), then it's possible to set a custom redis pool capable of sending async requests into redis. \
378
+ The pool should be compatible with `Async::Pool` - support `acquire` method.
379
+
380
+ ```ruby
381
+ Jiggler.configure_client do |config|
382
+ config[:client_redis_pool] = my_async_redis_pool
383
+ end
384
+
385
+ # or use build-in async pool with
386
+ require "async/pool"
387
+
388
+ Jiggler.configure_client do |config|
389
+ config[:client_async] = true
390
+ end
391
+ ```
392
+
393
+ Then, the client methods could be called with something like:
394
+ ```ruby
395
+ Sync { Jiggler.config.cleaner.prune_all }
396
+ Async { MyJob.enqueue }
397
+ ```
398
+
399
+ ### Local development
400
+
401
+ Docker! You can spin up a local development environment without the need to install dependencies directly on your local machine.
402
+
403
+ To get started, make sure you have Docker installed on your system. Then, simply run the following command to build the Docker image and start a development server:
404
+ ```
405
+ docker-compose up --build
406
+ ```
407
+
408
+ Debug:
409
+ ```
410
+ docker-compose up -d && docker attach jiggler_app
411
+ ```
412
+
413
+ Start irb:
414
+ ```
415
+ docker-compose exec app bundle exec irb
416
+ ```
417
+
418
+ Run tests:
419
+ ```
420
+ docker-compose run --rm web -- bundle exec rspec
421
+ ```
422
+
423
+ To run the load tests modify the `docker-compose.yml` to point to `bin/jigglerload`
data/bin/jiggler ADDED
@@ -0,0 +1,31 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'jiggler'
4
+
5
+ # require jiggler server classes
6
+ require 'jiggler/support/helper'
7
+ require 'jiggler/scheduled/enqueuer'
8
+ require 'jiggler/scheduled/poller'
9
+ require 'jiggler/stats/collection'
10
+ require 'jiggler/stats/monitor'
11
+ require 'jiggler/errors'
12
+ require 'jiggler/retrier'
13
+ require 'jiggler/launcher'
14
+ require 'jiggler/manager'
15
+ require 'jiggler/worker'
16
+
17
+ require 'jiggler/cli'
18
+
19
+ begin
20
+ cli = Jiggler::CLI.instance
21
+ cli.parse_and_init
22
+
23
+ cli.config.logger.info("Jiggler is starting in #{Jiggler.config[:environment].upcase} ✯⸜(*❛‿❛)⸝✯")
24
+ cli.config.logger.info("Jiggler version=#{Jiggler::VERSION} pid=#{Process.pid} concurrency=#{cli.config[:concurrency]} queues=#{cli.config[:queues].join(',')}")
25
+
26
+ cli.start
27
+ rescue => e
28
+ warn e.message
29
+ warn e.backtrace.join("\n")
30
+ exit 1
31
+ end
@@ -0,0 +1,130 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Jiggler
4
+ class Cleaner
5
+ attr_reader :config
6
+
7
+ def initialize(config)
8
+ @config = config
9
+ end
10
+
11
+ def prune_all(pool: config.client_redis_pool)
12
+ pool.acquire do |conn|
13
+ conn.pipelined do |pipeline|
14
+ prn_retries_set(pipeline)
15
+ prn_scheduled_set(pipeline)
16
+ prn_dead_set(pipeline)
17
+ prn_failures_counter(pipeline)
18
+ prn_processed_counter(pipeline)
19
+ end
20
+ prn_all_queues(conn)
21
+ prn_all_processes(conn)
22
+ end
23
+ end
24
+
25
+ def prune_failures_counter(pool: config.client_redis_pool)
26
+ pool.acquire do |conn|
27
+ prn_failures_counter(conn)
28
+ end
29
+ end
30
+
31
+ def prune_processed_counter(pool: config.client_redis_pool)
32
+ pool.acquire do |conn|
33
+ prn_processed_counter(conn)
34
+ end
35
+ end
36
+
37
+ def prune_all_processes(pool: config.client_redis_pool)
38
+ pool.acquire do |conn|
39
+ prn_all_processes(conn)
40
+ end
41
+ end
42
+
43
+ # uses full process uuid, it's not exposed in the web UI
44
+ # can be seen in raw Jiggler.summary
45
+ def prune_process(uuid:, pool: config.client_redis_pool)
46
+ pool.acquire do |conn|
47
+ conn.call('DEL', uuid)
48
+ end
49
+ end
50
+
51
+ # hex is exposed in the web UI
52
+ # should look like jiggler:svr:74426a5e67db
53
+ def prune_process_by_hex(hex:, pool: config.client_redis_pool)
54
+ pool.acquire do |conn|
55
+ processes = conn.call('SCAN', '0', 'MATCH', "#{hex}*").last
56
+ count = processes.count
57
+ if count == 0
58
+ config.logger.error("No process found for #{hex}")
59
+ return
60
+ elsif count > 1
61
+ config.logger.error("Multiple processes found for #{hex}, not pruning #{processes}")
62
+ return
63
+ end
64
+ conn.call('DEL', processes.first)
65
+ end
66
+ end
67
+
68
+ def prune_dead_set(pool: config.client_redis_pool)
69
+ pool.acquire do |conn|
70
+ prn_dead_set(conn)
71
+ end
72
+ end
73
+
74
+ def prune_retries_set(pool: config.client_redis_pool)
75
+ pool.acquire do |conn|
76
+ prn_retries_set(conn)
77
+ end
78
+ end
79
+
80
+ def prune_scheduled_set(pool: config.client_redis_pool)
81
+ pool.acquire do |conn|
82
+ prn_scheduled_set(conn)
83
+ end
84
+ end
85
+
86
+ def prune_all_queues(pool: config.client_redis_pool)
87
+ pool.acquire do |conn|
88
+ prn_all_queues(conn)
89
+ end
90
+ end
91
+
92
+ def prune_queue(name:, pool: config.client_redis_pool)
93
+ pool.acquire do |conn|
94
+ conn.call('DEL', "#{config.queue_prefix}#{name}")
95
+ end
96
+ end
97
+
98
+ private
99
+
100
+ def prn_retries_set(conn)
101
+ conn.call('DEL', config.retries_set)
102
+ end
103
+
104
+ def prn_scheduled_set(conn)
105
+ conn.call('DEL', config.scheduled_set)
106
+ end
107
+
108
+ def prn_dead_set(conn)
109
+ conn.call('DEL', config.dead_set)
110
+ end
111
+
112
+ def prn_all_queues(conn)
113
+ queues = conn.call('SCAN', '0', 'MATCH', config.queue_scan_key).last
114
+ conn.call('DEL', *queues) unless queues.empty?
115
+ end
116
+
117
+ def prn_all_processes(conn)
118
+ processes = conn.call('SCAN', '0', 'MATCH', config.process_scan_key).last
119
+ conn.call('DEL', *processes) unless processes.empty?
120
+ end
121
+
122
+ def prn_failures_counter(conn)
123
+ conn.call('DEL', config.failures_counter)
124
+ end
125
+
126
+ def prn_processed_counter(conn)
127
+ conn.call('DEL', config.processed_counter)
128
+ end
129
+ end
130
+ end