cosmonats 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +73 -22
- data/lib/cosmo/active_job/adapter.rb +2 -0
- data/lib/cosmo/active_job/executor.rb +7 -0
- data/lib/cosmo/active_job/options.rb +6 -0
- data/lib/cosmo/api/batch.rb +26 -0
- data/lib/cosmo/api/counter.rb +52 -4
- data/lib/cosmo/api/cron/entry.rb +34 -20
- data/lib/cosmo/api/cron.rb +24 -10
- data/lib/cosmo/api/job.rb +15 -1
- data/lib/cosmo/api/kv.rb +141 -12
- data/lib/cosmo/api/registry.rb +47 -6
- data/lib/cosmo/api/stats/jobs.rb +84 -0
- data/lib/cosmo/api/stats/metrics.rb +20 -9
- data/lib/cosmo/api/stats/processes.rb +27 -11
- data/lib/cosmo/api/stats/totals.rb +1 -0
- data/lib/cosmo/api/stats.rb +58 -7
- data/lib/cosmo/api/stream.rb +73 -24
- data/lib/cosmo/batch/callback.rb +5 -0
- data/lib/cosmo/batch/dispatcher.rb +4 -4
- data/lib/cosmo/batch.rb +34 -11
- data/lib/cosmo/cli.rb +6 -0
- data/lib/cosmo/client.rb +217 -23
- data/lib/cosmo/config/cosmo.yml +1 -0
- data/lib/cosmo/config/settings.rb +4 -0
- data/lib/cosmo/config.rb +58 -10
- data/lib/cosmo/engine.rb +28 -1
- data/lib/cosmo/heartbeat.rb +12 -3
- data/lib/cosmo/http_server/handler.rb +5 -0
- data/lib/cosmo/http_server/health.rb +2 -0
- data/lib/cosmo/http_server/ping.rb +2 -0
- data/lib/cosmo/http_server.rb +8 -0
- data/lib/cosmo/job/data.rb +12 -1
- data/lib/cosmo/job/enqueuer.rb +100 -1
- data/lib/cosmo/job/limit.rb +5 -1
- data/lib/cosmo/job/processor.rb +56 -21
- data/lib/cosmo/job/setter.rb +53 -0
- data/lib/cosmo/job.rb +71 -2
- data/lib/cosmo/logger.rb +37 -0
- data/lib/cosmo/middleware/chain.rb +12 -0
- data/lib/cosmo/middleware/limit.rb +12 -3
- data/lib/cosmo/middleware/metrics.rb +6 -4
- data/lib/cosmo/middleware/sentry.rb +6 -4
- data/lib/cosmo/middleware/totals.rb +5 -11
- data/lib/cosmo/middleware.rb +0 -1
- data/lib/cosmo/processor.rb +60 -3
- data/lib/cosmo/publisher.rb +8 -0
- data/lib/cosmo/services.rb +7 -2
- data/lib/cosmo/stream/message.rb +5 -0
- data/lib/cosmo/stream/processor.rb +7 -0
- data/lib/cosmo/stream/serializer.rb +4 -0
- data/lib/cosmo/stream.rb +35 -1
- data/lib/cosmo/utils/duration.rb +3 -0
- data/lib/cosmo/utils/hash.rb +12 -0
- data/lib/cosmo/utils/json.rb +8 -0
- data/lib/cosmo/utils/signal.rb +9 -0
- data/lib/cosmo/utils/stopwatch.rb +1 -0
- data/lib/cosmo/utils/string.rb +4 -0
- data/lib/cosmo/utils/thread_pool.rb +8 -0
- data/lib/cosmo/utils/ttl_cache.rb +10 -0
- data/lib/cosmo/utils.rb +0 -1
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +273 -629
- data/lib/cosmo/web/assets/app.js +158 -0
- data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
- data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
- data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
- data/lib/cosmo/web/assets/logo.svg +10 -0
- data/lib/cosmo/web/cache.rb +77 -0
- data/lib/cosmo/web/chart.rb +63 -39
- data/lib/cosmo/web/circuit.rb +56 -0
- data/lib/cosmo/web/context.rb +6 -4
- data/lib/cosmo/web/controllers/application.rb +42 -0
- data/lib/cosmo/web/controllers/batches.rb +8 -1
- data/lib/cosmo/web/controllers/crons.rb +24 -3
- data/lib/cosmo/web/controllers/jobs.rb +88 -68
- data/lib/cosmo/web/controllers/metrics.rb +6 -0
- data/lib/cosmo/web/controllers/overview.rb +88 -0
- data/lib/cosmo/web/controllers/processes.rb +36 -0
- data/lib/cosmo/web/controllers/streams.rb +38 -28
- data/lib/cosmo/web/helpers/application.rb +101 -0
- data/lib/cosmo/web/renderer.rb +36 -6
- data/lib/cosmo/web/throughput.rb +104 -0
- data/lib/cosmo/web/views/batches/_table.erb +42 -54
- data/lib/cosmo/web/views/batches/index.erb +3 -10
- data/lib/cosmo/web/views/crons/_table.erb +36 -55
- data/lib/cosmo/web/views/crons/index.erb +2 -9
- data/lib/cosmo/web/views/jobs/_args.erb +5 -0
- data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
- data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
- data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
- data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
- data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
- data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
- data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
- data/lib/cosmo/web/views/jobs/index.erb +4 -17
- data/lib/cosmo/web/views/layout.erb +73 -22
- data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
- data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
- data/lib/cosmo/web/views/metrics/_table.erb +24 -40
- data/lib/cosmo/web/views/metrics/index.erb +8 -16
- data/lib/cosmo/web/views/overview/_badges.erb +3 -0
- data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
- data/lib/cosmo/web/views/overview/_running.erb +18 -0
- data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
- data/lib/cosmo/web/views/overview/_workers.erb +25 -0
- data/lib/cosmo/web/views/overview/index.erb +28 -0
- data/lib/cosmo/web/views/processes/_table.erb +67 -0
- data/lib/cosmo/web/views/processes/index.erb +1 -0
- data/lib/cosmo/web/views/shared/_pager.erb +22 -0
- data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
- data/lib/cosmo/web/views/streams/_info.erb +29 -86
- data/lib/cosmo/web/views/streams/_list.erb +7 -0
- data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
- data/lib/cosmo/web/views/streams/_stream.erb +53 -0
- data/lib/cosmo/web/views/streams/index.erb +3 -11
- data/lib/cosmo/web/views/streams/info.erb +3 -11
- data/lib/cosmo/web.rb +73 -35
- data/lib/cosmo.rb +33 -0
- data/sig/cosmo/active_job/adapter.rbs +3 -3
- data/sig/cosmo/active_job/options.rbs +4 -2
- data/sig/cosmo/api/batch.rbs +5 -3
- data/sig/cosmo/api/counter.rbs +16 -6
- data/sig/cosmo/api/cron/entry.rbs +6 -3
- data/sig/cosmo/api/cron.rbs +10 -4
- data/sig/cosmo/api/kv.rbs +21 -4
- data/sig/cosmo/api/registry.rbs +10 -0
- data/sig/cosmo/api/stats/jobs.rbs +34 -0
- data/sig/cosmo/api/stats/metrics.rbs +4 -1
- data/sig/cosmo/api/stats/processes.rbs +9 -1
- data/sig/cosmo/api/stats.rbs +12 -2
- data/sig/cosmo/api/stream.rbs +12 -5
- data/sig/cosmo/batch/callback.rbs +1 -1
- data/sig/cosmo/batch/dispatcher.rbs +1 -1
- data/sig/cosmo/batch.rbs +5 -5
- data/sig/cosmo/cli.rbs +2 -0
- data/sig/cosmo/client.rbs +24 -4
- data/sig/cosmo/config/settings.rbs +6 -0
- data/sig/cosmo/config.rbs +5 -5
- data/sig/cosmo/engine.rbs +5 -0
- data/sig/cosmo/heartbeat.rbs +2 -0
- data/sig/cosmo/job/data.rbs +3 -1
- data/sig/cosmo/job/enqueuer.rbs +12 -0
- data/sig/cosmo/job/processor.rbs +18 -3
- data/sig/cosmo/job/setter.rbs +20 -0
- data/sig/cosmo/job.rbs +10 -4
- data/sig/cosmo/logger.rbs +5 -3
- data/sig/cosmo/middleware.rbs +0 -8
- data/sig/cosmo/processor.rbs +14 -2
- data/sig/cosmo/publisher.rbs +1 -0
- data/sig/cosmo/stream/processor.rbs +4 -0
- data/sig/cosmo/stream.rbs +4 -3
- data/sig/cosmo/utils/hash.rbs +2 -1
- data/sig/cosmo/utils/signal.rbs +5 -5
- data/sig/cosmo/utils/string.rbs +2 -2
- data/sig/cosmo/utils/thread_pool.rbs +1 -1
- data/sig/cosmo/web/cache.rbs +35 -0
- data/sig/cosmo/web/chart.rbs +48 -0
- data/sig/cosmo/web/circuit.rbs +29 -0
- data/sig/cosmo/web/context.rbs +15 -0
- data/sig/cosmo/web/controllers/application.rbs +32 -0
- data/sig/cosmo/web/controllers/batches.rbs +11 -0
- data/sig/cosmo/web/controllers/crons.rbs +23 -0
- data/sig/cosmo/web/controllers/jobs.rbs +47 -0
- data/sig/cosmo/web/controllers/metrics.rbs +21 -0
- data/sig/cosmo/web/controllers/overview.rbs +31 -0
- data/sig/cosmo/web/controllers/processes.rbs +13 -0
- data/sig/cosmo/web/controllers/streams.rbs +27 -0
- data/sig/cosmo/web/helpers/application.rbs +61 -0
- data/sig/cosmo/web/renderer.rbs +33 -0
- data/sig/cosmo/web/throughput.rbs +45 -0
- data/sig/cosmo/web.rbs +25 -0
- data/sig/cosmo.rbs +19 -0
- metadata +53 -23
- data/lib/cosmo/api/stats/busy.rb +0 -63
- data/lib/cosmo/middleware/busy.rb +0 -15
- data/lib/cosmo/utils/overrides.rb +0 -206
- data/lib/cosmo/utils/warnings.rb +0 -17
- data/lib/cosmo/web/controllers/actions.rb +0 -16
- data/lib/cosmo/web/views/actions/index.erb +0 -7
- data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
- data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
- data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
- data/lib/cosmo/web/views/jobs/busy.erb +0 -18
- data/lib/cosmo/web/views/jobs/dead.erb +0 -18
- data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
- data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
- data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
- data/lib/cosmo/web/views/streams/_table.erb +0 -25
- data/sig/cosmo/api/stats/busy.rbs +0 -31
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0b5ab042bb1128af2b4413df2e6cc413c5542a610a3d8eee24fc0933730ffb75
|
|
4
|
+
data.tar.gz: 7b30b98026f0fe77324126f527666c95cd5a9d19ac2b8072cf6b4b8113ea9c0a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b023b71e5a7a11a93e1c2cc8a27bf5f321fc7832ecc2b9048737680fbe7feb21dd6ef30d1379594f4afe6a9b141e338e815d0a295778526a63f97b3bf73f231c
|
|
7
|
+
data.tar.gz: df2cfd84f9c066d711d7ed0f3ed3438997089ee093871c545c8259907462a4ed2608ae1740cfabd9861322bc50eb44c5cc69b114d74349997d54a2352d2329e6
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# <img src="lib/cosmo/web/assets/logo.svg" alt="" width="32" height="32"> Cosmonats
|
|
2
2
|
|
|
3
3
|
Background jobs + real-time event streaming for Ruby — unified, in one gem, backed by NATS.
|
|
4
4
|
**No Redis. No DB polling. Disk-backed, horizontally scalable — no message is ever silently dropped.**
|
|
@@ -9,7 +9,7 @@ Background jobs + real-time event streaming for Ruby — unified, in one gem, ba
|
|
|
9
9
|
|
|
10
10
|
[](https://rubygems.org/gems/cosmonats)
|
|
11
11
|
[](https://rubygems.org/gems/cosmonats)
|
|
12
|
-
[](https://www.ruby-lang.org)
|
|
13
13
|
[](LICENSE.txt)
|
|
14
14
|
[](https://github.com/bitsbeam/cosmonats/actions)
|
|
15
15
|
|
|
@@ -58,8 +58,6 @@ bundle exec cosmo -C config/cosmo.yml -c 20 jobs # Jobs only
|
|
|
58
58
|
bundle exec cosmo -C config/cosmo.yml -c 20 streams # Streams only
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
-

|
|
62
|
-
|
|
63
61
|
## 📖 Index
|
|
64
62
|
|
|
65
63
|
- [Why?](#-why)
|
|
@@ -167,7 +165,7 @@ nothing else to run.
|
|
|
167
165
|
gem "cosmonats"
|
|
168
166
|
```
|
|
169
167
|
|
|
170
|
-
**Requirements:** Ruby ≥ 3.
|
|
168
|
+
**Requirements:** Ruby ≥ 3.2, NATS Server ≥ 2.14 with JetStream enabled ([install guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
|
|
171
169
|
|
|
172
170
|
**Optional gems** — not installed with `cosmonats`, add them only for the features you use:
|
|
173
171
|
|
|
@@ -298,7 +296,26 @@ ReportJob.perform_async(42) # Enqueue now
|
|
|
298
296
|
ReportJob.perform_in(30.minutes, 42) # Delayed
|
|
299
297
|
ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
|
|
300
298
|
ReportJob.perform_sync(42) # Inline, no NATS (great for tests)
|
|
299
|
+
ReportJob.set(stream: :critical, retry: 0).perform_async(42) # Per-call stream, retry, dead or subject
|
|
300
|
+
ReportJob.perform_bulk([[1], [2], [3]]) # One job per argument list, see below
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
`perform_bulk` publishes a thousand jobs at a time without waiting on each ack, which is many times faster than
|
|
304
|
+
calling `perform_async` in a loop. It takes the same options (`perform_bulk(list, in: 60)`, `set(...).perform_bulk`),
|
|
305
|
+
joins the current batch, and returns the jids in order. When the stream refuses some jobs, e.g. a full `discard: new`
|
|
306
|
+
stream, it raises `Cosmo::BulkEnqueueError`: its `jids` are `nil` for the jobs that weren't enqueued, `errors` says why
|
|
307
|
+
by index, and the chunks after the failing one aren't published. Inside a [batch](#batches), the refused jobs never
|
|
308
|
+
join it, so rescue within `jobs` to resend them while the batch is still open:
|
|
309
|
+
```ruby
|
|
310
|
+
batch.jobs do
|
|
311
|
+
list = ids.zip
|
|
312
|
+
ReportJob.perform_bulk(list)
|
|
313
|
+
rescue Cosmo::BulkEnqueueError => e
|
|
314
|
+
ReportJob.perform_bulk(list.reject.with_index { |_, i| e.jids[i] })
|
|
315
|
+
end
|
|
301
316
|
```
|
|
317
|
+
An error that is a `NATS::Timeout` means no ack came back, not that the job wasn't stored, so resending it can run that
|
|
318
|
+
job twice.
|
|
302
319
|
|
|
303
320
|
Inside `perform`, the job instance knows about the message it came from:
|
|
304
321
|
|
|
@@ -368,6 +385,7 @@ stream_config: &stream_config
|
|
|
368
385
|
duplicate_window: 120 # seconds
|
|
369
386
|
discard: new
|
|
370
387
|
allow_direct: true # required by the web UI
|
|
388
|
+
num_replicas: 1 # e.g. 3 on a NATS cluster
|
|
371
389
|
subjects: ["jobs.%{name}.>"]
|
|
372
390
|
|
|
373
391
|
consumer_config: &consumer_config
|
|
@@ -396,12 +414,22 @@ consumers:
|
|
|
396
414
|
`max_age`, `duplicate_window`, and `ack_wait` are in seconds. Without `setup.jobs`, `consumers.jobs.default` still
|
|
397
415
|
tunes the built-in consumer (e.g. just `ack_wait: 300`).
|
|
398
416
|
|
|
417
|
+
`consumers.jobs` can also be a list of groups, e.g. to mirror which streams each worker fleet runs together. It is
|
|
418
|
+
flattened on load, so it only organizes the file; a stream may appear in one group only:
|
|
419
|
+
```yaml
|
|
420
|
+
consumers:
|
|
421
|
+
jobs:
|
|
422
|
+
- critical: { <<: *consumer_config, priority: 50 }
|
|
423
|
+
default: { <<: *consumer_config, priority: 15 }
|
|
424
|
+
- crawlers: { <<: *consumer_config, ack_wait: 1860 }
|
|
425
|
+
```
|
|
426
|
+
|
|
399
427
|
**Service streams** — the `scheduled` stream (delayed jobs, crons), the `dead` stream (dead letters), and the stats
|
|
400
428
|
counters and KV buckets are Cosmo's own: they are not part of `cosmo.yml`, and `cosmo --setup` creates them next to
|
|
401
429
|
yours. Tune them in `Cosmo.configure`:
|
|
402
430
|
```ruby
|
|
403
431
|
Cosmo.configure do |config|
|
|
404
|
-
config.
|
|
432
|
+
config.services.num_replicas = 3 # every service stream and bucket, for a NATS cluster (default 1)
|
|
405
433
|
config.dead.max_age = "14d" # dead-letter retention: max_age, max_msgs (10_000), max_bytes (-1); default 7d
|
|
406
434
|
config.dead.enabled = false # drop jobs that give up instead of parking them
|
|
407
435
|
config.scheduled.enabled = false # no scheduled stream or scheduler: perform_in/perform_at and crons raise
|
|
@@ -433,9 +461,11 @@ Cosmo leaves a custom logger's formatter alone, so its lines lack Cosmo's `jid`/
|
|
|
433
461
|
| `COSMO_CLIENT_NAME` | `cosmo-<program>-<host>-<pid>` | NATS connection name, shown in `nats server report connections` and `/connz` |
|
|
434
462
|
| `COSMO_JS_TIMEOUT` | `5` | JetStream API timeout in seconds; raise it for the web UI on a busy server |
|
|
435
463
|
| `COSMO_CONNECT_TIMEOUT` | `2` | NATS TCP connect timeout in seconds |
|
|
464
|
+
| `NATS_MAX_RECONNECT_ATTEMPTS` | `-1` | Reconnect attempts before giving up on NATS; `-1` keeps trying through any outage |
|
|
436
465
|
| `COSMO_JOBS_STREAMS` | all configured | Comma-separated job streams to subscribe to, when `--streams`/`--stream` isn't given |
|
|
437
466
|
| `COSMO_JOBS_FETCH_TIMEOUT` | `0.1` | Seconds a jobs worker waits on each fetch |
|
|
438
467
|
| `COSMO_WEB_POLL_INTERVAL` | `5` | Web UI auto-refresh interval in seconds |
|
|
468
|
+
| `COSMO_WEB_CACHE_TTL` | `2` | Seconds the web UI shares a refresh between viewers; `0` turns sharing off |
|
|
439
469
|
|
|
440
470
|
|
|
441
471
|
## 🔧 Advanced Usage
|
|
@@ -471,6 +501,9 @@ setup:
|
|
|
471
501
|
timezone: America/New_York # optional, cron expressions only
|
|
472
502
|
```
|
|
473
503
|
|
|
504
|
+
Like `consumers.jobs`, `setup.cron` can be a list of groups (`- daily_report: {...}`) to keep related schedules together.
|
|
505
|
+
They are flattened on load, so a name may appear in one group only.
|
|
506
|
+
|
|
474
507
|
`cosmo -C config/cosmo.yml -S` syncs it — whatever's in the file is exactly what ends up scheduled in NATS, same as streams.
|
|
475
508
|
|
|
476
509
|
Prefer to manage schedules at runtime instead? The same operations are available from Ruby:
|
|
@@ -543,7 +576,7 @@ end
|
|
|
543
576
|
```
|
|
544
577
|
- Batch tracking data (pending counts, callbacks, results) expires automatically after
|
|
545
578
|
`config.batches.expiry` (default: 3 days), set in `Cosmo.configure`.
|
|
546
|
-
- Open and finished batches — with pending/succeeded/failed counts — are listed live
|
|
579
|
+
- Open and finished batches — with pending/succeeded/failed counts — are listed live on the web UI's **Batches** page.
|
|
547
580
|
- Only `Cosmo::Job`-based jobs are tracked; the ActiveJob adapter doesn't currently participate in
|
|
548
581
|
batches (see [`docs/active_job.md`](docs/active_job.md)).
|
|
549
582
|
|
|
@@ -553,11 +586,17 @@ end
|
|
|
553
586
|
class ThirdPartyApiJob
|
|
554
587
|
include Cosmo::Job
|
|
555
588
|
# At most 3 instances of this job run at once, cluster-wide.
|
|
556
|
-
# Jobs that lose the race are NAK'd
|
|
557
|
-
#
|
|
589
|
+
# Jobs that lose the race are NAK'd and redelivered after half of `duration`,
|
|
590
|
+
# or after `retry_in` when set (e.g. limit: { duration: 30, concurrency: 3, retry_in: 5 }).
|
|
558
591
|
options limit: { duration: 30, concurrency: 3 }
|
|
559
592
|
end
|
|
560
593
|
|
|
594
|
+
class CrawlJob
|
|
595
|
+
include Cosmo::Job
|
|
596
|
+
# Durations take seconds or strings such as "82m" or "2h" (also for limit retry_in).
|
|
597
|
+
options limit: { duration: "82m" }
|
|
598
|
+
end
|
|
599
|
+
|
|
561
600
|
class PerAccountSyncJob
|
|
562
601
|
include Cosmo::Job
|
|
563
602
|
# Scope the cap per key instead of class-wide — e.g. one concurrent sync per account.
|
|
@@ -631,8 +670,8 @@ a stream batch its `stream`, `processor`, and `size`. A handler that raises is l
|
|
|
631
670
|
|
|
632
671
|
### Middleware
|
|
633
672
|
|
|
634
|
-
Every job execution on a worker runs through the server middleware chain. A middleware gets the
|
|
635
|
-
|
|
673
|
+
Every job execution on a worker runs through the server middleware chain. A middleware gets the job instance (`nil`
|
|
674
|
+
when class can't be loaded), its parsed payload, and the NATS message, and yields to run the rest of the chain:
|
|
636
675
|
```ruby
|
|
637
676
|
class TimingMiddleware
|
|
638
677
|
def initialize(threshold:)
|
|
@@ -656,7 +695,7 @@ end
|
|
|
656
695
|
```
|
|
657
696
|
Register middleware at boot, before workers start (e.g. in a Rails initializer; loading `cosmo.yml` afterwards keeps
|
|
658
697
|
it). The chain starts with Cosmo's required middleware: `Cosmo::Middleware::Limit` (a job class's `limit:`
|
|
659
|
-
option), `Cosmo::Middleware::
|
|
698
|
+
option), `Cosmo::Middleware::Totals` (processed/failed counts,
|
|
660
699
|
one per execution), and `Cosmo::Middleware::Metrics` (the Metrics tab). `add` appends innermost, after them; `prepend`, `insert_before`, `insert_after`, and `remove`
|
|
661
700
|
reorder the chain, so moving or removing the built-ins is at your own risk.
|
|
662
701
|
A middleware that raises sends the job through the usual retry/DLQ path, and one that doesn't yield skips the job
|
|
@@ -664,8 +703,8 @@ and acks it. To put a job back for later instead, raise `Cosmo::Job::Requeue.new
|
|
|
664
703
|
redelivered without failing, though the redelivery still counts toward its `retry` and `max_deliver`.
|
|
665
704
|
A fresh instance is built for every job. `perform_sync` doesn't run the chain.
|
|
666
705
|
|
|
667
|
-
The **client** chain runs where a job is enqueued (`perform_async`, `perform_in`, `perform_at`, and
|
|
668
|
-
adapter), around the publish. It starts empty, and a middleware gets the job class name, the payload Hash that will be
|
|
706
|
+
The **client** chain runs where a job is enqueued (`perform_async`, `perform_in`, `perform_at`, `perform_bulk`, and
|
|
707
|
+
the ActiveJob adapter), around the publish; for `perform_bulk`, yielding marks the job for its chunk's publish. It starts empty, and a middleware gets the job class name, the payload Hash that will be
|
|
669
708
|
published, and the stream the job runs on:
|
|
670
709
|
```ruby
|
|
671
710
|
class RequestIdMiddleware
|
|
@@ -896,19 +935,31 @@ sudo systemctl enable cosmo && sudo systemctl start cosmo
|
|
|
896
935
|
|
|
897
936
|
## 📊 Monitoring
|
|
898
937
|
|
|
899
|
-
**Web UI** — mount `Cosmo::Web` (see [Installation](#-installation)) for a live, htmx-powered dashboard
|
|
900
|
-
|
|
901
|
-
- **
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
- **
|
|
938
|
+
**Web UI** — mount `Cosmo::Web` (see [Installation](#-installation)) for a live, htmx-powered dashboard with a
|
|
939
|
+
light, dark, or automatic (system) theme. Fonts and assets are bundled, so it loads nothing from a CDN:
|
|
940
|
+
- **Overview** — processed, failed, busy, enqueued, retries, scheduled and dead totals; a 7- or 30-day throughput
|
|
941
|
+
chart of successful and failed runs; a grid of every worker thread, busy ones colored by stream; the queues; and the
|
|
942
|
+
a sample of the jobs running right now
|
|
943
|
+
- **Jobs** — each job queue with its size, busy jobs and redeliveries, paused and resumed in place; then busy,
|
|
944
|
+
enqueued (per queue), scheduled and dead jobs, with retry and delete for dead ones and their error and backtrace
|
|
945
|
+
- **Processes** — every live worker (idle ones included): host, pid, IP, NATS connection name, command line,
|
|
946
|
+
subscribed streams, busy/concurrency, running/quiet/stopping state, memory, NATS RTT and reconnects. Each worker
|
|
947
|
+
beats into the `_cosmoprocesses` KV bucket every 10s, and the jobs it's running into `_cosmojobs`; a crashed worker
|
|
948
|
+
drops out within 60s. Busy jobs come from those heartbeats, so they show up to 10s late, and a job shorter than that
|
|
949
|
+
may not show at all. The page reads only the processes it shows, and every count is answered by NATS from the keys.
|
|
950
|
+
- **Streams** — every stream with its subjects, messages, size, retention and replicas, its consumers' pending,
|
|
951
|
+
ack-pending and redelivered counts, and pause/resume
|
|
905
952
|
- **Crons** — every schedule deployed in NATS, with run-now and delete
|
|
906
|
-
- **Batches** — open and finished batches with
|
|
953
|
+
- **Batches** — open and finished batches with their progress, succeeded/failed/pending counts and callbacks
|
|
907
954
|
- **Metrics** — per job class over today, 7, or 30 days: runs, failures, average execution time (successful runs), and
|
|
908
955
|
average wait from enqueue to first delivery, plus a per-day chart of runs and average execution time. Recorded by
|
|
909
956
|
`Cosmo::Middleware::Metrics`, flushed by every worker's heartbeat into the internal `_cosmometrics` counters stream;
|
|
910
957
|
turn it off with `config.metrics.enabled = false`
|
|
911
|
-
|
|
958
|
+
|
|
959
|
+
Each panel refreshes `COSMO_WEB_POLL_INTERVAL` seconds after its previous response, never on top of one still
|
|
960
|
+
running, backs off while refreshes fail, and pauses in a hidden tab. Identical refreshes from several viewers share a
|
|
961
|
+
single NATS read for `COSMO_WEB_CACHE_TTL` seconds, and while NATS is unreachable the UI answers at once instead of
|
|
962
|
+
holding your app server's threads.
|
|
912
963
|
|
|
913
964
|
**Structured logs:**
|
|
914
965
|
```
|
|
@@ -13,6 +13,7 @@ module Cosmo
|
|
|
13
13
|
class Adapter
|
|
14
14
|
# Enqueue a job to be run as soon as possible.
|
|
15
15
|
# @param job [ActiveJob::Base]
|
|
16
|
+
# @return [String, nil] the job id, nil when client middleware stopped the publish
|
|
16
17
|
def enqueue(job)
|
|
17
18
|
publish(job, nil)
|
|
18
19
|
end
|
|
@@ -20,6 +21,7 @@ module Cosmo
|
|
|
20
21
|
# Enqueue a job to be run at (or after) a given time.
|
|
21
22
|
# @param job [ActiveJob::Base]
|
|
22
23
|
# @param timestamp [Numeric] Unix timestamp (seconds, float)
|
|
24
|
+
# @return [String, nil] the job id, nil when client middleware stopped the publish
|
|
23
25
|
def enqueue_at(job, timestamp)
|
|
24
26
|
publish(job, timestamp)
|
|
25
27
|
end
|
|
@@ -10,6 +10,9 @@ module Cosmo
|
|
|
10
10
|
|
|
11
11
|
# Resolves +retry_in+ from the underlying ActiveJob class (declared via +cosmo_options+),
|
|
12
12
|
# since the processor only ever sees +Executor+ as the worker class for ActiveJob-dispatched jobs.
|
|
13
|
+
#
|
|
14
|
+
# @param data [Hash] the job's payload
|
|
15
|
+
# @return [Proc, nil]
|
|
13
16
|
def self.retry_in(data)
|
|
14
17
|
job_class = Utils::String.safe_constantize(data.dig(:args, 0, :job_class))
|
|
15
18
|
return super unless job_class.respond_to?(:get_cosmo_options)
|
|
@@ -17,6 +20,10 @@ module Cosmo
|
|
|
17
20
|
job_class.get_cosmo_options[:retry_in] || super
|
|
18
21
|
end
|
|
19
22
|
|
|
23
|
+
# Runs the serialized ActiveJob.
|
|
24
|
+
#
|
|
25
|
+
# @param job_data [Hash] what +ActiveJob::Base#serialize+ returned
|
|
26
|
+
# @return [void]
|
|
20
27
|
def perform(job_data)
|
|
21
28
|
::ActiveJob::Base.execute(Utils::Hash.stringify_keys(job_data))
|
|
22
29
|
end
|
|
@@ -22,6 +22,8 @@ module Cosmo
|
|
|
22
22
|
module Options
|
|
23
23
|
VALID_OPTIONS = %i[retry dead stream retry_in].freeze
|
|
24
24
|
|
|
25
|
+
# @param base [Class]
|
|
26
|
+
# @return [void]
|
|
25
27
|
def self.included(base)
|
|
26
28
|
base.extend(ClassMethods)
|
|
27
29
|
end
|
|
@@ -30,6 +32,9 @@ module Cosmo
|
|
|
30
32
|
# Set Cosmo-specific options for this job class.
|
|
31
33
|
# Merges with any options inherited from a superclass.
|
|
32
34
|
# Raises +ArgumentError+ for unknown keys.
|
|
35
|
+
#
|
|
36
|
+
# @param opts [Hash] see {Options}
|
|
37
|
+
# @return [Hash] the class's options
|
|
33
38
|
def cosmo_options(**opts)
|
|
34
39
|
unknown = opts.keys - Cosmo::ActiveJobAdapter::Options::VALID_OPTIONS
|
|
35
40
|
raise ::ArgumentError, "Unknown cosmo_options key(s): #{unknown.join(", ")}" if unknown.any?
|
|
@@ -38,6 +43,7 @@ module Cosmo
|
|
|
38
43
|
end
|
|
39
44
|
|
|
40
45
|
# Returns the resolved options, walking up the inheritance chain.
|
|
46
|
+
# @return [Hash]
|
|
41
47
|
def get_cosmo_options # rubocop:disable Naming/AccessorMethodName
|
|
42
48
|
if @cosmo_options
|
|
43
49
|
@cosmo_options.dup
|
data/lib/cosmo/api/batch.rb
CHANGED
|
@@ -6,36 +6,55 @@ module Cosmo
|
|
|
6
6
|
class Batch
|
|
7
7
|
LIMIT = 25
|
|
8
8
|
|
|
9
|
+
# @param limit [Integer, nil] the most batches to return, nil for every one
|
|
10
|
+
# @return [Array<Batch>]
|
|
9
11
|
def self.all(limit: LIMIT)
|
|
10
12
|
kv.keys("*.meta", limit: limit).map { |key| new(key.delete_suffix(".meta")) }
|
|
11
13
|
end
|
|
12
14
|
|
|
15
|
+
# Counts batches by direct gets, reading no further than +limit+.
|
|
16
|
+
#
|
|
17
|
+
# @param limit [Integer] the most batches to count
|
|
18
|
+
# @return [Integer]
|
|
19
|
+
def self.count(limit:)
|
|
20
|
+
kv.direct_entries("*.meta", limit:).size
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @return [KV] the bucket holding each batch's meta, callbacks and finalized snapshot
|
|
13
24
|
def self.kv
|
|
14
25
|
Cosmo::Batch.kv
|
|
15
26
|
end
|
|
16
27
|
|
|
28
|
+
# @return [Cosmo::Batch::Counters] the per-batch total, pending and failed counters
|
|
17
29
|
def self.counter
|
|
18
30
|
Cosmo::Batch.counter
|
|
19
31
|
end
|
|
20
32
|
|
|
21
33
|
attr_reader :bid
|
|
22
34
|
|
|
35
|
+
# @param bid [String] the batch id
|
|
23
36
|
def initialize(bid)
|
|
24
37
|
@bid = bid
|
|
25
38
|
end
|
|
26
39
|
|
|
40
|
+
# @return [Hash] the batch's stored metadata, empty when it has none
|
|
27
41
|
def meta
|
|
28
42
|
Utils::Json.parse(kv.get("#{bid}.meta")&.value) || {}
|
|
29
43
|
end
|
|
30
44
|
|
|
45
|
+
# @return [String, nil] the enclosing batch's id for a nested batch
|
|
31
46
|
def parent_id
|
|
32
47
|
meta[:parent_id]
|
|
33
48
|
end
|
|
34
49
|
|
|
50
|
+
# @return [Integer, nil]
|
|
35
51
|
def created_at
|
|
36
52
|
meta[:created_at]
|
|
37
53
|
end
|
|
38
54
|
|
|
55
|
+
# Whether the batch has been finalized.
|
|
56
|
+
#
|
|
57
|
+
# @return [Boolean]
|
|
39
58
|
def ready?
|
|
40
59
|
!!kv.get("#{bid}.ready")
|
|
41
60
|
end
|
|
@@ -43,6 +62,8 @@ module Cosmo
|
|
|
43
62
|
# { total:, pending:, succeeded:, failed: } -- read from the +ready+
|
|
44
63
|
# snapshot once finalized (counters are reset by then), or live off the
|
|
45
64
|
# counters (with +pending+ still moving) while the batch is still open.
|
|
65
|
+
#
|
|
66
|
+
# @return [Hash]
|
|
46
67
|
def stats
|
|
47
68
|
entry = kv.get("#{bid}.ready")
|
|
48
69
|
return Utils::Json.parse(entry.value).merge(pending: 0) if entry
|
|
@@ -50,15 +71,20 @@ module Cosmo
|
|
|
50
71
|
live_stats
|
|
51
72
|
end
|
|
52
73
|
|
|
74
|
+
# @param event [Symbol] +:success+ or +:complete+
|
|
75
|
+
# @return [Hash, nil] the callback registered for +event+
|
|
53
76
|
def callback(event)
|
|
54
77
|
entry = kv.get("#{bid}.callback.#{event}")
|
|
55
78
|
Utils::Json.parse(entry&.value)
|
|
56
79
|
end
|
|
57
80
|
|
|
81
|
+
# @return [Hash{Symbol => Hash, nil}] the +:success+ and +:complete+ callbacks
|
|
58
82
|
def callbacks
|
|
59
83
|
{ success: callback(:success), complete: callback(:complete) }
|
|
60
84
|
end
|
|
61
85
|
|
|
86
|
+
# @param event [Symbol] +:success+ or +:complete+
|
|
87
|
+
# @return [Boolean] whether the callback for +event+ has already fired
|
|
62
88
|
def fired?(event)
|
|
63
89
|
!!kv.get("#{bid}.fired.#{event}")
|
|
64
90
|
end
|
data/lib/cosmo/api/counter.rb
CHANGED
|
@@ -17,28 +17,69 @@ module Cosmo
|
|
|
17
17
|
# @return [Hash]
|
|
18
18
|
def self.stream_config
|
|
19
19
|
{ subjects: ["#{self::STREAM_NAME}.>"], allow_msg_counter: true, allow_direct: true, max_msgs_per_subject: 1,
|
|
20
|
-
num_replicas: Config.
|
|
20
|
+
num_replicas: Config.services.num_replicas, description: self::DESCRIPTION }
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# @param namespace [String] the subject segment every key of this instance counts under
|
|
23
24
|
def initialize(namespace)
|
|
24
25
|
@namespace = namespace
|
|
25
26
|
end
|
|
26
27
|
|
|
27
|
-
|
|
28
|
+
# Adds +by+ to +key+, raising when the counter can't be written.
|
|
29
|
+
#
|
|
30
|
+
# @param key [Symbol, String]
|
|
31
|
+
# @param by [Integer]
|
|
32
|
+
# @param msg_id [String, nil] see {#publish}
|
|
33
|
+
# @return [Integer, nil] see {#publish}
|
|
34
|
+
def increment!(key, by: 1, msg_id: nil)
|
|
28
35
|
publish(key, "+#{by}", msg_id: msg_id)
|
|
29
36
|
end
|
|
37
|
+
alias incr! increment!
|
|
38
|
+
|
|
39
|
+
# Adds +by+ to +key+ like {#increment!}, but logs a counter that can't be written instead of raising.
|
|
40
|
+
#
|
|
41
|
+
# @param key [Symbol, String]
|
|
42
|
+
# @param by [Integer]
|
|
43
|
+
# @param msg_id [String, nil]
|
|
44
|
+
# @return [Integer, nil] nil also when the counter can't be written
|
|
45
|
+
def increment(key, by: 1, msg_id: nil)
|
|
46
|
+
safely(key) { increment!(key, by:, msg_id:) }
|
|
47
|
+
end
|
|
30
48
|
alias incr increment
|
|
31
49
|
|
|
32
|
-
|
|
50
|
+
# Subtracts +by+ from +key+, raising when the counter can't be written.
|
|
51
|
+
#
|
|
52
|
+
# @param key [Symbol, String]
|
|
53
|
+
# @param by [Integer]
|
|
54
|
+
# @param msg_id [String, nil] see {#publish}
|
|
55
|
+
# @return [Integer, nil] see {#publish}
|
|
56
|
+
def decrement!(key, by: 1, msg_id: nil)
|
|
33
57
|
publish(key, "-#{by}", msg_id: msg_id)
|
|
34
58
|
end
|
|
59
|
+
alias decr! decrement!
|
|
60
|
+
|
|
61
|
+
# Subtracts +by+ from +key+ like {#decrement!}, but logs a counter that can't be written instead of raising.
|
|
62
|
+
#
|
|
63
|
+
# @param key [Symbol, String]
|
|
64
|
+
# @param by [Integer]
|
|
65
|
+
# @param msg_id [String, nil]
|
|
66
|
+
# @return [Integer, nil] nil also when the counter can't be written
|
|
67
|
+
def decrement(key, by: 1, msg_id: nil)
|
|
68
|
+
safely(key) { decrement!(key, by:, msg_id:) }
|
|
69
|
+
end
|
|
35
70
|
alias decr decrement
|
|
36
71
|
|
|
72
|
+
# Drops +key+'s counter, so it reads 0 again.
|
|
73
|
+
#
|
|
74
|
+
# @param key [Symbol, String]
|
|
75
|
+
# @return [Integer, nil] the number of messages purged
|
|
37
76
|
def reset(key)
|
|
38
77
|
client.purge(self.class::STREAM_NAME, subject(key))
|
|
39
78
|
end
|
|
40
79
|
alias purge reset
|
|
41
80
|
|
|
81
|
+
# @param key [Symbol, String]
|
|
82
|
+
# @return [Integer] the counter's value, 0 when it was never written or can't be read
|
|
42
83
|
def get(key)
|
|
43
84
|
raw = client.get_message(self.class::STREAM_NAME, direct: true, subject: subject(key))
|
|
44
85
|
Utils::Json.parse(raw.data, default: { "val" => 0 })[:val].to_i
|
|
@@ -48,6 +89,13 @@ module Cosmo
|
|
|
48
89
|
|
|
49
90
|
private
|
|
50
91
|
|
|
92
|
+
def safely(key)
|
|
93
|
+
yield
|
|
94
|
+
rescue StandardError => e
|
|
95
|
+
Logger.debug "#{self.class::STREAM_NAME} #{key} counter error: #{e.class} #{e.message}"
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
51
99
|
# @param msg_id [String, nil] when given, sent as Nats-Msg-Id so a redelivered
|
|
52
100
|
# caller (e.g. a job whose ack was lost and retried) collapses onto the
|
|
53
101
|
# stream's duplicate_window instead of double-applying the +/- delta.
|
|
@@ -58,7 +106,7 @@ module Cosmo
|
|
|
58
106
|
def publish(key, value, msg_id: nil)
|
|
59
107
|
rescued = nil
|
|
60
108
|
headers = { "Nats-Incr" => value }
|
|
61
|
-
headers[
|
|
109
|
+
headers[NATS::JetStream::Header::MSG_ID] = msg_id if msg_id
|
|
62
110
|
|
|
63
111
|
begin
|
|
64
112
|
ack = client.publish(subject(key), "", header: headers)
|
data/lib/cosmo/api/cron/entry.rb
CHANGED
|
@@ -21,6 +21,10 @@ module Cosmo
|
|
|
21
21
|
class Entry
|
|
22
22
|
SUBJECT_PREFIX = "cosmo.cron"
|
|
23
23
|
|
|
24
|
+
# Prepends the seconds field to a 5-field cron expression, leaving @-shortcuts and 6-field ones as they are.
|
|
25
|
+
#
|
|
26
|
+
# @param expr [String, nil]
|
|
27
|
+
# @return [String]
|
|
24
28
|
def self.normalize_expression(expr)
|
|
25
29
|
str = expr.to_s.strip
|
|
26
30
|
return str if str.start_with?("@")
|
|
@@ -48,6 +52,8 @@ module Cosmo
|
|
|
48
52
|
|
|
49
53
|
# Subject where the schedule message lives in NATS (one per unique schedule).
|
|
50
54
|
# e.g. "cosmo.cron.default.report_job" or "cosmo.cron.default.report_job.monthly"
|
|
55
|
+
#
|
|
56
|
+
# @return [String]
|
|
51
57
|
def schedule_subject
|
|
52
58
|
parts = [SUBJECT_PREFIX, @stream, job_name]
|
|
53
59
|
parts << @name if @name
|
|
@@ -57,32 +63,41 @@ module Cosmo
|
|
|
57
63
|
# Subject where NATS fires the generated job message. A schedule may only target a
|
|
58
64
|
# subject its own stream covers, so every schedule fires into the scheduled stream and
|
|
59
65
|
# the scheduler forwards the job on to {#dispatch_subject}.
|
|
66
|
+
#
|
|
67
|
+
# @return [String]
|
|
60
68
|
def target_subject
|
|
61
69
|
"jobs.#{::Cosmo::Job::StreamFilter::SCHEDULED}.#{job_name}"
|
|
62
70
|
end
|
|
63
71
|
|
|
64
72
|
# Subject the scheduler dispatches the fired job to, on the stream that runs it.
|
|
73
|
+
#
|
|
74
|
+
# @return [String]
|
|
65
75
|
def dispatch_subject
|
|
66
76
|
"jobs.#{@stream}.#{job_name}"
|
|
67
77
|
end
|
|
68
78
|
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"X-Stream" => @stream,
|
|
76
|
-
"X-Subject" => dispatch_subject
|
|
77
|
-
}
|
|
78
|
-
headers["Nats-Schedule-Time-Zone"] = @timezone if @timezone
|
|
79
|
-
headers
|
|
79
|
+
# The +schedule:+ option of nats-pure's JetStream#publish that makes the template a schedule.
|
|
80
|
+
# @-shortcuts (+@daily+, +@every 5m+) travel as the cron expression, which the server understands.
|
|
81
|
+
#
|
|
82
|
+
# @return [Hash{Symbol => String}]
|
|
83
|
+
def schedule
|
|
84
|
+
{ cron: @expression, target: target_subject, time_zone: @timezone }.compact
|
|
80
85
|
end
|
|
81
86
|
|
|
87
|
+
# Headers stored on the schedule template. NATS copies them onto every message it fires,
|
|
88
|
+
# which is how the scheduler learns where the job belongs.
|
|
89
|
+
#
|
|
90
|
+
# @return [Hash{String => String}]
|
|
91
|
+
def headers
|
|
92
|
+
{ "X-Stream" => @stream, "X-Subject" => dispatch_subject }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# @return [String] the underscored job class name used in subjects
|
|
82
96
|
def job_name
|
|
83
97
|
@job_name ||= Utils::String.underscore(@class_name)
|
|
84
98
|
end
|
|
85
99
|
|
|
100
|
+
# @return [Hash] the entry's attributes and subjects, without nil values
|
|
86
101
|
def as_json
|
|
87
102
|
{
|
|
88
103
|
class: @class_name,
|
|
@@ -97,18 +112,17 @@ module Cosmo
|
|
|
97
112
|
}.compact
|
|
98
113
|
end
|
|
99
114
|
|
|
100
|
-
# JSON payload sent as the body of the schedule message
|
|
101
|
-
#
|
|
115
|
+
# JSON payload sent as the body of the schedule message, built by Job::Data from the job class's own +retry+
|
|
116
|
+
# and +dead+ options, or the defaults when the class can't be loaded here.
|
|
117
|
+
#
|
|
118
|
+
# @return [String]
|
|
102
119
|
def job_payload
|
|
103
|
-
Utils::
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
args: @args,
|
|
107
|
-
retry: ::Cosmo::Job::Data.default_retry,
|
|
108
|
-
dead: ::Cosmo::Job::Data::DEFAULTS[:dead]
|
|
109
|
-
})
|
|
120
|
+
job_class = Utils::String.safe_constantize(@class_name)
|
|
121
|
+
options = job_class.respond_to?(:default_options) ? job_class.default_options : {}
|
|
122
|
+
::Cosmo::Job::Data.new(@class_name, @args, options.merge(stream: @stream)).to_json
|
|
110
123
|
end
|
|
111
124
|
|
|
125
|
+
# @return [String]
|
|
112
126
|
def to_s
|
|
113
127
|
"#<Cosmo::API::Cron::Entry class=#{@class_name} expression=#{@expression} stream=#{@stream}>"
|
|
114
128
|
end
|
data/lib/cosmo/api/cron.rb
CHANGED
|
@@ -19,11 +19,12 @@ module Cosmo
|
|
|
19
19
|
class Cron
|
|
20
20
|
STREAM = ::Cosmo::Job::StreamFilter::SCHEDULED.to_s
|
|
21
21
|
|
|
22
|
+
# @return [Cron]
|
|
22
23
|
def self.instance
|
|
23
24
|
@instance ||= new
|
|
24
25
|
end
|
|
25
26
|
|
|
26
|
-
# @return [Array<Hash>] every cron schedule currently deployed in NATS
|
|
27
|
+
# @return [Array<Hash>] every cron schedule currently deployed in NATS, sorted by stream, class and name
|
|
27
28
|
def all
|
|
28
29
|
schedules
|
|
29
30
|
rescue StandardError
|
|
@@ -31,6 +32,12 @@ module Cosmo
|
|
|
31
32
|
end
|
|
32
33
|
|
|
33
34
|
# Publish (or replace) a schedule message in NATS.
|
|
35
|
+
# @param class_name [String, nil] the job class to run
|
|
36
|
+
# @param stream [String, nil] the job stream that runs it
|
|
37
|
+
# @param schedule [String, nil] the cron expression or @-shortcut, see {Entry.normalize_expression}
|
|
38
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
39
|
+
# @param timezone [String, nil] IANA timezone name for cron expressions
|
|
40
|
+
# @param name [String, Symbol, nil] disambiguates several schedules of one class
|
|
34
41
|
# @return [Hash, nil] the persisted schedule as a hash, or nil on failure
|
|
35
42
|
# @raise [SchedulingDisabledError] when scheduling is turned off
|
|
36
43
|
def upsert!(class_name: nil, stream: nil, schedule: nil, args: [], timezone: nil, name: nil)
|
|
@@ -38,12 +45,13 @@ module Cosmo
|
|
|
38
45
|
|
|
39
46
|
e = Entry.new(class_name: class_name, stream: stream, expression: schedule,
|
|
40
47
|
args: args, timezone: timezone, name: name)
|
|
41
|
-
client.publish(e.schedule_subject, e.job_payload, stream: STREAM, header: e.
|
|
48
|
+
client.publish(e.schedule_subject, e.job_payload, stream: STREAM, header: e.headers, schedule: e.schedule)
|
|
42
49
|
build_from_nats(e.schedule_subject)
|
|
43
50
|
end
|
|
44
51
|
|
|
45
52
|
# Purge the schedule message from NATS (stops future firings).
|
|
46
53
|
# @param subject [String]
|
|
54
|
+
# @return [void]
|
|
47
55
|
def delete!(subject)
|
|
48
56
|
client.purge(STREAM, subject)
|
|
49
57
|
rescue NATS::JetStream::Error::NotFound, NATS::IO::Timeout
|
|
@@ -52,7 +60,8 @@ module Cosmo
|
|
|
52
60
|
|
|
53
61
|
# Dispatch the job immediately to the target stream, bypassing the timer.
|
|
54
62
|
# @param schedule_subject [String] e.g. "cosmo.cron.default.report_job.daily"
|
|
55
|
-
|
|
63
|
+
# @return [void]
|
|
64
|
+
def run_now!(schedule_subject) # rubocop:disable Metrics/AbcSize
|
|
56
65
|
msg = client.get_message(STREAM, subject: schedule_subject)
|
|
57
66
|
return unless msg
|
|
58
67
|
|
|
@@ -80,22 +89,27 @@ module Cosmo
|
|
|
80
89
|
end
|
|
81
90
|
|
|
82
91
|
def schedules
|
|
83
|
-
|
|
84
|
-
|
|
92
|
+
client.last_messages(STREAM, ["#{Entry::SUBJECT_PREFIX}.>"])
|
|
93
|
+
.filter_map { |subject, msg| build(subject, msg.header, msg.data) }
|
|
94
|
+
.sort_by { [_1[:stream].to_s, _1[:class].to_s, _1[:name].to_s] }
|
|
85
95
|
end
|
|
86
96
|
|
|
87
97
|
def build_from_nats(subject)
|
|
88
98
|
msg = client.get_message(STREAM, subject: subject)
|
|
89
|
-
|
|
99
|
+
build(subject, msg.headers, msg.data) if msg
|
|
100
|
+
rescue StandardError
|
|
101
|
+
nil
|
|
102
|
+
end
|
|
90
103
|
|
|
91
|
-
|
|
92
|
-
|
|
104
|
+
def build(subject, headers, data)
|
|
105
|
+
headers ||= {}
|
|
106
|
+
body = Utils::Json.parse(data) || {}
|
|
93
107
|
|
|
94
108
|
{
|
|
95
109
|
class: body[:class],
|
|
96
110
|
stream: headers["X-Stream"],
|
|
97
|
-
schedule: headers[
|
|
98
|
-
timezone: headers[
|
|
111
|
+
schedule: headers[NATS::JetStream::Header::SCHEDULE],
|
|
112
|
+
timezone: headers[NATS::JetStream::Header::SCHEDULE_TIME_ZONE],
|
|
99
113
|
args: body[:args] || [],
|
|
100
114
|
name: name_from_subject(subject),
|
|
101
115
|
schedule_subject: subject,
|