cosmonats 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -13
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +51 -3
  8. data/lib/cosmo/api/cron/entry.rb +24 -9
  9. data/lib/cosmo/api/cron.rb +21 -7
  10. data/lib/cosmo/api/job.rb +14 -0
  11. data/lib/cosmo/api/kv.rb +138 -9
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +18 -7
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +47 -11
  18. data/lib/cosmo/api/stream.rb +66 -22
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +208 -11
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +11 -0
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +51 -15
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/version.rb +1 -1
  62. data/lib/cosmo/web/assets/app.css +5 -3
  63. data/lib/cosmo/web/cache.rb +2 -0
  64. data/lib/cosmo/web/chart.rb +2 -2
  65. data/lib/cosmo/web/circuit.rb +1 -0
  66. data/lib/cosmo/web/context.rb +6 -0
  67. data/lib/cosmo/web/controllers/application.rb +26 -0
  68. data/lib/cosmo/web/controllers/batches.rb +7 -1
  69. data/lib/cosmo/web/controllers/crons.rb +16 -2
  70. data/lib/cosmo/web/controllers/jobs.rb +45 -16
  71. data/lib/cosmo/web/controllers/metrics.rb +5 -0
  72. data/lib/cosmo/web/controllers/overview.rb +36 -18
  73. data/lib/cosmo/web/controllers/processes.rb +8 -5
  74. data/lib/cosmo/web/controllers/streams.rb +12 -0
  75. data/lib/cosmo/web/helpers/application.rb +44 -0
  76. data/lib/cosmo/web/renderer.rb +17 -0
  77. data/lib/cosmo/web/throughput.rb +1 -1
  78. data/lib/cosmo/web/views/crons/_table.erb +29 -27
  79. data/lib/cosmo/web/views/jobs/_args.erb +2 -2
  80. data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
  81. data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
  82. data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
  83. data/lib/cosmo/web/views/layout.erb +1 -1
  84. data/lib/cosmo/web/views/overview/_badges.erb +1 -1
  85. data/lib/cosmo/web/views/overview/_running.erb +3 -3
  86. data/lib/cosmo/web/views/overview/index.erb +1 -1
  87. data/lib/cosmo/web/views/streams/_info.erb +1 -0
  88. data/lib/cosmo/web/views/streams/_stream.erb +1 -0
  89. data/lib/cosmo/web.rb +8 -1
  90. data/lib/cosmo.rb +33 -0
  91. data/sig/cosmo/active_job/adapter.rbs +3 -3
  92. data/sig/cosmo/active_job/options.rbs +4 -2
  93. data/sig/cosmo/api/batch.rbs +5 -3
  94. data/sig/cosmo/api/counter.rbs +16 -6
  95. data/sig/cosmo/api/cron/entry.rbs +4 -2
  96. data/sig/cosmo/api/cron.rbs +10 -4
  97. data/sig/cosmo/api/kv.rbs +21 -4
  98. data/sig/cosmo/api/registry.rbs +10 -0
  99. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  100. data/sig/cosmo/api/stats/metrics.rbs +3 -0
  101. data/sig/cosmo/api/stats/processes.rbs +9 -1
  102. data/sig/cosmo/api/stats.rbs +8 -2
  103. data/sig/cosmo/api/stream.rbs +10 -5
  104. data/sig/cosmo/batch/callback.rbs +1 -1
  105. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  106. data/sig/cosmo/batch.rbs +5 -5
  107. data/sig/cosmo/cli.rbs +2 -0
  108. data/sig/cosmo/client.rbs +23 -4
  109. data/sig/cosmo/config/settings.rbs +6 -0
  110. data/sig/cosmo/config.rbs +5 -5
  111. data/sig/cosmo/engine.rbs +5 -0
  112. data/sig/cosmo/heartbeat.rbs +2 -0
  113. data/sig/cosmo/job/data.rbs +3 -1
  114. data/sig/cosmo/job/enqueuer.rbs +12 -0
  115. data/sig/cosmo/job/processor.rbs +18 -3
  116. data/sig/cosmo/job/setter.rbs +20 -0
  117. data/sig/cosmo/job.rbs +10 -4
  118. data/sig/cosmo/logger.rbs +5 -3
  119. data/sig/cosmo/middleware.rbs +0 -8
  120. data/sig/cosmo/processor.rbs +14 -2
  121. data/sig/cosmo/publisher.rbs +1 -0
  122. data/sig/cosmo/stream/processor.rbs +4 -0
  123. data/sig/cosmo/stream.rbs +4 -3
  124. data/sig/cosmo/utils/hash.rbs +2 -1
  125. data/sig/cosmo/utils/signal.rbs +5 -5
  126. data/sig/cosmo/utils/string.rbs +2 -2
  127. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  128. data/sig/cosmo/web/cache.rbs +35 -0
  129. data/sig/cosmo/web/chart.rbs +48 -0
  130. data/sig/cosmo/web/circuit.rbs +29 -0
  131. data/sig/cosmo/web/context.rbs +15 -0
  132. data/sig/cosmo/web/controllers/application.rbs +32 -0
  133. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  134. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  135. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  136. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  137. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  138. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  139. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  140. data/sig/cosmo/web/helpers/application.rbs +61 -0
  141. data/sig/cosmo/web/renderer.rbs +33 -0
  142. data/sig/cosmo/web/throughput.rbs +45 -0
  143. data/sig/cosmo/web.rbs +25 -0
  144. data/sig/cosmo.rbs +19 -0
  145. metadata +23 -6
  146. data/lib/cosmo/api/stats/busy.rb +0 -68
  147. data/lib/cosmo/middleware/busy.rb +0 -15
  148. data/sig/cosmo/api/stats/busy.rbs +0 -33
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 40e30ee2fff77d85db1a3deedf05e040c72dca39d64da01220517d3d5ae8048b
4
- data.tar.gz: 6f0e5a50448674483f09f54938336940899ea81fe46ce66da23478eeee871bac
3
+ metadata.gz: 0b5ab042bb1128af2b4413df2e6cc413c5542a610a3d8eee24fc0933730ffb75
4
+ data.tar.gz: 7b30b98026f0fe77324126f527666c95cd5a9d19ac2b8072cf6b4b8113ea9c0a
5
5
  SHA512:
6
- metadata.gz: 6d57b79ee594fd7a7ace94e1e3f5d70de87b9a413b85389cddb9db51f39ed09542bd0f0a7d3c4308e7009b7a2d9997b9be0c2965efa61e259bd07baf9bfa4734
7
- data.tar.gz: 8122f92e84e4176a403678225ab66b753270ab5e49334568bc3e88cfa292451bc63ea6f5383bdb458ab89f8b319a6ed33b7af471f636ffe7314978c956b6acb1
6
+ metadata.gz: b023b71e5a7a11a93e1c2cc8a27bf5f321fc7832ecc2b9048737680fbe7feb21dd6ef30d1379594f4afe6a9b141e338e815d0a295778526a63f97b3bf73f231c
7
+ data.tar.gz: df2cfd84f9c066d711d7ed0f3ed3438997089ee093871c545c8259907462a4ed2608ae1740cfabd9861322bc50eb44c5cc69b114d74349997d54a2352d2329e6
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # 🚀 Cosmonats
1
+ # <img src="lib/cosmo/web/assets/logo.svg" alt="" width="32" height="32"> Cosmonats
2
2
 
3
3
  Background jobs + real-time event streaming for Ruby — unified, in one gem, backed by NATS.
4
4
  **No Redis. No DB polling. Disk-backed, horizontally scalable — no message is ever silently dropped.**
@@ -9,7 +9,7 @@ Background jobs + real-time event streaming for Ruby — unified, in one gem, ba
9
9
 
10
10
  [![Gem Version](https://badge.fury.io/rb/cosmonats.svg)](https://rubygems.org/gems/cosmonats)
11
11
  [![Downloads](https://img.shields.io/gem/dt/cosmonats.svg)](https://rubygems.org/gems/cosmonats)
12
- [![Ruby](https://img.shields.io/badge/ruby-%3E%3D%203.1-red)](https://www.ruby-lang.org)
12
+ [![Ruby](https://img.shields.io/badge/ruby-%3E%3D%203.2-red)](https://www.ruby-lang.org)
13
13
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.txt)
14
14
  [![Build Status](https://github.com/bitsbeam/cosmonats/actions/workflows/ci.yml/badge.svg)](https://github.com/bitsbeam/cosmonats/actions)
15
15
 
@@ -165,7 +165,7 @@ nothing else to run.
165
165
  gem "cosmonats"
166
166
  ```
167
167
 
168
- **Requirements:** Ruby ≥ 3.1, NATS Server ≥ 2.14 with JetStream enabled ([install guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
168
+ **Requirements:** Ruby ≥ 3.2, NATS Server ≥ 2.14 with JetStream enabled ([install guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
169
169
 
170
170
  **Optional gems** — not installed with `cosmonats`, add them only for the features you use:
171
171
 
@@ -296,8 +296,27 @@ ReportJob.perform_async(42) # Enqueue now
296
296
  ReportJob.perform_in(30.minutes, 42) # Delayed
297
297
  ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
298
298
  ReportJob.perform_sync(42) # Inline, no NATS (great for tests)
299
+ ReportJob.set(stream: :critical, retry: 0).perform_async(42) # Per-call stream, retry, dead or subject
300
+ ReportJob.perform_bulk([[1], [2], [3]]) # One job per argument list, see below
299
301
  ```
300
302
 
303
+ `perform_bulk` publishes a thousand jobs at a time without waiting on each ack, which is many times faster than
304
+ calling `perform_async` in a loop. It takes the same options (`perform_bulk(list, in: 60)`, `set(...).perform_bulk`),
305
+ joins the current batch, and returns the jids in order. When the stream refuses some jobs, e.g. a full `discard: new`
306
+ stream, it raises `Cosmo::BulkEnqueueError`: its `jids` are `nil` for the jobs that weren't enqueued, `errors` says why
307
+ by index, and the chunks after the failing one aren't published. Inside a [batch](#batches), the refused jobs never
308
+ join it, so rescue within `jobs` to resend them while the batch is still open:
309
+ ```ruby
310
+ batch.jobs do
311
+ list = ids.zip
312
+ ReportJob.perform_bulk(list)
313
+ rescue Cosmo::BulkEnqueueError => e
314
+ ReportJob.perform_bulk(list.reject.with_index { |_, i| e.jids[i] })
315
+ end
316
+ ```
317
+ An error that is a `NATS::Timeout` means no ack came back, not that the job wasn't stored, so resending it can run that
318
+ job twice.
319
+
301
320
  Inside `perform`, the job instance knows about the message it came from:
302
321
 
303
322
  | Accessor | Value |
@@ -366,6 +385,7 @@ stream_config: &stream_config
366
385
  duplicate_window: 120 # seconds
367
386
  discard: new
368
387
  allow_direct: true # required by the web UI
388
+ num_replicas: 1 # e.g. 3 on a NATS cluster
369
389
  subjects: ["jobs.%{name}.>"]
370
390
 
371
391
  consumer_config: &consumer_config
@@ -394,12 +414,22 @@ consumers:
394
414
  `max_age`, `duplicate_window`, and `ack_wait` are in seconds. Without `setup.jobs`, `consumers.jobs.default` still
395
415
  tunes the built-in consumer (e.g. just `ack_wait: 300`).
396
416
 
417
+ `consumers.jobs` can also be a list of groups, e.g. to mirror which streams each worker fleet runs together. It is
418
+ flattened on load, so it only organizes the file; a stream may appear in one group only:
419
+ ```yaml
420
+ consumers:
421
+ jobs:
422
+ - critical: { <<: *consumer_config, priority: 50 }
423
+ default: { <<: *consumer_config, priority: 15 }
424
+ - crawlers: { <<: *consumer_config, ack_wait: 1860 }
425
+ ```
426
+
397
427
  **Service streams** — the `scheduled` stream (delayed jobs, crons), the `dead` stream (dead letters), and the stats
398
428
  counters and KV buckets are Cosmo's own: they are not part of `cosmo.yml`, and `cosmo --setup` creates them next to
399
429
  yours. Tune them in `Cosmo.configure`:
400
430
  ```ruby
401
431
  Cosmo.configure do |config|
402
- config.replicas = 3 # every service stream and bucket, for a NATS cluster (default 1)
432
+ config.services.num_replicas = 3 # every service stream and bucket, for a NATS cluster (default 1)
403
433
  config.dead.max_age = "14d" # dead-letter retention: max_age, max_msgs (10_000), max_bytes (-1); default 7d
404
434
  config.dead.enabled = false # drop jobs that give up instead of parking them
405
435
  config.scheduled.enabled = false # no scheduled stream or scheduler: perform_in/perform_at and crons raise
@@ -471,6 +501,9 @@ setup:
471
501
  timezone: America/New_York # optional, cron expressions only
472
502
  ```
473
503
 
504
+ Like `consumers.jobs`, `setup.cron` can be a list of groups (`- daily_report: {...}`) to keep related schedules together.
505
+ They are flattened on load, so a name may appear in one group only.
506
+
474
507
  `cosmo -C config/cosmo.yml -S` syncs it — whatever's in the file is exactly what ends up scheduled in NATS, same as streams.
475
508
 
476
509
  Prefer to manage schedules at runtime instead? The same operations are available from Ruby:
@@ -553,11 +586,17 @@ end
553
586
  class ThirdPartyApiJob
554
587
  include Cosmo::Job
555
588
  # At most 3 instances of this job run at once, cluster-wide.
556
- # Jobs that lose the race are NAK'd with a delay equal to `duration`
557
- # so they aren't redelivered until a slot is guaranteed free.
589
+ # Jobs that lose the race are NAK'd and redelivered after half of `duration`,
590
+ # or after `retry_in` when set (e.g. limit: { duration: 30, concurrency: 3, retry_in: 5 }).
558
591
  options limit: { duration: 30, concurrency: 3 }
559
592
  end
560
593
 
594
+ class CrawlJob
595
+ include Cosmo::Job
596
+ # Durations take seconds or strings such as "82m" or "2h" (also for limit retry_in).
597
+ options limit: { duration: "82m" }
598
+ end
599
+
561
600
  class PerAccountSyncJob
562
601
  include Cosmo::Job
563
602
  # Scope the cap per key instead of class-wide — e.g. one concurrent sync per account.
@@ -631,8 +670,8 @@ a stream batch its `stream`, `processor`, and `size`. A handler that raises is l
631
670
 
632
671
  ### Middleware
633
672
 
634
- Every job execution on a worker runs through the server middleware chain. A middleware gets the
635
- job instance, its parsed payload, and the NATS message, and yields to run the rest of the chain:
673
+ Every job execution on a worker runs through the server middleware chain. A middleware gets the job instance (`nil`
674
+ when class can't be loaded), its parsed payload, and the NATS message, and yields to run the rest of the chain:
636
675
  ```ruby
637
676
  class TimingMiddleware
638
677
  def initialize(threshold:)
@@ -656,7 +695,7 @@ end
656
695
  ```
657
696
  Register middleware at boot, before workers start (e.g. in a Rails initializer; loading `cosmo.yml` afterwards keeps
658
697
  it). The chain starts with Cosmo's required middleware: `Cosmo::Middleware::Limit` (a job class's `limit:`
659
- option), `Cosmo::Middleware::Busy` (the Web UI's Busy page), `Cosmo::Middleware::Totals` (processed/failed counts,
698
+ option), `Cosmo::Middleware::Totals` (processed/failed counts,
660
699
  one per execution), and `Cosmo::Middleware::Metrics` (the Metrics tab). `add` appends innermost, after them; `prepend`, `insert_before`, `insert_after`, and `remove`
661
700
  reorder the chain, so moving or removing the built-ins is at your own risk.
662
701
  A middleware that raises sends the job through the usual retry/DLQ path, and one that doesn't yield skips the job
@@ -664,8 +703,8 @@ and acks it. To put a job back for later instead, raise `Cosmo::Job::Requeue.new
664
703
  redelivered without failing, though the redelivery still counts toward its `retry` and `max_deliver`.
665
704
  A fresh instance is built for every job. `perform_sync` doesn't run the chain.
666
705
 
667
- The **client** chain runs where a job is enqueued (`perform_async`, `perform_in`, `perform_at`, and the ActiveJob
668
- adapter), around the publish. It starts empty, and a middleware gets the job class name, the payload Hash that will be
706
+ The **client** chain runs where a job is enqueued (`perform_async`, `perform_in`, `perform_at`, `perform_bulk`, and
707
+ the ActiveJob adapter), around the publish; for `perform_bulk`, yielding marks the job for its chunk's publish. It starts empty, and a middleware gets the job class name, the payload Hash that will be
669
708
  published, and the stream the job runs on:
670
709
  ```ruby
671
710
  class RequestIdMiddleware
@@ -900,12 +939,14 @@ sudo systemctl enable cosmo && sudo systemctl start cosmo
900
939
  light, dark, or automatic (system) theme. Fonts and assets are bundled, so it loads nothing from a CDN:
901
940
  - **Overview** — processed, failed, busy, enqueued, retries, scheduled and dead totals; a 7- or 30-day throughput
902
941
  chart of successful and failed runs; a grid of every worker thread, busy ones colored by stream; the queues; and the
903
- jobs running right now, longest first
942
+ a sample of the jobs running right now
904
943
  - **Jobs** — each job queue with its size, busy jobs and redeliveries, paused and resumed in place; then busy,
905
944
  enqueued (per queue), scheduled and dead jobs, with retry and delete for dead ones and their error and backtrace
906
945
  - **Processes** — every live worker (idle ones included): host, pid, IP, NATS connection name, command line,
907
946
  subscribed streams, busy/concurrency, running/quiet/stopping state, memory, NATS RTT and reconnects. Each worker
908
- beats into the `_cosmoprocesses` KV bucket every 10s, and a crashed worker drops out within 60s
947
+ beats into the `_cosmoprocesses` KV bucket every 10s, and the jobs it's running into `_cosmojobs`; a crashed worker
948
+ drops out within 60s. Busy jobs come from those heartbeats, so they show up to 10s late, and a job shorter than that
949
+ may not show at all. The page reads only the processes it shows, and every count is answered by NATS from the keys.
909
950
  - **Streams** — every stream with its subjects, messages, size, retention and replicas, its consumers' pending,
910
951
  ack-pending and redelivered counts, and pause/resume
911
952
  - **Crons** — every schedule deployed in NATS, with run-now and delete
@@ -13,6 +13,7 @@ module Cosmo
13
13
  class Adapter
14
14
  # Enqueue a job to be run as soon as possible.
15
15
  # @param job [ActiveJob::Base]
16
+ # @return [String, nil] the job id, nil when client middleware stopped the publish
16
17
  def enqueue(job)
17
18
  publish(job, nil)
18
19
  end
@@ -20,6 +21,7 @@ module Cosmo
20
21
  # Enqueue a job to be run at (or after) a given time.
21
22
  # @param job [ActiveJob::Base]
22
23
  # @param timestamp [Numeric] Unix timestamp (seconds, float)
24
+ # @return [String, nil] the job id, nil when client middleware stopped the publish
23
25
  def enqueue_at(job, timestamp)
24
26
  publish(job, timestamp)
25
27
  end
@@ -10,6 +10,9 @@ module Cosmo
10
10
 
11
11
  # Resolves +retry_in+ from the underlying ActiveJob class (declared via +cosmo_options+),
12
12
  # since the processor only ever sees +Executor+ as the worker class for ActiveJob-dispatched jobs.
13
+ #
14
+ # @param data [Hash] the job's payload
15
+ # @return [Proc, nil]
13
16
  def self.retry_in(data)
14
17
  job_class = Utils::String.safe_constantize(data.dig(:args, 0, :job_class))
15
18
  return super unless job_class.respond_to?(:get_cosmo_options)
@@ -17,6 +20,10 @@ module Cosmo
17
20
  job_class.get_cosmo_options[:retry_in] || super
18
21
  end
19
22
 
23
+ # Runs the serialized ActiveJob.
24
+ #
25
+ # @param job_data [Hash] what +ActiveJob::Base#serialize+ returned
26
+ # @return [void]
20
27
  def perform(job_data)
21
28
  ::ActiveJob::Base.execute(Utils::Hash.stringify_keys(job_data))
22
29
  end
@@ -22,6 +22,8 @@ module Cosmo
22
22
  module Options
23
23
  VALID_OPTIONS = %i[retry dead stream retry_in].freeze
24
24
 
25
+ # @param base [Class]
26
+ # @return [void]
25
27
  def self.included(base)
26
28
  base.extend(ClassMethods)
27
29
  end
@@ -30,6 +32,9 @@ module Cosmo
30
32
  # Set Cosmo-specific options for this job class.
31
33
  # Merges with any options inherited from a superclass.
32
34
  # Raises +ArgumentError+ for unknown keys.
35
+ #
36
+ # @param opts [Hash] see {Options}
37
+ # @return [Hash] the class's options
33
38
  def cosmo_options(**opts)
34
39
  unknown = opts.keys - Cosmo::ActiveJobAdapter::Options::VALID_OPTIONS
35
40
  raise ::ArgumentError, "Unknown cosmo_options key(s): #{unknown.join(", ")}" if unknown.any?
@@ -38,6 +43,7 @@ module Cosmo
38
43
  end
39
44
 
40
45
  # Returns the resolved options, walking up the inheritance chain.
46
+ # @return [Hash]
41
47
  def get_cosmo_options # rubocop:disable Naming/AccessorMethodName
42
48
  if @cosmo_options
43
49
  @cosmo_options.dup
@@ -6,36 +6,55 @@ module Cosmo
6
6
  class Batch
7
7
  LIMIT = 25
8
8
 
9
+ # @param limit [Integer, nil] the most batches to return, nil for every one
10
+ # @return [Array<Batch>]
9
11
  def self.all(limit: LIMIT)
10
12
  kv.keys("*.meta", limit: limit).map { |key| new(key.delete_suffix(".meta")) }
11
13
  end
12
14
 
15
+ # Counts batches by direct gets, reading no further than +limit+.
16
+ #
17
+ # @param limit [Integer] the most batches to count
18
+ # @return [Integer]
19
+ def self.count(limit:)
20
+ kv.direct_entries("*.meta", limit:).size
21
+ end
22
+
23
+ # @return [KV] the bucket holding each batch's meta, callbacks and finalized snapshot
13
24
  def self.kv
14
25
  Cosmo::Batch.kv
15
26
  end
16
27
 
28
+ # @return [Cosmo::Batch::Counters] the per-batch total, pending and failed counters
17
29
  def self.counter
18
30
  Cosmo::Batch.counter
19
31
  end
20
32
 
21
33
  attr_reader :bid
22
34
 
35
+ # @param bid [String] the batch id
23
36
  def initialize(bid)
24
37
  @bid = bid
25
38
  end
26
39
 
40
+ # @return [Hash] the batch's stored metadata, empty when it has none
27
41
  def meta
28
42
  Utils::Json.parse(kv.get("#{bid}.meta")&.value) || {}
29
43
  end
30
44
 
45
+ # @return [String, nil] the enclosing batch's id for a nested batch
31
46
  def parent_id
32
47
  meta[:parent_id]
33
48
  end
34
49
 
50
+ # @return [Integer, nil]
35
51
  def created_at
36
52
  meta[:created_at]
37
53
  end
38
54
 
55
+ # Whether the batch has been finalized.
56
+ #
57
+ # @return [Boolean]
39
58
  def ready?
40
59
  !!kv.get("#{bid}.ready")
41
60
  end
@@ -43,6 +62,8 @@ module Cosmo
43
62
  # { total:, pending:, succeeded:, failed: } -- read from the +ready+
44
63
  # snapshot once finalized (counters are reset by then), or live off the
45
64
  # counters (with +pending+ still moving) while the batch is still open.
65
+ #
66
+ # @return [Hash]
46
67
  def stats
47
68
  entry = kv.get("#{bid}.ready")
48
69
  return Utils::Json.parse(entry.value).merge(pending: 0) if entry
@@ -50,15 +71,20 @@ module Cosmo
50
71
  live_stats
51
72
  end
52
73
 
74
+ # @param event [Symbol] +:success+ or +:complete+
75
+ # @return [Hash, nil] the callback registered for +event+
53
76
  def callback(event)
54
77
  entry = kv.get("#{bid}.callback.#{event}")
55
78
  Utils::Json.parse(entry&.value)
56
79
  end
57
80
 
81
+ # @return [Hash{Symbol => Hash, nil}] the +:success+ and +:complete+ callbacks
58
82
  def callbacks
59
83
  { success: callback(:success), complete: callback(:complete) }
60
84
  end
61
85
 
86
+ # @param event [Symbol] +:success+ or +:complete+
87
+ # @return [Boolean] whether the callback for +event+ has already fired
62
88
  def fired?(event)
63
89
  !!kv.get("#{bid}.fired.#{event}")
64
90
  end
@@ -17,28 +17,69 @@ module Cosmo
17
17
  # @return [Hash]
18
18
  def self.stream_config
19
19
  { subjects: ["#{self::STREAM_NAME}.>"], allow_msg_counter: true, allow_direct: true, max_msgs_per_subject: 1,
20
- num_replicas: Config.replicas, description: self::DESCRIPTION }
20
+ num_replicas: Config.services.num_replicas, description: self::DESCRIPTION }
21
21
  end
22
22
 
23
+ # @param namespace [String] the subject segment every key of this instance counts under
23
24
  def initialize(namespace)
24
25
  @namespace = namespace
25
26
  end
26
27
 
27
- def increment(key, by: 1, msg_id: nil)
28
+ # Adds +by+ to +key+, raising when the counter can't be written.
29
+ #
30
+ # @param key [Symbol, String]
31
+ # @param by [Integer]
32
+ # @param msg_id [String, nil] see {#publish}
33
+ # @return [Integer, nil] see {#publish}
34
+ def increment!(key, by: 1, msg_id: nil)
28
35
  publish(key, "+#{by}", msg_id: msg_id)
29
36
  end
37
+ alias incr! increment!
38
+
39
+ # Adds +by+ to +key+ like {#increment!}, but logs a counter that can't be written instead of raising.
40
+ #
41
+ # @param key [Symbol, String]
42
+ # @param by [Integer]
43
+ # @param msg_id [String, nil]
44
+ # @return [Integer, nil] nil also when the counter can't be written
45
+ def increment(key, by: 1, msg_id: nil)
46
+ safely(key) { increment!(key, by:, msg_id:) }
47
+ end
30
48
  alias incr increment
31
49
 
32
- def decrement(key, by: 1, msg_id: nil)
50
+ # Subtracts +by+ from +key+, raising when the counter can't be written.
51
+ #
52
+ # @param key [Symbol, String]
53
+ # @param by [Integer]
54
+ # @param msg_id [String, nil] see {#publish}
55
+ # @return [Integer, nil] see {#publish}
56
+ def decrement!(key, by: 1, msg_id: nil)
33
57
  publish(key, "-#{by}", msg_id: msg_id)
34
58
  end
59
+ alias decr! decrement!
60
+
61
+ # Subtracts +by+ from +key+ like {#decrement!}, but logs a counter that can't be written instead of raising.
62
+ #
63
+ # @param key [Symbol, String]
64
+ # @param by [Integer]
65
+ # @param msg_id [String, nil]
66
+ # @return [Integer, nil] nil also when the counter can't be written
67
+ def decrement(key, by: 1, msg_id: nil)
68
+ safely(key) { decrement!(key, by:, msg_id:) }
69
+ end
35
70
  alias decr decrement
36
71
 
72
+ # Drops +key+'s counter, so it reads 0 again.
73
+ #
74
+ # @param key [Symbol, String]
75
+ # @return [Integer, nil] the number of messages purged
37
76
  def reset(key)
38
77
  client.purge(self.class::STREAM_NAME, subject(key))
39
78
  end
40
79
  alias purge reset
41
80
 
81
+ # @param key [Symbol, String]
82
+ # @return [Integer] the counter's value, 0 when it was never written or can't be read
42
83
  def get(key)
43
84
  raw = client.get_message(self.class::STREAM_NAME, direct: true, subject: subject(key))
44
85
  Utils::Json.parse(raw.data, default: { "val" => 0 })[:val].to_i
@@ -48,6 +89,13 @@ module Cosmo
48
89
 
49
90
  private
50
91
 
92
+ def safely(key)
93
+ yield
94
+ rescue StandardError => e
95
+ Logger.debug "#{self.class::STREAM_NAME} #{key} counter error: #{e.class} #{e.message}"
96
+ nil
97
+ end
98
+
51
99
  # @param msg_id [String, nil] when given, sent as Nats-Msg-Id so a redelivered
52
100
  # caller (e.g. a job whose ack was lost and retried) collapses onto the
53
101
  # stream's duplicate_window instead of double-applying the +/- delta.
@@ -21,6 +21,10 @@ module Cosmo
21
21
  class Entry
22
22
  SUBJECT_PREFIX = "cosmo.cron"
23
23
 
24
+ # Prepends the seconds field to a 5-field cron expression, leaving @-shortcuts and 6-field ones as they are.
25
+ #
26
+ # @param expr [String, nil]
27
+ # @return [String]
24
28
  def self.normalize_expression(expr)
25
29
  str = expr.to_s.strip
26
30
  return str if str.start_with?("@")
@@ -48,6 +52,8 @@ module Cosmo
48
52
 
49
53
  # Subject where the schedule message lives in NATS (one per unique schedule).
50
54
  # e.g. "cosmo.cron.default.report_job" or "cosmo.cron.default.report_job.monthly"
55
+ #
56
+ # @return [String]
51
57
  def schedule_subject
52
58
  parts = [SUBJECT_PREFIX, @stream, job_name]
53
59
  parts << @name if @name
@@ -57,31 +63,41 @@ module Cosmo
57
63
  # Subject where NATS fires the generated job message. A schedule may only target a
58
64
  # subject its own stream covers, so every schedule fires into the scheduled stream and
59
65
  # the scheduler forwards the job on to {#dispatch_subject}.
66
+ #
67
+ # @return [String]
60
68
  def target_subject
61
69
  "jobs.#{::Cosmo::Job::StreamFilter::SCHEDULED}.#{job_name}"
62
70
  end
63
71
 
64
72
  # Subject the scheduler dispatches the fired job to, on the stream that runs it.
73
+ #
74
+ # @return [String]
65
75
  def dispatch_subject
66
76
  "jobs.#{@stream}.#{job_name}"
67
77
  end
68
78
 
69
79
  # The +schedule:+ option of nats-pure's JetStream#publish that makes the template a schedule.
70
80
  # @-shortcuts (+@daily+, +@every 5m+) travel as the cron expression, which the server understands.
81
+ #
82
+ # @return [Hash{Symbol => String}]
71
83
  def schedule
72
84
  { cron: @expression, target: target_subject, time_zone: @timezone }.compact
73
85
  end
74
86
 
75
87
  # Headers stored on the schedule template. NATS copies them onto every message it fires,
76
88
  # which is how the scheduler learns where the job belongs.
89
+ #
90
+ # @return [Hash{String => String}]
77
91
  def headers
78
92
  { "X-Stream" => @stream, "X-Subject" => dispatch_subject }
79
93
  end
80
94
 
95
+ # @return [String] the underscored job class name used in subjects
81
96
  def job_name
82
97
  @job_name ||= Utils::String.underscore(@class_name)
83
98
  end
84
99
 
100
+ # @return [Hash] the entry's attributes and subjects, without nil values
85
101
  def as_json
86
102
  {
87
103
  class: @class_name,
@@ -96,18 +112,17 @@ module Cosmo
96
112
  }.compact
97
113
  end
98
114
 
99
- # JSON payload sent as the body of the schedule message.
100
- # Mirrors the format produced by Job::Data so the job processor can handle it.
115
+ # JSON payload sent as the body of the schedule message, built by Job::Data from the job class's own +retry+
116
+ # and +dead+ options, or the defaults when the class can't be loaded here.
117
+ #
118
+ # @return [String]
101
119
  def job_payload
102
- Utils::Json.dump({
103
- jid: SecureRandom.hex(12),
104
- class: @class_name,
105
- args: @args,
106
- retry: ::Cosmo::Job::Data.default_retry,
107
- dead: ::Cosmo::Job::Data::DEFAULTS[:dead]
108
- })
120
+ job_class = Utils::String.safe_constantize(@class_name)
121
+ options = job_class.respond_to?(:default_options) ? job_class.default_options : {}
122
+ ::Cosmo::Job::Data.new(@class_name, @args, options.merge(stream: @stream)).to_json
109
123
  end
110
124
 
125
+ # @return [String]
111
126
  def to_s
112
127
  "#<Cosmo::API::Cron::Entry class=#{@class_name} expression=#{@expression} stream=#{@stream}>"
113
128
  end
@@ -19,11 +19,12 @@ module Cosmo
19
19
  class Cron
20
20
  STREAM = ::Cosmo::Job::StreamFilter::SCHEDULED.to_s
21
21
 
22
+ # @return [Cron]
22
23
  def self.instance
23
24
  @instance ||= new
24
25
  end
25
26
 
26
- # @return [Array<Hash>] every cron schedule currently deployed in NATS
27
+ # @return [Array<Hash>] every cron schedule currently deployed in NATS, sorted by stream, class and name
27
28
  def all
28
29
  schedules
29
30
  rescue StandardError
@@ -31,6 +32,12 @@ module Cosmo
31
32
  end
32
33
 
33
34
  # Publish (or replace) a schedule message in NATS.
35
+ # @param class_name [String, nil] the job class to run
36
+ # @param stream [String, nil] the job stream that runs it
37
+ # @param schedule [String, nil] the cron expression or @-shortcut, see {Entry.normalize_expression}
38
+ # @param args [Array] arguments passed to the job's +perform+
39
+ # @param timezone [String, nil] IANA timezone name for cron expressions
40
+ # @param name [String, Symbol, nil] disambiguates several schedules of one class
34
41
  # @return [Hash, nil] the persisted schedule as a hash, or nil on failure
35
42
  # @raise [SchedulingDisabledError] when scheduling is turned off
36
43
  def upsert!(class_name: nil, stream: nil, schedule: nil, args: [], timezone: nil, name: nil)
@@ -44,6 +51,7 @@ module Cosmo
44
51
 
45
52
  # Purge the schedule message from NATS (stops future firings).
46
53
  # @param subject [String]
54
+ # @return [void]
47
55
  def delete!(subject)
48
56
  client.purge(STREAM, subject)
49
57
  rescue NATS::JetStream::Error::NotFound, NATS::IO::Timeout
@@ -52,7 +60,8 @@ module Cosmo
52
60
 
53
61
  # Dispatch the job immediately to the target stream, bypassing the timer.
54
62
  # @param schedule_subject [String] e.g. "cosmo.cron.default.report_job.daily"
55
- def run_now!(schedule_subject) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
63
+ # @return [void]
64
+ def run_now!(schedule_subject) # rubocop:disable Metrics/AbcSize
56
65
  msg = client.get_message(STREAM, subject: schedule_subject)
57
66
  return unless msg
58
67
 
@@ -80,16 +89,21 @@ module Cosmo
80
89
  end
81
90
 
82
91
  def schedules
83
- subjects = client.cron_subjects_in_stream(STREAM, "#{Entry::SUBJECT_PREFIX}.>")
84
- subjects.filter_map { |subj| build_from_nats(subj) }
92
+ client.last_messages(STREAM, ["#{Entry::SUBJECT_PREFIX}.>"])
93
+ .filter_map { |subject, msg| build(subject, msg.header, msg.data) }
94
+ .sort_by { [_1[:stream].to_s, _1[:class].to_s, _1[:name].to_s] }
85
95
  end
86
96
 
87
97
  def build_from_nats(subject)
88
98
  msg = client.get_message(STREAM, subject: subject)
89
- return unless msg
99
+ build(subject, msg.headers, msg.data) if msg
100
+ rescue StandardError
101
+ nil
102
+ end
90
103
 
91
- headers = msg.headers || {}
92
- body = Utils::Json.parse(msg.data) || {}
104
+ def build(subject, headers, data)
105
+ headers ||= {}
106
+ body = Utils::Json.parse(data) || {}
93
107
 
94
108
  {
95
109
  class: body[:class],
data/lib/cosmo/api/job.rb CHANGED
@@ -7,47 +7,59 @@ module Cosmo
7
7
  class Job
8
8
  attr_reader :message, :stream
9
9
 
10
+ # @param stream [String] the stream the message was read from
11
+ # @param message [NATS::JetStream::API::RawStreamMsg]
10
12
  def initialize(stream, message)
11
13
  @stream = stream
12
14
  @message = message
13
15
  end
14
16
 
17
+ # @return [Hash, nil] the parsed job payload
15
18
  def data
16
19
  @data ||= Utils::Json.parse(@message.data)
17
20
  end
18
21
 
22
+ # @return [Integer, String] the message's stream sequence, a String when read by direct get
19
23
  def seq
20
24
  @message.seq
21
25
  end
22
26
 
27
+ # @return [Hash{String => String}, nil]
23
28
  def headers
24
29
  @message.headers
25
30
  end
26
31
 
32
+ # @return [Integer, nil] when a scheduled job is due, from +X-Execute-At+
27
33
  def execute_at
28
34
  headers&.dig("X-Execute-At")&.to_i
29
35
  end
30
36
 
37
+ # @return [String, nil] the stream a scheduled or dead job belongs to, from +X-Stream+
31
38
  def x_stream
32
39
  headers&.dig("X-Stream")
33
40
  end
34
41
 
42
+ # @return [String, nil] the subject a scheduled or dead job is dispatched to, from +X-Subject+
35
43
  def x_subject
36
44
  headers&.dig("X-Subject")
37
45
  end
38
46
 
47
+ # @return [String]
39
48
  def subject
40
49
  @message.subject
41
50
  end
42
51
 
52
+ # @return [String, nil] the last failure's exception class, from +X-Error-Class+
43
53
  def error_class
44
54
  headers&.dig("X-Error-Class")
45
55
  end
46
56
 
57
+ # @return [String, nil] the last failure's message, from +X-Error-Message+
47
58
  def error_message
48
59
  headers&.dig("X-Error-Message")
49
60
  end
50
61
 
62
+ # @return [String, nil] the last failure's backtrace as one line, from +X-Error-Backtrace+
51
63
  def error_backtrace
52
64
  headers&.dig("X-Error-Backtrace")
53
65
  end
@@ -57,10 +69,12 @@ module Cosmo
57
69
  error_backtrace.to_s.split(::Cosmo::Job::Failure::BACKTRACE_SEPARATOR)
58
70
  end
59
71
 
72
+ # @return [Boolean] whether the message carries a failure
60
73
  def error?
61
74
  !error_class.to_s.empty? || !error_message.to_s.empty?
62
75
  end
63
76
 
77
+ # @return [String, nil] the +Nats-Time-Stamp+ header
64
78
  def timestamp
65
79
  headers&.dig(NATS::JetStream::Header::TIME_STAMP)
66
80
  end