cosmonats 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +73 -22
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +52 -4
  8. data/lib/cosmo/api/cron/entry.rb +34 -20
  9. data/lib/cosmo/api/cron.rb +24 -10
  10. data/lib/cosmo/api/job.rb +15 -1
  11. data/lib/cosmo/api/kv.rb +141 -12
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +20 -9
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +58 -7
  18. data/lib/cosmo/api/stream.rb +73 -24
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +217 -23
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +12 -1
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +56 -21
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/utils.rb +0 -1
  62. data/lib/cosmo/version.rb +1 -1
  63. data/lib/cosmo/web/assets/app.css +273 -629
  64. data/lib/cosmo/web/assets/app.js +158 -0
  65. data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
  66. data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
  67. data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
  68. data/lib/cosmo/web/assets/logo.svg +10 -0
  69. data/lib/cosmo/web/cache.rb +77 -0
  70. data/lib/cosmo/web/chart.rb +63 -39
  71. data/lib/cosmo/web/circuit.rb +56 -0
  72. data/lib/cosmo/web/context.rb +6 -4
  73. data/lib/cosmo/web/controllers/application.rb +42 -0
  74. data/lib/cosmo/web/controllers/batches.rb +8 -1
  75. data/lib/cosmo/web/controllers/crons.rb +24 -3
  76. data/lib/cosmo/web/controllers/jobs.rb +88 -68
  77. data/lib/cosmo/web/controllers/metrics.rb +6 -0
  78. data/lib/cosmo/web/controllers/overview.rb +88 -0
  79. data/lib/cosmo/web/controllers/processes.rb +36 -0
  80. data/lib/cosmo/web/controllers/streams.rb +38 -28
  81. data/lib/cosmo/web/helpers/application.rb +101 -0
  82. data/lib/cosmo/web/renderer.rb +36 -6
  83. data/lib/cosmo/web/throughput.rb +104 -0
  84. data/lib/cosmo/web/views/batches/_table.erb +42 -54
  85. data/lib/cosmo/web/views/batches/index.erb +3 -10
  86. data/lib/cosmo/web/views/crons/_table.erb +36 -55
  87. data/lib/cosmo/web/views/crons/index.erb +2 -9
  88. data/lib/cosmo/web/views/jobs/_args.erb +5 -0
  89. data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
  90. data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
  91. data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
  92. data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
  93. data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
  94. data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
  95. data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
  96. data/lib/cosmo/web/views/jobs/index.erb +4 -17
  97. data/lib/cosmo/web/views/layout.erb +73 -22
  98. data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
  99. data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
  100. data/lib/cosmo/web/views/metrics/_table.erb +24 -40
  101. data/lib/cosmo/web/views/metrics/index.erb +8 -16
  102. data/lib/cosmo/web/views/overview/_badges.erb +3 -0
  103. data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
  104. data/lib/cosmo/web/views/overview/_running.erb +18 -0
  105. data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
  106. data/lib/cosmo/web/views/overview/_workers.erb +25 -0
  107. data/lib/cosmo/web/views/overview/index.erb +28 -0
  108. data/lib/cosmo/web/views/processes/_table.erb +67 -0
  109. data/lib/cosmo/web/views/processes/index.erb +1 -0
  110. data/lib/cosmo/web/views/shared/_pager.erb +22 -0
  111. data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
  112. data/lib/cosmo/web/views/streams/_info.erb +29 -86
  113. data/lib/cosmo/web/views/streams/_list.erb +7 -0
  114. data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
  115. data/lib/cosmo/web/views/streams/_stream.erb +53 -0
  116. data/lib/cosmo/web/views/streams/index.erb +3 -11
  117. data/lib/cosmo/web/views/streams/info.erb +3 -11
  118. data/lib/cosmo/web.rb +73 -35
  119. data/lib/cosmo.rb +33 -0
  120. data/sig/cosmo/active_job/adapter.rbs +3 -3
  121. data/sig/cosmo/active_job/options.rbs +4 -2
  122. data/sig/cosmo/api/batch.rbs +5 -3
  123. data/sig/cosmo/api/counter.rbs +16 -6
  124. data/sig/cosmo/api/cron/entry.rbs +6 -3
  125. data/sig/cosmo/api/cron.rbs +10 -4
  126. data/sig/cosmo/api/kv.rbs +21 -4
  127. data/sig/cosmo/api/registry.rbs +10 -0
  128. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  129. data/sig/cosmo/api/stats/metrics.rbs +4 -1
  130. data/sig/cosmo/api/stats/processes.rbs +9 -1
  131. data/sig/cosmo/api/stats.rbs +12 -2
  132. data/sig/cosmo/api/stream.rbs +12 -5
  133. data/sig/cosmo/batch/callback.rbs +1 -1
  134. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  135. data/sig/cosmo/batch.rbs +5 -5
  136. data/sig/cosmo/cli.rbs +2 -0
  137. data/sig/cosmo/client.rbs +24 -4
  138. data/sig/cosmo/config/settings.rbs +6 -0
  139. data/sig/cosmo/config.rbs +5 -5
  140. data/sig/cosmo/engine.rbs +5 -0
  141. data/sig/cosmo/heartbeat.rbs +2 -0
  142. data/sig/cosmo/job/data.rbs +3 -1
  143. data/sig/cosmo/job/enqueuer.rbs +12 -0
  144. data/sig/cosmo/job/processor.rbs +18 -3
  145. data/sig/cosmo/job/setter.rbs +20 -0
  146. data/sig/cosmo/job.rbs +10 -4
  147. data/sig/cosmo/logger.rbs +5 -3
  148. data/sig/cosmo/middleware.rbs +0 -8
  149. data/sig/cosmo/processor.rbs +14 -2
  150. data/sig/cosmo/publisher.rbs +1 -0
  151. data/sig/cosmo/stream/processor.rbs +4 -0
  152. data/sig/cosmo/stream.rbs +4 -3
  153. data/sig/cosmo/utils/hash.rbs +2 -1
  154. data/sig/cosmo/utils/signal.rbs +5 -5
  155. data/sig/cosmo/utils/string.rbs +2 -2
  156. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  157. data/sig/cosmo/web/cache.rbs +35 -0
  158. data/sig/cosmo/web/chart.rbs +48 -0
  159. data/sig/cosmo/web/circuit.rbs +29 -0
  160. data/sig/cosmo/web/context.rbs +15 -0
  161. data/sig/cosmo/web/controllers/application.rbs +32 -0
  162. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  163. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  164. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  165. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  166. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  167. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  168. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  169. data/sig/cosmo/web/helpers/application.rbs +61 -0
  170. data/sig/cosmo/web/renderer.rbs +33 -0
  171. data/sig/cosmo/web/throughput.rbs +45 -0
  172. data/sig/cosmo/web.rbs +25 -0
  173. data/sig/cosmo.rbs +19 -0
  174. metadata +53 -23
  175. data/lib/cosmo/api/stats/busy.rb +0 -63
  176. data/lib/cosmo/middleware/busy.rb +0 -15
  177. data/lib/cosmo/utils/overrides.rb +0 -206
  178. data/lib/cosmo/utils/warnings.rb +0 -17
  179. data/lib/cosmo/web/controllers/actions.rb +0 -16
  180. data/lib/cosmo/web/views/actions/index.erb +0 -7
  181. data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
  182. data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
  183. data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
  184. data/lib/cosmo/web/views/jobs/busy.erb +0 -18
  185. data/lib/cosmo/web/views/jobs/dead.erb +0 -18
  186. data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
  187. data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
  188. data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
  189. data/lib/cosmo/web/views/streams/_table.erb +0 -25
  190. data/sig/cosmo/api/stats/busy.rbs +0 -31
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "cosmo/api/stats/totals"
4
- require "cosmo/api/stats/busy"
5
4
  require "cosmo/api/stats/processes"
5
+ require "cosmo/api/stats/jobs"
6
6
  require "cosmo/api/stats/metrics"
7
7
 
8
8
  module Cosmo
@@ -10,37 +10,88 @@ module Cosmo
10
10
  module Stats
11
11
  module_function
12
12
 
13
+ # @return [Hash{Symbol => Integer}] the dashboard counts: +:processed+, +:failed+, +:busy+, +:enqueued+,
14
+ # +:retries+, +:scheduled+ and +:dead+
13
15
  def summary
14
- { processed:, failed:, busy:, enqueued:, retries:, scheduled:, dead: }
16
+ jobs = Stream.jobs
17
+ { processed:, failed:, busy:, enqueued: enqueued(jobs), retries: retries(jobs), scheduled:, dead: }
15
18
  end
16
19
 
20
+ # @return [Integer] jobs run successfully, ever
17
21
  def processed
18
22
  Totals.instance.processed
19
23
  end
20
24
 
25
+ # @return [Integer] failed job attempts, ever
21
26
  def failed
22
27
  Totals.instance.failed
23
28
  end
24
29
 
30
+ # @param limit [Integer]
31
+ # @return [Array<Hash>] up to +limit+ of the jobs running right now, which ones is not defined, each with the
32
+ # +:worker+ running it; up to Heartbeat::INTERVAL old, and missing a job shorter than that
33
+ def running(limit)
34
+ Jobs.instance.sample(limit)
35
+ end
36
+
37
+ # @param page [Integer]
38
+ # @param limit [Integer]
39
+ # @param longest [Boolean] longest running first, which reads every running job (at most once per
40
+ # Jobs::LONGEST_TTL) instead of only the page
41
+ # @return [Array<Hash>] the jobs running right now on a page, in stream, worker and job id order unless
42
+ # +longest+, each with the +:worker+ running it; see {.busy} for how many there are
43
+ def running_page(page:, limit:, longest: false)
44
+ longest ? Jobs.instance.longest(page:, limit:) : Jobs.instance.list(page:, limit:)
45
+ end
46
+
47
+ # @return [Integer] jobs running right now across all workers, counted by the server
25
48
  def busy
26
- Busy.instance.size
49
+ Jobs.instance.count
50
+ end
51
+
52
+ # @param streams [Array<Stream>] the job streams
53
+ # @return [Hash{String => Integer}] the number of jobs running per stream
54
+ def busy_streams(streams = Stream.jobs)
55
+ streams.to_h { [_1.name, Jobs.instance.count(_1.name)] }
27
56
  end
28
57
 
29
- def enqueued
30
- Stream.jobs.sum(&:size)
58
+ # @param streams [Array<Stream>] the job streams
59
+ # @return [Integer]
60
+ def enqueued(streams = Stream.jobs)
61
+ streams.sum(&:size)
31
62
  end
32
63
 
33
- def retries
34
- Stream.jobs.sum(&:retries)
64
+ # @param streams [Array<Stream>] the job streams
65
+ # @return [Integer]
66
+ def retries(streams = Stream.jobs)
67
+ streams.sum(&:retries)
35
68
  end
36
69
 
70
+ # @return [Integer] messages in the +scheduled+ stream, 0 while scheduling is off
37
71
  def scheduled
38
72
  Config.scheduled.enabled ? Stream.new("scheduled").size : 0
39
73
  end
40
74
 
75
+ # @return [Integer] messages in the +dead+ stream, 0 while it is off
41
76
  def dead
42
77
  Config.dead.enabled ? Stream.new("dead").size : 0
43
78
  end
79
+
80
+ # @return [Array<Hash>] a {.queue} row per job stream
81
+ def queues
82
+ streams = Stream.jobs
83
+ busy = busy_streams(streams)
84
+ streams.map { queue(_1, busy) }
85
+ end
86
+
87
+ # @param stream [Stream] a job stream
88
+ # @param busy [Hash{String => Integer}] running jobs per stream, as {.busy_streams} counts them
89
+ # @return [Hash] the stream's +:name+, waiting messages as +:size+, running jobs as +:busy+, redeliveries as
90
+ # +:retries+, and whether it is +:paused+
91
+ def queue(stream, busy = busy_streams([stream]))
92
+ { name: stream.name, size: stream.size, busy: busy.fetch(stream.name, 0), retries: stream.retries,
93
+ paused: stream.paused? }
94
+ end
44
95
  end
45
96
  end
46
97
  end
@@ -9,46 +9,68 @@ module Cosmo
9
9
 
10
10
  include Enumerable
11
11
 
12
+ # @return [Array<Stream>] every stream, each answering #info from this one listing
12
13
  def self.all
13
- client.list_streams.map { new(_1.dig("config", "name")) }
14
+ client.list_streams.map { new(_1.config.name, _1) }
14
15
  end
15
16
 
17
+ # @return [Array<Stream>] the job streams, each answering #info from this one listing
16
18
  def self.jobs
17
- client.list_streams.select { _1.dig("config", "metadata", "_cosmo.type") == "jobs" }
18
- .reject { %w[scheduled dead].include?(_1.dig("config", "name")) }
19
- .map { new(_1.dig("config", "name")) }
19
+ client.list_streams.select { _1.config.metadata&.[](:"_cosmo.type") == "jobs" }
20
+ .reject { %w[scheduled dead].include?(_1.config.name) }
21
+ .map { new(_1.config.name, _1) }
20
22
  end
21
23
 
24
+ # @return [Client]
22
25
  def self.client
23
26
  @client ||= Client.instance
24
27
  end
25
28
 
26
29
  attr_reader :name
27
30
 
28
- def initialize(name)
31
+ # @param name [String]
32
+ # @param info [NATS::JetStream::API::StreamInfo, nil] the stream's info from a listing, kept instead of fetched
33
+ def initialize(name, info = nil)
29
34
  @name = name
35
+ @info = info
30
36
  end
31
37
 
38
+ # @return [Hash] the stream's +:state+ and +:config+ as JetStream reports them, from the listing the stream
39
+ # came from or else fetched fresh on every call
32
40
  def info
33
- info = client.stream_info(name)
41
+ info = @info || client.stream_info(name)
34
42
  { state: info.state, config: info.config }
35
43
  end
36
44
 
45
+ # @return [Integer] the stream's messages, not counting cron schedule templates; 0 when NATS can't be read
37
46
  def total
38
- all_msgs = info[:state].messages.to_i
39
- cron_count = Client.instance.cron_subjects_in_stream(name, "#{Cron::Entry::SUBJECT_PREFIX}.>").size
40
- [all_msgs - cron_count, 0].max
47
+ state, config = info.values_at(:state, :config)
48
+ messages = state.messages.to_i
49
+ return messages unless Array(config.subjects).any? { _1.start_with?(Cron::Entry::SUBJECT_PREFIX) }
50
+
51
+ cron_count = client.cron_subjects_in_stream(name, "#{Cron::Entry::SUBJECT_PREFIX}.>").size
52
+ [messages - cron_count, 0].max
41
53
  rescue NATS::Error
42
54
  0
43
55
  end
44
56
  alias size total
45
57
 
58
+ # @return [Integer] redeliveries across the stream's consumers
46
59
  def retries
47
- client.list_consumers(name).sum { _1["num_redelivered"].to_i }
60
+ consumers.sum { _1["num_redelivered"].to_i }
61
+ end
62
+
63
+ # @return [Array<Hash>] the stream's consumers as JetStream reports them, with their config and delivery state
64
+ def consumers
65
+ client.list_consumers(name)
48
66
  rescue NATS::Error
49
- 0
67
+ []
50
68
  end
51
69
 
70
+ # Walks the stream's job messages in sequence order, from {#offset} or the first message.
71
+ #
72
+ # @yieldparam job [Job]
73
+ # @return [void]
52
74
  def each
53
75
  return if total.zero?
54
76
 
@@ -68,26 +90,27 @@ module Cosmo
68
90
  end
69
91
  end
70
92
 
93
+ # Sets the sequence {#each} starts at.
94
+ #
95
+ # @param value [Integer]
96
+ # @return [self]
71
97
  def offset(value)
72
98
  @offset = value.to_i
73
99
  self
74
100
  end
75
101
 
76
- def messages(page: nil, limit: nil)
77
- jobs = []
102
+ # @param page [Integer, nil] 1-based page, counted in sequences from the first message (or the last, for +:desc+)
103
+ # @param limit [Integer, nil] jobs per page, {LIMIT} when nil
104
+ # @param order [Symbol] +:asc+ for oldest first, +:desc+ for newest first
105
+ # @return [Array<Job>]
106
+ def messages(page: nil, limit: nil, order: :asc)
78
107
  limit = (limit || LIMIT).to_i
79
- state = info[:state]
80
- start = state.first_seq.to_i
81
- start += (page.to_i - 1) * limit if page
82
-
83
- offset(start).each do |message|
84
- jobs << message
85
- break if jobs.size >= limit
86
- end
87
-
88
- jobs
108
+ page = [page.to_i, 1].max
109
+ order == :desc ? messages_desc(page, limit) : messages_asc(page, limit)
89
110
  end
90
111
 
112
+ # @param seq [Integer]
113
+ # @return [Job, nil] nil when the message is gone or is a cron schedule template
91
114
  def message(seq)
92
115
  job = Job.new(name, client.get_message(name, seq: seq, direct: true))
93
116
  return if job.subject.to_s.start_with?(Cron::Entry::SUBJECT_PREFIX)
@@ -97,6 +120,10 @@ module Cosmo
97
120
  # nop, acked/nacked
98
121
  end
99
122
 
123
+ # Republishes the job at +seq+ to its +X-Subject+ and deletes it here.
124
+ #
125
+ # @param seq [Integer]
126
+ # @return [void]
100
127
  def retry(seq)
101
128
  job = message(seq)
102
129
  return unless job
@@ -105,24 +132,46 @@ module Cosmo
105
132
  delete(seq)
106
133
  end
107
134
 
135
+ # @param seq [Integer]
136
+ # @return [Hash] the JetStream API response
108
137
  def delete(seq)
109
138
  client.delete_message(name, seq)
110
139
  end
111
140
 
141
+ # Marks the stream paused, so workers stop fetching from it.
142
+ #
143
+ # @return [void]
112
144
  def pause!
113
145
  client.pause_stream(name)
114
146
  end
115
147
 
148
+ # @return [void]
116
149
  def unpause!
117
150
  client.unpause_stream(name)
118
151
  end
119
152
 
153
+ # @return [Boolean] false when the server doesn't answer in time
120
154
  def paused?
121
- client.stream_paused?(name)
155
+ info[:config].metadata&.[](:"_cosmo.paused") == "true"
156
+ rescue NATS::IO::Timeout
157
+ false
122
158
  end
123
159
 
124
160
  private
125
161
 
162
+ # Page +page+ holds the first +limit+ jobs from +(page - 1) * limit+ sequences after the first one.
163
+ def messages_asc(page, limit)
164
+ offset(info[:state].first_seq.to_i + ((page - 1) * limit)).first(limit)
165
+ end
166
+
167
+ # Page +page+ covers the +limit+ sequences ending +(page - 1) * limit+ below the last one, read forward and
168
+ # reversed, as direct gets can't walk a stream backwards.
169
+ def messages_desc(page, limit)
170
+ state = info[:state]
171
+ upto = state.last_seq.to_i - ((page - 1) * limit)
172
+ offset([upto - limit + 1, state.first_seq.to_i].max).take_while { _1.seq.to_i <= upto }.reverse
173
+ end
174
+
126
175
  def scan_range
127
176
  data = info
128
177
  state = data[:state]
@@ -8,6 +8,11 @@ module Cosmo
8
8
  class Callback
9
9
  include Cosmo::Job
10
10
 
11
+ # @param class_name [String] the class given to {Batch#on}
12
+ # @param event [String, Symbol] +success+ or +complete+
13
+ # @param status [Hash] +bid+, +total+, +succeeded+, and +failed+
14
+ # @param opts [Hash] the options given to {Batch#on}
15
+ # @return [void]
11
16
  def perform(class_name, event, status, opts)
12
17
  klass = Utils::String.safe_constantize(class_name)
13
18
  klass&.new&.public_send("on_#{event}", status, opts)
@@ -11,8 +11,8 @@ module Cosmo
11
11
  # finalize the batch. This never touches total or failed - that's what
12
12
  # lets #jobs release its placeholder slot without messing up the stats
13
13
  # we report back.
14
- def release_pending(bid, msg_id: nil)
15
- pending = counter.decrement("#{bid}.pending", msg_id: msg_id)
14
+ def release_pending(bid, by: 1, msg_id: nil)
15
+ pending = counter.decrement!("#{bid}.pending", by: by, msg_id: msg_id)
16
16
  finalize(bid) if pending.to_i.zero?
17
17
  end
18
18
 
@@ -51,8 +51,8 @@ module Cosmo
51
51
 
52
52
  # Adds this batch as one job of the parent batch.
53
53
  def link(parent_id)
54
- counter.increment("#{parent_id}.total")
55
- counter.increment("#{parent_id}.pending")
54
+ counter.increment!("#{parent_id}.total")
55
+ counter.increment!("#{parent_id}.pending")
56
56
  end
57
57
 
58
58
  # Fires the callback for +event+, if one is registered and the batch is
data/lib/cosmo/batch.rb CHANGED
@@ -87,6 +87,7 @@ module Cosmo
87
87
  BUCKET = "_cosmobatches"
88
88
  EVENTS = %i[success complete].freeze
89
89
 
90
+ # @return [Batch, nil] the batch whose #jobs block runs on this thread
90
91
  def self.current
91
92
  Thread.current[:cosmo_batch]
92
93
  end
@@ -94,17 +95,24 @@ module Cosmo
94
95
  # Called by Job::Processor when a job is done for good - acked, or
95
96
  # dead-lettered/dropped. Never call this for a retry. JID is used to dedupe: if the same
96
97
  # job's ack gets lost, and it's redelivered, we don't want to count it twice.
98
+ #
99
+ # @param bid [String]
100
+ # @param jid [String]
101
+ # @param success [Boolean] false when the job was dead-lettered or dropped
102
+ # @return [void]
97
103
  def self.notify(bid, jid, success:)
98
- counter.increment("#{bid}.failed", msg_id: "batch.#{bid}.#{jid}.failed") unless success
104
+ counter.increment!("#{bid}.failed", msg_id: "batch.#{bid}.#{jid}.failed") unless success
99
105
  release_pending(bid, msg_id: "batch.#{bid}.#{jid}.pending")
100
106
  end
101
107
 
108
+ # @return [Counters]
102
109
  def self.counter
103
110
  @counter ||= Counters.new("batch")
104
111
  end
105
112
 
113
+ # @return [API::KV] the bucket holding every batch's metadata, callbacks, and results
106
114
  def self.kv
107
- @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config.batches.expiry), replicas: Config.replicas)
115
+ @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config.batches.expiry), replicas: Config.services.num_replicas)
108
116
  end
109
117
 
110
118
  attr_reader :bid, :parent_id
@@ -123,10 +131,13 @@ module Cosmo
123
131
  # toward total, so it doesn't skew the stats. Safe to call more than
124
132
  # once. If the block raises, we still release the placeholder (so the
125
133
  # batch isn't stuck) and let the error propagate as normal.
134
+ #
135
+ # @yieldreturn [Object]
136
+ # @return [Object] the block's result
126
137
  def jobs
127
138
  previous = Thread.current[:cosmo_batch]
128
139
  Thread.current[:cosmo_batch] = self
129
- counter.increment("#{bid}.pending")
140
+ counter.increment!("#{bid}.pending")
130
141
  begin
131
142
  yield
132
143
  ensure
@@ -138,6 +149,12 @@ module Cosmo
138
149
  # Registers a callback for +event+ (:success or :complete). Can be
139
150
  # called before or after #jobs - if the batch already finished, it
140
151
  # fires right away.
152
+ #
153
+ # @param event [Symbol] +:success+ or +:complete+
154
+ # @param klass [Class] a class whose instances respond to +on_success+/+on_complete+ with +(status, options)+
155
+ # @param opts [Hash] passed to the callback as +options+
156
+ # @return [void]
157
+ # @raise [ArgumentError] for any other event
141
158
  def on(event, klass, **opts)
142
159
  raise ArgumentError, "event must be :success or :complete" unless EVENTS.include?(event)
143
160
 
@@ -145,18 +162,24 @@ module Cosmo
145
162
  try_fire(event)
146
163
  end
147
164
 
148
- # Called by Job#perform for each job added inside #jobs.
149
- def register_job!
150
- counter.increment("#{bid}.total")
151
- counter.increment("#{bid}.pending")
165
+ # Called by Job#perform for each job added inside #jobs, and once per chunk by perform_bulk.
166
+ # @param count [Integer] how many jobs join the batch
167
+ # @return [void]
168
+ def register_job!(count = 1)
169
+ counter.increment!("#{bid}.total", by: count)
170
+ counter.increment!("#{bid}.pending", by: count)
152
171
  end
153
172
 
154
173
  # Undoes #register_job! when the publish itself failed, so the job
155
174
  # never really joined the batch. Unlike a real completion, this also
156
175
  # reverts total, not just pending.
157
- def rollback_job!
158
- counter.decrement("#{bid}.total")
159
- release_pending
176
+ # @param count [Integer] how many registered jobs were not published
177
+ # @return [void]
178
+ def rollback_job!(count = 1)
179
+ return unless count.positive?
180
+
181
+ counter.decrement!("#{bid}.total", by: count)
182
+ release_pending(count)
160
183
  end
161
184
 
162
185
  private
@@ -165,7 +188,7 @@ module Cosmo
165
188
  def counter = self.class.counter
166
189
  def kv = self.class.kv
167
190
  def link(parent_id) = self.class.send(:link, parent_id)
168
- def release_pending = self.class.send(:release_pending, bid)
191
+ def release_pending(count = 1) = self.class.send(:release_pending, bid, by: count)
169
192
  def try_fire(event) = self.class.send(:try_fire, bid, event)
170
193
  end
171
194
  end
data/lib/cosmo/cli.rb CHANGED
@@ -7,10 +7,12 @@ require "fileutils"
7
7
  module Cosmo
8
8
  # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
9
9
  class CLI
10
+ # @return [void]
10
11
  def self.run
11
12
  instance.run
12
13
  end
13
14
 
15
+ # @return [CLI]
14
16
  def self.instance
15
17
  @instance ||= new
16
18
  end
@@ -18,6 +20,9 @@ module Cosmo
18
20
  # @return [Array<String>, nil] the arguments the process was started with, before parsing consumed them
19
21
  attr_reader :argv
20
22
 
23
+ # Parses the command line, then runs +--init+ or +--setup+, or loads the config and the app and starts the {Engine}.
24
+ #
25
+ # @return [void]
21
26
  def run
22
27
  @argv = ARGV.dup
23
28
  flags, command, options = parse
@@ -252,6 +257,7 @@ module Cosmo
252
257
  require path
253
258
  end
254
259
 
260
+ # @return [String] the logo printed on start
255
261
  # rubocop:disable-next Layout/TrailingWhitespace,Lint/IneffectiveAccessModifier
256
262
  def self.banner
257
263
  <<-TEXT