cosmonats 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -13
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +51 -3
  8. data/lib/cosmo/api/cron/entry.rb +24 -9
  9. data/lib/cosmo/api/cron.rb +21 -7
  10. data/lib/cosmo/api/job.rb +14 -0
  11. data/lib/cosmo/api/kv.rb +138 -9
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +18 -7
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +47 -11
  18. data/lib/cosmo/api/stream.rb +66 -22
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +208 -11
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +11 -0
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +51 -15
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/version.rb +1 -1
  62. data/lib/cosmo/web/assets/app.css +5 -3
  63. data/lib/cosmo/web/cache.rb +2 -0
  64. data/lib/cosmo/web/chart.rb +2 -2
  65. data/lib/cosmo/web/circuit.rb +1 -0
  66. data/lib/cosmo/web/context.rb +6 -0
  67. data/lib/cosmo/web/controllers/application.rb +26 -0
  68. data/lib/cosmo/web/controllers/batches.rb +7 -1
  69. data/lib/cosmo/web/controllers/crons.rb +16 -2
  70. data/lib/cosmo/web/controllers/jobs.rb +45 -16
  71. data/lib/cosmo/web/controllers/metrics.rb +5 -0
  72. data/lib/cosmo/web/controllers/overview.rb +36 -18
  73. data/lib/cosmo/web/controllers/processes.rb +8 -5
  74. data/lib/cosmo/web/controllers/streams.rb +12 -0
  75. data/lib/cosmo/web/helpers/application.rb +44 -0
  76. data/lib/cosmo/web/renderer.rb +17 -0
  77. data/lib/cosmo/web/throughput.rb +1 -1
  78. data/lib/cosmo/web/views/crons/_table.erb +29 -27
  79. data/lib/cosmo/web/views/jobs/_args.erb +2 -2
  80. data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
  81. data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
  82. data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
  83. data/lib/cosmo/web/views/layout.erb +1 -1
  84. data/lib/cosmo/web/views/overview/_badges.erb +1 -1
  85. data/lib/cosmo/web/views/overview/_running.erb +3 -3
  86. data/lib/cosmo/web/views/overview/index.erb +1 -1
  87. data/lib/cosmo/web/views/streams/_info.erb +1 -0
  88. data/lib/cosmo/web/views/streams/_stream.erb +1 -0
  89. data/lib/cosmo/web.rb +8 -1
  90. data/lib/cosmo.rb +33 -0
  91. data/sig/cosmo/active_job/adapter.rbs +3 -3
  92. data/sig/cosmo/active_job/options.rbs +4 -2
  93. data/sig/cosmo/api/batch.rbs +5 -3
  94. data/sig/cosmo/api/counter.rbs +16 -6
  95. data/sig/cosmo/api/cron/entry.rbs +4 -2
  96. data/sig/cosmo/api/cron.rbs +10 -4
  97. data/sig/cosmo/api/kv.rbs +21 -4
  98. data/sig/cosmo/api/registry.rbs +10 -0
  99. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  100. data/sig/cosmo/api/stats/metrics.rbs +3 -0
  101. data/sig/cosmo/api/stats/processes.rbs +9 -1
  102. data/sig/cosmo/api/stats.rbs +8 -2
  103. data/sig/cosmo/api/stream.rbs +10 -5
  104. data/sig/cosmo/batch/callback.rbs +1 -1
  105. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  106. data/sig/cosmo/batch.rbs +5 -5
  107. data/sig/cosmo/cli.rbs +2 -0
  108. data/sig/cosmo/client.rbs +23 -4
  109. data/sig/cosmo/config/settings.rbs +6 -0
  110. data/sig/cosmo/config.rbs +5 -5
  111. data/sig/cosmo/engine.rbs +5 -0
  112. data/sig/cosmo/heartbeat.rbs +2 -0
  113. data/sig/cosmo/job/data.rbs +3 -1
  114. data/sig/cosmo/job/enqueuer.rbs +12 -0
  115. data/sig/cosmo/job/processor.rbs +18 -3
  116. data/sig/cosmo/job/setter.rbs +20 -0
  117. data/sig/cosmo/job.rbs +10 -4
  118. data/sig/cosmo/logger.rbs +5 -3
  119. data/sig/cosmo/middleware.rbs +0 -8
  120. data/sig/cosmo/processor.rbs +14 -2
  121. data/sig/cosmo/publisher.rbs +1 -0
  122. data/sig/cosmo/stream/processor.rbs +4 -0
  123. data/sig/cosmo/stream.rbs +4 -3
  124. data/sig/cosmo/utils/hash.rbs +2 -1
  125. data/sig/cosmo/utils/signal.rbs +5 -5
  126. data/sig/cosmo/utils/string.rbs +2 -2
  127. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  128. data/sig/cosmo/web/cache.rbs +35 -0
  129. data/sig/cosmo/web/chart.rbs +48 -0
  130. data/sig/cosmo/web/circuit.rbs +29 -0
  131. data/sig/cosmo/web/context.rbs +15 -0
  132. data/sig/cosmo/web/controllers/application.rbs +32 -0
  133. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  134. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  135. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  136. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  137. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  138. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  139. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  140. data/sig/cosmo/web/helpers/application.rbs +61 -0
  141. data/sig/cosmo/web/renderer.rbs +33 -0
  142. data/sig/cosmo/web/throughput.rbs +45 -0
  143. data/sig/cosmo/web.rbs +25 -0
  144. data/sig/cosmo.rbs +19 -0
  145. metadata +23 -6
  146. data/lib/cosmo/api/stats/busy.rb +0 -68
  147. data/lib/cosmo/middleware/busy.rb +0 -15
  148. data/sig/cosmo/api/stats/busy.rbs +0 -33
data/lib/cosmo/client.rb CHANGED
@@ -12,7 +12,9 @@ module Cosmo
12
12
  # Reconnect attempts before the connection closes for good; -1 never gives up, so a process outlives a NATS
13
13
  # outage. nats-pure lets NATS_MAX_RECONNECT_ATTEMPTS override it.
14
14
  MAX_RECONNECT_ATTEMPTS = -1
15
+ DIRECT_BATCH = 1000
15
16
 
17
+ # @return [Client]
16
18
  def self.instance
17
19
  @instance ||= Client.new
18
20
  end
@@ -21,19 +23,24 @@ module Cosmo
21
23
  # misbehaving process can be found without cross-referencing host IPs. Derived from
22
24
  # the running program, which separates a web process from a `cosmo` worker on the
23
25
  # same host. Override with COSMO_CLIENT_NAME.
26
+ #
27
+ # @return [String]
24
28
  def self.default_name
25
29
  ENV.fetch("COSMO_CLIENT_NAME") { "cosmo-#{program_name}-#{Socket.gethostname}-#{Process.pid}" }
26
30
  end
27
31
 
32
+ # @return [Integer] seconds to wait for a JetStream API reply
28
33
  def self.js_timeout
29
34
  ENV.fetch("COSMO_JS_TIMEOUT", JS_TIMEOUT).to_i
30
35
  end
31
36
 
37
+ # @return [Integer] seconds to wait for the connection to the server
32
38
  def self.connect_timeout
33
39
  ENV.fetch("COSMO_CONNECT_TIMEOUT", CONNECT_TIMEOUT).to_i
34
40
  end
35
41
 
36
42
  # $PROGRAM_NAME is "puma 7.2.1 (tcp://...)" under Puma and a path under the CLI.
43
+ # @return [String]
37
44
  def self.program_name
38
45
  File.basename($PROGRAM_NAME.to_s.split.first.to_s, ".*").gsub(/[^\w.-]/, "")
39
46
  end
@@ -49,31 +56,66 @@ module Cosmo
49
56
  @js = @nc.jetstream(timeout: self.class.js_timeout)
50
57
  end
51
58
 
59
+ # @param subject [String]
60
+ # @param payload [String]
61
+ # @param params [Hash] JetStream publish options, e.g. +header:+
62
+ # @return [NATS::JetStream::PubAck]
52
63
  def publish(subject, payload, **params)
53
64
  js.publish(subject, payload, **params)
54
65
  end
55
66
 
67
+ # Publishes every message before waiting for any ack, so the batch costs about one round trip instead of one each.
68
+ # Unlike #publish, a message the stream refuses doesn't raise: its error takes the ack's place in the result.
69
+ #
70
+ # @param messages [Array<Hash>] each with +:subject+, +:payload+ and +:header+
71
+ # @return [Array<NATS::JetStream::PubAck, StandardError>] each message's ack, or why it was not stored, in order;
72
+ # +NATS::Timeout+ for one whose ack didn't arrive within the JetStream timeout
73
+ def publish_many(messages)
74
+ inbox = nc.new_inbox
75
+ subscription = nc.subscribe("#{inbox}.*")
76
+ messages.each_with_index { |m, i| nc.publish(m[:subject], m[:payload], "#{inbox}.#{i}", header: m[:header]) }
77
+ nc.flush(self.class.js_timeout)
78
+
79
+ acks(subscription, messages.size)
80
+ ensure
81
+ subscription&.unsubscribe
82
+ end
83
+
56
84
  # Create a pull subscription. Durable with +consumer_name+, and ephemeral without.
57
85
  # @param config [Hash] Consumer config. Ephemeral consumers additionally require
58
86
  # +:stream+ and +:inactive_threshold+ (seconds the consumer survives without a fetch).
87
+ # @param subject [String, Array<String>]
88
+ # @param consumer_name [String, nil]
89
+ # @return [NATS::JetStream::PullSubscription]
59
90
  def subscribe(subject, consumer_name, config)
60
91
  return js.pull_subscribe(subject, consumer_name, config: config) if consumer_name
61
92
 
62
93
  ephemeral_subscribe(subject, config)
63
94
  end
64
95
 
96
+ # @param name [String, Symbol]
97
+ # @return [NATS::JetStream::API::StreamInfo]
65
98
  def stream_info(name)
66
99
  js.stream_info(name)
67
100
  end
68
101
 
102
+ # @param name [String, Symbol]
103
+ # @param config [Hash]
104
+ # @return [NATS::JetStream::API::StreamCreateResponse]
69
105
  def create_stream(name, config)
70
106
  js.add_stream(name: name, **config)
71
107
  end
72
108
 
109
+ # @param name [String, Symbol]
110
+ # @param params [Hash]
111
+ # @return [Boolean]
73
112
  def delete_stream(name, params = {})
74
113
  js.delete_stream(name, params)
75
114
  end
76
115
 
116
+ # @param name [String, Symbol]
117
+ # @param config [Hash]
118
+ # @return [NATS::JetStream::API::StreamCreateResponse]
77
119
  def update_stream(name, config)
78
120
  js.update_stream(name: name, **config)
79
121
  end
@@ -81,6 +123,7 @@ module Cosmo
81
123
  # Create/update a stream, falling back to create when there's no stream.
82
124
  # @param name [String] Stream name
83
125
  # @param config [Hash] Full desired stream configuration
126
+ # @return [NATS::JetStream::API::StreamCreateResponse]
84
127
  def setup_stream(name, config)
85
128
  update_stream(name, config)
86
129
  rescue NATS::JetStream::Error::StreamNotFound
@@ -89,6 +132,8 @@ module Cosmo
89
132
 
90
133
  # Return all subjects in +stream_name+ that match +filter+ using NATS's
91
134
  # subjects_filter on STREAM.INFO (requires NATS ≥ 2.9).
135
+ # @param stream_name [String]
136
+ # @param filter [String]
92
137
  # @return [Array<String>]
93
138
  def cron_subjects_in_stream(stream_name, filter)
94
139
  payload = Utils::Json.dump({ subjects_filter: filter })
@@ -99,16 +144,22 @@ module Cosmo
99
144
  []
100
145
  end
101
146
 
147
+ # @return [Array<NATS::JetStream::API::StreamInfo>] every stream's info, as {#stream_info} returns it; empty when
148
+ # the request fails
102
149
  def list_streams
103
150
  response = nc.request("$JS.API.STREAM.LIST", "")
104
- data = Utils::Json.parse(response.data, symbolize_names: false)
105
- return [] if data.nil? || data["streams"].nil?
151
+ data = Utils::Json.parse(response.data)
152
+ return [] if data.nil? || data[:streams].nil?
106
153
 
107
- data["streams"]
154
+ data[:streams].map { NATS::JetStream::API::StreamInfo.new(_1) }
108
155
  rescue NATS::Error
109
156
  []
110
157
  end
111
158
 
159
+ # Marks the stream paused in its metadata, so workers stop fetching from it.
160
+ #
161
+ # @param name [String]
162
+ # @return [void]
112
163
  def pause_stream(name)
113
164
  config = stream_info(name).config.to_h
114
165
  config[:metadata] ||= {}
@@ -116,6 +167,8 @@ module Cosmo
116
167
  update_stream(name, config)
117
168
  end
118
169
 
170
+ # @param name [String]
171
+ # @return [void]
119
172
  def unpause_stream(name)
120
173
  config = stream_info(name).config.to_h
121
174
  config[:metadata] ||= {}
@@ -123,47 +176,65 @@ module Cosmo
123
176
  update_stream(name, config)
124
177
  end
125
178
 
179
+ # @param name [String]
180
+ # @return [Boolean] false when the server doesn't answer in time
126
181
  def stream_paused?(name)
127
182
  stream_info(name).config.metadata&.[](:"_cosmo.paused") == "true"
128
183
  rescue NATS::IO::Timeout
129
184
  false
130
185
  end
131
186
 
187
+ # @param stream_name [String]
188
+ # @return [Array<Hash>] the stream's consumers as JetStream reports them
132
189
  def list_consumers(stream_name)
133
190
  response = nc.request("$JS.API.CONSUMER.LIST.#{stream_name}", "")
134
191
  data = Utils::Json.parse(response.data, default: {}, symbolize_names: false)
135
192
  Array(data["consumers"])
136
193
  end
137
194
 
195
+ # @param stream_name [String]
196
+ # @param consumer_name [String]
197
+ # @return [NATS::JetStream::API::ConsumerInfo]
138
198
  def consumer_info(stream_name, consumer_name)
139
199
  js.consumer_info(stream_name, consumer_name)
140
200
  end
141
201
 
202
+ # @param stream_name [String]
203
+ # @param consumer_name [String]
204
+ # @return [Boolean]
142
205
  def delete_consumer(stream_name, consumer_name)
143
206
  js.delete_consumer(stream_name, consumer_name)
144
207
  end
145
208
 
209
+ # @param stream_name [String, Symbol]
210
+ # @param options [Hash] e.g. +seq:+, or +subject:+ for the last message on it
211
+ # @return [NATS::JetStream::API::RawStreamMsg]
146
212
  def get_message(stream_name, **options)
147
213
  js.get_msg(stream_name, **options)
148
214
  end
149
215
 
150
216
  # The last message on every subject matching +filters+, in one batched direct get (NATS ≥ 2.11). Needs
151
- # +allow_direct+ on the stream.
217
+ # +allow_direct+ on the stream. NATS refuses to match more than 1024 subjects: use {#direct_messages} for those.
152
218
  #
153
219
  # @param stream_name [String]
154
220
  # @param filters [Array<String>] subject filters, wildcards allowed
155
- # @return [Hash{String => String}] message data by subject
156
- def last_messages(stream_name, filters) # rubocop:disable Metrics/AbcSize
221
+ # @return [Hash{String => NATS::Msg}] the message by subject, its data and headers
222
+ # @raise [NATS::JetStream::Error] when the stream refuses the read, such as for matching too many subjects
223
+ def last_messages(stream_name, filters)
157
224
  inbox = nc.new_inbox
158
225
  sub = nc.subscribe(inbox)
159
226
  nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump({ multi_last: filters, batch: 10_000 }), inbox)
160
227
  messages = {}
161
228
  loop do
162
229
  msg = sub.next_msg(timeout: self.class.js_timeout)
163
- subject = msg.header&.dig(NATS::JetStream::Header::SUBJECT)
164
- break unless subject
165
-
166
- messages[subject] = msg.data
230
+ header = msg.header || {}
231
+ subject = header[NATS::JetStream::Header::SUBJECT]
232
+ unless subject
233
+ direct_pending(stream_name, header)
234
+ break
235
+ end
236
+
237
+ messages[subject] = msg
167
238
  end
168
239
  messages
169
240
  rescue NATS::Timeout
@@ -172,11 +243,83 @@ module Cosmo
172
243
  sub&.unsubscribe
173
244
  end
174
245
 
246
+ # Every message on +filter+ in stream order, read by batched direct gets (NATS ≥ 2.11) of up to
247
+ # DIRECT_BATCH messages each, so it has no subject cap. On a stream keeping one message per subject, such as
248
+ # a KV bucket, that is the last message on every subject, as {#last_messages} reads them. Needs +allow_direct+
249
+ # on the stream.
250
+ #
251
+ # @param stream_name [String]
252
+ # @param filter [String] a subject filter, wildcards allowed
253
+ # @param limit [Integer, nil] the most messages to read, nil for every one
254
+ # @param since [Time, nil] start at the first message stored at or after it, instead of the first one
255
+ # @return [Array<NATS::Msg>]
256
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
257
+ # @raise [NATS::Timeout] when a batch stops arriving, or never starts: no stream, or none with +allow_direct+
258
+ def direct_messages(stream_name, filter, limit: nil, since: nil)
259
+ messages = []
260
+ start = since ? { start_time: since.utc.iso8601(9) } : { seq: 1 }
261
+ loop do
262
+ batch = [DIRECT_BATCH, limit && (limit - messages.size)].compact.min
263
+ seq, pending = direct_batch(stream_name, { **start, next_by_subj: filter, batch: }, messages)
264
+ break if pending.zero? || (limit && messages.size >= limit)
265
+
266
+ start = { seq: }
267
+ end
268
+ messages
269
+ end
270
+
271
+ # How many messages match +filter+, answered by the server without sending any of them: a direct get of one
272
+ # message reports the rest as pending. On a stream keeping one message per subject, such as a KV bucket, that is
273
+ # the number of subjects. Needs +allow_direct+ on the stream.
274
+ #
275
+ # @param stream_name [String]
276
+ # @param filter [String] a subject filter, wildcards allowed
277
+ # @return [Integer]
278
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
279
+ # @raise [NATS::Timeout] when no stream with +allow_direct+ answers
280
+ def count_messages(stream_name, filter)
281
+ messages = []
282
+ _, pending = direct_batch(stream_name, { seq: 1, next_by_subj: filter, batch: 1 }, messages)
283
+ messages.size + pending
284
+ end
285
+
286
+ # The sorted subjects that hold a message and match +filter+, without the messages: the stream info's subject
287
+ # listing, which has no way to take a page of it, so read it once and page the result.
288
+ #
289
+ # @param stream_name [String]
290
+ # @param filter [String] a subject filter, wildcards allowed
291
+ # @return [Array<String>]
292
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
293
+ def subjects(stream_name, filter)
294
+ names = []
295
+ loop do
296
+ response = Utils::Json.parse(
297
+ nc.request("$JS.API.STREAM.INFO.#{stream_name}", Utils::Json.dump({ subjects_filter: filter, offset: names.size })).data,
298
+ default: {}, symbolize_names: false
299
+ )
300
+ raise NATS::JetStream::Error, response.dig("error", "description") if response["error"]
301
+
302
+ page = response.dig("state", "subjects")&.keys.to_a
303
+ names.concat(page)
304
+ break if page.empty? || names.size >= response["total"].to_i
305
+ end
306
+ names.sort
307
+ end
308
+
309
+ # @param name [String] the stream
310
+ # @param seq [Integer]
311
+ # @return [Hash] the JetStream API reply
175
312
  def delete_message(name, seq)
176
313
  response = nc.request("$JS.API.STREAM.MSG.DELETE.#{name}", JSON.dump({ seq: seq }))
177
314
  Utils::Json.parse(response.data, symbolize_names: false)
178
315
  end
179
316
 
317
+ # Purges +stream_name+, only the messages on +subject+ when given.
318
+ #
319
+ # @param stream_name [String]
320
+ # @param subject [String, nil]
321
+ # @return [Integer] how many messages were purged
322
+ # @raise [NATS::JetStream::Error] when JetStream rejects the purge
180
323
  def purge(stream_name, subject)
181
324
  payload = subject ? Utils::Json.dump({ filter: subject }) : ""
182
325
  response = @nc.request("$JS.API.STREAM.PURGE.#{stream_name}", payload)
@@ -186,22 +329,76 @@ module Cosmo
186
329
  result["purged"] # number of messages purged
187
330
  end
188
331
 
332
+ # Looks up the +name+ KV bucket, creating it with +options+ when it doesn't exist.
333
+ #
334
+ # @param name [String]
335
+ # @param allow_msg_ttl [Boolean] lets the bucket's messages carry their own +Nats-TTL+
336
+ # @param options [Hash] the bucket's config, used only on creation, e.g. +ttl:+
337
+ # @return [NATS::KeyValue]
189
338
  def kv(name, allow_msg_ttl: false, **options)
190
339
  js.key_value(name)
191
340
  rescue NATS::KeyValue::BucketNotFoundError
192
341
  allow_msg_ttl ? create_kv_with_msg_ttl(name, **options) : js.create_key_value({ bucket: name }.merge(options))
193
342
  end
194
343
 
344
+ # @return [void]
195
345
  def close
196
346
  nc.close
197
347
  end
198
348
 
199
349
  private
200
350
 
351
+ def direct_batch(stream_name, request, messages)
352
+ inbox = nc.new_inbox
353
+ sub = nc.subscribe(inbox)
354
+ nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump(request), inbox)
355
+ seq = request[:seq]
356
+ loop do
357
+ msg = sub.next_msg(timeout: self.class.js_timeout)
358
+ header = msg.header || {}
359
+ return [seq, direct_pending(stream_name, header)] unless header[NATS::JetStream::Header::SUBJECT]
360
+
361
+ messages << msg
362
+ seq = header[NATS::JetStream::Header::SEQUENCE].to_i + 1
363
+ end
364
+ ensure
365
+ sub&.unsubscribe
366
+ end
367
+
368
+ def direct_pending(stream_name, header)
369
+ return header["Nats-Num-Pending"].to_i if %w[204 404].include?(header["Status"])
370
+
371
+ raise NATS::JetStream::Error, "direct get on #{stream_name}: #{header["Status"]} #{header["Description"]}"
372
+ end
373
+
374
+ def acks(subscription, count)
375
+ results = Array.new(count)
376
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + self.class.js_timeout
377
+ count.times do
378
+ reply = next_reply(subscription, deadline) or break
379
+ results[reply.subject.split(".").last.to_i] = pub_ack(reply)
380
+ end
381
+ results.map { _1 || NATS::Timeout.new("nats: timeout") }
382
+ end
383
+
384
+ def next_reply(subscription, deadline)
385
+ timeout = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
386
+ subscription.next_msg(timeout: timeout) if timeout.positive?
387
+ rescue NATS::Timeout
388
+ nil
389
+ end
390
+
391
+ def pub_ack(reply)
392
+ return NATS::JetStream::Error::NoStreamResponse.new("nats: no response from stream") if reply.header&.dig("Status") == "503"
393
+
394
+ result = JSON.parse(reply.data, symbolize_names: true)
395
+ result[:error] ? NATS::JetStream::Error::APIError.new(result[:error]) : NATS::JetStream::PubAck.new(result)
396
+ end
397
+
201
398
  # NOTE: nats-pure's #pull_subscribe can't create a true ephemeral pull consumer: its rescue branch
202
399
  # unconditionally sets `config[:durable_name] = durable` (jetstream.rb). So the consumer is created
203
400
  # here, and #pull_subscribe only binds to it by name.
204
- def ephemeral_subscribe(subject, config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
401
+ def ephemeral_subscribe(subject, config)
205
402
  config = config.dup
206
403
  stream = config.delete(:stream) or raise ArgumentError, "stream required for ephemeral consumers"
207
404
  raise ArgumentError, "inactive_threshold required for ephemeral consumers" unless config[:inactive_threshold]
@@ -17,6 +17,7 @@ stream_config: &stream_config
17
17
  duplicate_window: 120 # seconds a Nats-Msg-Id is remembered, so a resent job is stored once
18
18
  discard: new # what to drop when the stream is full: new rejects publishes, old evicts the oldest
19
19
  allow_direct: true # required by the web UI
20
+ num_replicas: 1 # copies of the stream on a NATS cluster
20
21
  subjects:
21
22
  - jobs.%{name}.> # %{name} is the stream name
22
23
 
@@ -2,6 +2,10 @@
2
2
 
3
3
  module Cosmo
4
4
  class Config < ::Hash
5
+ # +config.services+ in Cosmo.configure: +num_replicas+ for Cosmo's own streams and KV buckets, e.g. 3 on a NATS
6
+ # cluster. Streams in cosmo.yml set their own +num_replicas+.
7
+ Services = Struct.new(:num_replicas, keyword_init: true)
8
+
5
9
  # +config.scheduled+ in Cosmo.configure. Turned off, Cosmo neither creates the scheduled stream nor runs the
6
10
  # scheduler, and delayed jobs (+perform_in+/+perform_at+) and crons raise {SchedulingDisabledError}.
7
11
  Scheduled = Struct.new(:enabled, keyword_init: true)
data/lib/cosmo/config.rb CHANGED
@@ -10,21 +10,54 @@ module Cosmo
10
10
  DEFAULT_PATH = "config/cosmo.yml"
11
11
  DEFAULTS_FILE = File.expand_path(DEFAULT_PATH, __dir__)
12
12
  SERVICE_STREAMS = %i[scheduled dead].freeze
13
+ GROUPED = [%i[consumers jobs], %i[setup cron]].freeze
13
14
 
14
15
  class << self
15
16
  extend Forwardable
16
17
 
17
- delegate %i[[] fetch dig load server_middleware client_middleware error_handlers replicas scheduled dead batches metrics] => :instance
18
+ delegate %i[[] fetch dig load server_middleware client_middleware error_handlers services scheduled dead batches metrics] => :instance
18
19
  end
19
20
 
21
+ # @param seconds [Numeric]
22
+ # @return [Integer] nanoseconds
20
23
  def self.to_ns(seconds)
21
24
  (seconds.to_f * NANO).to_i
22
25
  end
23
26
 
27
+ # Parses the YAML file at +path+ without normalizing it, only flattening grouped job consumers.
28
+ #
29
+ # @param path [String]
30
+ # @return [Hash]
31
+ # @raise [ConfigError] when grouped job consumers are malformed or repeat a stream
24
32
  def self.read(path)
25
- Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {})
33
+ Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {}).tap { flatten_groups!(_1) }
34
+ end
35
+
36
+ # Flattens +consumers.jobs+ and +setup.cron+ given as an array of groups (hashes of entries) into one hash, in
37
+ # place. Groups only organize the file: their order and boundaries are dropped.
38
+ #
39
+ # @param config [Hash]
40
+ # @return [void]
41
+ # @raise [ConfigError] when a group is not a hash or a name appears in more than one group
42
+ def self.flatten_groups!(config)
43
+ GROUPED.each do |path|
44
+ groups = config.dig(*path)
45
+ next unless groups.is_a?(::Array)
46
+
47
+ key = path.join(".")
48
+ config.dig(*path[0..-2])[path.last] = groups.each_with_object({}) do |group, entries|
49
+ raise ConfigError, "`#{key}` groups must be hashes of entries, got #{group.inspect}" unless group.is_a?(::Hash)
50
+
51
+ repeated = entries.keys & group.keys
52
+ raise ConfigError, "`#{key}.#{repeated.first}` appears in more than one group" if repeated.any?
53
+
54
+ entries.merge!(group)
55
+ end
56
+ end
26
57
  end
27
58
 
59
+ # @param path [String]
60
+ # @return [Hash] the file's config, normalized
28
61
  def self.parse_file(path)
29
62
  read(path).tap { normalize!(_1) }
30
63
  end
@@ -46,6 +79,9 @@ module Cosmo
46
79
  Utils::Hash.deep_merge(defaults, user).tap { normalize!(_1) }
47
80
  end
48
81
 
82
+ # @param config [Hash] a parsed config file
83
+ # @return [void]
84
+ # @raise [ConfigError] when +config+ sets a service stream or +batch_expiry+, which Cosmo.configure owns
49
85
  def self.validate!(config)
50
86
  %i[setup consumers].each do |section|
51
87
  jobs = config.dig(section, :jobs)
@@ -60,7 +96,11 @@ module Cosmo
60
96
  raise ConfigError, "`batch_expiry` moved out of cosmo.yml: Cosmo.configure { |config| config.batches.expiry = ... }"
61
97
  end
62
98
 
63
- def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
99
+ # Parses durations (+max_age+ and +duplicate_window+ into nanoseconds) and interpolates +%{name}+ in subjects, in place.
100
+ #
101
+ # @param config [Hash]
102
+ # @return [void]
103
+ def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
64
104
  Utils::Hash.symbolize_keys!(config)
65
105
 
66
106
  config[:timeout] = Utils::Duration.parse(config[:timeout]) if config[:timeout]
@@ -87,6 +127,10 @@ module Cosmo
87
127
  end
88
128
  end
89
129
 
130
+ # The consumer delivery options for a stream processor's +start_position+.
131
+ #
132
+ # @param start_position [Symbol, String, Time, nil] +:last+, +:new+, a start time, or anything else for every message
133
+ # @return [Hash]
90
134
  def self.deliver_policy(start_position)
91
135
  case start_position
92
136
  when "last", :last
@@ -102,10 +146,14 @@ module Cosmo
102
146
  end
103
147
  end
104
148
 
149
+ # @return [Config]
105
150
  def self.instance
106
151
  @instance ||= new
107
152
  end
108
153
 
154
+ # Registrations made at runtime rather than read from YAML, e.g. the stream classes under +:streams+.
155
+ #
156
+ # @return [Hash]
109
157
  def self.internal
110
158
  @internal ||= {}
111
159
  end
@@ -115,16 +163,15 @@ module Cosmo
115
163
  #
116
164
  # @param path [String, nil]
117
165
  # @param overrides [Hash, nil] e.g. +{ concurrency: 5 }+
166
+ # @return [void]
118
167
  def load(path = nil, overrides: nil)
119
168
  user = path ? self.class.read(path) : {}
120
169
  replace(self.class.build(Utils::Hash.deep_merge(user, Hash(overrides))))
121
170
  end
122
171
 
123
- attr_writer :replicas
124
-
125
- # @return [Integer] replicas for every service stream and bucket, e.g. 3 on a NATS cluster
126
- def replicas
127
- @replicas || 1
172
+ # @return [Services]
173
+ def services
174
+ @services ||= Services.new(num_replicas: 1)
128
175
  end
129
176
 
130
177
  # @return [Scheduled]
@@ -169,6 +216,7 @@ module Cosmo
169
216
  # Cosmo's +jid+/+elapsed+ context in each line.
170
217
  #
171
218
  # @param logger [::Logger]
219
+ # @return [void]
172
220
  def logger=(logger)
173
221
  Logger.instance = logger
174
222
  end
@@ -176,11 +224,12 @@ module Cosmo
176
224
  # Ignored when +COSMO_LOG_LEVEL+ is set, which always wins.
177
225
  #
178
226
  # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
227
+ # @return [void]
179
228
  def log_level=(level)
180
229
  Logger.level = level
181
230
  end
182
231
 
183
- # The chain every job execution runs through, starting with the built-in {Middleware::Limit}, {Middleware::Busy},
232
+ # The chain every job execution runs through, starting with the built-in {Middleware::Limit},
184
233
  # {Middleware::Totals}, and {Middleware::Metrics}. It lives outside the loaded YAML, so {#load} keeps it. Register middleware
185
234
  # at boot, before workers start:
186
235
  #
@@ -195,7 +244,6 @@ module Cosmo
195
244
  def server_middleware
196
245
  @server_middleware ||= Middleware::Chain.new do |chain|
197
246
  chain.add Middleware::Limit
198
- chain.add Middleware::Busy
199
247
  chain.add Middleware::Totals
200
248
  chain.add Middleware::Metrics
201
249
  end
data/lib/cosmo/engine.rb CHANGED
@@ -9,6 +9,11 @@ module Cosmo
9
9
  streams: Stream::Processor
10
10
  }.freeze
11
11
 
12
+ INTERRUPT_TIMEOUT = 5 # Seconds each interrupted thread gets to run its ensure blocks after a timed out shutdown
13
+
14
+ # Runs the {.instance}, see {#run}.
15
+ #
16
+ # @return [void]
12
17
  def self.run(...)
13
18
  instance.run(...)
14
19
  end
@@ -21,6 +26,7 @@ module Cosmo
21
26
  type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
22
27
  end
23
28
 
29
+ # @return [Engine]
24
30
  def self.instance
25
31
  @instance ||= new
26
32
  end
@@ -36,6 +42,11 @@ module Cosmo
36
42
  @heartbeat = nil
37
43
  end
38
44
 
45
+ # Starts the processors for +type+, blocks until a shutdown signal arrives, then shuts down.
46
+ #
47
+ # @param type [String, nil] +jobs+, +streams+, or nil for both
48
+ # @param options [Hash] the parsed command options
49
+ # @return [void]
39
50
  def run(type, options)
40
51
  handler = Utils::Signal.trap(:INT, :TERM, :TSTP, :CONT, :USR1, :TTIN)
41
52
  Logger.info "Starting processing, hit Ctrl-C to stop [concurrency=#{@concurrency}]"
@@ -54,6 +65,7 @@ module Cosmo
54
65
  shutdown
55
66
  end
56
67
 
68
+ # @return [Boolean]
57
69
  def running?
58
70
  @running.true?
59
71
  end
@@ -70,18 +82,27 @@ module Cosmo
70
82
  Array(@processors).sum(&:busy)
71
83
  end
72
84
 
85
+ # @return [Array<Hash>] the jobs running right now, across all processors, see {Job::Processor#jobs}
86
+ def jobs
87
+ Array(@processors).flat_map(&:jobs)
88
+ end
89
+
73
90
  # @return [Hash{Symbol => Array<String>}] what each running processor pulls from, keyed by processor type
74
91
  def subscriptions
75
92
  Array(@processors).to_h { [PROCESSORS.key(_1.class), _1.subscriptions] }
76
93
  end
77
94
 
95
+ # Stops fetching, waits up to +Config[:timeout]+ seconds for in-flight work, hands whatever is still
96
+ # running back to NATS and interrupts it, and unregisters the process.
97
+ #
98
+ # @return [void]
78
99
  def shutdown
79
100
  @running.make_false
80
101
  @heartbeat&.beat
81
102
  @http_server&.stop
82
103
  @pool.shutdown
83
104
  Logger.info "Pausing to allow jobs to finish..."
84
- @pool.wait_for_termination(Config[:timeout])
105
+ terminate unless @pool.wait_for_termination(Config[:timeout])
85
106
  @heartbeat&.stop
86
107
  Logger.info "Bye!"
87
108
  end
@@ -136,6 +157,12 @@ module Cosmo
136
157
  end
137
158
  end
138
159
 
160
+ def terminate
161
+ released = Array(@processors).sum(&:release)
162
+ Logger.warn "Shutdown timed out after #{Config[:timeout]}s, released #{released} unfinished message(s) for redelivery"
163
+ Array(@processors).each { _1.interrupt(INTERRUPT_TIMEOUT) }
164
+ end
165
+
139
166
  def dump_threads
140
167
  threads = Thread.list
141
168
  Logger.warn "Received TTIN, dumping backtraces of #{threads.size} threads"
@@ -5,7 +5,7 @@ require "shellwords"
5
5
 
6
6
  module Cosmo
7
7
  # Publishes this process's identity and live state to API::Stats::Processes, so the Web UI lists
8
- # every running worker, the idle ones included.
8
+ # every running worker, the idle ones included, and the jobs it is running to API::Stats::Jobs.
9
9
  class Heartbeat
10
10
  INTERVAL = 10
11
11
 
@@ -32,14 +32,18 @@ module Cosmo
32
32
  self
33
33
  end
34
34
 
35
- # Stops beating and removes the entry right away, instead of leaving it to expire.
35
+ # Stops beating and removes the entries right away, instead of leaving them to expire.
36
36
  #
37
37
  # @return [void]
38
38
  def stop
39
39
  @thread&.kill&.join
40
40
  @thread = nil
41
41
  flush
42
- processes.unregister(identity)
42
+ begin
43
+ processes.unregister(identity)
44
+ ensure
45
+ jobs.clear(identity)
46
+ end
43
47
  rescue StandardError => e
44
48
  Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
45
49
  end
@@ -49,6 +53,7 @@ module Cosmo
49
53
  # @return [void]
50
54
  def beat
51
55
  processes.register(identity, info)
56
+ jobs.report(identity, @engine.jobs)
52
57
  rescue StandardError => e
53
58
  Logger.debug "Heartbeat error: #{e.class} #{e.message}"
54
59
  end
@@ -136,5 +141,9 @@ module Cosmo
136
141
  def processes
137
142
  API::Stats::Processes.instance
138
143
  end
144
+
145
+ def jobs
146
+ API::Stats::Jobs.instance
147
+ end
139
148
  end
140
149
  end