cosmonats 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +73 -22
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +52 -4
  8. data/lib/cosmo/api/cron/entry.rb +34 -20
  9. data/lib/cosmo/api/cron.rb +24 -10
  10. data/lib/cosmo/api/job.rb +15 -1
  11. data/lib/cosmo/api/kv.rb +141 -12
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +20 -9
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +58 -7
  18. data/lib/cosmo/api/stream.rb +73 -24
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +217 -23
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +12 -1
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +56 -21
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/utils.rb +0 -1
  62. data/lib/cosmo/version.rb +1 -1
  63. data/lib/cosmo/web/assets/app.css +273 -629
  64. data/lib/cosmo/web/assets/app.js +158 -0
  65. data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
  66. data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
  67. data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
  68. data/lib/cosmo/web/assets/logo.svg +10 -0
  69. data/lib/cosmo/web/cache.rb +77 -0
  70. data/lib/cosmo/web/chart.rb +63 -39
  71. data/lib/cosmo/web/circuit.rb +56 -0
  72. data/lib/cosmo/web/context.rb +6 -4
  73. data/lib/cosmo/web/controllers/application.rb +42 -0
  74. data/lib/cosmo/web/controllers/batches.rb +8 -1
  75. data/lib/cosmo/web/controllers/crons.rb +24 -3
  76. data/lib/cosmo/web/controllers/jobs.rb +88 -68
  77. data/lib/cosmo/web/controllers/metrics.rb +6 -0
  78. data/lib/cosmo/web/controllers/overview.rb +88 -0
  79. data/lib/cosmo/web/controllers/processes.rb +36 -0
  80. data/lib/cosmo/web/controllers/streams.rb +38 -28
  81. data/lib/cosmo/web/helpers/application.rb +101 -0
  82. data/lib/cosmo/web/renderer.rb +36 -6
  83. data/lib/cosmo/web/throughput.rb +104 -0
  84. data/lib/cosmo/web/views/batches/_table.erb +42 -54
  85. data/lib/cosmo/web/views/batches/index.erb +3 -10
  86. data/lib/cosmo/web/views/crons/_table.erb +36 -55
  87. data/lib/cosmo/web/views/crons/index.erb +2 -9
  88. data/lib/cosmo/web/views/jobs/_args.erb +5 -0
  89. data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
  90. data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
  91. data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
  92. data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
  93. data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
  94. data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
  95. data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
  96. data/lib/cosmo/web/views/jobs/index.erb +4 -17
  97. data/lib/cosmo/web/views/layout.erb +73 -22
  98. data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
  99. data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
  100. data/lib/cosmo/web/views/metrics/_table.erb +24 -40
  101. data/lib/cosmo/web/views/metrics/index.erb +8 -16
  102. data/lib/cosmo/web/views/overview/_badges.erb +3 -0
  103. data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
  104. data/lib/cosmo/web/views/overview/_running.erb +18 -0
  105. data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
  106. data/lib/cosmo/web/views/overview/_workers.erb +25 -0
  107. data/lib/cosmo/web/views/overview/index.erb +28 -0
  108. data/lib/cosmo/web/views/processes/_table.erb +67 -0
  109. data/lib/cosmo/web/views/processes/index.erb +1 -0
  110. data/lib/cosmo/web/views/shared/_pager.erb +22 -0
  111. data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
  112. data/lib/cosmo/web/views/streams/_info.erb +29 -86
  113. data/lib/cosmo/web/views/streams/_list.erb +7 -0
  114. data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
  115. data/lib/cosmo/web/views/streams/_stream.erb +53 -0
  116. data/lib/cosmo/web/views/streams/index.erb +3 -11
  117. data/lib/cosmo/web/views/streams/info.erb +3 -11
  118. data/lib/cosmo/web.rb +73 -35
  119. data/lib/cosmo.rb +33 -0
  120. data/sig/cosmo/active_job/adapter.rbs +3 -3
  121. data/sig/cosmo/active_job/options.rbs +4 -2
  122. data/sig/cosmo/api/batch.rbs +5 -3
  123. data/sig/cosmo/api/counter.rbs +16 -6
  124. data/sig/cosmo/api/cron/entry.rbs +6 -3
  125. data/sig/cosmo/api/cron.rbs +10 -4
  126. data/sig/cosmo/api/kv.rbs +21 -4
  127. data/sig/cosmo/api/registry.rbs +10 -0
  128. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  129. data/sig/cosmo/api/stats/metrics.rbs +4 -1
  130. data/sig/cosmo/api/stats/processes.rbs +9 -1
  131. data/sig/cosmo/api/stats.rbs +12 -2
  132. data/sig/cosmo/api/stream.rbs +12 -5
  133. data/sig/cosmo/batch/callback.rbs +1 -1
  134. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  135. data/sig/cosmo/batch.rbs +5 -5
  136. data/sig/cosmo/cli.rbs +2 -0
  137. data/sig/cosmo/client.rbs +24 -4
  138. data/sig/cosmo/config/settings.rbs +6 -0
  139. data/sig/cosmo/config.rbs +5 -5
  140. data/sig/cosmo/engine.rbs +5 -0
  141. data/sig/cosmo/heartbeat.rbs +2 -0
  142. data/sig/cosmo/job/data.rbs +3 -1
  143. data/sig/cosmo/job/enqueuer.rbs +12 -0
  144. data/sig/cosmo/job/processor.rbs +18 -3
  145. data/sig/cosmo/job/setter.rbs +20 -0
  146. data/sig/cosmo/job.rbs +10 -4
  147. data/sig/cosmo/logger.rbs +5 -3
  148. data/sig/cosmo/middleware.rbs +0 -8
  149. data/sig/cosmo/processor.rbs +14 -2
  150. data/sig/cosmo/publisher.rbs +1 -0
  151. data/sig/cosmo/stream/processor.rbs +4 -0
  152. data/sig/cosmo/stream.rbs +4 -3
  153. data/sig/cosmo/utils/hash.rbs +2 -1
  154. data/sig/cosmo/utils/signal.rbs +5 -5
  155. data/sig/cosmo/utils/string.rbs +2 -2
  156. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  157. data/sig/cosmo/web/cache.rbs +35 -0
  158. data/sig/cosmo/web/chart.rbs +48 -0
  159. data/sig/cosmo/web/circuit.rbs +29 -0
  160. data/sig/cosmo/web/context.rbs +15 -0
  161. data/sig/cosmo/web/controllers/application.rbs +32 -0
  162. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  163. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  164. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  165. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  166. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  167. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  168. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  169. data/sig/cosmo/web/helpers/application.rbs +61 -0
  170. data/sig/cosmo/web/renderer.rbs +33 -0
  171. data/sig/cosmo/web/throughput.rbs +45 -0
  172. data/sig/cosmo/web.rbs +25 -0
  173. data/sig/cosmo.rbs +19 -0
  174. metadata +53 -23
  175. data/lib/cosmo/api/stats/busy.rb +0 -63
  176. data/lib/cosmo/middleware/busy.rb +0 -15
  177. data/lib/cosmo/utils/overrides.rb +0 -206
  178. data/lib/cosmo/utils/warnings.rb +0 -17
  179. data/lib/cosmo/web/controllers/actions.rb +0 -16
  180. data/lib/cosmo/web/views/actions/index.erb +0 -7
  181. data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
  182. data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
  183. data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
  184. data/lib/cosmo/web/views/jobs/busy.erb +0 -18
  185. data/lib/cosmo/web/views/jobs/dead.erb +0 -18
  186. data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
  187. data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
  188. data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
  189. data/lib/cosmo/web/views/streams/_table.erb +0 -25
  190. data/sig/cosmo/api/stats/busy.rbs +0 -31
data/lib/cosmo/client.rb CHANGED
@@ -2,7 +2,6 @@
2
2
 
3
3
  require "socket"
4
4
  require "nats/client"
5
- require "cosmo/utils/overrides"
6
5
 
7
6
  module Cosmo
8
7
  class Client
@@ -10,7 +9,12 @@ module Cosmo
10
9
  JS_TIMEOUT = 5
11
10
  # Seconds to wait for the TCP connect+handshake. Override with COSMO_CONNECT_TIMEOUT.
12
11
  CONNECT_TIMEOUT = 2
12
+ # Reconnect attempts before the connection closes for good; -1 never gives up, so a process outlives a NATS
13
+ # outage. nats-pure lets NATS_MAX_RECONNECT_ATTEMPTS override it.
14
+ MAX_RECONNECT_ATTEMPTS = -1
15
+ DIRECT_BATCH = 1000
13
16
 
17
+ # @return [Client]
14
18
  def self.instance
15
19
  @instance ||= Client.new
16
20
  end
@@ -19,19 +23,24 @@ module Cosmo
19
23
  # misbehaving process can be found without cross-referencing host IPs. Derived from
20
24
  # the running program, which separates a web process from a `cosmo` worker on the
21
25
  # same host. Override with COSMO_CLIENT_NAME.
26
+ #
27
+ # @return [String]
22
28
  def self.default_name
23
29
  ENV.fetch("COSMO_CLIENT_NAME") { "cosmo-#{program_name}-#{Socket.gethostname}-#{Process.pid}" }
24
30
  end
25
31
 
32
+ # @return [Integer] seconds to wait for a JetStream API reply
26
33
  def self.js_timeout
27
34
  ENV.fetch("COSMO_JS_TIMEOUT", JS_TIMEOUT).to_i
28
35
  end
29
36
 
37
+ # @return [Integer] seconds to wait for the connection to the server
30
38
  def self.connect_timeout
31
39
  ENV.fetch("COSMO_CONNECT_TIMEOUT", CONNECT_TIMEOUT).to_i
32
40
  end
33
41
 
34
42
  # $PROGRAM_NAME is "puma 7.2.1 (tcp://...)" under Puma and a path under the CLI.
43
+ # @return [String]
35
44
  def self.program_name
36
45
  File.basename($PROGRAM_NAME.to_s.split.first.to_s, ".*").gsub(/[^\w.-]/, "")
37
46
  end
@@ -41,36 +50,72 @@ module Cosmo
41
50
  def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"), name: self.class.default_name)
42
51
  @name = name
43
52
  Logger.debug "Connecting to NATS server at #{nats_url} as #{@name}..."
44
- @nc = NATS.connect(nats_url, name: @name, connect_timeout: self.class.connect_timeout)
53
+ @nc = NATS.connect(nats_url, name: @name, connect_timeout: self.class.connect_timeout,
54
+ max_reconnect_attempts: MAX_RECONNECT_ATTEMPTS)
45
55
  Logger.debug "Connection established"
46
56
  @js = @nc.jetstream(timeout: self.class.js_timeout)
47
57
  end
48
58
 
59
+ # @param subject [String]
60
+ # @param payload [String]
61
+ # @param params [Hash] JetStream publish options, e.g. +header:+
62
+ # @return [NATS::JetStream::PubAck]
49
63
  def publish(subject, payload, **params)
50
64
  js.publish(subject, payload, **params)
51
65
  end
52
66
 
67
+ # Publishes every message before waiting for any ack, so the batch costs about one round trip instead of one each.
68
+ # Unlike #publish, a message the stream refuses doesn't raise: its error takes the ack's place in the result.
69
+ #
70
+ # @param messages [Array<Hash>] each with +:subject+, +:payload+ and +:header+
71
+ # @return [Array<NATS::JetStream::PubAck, StandardError>] each message's ack, or why it was not stored, in order;
72
+ # +NATS::Timeout+ for one whose ack didn't arrive within the JetStream timeout
73
+ def publish_many(messages)
74
+ inbox = nc.new_inbox
75
+ subscription = nc.subscribe("#{inbox}.*")
76
+ messages.each_with_index { |m, i| nc.publish(m[:subject], m[:payload], "#{inbox}.#{i}", header: m[:header]) }
77
+ nc.flush(self.class.js_timeout)
78
+
79
+ acks(subscription, messages.size)
80
+ ensure
81
+ subscription&.unsubscribe
82
+ end
83
+
53
84
  # Create a pull subscription. Durable with +consumer_name+, and ephemeral without.
54
85
  # @param config [Hash] Consumer config. Ephemeral consumers additionally require
55
86
  # +:stream+ and +:inactive_threshold+ (seconds the consumer survives without a fetch).
87
+ # @param subject [String, Array<String>]
88
+ # @param consumer_name [String, nil]
89
+ # @return [NATS::JetStream::PullSubscription]
56
90
  def subscribe(subject, consumer_name, config)
57
91
  return js.pull_subscribe(subject, consumer_name, config: config) if consumer_name
58
92
 
59
93
  ephemeral_subscribe(subject, config)
60
94
  end
61
95
 
96
+ # @param name [String, Symbol]
97
+ # @return [NATS::JetStream::API::StreamInfo]
62
98
  def stream_info(name)
63
99
  js.stream_info(name)
64
100
  end
65
101
 
102
+ # @param name [String, Symbol]
103
+ # @param config [Hash]
104
+ # @return [NATS::JetStream::API::StreamCreateResponse]
66
105
  def create_stream(name, config)
67
106
  js.add_stream(name: name, **config)
68
107
  end
69
108
 
109
+ # @param name [String, Symbol]
110
+ # @param params [Hash]
111
+ # @return [Boolean]
70
112
  def delete_stream(name, params = {})
71
113
  js.delete_stream(name, params)
72
114
  end
73
115
 
116
+ # @param name [String, Symbol]
117
+ # @param config [Hash]
118
+ # @return [NATS::JetStream::API::StreamCreateResponse]
74
119
  def update_stream(name, config)
75
120
  js.update_stream(name: name, **config)
76
121
  end
@@ -78,6 +123,7 @@ module Cosmo
78
123
  # Create/update a stream, falling back to create when there's no stream.
79
124
  # @param name [String] Stream name
80
125
  # @param config [Hash] Full desired stream configuration
126
+ # @return [NATS::JetStream::API::StreamCreateResponse]
81
127
  def setup_stream(name, config)
82
128
  update_stream(name, config)
83
129
  rescue NATS::JetStream::Error::StreamNotFound
@@ -86,6 +132,8 @@ module Cosmo
86
132
 
87
133
  # Return all subjects in +stream_name+ that match +filter+ using NATS's
88
134
  # subjects_filter on STREAM.INFO (requires NATS ≥ 2.9).
135
+ # @param stream_name [String]
136
+ # @param filter [String]
89
137
  # @return [Array<String>]
90
138
  def cron_subjects_in_stream(stream_name, filter)
91
139
  payload = Utils::Json.dump({ subjects_filter: filter })
@@ -96,16 +144,22 @@ module Cosmo
96
144
  []
97
145
  end
98
146
 
147
+ # @return [Array<NATS::JetStream::API::StreamInfo>] every stream's info, as {#stream_info} returns it; empty when
148
+ # the request fails
99
149
  def list_streams
100
150
  response = nc.request("$JS.API.STREAM.LIST", "")
101
- data = Utils::Json.parse(response.data, symbolize_names: false)
102
- return [] if data.nil? || data["streams"].nil?
151
+ data = Utils::Json.parse(response.data)
152
+ return [] if data.nil? || data[:streams].nil?
103
153
 
104
- data["streams"]
154
+ data[:streams].map { NATS::JetStream::API::StreamInfo.new(_1) }
105
155
  rescue NATS::Error
106
156
  []
107
157
  end
108
158
 
159
+ # Marks the stream paused in its metadata, so workers stop fetching from it.
160
+ #
161
+ # @param name [String]
162
+ # @return [void]
109
163
  def pause_stream(name)
110
164
  config = stream_info(name).config.to_h
111
165
  config[:metadata] ||= {}
@@ -113,6 +167,8 @@ module Cosmo
113
167
  update_stream(name, config)
114
168
  end
115
169
 
170
+ # @param name [String]
171
+ # @return [void]
116
172
  def unpause_stream(name)
117
173
  config = stream_info(name).config.to_h
118
174
  config[:metadata] ||= {}
@@ -120,47 +176,65 @@ module Cosmo
120
176
  update_stream(name, config)
121
177
  end
122
178
 
179
+ # @param name [String]
180
+ # @return [Boolean] false when the server doesn't answer in time
123
181
  def stream_paused?(name)
124
182
  stream_info(name).config.metadata&.[](:"_cosmo.paused") == "true"
125
183
  rescue NATS::IO::Timeout
126
184
  false
127
185
  end
128
186
 
187
+ # @param stream_name [String]
188
+ # @return [Array<Hash>] the stream's consumers as JetStream reports them
129
189
  def list_consumers(stream_name)
130
190
  response = nc.request("$JS.API.CONSUMER.LIST.#{stream_name}", "")
131
191
  data = Utils::Json.parse(response.data, default: {}, symbolize_names: false)
132
192
  Array(data["consumers"])
133
193
  end
134
194
 
195
+ # @param stream_name [String]
196
+ # @param consumer_name [String]
197
+ # @return [NATS::JetStream::API::ConsumerInfo]
135
198
  def consumer_info(stream_name, consumer_name)
136
199
  js.consumer_info(stream_name, consumer_name)
137
200
  end
138
201
 
202
+ # @param stream_name [String]
203
+ # @param consumer_name [String]
204
+ # @return [Boolean]
139
205
  def delete_consumer(stream_name, consumer_name)
140
206
  js.delete_consumer(stream_name, consumer_name)
141
207
  end
142
208
 
209
+ # @param stream_name [String, Symbol]
210
+ # @param options [Hash] e.g. +seq:+, or +subject:+ for the last message on it
211
+ # @return [NATS::JetStream::API::RawStreamMsg]
143
212
  def get_message(stream_name, **options)
144
213
  js.get_msg(stream_name, **options)
145
214
  end
146
215
 
147
216
  # The last message on every subject matching +filters+, in one batched direct get (NATS ≥ 2.11). Needs
148
- # +allow_direct+ on the stream.
217
+ # +allow_direct+ on the stream. NATS refuses to match more than 1024 subjects: use {#direct_messages} for those.
149
218
  #
150
219
  # @param stream_name [String]
151
220
  # @param filters [Array<String>] subject filters, wildcards allowed
152
- # @return [Hash{String => String}] message data by subject
153
- def last_messages(stream_name, filters) # rubocop:disable Metrics/AbcSize
221
+ # @return [Hash{String => NATS::Msg}] the message by subject, its data and headers
222
+ # @raise [NATS::JetStream::Error] when the stream refuses the read, such as for matching too many subjects
223
+ def last_messages(stream_name, filters)
154
224
  inbox = nc.new_inbox
155
225
  sub = nc.subscribe(inbox)
156
226
  nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump({ multi_last: filters, batch: 10_000 }), inbox)
157
227
  messages = {}
158
228
  loop do
159
229
  msg = sub.next_msg(timeout: self.class.js_timeout)
160
- subject = msg.header&.dig("Nats-Subject")
161
- break unless subject
162
-
163
- messages[subject] = msg.data
230
+ header = msg.header || {}
231
+ subject = header[NATS::JetStream::Header::SUBJECT]
232
+ unless subject
233
+ direct_pending(stream_name, header)
234
+ break
235
+ end
236
+
237
+ messages[subject] = msg
164
238
  end
165
239
  messages
166
240
  rescue NATS::Timeout
@@ -169,11 +243,83 @@ module Cosmo
169
243
  sub&.unsubscribe
170
244
  end
171
245
 
246
+ # Every message on +filter+ in stream order, read by batched direct gets (NATS ≥ 2.11) of up to
247
+ # DIRECT_BATCH messages each, so it has no subject cap. On a stream keeping one message per subject, such as
248
+ # a KV bucket, that is the last message on every subject, as {#last_messages} reads them. Needs +allow_direct+
249
+ # on the stream.
250
+ #
251
+ # @param stream_name [String]
252
+ # @param filter [String] a subject filter, wildcards allowed
253
+ # @param limit [Integer, nil] the most messages to read, nil for every one
254
+ # @param since [Time, nil] start at the first message stored at or after it, instead of the first one
255
+ # @return [Array<NATS::Msg>]
256
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
257
+ # @raise [NATS::Timeout] when a batch stops arriving, or never starts: no stream, or none with +allow_direct+
258
+ def direct_messages(stream_name, filter, limit: nil, since: nil)
259
+ messages = []
260
+ start = since ? { start_time: since.utc.iso8601(9) } : { seq: 1 }
261
+ loop do
262
+ batch = [DIRECT_BATCH, limit && (limit - messages.size)].compact.min
263
+ seq, pending = direct_batch(stream_name, { **start, next_by_subj: filter, batch: }, messages)
264
+ break if pending.zero? || (limit && messages.size >= limit)
265
+
266
+ start = { seq: }
267
+ end
268
+ messages
269
+ end
270
+
271
+ # How many messages match +filter+, answered by the server without sending any of them: a direct get of one
272
+ # message reports the rest as pending. On a stream keeping one message per subject, such as a KV bucket, that is
273
+ # the number of subjects. Needs +allow_direct+ on the stream.
274
+ #
275
+ # @param stream_name [String]
276
+ # @param filter [String] a subject filter, wildcards allowed
277
+ # @return [Integer]
278
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
279
+ # @raise [NATS::Timeout] when no stream with +allow_direct+ answers
280
+ def count_messages(stream_name, filter)
281
+ messages = []
282
+ _, pending = direct_batch(stream_name, { seq: 1, next_by_subj: filter, batch: 1 }, messages)
283
+ messages.size + pending
284
+ end
285
+
286
+ # The sorted subjects that hold a message and match +filter+, without the messages: the stream info's subject
287
+ # listing, which has no way to take a page of it, so read it once and page the result.
288
+ #
289
+ # @param stream_name [String]
290
+ # @param filter [String] a subject filter, wildcards allowed
291
+ # @return [Array<String>]
292
+ # @raise [NATS::JetStream::Error] when the stream refuses the read
293
+ def subjects(stream_name, filter)
294
+ names = []
295
+ loop do
296
+ response = Utils::Json.parse(
297
+ nc.request("$JS.API.STREAM.INFO.#{stream_name}", Utils::Json.dump({ subjects_filter: filter, offset: names.size })).data,
298
+ default: {}, symbolize_names: false
299
+ )
300
+ raise NATS::JetStream::Error, response.dig("error", "description") if response["error"]
301
+
302
+ page = response.dig("state", "subjects")&.keys.to_a
303
+ names.concat(page)
304
+ break if page.empty? || names.size >= response["total"].to_i
305
+ end
306
+ names.sort
307
+ end
308
+
309
+ # @param name [String] the stream
310
+ # @param seq [Integer]
311
+ # @return [Hash] the JetStream API reply
172
312
  def delete_message(name, seq)
173
313
  response = nc.request("$JS.API.STREAM.MSG.DELETE.#{name}", JSON.dump({ seq: seq }))
174
314
  Utils::Json.parse(response.data, symbolize_names: false)
175
315
  end
176
316
 
317
+ # Purges +stream_name+, only the messages on +subject+ when given.
318
+ #
319
+ # @param stream_name [String]
320
+ # @param subject [String, nil]
321
+ # @return [Integer] how many messages were purged
322
+ # @raise [NATS::JetStream::Error] when JetStream rejects the purge
177
323
  def purge(stream_name, subject)
178
324
  payload = subject ? Utils::Json.dump({ filter: subject }) : ""
179
325
  response = @nc.request("$JS.API.STREAM.PURGE.#{stream_name}", payload)
@@ -183,21 +329,76 @@ module Cosmo
183
329
  result["purged"] # number of messages purged
184
330
  end
185
331
 
332
+ # Looks up the +name+ KV bucket, creating it with +options+ when it doesn't exist.
333
+ #
334
+ # @param name [String]
335
+ # @param allow_msg_ttl [Boolean] lets the bucket's messages carry their own +Nats-TTL+
336
+ # @param options [Hash] the bucket's config, used only on creation, e.g. +ttl:+
337
+ # @return [NATS::KeyValue]
186
338
  def kv(name, allow_msg_ttl: false, **options)
187
339
  js.key_value(name)
188
340
  rescue NATS::KeyValue::BucketNotFoundError
189
341
  allow_msg_ttl ? create_kv_with_msg_ttl(name, **options) : js.create_key_value({ bucket: name }.merge(options))
190
342
  end
191
343
 
344
+ # @return [void]
192
345
  def close
193
346
  nc.close
194
347
  end
195
348
 
196
349
  private
197
350
 
198
- # NOTE: nats-pure's #pull_subscribe has no path to a true ephemeral pull consumer.
199
- # Its rescue branch unconditionally sets `config[:durable_name] = durable` (jetstream.rb).
200
- def ephemeral_subscribe(subject, config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
351
+ def direct_batch(stream_name, request, messages)
352
+ inbox = nc.new_inbox
353
+ sub = nc.subscribe(inbox)
354
+ nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump(request), inbox)
355
+ seq = request[:seq]
356
+ loop do
357
+ msg = sub.next_msg(timeout: self.class.js_timeout)
358
+ header = msg.header || {}
359
+ return [seq, direct_pending(stream_name, header)] unless header[NATS::JetStream::Header::SUBJECT]
360
+
361
+ messages << msg
362
+ seq = header[NATS::JetStream::Header::SEQUENCE].to_i + 1
363
+ end
364
+ ensure
365
+ sub&.unsubscribe
366
+ end
367
+
368
+ def direct_pending(stream_name, header)
369
+ return header["Nats-Num-Pending"].to_i if %w[204 404].include?(header["Status"])
370
+
371
+ raise NATS::JetStream::Error, "direct get on #{stream_name}: #{header["Status"]} #{header["Description"]}"
372
+ end
373
+
374
+ def acks(subscription, count)
375
+ results = Array.new(count)
376
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + self.class.js_timeout
377
+ count.times do
378
+ reply = next_reply(subscription, deadline) or break
379
+ results[reply.subject.split(".").last.to_i] = pub_ack(reply)
380
+ end
381
+ results.map { _1 || NATS::Timeout.new("nats: timeout") }
382
+ end
383
+
384
+ def next_reply(subscription, deadline)
385
+ timeout = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
386
+ subscription.next_msg(timeout: timeout) if timeout.positive?
387
+ rescue NATS::Timeout
388
+ nil
389
+ end
390
+
391
+ def pub_ack(reply)
392
+ return NATS::JetStream::Error::NoStreamResponse.new("nats: no response from stream") if reply.header&.dig("Status") == "503"
393
+
394
+ result = JSON.parse(reply.data, symbolize_names: true)
395
+ result[:error] ? NATS::JetStream::Error::APIError.new(result[:error]) : NATS::JetStream::PubAck.new(result)
396
+ end
397
+
398
+ # NOTE: nats-pure's #pull_subscribe can't create a true ephemeral pull consumer: its rescue branch
399
+ # unconditionally sets `config[:durable_name] = durable` (jetstream.rb). So the consumer is created
400
+ # here, and #pull_subscribe only binds to it by name.
401
+ def ephemeral_subscribe(subject, config)
201
402
  config = config.dup
202
403
  stream = config.delete(:stream) or raise ArgumentError, "stream required for ephemeral consumers"
203
404
  raise ArgumentError, "inactive_threshold required for ephemeral consumers" unless config[:inactive_threshold]
@@ -211,14 +412,7 @@ module Cosmo
211
412
  end
212
413
 
213
414
  info = js.add_consumer(stream, consumer_config)
214
-
215
- sub = nc.subscribe(nc.new_inbox)
216
- sub.extend(NATS::JetStream.const_get(:PullSubscription))
217
- sub.jsi = NATS::JetStream.const_get(:JS)::Sub.new(
218
- js: js, stream: stream, consumer: info.name,
219
- nms: "#{js.prefix}.CONSUMER.MSG.NEXT.#{stream}.#{info.name}"
220
- )
221
- sub
415
+ js.pull_subscribe(subject, nil, stream: stream, name: info.name)
222
416
  end
223
417
 
224
418
  # NOTE: KV manager in nats-pure hardcodes the fields it copies into StreamConfig,
@@ -17,6 +17,7 @@ stream_config: &stream_config
17
17
  duplicate_window: 120 # seconds a Nats-Msg-Id is remembered, so a resent job is stored once
18
18
  discard: new # what to drop when the stream is full: new rejects publishes, old evicts the oldest
19
19
  allow_direct: true # required by the web UI
20
+ num_replicas: 1 # copies of the stream on a NATS cluster
20
21
  subjects:
21
22
  - jobs.%{name}.> # %{name} is the stream name
22
23
 
@@ -2,6 +2,10 @@
2
2
 
3
3
  module Cosmo
4
4
  class Config < ::Hash
5
+ # +config.services+ in Cosmo.configure: +num_replicas+ for Cosmo's own streams and KV buckets, e.g. 3 on a NATS
6
+ # cluster. Streams in cosmo.yml set their own +num_replicas+.
7
+ Services = Struct.new(:num_replicas, keyword_init: true)
8
+
5
9
  # +config.scheduled+ in Cosmo.configure. Turned off, Cosmo neither creates the scheduled stream nor runs the
6
10
  # scheduler, and delayed jobs (+perform_in+/+perform_at+) and crons raise {SchedulingDisabledError}.
7
11
  Scheduled = Struct.new(:enabled, keyword_init: true)
data/lib/cosmo/config.rb CHANGED
@@ -10,21 +10,54 @@ module Cosmo
10
10
  DEFAULT_PATH = "config/cosmo.yml"
11
11
  DEFAULTS_FILE = File.expand_path(DEFAULT_PATH, __dir__)
12
12
  SERVICE_STREAMS = %i[scheduled dead].freeze
13
+ GROUPED = [%i[consumers jobs], %i[setup cron]].freeze
13
14
 
14
15
  class << self
15
16
  extend Forwardable
16
17
 
17
- delegate %i[[] fetch dig load server_middleware client_middleware error_handlers replicas scheduled dead batches metrics] => :instance
18
+ delegate %i[[] fetch dig load server_middleware client_middleware error_handlers services scheduled dead batches metrics] => :instance
18
19
  end
19
20
 
21
+ # @param seconds [Numeric]
22
+ # @return [Integer] nanoseconds
20
23
  def self.to_ns(seconds)
21
24
  (seconds.to_f * NANO).to_i
22
25
  end
23
26
 
27
+ # Parses the YAML file at +path+ without normalizing it, only flattening grouped job consumers.
28
+ #
29
+ # @param path [String]
30
+ # @return [Hash]
31
+ # @raise [ConfigError] when grouped job consumers are malformed or repeat a stream
24
32
  def self.read(path)
25
- Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {})
33
+ Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {}).tap { flatten_groups!(_1) }
34
+ end
35
+
36
+ # Flattens +consumers.jobs+ and +setup.cron+ given as an array of groups (hashes of entries) into one hash, in
37
+ # place. Groups only organize the file: their order and boundaries are dropped.
38
+ #
39
+ # @param config [Hash]
40
+ # @return [void]
41
+ # @raise [ConfigError] when a group is not a hash or a name appears in more than one group
42
+ def self.flatten_groups!(config)
43
+ GROUPED.each do |path|
44
+ groups = config.dig(*path)
45
+ next unless groups.is_a?(::Array)
46
+
47
+ key = path.join(".")
48
+ config.dig(*path[0..-2])[path.last] = groups.each_with_object({}) do |group, entries|
49
+ raise ConfigError, "`#{key}` groups must be hashes of entries, got #{group.inspect}" unless group.is_a?(::Hash)
50
+
51
+ repeated = entries.keys & group.keys
52
+ raise ConfigError, "`#{key}.#{repeated.first}` appears in more than one group" if repeated.any?
53
+
54
+ entries.merge!(group)
55
+ end
56
+ end
26
57
  end
27
58
 
59
+ # @param path [String]
60
+ # @return [Hash] the file's config, normalized
28
61
  def self.parse_file(path)
29
62
  read(path).tap { normalize!(_1) }
30
63
  end
@@ -46,6 +79,9 @@ module Cosmo
46
79
  Utils::Hash.deep_merge(defaults, user).tap { normalize!(_1) }
47
80
  end
48
81
 
82
+ # @param config [Hash] a parsed config file
83
+ # @return [void]
84
+ # @raise [ConfigError] when +config+ sets a service stream or +batch_expiry+, which Cosmo.configure owns
49
85
  def self.validate!(config)
50
86
  %i[setup consumers].each do |section|
51
87
  jobs = config.dig(section, :jobs)
@@ -60,7 +96,11 @@ module Cosmo
60
96
  raise ConfigError, "`batch_expiry` moved out of cosmo.yml: Cosmo.configure { |config| config.batches.expiry = ... }"
61
97
  end
62
98
 
63
- def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
99
+ # Parses durations (+max_age+ and +duplicate_window+ into nanoseconds) and interpolates +%{name}+ in subjects, in place.
100
+ #
101
+ # @param config [Hash]
102
+ # @return [void]
103
+ def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
64
104
  Utils::Hash.symbolize_keys!(config)
65
105
 
66
106
  config[:timeout] = Utils::Duration.parse(config[:timeout]) if config[:timeout]
@@ -87,6 +127,10 @@ module Cosmo
87
127
  end
88
128
  end
89
129
 
130
+ # The consumer delivery options for a stream processor's +start_position+.
131
+ #
132
+ # @param start_position [Symbol, String, Time, nil] +:last+, +:new+, a start time, or anything else for every message
133
+ # @return [Hash]
90
134
  def self.deliver_policy(start_position)
91
135
  case start_position
92
136
  when "last", :last
@@ -102,10 +146,14 @@ module Cosmo
102
146
  end
103
147
  end
104
148
 
149
+ # @return [Config]
105
150
  def self.instance
106
151
  @instance ||= new
107
152
  end
108
153
 
154
+ # Registrations made at runtime rather than read from YAML, e.g. the stream classes under +:streams+.
155
+ #
156
+ # @return [Hash]
109
157
  def self.internal
110
158
  @internal ||= {}
111
159
  end
@@ -115,16 +163,15 @@ module Cosmo
115
163
  #
116
164
  # @param path [String, nil]
117
165
  # @param overrides [Hash, nil] e.g. +{ concurrency: 5 }+
166
+ # @return [void]
118
167
  def load(path = nil, overrides: nil)
119
168
  user = path ? self.class.read(path) : {}
120
169
  replace(self.class.build(Utils::Hash.deep_merge(user, Hash(overrides))))
121
170
  end
122
171
 
123
- attr_writer :replicas
124
-
125
- # @return [Integer] replicas for every service stream and bucket, e.g. 3 on a NATS cluster
126
- def replicas
127
- @replicas || 1
172
+ # @return [Services]
173
+ def services
174
+ @services ||= Services.new(num_replicas: 1)
128
175
  end
129
176
 
130
177
  # @return [Scheduled]
@@ -169,6 +216,7 @@ module Cosmo
169
216
  # Cosmo's +jid+/+elapsed+ context in each line.
170
217
  #
171
218
  # @param logger [::Logger]
219
+ # @return [void]
172
220
  def logger=(logger)
173
221
  Logger.instance = logger
174
222
  end
@@ -176,11 +224,12 @@ module Cosmo
176
224
  # Ignored when +COSMO_LOG_LEVEL+ is set, which always wins.
177
225
  #
178
226
  # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
227
+ # @return [void]
179
228
  def log_level=(level)
180
229
  Logger.level = level
181
230
  end
182
231
 
183
- # The chain every job execution runs through, starting with the built-in {Middleware::Limit}, {Middleware::Busy},
232
+ # The chain every job execution runs through, starting with the built-in {Middleware::Limit},
184
233
  # {Middleware::Totals}, and {Middleware::Metrics}. It lives outside the loaded YAML, so {#load} keeps it. Register middleware
185
234
  # at boot, before workers start:
186
235
  #
@@ -195,7 +244,6 @@ module Cosmo
195
244
  def server_middleware
196
245
  @server_middleware ||= Middleware::Chain.new do |chain|
197
246
  chain.add Middleware::Limit
198
- chain.add Middleware::Busy
199
247
  chain.add Middleware::Totals
200
248
  chain.add Middleware::Metrics
201
249
  end
data/lib/cosmo/engine.rb CHANGED
@@ -9,6 +9,11 @@ module Cosmo
9
9
  streams: Stream::Processor
10
10
  }.freeze
11
11
 
12
+ INTERRUPT_TIMEOUT = 5 # Seconds each interrupted thread gets to run its ensure blocks after a timed out shutdown
13
+
14
+ # Runs the {.instance}, see {#run}.
15
+ #
16
+ # @return [void]
12
17
  def self.run(...)
13
18
  instance.run(...)
14
19
  end
@@ -21,6 +26,7 @@ module Cosmo
21
26
  type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
22
27
  end
23
28
 
29
+ # @return [Engine]
24
30
  def self.instance
25
31
  @instance ||= new
26
32
  end
@@ -36,6 +42,11 @@ module Cosmo
36
42
  @heartbeat = nil
37
43
  end
38
44
 
45
+ # Starts the processors for +type+, blocks until a shutdown signal arrives, then shuts down.
46
+ #
47
+ # @param type [String, nil] +jobs+, +streams+, or nil for both
48
+ # @param options [Hash] the parsed command options
49
+ # @return [void]
39
50
  def run(type, options)
40
51
  handler = Utils::Signal.trap(:INT, :TERM, :TSTP, :CONT, :USR1, :TTIN)
41
52
  Logger.info "Starting processing, hit Ctrl-C to stop [concurrency=#{@concurrency}]"
@@ -54,6 +65,7 @@ module Cosmo
54
65
  shutdown
55
66
  end
56
67
 
68
+ # @return [Boolean]
57
69
  def running?
58
70
  @running.true?
59
71
  end
@@ -70,18 +82,27 @@ module Cosmo
70
82
  Array(@processors).sum(&:busy)
71
83
  end
72
84
 
85
+ # @return [Array<Hash>] the jobs running right now, across all processors, see {Job::Processor#jobs}
86
+ def jobs
87
+ Array(@processors).flat_map(&:jobs)
88
+ end
89
+
73
90
  # @return [Hash{Symbol => Array<String>}] what each running processor pulls from, keyed by processor type
74
91
  def subscriptions
75
92
  Array(@processors).to_h { [PROCESSORS.key(_1.class), _1.subscriptions] }
76
93
  end
77
94
 
95
+ # Stops fetching, waits up to +Config[:timeout]+ seconds for in-flight work, hands whatever is still
96
+ # running back to NATS and interrupts it, and unregisters the process.
97
+ #
98
+ # @return [void]
78
99
  def shutdown
79
100
  @running.make_false
80
101
  @heartbeat&.beat
81
102
  @http_server&.stop
82
103
  @pool.shutdown
83
104
  Logger.info "Pausing to allow jobs to finish..."
84
- @pool.wait_for_termination(Config[:timeout])
105
+ terminate unless @pool.wait_for_termination(Config[:timeout])
85
106
  @heartbeat&.stop
86
107
  Logger.info "Bye!"
87
108
  end
@@ -136,6 +157,12 @@ module Cosmo
136
157
  end
137
158
  end
138
159
 
160
+ def terminate
161
+ released = Array(@processors).sum(&:release)
162
+ Logger.warn "Shutdown timed out after #{Config[:timeout]}s, released #{released} unfinished message(s) for redelivery"
163
+ Array(@processors).each { _1.interrupt(INTERRUPT_TIMEOUT) }
164
+ end
165
+
139
166
  def dump_threads
140
167
  threads = Thread.list
141
168
  Logger.warn "Received TTIN, dumping backtraces of #{threads.size} threads"