cosmonats 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +73 -22
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +52 -4
  8. data/lib/cosmo/api/cron/entry.rb +34 -20
  9. data/lib/cosmo/api/cron.rb +24 -10
  10. data/lib/cosmo/api/job.rb +15 -1
  11. data/lib/cosmo/api/kv.rb +141 -12
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +20 -9
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +58 -7
  18. data/lib/cosmo/api/stream.rb +73 -24
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +217 -23
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +12 -1
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +56 -21
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/utils.rb +0 -1
  62. data/lib/cosmo/version.rb +1 -1
  63. data/lib/cosmo/web/assets/app.css +273 -629
  64. data/lib/cosmo/web/assets/app.js +158 -0
  65. data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
  66. data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
  67. data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
  68. data/lib/cosmo/web/assets/logo.svg +10 -0
  69. data/lib/cosmo/web/cache.rb +77 -0
  70. data/lib/cosmo/web/chart.rb +63 -39
  71. data/lib/cosmo/web/circuit.rb +56 -0
  72. data/lib/cosmo/web/context.rb +6 -4
  73. data/lib/cosmo/web/controllers/application.rb +42 -0
  74. data/lib/cosmo/web/controllers/batches.rb +8 -1
  75. data/lib/cosmo/web/controllers/crons.rb +24 -3
  76. data/lib/cosmo/web/controllers/jobs.rb +88 -68
  77. data/lib/cosmo/web/controllers/metrics.rb +6 -0
  78. data/lib/cosmo/web/controllers/overview.rb +88 -0
  79. data/lib/cosmo/web/controllers/processes.rb +36 -0
  80. data/lib/cosmo/web/controllers/streams.rb +38 -28
  81. data/lib/cosmo/web/helpers/application.rb +101 -0
  82. data/lib/cosmo/web/renderer.rb +36 -6
  83. data/lib/cosmo/web/throughput.rb +104 -0
  84. data/lib/cosmo/web/views/batches/_table.erb +42 -54
  85. data/lib/cosmo/web/views/batches/index.erb +3 -10
  86. data/lib/cosmo/web/views/crons/_table.erb +36 -55
  87. data/lib/cosmo/web/views/crons/index.erb +2 -9
  88. data/lib/cosmo/web/views/jobs/_args.erb +5 -0
  89. data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
  90. data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
  91. data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
  92. data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
  93. data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
  94. data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
  95. data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
  96. data/lib/cosmo/web/views/jobs/index.erb +4 -17
  97. data/lib/cosmo/web/views/layout.erb +73 -22
  98. data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
  99. data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
  100. data/lib/cosmo/web/views/metrics/_table.erb +24 -40
  101. data/lib/cosmo/web/views/metrics/index.erb +8 -16
  102. data/lib/cosmo/web/views/overview/_badges.erb +3 -0
  103. data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
  104. data/lib/cosmo/web/views/overview/_running.erb +18 -0
  105. data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
  106. data/lib/cosmo/web/views/overview/_workers.erb +25 -0
  107. data/lib/cosmo/web/views/overview/index.erb +28 -0
  108. data/lib/cosmo/web/views/processes/_table.erb +67 -0
  109. data/lib/cosmo/web/views/processes/index.erb +1 -0
  110. data/lib/cosmo/web/views/shared/_pager.erb +22 -0
  111. data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
  112. data/lib/cosmo/web/views/streams/_info.erb +29 -86
  113. data/lib/cosmo/web/views/streams/_list.erb +7 -0
  114. data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
  115. data/lib/cosmo/web/views/streams/_stream.erb +53 -0
  116. data/lib/cosmo/web/views/streams/index.erb +3 -11
  117. data/lib/cosmo/web/views/streams/info.erb +3 -11
  118. data/lib/cosmo/web.rb +73 -35
  119. data/lib/cosmo.rb +33 -0
  120. data/sig/cosmo/active_job/adapter.rbs +3 -3
  121. data/sig/cosmo/active_job/options.rbs +4 -2
  122. data/sig/cosmo/api/batch.rbs +5 -3
  123. data/sig/cosmo/api/counter.rbs +16 -6
  124. data/sig/cosmo/api/cron/entry.rbs +6 -3
  125. data/sig/cosmo/api/cron.rbs +10 -4
  126. data/sig/cosmo/api/kv.rbs +21 -4
  127. data/sig/cosmo/api/registry.rbs +10 -0
  128. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  129. data/sig/cosmo/api/stats/metrics.rbs +4 -1
  130. data/sig/cosmo/api/stats/processes.rbs +9 -1
  131. data/sig/cosmo/api/stats.rbs +12 -2
  132. data/sig/cosmo/api/stream.rbs +12 -5
  133. data/sig/cosmo/batch/callback.rbs +1 -1
  134. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  135. data/sig/cosmo/batch.rbs +5 -5
  136. data/sig/cosmo/cli.rbs +2 -0
  137. data/sig/cosmo/client.rbs +24 -4
  138. data/sig/cosmo/config/settings.rbs +6 -0
  139. data/sig/cosmo/config.rbs +5 -5
  140. data/sig/cosmo/engine.rbs +5 -0
  141. data/sig/cosmo/heartbeat.rbs +2 -0
  142. data/sig/cosmo/job/data.rbs +3 -1
  143. data/sig/cosmo/job/enqueuer.rbs +12 -0
  144. data/sig/cosmo/job/processor.rbs +18 -3
  145. data/sig/cosmo/job/setter.rbs +20 -0
  146. data/sig/cosmo/job.rbs +10 -4
  147. data/sig/cosmo/logger.rbs +5 -3
  148. data/sig/cosmo/middleware.rbs +0 -8
  149. data/sig/cosmo/processor.rbs +14 -2
  150. data/sig/cosmo/publisher.rbs +1 -0
  151. data/sig/cosmo/stream/processor.rbs +4 -0
  152. data/sig/cosmo/stream.rbs +4 -3
  153. data/sig/cosmo/utils/hash.rbs +2 -1
  154. data/sig/cosmo/utils/signal.rbs +5 -5
  155. data/sig/cosmo/utils/string.rbs +2 -2
  156. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  157. data/sig/cosmo/web/cache.rbs +35 -0
  158. data/sig/cosmo/web/chart.rbs +48 -0
  159. data/sig/cosmo/web/circuit.rbs +29 -0
  160. data/sig/cosmo/web/context.rbs +15 -0
  161. data/sig/cosmo/web/controllers/application.rbs +32 -0
  162. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  163. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  164. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  165. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  166. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  167. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  168. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  169. data/sig/cosmo/web/helpers/application.rbs +61 -0
  170. data/sig/cosmo/web/renderer.rbs +33 -0
  171. data/sig/cosmo/web/throughput.rbs +45 -0
  172. data/sig/cosmo/web.rbs +25 -0
  173. data/sig/cosmo.rbs +19 -0
  174. metadata +53 -23
  175. data/lib/cosmo/api/stats/busy.rb +0 -63
  176. data/lib/cosmo/middleware/busy.rb +0 -15
  177. data/lib/cosmo/utils/overrides.rb +0 -206
  178. data/lib/cosmo/utils/warnings.rb +0 -17
  179. data/lib/cosmo/web/controllers/actions.rb +0 -16
  180. data/lib/cosmo/web/views/actions/index.erb +0 -7
  181. data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
  182. data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
  183. data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
  184. data/lib/cosmo/web/views/jobs/busy.erb +0 -18
  185. data/lib/cosmo/web/views/jobs/dead.erb +0 -18
  186. data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
  187. data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
  188. data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
  189. data/lib/cosmo/web/views/streams/_table.erb +0 -25
  190. data/sig/cosmo/api/stats/busy.rbs +0 -31
@@ -5,7 +5,7 @@ require "shellwords"
5
5
 
6
6
  module Cosmo
7
7
  # Publishes this process's identity and live state to API::Stats::Processes, so the Web UI lists
8
- # every running worker, the idle ones included.
8
+ # every running worker, the idle ones included, and the jobs it is running to API::Stats::Jobs.
9
9
  class Heartbeat
10
10
  INTERVAL = 10
11
11
 
@@ -32,14 +32,18 @@ module Cosmo
32
32
  self
33
33
  end
34
34
 
35
- # Stops beating and removes the entry right away, instead of leaving it to expire.
35
+ # Stops beating and removes the entries right away, instead of leaving them to expire.
36
36
  #
37
37
  # @return [void]
38
38
  def stop
39
39
  @thread&.kill&.join
40
40
  @thread = nil
41
41
  flush
42
- processes.unregister(identity)
42
+ begin
43
+ processes.unregister(identity)
44
+ ensure
45
+ jobs.clear(identity)
46
+ end
43
47
  rescue StandardError => e
44
48
  Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
45
49
  end
@@ -49,6 +53,7 @@ module Cosmo
49
53
  # @return [void]
50
54
  def beat
51
55
  processes.register(identity, info)
56
+ jobs.report(identity, @engine.jobs)
52
57
  rescue StandardError => e
53
58
  Logger.debug "Heartbeat error: #{e.class} #{e.message}"
54
59
  end
@@ -136,5 +141,9 @@ module Cosmo
136
141
  def processes
137
142
  API::Stats::Processes.instance
138
143
  end
144
+
145
+ def jobs
146
+ API::Stats::Jobs.instance
147
+ end
139
148
  end
140
149
  end
@@ -11,6 +11,11 @@ module Cosmo
11
11
  @app = app
12
12
  end
13
13
 
14
+ # Runs the Rack app for +req+ and writes its response to +res+.
15
+ #
16
+ # @param req [WEBrick::HTTPRequest]
17
+ # @param res [WEBrick::HTTPResponse]
18
+ # @return [void]
14
19
  def service(req, res)
15
20
  status, headers, body = @app.call(env_for(req))
16
21
  res.status = status
@@ -13,6 +13,8 @@ module Cosmo
13
13
  @check = check || method(:default_check)
14
14
  end
15
15
 
16
+ # @param env [Hash] the Rack env
17
+ # @return [Array] the Rack response
16
18
  def call(env)
17
19
  return @app.call(env) unless env[Rack::PATH_INFO] == PATH
18
20
  return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
@@ -12,6 +12,8 @@ module Cosmo
12
12
  @app = app
13
13
  end
14
14
 
15
+ # @param env [Hash] the Rack env
16
+ # @return [Array] the Rack response
15
17
  def call(env)
16
18
  return @app.call(env) unless env[Rack::PATH_INFO] == PATH
17
19
  return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
@@ -19,6 +19,8 @@ module Cosmo
19
19
  NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
20
20
 
21
21
  # Routing is done by middlewares, anything unmatched falls through to 404.
22
+ #
23
+ # @return [#call] the Rack app
22
24
  def self.app
23
25
  Rack::Builder.app do
24
26
  use Health
@@ -33,6 +35,9 @@ module Cosmo
33
35
  @port = port
34
36
  end
35
37
 
38
+ # Starts serving in a background thread.
39
+ #
40
+ # @return [self]
36
41
  def start
37
42
  @server = ::WEBrick::HTTPServer.new(
38
43
  BindAddress: @host,
@@ -46,6 +51,7 @@ module Cosmo
46
51
  self
47
52
  end
48
53
 
54
+ # @return [void]
49
55
  def stop
50
56
  return unless @server
51
57
 
@@ -55,6 +61,8 @@ module Cosmo
55
61
  end
56
62
 
57
63
  # Actual bound port, useful when started with port 0.
64
+ #
65
+ # @return [Integer]
58
66
  def port
59
67
  @server ? @server.listeners.first.addr[1] : @port
60
68
  end
@@ -7,12 +7,17 @@ module Cosmo
7
7
  class Data
8
8
  DEFAULTS = { stream: :default, retry: 3, dead: true, limit: nil }.freeze
9
9
 
10
+ # @return [Integer] +max_retries+ from the config, or the built-in default
10
11
  def self.default_retry
11
12
  Config[:max_retries] || DEFAULTS[:retry]
12
13
  end
13
14
 
14
15
  attr_reader :jid
15
16
 
17
+ # @param class_name [String]
18
+ # @param args [Array]
19
+ # @param options [Hash, nil] job options, +stream:+ is required
20
+ # @raise [ArgumentError] when no stream is given
16
21
  def initialize(class_name, args, options = nil)
17
22
  @class_name = class_name
18
23
  @args = args
@@ -26,10 +31,13 @@ module Cosmo
26
31
  @jid = SecureRandom.hex(12)
27
32
  end
28
33
 
34
+ # @return [String, nil] the id of the batch the job belongs to
29
35
  def batch_id
30
36
  @options[:batch_id]
31
37
  end
32
38
 
39
+ # @param target [Boolean] true returns the job's own stream even when it is delayed
40
+ # @return [Symbol] the stream the job is published to: +scheduled+ for a delayed job unless +target+
33
41
  def stream(target: false)
34
42
  return @options[:stream] if target
35
43
 
@@ -44,18 +52,21 @@ module Cosmo
44
52
 
45
53
  # @return [Hash{String => String, Integer}] dedup id, plus where to dispatch a delayed job once it is due
46
54
  def headers
47
- headers = { "Nats-Msg-Id" => jid }
55
+ headers = { NATS::JetStream::Header::MSG_ID => jid }
48
56
  return headers unless @at
49
57
 
50
58
  target = stream(target: true)
51
59
  headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
52
60
  end
53
61
 
62
+ # @return [Hash{Symbol => Object}] the published payload
54
63
  def as_json
55
64
  json = { jid: jid, class: @class_name, args: @args, retry: retries, dead: dead }
56
65
  batch_id ? json.merge(batch_id: batch_id) : json
57
66
  end
58
67
 
68
+ # @param _args [Array] ignored, accepted for +JSON.generate+ compatibility
69
+ # @return [String, nil]
59
70
  def to_json(*_args)
60
71
  Utils::Json.dump(as_json)
61
72
  end
@@ -4,6 +4,9 @@ module Cosmo
4
4
  module Job
5
5
  # The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
6
6
  module Enqueuer
7
+ # Jobs published at once by {.enqueue_many}, before waiting for their acks.
8
+ CHUNK_SIZE = 1_000
9
+
7
10
  module_function
8
11
 
9
12
  # Publishes the job through {Config#client_middleware}.
@@ -17,7 +20,7 @@ module Cosmo
17
20
  # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
18
21
  # @raise [StreamNotFoundError] when the target stream does not exist
19
22
  # @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
20
- def enqueue(class_name, args, options, batch: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
23
+ def enqueue(class_name, args, options, batch: nil)
21
24
  options = options.merge(batch_id: batch.bid) if batch
22
25
  data = Data.new(class_name, args, options)
23
26
  raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
@@ -33,6 +36,98 @@ module Cosmo
33
36
  data.jid if published
34
37
  end
35
38
 
39
+ # Enqueues one job per argument list, publishing them a chunk at a time with {Client#publish_many}. Each job
40
+ # still runs through {Config#client_middleware}, whose block only marks it for publishing: the chunk is
41
+ # published once every job in it went through the chain.
42
+ #
43
+ # @param class_name [String]
44
+ # @param list [Array<Array>] each job's arguments
45
+ # @param options [Hash] job options shared by every job, see {Data}
46
+ # @param batch [Batch, nil] the batch the jobs join
47
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
48
+ # @raise [ArgumentError] when +list+ is not an array of argument arrays
49
+ # @raise [BulkEnqueueError] when the stream refused jobs of a chunk; the chunks after it are not published
50
+ # @raise [SchedulingDisabledError] for delayed jobs while scheduling is turned off
51
+ def enqueue_many(class_name, list, options, batch: nil)
52
+ raise ArgumentError, "expected an array of argument arrays" unless list.is_a?(::Array) && list.all?(::Array)
53
+
54
+ options = options.merge(batch_id: batch.bid) if batch
55
+ jids = []
56
+ list.each_slice(CHUNK_SIZE) do |chunk|
57
+ offset = jids.size
58
+ chunk_jids, errors = publish_chunk(class_name, chunk, options, batch)
59
+ jids.concat(chunk_jids)
60
+ raise BulkEnqueueError.new(jids + Array.new(list.size - jids.size), errors.transform_keys { _1 + offset }) if errors.any?
61
+ end
62
+ jids
63
+ end
64
+
65
+ # @param class_name [String]
66
+ # @param chunk [Array<Array>] each job's arguments
67
+ # @param options [Hash]
68
+ # @param batch [Batch, nil]
69
+ # @return [Array(Array<String, nil>, Hash{Integer => StandardError})] each job's jid, and why the jobs the
70
+ # stream refused were not enqueued, by their index in +chunk+
71
+ def publish_chunk(class_name, chunk, options, batch)
72
+ jobs = chunk.map { |args| through_middleware(class_name, Data.new(class_name, args, options)) }
73
+ published = jobs.compact
74
+ return [jobs.map { nil }, {}] if published.empty?
75
+
76
+ batch&.register_job!(published.size)
77
+ results = publish_all(published, batch)
78
+ batch&.rollback_job!(results.count { !_1.is_a?(NATS::JetStream::PubAck) })
79
+ collect(jobs, results)
80
+ end
81
+
82
+ # @param class_name [String]
83
+ # @param data [Data]
84
+ # @return [Array(Data, Hash), nil] the job and the payload to publish, or nil when a client middleware stopped it
85
+ def through_middleware(class_name, data)
86
+ raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
87
+
88
+ payload = data.as_json
89
+ job = nil
90
+ Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) { job = [data, payload] }
91
+ job
92
+ end
93
+
94
+ # @param jobs [Array<Array(Data, Hash)>]
95
+ # @param batch [Batch, nil] whose slots for every job are released when the publish raises
96
+ # @return [Array<NATS::JetStream::PubAck, StandardError>]
97
+ def publish_all(jobs, batch)
98
+ messages = jobs.map do |(data, payload)|
99
+ header = data.headers.merge(NATS::JetStream::Header::EXPECTED_STREAM => data.stream.to_s)
100
+ { subject: data.subject, payload: Utils::Json.dump(payload), header: header }
101
+ end
102
+ Client.instance.publish_many(messages)
103
+ rescue StandardError
104
+ batch&.rollback_job!(jobs.size)
105
+ raise
106
+ end
107
+
108
+ # @param jobs [Array<Array(Data, Hash), nil>] nil for a job a client middleware stopped
109
+ # @param results [Array<NATS::JetStream::PubAck, StandardError>] one per published job
110
+ # @return [Array(Array<String, nil>, Hash{Integer => StandardError})]
111
+ def collect(jobs, results)
112
+ results = results.each
113
+ errors = {}
114
+ jids = jobs.each_with_index.map do |job, index|
115
+ next unless job
116
+
117
+ result = results.next
118
+ next job.first.jid if result.is_a?(NATS::JetStream::PubAck)
119
+
120
+ errors[index] = result.is_a?(NATS::JetStream::Error::NoStreamResponse) ? StreamNotFoundError.new(job.first.stream.to_s) : result
121
+ nil
122
+ end
123
+ [jids, errors]
124
+ end
125
+
126
+ # Runs the client middleware around the publish.
127
+ #
128
+ # @param class_name [String]
129
+ # @param data [Data]
130
+ # @return [Boolean] whether the job was published
36
131
  def pipe(class_name, data)
37
132
  payload = data.as_json
38
133
  published = false
@@ -43,6 +138,10 @@ module Cosmo
43
138
  published
44
139
  end
45
140
 
141
+ # @param data [Data]
142
+ # @param payload [Hash] the job's payload, possibly changed by a middleware
143
+ # @return [NATS::JetStream::PubAck]
144
+ # @raise [StreamNotFoundError] when the target stream does not exist
46
145
  def publish(data, payload)
47
146
  Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
48
147
  rescue NATS::JetStream::Error::NoStreamResponse
@@ -15,12 +15,13 @@ module Cosmo
15
15
  class Limit
16
16
  BUCKET = "_cosmolimits"
17
17
 
18
+ # @return [Limit]
18
19
  def self.instance
19
20
  @instance ||= new
20
21
  end
21
22
 
22
23
  def initialize
23
- @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.replicas)
24
+ @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.services.num_replicas)
24
25
  end
25
26
 
26
27
  # Try to acquire one of the numbered slots for +key+.
@@ -45,6 +46,9 @@ module Cosmo
45
46
  # tombstone left behind) so a released slot looks identical to one
46
47
  # reclaimed by Nats-TTL expiry -- callers never have to special-case a
47
48
  # delete marker.
49
+ #
50
+ # @param slot [String] the slot key returned by {#acquire}
51
+ # @return [void]
48
52
  def release(slot)
49
53
  @kv.erase(slot)
50
54
  rescue NATS::Error
@@ -8,9 +8,12 @@ module Cosmo
8
8
  # Headers the scheduler consumes rather than forwards. The +X-+ ones address the dispatch
9
9
  # itself, and NATS sets +Nats-Scheduler+/+Nats-Schedule-Next+ on a message it fires and
10
10
  # rejects a publish that carries them back.
11
- DISPATCH_HEADERS = %w[X-Stream X-Subject X-Execute-At Nats-Expected-Stream
12
- Nats-Scheduler Nats-Schedule-Next].freeze
11
+ DISPATCH_HEADERS = ["X-Stream", "X-Subject", "X-Execute-At", NATS::JetStream::Header::EXPECTED_STREAM,
12
+ NATS::JetStream::Header::SCHEDULER, NATS::JetStream::Header::SCHEDULE_NEXT].freeze
13
+ ARGS_LIMIT = 256
13
14
 
15
+ # @param options [Hash] the CLI options
16
+ # @return [void]
14
17
  # @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
15
18
  def self.validate_options!(options)
16
19
  StreamFilter.from(options[:streams]).validate!
@@ -23,13 +26,33 @@ module Cosmo
23
26
  names
24
27
  end
25
28
 
29
+ # @return [Array<Hash>] the jobs running right now, each with its +:jid+, +:class+, +:args+ (cut to
30
+ # ARGS_LIMIT bytes), +:retry+, +:stream+, delivery attempt as +:delivery+ and +:started_at+
31
+ def jobs
32
+ @in_flight.each_pair.flat_map do |thread, messages|
33
+ started_at = @started_at[thread]
34
+ messages.map { job_entry(_1, started_at) }
35
+ end
36
+ end
37
+
26
38
  private
27
39
 
40
+ def job_entry(message, started_at)
41
+ data = Utils::Json.parse(message.data) || {}
42
+ meta = message.metadata
43
+ { jid: data[:jid], class: data[:class], args: capped_args(data[:args]), retry: data[:retry],
44
+ stream: meta.stream, delivery: meta.num_delivered, started_at: }
45
+ end
46
+
47
+ def capped_args(args)
48
+ json = Utils::Json.dump(args)
49
+ json.bytesize > ARGS_LIMIT ? "#{json.byteslice(0, ARGS_LIMIT).scrub("")}…" : args
50
+ end
51
+
28
52
  def setup
29
53
  filter = StreamFilter.from(@options[:streams]).validate!
30
54
 
31
55
  # Initialize singletons before starting to process messages
32
- API::Stats::Busy.instance
33
56
  API::Stats::Totals.instance
34
57
  Limit.instance
35
58
  Config.server_middleware
@@ -40,6 +63,8 @@ module Cosmo
40
63
  next unless filter.include?(stream_name)
41
64
 
42
65
  @consumers << subscribe(stream_name, config)
66
+ rescue NATS::JetStream::Error::NotFound
67
+ Logger.warn "stream #{stream_name} does not exist, skipping it (run `cosmo --setup` to create it)"
43
68
  end
44
69
 
45
70
  log_subscriptions
@@ -71,8 +96,7 @@ module Cosmo
71
96
  messages&.each do |message|
72
97
  headers = message.header.except(*DISPATCH_HEADERS)
73
98
  stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
74
- headers["Nats-Expected-Stream"] = stream
75
- scheduler = message.header["Nats-Scheduler"]
99
+ scheduler = message.header[NATS::JetStream::Header::SCHEDULER]
76
100
  if scheduler
77
101
  headers["X-Scheduled-By"] = scheduler
78
102
  headers["X-Enqueued-At"] = message.metadata.timestamp.to_f.to_s
@@ -80,7 +104,7 @@ module Cosmo
80
104
  execute_at = execute_at.to_i
81
105
 
82
106
  if now >= execute_at
83
- client.publish(subject, message.data, header: headers)
107
+ client.publish(subject, message.data, header: headers, stream:)
84
108
  message.ack
85
109
  else
86
110
  message.nak(delay: Config.to_ns(execute_at - now))
@@ -104,19 +128,13 @@ module Cosmo
104
128
  data = Utils::Json.parse(message.data)
105
129
  return reject_message(message, ArgumentError.new("malformed payload")) unless data
106
130
 
107
- worker_class = Utils::String.safe_constantize(data[:class])
108
- unless worker_class
109
- reject_message(message, ArgumentError.new("#{data[:class]} class not found"), data)
110
- notify_batch(data, success: false)
111
- return
112
- end
113
-
114
131
  begin
115
132
  sw = stopwatch
116
133
  Logger.with(jid: data[:jid])
117
134
  Logger.info "start"
118
135
 
119
- instance = build_worker(worker_class, data, message)
136
+ worker_class = Utils::String.safe_constantize(data[:class])
137
+ instance = build_worker(worker_class, data, message) if worker_class
120
138
  Config.server_middleware.invoke(instance, data, message) do
121
139
  perform_job(instance, data: data, message: message)
122
140
  end
@@ -127,7 +145,9 @@ module Cosmo
127
145
  rescue Requeue => e
128
146
  message.nak(delay: Config.to_ns(e.delay))
129
147
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "requeue[#{e.delay}s]" }
130
- rescue Timeout::Error => e
148
+ rescue NATS::JetStream::Error::MsgAlreadyAckd
149
+ Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "released[shutdown]" }
150
+ rescue Timeout::Error, JobTimeoutError => e
131
151
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
132
152
  Cosmo.handle_error(e, error_context(message, data))
133
153
  handle_failure(worker_class, message, data, e)
@@ -171,6 +191,16 @@ module Cosmo
171
191
  true
172
192
  end
173
193
 
194
+ # A job still running when the shutdown timed out counts as a failed attempt: it's retried after the
195
+ # usual delay, or moved to the DLQ (dropped when +dead+ is off) once it has no attempts left.
196
+ def abandon(message)
197
+ data = Utils::Json.parse(message.data)
198
+ return super unless data
199
+
200
+ API::Stats::Totals.instance.increment(:failed)
201
+ handle_failure(Utils::String.safe_constantize(data[:class]), message, data, JobInterruptedError.new)
202
+ end
203
+
174
204
  def notify_batch(data, success:)
175
205
  return unless data[:batch_id]
176
206
 
@@ -201,7 +231,7 @@ module Cosmo
201
231
  end
202
232
 
203
233
  def retry_delay(worker_class, data, current_attempt, exception)
204
- handler = worker_class.retry_in(data)
234
+ handler = worker_class&.retry_in(data)
205
235
  return default_retry_delay(current_attempt) unless handler
206
236
 
207
237
  delay = handler.call(current_attempt, exception)
@@ -231,10 +261,11 @@ module Cosmo
231
261
  end
232
262
 
233
263
  # Logs why a message can't be processed at all and parks it in the DLQ.
234
- def reject_message(message, error, data = nil)
264
+ def reject_message(message, error)
235
265
  Logger.error error
236
- Cosmo.handle_error(error, error_context(message, data, source: :reject))
237
- move_message(message, data, error)
266
+ Cosmo.handle_error(error, error_context(message, nil, source: :reject))
267
+ API::Stats::Totals.instance.increment(:failed)
268
+ move_message(message, nil, error)
238
269
  end
239
270
 
240
271
  def error_context(message, data, source: :job)
@@ -262,7 +293,7 @@ module Cosmo
262
293
  def scheduled_by(message)
263
294
  header = message.header or return
264
295
 
265
- header["X-Scheduled-By"] || header["Nats-Scheduler"]
296
+ header["X-Scheduled-By"] || header[NATS::JetStream::Header::SCHEDULER]
266
297
  end
267
298
 
268
299
  # A cron message is enqueued when NATS fires it, not when the scheduler dispatches it on, so
@@ -291,12 +322,16 @@ module Cosmo
291
322
  ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
292
323
  end
293
324
 
294
- # @param job_instance [Cosmo::Job]
325
+ # @param job_instance [Cosmo::Job, nil] nil when this worker can't load the job's class
295
326
  # @param data [Hash]
296
327
  # @param message [NATS::Msg]
328
+ # @return [Object] what the job's +perform+ returns
329
+ # @raise [JobClassNotFoundError] when +job_instance+ is nil
297
330
  #
298
331
  # rubocop:disable-next Lint/UnusedMethodArgument
299
332
  def perform_job(job_instance, data:, message:)
333
+ raise JobClassNotFoundError, data[:class] unless job_instance
334
+
300
335
  job_instance.perform(*data[:args])
301
336
  end
302
337
  end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cosmo
4
+ module Job
5
+ # Per-call job options, returned by {ClassMethods#set}: the enqueue methods merge them over the class options.
6
+ #
7
+ # ReportJob.set(stream: :critical, retry: 0).perform_async(42)
8
+ class Setter
9
+ # Options carried by the job's message. The rest (+limit+, +retry_in+) are read from the class by the worker.
10
+ OPTIONS = %i[stream retry dead subject].freeze
11
+
12
+ # @param job_class [Class] the job class including {Job}
13
+ # @param options [Hash] a subset of {OPTIONS}
14
+ # @raise [ArgumentError] when an option is not one of {OPTIONS}
15
+ def initialize(job_class, options)
16
+ unknown = options.keys - OPTIONS
17
+ if unknown.any?
18
+ hint = " (use stream: instead of queue:)" if unknown.include?(:queue)
19
+ raise ArgumentError, "unknown job options #{unknown.join(", ")}, expected #{OPTIONS.join(", ")}#{hint}"
20
+ end
21
+
22
+ @job_class = job_class
23
+ @options = options
24
+ end
25
+
26
+ # @param args [Array] arguments passed to the job's +perform+
27
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
28
+ def perform_async(*args)
29
+ @job_class.perform(*args, **@options)
30
+ end
31
+
32
+ # @param list [Array<Array>] each job's arguments
33
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
34
+ def perform_bulk(list)
35
+ @job_class.perform_bulk(list, **@options)
36
+ end
37
+
38
+ # @param timestamp [Time, Integer] when the job runs
39
+ # @param args [Array] arguments passed to the job's +perform+
40
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
41
+ def perform_at(timestamp, *args)
42
+ @job_class.perform(*args, **@options, at: timestamp)
43
+ end
44
+
45
+ # @param interval [Numeric] seconds from now until the job runs
46
+ # @param args [Array] arguments passed to the job's +perform+
47
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
48
+ def perform_in(interval, *args)
49
+ @job_class.perform(*args, **@options, in: interval)
50
+ end
51
+ end
52
+ end
53
+ end