cosmonats 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +73 -22
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +52 -4
  8. data/lib/cosmo/api/cron/entry.rb +34 -20
  9. data/lib/cosmo/api/cron.rb +24 -10
  10. data/lib/cosmo/api/job.rb +15 -1
  11. data/lib/cosmo/api/kv.rb +141 -12
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +20 -9
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +58 -7
  18. data/lib/cosmo/api/stream.rb +73 -24
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +217 -23
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +12 -1
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +56 -21
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/utils.rb +0 -1
  62. data/lib/cosmo/version.rb +1 -1
  63. data/lib/cosmo/web/assets/app.css +273 -629
  64. data/lib/cosmo/web/assets/app.js +158 -0
  65. data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
  66. data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
  67. data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
  68. data/lib/cosmo/web/assets/logo.svg +10 -0
  69. data/lib/cosmo/web/cache.rb +77 -0
  70. data/lib/cosmo/web/chart.rb +63 -39
  71. data/lib/cosmo/web/circuit.rb +56 -0
  72. data/lib/cosmo/web/context.rb +6 -4
  73. data/lib/cosmo/web/controllers/application.rb +42 -0
  74. data/lib/cosmo/web/controllers/batches.rb +8 -1
  75. data/lib/cosmo/web/controllers/crons.rb +24 -3
  76. data/lib/cosmo/web/controllers/jobs.rb +88 -68
  77. data/lib/cosmo/web/controllers/metrics.rb +6 -0
  78. data/lib/cosmo/web/controllers/overview.rb +88 -0
  79. data/lib/cosmo/web/controllers/processes.rb +36 -0
  80. data/lib/cosmo/web/controllers/streams.rb +38 -28
  81. data/lib/cosmo/web/helpers/application.rb +101 -0
  82. data/lib/cosmo/web/renderer.rb +36 -6
  83. data/lib/cosmo/web/throughput.rb +104 -0
  84. data/lib/cosmo/web/views/batches/_table.erb +42 -54
  85. data/lib/cosmo/web/views/batches/index.erb +3 -10
  86. data/lib/cosmo/web/views/crons/_table.erb +36 -55
  87. data/lib/cosmo/web/views/crons/index.erb +2 -9
  88. data/lib/cosmo/web/views/jobs/_args.erb +5 -0
  89. data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
  90. data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
  91. data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
  92. data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
  93. data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
  94. data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
  95. data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
  96. data/lib/cosmo/web/views/jobs/index.erb +4 -17
  97. data/lib/cosmo/web/views/layout.erb +73 -22
  98. data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
  99. data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
  100. data/lib/cosmo/web/views/metrics/_table.erb +24 -40
  101. data/lib/cosmo/web/views/metrics/index.erb +8 -16
  102. data/lib/cosmo/web/views/overview/_badges.erb +3 -0
  103. data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
  104. data/lib/cosmo/web/views/overview/_running.erb +18 -0
  105. data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
  106. data/lib/cosmo/web/views/overview/_workers.erb +25 -0
  107. data/lib/cosmo/web/views/overview/index.erb +28 -0
  108. data/lib/cosmo/web/views/processes/_table.erb +67 -0
  109. data/lib/cosmo/web/views/processes/index.erb +1 -0
  110. data/lib/cosmo/web/views/shared/_pager.erb +22 -0
  111. data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
  112. data/lib/cosmo/web/views/streams/_info.erb +29 -86
  113. data/lib/cosmo/web/views/streams/_list.erb +7 -0
  114. data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
  115. data/lib/cosmo/web/views/streams/_stream.erb +53 -0
  116. data/lib/cosmo/web/views/streams/index.erb +3 -11
  117. data/lib/cosmo/web/views/streams/info.erb +3 -11
  118. data/lib/cosmo/web.rb +73 -35
  119. data/lib/cosmo.rb +33 -0
  120. data/sig/cosmo/active_job/adapter.rbs +3 -3
  121. data/sig/cosmo/active_job/options.rbs +4 -2
  122. data/sig/cosmo/api/batch.rbs +5 -3
  123. data/sig/cosmo/api/counter.rbs +16 -6
  124. data/sig/cosmo/api/cron/entry.rbs +6 -3
  125. data/sig/cosmo/api/cron.rbs +10 -4
  126. data/sig/cosmo/api/kv.rbs +21 -4
  127. data/sig/cosmo/api/registry.rbs +10 -0
  128. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  129. data/sig/cosmo/api/stats/metrics.rbs +4 -1
  130. data/sig/cosmo/api/stats/processes.rbs +9 -1
  131. data/sig/cosmo/api/stats.rbs +12 -2
  132. data/sig/cosmo/api/stream.rbs +12 -5
  133. data/sig/cosmo/batch/callback.rbs +1 -1
  134. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  135. data/sig/cosmo/batch.rbs +5 -5
  136. data/sig/cosmo/cli.rbs +2 -0
  137. data/sig/cosmo/client.rbs +24 -4
  138. data/sig/cosmo/config/settings.rbs +6 -0
  139. data/sig/cosmo/config.rbs +5 -5
  140. data/sig/cosmo/engine.rbs +5 -0
  141. data/sig/cosmo/heartbeat.rbs +2 -0
  142. data/sig/cosmo/job/data.rbs +3 -1
  143. data/sig/cosmo/job/enqueuer.rbs +12 -0
  144. data/sig/cosmo/job/processor.rbs +18 -3
  145. data/sig/cosmo/job/setter.rbs +20 -0
  146. data/sig/cosmo/job.rbs +10 -4
  147. data/sig/cosmo/logger.rbs +5 -3
  148. data/sig/cosmo/middleware.rbs +0 -8
  149. data/sig/cosmo/processor.rbs +14 -2
  150. data/sig/cosmo/publisher.rbs +1 -0
  151. data/sig/cosmo/stream/processor.rbs +4 -0
  152. data/sig/cosmo/stream.rbs +4 -3
  153. data/sig/cosmo/utils/hash.rbs +2 -1
  154. data/sig/cosmo/utils/signal.rbs +5 -5
  155. data/sig/cosmo/utils/string.rbs +2 -2
  156. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  157. data/sig/cosmo/web/cache.rbs +35 -0
  158. data/sig/cosmo/web/chart.rbs +48 -0
  159. data/sig/cosmo/web/circuit.rbs +29 -0
  160. data/sig/cosmo/web/context.rbs +15 -0
  161. data/sig/cosmo/web/controllers/application.rbs +32 -0
  162. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  163. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  164. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  165. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  166. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  167. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  168. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  169. data/sig/cosmo/web/helpers/application.rbs +61 -0
  170. data/sig/cosmo/web/renderer.rbs +33 -0
  171. data/sig/cosmo/web/throughput.rbs +45 -0
  172. data/sig/cosmo/web.rbs +25 -0
  173. data/sig/cosmo.rbs +19 -0
  174. metadata +53 -23
  175. data/lib/cosmo/api/stats/busy.rb +0 -63
  176. data/lib/cosmo/middleware/busy.rb +0 -15
  177. data/lib/cosmo/utils/overrides.rb +0 -206
  178. data/lib/cosmo/utils/warnings.rb +0 -17
  179. data/lib/cosmo/web/controllers/actions.rb +0 -16
  180. data/lib/cosmo/web/views/actions/index.erb +0 -7
  181. data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
  182. data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
  183. data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
  184. data/lib/cosmo/web/views/jobs/busy.erb +0 -18
  185. data/lib/cosmo/web/views/jobs/dead.erb +0 -18
  186. data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
  187. data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
  188. data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
  189. data/lib/cosmo/web/views/streams/_table.erb +0 -25
  190. data/sig/cosmo/api/stats/busy.rbs +0 -31
data/lib/cosmo/job.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "cosmo/job/data"
4
4
  require "cosmo/job/enqueuer"
5
+ require "cosmo/job/setter"
5
6
  require "cosmo/job/requeue"
6
7
  require "cosmo/job/failure"
7
8
  require "cosmo/job/stream_filter"
@@ -10,6 +11,8 @@ require "cosmo/job/processor"
10
11
 
11
12
  module Cosmo
12
13
  module Job
14
+ # @param base [Class] the job class including Job
15
+ # @return [void]
13
16
  def self.included(base)
14
17
  base.extend(ClassMethods)
15
18
  end
@@ -28,14 +31,15 @@ module Cosmo
28
31
  # limit: { duration: 30, concurrency: { to: 3, key: ->(id) { id } } }
29
32
  # limit: { duration: 30, concurrency: 3, retry_in: 5 }
30
33
  #
31
- # @option config [Integer] :"limit[:duration]" hard execution timeout in seconds. The job thread is
34
+ # @option config [Integer, String] :"limit[:duration]" hard execution timeout in seconds, or a duration such as
35
+ # +"82m"+ (see {Utils::Duration.parse}). The job thread is
32
36
  # killed after this many seconds and counts as a failed attempt (retried with exponential backoff,
33
37
  # moved to DLQ after retries exhausted).
34
38
  # @option config [Integer, Hash] :"limit[:concurrency]" caps how many instances run at once across all
35
39
  # workers. Jobs that cannot acquire a slot are NAK'd (see +retry_in+) so they are not re-delivered until
36
40
  # the slot is likely free. Requires +duration+.
37
41
  # Pass an Integer for a class-wide cap, or <tt>{ to: N, key: ->(args) {} }</tt> to scope per key.
38
- # @option config [Integer] :"limit[:retry_in]" seconds to wait before NATS redelivers a job that was
42
+ # @option config [Integer, String] :"limit[:retry_in]" seconds, or a duration such as +"30s"+, to wait before NATS redelivers a job that was
39
43
  # NAK'd for lack of a concurrency slot (default: half of +duration+). Counts against the same delivery
40
44
  # counter as any other retry -- a job stuck behind the concurrency limit for enough consecutive
41
45
  # attempts is dropped/DLQ'd exactly like one that keeps failing outright.
@@ -50,7 +54,14 @@ module Cosmo
50
54
  # +limit[:retry_in]+) or actually ran and failed. So a job that gets turned away twice for lack of a
51
55
  # slot, then finally runs and fails, calls this handler with +count == 3+, not 1. Don't read +count+ as
52
56
  # "how many times perform has actually run and failed" when concurrency limits are in play.
57
+ # @return [Hash{Symbol => Object}] the class's merged options
58
+ # @raise [ArgumentError] when +limit[:concurrency]+ lacks a +duration+, a limit duration is invalid, or +retry_in+
59
+ # is not callable
53
60
  def options(**config)
61
+ if (limit = config[:limit])
62
+ config[:limit] = limit.merge(limit.slice(:duration, :retry_in).compact.transform_values { Utils::Duration.parse(_1) })
63
+ end
64
+
54
65
  if config[:limit] && config.dig(:limit, :concurrency) && !config.dig(:limit, :duration).to_i.positive?
55
66
  raise ArgumentError, "limit: duration is required when concurrency is set"
56
67
  end
@@ -61,6 +72,7 @@ module Cosmo
61
72
  end
62
73
  alias cosmo_options options
63
74
 
75
+ # @return [Boolean] whether +limit[:concurrency]+ is configured
64
76
  def limits_concurrency?
65
77
  !!concurrency_options
66
78
  end
@@ -68,12 +80,17 @@ module Cosmo
68
80
  # Returns the +retry_in+ Proc/lambda (taking +(count, exception)+) configured for this job class, or
69
81
  # +nil+ when unset. Overridable by wrapper job classes (e.g. the ActiveJob executor) that need to
70
82
  # resolve it from something other than +self+.
83
+ #
84
+ # @param _data [Hash, nil] the job payload, unused here
85
+ # @return [Proc, nil]
71
86
  def retry_in(_data = nil)
72
87
  default_options[:retry_in]
73
88
  end
74
89
 
75
90
  # Returns a normalized concurrency config hash, or +nil+ when not configured.
76
91
  # Always contains +:limit+, +:key+, +:duration+, and +:retry_in+.
92
+ #
93
+ # @return [Hash{Symbol => Object}, nil]
77
94
  def concurrency_options
78
95
  value = default_options.dig(:limit, :concurrency)
79
96
  return unless value
@@ -88,6 +105,9 @@ module Cosmo
88
105
  end
89
106
 
90
107
  # Derive the fully-scoped concurrency key for a given args array.
108
+ #
109
+ # @param args [Array] the job's arguments
110
+ # @return [String, nil] nil when concurrency is not limited
91
111
  def concurrency_key(args)
92
112
  config = concurrency_options
93
113
  return unless config
@@ -97,6 +117,12 @@ module Cosmo
97
117
  suffix ? "#{base}/#{suffix}" : base
98
118
  end
99
119
 
120
+ # Enqueues the job, or runs it inline when +async+ is false.
121
+ #
122
+ # @param args [Array] arguments passed to the job's +perform+
123
+ # @param async [Boolean] false runs the job in the current thread, without NATS
124
+ # @param options [Hash] job options merged over {#default_options}, e.g. +at:+ or +in:+
125
+ # @return [String, nil] the job's jid, or nil when run inline or a client middleware stopped the publish
100
126
  def perform(*args, async: true, **options)
101
127
  options = default_options.merge(options)
102
128
  return Enqueuer.enqueue(name, args, options, batch: Batch.current) if async
@@ -108,22 +134,60 @@ module Cosmo
108
134
  nil
109
135
  end
110
136
 
137
+ # Options for a single enqueue, merged over the class options.
138
+ #
139
+ # ReportJob.set(stream: :critical, retry: 0).perform_async(42)
140
+ #
141
+ # @param options [Hash] +stream+, +retry+, +dead+ or +subject+, see {#options}
142
+ # @return [Setter]
143
+ # @raise [ArgumentError] for any other option
144
+ def set(**options)
145
+ Setter.new(self, options)
146
+ end
147
+
148
+ # @param args [Array] arguments passed to the job's +perform+
149
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
111
150
  def perform_async(*args)
112
151
  perform(*args)
113
152
  end
114
153
 
154
+ # Enqueues one job per argument list, publishing a thousand at a time without waiting on each ack.
155
+ #
156
+ # ReportJob.perform_bulk([[1], [2], [3]])
157
+ #
158
+ # @param list [Array<Array>] each job's arguments
159
+ # @param options [Hash] job options for every job, merged over {#default_options}, e.g. +in:+
160
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
161
+ # @raise [BulkEnqueueError] when the stream refused some jobs; its +jids+ tell which were enqueued
162
+ def perform_bulk(list, **options)
163
+ Enqueuer.enqueue_many(name, list, default_options.merge(options), batch: Batch.current)
164
+ end
165
+
166
+ # @param timestamp [Time, Integer] when the job runs
167
+ # @param args [Array] arguments passed to the job's +perform+
168
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
115
169
  def perform_at(timestamp, *args)
116
170
  perform(*args, at: timestamp)
117
171
  end
118
172
 
173
+ # @param interval [Numeric] seconds from now until the job runs
174
+ # @param args [Array] arguments passed to the job's +perform+
175
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
119
176
  def perform_in(interval, *args)
120
177
  perform(*args, in: interval)
121
178
  end
122
179
 
180
+ # Runs the job inline, without NATS.
181
+ #
182
+ # @param args [Array] arguments passed to the job's +perform+
183
+ # @return [nil]
123
184
  def perform_sync(*args)
124
185
  perform(*args, async: false)
125
186
  end
126
187
 
188
+ # Options inherited from the superclass job, or the built-in defaults.
189
+ #
190
+ # @return [Hash{Symbol => Object}]
127
191
  def default_options
128
192
  @default_options ||= (superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS.merge(retry: Data.default_retry)).dup
129
193
  end
@@ -137,10 +201,15 @@ module Cosmo
137
201
 
138
202
  attr_accessor :jid, :batch_id, :enqueued_at, :attempt, :scheduled_by
139
203
 
204
+ # Runs the job with the arguments it was enqueued with.
205
+ #
206
+ # @return [Object]
207
+ # @raise [NotImplementedError] unless the job class overrides it
140
208
  def perform(...)
141
209
  raise NotImplementedError, "#{self.class}#perform must be implemented"
142
210
  end
143
211
 
212
+ # @return [::Logger]
144
213
  def logger
145
214
  Logger.instance
146
215
  end
data/lib/cosmo/logger.rb CHANGED
@@ -10,6 +10,11 @@ module Cosmo
10
10
  module Context
11
11
  KEY = :cosmo_context
12
12
 
13
+ # Merges +options+ into the current thread's log context, restoring the previous one after the block when given.
14
+ #
15
+ # @param options [Hash]
16
+ # @yieldreturn [Object]
17
+ # @return [Object, nil] the block's result
13
18
  def self.with(**options)
14
19
  prev = current
15
20
  Thread.current[KEY] = prev.merge(options)
@@ -18,11 +23,16 @@ module Cosmo
18
23
  Thread.current[KEY] = prev if block_given?
19
24
  end
20
25
 
26
+ # Removes +keys+ from the current thread's log context.
27
+ #
28
+ # @param keys [Array<Symbol>]
29
+ # @return [nil]
21
30
  def self.without(*keys)
22
31
  Thread.current[KEY] = current.except(*keys)
23
32
  nil
24
33
  end
25
34
 
35
+ # @return [Hash] the current thread's log context
26
36
  def self.current
27
37
  Thread.current[KEY] ||= {}
28
38
  Thread.current[KEY]
@@ -30,16 +40,23 @@ module Cosmo
30
40
  end
31
41
 
32
42
  class BaseFormatter < ::Logger::Formatter
43
+ # @return [String] the current thread's id, in base 36
33
44
  def tid
34
45
  (Thread.current.object_id ^ pid).to_s(36)
35
46
  end
36
47
 
48
+ # @return [Integer]
37
49
  def pid
38
50
  ::Process.pid
39
51
  end
40
52
  end
41
53
 
42
54
  class SimpleFormatter < BaseFormatter
55
+ # @param severity [String]
56
+ # @param time [Time]
57
+ # @param _ [String, nil] the progname, unused
58
+ # @param msg [String, Exception, Object]
59
+ # @return [String] the line, with the current {Context}
43
60
  def call(severity, time, _, msg)
44
61
  options = Context.current.compact.map { |k, v| "#{k}=#{v}" }.join(" ")
45
62
  options &&= " #{options}" unless options.empty?
@@ -49,10 +66,17 @@ module Cosmo
49
66
 
50
67
  # Adds TRACE (below DEBUG) on top of stdlib's fixed DEBUG..UNKNOWN severities.
51
68
  class Instance < ::Logger
69
+ # Logs at TRACE.
70
+ #
71
+ # @param progname [String, nil]
72
+ # @yieldreturn [String] the message, built only when TRACE is enabled
73
+ # @return [true]
52
74
  def trace(progname = nil, &)
53
75
  add(TRACE, nil, progname, &)
54
76
  end
55
77
 
78
+ # @param severity [Integer]
79
+ # @return [String]
56
80
  def format_severity(severity)
57
81
  severity == TRACE ? "TRACE" : super
58
82
  end
@@ -65,18 +89,27 @@ module Cosmo
65
89
  end
66
90
 
67
91
  # Dropped when the logger has no TRACE level (a plain +::Logger+), rather than flooding its DEBUG.
92
+ #
93
+ # @return [void]
68
94
  def self.trace(...)
69
95
  instance.trace(...) if instance.respond_to?(:trace)
70
96
  end
71
97
 
98
+ # Same as {Context.with}.
99
+ #
100
+ # @return [Object, nil]
72
101
  def self.with(...)
73
102
  Context.with(...)
74
103
  end
75
104
 
105
+ # Same as {Context.without}.
106
+ #
107
+ # @return [nil]
76
108
  def self.without(...)
77
109
  Context.without(...)
78
110
  end
79
111
 
112
+ # @return [::Logger] the logger Cosmo writes to, stdout by default
80
113
  def self.instance
81
114
  @instance ||= Instance.new($stdout).tap do |logger|
82
115
  logger.formatter = SimpleFormatter.new
@@ -84,12 +117,15 @@ module Cosmo
84
117
  end
85
118
  end
86
119
 
120
+ # @param level [Symbol, String, Integer]
121
+ # @return [Integer] the severity, {TRACE} included
87
122
  def self.coerce_level(level)
88
123
  level.to_s.downcase == "trace" ? TRACE : ::Logger::Severity.coerce(level)
89
124
  end
90
125
 
91
126
  # @param logger [::Logger] any stdlib-compatible logger. Its formatter is left alone, so context such as
92
127
  # +jid+ and +elapsed+ is only printed when it uses {SimpleFormatter}.
128
+ # @return [void]
93
129
  def self.instance=(logger)
94
130
  @instance = logger
95
131
  end
@@ -97,6 +133,7 @@ module Cosmo
97
133
  # +COSMO_LOG_LEVEL+, when set, wins over +level+.
98
134
  #
99
135
  # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
136
+ # @return [void]
100
137
  def self.level=(level)
101
138
  instance.level = coerce_level(ENV.fetch("COSMO_LOG_LEVEL", level))
102
139
  end
@@ -23,6 +23,7 @@ module Cosmo
23
23
  include Enumerable
24
24
 
25
25
  Entry = Struct.new(:klass, :args, :kwargs) do
26
+ # @return [Object] a new instance of the middleware class
26
27
  def build
27
28
  klass.new(*args, **kwargs)
28
29
  end
@@ -35,6 +36,7 @@ module Cosmo
35
36
  end
36
37
 
37
38
  # @yieldparam entry [Entry]
39
+ # @return [Array<Entry>]
38
40
  def each(&)
39
41
  @entries.each(&)
40
42
  end
@@ -42,6 +44,8 @@ module Cosmo
42
44
  # Appends +klass+, so it runs innermost, closest to the job.
43
45
  #
44
46
  # @param klass [Class]
47
+ # @param args [Array] passed to the middleware's +new+
48
+ # @param kwargs [Hash] passed to the middleware's +new+
45
49
  # @return [Chain]
46
50
  def add(klass, *args, **kwargs)
47
51
  remove(klass)
@@ -52,6 +56,8 @@ module Cosmo
52
56
  # Inserts +klass+ first, so it runs outermost.
53
57
  #
54
58
  # @param klass [Class]
59
+ # @param args [Array] passed to the middleware's +new+
60
+ # @param kwargs [Hash] passed to the middleware's +new+
55
61
  # @return [Chain]
56
62
  def prepend(klass, *args, **kwargs)
57
63
  remove(klass)
@@ -63,6 +69,8 @@ module Cosmo
63
69
  #
64
70
  # @param existing [Class]
65
71
  # @param klass [Class]
72
+ # @param args [Array] passed to the middleware's +new+
73
+ # @param kwargs [Hash] passed to the middleware's +new+
66
74
  # @return [Chain]
67
75
  def insert_before(existing, klass, *args, **kwargs)
68
76
  entry = take(klass, args, kwargs)
@@ -74,6 +82,8 @@ module Cosmo
74
82
  #
75
83
  # @param existing [Class]
76
84
  # @param klass [Class]
85
+ # @param args [Array] passed to the middleware's +new+
86
+ # @param kwargs [Hash] passed to the middleware's +new+
77
87
  # @return [Chain]
78
88
  def insert_after(existing, klass, *args, **kwargs)
79
89
  entry = take(klass, args, kwargs)
@@ -109,6 +119,8 @@ module Cosmo
109
119
 
110
120
  # Runs the block wrapped in every middleware, passing +args+ to each one's +call+.
111
121
  #
122
+ # @param args [Array] passed to every middleware's +call+
123
+ # @yieldreturn [Object]
112
124
  # @return [Object] the outermost middleware's return value, normally the block's
113
125
  def invoke(*args, &)
114
126
  return yield if empty?
@@ -8,12 +8,19 @@ module Cosmo
8
8
  # taken, and the +duration+ timeout. Jobs without +limit:+ pass straight through. It runs first in the chain,
9
9
  # so a requeued job never reaches the stats or user middleware.
10
10
  class Limit
11
- # @param job [Cosmo::Job]
11
+ # Raised into the job thread when its duration runs out. It is not a StandardError, so job code rescuing
12
+ # Timeout::Error or StandardError can't swallow it, and it is told apart from a Timeout::Error raised by the
13
+ # job's own code.
14
+ class Expired < Exception; end # rubocop:disable Lint/InheritException
15
+
16
+ # @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
12
17
  # @param data [Hash]
13
18
  # @param _message [NATS::Msg]
19
+ # @yieldreturn [Object] the job's result
20
+ # @return [Object] the block's result
14
21
  # @raise [Job::Requeue] while every concurrency slot is taken
15
22
  def call(job, data, _message, &)
16
- limit = job.class.default_options[:limit]
23
+ limit = job.class.default_options[:limit] if job
17
24
  return yield unless limit
18
25
 
19
26
  slot = acquire(job.class, data)
@@ -43,7 +50,9 @@ module Cosmo
43
50
  return yield unless duration
44
51
 
45
52
  seconds = duration.to_i
46
- Timeout.timeout(seconds, Timeout::Error, "execution expired after the #{seconds}s duration limit", &)
53
+ Timeout.timeout(seconds, Expired, &)
54
+ rescue Expired
55
+ raise JobTimeoutError, seconds
47
56
  end
48
57
  end
49
58
  end
@@ -6,14 +6,16 @@ module Cosmo
6
6
  # succeeds, a failure otherwise, and the wait from enqueue to the job's first delivery. ActiveJob jobs count under
7
7
  # their own class.
8
8
  class Metrics
9
- # @param job [Cosmo::Job]
9
+ # @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
10
10
  # @param data [Hash]
11
11
  # @param _message [NATS::Msg]
12
- def call(job, data, _message) # rubocop:disable Metrics/AbcSize
12
+ # @yieldreturn [Object] the job's result
13
+ # @return [Object] the block's result
14
+ def call(job, data, _message)
13
15
  return yield unless Config.metrics.enabled
14
16
 
15
17
  name = job_name(job, data)
16
- wait_ms = (Time.now - job.enqueued_at) * 1000 if job.attempt == 1 && job.enqueued_at
18
+ wait_ms = (Time.now - job.enqueued_at) * 1000 if job&.attempt == 1 && job.enqueued_at
17
19
  started = clock
18
20
  result = yield
19
21
  API::Stats::Metrics.instance.record(name, exec_ms: clock - started, wait_ms:)
@@ -26,7 +28,7 @@ module Cosmo
26
28
  private
27
29
 
28
30
  def job_name(job, data)
29
- return job.class.name unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
31
+ return data[:class].to_s unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
30
32
 
31
33
  data.dig(:args, 0, :job_class).to_s
32
34
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize
3
+ # rubocop:disable-next Metrics/AbcSize
4
4
  module Cosmo
5
5
  module Middleware
6
6
  # Wraps every job execution in a Sentry transaction. Errors reach Sentry through {ERROR_HANDLER}, which also
@@ -26,14 +26,16 @@ module Cosmo
26
26
  ::Sentry.capture_exception(error, contexts: { cosmonats: context }, hint: { background: true, integration: "cosmonats" })
27
27
  end
28
28
 
29
- # @param job [Cosmo::Job]
29
+ # @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
30
30
  # @param data [Hash]
31
31
  # @param message [NATS::Msg]
32
- def call(job, data, message)
32
+ # @yieldreturn [Object] the job's result
33
+ # @return [Object] the block's result
34
+ def call(_job, data, message)
33
35
  return yield unless ::Sentry.initialized?
34
36
 
35
37
  scope = ::Sentry.get_current_scope
36
- transaction_name = "#{NAME_PREFIX}/#{job.class.name}"
38
+ transaction_name = "#{NAME_PREFIX}/#{data[:class]}"
37
39
  scope.set_transaction_name(transaction_name, source: :task)
38
40
  transaction = ::Sentry.start_transaction(
39
41
  name: scope.transaction_name,
@@ -5,25 +5,19 @@ module Cosmo
5
5
  # Counts every execution as processed or failed, so a job retried twice and then succeeding adds two failures
6
6
  # and one processed. A counter that cannot be written is logged rather than raised, so stats never fail a job.
7
7
  class Totals
8
- # @param _job [Cosmo::Job]
8
+ # @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
9
9
  # @param _data [Hash]
10
10
  # @param _message [NATS::Msg]
11
+ # @yieldreturn [Object] the job's result
12
+ # @return [Object] the block's result
11
13
  def call(_job, _data, _message)
12
14
  result = yield
13
- increment(:processed)
15
+ API::Stats::Totals.instance.increment(:processed)
14
16
  result
15
17
  rescue Exception # rubocop:disable Lint/RescueException
16
- increment(:failed)
18
+ API::Stats::Totals.instance.increment(:failed)
17
19
  raise
18
20
  end
19
-
20
- private
21
-
22
- def increment(key)
23
- API::Stats::Totals.instance.increment(key)
24
- rescue StandardError => e
25
- Logger.debug "Totals #{key} counter error: #{e.class} #{e.message}"
26
- end
27
21
  end
28
22
  end
29
23
  end
@@ -2,6 +2,5 @@
2
2
 
3
3
  require "cosmo/middleware/chain"
4
4
  require "cosmo/middleware/limit"
5
- require "cosmo/middleware/busy"
6
5
  require "cosmo/middleware/totals"
7
6
  require "cosmo/middleware/metrics"
@@ -7,13 +7,16 @@ module Cosmo
7
7
  STREAM_EMPTY_BACKOFF_MAX = "5s" # Max sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
8
8
  QUIET_SLEEP = "5s" # How long to sleep between checks while in quiet mode (override via COSMO_QUIET_IDLE_SLEEP)
9
9
 
10
+ # Builds a processor and starts it, see {#run}.
11
+ #
12
+ # @return [Processor]
10
13
  def self.run(...)
11
14
  new(...).tap(&:run)
12
15
  end
13
16
 
14
17
  # Checks the command-line options this processor understands, before anything is booted or connected.
15
18
  #
16
- # @param options [Hash]
19
+ # @param _options [Hash]
17
20
  # @return [void]
18
21
  def self.validate_options!(_options); end
19
22
 
@@ -28,8 +31,13 @@ module Cosmo
28
31
  @consumers = []
29
32
  @cache = Utils::TTLCache.new
30
33
  @busy = Concurrent::AtomicFixnum.new
34
+ @in_flight = Concurrent::Map.new
35
+ @started_at = Concurrent::Map.new
31
36
  end
32
37
 
38
+ # Subscribes the consumers and, when there are any, starts fetching in the background.
39
+ #
40
+ # @return [void]
33
41
  def run
34
42
  setup
35
43
  return unless @consumers.any?
@@ -45,6 +53,37 @@ module Cosmo
45
53
  @busy.value
46
54
  end
47
55
 
56
+ # What the Web UI lists as running on this process, sent with each heartbeat.
57
+ #
58
+ # @return [Array<Hash>]
59
+ def jobs
60
+ []
61
+ end
62
+
63
+ # Hands every message still being processed back to NATS instead of leaving it for +ack_wait+: a job
64
+ # fails its attempt as usual, a stream message is NAK'd with a delay. Meant for a shutdown that timed
65
+ # out, right before the process exits; a job that finishes afterwards can't ack its released message.
66
+ #
67
+ # @return [Integer] number of messages released
68
+ def release
69
+ @in_flight.values.flatten.count { release_message(_1) }
70
+ end
71
+
72
+ # Kills the threads still processing messages and waits up to +timeout+ seconds for each to unwind,
73
+ # so their +ensure+ blocks run while the NATS connection is still up instead of during process exit.
74
+ #
75
+ # @param timeout [Numeric]
76
+ # @return [void]
77
+ def interrupt(timeout)
78
+ threads = @in_flight.keys
79
+ threads.each(&:kill)
80
+ threads.each { _1.join(timeout) }
81
+ end
82
+
83
+ # Stops fetching, unsubscribes, and waits up to +timeout+ seconds for in-flight work.
84
+ #
85
+ # @param timeout [Numeric]
86
+ # @return [void]
48
87
  def stop(timeout = Config[:timeout])
49
88
  @running.make_false
50
89
  @pool.shutdown
@@ -107,7 +146,7 @@ module Cosmo
107
146
  Logger.trace "fetched (#{messages&.size.to_i}) messages"
108
147
  if messages&.any?
109
148
  consumer_state.delete(key)
110
- track_busy { process(messages, processor) }
149
+ track_busy(messages) { process(messages, processor) }
111
150
  else
112
151
  max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
113
152
  consumer_state.compute(key) do |(count, wake_at)|
@@ -154,13 +193,31 @@ module Cosmo
154
193
  raise NotImplementedError
155
194
  end
156
195
 
157
- def track_busy
196
+ def track_busy(messages)
158
197
  @busy.increment
198
+ @started_at[Thread.current] = Time.now.to_i
199
+ @in_flight[Thread.current] = messages
159
200
  yield
160
201
  ensure
202
+ @in_flight.delete(Thread.current)
203
+ @started_at.delete(Thread.current)
161
204
  @busy.decrement
162
205
  end
163
206
 
207
+ def release_message(message)
208
+ abandon(message)
209
+ true
210
+ rescue NATS::JetStream::Error::MsgAlreadyAckd
211
+ false
212
+ rescue StandardError => e
213
+ Logger.error "Failed to release message #{message.subject}: #{e.class}: #{e.message}"
214
+ false
215
+ end
216
+
217
+ def abandon(...)
218
+ raise NotImplementedError
219
+ end
220
+
164
221
  def running?
165
222
  @running.true?
166
223
  end
@@ -10,6 +10,7 @@ module Cosmo
10
10
  delegate %i[publish] => :instance
11
11
  end
12
12
 
13
+ # @return [Publisher]
13
14
  def self.instance
14
15
  @instance ||= new
15
16
  end
@@ -18,6 +19,13 @@ module Cosmo
18
19
  @client = Client.instance
19
20
  end
20
21
 
22
+ # Serializes +data+ and publishes it to +subject+.
23
+ #
24
+ # @param subject [String]
25
+ # @param data [Object]
26
+ # @param serializer [#serialize, nil] defaults to {Stream::Serializer}
27
+ # @param options [Hash] JetStream publish options, e.g. +header:+
28
+ # @return [NATS::JetStream::PubAck]
21
29
  def publish(subject, data, serializer: nil, **options)
22
30
  payload = (serializer || Stream::Serializer).serialize(data)
23
31
  @client.publish(subject, payload, **options)
@@ -20,7 +20,7 @@ module Cosmo
20
20
 
21
21
  { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, allow_msg_schedules: true,
22
22
  duplicate_window: Config.to_ns(120), subjects: ["jobs.#{SCHEDULED}.>", "#{API::Cron::Entry::SUBJECT_PREFIX}.>"],
23
- num_replicas: Config.replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
23
+ num_replicas: Config.services.num_replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
24
24
  end
25
25
 
26
26
  # +max_deliver+ stays above 1: the scheduler naks a message that is not yet due, and one whose dispatch failed,
@@ -40,7 +40,7 @@ module Cosmo
40
40
 
41
41
  { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, duplicate_window: Config.to_ns(120),
42
42
  max_age: Config.to_ns(Utils::Duration.parse(dead.max_age)), max_msgs: dead.max_msgs, max_bytes: dead.max_bytes,
43
- subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.replicas, description: "Dead jobs", metadata: METADATA }
43
+ subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.services.num_replicas, description: "Dead jobs", metadata: METADATA }
44
44
  end
45
45
 
46
46
  # Creates or updates every enabled service stream, plus the internal totals and batch counters and, while enabled,
@@ -56,6 +56,11 @@ module Cosmo
56
56
  streams.keys
57
57
  end
58
58
 
59
+ # Creates or updates the +name+ stream.
60
+ #
61
+ # @param name [String]
62
+ # @param config [Hash]
63
+ # @return [void]
59
64
  # @raise [Error] when an existing stream has another retention policy, which NATS can't change in place
60
65
  def setup_stream(name, config)
61
66
  Client.instance.setup_stream(name, config)
@@ -11,19 +11,24 @@ module Cosmo
11
11
  delegate %i[timestamp num_delivered num_pending] => :metadata
12
12
  alias nack nak
13
13
 
14
+ # @param msg [NATS::Msg]
15
+ # @param serializer [#deserialize, nil] defaults to {Serializer}
14
16
  def initialize(msg, serializer: nil)
15
17
  @msg = msg
16
18
  @serializer = serializer || Serializer
17
19
  end
18
20
 
21
+ # @return [Object] the deserialized payload
19
22
  def data
20
23
  @serializer.deserialize(@msg.data)
21
24
  end
22
25
 
26
+ # @return [Integer]
23
27
  def stream_sequence
24
28
  metadata.sequence.stream
25
29
  end
26
30
 
31
+ # @return [Integer]
27
32
  def consumer_sequence
28
33
  metadata.sequence.consumer
29
34
  end