cosmonats 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -13
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +51 -3
  8. data/lib/cosmo/api/cron/entry.rb +24 -9
  9. data/lib/cosmo/api/cron.rb +21 -7
  10. data/lib/cosmo/api/job.rb +14 -0
  11. data/lib/cosmo/api/kv.rb +138 -9
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +18 -7
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +47 -11
  18. data/lib/cosmo/api/stream.rb +66 -22
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +208 -11
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +11 -0
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +51 -15
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/version.rb +1 -1
  62. data/lib/cosmo/web/assets/app.css +5 -3
  63. data/lib/cosmo/web/cache.rb +2 -0
  64. data/lib/cosmo/web/chart.rb +2 -2
  65. data/lib/cosmo/web/circuit.rb +1 -0
  66. data/lib/cosmo/web/context.rb +6 -0
  67. data/lib/cosmo/web/controllers/application.rb +26 -0
  68. data/lib/cosmo/web/controllers/batches.rb +7 -1
  69. data/lib/cosmo/web/controllers/crons.rb +16 -2
  70. data/lib/cosmo/web/controllers/jobs.rb +45 -16
  71. data/lib/cosmo/web/controllers/metrics.rb +5 -0
  72. data/lib/cosmo/web/controllers/overview.rb +36 -18
  73. data/lib/cosmo/web/controllers/processes.rb +8 -5
  74. data/lib/cosmo/web/controllers/streams.rb +12 -0
  75. data/lib/cosmo/web/helpers/application.rb +44 -0
  76. data/lib/cosmo/web/renderer.rb +17 -0
  77. data/lib/cosmo/web/throughput.rb +1 -1
  78. data/lib/cosmo/web/views/crons/_table.erb +29 -27
  79. data/lib/cosmo/web/views/jobs/_args.erb +2 -2
  80. data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
  81. data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
  82. data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
  83. data/lib/cosmo/web/views/layout.erb +1 -1
  84. data/lib/cosmo/web/views/overview/_badges.erb +1 -1
  85. data/lib/cosmo/web/views/overview/_running.erb +3 -3
  86. data/lib/cosmo/web/views/overview/index.erb +1 -1
  87. data/lib/cosmo/web/views/streams/_info.erb +1 -0
  88. data/lib/cosmo/web/views/streams/_stream.erb +1 -0
  89. data/lib/cosmo/web.rb +8 -1
  90. data/lib/cosmo.rb +33 -0
  91. data/sig/cosmo/active_job/adapter.rbs +3 -3
  92. data/sig/cosmo/active_job/options.rbs +4 -2
  93. data/sig/cosmo/api/batch.rbs +5 -3
  94. data/sig/cosmo/api/counter.rbs +16 -6
  95. data/sig/cosmo/api/cron/entry.rbs +4 -2
  96. data/sig/cosmo/api/cron.rbs +10 -4
  97. data/sig/cosmo/api/kv.rbs +21 -4
  98. data/sig/cosmo/api/registry.rbs +10 -0
  99. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  100. data/sig/cosmo/api/stats/metrics.rbs +3 -0
  101. data/sig/cosmo/api/stats/processes.rbs +9 -1
  102. data/sig/cosmo/api/stats.rbs +8 -2
  103. data/sig/cosmo/api/stream.rbs +10 -5
  104. data/sig/cosmo/batch/callback.rbs +1 -1
  105. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  106. data/sig/cosmo/batch.rbs +5 -5
  107. data/sig/cosmo/cli.rbs +2 -0
  108. data/sig/cosmo/client.rbs +23 -4
  109. data/sig/cosmo/config/settings.rbs +6 -0
  110. data/sig/cosmo/config.rbs +5 -5
  111. data/sig/cosmo/engine.rbs +5 -0
  112. data/sig/cosmo/heartbeat.rbs +2 -0
  113. data/sig/cosmo/job/data.rbs +3 -1
  114. data/sig/cosmo/job/enqueuer.rbs +12 -0
  115. data/sig/cosmo/job/processor.rbs +18 -3
  116. data/sig/cosmo/job/setter.rbs +20 -0
  117. data/sig/cosmo/job.rbs +10 -4
  118. data/sig/cosmo/logger.rbs +5 -3
  119. data/sig/cosmo/middleware.rbs +0 -8
  120. data/sig/cosmo/processor.rbs +14 -2
  121. data/sig/cosmo/publisher.rbs +1 -0
  122. data/sig/cosmo/stream/processor.rbs +4 -0
  123. data/sig/cosmo/stream.rbs +4 -3
  124. data/sig/cosmo/utils/hash.rbs +2 -1
  125. data/sig/cosmo/utils/signal.rbs +5 -5
  126. data/sig/cosmo/utils/string.rbs +2 -2
  127. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  128. data/sig/cosmo/web/cache.rbs +35 -0
  129. data/sig/cosmo/web/chart.rbs +48 -0
  130. data/sig/cosmo/web/circuit.rbs +29 -0
  131. data/sig/cosmo/web/context.rbs +15 -0
  132. data/sig/cosmo/web/controllers/application.rbs +32 -0
  133. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  134. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  135. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  136. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  137. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  138. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  139. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  140. data/sig/cosmo/web/helpers/application.rbs +61 -0
  141. data/sig/cosmo/web/renderer.rbs +33 -0
  142. data/sig/cosmo/web/throughput.rbs +45 -0
  143. data/sig/cosmo/web.rbs +25 -0
  144. data/sig/cosmo.rbs +19 -0
  145. metadata +23 -6
  146. data/lib/cosmo/api/stats/busy.rb +0 -68
  147. data/lib/cosmo/middleware/busy.rb +0 -15
  148. data/sig/cosmo/api/stats/busy.rbs +0 -33
data/lib/cosmo/logger.rb CHANGED
@@ -10,6 +10,11 @@ module Cosmo
10
10
  module Context
11
11
  KEY = :cosmo_context
12
12
 
13
+ # Merges +options+ into the current thread's log context, restoring the previous one after the block when given.
14
+ #
15
+ # @param options [Hash]
16
+ # @yieldreturn [Object]
17
+ # @return [Object, nil] the block's result
13
18
  def self.with(**options)
14
19
  prev = current
15
20
  Thread.current[KEY] = prev.merge(options)
@@ -18,11 +23,16 @@ module Cosmo
18
23
  Thread.current[KEY] = prev if block_given?
19
24
  end
20
25
 
26
+ # Removes +keys+ from the current thread's log context.
27
+ #
28
+ # @param keys [Array<Symbol>]
29
+ # @return [nil]
21
30
  def self.without(*keys)
22
31
  Thread.current[KEY] = current.except(*keys)
23
32
  nil
24
33
  end
25
34
 
35
+ # @return [Hash] the current thread's log context
26
36
  def self.current
27
37
  Thread.current[KEY] ||= {}
28
38
  Thread.current[KEY]
@@ -30,16 +40,23 @@ module Cosmo
30
40
  end
31
41
 
32
42
  class BaseFormatter < ::Logger::Formatter
43
+ # @return [String] the current thread's id, in base 36
33
44
  def tid
34
45
  (Thread.current.object_id ^ pid).to_s(36)
35
46
  end
36
47
 
48
+ # @return [Integer]
37
49
  def pid
38
50
  ::Process.pid
39
51
  end
40
52
  end
41
53
 
42
54
  class SimpleFormatter < BaseFormatter
55
+ # @param severity [String]
56
+ # @param time [Time]
57
+ # @param _ [String, nil] the progname, unused
58
+ # @param msg [String, Exception, Object]
59
+ # @return [String] the line, with the current {Context}
43
60
  def call(severity, time, _, msg)
44
61
  options = Context.current.compact.map { |k, v| "#{k}=#{v}" }.join(" ")
45
62
  options &&= " #{options}" unless options.empty?
@@ -49,10 +66,17 @@ module Cosmo
49
66
 
50
67
  # Adds TRACE (below DEBUG) on top of stdlib's fixed DEBUG..UNKNOWN severities.
51
68
  class Instance < ::Logger
69
+ # Logs at TRACE.
70
+ #
71
+ # @param progname [String, nil]
72
+ # @yieldreturn [String] the message, built only when TRACE is enabled
73
+ # @return [true]
52
74
  def trace(progname = nil, &)
53
75
  add(TRACE, nil, progname, &)
54
76
  end
55
77
 
78
+ # @param severity [Integer]
79
+ # @return [String]
56
80
  def format_severity(severity)
57
81
  severity == TRACE ? "TRACE" : super
58
82
  end
@@ -65,18 +89,27 @@ module Cosmo
65
89
  end
66
90
 
67
91
  # Dropped when the logger has no TRACE level (a plain +::Logger+), rather than flooding its DEBUG.
92
+ #
93
+ # @return [void]
68
94
  def self.trace(...)
69
95
  instance.trace(...) if instance.respond_to?(:trace)
70
96
  end
71
97
 
98
+ # Same as {Context.with}.
99
+ #
100
+ # @return [Object, nil]
72
101
  def self.with(...)
73
102
  Context.with(...)
74
103
  end
75
104
 
105
+ # Same as {Context.without}.
106
+ #
107
+ # @return [nil]
76
108
  def self.without(...)
77
109
  Context.without(...)
78
110
  end
79
111
 
112
+ # @return [::Logger] the logger Cosmo writes to, stdout by default
80
113
  def self.instance
81
114
  @instance ||= Instance.new($stdout).tap do |logger|
82
115
  logger.formatter = SimpleFormatter.new
@@ -84,12 +117,15 @@ module Cosmo
84
117
  end
85
118
  end
86
119
 
120
+ # @param level [Symbol, String, Integer]
121
+ # @return [Integer] the severity, {TRACE} included
87
122
  def self.coerce_level(level)
88
123
  level.to_s.downcase == "trace" ? TRACE : ::Logger::Severity.coerce(level)
89
124
  end
90
125
 
91
126
  # @param logger [::Logger] any stdlib-compatible logger. Its formatter is left alone, so context such as
92
127
  # +jid+ and +elapsed+ is only printed when it uses {SimpleFormatter}.
128
+ # @return [void]
93
129
  def self.instance=(logger)
94
130
  @instance = logger
95
131
  end
@@ -97,6 +133,7 @@ module Cosmo
97
133
  # +COSMO_LOG_LEVEL+, when set, wins over +level+.
98
134
  #
99
135
  # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
136
+ # @return [void]
100
137
  def self.level=(level)
101
138
  instance.level = coerce_level(ENV.fetch("COSMO_LOG_LEVEL", level))
102
139
  end
@@ -23,6 +23,7 @@ module Cosmo
23
23
  include Enumerable
24
24
 
25
25
  Entry = Struct.new(:klass, :args, :kwargs) do
26
+ # @return [Object] a new instance of the middleware class
26
27
  def build
27
28
  klass.new(*args, **kwargs)
28
29
  end
@@ -35,6 +36,7 @@ module Cosmo
35
36
  end
36
37
 
37
38
  # @yieldparam entry [Entry]
39
+ # @return [Array<Entry>]
38
40
  def each(&)
39
41
  @entries.each(&)
40
42
  end
@@ -42,6 +44,8 @@ module Cosmo
42
44
  # Appends +klass+, so it runs innermost, closest to the job.
43
45
  #
44
46
  # @param klass [Class]
47
+ # @param args [Array] passed to the middleware's +new+
48
+ # @param kwargs [Hash] passed to the middleware's +new+
45
49
  # @return [Chain]
46
50
  def add(klass, *args, **kwargs)
47
51
  remove(klass)
@@ -52,6 +56,8 @@ module Cosmo
52
56
  # Inserts +klass+ first, so it runs outermost.
53
57
  #
54
58
  # @param klass [Class]
59
+ # @param args [Array] passed to the middleware's +new+
60
+ # @param kwargs [Hash] passed to the middleware's +new+
55
61
  # @return [Chain]
56
62
  def prepend(klass, *args, **kwargs)
57
63
  remove(klass)
@@ -63,6 +69,8 @@ module Cosmo
63
69
  #
64
70
  # @param existing [Class]
65
71
  # @param klass [Class]
72
+ # @param args [Array] passed to the middleware's +new+
73
+ # @param kwargs [Hash] passed to the middleware's +new+
66
74
  # @return [Chain]
67
75
  def insert_before(existing, klass, *args, **kwargs)
68
76
  entry = take(klass, args, kwargs)
@@ -74,6 +82,8 @@ module Cosmo
74
82
  #
75
83
  # @param existing [Class]
76
84
  # @param klass [Class]
85
+ # @param args [Array] passed to the middleware's +new+
86
+ # @param kwargs [Hash] passed to the middleware's +new+
77
87
  # @return [Chain]
78
88
  def insert_after(existing, klass, *args, **kwargs)
79
89
  entry = take(klass, args, kwargs)
@@ -109,6 +119,8 @@ module Cosmo
109
119
 
110
120
  # Runs the block wrapped in every middleware, passing +args+ to each one's +call+.
111
121
  #
122
+ # @param args [Array] passed to every middleware's +call+
123
+ # @yieldreturn [Object]
112
124
  # @return [Object] the outermost middleware's return value, normally the block's
113
125
  def invoke(*args, &)
114
126
  return yield if empty?
@@ -8,12 +8,19 @@ module Cosmo
8
8
  # taken, and the +duration+ timeout. Jobs without +limit:+ pass straight through. It runs first in the chain,
9
9
  # so a requeued job never reaches the stats or user middleware.
10
10
  class Limit
11
- # @param job [Cosmo::Job]
11
+ # Raised into the job thread when its duration runs out. It is not a StandardError, so job code rescuing
12
+ # Timeout::Error or StandardError can't swallow it, and it is told apart from a Timeout::Error raised by the
13
+ # job's own code.
14
+ class Expired < Exception; end # rubocop:disable Lint/InheritException
15
+
16
+ # @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
12
17
  # @param data [Hash]
13
18
  # @param _message [NATS::Msg]
19
+ # @yieldreturn [Object] the job's result
20
+ # @return [Object] the block's result
14
21
  # @raise [Job::Requeue] while every concurrency slot is taken
15
22
  def call(job, data, _message, &)
16
- limit = job.class.default_options[:limit]
23
+ limit = job.class.default_options[:limit] if job
17
24
  return yield unless limit
18
25
 
19
26
  slot = acquire(job.class, data)
@@ -43,7 +50,9 @@ module Cosmo
43
50
  return yield unless duration
44
51
 
45
52
  seconds = duration.to_i
46
- Timeout.timeout(seconds, Timeout::Error, "execution expired after the #{seconds}s duration limit", &)
53
+ Timeout.timeout(seconds, Expired, &)
54
+ rescue Expired
55
+ raise JobTimeoutError, seconds
47
56
  end
48
57
  end
49
58
  end
@@ -6,14 +6,16 @@ module Cosmo
6
6
  # succeeds, a failure otherwise, and the wait from enqueue to the job's first delivery. ActiveJob jobs count under
7
7
  # their own class.
8
8
  class Metrics
9
- # @param job [Cosmo::Job]
9
+ # @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
10
10
  # @param data [Hash]
11
11
  # @param _message [NATS::Msg]
12
- def call(job, data, _message) # rubocop:disable Metrics/AbcSize
12
+ # @yieldreturn [Object] the job's result
13
+ # @return [Object] the block's result
14
+ def call(job, data, _message)
13
15
  return yield unless Config.metrics.enabled
14
16
 
15
17
  name = job_name(job, data)
16
- wait_ms = (Time.now - job.enqueued_at) * 1000 if job.attempt == 1 && job.enqueued_at
18
+ wait_ms = (Time.now - job.enqueued_at) * 1000 if job&.attempt == 1 && job.enqueued_at
17
19
  started = clock
18
20
  result = yield
19
21
  API::Stats::Metrics.instance.record(name, exec_ms: clock - started, wait_ms:)
@@ -26,7 +28,7 @@ module Cosmo
26
28
  private
27
29
 
28
30
  def job_name(job, data)
29
- return job.class.name unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
31
+ return data[:class].to_s unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
30
32
 
31
33
  data.dig(:args, 0, :job_class).to_s
32
34
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize
3
+ # rubocop:disable-next Metrics/AbcSize
4
4
  module Cosmo
5
5
  module Middleware
6
6
  # Wraps every job execution in a Sentry transaction. Errors reach Sentry through {ERROR_HANDLER}, which also
@@ -26,14 +26,16 @@ module Cosmo
26
26
  ::Sentry.capture_exception(error, contexts: { cosmonats: context }, hint: { background: true, integration: "cosmonats" })
27
27
  end
28
28
 
29
- # @param job [Cosmo::Job]
29
+ # @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
30
30
  # @param data [Hash]
31
31
  # @param message [NATS::Msg]
32
- def call(job, data, message)
32
+ # @yieldreturn [Object] the job's result
33
+ # @return [Object] the block's result
34
+ def call(_job, data, message)
33
35
  return yield unless ::Sentry.initialized?
34
36
 
35
37
  scope = ::Sentry.get_current_scope
36
- transaction_name = "#{NAME_PREFIX}/#{job.class.name}"
38
+ transaction_name = "#{NAME_PREFIX}/#{data[:class]}"
37
39
  scope.set_transaction_name(transaction_name, source: :task)
38
40
  transaction = ::Sentry.start_transaction(
39
41
  name: scope.transaction_name,
@@ -5,25 +5,19 @@ module Cosmo
5
5
  # Counts every execution as processed or failed, so a job retried twice and then succeeding adds two failures
6
6
  # and one processed. A counter that cannot be written is logged rather than raised, so stats never fail a job.
7
7
  class Totals
8
- # @param _job [Cosmo::Job]
8
+ # @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
9
9
  # @param _data [Hash]
10
10
  # @param _message [NATS::Msg]
11
+ # @yieldreturn [Object] the job's result
12
+ # @return [Object] the block's result
11
13
  def call(_job, _data, _message)
12
14
  result = yield
13
- increment(:processed)
15
+ API::Stats::Totals.instance.increment(:processed)
14
16
  result
15
17
  rescue Exception # rubocop:disable Lint/RescueException
16
- increment(:failed)
18
+ API::Stats::Totals.instance.increment(:failed)
17
19
  raise
18
20
  end
19
-
20
- private
21
-
22
- def increment(key)
23
- API::Stats::Totals.instance.increment(key)
24
- rescue StandardError => e
25
- Logger.debug "Totals #{key} counter error: #{e.class} #{e.message}"
26
- end
27
21
  end
28
22
  end
29
23
  end
@@ -2,6 +2,5 @@
2
2
 
3
3
  require "cosmo/middleware/chain"
4
4
  require "cosmo/middleware/limit"
5
- require "cosmo/middleware/busy"
6
5
  require "cosmo/middleware/totals"
7
6
  require "cosmo/middleware/metrics"
@@ -7,13 +7,16 @@ module Cosmo
7
7
  STREAM_EMPTY_BACKOFF_MAX = "5s" # Max sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
8
8
  QUIET_SLEEP = "5s" # How long to sleep between checks while in quiet mode (override via COSMO_QUIET_IDLE_SLEEP)
9
9
 
10
+ # Builds a processor and starts it, see {#run}.
11
+ #
12
+ # @return [Processor]
10
13
  def self.run(...)
11
14
  new(...).tap(&:run)
12
15
  end
13
16
 
14
17
  # Checks the command-line options this processor understands, before anything is booted or connected.
15
18
  #
16
- # @param options [Hash]
19
+ # @param _options [Hash]
17
20
  # @return [void]
18
21
  def self.validate_options!(_options); end
19
22
 
@@ -28,8 +31,13 @@ module Cosmo
28
31
  @consumers = []
29
32
  @cache = Utils::TTLCache.new
30
33
  @busy = Concurrent::AtomicFixnum.new
34
+ @in_flight = Concurrent::Map.new
35
+ @started_at = Concurrent::Map.new
31
36
  end
32
37
 
38
+ # Subscribes the consumers and, when there are any, starts fetching in the background.
39
+ #
40
+ # @return [void]
33
41
  def run
34
42
  setup
35
43
  return unless @consumers.any?
@@ -45,6 +53,37 @@ module Cosmo
45
53
  @busy.value
46
54
  end
47
55
 
56
+ # What the Web UI lists as running on this process, sent with each heartbeat.
57
+ #
58
+ # @return [Array<Hash>]
59
+ def jobs
60
+ []
61
+ end
62
+
63
+ # Hands every message still being processed back to NATS instead of leaving it for +ack_wait+: a job
64
+ # fails its attempt as usual, a stream message is NAK'd with a delay. Meant for a shutdown that timed
65
+ # out, right before the process exits; a job that finishes afterwards can't ack its released message.
66
+ #
67
+ # @return [Integer] number of messages released
68
+ def release
69
+ @in_flight.values.flatten.count { release_message(_1) }
70
+ end
71
+
72
+ # Kills the threads still processing messages and waits up to +timeout+ seconds for each to unwind,
73
+ # so their +ensure+ blocks run while the NATS connection is still up instead of during process exit.
74
+ #
75
+ # @param timeout [Numeric]
76
+ # @return [void]
77
+ def interrupt(timeout)
78
+ threads = @in_flight.keys
79
+ threads.each(&:kill)
80
+ threads.each { _1.join(timeout) }
81
+ end
82
+
83
+ # Stops fetching, unsubscribes, and waits up to +timeout+ seconds for in-flight work.
84
+ #
85
+ # @param timeout [Numeric]
86
+ # @return [void]
48
87
  def stop(timeout = Config[:timeout])
49
88
  @running.make_false
50
89
  @pool.shutdown
@@ -107,7 +146,7 @@ module Cosmo
107
146
  Logger.trace "fetched (#{messages&.size.to_i}) messages"
108
147
  if messages&.any?
109
148
  consumer_state.delete(key)
110
- track_busy { process(messages, processor) }
149
+ track_busy(messages) { process(messages, processor) }
111
150
  else
112
151
  max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
113
152
  consumer_state.compute(key) do |(count, wake_at)|
@@ -154,13 +193,31 @@ module Cosmo
154
193
  raise NotImplementedError
155
194
  end
156
195
 
157
- def track_busy
196
+ def track_busy(messages)
158
197
  @busy.increment
198
+ @started_at[Thread.current] = Time.now.to_i
199
+ @in_flight[Thread.current] = messages
159
200
  yield
160
201
  ensure
202
+ @in_flight.delete(Thread.current)
203
+ @started_at.delete(Thread.current)
161
204
  @busy.decrement
162
205
  end
163
206
 
207
+ def release_message(message)
208
+ abandon(message)
209
+ true
210
+ rescue NATS::JetStream::Error::MsgAlreadyAckd
211
+ false
212
+ rescue StandardError => e
213
+ Logger.error "Failed to release message #{message.subject}: #{e.class}: #{e.message}"
214
+ false
215
+ end
216
+
217
+ def abandon(...)
218
+ raise NotImplementedError
219
+ end
220
+
164
221
  def running?
165
222
  @running.true?
166
223
  end
@@ -10,6 +10,7 @@ module Cosmo
10
10
  delegate %i[publish] => :instance
11
11
  end
12
12
 
13
+ # @return [Publisher]
13
14
  def self.instance
14
15
  @instance ||= new
15
16
  end
@@ -18,6 +19,13 @@ module Cosmo
18
19
  @client = Client.instance
19
20
  end
20
21
 
22
+ # Serializes +data+ and publishes it to +subject+.
23
+ #
24
+ # @param subject [String]
25
+ # @param data [Object]
26
+ # @param serializer [#serialize, nil] defaults to {Stream::Serializer}
27
+ # @param options [Hash] JetStream publish options, e.g. +header:+
28
+ # @return [NATS::JetStream::PubAck]
21
29
  def publish(subject, data, serializer: nil, **options)
22
30
  payload = (serializer || Stream::Serializer).serialize(data)
23
31
  @client.publish(subject, payload, **options)
@@ -20,7 +20,7 @@ module Cosmo
20
20
 
21
21
  { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, allow_msg_schedules: true,
22
22
  duplicate_window: Config.to_ns(120), subjects: ["jobs.#{SCHEDULED}.>", "#{API::Cron::Entry::SUBJECT_PREFIX}.>"],
23
- num_replicas: Config.replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
23
+ num_replicas: Config.services.num_replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
24
24
  end
25
25
 
26
26
  # +max_deliver+ stays above 1: the scheduler naks a message that is not yet due, and one whose dispatch failed,
@@ -40,7 +40,7 @@ module Cosmo
40
40
 
41
41
  { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, duplicate_window: Config.to_ns(120),
42
42
  max_age: Config.to_ns(Utils::Duration.parse(dead.max_age)), max_msgs: dead.max_msgs, max_bytes: dead.max_bytes,
43
- subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.replicas, description: "Dead jobs", metadata: METADATA }
43
+ subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.services.num_replicas, description: "Dead jobs", metadata: METADATA }
44
44
  end
45
45
 
46
46
  # Creates or updates every enabled service stream, plus the internal totals and batch counters and, while enabled,
@@ -56,6 +56,11 @@ module Cosmo
56
56
  streams.keys
57
57
  end
58
58
 
59
+ # Creates or updates the +name+ stream.
60
+ #
61
+ # @param name [String]
62
+ # @param config [Hash]
63
+ # @return [void]
59
64
  # @raise [Error] when an existing stream has another retention policy, which NATS can't change in place
60
65
  def setup_stream(name, config)
61
66
  Client.instance.setup_stream(name, config)
@@ -11,19 +11,24 @@ module Cosmo
11
11
  delegate %i[timestamp num_delivered num_pending] => :metadata
12
12
  alias nack nak
13
13
 
14
+ # @param msg [NATS::Msg]
15
+ # @param serializer [#deserialize, nil] defaults to {Serializer}
14
16
  def initialize(msg, serializer: nil)
15
17
  @msg = msg
16
18
  @serializer = serializer || Serializer
17
19
  end
18
20
 
21
+ # @return [Object] the deserialized payload
19
22
  def data
20
23
  @serializer.deserialize(@msg.data)
21
24
  end
22
25
 
26
+ # @return [Integer]
23
27
  def stream_sequence
24
28
  metadata.sequence.stream
25
29
  end
26
30
 
31
+ # @return [Integer]
27
32
  def consumer_sequence
28
33
  metadata.sequence.consumer
29
34
  end
@@ -3,6 +3,8 @@
3
3
  module Cosmo
4
4
  module Stream
5
5
  class Processor < ::Cosmo::Processor
6
+ RELEASE_DELAY = "10s" # How long NATS holds a message released by a timed out shutdown before redelivering it (override via COSMO_STREAMS_RELEASE_DELAY)
7
+
6
8
  # @return [Array<String>] names of the stream classes being processed
7
9
  def subscriptions
8
10
  @consumers.map { |(_, config, _)| config[:class].name }
@@ -10,6 +12,11 @@ module Cosmo
10
12
 
11
13
  private
12
14
 
15
+ def abandon(message)
16
+ delay = Utils::Duration.parse(ENV.fetch("COSMO_STREAMS_RELEASE_DELAY", RELEASE_DELAY))
17
+ message.nak(delay: Config.to_ns(delay))
18
+ end
19
+
13
20
  def setup
14
21
  @configs ||= []
15
22
  @configs = static_config + dynamic_config
@@ -7,10 +7,14 @@ module Cosmo
7
7
  module Serializer
8
8
  module_function
9
9
 
10
+ # @param data [Object]
11
+ # @return [String, nil] JSON, or nil when +data+ cannot be dumped
10
12
  def serialize(data)
11
13
  Utils::Json.dump(data)
12
14
  end
13
15
 
16
+ # @param payload [String]
17
+ # @return [Object, nil] the parsed JSON with string keys, or nil when it is malformed
14
18
  def deserialize(payload)
15
19
  Utils::Json.parse(payload, symbolize_names: false)
16
20
  end
data/lib/cosmo/stream.rb CHANGED
@@ -7,28 +7,50 @@ require "cosmo/stream/serializer"
7
7
 
8
8
  module Cosmo
9
9
  module Stream
10
+ # @param base [Class] the stream class including Stream
11
+ # @return [void]
10
12
  def self.included(base)
11
13
  base.extend(ClassMethods)
12
14
  end
13
15
 
14
16
  module ClassMethods
17
+ # Registers the class as a stream processor and merges the given options over its defaults.
18
+ #
19
+ # @param stream [Symbol, String, nil] NATS stream to consume
20
+ # @param consumer_name [String, nil] durable consumer name
21
+ # @param batch_size [Integer, nil] messages fetched per batch
22
+ # @param fetch_timeout [Numeric, nil] seconds to wait for a batch
23
+ # @param start_position [Symbol, String, Time, nil] where a new consumer starts delivering from
24
+ # @param consumer [Hash, nil] consumer config, e.g. +subjects:+
25
+ # @param publisher [Hash, nil] publisher config: +subject:+ and +serializer:+
26
+ # @return [Hash{Symbol => Object}] the class's merged options
15
27
  def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil)
16
28
  register
17
29
  default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
18
30
  end
19
31
  alias cosmo_options options
20
32
 
33
+ # @param data [Object] serialized with the publisher's serializer
34
+ # @param subject [String, nil] defaults to the publisher's subject
35
+ # @param options [Hash] passed on to the JetStream publish
36
+ # @return [NATS::JetStream::PubAck]
21
37
  def publish(data, subject: nil, **options)
22
38
  stream = default_options[:stream]
23
39
  subject ||= default_options.dig(:publisher, :subject)
24
40
  Publisher.publish(subject, data, stream: stream, serializer: default_options.dig(:publisher, :serializer), **options)
25
41
  end
26
42
 
43
+ # Options inherited from the superclass stream, or the built-in defaults.
44
+ #
45
+ # @return [Hash{Symbol => Object}]
27
46
  def default_options
28
47
  @default_options ||= Utils::Hash.dup(superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS)
29
48
  end
30
49
 
31
- def register # rubocop:disable Metrics/AbcSize
50
+ # Adds the class to the processed streams and derives its stream, consumer and subjects from its name.
51
+ #
52
+ # @return [void]
53
+ def register
32
54
  Config.internal[:streams] ||= []
33
55
  Config.internal[:streams] << self
34
56
 
@@ -47,6 +69,10 @@ module Cosmo
47
69
  end
48
70
  end
49
71
 
72
+ # Calls {#process_one} for each message, exposing it through {#message}.
73
+ #
74
+ # @param messages [Array<Message>]
75
+ # @return [void]
50
76
  def process(messages)
51
77
  messages.each do |message|
52
78
  Thread.current[:cosmo_message] = message
@@ -58,18 +84,26 @@ module Cosmo
58
84
  alias process_many process
59
85
  alias process_batch process
60
86
 
87
+ # @return [void]
88
+ # @raise [NotImplementedError] unless the stream class overrides it or {#process}
61
89
  def process_one
62
90
  raise NotImplementedError, "#{self.class}#process_one must be implemented"
63
91
  end
64
92
 
93
+ # @return [::Logger]
65
94
  def logger
66
95
  Logger.instance
67
96
  end
68
97
 
98
+ # @return [Message, nil] the message being processed by {#process_one}
69
99
  def message
70
100
  Thread.current[:cosmo_message]
71
101
  end
72
102
 
103
+ # @param data [Object] serialized with the publisher's serializer
104
+ # @param subject [String, nil] defaults to the publisher's subject
105
+ # @param options [Hash] passed on to the JetStream publish
106
+ # @return [NATS::JetStream::PubAck]
73
107
  def publish(data, subject, **options)
74
108
  self.class.publish(data, subject:, **options)
75
109
  end
@@ -18,6 +18,9 @@ module Cosmo
18
18
 
19
19
  module_function
20
20
 
21
+ # @param value [Numeric, String] seconds, or a duration such as "20s", "1h", "7d"
22
+ # @return [Numeric] seconds
23
+ # @raise [ArgumentError] when +value+ is not a valid duration
21
24
  def parse(value)
22
25
  return value if value.is_a?(Numeric)
23
26
 
@@ -5,6 +5,11 @@ module Cosmo
5
5
  module Hash
6
6
  module_function
7
7
 
8
+ # Symbolizes hash keys in place, recursing into nested hashes and arrays.
9
+ #
10
+ # @param obj [Object]
11
+ # @return [Object] +obj+, mutated
12
+ # @raise [ArgumentError] when a key cannot be converted to a symbol
8
13
  def symbolize_keys!(obj)
9
14
  case obj
10
15
  when ::Hash
@@ -23,6 +28,10 @@ module Cosmo
23
28
  end
24
29
  end
25
30
 
31
+ # Stringifies hash keys, recursing into nested hashes and arrays.
32
+ #
33
+ # @param obj [Object]
34
+ # @return [Object] a copy of +obj+
26
35
  def stringify_keys(obj)
27
36
  case obj
28
37
  when ::Hash
@@ -46,6 +55,9 @@ module Cosmo
46
55
  end
47
56
 
48
57
  # deep dup
58
+ #
59
+ # @param hash [Object] a Marshal-able value
60
+ # @return [Object] a deep copy of +hash+
49
61
  def dup(hash)
50
62
  Marshal.load(Marshal.dump(hash))
51
63
  end