cosmonats 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -13
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +51 -3
  8. data/lib/cosmo/api/cron/entry.rb +24 -9
  9. data/lib/cosmo/api/cron.rb +21 -7
  10. data/lib/cosmo/api/job.rb +14 -0
  11. data/lib/cosmo/api/kv.rb +138 -9
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +18 -7
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +47 -11
  18. data/lib/cosmo/api/stream.rb +66 -22
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +208 -11
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +11 -0
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +51 -15
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/version.rb +1 -1
  62. data/lib/cosmo/web/assets/app.css +5 -3
  63. data/lib/cosmo/web/cache.rb +2 -0
  64. data/lib/cosmo/web/chart.rb +2 -2
  65. data/lib/cosmo/web/circuit.rb +1 -0
  66. data/lib/cosmo/web/context.rb +6 -0
  67. data/lib/cosmo/web/controllers/application.rb +26 -0
  68. data/lib/cosmo/web/controllers/batches.rb +7 -1
  69. data/lib/cosmo/web/controllers/crons.rb +16 -2
  70. data/lib/cosmo/web/controllers/jobs.rb +45 -16
  71. data/lib/cosmo/web/controllers/metrics.rb +5 -0
  72. data/lib/cosmo/web/controllers/overview.rb +36 -18
  73. data/lib/cosmo/web/controllers/processes.rb +8 -5
  74. data/lib/cosmo/web/controllers/streams.rb +12 -0
  75. data/lib/cosmo/web/helpers/application.rb +44 -0
  76. data/lib/cosmo/web/renderer.rb +17 -0
  77. data/lib/cosmo/web/throughput.rb +1 -1
  78. data/lib/cosmo/web/views/crons/_table.erb +29 -27
  79. data/lib/cosmo/web/views/jobs/_args.erb +2 -2
  80. data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
  81. data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
  82. data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
  83. data/lib/cosmo/web/views/layout.erb +1 -1
  84. data/lib/cosmo/web/views/overview/_badges.erb +1 -1
  85. data/lib/cosmo/web/views/overview/_running.erb +3 -3
  86. data/lib/cosmo/web/views/overview/index.erb +1 -1
  87. data/lib/cosmo/web/views/streams/_info.erb +1 -0
  88. data/lib/cosmo/web/views/streams/_stream.erb +1 -0
  89. data/lib/cosmo/web.rb +8 -1
  90. data/lib/cosmo.rb +33 -0
  91. data/sig/cosmo/active_job/adapter.rbs +3 -3
  92. data/sig/cosmo/active_job/options.rbs +4 -2
  93. data/sig/cosmo/api/batch.rbs +5 -3
  94. data/sig/cosmo/api/counter.rbs +16 -6
  95. data/sig/cosmo/api/cron/entry.rbs +4 -2
  96. data/sig/cosmo/api/cron.rbs +10 -4
  97. data/sig/cosmo/api/kv.rbs +21 -4
  98. data/sig/cosmo/api/registry.rbs +10 -0
  99. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  100. data/sig/cosmo/api/stats/metrics.rbs +3 -0
  101. data/sig/cosmo/api/stats/processes.rbs +9 -1
  102. data/sig/cosmo/api/stats.rbs +8 -2
  103. data/sig/cosmo/api/stream.rbs +10 -5
  104. data/sig/cosmo/batch/callback.rbs +1 -1
  105. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  106. data/sig/cosmo/batch.rbs +5 -5
  107. data/sig/cosmo/cli.rbs +2 -0
  108. data/sig/cosmo/client.rbs +23 -4
  109. data/sig/cosmo/config/settings.rbs +6 -0
  110. data/sig/cosmo/config.rbs +5 -5
  111. data/sig/cosmo/engine.rbs +5 -0
  112. data/sig/cosmo/heartbeat.rbs +2 -0
  113. data/sig/cosmo/job/data.rbs +3 -1
  114. data/sig/cosmo/job/enqueuer.rbs +12 -0
  115. data/sig/cosmo/job/processor.rbs +18 -3
  116. data/sig/cosmo/job/setter.rbs +20 -0
  117. data/sig/cosmo/job.rbs +10 -4
  118. data/sig/cosmo/logger.rbs +5 -3
  119. data/sig/cosmo/middleware.rbs +0 -8
  120. data/sig/cosmo/processor.rbs +14 -2
  121. data/sig/cosmo/publisher.rbs +1 -0
  122. data/sig/cosmo/stream/processor.rbs +4 -0
  123. data/sig/cosmo/stream.rbs +4 -3
  124. data/sig/cosmo/utils/hash.rbs +2 -1
  125. data/sig/cosmo/utils/signal.rbs +5 -5
  126. data/sig/cosmo/utils/string.rbs +2 -2
  127. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  128. data/sig/cosmo/web/cache.rbs +35 -0
  129. data/sig/cosmo/web/chart.rbs +48 -0
  130. data/sig/cosmo/web/circuit.rbs +29 -0
  131. data/sig/cosmo/web/context.rbs +15 -0
  132. data/sig/cosmo/web/controllers/application.rbs +32 -0
  133. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  134. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  135. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  136. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  137. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  138. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  139. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  140. data/sig/cosmo/web/helpers/application.rbs +61 -0
  141. data/sig/cosmo/web/renderer.rbs +33 -0
  142. data/sig/cosmo/web/throughput.rbs +45 -0
  143. data/sig/cosmo/web.rbs +25 -0
  144. data/sig/cosmo.rbs +19 -0
  145. metadata +23 -6
  146. data/lib/cosmo/api/stats/busy.rb +0 -68
  147. data/lib/cosmo/middleware/busy.rb +0 -15
  148. data/sig/cosmo/api/stats/busy.rbs +0 -33
@@ -11,6 +11,11 @@ module Cosmo
11
11
  @app = app
12
12
  end
13
13
 
14
+ # Runs the Rack app for +req+ and writes its response to +res+.
15
+ #
16
+ # @param req [WEBrick::HTTPRequest]
17
+ # @param res [WEBrick::HTTPResponse]
18
+ # @return [void]
14
19
  def service(req, res)
15
20
  status, headers, body = @app.call(env_for(req))
16
21
  res.status = status
@@ -13,6 +13,8 @@ module Cosmo
13
13
  @check = check || method(:default_check)
14
14
  end
15
15
 
16
+ # @param env [Hash] the Rack env
17
+ # @return [Array] the Rack response
16
18
  def call(env)
17
19
  return @app.call(env) unless env[Rack::PATH_INFO] == PATH
18
20
  return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
@@ -12,6 +12,8 @@ module Cosmo
12
12
  @app = app
13
13
  end
14
14
 
15
+ # @param env [Hash] the Rack env
16
+ # @return [Array] the Rack response
15
17
  def call(env)
16
18
  return @app.call(env) unless env[Rack::PATH_INFO] == PATH
17
19
  return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
@@ -19,6 +19,8 @@ module Cosmo
19
19
  NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
20
20
 
21
21
  # Routing is done by middlewares, anything unmatched falls through to 404.
22
+ #
23
+ # @return [#call] the Rack app
22
24
  def self.app
23
25
  Rack::Builder.app do
24
26
  use Health
@@ -33,6 +35,9 @@ module Cosmo
33
35
  @port = port
34
36
  end
35
37
 
38
+ # Starts serving in a background thread.
39
+ #
40
+ # @return [self]
36
41
  def start
37
42
  @server = ::WEBrick::HTTPServer.new(
38
43
  BindAddress: @host,
@@ -46,6 +51,7 @@ module Cosmo
46
51
  self
47
52
  end
48
53
 
54
+ # @return [void]
49
55
  def stop
50
56
  return unless @server
51
57
 
@@ -55,6 +61,8 @@ module Cosmo
55
61
  end
56
62
 
57
63
  # Actual bound port, useful when started with port 0.
64
+ #
65
+ # @return [Integer]
58
66
  def port
59
67
  @server ? @server.listeners.first.addr[1] : @port
60
68
  end
@@ -7,12 +7,17 @@ module Cosmo
7
7
  class Data
8
8
  DEFAULTS = { stream: :default, retry: 3, dead: true, limit: nil }.freeze
9
9
 
10
+ # @return [Integer] +max_retries+ from the config, or the built-in default
10
11
  def self.default_retry
11
12
  Config[:max_retries] || DEFAULTS[:retry]
12
13
  end
13
14
 
14
15
  attr_reader :jid
15
16
 
17
+ # @param class_name [String]
18
+ # @param args [Array]
19
+ # @param options [Hash, nil] job options, +stream:+ is required
20
+ # @raise [ArgumentError] when no stream is given
16
21
  def initialize(class_name, args, options = nil)
17
22
  @class_name = class_name
18
23
  @args = args
@@ -26,10 +31,13 @@ module Cosmo
26
31
  @jid = SecureRandom.hex(12)
27
32
  end
28
33
 
34
+ # @return [String, nil] the id of the batch the job belongs to
29
35
  def batch_id
30
36
  @options[:batch_id]
31
37
  end
32
38
 
39
+ # @param target [Boolean] true returns the job's own stream even when it is delayed
40
+ # @return [Symbol] the stream the job is published to: +scheduled+ for a delayed job unless +target+
33
41
  def stream(target: false)
34
42
  return @options[:stream] if target
35
43
 
@@ -51,11 +59,14 @@ module Cosmo
51
59
  headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
52
60
  end
53
61
 
62
+ # @return [Hash{Symbol => Object}] the published payload
54
63
  def as_json
55
64
  json = { jid: jid, class: @class_name, args: @args, retry: retries, dead: dead }
56
65
  batch_id ? json.merge(batch_id: batch_id) : json
57
66
  end
58
67
 
68
+ # @param _args [Array] ignored, accepted for +JSON.generate+ compatibility
69
+ # @return [String, nil]
59
70
  def to_json(*_args)
60
71
  Utils::Json.dump(as_json)
61
72
  end
@@ -4,6 +4,9 @@ module Cosmo
4
4
  module Job
5
5
  # The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
6
6
  module Enqueuer
7
+ # Jobs published at once by {.enqueue_many}, before waiting for their acks.
8
+ CHUNK_SIZE = 1_000
9
+
7
10
  module_function
8
11
 
9
12
  # Publishes the job through {Config#client_middleware}.
@@ -17,7 +20,7 @@ module Cosmo
17
20
  # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
18
21
  # @raise [StreamNotFoundError] when the target stream does not exist
19
22
  # @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
20
- def enqueue(class_name, args, options, batch: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
23
+ def enqueue(class_name, args, options, batch: nil)
21
24
  options = options.merge(batch_id: batch.bid) if batch
22
25
  data = Data.new(class_name, args, options)
23
26
  raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
@@ -33,6 +36,98 @@ module Cosmo
33
36
  data.jid if published
34
37
  end
35
38
 
39
+ # Enqueues one job per argument list, publishing them a chunk at a time with {Client#publish_many}. Each job
40
+ # still runs through {Config#client_middleware}, whose block only marks it for publishing: the chunk is
41
+ # published once every job in it went through the chain.
42
+ #
43
+ # @param class_name [String]
44
+ # @param list [Array<Array>] each job's arguments
45
+ # @param options [Hash] job options shared by every job, see {Data}
46
+ # @param batch [Batch, nil] the batch the jobs join
47
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
48
+ # @raise [ArgumentError] when +list+ is not an array of argument arrays
49
+ # @raise [BulkEnqueueError] when the stream refused jobs of a chunk; the chunks after it are not published
50
+ # @raise [SchedulingDisabledError] for delayed jobs while scheduling is turned off
51
+ def enqueue_many(class_name, list, options, batch: nil)
52
+ raise ArgumentError, "expected an array of argument arrays" unless list.is_a?(::Array) && list.all?(::Array)
53
+
54
+ options = options.merge(batch_id: batch.bid) if batch
55
+ jids = []
56
+ list.each_slice(CHUNK_SIZE) do |chunk|
57
+ offset = jids.size
58
+ chunk_jids, errors = publish_chunk(class_name, chunk, options, batch)
59
+ jids.concat(chunk_jids)
60
+ raise BulkEnqueueError.new(jids + Array.new(list.size - jids.size), errors.transform_keys { _1 + offset }) if errors.any?
61
+ end
62
+ jids
63
+ end
64
+
65
+ # @param class_name [String]
66
+ # @param chunk [Array<Array>] each job's arguments
67
+ # @param options [Hash]
68
+ # @param batch [Batch, nil]
69
+ # @return [Array(Array<String, nil>, Hash{Integer => StandardError})] each job's jid, and why the jobs the
70
+ # stream refused were not enqueued, by their index in +chunk+
71
+ def publish_chunk(class_name, chunk, options, batch)
72
+ jobs = chunk.map { |args| through_middleware(class_name, Data.new(class_name, args, options)) }
73
+ published = jobs.compact
74
+ return [jobs.map { nil }, {}] if published.empty?
75
+
76
+ batch&.register_job!(published.size)
77
+ results = publish_all(published, batch)
78
+ batch&.rollback_job!(results.count { !_1.is_a?(NATS::JetStream::PubAck) })
79
+ collect(jobs, results)
80
+ end
81
+
82
+ # @param class_name [String]
83
+ # @param data [Data]
84
+ # @return [Array(Data, Hash), nil] the job and the payload to publish, or nil when a client middleware stopped it
85
+ def through_middleware(class_name, data)
86
+ raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
87
+
88
+ payload = data.as_json
89
+ job = nil
90
+ Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) { job = [data, payload] }
91
+ job
92
+ end
93
+
94
+ # @param jobs [Array<Array(Data, Hash)>]
95
+ # @param batch [Batch, nil] whose slots for every job are released when the publish raises
96
+ # @return [Array<NATS::JetStream::PubAck, StandardError>]
97
+ def publish_all(jobs, batch)
98
+ messages = jobs.map do |(data, payload)|
99
+ header = data.headers.merge(NATS::JetStream::Header::EXPECTED_STREAM => data.stream.to_s)
100
+ { subject: data.subject, payload: Utils::Json.dump(payload), header: header }
101
+ end
102
+ Client.instance.publish_many(messages)
103
+ rescue StandardError
104
+ batch&.rollback_job!(jobs.size)
105
+ raise
106
+ end
107
+
108
+ # @param jobs [Array<Array(Data, Hash), nil>] nil for a job a client middleware stopped
109
+ # @param results [Array<NATS::JetStream::PubAck, StandardError>] one per published job
110
+ # @return [Array(Array<String, nil>, Hash{Integer => StandardError})]
111
+ def collect(jobs, results)
112
+ results = results.each
113
+ errors = {}
114
+ jids = jobs.each_with_index.map do |job, index|
115
+ next unless job
116
+
117
+ result = results.next
118
+ next job.first.jid if result.is_a?(NATS::JetStream::PubAck)
119
+
120
+ errors[index] = result.is_a?(NATS::JetStream::Error::NoStreamResponse) ? StreamNotFoundError.new(job.first.stream.to_s) : result
121
+ nil
122
+ end
123
+ [jids, errors]
124
+ end
125
+
126
+ # Runs the client middleware around the publish.
127
+ #
128
+ # @param class_name [String]
129
+ # @param data [Data]
130
+ # @return [Boolean] whether the job was published
36
131
  def pipe(class_name, data)
37
132
  payload = data.as_json
38
133
  published = false
@@ -43,6 +138,10 @@ module Cosmo
43
138
  published
44
139
  end
45
140
 
141
+ # @param data [Data]
142
+ # @param payload [Hash] the job's payload, possibly changed by a middleware
143
+ # @return [NATS::JetStream::PubAck]
144
+ # @raise [StreamNotFoundError] when the target stream does not exist
46
145
  def publish(data, payload)
47
146
  Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
48
147
  rescue NATS::JetStream::Error::NoStreamResponse
@@ -15,12 +15,13 @@ module Cosmo
15
15
  class Limit
16
16
  BUCKET = "_cosmolimits"
17
17
 
18
+ # @return [Limit]
18
19
  def self.instance
19
20
  @instance ||= new
20
21
  end
21
22
 
22
23
  def initialize
23
- @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.replicas)
24
+ @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.services.num_replicas)
24
25
  end
25
26
 
26
27
  # Try to acquire one of the numbered slots for +key+.
@@ -45,6 +46,9 @@ module Cosmo
45
46
  # tombstone left behind) so a released slot looks identical to one
46
47
  # reclaimed by Nats-TTL expiry -- callers never have to special-case a
47
48
  # delete marker.
49
+ #
50
+ # @param slot [String] the slot key returned by {#acquire}
51
+ # @return [void]
48
52
  def release(slot)
49
53
  @kv.erase(slot)
50
54
  rescue NATS::Error
@@ -10,7 +10,10 @@ module Cosmo
10
10
  # rejects a publish that carries them back.
11
11
  DISPATCH_HEADERS = ["X-Stream", "X-Subject", "X-Execute-At", NATS::JetStream::Header::EXPECTED_STREAM,
12
12
  NATS::JetStream::Header::SCHEDULER, NATS::JetStream::Header::SCHEDULE_NEXT].freeze
13
+ ARGS_LIMIT = 256
13
14
 
15
+ # @param options [Hash] the CLI options
16
+ # @return [void]
14
17
  # @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
15
18
  def self.validate_options!(options)
16
19
  StreamFilter.from(options[:streams]).validate!
@@ -23,13 +26,33 @@ module Cosmo
23
26
  names
24
27
  end
25
28
 
29
+ # @return [Array<Hash>] the jobs running right now, each with its +:jid+, +:class+, +:args+ (cut to
30
+ # ARGS_LIMIT bytes), +:retry+, +:stream+, delivery attempt as +:delivery+ and +:started_at+
31
+ def jobs
32
+ @in_flight.each_pair.flat_map do |thread, messages|
33
+ started_at = @started_at[thread]
34
+ messages.map { job_entry(_1, started_at) }
35
+ end
36
+ end
37
+
26
38
  private
27
39
 
40
+ def job_entry(message, started_at)
41
+ data = Utils::Json.parse(message.data) || {}
42
+ meta = message.metadata
43
+ { jid: data[:jid], class: data[:class], args: capped_args(data[:args]), retry: data[:retry],
44
+ stream: meta.stream, delivery: meta.num_delivered, started_at: }
45
+ end
46
+
47
+ def capped_args(args)
48
+ json = Utils::Json.dump(args)
49
+ json.bytesize > ARGS_LIMIT ? "#{json.byteslice(0, ARGS_LIMIT).scrub("")}…" : args
50
+ end
51
+
28
52
  def setup
29
53
  filter = StreamFilter.from(@options[:streams]).validate!
30
54
 
31
55
  # Initialize singletons before starting to process messages
32
- API::Stats::Busy.instance
33
56
  API::Stats::Totals.instance
34
57
  Limit.instance
35
58
  Config.server_middleware
@@ -40,6 +63,8 @@ module Cosmo
40
63
  next unless filter.include?(stream_name)
41
64
 
42
65
  @consumers << subscribe(stream_name, config)
66
+ rescue NATS::JetStream::Error::NotFound
67
+ Logger.warn "stream #{stream_name} does not exist, skipping it (run `cosmo --setup` to create it)"
43
68
  end
44
69
 
45
70
  log_subscriptions
@@ -103,19 +128,13 @@ module Cosmo
103
128
  data = Utils::Json.parse(message.data)
104
129
  return reject_message(message, ArgumentError.new("malformed payload")) unless data
105
130
 
106
- worker_class = Utils::String.safe_constantize(data[:class])
107
- unless worker_class
108
- reject_message(message, ArgumentError.new("#{data[:class]} class not found"), data)
109
- notify_batch(data, success: false)
110
- return
111
- end
112
-
113
131
  begin
114
132
  sw = stopwatch
115
133
  Logger.with(jid: data[:jid])
116
134
  Logger.info "start"
117
135
 
118
- instance = build_worker(worker_class, data, message)
136
+ worker_class = Utils::String.safe_constantize(data[:class])
137
+ instance = build_worker(worker_class, data, message) if worker_class
119
138
  Config.server_middleware.invoke(instance, data, message) do
120
139
  perform_job(instance, data: data, message: message)
121
140
  end
@@ -126,7 +145,9 @@ module Cosmo
126
145
  rescue Requeue => e
127
146
  message.nak(delay: Config.to_ns(e.delay))
128
147
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "requeue[#{e.delay}s]" }
129
- rescue Timeout::Error => e
148
+ rescue NATS::JetStream::Error::MsgAlreadyAckd
149
+ Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "released[shutdown]" }
150
+ rescue Timeout::Error, JobTimeoutError => e
130
151
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
131
152
  Cosmo.handle_error(e, error_context(message, data))
132
153
  handle_failure(worker_class, message, data, e)
@@ -170,6 +191,16 @@ module Cosmo
170
191
  true
171
192
  end
172
193
 
194
+ # A job still running when the shutdown timed out counts as a failed attempt: it's retried after the
195
+ # usual delay, or moved to the DLQ (dropped when +dead+ is off) once it has no attempts left.
196
+ def abandon(message)
197
+ data = Utils::Json.parse(message.data)
198
+ return super unless data
199
+
200
+ API::Stats::Totals.instance.increment(:failed)
201
+ handle_failure(Utils::String.safe_constantize(data[:class]), message, data, JobInterruptedError.new)
202
+ end
203
+
173
204
  def notify_batch(data, success:)
174
205
  return unless data[:batch_id]
175
206
 
@@ -200,7 +231,7 @@ module Cosmo
200
231
  end
201
232
 
202
233
  def retry_delay(worker_class, data, current_attempt, exception)
203
- handler = worker_class.retry_in(data)
234
+ handler = worker_class&.retry_in(data)
204
235
  return default_retry_delay(current_attempt) unless handler
205
236
 
206
237
  delay = handler.call(current_attempt, exception)
@@ -230,10 +261,11 @@ module Cosmo
230
261
  end
231
262
 
232
263
  # Logs why a message can't be processed at all and parks it in the DLQ.
233
- def reject_message(message, error, data = nil)
264
+ def reject_message(message, error)
234
265
  Logger.error error
235
- Cosmo.handle_error(error, error_context(message, data, source: :reject))
236
- move_message(message, data, error)
266
+ Cosmo.handle_error(error, error_context(message, nil, source: :reject))
267
+ API::Stats::Totals.instance.increment(:failed)
268
+ move_message(message, nil, error)
237
269
  end
238
270
 
239
271
  def error_context(message, data, source: :job)
@@ -290,12 +322,16 @@ module Cosmo
290
322
  ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
291
323
  end
292
324
 
293
- # @param job_instance [Cosmo::Job]
325
+ # @param job_instance [Cosmo::Job, nil] nil when this worker can't load the job's class
294
326
  # @param data [Hash]
295
327
  # @param message [NATS::Msg]
328
+ # @return [Object] what the job's +perform+ returns
329
+ # @raise [JobClassNotFoundError] when +job_instance+ is nil
296
330
  #
297
331
  # rubocop:disable-next Lint/UnusedMethodArgument
298
332
  def perform_job(job_instance, data:, message:)
333
+ raise JobClassNotFoundError, data[:class] unless job_instance
334
+
299
335
  job_instance.perform(*data[:args])
300
336
  end
301
337
  end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cosmo
4
+ module Job
5
+ # Per-call job options, returned by {ClassMethods#set}: the enqueue methods merge them over the class options.
6
+ #
7
+ # ReportJob.set(stream: :critical, retry: 0).perform_async(42)
8
+ class Setter
9
+ # Options carried by the job's message. The rest (+limit+, +retry_in+) are read from the class by the worker.
10
+ OPTIONS = %i[stream retry dead subject].freeze
11
+
12
+ # @param job_class [Class] the job class including {Job}
13
+ # @param options [Hash] a subset of {OPTIONS}
14
+ # @raise [ArgumentError] when an option is not one of {OPTIONS}
15
+ def initialize(job_class, options)
16
+ unknown = options.keys - OPTIONS
17
+ if unknown.any?
18
+ hint = " (use stream: instead of queue:)" if unknown.include?(:queue)
19
+ raise ArgumentError, "unknown job options #{unknown.join(", ")}, expected #{OPTIONS.join(", ")}#{hint}"
20
+ end
21
+
22
+ @job_class = job_class
23
+ @options = options
24
+ end
25
+
26
+ # @param args [Array] arguments passed to the job's +perform+
27
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
28
+ def perform_async(*args)
29
+ @job_class.perform(*args, **@options)
30
+ end
31
+
32
+ # @param list [Array<Array>] each job's arguments
33
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
34
+ def perform_bulk(list)
35
+ @job_class.perform_bulk(list, **@options)
36
+ end
37
+
38
+ # @param timestamp [Time, Integer] when the job runs
39
+ # @param args [Array] arguments passed to the job's +perform+
40
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
41
+ def perform_at(timestamp, *args)
42
+ @job_class.perform(*args, **@options, at: timestamp)
43
+ end
44
+
45
+ # @param interval [Numeric] seconds from now until the job runs
46
+ # @param args [Array] arguments passed to the job's +perform+
47
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
48
+ def perform_in(interval, *args)
49
+ @job_class.perform(*args, **@options, in: interval)
50
+ end
51
+ end
52
+ end
53
+ end
data/lib/cosmo/job.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "cosmo/job/data"
4
4
  require "cosmo/job/enqueuer"
5
+ require "cosmo/job/setter"
5
6
  require "cosmo/job/requeue"
6
7
  require "cosmo/job/failure"
7
8
  require "cosmo/job/stream_filter"
@@ -10,6 +11,8 @@ require "cosmo/job/processor"
10
11
 
11
12
  module Cosmo
12
13
  module Job
14
+ # @param base [Class] the job class including Job
15
+ # @return [void]
13
16
  def self.included(base)
14
17
  base.extend(ClassMethods)
15
18
  end
@@ -28,14 +31,15 @@ module Cosmo
28
31
  # limit: { duration: 30, concurrency: { to: 3, key: ->(id) { id } } }
29
32
  # limit: { duration: 30, concurrency: 3, retry_in: 5 }
30
33
  #
31
- # @option config [Integer] :"limit[:duration]" hard execution timeout in seconds. The job thread is
34
+ # @option config [Integer, String] :"limit[:duration]" hard execution timeout in seconds, or a duration such as
35
+ # +"82m"+ (see {Utils::Duration.parse}). The job thread is
32
36
  # killed after this many seconds and counts as a failed attempt (retried with exponential backoff,
33
37
  # moved to DLQ after retries exhausted).
34
38
  # @option config [Integer, Hash] :"limit[:concurrency]" caps how many instances run at once across all
35
39
  # workers. Jobs that cannot acquire a slot are NAK'd (see +retry_in+) so they are not re-delivered until
36
40
  # the slot is likely free. Requires +duration+.
37
41
  # Pass an Integer for a class-wide cap, or <tt>{ to: N, key: ->(args) {} }</tt> to scope per key.
38
- # @option config [Integer] :"limit[:retry_in]" seconds to wait before NATS redelivers a job that was
42
+ # @option config [Integer, String] :"limit[:retry_in]" seconds, or a duration such as +"30s"+, to wait before NATS redelivers a job that was
39
43
  # NAK'd for lack of a concurrency slot (default: half of +duration+). Counts against the same delivery
40
44
  # counter as any other retry -- a job stuck behind the concurrency limit for enough consecutive
41
45
  # attempts is dropped/DLQ'd exactly like one that keeps failing outright.
@@ -50,7 +54,14 @@ module Cosmo
50
54
  # +limit[:retry_in]+) or actually ran and failed. So a job that gets turned away twice for lack of a
51
55
  # slot, then finally runs and fails, calls this handler with +count == 3+, not 1. Don't read +count+ as
52
56
  # "how many times perform has actually run and failed" when concurrency limits are in play.
57
+ # @return [Hash{Symbol => Object}] the class's merged options
58
+ # @raise [ArgumentError] when +limit[:concurrency]+ lacks a +duration+, a limit duration is invalid, or +retry_in+
59
+ # is not callable
53
60
  def options(**config)
61
+ if (limit = config[:limit])
62
+ config[:limit] = limit.merge(limit.slice(:duration, :retry_in).compact.transform_values { Utils::Duration.parse(_1) })
63
+ end
64
+
54
65
  if config[:limit] && config.dig(:limit, :concurrency) && !config.dig(:limit, :duration).to_i.positive?
55
66
  raise ArgumentError, "limit: duration is required when concurrency is set"
56
67
  end
@@ -61,6 +72,7 @@ module Cosmo
61
72
  end
62
73
  alias cosmo_options options
63
74
 
75
+ # @return [Boolean] whether +limit[:concurrency]+ is configured
64
76
  def limits_concurrency?
65
77
  !!concurrency_options
66
78
  end
@@ -68,12 +80,17 @@ module Cosmo
68
80
  # Returns the +retry_in+ Proc/lambda (taking +(count, exception)+) configured for this job class, or
69
81
  # +nil+ when unset. Overridable by wrapper job classes (e.g. the ActiveJob executor) that need to
70
82
  # resolve it from something other than +self+.
83
+ #
84
+ # @param _data [Hash, nil] the job payload, unused here
85
+ # @return [Proc, nil]
71
86
  def retry_in(_data = nil)
72
87
  default_options[:retry_in]
73
88
  end
74
89
 
75
90
  # Returns a normalized concurrency config hash, or +nil+ when not configured.
76
91
  # Always contains +:limit+, +:key+, +:duration+, and +:retry_in+.
92
+ #
93
+ # @return [Hash{Symbol => Object}, nil]
77
94
  def concurrency_options
78
95
  value = default_options.dig(:limit, :concurrency)
79
96
  return unless value
@@ -88,6 +105,9 @@ module Cosmo
88
105
  end
89
106
 
90
107
  # Derive the fully-scoped concurrency key for a given args array.
108
+ #
109
+ # @param args [Array] the job's arguments
110
+ # @return [String, nil] nil when concurrency is not limited
91
111
  def concurrency_key(args)
92
112
  config = concurrency_options
93
113
  return unless config
@@ -97,6 +117,12 @@ module Cosmo
97
117
  suffix ? "#{base}/#{suffix}" : base
98
118
  end
99
119
 
120
+ # Enqueues the job, or runs it inline when +async+ is false.
121
+ #
122
+ # @param args [Array] arguments passed to the job's +perform+
123
+ # @param async [Boolean] false runs the job in the current thread, without NATS
124
+ # @param options [Hash] job options merged over {#default_options}, e.g. +at:+ or +in:+
125
+ # @return [String, nil] the job's jid, or nil when run inline or a client middleware stopped the publish
100
126
  def perform(*args, async: true, **options)
101
127
  options = default_options.merge(options)
102
128
  return Enqueuer.enqueue(name, args, options, batch: Batch.current) if async
@@ -108,22 +134,60 @@ module Cosmo
108
134
  nil
109
135
  end
110
136
 
137
+ # Options for a single enqueue, merged over the class options.
138
+ #
139
+ # ReportJob.set(stream: :critical, retry: 0).perform_async(42)
140
+ #
141
+ # @param options [Hash] +stream+, +retry+, +dead+ or +subject+, see {#options}
142
+ # @return [Setter]
143
+ # @raise [ArgumentError] for any other option
144
+ def set(**options)
145
+ Setter.new(self, options)
146
+ end
147
+
148
+ # @param args [Array] arguments passed to the job's +perform+
149
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
111
150
  def perform_async(*args)
112
151
  perform(*args)
113
152
  end
114
153
 
154
+ # Enqueues one job per argument list, publishing a thousand at a time without waiting on each ack.
155
+ #
156
+ # ReportJob.perform_bulk([[1], [2], [3]])
157
+ #
158
+ # @param list [Array<Array>] each job's arguments
159
+ # @param options [Hash] job options for every job, merged over {#default_options}, e.g. +in:+
160
+ # @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
161
+ # @raise [BulkEnqueueError] when the stream refused some jobs; its +jids+ tell which were enqueued
162
+ def perform_bulk(list, **options)
163
+ Enqueuer.enqueue_many(name, list, default_options.merge(options), batch: Batch.current)
164
+ end
165
+
166
+ # @param timestamp [Time, Integer] when the job runs
167
+ # @param args [Array] arguments passed to the job's +perform+
168
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
115
169
  def perform_at(timestamp, *args)
116
170
  perform(*args, at: timestamp)
117
171
  end
118
172
 
173
+ # @param interval [Numeric] seconds from now until the job runs
174
+ # @param args [Array] arguments passed to the job's +perform+
175
+ # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
119
176
  def perform_in(interval, *args)
120
177
  perform(*args, in: interval)
121
178
  end
122
179
 
180
+ # Runs the job inline, without NATS.
181
+ #
182
+ # @param args [Array] arguments passed to the job's +perform+
183
+ # @return [nil]
123
184
  def perform_sync(*args)
124
185
  perform(*args, async: false)
125
186
  end
126
187
 
188
+ # Options inherited from the superclass job, or the built-in defaults.
189
+ #
190
+ # @return [Hash{Symbol => Object}]
127
191
  def default_options
128
192
  @default_options ||= (superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS.merge(retry: Data.default_retry)).dup
129
193
  end
@@ -137,10 +201,15 @@ module Cosmo
137
201
 
138
202
  attr_accessor :jid, :batch_id, :enqueued_at, :attempt, :scheduled_by
139
203
 
204
+ # Runs the job with the arguments it was enqueued with.
205
+ #
206
+ # @return [Object]
207
+ # @raise [NotImplementedError] unless the job class overrides it
140
208
  def perform(...)
141
209
  raise NotImplementedError, "#{self.class}#perform must be implemented"
142
210
  end
143
211
 
212
+ # @return [::Logger]
144
213
  def logger
145
214
  Logger.instance
146
215
  end