cosmonats 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +73 -22
- data/lib/cosmo/active_job/adapter.rb +2 -0
- data/lib/cosmo/active_job/executor.rb +7 -0
- data/lib/cosmo/active_job/options.rb +6 -0
- data/lib/cosmo/api/batch.rb +26 -0
- data/lib/cosmo/api/counter.rb +52 -4
- data/lib/cosmo/api/cron/entry.rb +34 -20
- data/lib/cosmo/api/cron.rb +24 -10
- data/lib/cosmo/api/job.rb +15 -1
- data/lib/cosmo/api/kv.rb +141 -12
- data/lib/cosmo/api/registry.rb +47 -6
- data/lib/cosmo/api/stats/jobs.rb +84 -0
- data/lib/cosmo/api/stats/metrics.rb +20 -9
- data/lib/cosmo/api/stats/processes.rb +27 -11
- data/lib/cosmo/api/stats/totals.rb +1 -0
- data/lib/cosmo/api/stats.rb +58 -7
- data/lib/cosmo/api/stream.rb +73 -24
- data/lib/cosmo/batch/callback.rb +5 -0
- data/lib/cosmo/batch/dispatcher.rb +4 -4
- data/lib/cosmo/batch.rb +34 -11
- data/lib/cosmo/cli.rb +6 -0
- data/lib/cosmo/client.rb +217 -23
- data/lib/cosmo/config/cosmo.yml +1 -0
- data/lib/cosmo/config/settings.rb +4 -0
- data/lib/cosmo/config.rb +58 -10
- data/lib/cosmo/engine.rb +28 -1
- data/lib/cosmo/heartbeat.rb +12 -3
- data/lib/cosmo/http_server/handler.rb +5 -0
- data/lib/cosmo/http_server/health.rb +2 -0
- data/lib/cosmo/http_server/ping.rb +2 -0
- data/lib/cosmo/http_server.rb +8 -0
- data/lib/cosmo/job/data.rb +12 -1
- data/lib/cosmo/job/enqueuer.rb +100 -1
- data/lib/cosmo/job/limit.rb +5 -1
- data/lib/cosmo/job/processor.rb +56 -21
- data/lib/cosmo/job/setter.rb +53 -0
- data/lib/cosmo/job.rb +71 -2
- data/lib/cosmo/logger.rb +37 -0
- data/lib/cosmo/middleware/chain.rb +12 -0
- data/lib/cosmo/middleware/limit.rb +12 -3
- data/lib/cosmo/middleware/metrics.rb +6 -4
- data/lib/cosmo/middleware/sentry.rb +6 -4
- data/lib/cosmo/middleware/totals.rb +5 -11
- data/lib/cosmo/middleware.rb +0 -1
- data/lib/cosmo/processor.rb +60 -3
- data/lib/cosmo/publisher.rb +8 -0
- data/lib/cosmo/services.rb +7 -2
- data/lib/cosmo/stream/message.rb +5 -0
- data/lib/cosmo/stream/processor.rb +7 -0
- data/lib/cosmo/stream/serializer.rb +4 -0
- data/lib/cosmo/stream.rb +35 -1
- data/lib/cosmo/utils/duration.rb +3 -0
- data/lib/cosmo/utils/hash.rb +12 -0
- data/lib/cosmo/utils/json.rb +8 -0
- data/lib/cosmo/utils/signal.rb +9 -0
- data/lib/cosmo/utils/stopwatch.rb +1 -0
- data/lib/cosmo/utils/string.rb +4 -0
- data/lib/cosmo/utils/thread_pool.rb +8 -0
- data/lib/cosmo/utils/ttl_cache.rb +10 -0
- data/lib/cosmo/utils.rb +0 -1
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +273 -629
- data/lib/cosmo/web/assets/app.js +158 -0
- data/lib/cosmo/web/assets/fonts/OFL.txt +92 -0
- data/lib/cosmo/web/assets/fonts/geist-mono.woff2 +0 -0
- data/lib/cosmo/web/assets/fonts/geist.woff2 +0 -0
- data/lib/cosmo/web/assets/logo.svg +10 -0
- data/lib/cosmo/web/cache.rb +77 -0
- data/lib/cosmo/web/chart.rb +63 -39
- data/lib/cosmo/web/circuit.rb +56 -0
- data/lib/cosmo/web/context.rb +6 -4
- data/lib/cosmo/web/controllers/application.rb +42 -0
- data/lib/cosmo/web/controllers/batches.rb +8 -1
- data/lib/cosmo/web/controllers/crons.rb +24 -3
- data/lib/cosmo/web/controllers/jobs.rb +88 -68
- data/lib/cosmo/web/controllers/metrics.rb +6 -0
- data/lib/cosmo/web/controllers/overview.rb +88 -0
- data/lib/cosmo/web/controllers/processes.rb +36 -0
- data/lib/cosmo/web/controllers/streams.rb +38 -28
- data/lib/cosmo/web/helpers/application.rb +101 -0
- data/lib/cosmo/web/renderer.rb +36 -6
- data/lib/cosmo/web/throughput.rb +104 -0
- data/lib/cosmo/web/views/batches/_table.erb +42 -54
- data/lib/cosmo/web/views/batches/index.erb +3 -10
- data/lib/cosmo/web/views/crons/_table.erb +36 -55
- data/lib/cosmo/web/views/crons/index.erb +2 -9
- data/lib/cosmo/web/views/jobs/_args.erb +5 -0
- data/lib/cosmo/web/views/jobs/_busy.erb +25 -121
- data/lib/cosmo/web/views/jobs/_dead.erb +42 -82
- data/lib/cosmo/web/views/jobs/_enqueued.erb +33 -113
- data/lib/cosmo/web/views/jobs/_panel.erb +34 -0
- data/lib/cosmo/web/views/jobs/_queue_row.erb +18 -0
- data/lib/cosmo/web/views/jobs/_queues.erb +20 -0
- data/lib/cosmo/web/views/jobs/_scheduled.erb +20 -54
- data/lib/cosmo/web/views/jobs/index.erb +4 -17
- data/lib/cosmo/web/views/layout.erb +73 -22
- data/lib/cosmo/web/views/metrics/_chart.erb +37 -32
- data/lib/cosmo/web/views/metrics/_panel.erb +3 -3
- data/lib/cosmo/web/views/metrics/_table.erb +24 -40
- data/lib/cosmo/web/views/metrics/index.erb +8 -16
- data/lib/cosmo/web/views/overview/_badges.erb +3 -0
- data/lib/cosmo/web/views/overview/_kpis.erb +19 -0
- data/lib/cosmo/web/views/overview/_running.erb +18 -0
- data/lib/cosmo/web/views/overview/_throughput.erb +50 -0
- data/lib/cosmo/web/views/overview/_workers.erb +25 -0
- data/lib/cosmo/web/views/overview/index.erb +28 -0
- data/lib/cosmo/web/views/processes/_table.erb +67 -0
- data/lib/cosmo/web/views/processes/index.erb +1 -0
- data/lib/cosmo/web/views/shared/_pager.erb +22 -0
- data/lib/cosmo/web/views/shared/unavailable.erb +7 -0
- data/lib/cosmo/web/views/streams/_info.erb +29 -86
- data/lib/cosmo/web/views/streams/_list.erb +7 -0
- data/lib/cosmo/web/views/streams/_pause_banner.erb +8 -16
- data/lib/cosmo/web/views/streams/_stream.erb +53 -0
- data/lib/cosmo/web/views/streams/index.erb +3 -11
- data/lib/cosmo/web/views/streams/info.erb +3 -11
- data/lib/cosmo/web.rb +73 -35
- data/lib/cosmo.rb +33 -0
- data/sig/cosmo/active_job/adapter.rbs +3 -3
- data/sig/cosmo/active_job/options.rbs +4 -2
- data/sig/cosmo/api/batch.rbs +5 -3
- data/sig/cosmo/api/counter.rbs +16 -6
- data/sig/cosmo/api/cron/entry.rbs +6 -3
- data/sig/cosmo/api/cron.rbs +10 -4
- data/sig/cosmo/api/kv.rbs +21 -4
- data/sig/cosmo/api/registry.rbs +10 -0
- data/sig/cosmo/api/stats/jobs.rbs +34 -0
- data/sig/cosmo/api/stats/metrics.rbs +4 -1
- data/sig/cosmo/api/stats/processes.rbs +9 -1
- data/sig/cosmo/api/stats.rbs +12 -2
- data/sig/cosmo/api/stream.rbs +12 -5
- data/sig/cosmo/batch/callback.rbs +1 -1
- data/sig/cosmo/batch/dispatcher.rbs +1 -1
- data/sig/cosmo/batch.rbs +5 -5
- data/sig/cosmo/cli.rbs +2 -0
- data/sig/cosmo/client.rbs +24 -4
- data/sig/cosmo/config/settings.rbs +6 -0
- data/sig/cosmo/config.rbs +5 -5
- data/sig/cosmo/engine.rbs +5 -0
- data/sig/cosmo/heartbeat.rbs +2 -0
- data/sig/cosmo/job/data.rbs +3 -1
- data/sig/cosmo/job/enqueuer.rbs +12 -0
- data/sig/cosmo/job/processor.rbs +18 -3
- data/sig/cosmo/job/setter.rbs +20 -0
- data/sig/cosmo/job.rbs +10 -4
- data/sig/cosmo/logger.rbs +5 -3
- data/sig/cosmo/middleware.rbs +0 -8
- data/sig/cosmo/processor.rbs +14 -2
- data/sig/cosmo/publisher.rbs +1 -0
- data/sig/cosmo/stream/processor.rbs +4 -0
- data/sig/cosmo/stream.rbs +4 -3
- data/sig/cosmo/utils/hash.rbs +2 -1
- data/sig/cosmo/utils/signal.rbs +5 -5
- data/sig/cosmo/utils/string.rbs +2 -2
- data/sig/cosmo/utils/thread_pool.rbs +1 -1
- data/sig/cosmo/web/cache.rbs +35 -0
- data/sig/cosmo/web/chart.rbs +48 -0
- data/sig/cosmo/web/circuit.rbs +29 -0
- data/sig/cosmo/web/context.rbs +15 -0
- data/sig/cosmo/web/controllers/application.rbs +32 -0
- data/sig/cosmo/web/controllers/batches.rbs +11 -0
- data/sig/cosmo/web/controllers/crons.rbs +23 -0
- data/sig/cosmo/web/controllers/jobs.rbs +47 -0
- data/sig/cosmo/web/controllers/metrics.rbs +21 -0
- data/sig/cosmo/web/controllers/overview.rbs +31 -0
- data/sig/cosmo/web/controllers/processes.rbs +13 -0
- data/sig/cosmo/web/controllers/streams.rbs +27 -0
- data/sig/cosmo/web/helpers/application.rbs +61 -0
- data/sig/cosmo/web/renderer.rbs +33 -0
- data/sig/cosmo/web/throughput.rbs +45 -0
- data/sig/cosmo/web.rbs +25 -0
- data/sig/cosmo.rbs +19 -0
- metadata +53 -23
- data/lib/cosmo/api/stats/busy.rb +0 -63
- data/lib/cosmo/middleware/busy.rb +0 -15
- data/lib/cosmo/utils/overrides.rb +0 -206
- data/lib/cosmo/utils/warnings.rb +0 -17
- data/lib/cosmo/web/controllers/actions.rb +0 -16
- data/lib/cosmo/web/views/actions/index.erb +0 -7
- data/lib/cosmo/web/views/jobs/_processes.erb +0 -145
- data/lib/cosmo/web/views/jobs/_stats.erb +0 -77
- data/lib/cosmo/web/views/jobs/_tabs.erb +0 -14
- data/lib/cosmo/web/views/jobs/busy.erb +0 -18
- data/lib/cosmo/web/views/jobs/dead.erb +0 -18
- data/lib/cosmo/web/views/jobs/enqueued.erb +0 -18
- data/lib/cosmo/web/views/jobs/scheduled.erb +0 -18
- data/lib/cosmo/web/views/streams/_stream_row.erb +0 -42
- data/lib/cosmo/web/views/streams/_table.erb +0 -25
- data/sig/cosmo/api/stats/busy.rbs +0 -31
data/lib/cosmo/heartbeat.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "shellwords"
|
|
|
5
5
|
|
|
6
6
|
module Cosmo
|
|
7
7
|
# Publishes this process's identity and live state to API::Stats::Processes, so the Web UI lists
|
|
8
|
-
# every running worker, the idle ones included.
|
|
8
|
+
# every running worker, the idle ones included, and the jobs it is running to API::Stats::Jobs.
|
|
9
9
|
class Heartbeat
|
|
10
10
|
INTERVAL = 10
|
|
11
11
|
|
|
@@ -32,14 +32,18 @@ module Cosmo
|
|
|
32
32
|
self
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
# Stops beating and removes the
|
|
35
|
+
# Stops beating and removes the entries right away, instead of leaving them to expire.
|
|
36
36
|
#
|
|
37
37
|
# @return [void]
|
|
38
38
|
def stop
|
|
39
39
|
@thread&.kill&.join
|
|
40
40
|
@thread = nil
|
|
41
41
|
flush
|
|
42
|
-
|
|
42
|
+
begin
|
|
43
|
+
processes.unregister(identity)
|
|
44
|
+
ensure
|
|
45
|
+
jobs.clear(identity)
|
|
46
|
+
end
|
|
43
47
|
rescue StandardError => e
|
|
44
48
|
Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
|
|
45
49
|
end
|
|
@@ -49,6 +53,7 @@ module Cosmo
|
|
|
49
53
|
# @return [void]
|
|
50
54
|
def beat
|
|
51
55
|
processes.register(identity, info)
|
|
56
|
+
jobs.report(identity, @engine.jobs)
|
|
52
57
|
rescue StandardError => e
|
|
53
58
|
Logger.debug "Heartbeat error: #{e.class} #{e.message}"
|
|
54
59
|
end
|
|
@@ -136,5 +141,9 @@ module Cosmo
|
|
|
136
141
|
def processes
|
|
137
142
|
API::Stats::Processes.instance
|
|
138
143
|
end
|
|
144
|
+
|
|
145
|
+
def jobs
|
|
146
|
+
API::Stats::Jobs.instance
|
|
147
|
+
end
|
|
139
148
|
end
|
|
140
149
|
end
|
|
@@ -11,6 +11,11 @@ module Cosmo
|
|
|
11
11
|
@app = app
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# Runs the Rack app for +req+ and writes its response to +res+.
|
|
15
|
+
#
|
|
16
|
+
# @param req [WEBrick::HTTPRequest]
|
|
17
|
+
# @param res [WEBrick::HTTPResponse]
|
|
18
|
+
# @return [void]
|
|
14
19
|
def service(req, res)
|
|
15
20
|
status, headers, body = @app.call(env_for(req))
|
|
16
21
|
res.status = status
|
|
@@ -13,6 +13,8 @@ module Cosmo
|
|
|
13
13
|
@check = check || method(:default_check)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
# @param env [Hash] the Rack env
|
|
17
|
+
# @return [Array] the Rack response
|
|
16
18
|
def call(env)
|
|
17
19
|
return @app.call(env) unless env[Rack::PATH_INFO] == PATH
|
|
18
20
|
return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
|
|
@@ -12,6 +12,8 @@ module Cosmo
|
|
|
12
12
|
@app = app
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
+
# @param env [Hash] the Rack env
|
|
16
|
+
# @return [Array] the Rack response
|
|
15
17
|
def call(env)
|
|
16
18
|
return @app.call(env) unless env[Rack::PATH_INFO] == PATH
|
|
17
19
|
return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
|
data/lib/cosmo/http_server.rb
CHANGED
|
@@ -19,6 +19,8 @@ module Cosmo
|
|
|
19
19
|
NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
|
|
20
20
|
|
|
21
21
|
# Routing is done by middlewares, anything unmatched falls through to 404.
|
|
22
|
+
#
|
|
23
|
+
# @return [#call] the Rack app
|
|
22
24
|
def self.app
|
|
23
25
|
Rack::Builder.app do
|
|
24
26
|
use Health
|
|
@@ -33,6 +35,9 @@ module Cosmo
|
|
|
33
35
|
@port = port
|
|
34
36
|
end
|
|
35
37
|
|
|
38
|
+
# Starts serving in a background thread.
|
|
39
|
+
#
|
|
40
|
+
# @return [self]
|
|
36
41
|
def start
|
|
37
42
|
@server = ::WEBrick::HTTPServer.new(
|
|
38
43
|
BindAddress: @host,
|
|
@@ -46,6 +51,7 @@ module Cosmo
|
|
|
46
51
|
self
|
|
47
52
|
end
|
|
48
53
|
|
|
54
|
+
# @return [void]
|
|
49
55
|
def stop
|
|
50
56
|
return unless @server
|
|
51
57
|
|
|
@@ -55,6 +61,8 @@ module Cosmo
|
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
# Actual bound port, useful when started with port 0.
|
|
64
|
+
#
|
|
65
|
+
# @return [Integer]
|
|
58
66
|
def port
|
|
59
67
|
@server ? @server.listeners.first.addr[1] : @port
|
|
60
68
|
end
|
data/lib/cosmo/job/data.rb
CHANGED
|
@@ -7,12 +7,17 @@ module Cosmo
|
|
|
7
7
|
class Data
|
|
8
8
|
DEFAULTS = { stream: :default, retry: 3, dead: true, limit: nil }.freeze
|
|
9
9
|
|
|
10
|
+
# @return [Integer] +max_retries+ from the config, or the built-in default
|
|
10
11
|
def self.default_retry
|
|
11
12
|
Config[:max_retries] || DEFAULTS[:retry]
|
|
12
13
|
end
|
|
13
14
|
|
|
14
15
|
attr_reader :jid
|
|
15
16
|
|
|
17
|
+
# @param class_name [String]
|
|
18
|
+
# @param args [Array]
|
|
19
|
+
# @param options [Hash, nil] job options, +stream:+ is required
|
|
20
|
+
# @raise [ArgumentError] when no stream is given
|
|
16
21
|
def initialize(class_name, args, options = nil)
|
|
17
22
|
@class_name = class_name
|
|
18
23
|
@args = args
|
|
@@ -26,10 +31,13 @@ module Cosmo
|
|
|
26
31
|
@jid = SecureRandom.hex(12)
|
|
27
32
|
end
|
|
28
33
|
|
|
34
|
+
# @return [String, nil] the id of the batch the job belongs to
|
|
29
35
|
def batch_id
|
|
30
36
|
@options[:batch_id]
|
|
31
37
|
end
|
|
32
38
|
|
|
39
|
+
# @param target [Boolean] true returns the job's own stream even when it is delayed
|
|
40
|
+
# @return [Symbol] the stream the job is published to: +scheduled+ for a delayed job unless +target+
|
|
33
41
|
def stream(target: false)
|
|
34
42
|
return @options[:stream] if target
|
|
35
43
|
|
|
@@ -44,18 +52,21 @@ module Cosmo
|
|
|
44
52
|
|
|
45
53
|
# @return [Hash{String => String, Integer}] dedup id, plus where to dispatch a delayed job once it is due
|
|
46
54
|
def headers
|
|
47
|
-
headers = {
|
|
55
|
+
headers = { NATS::JetStream::Header::MSG_ID => jid }
|
|
48
56
|
return headers unless @at
|
|
49
57
|
|
|
50
58
|
target = stream(target: true)
|
|
51
59
|
headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
|
|
52
60
|
end
|
|
53
61
|
|
|
62
|
+
# @return [Hash{Symbol => Object}] the published payload
|
|
54
63
|
def as_json
|
|
55
64
|
json = { jid: jid, class: @class_name, args: @args, retry: retries, dead: dead }
|
|
56
65
|
batch_id ? json.merge(batch_id: batch_id) : json
|
|
57
66
|
end
|
|
58
67
|
|
|
68
|
+
# @param _args [Array] ignored, accepted for +JSON.generate+ compatibility
|
|
69
|
+
# @return [String, nil]
|
|
59
70
|
def to_json(*_args)
|
|
60
71
|
Utils::Json.dump(as_json)
|
|
61
72
|
end
|
data/lib/cosmo/job/enqueuer.rb
CHANGED
|
@@ -4,6 +4,9 @@ module Cosmo
|
|
|
4
4
|
module Job
|
|
5
5
|
# The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
|
|
6
6
|
module Enqueuer
|
|
7
|
+
# Jobs published at once by {.enqueue_many}, before waiting for their acks.
|
|
8
|
+
CHUNK_SIZE = 1_000
|
|
9
|
+
|
|
7
10
|
module_function
|
|
8
11
|
|
|
9
12
|
# Publishes the job through {Config#client_middleware}.
|
|
@@ -17,7 +20,7 @@ module Cosmo
|
|
|
17
20
|
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
18
21
|
# @raise [StreamNotFoundError] when the target stream does not exist
|
|
19
22
|
# @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
|
|
20
|
-
def enqueue(class_name, args, options, batch: nil)
|
|
23
|
+
def enqueue(class_name, args, options, batch: nil)
|
|
21
24
|
options = options.merge(batch_id: batch.bid) if batch
|
|
22
25
|
data = Data.new(class_name, args, options)
|
|
23
26
|
raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
|
|
@@ -33,6 +36,98 @@ module Cosmo
|
|
|
33
36
|
data.jid if published
|
|
34
37
|
end
|
|
35
38
|
|
|
39
|
+
# Enqueues one job per argument list, publishing them a chunk at a time with {Client#publish_many}. Each job
|
|
40
|
+
# still runs through {Config#client_middleware}, whose block only marks it for publishing: the chunk is
|
|
41
|
+
# published once every job in it went through the chain.
|
|
42
|
+
#
|
|
43
|
+
# @param class_name [String]
|
|
44
|
+
# @param list [Array<Array>] each job's arguments
|
|
45
|
+
# @param options [Hash] job options shared by every job, see {Data}
|
|
46
|
+
# @param batch [Batch, nil] the batch the jobs join
|
|
47
|
+
# @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
|
|
48
|
+
# @raise [ArgumentError] when +list+ is not an array of argument arrays
|
|
49
|
+
# @raise [BulkEnqueueError] when the stream refused jobs of a chunk; the chunks after it are not published
|
|
50
|
+
# @raise [SchedulingDisabledError] for delayed jobs while scheduling is turned off
|
|
51
|
+
def enqueue_many(class_name, list, options, batch: nil)
|
|
52
|
+
raise ArgumentError, "expected an array of argument arrays" unless list.is_a?(::Array) && list.all?(::Array)
|
|
53
|
+
|
|
54
|
+
options = options.merge(batch_id: batch.bid) if batch
|
|
55
|
+
jids = []
|
|
56
|
+
list.each_slice(CHUNK_SIZE) do |chunk|
|
|
57
|
+
offset = jids.size
|
|
58
|
+
chunk_jids, errors = publish_chunk(class_name, chunk, options, batch)
|
|
59
|
+
jids.concat(chunk_jids)
|
|
60
|
+
raise BulkEnqueueError.new(jids + Array.new(list.size - jids.size), errors.transform_keys { _1 + offset }) if errors.any?
|
|
61
|
+
end
|
|
62
|
+
jids
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @param class_name [String]
|
|
66
|
+
# @param chunk [Array<Array>] each job's arguments
|
|
67
|
+
# @param options [Hash]
|
|
68
|
+
# @param batch [Batch, nil]
|
|
69
|
+
# @return [Array(Array<String, nil>, Hash{Integer => StandardError})] each job's jid, and why the jobs the
|
|
70
|
+
# stream refused were not enqueued, by their index in +chunk+
|
|
71
|
+
def publish_chunk(class_name, chunk, options, batch)
|
|
72
|
+
jobs = chunk.map { |args| through_middleware(class_name, Data.new(class_name, args, options)) }
|
|
73
|
+
published = jobs.compact
|
|
74
|
+
return [jobs.map { nil }, {}] if published.empty?
|
|
75
|
+
|
|
76
|
+
batch&.register_job!(published.size)
|
|
77
|
+
results = publish_all(published, batch)
|
|
78
|
+
batch&.rollback_job!(results.count { !_1.is_a?(NATS::JetStream::PubAck) })
|
|
79
|
+
collect(jobs, results)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# @param class_name [String]
|
|
83
|
+
# @param data [Data]
|
|
84
|
+
# @return [Array(Data, Hash), nil] the job and the payload to publish, or nil when a client middleware stopped it
|
|
85
|
+
def through_middleware(class_name, data)
|
|
86
|
+
raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
|
|
87
|
+
|
|
88
|
+
payload = data.as_json
|
|
89
|
+
job = nil
|
|
90
|
+
Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) { job = [data, payload] }
|
|
91
|
+
job
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @param jobs [Array<Array(Data, Hash)>]
|
|
95
|
+
# @param batch [Batch, nil] whose slots for every job are released when the publish raises
|
|
96
|
+
# @return [Array<NATS::JetStream::PubAck, StandardError>]
|
|
97
|
+
def publish_all(jobs, batch)
|
|
98
|
+
messages = jobs.map do |(data, payload)|
|
|
99
|
+
header = data.headers.merge(NATS::JetStream::Header::EXPECTED_STREAM => data.stream.to_s)
|
|
100
|
+
{ subject: data.subject, payload: Utils::Json.dump(payload), header: header }
|
|
101
|
+
end
|
|
102
|
+
Client.instance.publish_many(messages)
|
|
103
|
+
rescue StandardError
|
|
104
|
+
batch&.rollback_job!(jobs.size)
|
|
105
|
+
raise
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @param jobs [Array<Array(Data, Hash), nil>] nil for a job a client middleware stopped
|
|
109
|
+
# @param results [Array<NATS::JetStream::PubAck, StandardError>] one per published job
|
|
110
|
+
# @return [Array(Array<String, nil>, Hash{Integer => StandardError})]
|
|
111
|
+
def collect(jobs, results)
|
|
112
|
+
results = results.each
|
|
113
|
+
errors = {}
|
|
114
|
+
jids = jobs.each_with_index.map do |job, index|
|
|
115
|
+
next unless job
|
|
116
|
+
|
|
117
|
+
result = results.next
|
|
118
|
+
next job.first.jid if result.is_a?(NATS::JetStream::PubAck)
|
|
119
|
+
|
|
120
|
+
errors[index] = result.is_a?(NATS::JetStream::Error::NoStreamResponse) ? StreamNotFoundError.new(job.first.stream.to_s) : result
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
123
|
+
[jids, errors]
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Runs the client middleware around the publish.
|
|
127
|
+
#
|
|
128
|
+
# @param class_name [String]
|
|
129
|
+
# @param data [Data]
|
|
130
|
+
# @return [Boolean] whether the job was published
|
|
36
131
|
def pipe(class_name, data)
|
|
37
132
|
payload = data.as_json
|
|
38
133
|
published = false
|
|
@@ -43,6 +138,10 @@ module Cosmo
|
|
|
43
138
|
published
|
|
44
139
|
end
|
|
45
140
|
|
|
141
|
+
# @param data [Data]
|
|
142
|
+
# @param payload [Hash] the job's payload, possibly changed by a middleware
|
|
143
|
+
# @return [NATS::JetStream::PubAck]
|
|
144
|
+
# @raise [StreamNotFoundError] when the target stream does not exist
|
|
46
145
|
def publish(data, payload)
|
|
47
146
|
Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
|
|
48
147
|
rescue NATS::JetStream::Error::NoStreamResponse
|
data/lib/cosmo/job/limit.rb
CHANGED
|
@@ -15,12 +15,13 @@ module Cosmo
|
|
|
15
15
|
class Limit
|
|
16
16
|
BUCKET = "_cosmolimits"
|
|
17
17
|
|
|
18
|
+
# @return [Limit]
|
|
18
19
|
def self.instance
|
|
19
20
|
@instance ||= new
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
def initialize
|
|
23
|
-
@kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.
|
|
24
|
+
@kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.services.num_replicas)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
# Try to acquire one of the numbered slots for +key+.
|
|
@@ -45,6 +46,9 @@ module Cosmo
|
|
|
45
46
|
# tombstone left behind) so a released slot looks identical to one
|
|
46
47
|
# reclaimed by Nats-TTL expiry -- callers never have to special-case a
|
|
47
48
|
# delete marker.
|
|
49
|
+
#
|
|
50
|
+
# @param slot [String] the slot key returned by {#acquire}
|
|
51
|
+
# @return [void]
|
|
48
52
|
def release(slot)
|
|
49
53
|
@kv.erase(slot)
|
|
50
54
|
rescue NATS::Error
|
data/lib/cosmo/job/processor.rb
CHANGED
|
@@ -8,9 +8,12 @@ module Cosmo
|
|
|
8
8
|
# Headers the scheduler consumes rather than forwards. The +X-+ ones address the dispatch
|
|
9
9
|
# itself, and NATS sets +Nats-Scheduler+/+Nats-Schedule-Next+ on a message it fires and
|
|
10
10
|
# rejects a publish that carries them back.
|
|
11
|
-
DISPATCH_HEADERS =
|
|
12
|
-
|
|
11
|
+
DISPATCH_HEADERS = ["X-Stream", "X-Subject", "X-Execute-At", NATS::JetStream::Header::EXPECTED_STREAM,
|
|
12
|
+
NATS::JetStream::Header::SCHEDULER, NATS::JetStream::Header::SCHEDULE_NEXT].freeze
|
|
13
|
+
ARGS_LIMIT = 256
|
|
13
14
|
|
|
15
|
+
# @param options [Hash] the CLI options
|
|
16
|
+
# @return [void]
|
|
14
17
|
# @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
|
|
15
18
|
def self.validate_options!(options)
|
|
16
19
|
StreamFilter.from(options[:streams]).validate!
|
|
@@ -23,13 +26,33 @@ module Cosmo
|
|
|
23
26
|
names
|
|
24
27
|
end
|
|
25
28
|
|
|
29
|
+
# @return [Array<Hash>] the jobs running right now, each with its +:jid+, +:class+, +:args+ (cut to
|
|
30
|
+
# ARGS_LIMIT bytes), +:retry+, +:stream+, delivery attempt as +:delivery+ and +:started_at+
|
|
31
|
+
def jobs
|
|
32
|
+
@in_flight.each_pair.flat_map do |thread, messages|
|
|
33
|
+
started_at = @started_at[thread]
|
|
34
|
+
messages.map { job_entry(_1, started_at) }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
26
38
|
private
|
|
27
39
|
|
|
40
|
+
def job_entry(message, started_at)
|
|
41
|
+
data = Utils::Json.parse(message.data) || {}
|
|
42
|
+
meta = message.metadata
|
|
43
|
+
{ jid: data[:jid], class: data[:class], args: capped_args(data[:args]), retry: data[:retry],
|
|
44
|
+
stream: meta.stream, delivery: meta.num_delivered, started_at: }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def capped_args(args)
|
|
48
|
+
json = Utils::Json.dump(args)
|
|
49
|
+
json.bytesize > ARGS_LIMIT ? "#{json.byteslice(0, ARGS_LIMIT).scrub("")}…" : args
|
|
50
|
+
end
|
|
51
|
+
|
|
28
52
|
def setup
|
|
29
53
|
filter = StreamFilter.from(@options[:streams]).validate!
|
|
30
54
|
|
|
31
55
|
# Initialize singletons before starting to process messages
|
|
32
|
-
API::Stats::Busy.instance
|
|
33
56
|
API::Stats::Totals.instance
|
|
34
57
|
Limit.instance
|
|
35
58
|
Config.server_middleware
|
|
@@ -40,6 +63,8 @@ module Cosmo
|
|
|
40
63
|
next unless filter.include?(stream_name)
|
|
41
64
|
|
|
42
65
|
@consumers << subscribe(stream_name, config)
|
|
66
|
+
rescue NATS::JetStream::Error::NotFound
|
|
67
|
+
Logger.warn "stream #{stream_name} does not exist, skipping it (run `cosmo --setup` to create it)"
|
|
43
68
|
end
|
|
44
69
|
|
|
45
70
|
log_subscriptions
|
|
@@ -71,8 +96,7 @@ module Cosmo
|
|
|
71
96
|
messages&.each do |message|
|
|
72
97
|
headers = message.header.except(*DISPATCH_HEADERS)
|
|
73
98
|
stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
|
|
74
|
-
|
|
75
|
-
scheduler = message.header["Nats-Scheduler"]
|
|
99
|
+
scheduler = message.header[NATS::JetStream::Header::SCHEDULER]
|
|
76
100
|
if scheduler
|
|
77
101
|
headers["X-Scheduled-By"] = scheduler
|
|
78
102
|
headers["X-Enqueued-At"] = message.metadata.timestamp.to_f.to_s
|
|
@@ -80,7 +104,7 @@ module Cosmo
|
|
|
80
104
|
execute_at = execute_at.to_i
|
|
81
105
|
|
|
82
106
|
if now >= execute_at
|
|
83
|
-
client.publish(subject, message.data, header: headers)
|
|
107
|
+
client.publish(subject, message.data, header: headers, stream:)
|
|
84
108
|
message.ack
|
|
85
109
|
else
|
|
86
110
|
message.nak(delay: Config.to_ns(execute_at - now))
|
|
@@ -104,19 +128,13 @@ module Cosmo
|
|
|
104
128
|
data = Utils::Json.parse(message.data)
|
|
105
129
|
return reject_message(message, ArgumentError.new("malformed payload")) unless data
|
|
106
130
|
|
|
107
|
-
worker_class = Utils::String.safe_constantize(data[:class])
|
|
108
|
-
unless worker_class
|
|
109
|
-
reject_message(message, ArgumentError.new("#{data[:class]} class not found"), data)
|
|
110
|
-
notify_batch(data, success: false)
|
|
111
|
-
return
|
|
112
|
-
end
|
|
113
|
-
|
|
114
131
|
begin
|
|
115
132
|
sw = stopwatch
|
|
116
133
|
Logger.with(jid: data[:jid])
|
|
117
134
|
Logger.info "start"
|
|
118
135
|
|
|
119
|
-
|
|
136
|
+
worker_class = Utils::String.safe_constantize(data[:class])
|
|
137
|
+
instance = build_worker(worker_class, data, message) if worker_class
|
|
120
138
|
Config.server_middleware.invoke(instance, data, message) do
|
|
121
139
|
perform_job(instance, data: data, message: message)
|
|
122
140
|
end
|
|
@@ -127,7 +145,9 @@ module Cosmo
|
|
|
127
145
|
rescue Requeue => e
|
|
128
146
|
message.nak(delay: Config.to_ns(e.delay))
|
|
129
147
|
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "requeue[#{e.delay}s]" }
|
|
130
|
-
rescue
|
|
148
|
+
rescue NATS::JetStream::Error::MsgAlreadyAckd
|
|
149
|
+
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "released[shutdown]" }
|
|
150
|
+
rescue Timeout::Error, JobTimeoutError => e
|
|
131
151
|
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
|
|
132
152
|
Cosmo.handle_error(e, error_context(message, data))
|
|
133
153
|
handle_failure(worker_class, message, data, e)
|
|
@@ -171,6 +191,16 @@ module Cosmo
|
|
|
171
191
|
true
|
|
172
192
|
end
|
|
173
193
|
|
|
194
|
+
# A job still running when the shutdown timed out counts as a failed attempt: it's retried after the
|
|
195
|
+
# usual delay, or moved to the DLQ (dropped when +dead+ is off) once it has no attempts left.
|
|
196
|
+
def abandon(message)
|
|
197
|
+
data = Utils::Json.parse(message.data)
|
|
198
|
+
return super unless data
|
|
199
|
+
|
|
200
|
+
API::Stats::Totals.instance.increment(:failed)
|
|
201
|
+
handle_failure(Utils::String.safe_constantize(data[:class]), message, data, JobInterruptedError.new)
|
|
202
|
+
end
|
|
203
|
+
|
|
174
204
|
def notify_batch(data, success:)
|
|
175
205
|
return unless data[:batch_id]
|
|
176
206
|
|
|
@@ -201,7 +231,7 @@ module Cosmo
|
|
|
201
231
|
end
|
|
202
232
|
|
|
203
233
|
def retry_delay(worker_class, data, current_attempt, exception)
|
|
204
|
-
handler = worker_class
|
|
234
|
+
handler = worker_class&.retry_in(data)
|
|
205
235
|
return default_retry_delay(current_attempt) unless handler
|
|
206
236
|
|
|
207
237
|
delay = handler.call(current_attempt, exception)
|
|
@@ -231,10 +261,11 @@ module Cosmo
|
|
|
231
261
|
end
|
|
232
262
|
|
|
233
263
|
# Logs why a message can't be processed at all and parks it in the DLQ.
|
|
234
|
-
def reject_message(message, error
|
|
264
|
+
def reject_message(message, error)
|
|
235
265
|
Logger.error error
|
|
236
|
-
Cosmo.handle_error(error, error_context(message,
|
|
237
|
-
|
|
266
|
+
Cosmo.handle_error(error, error_context(message, nil, source: :reject))
|
|
267
|
+
API::Stats::Totals.instance.increment(:failed)
|
|
268
|
+
move_message(message, nil, error)
|
|
238
269
|
end
|
|
239
270
|
|
|
240
271
|
def error_context(message, data, source: :job)
|
|
@@ -262,7 +293,7 @@ module Cosmo
|
|
|
262
293
|
def scheduled_by(message)
|
|
263
294
|
header = message.header or return
|
|
264
295
|
|
|
265
|
-
header["X-Scheduled-By"] || header[
|
|
296
|
+
header["X-Scheduled-By"] || header[NATS::JetStream::Header::SCHEDULER]
|
|
266
297
|
end
|
|
267
298
|
|
|
268
299
|
# A cron message is enqueued when NATS fires it, not when the scheduler dispatches it on, so
|
|
@@ -291,12 +322,16 @@ module Cosmo
|
|
|
291
322
|
ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
|
|
292
323
|
end
|
|
293
324
|
|
|
294
|
-
# @param job_instance [Cosmo::Job]
|
|
325
|
+
# @param job_instance [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
295
326
|
# @param data [Hash]
|
|
296
327
|
# @param message [NATS::Msg]
|
|
328
|
+
# @return [Object] what the job's +perform+ returns
|
|
329
|
+
# @raise [JobClassNotFoundError] when +job_instance+ is nil
|
|
297
330
|
#
|
|
298
331
|
# rubocop:disable-next Lint/UnusedMethodArgument
|
|
299
332
|
def perform_job(job_instance, data:, message:)
|
|
333
|
+
raise JobClassNotFoundError, data[:class] unless job_instance
|
|
334
|
+
|
|
300
335
|
job_instance.perform(*data[:args])
|
|
301
336
|
end
|
|
302
337
|
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cosmo
|
|
4
|
+
module Job
|
|
5
|
+
# Per-call job options, returned by {ClassMethods#set}: the enqueue methods merge them over the class options.
|
|
6
|
+
#
|
|
7
|
+
# ReportJob.set(stream: :critical, retry: 0).perform_async(42)
|
|
8
|
+
class Setter
|
|
9
|
+
# Options carried by the job's message. The rest (+limit+, +retry_in+) are read from the class by the worker.
|
|
10
|
+
OPTIONS = %i[stream retry dead subject].freeze
|
|
11
|
+
|
|
12
|
+
# @param job_class [Class] the job class including {Job}
|
|
13
|
+
# @param options [Hash] a subset of {OPTIONS}
|
|
14
|
+
# @raise [ArgumentError] when an option is not one of {OPTIONS}
|
|
15
|
+
def initialize(job_class, options)
|
|
16
|
+
unknown = options.keys - OPTIONS
|
|
17
|
+
if unknown.any?
|
|
18
|
+
hint = " (use stream: instead of queue:)" if unknown.include?(:queue)
|
|
19
|
+
raise ArgumentError, "unknown job options #{unknown.join(", ")}, expected #{OPTIONS.join(", ")}#{hint}"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
@job_class = job_class
|
|
23
|
+
@options = options
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
27
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
28
|
+
def perform_async(*args)
|
|
29
|
+
@job_class.perform(*args, **@options)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param list [Array<Array>] each job's arguments
|
|
33
|
+
# @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
|
|
34
|
+
def perform_bulk(list)
|
|
35
|
+
@job_class.perform_bulk(list, **@options)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param timestamp [Time, Integer] when the job runs
|
|
39
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
40
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
41
|
+
def perform_at(timestamp, *args)
|
|
42
|
+
@job_class.perform(*args, **@options, at: timestamp)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @param interval [Numeric] seconds from now until the job runs
|
|
46
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
47
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
48
|
+
def perform_in(interval, *args)
|
|
49
|
+
@job_class.perform(*args, **@options, in: interval)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|