cosmonats 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -13
- data/lib/cosmo/active_job/adapter.rb +2 -0
- data/lib/cosmo/active_job/executor.rb +7 -0
- data/lib/cosmo/active_job/options.rb +6 -0
- data/lib/cosmo/api/batch.rb +26 -0
- data/lib/cosmo/api/counter.rb +51 -3
- data/lib/cosmo/api/cron/entry.rb +24 -9
- data/lib/cosmo/api/cron.rb +21 -7
- data/lib/cosmo/api/job.rb +14 -0
- data/lib/cosmo/api/kv.rb +138 -9
- data/lib/cosmo/api/registry.rb +47 -6
- data/lib/cosmo/api/stats/jobs.rb +84 -0
- data/lib/cosmo/api/stats/metrics.rb +18 -7
- data/lib/cosmo/api/stats/processes.rb +27 -11
- data/lib/cosmo/api/stats/totals.rb +1 -0
- data/lib/cosmo/api/stats.rb +47 -11
- data/lib/cosmo/api/stream.rb +66 -22
- data/lib/cosmo/batch/callback.rb +5 -0
- data/lib/cosmo/batch/dispatcher.rb +4 -4
- data/lib/cosmo/batch.rb +34 -11
- data/lib/cosmo/cli.rb +6 -0
- data/lib/cosmo/client.rb +208 -11
- data/lib/cosmo/config/cosmo.yml +1 -0
- data/lib/cosmo/config/settings.rb +4 -0
- data/lib/cosmo/config.rb +58 -10
- data/lib/cosmo/engine.rb +28 -1
- data/lib/cosmo/heartbeat.rb +12 -3
- data/lib/cosmo/http_server/handler.rb +5 -0
- data/lib/cosmo/http_server/health.rb +2 -0
- data/lib/cosmo/http_server/ping.rb +2 -0
- data/lib/cosmo/http_server.rb +8 -0
- data/lib/cosmo/job/data.rb +11 -0
- data/lib/cosmo/job/enqueuer.rb +100 -1
- data/lib/cosmo/job/limit.rb +5 -1
- data/lib/cosmo/job/processor.rb +51 -15
- data/lib/cosmo/job/setter.rb +53 -0
- data/lib/cosmo/job.rb +71 -2
- data/lib/cosmo/logger.rb +37 -0
- data/lib/cosmo/middleware/chain.rb +12 -0
- data/lib/cosmo/middleware/limit.rb +12 -3
- data/lib/cosmo/middleware/metrics.rb +6 -4
- data/lib/cosmo/middleware/sentry.rb +6 -4
- data/lib/cosmo/middleware/totals.rb +5 -11
- data/lib/cosmo/middleware.rb +0 -1
- data/lib/cosmo/processor.rb +60 -3
- data/lib/cosmo/publisher.rb +8 -0
- data/lib/cosmo/services.rb +7 -2
- data/lib/cosmo/stream/message.rb +5 -0
- data/lib/cosmo/stream/processor.rb +7 -0
- data/lib/cosmo/stream/serializer.rb +4 -0
- data/lib/cosmo/stream.rb +35 -1
- data/lib/cosmo/utils/duration.rb +3 -0
- data/lib/cosmo/utils/hash.rb +12 -0
- data/lib/cosmo/utils/json.rb +8 -0
- data/lib/cosmo/utils/signal.rb +9 -0
- data/lib/cosmo/utils/stopwatch.rb +1 -0
- data/lib/cosmo/utils/string.rb +4 -0
- data/lib/cosmo/utils/thread_pool.rb +8 -0
- data/lib/cosmo/utils/ttl_cache.rb +10 -0
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +5 -3
- data/lib/cosmo/web/cache.rb +2 -0
- data/lib/cosmo/web/chart.rb +2 -2
- data/lib/cosmo/web/circuit.rb +1 -0
- data/lib/cosmo/web/context.rb +6 -0
- data/lib/cosmo/web/controllers/application.rb +26 -0
- data/lib/cosmo/web/controllers/batches.rb +7 -1
- data/lib/cosmo/web/controllers/crons.rb +16 -2
- data/lib/cosmo/web/controllers/jobs.rb +45 -16
- data/lib/cosmo/web/controllers/metrics.rb +5 -0
- data/lib/cosmo/web/controllers/overview.rb +36 -18
- data/lib/cosmo/web/controllers/processes.rb +8 -5
- data/lib/cosmo/web/controllers/streams.rb +12 -0
- data/lib/cosmo/web/helpers/application.rb +44 -0
- data/lib/cosmo/web/renderer.rb +17 -0
- data/lib/cosmo/web/throughput.rb +1 -1
- data/lib/cosmo/web/views/crons/_table.erb +29 -27
- data/lib/cosmo/web/views/jobs/_args.erb +2 -2
- data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
- data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
- data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
- data/lib/cosmo/web/views/layout.erb +1 -1
- data/lib/cosmo/web/views/overview/_badges.erb +1 -1
- data/lib/cosmo/web/views/overview/_running.erb +3 -3
- data/lib/cosmo/web/views/overview/index.erb +1 -1
- data/lib/cosmo/web/views/streams/_info.erb +1 -0
- data/lib/cosmo/web/views/streams/_stream.erb +1 -0
- data/lib/cosmo/web.rb +8 -1
- data/lib/cosmo.rb +33 -0
- data/sig/cosmo/active_job/adapter.rbs +3 -3
- data/sig/cosmo/active_job/options.rbs +4 -2
- data/sig/cosmo/api/batch.rbs +5 -3
- data/sig/cosmo/api/counter.rbs +16 -6
- data/sig/cosmo/api/cron/entry.rbs +4 -2
- data/sig/cosmo/api/cron.rbs +10 -4
- data/sig/cosmo/api/kv.rbs +21 -4
- data/sig/cosmo/api/registry.rbs +10 -0
- data/sig/cosmo/api/stats/jobs.rbs +34 -0
- data/sig/cosmo/api/stats/metrics.rbs +3 -0
- data/sig/cosmo/api/stats/processes.rbs +9 -1
- data/sig/cosmo/api/stats.rbs +8 -2
- data/sig/cosmo/api/stream.rbs +10 -5
- data/sig/cosmo/batch/callback.rbs +1 -1
- data/sig/cosmo/batch/dispatcher.rbs +1 -1
- data/sig/cosmo/batch.rbs +5 -5
- data/sig/cosmo/cli.rbs +2 -0
- data/sig/cosmo/client.rbs +23 -4
- data/sig/cosmo/config/settings.rbs +6 -0
- data/sig/cosmo/config.rbs +5 -5
- data/sig/cosmo/engine.rbs +5 -0
- data/sig/cosmo/heartbeat.rbs +2 -0
- data/sig/cosmo/job/data.rbs +3 -1
- data/sig/cosmo/job/enqueuer.rbs +12 -0
- data/sig/cosmo/job/processor.rbs +18 -3
- data/sig/cosmo/job/setter.rbs +20 -0
- data/sig/cosmo/job.rbs +10 -4
- data/sig/cosmo/logger.rbs +5 -3
- data/sig/cosmo/middleware.rbs +0 -8
- data/sig/cosmo/processor.rbs +14 -2
- data/sig/cosmo/publisher.rbs +1 -0
- data/sig/cosmo/stream/processor.rbs +4 -0
- data/sig/cosmo/stream.rbs +4 -3
- data/sig/cosmo/utils/hash.rbs +2 -1
- data/sig/cosmo/utils/signal.rbs +5 -5
- data/sig/cosmo/utils/string.rbs +2 -2
- data/sig/cosmo/utils/thread_pool.rbs +1 -1
- data/sig/cosmo/web/cache.rbs +35 -0
- data/sig/cosmo/web/chart.rbs +48 -0
- data/sig/cosmo/web/circuit.rbs +29 -0
- data/sig/cosmo/web/context.rbs +15 -0
- data/sig/cosmo/web/controllers/application.rbs +32 -0
- data/sig/cosmo/web/controllers/batches.rbs +11 -0
- data/sig/cosmo/web/controllers/crons.rbs +23 -0
- data/sig/cosmo/web/controllers/jobs.rbs +47 -0
- data/sig/cosmo/web/controllers/metrics.rbs +21 -0
- data/sig/cosmo/web/controllers/overview.rbs +31 -0
- data/sig/cosmo/web/controllers/processes.rbs +13 -0
- data/sig/cosmo/web/controllers/streams.rbs +27 -0
- data/sig/cosmo/web/helpers/application.rbs +61 -0
- data/sig/cosmo/web/renderer.rbs +33 -0
- data/sig/cosmo/web/throughput.rbs +45 -0
- data/sig/cosmo/web.rbs +25 -0
- data/sig/cosmo.rbs +19 -0
- metadata +23 -6
- data/lib/cosmo/api/stats/busy.rb +0 -68
- data/lib/cosmo/middleware/busy.rb +0 -15
- data/sig/cosmo/api/stats/busy.rbs +0 -33
|
@@ -11,6 +11,11 @@ module Cosmo
|
|
|
11
11
|
@app = app
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# Runs the Rack app for +req+ and writes its response to +res+.
|
|
15
|
+
#
|
|
16
|
+
# @param req [WEBrick::HTTPRequest]
|
|
17
|
+
# @param res [WEBrick::HTTPResponse]
|
|
18
|
+
# @return [void]
|
|
14
19
|
def service(req, res)
|
|
15
20
|
status, headers, body = @app.call(env_for(req))
|
|
16
21
|
res.status = status
|
|
@@ -13,6 +13,8 @@ module Cosmo
|
|
|
13
13
|
@check = check || method(:default_check)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
# @param env [Hash] the Rack env
|
|
17
|
+
# @return [Array] the Rack response
|
|
16
18
|
def call(env)
|
|
17
19
|
return @app.call(env) unless env[Rack::PATH_INFO] == PATH
|
|
18
20
|
return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
|
|
@@ -12,6 +12,8 @@ module Cosmo
|
|
|
12
12
|
@app = app
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
+
# @param env [Hash] the Rack env
|
|
16
|
+
# @return [Array] the Rack response
|
|
15
17
|
def call(env)
|
|
16
18
|
return @app.call(env) unless env[Rack::PATH_INFO] == PATH
|
|
17
19
|
return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
|
data/lib/cosmo/http_server.rb
CHANGED
|
@@ -19,6 +19,8 @@ module Cosmo
|
|
|
19
19
|
NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
|
|
20
20
|
|
|
21
21
|
# Routing is done by middlewares, anything unmatched falls through to 404.
|
|
22
|
+
#
|
|
23
|
+
# @return [#call] the Rack app
|
|
22
24
|
def self.app
|
|
23
25
|
Rack::Builder.app do
|
|
24
26
|
use Health
|
|
@@ -33,6 +35,9 @@ module Cosmo
|
|
|
33
35
|
@port = port
|
|
34
36
|
end
|
|
35
37
|
|
|
38
|
+
# Starts serving in a background thread.
|
|
39
|
+
#
|
|
40
|
+
# @return [self]
|
|
36
41
|
def start
|
|
37
42
|
@server = ::WEBrick::HTTPServer.new(
|
|
38
43
|
BindAddress: @host,
|
|
@@ -46,6 +51,7 @@ module Cosmo
|
|
|
46
51
|
self
|
|
47
52
|
end
|
|
48
53
|
|
|
54
|
+
# @return [void]
|
|
49
55
|
def stop
|
|
50
56
|
return unless @server
|
|
51
57
|
|
|
@@ -55,6 +61,8 @@ module Cosmo
|
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
# Actual bound port, useful when started with port 0.
|
|
64
|
+
#
|
|
65
|
+
# @return [Integer]
|
|
58
66
|
def port
|
|
59
67
|
@server ? @server.listeners.first.addr[1] : @port
|
|
60
68
|
end
|
data/lib/cosmo/job/data.rb
CHANGED
|
@@ -7,12 +7,17 @@ module Cosmo
|
|
|
7
7
|
class Data
|
|
8
8
|
DEFAULTS = { stream: :default, retry: 3, dead: true, limit: nil }.freeze
|
|
9
9
|
|
|
10
|
+
# @return [Integer] +max_retries+ from the config, or the built-in default
|
|
10
11
|
def self.default_retry
|
|
11
12
|
Config[:max_retries] || DEFAULTS[:retry]
|
|
12
13
|
end
|
|
13
14
|
|
|
14
15
|
attr_reader :jid
|
|
15
16
|
|
|
17
|
+
# @param class_name [String]
|
|
18
|
+
# @param args [Array]
|
|
19
|
+
# @param options [Hash, nil] job options, +stream:+ is required
|
|
20
|
+
# @raise [ArgumentError] when no stream is given
|
|
16
21
|
def initialize(class_name, args, options = nil)
|
|
17
22
|
@class_name = class_name
|
|
18
23
|
@args = args
|
|
@@ -26,10 +31,13 @@ module Cosmo
|
|
|
26
31
|
@jid = SecureRandom.hex(12)
|
|
27
32
|
end
|
|
28
33
|
|
|
34
|
+
# @return [String, nil] the id of the batch the job belongs to
|
|
29
35
|
def batch_id
|
|
30
36
|
@options[:batch_id]
|
|
31
37
|
end
|
|
32
38
|
|
|
39
|
+
# @param target [Boolean] true returns the job's own stream even when it is delayed
|
|
40
|
+
# @return [Symbol] the stream the job is published to: +scheduled+ for a delayed job unless +target+
|
|
33
41
|
def stream(target: false)
|
|
34
42
|
return @options[:stream] if target
|
|
35
43
|
|
|
@@ -51,11 +59,14 @@ module Cosmo
|
|
|
51
59
|
headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
|
|
52
60
|
end
|
|
53
61
|
|
|
62
|
+
# @return [Hash{Symbol => Object}] the published payload
|
|
54
63
|
def as_json
|
|
55
64
|
json = { jid: jid, class: @class_name, args: @args, retry: retries, dead: dead }
|
|
56
65
|
batch_id ? json.merge(batch_id: batch_id) : json
|
|
57
66
|
end
|
|
58
67
|
|
|
68
|
+
# @param _args [Array] ignored, accepted for +JSON.generate+ compatibility
|
|
69
|
+
# @return [String, nil]
|
|
59
70
|
def to_json(*_args)
|
|
60
71
|
Utils::Json.dump(as_json)
|
|
61
72
|
end
|
data/lib/cosmo/job/enqueuer.rb
CHANGED
|
@@ -4,6 +4,9 @@ module Cosmo
|
|
|
4
4
|
module Job
|
|
5
5
|
# The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
|
|
6
6
|
module Enqueuer
|
|
7
|
+
# Jobs published at once by {.enqueue_many}, before waiting for their acks.
|
|
8
|
+
CHUNK_SIZE = 1_000
|
|
9
|
+
|
|
7
10
|
module_function
|
|
8
11
|
|
|
9
12
|
# Publishes the job through {Config#client_middleware}.
|
|
@@ -17,7 +20,7 @@ module Cosmo
|
|
|
17
20
|
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
18
21
|
# @raise [StreamNotFoundError] when the target stream does not exist
|
|
19
22
|
# @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
|
|
20
|
-
def enqueue(class_name, args, options, batch: nil)
|
|
23
|
+
def enqueue(class_name, args, options, batch: nil)
|
|
21
24
|
options = options.merge(batch_id: batch.bid) if batch
|
|
22
25
|
data = Data.new(class_name, args, options)
|
|
23
26
|
raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
|
|
@@ -33,6 +36,98 @@ module Cosmo
|
|
|
33
36
|
data.jid if published
|
|
34
37
|
end
|
|
35
38
|
|
|
39
|
+
# Enqueues one job per argument list, publishing them a chunk at a time with {Client#publish_many}. Each job
|
|
40
|
+
# still runs through {Config#client_middleware}, whose block only marks it for publishing: the chunk is
|
|
41
|
+
# published once every job in it went through the chain.
|
|
42
|
+
#
|
|
43
|
+
# @param class_name [String]
|
|
44
|
+
# @param list [Array<Array>] each job's arguments
|
|
45
|
+
# @param options [Hash] job options shared by every job, see {Data}
|
|
46
|
+
# @param batch [Batch, nil] the batch the jobs join
|
|
47
|
+
# @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
|
|
48
|
+
# @raise [ArgumentError] when +list+ is not an array of argument arrays
|
|
49
|
+
# @raise [BulkEnqueueError] when the stream refused jobs of a chunk; the chunks after it are not published
|
|
50
|
+
# @raise [SchedulingDisabledError] for delayed jobs while scheduling is turned off
|
|
51
|
+
def enqueue_many(class_name, list, options, batch: nil)
|
|
52
|
+
raise ArgumentError, "expected an array of argument arrays" unless list.is_a?(::Array) && list.all?(::Array)
|
|
53
|
+
|
|
54
|
+
options = options.merge(batch_id: batch.bid) if batch
|
|
55
|
+
jids = []
|
|
56
|
+
list.each_slice(CHUNK_SIZE) do |chunk|
|
|
57
|
+
offset = jids.size
|
|
58
|
+
chunk_jids, errors = publish_chunk(class_name, chunk, options, batch)
|
|
59
|
+
jids.concat(chunk_jids)
|
|
60
|
+
raise BulkEnqueueError.new(jids + Array.new(list.size - jids.size), errors.transform_keys { _1 + offset }) if errors.any?
|
|
61
|
+
end
|
|
62
|
+
jids
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @param class_name [String]
|
|
66
|
+
# @param chunk [Array<Array>] each job's arguments
|
|
67
|
+
# @param options [Hash]
|
|
68
|
+
# @param batch [Batch, nil]
|
|
69
|
+
# @return [Array(Array<String, nil>, Hash{Integer => StandardError})] each job's jid, and why the jobs the
|
|
70
|
+
# stream refused were not enqueued, by their index in +chunk+
|
|
71
|
+
def publish_chunk(class_name, chunk, options, batch)
|
|
72
|
+
jobs = chunk.map { |args| through_middleware(class_name, Data.new(class_name, args, options)) }
|
|
73
|
+
published = jobs.compact
|
|
74
|
+
return [jobs.map { nil }, {}] if published.empty?
|
|
75
|
+
|
|
76
|
+
batch&.register_job!(published.size)
|
|
77
|
+
results = publish_all(published, batch)
|
|
78
|
+
batch&.rollback_job!(results.count { !_1.is_a?(NATS::JetStream::PubAck) })
|
|
79
|
+
collect(jobs, results)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# @param class_name [String]
|
|
83
|
+
# @param data [Data]
|
|
84
|
+
# @return [Array(Data, Hash), nil] the job and the payload to publish, or nil when a client middleware stopped it
|
|
85
|
+
def through_middleware(class_name, data)
|
|
86
|
+
raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
|
|
87
|
+
|
|
88
|
+
payload = data.as_json
|
|
89
|
+
job = nil
|
|
90
|
+
Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) { job = [data, payload] }
|
|
91
|
+
job
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @param jobs [Array<Array(Data, Hash)>]
|
|
95
|
+
# @param batch [Batch, nil] whose slots for every job are released when the publish raises
|
|
96
|
+
# @return [Array<NATS::JetStream::PubAck, StandardError>]
|
|
97
|
+
def publish_all(jobs, batch)
|
|
98
|
+
messages = jobs.map do |(data, payload)|
|
|
99
|
+
header = data.headers.merge(NATS::JetStream::Header::EXPECTED_STREAM => data.stream.to_s)
|
|
100
|
+
{ subject: data.subject, payload: Utils::Json.dump(payload), header: header }
|
|
101
|
+
end
|
|
102
|
+
Client.instance.publish_many(messages)
|
|
103
|
+
rescue StandardError
|
|
104
|
+
batch&.rollback_job!(jobs.size)
|
|
105
|
+
raise
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @param jobs [Array<Array(Data, Hash), nil>] nil for a job a client middleware stopped
|
|
109
|
+
# @param results [Array<NATS::JetStream::PubAck, StandardError>] one per published job
|
|
110
|
+
# @return [Array(Array<String, nil>, Hash{Integer => StandardError})]
|
|
111
|
+
def collect(jobs, results)
|
|
112
|
+
results = results.each
|
|
113
|
+
errors = {}
|
|
114
|
+
jids = jobs.each_with_index.map do |job, index|
|
|
115
|
+
next unless job
|
|
116
|
+
|
|
117
|
+
result = results.next
|
|
118
|
+
next job.first.jid if result.is_a?(NATS::JetStream::PubAck)
|
|
119
|
+
|
|
120
|
+
errors[index] = result.is_a?(NATS::JetStream::Error::NoStreamResponse) ? StreamNotFoundError.new(job.first.stream.to_s) : result
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
123
|
+
[jids, errors]
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Runs the client middleware around the publish.
|
|
127
|
+
#
|
|
128
|
+
# @param class_name [String]
|
|
129
|
+
# @param data [Data]
|
|
130
|
+
# @return [Boolean] whether the job was published
|
|
36
131
|
def pipe(class_name, data)
|
|
37
132
|
payload = data.as_json
|
|
38
133
|
published = false
|
|
@@ -43,6 +138,10 @@ module Cosmo
|
|
|
43
138
|
published
|
|
44
139
|
end
|
|
45
140
|
|
|
141
|
+
# @param data [Data]
|
|
142
|
+
# @param payload [Hash] the job's payload, possibly changed by a middleware
|
|
143
|
+
# @return [NATS::JetStream::PubAck]
|
|
144
|
+
# @raise [StreamNotFoundError] when the target stream does not exist
|
|
46
145
|
def publish(data, payload)
|
|
47
146
|
Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
|
|
48
147
|
rescue NATS::JetStream::Error::NoStreamResponse
|
data/lib/cosmo/job/limit.rb
CHANGED
|
@@ -15,12 +15,13 @@ module Cosmo
|
|
|
15
15
|
class Limit
|
|
16
16
|
BUCKET = "_cosmolimits"
|
|
17
17
|
|
|
18
|
+
# @return [Limit]
|
|
18
19
|
def self.instance
|
|
19
20
|
@instance ||= new
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
def initialize
|
|
23
|
-
@kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.
|
|
24
|
+
@kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.services.num_replicas)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
# Try to acquire one of the numbered slots for +key+.
|
|
@@ -45,6 +46,9 @@ module Cosmo
|
|
|
45
46
|
# tombstone left behind) so a released slot looks identical to one
|
|
46
47
|
# reclaimed by Nats-TTL expiry -- callers never have to special-case a
|
|
47
48
|
# delete marker.
|
|
49
|
+
#
|
|
50
|
+
# @param slot [String] the slot key returned by {#acquire}
|
|
51
|
+
# @return [void]
|
|
48
52
|
def release(slot)
|
|
49
53
|
@kv.erase(slot)
|
|
50
54
|
rescue NATS::Error
|
data/lib/cosmo/job/processor.rb
CHANGED
|
@@ -10,7 +10,10 @@ module Cosmo
|
|
|
10
10
|
# rejects a publish that carries them back.
|
|
11
11
|
DISPATCH_HEADERS = ["X-Stream", "X-Subject", "X-Execute-At", NATS::JetStream::Header::EXPECTED_STREAM,
|
|
12
12
|
NATS::JetStream::Header::SCHEDULER, NATS::JetStream::Header::SCHEDULE_NEXT].freeze
|
|
13
|
+
ARGS_LIMIT = 256
|
|
13
14
|
|
|
15
|
+
# @param options [Hash] the CLI options
|
|
16
|
+
# @return [void]
|
|
14
17
|
# @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
|
|
15
18
|
def self.validate_options!(options)
|
|
16
19
|
StreamFilter.from(options[:streams]).validate!
|
|
@@ -23,13 +26,33 @@ module Cosmo
|
|
|
23
26
|
names
|
|
24
27
|
end
|
|
25
28
|
|
|
29
|
+
# @return [Array<Hash>] the jobs running right now, each with its +:jid+, +:class+, +:args+ (cut to
|
|
30
|
+
# ARGS_LIMIT bytes), +:retry+, +:stream+, delivery attempt as +:delivery+ and +:started_at+
|
|
31
|
+
def jobs
|
|
32
|
+
@in_flight.each_pair.flat_map do |thread, messages|
|
|
33
|
+
started_at = @started_at[thread]
|
|
34
|
+
messages.map { job_entry(_1, started_at) }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
26
38
|
private
|
|
27
39
|
|
|
40
|
+
def job_entry(message, started_at)
|
|
41
|
+
data = Utils::Json.parse(message.data) || {}
|
|
42
|
+
meta = message.metadata
|
|
43
|
+
{ jid: data[:jid], class: data[:class], args: capped_args(data[:args]), retry: data[:retry],
|
|
44
|
+
stream: meta.stream, delivery: meta.num_delivered, started_at: }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def capped_args(args)
|
|
48
|
+
json = Utils::Json.dump(args)
|
|
49
|
+
json.bytesize > ARGS_LIMIT ? "#{json.byteslice(0, ARGS_LIMIT).scrub("")}…" : args
|
|
50
|
+
end
|
|
51
|
+
|
|
28
52
|
def setup
|
|
29
53
|
filter = StreamFilter.from(@options[:streams]).validate!
|
|
30
54
|
|
|
31
55
|
# Initialize singletons before starting to process messages
|
|
32
|
-
API::Stats::Busy.instance
|
|
33
56
|
API::Stats::Totals.instance
|
|
34
57
|
Limit.instance
|
|
35
58
|
Config.server_middleware
|
|
@@ -40,6 +63,8 @@ module Cosmo
|
|
|
40
63
|
next unless filter.include?(stream_name)
|
|
41
64
|
|
|
42
65
|
@consumers << subscribe(stream_name, config)
|
|
66
|
+
rescue NATS::JetStream::Error::NotFound
|
|
67
|
+
Logger.warn "stream #{stream_name} does not exist, skipping it (run `cosmo --setup` to create it)"
|
|
43
68
|
end
|
|
44
69
|
|
|
45
70
|
log_subscriptions
|
|
@@ -103,19 +128,13 @@ module Cosmo
|
|
|
103
128
|
data = Utils::Json.parse(message.data)
|
|
104
129
|
return reject_message(message, ArgumentError.new("malformed payload")) unless data
|
|
105
130
|
|
|
106
|
-
worker_class = Utils::String.safe_constantize(data[:class])
|
|
107
|
-
unless worker_class
|
|
108
|
-
reject_message(message, ArgumentError.new("#{data[:class]} class not found"), data)
|
|
109
|
-
notify_batch(data, success: false)
|
|
110
|
-
return
|
|
111
|
-
end
|
|
112
|
-
|
|
113
131
|
begin
|
|
114
132
|
sw = stopwatch
|
|
115
133
|
Logger.with(jid: data[:jid])
|
|
116
134
|
Logger.info "start"
|
|
117
135
|
|
|
118
|
-
|
|
136
|
+
worker_class = Utils::String.safe_constantize(data[:class])
|
|
137
|
+
instance = build_worker(worker_class, data, message) if worker_class
|
|
119
138
|
Config.server_middleware.invoke(instance, data, message) do
|
|
120
139
|
perform_job(instance, data: data, message: message)
|
|
121
140
|
end
|
|
@@ -126,7 +145,9 @@ module Cosmo
|
|
|
126
145
|
rescue Requeue => e
|
|
127
146
|
message.nak(delay: Config.to_ns(e.delay))
|
|
128
147
|
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "requeue[#{e.delay}s]" }
|
|
129
|
-
rescue
|
|
148
|
+
rescue NATS::JetStream::Error::MsgAlreadyAckd
|
|
149
|
+
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "released[shutdown]" }
|
|
150
|
+
rescue Timeout::Error, JobTimeoutError => e
|
|
130
151
|
Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
|
|
131
152
|
Cosmo.handle_error(e, error_context(message, data))
|
|
132
153
|
handle_failure(worker_class, message, data, e)
|
|
@@ -170,6 +191,16 @@ module Cosmo
|
|
|
170
191
|
true
|
|
171
192
|
end
|
|
172
193
|
|
|
194
|
+
# A job still running when the shutdown timed out counts as a failed attempt: it's retried after the
|
|
195
|
+
# usual delay, or moved to the DLQ (dropped when +dead+ is off) once it has no attempts left.
|
|
196
|
+
def abandon(message)
|
|
197
|
+
data = Utils::Json.parse(message.data)
|
|
198
|
+
return super unless data
|
|
199
|
+
|
|
200
|
+
API::Stats::Totals.instance.increment(:failed)
|
|
201
|
+
handle_failure(Utils::String.safe_constantize(data[:class]), message, data, JobInterruptedError.new)
|
|
202
|
+
end
|
|
203
|
+
|
|
173
204
|
def notify_batch(data, success:)
|
|
174
205
|
return unless data[:batch_id]
|
|
175
206
|
|
|
@@ -200,7 +231,7 @@ module Cosmo
|
|
|
200
231
|
end
|
|
201
232
|
|
|
202
233
|
def retry_delay(worker_class, data, current_attempt, exception)
|
|
203
|
-
handler = worker_class
|
|
234
|
+
handler = worker_class&.retry_in(data)
|
|
204
235
|
return default_retry_delay(current_attempt) unless handler
|
|
205
236
|
|
|
206
237
|
delay = handler.call(current_attempt, exception)
|
|
@@ -230,10 +261,11 @@ module Cosmo
|
|
|
230
261
|
end
|
|
231
262
|
|
|
232
263
|
# Logs why a message can't be processed at all and parks it in the DLQ.
|
|
233
|
-
def reject_message(message, error
|
|
264
|
+
def reject_message(message, error)
|
|
234
265
|
Logger.error error
|
|
235
|
-
Cosmo.handle_error(error, error_context(message,
|
|
236
|
-
|
|
266
|
+
Cosmo.handle_error(error, error_context(message, nil, source: :reject))
|
|
267
|
+
API::Stats::Totals.instance.increment(:failed)
|
|
268
|
+
move_message(message, nil, error)
|
|
237
269
|
end
|
|
238
270
|
|
|
239
271
|
def error_context(message, data, source: :job)
|
|
@@ -290,12 +322,16 @@ module Cosmo
|
|
|
290
322
|
ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
|
|
291
323
|
end
|
|
292
324
|
|
|
293
|
-
# @param job_instance [Cosmo::Job]
|
|
325
|
+
# @param job_instance [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
294
326
|
# @param data [Hash]
|
|
295
327
|
# @param message [NATS::Msg]
|
|
328
|
+
# @return [Object] what the job's +perform+ returns
|
|
329
|
+
# @raise [JobClassNotFoundError] when +job_instance+ is nil
|
|
296
330
|
#
|
|
297
331
|
# rubocop:disable-next Lint/UnusedMethodArgument
|
|
298
332
|
def perform_job(job_instance, data:, message:)
|
|
333
|
+
raise JobClassNotFoundError, data[:class] unless job_instance
|
|
334
|
+
|
|
299
335
|
job_instance.perform(*data[:args])
|
|
300
336
|
end
|
|
301
337
|
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cosmo
|
|
4
|
+
module Job
|
|
5
|
+
# Per-call job options, returned by {ClassMethods#set}: the enqueue methods merge them over the class options.
|
|
6
|
+
#
|
|
7
|
+
# ReportJob.set(stream: :critical, retry: 0).perform_async(42)
|
|
8
|
+
class Setter
|
|
9
|
+
# Options carried by the job's message. The rest (+limit+, +retry_in+) are read from the class by the worker.
|
|
10
|
+
OPTIONS = %i[stream retry dead subject].freeze
|
|
11
|
+
|
|
12
|
+
# @param job_class [Class] the job class including {Job}
|
|
13
|
+
# @param options [Hash] a subset of {OPTIONS}
|
|
14
|
+
# @raise [ArgumentError] when an option is not one of {OPTIONS}
|
|
15
|
+
def initialize(job_class, options)
|
|
16
|
+
unknown = options.keys - OPTIONS
|
|
17
|
+
if unknown.any?
|
|
18
|
+
hint = " (use stream: instead of queue:)" if unknown.include?(:queue)
|
|
19
|
+
raise ArgumentError, "unknown job options #{unknown.join(", ")}, expected #{OPTIONS.join(", ")}#{hint}"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
@job_class = job_class
|
|
23
|
+
@options = options
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
27
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
28
|
+
def perform_async(*args)
|
|
29
|
+
@job_class.perform(*args, **@options)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param list [Array<Array>] each job's arguments
|
|
33
|
+
# @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
|
|
34
|
+
def perform_bulk(list)
|
|
35
|
+
@job_class.perform_bulk(list, **@options)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param timestamp [Time, Integer] when the job runs
|
|
39
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
40
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
41
|
+
def perform_at(timestamp, *args)
|
|
42
|
+
@job_class.perform(*args, **@options, at: timestamp)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @param interval [Numeric] seconds from now until the job runs
|
|
46
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
47
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
48
|
+
def perform_in(interval, *args)
|
|
49
|
+
@job_class.perform(*args, **@options, in: interval)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
data/lib/cosmo/job.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "cosmo/job/data"
|
|
4
4
|
require "cosmo/job/enqueuer"
|
|
5
|
+
require "cosmo/job/setter"
|
|
5
6
|
require "cosmo/job/requeue"
|
|
6
7
|
require "cosmo/job/failure"
|
|
7
8
|
require "cosmo/job/stream_filter"
|
|
@@ -10,6 +11,8 @@ require "cosmo/job/processor"
|
|
|
10
11
|
|
|
11
12
|
module Cosmo
|
|
12
13
|
module Job
|
|
14
|
+
# @param base [Class] the job class including Job
|
|
15
|
+
# @return [void]
|
|
13
16
|
def self.included(base)
|
|
14
17
|
base.extend(ClassMethods)
|
|
15
18
|
end
|
|
@@ -28,14 +31,15 @@ module Cosmo
|
|
|
28
31
|
# limit: { duration: 30, concurrency: { to: 3, key: ->(id) { id } } }
|
|
29
32
|
# limit: { duration: 30, concurrency: 3, retry_in: 5 }
|
|
30
33
|
#
|
|
31
|
-
# @option config [Integer] :"limit[:duration]" hard execution timeout in seconds
|
|
34
|
+
# @option config [Integer, String] :"limit[:duration]" hard execution timeout in seconds, or a duration such as
|
|
35
|
+
# +"82m"+ (see {Utils::Duration.parse}). The job thread is
|
|
32
36
|
# killed after this many seconds and counts as a failed attempt (retried with exponential backoff,
|
|
33
37
|
# moved to DLQ after retries exhausted).
|
|
34
38
|
# @option config [Integer, Hash] :"limit[:concurrency]" caps how many instances run at once across all
|
|
35
39
|
# workers. Jobs that cannot acquire a slot are NAK'd (see +retry_in+) so they are not re-delivered until
|
|
36
40
|
# the slot is likely free. Requires +duration+.
|
|
37
41
|
# Pass an Integer for a class-wide cap, or <tt>{ to: N, key: ->(args) {} }</tt> to scope per key.
|
|
38
|
-
# @option config [Integer] :"limit[:retry_in]" seconds to wait before NATS redelivers a job that was
|
|
42
|
+
# @option config [Integer, String] :"limit[:retry_in]" seconds, or a duration such as +"30s"+, to wait before NATS redelivers a job that was
|
|
39
43
|
# NAK'd for lack of a concurrency slot (default: half of +duration+). Counts against the same delivery
|
|
40
44
|
# counter as any other retry -- a job stuck behind the concurrency limit for enough consecutive
|
|
41
45
|
# attempts is dropped/DLQ'd exactly like one that keeps failing outright.
|
|
@@ -50,7 +54,14 @@ module Cosmo
|
|
|
50
54
|
# +limit[:retry_in]+) or actually ran and failed. So a job that gets turned away twice for lack of a
|
|
51
55
|
# slot, then finally runs and fails, calls this handler with +count == 3+, not 1. Don't read +count+ as
|
|
52
56
|
# "how many times perform has actually run and failed" when concurrency limits are in play.
|
|
57
|
+
# @return [Hash{Symbol => Object}] the class's merged options
|
|
58
|
+
# @raise [ArgumentError] when +limit[:concurrency]+ lacks a +duration+, a limit duration is invalid, or +retry_in+
|
|
59
|
+
# is not callable
|
|
53
60
|
def options(**config)
|
|
61
|
+
if (limit = config[:limit])
|
|
62
|
+
config[:limit] = limit.merge(limit.slice(:duration, :retry_in).compact.transform_values { Utils::Duration.parse(_1) })
|
|
63
|
+
end
|
|
64
|
+
|
|
54
65
|
if config[:limit] && config.dig(:limit, :concurrency) && !config.dig(:limit, :duration).to_i.positive?
|
|
55
66
|
raise ArgumentError, "limit: duration is required when concurrency is set"
|
|
56
67
|
end
|
|
@@ -61,6 +72,7 @@ module Cosmo
|
|
|
61
72
|
end
|
|
62
73
|
alias cosmo_options options
|
|
63
74
|
|
|
75
|
+
# @return [Boolean] whether +limit[:concurrency]+ is configured
|
|
64
76
|
def limits_concurrency?
|
|
65
77
|
!!concurrency_options
|
|
66
78
|
end
|
|
@@ -68,12 +80,17 @@ module Cosmo
|
|
|
68
80
|
# Returns the +retry_in+ Proc/lambda (taking +(count, exception)+) configured for this job class, or
|
|
69
81
|
# +nil+ when unset. Overridable by wrapper job classes (e.g. the ActiveJob executor) that need to
|
|
70
82
|
# resolve it from something other than +self+.
|
|
83
|
+
#
|
|
84
|
+
# @param _data [Hash, nil] the job payload, unused here
|
|
85
|
+
# @return [Proc, nil]
|
|
71
86
|
def retry_in(_data = nil)
|
|
72
87
|
default_options[:retry_in]
|
|
73
88
|
end
|
|
74
89
|
|
|
75
90
|
# Returns a normalized concurrency config hash, or +nil+ when not configured.
|
|
76
91
|
# Always contains +:limit+, +:key+, +:duration+, and +:retry_in+.
|
|
92
|
+
#
|
|
93
|
+
# @return [Hash{Symbol => Object}, nil]
|
|
77
94
|
def concurrency_options
|
|
78
95
|
value = default_options.dig(:limit, :concurrency)
|
|
79
96
|
return unless value
|
|
@@ -88,6 +105,9 @@ module Cosmo
|
|
|
88
105
|
end
|
|
89
106
|
|
|
90
107
|
# Derive the fully-scoped concurrency key for a given args array.
|
|
108
|
+
#
|
|
109
|
+
# @param args [Array] the job's arguments
|
|
110
|
+
# @return [String, nil] nil when concurrency is not limited
|
|
91
111
|
def concurrency_key(args)
|
|
92
112
|
config = concurrency_options
|
|
93
113
|
return unless config
|
|
@@ -97,6 +117,12 @@ module Cosmo
|
|
|
97
117
|
suffix ? "#{base}/#{suffix}" : base
|
|
98
118
|
end
|
|
99
119
|
|
|
120
|
+
# Enqueues the job, or runs it inline when +async+ is false.
|
|
121
|
+
#
|
|
122
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
123
|
+
# @param async [Boolean] false runs the job in the current thread, without NATS
|
|
124
|
+
# @param options [Hash] job options merged over {#default_options}, e.g. +at:+ or +in:+
|
|
125
|
+
# @return [String, nil] the job's jid, or nil when run inline or a client middleware stopped the publish
|
|
100
126
|
def perform(*args, async: true, **options)
|
|
101
127
|
options = default_options.merge(options)
|
|
102
128
|
return Enqueuer.enqueue(name, args, options, batch: Batch.current) if async
|
|
@@ -108,22 +134,60 @@ module Cosmo
|
|
|
108
134
|
nil
|
|
109
135
|
end
|
|
110
136
|
|
|
137
|
+
# Options for a single enqueue, merged over the class options.
|
|
138
|
+
#
|
|
139
|
+
# ReportJob.set(stream: :critical, retry: 0).perform_async(42)
|
|
140
|
+
#
|
|
141
|
+
# @param options [Hash] +stream+, +retry+, +dead+ or +subject+, see {#options}
|
|
142
|
+
# @return [Setter]
|
|
143
|
+
# @raise [ArgumentError] for any other option
|
|
144
|
+
def set(**options)
|
|
145
|
+
Setter.new(self, options)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
149
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
111
150
|
def perform_async(*args)
|
|
112
151
|
perform(*args)
|
|
113
152
|
end
|
|
114
153
|
|
|
154
|
+
# Enqueues one job per argument list, publishing a thousand at a time without waiting on each ack.
|
|
155
|
+
#
|
|
156
|
+
# ReportJob.perform_bulk([[1], [2], [3]])
|
|
157
|
+
#
|
|
158
|
+
# @param list [Array<Array>] each job's arguments
|
|
159
|
+
# @param options [Hash] job options for every job, merged over {#default_options}, e.g. +in:+
|
|
160
|
+
# @return [Array<String, nil>] each job's jid, nil for one a client middleware stopped
|
|
161
|
+
# @raise [BulkEnqueueError] when the stream refused some jobs; its +jids+ tell which were enqueued
|
|
162
|
+
def perform_bulk(list, **options)
|
|
163
|
+
Enqueuer.enqueue_many(name, list, default_options.merge(options), batch: Batch.current)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# @param timestamp [Time, Integer] when the job runs
|
|
167
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
168
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
115
169
|
def perform_at(timestamp, *args)
|
|
116
170
|
perform(*args, at: timestamp)
|
|
117
171
|
end
|
|
118
172
|
|
|
173
|
+
# @param interval [Numeric] seconds from now until the job runs
|
|
174
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
175
|
+
# @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
|
|
119
176
|
def perform_in(interval, *args)
|
|
120
177
|
perform(*args, in: interval)
|
|
121
178
|
end
|
|
122
179
|
|
|
180
|
+
# Runs the job inline, without NATS.
|
|
181
|
+
#
|
|
182
|
+
# @param args [Array] arguments passed to the job's +perform+
|
|
183
|
+
# @return [nil]
|
|
123
184
|
def perform_sync(*args)
|
|
124
185
|
perform(*args, async: false)
|
|
125
186
|
end
|
|
126
187
|
|
|
188
|
+
# Options inherited from the superclass job, or the built-in defaults.
|
|
189
|
+
#
|
|
190
|
+
# @return [Hash{Symbol => Object}]
|
|
127
191
|
def default_options
|
|
128
192
|
@default_options ||= (superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS.merge(retry: Data.default_retry)).dup
|
|
129
193
|
end
|
|
@@ -137,10 +201,15 @@ module Cosmo
|
|
|
137
201
|
|
|
138
202
|
attr_accessor :jid, :batch_id, :enqueued_at, :attempt, :scheduled_by
|
|
139
203
|
|
|
204
|
+
# Runs the job with the arguments it was enqueued with.
|
|
205
|
+
#
|
|
206
|
+
# @return [Object]
|
|
207
|
+
# @raise [NotImplementedError] unless the job class overrides it
|
|
140
208
|
def perform(...)
|
|
141
209
|
raise NotImplementedError, "#{self.class}#perform must be implemented"
|
|
142
210
|
end
|
|
143
211
|
|
|
212
|
+
# @return [::Logger]
|
|
144
213
|
def logger
|
|
145
214
|
Logger.instance
|
|
146
215
|
end
|