cosmonats 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -13
- data/lib/cosmo/active_job/adapter.rb +2 -0
- data/lib/cosmo/active_job/executor.rb +7 -0
- data/lib/cosmo/active_job/options.rb +6 -0
- data/lib/cosmo/api/batch.rb +26 -0
- data/lib/cosmo/api/counter.rb +51 -3
- data/lib/cosmo/api/cron/entry.rb +24 -9
- data/lib/cosmo/api/cron.rb +21 -7
- data/lib/cosmo/api/job.rb +14 -0
- data/lib/cosmo/api/kv.rb +138 -9
- data/lib/cosmo/api/registry.rb +47 -6
- data/lib/cosmo/api/stats/jobs.rb +84 -0
- data/lib/cosmo/api/stats/metrics.rb +18 -7
- data/lib/cosmo/api/stats/processes.rb +27 -11
- data/lib/cosmo/api/stats/totals.rb +1 -0
- data/lib/cosmo/api/stats.rb +47 -11
- data/lib/cosmo/api/stream.rb +66 -22
- data/lib/cosmo/batch/callback.rb +5 -0
- data/lib/cosmo/batch/dispatcher.rb +4 -4
- data/lib/cosmo/batch.rb +34 -11
- data/lib/cosmo/cli.rb +6 -0
- data/lib/cosmo/client.rb +208 -11
- data/lib/cosmo/config/cosmo.yml +1 -0
- data/lib/cosmo/config/settings.rb +4 -0
- data/lib/cosmo/config.rb +58 -10
- data/lib/cosmo/engine.rb +28 -1
- data/lib/cosmo/heartbeat.rb +12 -3
- data/lib/cosmo/http_server/handler.rb +5 -0
- data/lib/cosmo/http_server/health.rb +2 -0
- data/lib/cosmo/http_server/ping.rb +2 -0
- data/lib/cosmo/http_server.rb +8 -0
- data/lib/cosmo/job/data.rb +11 -0
- data/lib/cosmo/job/enqueuer.rb +100 -1
- data/lib/cosmo/job/limit.rb +5 -1
- data/lib/cosmo/job/processor.rb +51 -15
- data/lib/cosmo/job/setter.rb +53 -0
- data/lib/cosmo/job.rb +71 -2
- data/lib/cosmo/logger.rb +37 -0
- data/lib/cosmo/middleware/chain.rb +12 -0
- data/lib/cosmo/middleware/limit.rb +12 -3
- data/lib/cosmo/middleware/metrics.rb +6 -4
- data/lib/cosmo/middleware/sentry.rb +6 -4
- data/lib/cosmo/middleware/totals.rb +5 -11
- data/lib/cosmo/middleware.rb +0 -1
- data/lib/cosmo/processor.rb +60 -3
- data/lib/cosmo/publisher.rb +8 -0
- data/lib/cosmo/services.rb +7 -2
- data/lib/cosmo/stream/message.rb +5 -0
- data/lib/cosmo/stream/processor.rb +7 -0
- data/lib/cosmo/stream/serializer.rb +4 -0
- data/lib/cosmo/stream.rb +35 -1
- data/lib/cosmo/utils/duration.rb +3 -0
- data/lib/cosmo/utils/hash.rb +12 -0
- data/lib/cosmo/utils/json.rb +8 -0
- data/lib/cosmo/utils/signal.rb +9 -0
- data/lib/cosmo/utils/stopwatch.rb +1 -0
- data/lib/cosmo/utils/string.rb +4 -0
- data/lib/cosmo/utils/thread_pool.rb +8 -0
- data/lib/cosmo/utils/ttl_cache.rb +10 -0
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +5 -3
- data/lib/cosmo/web/cache.rb +2 -0
- data/lib/cosmo/web/chart.rb +2 -2
- data/lib/cosmo/web/circuit.rb +1 -0
- data/lib/cosmo/web/context.rb +6 -0
- data/lib/cosmo/web/controllers/application.rb +26 -0
- data/lib/cosmo/web/controllers/batches.rb +7 -1
- data/lib/cosmo/web/controllers/crons.rb +16 -2
- data/lib/cosmo/web/controllers/jobs.rb +45 -16
- data/lib/cosmo/web/controllers/metrics.rb +5 -0
- data/lib/cosmo/web/controllers/overview.rb +36 -18
- data/lib/cosmo/web/controllers/processes.rb +8 -5
- data/lib/cosmo/web/controllers/streams.rb +12 -0
- data/lib/cosmo/web/helpers/application.rb +44 -0
- data/lib/cosmo/web/renderer.rb +17 -0
- data/lib/cosmo/web/throughput.rb +1 -1
- data/lib/cosmo/web/views/crons/_table.erb +29 -27
- data/lib/cosmo/web/views/jobs/_args.erb +2 -2
- data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
- data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
- data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
- data/lib/cosmo/web/views/layout.erb +1 -1
- data/lib/cosmo/web/views/overview/_badges.erb +1 -1
- data/lib/cosmo/web/views/overview/_running.erb +3 -3
- data/lib/cosmo/web/views/overview/index.erb +1 -1
- data/lib/cosmo/web/views/streams/_info.erb +1 -0
- data/lib/cosmo/web/views/streams/_stream.erb +1 -0
- data/lib/cosmo/web.rb +8 -1
- data/lib/cosmo.rb +33 -0
- data/sig/cosmo/active_job/adapter.rbs +3 -3
- data/sig/cosmo/active_job/options.rbs +4 -2
- data/sig/cosmo/api/batch.rbs +5 -3
- data/sig/cosmo/api/counter.rbs +16 -6
- data/sig/cosmo/api/cron/entry.rbs +4 -2
- data/sig/cosmo/api/cron.rbs +10 -4
- data/sig/cosmo/api/kv.rbs +21 -4
- data/sig/cosmo/api/registry.rbs +10 -0
- data/sig/cosmo/api/stats/jobs.rbs +34 -0
- data/sig/cosmo/api/stats/metrics.rbs +3 -0
- data/sig/cosmo/api/stats/processes.rbs +9 -1
- data/sig/cosmo/api/stats.rbs +8 -2
- data/sig/cosmo/api/stream.rbs +10 -5
- data/sig/cosmo/batch/callback.rbs +1 -1
- data/sig/cosmo/batch/dispatcher.rbs +1 -1
- data/sig/cosmo/batch.rbs +5 -5
- data/sig/cosmo/cli.rbs +2 -0
- data/sig/cosmo/client.rbs +23 -4
- data/sig/cosmo/config/settings.rbs +6 -0
- data/sig/cosmo/config.rbs +5 -5
- data/sig/cosmo/engine.rbs +5 -0
- data/sig/cosmo/heartbeat.rbs +2 -0
- data/sig/cosmo/job/data.rbs +3 -1
- data/sig/cosmo/job/enqueuer.rbs +12 -0
- data/sig/cosmo/job/processor.rbs +18 -3
- data/sig/cosmo/job/setter.rbs +20 -0
- data/sig/cosmo/job.rbs +10 -4
- data/sig/cosmo/logger.rbs +5 -3
- data/sig/cosmo/middleware.rbs +0 -8
- data/sig/cosmo/processor.rbs +14 -2
- data/sig/cosmo/publisher.rbs +1 -0
- data/sig/cosmo/stream/processor.rbs +4 -0
- data/sig/cosmo/stream.rbs +4 -3
- data/sig/cosmo/utils/hash.rbs +2 -1
- data/sig/cosmo/utils/signal.rbs +5 -5
- data/sig/cosmo/utils/string.rbs +2 -2
- data/sig/cosmo/utils/thread_pool.rbs +1 -1
- data/sig/cosmo/web/cache.rbs +35 -0
- data/sig/cosmo/web/chart.rbs +48 -0
- data/sig/cosmo/web/circuit.rbs +29 -0
- data/sig/cosmo/web/context.rbs +15 -0
- data/sig/cosmo/web/controllers/application.rbs +32 -0
- data/sig/cosmo/web/controllers/batches.rbs +11 -0
- data/sig/cosmo/web/controllers/crons.rbs +23 -0
- data/sig/cosmo/web/controllers/jobs.rbs +47 -0
- data/sig/cosmo/web/controllers/metrics.rbs +21 -0
- data/sig/cosmo/web/controllers/overview.rbs +31 -0
- data/sig/cosmo/web/controllers/processes.rbs +13 -0
- data/sig/cosmo/web/controllers/streams.rbs +27 -0
- data/sig/cosmo/web/helpers/application.rbs +61 -0
- data/sig/cosmo/web/renderer.rbs +33 -0
- data/sig/cosmo/web/throughput.rbs +45 -0
- data/sig/cosmo/web.rbs +25 -0
- data/sig/cosmo.rbs +19 -0
- metadata +23 -6
- data/lib/cosmo/api/stats/busy.rb +0 -68
- data/lib/cosmo/middleware/busy.rb +0 -15
- data/sig/cosmo/api/stats/busy.rbs +0 -33
data/lib/cosmo/logger.rb
CHANGED
|
@@ -10,6 +10,11 @@ module Cosmo
|
|
|
10
10
|
module Context
|
|
11
11
|
KEY = :cosmo_context
|
|
12
12
|
|
|
13
|
+
# Merges +options+ into the current thread's log context, restoring the previous one after the block when given.
|
|
14
|
+
#
|
|
15
|
+
# @param options [Hash]
|
|
16
|
+
# @yieldreturn [Object]
|
|
17
|
+
# @return [Object, nil] the block's result
|
|
13
18
|
def self.with(**options)
|
|
14
19
|
prev = current
|
|
15
20
|
Thread.current[KEY] = prev.merge(options)
|
|
@@ -18,11 +23,16 @@ module Cosmo
|
|
|
18
23
|
Thread.current[KEY] = prev if block_given?
|
|
19
24
|
end
|
|
20
25
|
|
|
26
|
+
# Removes +keys+ from the current thread's log context.
|
|
27
|
+
#
|
|
28
|
+
# @param keys [Array<Symbol>]
|
|
29
|
+
# @return [nil]
|
|
21
30
|
def self.without(*keys)
|
|
22
31
|
Thread.current[KEY] = current.except(*keys)
|
|
23
32
|
nil
|
|
24
33
|
end
|
|
25
34
|
|
|
35
|
+
# @return [Hash] the current thread's log context
|
|
26
36
|
def self.current
|
|
27
37
|
Thread.current[KEY] ||= {}
|
|
28
38
|
Thread.current[KEY]
|
|
@@ -30,16 +40,23 @@ module Cosmo
|
|
|
30
40
|
end
|
|
31
41
|
|
|
32
42
|
class BaseFormatter < ::Logger::Formatter
|
|
43
|
+
# @return [String] the current thread's id, in base 36
|
|
33
44
|
def tid
|
|
34
45
|
(Thread.current.object_id ^ pid).to_s(36)
|
|
35
46
|
end
|
|
36
47
|
|
|
48
|
+
# @return [Integer]
|
|
37
49
|
def pid
|
|
38
50
|
::Process.pid
|
|
39
51
|
end
|
|
40
52
|
end
|
|
41
53
|
|
|
42
54
|
class SimpleFormatter < BaseFormatter
|
|
55
|
+
# @param severity [String]
|
|
56
|
+
# @param time [Time]
|
|
57
|
+
# @param _ [String, nil] the progname, unused
|
|
58
|
+
# @param msg [String, Exception, Object]
|
|
59
|
+
# @return [String] the line, with the current {Context}
|
|
43
60
|
def call(severity, time, _, msg)
|
|
44
61
|
options = Context.current.compact.map { |k, v| "#{k}=#{v}" }.join(" ")
|
|
45
62
|
options &&= " #{options}" unless options.empty?
|
|
@@ -49,10 +66,17 @@ module Cosmo
|
|
|
49
66
|
|
|
50
67
|
# Adds TRACE (below DEBUG) on top of stdlib's fixed DEBUG..UNKNOWN severities.
|
|
51
68
|
class Instance < ::Logger
|
|
69
|
+
# Logs at TRACE.
|
|
70
|
+
#
|
|
71
|
+
# @param progname [String, nil]
|
|
72
|
+
# @yieldreturn [String] the message, built only when TRACE is enabled
|
|
73
|
+
# @return [true]
|
|
52
74
|
def trace(progname = nil, &)
|
|
53
75
|
add(TRACE, nil, progname, &)
|
|
54
76
|
end
|
|
55
77
|
|
|
78
|
+
# @param severity [Integer]
|
|
79
|
+
# @return [String]
|
|
56
80
|
def format_severity(severity)
|
|
57
81
|
severity == TRACE ? "TRACE" : super
|
|
58
82
|
end
|
|
@@ -65,18 +89,27 @@ module Cosmo
|
|
|
65
89
|
end
|
|
66
90
|
|
|
67
91
|
# Dropped when the logger has no TRACE level (a plain +::Logger+), rather than flooding its DEBUG.
|
|
92
|
+
#
|
|
93
|
+
# @return [void]
|
|
68
94
|
def self.trace(...)
|
|
69
95
|
instance.trace(...) if instance.respond_to?(:trace)
|
|
70
96
|
end
|
|
71
97
|
|
|
98
|
+
# Same as {Context.with}.
|
|
99
|
+
#
|
|
100
|
+
# @return [Object, nil]
|
|
72
101
|
def self.with(...)
|
|
73
102
|
Context.with(...)
|
|
74
103
|
end
|
|
75
104
|
|
|
105
|
+
# Same as {Context.without}.
|
|
106
|
+
#
|
|
107
|
+
# @return [nil]
|
|
76
108
|
def self.without(...)
|
|
77
109
|
Context.without(...)
|
|
78
110
|
end
|
|
79
111
|
|
|
112
|
+
# @return [::Logger] the logger Cosmo writes to, stdout by default
|
|
80
113
|
def self.instance
|
|
81
114
|
@instance ||= Instance.new($stdout).tap do |logger|
|
|
82
115
|
logger.formatter = SimpleFormatter.new
|
|
@@ -84,12 +117,15 @@ module Cosmo
|
|
|
84
117
|
end
|
|
85
118
|
end
|
|
86
119
|
|
|
120
|
+
# @param level [Symbol, String, Integer]
|
|
121
|
+
# @return [Integer] the severity, {TRACE} included
|
|
87
122
|
def self.coerce_level(level)
|
|
88
123
|
level.to_s.downcase == "trace" ? TRACE : ::Logger::Severity.coerce(level)
|
|
89
124
|
end
|
|
90
125
|
|
|
91
126
|
# @param logger [::Logger] any stdlib-compatible logger. Its formatter is left alone, so context such as
|
|
92
127
|
# +jid+ and +elapsed+ is only printed when it uses {SimpleFormatter}.
|
|
128
|
+
# @return [void]
|
|
93
129
|
def self.instance=(logger)
|
|
94
130
|
@instance = logger
|
|
95
131
|
end
|
|
@@ -97,6 +133,7 @@ module Cosmo
|
|
|
97
133
|
# +COSMO_LOG_LEVEL+, when set, wins over +level+.
|
|
98
134
|
#
|
|
99
135
|
# @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
|
|
136
|
+
# @return [void]
|
|
100
137
|
def self.level=(level)
|
|
101
138
|
instance.level = coerce_level(ENV.fetch("COSMO_LOG_LEVEL", level))
|
|
102
139
|
end
|
|
@@ -23,6 +23,7 @@ module Cosmo
|
|
|
23
23
|
include Enumerable
|
|
24
24
|
|
|
25
25
|
Entry = Struct.new(:klass, :args, :kwargs) do
|
|
26
|
+
# @return [Object] a new instance of the middleware class
|
|
26
27
|
def build
|
|
27
28
|
klass.new(*args, **kwargs)
|
|
28
29
|
end
|
|
@@ -35,6 +36,7 @@ module Cosmo
|
|
|
35
36
|
end
|
|
36
37
|
|
|
37
38
|
# @yieldparam entry [Entry]
|
|
39
|
+
# @return [Array<Entry>]
|
|
38
40
|
def each(&)
|
|
39
41
|
@entries.each(&)
|
|
40
42
|
end
|
|
@@ -42,6 +44,8 @@ module Cosmo
|
|
|
42
44
|
# Appends +klass+, so it runs innermost, closest to the job.
|
|
43
45
|
#
|
|
44
46
|
# @param klass [Class]
|
|
47
|
+
# @param args [Array] passed to the middleware's +new+
|
|
48
|
+
# @param kwargs [Hash] passed to the middleware's +new+
|
|
45
49
|
# @return [Chain]
|
|
46
50
|
def add(klass, *args, **kwargs)
|
|
47
51
|
remove(klass)
|
|
@@ -52,6 +56,8 @@ module Cosmo
|
|
|
52
56
|
# Inserts +klass+ first, so it runs outermost.
|
|
53
57
|
#
|
|
54
58
|
# @param klass [Class]
|
|
59
|
+
# @param args [Array] passed to the middleware's +new+
|
|
60
|
+
# @param kwargs [Hash] passed to the middleware's +new+
|
|
55
61
|
# @return [Chain]
|
|
56
62
|
def prepend(klass, *args, **kwargs)
|
|
57
63
|
remove(klass)
|
|
@@ -63,6 +69,8 @@ module Cosmo
|
|
|
63
69
|
#
|
|
64
70
|
# @param existing [Class]
|
|
65
71
|
# @param klass [Class]
|
|
72
|
+
# @param args [Array] passed to the middleware's +new+
|
|
73
|
+
# @param kwargs [Hash] passed to the middleware's +new+
|
|
66
74
|
# @return [Chain]
|
|
67
75
|
def insert_before(existing, klass, *args, **kwargs)
|
|
68
76
|
entry = take(klass, args, kwargs)
|
|
@@ -74,6 +82,8 @@ module Cosmo
|
|
|
74
82
|
#
|
|
75
83
|
# @param existing [Class]
|
|
76
84
|
# @param klass [Class]
|
|
85
|
+
# @param args [Array] passed to the middleware's +new+
|
|
86
|
+
# @param kwargs [Hash] passed to the middleware's +new+
|
|
77
87
|
# @return [Chain]
|
|
78
88
|
def insert_after(existing, klass, *args, **kwargs)
|
|
79
89
|
entry = take(klass, args, kwargs)
|
|
@@ -109,6 +119,8 @@ module Cosmo
|
|
|
109
119
|
|
|
110
120
|
# Runs the block wrapped in every middleware, passing +args+ to each one's +call+.
|
|
111
121
|
#
|
|
122
|
+
# @param args [Array] passed to every middleware's +call+
|
|
123
|
+
# @yieldreturn [Object]
|
|
112
124
|
# @return [Object] the outermost middleware's return value, normally the block's
|
|
113
125
|
def invoke(*args, &)
|
|
114
126
|
return yield if empty?
|
|
@@ -8,12 +8,19 @@ module Cosmo
|
|
|
8
8
|
# taken, and the +duration+ timeout. Jobs without +limit:+ pass straight through. It runs first in the chain,
|
|
9
9
|
# so a requeued job never reaches the stats or user middleware.
|
|
10
10
|
class Limit
|
|
11
|
-
#
|
|
11
|
+
# Raised into the job thread when its duration runs out. It is not a StandardError, so job code rescuing
|
|
12
|
+
# Timeout::Error or StandardError can't swallow it, and it is told apart from a Timeout::Error raised by the
|
|
13
|
+
# job's own code.
|
|
14
|
+
class Expired < Exception; end # rubocop:disable Lint/InheritException
|
|
15
|
+
|
|
16
|
+
# @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
12
17
|
# @param data [Hash]
|
|
13
18
|
# @param _message [NATS::Msg]
|
|
19
|
+
# @yieldreturn [Object] the job's result
|
|
20
|
+
# @return [Object] the block's result
|
|
14
21
|
# @raise [Job::Requeue] while every concurrency slot is taken
|
|
15
22
|
def call(job, data, _message, &)
|
|
16
|
-
limit = job.class.default_options[:limit]
|
|
23
|
+
limit = job.class.default_options[:limit] if job
|
|
17
24
|
return yield unless limit
|
|
18
25
|
|
|
19
26
|
slot = acquire(job.class, data)
|
|
@@ -43,7 +50,9 @@ module Cosmo
|
|
|
43
50
|
return yield unless duration
|
|
44
51
|
|
|
45
52
|
seconds = duration.to_i
|
|
46
|
-
Timeout.timeout(seconds,
|
|
53
|
+
Timeout.timeout(seconds, Expired, &)
|
|
54
|
+
rescue Expired
|
|
55
|
+
raise JobTimeoutError, seconds
|
|
47
56
|
end
|
|
48
57
|
end
|
|
49
58
|
end
|
|
@@ -6,14 +6,16 @@ module Cosmo
|
|
|
6
6
|
# succeeds, a failure otherwise, and the wait from enqueue to the job's first delivery. ActiveJob jobs count under
|
|
7
7
|
# their own class.
|
|
8
8
|
class Metrics
|
|
9
|
-
# @param job [Cosmo::Job]
|
|
9
|
+
# @param job [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
10
10
|
# @param data [Hash]
|
|
11
11
|
# @param _message [NATS::Msg]
|
|
12
|
-
|
|
12
|
+
# @yieldreturn [Object] the job's result
|
|
13
|
+
# @return [Object] the block's result
|
|
14
|
+
def call(job, data, _message)
|
|
13
15
|
return yield unless Config.metrics.enabled
|
|
14
16
|
|
|
15
17
|
name = job_name(job, data)
|
|
16
|
-
wait_ms = (Time.now - job.enqueued_at) * 1000 if job
|
|
18
|
+
wait_ms = (Time.now - job.enqueued_at) * 1000 if job&.attempt == 1 && job.enqueued_at
|
|
17
19
|
started = clock
|
|
18
20
|
result = yield
|
|
19
21
|
API::Stats::Metrics.instance.record(name, exec_ms: clock - started, wait_ms:)
|
|
@@ -26,7 +28,7 @@ module Cosmo
|
|
|
26
28
|
private
|
|
27
29
|
|
|
28
30
|
def job_name(job, data)
|
|
29
|
-
return
|
|
31
|
+
return data[:class].to_s unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
|
|
30
32
|
|
|
31
33
|
data.dig(:args, 0, :job_class).to_s
|
|
32
34
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# rubocop:disable-next Metrics/
|
|
3
|
+
# rubocop:disable-next Metrics/AbcSize
|
|
4
4
|
module Cosmo
|
|
5
5
|
module Middleware
|
|
6
6
|
# Wraps every job execution in a Sentry transaction. Errors reach Sentry through {ERROR_HANDLER}, which also
|
|
@@ -26,14 +26,16 @@ module Cosmo
|
|
|
26
26
|
::Sentry.capture_exception(error, contexts: { cosmonats: context }, hint: { background: true, integration: "cosmonats" })
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
-
# @param
|
|
29
|
+
# @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
30
30
|
# @param data [Hash]
|
|
31
31
|
# @param message [NATS::Msg]
|
|
32
|
-
|
|
32
|
+
# @yieldreturn [Object] the job's result
|
|
33
|
+
# @return [Object] the block's result
|
|
34
|
+
def call(_job, data, message)
|
|
33
35
|
return yield unless ::Sentry.initialized?
|
|
34
36
|
|
|
35
37
|
scope = ::Sentry.get_current_scope
|
|
36
|
-
transaction_name = "#{NAME_PREFIX}/#{
|
|
38
|
+
transaction_name = "#{NAME_PREFIX}/#{data[:class]}"
|
|
37
39
|
scope.set_transaction_name(transaction_name, source: :task)
|
|
38
40
|
transaction = ::Sentry.start_transaction(
|
|
39
41
|
name: scope.transaction_name,
|
|
@@ -5,25 +5,19 @@ module Cosmo
|
|
|
5
5
|
# Counts every execution as processed or failed, so a job retried twice and then succeeding adds two failures
|
|
6
6
|
# and one processed. A counter that cannot be written is logged rather than raised, so stats never fail a job.
|
|
7
7
|
class Totals
|
|
8
|
-
# @param _job [Cosmo::Job]
|
|
8
|
+
# @param _job [Cosmo::Job, nil] nil when this worker can't load the job's class
|
|
9
9
|
# @param _data [Hash]
|
|
10
10
|
# @param _message [NATS::Msg]
|
|
11
|
+
# @yieldreturn [Object] the job's result
|
|
12
|
+
# @return [Object] the block's result
|
|
11
13
|
def call(_job, _data, _message)
|
|
12
14
|
result = yield
|
|
13
|
-
increment(:processed)
|
|
15
|
+
API::Stats::Totals.instance.increment(:processed)
|
|
14
16
|
result
|
|
15
17
|
rescue Exception # rubocop:disable Lint/RescueException
|
|
16
|
-
increment(:failed)
|
|
18
|
+
API::Stats::Totals.instance.increment(:failed)
|
|
17
19
|
raise
|
|
18
20
|
end
|
|
19
|
-
|
|
20
|
-
private
|
|
21
|
-
|
|
22
|
-
def increment(key)
|
|
23
|
-
API::Stats::Totals.instance.increment(key)
|
|
24
|
-
rescue StandardError => e
|
|
25
|
-
Logger.debug "Totals #{key} counter error: #{e.class} #{e.message}"
|
|
26
|
-
end
|
|
27
21
|
end
|
|
28
22
|
end
|
|
29
23
|
end
|
data/lib/cosmo/middleware.rb
CHANGED
data/lib/cosmo/processor.rb
CHANGED
|
@@ -7,13 +7,16 @@ module Cosmo
|
|
|
7
7
|
STREAM_EMPTY_BACKOFF_MAX = "5s" # Max sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
|
|
8
8
|
QUIET_SLEEP = "5s" # How long to sleep between checks while in quiet mode (override via COSMO_QUIET_IDLE_SLEEP)
|
|
9
9
|
|
|
10
|
+
# Builds a processor and starts it, see {#run}.
|
|
11
|
+
#
|
|
12
|
+
# @return [Processor]
|
|
10
13
|
def self.run(...)
|
|
11
14
|
new(...).tap(&:run)
|
|
12
15
|
end
|
|
13
16
|
|
|
14
17
|
# Checks the command-line options this processor understands, before anything is booted or connected.
|
|
15
18
|
#
|
|
16
|
-
# @param
|
|
19
|
+
# @param _options [Hash]
|
|
17
20
|
# @return [void]
|
|
18
21
|
def self.validate_options!(_options); end
|
|
19
22
|
|
|
@@ -28,8 +31,13 @@ module Cosmo
|
|
|
28
31
|
@consumers = []
|
|
29
32
|
@cache = Utils::TTLCache.new
|
|
30
33
|
@busy = Concurrent::AtomicFixnum.new
|
|
34
|
+
@in_flight = Concurrent::Map.new
|
|
35
|
+
@started_at = Concurrent::Map.new
|
|
31
36
|
end
|
|
32
37
|
|
|
38
|
+
# Subscribes the consumers and, when there are any, starts fetching in the background.
|
|
39
|
+
#
|
|
40
|
+
# @return [void]
|
|
33
41
|
def run
|
|
34
42
|
setup
|
|
35
43
|
return unless @consumers.any?
|
|
@@ -45,6 +53,37 @@ module Cosmo
|
|
|
45
53
|
@busy.value
|
|
46
54
|
end
|
|
47
55
|
|
|
56
|
+
# What the Web UI lists as running on this process, sent with each heartbeat.
|
|
57
|
+
#
|
|
58
|
+
# @return [Array<Hash>]
|
|
59
|
+
def jobs
|
|
60
|
+
[]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Hands every message still being processed back to NATS instead of leaving it for +ack_wait+: a job
|
|
64
|
+
# fails its attempt as usual, a stream message is NAK'd with a delay. Meant for a shutdown that timed
|
|
65
|
+
# out, right before the process exits; a job that finishes afterwards can't ack its released message.
|
|
66
|
+
#
|
|
67
|
+
# @return [Integer] number of messages released
|
|
68
|
+
def release
|
|
69
|
+
@in_flight.values.flatten.count { release_message(_1) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Kills the threads still processing messages and waits up to +timeout+ seconds for each to unwind,
|
|
73
|
+
# so their +ensure+ blocks run while the NATS connection is still up instead of during process exit.
|
|
74
|
+
#
|
|
75
|
+
# @param timeout [Numeric]
|
|
76
|
+
# @return [void]
|
|
77
|
+
def interrupt(timeout)
|
|
78
|
+
threads = @in_flight.keys
|
|
79
|
+
threads.each(&:kill)
|
|
80
|
+
threads.each { _1.join(timeout) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Stops fetching, unsubscribes, and waits up to +timeout+ seconds for in-flight work.
|
|
84
|
+
#
|
|
85
|
+
# @param timeout [Numeric]
|
|
86
|
+
# @return [void]
|
|
48
87
|
def stop(timeout = Config[:timeout])
|
|
49
88
|
@running.make_false
|
|
50
89
|
@pool.shutdown
|
|
@@ -107,7 +146,7 @@ module Cosmo
|
|
|
107
146
|
Logger.trace "fetched (#{messages&.size.to_i}) messages"
|
|
108
147
|
if messages&.any?
|
|
109
148
|
consumer_state.delete(key)
|
|
110
|
-
track_busy { process(messages, processor) }
|
|
149
|
+
track_busy(messages) { process(messages, processor) }
|
|
111
150
|
else
|
|
112
151
|
max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
|
|
113
152
|
consumer_state.compute(key) do |(count, wake_at)|
|
|
@@ -154,13 +193,31 @@ module Cosmo
|
|
|
154
193
|
raise NotImplementedError
|
|
155
194
|
end
|
|
156
195
|
|
|
157
|
-
def track_busy
|
|
196
|
+
def track_busy(messages)
|
|
158
197
|
@busy.increment
|
|
198
|
+
@started_at[Thread.current] = Time.now.to_i
|
|
199
|
+
@in_flight[Thread.current] = messages
|
|
159
200
|
yield
|
|
160
201
|
ensure
|
|
202
|
+
@in_flight.delete(Thread.current)
|
|
203
|
+
@started_at.delete(Thread.current)
|
|
161
204
|
@busy.decrement
|
|
162
205
|
end
|
|
163
206
|
|
|
207
|
+
def release_message(message)
|
|
208
|
+
abandon(message)
|
|
209
|
+
true
|
|
210
|
+
rescue NATS::JetStream::Error::MsgAlreadyAckd
|
|
211
|
+
false
|
|
212
|
+
rescue StandardError => e
|
|
213
|
+
Logger.error "Failed to release message #{message.subject}: #{e.class}: #{e.message}"
|
|
214
|
+
false
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def abandon(...)
|
|
218
|
+
raise NotImplementedError
|
|
219
|
+
end
|
|
220
|
+
|
|
164
221
|
def running?
|
|
165
222
|
@running.true?
|
|
166
223
|
end
|
data/lib/cosmo/publisher.rb
CHANGED
|
@@ -10,6 +10,7 @@ module Cosmo
|
|
|
10
10
|
delegate %i[publish] => :instance
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
+
# @return [Publisher]
|
|
13
14
|
def self.instance
|
|
14
15
|
@instance ||= new
|
|
15
16
|
end
|
|
@@ -18,6 +19,13 @@ module Cosmo
|
|
|
18
19
|
@client = Client.instance
|
|
19
20
|
end
|
|
20
21
|
|
|
22
|
+
# Serializes +data+ and publishes it to +subject+.
|
|
23
|
+
#
|
|
24
|
+
# @param subject [String]
|
|
25
|
+
# @param data [Object]
|
|
26
|
+
# @param serializer [#serialize, nil] defaults to {Stream::Serializer}
|
|
27
|
+
# @param options [Hash] JetStream publish options, e.g. +header:+
|
|
28
|
+
# @return [NATS::JetStream::PubAck]
|
|
21
29
|
def publish(subject, data, serializer: nil, **options)
|
|
22
30
|
payload = (serializer || Stream::Serializer).serialize(data)
|
|
23
31
|
@client.publish(subject, payload, **options)
|
data/lib/cosmo/services.rb
CHANGED
|
@@ -20,7 +20,7 @@ module Cosmo
|
|
|
20
20
|
|
|
21
21
|
{ storage: "file", retention: "workqueue", discard: "old", allow_direct: true, allow_msg_schedules: true,
|
|
22
22
|
duplicate_window: Config.to_ns(120), subjects: ["jobs.#{SCHEDULED}.>", "#{API::Cron::Entry::SUBJECT_PREFIX}.>"],
|
|
23
|
-
num_replicas: Config.
|
|
23
|
+
num_replicas: Config.services.num_replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
# +max_deliver+ stays above 1: the scheduler naks a message that is not yet due, and one whose dispatch failed,
|
|
@@ -40,7 +40,7 @@ module Cosmo
|
|
|
40
40
|
|
|
41
41
|
{ storage: "file", retention: "workqueue", discard: "old", allow_direct: true, duplicate_window: Config.to_ns(120),
|
|
42
42
|
max_age: Config.to_ns(Utils::Duration.parse(dead.max_age)), max_msgs: dead.max_msgs, max_bytes: dead.max_bytes,
|
|
43
|
-
subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.
|
|
43
|
+
subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.services.num_replicas, description: "Dead jobs", metadata: METADATA }
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
# Creates or updates every enabled service stream, plus the internal totals and batch counters and, while enabled,
|
|
@@ -56,6 +56,11 @@ module Cosmo
|
|
|
56
56
|
streams.keys
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
# Creates or updates the +name+ stream.
|
|
60
|
+
#
|
|
61
|
+
# @param name [String]
|
|
62
|
+
# @param config [Hash]
|
|
63
|
+
# @return [void]
|
|
59
64
|
# @raise [Error] when an existing stream has another retention policy, which NATS can't change in place
|
|
60
65
|
def setup_stream(name, config)
|
|
61
66
|
Client.instance.setup_stream(name, config)
|
data/lib/cosmo/stream/message.rb
CHANGED
|
@@ -11,19 +11,24 @@ module Cosmo
|
|
|
11
11
|
delegate %i[timestamp num_delivered num_pending] => :metadata
|
|
12
12
|
alias nack nak
|
|
13
13
|
|
|
14
|
+
# @param msg [NATS::Msg]
|
|
15
|
+
# @param serializer [#deserialize, nil] defaults to {Serializer}
|
|
14
16
|
def initialize(msg, serializer: nil)
|
|
15
17
|
@msg = msg
|
|
16
18
|
@serializer = serializer || Serializer
|
|
17
19
|
end
|
|
18
20
|
|
|
21
|
+
# @return [Object] the deserialized payload
|
|
19
22
|
def data
|
|
20
23
|
@serializer.deserialize(@msg.data)
|
|
21
24
|
end
|
|
22
25
|
|
|
26
|
+
# @return [Integer]
|
|
23
27
|
def stream_sequence
|
|
24
28
|
metadata.sequence.stream
|
|
25
29
|
end
|
|
26
30
|
|
|
31
|
+
# @return [Integer]
|
|
27
32
|
def consumer_sequence
|
|
28
33
|
metadata.sequence.consumer
|
|
29
34
|
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
module Cosmo
|
|
4
4
|
module Stream
|
|
5
5
|
class Processor < ::Cosmo::Processor
|
|
6
|
+
RELEASE_DELAY = "10s" # How long NATS holds a message released by a timed out shutdown before redelivering it (override via COSMO_STREAMS_RELEASE_DELAY)
|
|
7
|
+
|
|
6
8
|
# @return [Array<String>] names of the stream classes being processed
|
|
7
9
|
def subscriptions
|
|
8
10
|
@consumers.map { |(_, config, _)| config[:class].name }
|
|
@@ -10,6 +12,11 @@ module Cosmo
|
|
|
10
12
|
|
|
11
13
|
private
|
|
12
14
|
|
|
15
|
+
def abandon(message)
|
|
16
|
+
delay = Utils::Duration.parse(ENV.fetch("COSMO_STREAMS_RELEASE_DELAY", RELEASE_DELAY))
|
|
17
|
+
message.nak(delay: Config.to_ns(delay))
|
|
18
|
+
end
|
|
19
|
+
|
|
13
20
|
def setup
|
|
14
21
|
@configs ||= []
|
|
15
22
|
@configs = static_config + dynamic_config
|
|
@@ -7,10 +7,14 @@ module Cosmo
|
|
|
7
7
|
module Serializer
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
|
+
# @param data [Object]
|
|
11
|
+
# @return [String, nil] JSON, or nil when +data+ cannot be dumped
|
|
10
12
|
def serialize(data)
|
|
11
13
|
Utils::Json.dump(data)
|
|
12
14
|
end
|
|
13
15
|
|
|
16
|
+
# @param payload [String]
|
|
17
|
+
# @return [Object, nil] the parsed JSON with string keys, or nil when it is malformed
|
|
14
18
|
def deserialize(payload)
|
|
15
19
|
Utils::Json.parse(payload, symbolize_names: false)
|
|
16
20
|
end
|
data/lib/cosmo/stream.rb
CHANGED
|
@@ -7,28 +7,50 @@ require "cosmo/stream/serializer"
|
|
|
7
7
|
|
|
8
8
|
module Cosmo
|
|
9
9
|
module Stream
|
|
10
|
+
# @param base [Class] the stream class including Stream
|
|
11
|
+
# @return [void]
|
|
10
12
|
def self.included(base)
|
|
11
13
|
base.extend(ClassMethods)
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
module ClassMethods
|
|
17
|
+
# Registers the class as a stream processor and merges the given options over its defaults.
|
|
18
|
+
#
|
|
19
|
+
# @param stream [Symbol, String, nil] NATS stream to consume
|
|
20
|
+
# @param consumer_name [String, nil] durable consumer name
|
|
21
|
+
# @param batch_size [Integer, nil] messages fetched per batch
|
|
22
|
+
# @param fetch_timeout [Numeric, nil] seconds to wait for a batch
|
|
23
|
+
# @param start_position [Symbol, String, Time, nil] where a new consumer starts delivering from
|
|
24
|
+
# @param consumer [Hash, nil] consumer config, e.g. +subjects:+
|
|
25
|
+
# @param publisher [Hash, nil] publisher config: +subject:+ and +serializer:+
|
|
26
|
+
# @return [Hash{Symbol => Object}] the class's merged options
|
|
15
27
|
def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil)
|
|
16
28
|
register
|
|
17
29
|
default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
|
|
18
30
|
end
|
|
19
31
|
alias cosmo_options options
|
|
20
32
|
|
|
33
|
+
# @param data [Object] serialized with the publisher's serializer
|
|
34
|
+
# @param subject [String, nil] defaults to the publisher's subject
|
|
35
|
+
# @param options [Hash] passed on to the JetStream publish
|
|
36
|
+
# @return [NATS::JetStream::PubAck]
|
|
21
37
|
def publish(data, subject: nil, **options)
|
|
22
38
|
stream = default_options[:stream]
|
|
23
39
|
subject ||= default_options.dig(:publisher, :subject)
|
|
24
40
|
Publisher.publish(subject, data, stream: stream, serializer: default_options.dig(:publisher, :serializer), **options)
|
|
25
41
|
end
|
|
26
42
|
|
|
43
|
+
# Options inherited from the superclass stream, or the built-in defaults.
|
|
44
|
+
#
|
|
45
|
+
# @return [Hash{Symbol => Object}]
|
|
27
46
|
def default_options
|
|
28
47
|
@default_options ||= Utils::Hash.dup(superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS)
|
|
29
48
|
end
|
|
30
49
|
|
|
31
|
-
|
|
50
|
+
# Adds the class to the processed streams and derives its stream, consumer and subjects from its name.
|
|
51
|
+
#
|
|
52
|
+
# @return [void]
|
|
53
|
+
def register
|
|
32
54
|
Config.internal[:streams] ||= []
|
|
33
55
|
Config.internal[:streams] << self
|
|
34
56
|
|
|
@@ -47,6 +69,10 @@ module Cosmo
|
|
|
47
69
|
end
|
|
48
70
|
end
|
|
49
71
|
|
|
72
|
+
# Calls {#process_one} for each message, exposing it through {#message}.
|
|
73
|
+
#
|
|
74
|
+
# @param messages [Array<Message>]
|
|
75
|
+
# @return [void]
|
|
50
76
|
def process(messages)
|
|
51
77
|
messages.each do |message|
|
|
52
78
|
Thread.current[:cosmo_message] = message
|
|
@@ -58,18 +84,26 @@ module Cosmo
|
|
|
58
84
|
alias process_many process
|
|
59
85
|
alias process_batch process
|
|
60
86
|
|
|
87
|
+
# @return [void]
|
|
88
|
+
# @raise [NotImplementedError] unless the stream class overrides it or {#process}
|
|
61
89
|
def process_one
|
|
62
90
|
raise NotImplementedError, "#{self.class}#process_one must be implemented"
|
|
63
91
|
end
|
|
64
92
|
|
|
93
|
+
# @return [::Logger]
|
|
65
94
|
def logger
|
|
66
95
|
Logger.instance
|
|
67
96
|
end
|
|
68
97
|
|
|
98
|
+
# @return [Message, nil] the message being processed by {#process_one}
|
|
69
99
|
def message
|
|
70
100
|
Thread.current[:cosmo_message]
|
|
71
101
|
end
|
|
72
102
|
|
|
103
|
+
# @param data [Object] serialized with the publisher's serializer
|
|
104
|
+
# @param subject [String, nil] defaults to the publisher's subject
|
|
105
|
+
# @param options [Hash] passed on to the JetStream publish
|
|
106
|
+
# @return [NATS::JetStream::PubAck]
|
|
73
107
|
def publish(data, subject, **options)
|
|
74
108
|
self.class.publish(data, subject:, **options)
|
|
75
109
|
end
|
data/lib/cosmo/utils/duration.rb
CHANGED
|
@@ -18,6 +18,9 @@ module Cosmo
|
|
|
18
18
|
|
|
19
19
|
module_function
|
|
20
20
|
|
|
21
|
+
# @param value [Numeric, String] seconds, or a duration such as "20s", "1h", "7d"
|
|
22
|
+
# @return [Numeric] seconds
|
|
23
|
+
# @raise [ArgumentError] when +value+ is not a valid duration
|
|
21
24
|
def parse(value)
|
|
22
25
|
return value if value.is_a?(Numeric)
|
|
23
26
|
|
data/lib/cosmo/utils/hash.rb
CHANGED
|
@@ -5,6 +5,11 @@ module Cosmo
|
|
|
5
5
|
module Hash
|
|
6
6
|
module_function
|
|
7
7
|
|
|
8
|
+
# Symbolizes hash keys in place, recursing into nested hashes and arrays.
|
|
9
|
+
#
|
|
10
|
+
# @param obj [Object]
|
|
11
|
+
# @return [Object] +obj+, mutated
|
|
12
|
+
# @raise [ArgumentError] when a key cannot be converted to a symbol
|
|
8
13
|
def symbolize_keys!(obj)
|
|
9
14
|
case obj
|
|
10
15
|
when ::Hash
|
|
@@ -23,6 +28,10 @@ module Cosmo
|
|
|
23
28
|
end
|
|
24
29
|
end
|
|
25
30
|
|
|
31
|
+
# Stringifies hash keys, recursing into nested hashes and arrays.
|
|
32
|
+
#
|
|
33
|
+
# @param obj [Object]
|
|
34
|
+
# @return [Object] a copy of +obj+
|
|
26
35
|
def stringify_keys(obj)
|
|
27
36
|
case obj
|
|
28
37
|
when ::Hash
|
|
@@ -46,6 +55,9 @@ module Cosmo
|
|
|
46
55
|
end
|
|
47
56
|
|
|
48
57
|
# deep dup
|
|
58
|
+
#
|
|
59
|
+
# @param hash [Object] a Marshal-able value
|
|
60
|
+
# @return [Object] a deep copy of +hash+
|
|
49
61
|
def dup(hash)
|
|
50
62
|
Marshal.load(Marshal.dump(hash))
|
|
51
63
|
end
|