cosmonats 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -13
- data/lib/cosmo/active_job/adapter.rb +2 -0
- data/lib/cosmo/active_job/executor.rb +7 -0
- data/lib/cosmo/active_job/options.rb +6 -0
- data/lib/cosmo/api/batch.rb +26 -0
- data/lib/cosmo/api/counter.rb +51 -3
- data/lib/cosmo/api/cron/entry.rb +24 -9
- data/lib/cosmo/api/cron.rb +21 -7
- data/lib/cosmo/api/job.rb +14 -0
- data/lib/cosmo/api/kv.rb +138 -9
- data/lib/cosmo/api/registry.rb +47 -6
- data/lib/cosmo/api/stats/jobs.rb +84 -0
- data/lib/cosmo/api/stats/metrics.rb +18 -7
- data/lib/cosmo/api/stats/processes.rb +27 -11
- data/lib/cosmo/api/stats/totals.rb +1 -0
- data/lib/cosmo/api/stats.rb +47 -11
- data/lib/cosmo/api/stream.rb +66 -22
- data/lib/cosmo/batch/callback.rb +5 -0
- data/lib/cosmo/batch/dispatcher.rb +4 -4
- data/lib/cosmo/batch.rb +34 -11
- data/lib/cosmo/cli.rb +6 -0
- data/lib/cosmo/client.rb +208 -11
- data/lib/cosmo/config/cosmo.yml +1 -0
- data/lib/cosmo/config/settings.rb +4 -0
- data/lib/cosmo/config.rb +58 -10
- data/lib/cosmo/engine.rb +28 -1
- data/lib/cosmo/heartbeat.rb +12 -3
- data/lib/cosmo/http_server/handler.rb +5 -0
- data/lib/cosmo/http_server/health.rb +2 -0
- data/lib/cosmo/http_server/ping.rb +2 -0
- data/lib/cosmo/http_server.rb +8 -0
- data/lib/cosmo/job/data.rb +11 -0
- data/lib/cosmo/job/enqueuer.rb +100 -1
- data/lib/cosmo/job/limit.rb +5 -1
- data/lib/cosmo/job/processor.rb +51 -15
- data/lib/cosmo/job/setter.rb +53 -0
- data/lib/cosmo/job.rb +71 -2
- data/lib/cosmo/logger.rb +37 -0
- data/lib/cosmo/middleware/chain.rb +12 -0
- data/lib/cosmo/middleware/limit.rb +12 -3
- data/lib/cosmo/middleware/metrics.rb +6 -4
- data/lib/cosmo/middleware/sentry.rb +6 -4
- data/lib/cosmo/middleware/totals.rb +5 -11
- data/lib/cosmo/middleware.rb +0 -1
- data/lib/cosmo/processor.rb +60 -3
- data/lib/cosmo/publisher.rb +8 -0
- data/lib/cosmo/services.rb +7 -2
- data/lib/cosmo/stream/message.rb +5 -0
- data/lib/cosmo/stream/processor.rb +7 -0
- data/lib/cosmo/stream/serializer.rb +4 -0
- data/lib/cosmo/stream.rb +35 -1
- data/lib/cosmo/utils/duration.rb +3 -0
- data/lib/cosmo/utils/hash.rb +12 -0
- data/lib/cosmo/utils/json.rb +8 -0
- data/lib/cosmo/utils/signal.rb +9 -0
- data/lib/cosmo/utils/stopwatch.rb +1 -0
- data/lib/cosmo/utils/string.rb +4 -0
- data/lib/cosmo/utils/thread_pool.rb +8 -0
- data/lib/cosmo/utils/ttl_cache.rb +10 -0
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +5 -3
- data/lib/cosmo/web/cache.rb +2 -0
- data/lib/cosmo/web/chart.rb +2 -2
- data/lib/cosmo/web/circuit.rb +1 -0
- data/lib/cosmo/web/context.rb +6 -0
- data/lib/cosmo/web/controllers/application.rb +26 -0
- data/lib/cosmo/web/controllers/batches.rb +7 -1
- data/lib/cosmo/web/controllers/crons.rb +16 -2
- data/lib/cosmo/web/controllers/jobs.rb +45 -16
- data/lib/cosmo/web/controllers/metrics.rb +5 -0
- data/lib/cosmo/web/controllers/overview.rb +36 -18
- data/lib/cosmo/web/controllers/processes.rb +8 -5
- data/lib/cosmo/web/controllers/streams.rb +12 -0
- data/lib/cosmo/web/helpers/application.rb +44 -0
- data/lib/cosmo/web/renderer.rb +17 -0
- data/lib/cosmo/web/throughput.rb +1 -1
- data/lib/cosmo/web/views/crons/_table.erb +29 -27
- data/lib/cosmo/web/views/jobs/_args.erb +2 -2
- data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
- data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
- data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
- data/lib/cosmo/web/views/layout.erb +1 -1
- data/lib/cosmo/web/views/overview/_badges.erb +1 -1
- data/lib/cosmo/web/views/overview/_running.erb +3 -3
- data/lib/cosmo/web/views/overview/index.erb +1 -1
- data/lib/cosmo/web/views/streams/_info.erb +1 -0
- data/lib/cosmo/web/views/streams/_stream.erb +1 -0
- data/lib/cosmo/web.rb +8 -1
- data/lib/cosmo.rb +33 -0
- data/sig/cosmo/active_job/adapter.rbs +3 -3
- data/sig/cosmo/active_job/options.rbs +4 -2
- data/sig/cosmo/api/batch.rbs +5 -3
- data/sig/cosmo/api/counter.rbs +16 -6
- data/sig/cosmo/api/cron/entry.rbs +4 -2
- data/sig/cosmo/api/cron.rbs +10 -4
- data/sig/cosmo/api/kv.rbs +21 -4
- data/sig/cosmo/api/registry.rbs +10 -0
- data/sig/cosmo/api/stats/jobs.rbs +34 -0
- data/sig/cosmo/api/stats/metrics.rbs +3 -0
- data/sig/cosmo/api/stats/processes.rbs +9 -1
- data/sig/cosmo/api/stats.rbs +8 -2
- data/sig/cosmo/api/stream.rbs +10 -5
- data/sig/cosmo/batch/callback.rbs +1 -1
- data/sig/cosmo/batch/dispatcher.rbs +1 -1
- data/sig/cosmo/batch.rbs +5 -5
- data/sig/cosmo/cli.rbs +2 -0
- data/sig/cosmo/client.rbs +23 -4
- data/sig/cosmo/config/settings.rbs +6 -0
- data/sig/cosmo/config.rbs +5 -5
- data/sig/cosmo/engine.rbs +5 -0
- data/sig/cosmo/heartbeat.rbs +2 -0
- data/sig/cosmo/job/data.rbs +3 -1
- data/sig/cosmo/job/enqueuer.rbs +12 -0
- data/sig/cosmo/job/processor.rbs +18 -3
- data/sig/cosmo/job/setter.rbs +20 -0
- data/sig/cosmo/job.rbs +10 -4
- data/sig/cosmo/logger.rbs +5 -3
- data/sig/cosmo/middleware.rbs +0 -8
- data/sig/cosmo/processor.rbs +14 -2
- data/sig/cosmo/publisher.rbs +1 -0
- data/sig/cosmo/stream/processor.rbs +4 -0
- data/sig/cosmo/stream.rbs +4 -3
- data/sig/cosmo/utils/hash.rbs +2 -1
- data/sig/cosmo/utils/signal.rbs +5 -5
- data/sig/cosmo/utils/string.rbs +2 -2
- data/sig/cosmo/utils/thread_pool.rbs +1 -1
- data/sig/cosmo/web/cache.rbs +35 -0
- data/sig/cosmo/web/chart.rbs +48 -0
- data/sig/cosmo/web/circuit.rbs +29 -0
- data/sig/cosmo/web/context.rbs +15 -0
- data/sig/cosmo/web/controllers/application.rbs +32 -0
- data/sig/cosmo/web/controllers/batches.rbs +11 -0
- data/sig/cosmo/web/controllers/crons.rbs +23 -0
- data/sig/cosmo/web/controllers/jobs.rbs +47 -0
- data/sig/cosmo/web/controllers/metrics.rbs +21 -0
- data/sig/cosmo/web/controllers/overview.rbs +31 -0
- data/sig/cosmo/web/controllers/processes.rbs +13 -0
- data/sig/cosmo/web/controllers/streams.rbs +27 -0
- data/sig/cosmo/web/helpers/application.rbs +61 -0
- data/sig/cosmo/web/renderer.rbs +33 -0
- data/sig/cosmo/web/throughput.rbs +45 -0
- data/sig/cosmo/web.rbs +25 -0
- data/sig/cosmo.rbs +19 -0
- metadata +23 -6
- data/lib/cosmo/api/stats/busy.rb +0 -68
- data/lib/cosmo/middleware/busy.rb +0 -15
- data/sig/cosmo/api/stats/busy.rbs +0 -33
data/lib/cosmo/api/kv.rb
CHANGED
|
@@ -7,12 +7,21 @@ module Cosmo
|
|
|
7
7
|
|
|
8
8
|
attr_reader :kv
|
|
9
9
|
|
|
10
|
+
# Opens the +name+ bucket, creating it with +options+ when it doesn't exist.
|
|
11
|
+
#
|
|
12
|
+
# @param name [String]
|
|
13
|
+
# @param options [Hash, nil] bucket settings, see {Client#kv}
|
|
10
14
|
def initialize(name, options = nil)
|
|
11
15
|
@name = name
|
|
12
16
|
@options = Hash(options)
|
|
13
17
|
@kv = client.kv(@name, **@options)
|
|
14
18
|
end
|
|
15
19
|
|
|
20
|
+
# @param key [String, Integer]
|
|
21
|
+
# @param value [#to_s]
|
|
22
|
+
# @param ttl [Integer, nil] seconds before the key expires; with it the key is only created when absent
|
|
23
|
+
# @return [Integer] the stored message's sequence
|
|
24
|
+
# @raise [NATS::KeyValue::KeyWrongLastSequenceError] when +ttl+ is given and the key is live
|
|
16
25
|
def set(key, value, ttl: nil)
|
|
17
26
|
return kv.put(key, value.to_s) unless ttl
|
|
18
27
|
|
|
@@ -21,6 +30,8 @@ module Cosmo
|
|
|
21
30
|
publish_cas(key, value.to_s, ttl, last_seq: 0).seq
|
|
22
31
|
end
|
|
23
32
|
|
|
33
|
+
# @param key [String, Integer]
|
|
34
|
+
# @return [NATS::KeyValue::Entry, nil] nil when the key doesn't exist
|
|
24
35
|
def get(key)
|
|
25
36
|
kv.get(key)
|
|
26
37
|
rescue NATS::KeyValue::KeyNotFoundError
|
|
@@ -30,6 +41,10 @@ module Cosmo
|
|
|
30
41
|
# Creates +key+ if it does not already exist -- a plain CAS-if-absent with
|
|
31
42
|
# no per-message TTL involved. Raises NATS::KeyValue::KeyWrongLastSequenceError
|
|
32
43
|
# when the key is already live, so callers can use it as a fire-once gate.
|
|
44
|
+
#
|
|
45
|
+
# @param key [String, Integer]
|
|
46
|
+
# @param value [#to_s]
|
|
47
|
+
# @return [Integer] the key's revision
|
|
33
48
|
def create(key, value)
|
|
34
49
|
kv.create(key, value.to_s)
|
|
35
50
|
end
|
|
@@ -37,6 +52,9 @@ module Cosmo
|
|
|
37
52
|
# Writes a KV-Operation tombstone. On a ttl-bearing bucket this leaves the
|
|
38
53
|
# subject occupied, so a subsequent #set(ttl:) CAS with last_seq: 0 will
|
|
39
54
|
# keep failing -- use #erase on those buckets instead.
|
|
55
|
+
#
|
|
56
|
+
# @param key [String, Integer]
|
|
57
|
+
# @return [Integer] the tombstone's sequence
|
|
40
58
|
def delete(key)
|
|
41
59
|
kv.delete(key)
|
|
42
60
|
end
|
|
@@ -44,6 +62,11 @@ module Cosmo
|
|
|
44
62
|
# Walks the bucket and collects live keys. Pass +limit: nil+ to walk the whole
|
|
45
63
|
# bucket -- a bounded +limit+ stops the watch early, so it can never be used
|
|
46
64
|
# to count (see #count).
|
|
65
|
+
#
|
|
66
|
+
# @param subject [String, nil] a key filter, every key when nil
|
|
67
|
+
# @param limit [Integer, nil]
|
|
68
|
+
# @param offset [Integer] live keys to skip first
|
|
69
|
+
# @return [Array<String>]
|
|
47
70
|
def keys(subject = nil, limit: LIMIT, offset: 0)
|
|
48
71
|
walk(subject, limit:, offset:, meta_only: true).map(&:first)
|
|
49
72
|
end
|
|
@@ -51,12 +74,91 @@ module Cosmo
|
|
|
51
74
|
# Same walk as #keys, but carries each value back with its key. The watch
|
|
52
75
|
# already has the values in hand, so callers that need them get one pass
|
|
53
76
|
# instead of a #keys walk followed by a #get per key.
|
|
77
|
+
# @param subject [String, nil] a key filter, every key when nil
|
|
78
|
+
# @param limit [Integer, nil]
|
|
79
|
+
# @param offset [Integer] live entries to skip first
|
|
54
80
|
# @return [Array<Array(String, String)>] +[key, value]+ pairs
|
|
55
81
|
def entries(subject = nil, limit: LIMIT, offset: 0)
|
|
56
82
|
walk(subject, limit:, offset:, meta_only: false)
|
|
57
83
|
end
|
|
58
84
|
|
|
85
|
+
# Same pairs as #entries, read by batched direct gets instead of a watch: no consumer is created, any replica
|
|
86
|
+
# can answer, and the cost doesn't grow with the writes landing on the bucket while it's read. Keys come in
|
|
87
|
+
# the order they were last written. Turns on direct gets for the bucket first, see #direct!.
|
|
88
|
+
#
|
|
89
|
+
# @param subject [String, nil] a key filter, every key when nil
|
|
90
|
+
# @param limit [Integer, nil] the most messages to read, tombstones included; nil for every one
|
|
91
|
+
# @return [Array<Array(String, String)>] +[key, value]+ pairs
|
|
92
|
+
def direct_entries(subject = nil, limit: nil)
|
|
93
|
+
direct!
|
|
94
|
+
messages = client.direct_messages("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}", limit:)
|
|
95
|
+
messages.filter_map do |msg|
|
|
96
|
+
header = msg.header
|
|
97
|
+
[header[NATS::JetStream::Header::SUBJECT].delete_prefix("$KV.#{@name}."), msg.data] unless header["KV-Operation"]
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# How many keys match +subject+, counted by the server without reading any value: one request however many
|
|
102
|
+
# keys there are. Turns on direct gets for the bucket first, see #direct!.
|
|
103
|
+
#
|
|
104
|
+
# @param subject [String, nil] a key filter, every key when nil
|
|
105
|
+
# @return [Integer]
|
|
106
|
+
def direct_count(subject = nil)
|
|
107
|
+
direct!
|
|
108
|
+
client.count_messages("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}")
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# The sorted keys matching +subject+, without their values, for paging through a bucket in a stable order.
|
|
112
|
+
# Turns on direct gets for the bucket first, see #direct!.
|
|
113
|
+
#
|
|
114
|
+
# @param subject [String, nil] a key filter, every key when nil
|
|
115
|
+
# @return [Array<String>]
|
|
116
|
+
def direct_keys(subject = nil)
|
|
117
|
+
direct!
|
|
118
|
+
client.subjects("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}").map { _1.delete_prefix("$KV.#{@name}.") }
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The values of +keys+, in one batched direct get. NATS answers for at most 1024 keys at a time.
|
|
122
|
+
#
|
|
123
|
+
# @param keys [Array<String>]
|
|
124
|
+
# @return [Hash{String => String}] the value by key, without the keys that don't exist
|
|
125
|
+
def direct_values(keys)
|
|
126
|
+
direct!
|
|
127
|
+
messages = client.last_messages("KV_#{@name}", keys.map { "$KV.#{@name}.#{_1}" })
|
|
128
|
+
messages.each_with_object({}) do |(subject, msg), values|
|
|
129
|
+
values[subject.delete_prefix("$KV.#{@name}.")] = msg.data unless msg.header&.key?("KV-Operation")
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Writes every pair before waiting for any ack, so a batch costs about one round trip instead of one each.
|
|
134
|
+
#
|
|
135
|
+
# @param pairs [Hash{String => #to_s}] the value by key
|
|
136
|
+
# @param ttl [Integer, nil] seconds before each key expires, on a bucket created with +allow_msg_ttl+
|
|
137
|
+
# @return [void]
|
|
138
|
+
# @raise [StandardError] the first error among the acks
|
|
139
|
+
def write_many(pairs, ttl: nil)
|
|
140
|
+
header = ttl ? { "Nats-TTL" => [ttl.to_i, 1].max.to_s } : {}
|
|
141
|
+
acks = client.publish_many(pairs.map { |key, value| { subject: "$KV.#{@name}.#{key}", payload: value.to_s, header: } })
|
|
142
|
+
error = acks.find { _1.is_a?(StandardError) }
|
|
143
|
+
raise error if error
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Turns on +allow_direct+ for a bucket created without it, as nats-pure creates them. Checked once per instance.
|
|
147
|
+
#
|
|
148
|
+
# @return [true]
|
|
149
|
+
def direct!
|
|
150
|
+
@direct ||= begin
|
|
151
|
+
stream = "KV_#{@name}"
|
|
152
|
+
config = client.stream_info(stream).config
|
|
153
|
+
client.update_stream(stream, config.to_h.compact.except(:name).merge(allow_direct: true)) unless config.allow_direct
|
|
154
|
+
true
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
59
158
|
# Writes a KV-Operation tombstone (same issue as #delete on ttl buckets).
|
|
159
|
+
#
|
|
160
|
+
# @param key [String, Integer]
|
|
161
|
+
# @return [NATS::JetStream::PubAck]
|
|
60
162
|
def purge(key)
|
|
61
163
|
kv.purge(key)
|
|
62
164
|
end
|
|
@@ -65,14 +167,21 @@ module Cosmo
|
|
|
65
167
|
# write a KV-Operation tombstone message. Mirrors how per-message
|
|
66
168
|
# Nats-TTL expiry removes a key, so callers never have to distinguish
|
|
67
169
|
# "deleted" from "TTL-expired" on read.
|
|
170
|
+
#
|
|
171
|
+
# @param key [String, Integer]
|
|
172
|
+
# @return [Integer, nil] the number of messages purged
|
|
68
173
|
def erase(key)
|
|
69
174
|
client.purge("KV_#{@name}", "$KV.#{@name}.#{key}")
|
|
70
175
|
end
|
|
71
176
|
|
|
177
|
+
# Removes every key in the bucket.
|
|
178
|
+
#
|
|
179
|
+
# @return [Integer, nil] the number of messages purged
|
|
72
180
|
def clean
|
|
73
181
|
client.purge("KV_#{@name}", ">")
|
|
74
182
|
end
|
|
75
183
|
|
|
184
|
+
# @return [Integer] the number of live keys
|
|
76
185
|
def count
|
|
77
186
|
keys(limit: nil).size
|
|
78
187
|
rescue NATS::KeyValue::NoKeysFoundError, NATS::JetStream::Error::NotFound
|
|
@@ -85,19 +194,39 @@ module Cosmo
|
|
|
85
194
|
def walk(subject, limit:, offset:, meta_only:)
|
|
86
195
|
results = []
|
|
87
196
|
skipped = 0
|
|
88
|
-
|
|
89
|
-
watcher.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
197
|
+
seen = 0
|
|
198
|
+
watcher = kv.watch(subject || ">", meta_only:)
|
|
199
|
+
begin
|
|
200
|
+
watcher.each do |entry|
|
|
201
|
+
if entry.nil?
|
|
202
|
+
break if drained?(watcher, seen)
|
|
203
|
+
|
|
204
|
+
next
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
seen += 1
|
|
208
|
+
unless entry.operation
|
|
209
|
+
next skipped += 1 if skipped < offset.to_i
|
|
210
|
+
|
|
211
|
+
results << [entry.key, entry.value]
|
|
212
|
+
break if limit && results.size >= limit
|
|
213
|
+
end
|
|
214
|
+
break if entry.delta.zero?
|
|
215
|
+
end
|
|
216
|
+
ensure
|
|
217
|
+
watcher.stop
|
|
96
218
|
end
|
|
97
|
-
watcher.stop
|
|
98
219
|
results
|
|
99
220
|
end
|
|
100
221
|
|
|
222
|
+
# nats-pure ends the initial walk with a nil once consumer info reports nothing pending and
|
|
223
|
+
# nothing received yet, but the server can answer that info before the pushed entries arrive.
|
|
224
|
+
# The nil is only trustworthy when the consumer really delivered no more than was seen.
|
|
225
|
+
# https://github.com/nats-io/nats-pure.rb/issues/197
|
|
226
|
+
def drained?(watcher, seen)
|
|
227
|
+
watcher._sub.consumer_info.delivered.consumer_seq <= seen
|
|
228
|
+
end
|
|
229
|
+
|
|
101
230
|
# CAS = Compare-And-Swap: publish +value+ with a per-message TTL of +ttl+ whole seconds (at least 1),
|
|
102
231
|
# but only if the subject's current last sequence matches +last_seq+
|
|
103
232
|
# (sent as the Nats-Expected-Last-Subject-Sequence header). Raises
|
data/lib/cosmo/api/registry.rb
CHANGED
|
@@ -5,15 +5,29 @@ module Cosmo
|
|
|
5
5
|
# Base for a KV bucket of JSON entries that live only while their writer keeps refreshing them:
|
|
6
6
|
# a writer re-puts its entries faster than the bucket's TTL, and an entry whose writer is gone expires.
|
|
7
7
|
# Subclasses define +BUCKET+ and +TTL+.
|
|
8
|
+
#
|
|
9
|
+
# An entry's key is what the Web UI counts and pages by: the server counts keys and lists them sorted, so only
|
|
10
|
+
# the entries on the page being shown are ever read.
|
|
8
11
|
class Registry
|
|
9
12
|
LIMIT = 25
|
|
13
|
+
KEYS_TTL = 2
|
|
10
14
|
|
|
15
|
+
# @return [Registry]
|
|
11
16
|
def self.instance
|
|
12
17
|
@instance ||= new
|
|
13
18
|
end
|
|
14
19
|
|
|
20
|
+
# Makes +value+ one subject token: a key is a dotted path that NATS filters by token.
|
|
21
|
+
#
|
|
22
|
+
# @param value [#to_s]
|
|
23
|
+
# @return [String]
|
|
24
|
+
def self.token(value)
|
|
25
|
+
value.to_s.gsub(/[^\w=-]/, "_").then { _1.empty? ? "_" : _1 }
|
|
26
|
+
end
|
|
27
|
+
|
|
15
28
|
def initialize
|
|
16
29
|
@kv = bucket
|
|
30
|
+
@keys_mutex = Mutex.new
|
|
17
31
|
end
|
|
18
32
|
|
|
19
33
|
# Writes or refreshes an entry, recreating the bucket if it was deleted under a running writer.
|
|
@@ -28,6 +42,19 @@ module Cosmo
|
|
|
28
42
|
@kv.set(sanitize(key), Utils::Json.dump(value))
|
|
29
43
|
end
|
|
30
44
|
|
|
45
|
+
# Writes or refreshes several entries in one round trip, recreating the bucket like {#put}.
|
|
46
|
+
#
|
|
47
|
+
# @param entries [Hash{String => Hash}] the value by key
|
|
48
|
+
# @param ttl [Integer, nil] seconds before each entry expires, on a bucket that allows per-message TTLs
|
|
49
|
+
# @return [void]
|
|
50
|
+
def put_many(entries, ttl: nil)
|
|
51
|
+
pairs = entries.to_h { |key, value| [sanitize(key), Utils::Json.dump(value)] }
|
|
52
|
+
@kv.write_many(pairs, ttl:)
|
|
53
|
+
rescue NATS::JetStream::Error::NoStreamResponse
|
|
54
|
+
@kv = bucket
|
|
55
|
+
@kv.write_many(pairs, ttl:)
|
|
56
|
+
end
|
|
57
|
+
|
|
31
58
|
# Removes an entry, leaving no tombstone behind.
|
|
32
59
|
#
|
|
33
60
|
# @param key [String, Integer]
|
|
@@ -38,26 +65,40 @@ module Cosmo
|
|
|
38
65
|
|
|
39
66
|
# @param page [Integer, nil]
|
|
40
67
|
# @param limit [Integer]
|
|
41
|
-
# @return [Array<Hash>]
|
|
68
|
+
# @return [Array<Hash>] the entries on the page, in key order, read in one batched get
|
|
42
69
|
def list(page: nil, limit: LIMIT)
|
|
43
70
|
offset = ([page.to_i, 1].max - 1) * limit
|
|
44
|
-
|
|
71
|
+
page_keys = keys.slice(offset, limit).to_a
|
|
72
|
+
@kv.direct_values(page_keys).values_at(*page_keys).filter_map { _1 && Utils::Json.parse(_1) }
|
|
45
73
|
end
|
|
46
74
|
|
|
75
|
+
# Read by direct gets rather than a KV watch, so a read costs the same however often the writers refresh.
|
|
76
|
+
#
|
|
47
77
|
# @return [Array<Hash>] every live entry
|
|
48
78
|
def all
|
|
49
|
-
@kv.
|
|
79
|
+
@kv.direct_entries.filter_map { Utils::Json.parse(_1.last) }
|
|
50
80
|
end
|
|
51
81
|
|
|
52
|
-
# @return [Integer] the number of live entries
|
|
82
|
+
# @return [Integer] the number of live entries, counted by the server
|
|
53
83
|
def size
|
|
54
|
-
|
|
84
|
+
keys.size
|
|
55
85
|
end
|
|
56
86
|
|
|
57
87
|
private
|
|
58
88
|
|
|
89
|
+
def keys
|
|
90
|
+
@keys_mutex.synchronize do
|
|
91
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
92
|
+
unless @keys && now - @keys_at < self.class::KEYS_TTL
|
|
93
|
+
@keys = @kv.direct_keys.freeze
|
|
94
|
+
@keys_at = now
|
|
95
|
+
end
|
|
96
|
+
@keys
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
59
100
|
def bucket
|
|
60
|
-
KV.new(self.class::BUCKET, { ttl: self.class::TTL, replicas: Config.
|
|
101
|
+
KV.new(self.class::BUCKET, { ttl: self.class::TTL, replicas: Config.services.num_replicas })
|
|
61
102
|
end
|
|
62
103
|
|
|
63
104
|
def sanitize(key)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cosmo
|
|
4
|
+
module API
|
|
5
|
+
module Stats
|
|
6
|
+
# The jobs running right now, one entry per job, refreshed by the worker running it through Cosmo::Heartbeat.
|
|
7
|
+
#
|
|
8
|
+
# An entry's key is +<stream>.<worker>.<jid>+, so how many jobs run, in all or on one stream, is a count the
|
|
9
|
+
# server answers without reading an entry. Each entry expires ENTRY_TTL after its last heartbeat, so a finished
|
|
10
|
+
# job or a dead worker's jobs drop out without a delete, and the counts never drift.
|
|
11
|
+
class Jobs < Registry
|
|
12
|
+
TTL = 60
|
|
13
|
+
BUCKET = "_cosmojobs"
|
|
14
|
+
LONGEST_TTL = 5
|
|
15
|
+
ENTRY_TTL = 2 * Heartbeat::INTERVAL
|
|
16
|
+
|
|
17
|
+
def initialize
|
|
18
|
+
super
|
|
19
|
+
@mutex = Mutex.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Writes the jobs +identity+ is running, each to expire ENTRY_TTL from now unless the next report refreshes it.
|
|
23
|
+
#
|
|
24
|
+
# @param identity [String] the worker, e.g. +hostname-pid+
|
|
25
|
+
# @param jobs [Array<Hash>] each with +:jid+ and +:stream+
|
|
26
|
+
# @return [void]
|
|
27
|
+
def report(identity, jobs)
|
|
28
|
+
entries = jobs.to_h { |job| [key_for(identity, job), job.merge(worker: identity)] }
|
|
29
|
+
put_many(entries, ttl: ENTRY_TTL) unless entries.empty?
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Removes every job +identity+ reported, in one request.
|
|
33
|
+
#
|
|
34
|
+
# @param identity [String]
|
|
35
|
+
# @return [void]
|
|
36
|
+
def clear(identity)
|
|
37
|
+
@kv.erase("*.#{Registry.token(identity)}.*")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @param stream [String, nil] only the jobs on this stream
|
|
41
|
+
# @return [Integer] the jobs running, counted by the server
|
|
42
|
+
def count(stream = nil)
|
|
43
|
+
@kv.direct_count(stream && "#{Registry.token(stream)}.>")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Some running jobs, as many as +limit+, with no say in which: the first ones NATS holds. Costs one small read
|
|
47
|
+
# however many jobs run, so it suits a glance at what is running, not a listing.
|
|
48
|
+
#
|
|
49
|
+
# @param limit [Integer]
|
|
50
|
+
# @return [Array<Hash>]
|
|
51
|
+
def sample(limit)
|
|
52
|
+
@kv.direct_entries(nil, limit:).filter_map { Utils::Json.parse(_1.last) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# One page of the running jobs, longest running first. Ordering them takes reading every job, so that read
|
|
56
|
+
# happens at most once per LONGEST_TTL and is shared by every caller; concurrent callers wait for it.
|
|
57
|
+
#
|
|
58
|
+
# @param page [Integer, nil]
|
|
59
|
+
# @param limit [Integer]
|
|
60
|
+
# @return [Array<Hash>]
|
|
61
|
+
def longest(page: nil, limit: LIMIT)
|
|
62
|
+
@mutex.synchronize do
|
|
63
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
64
|
+
unless @longest && now - @longest_at < LONGEST_TTL
|
|
65
|
+
@longest = all.sort_by { _1[:started_at].to_i }.freeze
|
|
66
|
+
@longest_at = now
|
|
67
|
+
end
|
|
68
|
+
@longest.slice(([page.to_i, 1].max - 1) * limit, limit).to_a
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def bucket
|
|
75
|
+
KV.new(BUCKET, { allow_msg_ttl: true, max_age: Config.to_ns(TTL), replicas: Config.services.num_replicas })
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def key_for(identity, job)
|
|
79
|
+
[job[:stream], identity, job[:jid]].map { Registry.token(_1) }.join(".")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -5,12 +5,16 @@ module Cosmo
|
|
|
5
5
|
module Stats
|
|
6
6
|
# Per-day metrics for every job class: runs, failures, and execution and wait time, kept in their own counters
|
|
7
7
|
# stream for +config.metrics.retention+. Workers {#record} executions in memory and the heartbeat {#flush}es them,
|
|
8
|
-
# so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth.
|
|
8
|
+
# so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth. A counter keeps only
|
|
9
|
+
# its last update, so a day's counters all sit at or after that day's start: reads begin there.
|
|
9
10
|
class Metrics < Counter
|
|
10
11
|
STREAM_NAME = "_cosmometrics"
|
|
11
12
|
DESCRIPTION = "Cosmo metrics"
|
|
12
13
|
DAY = 86_400
|
|
14
|
+
# Seconds a worker's clock, which names the day, may run ahead of the NATS server's, which stamps the message.
|
|
15
|
+
CLOCK_SKEW = 60
|
|
13
16
|
|
|
17
|
+
# @return [Metrics]
|
|
14
18
|
def self.instance
|
|
15
19
|
@instance ||= new
|
|
16
20
|
end
|
|
@@ -54,7 +58,7 @@ module Cosmo
|
|
|
54
58
|
def flush
|
|
55
59
|
pending = @lock.synchronize { @buffer.tap { @buffer = buffer } }
|
|
56
60
|
pending.each do |(day, job), totals|
|
|
57
|
-
totals.each { |field, value| increment("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
|
|
61
|
+
totals.each { |field, value| increment!("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
|
|
58
62
|
end
|
|
59
63
|
end
|
|
60
64
|
|
|
@@ -70,7 +74,7 @@ module Cosmo
|
|
|
70
74
|
# @param job_class [String, nil] nil sums every job class
|
|
71
75
|
# @param days [Integer]
|
|
72
76
|
# @return [Array<Hash>] one +{ date:, count:, failed:, exec_ms:, wait_ms: }+ per day, oldest first
|
|
73
|
-
def daily(job_class = nil, days: 30)
|
|
77
|
+
def daily(job_class = nil, days: 30)
|
|
74
78
|
sums = Hash.new { |hash, day| hash[day] = Hash.new(0) }
|
|
75
79
|
read(days).each { |(day, job, field), value| sums[day][field] += value if job_class.nil? || job == job_class }
|
|
76
80
|
dates(days).map { |day| row(sums[day]).merge(date: day) }
|
|
@@ -83,13 +87,20 @@ module Cosmo
|
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
def read(days)
|
|
86
|
-
|
|
87
|
-
client.
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
window = dates(days)
|
|
91
|
+
messages = client.direct_messages(STREAM_NAME, "#{STREAM_NAME}.#{@namespace}.>", since: since(window.first))
|
|
92
|
+
messages.filter_map do |message|
|
|
93
|
+
day, job, field = message.header[NATS::JetStream::Header::SUBJECT].split(".").last(3)
|
|
94
|
+
next unless window.include?(day)
|
|
95
|
+
|
|
96
|
+
[[day, job.gsub("-", "::"), field.to_sym], Utils::Json.parse(message.data, default: {})[:val].to_i]
|
|
90
97
|
end
|
|
91
98
|
end
|
|
92
99
|
|
|
100
|
+
def since(day)
|
|
101
|
+
Time.utc(day[0, 4].to_i, day[4, 2].to_i, day[6, 2].to_i) - CLOCK_SKEW
|
|
102
|
+
end
|
|
103
|
+
|
|
93
104
|
def dates(days)
|
|
94
105
|
now = Time.now.utc
|
|
95
106
|
(days - 1).downto(0).map { (now - (_1 * DAY)).strftime("%Y%m%d") }
|
|
@@ -4,6 +4,10 @@ module Cosmo
|
|
|
4
4
|
module API
|
|
5
5
|
module Stats
|
|
6
6
|
# Live worker processes, one entry per process, refreshed by Cosmo::Heartbeat.
|
|
7
|
+
#
|
|
8
|
+
# An entry's key is +<kind>.<host>.<pid>.<concurrency>+: the sorted keys are the stable order to page by (a
|
|
9
|
+
# heartbeat rewrites an entry, so the bucket's own order shifts constantly), and they alone give the process
|
|
10
|
+
# count and the fleet's thread count, so only a page's worth of entries is ever read.
|
|
7
11
|
class Processes < Registry
|
|
8
12
|
TTL = 60
|
|
9
13
|
BUCKET = "_cosmoprocesses"
|
|
@@ -18,28 +22,40 @@ module Cosmo
|
|
|
18
22
|
types.empty? ? "none" : types.join("+")
|
|
19
23
|
end
|
|
20
24
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
# @return [
|
|
27
|
-
def
|
|
28
|
-
|
|
29
|
-
all.sort_by { [KINDS.index(self.class.kind(_1)) || KINDS.size, _1[:hostname].to_s, _1[:pid].to_i] }.slice(offset, limit).to_a
|
|
25
|
+
def initialize
|
|
26
|
+
super
|
|
27
|
+
@registered = {}
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @return [Integer] the worker threads across every live process
|
|
31
|
+
def threads
|
|
32
|
+
keys.sum { _1.split(".").last.to_i }
|
|
30
33
|
end
|
|
31
34
|
|
|
32
35
|
# @param identity [String] a unique process id, e.g. +hostname-pid+
|
|
33
36
|
# @param info [Hash] the process details shown in the Web UI
|
|
34
37
|
# @return [void]
|
|
35
38
|
def register(identity, info)
|
|
36
|
-
|
|
39
|
+
key = key_for(identity, info)
|
|
40
|
+
put(key, info)
|
|
41
|
+
stale = @registered[identity]
|
|
42
|
+
@registered[identity] = key
|
|
43
|
+
remove(stale) if stale && stale != key
|
|
37
44
|
end
|
|
38
45
|
|
|
39
46
|
# @param identity [String]
|
|
40
47
|
# @return [void]
|
|
41
48
|
def unregister(identity)
|
|
42
|
-
|
|
49
|
+
key = @registered.delete(identity)
|
|
50
|
+
remove(key) if key
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def key_for(identity, info)
|
|
56
|
+
kind = KINDS.index(self.class.kind(info))
|
|
57
|
+
host = Registry.token(info[:hostname] || identity)
|
|
58
|
+
[kind, host, format("%08d", info[:pid].to_i), info[:concurrency].to_i].join(".")
|
|
43
59
|
end
|
|
44
60
|
end
|
|
45
61
|
end
|
data/lib/cosmo/api/stats.rb
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "cosmo/api/stats/totals"
|
|
4
|
-
require "cosmo/api/stats/busy"
|
|
5
4
|
require "cosmo/api/stats/processes"
|
|
5
|
+
require "cosmo/api/stats/jobs"
|
|
6
6
|
require "cosmo/api/stats/metrics"
|
|
7
7
|
|
|
8
8
|
module Cosmo
|
|
@@ -10,49 +10,85 @@ module Cosmo
|
|
|
10
10
|
module Stats
|
|
11
11
|
module_function
|
|
12
12
|
|
|
13
|
+
# @return [Hash{Symbol => Integer}] the dashboard counts: +:processed+, +:failed+, +:busy+, +:enqueued+,
|
|
14
|
+
# +:retries+, +:scheduled+ and +:dead+
|
|
13
15
|
def summary
|
|
14
|
-
|
|
16
|
+
jobs = Stream.jobs
|
|
17
|
+
{ processed:, failed:, busy:, enqueued: enqueued(jobs), retries: retries(jobs), scheduled:, dead: }
|
|
15
18
|
end
|
|
16
19
|
|
|
20
|
+
# @return [Integer] jobs run successfully, ever
|
|
17
21
|
def processed
|
|
18
22
|
Totals.instance.processed
|
|
19
23
|
end
|
|
20
24
|
|
|
25
|
+
# @return [Integer] failed job attempts, ever
|
|
21
26
|
def failed
|
|
22
27
|
Totals.instance.failed
|
|
23
28
|
end
|
|
24
29
|
|
|
30
|
+
# @param limit [Integer]
|
|
31
|
+
# @return [Array<Hash>] up to +limit+ of the jobs running right now, which ones is not defined, each with the
|
|
32
|
+
# +:worker+ running it; up to Heartbeat::INTERVAL old, and missing a job shorter than that
|
|
33
|
+
def running(limit)
|
|
34
|
+
Jobs.instance.sample(limit)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @param page [Integer]
|
|
38
|
+
# @param limit [Integer]
|
|
39
|
+
# @param longest [Boolean] longest running first, which reads every running job (at most once per
|
|
40
|
+
# Jobs::LONGEST_TTL) instead of only the page
|
|
41
|
+
# @return [Array<Hash>] the jobs running right now on a page, in stream, worker and job id order unless
|
|
42
|
+
# +longest+, each with the +:worker+ running it; see {.busy} for how many there are
|
|
43
|
+
def running_page(page:, limit:, longest: false)
|
|
44
|
+
longest ? Jobs.instance.longest(page:, limit:) : Jobs.instance.list(page:, limit:)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# @return [Integer] jobs running right now across all workers, counted by the server
|
|
25
48
|
def busy
|
|
26
|
-
|
|
49
|
+
Jobs.instance.count
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @param streams [Array<Stream>] the job streams
|
|
53
|
+
# @return [Hash{String => Integer}] the number of jobs running per stream
|
|
54
|
+
def busy_streams(streams = Stream.jobs)
|
|
55
|
+
streams.to_h { [_1.name, Jobs.instance.count(_1.name)] }
|
|
27
56
|
end
|
|
28
57
|
|
|
29
|
-
|
|
30
|
-
|
|
58
|
+
# @param streams [Array<Stream>] the job streams
|
|
59
|
+
# @return [Integer]
|
|
60
|
+
def enqueued(streams = Stream.jobs)
|
|
61
|
+
streams.sum(&:size)
|
|
31
62
|
end
|
|
32
63
|
|
|
33
|
-
|
|
34
|
-
|
|
64
|
+
# @param streams [Array<Stream>] the job streams
|
|
65
|
+
# @return [Integer]
|
|
66
|
+
def retries(streams = Stream.jobs)
|
|
67
|
+
streams.sum(&:retries)
|
|
35
68
|
end
|
|
36
69
|
|
|
70
|
+
# @return [Integer] messages in the +scheduled+ stream, 0 while scheduling is off
|
|
37
71
|
def scheduled
|
|
38
72
|
Config.scheduled.enabled ? Stream.new("scheduled").size : 0
|
|
39
73
|
end
|
|
40
74
|
|
|
75
|
+
# @return [Integer] messages in the +dead+ stream, 0 while it is off
|
|
41
76
|
def dead
|
|
42
77
|
Config.dead.enabled ? Stream.new("dead").size : 0
|
|
43
78
|
end
|
|
44
79
|
|
|
45
80
|
# @return [Array<Hash>] a {.queue} row per job stream
|
|
46
81
|
def queues
|
|
47
|
-
|
|
48
|
-
|
|
82
|
+
streams = Stream.jobs
|
|
83
|
+
busy = busy_streams(streams)
|
|
84
|
+
streams.map { queue(_1, busy) }
|
|
49
85
|
end
|
|
50
86
|
|
|
51
87
|
# @param stream [Stream] a job stream
|
|
52
|
-
# @param busy [Hash{String => Integer}] running jobs per stream, as {
|
|
88
|
+
# @param busy [Hash{String => Integer}] running jobs per stream, as {.busy_streams} counts them
|
|
53
89
|
# @return [Hash] the stream's +:name+, waiting messages as +:size+, running jobs as +:busy+, redeliveries as
|
|
54
90
|
# +:retries+, and whether it is +:paused+
|
|
55
|
-
def queue(stream, busy =
|
|
91
|
+
def queue(stream, busy = busy_streams([stream]))
|
|
56
92
|
{ name: stream.name, size: stream.size, busy: busy.fetch(stream.name, 0), retries: stream.retries,
|
|
57
93
|
paused: stream.paused? }
|
|
58
94
|
end
|