cosmonats 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -13
  3. data/lib/cosmo/active_job/adapter.rb +2 -0
  4. data/lib/cosmo/active_job/executor.rb +7 -0
  5. data/lib/cosmo/active_job/options.rb +6 -0
  6. data/lib/cosmo/api/batch.rb +26 -0
  7. data/lib/cosmo/api/counter.rb +51 -3
  8. data/lib/cosmo/api/cron/entry.rb +24 -9
  9. data/lib/cosmo/api/cron.rb +21 -7
  10. data/lib/cosmo/api/job.rb +14 -0
  11. data/lib/cosmo/api/kv.rb +138 -9
  12. data/lib/cosmo/api/registry.rb +47 -6
  13. data/lib/cosmo/api/stats/jobs.rb +84 -0
  14. data/lib/cosmo/api/stats/metrics.rb +18 -7
  15. data/lib/cosmo/api/stats/processes.rb +27 -11
  16. data/lib/cosmo/api/stats/totals.rb +1 -0
  17. data/lib/cosmo/api/stats.rb +47 -11
  18. data/lib/cosmo/api/stream.rb +66 -22
  19. data/lib/cosmo/batch/callback.rb +5 -0
  20. data/lib/cosmo/batch/dispatcher.rb +4 -4
  21. data/lib/cosmo/batch.rb +34 -11
  22. data/lib/cosmo/cli.rb +6 -0
  23. data/lib/cosmo/client.rb +208 -11
  24. data/lib/cosmo/config/cosmo.yml +1 -0
  25. data/lib/cosmo/config/settings.rb +4 -0
  26. data/lib/cosmo/config.rb +58 -10
  27. data/lib/cosmo/engine.rb +28 -1
  28. data/lib/cosmo/heartbeat.rb +12 -3
  29. data/lib/cosmo/http_server/handler.rb +5 -0
  30. data/lib/cosmo/http_server/health.rb +2 -0
  31. data/lib/cosmo/http_server/ping.rb +2 -0
  32. data/lib/cosmo/http_server.rb +8 -0
  33. data/lib/cosmo/job/data.rb +11 -0
  34. data/lib/cosmo/job/enqueuer.rb +100 -1
  35. data/lib/cosmo/job/limit.rb +5 -1
  36. data/lib/cosmo/job/processor.rb +51 -15
  37. data/lib/cosmo/job/setter.rb +53 -0
  38. data/lib/cosmo/job.rb +71 -2
  39. data/lib/cosmo/logger.rb +37 -0
  40. data/lib/cosmo/middleware/chain.rb +12 -0
  41. data/lib/cosmo/middleware/limit.rb +12 -3
  42. data/lib/cosmo/middleware/metrics.rb +6 -4
  43. data/lib/cosmo/middleware/sentry.rb +6 -4
  44. data/lib/cosmo/middleware/totals.rb +5 -11
  45. data/lib/cosmo/middleware.rb +0 -1
  46. data/lib/cosmo/processor.rb +60 -3
  47. data/lib/cosmo/publisher.rb +8 -0
  48. data/lib/cosmo/services.rb +7 -2
  49. data/lib/cosmo/stream/message.rb +5 -0
  50. data/lib/cosmo/stream/processor.rb +7 -0
  51. data/lib/cosmo/stream/serializer.rb +4 -0
  52. data/lib/cosmo/stream.rb +35 -1
  53. data/lib/cosmo/utils/duration.rb +3 -0
  54. data/lib/cosmo/utils/hash.rb +12 -0
  55. data/lib/cosmo/utils/json.rb +8 -0
  56. data/lib/cosmo/utils/signal.rb +9 -0
  57. data/lib/cosmo/utils/stopwatch.rb +1 -0
  58. data/lib/cosmo/utils/string.rb +4 -0
  59. data/lib/cosmo/utils/thread_pool.rb +8 -0
  60. data/lib/cosmo/utils/ttl_cache.rb +10 -0
  61. data/lib/cosmo/version.rb +1 -1
  62. data/lib/cosmo/web/assets/app.css +5 -3
  63. data/lib/cosmo/web/cache.rb +2 -0
  64. data/lib/cosmo/web/chart.rb +2 -2
  65. data/lib/cosmo/web/circuit.rb +1 -0
  66. data/lib/cosmo/web/context.rb +6 -0
  67. data/lib/cosmo/web/controllers/application.rb +26 -0
  68. data/lib/cosmo/web/controllers/batches.rb +7 -1
  69. data/lib/cosmo/web/controllers/crons.rb +16 -2
  70. data/lib/cosmo/web/controllers/jobs.rb +45 -16
  71. data/lib/cosmo/web/controllers/metrics.rb +5 -0
  72. data/lib/cosmo/web/controllers/overview.rb +36 -18
  73. data/lib/cosmo/web/controllers/processes.rb +8 -5
  74. data/lib/cosmo/web/controllers/streams.rb +12 -0
  75. data/lib/cosmo/web/helpers/application.rb +44 -0
  76. data/lib/cosmo/web/renderer.rb +17 -0
  77. data/lib/cosmo/web/throughput.rb +1 -1
  78. data/lib/cosmo/web/views/crons/_table.erb +29 -27
  79. data/lib/cosmo/web/views/jobs/_args.erb +2 -2
  80. data/lib/cosmo/web/views/jobs/_busy.erb +4 -4
  81. data/lib/cosmo/web/views/jobs/_enqueued.erb +2 -2
  82. data/lib/cosmo/web/views/jobs/_panel.erb +6 -1
  83. data/lib/cosmo/web/views/layout.erb +1 -1
  84. data/lib/cosmo/web/views/overview/_badges.erb +1 -1
  85. data/lib/cosmo/web/views/overview/_running.erb +3 -3
  86. data/lib/cosmo/web/views/overview/index.erb +1 -1
  87. data/lib/cosmo/web/views/streams/_info.erb +1 -0
  88. data/lib/cosmo/web/views/streams/_stream.erb +1 -0
  89. data/lib/cosmo/web.rb +8 -1
  90. data/lib/cosmo.rb +33 -0
  91. data/sig/cosmo/active_job/adapter.rbs +3 -3
  92. data/sig/cosmo/active_job/options.rbs +4 -2
  93. data/sig/cosmo/api/batch.rbs +5 -3
  94. data/sig/cosmo/api/counter.rbs +16 -6
  95. data/sig/cosmo/api/cron/entry.rbs +4 -2
  96. data/sig/cosmo/api/cron.rbs +10 -4
  97. data/sig/cosmo/api/kv.rbs +21 -4
  98. data/sig/cosmo/api/registry.rbs +10 -0
  99. data/sig/cosmo/api/stats/jobs.rbs +34 -0
  100. data/sig/cosmo/api/stats/metrics.rbs +3 -0
  101. data/sig/cosmo/api/stats/processes.rbs +9 -1
  102. data/sig/cosmo/api/stats.rbs +8 -2
  103. data/sig/cosmo/api/stream.rbs +10 -5
  104. data/sig/cosmo/batch/callback.rbs +1 -1
  105. data/sig/cosmo/batch/dispatcher.rbs +1 -1
  106. data/sig/cosmo/batch.rbs +5 -5
  107. data/sig/cosmo/cli.rbs +2 -0
  108. data/sig/cosmo/client.rbs +23 -4
  109. data/sig/cosmo/config/settings.rbs +6 -0
  110. data/sig/cosmo/config.rbs +5 -5
  111. data/sig/cosmo/engine.rbs +5 -0
  112. data/sig/cosmo/heartbeat.rbs +2 -0
  113. data/sig/cosmo/job/data.rbs +3 -1
  114. data/sig/cosmo/job/enqueuer.rbs +12 -0
  115. data/sig/cosmo/job/processor.rbs +18 -3
  116. data/sig/cosmo/job/setter.rbs +20 -0
  117. data/sig/cosmo/job.rbs +10 -4
  118. data/sig/cosmo/logger.rbs +5 -3
  119. data/sig/cosmo/middleware.rbs +0 -8
  120. data/sig/cosmo/processor.rbs +14 -2
  121. data/sig/cosmo/publisher.rbs +1 -0
  122. data/sig/cosmo/stream/processor.rbs +4 -0
  123. data/sig/cosmo/stream.rbs +4 -3
  124. data/sig/cosmo/utils/hash.rbs +2 -1
  125. data/sig/cosmo/utils/signal.rbs +5 -5
  126. data/sig/cosmo/utils/string.rbs +2 -2
  127. data/sig/cosmo/utils/thread_pool.rbs +1 -1
  128. data/sig/cosmo/web/cache.rbs +35 -0
  129. data/sig/cosmo/web/chart.rbs +48 -0
  130. data/sig/cosmo/web/circuit.rbs +29 -0
  131. data/sig/cosmo/web/context.rbs +15 -0
  132. data/sig/cosmo/web/controllers/application.rbs +32 -0
  133. data/sig/cosmo/web/controllers/batches.rbs +11 -0
  134. data/sig/cosmo/web/controllers/crons.rbs +23 -0
  135. data/sig/cosmo/web/controllers/jobs.rbs +47 -0
  136. data/sig/cosmo/web/controllers/metrics.rbs +21 -0
  137. data/sig/cosmo/web/controllers/overview.rbs +31 -0
  138. data/sig/cosmo/web/controllers/processes.rbs +13 -0
  139. data/sig/cosmo/web/controllers/streams.rbs +27 -0
  140. data/sig/cosmo/web/helpers/application.rbs +61 -0
  141. data/sig/cosmo/web/renderer.rbs +33 -0
  142. data/sig/cosmo/web/throughput.rbs +45 -0
  143. data/sig/cosmo/web.rbs +25 -0
  144. data/sig/cosmo.rbs +19 -0
  145. metadata +23 -6
  146. data/lib/cosmo/api/stats/busy.rb +0 -68
  147. data/lib/cosmo/middleware/busy.rb +0 -15
  148. data/sig/cosmo/api/stats/busy.rbs +0 -33
data/lib/cosmo/api/kv.rb CHANGED
@@ -7,12 +7,21 @@ module Cosmo
7
7
 
8
8
  attr_reader :kv
9
9
 
10
+ # Opens the +name+ bucket, creating it with +options+ when it doesn't exist.
11
+ #
12
+ # @param name [String]
13
+ # @param options [Hash, nil] bucket settings, see {Client#kv}
10
14
  def initialize(name, options = nil)
11
15
  @name = name
12
16
  @options = Hash(options)
13
17
  @kv = client.kv(@name, **@options)
14
18
  end
15
19
 
20
+ # @param key [String, Integer]
21
+ # @param value [#to_s]
22
+ # @param ttl [Integer, nil] seconds before the key expires; with it the key is only created when absent
23
+ # @return [Integer] the stored message's sequence
24
+ # @raise [NATS::KeyValue::KeyWrongLastSequenceError] when +ttl+ is given and the key is live
16
25
  def set(key, value, ttl: nil)
17
26
  return kv.put(key, value.to_s) unless ttl
18
27
 
@@ -21,6 +30,8 @@ module Cosmo
21
30
  publish_cas(key, value.to_s, ttl, last_seq: 0).seq
22
31
  end
23
32
 
33
+ # @param key [String, Integer]
34
+ # @return [NATS::KeyValue::Entry, nil] nil when the key doesn't exist
24
35
  def get(key)
25
36
  kv.get(key)
26
37
  rescue NATS::KeyValue::KeyNotFoundError
@@ -30,6 +41,10 @@ module Cosmo
30
41
  # Creates +key+ if it does not already exist -- a plain CAS-if-absent with
31
42
  # no per-message TTL involved. Raises NATS::KeyValue::KeyWrongLastSequenceError
32
43
  # when the key is already live, so callers can use it as a fire-once gate.
44
+ #
45
+ # @param key [String, Integer]
46
+ # @param value [#to_s]
47
+ # @return [Integer] the key's revision
33
48
  def create(key, value)
34
49
  kv.create(key, value.to_s)
35
50
  end
@@ -37,6 +52,9 @@ module Cosmo
37
52
  # Writes a KV-Operation tombstone. On a ttl-bearing bucket this leaves the
38
53
  # subject occupied, so a subsequent #set(ttl:) CAS with last_seq: 0 will
39
54
  # keep failing -- use #erase on those buckets instead.
55
+ #
56
+ # @param key [String, Integer]
57
+ # @return [Integer] the tombstone's sequence
40
58
  def delete(key)
41
59
  kv.delete(key)
42
60
  end
@@ -44,6 +62,11 @@ module Cosmo
44
62
  # Walks the bucket and collects live keys. Pass +limit: nil+ to walk the whole
45
63
  # bucket -- a bounded +limit+ stops the watch early, so it can never be used
46
64
  # to count (see #count).
65
+ #
66
+ # @param subject [String, nil] a key filter, every key when nil
67
+ # @param limit [Integer, nil]
68
+ # @param offset [Integer] live keys to skip first
69
+ # @return [Array<String>]
47
70
  def keys(subject = nil, limit: LIMIT, offset: 0)
48
71
  walk(subject, limit:, offset:, meta_only: true).map(&:first)
49
72
  end
@@ -51,12 +74,91 @@ module Cosmo
51
74
  # Same walk as #keys, but carries each value back with its key. The watch
52
75
  # already has the values in hand, so callers that need them get one pass
53
76
  # instead of a #keys walk followed by a #get per key.
77
+ # @param subject [String, nil] a key filter, every key when nil
78
+ # @param limit [Integer, nil]
79
+ # @param offset [Integer] live entries to skip first
54
80
  # @return [Array<Array(String, String)>] +[key, value]+ pairs
55
81
  def entries(subject = nil, limit: LIMIT, offset: 0)
56
82
  walk(subject, limit:, offset:, meta_only: false)
57
83
  end
58
84
 
85
+ # Same pairs as #entries, read by batched direct gets instead of a watch: no consumer is created, any replica
86
+ # can answer, and the cost doesn't grow with the writes landing on the bucket while it's read. Keys come in
87
+ # the order they were last written. Turns on direct gets for the bucket first, see #direct!.
88
+ #
89
+ # @param subject [String, nil] a key filter, every key when nil
90
+ # @param limit [Integer, nil] the most messages to read, tombstones included; nil for every one
91
+ # @return [Array<Array(String, String)>] +[key, value]+ pairs
92
+ def direct_entries(subject = nil, limit: nil)
93
+ direct!
94
+ messages = client.direct_messages("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}", limit:)
95
+ messages.filter_map do |msg|
96
+ header = msg.header
97
+ [header[NATS::JetStream::Header::SUBJECT].delete_prefix("$KV.#{@name}."), msg.data] unless header["KV-Operation"]
98
+ end
99
+ end
100
+
101
+ # How many keys match +subject+, counted by the server without reading any value: one request however many
102
+ # keys there are. Turns on direct gets for the bucket first, see #direct!.
103
+ #
104
+ # @param subject [String, nil] a key filter, every key when nil
105
+ # @return [Integer]
106
+ def direct_count(subject = nil)
107
+ direct!
108
+ client.count_messages("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}")
109
+ end
110
+
111
+ # The sorted keys matching +subject+, without their values, for paging through a bucket in a stable order.
112
+ # Turns on direct gets for the bucket first, see #direct!.
113
+ #
114
+ # @param subject [String, nil] a key filter, every key when nil
115
+ # @return [Array<String>]
116
+ def direct_keys(subject = nil)
117
+ direct!
118
+ client.subjects("KV_#{@name}", "$KV.#{@name}.#{subject || ">"}").map { _1.delete_prefix("$KV.#{@name}.") }
119
+ end
120
+
121
+ # The values of +keys+, in one batched direct get. NATS answers for at most 1024 keys at a time.
122
+ #
123
+ # @param keys [Array<String>]
124
+ # @return [Hash{String => String}] the value by key, without the keys that don't exist
125
+ def direct_values(keys)
126
+ direct!
127
+ messages = client.last_messages("KV_#{@name}", keys.map { "$KV.#{@name}.#{_1}" })
128
+ messages.each_with_object({}) do |(subject, msg), values|
129
+ values[subject.delete_prefix("$KV.#{@name}.")] = msg.data unless msg.header&.key?("KV-Operation")
130
+ end
131
+ end
132
+
133
+ # Writes every pair before waiting for any ack, so a batch costs about one round trip instead of one each.
134
+ #
135
+ # @param pairs [Hash{String => #to_s}] the value by key
136
+ # @param ttl [Integer, nil] seconds before each key expires, on a bucket created with +allow_msg_ttl+
137
+ # @return [void]
138
+ # @raise [StandardError] the first error among the acks
139
+ def write_many(pairs, ttl: nil)
140
+ header = ttl ? { "Nats-TTL" => [ttl.to_i, 1].max.to_s } : {}
141
+ acks = client.publish_many(pairs.map { |key, value| { subject: "$KV.#{@name}.#{key}", payload: value.to_s, header: } })
142
+ error = acks.find { _1.is_a?(StandardError) }
143
+ raise error if error
144
+ end
145
+
146
+ # Turns on +allow_direct+ for a bucket created without it, as nats-pure creates them. Checked once per instance.
147
+ #
148
+ # @return [true]
149
+ def direct!
150
+ @direct ||= begin
151
+ stream = "KV_#{@name}"
152
+ config = client.stream_info(stream).config
153
+ client.update_stream(stream, config.to_h.compact.except(:name).merge(allow_direct: true)) unless config.allow_direct
154
+ true
155
+ end
156
+ end
157
+
59
158
  # Writes a KV-Operation tombstone (same issue as #delete on ttl buckets).
159
+ #
160
+ # @param key [String, Integer]
161
+ # @return [NATS::JetStream::PubAck]
60
162
  def purge(key)
61
163
  kv.purge(key)
62
164
  end
@@ -65,14 +167,21 @@ module Cosmo
65
167
  # write a KV-Operation tombstone message. Mirrors how per-message
66
168
  # Nats-TTL expiry removes a key, so callers never have to distinguish
67
169
  # "deleted" from "TTL-expired" on read.
170
+ #
171
+ # @param key [String, Integer]
172
+ # @return [Integer, nil] the number of messages purged
68
173
  def erase(key)
69
174
  client.purge("KV_#{@name}", "$KV.#{@name}.#{key}")
70
175
  end
71
176
 
177
+ # Removes every key in the bucket.
178
+ #
179
+ # @return [Integer, nil] the number of messages purged
72
180
  def clean
73
181
  client.purge("KV_#{@name}", ">")
74
182
  end
75
183
 
184
+ # @return [Integer] the number of live keys
76
185
  def count
77
186
  keys(limit: nil).size
78
187
  rescue NATS::KeyValue::NoKeysFoundError, NATS::JetStream::Error::NotFound
@@ -85,19 +194,39 @@ module Cosmo
85
194
  def walk(subject, limit:, offset:, meta_only:)
86
195
  results = []
87
196
  skipped = 0
88
- watcher = kv.watch(subject || ">", ignore_deletes: true, meta_only:)
89
- watcher.each do |entry|
90
- break unless entry
91
-
92
- next skipped += 1 if skipped < offset.to_i
93
-
94
- results << [entry.key, entry.value]
95
- break if limit && results.size >= limit
197
+ seen = 0
198
+ watcher = kv.watch(subject || ">", meta_only:)
199
+ begin
200
+ watcher.each do |entry|
201
+ if entry.nil?
202
+ break if drained?(watcher, seen)
203
+
204
+ next
205
+ end
206
+
207
+ seen += 1
208
+ unless entry.operation
209
+ next skipped += 1 if skipped < offset.to_i
210
+
211
+ results << [entry.key, entry.value]
212
+ break if limit && results.size >= limit
213
+ end
214
+ break if entry.delta.zero?
215
+ end
216
+ ensure
217
+ watcher.stop
96
218
  end
97
- watcher.stop
98
219
  results
99
220
  end
100
221
 
222
+ # nats-pure ends the initial walk with a nil once consumer info reports nothing pending and
223
+ # nothing received yet, but the server can answer that info before the pushed entries arrive.
224
+ # The nil is only trustworthy when the consumer really delivered no more than was seen.
225
+ # https://github.com/nats-io/nats-pure.rb/issues/197
226
+ def drained?(watcher, seen)
227
+ watcher._sub.consumer_info.delivered.consumer_seq <= seen
228
+ end
229
+
101
230
  # CAS = Compare-And-Swap: publish +value+ with a per-message TTL of +ttl+ whole seconds (at least 1),
102
231
  # but only if the subject's current last sequence matches +last_seq+
103
232
  # (sent as the Nats-Expected-Last-Subject-Sequence header). Raises
@@ -5,15 +5,29 @@ module Cosmo
5
5
  # Base for a KV bucket of JSON entries that live only while their writer keeps refreshing them:
6
6
  # a writer re-puts its entries faster than the bucket's TTL, and an entry whose writer is gone expires.
7
7
  # Subclasses define +BUCKET+ and +TTL+.
8
+ #
9
+ # An entry's key is what the Web UI counts and pages by: the server counts keys and lists them sorted, so only
10
+ # the entries on the page being shown are ever read.
8
11
  class Registry
9
12
  LIMIT = 25
13
+ KEYS_TTL = 2
10
14
 
15
+ # @return [Registry]
11
16
  def self.instance
12
17
  @instance ||= new
13
18
  end
14
19
 
20
+ # Makes +value+ one subject token: a key is a dotted path that NATS filters by token.
21
+ #
22
+ # @param value [#to_s]
23
+ # @return [String]
24
+ def self.token(value)
25
+ value.to_s.gsub(/[^\w=-]/, "_").then { _1.empty? ? "_" : _1 }
26
+ end
27
+
15
28
  def initialize
16
29
  @kv = bucket
30
+ @keys_mutex = Mutex.new
17
31
  end
18
32
 
19
33
  # Writes or refreshes an entry, recreating the bucket if it was deleted under a running writer.
@@ -28,6 +42,19 @@ module Cosmo
28
42
  @kv.set(sanitize(key), Utils::Json.dump(value))
29
43
  end
30
44
 
45
+ # Writes or refreshes several entries in one round trip, recreating the bucket like {#put}.
46
+ #
47
+ # @param entries [Hash{String => Hash}] the value by key
48
+ # @param ttl [Integer, nil] seconds before each entry expires, on a bucket that allows per-message TTLs
49
+ # @return [void]
50
+ def put_many(entries, ttl: nil)
51
+ pairs = entries.to_h { |key, value| [sanitize(key), Utils::Json.dump(value)] }
52
+ @kv.write_many(pairs, ttl:)
53
+ rescue NATS::JetStream::Error::NoStreamResponse
54
+ @kv = bucket
55
+ @kv.write_many(pairs, ttl:)
56
+ end
57
+
31
58
  # Removes an entry, leaving no tombstone behind.
32
59
  #
33
60
  # @param key [String, Integer]
@@ -38,26 +65,40 @@ module Cosmo
38
65
 
39
66
  # @param page [Integer, nil]
40
67
  # @param limit [Integer]
41
- # @return [Array<Hash>]
68
+ # @return [Array<Hash>] the entries on the page, in key order, read in one batched get
42
69
  def list(page: nil, limit: LIMIT)
43
70
  offset = ([page.to_i, 1].max - 1) * limit
44
- @kv.entries(limit:, offset:).filter_map { Utils::Json.parse(_1.last) }
71
+ page_keys = keys.slice(offset, limit).to_a
72
+ @kv.direct_values(page_keys).values_at(*page_keys).filter_map { _1 && Utils::Json.parse(_1) }
45
73
  end
46
74
 
75
+ # Read by direct gets rather than a KV watch, so a read costs the same however often the writers refresh.
76
+ #
47
77
  # @return [Array<Hash>] every live entry
48
78
  def all
49
- @kv.entries(limit: nil).filter_map { Utils::Json.parse(_1.last) }
79
+ @kv.direct_entries.filter_map { Utils::Json.parse(_1.last) }
50
80
  end
51
81
 
52
- # @return [Integer] the number of live entries
82
+ # @return [Integer] the number of live entries, counted by the server
53
83
  def size
54
- @kv.size
84
+ keys.size
55
85
  end
56
86
 
57
87
  private
58
88
 
89
+ def keys
90
+ @keys_mutex.synchronize do
91
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
92
+ unless @keys && now - @keys_at < self.class::KEYS_TTL
93
+ @keys = @kv.direct_keys.freeze
94
+ @keys_at = now
95
+ end
96
+ @keys
97
+ end
98
+ end
99
+
59
100
  def bucket
60
- KV.new(self.class::BUCKET, { ttl: self.class::TTL, replicas: Config.replicas })
101
+ KV.new(self.class::BUCKET, { ttl: self.class::TTL, replicas: Config.services.num_replicas })
61
102
  end
62
103
 
63
104
  def sanitize(key)
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cosmo
4
+ module API
5
+ module Stats
6
+ # The jobs running right now, one entry per job, refreshed by the worker running it through Cosmo::Heartbeat.
7
+ #
8
+ # An entry's key is +<stream>.<worker>.<jid>+, so how many jobs run, in all or on one stream, is a count the
9
+ # server answers without reading an entry. Each entry expires ENTRY_TTL after its last heartbeat, so a finished
10
+ # job or a dead worker's jobs drop out without a delete, and the counts never drift.
11
+ class Jobs < Registry
12
+ TTL = 60
13
+ BUCKET = "_cosmojobs"
14
+ LONGEST_TTL = 5
15
+ ENTRY_TTL = 2 * Heartbeat::INTERVAL
16
+
17
+ def initialize
18
+ super
19
+ @mutex = Mutex.new
20
+ end
21
+
22
+ # Writes the jobs +identity+ is running, each to expire ENTRY_TTL from now unless the next report refreshes it.
23
+ #
24
+ # @param identity [String] the worker, e.g. +hostname-pid+
25
+ # @param jobs [Array<Hash>] each with +:jid+ and +:stream+
26
+ # @return [void]
27
+ def report(identity, jobs)
28
+ entries = jobs.to_h { |job| [key_for(identity, job), job.merge(worker: identity)] }
29
+ put_many(entries, ttl: ENTRY_TTL) unless entries.empty?
30
+ end
31
+
32
+ # Removes every job +identity+ reported, in one request.
33
+ #
34
+ # @param identity [String]
35
+ # @return [void]
36
+ def clear(identity)
37
+ @kv.erase("*.#{Registry.token(identity)}.*")
38
+ end
39
+
40
+ # @param stream [String, nil] only the jobs on this stream
41
+ # @return [Integer] the jobs running, counted by the server
42
+ def count(stream = nil)
43
+ @kv.direct_count(stream && "#{Registry.token(stream)}.>")
44
+ end
45
+
46
+ # Some running jobs, as many as +limit+, with no say in which: the first ones NATS holds. Costs one small read
47
+ # however many jobs run, so it suits a glance at what is running, not a listing.
48
+ #
49
+ # @param limit [Integer]
50
+ # @return [Array<Hash>]
51
+ def sample(limit)
52
+ @kv.direct_entries(nil, limit:).filter_map { Utils::Json.parse(_1.last) }
53
+ end
54
+
55
+ # One page of the running jobs, longest running first. Ordering them takes reading every job, so that read
56
+ # happens at most once per LONGEST_TTL and is shared by every caller; concurrent callers wait for it.
57
+ #
58
+ # @param page [Integer, nil]
59
+ # @param limit [Integer]
60
+ # @return [Array<Hash>]
61
+ def longest(page: nil, limit: LIMIT)
62
+ @mutex.synchronize do
63
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
64
+ unless @longest && now - @longest_at < LONGEST_TTL
65
+ @longest = all.sort_by { _1[:started_at].to_i }.freeze
66
+ @longest_at = now
67
+ end
68
+ @longest.slice(([page.to_i, 1].max - 1) * limit, limit).to_a
69
+ end
70
+ end
71
+
72
+ private
73
+
74
+ def bucket
75
+ KV.new(BUCKET, { allow_msg_ttl: true, max_age: Config.to_ns(TTL), replicas: Config.services.num_replicas })
76
+ end
77
+
78
+ def key_for(identity, job)
79
+ [job[:stream], identity, job[:jid]].map { Registry.token(_1) }.join(".")
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -5,12 +5,16 @@ module Cosmo
5
5
  module Stats
6
6
  # Per-day metrics for every job class: runs, failures, and execution and wait time, kept in their own counters
7
7
  # stream for +config.metrics.retention+. Workers {#record} executions in memory and the heartbeat {#flush}es them,
8
- # so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth.
8
+ # so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth. A counter keeps only
9
+ # its last update, so a day's counters all sit at or after that day's start: reads begin there.
9
10
  class Metrics < Counter
10
11
  STREAM_NAME = "_cosmometrics"
11
12
  DESCRIPTION = "Cosmo metrics"
12
13
  DAY = 86_400
14
+ # Seconds a worker's clock, which names the day, may run ahead of the NATS server's, which stamps the message.
15
+ CLOCK_SKEW = 60
13
16
 
17
+ # @return [Metrics]
14
18
  def self.instance
15
19
  @instance ||= new
16
20
  end
@@ -54,7 +58,7 @@ module Cosmo
54
58
  def flush
55
59
  pending = @lock.synchronize { @buffer.tap { @buffer = buffer } }
56
60
  pending.each do |(day, job), totals|
57
- totals.each { |field, value| increment("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
61
+ totals.each { |field, value| increment!("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
58
62
  end
59
63
  end
60
64
 
@@ -70,7 +74,7 @@ module Cosmo
70
74
  # @param job_class [String, nil] nil sums every job class
71
75
  # @param days [Integer]
72
76
  # @return [Array<Hash>] one +{ date:, count:, failed:, exec_ms:, wait_ms: }+ per day, oldest first
73
- def daily(job_class = nil, days: 30) # rubocop:disable Metrics/AbcSize
77
+ def daily(job_class = nil, days: 30)
74
78
  sums = Hash.new { |hash, day| hash[day] = Hash.new(0) }
75
79
  read(days).each { |(day, job, field), value| sums[day][field] += value if job_class.nil? || job == job_class }
76
80
  dates(days).map { |day| row(sums[day]).merge(date: day) }
@@ -83,13 +87,20 @@ module Cosmo
83
87
  end
84
88
 
85
89
  def read(days)
86
- filters = dates(days).map { "#{STREAM_NAME}.#{@namespace}.#{_1}.>" }
87
- client.last_messages(STREAM_NAME, filters).map do |subject, data|
88
- day, job, field = subject.split(".").last(3)
89
- [[day, job.gsub("-", "::"), field.to_sym], Utils::Json.parse(data, default: {})[:val].to_i]
90
+ window = dates(days)
91
+ messages = client.direct_messages(STREAM_NAME, "#{STREAM_NAME}.#{@namespace}.>", since: since(window.first))
92
+ messages.filter_map do |message|
93
+ day, job, field = message.header[NATS::JetStream::Header::SUBJECT].split(".").last(3)
94
+ next unless window.include?(day)
95
+
96
+ [[day, job.gsub("-", "::"), field.to_sym], Utils::Json.parse(message.data, default: {})[:val].to_i]
90
97
  end
91
98
  end
92
99
 
100
+ def since(day)
101
+ Time.utc(day[0, 4].to_i, day[4, 2].to_i, day[6, 2].to_i) - CLOCK_SKEW
102
+ end
103
+
93
104
  def dates(days)
94
105
  now = Time.now.utc
95
106
  (days - 1).downto(0).map { (now - (_1 * DAY)).strftime("%Y%m%d") }
@@ -4,6 +4,10 @@ module Cosmo
4
4
  module API
5
5
  module Stats
6
6
  # Live worker processes, one entry per process, refreshed by Cosmo::Heartbeat.
7
+ #
8
+ # An entry's key is +<kind>.<host>.<pid>.<concurrency>+: the sorted keys are the stable order to page by (a
9
+ # heartbeat rewrites an entry, so the bucket's own order shifts constantly), and they alone give the process
10
+ # count and the fleet's thread count, so only a page's worth of entries is ever read.
7
11
  class Processes < Registry
8
12
  TTL = 60
9
13
  BUCKET = "_cosmoprocesses"
@@ -18,28 +22,40 @@ module Cosmo
18
22
  types.empty? ? "none" : types.join("+")
19
23
  end
20
24
 
21
- # Sorted by kind, host and pid before paging: every heartbeat rewrites an entry, so the bucket's own
22
- # order shifts constantly and would move processes between pages, and each kind stays contiguous.
23
- #
24
- # @param page [Integer, nil]
25
- # @param limit [Integer]
26
- # @return [Array<Hash>]
27
- def list(page: nil, limit: LIMIT)
28
- offset = ([page.to_i, 1].max - 1) * limit
29
- all.sort_by { [KINDS.index(self.class.kind(_1)) || KINDS.size, _1[:hostname].to_s, _1[:pid].to_i] }.slice(offset, limit).to_a
25
+ def initialize
26
+ super
27
+ @registered = {}
28
+ end
29
+
30
+ # @return [Integer] the worker threads across every live process
31
+ def threads
32
+ keys.sum { _1.split(".").last.to_i }
30
33
  end
31
34
 
32
35
  # @param identity [String] a unique process id, e.g. +hostname-pid+
33
36
  # @param info [Hash] the process details shown in the Web UI
34
37
  # @return [void]
35
38
  def register(identity, info)
36
- put(identity, info)
39
+ key = key_for(identity, info)
40
+ put(key, info)
41
+ stale = @registered[identity]
42
+ @registered[identity] = key
43
+ remove(stale) if stale && stale != key
37
44
  end
38
45
 
39
46
  # @param identity [String]
40
47
  # @return [void]
41
48
  def unregister(identity)
42
- remove(identity)
49
+ key = @registered.delete(identity)
50
+ remove(key) if key
51
+ end
52
+
53
+ private
54
+
55
+ def key_for(identity, info)
56
+ kind = KINDS.index(self.class.kind(info))
57
+ host = Registry.token(info[:hostname] || identity)
58
+ [kind, host, format("%08d", info[:pid].to_i), info[:concurrency].to_i].join(".")
43
59
  end
44
60
  end
45
61
  end
@@ -8,6 +8,7 @@ module Cosmo
8
8
  STREAM_NAME = "_cosmototals"
9
9
  DESCRIPTION = "Cosmo statistics"
10
10
 
11
+ # @return [Totals]
11
12
  def self.instance
12
13
  @instance ||= new("jobs")
13
14
  end
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "cosmo/api/stats/totals"
4
- require "cosmo/api/stats/busy"
5
4
  require "cosmo/api/stats/processes"
5
+ require "cosmo/api/stats/jobs"
6
6
  require "cosmo/api/stats/metrics"
7
7
 
8
8
  module Cosmo
@@ -10,49 +10,85 @@ module Cosmo
10
10
  module Stats
11
11
  module_function
12
12
 
13
+ # @return [Hash{Symbol => Integer}] the dashboard counts: +:processed+, +:failed+, +:busy+, +:enqueued+,
14
+ # +:retries+, +:scheduled+ and +:dead+
13
15
  def summary
14
- { processed:, failed:, busy:, enqueued:, retries:, scheduled:, dead: }
16
+ jobs = Stream.jobs
17
+ { processed:, failed:, busy:, enqueued: enqueued(jobs), retries: retries(jobs), scheduled:, dead: }
15
18
  end
16
19
 
20
+ # @return [Integer] jobs run successfully, ever
17
21
  def processed
18
22
  Totals.instance.processed
19
23
  end
20
24
 
25
+ # @return [Integer] failed job attempts, ever
21
26
  def failed
22
27
  Totals.instance.failed
23
28
  end
24
29
 
30
+ # @param limit [Integer]
31
+ # @return [Array<Hash>] up to +limit+ of the jobs running right now, which ones is not defined, each with the
32
+ # +:worker+ running it; up to Heartbeat::INTERVAL old, and missing a job shorter than that
33
+ def running(limit)
34
+ Jobs.instance.sample(limit)
35
+ end
36
+
37
+ # @param page [Integer]
38
+ # @param limit [Integer]
39
+ # @param longest [Boolean] longest running first, which reads every running job (at most once per
40
+ # Jobs::LONGEST_TTL) instead of only the page
41
+ # @return [Array<Hash>] the jobs running right now on a page, in stream, worker and job id order unless
42
+ # +longest+, each with the +:worker+ running it; see {.busy} for how many there are
43
+ def running_page(page:, limit:, longest: false)
44
+ longest ? Jobs.instance.longest(page:, limit:) : Jobs.instance.list(page:, limit:)
45
+ end
46
+
47
+ # @return [Integer] jobs running right now across all workers, counted by the server
25
48
  def busy
26
- Busy.instance.size
49
+ Jobs.instance.count
50
+ end
51
+
52
+ # @param streams [Array<Stream>] the job streams
53
+ # @return [Hash{String => Integer}] the number of jobs running per stream
54
+ def busy_streams(streams = Stream.jobs)
55
+ streams.to_h { [_1.name, Jobs.instance.count(_1.name)] }
27
56
  end
28
57
 
29
- def enqueued
30
- Stream.jobs.sum(&:size)
58
+ # @param streams [Array<Stream>] the job streams
59
+ # @return [Integer]
60
+ def enqueued(streams = Stream.jobs)
61
+ streams.sum(&:size)
31
62
  end
32
63
 
33
- def retries
34
- Stream.jobs.sum(&:retries)
64
+ # @param streams [Array<Stream>] the job streams
65
+ # @return [Integer]
66
+ def retries(streams = Stream.jobs)
67
+ streams.sum(&:retries)
35
68
  end
36
69
 
70
+ # @return [Integer] messages in the +scheduled+ stream, 0 while scheduling is off
37
71
  def scheduled
38
72
  Config.scheduled.enabled ? Stream.new("scheduled").size : 0
39
73
  end
40
74
 
75
+ # @return [Integer] messages in the +dead+ stream, 0 while it is off
41
76
  def dead
42
77
  Config.dead.enabled ? Stream.new("dead").size : 0
43
78
  end
44
79
 
45
80
  # @return [Array<Hash>] a {.queue} row per job stream
46
81
  def queues
47
- busy = Busy.instance.by_stream
48
- Stream.jobs.map { queue(_1, busy) }
82
+ streams = Stream.jobs
83
+ busy = busy_streams(streams)
84
+ streams.map { queue(_1, busy) }
49
85
  end
50
86
 
51
87
  # @param stream [Stream] a job stream
52
- # @param busy [Hash{String => Integer}] running jobs per stream, as {Busy#by_stream} counts them
88
+ # @param busy [Hash{String => Integer}] running jobs per stream, as {.busy_streams} counts them
53
89
  # @return [Hash] the stream's +:name+, waiting messages as +:size+, running jobs as +:busy+, redeliveries as
54
90
  # +:retries+, and whether it is +:paused+
55
- def queue(stream, busy = Busy.instance.by_stream)
91
+ def queue(stream, busy = busy_streams([stream]))
56
92
  { name: stream.name, size: stream.size, busy: busy.fetch(stream.name, 0), retries: stream.retries,
57
93
  paused: stream.paused? }
58
94
  end