natswork-server 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +0 -0
- data/LICENSE +21 -0
- data/README.md +286 -0
- data/lib/natswork/cli.rb +420 -0
- data/lib/natswork/error_tracker.rb +338 -0
- data/lib/natswork/health_check.rb +252 -0
- data/lib/natswork/instrumentation.rb +141 -0
- data/lib/natswork/job_executor.rb +271 -0
- data/lib/natswork/job_hooks.rb +63 -0
- data/lib/natswork/logger.rb +183 -0
- data/lib/natswork/metrics.rb +241 -0
- data/lib/natswork/middleware.rb +142 -0
- data/lib/natswork/middleware_chain.rb +40 -0
- data/lib/natswork/monitoring.rb +397 -0
- data/lib/natswork/protocol.rb +454 -0
- data/lib/natswork/queue_manager.rb +164 -0
- data/lib/natswork/retry_handler.rb +125 -0
- data/lib/natswork/server/version.rb +7 -0
- data/lib/natswork/server.rb +47 -0
- data/lib/natswork/simple_worker.rb +101 -0
- data/lib/natswork/thread_pool.rb +192 -0
- data/lib/natswork/worker.rb +217 -0
- data/lib/natswork/worker_manager.rb +62 -0
- data/lib/natswork-server.rb +5 -0
- metadata +151 -0
@@ -0,0 +1,241 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'concurrent'
|
4
|
+
|
5
|
+
module NatsWork
|
6
|
+
class Metrics
|
7
|
+
attr_reader :counters, :gauges, :histograms, :timers
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@counters = Concurrent::Hash.new(0)
|
11
|
+
@gauges = Concurrent::Hash.new(0)
|
12
|
+
@histograms = Concurrent::Hash.new { |h, k| h[k] = [] }
|
13
|
+
@timers = Concurrent::Hash.new { |h, k| h[k] = [] }
|
14
|
+
|
15
|
+
@collectors = []
|
16
|
+
@enabled = true
|
17
|
+
end
|
18
|
+
|
19
|
+
# Counter - always increasing value
|
20
|
+
def increment(metric, value = 1, tags = {})
|
21
|
+
return unless @enabled
|
22
|
+
|
23
|
+
key = metric_key(metric, tags)
|
24
|
+
@counters[key] += value
|
25
|
+
|
26
|
+
notify_collectors(:counter, metric, @counters[key], tags)
|
27
|
+
end
|
28
|
+
|
29
|
+
def counter(metric, tags = {})
|
30
|
+
@counters[metric_key(metric, tags)]
|
31
|
+
end
|
32
|
+
|
33
|
+
# Gauge - point-in-time value
|
34
|
+
def gauge(metric, value, tags = {})
|
35
|
+
return unless @enabled
|
36
|
+
|
37
|
+
key = metric_key(metric, tags)
|
38
|
+
@gauges[key] = value
|
39
|
+
|
40
|
+
notify_collectors(:gauge, metric, value, tags)
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_gauge(metric, tags = {})
|
44
|
+
@gauges[metric_key(metric, tags)]
|
45
|
+
end
|
46
|
+
|
47
|
+
# Histogram - distribution of values
|
48
|
+
def histogram(metric, value, tags = {})
|
49
|
+
return unless @enabled
|
50
|
+
|
51
|
+
key = metric_key(metric, tags)
|
52
|
+
@histograms[key] << value
|
53
|
+
|
54
|
+
# Keep only last 1000 values to prevent memory bloat
|
55
|
+
@histograms[key] = @histograms[key].last(1000) if @histograms[key].size > 1000
|
56
|
+
|
57
|
+
notify_collectors(:histogram, metric, value, tags)
|
58
|
+
end
|
59
|
+
|
60
|
+
def histogram_stats(metric, tags = {})
|
61
|
+
values = @histograms[metric_key(metric, tags)]
|
62
|
+
return nil if values.empty?
|
63
|
+
|
64
|
+
sorted = values.sort
|
65
|
+
{
|
66
|
+
count: values.size,
|
67
|
+
min: sorted.first,
|
68
|
+
max: sorted.last,
|
69
|
+
mean: values.sum.to_f / values.size,
|
70
|
+
p50: percentile(sorted, 0.5),
|
71
|
+
p95: percentile(sorted, 0.95),
|
72
|
+
p99: percentile(sorted, 0.99)
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
# Timer - measure duration
|
77
|
+
def time(metric, tags = {})
|
78
|
+
return yield unless @enabled
|
79
|
+
|
80
|
+
start_time = Time.now
|
81
|
+
begin
|
82
|
+
result = yield
|
83
|
+
duration = (Time.now - start_time) * 1000 # milliseconds
|
84
|
+
|
85
|
+
key = metric_key(metric, tags)
|
86
|
+
@timers[key] << duration
|
87
|
+
|
88
|
+
# Keep only last 1000 values
|
89
|
+
@timers[key] = @timers[key].last(1000) if @timers[key].size > 1000
|
90
|
+
|
91
|
+
notify_collectors(:timer, metric, duration, tags)
|
92
|
+
result
|
93
|
+
rescue StandardError
|
94
|
+
duration = (Time.now - start_time) * 1000
|
95
|
+
|
96
|
+
key = metric_key(metric, tags)
|
97
|
+
@timers[key] << duration
|
98
|
+
|
99
|
+
# Keep only last 1000 values
|
100
|
+
@timers[key] = @timers[key].last(1000) if @timers[key].size > 1000
|
101
|
+
|
102
|
+
notify_collectors(:timer, metric, duration, tags.merge(status: 'error'))
|
103
|
+
raise
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def timer_stats(metric, tags = {})
|
108
|
+
key = metric_key(metric, tags)
|
109
|
+
values = @timers[key]
|
110
|
+
return nil if values.empty?
|
111
|
+
|
112
|
+
histogram_stats_from_values(values)
|
113
|
+
end
|
114
|
+
|
115
|
+
# Add a metrics collector (e.g., StatsD, Prometheus)
|
116
|
+
def add_collector(collector)
|
117
|
+
@collectors << collector if collector.respond_to?(:collect)
|
118
|
+
end
|
119
|
+
|
120
|
+
def remove_collector(collector)
|
121
|
+
@collectors.delete(collector)
|
122
|
+
end
|
123
|
+
|
124
|
+
def enable!
|
125
|
+
@enabled = true
|
126
|
+
end
|
127
|
+
|
128
|
+
def disable!
|
129
|
+
@enabled = false
|
130
|
+
end
|
131
|
+
|
132
|
+
def reset!
|
133
|
+
@counters.clear
|
134
|
+
@gauges.clear
|
135
|
+
@histograms.clear
|
136
|
+
@timers.clear
|
137
|
+
end
|
138
|
+
|
139
|
+
def snapshot
|
140
|
+
{
|
141
|
+
counters: @counters.to_h,
|
142
|
+
gauges: @gauges.to_h,
|
143
|
+
histograms: @histograms.transform_values { |v| histogram_stats_from_values(v) },
|
144
|
+
timers: @timers.transform_values { |v| histogram_stats_from_values(v) }
|
145
|
+
}
|
146
|
+
end
|
147
|
+
|
148
|
+
private
|
149
|
+
|
150
|
+
def metric_key(metric, tags)
|
151
|
+
return metric if tags.empty?
|
152
|
+
|
153
|
+
tag_str = tags.sort.map { |k, v| "#{k}:#{v}" }.join(',')
|
154
|
+
"#{metric}{#{tag_str}}"
|
155
|
+
end
|
156
|
+
|
157
|
+
def percentile(sorted_values, pct)
|
158
|
+
return nil if sorted_values.empty?
|
159
|
+
|
160
|
+
index = (sorted_values.size * pct).ceil - 1
|
161
|
+
index = 0 if index.negative?
|
162
|
+
sorted_values[index]
|
163
|
+
end
|
164
|
+
|
165
|
+
def histogram_stats_from_values(values)
|
166
|
+
return nil if values.empty?
|
167
|
+
|
168
|
+
sorted = values.sort
|
169
|
+
{
|
170
|
+
count: values.size,
|
171
|
+
min: sorted.first,
|
172
|
+
max: sorted.last,
|
173
|
+
mean: values.sum.to_f / values.size,
|
174
|
+
p50: percentile(sorted, 0.5),
|
175
|
+
p95: percentile(sorted, 0.95),
|
176
|
+
p99: percentile(sorted, 0.99)
|
177
|
+
}
|
178
|
+
end
|
179
|
+
|
180
|
+
def notify_collectors(type, metric, value, tags)
|
181
|
+
@collectors.each do |collector|
|
182
|
+
collector.collect(type, metric, value, tags)
|
183
|
+
rescue StandardError => e
|
184
|
+
# Don't let collector errors break metrics collection
|
185
|
+
NatsWork::Logger.error('Metrics collector error', {
|
186
|
+
collector: collector.class.name,
|
187
|
+
error: e.message
|
188
|
+
})
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class << self
|
193
|
+
def global
|
194
|
+
@global ||= new
|
195
|
+
end
|
196
|
+
|
197
|
+
def method_missing(method, *args, &block)
|
198
|
+
if global.respond_to?(method)
|
199
|
+
global.send(method, *args, &block)
|
200
|
+
else
|
201
|
+
super
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def respond_to_missing?(method, include_private = false)
|
206
|
+
global.respond_to?(method, include_private) || super
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
# Base class for metrics collectors
|
212
|
+
class MetricsCollector
|
213
|
+
def collect(type, metric, value, tags)
|
214
|
+
raise NotImplementedError, "#{self.class} must implement #collect"
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# StatsD metrics collector
|
219
|
+
class StatsDCollector < MetricsCollector
|
220
|
+
def initialize(client)
|
221
|
+
@client = client
|
222
|
+
end
|
223
|
+
|
224
|
+
def collect(type, metric, value, tags)
|
225
|
+
metric_name = tags.empty? ? metric : "#{metric}.#{tags.map { |k, v| "#{k}_#{v}" }.join('.')}"
|
226
|
+
|
227
|
+
case type
|
228
|
+
when :counter
|
229
|
+
@client.increment(metric_name, value)
|
230
|
+
when :gauge
|
231
|
+
@client.gauge(metric_name, value)
|
232
|
+
when :histogram
|
233
|
+
@client.histogram(metric_name, value)
|
234
|
+
when :timer
|
235
|
+
@client.timing(metric_name, value)
|
236
|
+
end
|
237
|
+
rescue StandardError
|
238
|
+
# Log but don't raise
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
module NatsWork
|
6
|
+
module Middleware
|
7
|
+
class Entry
|
8
|
+
attr_reader :klass, :args
|
9
|
+
|
10
|
+
def initialize(klass, args = [])
|
11
|
+
@klass = klass
|
12
|
+
@args = args
|
13
|
+
end
|
14
|
+
|
15
|
+
def build
|
16
|
+
@klass.new(*@args)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class MiddlewareChain
|
21
|
+
attr_reader :entries
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
@entries = []
|
25
|
+
end
|
26
|
+
|
27
|
+
def add(klass, *args)
|
28
|
+
@entries << Entry.new(klass, args)
|
29
|
+
end
|
30
|
+
|
31
|
+
def remove(klass)
|
32
|
+
@entries.delete_if { |entry| entry.klass == klass }
|
33
|
+
end
|
34
|
+
|
35
|
+
def clear
|
36
|
+
@entries.clear
|
37
|
+
end
|
38
|
+
|
39
|
+
def invoke(job, message, &block)
|
40
|
+
chain = @entries.map(&:build)
|
41
|
+
traverse(chain, job, message, &block)
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def traverse(chain, job, message, &block)
|
47
|
+
if chain.empty?
|
48
|
+
block.call
|
49
|
+
else
|
50
|
+
middleware = chain.shift
|
51
|
+
middleware.call(job, message) do
|
52
|
+
traverse(chain, job, message, &block)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# Built-in Middleware
|
59
|
+
class Logger
|
60
|
+
def initialize(logger: ::Logger.new($stdout))
|
61
|
+
@logger = logger
|
62
|
+
end
|
63
|
+
|
64
|
+
def call(job, message)
|
65
|
+
job_class = job.class.name
|
66
|
+
job_id = message['job_id']
|
67
|
+
|
68
|
+
@logger.info "Starting job #{job_class} (#{job_id})"
|
69
|
+
started_at = Time.now
|
70
|
+
|
71
|
+
begin
|
72
|
+
result = yield
|
73
|
+
elapsed = ((Time.now - started_at) * 1000).round(2)
|
74
|
+
@logger.info "Completed job #{job_class} (#{job_id}) in #{elapsed}ms"
|
75
|
+
result
|
76
|
+
rescue StandardError => e
|
77
|
+
elapsed = ((Time.now - started_at) * 1000).round(2)
|
78
|
+
@logger.error "Failed job #{job_class} (#{job_id}) after #{elapsed}ms: #{e.message}"
|
79
|
+
raise
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
class Timer
|
85
|
+
def call(_job, message)
|
86
|
+
started_at = Time.now
|
87
|
+
|
88
|
+
result = yield
|
89
|
+
|
90
|
+
completed_at = Time.now
|
91
|
+
message['started_at'] = started_at.iso8601
|
92
|
+
message['completed_at'] = completed_at.iso8601
|
93
|
+
message['execution_time'] = completed_at - started_at
|
94
|
+
|
95
|
+
result
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class ErrorHandler
|
100
|
+
def initialize(on_error: nil)
|
101
|
+
@on_error = on_error
|
102
|
+
end
|
103
|
+
|
104
|
+
def call(job, message)
|
105
|
+
yield
|
106
|
+
rescue StandardError => e
|
107
|
+
message['error'] = {
|
108
|
+
'type' => e.class.name,
|
109
|
+
'message' => e.message
|
110
|
+
}
|
111
|
+
|
112
|
+
@on_error&.call(job, message, e)
|
113
|
+
|
114
|
+
raise
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
class Retry
|
119
|
+
def initialize(retry_handler:, connection: nil)
|
120
|
+
@retry_handler = retry_handler
|
121
|
+
@connection = connection
|
122
|
+
end
|
123
|
+
|
124
|
+
def call(_job, message)
|
125
|
+
yield
|
126
|
+
rescue StandardError => e
|
127
|
+
handle_error(message, e)
|
128
|
+
nil # Don't propagate error
|
129
|
+
end
|
130
|
+
|
131
|
+
private
|
132
|
+
|
133
|
+
def handle_error(message, error)
|
134
|
+
if @retry_handler.should_retry?(message)
|
135
|
+
@retry_handler.schedule_retry(@connection, message, error)
|
136
|
+
else
|
137
|
+
@retry_handler.send_to_dead_letter(@connection, message, error)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module NatsWork
|
4
|
+
module Server
|
5
|
+
class MiddlewareChain
|
6
|
+
def initialize
|
7
|
+
@entries = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def add(middleware, *args)
|
11
|
+
@entries << [middleware, args]
|
12
|
+
self
|
13
|
+
end
|
14
|
+
|
15
|
+
def invoke(*args)
|
16
|
+
chain = @entries.map do |middleware, options|
|
17
|
+
middleware.new(*options)
|
18
|
+
end
|
19
|
+
|
20
|
+
traverse_chain = lambda do
|
21
|
+
if chain.empty?
|
22
|
+
yield if block_given?
|
23
|
+
else
|
24
|
+
chain.shift.call(*args, &traverse_chain)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
traverse_chain.call
|
29
|
+
end
|
30
|
+
|
31
|
+
def clear
|
32
|
+
@entries.clear
|
33
|
+
end
|
34
|
+
|
35
|
+
def empty?
|
36
|
+
@entries.empty?
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|