fastly_nsq 1.15.0 → 1.17.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.git-blame-ignore-revs +6 -0
- data/.ruby-version +1 -1
- data/.travis.yml +4 -3
- data/ChangeLog.md +24 -1
- data/Gemfile +8 -8
- data/README.md +5 -1
- data/Rakefile +10 -11
- data/fastly_nsq.gemspec +26 -26
- data/lib/fastly_nsq/cli.rb +43 -50
- data/lib/fastly_nsq/consumer.rb +6 -6
- data/lib/fastly_nsq/feeder.rb +5 -7
- data/lib/fastly_nsq/http/nsqd.rb +28 -28
- data/lib/fastly_nsq/http/nsqlookupd.rb +11 -11
- data/lib/fastly_nsq/http.rb +4 -4
- data/lib/fastly_nsq/launcher.rb +16 -16
- data/lib/fastly_nsq/listener.rb +17 -16
- data/lib/fastly_nsq/manager.rb +13 -12
- data/lib/fastly_nsq/message.rb +9 -5
- data/lib/fastly_nsq/messenger.rb +25 -15
- data/lib/fastly_nsq/new_relic.rb +8 -8
- data/lib/fastly_nsq/priority_queue.rb +2 -2
- data/lib/fastly_nsq/priority_thread_pool.rb +3 -3
- data/lib/fastly_nsq/producer.rb +7 -7
- data/lib/fastly_nsq/safe_thread.rb +1 -1
- data/lib/fastly_nsq/testing.rb +12 -2
- data/lib/fastly_nsq/tls_options.rb +6 -6
- data/lib/fastly_nsq/version.rb +1 -1
- data/lib/fastly_nsq.rb +27 -29
- data/spec/cli_spec.rb +2 -2
- data/spec/consumer_spec.rb +12 -12
- data/spec/fastly_nsq_spec.rb +31 -31
- data/spec/feeder_spec.rb +4 -4
- data/spec/http/nsqd_spec.rb +23 -23
- data/spec/http/nsqlookupd_spec.rb +19 -19
- data/spec/http_spec.rb +22 -22
- data/spec/integration_spec.rb +10 -10
- data/spec/launcher_spec.rb +21 -21
- data/spec/listener_spec.rb +50 -50
- data/spec/manager_spec.rb +27 -27
- data/spec/matchers/delegate.rb +4 -4
- data/spec/message_spec.rb +20 -20
- data/spec/messenger_spec.rb +71 -59
- data/spec/new_relic.rb +27 -27
- data/spec/priority_thread_pool_spec.rb +2 -2
- data/spec/producer_spec.rb +30 -30
- data/spec/spec_helper.rb +12 -12
- data/spec/support/http.rb +2 -2
- data/spec/support/webmock.rb +1 -1
- data/spec/testing_spec.rb +12 -12
- data/spec/tls_options_spec.rb +47 -47
- metadata +14 -15
- data/.rubocop.yml +0 -68
data/lib/fastly_nsq/http/nsqd.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/http"
|
4
4
|
|
5
5
|
class FastlyNsq::Http
|
6
6
|
##
|
@@ -13,11 +13,11 @@ class FastlyNsq::Http
|
|
13
13
|
def_delegator :client, :get
|
14
14
|
def_delegator :client, :post
|
15
15
|
|
16
|
-
BASE_NSQD_URL = ENV.fetch(
|
17
|
-
if ENV[
|
18
|
-
"https://#{ENV.fetch(
|
16
|
+
BASE_NSQD_URL = ENV.fetch("NSQD_URL") do
|
17
|
+
if ENV["NSQD_HTTPS_ADDRESS"]
|
18
|
+
"https://#{ENV.fetch("NSQD_HTTPS_ADDRESS")}"
|
19
19
|
else
|
20
|
-
"http://#{ENV.fetch(
|
20
|
+
"http://#{ENV.fetch("NSQD_HTTP_ADDRESS")}"
|
21
21
|
end
|
22
22
|
end
|
23
23
|
VALID_FORMATS = %w[text json].freeze
|
@@ -27,13 +27,13 @@ class FastlyNsq::Http
|
|
27
27
|
#
|
28
28
|
# NOTE: The only "unhealthy" state is if nsqd failed to write messages to disk when overflow occurred.
|
29
29
|
def self.ping(**args)
|
30
|
-
new(request_uri:
|
30
|
+
new(request_uri: "/ping", **args).get
|
31
31
|
end
|
32
32
|
|
33
33
|
##
|
34
34
|
# NSQ version information
|
35
35
|
def self.info(**args)
|
36
|
-
new(request_uri:
|
36
|
+
new(request_uri: "/info", **args).get
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
@@ -47,12 +47,12 @@ class FastlyNsq::Http
|
|
47
47
|
#
|
48
48
|
# @example Fetch Statistics for topic: 'foo', channel: 'bar' as text
|
49
49
|
# Nsqd.stats(topic: 'foo', channel: 'bar', format: 'text')
|
50
|
-
def self.stats(topic: nil, channel: nil, format:
|
50
|
+
def self.stats(topic: nil, channel: nil, format: "json", **args)
|
51
51
|
raise InvalidFormatError unless VALID_FORMATS.include?(format)
|
52
|
-
params = {
|
52
|
+
params = {format: format}
|
53
53
|
params[:topic] = topic if topic
|
54
54
|
params[:channel] = channel if channel
|
55
|
-
new(request_uri:
|
55
|
+
new(request_uri: "/stats", **args).get(params)
|
56
56
|
end
|
57
57
|
|
58
58
|
##
|
@@ -62,9 +62,9 @@ class FastlyNsq::Http
|
|
62
62
|
# @param defer [String] the time in ms to delay message delivery
|
63
63
|
# @param message the message body
|
64
64
|
def self.pub(topic:, message:, defer: nil, **args)
|
65
|
-
params = {
|
65
|
+
params = {topic: topic}
|
66
66
|
params[:defer] = defer if defer
|
67
|
-
new(request_uri:
|
67
|
+
new(request_uri: "/pub", **args).post(params, message)
|
68
68
|
end
|
69
69
|
|
70
70
|
##
|
@@ -83,17 +83,17 @@ class FastlyNsq::Http
|
|
83
83
|
# @param topic [String] the topic to publish to
|
84
84
|
# @param binary [Boolean] enables binary mode
|
85
85
|
# @param message the messages to send with \n used to seperate messages
|
86
|
-
def self.mpub(topic:, binary: false,
|
87
|
-
binary_param = binary ?
|
88
|
-
raise NotImplementedError,
|
89
|
-
params = {
|
90
|
-
new(request_uri:
|
86
|
+
def self.mpub(topic:, message:, binary: false, **args)
|
87
|
+
binary_param = binary ? "true" : "false"
|
88
|
+
raise NotImplementedError, "binary mode has yet to be implemented" if binary
|
89
|
+
params = {topic: topic, binary: binary_param}
|
90
|
+
new(request_uri: "/mpub", **args).post(params, message)
|
91
91
|
end
|
92
92
|
|
93
93
|
##
|
94
94
|
# List of nsqlookupd TCP addresses
|
95
95
|
def self.config_nsqlookupd_tcp_addresses(**args)
|
96
|
-
new(request_uri:
|
96
|
+
new(request_uri: "/config/nsqlookupd_tcp_addresses", **args).get
|
97
97
|
end
|
98
98
|
|
99
99
|
##
|
@@ -101,7 +101,7 @@ class FastlyNsq::Http
|
|
101
101
|
#
|
102
102
|
# @param topic [String] the topic to create
|
103
103
|
def self.topic_create(topic:, **args)
|
104
|
-
new(request_uri:
|
104
|
+
new(request_uri: "/topic/create", **args).post(topic: topic)
|
105
105
|
end
|
106
106
|
|
107
107
|
##
|
@@ -109,7 +109,7 @@ class FastlyNsq::Http
|
|
109
109
|
#
|
110
110
|
# @param topic [String] the existing topic to delete
|
111
111
|
def self.topic_delete(topic:, **args)
|
112
|
-
new(request_uri:
|
112
|
+
new(request_uri: "/topic/delete", **args).post(topic: topic)
|
113
113
|
end
|
114
114
|
|
115
115
|
##
|
@@ -117,7 +117,7 @@ class FastlyNsq::Http
|
|
117
117
|
#
|
118
118
|
# @param topic [String] the existing topic to empty
|
119
119
|
def self.topic_empty(topic:, **args)
|
120
|
-
new(request_uri:
|
120
|
+
new(request_uri: "/topic/empty", **args).post(topic: topic)
|
121
121
|
end
|
122
122
|
|
123
123
|
##
|
@@ -126,7 +126,7 @@ class FastlyNsq::Http
|
|
126
126
|
#
|
127
127
|
# @param topic [String] the existing topic to pause
|
128
128
|
def self.topic_pause(topic:, **args)
|
129
|
-
new(request_uri:
|
129
|
+
new(request_uri: "/topic/pause", **args).post(topic: topic)
|
130
130
|
end
|
131
131
|
|
132
132
|
##
|
@@ -134,7 +134,7 @@ class FastlyNsq::Http
|
|
134
134
|
#
|
135
135
|
# @param topic [String] the existing, paused topic to unpause
|
136
136
|
def self.topic_unpause(topic:, **args)
|
137
|
-
new(request_uri:
|
137
|
+
new(request_uri: "/topic/unpause", **args).post(topic: topic)
|
138
138
|
end
|
139
139
|
|
140
140
|
##
|
@@ -143,7 +143,7 @@ class FastlyNsq::Http
|
|
143
143
|
# @param topic [String] the existing topic
|
144
144
|
# @param channel [String] the channel to create
|
145
145
|
def self.channel_create(topic:, channel:, **args)
|
146
|
-
new(request_uri:
|
146
|
+
new(request_uri: "/channel/create", **args).post(topic: topic, channel: channel)
|
147
147
|
end
|
148
148
|
|
149
149
|
##
|
@@ -152,7 +152,7 @@ class FastlyNsq::Http
|
|
152
152
|
# @param topic [String] the existing topic
|
153
153
|
# @param channel [String] the channel to delete
|
154
154
|
def self.channel_delete(topic:, channel:, **args)
|
155
|
-
new(request_uri:
|
155
|
+
new(request_uri: "/channel/delete", **args).post(topic: topic, channel: channel)
|
156
156
|
end
|
157
157
|
|
158
158
|
##
|
@@ -161,7 +161,7 @@ class FastlyNsq::Http
|
|
161
161
|
# @param topic [String] the existing topic
|
162
162
|
# @param channel [String] the channel to empty
|
163
163
|
def self.channel_empty(topic:, channel:, **args)
|
164
|
-
new(request_uri:
|
164
|
+
new(request_uri: "/channel/empty", **args).post(topic: topic, channel: channel)
|
165
165
|
end
|
166
166
|
|
167
167
|
##
|
@@ -171,7 +171,7 @@ class FastlyNsq::Http
|
|
171
171
|
# @param topic [String] the existing topic
|
172
172
|
# @param channel [String] the channel to pause
|
173
173
|
def self.channel_pause(topic:, channel:, **args)
|
174
|
-
new(request_uri:
|
174
|
+
new(request_uri: "/channel/pause", **args).post(topic: topic, channel: channel)
|
175
175
|
end
|
176
176
|
|
177
177
|
##
|
@@ -180,7 +180,7 @@ class FastlyNsq::Http
|
|
180
180
|
# @param topic [String] the existing topic
|
181
181
|
# @param channel [String] the existing, paused, channel to unpause
|
182
182
|
def self.channel_unpause(topic:, channel:, **args)
|
183
|
-
new(request_uri:
|
183
|
+
new(request_uri: "/channel/unpause", **args).post(topic: topic, channel: channel)
|
184
184
|
end
|
185
185
|
|
186
186
|
##
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/http"
|
4
4
|
|
5
5
|
class FastlyNsq::Http
|
6
6
|
##
|
@@ -12,20 +12,20 @@ class FastlyNsq::Http
|
|
12
12
|
extend Forwardable
|
13
13
|
def_delegator :client, :get
|
14
14
|
|
15
|
-
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch(
|
15
|
+
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch("NSQLOOKUPD_HTTP_ADDRESS", "").split(",")[0]}"
|
16
16
|
|
17
17
|
##
|
18
18
|
# List of producers for a given topic
|
19
19
|
#
|
20
20
|
# @param topic [String] the topic for which to list producers
|
21
21
|
def self.lookup(topic:, **args)
|
22
|
-
new(request_uri:
|
22
|
+
new(request_uri: "/lookup", **args).get(topic: topic)
|
23
23
|
end
|
24
24
|
|
25
25
|
##
|
26
26
|
# List of all known topics
|
27
27
|
def self.topics(**args)
|
28
|
-
new(request_uri:
|
28
|
+
new(request_uri: "/topics", **args).get
|
29
29
|
end
|
30
30
|
|
31
31
|
##
|
@@ -33,13 +33,13 @@ class FastlyNsq::Http
|
|
33
33
|
#
|
34
34
|
# @param topic [String] the topic for which to list channels
|
35
35
|
def self.channels(topic:, **args)
|
36
|
-
new(request_uri:
|
36
|
+
new(request_uri: "/channels", **args).get(topic: topic)
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
40
40
|
# List all known nsqd nodes
|
41
41
|
def self.nodes(**args)
|
42
|
-
new(request_uri:
|
42
|
+
new(request_uri: "/nodes", **args).get
|
43
43
|
end
|
44
44
|
|
45
45
|
##
|
@@ -47,7 +47,7 @@ class FastlyNsq::Http
|
|
47
47
|
#
|
48
48
|
# @param topic [String] the exsiting topic to delete
|
49
49
|
def self.delete_topic(topic:, **args)
|
50
|
-
new(request_uri:
|
50
|
+
new(request_uri: "/delete_topic", **args).get(topic: topic)
|
51
51
|
end
|
52
52
|
|
53
53
|
##
|
@@ -56,7 +56,7 @@ class FastlyNsq::Http
|
|
56
56
|
# @param topic [String] an exsiting topic
|
57
57
|
# @param channel [String] the exsiting channel to delete
|
58
58
|
def self.delete_channel(topic:, channel:, **args)
|
59
|
-
new(request_uri:
|
59
|
+
new(request_uri: "/delete_channel", **args).get(topic: topic, channel: channel)
|
60
60
|
end
|
61
61
|
|
62
62
|
##
|
@@ -67,19 +67,19 @@ class FastlyNsq::Http
|
|
67
67
|
# @param topic [String] the existing topic
|
68
68
|
# @param node [String] the producer (nsqd) to tombstone (identified by <broadcast_address>:<http_port>)
|
69
69
|
def self.tombstone_topic_producer(topic:, node:, **args)
|
70
|
-
new(request_uri:
|
70
|
+
new(request_uri: "/tombstone_topic_producer", **args).get(topic: topic, node: node)
|
71
71
|
end
|
72
72
|
|
73
73
|
##
|
74
74
|
# Monitoring endpoint, should return +OK+
|
75
75
|
def self.ping(**args)
|
76
|
-
new(request_uri:
|
76
|
+
new(request_uri: "/ping", **args).get
|
77
77
|
end
|
78
78
|
|
79
79
|
##
|
80
80
|
# Returns nsqlookupd version information
|
81
81
|
def self.info(**args)
|
82
|
-
new(request_uri:
|
82
|
+
new(request_uri: "/info", **args).get
|
83
83
|
end
|
84
84
|
|
85
85
|
##
|
data/lib/fastly_nsq/http.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "net/https"
|
4
|
+
require "fastly_nsq/http/nsqd"
|
5
|
+
require "fastly_nsq/http/nsqlookupd"
|
6
6
|
|
7
7
|
##
|
8
8
|
# Adapter class for HTTP requests to NSQD
|
@@ -15,7 +15,7 @@ require 'fastly_nsq/http/nsqlookupd'
|
|
15
15
|
# @see FastlyNsq::Http::Nsqd
|
16
16
|
# @see FastlyNsq::Http::Nsqlookupd
|
17
17
|
class FastlyNsq::Http
|
18
|
-
def initialize(uri:, cert_filename: ENV[
|
18
|
+
def initialize(uri:, cert_filename: ENV["NSQ_SSL_CERTIFICATE"], key_filename: ENV["NSQ_SSL_KEY"])
|
19
19
|
@uri = uri.is_a?(URI) ? uri : URI.parse(uri)
|
20
20
|
@cert_filename = cert_filename
|
21
21
|
@key_filename = key_filename
|
data/lib/fastly_nsq/launcher.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/safe_thread"
|
4
4
|
|
5
5
|
##
|
6
6
|
# FastlyNsq::Launcher is a lighweight wrapper of a thread manager
|
@@ -17,18 +17,18 @@ class FastlyNsq::Launcher
|
|
17
17
|
FastlyNsq.manager
|
18
18
|
end
|
19
19
|
|
20
|
-
def initialize(
|
21
|
-
@done
|
22
|
-
@timeout = timeout
|
23
|
-
@pulse
|
24
|
-
@logger
|
20
|
+
def initialize(**options)
|
21
|
+
@done = false
|
22
|
+
@timeout = options[:timeout] || 5
|
23
|
+
@pulse = options[:pulse] || 5
|
24
|
+
@logger = options[:logger] || FastlyNsq.logger
|
25
25
|
|
26
|
-
FastlyNsq.manager = FastlyNsq::Manager.new(options)
|
26
|
+
FastlyNsq.manager = FastlyNsq::Manager.new(**options)
|
27
27
|
FastlyNsq.fire_event :startup
|
28
28
|
end
|
29
29
|
|
30
30
|
def beat
|
31
|
-
@heartbeat ||= safe_thread(
|
31
|
+
@heartbeat ||= safe_thread("heartbeat", &method(:start_heartbeat))
|
32
32
|
end
|
33
33
|
|
34
34
|
def stop
|
@@ -51,13 +51,13 @@ class FastlyNsq::Launcher
|
|
51
51
|
def heartbeat
|
52
52
|
logger.debug do
|
53
53
|
[
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
].join(
|
54
|
+
"HEARTBEAT:",
|
55
|
+
"busy:", manager.pool.length,
|
56
|
+
"processed:", manager.pool.completed_task_count,
|
57
|
+
"max_threads:", manager.pool.max_length,
|
58
|
+
"max_queue_size:", manager.pool.largest_length,
|
59
|
+
"listeners:", manager.listeners.count
|
60
|
+
].join(" ")
|
61
61
|
end
|
62
62
|
|
63
63
|
# TODO: Check the health of the system overall and kill it if needed
|
@@ -73,6 +73,6 @@ class FastlyNsq::Launcher
|
|
73
73
|
heartbeat
|
74
74
|
sleep pulse
|
75
75
|
end
|
76
|
-
logger.info(
|
76
|
+
logger.info("Heartbeat stopping...")
|
77
77
|
end
|
78
78
|
end
|
data/lib/fastly_nsq/listener.rb
CHANGED
@@ -77,25 +77,25 @@ class FastlyNsq::Listener
|
|
77
77
|
# max_attempts: 15,
|
78
78
|
# )
|
79
79
|
def initialize(topic:, processor:, preprocessor: FastlyNsq.preprocessor, channel: FastlyNsq.channel, consumer: nil,
|
80
|
-
|
81
|
-
|
80
|
+
logger: FastlyNsq.logger, priority: DEFAULT_PRIORITY, connect_timeout: DEFAULT_CONNECTION_TIMEOUT,
|
81
|
+
max_attempts: FastlyNsq.max_attempts, **consumer_options)
|
82
82
|
|
83
83
|
raise ArgumentError, "processor #{processor.inspect} does not respond to #call" unless processor.respond_to?(:call)
|
84
84
|
raise ArgumentError, "priority #{priority.inspect} must be a Integer" unless priority.is_a?(Integer)
|
85
85
|
|
86
|
-
@channel
|
87
|
-
@logger
|
86
|
+
@channel = channel
|
87
|
+
@logger = logger
|
88
88
|
@max_attempts = max_attempts
|
89
89
|
@preprocessor = preprocessor
|
90
|
-
@priority
|
91
|
-
@processor
|
92
|
-
@topic
|
90
|
+
@priority = priority
|
91
|
+
@processor = processor
|
92
|
+
@topic = topic
|
93
93
|
|
94
94
|
@consumer = consumer || FastlyNsq::Consumer.new(topic: topic,
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
95
|
+
connect_timeout: connect_timeout,
|
96
|
+
channel: channel,
|
97
|
+
queue: FastlyNsq::Feeder.new(self, priority),
|
98
|
+
max_attempts: max_attempts,
|
99
99
|
**consumer_options)
|
100
100
|
|
101
101
|
FastlyNsq.manager.add_listener(self)
|
@@ -114,11 +114,12 @@ class FastlyNsq::Listener
|
|
114
114
|
message = FastlyNsq::Message.new(nsq_message)
|
115
115
|
|
116
116
|
msg_info = {
|
117
|
-
channel:
|
118
|
-
topic:
|
119
|
-
attempts:
|
120
|
-
id:
|
121
|
-
|
117
|
+
channel: channel,
|
118
|
+
topic: topic,
|
119
|
+
attempts: message.attempts,
|
120
|
+
id: Digest::MD5.hexdigest(nsq_message.body.to_s),
|
121
|
+
nsq_id: message.id,
|
122
|
+
metadata: message.meta
|
122
123
|
}
|
123
124
|
|
124
125
|
logger.info do
|
data/lib/fastly_nsq/manager.rb
CHANGED
@@ -17,14 +17,15 @@ class FastlyNsq::Manager
|
|
17
17
|
##
|
18
18
|
# Create a FastlyNsq::Manager
|
19
19
|
#
|
20
|
-
# @param
|
21
|
-
#
|
22
|
-
#
|
23
|
-
def initialize(logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads
|
24
|
-
@done
|
25
|
-
@logger
|
26
|
-
|
27
|
-
|
20
|
+
# @param opts [Hash] Set of options passed to FastlyNsqw::PriorityThreadPool. valid options include:
|
21
|
+
# * max_threads [Integer] Maxiumum number of threads to be used by {FastlyNsq::PriorityThreadPool}
|
22
|
+
# * logger [Logger]
|
23
|
+
def initialize(**opts) # logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads)
|
24
|
+
@done = false
|
25
|
+
@logger = opts[:logger] || FastlyNsq.logger
|
26
|
+
max_threads = opts[:max_threads] || FastlyNsq.max_processing_pool_threads
|
27
|
+
@pool = FastlyNsq::PriorityThreadPool.new(
|
28
|
+
{fallback_policy: :caller_runs, max_threads: max_threads}.merge(opts)
|
28
29
|
)
|
29
30
|
end
|
30
31
|
|
@@ -99,7 +100,7 @@ class FastlyNsq::Manager
|
|
99
100
|
##
|
100
101
|
# Terminate all listeners
|
101
102
|
def stop_listeners
|
102
|
-
logger.info {
|
103
|
+
logger.info { "Stopping listeners" }
|
103
104
|
listeners.each(&:terminate)
|
104
105
|
topic_listeners.clear
|
105
106
|
end
|
@@ -110,13 +111,13 @@ class FastlyNsq::Manager
|
|
110
111
|
# Shutdown the pool
|
111
112
|
# @param deadline [Integer] Number of seconds to wait for pool to stop processing
|
112
113
|
def stop_processing(deadline)
|
113
|
-
logger.info {
|
114
|
+
logger.info { "Stopping processors" }
|
114
115
|
pool.shutdown
|
115
116
|
|
116
|
-
logger.info {
|
117
|
+
logger.info { "Waiting for processors to finish..." }
|
117
118
|
return if pool.wait_for_termination(deadline)
|
118
119
|
|
119
|
-
logger.info {
|
120
|
+
logger.info { "Killing processors..." }
|
120
121
|
pool.kill
|
121
122
|
end
|
122
123
|
end
|
data/lib/fastly_nsq/message.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "json"
|
4
4
|
|
5
5
|
##
|
6
6
|
# Adapter to Nsq::Message. Provides convenience methods for interacting
|
@@ -20,7 +20,11 @@ class FastlyNsq::Message
|
|
20
20
|
# Delegated to +self.nsq_message+
|
21
21
|
# @return [Nsq::Message#timestamp]
|
22
22
|
# @see https://www.rubydoc.info/gems/nsq-ruby/Nsq/Message#timestamp-instance_method
|
23
|
-
|
23
|
+
# @!method id
|
24
|
+
# Delegated to +self.nsq_message+
|
25
|
+
# @return [Nsq::Message#id]
|
26
|
+
# @see https://www.rubydoc.info/gems/nsq-ruby/Nsq/Message#id-instance_method
|
27
|
+
def_delegators :@nsq_message, :attempts, :touch, :timestamp, :id
|
24
28
|
|
25
29
|
# @return [Symbol] Message state. Returns +nil+ if message has not been requeued or finished.
|
26
30
|
attr_reader :managed
|
@@ -32,7 +36,7 @@ class FastlyNsq::Message
|
|
32
36
|
# @return [String] Nsq::Message body
|
33
37
|
attr_reader :raw_body
|
34
38
|
|
35
|
-
|
39
|
+
alias_method :to_s, :raw_body
|
36
40
|
|
37
41
|
##
|
38
42
|
# @param nsq_message [Nsq::Message]
|
@@ -42,11 +46,11 @@ class FastlyNsq::Message
|
|
42
46
|
end
|
43
47
|
|
44
48
|
def data
|
45
|
-
@data ||= body[
|
49
|
+
@data ||= body["data"]
|
46
50
|
end
|
47
51
|
|
48
52
|
def meta
|
49
|
-
@meta ||= body[
|
53
|
+
@meta ||= body["meta"]
|
50
54
|
end
|
51
55
|
|
52
56
|
def body
|
data/lib/fastly_nsq/messenger.rb
CHANGED
@@ -10,7 +10,7 @@
|
|
10
10
|
# meta: metadata_hash,
|
11
11
|
# )
|
12
12
|
module FastlyNsq::Messenger
|
13
|
-
DEFAULT_ORIGIN =
|
13
|
+
DEFAULT_ORIGIN = "Unknown"
|
14
14
|
@originating_service = DEFAULT_ORIGIN
|
15
15
|
|
16
16
|
module_function
|
@@ -18,23 +18,26 @@ module FastlyNsq::Messenger
|
|
18
18
|
##
|
19
19
|
# Deliver an NSQ message. Uses +pub+
|
20
20
|
#
|
21
|
-
#
|
22
|
-
# +originating_service+ which defaults to {FastlyNsq#originating_service}
|
23
|
-
# +sent_at+ which will be set to +Time.now.iso8601(5)+
|
21
|
+
# Adds keys to the `+meta+:
|
22
|
+
# +originating_service+ which defaults to {FastlyNsq#originating_service}.
|
23
|
+
# +sent_at+ which will be set to +Time.now.iso8601(5)+ if the +sent_at+ param is nil OR
|
24
|
+
# if the passed +sent_at+ is not a valid timestamp.
|
24
25
|
# @param message [#to_json(*)] written to the +data+ key of the NSQ message payload
|
25
26
|
# @param topic [String] NSQ topic on which to deliver the message
|
26
27
|
# @param originating_service [String] added to meta key of message payload
|
28
|
+
# @param sent_at [Time] Timestamp that will be added to the meta payload
|
27
29
|
# @param meta [Hash]
|
28
30
|
# @return [Void]
|
29
31
|
# @example
|
30
32
|
# FastlyNsq::Messenger.deliver(
|
31
33
|
# message: {a: 1, count: 123},
|
32
34
|
# topic: 'count',
|
35
|
+
# meta: { sent_at: Time.now }
|
33
36
|
# )
|
34
|
-
def deliver(message:, topic:, originating_service: nil, meta: {})
|
37
|
+
def deliver(message:, topic:, originating_service: nil, sent_at: nil, meta: {})
|
35
38
|
payload = {
|
36
39
|
data: message,
|
37
|
-
meta: populate_meta(originating_service: originating_service, meta: meta)
|
40
|
+
meta: populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
38
41
|
}
|
39
42
|
|
40
43
|
deliver_payload(topic: topic, payload: payload.to_json)
|
@@ -43,15 +46,16 @@ module FastlyNsq::Messenger
|
|
43
46
|
##
|
44
47
|
# Deliver many NSQ messages at once. Uses +mpub+
|
45
48
|
#
|
46
|
-
# For each message will add two keys to the `+meta+ payload of each message
|
47
|
-
#
|
48
|
-
# +
|
49
|
-
# +
|
49
|
+
# For each message will add two keys to the `+meta+ payload of each message:
|
50
|
+
# +originating_service+ which defaults to {FastlyNsq#originating_service}
|
51
|
+
# +sent_at+ which will be set to +Time.now.iso8601(5)+ when messages are processed if not included
|
52
|
+
# in the +meta+ param OR if the pased +sent_at+ is not a valid timestamp.
|
50
53
|
# The +sent_at+ time and +originating_service+ will be the same for every message.
|
51
54
|
# @param messages [Array] Array of message which will be written to +data+ key of the
|
52
55
|
# individual NSQ message payload. Each message needs to respond to +to_json(*)+.
|
53
56
|
# @param topic [String] NSQ topic on which to deliver the message
|
54
57
|
# @param originating_service [String] added to meta key of message payload
|
58
|
+
# @param sent_at [Time] Timestamp that will be added to the meta payload
|
55
59
|
# @param meta [Hash]
|
56
60
|
# @return [Void]
|
57
61
|
# @example
|
@@ -59,13 +63,13 @@ module FastlyNsq::Messenger
|
|
59
63
|
# messages: [{a: 1, count: 11}, {a: 2, count: 22}],
|
60
64
|
# topic: 'counts',
|
61
65
|
# )
|
62
|
-
def deliver_multi(messages:, topic:, originating_service: nil, meta: {})
|
63
|
-
meta = populate_meta(originating_service: originating_service, meta: meta)
|
66
|
+
def deliver_multi(messages:, topic:, originating_service: nil, sent_at: nil, meta: {})
|
67
|
+
meta = populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
64
68
|
|
65
69
|
payload = messages.each_with_object([]) do |message, a|
|
66
70
|
msg = {
|
67
71
|
data: message,
|
68
|
-
meta: meta
|
72
|
+
meta: meta
|
69
73
|
}
|
70
74
|
|
71
75
|
a << msg.to_json
|
@@ -121,9 +125,15 @@ module FastlyNsq::Messenger
|
|
121
125
|
producer_for(topic: topic) { |producer| producer.write payload }
|
122
126
|
end
|
123
127
|
|
124
|
-
def populate_meta(originating_service: nil, meta: {})
|
128
|
+
def populate_meta(originating_service: nil, sent_at: nil, meta: {})
|
125
129
|
meta[:originating_service] = originating_service || self.originating_service
|
126
|
-
|
130
|
+
|
131
|
+
meta[:sent_at] = if sent_at && sent_at.respond_to?(:iso8601)
|
132
|
+
sent_at.iso8601(5)
|
133
|
+
else
|
134
|
+
Time.now.iso8601(5)
|
135
|
+
end
|
136
|
+
|
127
137
|
meta
|
128
138
|
end
|
129
139
|
end
|
data/lib/fastly_nsq/new_relic.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
5
|
-
rescue LoadError
|
4
|
+
require "newrelic_rpm"
|
5
|
+
rescue LoadError
|
6
6
|
end
|
7
7
|
|
8
8
|
##
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
class FastlyNsq::NewRelic
|
12
12
|
include NewRelic::Agent::Instrumentation::ControllerInstrumentation if defined?(::NewRelic)
|
13
13
|
|
14
|
-
CATEGORY =
|
14
|
+
CATEGORY = "OtherTransaction/FastlyNsqProcessor"
|
15
15
|
|
16
16
|
attr_reader :agent
|
17
17
|
|
@@ -22,14 +22,14 @@ class FastlyNsq::NewRelic
|
|
22
22
|
# tracer = FastlyNsq::NewRelic.new
|
23
23
|
# tracer.notice_error(exception)
|
24
24
|
def initialize(agent = nil)
|
25
|
-
@agent = agent || (Object.const_defined?(
|
25
|
+
@agent = agent || (Object.const_defined?(:NewRelic) ? NewRelic::Agent : nil)
|
26
26
|
end
|
27
27
|
|
28
28
|
##
|
29
29
|
# Returns true if NewRelic is loaded and available.
|
30
30
|
# @return [Boolean]
|
31
31
|
def enabled?
|
32
|
-
@enabled ||= Object.const_defined?(
|
32
|
+
@enabled ||= Object.const_defined?(:NewRelic)
|
33
33
|
end
|
34
34
|
|
35
35
|
##
|
@@ -49,7 +49,7 @@ class FastlyNsq::NewRelic
|
|
49
49
|
# @see {https://www.rubydoc.info/github/newrelic/rpm/NewRelic%2FAgent%2FInstrumentation%2FControllerInstrumentation:perform_action_with_newrelic_trace}
|
50
50
|
def trace_with_newrelic(**args)
|
51
51
|
if enabled?
|
52
|
-
perform_action_with_newrelic_trace(trace_args(args)) do
|
52
|
+
perform_action_with_newrelic_trace(trace_args(**args)) do
|
53
53
|
yield
|
54
54
|
end
|
55
55
|
else
|
@@ -61,8 +61,8 @@ class FastlyNsq::NewRelic
|
|
61
61
|
|
62
62
|
def trace_args(**args)
|
63
63
|
{
|
64
|
-
name:
|
65
|
-
category: CATEGORY
|
64
|
+
name: "call",
|
65
|
+
category: CATEGORY
|
66
66
|
}.merge(args)
|
67
67
|
end
|
68
68
|
end
|