fastly_nsq 1.16.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.env +2 -2
- data/.git-blame-ignore-revs +6 -0
- data/.ruby-version +1 -1
- data/.travis.yml +4 -3
- data/ChangeLog.md +31 -1
- data/Gemfile +8 -8
- data/README.md +84 -6
- data/Rakefile +10 -11
- data/fastly_nsq.gemspec +26 -26
- data/lib/fastly_nsq/cli.rb +43 -50
- data/lib/fastly_nsq/consumer.rb +27 -14
- data/lib/fastly_nsq/feeder.rb +5 -7
- data/lib/fastly_nsq/http/nsqd.rb +28 -28
- data/lib/fastly_nsq/http/nsqlookupd.rb +11 -11
- data/lib/fastly_nsq/http.rb +4 -4
- data/lib/fastly_nsq/launcher.rb +16 -16
- data/lib/fastly_nsq/listener.rb +16 -16
- data/lib/fastly_nsq/manager.rb +13 -12
- data/lib/fastly_nsq/message.rb +4 -4
- data/lib/fastly_nsq/messenger.rb +25 -15
- data/lib/fastly_nsq/new_relic.rb +8 -8
- data/lib/fastly_nsq/priority_queue.rb +2 -2
- data/lib/fastly_nsq/priority_thread_pool.rb +3 -3
- data/lib/fastly_nsq/producer.rb +23 -14
- data/lib/fastly_nsq/safe_thread.rb +1 -1
- data/lib/fastly_nsq/testing.rb +4 -3
- data/lib/fastly_nsq/tls_options.rb +6 -6
- data/lib/fastly_nsq/version.rb +1 -1
- data/lib/fastly_nsq.rb +64 -29
- data/spec/cli_spec.rb +2 -2
- data/spec/consumer_spec.rb +53 -12
- data/spec/fastly_nsq_spec.rb +108 -32
- data/spec/feeder_spec.rb +4 -4
- data/spec/http/nsqd_spec.rb +23 -23
- data/spec/http/nsqlookupd_spec.rb +19 -19
- data/spec/http_spec.rb +22 -22
- data/spec/integration_spec.rb +10 -10
- data/spec/launcher_spec.rb +21 -21
- data/spec/listener_spec.rb +50 -50
- data/spec/manager_spec.rb +27 -27
- data/spec/matchers/delegate.rb +4 -4
- data/spec/message_spec.rb +19 -19
- data/spec/messenger_spec.rb +71 -59
- data/spec/new_relic.rb +27 -27
- data/spec/priority_thread_pool_spec.rb +2 -2
- data/spec/producer_spec.rb +70 -31
- data/spec/spec_helper.rb +12 -12
- data/spec/support/http.rb +2 -2
- data/spec/support/webmock.rb +1 -1
- data/spec/testing_spec.rb +12 -12
- data/spec/tls_options_spec.rb +47 -47
- metadata +10 -11
- data/.rubocop.yml +0 -68
data/lib/fastly_nsq/http/nsqd.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/http"
|
4
4
|
|
5
5
|
class FastlyNsq::Http
|
6
6
|
##
|
@@ -13,11 +13,11 @@ class FastlyNsq::Http
|
|
13
13
|
def_delegator :client, :get
|
14
14
|
def_delegator :client, :post
|
15
15
|
|
16
|
-
BASE_NSQD_URL = ENV.fetch(
|
17
|
-
if ENV[
|
18
|
-
"https://#{ENV.fetch(
|
16
|
+
BASE_NSQD_URL = ENV.fetch("NSQD_URL") do
|
17
|
+
if ENV["NSQD_HTTPS_ADDRESS"]
|
18
|
+
"https://#{ENV.fetch("NSQD_HTTPS_ADDRESS")}"
|
19
19
|
else
|
20
|
-
"http://#{ENV.fetch(
|
20
|
+
"http://#{ENV.fetch("NSQD_HTTP_ADDRESS")}"
|
21
21
|
end
|
22
22
|
end
|
23
23
|
VALID_FORMATS = %w[text json].freeze
|
@@ -27,13 +27,13 @@ class FastlyNsq::Http
|
|
27
27
|
#
|
28
28
|
# NOTE: The only "unhealthy" state is if nsqd failed to write messages to disk when overflow occurred.
|
29
29
|
def self.ping(**args)
|
30
|
-
new(request_uri:
|
30
|
+
new(request_uri: "/ping", **args).get
|
31
31
|
end
|
32
32
|
|
33
33
|
##
|
34
34
|
# NSQ version information
|
35
35
|
def self.info(**args)
|
36
|
-
new(request_uri:
|
36
|
+
new(request_uri: "/info", **args).get
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
@@ -47,12 +47,12 @@ class FastlyNsq::Http
|
|
47
47
|
#
|
48
48
|
# @example Fetch Statistics for topic: 'foo', channel: 'bar' as text
|
49
49
|
# Nsqd.stats(topic: 'foo', channel: 'bar', format: 'text')
|
50
|
-
def self.stats(topic: nil, channel: nil, format:
|
50
|
+
def self.stats(topic: nil, channel: nil, format: "json", **args)
|
51
51
|
raise InvalidFormatError unless VALID_FORMATS.include?(format)
|
52
|
-
params = {
|
52
|
+
params = {format: format}
|
53
53
|
params[:topic] = topic if topic
|
54
54
|
params[:channel] = channel if channel
|
55
|
-
new(request_uri:
|
55
|
+
new(request_uri: "/stats", **args).get(params)
|
56
56
|
end
|
57
57
|
|
58
58
|
##
|
@@ -62,9 +62,9 @@ class FastlyNsq::Http
|
|
62
62
|
# @param defer [String] the time in ms to delay message delivery
|
63
63
|
# @param message the message body
|
64
64
|
def self.pub(topic:, message:, defer: nil, **args)
|
65
|
-
params = {
|
65
|
+
params = {topic: topic}
|
66
66
|
params[:defer] = defer if defer
|
67
|
-
new(request_uri:
|
67
|
+
new(request_uri: "/pub", **args).post(params, message)
|
68
68
|
end
|
69
69
|
|
70
70
|
##
|
@@ -83,17 +83,17 @@ class FastlyNsq::Http
|
|
83
83
|
# @param topic [String] the topic to publish to
|
84
84
|
# @param binary [Boolean] enables binary mode
|
85
85
|
# @param message the messages to send with \n used to seperate messages
|
86
|
-
def self.mpub(topic:, binary: false,
|
87
|
-
binary_param = binary ?
|
88
|
-
raise NotImplementedError,
|
89
|
-
params = {
|
90
|
-
new(request_uri:
|
86
|
+
def self.mpub(topic:, message:, binary: false, **args)
|
87
|
+
binary_param = binary ? "true" : "false"
|
88
|
+
raise NotImplementedError, "binary mode has yet to be implemented" if binary
|
89
|
+
params = {topic: topic, binary: binary_param}
|
90
|
+
new(request_uri: "/mpub", **args).post(params, message)
|
91
91
|
end
|
92
92
|
|
93
93
|
##
|
94
94
|
# List of nsqlookupd TCP addresses
|
95
95
|
def self.config_nsqlookupd_tcp_addresses(**args)
|
96
|
-
new(request_uri:
|
96
|
+
new(request_uri: "/config/nsqlookupd_tcp_addresses", **args).get
|
97
97
|
end
|
98
98
|
|
99
99
|
##
|
@@ -101,7 +101,7 @@ class FastlyNsq::Http
|
|
101
101
|
#
|
102
102
|
# @param topic [String] the topic to create
|
103
103
|
def self.topic_create(topic:, **args)
|
104
|
-
new(request_uri:
|
104
|
+
new(request_uri: "/topic/create", **args).post(topic: topic)
|
105
105
|
end
|
106
106
|
|
107
107
|
##
|
@@ -109,7 +109,7 @@ class FastlyNsq::Http
|
|
109
109
|
#
|
110
110
|
# @param topic [String] the existing topic to delete
|
111
111
|
def self.topic_delete(topic:, **args)
|
112
|
-
new(request_uri:
|
112
|
+
new(request_uri: "/topic/delete", **args).post(topic: topic)
|
113
113
|
end
|
114
114
|
|
115
115
|
##
|
@@ -117,7 +117,7 @@ class FastlyNsq::Http
|
|
117
117
|
#
|
118
118
|
# @param topic [String] the existing topic to empty
|
119
119
|
def self.topic_empty(topic:, **args)
|
120
|
-
new(request_uri:
|
120
|
+
new(request_uri: "/topic/empty", **args).post(topic: topic)
|
121
121
|
end
|
122
122
|
|
123
123
|
##
|
@@ -126,7 +126,7 @@ class FastlyNsq::Http
|
|
126
126
|
#
|
127
127
|
# @param topic [String] the existing topic to pause
|
128
128
|
def self.topic_pause(topic:, **args)
|
129
|
-
new(request_uri:
|
129
|
+
new(request_uri: "/topic/pause", **args).post(topic: topic)
|
130
130
|
end
|
131
131
|
|
132
132
|
##
|
@@ -134,7 +134,7 @@ class FastlyNsq::Http
|
|
134
134
|
#
|
135
135
|
# @param topic [String] the existing, paused topic to unpause
|
136
136
|
def self.topic_unpause(topic:, **args)
|
137
|
-
new(request_uri:
|
137
|
+
new(request_uri: "/topic/unpause", **args).post(topic: topic)
|
138
138
|
end
|
139
139
|
|
140
140
|
##
|
@@ -143,7 +143,7 @@ class FastlyNsq::Http
|
|
143
143
|
# @param topic [String] the existing topic
|
144
144
|
# @param channel [String] the channel to create
|
145
145
|
def self.channel_create(topic:, channel:, **args)
|
146
|
-
new(request_uri:
|
146
|
+
new(request_uri: "/channel/create", **args).post(topic: topic, channel: channel)
|
147
147
|
end
|
148
148
|
|
149
149
|
##
|
@@ -152,7 +152,7 @@ class FastlyNsq::Http
|
|
152
152
|
# @param topic [String] the existing topic
|
153
153
|
# @param channel [String] the channel to delete
|
154
154
|
def self.channel_delete(topic:, channel:, **args)
|
155
|
-
new(request_uri:
|
155
|
+
new(request_uri: "/channel/delete", **args).post(topic: topic, channel: channel)
|
156
156
|
end
|
157
157
|
|
158
158
|
##
|
@@ -161,7 +161,7 @@ class FastlyNsq::Http
|
|
161
161
|
# @param topic [String] the existing topic
|
162
162
|
# @param channel [String] the channel to empty
|
163
163
|
def self.channel_empty(topic:, channel:, **args)
|
164
|
-
new(request_uri:
|
164
|
+
new(request_uri: "/channel/empty", **args).post(topic: topic, channel: channel)
|
165
165
|
end
|
166
166
|
|
167
167
|
##
|
@@ -171,7 +171,7 @@ class FastlyNsq::Http
|
|
171
171
|
# @param topic [String] the existing topic
|
172
172
|
# @param channel [String] the channel to pause
|
173
173
|
def self.channel_pause(topic:, channel:, **args)
|
174
|
-
new(request_uri:
|
174
|
+
new(request_uri: "/channel/pause", **args).post(topic: topic, channel: channel)
|
175
175
|
end
|
176
176
|
|
177
177
|
##
|
@@ -180,7 +180,7 @@ class FastlyNsq::Http
|
|
180
180
|
# @param topic [String] the existing topic
|
181
181
|
# @param channel [String] the existing, paused, channel to unpause
|
182
182
|
def self.channel_unpause(topic:, channel:, **args)
|
183
|
-
new(request_uri:
|
183
|
+
new(request_uri: "/channel/unpause", **args).post(topic: topic, channel: channel)
|
184
184
|
end
|
185
185
|
|
186
186
|
##
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/http"
|
4
4
|
|
5
5
|
class FastlyNsq::Http
|
6
6
|
##
|
@@ -12,20 +12,20 @@ class FastlyNsq::Http
|
|
12
12
|
extend Forwardable
|
13
13
|
def_delegator :client, :get
|
14
14
|
|
15
|
-
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch(
|
15
|
+
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch("NSQLOOKUPD_HTTP_ADDRESS", "").split(",")[0]}"
|
16
16
|
|
17
17
|
##
|
18
18
|
# List of producers for a given topic
|
19
19
|
#
|
20
20
|
# @param topic [String] the topic for which to list producers
|
21
21
|
def self.lookup(topic:, **args)
|
22
|
-
new(request_uri:
|
22
|
+
new(request_uri: "/lookup", **args).get(topic: topic)
|
23
23
|
end
|
24
24
|
|
25
25
|
##
|
26
26
|
# List of all known topics
|
27
27
|
def self.topics(**args)
|
28
|
-
new(request_uri:
|
28
|
+
new(request_uri: "/topics", **args).get
|
29
29
|
end
|
30
30
|
|
31
31
|
##
|
@@ -33,13 +33,13 @@ class FastlyNsq::Http
|
|
33
33
|
#
|
34
34
|
# @param topic [String] the topic for which to list channels
|
35
35
|
def self.channels(topic:, **args)
|
36
|
-
new(request_uri:
|
36
|
+
new(request_uri: "/channels", **args).get(topic: topic)
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
40
40
|
# List all known nsqd nodes
|
41
41
|
def self.nodes(**args)
|
42
|
-
new(request_uri:
|
42
|
+
new(request_uri: "/nodes", **args).get
|
43
43
|
end
|
44
44
|
|
45
45
|
##
|
@@ -47,7 +47,7 @@ class FastlyNsq::Http
|
|
47
47
|
#
|
48
48
|
# @param topic [String] the exsiting topic to delete
|
49
49
|
def self.delete_topic(topic:, **args)
|
50
|
-
new(request_uri:
|
50
|
+
new(request_uri: "/delete_topic", **args).get(topic: topic)
|
51
51
|
end
|
52
52
|
|
53
53
|
##
|
@@ -56,7 +56,7 @@ class FastlyNsq::Http
|
|
56
56
|
# @param topic [String] an exsiting topic
|
57
57
|
# @param channel [String] the exsiting channel to delete
|
58
58
|
def self.delete_channel(topic:, channel:, **args)
|
59
|
-
new(request_uri:
|
59
|
+
new(request_uri: "/delete_channel", **args).get(topic: topic, channel: channel)
|
60
60
|
end
|
61
61
|
|
62
62
|
##
|
@@ -67,19 +67,19 @@ class FastlyNsq::Http
|
|
67
67
|
# @param topic [String] the existing topic
|
68
68
|
# @param node [String] the producer (nsqd) to tombstone (identified by <broadcast_address>:<http_port>)
|
69
69
|
def self.tombstone_topic_producer(topic:, node:, **args)
|
70
|
-
new(request_uri:
|
70
|
+
new(request_uri: "/tombstone_topic_producer", **args).get(topic: topic, node: node)
|
71
71
|
end
|
72
72
|
|
73
73
|
##
|
74
74
|
# Monitoring endpoint, should return +OK+
|
75
75
|
def self.ping(**args)
|
76
|
-
new(request_uri:
|
76
|
+
new(request_uri: "/ping", **args).get
|
77
77
|
end
|
78
78
|
|
79
79
|
##
|
80
80
|
# Returns nsqlookupd version information
|
81
81
|
def self.info(**args)
|
82
|
-
new(request_uri:
|
82
|
+
new(request_uri: "/info", **args).get
|
83
83
|
end
|
84
84
|
|
85
85
|
##
|
data/lib/fastly_nsq/http.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "net/https"
|
4
|
+
require "fastly_nsq/http/nsqd"
|
5
|
+
require "fastly_nsq/http/nsqlookupd"
|
6
6
|
|
7
7
|
##
|
8
8
|
# Adapter class for HTTP requests to NSQD
|
@@ -15,7 +15,7 @@ require 'fastly_nsq/http/nsqlookupd'
|
|
15
15
|
# @see FastlyNsq::Http::Nsqd
|
16
16
|
# @see FastlyNsq::Http::Nsqlookupd
|
17
17
|
class FastlyNsq::Http
|
18
|
-
def initialize(uri:, cert_filename: ENV[
|
18
|
+
def initialize(uri:, cert_filename: ENV["NSQ_SSL_CERTIFICATE"], key_filename: ENV["NSQ_SSL_KEY"])
|
19
19
|
@uri = uri.is_a?(URI) ? uri : URI.parse(uri)
|
20
20
|
@cert_filename = cert_filename
|
21
21
|
@key_filename = key_filename
|
data/lib/fastly_nsq/launcher.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/safe_thread"
|
4
4
|
|
5
5
|
##
|
6
6
|
# FastlyNsq::Launcher is a lighweight wrapper of a thread manager
|
@@ -17,18 +17,18 @@ class FastlyNsq::Launcher
|
|
17
17
|
FastlyNsq.manager
|
18
18
|
end
|
19
19
|
|
20
|
-
def initialize(
|
21
|
-
@done
|
22
|
-
@timeout = timeout
|
23
|
-
@pulse
|
24
|
-
@logger
|
20
|
+
def initialize(**options)
|
21
|
+
@done = false
|
22
|
+
@timeout = options[:timeout] || 5
|
23
|
+
@pulse = options[:pulse] || 5
|
24
|
+
@logger = options[:logger] || FastlyNsq.logger
|
25
25
|
|
26
|
-
FastlyNsq.manager = FastlyNsq::Manager.new(options)
|
26
|
+
FastlyNsq.manager = FastlyNsq::Manager.new(**options)
|
27
27
|
FastlyNsq.fire_event :startup
|
28
28
|
end
|
29
29
|
|
30
30
|
def beat
|
31
|
-
@heartbeat ||= safe_thread(
|
31
|
+
@heartbeat ||= safe_thread("heartbeat", &method(:start_heartbeat))
|
32
32
|
end
|
33
33
|
|
34
34
|
def stop
|
@@ -51,13 +51,13 @@ class FastlyNsq::Launcher
|
|
51
51
|
def heartbeat
|
52
52
|
logger.debug do
|
53
53
|
[
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
].join(
|
54
|
+
"HEARTBEAT:",
|
55
|
+
"busy:", manager.pool.length,
|
56
|
+
"processed:", manager.pool.completed_task_count,
|
57
|
+
"max_threads:", manager.pool.max_length,
|
58
|
+
"max_queue_size:", manager.pool.largest_length,
|
59
|
+
"listeners:", manager.listeners.count
|
60
|
+
].join(" ")
|
61
61
|
end
|
62
62
|
|
63
63
|
# TODO: Check the health of the system overall and kill it if needed
|
@@ -73,6 +73,6 @@ class FastlyNsq::Launcher
|
|
73
73
|
heartbeat
|
74
74
|
sleep pulse
|
75
75
|
end
|
76
|
-
logger.info(
|
76
|
+
logger.info("Heartbeat stopping...")
|
77
77
|
end
|
78
78
|
end
|
data/lib/fastly_nsq/listener.rb
CHANGED
@@ -77,25 +77,25 @@ class FastlyNsq::Listener
|
|
77
77
|
# max_attempts: 15,
|
78
78
|
# )
|
79
79
|
def initialize(topic:, processor:, preprocessor: FastlyNsq.preprocessor, channel: FastlyNsq.channel, consumer: nil,
|
80
|
-
|
81
|
-
|
80
|
+
logger: FastlyNsq.logger, priority: DEFAULT_PRIORITY, connect_timeout: DEFAULT_CONNECTION_TIMEOUT,
|
81
|
+
max_attempts: FastlyNsq.max_attempts, **consumer_options)
|
82
82
|
|
83
83
|
raise ArgumentError, "processor #{processor.inspect} does not respond to #call" unless processor.respond_to?(:call)
|
84
84
|
raise ArgumentError, "priority #{priority.inspect} must be a Integer" unless priority.is_a?(Integer)
|
85
85
|
|
86
|
-
@channel
|
87
|
-
@logger
|
86
|
+
@channel = channel
|
87
|
+
@logger = logger
|
88
88
|
@max_attempts = max_attempts
|
89
89
|
@preprocessor = preprocessor
|
90
|
-
@priority
|
91
|
-
@processor
|
92
|
-
@topic
|
90
|
+
@priority = priority
|
91
|
+
@processor = processor
|
92
|
+
@topic = topic
|
93
93
|
|
94
94
|
@consumer = consumer || FastlyNsq::Consumer.new(topic: topic,
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
95
|
+
connect_timeout: connect_timeout,
|
96
|
+
channel: channel,
|
97
|
+
queue: FastlyNsq::Feeder.new(self, priority),
|
98
|
+
max_attempts: max_attempts,
|
99
99
|
**consumer_options)
|
100
100
|
|
101
101
|
FastlyNsq.manager.add_listener(self)
|
@@ -114,12 +114,12 @@ class FastlyNsq::Listener
|
|
114
114
|
message = FastlyNsq::Message.new(nsq_message)
|
115
115
|
|
116
116
|
msg_info = {
|
117
|
-
channel:
|
118
|
-
topic:
|
117
|
+
channel: channel,
|
118
|
+
topic: topic,
|
119
119
|
attempts: message.attempts,
|
120
|
-
id:
|
121
|
-
nsq_id:
|
122
|
-
metadata: message.meta
|
120
|
+
id: Digest::MD5.hexdigest(nsq_message.body.to_s),
|
121
|
+
nsq_id: message.id,
|
122
|
+
metadata: message.meta
|
123
123
|
}
|
124
124
|
|
125
125
|
logger.info do
|
data/lib/fastly_nsq/manager.rb
CHANGED
@@ -17,14 +17,15 @@ class FastlyNsq::Manager
|
|
17
17
|
##
|
18
18
|
# Create a FastlyNsq::Manager
|
19
19
|
#
|
20
|
-
# @param
|
21
|
-
#
|
22
|
-
#
|
23
|
-
def initialize(logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads
|
24
|
-
@done
|
25
|
-
@logger
|
26
|
-
|
27
|
-
|
20
|
+
# @param opts [Hash] Set of options passed to FastlyNsqw::PriorityThreadPool. valid options include:
|
21
|
+
# * max_threads [Integer] Maxiumum number of threads to be used by {FastlyNsq::PriorityThreadPool}
|
22
|
+
# * logger [Logger]
|
23
|
+
def initialize(**opts) # logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads)
|
24
|
+
@done = false
|
25
|
+
@logger = opts[:logger] || FastlyNsq.logger
|
26
|
+
max_threads = opts[:max_threads] || FastlyNsq.max_processing_pool_threads
|
27
|
+
@pool = FastlyNsq::PriorityThreadPool.new(
|
28
|
+
{fallback_policy: :caller_runs, max_threads: max_threads}.merge(opts)
|
28
29
|
)
|
29
30
|
end
|
30
31
|
|
@@ -99,7 +100,7 @@ class FastlyNsq::Manager
|
|
99
100
|
##
|
100
101
|
# Terminate all listeners
|
101
102
|
def stop_listeners
|
102
|
-
logger.info {
|
103
|
+
logger.info { "Stopping listeners" }
|
103
104
|
listeners.each(&:terminate)
|
104
105
|
topic_listeners.clear
|
105
106
|
end
|
@@ -110,13 +111,13 @@ class FastlyNsq::Manager
|
|
110
111
|
# Shutdown the pool
|
111
112
|
# @param deadline [Integer] Number of seconds to wait for pool to stop processing
|
112
113
|
def stop_processing(deadline)
|
113
|
-
logger.info {
|
114
|
+
logger.info { "Stopping processors" }
|
114
115
|
pool.shutdown
|
115
116
|
|
116
|
-
logger.info {
|
117
|
+
logger.info { "Waiting for processors to finish..." }
|
117
118
|
return if pool.wait_for_termination(deadline)
|
118
119
|
|
119
|
-
logger.info {
|
120
|
+
logger.info { "Killing processors..." }
|
120
121
|
pool.kill
|
121
122
|
end
|
122
123
|
end
|
data/lib/fastly_nsq/message.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "json"
|
4
4
|
|
5
5
|
##
|
6
6
|
# Adapter to Nsq::Message. Provides convenience methods for interacting
|
@@ -36,7 +36,7 @@ class FastlyNsq::Message
|
|
36
36
|
# @return [String] Nsq::Message body
|
37
37
|
attr_reader :raw_body
|
38
38
|
|
39
|
-
|
39
|
+
alias_method :to_s, :raw_body
|
40
40
|
|
41
41
|
##
|
42
42
|
# @param nsq_message [Nsq::Message]
|
@@ -46,11 +46,11 @@ class FastlyNsq::Message
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def data
|
49
|
-
@data ||= body[
|
49
|
+
@data ||= body["data"]
|
50
50
|
end
|
51
51
|
|
52
52
|
def meta
|
53
|
-
@meta ||= body[
|
53
|
+
@meta ||= body["meta"]
|
54
54
|
end
|
55
55
|
|
56
56
|
def body
|
data/lib/fastly_nsq/messenger.rb
CHANGED
@@ -10,7 +10,7 @@
|
|
10
10
|
# meta: metadata_hash,
|
11
11
|
# )
|
12
12
|
module FastlyNsq::Messenger
|
13
|
-
DEFAULT_ORIGIN =
|
13
|
+
DEFAULT_ORIGIN = "Unknown"
|
14
14
|
@originating_service = DEFAULT_ORIGIN
|
15
15
|
|
16
16
|
module_function
|
@@ -18,23 +18,26 @@ module FastlyNsq::Messenger
|
|
18
18
|
##
|
19
19
|
# Deliver an NSQ message. Uses +pub+
|
20
20
|
#
|
21
|
-
#
|
22
|
-
# +originating_service+ which defaults to {FastlyNsq#originating_service}
|
23
|
-
# +sent_at+ which will be set to +Time.now.iso8601(5)+
|
21
|
+
# Adds keys to the `+meta+:
|
22
|
+
# +originating_service+ which defaults to {FastlyNsq#originating_service}.
|
23
|
+
# +sent_at+ which will be set to +Time.now.iso8601(5)+ if the +sent_at+ param is nil OR
|
24
|
+
# if the passed +sent_at+ is not a valid timestamp.
|
24
25
|
# @param message [#to_json(*)] written to the +data+ key of the NSQ message payload
|
25
26
|
# @param topic [String] NSQ topic on which to deliver the message
|
26
27
|
# @param originating_service [String] added to meta key of message payload
|
28
|
+
# @param sent_at [Time] Timestamp that will be added to the meta payload
|
27
29
|
# @param meta [Hash]
|
28
30
|
# @return [Void]
|
29
31
|
# @example
|
30
32
|
# FastlyNsq::Messenger.deliver(
|
31
33
|
# message: {a: 1, count: 123},
|
32
34
|
# topic: 'count',
|
35
|
+
# meta: { sent_at: Time.now }
|
33
36
|
# )
|
34
|
-
def deliver(message:, topic:, originating_service: nil, meta: {})
|
37
|
+
def deliver(message:, topic:, originating_service: nil, sent_at: nil, meta: {})
|
35
38
|
payload = {
|
36
39
|
data: message,
|
37
|
-
meta: populate_meta(originating_service: originating_service, meta: meta)
|
40
|
+
meta: populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
38
41
|
}
|
39
42
|
|
40
43
|
deliver_payload(topic: topic, payload: payload.to_json)
|
@@ -43,15 +46,16 @@ module FastlyNsq::Messenger
|
|
43
46
|
##
|
44
47
|
# Deliver many NSQ messages at once. Uses +mpub+
|
45
48
|
#
|
46
|
-
# For each message will add two keys to the `+meta+ payload of each message
|
47
|
-
#
|
48
|
-
# +
|
49
|
-
# +
|
49
|
+
# For each message will add two keys to the `+meta+ payload of each message:
|
50
|
+
# +originating_service+ which defaults to {FastlyNsq#originating_service}
|
51
|
+
# +sent_at+ which will be set to +Time.now.iso8601(5)+ when messages are processed if not included
|
52
|
+
# in the +meta+ param OR if the pased +sent_at+ is not a valid timestamp.
|
50
53
|
# The +sent_at+ time and +originating_service+ will be the same for every message.
|
51
54
|
# @param messages [Array] Array of message which will be written to +data+ key of the
|
52
55
|
# individual NSQ message payload. Each message needs to respond to +to_json(*)+.
|
53
56
|
# @param topic [String] NSQ topic on which to deliver the message
|
54
57
|
# @param originating_service [String] added to meta key of message payload
|
58
|
+
# @param sent_at [Time] Timestamp that will be added to the meta payload
|
55
59
|
# @param meta [Hash]
|
56
60
|
# @return [Void]
|
57
61
|
# @example
|
@@ -59,13 +63,13 @@ module FastlyNsq::Messenger
|
|
59
63
|
# messages: [{a: 1, count: 11}, {a: 2, count: 22}],
|
60
64
|
# topic: 'counts',
|
61
65
|
# )
|
62
|
-
def deliver_multi(messages:, topic:, originating_service: nil, meta: {})
|
63
|
-
meta = populate_meta(originating_service: originating_service, meta: meta)
|
66
|
+
def deliver_multi(messages:, topic:, originating_service: nil, sent_at: nil, meta: {})
|
67
|
+
meta = populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
64
68
|
|
65
69
|
payload = messages.each_with_object([]) do |message, a|
|
66
70
|
msg = {
|
67
71
|
data: message,
|
68
|
-
meta: meta
|
72
|
+
meta: meta
|
69
73
|
}
|
70
74
|
|
71
75
|
a << msg.to_json
|
@@ -121,9 +125,15 @@ module FastlyNsq::Messenger
|
|
121
125
|
producer_for(topic: topic) { |producer| producer.write payload }
|
122
126
|
end
|
123
127
|
|
124
|
-
def populate_meta(originating_service: nil, meta: {})
|
128
|
+
def populate_meta(originating_service: nil, sent_at: nil, meta: {})
|
125
129
|
meta[:originating_service] = originating_service || self.originating_service
|
126
|
-
|
130
|
+
|
131
|
+
meta[:sent_at] = if sent_at && sent_at.respond_to?(:iso8601)
|
132
|
+
sent_at.iso8601(5)
|
133
|
+
else
|
134
|
+
Time.now.iso8601(5)
|
135
|
+
end
|
136
|
+
|
127
137
|
meta
|
128
138
|
end
|
129
139
|
end
|
data/lib/fastly_nsq/new_relic.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
5
|
-
rescue LoadError
|
4
|
+
require "newrelic_rpm"
|
5
|
+
rescue LoadError
|
6
6
|
end
|
7
7
|
|
8
8
|
##
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
class FastlyNsq::NewRelic
|
12
12
|
include NewRelic::Agent::Instrumentation::ControllerInstrumentation if defined?(::NewRelic)
|
13
13
|
|
14
|
-
CATEGORY =
|
14
|
+
CATEGORY = "OtherTransaction/FastlyNsqProcessor"
|
15
15
|
|
16
16
|
attr_reader :agent
|
17
17
|
|
@@ -22,14 +22,14 @@ class FastlyNsq::NewRelic
|
|
22
22
|
# tracer = FastlyNsq::NewRelic.new
|
23
23
|
# tracer.notice_error(exception)
|
24
24
|
def initialize(agent = nil)
|
25
|
-
@agent = agent || (Object.const_defined?(
|
25
|
+
@agent = agent || (Object.const_defined?(:NewRelic) ? NewRelic::Agent : nil)
|
26
26
|
end
|
27
27
|
|
28
28
|
##
|
29
29
|
# Returns true if NewRelic is loaded and available.
|
30
30
|
# @return [Boolean]
|
31
31
|
def enabled?
|
32
|
-
@enabled ||= Object.const_defined?(
|
32
|
+
@enabled ||= Object.const_defined?(:NewRelic)
|
33
33
|
end
|
34
34
|
|
35
35
|
##
|
@@ -49,7 +49,7 @@ class FastlyNsq::NewRelic
|
|
49
49
|
# @see {https://www.rubydoc.info/github/newrelic/rpm/NewRelic%2FAgent%2FInstrumentation%2FControllerInstrumentation:perform_action_with_newrelic_trace}
|
50
50
|
def trace_with_newrelic(**args)
|
51
51
|
if enabled?
|
52
|
-
perform_action_with_newrelic_trace(trace_args(args)) do
|
52
|
+
perform_action_with_newrelic_trace(trace_args(**args)) do
|
53
53
|
yield
|
54
54
|
end
|
55
55
|
else
|
@@ -61,8 +61,8 @@ class FastlyNsq::NewRelic
|
|
61
61
|
|
62
62
|
def trace_args(**args)
|
63
63
|
{
|
64
|
-
name:
|
65
|
-
category: CATEGORY
|
64
|
+
name: "call",
|
65
|
+
category: CATEGORY
|
66
66
|
}.merge(args)
|
67
67
|
end
|
68
68
|
end
|