fastly_nsq 1.17.0 → 1.17.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.git-blame-ignore-revs +6 -0
- data/.ruby-version +1 -1
- data/.travis.yml +4 -2
- data/ChangeLog.md +8 -1
- data/Gemfile +8 -8
- data/README.md +1 -1
- data/Rakefile +10 -11
- data/fastly_nsq.gemspec +26 -26
- data/lib/fastly_nsq/cli.rb +43 -50
- data/lib/fastly_nsq/consumer.rb +6 -6
- data/lib/fastly_nsq/feeder.rb +5 -7
- data/lib/fastly_nsq/http/nsqd.rb +28 -28
- data/lib/fastly_nsq/http/nsqlookupd.rb +11 -11
- data/lib/fastly_nsq/http.rb +4 -4
- data/lib/fastly_nsq/launcher.rb +16 -16
- data/lib/fastly_nsq/listener.rb +16 -16
- data/lib/fastly_nsq/manager.rb +13 -12
- data/lib/fastly_nsq/message.rb +4 -4
- data/lib/fastly_nsq/messenger.rb +7 -7
- data/lib/fastly_nsq/new_relic.rb +8 -8
- data/lib/fastly_nsq/priority_queue.rb +2 -2
- data/lib/fastly_nsq/priority_thread_pool.rb +3 -3
- data/lib/fastly_nsq/producer.rb +7 -7
- data/lib/fastly_nsq/safe_thread.rb +1 -1
- data/lib/fastly_nsq/testing.rb +4 -3
- data/lib/fastly_nsq/tls_options.rb +6 -6
- data/lib/fastly_nsq/version.rb +1 -1
- data/lib/fastly_nsq.rb +27 -29
- data/spec/cli_spec.rb +2 -2
- data/spec/consumer_spec.rb +12 -12
- data/spec/fastly_nsq_spec.rb +31 -31
- data/spec/feeder_spec.rb +4 -4
- data/spec/http/nsqd_spec.rb +23 -23
- data/spec/http/nsqlookupd_spec.rb +19 -19
- data/spec/http_spec.rb +22 -22
- data/spec/integration_spec.rb +10 -10
- data/spec/launcher_spec.rb +21 -21
- data/spec/listener_spec.rb +50 -50
- data/spec/manager_spec.rb +27 -27
- data/spec/matchers/delegate.rb +4 -4
- data/spec/message_spec.rb +19 -19
- data/spec/messenger_spec.rb +63 -64
- data/spec/new_relic.rb +27 -27
- data/spec/priority_thread_pool_spec.rb +2 -2
- data/spec/producer_spec.rb +30 -30
- data/spec/spec_helper.rb +12 -12
- data/spec/support/http.rb +2 -2
- data/spec/support/webmock.rb +1 -1
- data/spec/testing_spec.rb +12 -12
- data/spec/tls_options_spec.rb +47 -47
- metadata +7 -8
- data/.rubocop.yml +0 -68
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/http"
|
4
4
|
|
5
5
|
class FastlyNsq::Http
|
6
6
|
##
|
@@ -12,20 +12,20 @@ class FastlyNsq::Http
|
|
12
12
|
extend Forwardable
|
13
13
|
def_delegator :client, :get
|
14
14
|
|
15
|
-
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch(
|
15
|
+
BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch("NSQLOOKUPD_HTTP_ADDRESS", "").split(",")[0]}"
|
16
16
|
|
17
17
|
##
|
18
18
|
# List of producers for a given topic
|
19
19
|
#
|
20
20
|
# @param topic [String] the topic for which to list producers
|
21
21
|
def self.lookup(topic:, **args)
|
22
|
-
new(request_uri:
|
22
|
+
new(request_uri: "/lookup", **args).get(topic: topic)
|
23
23
|
end
|
24
24
|
|
25
25
|
##
|
26
26
|
# List of all known topics
|
27
27
|
def self.topics(**args)
|
28
|
-
new(request_uri:
|
28
|
+
new(request_uri: "/topics", **args).get
|
29
29
|
end
|
30
30
|
|
31
31
|
##
|
@@ -33,13 +33,13 @@ class FastlyNsq::Http
|
|
33
33
|
#
|
34
34
|
# @param topic [String] the topic for which to list channels
|
35
35
|
def self.channels(topic:, **args)
|
36
|
-
new(request_uri:
|
36
|
+
new(request_uri: "/channels", **args).get(topic: topic)
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
40
40
|
# List all known nsqd nodes
|
41
41
|
def self.nodes(**args)
|
42
|
-
new(request_uri:
|
42
|
+
new(request_uri: "/nodes", **args).get
|
43
43
|
end
|
44
44
|
|
45
45
|
##
|
@@ -47,7 +47,7 @@ class FastlyNsq::Http
|
|
47
47
|
#
|
48
48
|
# @param topic [String] the exsiting topic to delete
|
49
49
|
def self.delete_topic(topic:, **args)
|
50
|
-
new(request_uri:
|
50
|
+
new(request_uri: "/delete_topic", **args).get(topic: topic)
|
51
51
|
end
|
52
52
|
|
53
53
|
##
|
@@ -56,7 +56,7 @@ class FastlyNsq::Http
|
|
56
56
|
# @param topic [String] an exsiting topic
|
57
57
|
# @param channel [String] the exsiting channel to delete
|
58
58
|
def self.delete_channel(topic:, channel:, **args)
|
59
|
-
new(request_uri:
|
59
|
+
new(request_uri: "/delete_channel", **args).get(topic: topic, channel: channel)
|
60
60
|
end
|
61
61
|
|
62
62
|
##
|
@@ -67,19 +67,19 @@ class FastlyNsq::Http
|
|
67
67
|
# @param topic [String] the existing topic
|
68
68
|
# @param node [String] the producer (nsqd) to tombstone (identified by <broadcast_address>:<http_port>)
|
69
69
|
def self.tombstone_topic_producer(topic:, node:, **args)
|
70
|
-
new(request_uri:
|
70
|
+
new(request_uri: "/tombstone_topic_producer", **args).get(topic: topic, node: node)
|
71
71
|
end
|
72
72
|
|
73
73
|
##
|
74
74
|
# Monitoring endpoint, should return +OK+
|
75
75
|
def self.ping(**args)
|
76
|
-
new(request_uri:
|
76
|
+
new(request_uri: "/ping", **args).get
|
77
77
|
end
|
78
78
|
|
79
79
|
##
|
80
80
|
# Returns nsqlookupd version information
|
81
81
|
def self.info(**args)
|
82
|
-
new(request_uri:
|
82
|
+
new(request_uri: "/info", **args).get
|
83
83
|
end
|
84
84
|
|
85
85
|
##
|
data/lib/fastly_nsq/http.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "net/https"
|
4
|
+
require "fastly_nsq/http/nsqd"
|
5
|
+
require "fastly_nsq/http/nsqlookupd"
|
6
6
|
|
7
7
|
##
|
8
8
|
# Adapter class for HTTP requests to NSQD
|
@@ -15,7 +15,7 @@ require 'fastly_nsq/http/nsqlookupd'
|
|
15
15
|
# @see FastlyNsq::Http::Nsqd
|
16
16
|
# @see FastlyNsq::Http::Nsqlookupd
|
17
17
|
class FastlyNsq::Http
|
18
|
-
def initialize(uri:, cert_filename: ENV[
|
18
|
+
def initialize(uri:, cert_filename: ENV["NSQ_SSL_CERTIFICATE"], key_filename: ENV["NSQ_SSL_KEY"])
|
19
19
|
@uri = uri.is_a?(URI) ? uri : URI.parse(uri)
|
20
20
|
@cert_filename = cert_filename
|
21
21
|
@key_filename = key_filename
|
data/lib/fastly_nsq/launcher.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "fastly_nsq/safe_thread"
|
4
4
|
|
5
5
|
##
|
6
6
|
# FastlyNsq::Launcher is a lighweight wrapper of a thread manager
|
@@ -17,18 +17,18 @@ class FastlyNsq::Launcher
|
|
17
17
|
FastlyNsq.manager
|
18
18
|
end
|
19
19
|
|
20
|
-
def initialize(
|
21
|
-
@done
|
22
|
-
@timeout = timeout
|
23
|
-
@pulse
|
24
|
-
@logger
|
20
|
+
def initialize(**options)
|
21
|
+
@done = false
|
22
|
+
@timeout = options[:timeout] || 5
|
23
|
+
@pulse = options[:pulse] || 5
|
24
|
+
@logger = options[:logger] || FastlyNsq.logger
|
25
25
|
|
26
|
-
FastlyNsq.manager = FastlyNsq::Manager.new(options)
|
26
|
+
FastlyNsq.manager = FastlyNsq::Manager.new(**options)
|
27
27
|
FastlyNsq.fire_event :startup
|
28
28
|
end
|
29
29
|
|
30
30
|
def beat
|
31
|
-
@heartbeat ||= safe_thread(
|
31
|
+
@heartbeat ||= safe_thread("heartbeat", &method(:start_heartbeat))
|
32
32
|
end
|
33
33
|
|
34
34
|
def stop
|
@@ -51,13 +51,13 @@ class FastlyNsq::Launcher
|
|
51
51
|
def heartbeat
|
52
52
|
logger.debug do
|
53
53
|
[
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
].join(
|
54
|
+
"HEARTBEAT:",
|
55
|
+
"busy:", manager.pool.length,
|
56
|
+
"processed:", manager.pool.completed_task_count,
|
57
|
+
"max_threads:", manager.pool.max_length,
|
58
|
+
"max_queue_size:", manager.pool.largest_length,
|
59
|
+
"listeners:", manager.listeners.count
|
60
|
+
].join(" ")
|
61
61
|
end
|
62
62
|
|
63
63
|
# TODO: Check the health of the system overall and kill it if needed
|
@@ -73,6 +73,6 @@ class FastlyNsq::Launcher
|
|
73
73
|
heartbeat
|
74
74
|
sleep pulse
|
75
75
|
end
|
76
|
-
logger.info(
|
76
|
+
logger.info("Heartbeat stopping...")
|
77
77
|
end
|
78
78
|
end
|
data/lib/fastly_nsq/listener.rb
CHANGED
@@ -77,25 +77,25 @@ class FastlyNsq::Listener
|
|
77
77
|
# max_attempts: 15,
|
78
78
|
# )
|
79
79
|
def initialize(topic:, processor:, preprocessor: FastlyNsq.preprocessor, channel: FastlyNsq.channel, consumer: nil,
|
80
|
-
|
81
|
-
|
80
|
+
logger: FastlyNsq.logger, priority: DEFAULT_PRIORITY, connect_timeout: DEFAULT_CONNECTION_TIMEOUT,
|
81
|
+
max_attempts: FastlyNsq.max_attempts, **consumer_options)
|
82
82
|
|
83
83
|
raise ArgumentError, "processor #{processor.inspect} does not respond to #call" unless processor.respond_to?(:call)
|
84
84
|
raise ArgumentError, "priority #{priority.inspect} must be a Integer" unless priority.is_a?(Integer)
|
85
85
|
|
86
|
-
@channel
|
87
|
-
@logger
|
86
|
+
@channel = channel
|
87
|
+
@logger = logger
|
88
88
|
@max_attempts = max_attempts
|
89
89
|
@preprocessor = preprocessor
|
90
|
-
@priority
|
91
|
-
@processor
|
92
|
-
@topic
|
90
|
+
@priority = priority
|
91
|
+
@processor = processor
|
92
|
+
@topic = topic
|
93
93
|
|
94
94
|
@consumer = consumer || FastlyNsq::Consumer.new(topic: topic,
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
95
|
+
connect_timeout: connect_timeout,
|
96
|
+
channel: channel,
|
97
|
+
queue: FastlyNsq::Feeder.new(self, priority),
|
98
|
+
max_attempts: max_attempts,
|
99
99
|
**consumer_options)
|
100
100
|
|
101
101
|
FastlyNsq.manager.add_listener(self)
|
@@ -114,12 +114,12 @@ class FastlyNsq::Listener
|
|
114
114
|
message = FastlyNsq::Message.new(nsq_message)
|
115
115
|
|
116
116
|
msg_info = {
|
117
|
-
channel:
|
118
|
-
topic:
|
117
|
+
channel: channel,
|
118
|
+
topic: topic,
|
119
119
|
attempts: message.attempts,
|
120
|
-
id:
|
121
|
-
nsq_id:
|
122
|
-
metadata: message.meta
|
120
|
+
id: Digest::MD5.hexdigest(nsq_message.body.to_s),
|
121
|
+
nsq_id: message.id,
|
122
|
+
metadata: message.meta
|
123
123
|
}
|
124
124
|
|
125
125
|
logger.info do
|
data/lib/fastly_nsq/manager.rb
CHANGED
@@ -17,14 +17,15 @@ class FastlyNsq::Manager
|
|
17
17
|
##
|
18
18
|
# Create a FastlyNsq::Manager
|
19
19
|
#
|
20
|
-
# @param
|
21
|
-
#
|
22
|
-
#
|
23
|
-
def initialize(logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads
|
24
|
-
@done
|
25
|
-
@logger
|
26
|
-
|
27
|
-
|
20
|
+
# @param opts [Hash] Set of options passed to FastlyNsqw::PriorityThreadPool. valid options include:
|
21
|
+
# * max_threads [Integer] Maxiumum number of threads to be used by {FastlyNsq::PriorityThreadPool}
|
22
|
+
# * logger [Logger]
|
23
|
+
def initialize(**opts) # logger: FastlyNsq.logger, max_threads: FastlyNsq.max_processing_pool_threads)
|
24
|
+
@done = false
|
25
|
+
@logger = opts[:logger] || FastlyNsq.logger
|
26
|
+
max_threads = opts[:max_threads] || FastlyNsq.max_processing_pool_threads
|
27
|
+
@pool = FastlyNsq::PriorityThreadPool.new(
|
28
|
+
{fallback_policy: :caller_runs, max_threads: max_threads}.merge(opts)
|
28
29
|
)
|
29
30
|
end
|
30
31
|
|
@@ -99,7 +100,7 @@ class FastlyNsq::Manager
|
|
99
100
|
##
|
100
101
|
# Terminate all listeners
|
101
102
|
def stop_listeners
|
102
|
-
logger.info {
|
103
|
+
logger.info { "Stopping listeners" }
|
103
104
|
listeners.each(&:terminate)
|
104
105
|
topic_listeners.clear
|
105
106
|
end
|
@@ -110,13 +111,13 @@ class FastlyNsq::Manager
|
|
110
111
|
# Shutdown the pool
|
111
112
|
# @param deadline [Integer] Number of seconds to wait for pool to stop processing
|
112
113
|
def stop_processing(deadline)
|
113
|
-
logger.info {
|
114
|
+
logger.info { "Stopping processors" }
|
114
115
|
pool.shutdown
|
115
116
|
|
116
|
-
logger.info {
|
117
|
+
logger.info { "Waiting for processors to finish..." }
|
117
118
|
return if pool.wait_for_termination(deadline)
|
118
119
|
|
119
|
-
logger.info {
|
120
|
+
logger.info { "Killing processors..." }
|
120
121
|
pool.kill
|
121
122
|
end
|
122
123
|
end
|
data/lib/fastly_nsq/message.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "json"
|
4
4
|
|
5
5
|
##
|
6
6
|
# Adapter to Nsq::Message. Provides convenience methods for interacting
|
@@ -36,7 +36,7 @@ class FastlyNsq::Message
|
|
36
36
|
# @return [String] Nsq::Message body
|
37
37
|
attr_reader :raw_body
|
38
38
|
|
39
|
-
|
39
|
+
alias_method :to_s, :raw_body
|
40
40
|
|
41
41
|
##
|
42
42
|
# @param nsq_message [Nsq::Message]
|
@@ -46,11 +46,11 @@ class FastlyNsq::Message
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def data
|
49
|
-
@data ||= body[
|
49
|
+
@data ||= body["data"]
|
50
50
|
end
|
51
51
|
|
52
52
|
def meta
|
53
|
-
@meta ||= body[
|
53
|
+
@meta ||= body["meta"]
|
54
54
|
end
|
55
55
|
|
56
56
|
def body
|
data/lib/fastly_nsq/messenger.rb
CHANGED
@@ -10,7 +10,7 @@
|
|
10
10
|
# meta: metadata_hash,
|
11
11
|
# )
|
12
12
|
module FastlyNsq::Messenger
|
13
|
-
DEFAULT_ORIGIN =
|
13
|
+
DEFAULT_ORIGIN = "Unknown"
|
14
14
|
@originating_service = DEFAULT_ORIGIN
|
15
15
|
|
16
16
|
module_function
|
@@ -37,7 +37,7 @@ module FastlyNsq::Messenger
|
|
37
37
|
def deliver(message:, topic:, originating_service: nil, sent_at: nil, meta: {})
|
38
38
|
payload = {
|
39
39
|
data: message,
|
40
|
-
meta: populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
40
|
+
meta: populate_meta(originating_service: originating_service, sent_at: sent_at, meta: meta)
|
41
41
|
}
|
42
42
|
|
43
43
|
deliver_payload(topic: topic, payload: payload.to_json)
|
@@ -69,7 +69,7 @@ module FastlyNsq::Messenger
|
|
69
69
|
payload = messages.each_with_object([]) do |message, a|
|
70
70
|
msg = {
|
71
71
|
data: message,
|
72
|
-
meta: meta
|
72
|
+
meta: meta
|
73
73
|
}
|
74
74
|
|
75
75
|
a << msg.to_json
|
@@ -129,10 +129,10 @@ module FastlyNsq::Messenger
|
|
129
129
|
meta[:originating_service] = originating_service || self.originating_service
|
130
130
|
|
131
131
|
meta[:sent_at] = if sent_at && sent_at.respond_to?(:iso8601)
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
132
|
+
sent_at.iso8601(5)
|
133
|
+
else
|
134
|
+
Time.now.iso8601(5)
|
135
|
+
end
|
136
136
|
|
137
137
|
meta
|
138
138
|
end
|
data/lib/fastly_nsq/new_relic.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
5
|
-
rescue LoadError
|
4
|
+
require "newrelic_rpm"
|
5
|
+
rescue LoadError
|
6
6
|
end
|
7
7
|
|
8
8
|
##
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
class FastlyNsq::NewRelic
|
12
12
|
include NewRelic::Agent::Instrumentation::ControllerInstrumentation if defined?(::NewRelic)
|
13
13
|
|
14
|
-
CATEGORY =
|
14
|
+
CATEGORY = "OtherTransaction/FastlyNsqProcessor"
|
15
15
|
|
16
16
|
attr_reader :agent
|
17
17
|
|
@@ -22,14 +22,14 @@ class FastlyNsq::NewRelic
|
|
22
22
|
# tracer = FastlyNsq::NewRelic.new
|
23
23
|
# tracer.notice_error(exception)
|
24
24
|
def initialize(agent = nil)
|
25
|
-
@agent = agent || (Object.const_defined?(
|
25
|
+
@agent = agent || (Object.const_defined?(:NewRelic) ? NewRelic::Agent : nil)
|
26
26
|
end
|
27
27
|
|
28
28
|
##
|
29
29
|
# Returns true if NewRelic is loaded and available.
|
30
30
|
# @return [Boolean]
|
31
31
|
def enabled?
|
32
|
-
@enabled ||= Object.const_defined?(
|
32
|
+
@enabled ||= Object.const_defined?(:NewRelic)
|
33
33
|
end
|
34
34
|
|
35
35
|
##
|
@@ -49,7 +49,7 @@ class FastlyNsq::NewRelic
|
|
49
49
|
# @see {https://www.rubydoc.info/github/newrelic/rpm/NewRelic%2FAgent%2FInstrumentation%2FControllerInstrumentation:perform_action_with_newrelic_trace}
|
50
50
|
def trace_with_newrelic(**args)
|
51
51
|
if enabled?
|
52
|
-
perform_action_with_newrelic_trace(trace_args(args)) do
|
52
|
+
perform_action_with_newrelic_trace(trace_args(**args)) do
|
53
53
|
yield
|
54
54
|
end
|
55
55
|
else
|
@@ -61,8 +61,8 @@ class FastlyNsq::NewRelic
|
|
61
61
|
|
62
62
|
def trace_args(**args)
|
63
63
|
{
|
64
|
-
name:
|
65
|
-
category: CATEGORY
|
64
|
+
name: "call",
|
65
|
+
category: CATEGORY
|
66
66
|
}.merge(args)
|
67
67
|
end
|
68
68
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
|
4
|
-
|
4
|
+
alias_method :max_threads, :max_length
|
5
5
|
|
6
6
|
def initialize(*)
|
7
7
|
super
|
@@ -15,7 +15,7 @@ class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
|
|
15
15
|
# @!visibility private
|
16
16
|
def ns_enqueue(*args, &task)
|
17
17
|
if !ns_limited_queue? || @queue.size < @max_queue
|
18
|
-
@queue.push([task, args[1
|
18
|
+
@queue.push([task, args[1..]], args[0])
|
19
19
|
true
|
20
20
|
else
|
21
21
|
false
|
@@ -27,6 +27,6 @@ class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
|
|
27
27
|
#
|
28
28
|
# @!visibility private
|
29
29
|
def ns_assign_worker(*args, &task)
|
30
|
-
super(args[1
|
30
|
+
super(args[1..], &task)
|
31
31
|
end
|
32
32
|
end
|
data/lib/fastly_nsq/producer.rb
CHANGED
@@ -31,10 +31,10 @@ class FastlyNsq::Producer
|
|
31
31
|
# @param logger [Logger] defaults to FastlyNsq.logger
|
32
32
|
# @param connect_timeout [Integer] NSQ connection timeout in seconds
|
33
33
|
def initialize(topic:, tls_options: nil, logger: FastlyNsq.logger, connect_timeout: DEFAULT_CONNECTION_TIMEOUT)
|
34
|
-
@topic
|
35
|
-
@tls_options
|
34
|
+
@topic = topic
|
35
|
+
@tls_options = FastlyNsq::TlsOptions.as_hash(tls_options)
|
36
36
|
@connect_timeout = connect_timeout
|
37
|
-
@logger
|
37
|
+
@logger = logger
|
38
38
|
|
39
39
|
connect
|
40
40
|
end
|
@@ -75,14 +75,14 @@ class FastlyNsq::Producer
|
|
75
75
|
|
76
76
|
@connection ||= Nsq::Producer.new(
|
77
77
|
tls_options.merge(
|
78
|
-
nsqlookupd:
|
79
|
-
topic:
|
80
|
-
)
|
78
|
+
nsqlookupd: lookupd,
|
79
|
+
topic: topic
|
80
|
+
)
|
81
81
|
)
|
82
82
|
|
83
83
|
timeout_args = [connect_timeout, FastlyNsq::ConnectionFailed]
|
84
84
|
|
85
|
-
if RUBY_VERSION >
|
85
|
+
if RUBY_VERSION > "2.4.0"
|
86
86
|
timeout_args << "Failed connection to #{lookupd} within #{connect_timeout} seconds"
|
87
87
|
end
|
88
88
|
|
data/lib/fastly_nsq/testing.rb
CHANGED
@@ -98,7 +98,7 @@ module FastlyNsq
|
|
98
98
|
# processor_klass.call(test_message)
|
99
99
|
# expect(Post.find(post_data['id']).not_to be nil
|
100
100
|
def message(data:, meta: nil)
|
101
|
-
test_message = FastlyNsq::TestMessage.new(JSON.dump(
|
101
|
+
test_message = FastlyNsq::TestMessage.new(JSON.dump("data" => data, "meta" => meta))
|
102
102
|
FastlyNsq::Message.new(test_message)
|
103
103
|
end
|
104
104
|
end
|
@@ -121,7 +121,7 @@ module FastlyNsq
|
|
121
121
|
|
122
122
|
def initialize(raw_body)
|
123
123
|
@raw_body = raw_body
|
124
|
-
@id
|
124
|
+
@id = Digest::SHA1.hexdigest(raw_body.to_s + Time.now.to_s)
|
125
125
|
@attempts = 0
|
126
126
|
end
|
127
127
|
|
@@ -250,7 +250,8 @@ module FastlyNsq
|
|
250
250
|
FastlyNsq::Listener.prepend(ListenerTesting)
|
251
251
|
|
252
252
|
class FakeConnection
|
253
|
-
def connected
|
253
|
+
def connected?
|
254
|
+
end
|
254
255
|
end
|
255
256
|
|
256
257
|
module ConsumerTesting
|
@@ -20,7 +20,7 @@ module FastlyNsq
|
|
20
20
|
else
|
21
21
|
{
|
22
22
|
tls_v1: true,
|
23
|
-
tls_options: @context
|
23
|
+
tls_options: @context
|
24
24
|
}
|
25
25
|
end
|
26
26
|
end
|
@@ -28,19 +28,19 @@ module FastlyNsq
|
|
28
28
|
private
|
29
29
|
|
30
30
|
def env_key
|
31
|
-
ENV.fetch(
|
31
|
+
ENV.fetch("NSQ_SSL_KEY", nil)
|
32
32
|
end
|
33
33
|
|
34
34
|
def env_certificate
|
35
|
-
ENV.fetch(
|
35
|
+
ENV.fetch("NSQ_SSL_CERTIFICATE", nil)
|
36
36
|
end
|
37
37
|
|
38
38
|
def env_ca_certificate
|
39
|
-
ENV.fetch(
|
39
|
+
ENV.fetch("NSQ_SSL_CA_CERTIFICATE", nil)
|
40
40
|
end
|
41
41
|
|
42
42
|
def verify_mode
|
43
|
-
ENV.fetch(
|
43
|
+
ENV.fetch("NSQ_SSL_VERIFY_MODE", nil)
|
44
44
|
end
|
45
45
|
|
46
46
|
def env_default_hash
|
@@ -48,7 +48,7 @@ module FastlyNsq
|
|
48
48
|
key: env_key,
|
49
49
|
certificate: env_certificate,
|
50
50
|
ca_certificate: env_ca_certificate,
|
51
|
-
verify_mode: verify_mode
|
51
|
+
verify_mode: verify_mode
|
52
52
|
}
|
53
53
|
end
|
54
54
|
|
data/lib/fastly_nsq/version.rb
CHANGED
data/lib/fastly_nsq.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
3
|
+
require "nsq"
|
4
|
+
require "concurrent"
|
5
|
+
require "fc"
|
6
|
+
require "set"
|
7
|
+
require "logger"
|
8
|
+
require "forwardable"
|
9
|
+
require "digest/md5"
|
10
10
|
|
11
11
|
module FastlyNsq
|
12
12
|
NotConnectedError = Class.new(StandardError)
|
@@ -58,7 +58,7 @@ module FastlyNsq
|
|
58
58
|
def logger
|
59
59
|
return @logger if @logger
|
60
60
|
|
61
|
-
self.logger = Logger.new(
|
61
|
+
self.logger = Logger.new($stderr)
|
62
62
|
end
|
63
63
|
|
64
64
|
##
|
@@ -112,21 +112,21 @@ module FastlyNsq
|
|
112
112
|
# @return [Integer]
|
113
113
|
# @see https://nsq.io/components/nsqd.html#command-line-options
|
114
114
|
def max_req_timeout
|
115
|
-
@max_req_timeout ||= ENV.fetch(
|
115
|
+
@max_req_timeout ||= ENV.fetch("MAX_REQ_TIMEOUT", 60 * 60 * 1_000).to_i
|
116
116
|
end
|
117
117
|
|
118
118
|
# Maximum number of threads for FastlyNsq::PriorityThreadPool
|
119
119
|
# Default setting is 5 and can be set via ENV['MAX_PROCESSING_POOL_THREADS']
|
120
120
|
# @return [Integer]
|
121
121
|
def max_processing_pool_threads
|
122
|
-
@max_processing_pool_threads ||= ENV.fetch(
|
122
|
+
@max_processing_pool_threads ||= ENV.fetch("MAX_PROCESSING_POOL_THREADS", 5).to_i
|
123
123
|
end
|
124
124
|
|
125
125
|
##
|
126
126
|
# Return an array of NSQ lookupd http addresses sourced from ENV['NSQLOOKUPD_HTTP_ADDRESS']
|
127
127
|
# @return [Array<String>] list of nsqlookupd http addresses
|
128
128
|
def lookupd_http_addresses
|
129
|
-
ENV.fetch(
|
129
|
+
ENV.fetch("NSQLOOKUPD_HTTP_ADDRESS").split(",").map(&:strip)
|
130
130
|
end
|
131
131
|
|
132
132
|
# Register a block to run at a point in the lifecycle.
|
@@ -151,11 +151,9 @@ module FastlyNsq
|
|
151
151
|
def fire_event(event)
|
152
152
|
blocks = FastlyNsq.events.fetch(event)
|
153
153
|
blocks.each do |block|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
logger.error "[#{event}] #{e.inspect}"
|
158
|
-
end
|
154
|
+
block.call
|
155
|
+
rescue => e
|
156
|
+
logger.error "[#{event}] #{e.inspect}"
|
159
157
|
end
|
160
158
|
end
|
161
159
|
|
@@ -168,16 +166,16 @@ module FastlyNsq
|
|
168
166
|
end
|
169
167
|
end
|
170
168
|
|
171
|
-
require
|
172
|
-
require
|
173
|
-
require
|
174
|
-
require
|
175
|
-
require
|
176
|
-
require
|
177
|
-
require
|
178
|
-
require
|
179
|
-
require
|
180
|
-
require
|
181
|
-
require
|
182
|
-
require
|
183
|
-
require
|
169
|
+
require "fastly_nsq/consumer"
|
170
|
+
require "fastly_nsq/feeder"
|
171
|
+
require "fastly_nsq/launcher"
|
172
|
+
require "fastly_nsq/listener"
|
173
|
+
require "fastly_nsq/manager"
|
174
|
+
require "fastly_nsq/message"
|
175
|
+
require "fastly_nsq/messenger"
|
176
|
+
require "fastly_nsq/new_relic"
|
177
|
+
require "fastly_nsq/priority_queue"
|
178
|
+
require "fastly_nsq/priority_thread_pool"
|
179
|
+
require "fastly_nsq/producer"
|
180
|
+
require "fastly_nsq/tls_options"
|
181
|
+
require "fastly_nsq/version"
|