nsq-ruby 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -4
- data/lib/nsq/client_base.rb +111 -0
- data/lib/nsq/consumer.rb +17 -79
- data/lib/nsq/discovery.rb +28 -8
- data/lib/nsq/producer.rb +36 -21
- data/lib/version.rb +1 -1
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16ea2d920538289564a71a48e051589bd8977a53
|
4
|
+
data.tar.gz: 4ac36aa2fcf6178267d75f46db965555756337bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d528a2b2b3752cc9ded663ea4bcc1f8343bb6300481cbeb9587b4e543acccda9a469e8463a1060f951937accb751dfb0e214b6aab86de8d25f5f8b3bd5301b3
|
7
|
+
data.tar.gz: 73f9c978d51d10b3e2a7a61092cf96c1e7d1d5b1c4899f943c0a2a331e35f94c741a1f765831d8bd718b534c7983a71a556851a8bf1bcebb9406fb65dee34563
|
data/README.md
CHANGED
@@ -50,7 +50,7 @@ consumer.terminate
|
|
50
50
|
|
51
51
|
## Producer
|
52
52
|
|
53
|
-
###
|
53
|
+
### Initialization
|
54
54
|
|
55
55
|
The Nsq::Producer constructor takes the following options:
|
56
56
|
|
@@ -58,8 +58,9 @@ The Nsq::Producer constructor takes the following options:
|
|
58
58
|
|---------------|----------------------------------------|--------------------|
|
59
59
|
| `topic` | Topic to which to publish messages | |
|
60
60
|
| `nsqd` | Host and port of the nsqd instance | '127.0.0.1:4150' |
|
61
|
+
| `nsqlookupd` | Use lookupd to automatically discover nsqds | |
|
61
62
|
|
62
|
-
For example
|
63
|
+
For example, if you'd like to publish messages to a single nsqd.
|
63
64
|
|
64
65
|
```Ruby
|
65
66
|
producer = Nsq::Producer.new(
|
@@ -68,6 +69,19 @@ producer = Nsq::Producer.new(
|
|
68
69
|
)
|
69
70
|
```
|
70
71
|
|
72
|
+
Alternatively, you can use nsqlookupd to find all nsqd nodes in the cluster.
|
73
|
+
When you instantiate Nsq::Producer in this way, it will automatically maintain
|
74
|
+
connections to all nsqd instances. When you publish a message, it will be sent
|
75
|
+
to a random nsqd instance.
|
76
|
+
|
77
|
+
```Ruby
|
78
|
+
producer = Nsq::Producer.new(
|
79
|
+
nsqlookupd: ['1.2.3.4:4161', '6.7.8.9:4161'],
|
80
|
+
topic: 'topic-of-great-esteem'
|
81
|
+
)
|
82
|
+
```
|
83
|
+
|
84
|
+
|
71
85
|
### `#write`
|
72
86
|
|
73
87
|
Publishes one or more message to nsqd. If you give it a single argument, it will
|
@@ -107,7 +121,7 @@ producers when you're done with them.
|
|
107
121
|
|
108
122
|
## Consumer
|
109
123
|
|
110
|
-
###
|
124
|
+
### Initialization
|
111
125
|
|
112
126
|
| Option | Description | Default |
|
113
127
|
|----------------------|-----------------------------------------------|--------------------|
|
@@ -215,7 +229,7 @@ Nsq.logger = Logger.new(STDOUT)
|
|
215
229
|
|
216
230
|
## Requirements
|
217
231
|
|
218
|
-
NSQ v0.2.29 or later
|
232
|
+
NSQ v0.2.29 or later due for IDENTITY metadata specification (0.2.28) and per-
|
219
233
|
connection timeout support (0.2.29).
|
220
234
|
|
221
235
|
|
@@ -252,6 +266,13 @@ VERBOSE=true rake spec
|
|
252
266
|
```
|
253
267
|
|
254
268
|
|
269
|
+
## Authors
|
270
|
+
|
271
|
+
- Robby Grossman (@freerobby)
|
272
|
+
- Brendan Schwartz (@bschwartz)
|
273
|
+
- Marshall Moutenot (@mmoutenot)
|
274
|
+
|
275
|
+
|
255
276
|
## MIT License
|
256
277
|
|
257
278
|
Copyright (C) 2014 Wistia, Inc.
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require_relative 'discovery'
|
2
|
+
require_relative 'connection'
|
3
|
+
require_relative 'logger'
|
4
|
+
|
5
|
+
module Nsq
|
6
|
+
class ClientBase
|
7
|
+
include Nsq::AttributeLogger
|
8
|
+
@@log_attributes = [:topic]
|
9
|
+
|
10
|
+
attr_reader :topic
|
11
|
+
attr_reader :connections
|
12
|
+
|
13
|
+
def connected?
|
14
|
+
@connections.values.any?(&:connected?)
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
def terminate
|
19
|
+
@discovery_thread.kill if @discovery_thread
|
20
|
+
drop_all_connections
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# discovers nsqds from an nsqlookupd repeatedly
|
27
|
+
#
|
28
|
+
# opts:
|
29
|
+
# nsqlookups: ['127.0.0.1:4161'],
|
30
|
+
# topic: 'topic-to-find-nsqds-for',
|
31
|
+
# interval: 60
|
32
|
+
#
|
33
|
+
def discover_repeatedly(opts = {})
|
34
|
+
@discovery_thread = Thread.new do
|
35
|
+
|
36
|
+
@discovery = Discovery.new(opts[:nsqlookupds])
|
37
|
+
|
38
|
+
loop do
|
39
|
+
nsqds = nsqds_from_lookupd(opts[:topic])
|
40
|
+
drop_and_add_connections(nsqds)
|
41
|
+
sleep opts[:interval]
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
@discovery_thread.abort_on_exception = true
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
def nsqds_from_lookupd(topic = nil)
|
51
|
+
if topic
|
52
|
+
@discovery.nsqds_for_topic(topic)
|
53
|
+
else
|
54
|
+
@discovery.nsqds
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
|
59
|
+
def drop_and_add_connections(nsqds)
|
60
|
+
# drop nsqd connections that are no longer in lookupd
|
61
|
+
missing_nsqds = @connections.keys - nsqds
|
62
|
+
missing_nsqds.each do |nsqd|
|
63
|
+
drop_connection(nsqd)
|
64
|
+
end
|
65
|
+
|
66
|
+
# add new ones
|
67
|
+
new_nsqds = nsqds - @connections.keys
|
68
|
+
new_nsqds.each do |nsqd|
|
69
|
+
begin
|
70
|
+
add_connection(nsqd)
|
71
|
+
rescue Exception => ex
|
72
|
+
error "Failed to connect to nsqd @ #{nsqd}: #{ex}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# balance RDY state amongst the connections
|
77
|
+
connections_changed
|
78
|
+
end
|
79
|
+
|
80
|
+
|
81
|
+
def add_connection(nsqd, options = {})
|
82
|
+
info "+ Adding connection #{nsqd}"
|
83
|
+
host, port = nsqd.split(':')
|
84
|
+
connection = Connection.new({
|
85
|
+
host: host,
|
86
|
+
port: port
|
87
|
+
}.merge(options))
|
88
|
+
@connections[nsqd] = connection
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
def drop_connection(nsqd)
|
93
|
+
info "- Dropping connection #{nsqd}"
|
94
|
+
connection = @connections.delete(nsqd)
|
95
|
+
connection.close if connection
|
96
|
+
connections_changed
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
def drop_all_connections
|
101
|
+
@connections.keys.each do |nsqd|
|
102
|
+
drop_connection(nsqd)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
|
107
|
+
# optional subclass hook
|
108
|
+
def connections_changed
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
data/lib/nsq/consumer.rb
CHANGED
@@ -1,16 +1,9 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'discovery'
|
3
|
-
require_relative 'logger'
|
1
|
+
require_relative 'client_base'
|
4
2
|
|
5
3
|
module Nsq
|
6
|
-
class Consumer
|
7
|
-
include Nsq::AttributeLogger
|
8
|
-
@@log_attributes = [:topic]
|
4
|
+
class Consumer < ClientBase
|
9
5
|
|
10
|
-
attr_reader :topic
|
11
6
|
attr_reader :max_in_flight
|
12
|
-
attr_reader :discovery_interval
|
13
|
-
attr_reader :connections
|
14
7
|
|
15
8
|
def initialize(opts = {})
|
16
9
|
if opts[:nsqlookupd]
|
@@ -34,24 +27,21 @@ module Nsq
|
|
34
27
|
@connections = {}
|
35
28
|
|
36
29
|
if !@nsqlookupds.empty?
|
37
|
-
|
38
|
-
|
30
|
+
discover_repeatedly(
|
31
|
+
nsqlookupds: @nsqlookupds,
|
32
|
+
topic: @topic,
|
33
|
+
interval: @discovery_interval
|
34
|
+
)
|
39
35
|
else
|
40
36
|
# normally, we find nsqd instances to connect to via nsqlookupd(s)
|
41
37
|
# in this case let's connect to an nsqd instance directly
|
42
|
-
add_connection(opts[:nsqd] || '127.0.0.1:4150', @max_in_flight)
|
38
|
+
add_connection(opts[:nsqd] || '127.0.0.1:4150', max_in_flight: @max_in_flight)
|
43
39
|
end
|
44
40
|
|
45
41
|
at_exit{terminate}
|
46
42
|
end
|
47
43
|
|
48
44
|
|
49
|
-
def terminate
|
50
|
-
@discovery_thread.kill if @discovery_thread
|
51
|
-
drop_all_connections
|
52
|
-
end
|
53
|
-
|
54
|
-
|
55
45
|
# pop the next message off the queue
|
56
46
|
def pop
|
57
47
|
@messages.pop
|
@@ -65,82 +55,30 @@ module Nsq
|
|
65
55
|
|
66
56
|
|
67
57
|
private
|
68
|
-
def
|
69
|
-
|
70
|
-
loop do
|
71
|
-
discover
|
72
|
-
sleep @discovery_interval
|
73
|
-
end
|
74
|
-
end
|
75
|
-
@discovery_thread.abort_on_exception = true
|
76
|
-
end
|
77
|
-
|
78
|
-
|
79
|
-
def discover
|
80
|
-
nsqds = @discovery.nsqds_for_topic(@topic)
|
81
|
-
|
82
|
-
# drop nsqd connections that are no longer in lookupd
|
83
|
-
missing_nsqds = @connections.keys - nsqds
|
84
|
-
missing_nsqds.each do |nsqd|
|
85
|
-
drop_connection(nsqd)
|
86
|
-
end
|
87
|
-
|
88
|
-
# add new ones
|
89
|
-
new_nsqds = nsqds - @connections.keys
|
90
|
-
new_nsqds.each do |nsqd|
|
91
|
-
# Be conservative and start new connections with RDY 1
|
92
|
-
# This helps ensure we don't exceed @max_in_flight across all our
|
93
|
-
# connections momentarily.
|
94
|
-
add_connection(nsqd, 1)
|
95
|
-
end
|
96
|
-
|
97
|
-
# balance RDY state amongst the connections
|
98
|
-
redistribute_ready
|
99
|
-
end
|
100
|
-
|
101
|
-
|
102
|
-
def add_connection(nsqd, max_in_flight)
|
103
|
-
info "+ Adding connection #{nsqd}"
|
104
|
-
host, port = nsqd.split(':')
|
105
|
-
connection = Connection.new(
|
106
|
-
host: host,
|
107
|
-
port: port,
|
58
|
+
def add_connection(nsqd, options = {})
|
59
|
+
super(nsqd, {
|
108
60
|
topic: @topic,
|
109
61
|
channel: @channel,
|
110
62
|
queue: @messages,
|
111
63
|
msg_timeout: @msg_timeout,
|
112
|
-
max_in_flight:
|
113
|
-
)
|
114
|
-
@connections[nsqd] = connection
|
64
|
+
max_in_flight: 1
|
65
|
+
}.merge(options))
|
115
66
|
end
|
116
67
|
|
68
|
+
# Be conservative, but don't set a connection's max_in_flight below 1
|
69
|
+
def max_in_flight_per_connection(number_of_connections = @connections.length)
|
70
|
+
[@max_in_flight / number_of_connections, 1].max
|
71
|
+
end
|
117
72
|
|
118
|
-
def
|
119
|
-
info "- Dropping connection #{nsqd}"
|
120
|
-
connection = @connections.delete(nsqd)
|
121
|
-
connection.close
|
73
|
+
def connections_changed
|
122
74
|
redistribute_ready
|
123
75
|
end
|
124
76
|
|
125
|
-
|
126
77
|
def redistribute_ready
|
127
78
|
@connections.values.each do |connection|
|
128
79
|
connection.max_in_flight = max_in_flight_per_connection
|
129
80
|
connection.re_up_ready
|
130
81
|
end
|
131
82
|
end
|
132
|
-
|
133
|
-
|
134
|
-
def drop_all_connections
|
135
|
-
@connections.keys.each do |nsqd|
|
136
|
-
drop_connection(nsqd)
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
|
141
|
-
# Be conservative, but don't set a connection's max_in_flight below 1
|
142
|
-
def max_in_flight_per_connection(number_of_connections = @connections.length)
|
143
|
-
[@max_in_flight / number_of_connections, 1].max
|
144
|
-
end
|
145
83
|
end
|
146
84
|
end
|
data/lib/nsq/discovery.rb
CHANGED
@@ -14,27 +14,47 @@ module Nsq
|
|
14
14
|
@lookupds = lookupds
|
15
15
|
end
|
16
16
|
|
17
|
-
#
|
17
|
+
# Returns an array of nsqds instances
|
18
|
+
#
|
19
|
+
# nsqd instances returned are strings in this format: '<host>:<tcp-port>'
|
20
|
+
#
|
21
|
+
# discovery.nsqds
|
22
|
+
# #=> ['127.0.0.1:4150', '127.0.0.1:4152']
|
23
|
+
#
|
24
|
+
def nsqds
|
25
|
+
@lookupds.map do |lookupd|
|
26
|
+
get_nsqds(lookupd)
|
27
|
+
end.flatten.uniq
|
28
|
+
end
|
29
|
+
|
30
|
+
# Returns an array of nsqds instances that have messages for
|
18
31
|
# that topic.
|
19
32
|
#
|
20
33
|
# nsqd instances returned are strings in this format: '<host>:<tcp-port>'
|
21
34
|
#
|
22
|
-
# discovery.nsqds_for_topic('
|
35
|
+
# discovery.nsqds_for_topic('a-topic')
|
23
36
|
# #=> ['127.0.0.1:4150', '127.0.0.1:4152']
|
24
37
|
#
|
25
38
|
def nsqds_for_topic(topic)
|
26
39
|
@lookupds.map do |lookupd|
|
27
|
-
|
40
|
+
get_nsqds(lookupd, topic)
|
28
41
|
end.flatten.uniq
|
29
42
|
end
|
30
43
|
|
31
|
-
|
32
44
|
private
|
33
45
|
|
34
|
-
def
|
35
|
-
|
36
|
-
uri
|
37
|
-
|
46
|
+
def get_nsqds(lookupd, topic = nil)
|
47
|
+
uri_scheme = 'http://' unless lookupd.match(%r(https?://))
|
48
|
+
uri = URI.parse("#{uri_scheme}#{lookupd}")
|
49
|
+
|
50
|
+
uri.query = "ts=#{Time.now.to_i}"
|
51
|
+
if topic
|
52
|
+
uri.path = '/lookup'
|
53
|
+
uri.query += "&topic=#{topic}"
|
54
|
+
else
|
55
|
+
uri.path = '/nodes'
|
56
|
+
end
|
57
|
+
|
38
58
|
begin
|
39
59
|
body = Net::HTTP.get(uri)
|
40
60
|
data = JSON.parse(body)
|
data/lib/nsq/producer.rb
CHANGED
@@ -1,47 +1,62 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'logger'
|
1
|
+
require_relative 'client_base'
|
3
2
|
|
4
3
|
module Nsq
|
5
|
-
class Producer
|
6
|
-
include Nsq::AttributeLogger
|
7
|
-
@@log_attributes = [:host, :port, :topic]
|
8
|
-
|
9
|
-
attr_reader :host
|
10
|
-
attr_reader :port
|
4
|
+
class Producer < ClientBase
|
11
5
|
attr_reader :topic
|
12
6
|
|
13
|
-
|
14
7
|
def initialize(opts = {})
|
15
|
-
@
|
16
|
-
@host, @port = @nsqd.split(':')
|
17
|
-
|
8
|
+
@connections = {}
|
18
9
|
@topic = opts[:topic] || raise(ArgumentError, 'topic is required')
|
10
|
+
@discovery_interval = opts[:discovery_interval] || 60
|
11
|
+
|
12
|
+
nsqlookupds = []
|
13
|
+
if opts[:nsqlookupd]
|
14
|
+
nsqlookupds = [opts[:nsqlookupd]].flatten
|
15
|
+
discover_repeatedly(
|
16
|
+
nsqlookupds: nsqlookupds,
|
17
|
+
interval: @discovery_interval
|
18
|
+
)
|
19
|
+
|
20
|
+
elsif opts[:nsqd]
|
21
|
+
nsqds = [opts[:nsqd]].flatten
|
22
|
+
nsqds.each{|d| add_connection(d)}
|
19
23
|
|
20
|
-
|
24
|
+
else
|
25
|
+
add_connection('127.0.0.1:4150')
|
26
|
+
end
|
21
27
|
|
22
28
|
at_exit{terminate}
|
23
29
|
end
|
24
30
|
|
25
31
|
|
26
32
|
def write(*raw_messages)
|
27
|
-
# stringify
|
33
|
+
# stringify the messages
|
28
34
|
messages = raw_messages.map(&:to_s)
|
29
35
|
|
36
|
+
# get a suitable connection to write to
|
37
|
+
connection = connection_for_write
|
38
|
+
|
30
39
|
if messages.length > 1
|
31
|
-
|
40
|
+
connection.mpub(@topic, messages)
|
32
41
|
else
|
33
|
-
|
42
|
+
connection.pub(@topic, messages.first)
|
34
43
|
end
|
35
44
|
end
|
36
45
|
|
37
46
|
|
38
|
-
|
39
|
-
|
40
|
-
|
47
|
+
private
|
48
|
+
def connection_for_write
|
49
|
+
# Choose a random Connection that's currently connected
|
50
|
+
# Or, if there's nothing connected, just take any random one
|
51
|
+
connections_currently_connected = connections.select{|_,c| c.connected?}
|
52
|
+
connection = connections_currently_connected.values.sample || connections.values.sample
|
41
53
|
|
54
|
+
# Raise an exception if there's no connection available
|
55
|
+
unless connection
|
56
|
+
raise 'No connections available'
|
57
|
+
end
|
42
58
|
|
43
|
-
|
44
|
-
@connection.close
|
59
|
+
connection
|
45
60
|
end
|
46
61
|
|
47
62
|
end
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nsq-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wistia
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-08-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 1.1.0
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 1.1.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rspec
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,6 +77,7 @@ files:
|
|
77
77
|
- LICENSE.txt
|
78
78
|
- README.md
|
79
79
|
- lib/nsq.rb
|
80
|
+
- lib/nsq/client_base.rb
|
80
81
|
- lib/nsq/connection.rb
|
81
82
|
- lib/nsq/consumer.rb
|
82
83
|
- lib/nsq/discovery.rb
|