ruby_nsq 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.md +9 -1
- data/README.md +21 -8
- data/examples/async/README +6 -0
- data/examples/async/publisher.rb +21 -0
- data/examples/async/reader.rb +13 -11
- data/examples/simple/README +5 -0
- data/examples/simple/publisher.rb +13 -0
- data/examples/simple/reader.rb +7 -4
- data/lib/nsq/backoff_timer.rb +2 -1
- data/lib/nsq/connection.rb +12 -15
- data/lib/nsq/message.rb +3 -1
- data/lib/nsq/publisher.rb +40 -0
- data/lib/nsq/queue_subscriber.rb +9 -3
- data/lib/nsq/reader.rb +46 -11
- data/lib/nsq/subscriber.rb +63 -14
- data/lib/nsq/timer.rb +1 -0
- data/lib/nsq/util.rb +17 -0
- data/lib/nsq.rb +4 -14
- metadata +11 -6
- data/examples/async/writer.rb +0 -17
data/History.md
CHANGED
data/README.md
CHANGED
@@ -2,15 +2,15 @@
|
|
2
2
|
|
3
3
|
https://github.com/ClarityServices/ruby_nsq
|
4
4
|
|
5
|
-
## Description
|
5
|
+
## Description
|
6
6
|
|
7
7
|
Ruby client for the [NSQ](https://github.com/bitly/nsq) realtime message processing system.
|
8
8
|
|
9
|
-
## Install
|
9
|
+
## Install
|
10
10
|
|
11
11
|
gem install ruby_nsq
|
12
12
|
|
13
|
-
## Usage
|
13
|
+
## Usage
|
14
14
|
|
15
15
|
See [examples](https://github.com/ClarityServices/ruby_nsq/tree/master/examples)
|
16
16
|
|
@@ -18,7 +18,7 @@ Simple example for synchronous message handling:
|
|
18
18
|
```
|
19
19
|
require 'nsq'
|
20
20
|
|
21
|
-
reader = NSQ.
|
21
|
+
reader = NSQ::Reader.new(:nsqd_tcp_addresses => '127.0.0.1:4150')
|
22
22
|
# Subscribe to topic=test channel=simple
|
23
23
|
reader.subscribe('test', 'simple') do |message|
|
24
24
|
# If this block raises an exception, then the message will be requeued.
|
@@ -36,7 +36,7 @@ foo_worker_count = 50
|
|
36
36
|
bar_worker_count = 30
|
37
37
|
baz_worker_count = 20
|
38
38
|
|
39
|
-
reader = NSQ.
|
39
|
+
reader = NSQ::Reader.new(:nsqd_tcp_addresses => '127.0.0.1:4150')
|
40
40
|
|
41
41
|
foo_subscriber = reader.subscribe('test', 'foo', :max_in_flight => foo_worker_count)
|
42
42
|
bar_subscriber = reader.subscribe('test2', 'bar', :max_in_flight => bar_worker_count)
|
@@ -62,10 +62,23 @@ bar_threads.each(&:join)
|
|
62
62
|
baz_threads.each(&:join)
|
63
63
|
```
|
64
64
|
|
65
|
-
## TODO
|
66
|
-
|
67
|
-
* Fix timestamp
|
65
|
+
## TODO
|
68
66
|
|
69
67
|
* Implement lookupd
|
70
68
|
|
71
69
|
* Tests!
|
70
|
+
|
71
|
+
* Documentation
|
72
|
+
|
73
|
+
## Meta
|
74
|
+
|
75
|
+
* Code: `git clone git://github.com/ClarityServices/ruby_nsq.git`
|
76
|
+
* Home: <https://github.com/ClarityServices/ruby_nsq>
|
77
|
+
* Bugs: <http://github.com/reidmorrison/ruby_nsq/issues>
|
78
|
+
* Gems: <http://rubygems.org/gems/ruby_nsq>
|
79
|
+
|
80
|
+
This project uses [Semantic Versioning](http://semver.org/).
|
81
|
+
|
82
|
+
## Authors
|
83
|
+
|
84
|
+
Brad Pardee :: bradpardee@gmail.com
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nsq'
|
4
|
+
|
5
|
+
if ARGV.length != 3
|
6
|
+
$stderr.puts "bundle exec ./publisher.rb <topic> <count> <eval-string>"
|
7
|
+
$stderr.puts " where <topic> is either test_xy or test_z"
|
8
|
+
$stderr.puts " and <eval-string> could be something like 'sleep rand(100)/10.0'"
|
9
|
+
$stderr.puts " Example: bundle exec ./publisher.rb test_xy 500 'sleep rand(100)/10.0'"
|
10
|
+
$stderr.puts " or: bundle exec ./publisher.rb test_z 5000 nil"
|
11
|
+
exit 1
|
12
|
+
end
|
13
|
+
topic = ARGV[0]
|
14
|
+
count = ARGV[1].to_i
|
15
|
+
eval_string = ARGV[2]
|
16
|
+
|
17
|
+
NSQ::Publisher.new('localhost', 4150) do |publisher|
|
18
|
+
count.times do
|
19
|
+
publisher.publish(topic, eval_string)
|
20
|
+
end
|
21
|
+
end
|
data/examples/async/reader.rb
CHANGED
@@ -11,9 +11,9 @@ z_worker_count = 20
|
|
11
11
|
puts 'Press enter to start and enter to finish'
|
12
12
|
$stdin.gets
|
13
13
|
|
14
|
-
reader = NSQ.
|
14
|
+
reader = NSQ::Reader.new(
|
15
15
|
:nsqd_tcp_addresses => '127.0.0.1:4150',
|
16
|
-
|
16
|
+
#:logger_level => Logger::DEBUG
|
17
17
|
)
|
18
18
|
|
19
19
|
x_subscriber = reader.subscribe('test_xy', 'x', :max_in_flight => x_worker_count)
|
@@ -46,13 +46,15 @@ end
|
|
46
46
|
main_thread = Thread.new do
|
47
47
|
reader.run
|
48
48
|
end
|
49
|
+
at_exit {
|
50
|
+
puts 'Exiting...'
|
51
|
+
reader.stop
|
52
|
+
main_thread.join
|
53
|
+
threads.each_value { |arr| arr.each(&:join) }
|
54
|
+
puts
|
55
|
+
puts "Summary of worker message counts"
|
56
|
+
threads.each do |char, arr|
|
57
|
+
puts "#{char} - #{arr.map(&:message_count).join(' ')} total=#{arr.map(&:message_count).inject(:+)}"
|
58
|
+
end
|
59
|
+
}
|
49
60
|
$stdin.gets
|
50
|
-
puts 'Exiting...'
|
51
|
-
reader.stop
|
52
|
-
main_thread.join
|
53
|
-
threads.each_value { |arr| arr.each(&:join) }
|
54
|
-
puts
|
55
|
-
puts "Summary of worker message counts"
|
56
|
-
threads.each do |char, arr|
|
57
|
-
puts "#{char} - #{arr.map(&:message_count).join(' ')}"
|
58
|
-
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nsq'
|
4
|
+
|
5
|
+
if ARGV.length == 0
|
6
|
+
$stderr.puts "bundle exec ./publisher.rb <message>*"
|
7
|
+
$stderr.puts " Example: bundle exec ./publisher.rb hello world"
|
8
|
+
exit 1
|
9
|
+
end
|
10
|
+
|
11
|
+
NSQ::Publisher.new('localhost', 4150) do |publisher|
|
12
|
+
publisher.publish('test', ARGV.join(' '))
|
13
|
+
end
|
data/examples/simple/reader.rb
CHANGED
@@ -3,16 +3,17 @@
|
|
3
3
|
require 'nsq'
|
4
4
|
require 'logger'
|
5
5
|
|
6
|
+
# Cntl-c doesn't run at_exit under jruby
|
6
7
|
puts 'Press enter to start and enter to finish'
|
7
8
|
gets
|
8
|
-
reader = NSQ.
|
9
|
+
reader = NSQ::Reader.new(
|
9
10
|
:nsqd_tcp_addresses => '127.0.0.1:4150',
|
10
11
|
#:logger_level => Logger::DEBUG
|
11
12
|
)
|
12
13
|
thread = Thread.new do
|
13
14
|
begin
|
14
15
|
reader.subscribe('test', 'simple') do |message|
|
15
|
-
puts "Read #{message.body}"
|
16
|
+
puts "Read: #{message.body.inspect}"
|
16
17
|
end
|
17
18
|
reader.run
|
18
19
|
rescue Exception => e
|
@@ -20,6 +21,8 @@ thread = Thread.new do
|
|
20
21
|
end
|
21
22
|
puts 'Reader exiting'
|
22
23
|
end
|
24
|
+
at_exit {
|
25
|
+
reader.stop
|
26
|
+
thread.join
|
27
|
+
}
|
23
28
|
gets
|
24
|
-
reader.stop
|
25
|
-
thread.join
|
data/lib/nsq/backoff_timer.rb
CHANGED
@@ -34,7 +34,8 @@ module NSQ
|
|
34
34
|
@short_interval = [@short_interval, @max_short_timer].min
|
35
35
|
@long_interval = [@long_interval, @max_long_timer].min
|
36
36
|
end
|
37
|
-
|
37
|
+
|
38
|
+
# Return the interval to wait based on the successes and failures
|
38
39
|
def interval
|
39
40
|
@min_interval + @short_interval + @long_interval
|
40
41
|
end
|
data/lib/nsq/connection.rb
CHANGED
@@ -2,6 +2,7 @@ require 'monitor'
|
|
2
2
|
require 'thread' #Mutex
|
3
3
|
|
4
4
|
module NSQ
|
5
|
+
# Represents a single subscribed connection to an nsqd server.
|
5
6
|
class Connection
|
6
7
|
attr_reader :name
|
7
8
|
|
@@ -27,19 +28,19 @@ module NSQ
|
|
27
28
|
connect
|
28
29
|
end
|
29
30
|
|
30
|
-
def send_init(topic, channel, short_id, long_id)
|
31
|
+
def send_init(topic, channel, short_id, long_id) #:nodoc:
|
31
32
|
write NSQ::MAGIC_V2
|
32
33
|
write "SUB #{topic} #{channel} #{short_id} #{long_id}\n"
|
33
34
|
self.send_ready
|
34
35
|
end
|
35
36
|
|
36
|
-
def send_ready
|
37
|
+
def send_ready #:nodoc:
|
37
38
|
@ready_count = @subscriber.ready_count
|
38
39
|
write "RDY #{@ready_count}\n" unless @subscriber.stopped?
|
39
40
|
@sending_ready = false
|
40
41
|
end
|
41
42
|
|
42
|
-
def send_finish(id, success)
|
43
|
+
def send_finish(id, success) #:nodoc:
|
43
44
|
write "FIN #{id}\n"
|
44
45
|
@ready_mutex.synchronize do
|
45
46
|
@ready_count -= 1
|
@@ -52,7 +53,7 @@ module NSQ
|
|
52
53
|
end
|
53
54
|
end
|
54
55
|
|
55
|
-
def send_requeue(id, time_ms)
|
56
|
+
def send_requeue(id, time_ms) #:nodoc:
|
56
57
|
write "REQ #{id} #{time_ms}\n"
|
57
58
|
@ready_mutex.synchronize do
|
58
59
|
@ready_count -= 1
|
@@ -61,7 +62,7 @@ module NSQ
|
|
61
62
|
end
|
62
63
|
end
|
63
64
|
|
64
|
-
def reset
|
65
|
+
def reset #:nodoc:
|
65
66
|
return unless verify_connect_state?(:connecting, :connected)
|
66
67
|
# Close with the hopes of re-establishing
|
67
68
|
close(false)
|
@@ -82,7 +83,7 @@ module NSQ
|
|
82
83
|
end
|
83
84
|
end
|
84
85
|
|
85
|
-
def close(permanent=true)
|
86
|
+
def close(permanent=true) #:nodoc:
|
86
87
|
NSQ.logger.debug {"#{@name}: Closing..."}
|
87
88
|
@write_monitor.synchronize do
|
88
89
|
begin
|
@@ -98,13 +99,11 @@ module NSQ
|
|
98
99
|
end
|
99
100
|
end
|
100
101
|
|
101
|
-
def connect
|
102
|
+
def connect #:nodoc:
|
102
103
|
return unless verify_connect_state?(:init, :interval)
|
103
104
|
NSQ.logger.debug {"#{self}: Beginning connect"}
|
104
105
|
@connect_state = :connecting
|
105
106
|
@buffer = ''
|
106
|
-
@connecting = false
|
107
|
-
@connected = false
|
108
107
|
@ready_count = 0
|
109
108
|
@socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
110
109
|
@sockaddr = Socket.pack_sockaddr_in(@port, @host)
|
@@ -113,6 +112,10 @@ module NSQ
|
|
113
112
|
do_connect
|
114
113
|
end
|
115
114
|
|
115
|
+
def to_s #:nodoc:
|
116
|
+
@name
|
117
|
+
end
|
118
|
+
|
116
119
|
private
|
117
120
|
|
118
121
|
def do_connect
|
@@ -200,12 +203,6 @@ module NSQ
|
|
200
203
|
end
|
201
204
|
end
|
202
205
|
|
203
|
-
def to_s
|
204
|
-
@name
|
205
|
-
end
|
206
|
-
|
207
|
-
private
|
208
|
-
|
209
206
|
def verify_connect_state?(*states)
|
210
207
|
return true if states.include?(@connect_state)
|
211
208
|
NSQ.logger.error("Unexpected connect state of #{@connect_state}, expected to be in #{states.inspect}\n\t#{caller[0]}")
|
data/lib/nsq/message.rb
CHANGED
@@ -12,7 +12,9 @@ module NSQ
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def timestamp
|
15
|
-
|
15
|
+
# TODO: Not really a nanosecond timestamp
|
16
|
+
#Time.at((@timestamp_high * 2**32 + @timestamp_low) / 1000000000.0)
|
17
|
+
Time.at(@timestamp_low)
|
16
18
|
end
|
17
19
|
|
18
20
|
def to_s
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module NSQ
|
4
|
+
class Publisher
|
5
|
+
def initialize(host, port, options={}, &block)
|
6
|
+
@socket = TCPSocket.open(host, port)
|
7
|
+
@socket.write(MAGIC_V2)
|
8
|
+
@response_timeout = options[:response_timeout] || 5
|
9
|
+
yield self if block_given?
|
10
|
+
ensure
|
11
|
+
close if block_given?
|
12
|
+
end
|
13
|
+
|
14
|
+
def publish(topic, message)
|
15
|
+
buf = ['PUB ', topic, "\n", message.length, message].pack('a*a*a*Na*')
|
16
|
+
@socket.write(buf)
|
17
|
+
response = ''
|
18
|
+
loop do
|
19
|
+
response += @socket.recv(4096)
|
20
|
+
size, frame, msg = response.unpack('NNa*')
|
21
|
+
if response.length == size+4
|
22
|
+
case msg
|
23
|
+
when 'OK' then return
|
24
|
+
when 'E_INVALID' then raise 'Invalid message'
|
25
|
+
when 'E_BAD_TOPIC' then raise 'Bad topic'
|
26
|
+
when 'E_BAD_MESSAGE' then raise 'Bad message'
|
27
|
+
when 'E_PUT_FAILED' then raise 'Put failed'
|
28
|
+
else raise "Unknown PUB response: #{msg}"
|
29
|
+
end
|
30
|
+
elsif response.length > size+4
|
31
|
+
raise "Unexpected PUB response - Expected size = #{size} actual size = #{response.length-4}: message=#{msg}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def close
|
37
|
+
@socket.close
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/nsq/queue_subscriber.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
1
|
require 'thread' #Mutex
|
2
2
|
|
3
3
|
module NSQ
|
4
|
+
# An asynchronous subscriber that can be run on multiple threads for reading messages from a subscribed channel.
|
4
5
|
class QueueSubscriber < Subscriber
|
5
|
-
def initialize(reader, topic, channel, options)
|
6
|
+
def initialize(reader, topic, channel, options) #:nodoc:
|
6
7
|
super
|
7
8
|
@queue = Queue.new
|
8
9
|
@run_mutex = Mutex.new
|
9
10
|
@run_count = 0
|
10
11
|
end
|
11
12
|
|
12
|
-
def ready_count
|
13
|
+
def ready_count #:nodoc:
|
13
14
|
# Return the minimum of Subscriber#ready_count and the amount of space left in the queue
|
14
15
|
[super, self.max_in_flight - @queue.size].min
|
15
16
|
end
|
16
17
|
|
17
|
-
def handle_message(connection, message)
|
18
|
+
def handle_message(connection, message) #:nodoc:
|
18
19
|
@queue << [connection, message]
|
19
20
|
end
|
20
21
|
|
22
|
+
# Processes messages from the subscribed connections. This will not return until #stop
|
23
|
+
# has been called in a separate thread. This can be called from multiple threads if you
|
24
|
+
# want multiple workers handling the incoming messages.
|
21
25
|
def run(&block)
|
22
26
|
@run_mutex.synchronize { @run_count += 1}
|
23
27
|
until @stopped
|
24
28
|
pair = @queue.pop
|
25
29
|
if pair == :stop
|
30
|
+
# Give the next thread something to pop
|
26
31
|
@queue << :stop
|
27
32
|
return
|
28
33
|
end
|
@@ -33,6 +38,7 @@ module NSQ
|
|
33
38
|
@run_mutex.synchronize { @run_count -= 1}
|
34
39
|
end
|
35
40
|
|
41
|
+
# Stop this subscriber once all the queued messages have been handled.
|
36
42
|
def stop
|
37
43
|
@stopped = true
|
38
44
|
# Give the threads something to pop
|
data/lib/nsq/reader.rb
CHANGED
@@ -2,16 +2,39 @@ require 'socket'
|
|
2
2
|
require 'thread'
|
3
3
|
require 'monitor'
|
4
4
|
require 'nio'
|
5
|
-
#require 'thread_safe'
|
6
5
|
|
7
6
|
module NSQ
|
7
|
+
# Maintains a collection of subscribers to topics and channels.
|
8
8
|
class Reader
|
9
9
|
attr_reader :name, :long_id, :short_id, :selector, :options
|
10
10
|
|
11
|
+
# Create a new NSQ Reader
|
12
|
+
#
|
13
|
+
# Options (Refer to NSQ::Subscriber::new for additional options which will be passed on to each subscriber):
|
14
|
+
# :nsqd_tcp_addresses [String or Array of Strings]
|
15
|
+
# Array of nsqd servers to connect to with port numbers
|
16
|
+
# ['server1:4150', 'server2:4150']
|
17
|
+
#
|
18
|
+
# :lookupd_tcp_addresses [String or Array of Strings] (Not implemented)
|
19
|
+
# Array of nsq_lookupd servers to connect to with port numbers
|
20
|
+
# ['server1:4160', 'server2:4160']
|
21
|
+
#
|
22
|
+
# :lookupd_poll_interval [Float] (Not implemented)
|
23
|
+
# How often to poll the lookupd_tcp_addresses for new nsqd servers
|
24
|
+
# Default: 120
|
25
|
+
#
|
26
|
+
# :long_id [String]
|
27
|
+
# The identifier used as a long-form descriptor
|
28
|
+
# Default: fully-qualified hostname
|
29
|
+
#
|
30
|
+
# :short_id [String]
|
31
|
+
# The identifier used as a short-form descriptor
|
32
|
+
# Default: short hostname
|
33
|
+
#
|
11
34
|
def initialize(options={})
|
12
35
|
@options = options
|
13
36
|
@nsqd_tcp_addresses = s_to_a(options[:nsqd_tcp_addresses])
|
14
|
-
@
|
37
|
+
@lookupd_tcp_addresses = s_to_a(options[:lookupd_tcp_addresses])
|
15
38
|
@lookupd_poll_interval = options[:lookupd_poll_interval] || 120
|
16
39
|
@long_id = options[:long_id] || Socket.gethostname
|
17
40
|
@short_id = options[:short_id] || @long_id.split('.')[0]
|
@@ -27,25 +50,30 @@ module NSQ
|
|
27
50
|
|
28
51
|
raise 'Must pass either option :nsqd_tcp_addresses or :lookupd_http_addresses' if @nsqd_tcp_addresses.empty? && @lookupd_http_addresses.empty?
|
29
52
|
|
30
|
-
# TODO: If the messages are failing, the backoff timer will exponentially increase a timeout before sending a RDY
|
31
|
-
#self.backoff_timer = dict((k, BackoffTimer.BackoffTimer(0, 120)) for k in self.task_lookup.keys())
|
32
|
-
|
33
53
|
@conns = {}
|
34
54
|
@last_lookup = nil
|
35
55
|
|
36
56
|
@logger.info("starting reader for topic '%s'..." % self.topic) if @logger
|
37
57
|
end
|
38
58
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
59
|
+
# Subscribes to a given topic and channel.
|
60
|
+
#
|
61
|
+
# If a block is passed, then within NSQ::Reader#run that block will be run synchronously whenever a message
|
62
|
+
# is received for this channel.
|
63
|
+
#
|
64
|
+
# If a block is not passed, then the QueueSubscriber that is returned from this method should have it's
|
65
|
+
# QueueSubscriber#run method executed within one or more separate threads for processing the messages.
|
66
|
+
#
|
67
|
+
# Refer to Subscriber::new for the options that can be passed to this method.
|
68
|
+
#
|
69
|
+
def subscribe(topic, channel, options={}, &block)
|
70
|
+
Util.assert_topic_and_channel_valid(topic, channel)
|
43
71
|
subscriber = nil
|
44
72
|
name = "#{topic}:#{channel}"
|
45
73
|
@subscriber_mutex.synchronize do
|
46
74
|
raise "Already subscribed to #{name}" if @subscribers[name]
|
47
75
|
subscriber_class = block_given? ? Subscriber : QueueSubscriber
|
48
|
-
subscriber = @subscribers[name] = subscriber_class.new(self, topic, channel,
|
76
|
+
subscriber = @subscribers[name] = subscriber_class.new(self, topic, channel, options, &block)
|
49
77
|
end
|
50
78
|
|
51
79
|
@nsqd_tcp_addresses.each do |addr|
|
@@ -55,6 +83,7 @@ module NSQ
|
|
55
83
|
subscriber
|
56
84
|
end
|
57
85
|
|
86
|
+
# Unsubscribe a given topic and channel.
|
58
87
|
def unsubscribe(topic, channel)
|
59
88
|
name = "#{topic}:#{channel}"
|
60
89
|
@subscriber_mutex.synchronize do
|
@@ -65,6 +94,8 @@ module NSQ
|
|
65
94
|
end
|
66
95
|
end
|
67
96
|
|
97
|
+
# Processes all the messages from the subscribed connections. This will not return until #stop
|
98
|
+
# has been called in a separate thread.
|
68
99
|
def run
|
69
100
|
@stopped = false
|
70
101
|
until @stopped do
|
@@ -75,6 +106,7 @@ module NSQ
|
|
75
106
|
end
|
76
107
|
end
|
77
108
|
|
109
|
+
# Stop this reader which will gracefully exit the run method after all current messages are processed.
|
78
110
|
def stop
|
79
111
|
NSQ.logger.info("#{self}: Reader stopping...")
|
80
112
|
@stopped = true
|
@@ -84,14 +116,17 @@ module NSQ
|
|
84
116
|
end
|
85
117
|
end
|
86
118
|
|
119
|
+
# Call the given block from within the #run thread when the given interval has passed.
|
87
120
|
def add_timeout(interval, &block)
|
88
121
|
@timer.add(interval, &block)
|
89
122
|
end
|
90
123
|
|
91
|
-
def to_s
|
124
|
+
def to_s #:nodoc:
|
92
125
|
@name
|
93
126
|
end
|
94
127
|
|
128
|
+
private
|
129
|
+
|
95
130
|
def s_to_a(val)
|
96
131
|
val.kind_of?(String) ? [val] : val
|
97
132
|
end
|
data/lib/nsq/subscriber.rb
CHANGED
@@ -3,6 +3,53 @@ module NSQ
|
|
3
3
|
attr_reader :selector, :name
|
4
4
|
attr_accessor :max_in_flight
|
5
5
|
|
6
|
+
# Creates a new subscriber which maintain connections to all the nsqd instances which publish
|
7
|
+
# the given topic. This is never called directly but instead called when Reader#subscribe is called.
|
8
|
+
#
|
9
|
+
# Options:
|
10
|
+
# :max_tries [Integer]
|
11
|
+
# The max number of attempts to process a given message at which point it will no longer be requeued.
|
12
|
+
# Defaults to nil which means it will be requeued forever if it continues to fail.
|
13
|
+
#
|
14
|
+
# :max_in_flight [Integer]
|
15
|
+
# The number used to determine the RDY count sent for each connection.
|
16
|
+
# Defaults to 1
|
17
|
+
#
|
18
|
+
# :requeue_delay (msec) [Integer]
|
19
|
+
# The delay that is sent along with the requeue when a message fails.
|
20
|
+
# Defaults to 90,000 msec
|
21
|
+
#
|
22
|
+
# :ready_backoff_timer [Hash of BackoffTimer options]
|
23
|
+
# Options passed to a BackoffTimer for increasing the interval between ready counts when
|
24
|
+
# messages are failing.
|
25
|
+
# Options:
|
26
|
+
# :min_interval (seconds) [Float]
|
27
|
+
# The minimum interval that the BackoffTimer will return.
|
28
|
+
# Defaults to 0
|
29
|
+
#
|
30
|
+
# :max_interval (seconds) [Float]
|
31
|
+
# The maximum interval that the BackoffTimer will return.
|
32
|
+
# Defaults to 120
|
33
|
+
#
|
34
|
+
# :ratio [Float]
|
35
|
+
# Defaults to 0.25
|
36
|
+
#
|
37
|
+
# :short_length [Float]
|
38
|
+
# Defaults to 10
|
39
|
+
#
|
40
|
+
# :long_length [Float]
|
41
|
+
# Defaults to 250
|
42
|
+
#
|
43
|
+
# :connection_backoff_timer [Hash of BackoffTimer options]
|
44
|
+
# Options passed to a BackoffTimer for increasing the interval between connection attempts
|
45
|
+
# when a connection to nsqd is failing.
|
46
|
+
# Options (Refer to :ready_backoff_timer above for the meaning of these options):
|
47
|
+
# :min_interval (seconds) [Float]
|
48
|
+
# Defaults to 0
|
49
|
+
#
|
50
|
+
# :max_interval (seconds) [Float]
|
51
|
+
# Defaults to 30
|
52
|
+
#
|
6
53
|
def initialize(reader, topic, channel, options, &block)
|
7
54
|
options = reader.options.merge(options)
|
8
55
|
@name = "#{reader.name}:#{topic}:#{channel}"
|
@@ -34,40 +81,41 @@ module NSQ
|
|
34
81
|
raise "Invalid value for max_in_flight, must be between 0 and 2500: #{@max_in_flight}" unless @max_in_flight.between?(1,2499)
|
35
82
|
end
|
36
83
|
|
37
|
-
def create_ready_backoff_timer
|
84
|
+
def create_ready_backoff_timer #:nodoc:
|
38
85
|
BackoffTimer.new(@ready_min_interval, @ready_max_interval, @ready_ratio, @ready_short_length, @ready_long_length)
|
39
86
|
end
|
40
87
|
|
41
|
-
def create_connection_backoff_timer
|
88
|
+
def create_connection_backoff_timer #:nodoc:
|
42
89
|
BackoffTimer.new(@connection_min_interval, @connection_max_interval, @connection_ratio, @connection_short_length, @connection_long_length)
|
43
90
|
end
|
44
91
|
|
45
92
|
# Threshold for a connection where it's time to send a new READY message
|
46
|
-
def ready_threshold
|
93
|
+
def ready_threshold #:nodoc:
|
47
94
|
@max_in_flight / @connection_hash.size / 4
|
48
95
|
end
|
49
96
|
|
50
97
|
# The actual value for the READY message
|
51
|
-
def ready_count
|
98
|
+
def ready_count #:nodoc:
|
52
99
|
# TODO: Should we take into account the last_ready_count minus the number of messages sent since then?
|
53
100
|
# Rounding up!
|
54
101
|
(@max_in_flight + @connection_hash.size - 1) / @connection_hash.size
|
55
102
|
end
|
56
103
|
|
57
|
-
def connection_count
|
104
|
+
def connection_count #:nodoc:
|
58
105
|
@connection_hash.size
|
59
106
|
end
|
60
107
|
|
61
|
-
def add_connection(host, port)
|
108
|
+
def add_connection(host, port) #:nodoc:
|
62
109
|
@connection_hash[[host, port]] = Connection.new(@reader, self, host, port)
|
63
110
|
end
|
64
111
|
|
65
|
-
def remove_connection(host, port)
|
112
|
+
def remove_connection(host, port) #:nodoc:
|
66
113
|
connection = @connection_hash.delete([host, port])
|
67
114
|
return unless connection
|
68
115
|
connection.close
|
69
116
|
end
|
70
117
|
|
118
|
+
# Stop this subscriber
|
71
119
|
def stop
|
72
120
|
@stopped = true
|
73
121
|
@connection_hash.each_value do |connection|
|
@@ -76,22 +124,23 @@ module NSQ
|
|
76
124
|
@connection_hash.clear
|
77
125
|
end
|
78
126
|
|
127
|
+
# Return true if this subscriber has been stopped
|
79
128
|
def stopped?
|
80
129
|
@stopped
|
81
130
|
end
|
82
131
|
|
83
|
-
def handle_connection(connection)
|
132
|
+
def handle_connection(connection) #:nodoc:
|
84
133
|
connection.send_init(@topic, @channel, @reader.short_id, @reader.long_id)
|
85
134
|
end
|
86
135
|
|
87
|
-
def handle_heartbeat(connection)
|
136
|
+
def handle_heartbeat(connection) #:nodoc:
|
88
137
|
end
|
89
138
|
|
90
|
-
def handle_message(connection, message)
|
139
|
+
def handle_message(connection, message) #:nodoc:
|
91
140
|
process_message(connection, message, &@block)
|
92
141
|
end
|
93
142
|
|
94
|
-
def process_message(connection, message, &block)
|
143
|
+
def process_message(connection, message, &block) #:nodoc:
|
95
144
|
yield message
|
96
145
|
connection.send_finish(message.id, true)
|
97
146
|
rescue Exception => e
|
@@ -104,17 +153,17 @@ module NSQ
|
|
104
153
|
end
|
105
154
|
end
|
106
155
|
|
107
|
-
def handle_frame_error(connection, error_message)
|
156
|
+
def handle_frame_error(connection, error_message) #:nodoc:
|
108
157
|
NSQ.logger.error("Received error from nsqd: #{error_message.inspect}")
|
109
158
|
connection.reset
|
110
159
|
end
|
111
160
|
|
112
|
-
def handle_io_error(connection, exception)
|
161
|
+
def handle_io_error(connection, exception) #:nodoc:
|
113
162
|
NSQ.logger.error("Socket error: #{exception.message}\n\t#{exception.backtrace[0,2].join("\n\t")}")
|
114
163
|
connection.reset
|
115
164
|
end
|
116
165
|
|
117
|
-
def to_s
|
166
|
+
def to_s #:nodoc:
|
118
167
|
@name
|
119
168
|
end
|
120
169
|
end
|
data/lib/nsq/timer.rb
CHANGED
data/lib/nsq/util.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
module NSQ
|
2
|
+
module Util
|
3
|
+
|
4
|
+
def self.assert_topic_and_channel_valid(topic, channel) #:nodoc:
|
5
|
+
raise "Invalid topic #{topic}" unless valid_topic_name?(topic)
|
6
|
+
raise "Invalid channel #{channel}" unless valid_channel_name?(channel)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.valid_topic_name?(topic) #:nodoc:
|
10
|
+
!!topic.match(/^[\.a-zA-Z0-9_-]+$/)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.valid_channel_name?(channel) #:nodoc:
|
14
|
+
!!channel.match(/^[\.a-zA-Z0-9_-]+(#ephemeral)?$/)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/nsq.rb
CHANGED
@@ -2,10 +2,12 @@ require 'nsq/loggable'
|
|
2
2
|
require 'nsq/message'
|
3
3
|
require 'nsq/reader'
|
4
4
|
require 'nsq/subscriber'
|
5
|
+
require 'nsq/publisher'
|
5
6
|
require 'nsq/queue_subscriber'
|
6
7
|
require 'nsq/connection'
|
7
8
|
require 'nsq/backoff_timer'
|
8
9
|
require 'nsq/timer'
|
10
|
+
require 'nsq/util'
|
9
11
|
|
10
12
|
module NSQ
|
11
13
|
extend NSQ::Loggable
|
@@ -16,20 +18,8 @@ module NSQ
|
|
16
18
|
FRAME_TYPE_ERROR = 1
|
17
19
|
FRAME_TYPE_MESSAGE = 2
|
18
20
|
|
19
|
-
def self.create_reader(options, &block)
|
21
|
+
def self.create_reader(options, &block) #:nodoc:
|
22
|
+
NSQ.logger.info('NSQ#create_reader has been deprecated, please use NSQ::Reader#new instead')
|
20
23
|
Reader.new(options, &block)
|
21
24
|
end
|
22
|
-
|
23
|
-
def self.assert_topic_and_channel_valid(topic, channel)
|
24
|
-
raise "Invalid topic #{topic}" unless valid_topic_name?(topic)
|
25
|
-
raise "Invalid channel #{channel}" unless valid_channel_name?(channel)
|
26
|
-
end
|
27
|
-
|
28
|
-
def self.valid_topic_name?(topic)
|
29
|
-
!!topic.match(/^[\.a-zA-Z0-9_-]+$/)
|
30
|
-
end
|
31
|
-
|
32
|
-
def self.valid_channel_name?(channel)
|
33
|
-
!!channel.match(/^[\.a-zA-Z0-9_-]+(#ephemeral)?$/)
|
34
|
-
end
|
35
25
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_nsq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: nio4r
|
@@ -107,7 +107,7 @@ dependencies:
|
|
107
107
|
- - ! '>='
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: '0'
|
110
|
-
description: Ruby client for NSQ
|
110
|
+
description: Ruby client for the NSQ realtime message processing system
|
111
111
|
email:
|
112
112
|
- bradpardee@gmail.com
|
113
113
|
executables: []
|
@@ -118,15 +118,20 @@ files:
|
|
118
118
|
- lib/nsq/connection.rb
|
119
119
|
- lib/nsq/loggable.rb
|
120
120
|
- lib/nsq/message.rb
|
121
|
+
- lib/nsq/publisher.rb
|
121
122
|
- lib/nsq/queue_subscriber.rb
|
122
123
|
- lib/nsq/reader.rb
|
123
124
|
- lib/nsq/subscriber.rb
|
124
125
|
- lib/nsq/timer.rb
|
126
|
+
- lib/nsq/util.rb
|
125
127
|
- lib/nsq.rb
|
126
128
|
- lib/ruby_nsq.rb
|
129
|
+
- examples/async/publisher.rb
|
127
130
|
- examples/async/reader.rb
|
128
|
-
- examples/async/
|
131
|
+
- examples/async/README
|
132
|
+
- examples/simple/publisher.rb
|
129
133
|
- examples/simple/reader.rb
|
134
|
+
- examples/simple/README
|
130
135
|
- LICENSE.txt
|
131
136
|
- Rakefile
|
132
137
|
- History.md
|
@@ -147,7 +152,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
147
152
|
version: '0'
|
148
153
|
segments:
|
149
154
|
- 0
|
150
|
-
hash:
|
155
|
+
hash: 1267684542890965568
|
151
156
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
152
157
|
none: false
|
153
158
|
requirements:
|
@@ -156,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
161
|
version: '0'
|
157
162
|
segments:
|
158
163
|
- 0
|
159
|
-
hash:
|
164
|
+
hash: 1267684542890965568
|
160
165
|
requirements: []
|
161
166
|
rubyforge_project:
|
162
167
|
rubygems_version: 1.8.23
|
data/examples/async/writer.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
if ARGV.length != 3
|
4
|
-
$stderr.puts "ruby writer.rb <topic> <count> <eval-string>"
|
5
|
-
$stderr.puts " where <topic> is either test_xy or test_z"
|
6
|
-
$stderr.puts " and <eval-string> could be something like 'sleep rand(100)/10.0'"
|
7
|
-
$stderr.puts " Example: ./writer.rb test_xy 500 'sleep rand(100)/10.0'"
|
8
|
-
$stderr.puts " or: ./writer.rb test_z 5000 nil"
|
9
|
-
exit 1
|
10
|
-
end
|
11
|
-
topic = ARGV[0]
|
12
|
-
count = ARGV[1].to_i
|
13
|
-
eval_string = ARGV[2]
|
14
|
-
# TODO: Figure out TCP protocol
|
15
|
-
count.times do
|
16
|
-
system "curl -d #{eval_string.inspect} 'http://127.0.0.1:4151/put?topic=#{topic}'"
|
17
|
-
end
|