ruby_nsq 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/History.md +9 -1
- data/README.md +21 -8
- data/examples/async/README +6 -0
- data/examples/async/publisher.rb +21 -0
- data/examples/async/reader.rb +13 -11
- data/examples/simple/README +5 -0
- data/examples/simple/publisher.rb +13 -0
- data/examples/simple/reader.rb +7 -4
- data/lib/nsq/backoff_timer.rb +2 -1
- data/lib/nsq/connection.rb +12 -15
- data/lib/nsq/message.rb +3 -1
- data/lib/nsq/publisher.rb +40 -0
- data/lib/nsq/queue_subscriber.rb +9 -3
- data/lib/nsq/reader.rb +46 -11
- data/lib/nsq/subscriber.rb +63 -14
- data/lib/nsq/timer.rb +1 -0
- data/lib/nsq/util.rb +17 -0
- data/lib/nsq.rb +4 -14
- metadata +11 -6
- data/examples/async/writer.rb +0 -17
data/History.md
CHANGED
data/README.md
CHANGED
@@ -2,15 +2,15 @@
|
|
2
2
|
|
3
3
|
https://github.com/ClarityServices/ruby_nsq
|
4
4
|
|
5
|
-
## Description
|
5
|
+
## Description
|
6
6
|
|
7
7
|
Ruby client for the [NSQ](https://github.com/bitly/nsq) realtime message processing system.
|
8
8
|
|
9
|
-
## Install
|
9
|
+
## Install
|
10
10
|
|
11
11
|
gem install ruby_nsq
|
12
12
|
|
13
|
-
## Usage
|
13
|
+
## Usage
|
14
14
|
|
15
15
|
See [examples](https://github.com/ClarityServices/ruby_nsq/tree/master/examples)
|
16
16
|
|
@@ -18,7 +18,7 @@ Simple example for synchronous message handling:
|
|
18
18
|
```
|
19
19
|
require 'nsq'
|
20
20
|
|
21
|
-
reader = NSQ.
|
21
|
+
reader = NSQ::Reader.new(:nsqd_tcp_addresses => '127.0.0.1:4150')
|
22
22
|
# Subscribe to topic=test channel=simple
|
23
23
|
reader.subscribe('test', 'simple') do |message|
|
24
24
|
# If this block raises an exception, then the message will be requeued.
|
@@ -36,7 +36,7 @@ foo_worker_count = 50
|
|
36
36
|
bar_worker_count = 30
|
37
37
|
baz_worker_count = 20
|
38
38
|
|
39
|
-
reader = NSQ.
|
39
|
+
reader = NSQ::Reader.new(:nsqd_tcp_addresses => '127.0.0.1:4150')
|
40
40
|
|
41
41
|
foo_subscriber = reader.subscribe('test', 'foo', :max_in_flight => foo_worker_count)
|
42
42
|
bar_subscriber = reader.subscribe('test2', 'bar', :max_in_flight => bar_worker_count)
|
@@ -62,10 +62,23 @@ bar_threads.each(&:join)
|
|
62
62
|
baz_threads.each(&:join)
|
63
63
|
```
|
64
64
|
|
65
|
-
## TODO
|
66
|
-
|
67
|
-
* Fix timestamp
|
65
|
+
## TODO
|
68
66
|
|
69
67
|
* Implement lookupd
|
70
68
|
|
71
69
|
* Tests!
|
70
|
+
|
71
|
+
* Documentation
|
72
|
+
|
73
|
+
## Meta
|
74
|
+
|
75
|
+
* Code: `git clone git://github.com/ClarityServices/ruby_nsq.git`
|
76
|
+
* Home: <https://github.com/ClarityServices/ruby_nsq>
|
77
|
+
* Bugs: <http://github.com/reidmorrison/ruby_nsq/issues>
|
78
|
+
* Gems: <http://rubygems.org/gems/ruby_nsq>
|
79
|
+
|
80
|
+
This project uses [Semantic Versioning](http://semver.org/).
|
81
|
+
|
82
|
+
## Authors
|
83
|
+
|
84
|
+
Brad Pardee :: bradpardee@gmail.com
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nsq'
|
4
|
+
|
5
|
+
if ARGV.length != 3
|
6
|
+
$stderr.puts "bundle exec ./publisher.rb <topic> <count> <eval-string>"
|
7
|
+
$stderr.puts " where <topic> is either test_xy or test_z"
|
8
|
+
$stderr.puts " and <eval-string> could be something like 'sleep rand(100)/10.0'"
|
9
|
+
$stderr.puts " Example: bundle exec ./publisher.rb test_xy 500 'sleep rand(100)/10.0'"
|
10
|
+
$stderr.puts " or: bundle exec ./publisher.rb test_z 5000 nil"
|
11
|
+
exit 1
|
12
|
+
end
|
13
|
+
topic = ARGV[0]
|
14
|
+
count = ARGV[1].to_i
|
15
|
+
eval_string = ARGV[2]
|
16
|
+
|
17
|
+
NSQ::Publisher.new('localhost', 4150) do |publisher|
|
18
|
+
count.times do
|
19
|
+
publisher.publish(topic, eval_string)
|
20
|
+
end
|
21
|
+
end
|
data/examples/async/reader.rb
CHANGED
@@ -11,9 +11,9 @@ z_worker_count = 20
|
|
11
11
|
puts 'Press enter to start and enter to finish'
|
12
12
|
$stdin.gets
|
13
13
|
|
14
|
-
reader = NSQ.
|
14
|
+
reader = NSQ::Reader.new(
|
15
15
|
:nsqd_tcp_addresses => '127.0.0.1:4150',
|
16
|
-
|
16
|
+
#:logger_level => Logger::DEBUG
|
17
17
|
)
|
18
18
|
|
19
19
|
x_subscriber = reader.subscribe('test_xy', 'x', :max_in_flight => x_worker_count)
|
@@ -46,13 +46,15 @@ end
|
|
46
46
|
main_thread = Thread.new do
|
47
47
|
reader.run
|
48
48
|
end
|
49
|
+
at_exit {
|
50
|
+
puts 'Exiting...'
|
51
|
+
reader.stop
|
52
|
+
main_thread.join
|
53
|
+
threads.each_value { |arr| arr.each(&:join) }
|
54
|
+
puts
|
55
|
+
puts "Summary of worker message counts"
|
56
|
+
threads.each do |char, arr|
|
57
|
+
puts "#{char} - #{arr.map(&:message_count).join(' ')} total=#{arr.map(&:message_count).inject(:+)}"
|
58
|
+
end
|
59
|
+
}
|
49
60
|
$stdin.gets
|
50
|
-
puts 'Exiting...'
|
51
|
-
reader.stop
|
52
|
-
main_thread.join
|
53
|
-
threads.each_value { |arr| arr.each(&:join) }
|
54
|
-
puts
|
55
|
-
puts "Summary of worker message counts"
|
56
|
-
threads.each do |char, arr|
|
57
|
-
puts "#{char} - #{arr.map(&:message_count).join(' ')}"
|
58
|
-
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nsq'
|
4
|
+
|
5
|
+
if ARGV.length == 0
|
6
|
+
$stderr.puts "bundle exec ./publisher.rb <message>*"
|
7
|
+
$stderr.puts " Example: bundle exec ./publisher.rb hello world"
|
8
|
+
exit 1
|
9
|
+
end
|
10
|
+
|
11
|
+
NSQ::Publisher.new('localhost', 4150) do |publisher|
|
12
|
+
publisher.publish('test', ARGV.join(' '))
|
13
|
+
end
|
data/examples/simple/reader.rb
CHANGED
@@ -3,16 +3,17 @@
|
|
3
3
|
require 'nsq'
|
4
4
|
require 'logger'
|
5
5
|
|
6
|
+
# Cntl-c doesn't run at_exit under jruby
|
6
7
|
puts 'Press enter to start and enter to finish'
|
7
8
|
gets
|
8
|
-
reader = NSQ.
|
9
|
+
reader = NSQ::Reader.new(
|
9
10
|
:nsqd_tcp_addresses => '127.0.0.1:4150',
|
10
11
|
#:logger_level => Logger::DEBUG
|
11
12
|
)
|
12
13
|
thread = Thread.new do
|
13
14
|
begin
|
14
15
|
reader.subscribe('test', 'simple') do |message|
|
15
|
-
puts "Read #{message.body}"
|
16
|
+
puts "Read: #{message.body.inspect}"
|
16
17
|
end
|
17
18
|
reader.run
|
18
19
|
rescue Exception => e
|
@@ -20,6 +21,8 @@ thread = Thread.new do
|
|
20
21
|
end
|
21
22
|
puts 'Reader exiting'
|
22
23
|
end
|
24
|
+
at_exit {
|
25
|
+
reader.stop
|
26
|
+
thread.join
|
27
|
+
}
|
23
28
|
gets
|
24
|
-
reader.stop
|
25
|
-
thread.join
|
data/lib/nsq/backoff_timer.rb
CHANGED
@@ -34,7 +34,8 @@ module NSQ
|
|
34
34
|
@short_interval = [@short_interval, @max_short_timer].min
|
35
35
|
@long_interval = [@long_interval, @max_long_timer].min
|
36
36
|
end
|
37
|
-
|
37
|
+
|
38
|
+
# Return the interval to wait based on the successes and failures
|
38
39
|
def interval
|
39
40
|
@min_interval + @short_interval + @long_interval
|
40
41
|
end
|
data/lib/nsq/connection.rb
CHANGED
@@ -2,6 +2,7 @@ require 'monitor'
|
|
2
2
|
require 'thread' #Mutex
|
3
3
|
|
4
4
|
module NSQ
|
5
|
+
# Represents a single subscribed connection to an nsqd server.
|
5
6
|
class Connection
|
6
7
|
attr_reader :name
|
7
8
|
|
@@ -27,19 +28,19 @@ module NSQ
|
|
27
28
|
connect
|
28
29
|
end
|
29
30
|
|
30
|
-
def send_init(topic, channel, short_id, long_id)
|
31
|
+
def send_init(topic, channel, short_id, long_id) #:nodoc:
|
31
32
|
write NSQ::MAGIC_V2
|
32
33
|
write "SUB #{topic} #{channel} #{short_id} #{long_id}\n"
|
33
34
|
self.send_ready
|
34
35
|
end
|
35
36
|
|
36
|
-
def send_ready
|
37
|
+
def send_ready #:nodoc:
|
37
38
|
@ready_count = @subscriber.ready_count
|
38
39
|
write "RDY #{@ready_count}\n" unless @subscriber.stopped?
|
39
40
|
@sending_ready = false
|
40
41
|
end
|
41
42
|
|
42
|
-
def send_finish(id, success)
|
43
|
+
def send_finish(id, success) #:nodoc:
|
43
44
|
write "FIN #{id}\n"
|
44
45
|
@ready_mutex.synchronize do
|
45
46
|
@ready_count -= 1
|
@@ -52,7 +53,7 @@ module NSQ
|
|
52
53
|
end
|
53
54
|
end
|
54
55
|
|
55
|
-
def send_requeue(id, time_ms)
|
56
|
+
def send_requeue(id, time_ms) #:nodoc:
|
56
57
|
write "REQ #{id} #{time_ms}\n"
|
57
58
|
@ready_mutex.synchronize do
|
58
59
|
@ready_count -= 1
|
@@ -61,7 +62,7 @@ module NSQ
|
|
61
62
|
end
|
62
63
|
end
|
63
64
|
|
64
|
-
def reset
|
65
|
+
def reset #:nodoc:
|
65
66
|
return unless verify_connect_state?(:connecting, :connected)
|
66
67
|
# Close with the hopes of re-establishing
|
67
68
|
close(false)
|
@@ -82,7 +83,7 @@ module NSQ
|
|
82
83
|
end
|
83
84
|
end
|
84
85
|
|
85
|
-
def close(permanent=true)
|
86
|
+
def close(permanent=true) #:nodoc:
|
86
87
|
NSQ.logger.debug {"#{@name}: Closing..."}
|
87
88
|
@write_monitor.synchronize do
|
88
89
|
begin
|
@@ -98,13 +99,11 @@ module NSQ
|
|
98
99
|
end
|
99
100
|
end
|
100
101
|
|
101
|
-
def connect
|
102
|
+
def connect #:nodoc:
|
102
103
|
return unless verify_connect_state?(:init, :interval)
|
103
104
|
NSQ.logger.debug {"#{self}: Beginning connect"}
|
104
105
|
@connect_state = :connecting
|
105
106
|
@buffer = ''
|
106
|
-
@connecting = false
|
107
|
-
@connected = false
|
108
107
|
@ready_count = 0
|
109
108
|
@socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
110
109
|
@sockaddr = Socket.pack_sockaddr_in(@port, @host)
|
@@ -113,6 +112,10 @@ module NSQ
|
|
113
112
|
do_connect
|
114
113
|
end
|
115
114
|
|
115
|
+
def to_s #:nodoc:
|
116
|
+
@name
|
117
|
+
end
|
118
|
+
|
116
119
|
private
|
117
120
|
|
118
121
|
def do_connect
|
@@ -200,12 +203,6 @@ module NSQ
|
|
200
203
|
end
|
201
204
|
end
|
202
205
|
|
203
|
-
def to_s
|
204
|
-
@name
|
205
|
-
end
|
206
|
-
|
207
|
-
private
|
208
|
-
|
209
206
|
def verify_connect_state?(*states)
|
210
207
|
return true if states.include?(@connect_state)
|
211
208
|
NSQ.logger.error("Unexpected connect state of #{@connect_state}, expected to be in #{states.inspect}\n\t#{caller[0]}")
|
data/lib/nsq/message.rb
CHANGED
@@ -12,7 +12,9 @@ module NSQ
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def timestamp
|
15
|
-
|
15
|
+
# TODO: Not really a nanosecond timestamp
|
16
|
+
#Time.at((@timestamp_high * 2**32 + @timestamp_low) / 1000000000.0)
|
17
|
+
Time.at(@timestamp_low)
|
16
18
|
end
|
17
19
|
|
18
20
|
def to_s
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module NSQ
|
4
|
+
class Publisher
|
5
|
+
def initialize(host, port, options={}, &block)
|
6
|
+
@socket = TCPSocket.open(host, port)
|
7
|
+
@socket.write(MAGIC_V2)
|
8
|
+
@response_timeout = options[:response_timeout] || 5
|
9
|
+
yield self if block_given?
|
10
|
+
ensure
|
11
|
+
close if block_given?
|
12
|
+
end
|
13
|
+
|
14
|
+
def publish(topic, message)
|
15
|
+
buf = ['PUB ', topic, "\n", message.length, message].pack('a*a*a*Na*')
|
16
|
+
@socket.write(buf)
|
17
|
+
response = ''
|
18
|
+
loop do
|
19
|
+
response += @socket.recv(4096)
|
20
|
+
size, frame, msg = response.unpack('NNa*')
|
21
|
+
if response.length == size+4
|
22
|
+
case msg
|
23
|
+
when 'OK' then return
|
24
|
+
when 'E_INVALID' then raise 'Invalid message'
|
25
|
+
when 'E_BAD_TOPIC' then raise 'Bad topic'
|
26
|
+
when 'E_BAD_MESSAGE' then raise 'Bad message'
|
27
|
+
when 'E_PUT_FAILED' then raise 'Put failed'
|
28
|
+
else raise "Unknown PUB response: #{msg}"
|
29
|
+
end
|
30
|
+
elsif response.length > size+4
|
31
|
+
raise "Unexpected PUB response - Expected size = #{size} actual size = #{response.length-4}: message=#{msg}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def close
|
37
|
+
@socket.close
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/nsq/queue_subscriber.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
1
|
require 'thread' #Mutex
|
2
2
|
|
3
3
|
module NSQ
|
4
|
+
# An asynchronous subscriber that can be run on multiple threads for reading messages from a subscribed channel.
|
4
5
|
class QueueSubscriber < Subscriber
|
5
|
-
def initialize(reader, topic, channel, options)
|
6
|
+
def initialize(reader, topic, channel, options) #:nodoc:
|
6
7
|
super
|
7
8
|
@queue = Queue.new
|
8
9
|
@run_mutex = Mutex.new
|
9
10
|
@run_count = 0
|
10
11
|
end
|
11
12
|
|
12
|
-
def ready_count
|
13
|
+
def ready_count #:nodoc:
|
13
14
|
# Return the minimum of Subscriber#ready_count and the amount of space left in the queue
|
14
15
|
[super, self.max_in_flight - @queue.size].min
|
15
16
|
end
|
16
17
|
|
17
|
-
def handle_message(connection, message)
|
18
|
+
def handle_message(connection, message) #:nodoc:
|
18
19
|
@queue << [connection, message]
|
19
20
|
end
|
20
21
|
|
22
|
+
# Processes messages from the subscribed connections. This will not return until #stop
|
23
|
+
# has been called in a separate thread. This can be called from multiple threads if you
|
24
|
+
# want multiple workers handling the incoming messages.
|
21
25
|
def run(&block)
|
22
26
|
@run_mutex.synchronize { @run_count += 1}
|
23
27
|
until @stopped
|
24
28
|
pair = @queue.pop
|
25
29
|
if pair == :stop
|
30
|
+
# Give the next thread something to pop
|
26
31
|
@queue << :stop
|
27
32
|
return
|
28
33
|
end
|
@@ -33,6 +38,7 @@ module NSQ
|
|
33
38
|
@run_mutex.synchronize { @run_count -= 1}
|
34
39
|
end
|
35
40
|
|
41
|
+
# Stop this subscriber once all the queued messages have been handled.
|
36
42
|
def stop
|
37
43
|
@stopped = true
|
38
44
|
# Give the threads something to pop
|
data/lib/nsq/reader.rb
CHANGED
@@ -2,16 +2,39 @@ require 'socket'
|
|
2
2
|
require 'thread'
|
3
3
|
require 'monitor'
|
4
4
|
require 'nio'
|
5
|
-
#require 'thread_safe'
|
6
5
|
|
7
6
|
module NSQ
|
7
|
+
# Maintains a collection of subscribers to topics and channels.
|
8
8
|
class Reader
|
9
9
|
attr_reader :name, :long_id, :short_id, :selector, :options
|
10
10
|
|
11
|
+
# Create a new NSQ Reader
|
12
|
+
#
|
13
|
+
# Options (Refer to NSQ::Subscriber::new for additional options which will be passed on to each subscriber):
|
14
|
+
# :nsqd_tcp_addresses [String or Array of Strings]
|
15
|
+
# Array of nsqd servers to connect to with port numbers
|
16
|
+
# ['server1:4150', 'server2:4150']
|
17
|
+
#
|
18
|
+
# :lookupd_tcp_addresses [String or Array of Strings] (Not implemented)
|
19
|
+
# Array of nsq_lookupd servers to connect to with port numbers
|
20
|
+
# ['server1:4160', 'server2:4160']
|
21
|
+
#
|
22
|
+
# :lookupd_poll_interval [Float] (Not implemented)
|
23
|
+
# How often to poll the lookupd_tcp_addresses for new nsqd servers
|
24
|
+
# Default: 120
|
25
|
+
#
|
26
|
+
# :long_id [String]
|
27
|
+
# The identifier used as a long-form descriptor
|
28
|
+
# Default: fully-qualified hostname
|
29
|
+
#
|
30
|
+
# :short_id [String]
|
31
|
+
# The identifier used as a short-form descriptor
|
32
|
+
# Default: short hostname
|
33
|
+
#
|
11
34
|
def initialize(options={})
|
12
35
|
@options = options
|
13
36
|
@nsqd_tcp_addresses = s_to_a(options[:nsqd_tcp_addresses])
|
14
|
-
@
|
37
|
+
@lookupd_tcp_addresses = s_to_a(options[:lookupd_tcp_addresses])
|
15
38
|
@lookupd_poll_interval = options[:lookupd_poll_interval] || 120
|
16
39
|
@long_id = options[:long_id] || Socket.gethostname
|
17
40
|
@short_id = options[:short_id] || @long_id.split('.')[0]
|
@@ -27,25 +50,30 @@ module NSQ
|
|
27
50
|
|
28
51
|
raise 'Must pass either option :nsqd_tcp_addresses or :lookupd_http_addresses' if @nsqd_tcp_addresses.empty? && @lookupd_http_addresses.empty?
|
29
52
|
|
30
|
-
# TODO: If the messages are failing, the backoff timer will exponentially increase a timeout before sending a RDY
|
31
|
-
#self.backoff_timer = dict((k, BackoffTimer.BackoffTimer(0, 120)) for k in self.task_lookup.keys())
|
32
|
-
|
33
53
|
@conns = {}
|
34
54
|
@last_lookup = nil
|
35
55
|
|
36
56
|
@logger.info("starting reader for topic '%s'..." % self.topic) if @logger
|
37
57
|
end
|
38
58
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
59
|
+
# Subscribes to a given topic and channel.
|
60
|
+
#
|
61
|
+
# If a block is passed, then within NSQ::Reader#run that block will be run synchronously whenever a message
|
62
|
+
# is received for this channel.
|
63
|
+
#
|
64
|
+
# If a block is not passed, then the QueueSubscriber that is returned from this method should have it's
|
65
|
+
# QueueSubscriber#run method executed within one or more separate threads for processing the messages.
|
66
|
+
#
|
67
|
+
# Refer to Subscriber::new for the options that can be passed to this method.
|
68
|
+
#
|
69
|
+
def subscribe(topic, channel, options={}, &block)
|
70
|
+
Util.assert_topic_and_channel_valid(topic, channel)
|
43
71
|
subscriber = nil
|
44
72
|
name = "#{topic}:#{channel}"
|
45
73
|
@subscriber_mutex.synchronize do
|
46
74
|
raise "Already subscribed to #{name}" if @subscribers[name]
|
47
75
|
subscriber_class = block_given? ? Subscriber : QueueSubscriber
|
48
|
-
subscriber = @subscribers[name] = subscriber_class.new(self, topic, channel,
|
76
|
+
subscriber = @subscribers[name] = subscriber_class.new(self, topic, channel, options, &block)
|
49
77
|
end
|
50
78
|
|
51
79
|
@nsqd_tcp_addresses.each do |addr|
|
@@ -55,6 +83,7 @@ module NSQ
|
|
55
83
|
subscriber
|
56
84
|
end
|
57
85
|
|
86
|
+
# Unsubscribe a given topic and channel.
|
58
87
|
def unsubscribe(topic, channel)
|
59
88
|
name = "#{topic}:#{channel}"
|
60
89
|
@subscriber_mutex.synchronize do
|
@@ -65,6 +94,8 @@ module NSQ
|
|
65
94
|
end
|
66
95
|
end
|
67
96
|
|
97
|
+
# Processes all the messages from the subscribed connections. This will not return until #stop
|
98
|
+
# has been called in a separate thread.
|
68
99
|
def run
|
69
100
|
@stopped = false
|
70
101
|
until @stopped do
|
@@ -75,6 +106,7 @@ module NSQ
|
|
75
106
|
end
|
76
107
|
end
|
77
108
|
|
109
|
+
# Stop this reader which will gracefully exit the run method after all current messages are processed.
|
78
110
|
def stop
|
79
111
|
NSQ.logger.info("#{self}: Reader stopping...")
|
80
112
|
@stopped = true
|
@@ -84,14 +116,17 @@ module NSQ
|
|
84
116
|
end
|
85
117
|
end
|
86
118
|
|
119
|
+
# Call the given block from within the #run thread when the given interval has passed.
|
87
120
|
def add_timeout(interval, &block)
|
88
121
|
@timer.add(interval, &block)
|
89
122
|
end
|
90
123
|
|
91
|
-
def to_s
|
124
|
+
def to_s #:nodoc:
|
92
125
|
@name
|
93
126
|
end
|
94
127
|
|
128
|
+
private
|
129
|
+
|
95
130
|
def s_to_a(val)
|
96
131
|
val.kind_of?(String) ? [val] : val
|
97
132
|
end
|
data/lib/nsq/subscriber.rb
CHANGED
@@ -3,6 +3,53 @@ module NSQ
|
|
3
3
|
attr_reader :selector, :name
|
4
4
|
attr_accessor :max_in_flight
|
5
5
|
|
6
|
+
# Creates a new subscriber which maintain connections to all the nsqd instances which publish
|
7
|
+
# the given topic. This is never called directly but instead called when Reader#subscribe is called.
|
8
|
+
#
|
9
|
+
# Options:
|
10
|
+
# :max_tries [Integer]
|
11
|
+
# The max number of attempts to process a given message at which point it will no longer be requeued.
|
12
|
+
# Defaults to nil which means it will be requeued forever if it continues to fail.
|
13
|
+
#
|
14
|
+
# :max_in_flight [Integer]
|
15
|
+
# The number used to determine the RDY count sent for each connection.
|
16
|
+
# Defaults to 1
|
17
|
+
#
|
18
|
+
# :requeue_delay (msec) [Integer]
|
19
|
+
# The delay that is sent along with the requeue when a message fails.
|
20
|
+
# Defaults to 90,000 msec
|
21
|
+
#
|
22
|
+
# :ready_backoff_timer [Hash of BackoffTimer options]
|
23
|
+
# Options passed to a BackoffTimer for increasing the interval between ready counts when
|
24
|
+
# messages are failing.
|
25
|
+
# Options:
|
26
|
+
# :min_interval (seconds) [Float]
|
27
|
+
# The minimum interval that the BackoffTimer will return.
|
28
|
+
# Defaults to 0
|
29
|
+
#
|
30
|
+
# :max_interval (seconds) [Float]
|
31
|
+
# The maximum interval that the BackoffTimer will return.
|
32
|
+
# Defaults to 120
|
33
|
+
#
|
34
|
+
# :ratio [Float]
|
35
|
+
# Defaults to 0.25
|
36
|
+
#
|
37
|
+
# :short_length [Float]
|
38
|
+
# Defaults to 10
|
39
|
+
#
|
40
|
+
# :long_length [Float]
|
41
|
+
# Defaults to 250
|
42
|
+
#
|
43
|
+
# :connection_backoff_timer [Hash of BackoffTimer options]
|
44
|
+
# Options passed to a BackoffTimer for increasing the interval between connection attempts
|
45
|
+
# when a connection to nsqd is failing.
|
46
|
+
# Options (Refer to :ready_backoff_timer above for the meaning of these options):
|
47
|
+
# :min_interval (seconds) [Float]
|
48
|
+
# Defaults to 0
|
49
|
+
#
|
50
|
+
# :max_interval (seconds) [Float]
|
51
|
+
# Defaults to 30
|
52
|
+
#
|
6
53
|
def initialize(reader, topic, channel, options, &block)
|
7
54
|
options = reader.options.merge(options)
|
8
55
|
@name = "#{reader.name}:#{topic}:#{channel}"
|
@@ -34,40 +81,41 @@ module NSQ
|
|
34
81
|
raise "Invalid value for max_in_flight, must be between 0 and 2500: #{@max_in_flight}" unless @max_in_flight.between?(1,2499)
|
35
82
|
end
|
36
83
|
|
37
|
-
def create_ready_backoff_timer
|
84
|
+
def create_ready_backoff_timer #:nodoc:
|
38
85
|
BackoffTimer.new(@ready_min_interval, @ready_max_interval, @ready_ratio, @ready_short_length, @ready_long_length)
|
39
86
|
end
|
40
87
|
|
41
|
-
def create_connection_backoff_timer
|
88
|
+
def create_connection_backoff_timer #:nodoc:
|
42
89
|
BackoffTimer.new(@connection_min_interval, @connection_max_interval, @connection_ratio, @connection_short_length, @connection_long_length)
|
43
90
|
end
|
44
91
|
|
45
92
|
# Threshold for a connection where it's time to send a new READY message
|
46
|
-
def ready_threshold
|
93
|
+
def ready_threshold #:nodoc:
|
47
94
|
@max_in_flight / @connection_hash.size / 4
|
48
95
|
end
|
49
96
|
|
50
97
|
# The actual value for the READY message
|
51
|
-
def ready_count
|
98
|
+
def ready_count #:nodoc:
|
52
99
|
# TODO: Should we take into account the last_ready_count minus the number of messages sent since then?
|
53
100
|
# Rounding up!
|
54
101
|
(@max_in_flight + @connection_hash.size - 1) / @connection_hash.size
|
55
102
|
end
|
56
103
|
|
57
|
-
def connection_count
|
104
|
+
def connection_count #:nodoc:
|
58
105
|
@connection_hash.size
|
59
106
|
end
|
60
107
|
|
61
|
-
def add_connection(host, port)
|
108
|
+
def add_connection(host, port) #:nodoc:
|
62
109
|
@connection_hash[[host, port]] = Connection.new(@reader, self, host, port)
|
63
110
|
end
|
64
111
|
|
65
|
-
def remove_connection(host, port)
|
112
|
+
def remove_connection(host, port) #:nodoc:
|
66
113
|
connection = @connection_hash.delete([host, port])
|
67
114
|
return unless connection
|
68
115
|
connection.close
|
69
116
|
end
|
70
117
|
|
118
|
+
# Stop this subscriber
|
71
119
|
def stop
|
72
120
|
@stopped = true
|
73
121
|
@connection_hash.each_value do |connection|
|
@@ -76,22 +124,23 @@ module NSQ
|
|
76
124
|
@connection_hash.clear
|
77
125
|
end
|
78
126
|
|
127
|
+
# Return true if this subscriber has been stopped
|
79
128
|
def stopped?
|
80
129
|
@stopped
|
81
130
|
end
|
82
131
|
|
83
|
-
def handle_connection(connection)
|
132
|
+
def handle_connection(connection) #:nodoc:
|
84
133
|
connection.send_init(@topic, @channel, @reader.short_id, @reader.long_id)
|
85
134
|
end
|
86
135
|
|
87
|
-
def handle_heartbeat(connection)
|
136
|
+
def handle_heartbeat(connection) #:nodoc:
|
88
137
|
end
|
89
138
|
|
90
|
-
def handle_message(connection, message)
|
139
|
+
def handle_message(connection, message) #:nodoc:
|
91
140
|
process_message(connection, message, &@block)
|
92
141
|
end
|
93
142
|
|
94
|
-
def process_message(connection, message, &block)
|
143
|
+
def process_message(connection, message, &block) #:nodoc:
|
95
144
|
yield message
|
96
145
|
connection.send_finish(message.id, true)
|
97
146
|
rescue Exception => e
|
@@ -104,17 +153,17 @@ module NSQ
|
|
104
153
|
end
|
105
154
|
end
|
106
155
|
|
107
|
-
def handle_frame_error(connection, error_message)
|
156
|
+
def handle_frame_error(connection, error_message) #:nodoc:
|
108
157
|
NSQ.logger.error("Received error from nsqd: #{error_message.inspect}")
|
109
158
|
connection.reset
|
110
159
|
end
|
111
160
|
|
112
|
-
def handle_io_error(connection, exception)
|
161
|
+
def handle_io_error(connection, exception) #:nodoc:
|
113
162
|
NSQ.logger.error("Socket error: #{exception.message}\n\t#{exception.backtrace[0,2].join("\n\t")}")
|
114
163
|
connection.reset
|
115
164
|
end
|
116
165
|
|
117
|
-
def to_s
|
166
|
+
def to_s #:nodoc:
|
118
167
|
@name
|
119
168
|
end
|
120
169
|
end
|
data/lib/nsq/timer.rb
CHANGED
data/lib/nsq/util.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
module NSQ
|
2
|
+
module Util
|
3
|
+
|
4
|
+
def self.assert_topic_and_channel_valid(topic, channel) #:nodoc:
|
5
|
+
raise "Invalid topic #{topic}" unless valid_topic_name?(topic)
|
6
|
+
raise "Invalid channel #{channel}" unless valid_channel_name?(channel)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.valid_topic_name?(topic) #:nodoc:
|
10
|
+
!!topic.match(/^[\.a-zA-Z0-9_-]+$/)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.valid_channel_name?(channel) #:nodoc:
|
14
|
+
!!channel.match(/^[\.a-zA-Z0-9_-]+(#ephemeral)?$/)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/nsq.rb
CHANGED
@@ -2,10 +2,12 @@ require 'nsq/loggable'
|
|
2
2
|
require 'nsq/message'
|
3
3
|
require 'nsq/reader'
|
4
4
|
require 'nsq/subscriber'
|
5
|
+
require 'nsq/publisher'
|
5
6
|
require 'nsq/queue_subscriber'
|
6
7
|
require 'nsq/connection'
|
7
8
|
require 'nsq/backoff_timer'
|
8
9
|
require 'nsq/timer'
|
10
|
+
require 'nsq/util'
|
9
11
|
|
10
12
|
module NSQ
|
11
13
|
extend NSQ::Loggable
|
@@ -16,20 +18,8 @@ module NSQ
|
|
16
18
|
FRAME_TYPE_ERROR = 1
|
17
19
|
FRAME_TYPE_MESSAGE = 2
|
18
20
|
|
19
|
-
def self.create_reader(options, &block)
|
21
|
+
def self.create_reader(options, &block) #:nodoc:
|
22
|
+
NSQ.logger.info('NSQ#create_reader has been deprecated, please use NSQ::Reader#new instead')
|
20
23
|
Reader.new(options, &block)
|
21
24
|
end
|
22
|
-
|
23
|
-
def self.assert_topic_and_channel_valid(topic, channel)
|
24
|
-
raise "Invalid topic #{topic}" unless valid_topic_name?(topic)
|
25
|
-
raise "Invalid channel #{channel}" unless valid_channel_name?(channel)
|
26
|
-
end
|
27
|
-
|
28
|
-
def self.valid_topic_name?(topic)
|
29
|
-
!!topic.match(/^[\.a-zA-Z0-9_-]+$/)
|
30
|
-
end
|
31
|
-
|
32
|
-
def self.valid_channel_name?(channel)
|
33
|
-
!!channel.match(/^[\.a-zA-Z0-9_-]+(#ephemeral)?$/)
|
34
|
-
end
|
35
25
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_nsq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: nio4r
|
@@ -107,7 +107,7 @@ dependencies:
|
|
107
107
|
- - ! '>='
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: '0'
|
110
|
-
description: Ruby client for NSQ
|
110
|
+
description: Ruby client for the NSQ realtime message processing system
|
111
111
|
email:
|
112
112
|
- bradpardee@gmail.com
|
113
113
|
executables: []
|
@@ -118,15 +118,20 @@ files:
|
|
118
118
|
- lib/nsq/connection.rb
|
119
119
|
- lib/nsq/loggable.rb
|
120
120
|
- lib/nsq/message.rb
|
121
|
+
- lib/nsq/publisher.rb
|
121
122
|
- lib/nsq/queue_subscriber.rb
|
122
123
|
- lib/nsq/reader.rb
|
123
124
|
- lib/nsq/subscriber.rb
|
124
125
|
- lib/nsq/timer.rb
|
126
|
+
- lib/nsq/util.rb
|
125
127
|
- lib/nsq.rb
|
126
128
|
- lib/ruby_nsq.rb
|
129
|
+
- examples/async/publisher.rb
|
127
130
|
- examples/async/reader.rb
|
128
|
-
- examples/async/
|
131
|
+
- examples/async/README
|
132
|
+
- examples/simple/publisher.rb
|
129
133
|
- examples/simple/reader.rb
|
134
|
+
- examples/simple/README
|
130
135
|
- LICENSE.txt
|
131
136
|
- Rakefile
|
132
137
|
- History.md
|
@@ -147,7 +152,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
147
152
|
version: '0'
|
148
153
|
segments:
|
149
154
|
- 0
|
150
|
-
hash:
|
155
|
+
hash: 1267684542890965568
|
151
156
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
152
157
|
none: false
|
153
158
|
requirements:
|
@@ -156,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
161
|
version: '0'
|
157
162
|
segments:
|
158
163
|
- 0
|
159
|
-
hash:
|
164
|
+
hash: 1267684542890965568
|
160
165
|
requirements: []
|
161
166
|
rubyforge_project:
|
162
167
|
rubygems_version: 1.8.23
|
data/examples/async/writer.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
if ARGV.length != 3
|
4
|
-
$stderr.puts "ruby writer.rb <topic> <count> <eval-string>"
|
5
|
-
$stderr.puts " where <topic> is either test_xy or test_z"
|
6
|
-
$stderr.puts " and <eval-string> could be something like 'sleep rand(100)/10.0'"
|
7
|
-
$stderr.puts " Example: ./writer.rb test_xy 500 'sleep rand(100)/10.0'"
|
8
|
-
$stderr.puts " or: ./writer.rb test_z 5000 nil"
|
9
|
-
exit 1
|
10
|
-
end
|
11
|
-
topic = ARGV[0]
|
12
|
-
count = ARGV[1].to_i
|
13
|
-
eval_string = ARGV[2]
|
14
|
-
# TODO: Figure out TCP protocol
|
15
|
-
count.times do
|
16
|
-
system "curl -d #{eval_string.inspect} 'http://127.0.0.1:4151/put?topic=#{topic}'"
|
17
|
-
end
|