nsq-krakow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +84 -0
- data/CONTRIBUTING.md +25 -0
- data/LICENSE +13 -0
- data/README.md +249 -0
- data/krakow.gemspec +22 -0
- data/lib/krakow.rb +25 -0
- data/lib/krakow/command.rb +89 -0
- data/lib/krakow/command/auth.rb +36 -0
- data/lib/krakow/command/cls.rb +24 -0
- data/lib/krakow/command/fin.rb +31 -0
- data/lib/krakow/command/identify.rb +55 -0
- data/lib/krakow/command/mpub.rb +39 -0
- data/lib/krakow/command/nop.rb +14 -0
- data/lib/krakow/command/pub.rb +37 -0
- data/lib/krakow/command/rdy.rb +31 -0
- data/lib/krakow/command/req.rb +32 -0
- data/lib/krakow/command/sub.rb +36 -0
- data/lib/krakow/command/touch.rb +31 -0
- data/lib/krakow/connection.rb +417 -0
- data/lib/krakow/connection_features.rb +10 -0
- data/lib/krakow/connection_features/deflate.rb +82 -0
- data/lib/krakow/connection_features/snappy_frames.rb +129 -0
- data/lib/krakow/connection_features/ssl.rb +75 -0
- data/lib/krakow/consumer.rb +355 -0
- data/lib/krakow/consumer/queue.rb +151 -0
- data/lib/krakow/discovery.rb +57 -0
- data/lib/krakow/distribution.rb +229 -0
- data/lib/krakow/distribution/default.rb +159 -0
- data/lib/krakow/exceptions.rb +30 -0
- data/lib/krakow/frame_type.rb +66 -0
- data/lib/krakow/frame_type/error.rb +26 -0
- data/lib/krakow/frame_type/message.rb +74 -0
- data/lib/krakow/frame_type/response.rb +26 -0
- data/lib/krakow/ksocket.rb +102 -0
- data/lib/krakow/producer.rb +162 -0
- data/lib/krakow/producer/http.rb +224 -0
- data/lib/krakow/utils.rb +9 -0
- data/lib/krakow/utils/lazy.rb +125 -0
- data/lib/krakow/utils/logging.rb +43 -0
- data/lib/krakow/version.rb +4 -0
- metadata +184 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
# Base error type
|
5
|
+
class Error < StandardError
|
6
|
+
|
7
|
+
# Failed to enable required feature on connection
|
8
|
+
class ConnectionFeatureFailure < Error; end
|
9
|
+
# Failed to perform lookup (not found)
|
10
|
+
class LookupFailed < Error; end
|
11
|
+
# Connection has failed
|
12
|
+
class ConnectionFailure < Error; end
|
13
|
+
# Configuration is not in valid state
|
14
|
+
class ConfigurationError < Error; end
|
15
|
+
# Connection is temporarily unavailable
|
16
|
+
class ConnectionUnavailable < Error; end
|
17
|
+
# Consumer was not set
|
18
|
+
class OriginNotFound < Error; end
|
19
|
+
|
20
|
+
# Invalid response
|
21
|
+
class BadResponse < Error
|
22
|
+
# @return [Response] error response
|
23
|
+
attr_accessor :result
|
24
|
+
# No response received
|
25
|
+
class NoResponse < BadResponse
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
# Received message
|
5
|
+
# @abstract
|
6
|
+
class FrameType
|
7
|
+
|
8
|
+
autoload :Error, 'krakow/frame_type/error'
|
9
|
+
autoload :Message, 'krakow/frame_type/message'
|
10
|
+
autoload :Response, 'krakow/frame_type/response'
|
11
|
+
|
12
|
+
include Utils::Lazy
|
13
|
+
# @!parse include Krakow::Utils::Lazy::InstanceMethods
|
14
|
+
# @!parse extend Krakow::Utils::Lazy::ClassMethods
|
15
|
+
|
16
|
+
# Registered frame types
|
17
|
+
FRAME_TYPE_MAP = [
|
18
|
+
FrameType::Response,
|
19
|
+
FrameType::Error,
|
20
|
+
FrameType::Message
|
21
|
+
]
|
22
|
+
# Size bytes
|
23
|
+
SIZE_BYTES = 4
|
24
|
+
|
25
|
+
class << self
|
26
|
+
|
27
|
+
# Information about incoming frame
|
28
|
+
# @param bytes [String]
|
29
|
+
# @return [Hash]
|
30
|
+
def decode(bytes)
|
31
|
+
size, type = bytes.unpack('l>l>')
|
32
|
+
{:size => size - SIZE_BYTES, :type => type}
|
33
|
+
end
|
34
|
+
|
35
|
+
# Build proper FrameType instance based on args
|
36
|
+
# @param args [Hash]
|
37
|
+
# @option args [FrameType] :type class of frame
|
38
|
+
# @option args [String] :data
|
39
|
+
# @option args [Integer] :size
|
40
|
+
# @return [FrameType]
|
41
|
+
def build(args={})
|
42
|
+
klass = FRAME_TYPE_MAP[args[:type].to_i]
|
43
|
+
if(klass == FrameType::Response)
|
44
|
+
klass.new(:response => args[:data])
|
45
|
+
elsif(klass == FrameType::Error)
|
46
|
+
klass.new(:error => args[:data])
|
47
|
+
elsif(klass == FrameType::Message)
|
48
|
+
unpacked = args[:data].unpack("Q>s>a16a#{args[:size]}")
|
49
|
+
klass.new(
|
50
|
+
Hash[*([:timestamp, :attempts, :message_id, :message].zip(unpacked).flatten)]
|
51
|
+
)
|
52
|
+
else
|
53
|
+
raise TypeError.new "Unknown frame type received: #{args[:type].inspect} - #{klass.inspect}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# Content of message
|
59
|
+
#
|
60
|
+
# @return [String]
|
61
|
+
def content
|
62
|
+
raise NotImplementedError.new 'Content method not properly defined!'
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
class FrameType
|
5
|
+
# Error from server
|
6
|
+
class Error < FrameType
|
7
|
+
|
8
|
+
# @!group Attributes
|
9
|
+
|
10
|
+
# @!macro [attach] attribute
|
11
|
+
# @!method $1
|
12
|
+
# @return [$2] the $1 $0
|
13
|
+
# @!method $1?
|
14
|
+
# @return [TrueClass, FalseClass] truthiness of the $1 $0
|
15
|
+
attribute :error, String, :required => true
|
16
|
+
|
17
|
+
# @!endgroup
|
18
|
+
|
19
|
+
# @return [String] content of error
|
20
|
+
def content
|
21
|
+
error
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
class FrameType
|
5
|
+
# Message received from server
|
6
|
+
class Message < FrameType
|
7
|
+
|
8
|
+
# @return [Float] time of message instance creation
|
9
|
+
attr_reader :instance_stamp
|
10
|
+
attr_accessor :origin, :connection
|
11
|
+
|
12
|
+
# @!group Attributes
|
13
|
+
|
14
|
+
# @!macro [attach] attribute
|
15
|
+
# @!method $1
|
16
|
+
# @return [$2] the $1 $0
|
17
|
+
# @!method $1?
|
18
|
+
# @return [TrueClass, FalseClass] truthiness of the $1 $0
|
19
|
+
attribute :attempts, Integer, :required => true
|
20
|
+
attribute :timestamp, Integer, :required => true
|
21
|
+
attribute :message_id, String, :required => true
|
22
|
+
attribute :message, String, :required => true
|
23
|
+
|
24
|
+
# @!endgroup
|
25
|
+
|
26
|
+
def initialize(*args)
|
27
|
+
super
|
28
|
+
@instance_stamp = Time.now.to_f
|
29
|
+
end
|
30
|
+
|
31
|
+
# Message content
|
32
|
+
#
|
33
|
+
# @return [String]
|
34
|
+
def content
|
35
|
+
message
|
36
|
+
end
|
37
|
+
|
38
|
+
# @return [Krakow::Consumer]
|
39
|
+
def origin
|
40
|
+
unless(@origin)
|
41
|
+
error 'No origin has been specified for this message'
|
42
|
+
abort Krakow::Error::OriginNotFound.new('No origin specified for this message')
|
43
|
+
end
|
44
|
+
@origin
|
45
|
+
end
|
46
|
+
|
47
|
+
# @return [Krakow::Connection]
|
48
|
+
def connection
|
49
|
+
unless(@connection)
|
50
|
+
error 'No origin connection has been specified for this message'
|
51
|
+
abort Krakow::Error::ConnectionNotFound.new('No connection specified for this message')
|
52
|
+
end
|
53
|
+
@connection
|
54
|
+
end
|
55
|
+
|
56
|
+
# Proxy to [Krakow::Consumer#confirm]
|
57
|
+
def confirm(*args)
|
58
|
+
origin.confirm(*[self, *args].compact)
|
59
|
+
end
|
60
|
+
alias_method :finish, :confirm
|
61
|
+
|
62
|
+
# Proxy to [Krakow::Consumer#requeue]
|
63
|
+
def requeue(*args)
|
64
|
+
origin.requeue(*[self, *args].compact)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Proxy to [Krakow::Consumer#touch]
|
68
|
+
def touch(*args)
|
69
|
+
origin.touch(*[self, *args].compact)
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
class FrameType
|
5
|
+
# Response from server
|
6
|
+
class Response < FrameType
|
7
|
+
|
8
|
+
# @!group Attributes
|
9
|
+
|
10
|
+
# @!macro [attach] attribute
|
11
|
+
# @!method $1
|
12
|
+
# @return [$2] the $1 $0
|
13
|
+
# @!method $1?
|
14
|
+
# @return [TrueClass, FalseClass] truthiness of the $1 $0
|
15
|
+
attribute :response, String, :required => true
|
16
|
+
|
17
|
+
# @!endgroup
|
18
|
+
|
19
|
+
# @return [String] content of response
|
20
|
+
def content
|
21
|
+
response
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
require 'socket'
|
3
|
+
|
4
|
+
module Krakow
|
5
|
+
class Ksocket
|
6
|
+
|
7
|
+
include Utils::Lazy
|
8
|
+
include Celluloid
|
9
|
+
|
10
|
+
# @return [String]
|
11
|
+
attr_reader :buffer
|
12
|
+
|
13
|
+
finalizer :closedown_socket
|
14
|
+
|
15
|
+
# Teardown helper
|
16
|
+
def closedown_socket
|
17
|
+
@writing = @reading = false
|
18
|
+
if(socket && !socket.closed?)
|
19
|
+
socket.close
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Create new socket wrapper
|
24
|
+
#
|
25
|
+
# @param args [Hash]
|
26
|
+
# @option args [Socket-ish] :socket
|
27
|
+
# @option args [String] :host
|
28
|
+
# @option args [Integer] :port
|
29
|
+
# @return [self]
|
30
|
+
def initialize(args={})
|
31
|
+
if(args[:socket])
|
32
|
+
@socket = args[:socket]
|
33
|
+
else
|
34
|
+
unless([:host, :port].all?{|k| args.include?(k)})
|
35
|
+
raise ArgumentError.new 'Missing required arguments. Expecting `:socket` or `:host` and `:port`.'
|
36
|
+
end
|
37
|
+
@socket = TCPSocket.new(args[:host], args[:port])
|
38
|
+
end
|
39
|
+
@buffer = ''
|
40
|
+
async.read_loop
|
41
|
+
end
|
42
|
+
|
43
|
+
# @return [TrueClass, FalseClass] read loop enabled
|
44
|
+
def reading?
|
45
|
+
!!@reading
|
46
|
+
end
|
47
|
+
|
48
|
+
# Read from socket and push into local Queue
|
49
|
+
def read_loop
|
50
|
+
unless(reading?)
|
51
|
+
@reading = true
|
52
|
+
while(reading?)
|
53
|
+
res = defer do
|
54
|
+
Kernel.select([socket], nil, nil, nil)
|
55
|
+
socket{|s| s.readpartial(1024)}
|
56
|
+
end
|
57
|
+
if(res)
|
58
|
+
debug "Received content from socket: #{res.inspect}"
|
59
|
+
buffer << res
|
60
|
+
signal(:content_read)
|
61
|
+
else
|
62
|
+
debug 'No content received from socket read. Ignoring.'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# Fetch bytes from socket
|
69
|
+
#
|
70
|
+
# @param n [Integer]
|
71
|
+
# @return [String]
|
72
|
+
def get(n)
|
73
|
+
until(buffer.length >= n)
|
74
|
+
wait(:content_read)
|
75
|
+
end
|
76
|
+
buffer.slice!(0, n)
|
77
|
+
end
|
78
|
+
alias_method :recv, :get
|
79
|
+
alias_method :read, :get
|
80
|
+
alias_method :sysread, :get
|
81
|
+
alias_method :readpartial, :get
|
82
|
+
|
83
|
+
# Push bytes to socket
|
84
|
+
#
|
85
|
+
# @param line [String]
|
86
|
+
# @return [Integer]
|
87
|
+
def put(line)
|
88
|
+
socket{|s| s.write(line)}
|
89
|
+
end
|
90
|
+
alias_method :write, :put
|
91
|
+
|
92
|
+
# @return [Socket]
|
93
|
+
def socket
|
94
|
+
if(block_given?)
|
95
|
+
yield @socket
|
96
|
+
else
|
97
|
+
@socket
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
require 'krakow'
|
2
|
+
|
3
|
+
module Krakow
|
4
|
+
|
5
|
+
# TCP based producer
|
6
|
+
class Producer
|
7
|
+
|
8
|
+
autoload :Http, 'krakow/producer/http'
|
9
|
+
|
10
|
+
include Utils::Lazy
|
11
|
+
# @!parse include Utils::Lazy::InstanceMethods
|
12
|
+
# @!parse extend Utils::Lazy::ClassMethods
|
13
|
+
|
14
|
+
include Celluloid
|
15
|
+
|
16
|
+
trap_exit :connection_failure
|
17
|
+
finalizer :producer_cleanup
|
18
|
+
|
19
|
+
# set exclusive methods
|
20
|
+
exclusive :write
|
21
|
+
|
22
|
+
attr_reader :connection
|
23
|
+
attr_reader :notifier
|
24
|
+
|
25
|
+
# @!group Attributes
|
26
|
+
|
27
|
+
# @!macro [attach] attribute
|
28
|
+
# @!method $1
|
29
|
+
# @return [$2] the $1 $0
|
30
|
+
# @!method $1?
|
31
|
+
# @return [TrueClass, FalseClass] truthiness of the $1 $0
|
32
|
+
attribute :host, String, :required => true
|
33
|
+
attribute :port, [String, Integer], :required => true
|
34
|
+
attribute :topic, String, :required => true
|
35
|
+
attribute :reconnect_retries, Integer, :default => 10
|
36
|
+
attribute :reconnect_interval, Integer, :default => 5
|
37
|
+
attribute :connection_options, Hash, :default => ->{ Hash.new }
|
38
|
+
|
39
|
+
# @!endgroup
|
40
|
+
|
41
|
+
def initialize(args={})
|
42
|
+
super
|
43
|
+
arguments[:connection_options] = {:features => {}, :config => {}, :options => {}}.merge(
|
44
|
+
arguments.fetch(:connection_options, {})
|
45
|
+
)
|
46
|
+
connect
|
47
|
+
end
|
48
|
+
|
49
|
+
# Establish connection to configured `host` and `port`
|
50
|
+
#
|
51
|
+
# @return nil
|
52
|
+
def connect
|
53
|
+
@connecting = true
|
54
|
+
info "Establishing connection to: #{host}:#{port}"
|
55
|
+
begin
|
56
|
+
con_args = connection_options[:options].dup.tap do |args|
|
57
|
+
args[:host] = host
|
58
|
+
args[:port] = port
|
59
|
+
if(connection_options[:features])
|
60
|
+
args[:features] = connection_options[:features]
|
61
|
+
end
|
62
|
+
if(connection_options[:config])
|
63
|
+
args[:features_args] = connection_options[:config]
|
64
|
+
end
|
65
|
+
end
|
66
|
+
@connection = Connection.new(con_args)
|
67
|
+
@connection.init!
|
68
|
+
self.link @connection
|
69
|
+
info "Connection established: #{@connection}"
|
70
|
+
nil
|
71
|
+
rescue => e
|
72
|
+
abort e
|
73
|
+
end
|
74
|
+
@connecting = false
|
75
|
+
end
|
76
|
+
|
77
|
+
# @return [String] stringify object
|
78
|
+
def to_s
|
79
|
+
"<#{self.class.name}:#{object_id} {#{host}:#{port}} T:#{topic}>"
|
80
|
+
end
|
81
|
+
|
82
|
+
# @return [TrueClass, FalseClass] currently connected to server
|
83
|
+
def connected?
|
84
|
+
begin
|
85
|
+
!!(!@connecting &&
|
86
|
+
connection &&
|
87
|
+
connection.alive? &&
|
88
|
+
connection.connected?)
|
89
|
+
rescue Celluloid::DeadActorError
|
90
|
+
false
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# Process connection failure and attempt reconnection
|
95
|
+
#
|
96
|
+
# @return [TrueClass]
|
97
|
+
def connection_failure(obj, reason)
|
98
|
+
if(obj == connection && !reason.nil?)
|
99
|
+
begin
|
100
|
+
@connection = nil
|
101
|
+
warn "Connection failure detected for #{host}:#{port} - #{reason}"
|
102
|
+
obj.terminate if obj.alive?
|
103
|
+
connect
|
104
|
+
rescue => reason
|
105
|
+
warn "Failed to establish connection to #{host}:#{port}. Pausing #{reconnect_interval} before retry"
|
106
|
+
sleep reconnect_interval
|
107
|
+
retry
|
108
|
+
end
|
109
|
+
end
|
110
|
+
true
|
111
|
+
end
|
112
|
+
|
113
|
+
# Instance destructor
|
114
|
+
# @return nil
|
115
|
+
def producer_cleanup
|
116
|
+
debug 'Tearing down producer'
|
117
|
+
if(connection && connection.alive?)
|
118
|
+
connection.terminate
|
119
|
+
end
|
120
|
+
@connection = nil
|
121
|
+
info 'Producer torn down'
|
122
|
+
nil
|
123
|
+
end
|
124
|
+
|
125
|
+
# Write message to server
|
126
|
+
#
|
127
|
+
# @param message [String] message to write
|
128
|
+
# @return [Krakow::FrameType, TrueClass]
|
129
|
+
# @note if connection response wait is set to 0, writes will
|
130
|
+
# return a `true` value on completion
|
131
|
+
# @raise [Krakow::Error::ConnectionUnavailable]
|
132
|
+
def write(*message)
|
133
|
+
if(message.empty?)
|
134
|
+
abort ArgumentError.new 'Expecting one or more messages to send. None provided.'
|
135
|
+
end
|
136
|
+
begin
|
137
|
+
if(message.size > 1)
|
138
|
+
debug 'Multiple message publish'
|
139
|
+
connection.transmit(
|
140
|
+
Command::Mpub.new(
|
141
|
+
:topic_name => topic,
|
142
|
+
:messages => message
|
143
|
+
)
|
144
|
+
)
|
145
|
+
else
|
146
|
+
debug 'Single message publish'
|
147
|
+
connection.transmit(
|
148
|
+
Command::Pub.new(
|
149
|
+
:message => message.first,
|
150
|
+
:topic_name => topic
|
151
|
+
)
|
152
|
+
)
|
153
|
+
end
|
154
|
+
rescue Celluloid::Task::TerminatedError
|
155
|
+
abort Error::ConnectionUnavailable.new 'Connection is currently unavailable'
|
156
|
+
rescue => e
|
157
|
+
abort e
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
end
|