krakow 0.1.2 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## v0.2.0
2
+ * Fix the rest of the namespacing issues
3
+ * Start adding some tests
4
+ * Use better exception types (NotImplementedError instead of NoMethodError)
5
+ * Be smart about responses within connections
6
+ * Add snappy support
7
+ * Add deflate support
8
+ * Add TLS support
9
+ * Prevent division by zero in distribution
10
+ * Add query methods to lazy helper (`attribute_name`?)
11
+
1
12
  ## v0.1.2
2
13
  * Include backoff support
3
14
  * Remove `method_missing` magic
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
+ gem 'minitest'
4
+
3
5
  gemspec
data/Gemfile.lock CHANGED
@@ -1,10 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- krakow (0.1.1)
4
+ krakow (0.1.3)
5
5
  celluloid-io
6
+ digest-crc
6
7
  http
7
8
  multi_json
9
+ snappy
8
10
 
9
11
  GEM
10
12
  remote: https://rubygems.org/
@@ -14,11 +16,14 @@ GEM
14
16
  celluloid-io (0.15.0)
15
17
  celluloid (>= 0.15.0)
16
18
  nio4r (>= 0.5.0)
19
+ digest-crc (0.4.0)
17
20
  http (0.5.0)
18
21
  http_parser.rb
19
22
  http_parser.rb (0.6.0)
23
+ minitest (5.0.8)
20
24
  multi_json (1.8.4)
21
- nio4r (0.5.0)
25
+ nio4r (1.0.0)
26
+ snappy (0.0.10)
22
27
  timers (1.1.0)
23
28
 
24
29
  PLATFORMS
@@ -26,3 +31,4 @@ PLATFORMS
26
31
 
27
32
  DEPENDENCIES
28
33
  krakow!
34
+ minitest
data/README.md CHANGED
@@ -124,6 +124,51 @@ consumer = Krakow::Consumer.new(
124
124
  )
125
125
  ```
126
126
 
127
+ ### I need TLS!
128
+
129
+ OK!
130
+
131
+ ```ruby
132
+ consumer = Krakow::Consumer.new(
133
+ :nsqlookupd => 'http://HOST:PORT',
134
+ :topic => 'target',
135
+ :channel => 'ship',
136
+ :connection_features => {
137
+ :tls_v1 => true
138
+ }
139
+ )
140
+ ```
141
+
142
+ ### I need Snappy compression!
143
+
144
+ OK!
145
+
146
+ ```ruby
147
+ consumer = Krakow::Consumer.new(
148
+ :nsqlookupd => 'http://HOST:PORT',
149
+ :topic => 'target',
150
+ :channel => 'ship',
151
+ :connection_features => {
152
+ :snappy => true
153
+ }
154
+ )
155
+ ```
156
+
157
+ ### I need Deflate compression!
158
+
159
+ OK!
160
+
161
+ ```ruby
162
+ consumer = Krakow::Consumer.new(
163
+ :nsqlookupd => 'http://HOST:PORT',
164
+ :topic => 'target',
165
+ :channel => 'ship',
166
+ :connection_features => {
167
+ :deflate => true
168
+ }
169
+ )
170
+ ```
171
+
127
172
  ### It doesn't work
128
173
 
129
174
  Create an issue on the github repository.
data/krakow.gemspec CHANGED
@@ -12,5 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.add_dependency 'celluloid-io'
13
13
  s.add_dependency 'http'
14
14
  s.add_dependency 'multi_json'
15
+ s.add_dependency 'snappy'
16
+ s.add_dependency 'digest-crc'
15
17
  s.files = Dir['**/*']
16
18
  end
@@ -34,6 +34,7 @@ module Krakow
34
34
  def error
35
35
  %w(E_INVALID E_BAD_BODY)
36
36
  end
37
+
37
38
  end
38
39
 
39
40
  end
@@ -13,6 +13,20 @@ module Krakow
13
13
  []
14
14
  end
15
15
 
16
+ # message:: Krakow::Message
17
+ # Returns type of response expected (:none, :error_only, :required)
18
+ def response_for(message)
19
+ if(message.class.ok.empty?)
20
+ if(message.class.error.empty?)
21
+ :none
22
+ else
23
+ :error_only
24
+ end
25
+ else
26
+ :required
27
+ end
28
+ end
29
+
16
30
  end
17
31
 
18
32
  attr_accessor :response
@@ -23,8 +37,8 @@ module Krakow
23
37
  end
24
38
 
25
39
  # Convert to line output
26
- def to_line
27
- raise NoMethodError.new 'No line conversion method defined!'
40
+ def to_line(*args)
41
+ raise NotImplementedError.new 'No line conversion method defined!'
28
42
  end
29
43
 
30
44
  def ok?(response)
@@ -1,3 +1,4 @@
1
+ require 'krakow/version'
1
2
  require 'celluloid/io'
2
3
  require 'celluloid/autostart'
3
4
 
@@ -7,18 +8,42 @@ module Krakow
7
8
  include Utils::Lazy
8
9
  include Celluloid::IO
9
10
 
11
+ FEATURES = [
12
+ :max_rdy_count,
13
+ :max_msg_timeout,
14
+ :msg_timeout,
15
+ :tls_v1,
16
+ :deflate,
17
+ :deflate_level,
18
+ :max_deflate_level,
19
+ :snappy,
20
+ :sample_rate
21
+ ]
22
+ EXCLUSIVE_FEATURES = [[:snappy, :deflate]]
23
+ ENABLEABLE_FEATURES = [:tls_v1, :snappy, :deflate]
24
+
10
25
  finalizer :goodbye_my_love!
11
26
 
12
- attr_reader :socket
27
+ attr_reader :socket, :endpoint_settings
13
28
 
14
29
  def initialize(args={})
15
30
  super
16
31
  required! :host, :port
17
- optional :version, :queue, :callback, :responses
32
+ optional(
33
+ :version, :queue, :callback, :responses, :notifier,
34
+ :features, :response_wait, :error_wait, :enforce_features
35
+ )
18
36
  arguments[:queue] ||= Queue.new
19
37
  arguments[:responses] ||= Queue.new
20
38
  arguments[:version] ||= 'v2'
39
+ arguments[:features] ||= {}
40
+ arguments[:response_wait] ||= 1
41
+ arguments[:error_wait] ||= 0.4
42
+ if(arguments[:enforce_features].nil?)
43
+ arguments[:enforce_features] = true
44
+ end
21
45
  @socket = TCPSocket.new(host, port)
46
+ @endpoint_settings = {}
22
47
  end
23
48
 
24
49
  def to_s
@@ -29,24 +54,44 @@ module Krakow
29
54
  def init!
30
55
  debug 'Initializing connection'
31
56
  socket.write version.rjust(4).upcase
57
+ identify_and_negotiate
32
58
  async.process_to_queue!
33
59
  info 'Connection initialized'
34
60
  end
35
61
 
36
62
  # message:: Command instance to send
37
63
  # Send the message
64
+ # TODO: Do we want to validate Command instance and abort if
65
+ # response is already set?
38
66
  def transmit(message)
39
67
  output = message.to_line
40
68
  debug ">>> #{output}"
41
69
  socket.write output
42
- unless(responses.empty?)
43
- response = responses.pop
44
- message.response = response
45
- if(message.error?(response))
46
- res = Error::BadResponse.new "Message transmission failed #{message}"
47
- res.result = response
48
- abort res
70
+ response_wait = wait_time_for(message)
71
+ responses.clear if response_wait
72
+ if(response_wait)
73
+ response = nil
74
+ (response_wait / 0.1).to_i.times do |i|
75
+ response = responses.pop unless responses.empty?
76
+ break if response
77
+ debug "Response wait sleep for 0.1 seconds (#{i+1} time)"
78
+ sleep(0.1)
79
+ end
80
+ if(response)
81
+ message.response = response
82
+ if(message.error?(response))
83
+ res = Error::BadResponse.new "Message transmission failed #{message}"
84
+ res.result = response
85
+ abort res
86
+ end
87
+ response
88
+ else
89
+ unless(Command.response_for(message) == :error_only)
90
+ abort Error::BadResponse::NoResponse.new "No response provided for message #{message}"
91
+ end
49
92
  end
93
+ else
94
+ true
50
95
  end
51
96
  end
52
97
 
@@ -96,6 +141,7 @@ module Krakow
96
141
  if(message)
97
142
  debug "Adding message to queue #{message}"
98
143
  queue << message
144
+ notifier.signal(message) if notifier
99
145
  end
100
146
  end
101
147
  end
@@ -122,5 +168,77 @@ module Krakow
122
168
  end
123
169
  end
124
170
  end
171
+
172
+ def wait_time_for(message)
173
+ case Command.response_for(message)
174
+ when :required
175
+ response_wait
176
+ when :error_only
177
+ error_wait
178
+ end
179
+ end
180
+
181
+ def identify_defaults
182
+ unless(@identify_defaults)
183
+ @identify_defaults = {
184
+ :short_id => Socket.gethostname,
185
+ :long_id => Socket.gethostbyname(Socket.gethostname).flatten.compact.first,
186
+ :user_agent => "krakow/#{Krakow::VERSION}",
187
+ :feature_negotiation => true
188
+ }
189
+ end
190
+ @identify_defaults
191
+ end
192
+
193
+ def identify_and_negotiate
194
+ expected_features = identify_defaults.merge(features)
195
+ ident = Command::Identify.new(
196
+ expected_features
197
+ )
198
+ socket.write(ident.to_line)
199
+ response = receive
200
+ if(expected_features[:feature_negotiation])
201
+ begin
202
+ @endpoint_settings = MultiJson.load(response.content, :symbolize_keys => true)
203
+ info "Connection settings: #{endpoint_settings.inspect}"
204
+ # Enable things we need to enable
205
+ ENABLEABLE_FEATURES.each do |key|
206
+ if(endpoint_settings[key])
207
+ send(key)
208
+ elsif(enforce_features && expected_features[key])
209
+ abort Error::ConnectionFeatureFailure.new("Failed to enable #{key} feature on connection!")
210
+ end
211
+ end
212
+ rescue MultiJson::LoadError => e
213
+ error "Failed to parse response from Identify request: #{e} - #{response}"
214
+ abort e
215
+ end
216
+ else
217
+ @endpoint_settings = {}
218
+ end
219
+ true
220
+ end
221
+
222
+ def snappy
223
+ info 'Loading support for snappy compression and converting connection'
224
+ @socket = ConnectionFeatures::SnappyFrames::Io.new(socket)
225
+ response = receive
226
+ info "Snappy connection conversion complete. Response: #{response.inspect}"
227
+ end
228
+
229
+ def deflate
230
+ debug 'Loading support for deflate compression and converting connection'
231
+ @socket = ConnectionFeatures::Deflate::Io.new(socket)
232
+ response = receive
233
+ info "Deflate connection conversion complete. Response: #{response.inspect}"
234
+ end
235
+
236
+ def tls_v1
237
+ info 'Enabling TLS for connection'
238
+ @socket = ConnectionFeatures::Ssl::Io.new(socket)
239
+ response = receive
240
+ info "TLS enable complete. Response: #{response.inspect}"
241
+ end
242
+
125
243
  end
126
244
  end
@@ -0,0 +1,57 @@
1
+ require 'zlib'
2
+
3
+ module Krakow
4
+ module ConnectionFeatures
5
+ module Deflate
6
+ class Io
7
+
8
+ attr_reader :io, :buffer, :headers, :inflator, :deflator
9
+
10
+ def initialize(io)
11
+ @io = io
12
+ @buffer = ''
13
+ @inflator = Zlib::Inflate.new(-Zlib::MAX_WBITS)
14
+ @deflator = Zlib::Deflate.new(nil, -Zlib::MAX_WBITS)
15
+ end
16
+
17
+ # Proxy to underlying socket
18
+ def method_missing(*args)
19
+ io.__send__(*args)
20
+ end
21
+
22
+ def recv(n)
23
+ until(buffer.length >= n)
24
+ read_stream
25
+ sleep(0.1) unless buffer.length >= n
26
+ end
27
+ buffer.slice!(0, n)
28
+ end
29
+ alias_method :read, :recv
30
+
31
+ def read_stream
32
+ str = io.read
33
+ unless(str.empty?)
34
+ buffer << inflator.inflate(str)
35
+ end
36
+ end
37
+
38
+ def write(string)
39
+ unless(string.empty?)
40
+ output = deflator.deflate(string)
41
+ output << deflator.flush
42
+ io.write(output)
43
+ else
44
+ 0
45
+ end
46
+ end
47
+
48
+ def close(*args)
49
+ super
50
+ deflator.deflate(nil, Zlib::FINISH)
51
+ deflator.close
52
+ end
53
+
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,93 @@
1
+ require 'snappy'
2
+ require 'digest/crc'
3
+
4
+ # TODO: Add support for max size + chunks
5
+ # TODO: Include support for remaining types
6
+ module Krakow
7
+ module ConnectionFeatures
8
+ module SnappyFrames
9
+ class Io
10
+
11
+ IDENTIFIER = "\x73\x4e\x61\x50\x70\x59".force_encoding('ASCII-8BIT')
12
+ ident_size = [IDENTIFIER.size].pack('L<')
13
+ ident_size.slice!(-1,1)
14
+ IDENTIFIER_SIZE = ident_size
15
+
16
+ CHUNK_TYPE = {
17
+ "\xff".force_encoding('ASCII-8BIT') => :identifier,
18
+ "\x00".force_encoding('ASCII-8BIT') => :compressed,
19
+ "\x01".force_encoding('ASCII-8BIT') => :uncompressed
20
+ }
21
+
22
+ attr_reader :io, :buffer
23
+
24
+ def initialize(io)
25
+ @io = io
26
+ @snappy_write_ident = false
27
+ @buffer = ''
28
+ end
29
+
30
+ # Proxy to underlying socket
31
+ def method_missing(*args)
32
+ io.__send__(*args)
33
+ end
34
+
35
+ def checksum_mask(checksum)
36
+ (((checksum >> 15) | (checksum << 17)) + 0xa282ead8) & 0xffffffff
37
+ end
38
+
39
+ def recv(n)
40
+ read_stream unless buffer.size >= n
41
+ result = buffer.slice!(0,n)
42
+ result.empty? ? nil : result
43
+ end
44
+ alias_method :read, :recv
45
+
46
+ def read_stream
47
+ header = io.recv(4)
48
+ ident = CHUNK_TYPE[header.slice!(0)]
49
+ size = (header << CHUNK_TYPE.key(:compressed)).unpack('L<').first
50
+ content = io.recv(size)
51
+ case ident
52
+ when :identifier
53
+ unless(content == IDENTIFIER)
54
+ raise "Invalid stream identification encountered (content: #{content.inspect})"
55
+ end
56
+ read_stream
57
+ when :compressed
58
+ checksum = content.slice!(0, 4).unpack('L<').first
59
+ deflated = Snappy.inflate(content)
60
+ digest = Digest::CRC32c.new
61
+ digest << deflated
62
+ unless(checksum == checksum_mask(digest.checksum))
63
+ raise 'Checksum mismatch!'
64
+ end
65
+ buffer << deflated
66
+ when :uncompressed
67
+ buffer << content
68
+ end
69
+ end
70
+
71
+ def write(string)
72
+ unless(@snappy_writer_ident)
73
+ send_snappy_identifier
74
+ end
75
+ digest = Digest::CRC32c.new
76
+ digest << string
77
+ content = Snappy.deflate(string)
78
+ size = content.length + 4
79
+ size = [size].pack('L<')
80
+ size.slice!(-1,1)
81
+ checksum = [checksum_mask(digest.checksum)].pack('L<')
82
+ output = [CHUNK_TYPE.key(:compressed), size, checksum, content].pack('a*a*a*a*')
83
+ io.write output
84
+ end
85
+
86
+ def send_snappy_identifier
87
+ io.write [CHUNK_TYPE.key(:identifier), IDENTIFIER_SIZE, IDENTIFIER].pack('a*a*a*')
88
+ end
89
+
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,33 @@
1
+ module Krakow
2
+ module ConnectionFeatures
3
+ module Ssl
4
+ class Io
5
+
6
+ attr_reader :_socket
7
+
8
+ def initialize(io, args={})
9
+ ssl_socket_arguments = [io]
10
+ if(args[:ssl_context])
11
+ # ssl_socket_arguments << SSLContext.new
12
+ end
13
+ @_socket = Celluloid::IO::SSLSocket.new(*ssl_socket_arguments)
14
+ _socket.sync = true
15
+ _socket.connect
16
+ end
17
+
18
+ def method_missing(*args)
19
+ _socket.send(*args)
20
+ end
21
+
22
+ def recv(len)
23
+ str = readpartial(len)
24
+ if(len > str.length)
25
+ str << sysread(len - str.length)
26
+ end
27
+ str
28
+ end
29
+
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,9 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ module ConnectionFeatures
5
+ autoload :SnappyFrames, 'krakow/connection_features/snappy_frames'
6
+ autoload :Deflate, 'krakow/connection_features/deflate'
7
+ autoload :Ssl, 'krakow/connection_features/ssl'
8
+ end
9
+ end
@@ -12,9 +12,10 @@ module Krakow
12
12
  def initialize(args={})
13
13
  super
14
14
  required! :topic, :channel
15
- optional :host, :port, :nslookupd, :max_in_flight, :backoff_interval, :discovery_interval
15
+ optional :host, :port, :nslookupd, :max_in_flight, :backoff_interval, :discovery_interval, :notifier, :connection_features
16
16
  arguments[:max_in_flight] ||= 1
17
17
  arguments[:discovery_interval] ||= 30
18
+ arguments[:connection_features] ||= {}
18
19
  @connections = {}
19
20
  @distribution = Distribution::Default.new(
20
21
  :max_in_flight => max_in_flight,
@@ -24,8 +25,8 @@ module Krakow
24
25
  if(nslookupd)
25
26
  debug "Connections will be established via lookup #{nslookupd.inspect}"
26
27
  @discovery = Discovery.new(:nslookupd => nslookupd)
27
- every(discovery_interval){ init! }
28
28
  init!
29
+ every(discovery_interval){ init! }
29
30
  elsif(host && port)
30
31
  debug "Connection will be established via direct connection #{host}:#{port}"
31
32
  connection = build_connection(host, port, queue)
@@ -33,10 +34,10 @@ module Krakow
33
34
  info "Registered new connection #{connection}"
34
35
  distribution.redistribute!
35
36
  else
36
- abort ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
37
+ abort Error::ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
37
38
  end
38
39
  else
39
- abort ConfigurationError.new('No connection information provided!')
40
+ abort Error::ConfigurationError.new('No connection information provided!')
40
41
  end
41
42
  end
42
43
 
@@ -62,6 +63,8 @@ module Krakow
62
63
  :host => host,
63
64
  :port => port,
64
65
  :queue => queue,
66
+ :notifier => notifier,
67
+ :features => connection_features,
65
68
  :callback => {
66
69
  :actor => current_actor,
67
70
  :method => :process_message
@@ -135,23 +138,30 @@ module Krakow
135
138
  distribution.redistribute!
136
139
  end
137
140
 
138
- # message_id:: Message ID
141
+ # message_id:: Message ID (or message if you want to be lazy)
139
142
  # Confirm message has been processed
140
143
  def confirm(message_id)
141
- distribution.in_flight_lookup(message_id) do |connection|
142
- connection.transmit(Command::Fin.new(:message_id => message_id))
143
- distribution.success(connection)
144
+ message_id = message_id.message_id if message_id.respond_to?(:message_id)
145
+ begin
146
+ distribution.in_flight_lookup(message_id) do |connection|
147
+ distribution.unregister_message(message_id)
148
+ connection.transmit(Command::Fin.new(:message_id => message_id))
149
+ distribution.success(connection)
150
+ update_ready!(connection)
151
+ end
152
+ true
153
+ rescue => e
154
+ abort e
144
155
  end
145
- connection = distribution.unregister_message(message_id)
146
- update_ready!(connection)
147
- true
148
156
  end
149
157
 
150
158
  # message_id:: Message ID
151
159
  # timeout:: Requeue timeout (default is none)
152
160
  # Requeue message (processing failure)
153
161
  def requeue(message_id, timeout=0)
162
+ message_id = message_id.message_id if message_id.respond_to?(:message_id)
154
163
  distribution.in_flight_lookup(message_id) do |connection|
164
+ distribution.unregister_message(message_id)
155
165
  connection.transmit(
156
166
  Command::Req.new(
157
167
  :message_id => message_id,
@@ -159,9 +169,8 @@ module Krakow
159
169
  )
160
170
  )
161
171
  distribution.failure(connection)
172
+ update_ready!(connection)
162
173
  end
163
- connection = distribution.unregister_message(message_id)
164
- update_ready!(connection)
165
174
  true
166
175
  end
167
176
 
@@ -6,7 +6,7 @@ module Krakow
6
6
 
7
7
  # recalculate `ideal` and update RDY on connections
8
8
  def redistribute!
9
- @ideal = max_in_flight / registry.size
9
+ @ideal = registry.size < 1 ? 0 : max_in_flight / registry.size
10
10
  debug "Distribution calculated ideal: #{ideal}"
11
11
  if(less_than_ideal?)
12
12
  registry.each do |connection, reg_info|
@@ -22,13 +22,13 @@ module Krakow
22
22
 
23
23
  # Reset flight distributions
24
24
  def redistribute!
25
- raise NoMethodError.new 'Custom `#redistrubute!` method must be provided!'
25
+ raise NotImplementedError.new 'Custom `#redistrubute!` method must be provided!'
26
26
  end
27
27
 
28
28
  # connection:: Connection
29
29
  # Determine RDY value for given connection
30
30
  def calculate_ready!(connection)
31
- raise NoMethodError.new 'Custom `#calculate_ready!` method must be provided!'
31
+ raise NotImplementedError.new 'Custom `#calculate_ready!` method must be provided!'
32
32
  end
33
33
 
34
34
  # message:: FrameType::Message or message ID string
@@ -1,12 +1,15 @@
1
1
  module Krakow
2
2
  class Error < StandardError
3
3
 
4
+ class ConnectionFeatureFailure < Error; end
4
5
  class LookupFailed < Error; end
5
6
  class ConnectionFailure < Error; end
6
7
  class ConfigurationError < Error; end
7
8
 
8
9
  class BadResponse < Error
9
10
  attr_accessor :result
11
+ class NoResponse < BadResponse
12
+ end
10
13
  end
11
14
 
12
15
  end
@@ -1,15 +1,20 @@
1
1
  require 'http'
2
2
  require 'uri'
3
+ require 'ostruct'
3
4
 
4
5
  module Krakow
5
6
  class Producer
6
7
  class Http
7
8
 
9
+ class Response < OpenStruct
10
+ end
11
+
8
12
  include Utils::Lazy
9
13
 
10
14
  attr_reader :uri
11
15
 
12
16
  def initialize(args={})
17
+ super
13
18
  required! :endpoint, :topic
14
19
  @uri = URI.parse(endpoint)
15
20
  end
@@ -17,7 +22,13 @@ module Krakow
17
22
  def send_message(method, path, args={})
18
23
  build = uri.dup
19
24
  build.path = "/#{path}"
20
- HTTP.send(method, build.to_s, args)
25
+ response = HTTP.send(method, build.to_s, args)
26
+ begin
27
+ response = MultiJson.load(response.response.body)
28
+ rescue MultiJson::LoadError
29
+ response = {'status_code' => response == 'OK' ? 200 : nil, 'status_txt' => response, 'data' => nil}
30
+ end
31
+ Response.new(response)
21
32
  end
22
33
 
23
34
  def write(*payload)
@@ -110,6 +121,10 @@ module Krakow
110
121
  send_message(:get, :ping)
111
122
  end
112
123
 
124
+ def info
125
+ send_message(:get, :info)
126
+ end
127
+
113
128
  end
114
129
  end
115
130
  end
@@ -14,9 +14,10 @@ module Krakow
14
14
  def initialize(args={})
15
15
  super
16
16
  required! :host, :port, :topic
17
- optional :connect_retries
17
+ optional :reconnect_retries, :reconnect_interval, :connection_features
18
18
  arguments[:reconnect_retries] ||= 10
19
19
  arguments[:reconnect_interval] = 5
20
+ arguments[:connection_features] ||= {}
20
21
  connect
21
22
  end
22
23
 
@@ -24,7 +25,11 @@ module Krakow
24
25
  def connect
25
26
  info "Establishing connection to: #{host}:#{port}"
26
27
  begin
27
- @connection = Connection.new(:host => host, :port => port)
28
+ @connection = Connection.new(
29
+ :host => host,
30
+ :port => port,
31
+ :features => connection_features
32
+ )
28
33
  self.link connection
29
34
  connection.init!
30
35
  info "Connection established: #{connection}"
@@ -45,21 +50,9 @@ module Krakow
45
50
  # Process connection failure and attempt reconnection
46
51
  def connection_failure(*args)
47
52
  warn "Connection has failed to #{host}:#{port}"
48
- retries = 0
49
- begin
50
- connect
51
- rescue => e
52
- retries += 1
53
- warn "Connection retry #{retries}/#{reconnect_retries} failed. #{e.class}: #{e}"
54
- if(retries < reconnect_retries)
55
- sleep_interval = retries * reconnect_interval
56
- debug "Sleeping for reconnect interval of #{sleep_interval} seconds"
57
- sleep sleep_interval
58
- retry
59
- else
60
- abort e
61
- end
62
- end
53
+ debug "Sleeping for reconnect interval of #{reconnect_interval} seconds"
54
+ sleep reconnect_interval
55
+ connect
63
56
  end
64
57
 
65
58
  def goodbye_my_love!
@@ -92,26 +85,10 @@ module Krakow
92
85
  )
93
86
  )
94
87
  end
95
- read(:validate)
96
88
  else
97
89
  abort Error.new 'Remote connection is unavailable!'
98
90
  end
99
91
  end
100
92
 
101
- # args:: Options (:validate)
102
- # Read response from connection. If :validate is included an
103
- # exception will be raised if `FrameType::Error` is received
104
- def read(*args)
105
- result = connection.responses.pop
106
- debug "Read response: #{result}"
107
- if(args.include?(:validate) && result.is_a?(FrameType::Error))
108
- error = Error::BadResponse.new('Write failed')
109
- error.result = result
110
- abort error
111
- else
112
- result
113
- end
114
- end
115
-
116
93
  end
117
94
  end
@@ -26,6 +26,11 @@ module Krakow
26
26
  define_singleton_method(key) do
27
27
  arguments[key]
28
28
  end
29
+ if(key.match(/\w$/))
30
+ define_singleton_method("#{key}?".to_sym) do
31
+ !!arguments[key]
32
+ end
33
+ end
29
34
  end
30
35
  end
31
36
 
@@ -40,6 +45,11 @@ module Krakow
40
45
  define_singleton_method(key) do
41
46
  arguments[key]
42
47
  end
48
+ if(key.match(/\w$/))
49
+ define_singleton_method("#{key}?".to_sym) do
50
+ !!arguments[key]
51
+ end
52
+ end
43
53
  end
44
54
  end
45
55
 
@@ -1,7 +1,5 @@
1
- require 'krakow'
2
-
3
1
  module Krakow
4
2
  class Version < Gem::Version
5
3
  end
6
- VERSION = Version.new('0.1.2')
4
+ VERSION = Version.new('0.2.0')
7
5
  end
data/lib/krakow.rb CHANGED
@@ -1,9 +1,11 @@
1
- autoload :Celluloid, 'celluloid'
1
+ require 'celluloid/autostart'
2
+ require 'multi_json'
2
3
 
3
4
  module Krakow
4
5
 
5
6
  autoload :Command, 'krakow/command'
6
7
  autoload :Connection, 'krakow/connection'
8
+ autoload :ConnectionFeatures, 'krakow/connection_features'
7
9
  autoload :Consumer, 'krakow/consumer'
8
10
  autoload :Discovery, 'krakow/discovery'
9
11
  autoload :Distribution, 'krakow/distribution'
data/test/spec.rb ADDED
@@ -0,0 +1,81 @@
1
+ require 'krakow'
2
+ require 'minitest/autorun'
3
+
4
+ module Krakow
5
+ class Test
6
+ class << self
7
+
8
+ def method_missing(*args)
9
+ name = method = args.first.to_s
10
+ if(name.end_with?('?'))
11
+ name = name.tr('?', '')
12
+ end
13
+ val = ENV[name.upcase] || ENV[name]
14
+ if(method.end_with?('?'))
15
+ !!val
16
+ else
17
+ val
18
+ end
19
+ end
20
+
21
+ def _topic
22
+ 'krakow-test'
23
+ end
24
+
25
+ def _scrub_topic!
26
+ _http_producer.delete_topic
27
+ end
28
+
29
+ def _http_producer
30
+ @http_producer ||= Krakow::Producer::Http.new(
31
+ :endpoint => 'http://127.0.0.1:4151',
32
+ :topic => _topic
33
+ )
34
+ end
35
+
36
+ def _producer(args={})
37
+ Krakow::Producer.new(
38
+ {
39
+ :host => Krakow::Test.nsq_producer_host || '127.0.0.1',
40
+ :port => Krakow::Test.nsq_producer_port || 4150,
41
+ :topic => 'krakow-test'
42
+ }.merge(args)
43
+ )
44
+ end
45
+
46
+ def _consumer(args={})
47
+ Krakow::Consumer.new(
48
+ {
49
+ :nslookupd => Krakow::Test.nsq_lookupd || 'http://127.0.0.1:4161',
50
+ :topic => 'krakow-test',
51
+ :channel => 'default',
52
+ :discovery_interval => 0.5,
53
+ :max_in_flight => 20
54
+ }.merge(args)
55
+ )
56
+ end
57
+
58
+ end
59
+ end
60
+ end
61
+
62
+ MiniTest::Spec.before do
63
+ Celluloid.boot
64
+ Krakow::Test._scrub_topic!
65
+ @consumer = Krakow::Test._consumer
66
+ @producer = Krakow::Test._producer
67
+ end
68
+
69
+ MiniTest::Spec.after do
70
+ @consumer.terminate if @consumer && @consumer.alive?
71
+ @producer.terminate if @producer && @producer.alive?
72
+ Krakow::Test._scrub_topic!
73
+ end
74
+
75
+ unless(Krakow::Test.debug?)
76
+ Celluloid.logger.level = 3
77
+ end
78
+
79
+ Dir.glob(File.join(File.dirname(__FILE__), 'specs', '*.rb')).each do |path|
80
+ require File.expand_path(path)
81
+ end
@@ -0,0 +1,49 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Consumer do
4
+
5
+ it 'should not have any connections' do
6
+ @consumer.connections.size.must_equal 0
7
+ end
8
+ it 'should have an empty queue' do
9
+ @consumer.queue.size.must_equal 0
10
+ end
11
+
12
+ describe 'with active producer' do
13
+
14
+ before do
15
+ @producer.write('msg1', 'msg2', 'msg3')
16
+ @inital_wait ||= sleep(0.8) # allow setup (topic creation, discovery, etc)
17
+ end
18
+
19
+ it 'should have one connection' do
20
+ @consumer.connections.size.must_equal 1
21
+ end
22
+ it 'should have three messages queued' do
23
+ @consumer.queue.size.must_equal 3
24
+ end
25
+ it 'should properly confirm messages' do
26
+ 3.times do
27
+ msg = @consumer.queue.pop
28
+ @consumer.confirm(msg).must_equal true
29
+ end
30
+ sleep(0.5) # pause to let everything settle
31
+ @consumer.queue.must_be :empty?
32
+ end
33
+ it 'should properly requeue messages' do
34
+ 2.times do
35
+ @consumer.confirm(@consumer.queue.pop)
36
+ end
37
+ @consumer.queue.size.must_equal 1
38
+ original_msg = @consumer.queue.pop
39
+ @consumer.queue.must_be :empty?
40
+ @consumer.requeue(original_msg).must_equal true
41
+ sleep(0.2)
42
+ @consumer.queue.size.must_equal 1
43
+ req_msg = @consumer.queue.pop
44
+ req_msg.message_id.must_equal original_msg.message_id
45
+ end
46
+
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,123 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Producer::Http do
4
+
5
+ before do
6
+ @http = Krakow::Test._http_producer
7
+ end
8
+
9
+ it 'should write single messages successfully' do
10
+ response = @http.write('msg')
11
+ Krakow::Command::Pub.ok.must_include response.status_txt
12
+ end
13
+ it 'should write multiple messages successfully' do
14
+ response = @http.write('msg1', 'msg2', 'msg3')
15
+ Krakow::Command::Mpub.ok.must_include response.status_txt
16
+ end
17
+ it 'should create topic' do
18
+ response = @http.create_topic
19
+ response.status_code.must_equal 200
20
+ response.status_txt.must_equal 'OK'
21
+ end
22
+ it 'should delete topic' do
23
+ c_response = @http.create_topic
24
+ c_response.status_code.must_equal 200
25
+ c_response.status_txt.must_equal 'OK'
26
+ d_response = @http.delete_topic
27
+ d_response.status_code.must_equal 200
28
+ d_response.status_txt.must_equal 'OK'
29
+ end
30
+ it 'should create channel' do
31
+ c_response = @http.create_topic
32
+ c_response.status_code.must_equal 200
33
+ c_response.status_txt.must_equal 'OK'
34
+ ch_response = @http.create_channel('fubar')
35
+ ch_response.status_code.must_equal 200
36
+ ch_response.status_txt.must_equal 'OK'
37
+ end
38
+ it 'should delete channel' do
39
+ c_response = @http.create_topic
40
+ c_response.status_code.must_equal 200
41
+ c_response.status_txt.must_equal 'OK'
42
+ ch_response = @http.create_channel('fubar')
43
+ ch_response.status_code.must_equal 200
44
+ ch_response.status_txt.must_equal 'OK'
45
+ dch_response = @http.delete_channel('fubar')
46
+ dch_response.status_code.must_equal 200
47
+ dch_response.status_txt.must_equal 'OK'
48
+ end
49
+ it 'should empty topic' do
50
+ @consumer.terminate
51
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
52
+ et_response = @http.empty_topic
53
+ et_response.status_code.must_equal 200
54
+ et_response.status_txt.must_equal 'OK'
55
+ @consumer = Krakow::Test._consumer
56
+ sleep(0.2)
57
+ @consumer.connections.wont_be :empty?
58
+ @consumer.queue.must_be :empty?
59
+ end
60
+ it 'should empty channel' do
61
+ @consumer.terminate
62
+ @http.write('msg0')
63
+ @http.pause_channel('chan2').status_code.must_equal 200
64
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
65
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
66
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
67
+ sleep(0.5)
68
+ consumer1.queue.size.must_equal 4
69
+ consumer2.queue.size.must_equal 0
70
+ @http.empty_channel('chan2').status_code.must_equal 200
71
+ @http.unpause_channel('chan2').status_code.must_equal 200
72
+ sleep(0.5)
73
+ consumer1.queue.size.must_equal 4
74
+ consumer2.queue.size.must_equal 0
75
+ consumer1.terminate
76
+ consumer2.terminate
77
+ end
78
+ it 'should pause channel' do
79
+ @consumer.terminate
80
+ @http.write('msg0')
81
+ @http.pause_channel('chan2').status_code.must_equal 200
82
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
83
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
84
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
85
+ sleep(0.5)
86
+ consumer1.queue.size.must_equal 4
87
+ consumer2.queue.size.must_equal 0
88
+ consumer1.terminate
89
+ consumer2.terminate
90
+ end
91
+ it 'should unpause channel' do
92
+ @consumer.terminate
93
+ @http.write('msg0')
94
+ @http.pause_channel('chan2').status_code.must_equal 200
95
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
96
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
97
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
98
+ sleep(0.5)
99
+ consumer1.queue.size.must_equal 4
100
+ consumer2.queue.size.must_equal 0
101
+ @http.unpause_channel('chan2').status_code.must_equal 200
102
+ sleep(0.5)
103
+ consumer2.queue.size.must_equal 4
104
+ consumer1.terminate
105
+ consumer2.terminate
106
+ end
107
+ it 'should return stats' do
108
+ stat = @http.stats
109
+ stat.status_code.must_equal 200
110
+ stat.data.must_be_kind_of Hash
111
+ end
112
+ it 'should ping' do
113
+ @http.ping.status_code.must_equal 200
114
+ end
115
+ it 'should fetch info' do
116
+ infos = @http.info
117
+ infos.status_code.must_equal 200
118
+ infos.data.must_be_kind_of Hash
119
+ end
120
+
121
+ end
122
+
123
+ end
@@ -0,0 +1,20 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Producer do
4
+
5
+ it 'should be connected' do
6
+ @producer.connected?.must_equal true
7
+ end
8
+ it 'should write single messages successfully' do
9
+ response = @producer.write('msg')
10
+ response.must_be_kind_of Krakow::FrameType::Response
11
+ Krakow::Command::Pub.ok.must_include response.content
12
+ end
13
+ it 'should write multiple messages successfully' do
14
+ response = @producer.write('msg1', 'msg2', 'msg3')
15
+ response.must_be_kind_of Krakow::FrameType::Response
16
+ Krakow::Command::Mpub.ok.must_include response.content
17
+ end
18
+
19
+ end
20
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krakow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-20 00:00:00.000000000 Z
12
+ date: 2014-03-08 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: celluloid-io
@@ -59,6 +59,38 @@ dependencies:
59
59
  - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: snappy
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: digest-crc
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
62
94
  description: NSQ ruby library
63
95
  email: code@chrisroberts.org
64
96
  executables: []
@@ -93,9 +125,18 @@ files:
93
125
  - lib/krakow/exceptions.rb
94
126
  - lib/krakow/utils.rb
95
127
  - lib/krakow/discovery.rb
128
+ - lib/krakow/connection_features.rb
129
+ - lib/krakow/connection_features/ssl.rb
130
+ - lib/krakow/connection_features/snappy_frames.rb
131
+ - lib/krakow/connection_features/deflate.rb
132
+ - test/spec.rb
133
+ - test/specs/consumer.rb
134
+ - test/specs/producer.rb
135
+ - test/specs/http_producer.rb
96
136
  - Gemfile
97
137
  - README.md
98
138
  - krakow.gemspec
139
+ - krakow-0.2.0.gem
99
140
  - CHANGELOG.md
100
141
  - Gemfile.lock
101
142
  homepage: http://github.com/chrisroberts/krakow