krakow 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## v0.2.0
2
+ * Fix the rest of the namespacing issues
3
+ * Start adding some tests
4
+ * Use better exception types (NotImplementedError instead of NoMethodError)
5
+ * Be smart about responses within connections
6
+ * Add snappy support
7
+ * Add deflate support
8
+ * Add TLS support
9
+ * Prevent division by zero in distribution
10
+ * Add query methods to lazy helper (`attribute_name`?)
11
+
1
12
  ## v0.1.2
2
13
  * Include backoff support
3
14
  * Remove `method_missing` magic
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
+ gem 'minitest'
4
+
3
5
  gemspec
data/Gemfile.lock CHANGED
@@ -1,10 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- krakow (0.1.1)
4
+ krakow (0.1.3)
5
5
  celluloid-io
6
+ digest-crc
6
7
  http
7
8
  multi_json
9
+ snappy
8
10
 
9
11
  GEM
10
12
  remote: https://rubygems.org/
@@ -14,11 +16,14 @@ GEM
14
16
  celluloid-io (0.15.0)
15
17
  celluloid (>= 0.15.0)
16
18
  nio4r (>= 0.5.0)
19
+ digest-crc (0.4.0)
17
20
  http (0.5.0)
18
21
  http_parser.rb
19
22
  http_parser.rb (0.6.0)
23
+ minitest (5.0.8)
20
24
  multi_json (1.8.4)
21
- nio4r (0.5.0)
25
+ nio4r (1.0.0)
26
+ snappy (0.0.10)
22
27
  timers (1.1.0)
23
28
 
24
29
  PLATFORMS
@@ -26,3 +31,4 @@ PLATFORMS
26
31
 
27
32
  DEPENDENCIES
28
33
  krakow!
34
+ minitest
data/README.md CHANGED
@@ -124,6 +124,51 @@ consumer = Krakow::Consumer.new(
124
124
  )
125
125
  ```
126
126
 
127
+ ### I need TLS!
128
+
129
+ OK!
130
+
131
+ ```ruby
132
+ consumer = Krakow::Consumer.new(
133
+ :nsqlookupd => 'http://HOST:PORT',
134
+ :topic => 'target',
135
+ :channel => 'ship',
136
+ :connection_features => {
137
+ :tls_v1 => true
138
+ }
139
+ )
140
+ ```
141
+
142
+ ### I need Snappy compression!
143
+
144
+ OK!
145
+
146
+ ```ruby
147
+ consumer = Krakow::Consumer.new(
148
+ :nsqlookupd => 'http://HOST:PORT',
149
+ :topic => 'target',
150
+ :channel => 'ship',
151
+ :connection_features => {
152
+ :snappy => true
153
+ }
154
+ )
155
+ ```
156
+
157
+ ### I need Deflate compression!
158
+
159
+ OK!
160
+
161
+ ```ruby
162
+ consumer = Krakow::Consumer.new(
163
+ :nsqlookupd => 'http://HOST:PORT',
164
+ :topic => 'target',
165
+ :channel => 'ship',
166
+ :connection_features => {
167
+ :deflate => true
168
+ }
169
+ )
170
+ ```
171
+
127
172
  ### It doesn't work
128
173
 
129
174
  Create an issue on the github repository.
data/krakow.gemspec CHANGED
@@ -12,5 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.add_dependency 'celluloid-io'
13
13
  s.add_dependency 'http'
14
14
  s.add_dependency 'multi_json'
15
+ s.add_dependency 'snappy'
16
+ s.add_dependency 'digest-crc'
15
17
  s.files = Dir['**/*']
16
18
  end
@@ -34,6 +34,7 @@ module Krakow
34
34
  def error
35
35
  %w(E_INVALID E_BAD_BODY)
36
36
  end
37
+
37
38
  end
38
39
 
39
40
  end
@@ -13,6 +13,20 @@ module Krakow
13
13
  []
14
14
  end
15
15
 
16
+ # message:: Krakow::Message
17
+ # Returns type of response expected (:none, :error_only, :required)
18
+ def response_for(message)
19
+ if(message.class.ok.empty?)
20
+ if(message.class.error.empty?)
21
+ :none
22
+ else
23
+ :error_only
24
+ end
25
+ else
26
+ :required
27
+ end
28
+ end
29
+
16
30
  end
17
31
 
18
32
  attr_accessor :response
@@ -23,8 +37,8 @@ module Krakow
23
37
  end
24
38
 
25
39
  # Convert to line output
26
- def to_line
27
- raise NoMethodError.new 'No line conversion method defined!'
40
+ def to_line(*args)
41
+ raise NotImplementedError.new 'No line conversion method defined!'
28
42
  end
29
43
 
30
44
  def ok?(response)
@@ -1,3 +1,4 @@
1
+ require 'krakow/version'
1
2
  require 'celluloid/io'
2
3
  require 'celluloid/autostart'
3
4
 
@@ -7,18 +8,42 @@ module Krakow
7
8
  include Utils::Lazy
8
9
  include Celluloid::IO
9
10
 
11
+ FEATURES = [
12
+ :max_rdy_count,
13
+ :max_msg_timeout,
14
+ :msg_timeout,
15
+ :tls_v1,
16
+ :deflate,
17
+ :deflate_level,
18
+ :max_deflate_level,
19
+ :snappy,
20
+ :sample_rate
21
+ ]
22
+ EXCLUSIVE_FEATURES = [[:snappy, :deflate]]
23
+ ENABLEABLE_FEATURES = [:tls_v1, :snappy, :deflate]
24
+
10
25
  finalizer :goodbye_my_love!
11
26
 
12
- attr_reader :socket
27
+ attr_reader :socket, :endpoint_settings
13
28
 
14
29
  def initialize(args={})
15
30
  super
16
31
  required! :host, :port
17
- optional :version, :queue, :callback, :responses
32
+ optional(
33
+ :version, :queue, :callback, :responses, :notifier,
34
+ :features, :response_wait, :error_wait, :enforce_features
35
+ )
18
36
  arguments[:queue] ||= Queue.new
19
37
  arguments[:responses] ||= Queue.new
20
38
  arguments[:version] ||= 'v2'
39
+ arguments[:features] ||= {}
40
+ arguments[:response_wait] ||= 1
41
+ arguments[:error_wait] ||= 0.4
42
+ if(arguments[:enforce_features].nil?)
43
+ arguments[:enforce_features] = true
44
+ end
21
45
  @socket = TCPSocket.new(host, port)
46
+ @endpoint_settings = {}
22
47
  end
23
48
 
24
49
  def to_s
@@ -29,24 +54,44 @@ module Krakow
29
54
  def init!
30
55
  debug 'Initializing connection'
31
56
  socket.write version.rjust(4).upcase
57
+ identify_and_negotiate
32
58
  async.process_to_queue!
33
59
  info 'Connection initialized'
34
60
  end
35
61
 
36
62
  # message:: Command instance to send
37
63
  # Send the message
64
+ # TODO: Do we want to validate Command instance and abort if
65
+ # response is already set?
38
66
  def transmit(message)
39
67
  output = message.to_line
40
68
  debug ">>> #{output}"
41
69
  socket.write output
42
- unless(responses.empty?)
43
- response = responses.pop
44
- message.response = response
45
- if(message.error?(response))
46
- res = Error::BadResponse.new "Message transmission failed #{message}"
47
- res.result = response
48
- abort res
70
+ response_wait = wait_time_for(message)
71
+ responses.clear if response_wait
72
+ if(response_wait)
73
+ response = nil
74
+ (response_wait / 0.1).to_i.times do |i|
75
+ response = responses.pop unless responses.empty?
76
+ break if response
77
+ debug "Response wait sleep for 0.1 seconds (#{i+1} time)"
78
+ sleep(0.1)
79
+ end
80
+ if(response)
81
+ message.response = response
82
+ if(message.error?(response))
83
+ res = Error::BadResponse.new "Message transmission failed #{message}"
84
+ res.result = response
85
+ abort res
86
+ end
87
+ response
88
+ else
89
+ unless(Command.response_for(message) == :error_only)
90
+ abort Error::BadResponse::NoResponse.new "No response provided for message #{message}"
91
+ end
49
92
  end
93
+ else
94
+ true
50
95
  end
51
96
  end
52
97
 
@@ -96,6 +141,7 @@ module Krakow
96
141
  if(message)
97
142
  debug "Adding message to queue #{message}"
98
143
  queue << message
144
+ notifier.signal(message) if notifier
99
145
  end
100
146
  end
101
147
  end
@@ -122,5 +168,77 @@ module Krakow
122
168
  end
123
169
  end
124
170
  end
171
+
172
+ def wait_time_for(message)
173
+ case Command.response_for(message)
174
+ when :required
175
+ response_wait
176
+ when :error_only
177
+ error_wait
178
+ end
179
+ end
180
+
181
+ def identify_defaults
182
+ unless(@identify_defaults)
183
+ @identify_defaults = {
184
+ :short_id => Socket.gethostname,
185
+ :long_id => Socket.gethostbyname(Socket.gethostname).flatten.compact.first,
186
+ :user_agent => "krakow/#{Krakow::VERSION}",
187
+ :feature_negotiation => true
188
+ }
189
+ end
190
+ @identify_defaults
191
+ end
192
+
193
+ def identify_and_negotiate
194
+ expected_features = identify_defaults.merge(features)
195
+ ident = Command::Identify.new(
196
+ expected_features
197
+ )
198
+ socket.write(ident.to_line)
199
+ response = receive
200
+ if(expected_features[:feature_negotiation])
201
+ begin
202
+ @endpoint_settings = MultiJson.load(response.content, :symbolize_keys => true)
203
+ info "Connection settings: #{endpoint_settings.inspect}"
204
+ # Enable things we need to enable
205
+ ENABLEABLE_FEATURES.each do |key|
206
+ if(endpoint_settings[key])
207
+ send(key)
208
+ elsif(enforce_features && expected_features[key])
209
+ abort Error::ConnectionFeatureFailure.new("Failed to enable #{key} feature on connection!")
210
+ end
211
+ end
212
+ rescue MultiJson::LoadError => e
213
+ error "Failed to parse response from Identify request: #{e} - #{response}"
214
+ abort e
215
+ end
216
+ else
217
+ @endpoint_settings = {}
218
+ end
219
+ true
220
+ end
221
+
222
+ def snappy
223
+ info 'Loading support for snappy compression and converting connection'
224
+ @socket = ConnectionFeatures::SnappyFrames::Io.new(socket)
225
+ response = receive
226
+ info "Snappy connection conversion complete. Response: #{response.inspect}"
227
+ end
228
+
229
+ def deflate
230
+ debug 'Loading support for deflate compression and converting connection'
231
+ @socket = ConnectionFeatures::Deflate::Io.new(socket)
232
+ response = receive
233
+ info "Deflate connection conversion complete. Response: #{response.inspect}"
234
+ end
235
+
236
+ def tls_v1
237
+ info 'Enabling TLS for connection'
238
+ @socket = ConnectionFeatures::Ssl::Io.new(socket)
239
+ response = receive
240
+ info "TLS enable complete. Response: #{response.inspect}"
241
+ end
242
+
125
243
  end
126
244
  end
@@ -0,0 +1,57 @@
1
+ require 'zlib'
2
+
3
+ module Krakow
4
+ module ConnectionFeatures
5
+ module Deflate
6
+ class Io
7
+
8
+ attr_reader :io, :buffer, :headers, :inflator, :deflator
9
+
10
+ def initialize(io)
11
+ @io = io
12
+ @buffer = ''
13
+ @inflator = Zlib::Inflate.new(-Zlib::MAX_WBITS)
14
+ @deflator = Zlib::Deflate.new(nil, -Zlib::MAX_WBITS)
15
+ end
16
+
17
+ # Proxy to underlying socket
18
+ def method_missing(*args)
19
+ io.__send__(*args)
20
+ end
21
+
22
+ def recv(n)
23
+ until(buffer.length >= n)
24
+ read_stream
25
+ sleep(0.1) unless buffer.length >= n
26
+ end
27
+ buffer.slice!(0, n)
28
+ end
29
+ alias_method :read, :recv
30
+
31
+ def read_stream
32
+ str = io.read
33
+ unless(str.empty?)
34
+ buffer << inflator.inflate(str)
35
+ end
36
+ end
37
+
38
+ def write(string)
39
+ unless(string.empty?)
40
+ output = deflator.deflate(string)
41
+ output << deflator.flush
42
+ io.write(output)
43
+ else
44
+ 0
45
+ end
46
+ end
47
+
48
+ def close(*args)
49
+ super
50
+ deflator.deflate(nil, Zlib::FINISH)
51
+ deflator.close
52
+ end
53
+
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,93 @@
1
+ require 'snappy'
2
+ require 'digest/crc'
3
+
4
+ # TODO: Add support for max size + chunks
5
+ # TODO: Include support for remaining types
6
+ module Krakow
7
+ module ConnectionFeatures
8
+ module SnappyFrames
9
+ class Io
10
+
11
+ IDENTIFIER = "\x73\x4e\x61\x50\x70\x59".force_encoding('ASCII-8BIT')
12
+ ident_size = [IDENTIFIER.size].pack('L<')
13
+ ident_size.slice!(-1,1)
14
+ IDENTIFIER_SIZE = ident_size
15
+
16
+ CHUNK_TYPE = {
17
+ "\xff".force_encoding('ASCII-8BIT') => :identifier,
18
+ "\x00".force_encoding('ASCII-8BIT') => :compressed,
19
+ "\x01".force_encoding('ASCII-8BIT') => :uncompressed
20
+ }
21
+
22
+ attr_reader :io, :buffer
23
+
24
+ def initialize(io)
25
+ @io = io
26
+ @snappy_write_ident = false
27
+ @buffer = ''
28
+ end
29
+
30
+ # Proxy to underlying socket
31
+ def method_missing(*args)
32
+ io.__send__(*args)
33
+ end
34
+
35
+ def checksum_mask(checksum)
36
+ (((checksum >> 15) | (checksum << 17)) + 0xa282ead8) & 0xffffffff
37
+ end
38
+
39
+ def recv(n)
40
+ read_stream unless buffer.size >= n
41
+ result = buffer.slice!(0,n)
42
+ result.empty? ? nil : result
43
+ end
44
+ alias_method :read, :recv
45
+
46
+ def read_stream
47
+ header = io.recv(4)
48
+ ident = CHUNK_TYPE[header.slice!(0)]
49
+ size = (header << CHUNK_TYPE.key(:compressed)).unpack('L<').first
50
+ content = io.recv(size)
51
+ case ident
52
+ when :identifier
53
+ unless(content == IDENTIFIER)
54
+ raise "Invalid stream identification encountered (content: #{content.inspect})"
55
+ end
56
+ read_stream
57
+ when :compressed
58
+ checksum = content.slice!(0, 4).unpack('L<').first
59
+ deflated = Snappy.inflate(content)
60
+ digest = Digest::CRC32c.new
61
+ digest << deflated
62
+ unless(checksum == checksum_mask(digest.checksum))
63
+ raise 'Checksum mismatch!'
64
+ end
65
+ buffer << deflated
66
+ when :uncompressed
67
+ buffer << content
68
+ end
69
+ end
70
+
71
+ def write(string)
72
+ unless(@snappy_writer_ident)
73
+ send_snappy_identifier
74
+ end
75
+ digest = Digest::CRC32c.new
76
+ digest << string
77
+ content = Snappy.deflate(string)
78
+ size = content.length + 4
79
+ size = [size].pack('L<')
80
+ size.slice!(-1,1)
81
+ checksum = [checksum_mask(digest.checksum)].pack('L<')
82
+ output = [CHUNK_TYPE.key(:compressed), size, checksum, content].pack('a*a*a*a*')
83
+ io.write output
84
+ end
85
+
86
+ def send_snappy_identifier
87
+ io.write [CHUNK_TYPE.key(:identifier), IDENTIFIER_SIZE, IDENTIFIER].pack('a*a*a*')
88
+ end
89
+
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,33 @@
1
+ module Krakow
2
+ module ConnectionFeatures
3
+ module Ssl
4
+ class Io
5
+
6
+ attr_reader :_socket
7
+
8
+ def initialize(io, args={})
9
+ ssl_socket_arguments = [io]
10
+ if(args[:ssl_context])
11
+ # ssl_socket_arguments << SSLContext.new
12
+ end
13
+ @_socket = Celluloid::IO::SSLSocket.new(*ssl_socket_arguments)
14
+ _socket.sync = true
15
+ _socket.connect
16
+ end
17
+
18
+ def method_missing(*args)
19
+ _socket.send(*args)
20
+ end
21
+
22
+ def recv(len)
23
+ str = readpartial(len)
24
+ if(len > str.length)
25
+ str << sysread(len - str.length)
26
+ end
27
+ str
28
+ end
29
+
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,9 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ module ConnectionFeatures
5
+ autoload :SnappyFrames, 'krakow/connection_features/snappy_frames'
6
+ autoload :Deflate, 'krakow/connection_features/deflate'
7
+ autoload :Ssl, 'krakow/connection_features/ssl'
8
+ end
9
+ end
@@ -12,9 +12,10 @@ module Krakow
12
12
  def initialize(args={})
13
13
  super
14
14
  required! :topic, :channel
15
- optional :host, :port, :nslookupd, :max_in_flight, :backoff_interval, :discovery_interval
15
+ optional :host, :port, :nslookupd, :max_in_flight, :backoff_interval, :discovery_interval, :notifier, :connection_features
16
16
  arguments[:max_in_flight] ||= 1
17
17
  arguments[:discovery_interval] ||= 30
18
+ arguments[:connection_features] ||= {}
18
19
  @connections = {}
19
20
  @distribution = Distribution::Default.new(
20
21
  :max_in_flight => max_in_flight,
@@ -24,8 +25,8 @@ module Krakow
24
25
  if(nslookupd)
25
26
  debug "Connections will be established via lookup #{nslookupd.inspect}"
26
27
  @discovery = Discovery.new(:nslookupd => nslookupd)
27
- every(discovery_interval){ init! }
28
28
  init!
29
+ every(discovery_interval){ init! }
29
30
  elsif(host && port)
30
31
  debug "Connection will be established via direct connection #{host}:#{port}"
31
32
  connection = build_connection(host, port, queue)
@@ -33,10 +34,10 @@ module Krakow
33
34
  info "Registered new connection #{connection}"
34
35
  distribution.redistribute!
35
36
  else
36
- abort ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
37
+ abort Error::ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
37
38
  end
38
39
  else
39
- abort ConfigurationError.new('No connection information provided!')
40
+ abort Error::ConfigurationError.new('No connection information provided!')
40
41
  end
41
42
  end
42
43
 
@@ -62,6 +63,8 @@ module Krakow
62
63
  :host => host,
63
64
  :port => port,
64
65
  :queue => queue,
66
+ :notifier => notifier,
67
+ :features => connection_features,
65
68
  :callback => {
66
69
  :actor => current_actor,
67
70
  :method => :process_message
@@ -135,23 +138,30 @@ module Krakow
135
138
  distribution.redistribute!
136
139
  end
137
140
 
138
- # message_id:: Message ID
141
+ # message_id:: Message ID (or message if you want to be lazy)
139
142
  # Confirm message has been processed
140
143
  def confirm(message_id)
141
- distribution.in_flight_lookup(message_id) do |connection|
142
- connection.transmit(Command::Fin.new(:message_id => message_id))
143
- distribution.success(connection)
144
+ message_id = message_id.message_id if message_id.respond_to?(:message_id)
145
+ begin
146
+ distribution.in_flight_lookup(message_id) do |connection|
147
+ distribution.unregister_message(message_id)
148
+ connection.transmit(Command::Fin.new(:message_id => message_id))
149
+ distribution.success(connection)
150
+ update_ready!(connection)
151
+ end
152
+ true
153
+ rescue => e
154
+ abort e
144
155
  end
145
- connection = distribution.unregister_message(message_id)
146
- update_ready!(connection)
147
- true
148
156
  end
149
157
 
150
158
  # message_id:: Message ID
151
159
  # timeout:: Requeue timeout (default is none)
152
160
  # Requeue message (processing failure)
153
161
  def requeue(message_id, timeout=0)
162
+ message_id = message_id.message_id if message_id.respond_to?(:message_id)
154
163
  distribution.in_flight_lookup(message_id) do |connection|
164
+ distribution.unregister_message(message_id)
155
165
  connection.transmit(
156
166
  Command::Req.new(
157
167
  :message_id => message_id,
@@ -159,9 +169,8 @@ module Krakow
159
169
  )
160
170
  )
161
171
  distribution.failure(connection)
172
+ update_ready!(connection)
162
173
  end
163
- connection = distribution.unregister_message(message_id)
164
- update_ready!(connection)
165
174
  true
166
175
  end
167
176
 
@@ -6,7 +6,7 @@ module Krakow
6
6
 
7
7
  # recalculate `ideal` and update RDY on connections
8
8
  def redistribute!
9
- @ideal = max_in_flight / registry.size
9
+ @ideal = registry.size < 1 ? 0 : max_in_flight / registry.size
10
10
  debug "Distribution calculated ideal: #{ideal}"
11
11
  if(less_than_ideal?)
12
12
  registry.each do |connection, reg_info|
@@ -22,13 +22,13 @@ module Krakow
22
22
 
23
23
  # Reset flight distributions
24
24
  def redistribute!
25
- raise NoMethodError.new 'Custom `#redistrubute!` method must be provided!'
25
+ raise NotImplementedError.new 'Custom `#redistrubute!` method must be provided!'
26
26
  end
27
27
 
28
28
  # connection:: Connection
29
29
  # Determine RDY value for given connection
30
30
  def calculate_ready!(connection)
31
- raise NoMethodError.new 'Custom `#calculate_ready!` method must be provided!'
31
+ raise NotImplementedError.new 'Custom `#calculate_ready!` method must be provided!'
32
32
  end
33
33
 
34
34
  # message:: FrameType::Message or message ID string
@@ -1,12 +1,15 @@
1
1
  module Krakow
2
2
  class Error < StandardError
3
3
 
4
+ class ConnectionFeatureFailure < Error; end
4
5
  class LookupFailed < Error; end
5
6
  class ConnectionFailure < Error; end
6
7
  class ConfigurationError < Error; end
7
8
 
8
9
  class BadResponse < Error
9
10
  attr_accessor :result
11
+ class NoResponse < BadResponse
12
+ end
10
13
  end
11
14
 
12
15
  end
@@ -1,15 +1,20 @@
1
1
  require 'http'
2
2
  require 'uri'
3
+ require 'ostruct'
3
4
 
4
5
  module Krakow
5
6
  class Producer
6
7
  class Http
7
8
 
9
+ class Response < OpenStruct
10
+ end
11
+
8
12
  include Utils::Lazy
9
13
 
10
14
  attr_reader :uri
11
15
 
12
16
  def initialize(args={})
17
+ super
13
18
  required! :endpoint, :topic
14
19
  @uri = URI.parse(endpoint)
15
20
  end
@@ -17,7 +22,13 @@ module Krakow
17
22
  def send_message(method, path, args={})
18
23
  build = uri.dup
19
24
  build.path = "/#{path}"
20
- HTTP.send(method, build.to_s, args)
25
+ response = HTTP.send(method, build.to_s, args)
26
+ begin
27
+ response = MultiJson.load(response.response.body)
28
+ rescue MultiJson::LoadError
29
+ response = {'status_code' => response == 'OK' ? 200 : nil, 'status_txt' => response, 'data' => nil}
30
+ end
31
+ Response.new(response)
21
32
  end
22
33
 
23
34
  def write(*payload)
@@ -110,6 +121,10 @@ module Krakow
110
121
  send_message(:get, :ping)
111
122
  end
112
123
 
124
+ def info
125
+ send_message(:get, :info)
126
+ end
127
+
113
128
  end
114
129
  end
115
130
  end
@@ -14,9 +14,10 @@ module Krakow
14
14
  def initialize(args={})
15
15
  super
16
16
  required! :host, :port, :topic
17
- optional :connect_retries
17
+ optional :reconnect_retries, :reconnect_interval, :connection_features
18
18
  arguments[:reconnect_retries] ||= 10
19
19
  arguments[:reconnect_interval] = 5
20
+ arguments[:connection_features] ||= {}
20
21
  connect
21
22
  end
22
23
 
@@ -24,7 +25,11 @@ module Krakow
24
25
  def connect
25
26
  info "Establishing connection to: #{host}:#{port}"
26
27
  begin
27
- @connection = Connection.new(:host => host, :port => port)
28
+ @connection = Connection.new(
29
+ :host => host,
30
+ :port => port,
31
+ :features => connection_features
32
+ )
28
33
  self.link connection
29
34
  connection.init!
30
35
  info "Connection established: #{connection}"
@@ -45,21 +50,9 @@ module Krakow
45
50
  # Process connection failure and attempt reconnection
46
51
  def connection_failure(*args)
47
52
  warn "Connection has failed to #{host}:#{port}"
48
- retries = 0
49
- begin
50
- connect
51
- rescue => e
52
- retries += 1
53
- warn "Connection retry #{retries}/#{reconnect_retries} failed. #{e.class}: #{e}"
54
- if(retries < reconnect_retries)
55
- sleep_interval = retries * reconnect_interval
56
- debug "Sleeping for reconnect interval of #{sleep_interval} seconds"
57
- sleep sleep_interval
58
- retry
59
- else
60
- abort e
61
- end
62
- end
53
+ debug "Sleeping for reconnect interval of #{reconnect_interval} seconds"
54
+ sleep reconnect_interval
55
+ connect
63
56
  end
64
57
 
65
58
  def goodbye_my_love!
@@ -92,26 +85,10 @@ module Krakow
92
85
  )
93
86
  )
94
87
  end
95
- read(:validate)
96
88
  else
97
89
  abort Error.new 'Remote connection is unavailable!'
98
90
  end
99
91
  end
100
92
 
101
- # args:: Options (:validate)
102
- # Read response from connection. If :validate is included an
103
- # exception will be raised if `FrameType::Error` is received
104
- def read(*args)
105
- result = connection.responses.pop
106
- debug "Read response: #{result}"
107
- if(args.include?(:validate) && result.is_a?(FrameType::Error))
108
- error = Error::BadResponse.new('Write failed')
109
- error.result = result
110
- abort error
111
- else
112
- result
113
- end
114
- end
115
-
116
93
  end
117
94
  end
@@ -26,6 +26,11 @@ module Krakow
26
26
  define_singleton_method(key) do
27
27
  arguments[key]
28
28
  end
29
+ if(key.match(/\w$/))
30
+ define_singleton_method("#{key}?".to_sym) do
31
+ !!arguments[key]
32
+ end
33
+ end
29
34
  end
30
35
  end
31
36
 
@@ -40,6 +45,11 @@ module Krakow
40
45
  define_singleton_method(key) do
41
46
  arguments[key]
42
47
  end
48
+ if(key.match(/\w$/))
49
+ define_singleton_method("#{key}?".to_sym) do
50
+ !!arguments[key]
51
+ end
52
+ end
43
53
  end
44
54
  end
45
55
 
@@ -1,7 +1,5 @@
1
- require 'krakow'
2
-
3
1
  module Krakow
4
2
  class Version < Gem::Version
5
3
  end
6
- VERSION = Version.new('0.1.2')
4
+ VERSION = Version.new('0.2.0')
7
5
  end
data/lib/krakow.rb CHANGED
@@ -1,9 +1,11 @@
1
- autoload :Celluloid, 'celluloid'
1
+ require 'celluloid/autostart'
2
+ require 'multi_json'
2
3
 
3
4
  module Krakow
4
5
 
5
6
  autoload :Command, 'krakow/command'
6
7
  autoload :Connection, 'krakow/connection'
8
+ autoload :ConnectionFeatures, 'krakow/connection_features'
7
9
  autoload :Consumer, 'krakow/consumer'
8
10
  autoload :Discovery, 'krakow/discovery'
9
11
  autoload :Distribution, 'krakow/distribution'
data/test/spec.rb ADDED
@@ -0,0 +1,81 @@
1
+ require 'krakow'
2
+ require 'minitest/autorun'
3
+
4
+ module Krakow
5
+ class Test
6
+ class << self
7
+
8
+ def method_missing(*args)
9
+ name = method = args.first.to_s
10
+ if(name.end_with?('?'))
11
+ name = name.tr('?', '')
12
+ end
13
+ val = ENV[name.upcase] || ENV[name]
14
+ if(method.end_with?('?'))
15
+ !!val
16
+ else
17
+ val
18
+ end
19
+ end
20
+
21
+ def _topic
22
+ 'krakow-test'
23
+ end
24
+
25
+ def _scrub_topic!
26
+ _http_producer.delete_topic
27
+ end
28
+
29
+ def _http_producer
30
+ @http_producer ||= Krakow::Producer::Http.new(
31
+ :endpoint => 'http://127.0.0.1:4151',
32
+ :topic => _topic
33
+ )
34
+ end
35
+
36
+ def _producer(args={})
37
+ Krakow::Producer.new(
38
+ {
39
+ :host => Krakow::Test.nsq_producer_host || '127.0.0.1',
40
+ :port => Krakow::Test.nsq_producer_port || 4150,
41
+ :topic => 'krakow-test'
42
+ }.merge(args)
43
+ )
44
+ end
45
+
46
+ def _consumer(args={})
47
+ Krakow::Consumer.new(
48
+ {
49
+ :nslookupd => Krakow::Test.nsq_lookupd || 'http://127.0.0.1:4161',
50
+ :topic => 'krakow-test',
51
+ :channel => 'default',
52
+ :discovery_interval => 0.5,
53
+ :max_in_flight => 20
54
+ }.merge(args)
55
+ )
56
+ end
57
+
58
+ end
59
+ end
60
+ end
61
+
62
+ MiniTest::Spec.before do
63
+ Celluloid.boot
64
+ Krakow::Test._scrub_topic!
65
+ @consumer = Krakow::Test._consumer
66
+ @producer = Krakow::Test._producer
67
+ end
68
+
69
+ MiniTest::Spec.after do
70
+ @consumer.terminate if @consumer && @consumer.alive?
71
+ @producer.terminate if @producer && @producer.alive?
72
+ Krakow::Test._scrub_topic!
73
+ end
74
+
75
+ unless(Krakow::Test.debug?)
76
+ Celluloid.logger.level = 3
77
+ end
78
+
79
+ Dir.glob(File.join(File.dirname(__FILE__), 'specs', '*.rb')).each do |path|
80
+ require File.expand_path(path)
81
+ end
@@ -0,0 +1,49 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Consumer do
4
+
5
+ it 'should not have any connections' do
6
+ @consumer.connections.size.must_equal 0
7
+ end
8
+ it 'should have an empty queue' do
9
+ @consumer.queue.size.must_equal 0
10
+ end
11
+
12
+ describe 'with active producer' do
13
+
14
+ before do
15
+ @producer.write('msg1', 'msg2', 'msg3')
16
+ @inital_wait ||= sleep(0.8) # allow setup (topic creation, discovery, etc)
17
+ end
18
+
19
+ it 'should have one connection' do
20
+ @consumer.connections.size.must_equal 1
21
+ end
22
+ it 'should have three messages queued' do
23
+ @consumer.queue.size.must_equal 3
24
+ end
25
+ it 'should properly confirm messages' do
26
+ 3.times do
27
+ msg = @consumer.queue.pop
28
+ @consumer.confirm(msg).must_equal true
29
+ end
30
+ sleep(0.5) # pause to let everything settle
31
+ @consumer.queue.must_be :empty?
32
+ end
33
+ it 'should properly requeue messages' do
34
+ 2.times do
35
+ @consumer.confirm(@consumer.queue.pop)
36
+ end
37
+ @consumer.queue.size.must_equal 1
38
+ original_msg = @consumer.queue.pop
39
+ @consumer.queue.must_be :empty?
40
+ @consumer.requeue(original_msg).must_equal true
41
+ sleep(0.2)
42
+ @consumer.queue.size.must_equal 1
43
+ req_msg = @consumer.queue.pop
44
+ req_msg.message_id.must_equal original_msg.message_id
45
+ end
46
+
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,123 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Producer::Http do
4
+
5
+ before do
6
+ @http = Krakow::Test._http_producer
7
+ end
8
+
9
+ it 'should write single messages successfully' do
10
+ response = @http.write('msg')
11
+ Krakow::Command::Pub.ok.must_include response.status_txt
12
+ end
13
+ it 'should write multiple messages successfully' do
14
+ response = @http.write('msg1', 'msg2', 'msg3')
15
+ Krakow::Command::Mpub.ok.must_include response.status_txt
16
+ end
17
+ it 'should create topic' do
18
+ response = @http.create_topic
19
+ response.status_code.must_equal 200
20
+ response.status_txt.must_equal 'OK'
21
+ end
22
+ it 'should delete topic' do
23
+ c_response = @http.create_topic
24
+ c_response.status_code.must_equal 200
25
+ c_response.status_txt.must_equal 'OK'
26
+ d_response = @http.delete_topic
27
+ d_response.status_code.must_equal 200
28
+ d_response.status_txt.must_equal 'OK'
29
+ end
30
+ it 'should create channel' do
31
+ c_response = @http.create_topic
32
+ c_response.status_code.must_equal 200
33
+ c_response.status_txt.must_equal 'OK'
34
+ ch_response = @http.create_channel('fubar')
35
+ ch_response.status_code.must_equal 200
36
+ ch_response.status_txt.must_equal 'OK'
37
+ end
38
+ it 'should delete channel' do
39
+ c_response = @http.create_topic
40
+ c_response.status_code.must_equal 200
41
+ c_response.status_txt.must_equal 'OK'
42
+ ch_response = @http.create_channel('fubar')
43
+ ch_response.status_code.must_equal 200
44
+ ch_response.status_txt.must_equal 'OK'
45
+ dch_response = @http.delete_channel('fubar')
46
+ dch_response.status_code.must_equal 200
47
+ dch_response.status_txt.must_equal 'OK'
48
+ end
49
+ it 'should empty topic' do
50
+ @consumer.terminate
51
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
52
+ et_response = @http.empty_topic
53
+ et_response.status_code.must_equal 200
54
+ et_response.status_txt.must_equal 'OK'
55
+ @consumer = Krakow::Test._consumer
56
+ sleep(0.2)
57
+ @consumer.connections.wont_be :empty?
58
+ @consumer.queue.must_be :empty?
59
+ end
60
+ it 'should empty channel' do
61
+ @consumer.terminate
62
+ @http.write('msg0')
63
+ @http.pause_channel('chan2').status_code.must_equal 200
64
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
65
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
66
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
67
+ sleep(0.5)
68
+ consumer1.queue.size.must_equal 4
69
+ consumer2.queue.size.must_equal 0
70
+ @http.empty_channel('chan2').status_code.must_equal 200
71
+ @http.unpause_channel('chan2').status_code.must_equal 200
72
+ sleep(0.5)
73
+ consumer1.queue.size.must_equal 4
74
+ consumer2.queue.size.must_equal 0
75
+ consumer1.terminate
76
+ consumer2.terminate
77
+ end
78
+ it 'should pause channel' do
79
+ @consumer.terminate
80
+ @http.write('msg0')
81
+ @http.pause_channel('chan2').status_code.must_equal 200
82
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
83
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
84
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
85
+ sleep(0.5)
86
+ consumer1.queue.size.must_equal 4
87
+ consumer2.queue.size.must_equal 0
88
+ consumer1.terminate
89
+ consumer2.terminate
90
+ end
91
+ it 'should unpause channel' do
92
+ @consumer.terminate
93
+ @http.write('msg0')
94
+ @http.pause_channel('chan2').status_code.must_equal 200
95
+ consumer1 = Krakow::Test._consumer(:channel => 'chan1')
96
+ consumer2 = Krakow::Test._consumer(:channel => 'chan2')
97
+ @http.write('msg1', 'msg2', 'msg3').status_code.must_equal 200
98
+ sleep(0.5)
99
+ consumer1.queue.size.must_equal 4
100
+ consumer2.queue.size.must_equal 0
101
+ @http.unpause_channel('chan2').status_code.must_equal 200
102
+ sleep(0.5)
103
+ consumer2.queue.size.must_equal 4
104
+ consumer1.terminate
105
+ consumer2.terminate
106
+ end
107
+ it 'should return stats' do
108
+ stat = @http.stats
109
+ stat.status_code.must_equal 200
110
+ stat.data.must_be_kind_of Hash
111
+ end
112
+ it 'should ping' do
113
+ @http.ping.status_code.must_equal 200
114
+ end
115
+ it 'should fetch info' do
116
+ infos = @http.info
117
+ infos.status_code.must_equal 200
118
+ infos.data.must_be_kind_of Hash
119
+ end
120
+
121
+ end
122
+
123
+ end
@@ -0,0 +1,20 @@
1
+ describe Krakow do
2
+
3
+ describe Krakow::Producer do
4
+
5
+ it 'should be connected' do
6
+ @producer.connected?.must_equal true
7
+ end
8
+ it 'should write single messages successfully' do
9
+ response = @producer.write('msg')
10
+ response.must_be_kind_of Krakow::FrameType::Response
11
+ Krakow::Command::Pub.ok.must_include response.content
12
+ end
13
+ it 'should write multiple messages successfully' do
14
+ response = @producer.write('msg1', 'msg2', 'msg3')
15
+ response.must_be_kind_of Krakow::FrameType::Response
16
+ Krakow::Command::Mpub.ok.must_include response.content
17
+ end
18
+
19
+ end
20
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krakow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-20 00:00:00.000000000 Z
12
+ date: 2014-03-08 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: celluloid-io
@@ -59,6 +59,38 @@ dependencies:
59
59
  - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: snappy
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: digest-crc
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
62
94
  description: NSQ ruby library
63
95
  email: code@chrisroberts.org
64
96
  executables: []
@@ -93,9 +125,18 @@ files:
93
125
  - lib/krakow/exceptions.rb
94
126
  - lib/krakow/utils.rb
95
127
  - lib/krakow/discovery.rb
128
+ - lib/krakow/connection_features.rb
129
+ - lib/krakow/connection_features/ssl.rb
130
+ - lib/krakow/connection_features/snappy_frames.rb
131
+ - lib/krakow/connection_features/deflate.rb
132
+ - test/spec.rb
133
+ - test/specs/consumer.rb
134
+ - test/specs/producer.rb
135
+ - test/specs/http_producer.rb
96
136
  - Gemfile
97
137
  - README.md
98
138
  - krakow.gemspec
139
+ - krakow-0.2.0.gem
99
140
  - CHANGELOG.md
100
141
  - Gemfile.lock
101
142
  homepage: http://github.com/chrisroberts/krakow