nsq-krakow 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +84 -0
  3. data/CONTRIBUTING.md +25 -0
  4. data/LICENSE +13 -0
  5. data/README.md +249 -0
  6. data/krakow.gemspec +22 -0
  7. data/lib/krakow.rb +25 -0
  8. data/lib/krakow/command.rb +89 -0
  9. data/lib/krakow/command/auth.rb +36 -0
  10. data/lib/krakow/command/cls.rb +24 -0
  11. data/lib/krakow/command/fin.rb +31 -0
  12. data/lib/krakow/command/identify.rb +55 -0
  13. data/lib/krakow/command/mpub.rb +39 -0
  14. data/lib/krakow/command/nop.rb +14 -0
  15. data/lib/krakow/command/pub.rb +37 -0
  16. data/lib/krakow/command/rdy.rb +31 -0
  17. data/lib/krakow/command/req.rb +32 -0
  18. data/lib/krakow/command/sub.rb +36 -0
  19. data/lib/krakow/command/touch.rb +31 -0
  20. data/lib/krakow/connection.rb +417 -0
  21. data/lib/krakow/connection_features.rb +10 -0
  22. data/lib/krakow/connection_features/deflate.rb +82 -0
  23. data/lib/krakow/connection_features/snappy_frames.rb +129 -0
  24. data/lib/krakow/connection_features/ssl.rb +75 -0
  25. data/lib/krakow/consumer.rb +355 -0
  26. data/lib/krakow/consumer/queue.rb +151 -0
  27. data/lib/krakow/discovery.rb +57 -0
  28. data/lib/krakow/distribution.rb +229 -0
  29. data/lib/krakow/distribution/default.rb +159 -0
  30. data/lib/krakow/exceptions.rb +30 -0
  31. data/lib/krakow/frame_type.rb +66 -0
  32. data/lib/krakow/frame_type/error.rb +26 -0
  33. data/lib/krakow/frame_type/message.rb +74 -0
  34. data/lib/krakow/frame_type/response.rb +26 -0
  35. data/lib/krakow/ksocket.rb +102 -0
  36. data/lib/krakow/producer.rb +162 -0
  37. data/lib/krakow/producer/http.rb +224 -0
  38. data/lib/krakow/utils.rb +9 -0
  39. data/lib/krakow/utils/lazy.rb +125 -0
  40. data/lib/krakow/utils/logging.rb +43 -0
  41. data/lib/krakow/version.rb +4 -0
  42. metadata +184 -0
@@ -0,0 +1,30 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ # Base error type
5
+ class Error < StandardError
6
+
7
+ # Failed to enable required feature on connection
8
+ class ConnectionFeatureFailure < Error; end
9
+ # Failed to perform lookup (not found)
10
+ class LookupFailed < Error; end
11
+ # Connection has failed
12
+ class ConnectionFailure < Error; end
13
+ # Configuration is not in valid state
14
+ class ConfigurationError < Error; end
15
+ # Connection is temporarily unavailable
16
+ class ConnectionUnavailable < Error; end
17
+ # Consumer was not set
18
+ class OriginNotFound < Error; end
19
+
20
+ # Invalid response
21
+ class BadResponse < Error
22
+ # @return [Response] error response
23
+ attr_accessor :result
24
+ # No response received
25
+ class NoResponse < BadResponse
26
+ end
27
+ end
28
+
29
+ end
30
+ end
@@ -0,0 +1,66 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ # Received message
5
+ # @abstract
6
+ class FrameType
7
+
8
+ autoload :Error, 'krakow/frame_type/error'
9
+ autoload :Message, 'krakow/frame_type/message'
10
+ autoload :Response, 'krakow/frame_type/response'
11
+
12
+ include Utils::Lazy
13
+ # @!parse include Krakow::Utils::Lazy::InstanceMethods
14
+ # @!parse extend Krakow::Utils::Lazy::ClassMethods
15
+
16
+ # Registered frame types
17
+ FRAME_TYPE_MAP = [
18
+ FrameType::Response,
19
+ FrameType::Error,
20
+ FrameType::Message
21
+ ]
22
+ # Size bytes
23
+ SIZE_BYTES = 4
24
+
25
+ class << self
26
+
27
+ # Information about incoming frame
28
+ # @param bytes [String]
29
+ # @return [Hash]
30
+ def decode(bytes)
31
+ size, type = bytes.unpack('l>l>')
32
+ {:size => size - SIZE_BYTES, :type => type}
33
+ end
34
+
35
+ # Build proper FrameType instance based on args
36
+ # @param args [Hash]
37
+ # @option args [FrameType] :type class of frame
38
+ # @option args [String] :data
39
+ # @option args [Integer] :size
40
+ # @return [FrameType]
41
+ def build(args={})
42
+ klass = FRAME_TYPE_MAP[args[:type].to_i]
43
+ if(klass == FrameType::Response)
44
+ klass.new(:response => args[:data])
45
+ elsif(klass == FrameType::Error)
46
+ klass.new(:error => args[:data])
47
+ elsif(klass == FrameType::Message)
48
+ unpacked = args[:data].unpack("Q>s>a16a#{args[:size]}")
49
+ klass.new(
50
+ Hash[*([:timestamp, :attempts, :message_id, :message].zip(unpacked).flatten)]
51
+ )
52
+ else
53
+ raise TypeError.new "Unknown frame type received: #{args[:type].inspect} - #{klass.inspect}"
54
+ end
55
+ end
56
+ end
57
+
58
+ # Content of message
59
+ #
60
+ # @return [String]
61
+ def content
62
+ raise NotImplementedError.new 'Content method not properly defined!'
63
+ end
64
+
65
+ end
66
+ end
@@ -0,0 +1,26 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ class FrameType
5
+ # Error from server
6
+ class Error < FrameType
7
+
8
+ # @!group Attributes
9
+
10
+ # @!macro [attach] attribute
11
+ # @!method $1
12
+ # @return [$2] the $1 $0
13
+ # @!method $1?
14
+ # @return [TrueClass, FalseClass] truthiness of the $1 $0
15
+ attribute :error, String, :required => true
16
+
17
+ # @!endgroup
18
+
19
+ # @return [String] content of error
20
+ def content
21
+ error
22
+ end
23
+
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,74 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ class FrameType
5
+ # Message received from server
6
+ class Message < FrameType
7
+
8
+ # @return [Float] time of message instance creation
9
+ attr_reader :instance_stamp
10
+ attr_accessor :origin, :connection
11
+
12
+ # @!group Attributes
13
+
14
+ # @!macro [attach] attribute
15
+ # @!method $1
16
+ # @return [$2] the $1 $0
17
+ # @!method $1?
18
+ # @return [TrueClass, FalseClass] truthiness of the $1 $0
19
+ attribute :attempts, Integer, :required => true
20
+ attribute :timestamp, Integer, :required => true
21
+ attribute :message_id, String, :required => true
22
+ attribute :message, String, :required => true
23
+
24
+ # @!endgroup
25
+
26
+ def initialize(*args)
27
+ super
28
+ @instance_stamp = Time.now.to_f
29
+ end
30
+
31
+ # Message content
32
+ #
33
+ # @return [String]
34
+ def content
35
+ message
36
+ end
37
+
38
+ # @return [Krakow::Consumer]
39
+ def origin
40
+ unless(@origin)
41
+ error 'No origin has been specified for this message'
42
+ abort Krakow::Error::OriginNotFound.new('No origin specified for this message')
43
+ end
44
+ @origin
45
+ end
46
+
47
+ # @return [Krakow::Connection]
48
+ def connection
49
+ unless(@connection)
50
+ error 'No origin connection has been specified for this message'
51
+ abort Krakow::Error::ConnectionNotFound.new('No connection specified for this message')
52
+ end
53
+ @connection
54
+ end
55
+
56
+ # Proxy to [Krakow::Consumer#confirm]
57
+ def confirm(*args)
58
+ origin.confirm(*[self, *args].compact)
59
+ end
60
+ alias_method :finish, :confirm
61
+
62
+ # Proxy to [Krakow::Consumer#requeue]
63
+ def requeue(*args)
64
+ origin.requeue(*[self, *args].compact)
65
+ end
66
+
67
+ # Proxy to [Krakow::Consumer#touch]
68
+ def touch(*args)
69
+ origin.touch(*[self, *args].compact)
70
+ end
71
+
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,26 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+ class FrameType
5
+ # Response from server
6
+ class Response < FrameType
7
+
8
+ # @!group Attributes
9
+
10
+ # @!macro [attach] attribute
11
+ # @!method $1
12
+ # @return [$2] the $1 $0
13
+ # @!method $1?
14
+ # @return [TrueClass, FalseClass] truthiness of the $1 $0
15
+ attribute :response, String, :required => true
16
+
17
+ # @!endgroup
18
+
19
+ # @return [String] content of response
20
+ def content
21
+ response
22
+ end
23
+
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,102 @@
1
+ require 'krakow'
2
+ require 'socket'
3
+
4
+ module Krakow
5
+ class Ksocket
6
+
7
+ include Utils::Lazy
8
+ include Celluloid
9
+
10
+ # @return [String]
11
+ attr_reader :buffer
12
+
13
+ finalizer :closedown_socket
14
+
15
+ # Teardown helper
16
+ def closedown_socket
17
+ @writing = @reading = false
18
+ if(socket && !socket.closed?)
19
+ socket.close
20
+ end
21
+ end
22
+
23
+ # Create new socket wrapper
24
+ #
25
+ # @param args [Hash]
26
+ # @option args [Socket-ish] :socket
27
+ # @option args [String] :host
28
+ # @option args [Integer] :port
29
+ # @return [self]
30
+ def initialize(args={})
31
+ if(args[:socket])
32
+ @socket = args[:socket]
33
+ else
34
+ unless([:host, :port].all?{|k| args.include?(k)})
35
+ raise ArgumentError.new 'Missing required arguments. Expecting `:socket` or `:host` and `:port`.'
36
+ end
37
+ @socket = TCPSocket.new(args[:host], args[:port])
38
+ end
39
+ @buffer = ''
40
+ async.read_loop
41
+ end
42
+
43
+ # @return [TrueClass, FalseClass] read loop enabled
44
+ def reading?
45
+ !!@reading
46
+ end
47
+
48
+ # Read from socket and push into local Queue
49
+ def read_loop
50
+ unless(reading?)
51
+ @reading = true
52
+ while(reading?)
53
+ res = defer do
54
+ Kernel.select([socket], nil, nil, nil)
55
+ socket{|s| s.readpartial(1024)}
56
+ end
57
+ if(res)
58
+ debug "Received content from socket: #{res.inspect}"
59
+ buffer << res
60
+ signal(:content_read)
61
+ else
62
+ debug 'No content received from socket read. Ignoring.'
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ # Fetch bytes from socket
69
+ #
70
+ # @param n [Integer]
71
+ # @return [String]
72
+ def get(n)
73
+ until(buffer.length >= n)
74
+ wait(:content_read)
75
+ end
76
+ buffer.slice!(0, n)
77
+ end
78
+ alias_method :recv, :get
79
+ alias_method :read, :get
80
+ alias_method :sysread, :get
81
+ alias_method :readpartial, :get
82
+
83
+ # Push bytes to socket
84
+ #
85
+ # @param line [String]
86
+ # @return [Integer]
87
+ def put(line)
88
+ socket{|s| s.write(line)}
89
+ end
90
+ alias_method :write, :put
91
+
92
+ # @return [Socket]
93
+ def socket
94
+ if(block_given?)
95
+ yield @socket
96
+ else
97
+ @socket
98
+ end
99
+ end
100
+
101
+ end
102
+ end
@@ -0,0 +1,162 @@
1
+ require 'krakow'
2
+
3
+ module Krakow
4
+
5
+ # TCP based producer
6
+ class Producer
7
+
8
+ autoload :Http, 'krakow/producer/http'
9
+
10
+ include Utils::Lazy
11
+ # @!parse include Utils::Lazy::InstanceMethods
12
+ # @!parse extend Utils::Lazy::ClassMethods
13
+
14
+ include Celluloid
15
+
16
+ trap_exit :connection_failure
17
+ finalizer :producer_cleanup
18
+
19
+ # set exclusive methods
20
+ exclusive :write
21
+
22
+ attr_reader :connection
23
+ attr_reader :notifier
24
+
25
+ # @!group Attributes
26
+
27
+ # @!macro [attach] attribute
28
+ # @!method $1
29
+ # @return [$2] the $1 $0
30
+ # @!method $1?
31
+ # @return [TrueClass, FalseClass] truthiness of the $1 $0
32
+ attribute :host, String, :required => true
33
+ attribute :port, [String, Integer], :required => true
34
+ attribute :topic, String, :required => true
35
+ attribute :reconnect_retries, Integer, :default => 10
36
+ attribute :reconnect_interval, Integer, :default => 5
37
+ attribute :connection_options, Hash, :default => ->{ Hash.new }
38
+
39
+ # @!endgroup
40
+
41
+ def initialize(args={})
42
+ super
43
+ arguments[:connection_options] = {:features => {}, :config => {}, :options => {}}.merge(
44
+ arguments.fetch(:connection_options, {})
45
+ )
46
+ connect
47
+ end
48
+
49
+ # Establish connection to configured `host` and `port`
50
+ #
51
+ # @return nil
52
+ def connect
53
+ @connecting = true
54
+ info "Establishing connection to: #{host}:#{port}"
55
+ begin
56
+ con_args = connection_options[:options].dup.tap do |args|
57
+ args[:host] = host
58
+ args[:port] = port
59
+ if(connection_options[:features])
60
+ args[:features] = connection_options[:features]
61
+ end
62
+ if(connection_options[:config])
63
+ args[:features_args] = connection_options[:config]
64
+ end
65
+ end
66
+ @connection = Connection.new(con_args)
67
+ @connection.init!
68
+ self.link @connection
69
+ info "Connection established: #{@connection}"
70
+ nil
71
+ rescue => e
72
+ abort e
73
+ end
74
+ @connecting = false
75
+ end
76
+
77
+ # @return [String] stringify object
78
+ def to_s
79
+ "<#{self.class.name}:#{object_id} {#{host}:#{port}} T:#{topic}>"
80
+ end
81
+
82
+ # @return [TrueClass, FalseClass] currently connected to server
83
+ def connected?
84
+ begin
85
+ !!(!@connecting &&
86
+ connection &&
87
+ connection.alive? &&
88
+ connection.connected?)
89
+ rescue Celluloid::DeadActorError
90
+ false
91
+ end
92
+ end
93
+
94
+ # Process connection failure and attempt reconnection
95
+ #
96
+ # @return [TrueClass]
97
+ def connection_failure(obj, reason)
98
+ if(obj == connection && !reason.nil?)
99
+ begin
100
+ @connection = nil
101
+ warn "Connection failure detected for #{host}:#{port} - #{reason}"
102
+ obj.terminate if obj.alive?
103
+ connect
104
+ rescue => reason
105
+ warn "Failed to establish connection to #{host}:#{port}. Pausing #{reconnect_interval} before retry"
106
+ sleep reconnect_interval
107
+ retry
108
+ end
109
+ end
110
+ true
111
+ end
112
+
113
+ # Instance destructor
114
+ # @return nil
115
+ def producer_cleanup
116
+ debug 'Tearing down producer'
117
+ if(connection && connection.alive?)
118
+ connection.terminate
119
+ end
120
+ @connection = nil
121
+ info 'Producer torn down'
122
+ nil
123
+ end
124
+
125
+ # Write message to server
126
+ #
127
+ # @param message [String] message to write
128
+ # @return [Krakow::FrameType, TrueClass]
129
+ # @note if connection response wait is set to 0, writes will
130
+ # return a `true` value on completion
131
+ # @raise [Krakow::Error::ConnectionUnavailable]
132
+ def write(*message)
133
+ if(message.empty?)
134
+ abort ArgumentError.new 'Expecting one or more messages to send. None provided.'
135
+ end
136
+ begin
137
+ if(message.size > 1)
138
+ debug 'Multiple message publish'
139
+ connection.transmit(
140
+ Command::Mpub.new(
141
+ :topic_name => topic,
142
+ :messages => message
143
+ )
144
+ )
145
+ else
146
+ debug 'Single message publish'
147
+ connection.transmit(
148
+ Command::Pub.new(
149
+ :message => message.first,
150
+ :topic_name => topic
151
+ )
152
+ )
153
+ end
154
+ rescue Celluloid::Task::TerminatedError
155
+ abort Error::ConnectionUnavailable.new 'Connection is currently unavailable'
156
+ rescue => e
157
+ abort e
158
+ end
159
+ end
160
+
161
+ end
162
+ end