elasticsearch-transport-sinneduy 1.0.12

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/Gemfile +16 -0
  4. data/LICENSE.txt +13 -0
  5. data/README.md +441 -0
  6. data/Rakefile +80 -0
  7. data/elasticsearch-transport.gemspec +74 -0
  8. data/lib/elasticsearch-transport.rb +1 -0
  9. data/lib/elasticsearch/transport.rb +30 -0
  10. data/lib/elasticsearch/transport/client.rb +195 -0
  11. data/lib/elasticsearch/transport/transport/base.rb +284 -0
  12. data/lib/elasticsearch/transport/transport/connections/collection.rb +93 -0
  13. data/lib/elasticsearch/transport/transport/connections/connection.rb +121 -0
  14. data/lib/elasticsearch/transport/transport/connections/selector.rb +63 -0
  15. data/lib/elasticsearch/transport/transport/errors.rb +73 -0
  16. data/lib/elasticsearch/transport/transport/http/curb.rb +87 -0
  17. data/lib/elasticsearch/transport/transport/http/faraday.rb +60 -0
  18. data/lib/elasticsearch/transport/transport/http/manticore.rb +124 -0
  19. data/lib/elasticsearch/transport/transport/response.rb +21 -0
  20. data/lib/elasticsearch/transport/transport/serializer/multi_json.rb +36 -0
  21. data/lib/elasticsearch/transport/transport/sniffer.rb +46 -0
  22. data/lib/elasticsearch/transport/version.rb +5 -0
  23. data/test/integration/client_test.rb +144 -0
  24. data/test/integration/transport_test.rb +73 -0
  25. data/test/profile/client_benchmark_test.rb +125 -0
  26. data/test/test_helper.rb +76 -0
  27. data/test/unit/client_test.rb +274 -0
  28. data/test/unit/connection_collection_test.rb +88 -0
  29. data/test/unit/connection_selector_test.rb +64 -0
  30. data/test/unit/connection_test.rb +100 -0
  31. data/test/unit/response_test.rb +15 -0
  32. data/test/unit/serializer_test.rb +16 -0
  33. data/test/unit/sniffer_test.rb +145 -0
  34. data/test/unit/transport_base_test.rb +478 -0
  35. data/test/unit/transport_curb_test.rb +97 -0
  36. data/test/unit/transport_faraday_test.rb +140 -0
  37. data/test/unit/transport_manticore_test.rb +118 -0
  38. metadata +408 -0
@@ -0,0 +1,93 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+
6
+ # Wraps the collection of connections for the transport object as an Enumerable object.
7
+ #
8
+ # @see Base#connections
9
+ # @see Selector::Base#select
10
+ # @see Connection
11
+ #
12
+ class Collection
13
+ include Enumerable
14
+
15
+ DEFAULT_SELECTOR = Selector::RoundRobin
16
+
17
+ attr_reader :selector
18
+
19
+ # @option arguments [Array] :connections An array of {Connection} objects.
20
+ # @option arguments [Constant] :selector_class The class to be used as a connection selector strategy.
21
+ # @option arguments [Object] :selector The selector strategy object.
22
+ #
23
+ def initialize(arguments={})
24
+ selector_class = arguments[:selector_class] || DEFAULT_SELECTOR
25
+ @connections = arguments[:connections] || []
26
+ @selector = arguments[:selector] || selector_class.new(arguments.merge(:connections => self))
27
+ end
28
+
29
+ # Returns an Array of hosts information in this collection as Hashes.
30
+ #
31
+ # @return [Array]
32
+ #
33
+ def hosts
34
+ @connections.to_a.map { |c| c.host }
35
+ end
36
+
37
+ # Returns an Array of alive connections.
38
+ #
39
+ # @return [Array]
40
+ #
41
+ def connections
42
+ @connections.reject { |c| c.dead? }
43
+ end
44
+ alias :alive :connections
45
+
46
+ # Returns an Array of dead connections.
47
+ #
48
+ # @return [Array]
49
+ #
50
+ def dead
51
+ @connections.select { |c| c.dead? }
52
+ end
53
+
54
+ # Returns an Array of all connections, both dead and alive
55
+ #
56
+ # @return [Array]
57
+ #
58
+ def all
59
+ @connections
60
+ end
61
+
62
+ # Returns a connection.
63
+ #
64
+ # If there are no alive connections, resurrects a connection with least failures.
65
+ # Delegates to selector's `#select` method to get the connection.
66
+ #
67
+ # @return [Connection]
68
+ #
69
+ def get_connection(options={})
70
+ if connections.empty? && dead_connection = dead.sort { |a,b| a.failures <=> b.failures }.first
71
+ dead_connection.alive!
72
+ end
73
+ selector.select(options)
74
+ end
75
+
76
+ def each(&block)
77
+ connections.each(&block)
78
+ end
79
+
80
+ def slice(*args)
81
+ connections.slice(*args)
82
+ end
83
+ alias :[] :slice
84
+
85
+ def size
86
+ connections.size
87
+ end
88
+ end
89
+
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,121 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+
6
+ # Wraps the connection information and logic.
7
+ #
8
+ # The Connection instance wraps the host information (hostname, port, attributes, etc),
9
+ # as well as the "session" (a transport client object, such as a {HTTP::Faraday} instance).
10
+ #
11
+ # It provides methods to construct and properly encode the URLs and paths for passing them
12
+ # to the transport client object.
13
+ #
14
+ # It provides methods to handle connection livecycle (dead, alive, healthy).
15
+ #
16
+ class Connection
17
+ DEFAULT_RESURRECT_TIMEOUT = 60
18
+
19
+ attr_reader :host, :connection, :options, :failures, :dead_since
20
+
21
+ # @option arguments [Hash] :host Host information (example: `{host: 'localhost', port: 9200}`)
22
+ # @option arguments [Object] :connection The transport-specific physical connection or "session"
23
+ # @option arguments [Hash] :options Options (usually passed in from transport)
24
+ #
25
+ def initialize(arguments={})
26
+ @host = arguments[:host]
27
+ @connection = arguments[:connection]
28
+ @options = arguments[:options] || {}
29
+
30
+ @options[:resurrect_timeout] ||= DEFAULT_RESURRECT_TIMEOUT
31
+ @failures = 0
32
+ end
33
+
34
+ # Returns the complete endpoint URL with host, port, path and serialized parameters.
35
+ #
36
+ # @return [String]
37
+ #
38
+ def full_url(path, params={})
39
+ url = "#{host[:protocol]}://"
40
+ url += "#{host[:user]}:#{host[:password]}@" if host[:user]
41
+ url += "#{host[:host]}:#{host[:port]}"
42
+ url += "#{host[:path]}" if host[:path]
43
+ url += "/#{full_path(path, params)}"
44
+ end
45
+
46
+ # Returns the complete endpoint path with serialized parameters.
47
+ #
48
+ # @return [String]
49
+ #
50
+ def full_path(path, params={})
51
+ path + (params.empty? ? '' : "?#{::Faraday::Utils::ParamsHash[params].to_query}")
52
+ end
53
+
54
+ # Returns true when this connection has been marked dead, false otherwise.
55
+ #
56
+ # @return [Boolean]
57
+ #
58
+ def dead?
59
+ @dead || false
60
+ end
61
+
62
+ # Marks this connection as dead, incrementing the `failures` counter and
63
+ # storing the current time as `dead_since`.
64
+ #
65
+ # @return [self]
66
+ #
67
+ def dead!
68
+ @dead = true
69
+ @failures += 1
70
+ @dead_since = Time.now
71
+ self
72
+ end
73
+
74
+ # Marks this connection as alive, ie. it is eligible to be returned from the pool by the selector.
75
+ #
76
+ # @return [self]
77
+ #
78
+ def alive!
79
+ @dead = false
80
+ self
81
+ end
82
+
83
+ # Marks this connection as healthy, ie. a request has been successfully performed with it.
84
+ #
85
+ # @return [self]
86
+ #
87
+ def healthy!
88
+ @dead = false
89
+ @failures = 0
90
+ self
91
+ end
92
+
93
+ # Marks this connection as alive, if the required timeout has passed.
94
+ #
95
+ # @return [self,nil]
96
+ # @see DEFAULT_RESURRECT_TIMEOUT
97
+ # @see #resurrectable?
98
+ #
99
+ def resurrect!
100
+ alive! if resurrectable?
101
+ end
102
+
103
+ # Returns true if the connection is eligible to be resurrected as alive, false otherwise.
104
+ #
105
+ # @return [Boolean]
106
+ #
107
+ def resurrectable?
108
+ Time.now > @dead_since + ( @options[:resurrect_timeout] * 2 ** (@failures-1) )
109
+ end
110
+
111
+ # @return [String]
112
+ #
113
+ def to_s
114
+ "<#{self.class.name} host: #{host} (#{dead? ? 'dead since ' + dead_since.to_s : 'alive'})>"
115
+ end
116
+ end
117
+
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,63 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+ module Selector
6
+
7
+ # @abstract Common functionality for connection selector implementations.
8
+ #
9
+ module Base
10
+ attr_reader :connections
11
+
12
+ # @option arguments [Connections::Collection] :connections Collection with connections.
13
+ #
14
+ def initialize(arguments={})
15
+ @connections = arguments[:connections]
16
+ end
17
+
18
+ # @abstract Selector strategies implement this method to
19
+ # select and return a connection from the pool.
20
+ #
21
+ # @return [Connection]
22
+ #
23
+ def select(options={})
24
+ raise NoMethodError, "Implement this method in the selector implementation."
25
+ end
26
+ end
27
+
28
+ # "Random connection" selector strategy.
29
+ #
30
+ class Random
31
+ include Base
32
+
33
+ # Returns a random connection from the collection.
34
+ #
35
+ # @return [Connections::Connection]
36
+ #
37
+ def select(options={})
38
+ connections.to_a.send( defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' ? :sample : :choice)
39
+ end
40
+ end
41
+
42
+ # "Round-robin" selector strategy (default).
43
+ #
44
+ class RoundRobin
45
+ include Base
46
+
47
+ # Returns the next connection from the collection, rotating them in round-robin fashion.
48
+ #
49
+ # @return [Connections::Connection]
50
+ #
51
+ def select(options={})
52
+ # On Ruby 1.9, Array#rotate could be used instead
53
+ @current = @current.nil? ? 0 : @current+1
54
+ @current = 0 if @current >= connections.size
55
+ connections[@current]
56
+ end
57
+ end
58
+
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,73 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+
5
+ # Generic client error
6
+ #
7
+ class Error < StandardError; end
8
+
9
+ # Reloading connections timeout (1 sec by default)
10
+ #
11
+ class SnifferTimeoutError < Timeout::Error; end
12
+
13
+ # Elasticsearch server error (HTTP status 5xx)
14
+ #
15
+ class ServerError < Error; end
16
+
17
+ module Errors; end
18
+
19
+ HTTP_STATUSES = {
20
+ 300 => 'MultipleChoices',
21
+ 301 => 'MovedPermanently',
22
+ 302 => 'Found',
23
+ 303 => 'SeeOther',
24
+ 304 => 'NotModified',
25
+ 305 => 'UseProxy',
26
+ 307 => 'TemporaryRedirect',
27
+ 308 => 'PermanentRedirect',
28
+
29
+ 400 => 'BadRequest',
30
+ 401 => 'Unauthorized',
31
+ 402 => 'PaymentRequired',
32
+ 403 => 'Forbidden',
33
+ 404 => 'NotFound',
34
+ 405 => 'MethodNotAllowed',
35
+ 406 => 'NotAcceptable',
36
+ 407 => 'ProxyAuthenticationRequired',
37
+ 408 => 'RequestTimeout',
38
+ 409 => 'Conflict',
39
+ 410 => 'Gone',
40
+ 411 => 'LengthRequired',
41
+ 412 => 'PreconditionFailed',
42
+ 413 => 'RequestEntityTooLarge',
43
+ 414 => 'RequestURITooLong',
44
+ 415 => 'UnsupportedMediaType',
45
+ 416 => 'RequestedRangeNotSatisfiable',
46
+ 417 => 'ExpectationFailed',
47
+ 418 => 'ImATeapot',
48
+ 421 => 'TooManyConnectionsFromThisIP',
49
+ 426 => 'UpgradeRequired',
50
+ 450 => 'BlockedByWindowsParentalControls',
51
+ 494 => 'RequestHeaderTooLarge',
52
+ 497 => 'HTTPToHTTPS',
53
+ 499 => 'ClientClosedRequest',
54
+
55
+ 500 => 'InternalServerError',
56
+ 501 => 'NotImplemented',
57
+ 502 => 'BadGateway',
58
+ 503 => 'ServiceUnavailable',
59
+ 504 => 'GatewayTimeout',
60
+ 505 => 'HTTPVersionNotSupported',
61
+ 506 => 'VariantAlsoNegotiates',
62
+ 510 => 'NotExtended'
63
+ }
64
+
65
+ ERRORS = HTTP_STATUSES.inject({}) do |sum, error|
66
+ status, name = error
67
+ sum[status] = Errors.const_set name, Class.new(ServerError)
68
+ sum
69
+ end
70
+
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,87 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module HTTP
5
+
6
+ # Alternative HTTP transport implementation, using the [_Curb_](https://rubygems.org/gems/curb) client.
7
+ #
8
+ # @see Transport::Base
9
+ #
10
+ class Curb
11
+ include Base
12
+
13
+ # Performs the request by invoking {Transport::Base#perform_request} with a block.
14
+ #
15
+ # @return [Response]
16
+ # @see Transport::Base#perform_request
17
+ #
18
+ def perform_request(method, path, params={}, body=nil)
19
+ super do |connection,url|
20
+ connection.connection.url = url
21
+
22
+ case method
23
+ when 'HEAD'
24
+ when 'GET', 'POST', 'PUT', 'DELETE'
25
+ connection.connection.put_data = __convert_to_json(body) if body
26
+ else raise ArgumentError, "Unsupported HTTP method: #{method}"
27
+ end
28
+
29
+ connection.connection.http(method.to_sym)
30
+
31
+ headers = {}
32
+ headers['content-type'] = 'application/json' if connection.connection.header_str =~ /\/json/
33
+
34
+ Response.new connection.connection.response_code,
35
+ connection.connection.body_str,
36
+ headers
37
+ end
38
+ end
39
+
40
+ # Builds and returns a collection of connections.
41
+ #
42
+ # @return [Connections::Collection]
43
+ #
44
+ def __build_connections
45
+ Connections::Collection.new \
46
+ :connections => hosts.map { |host|
47
+ host[:protocol] = host[:scheme] || DEFAULT_PROTOCOL
48
+ host[:port] ||= DEFAULT_PORT
49
+
50
+ client = ::Curl::Easy.new
51
+ client.headers = {'User-Agent' => "Curb #{Curl::CURB_VERSION}"}
52
+ client.url = __full_url(host)
53
+
54
+ if host[:user]
55
+ client.http_auth_types = host[:auth_type] || :basic
56
+ client.username = host[:user]
57
+ client.password = host[:password]
58
+ end
59
+
60
+ client.instance_eval &@block if @block
61
+
62
+ Connections::Connection.new :host => host, :connection => client
63
+ },
64
+ :selector_class => options[:selector_class],
65
+ :selector => options[:selector]
66
+ end
67
+
68
+ # Returns an array of implementation specific connection errors.
69
+ #
70
+ # @return [Array]
71
+ #
72
+ def host_unreachable_exceptions
73
+ [
74
+ ::Curl::Err::HostResolutionError,
75
+ ::Curl::Err::ConnectionFailedError,
76
+ ::Curl::Err::GotNothingError,
77
+ ::Curl::Err::RecvError,
78
+ ::Curl::Err::SendError,
79
+ ::Curl::Err::TimeoutError
80
+ ]
81
+ end
82
+ end
83
+
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,60 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module HTTP
5
+
6
+ # The default transport implementation, using the [_Faraday_](https://rubygems.org/gems/faraday)
7
+ # library for abstracting the HTTP client.
8
+ #
9
+ # @see Transport::Base
10
+ #
11
+ class Faraday
12
+ include Base
13
+
14
+ # Performs the request by invoking {Transport::Base#perform_request} with a block.
15
+ #
16
+ # @return [Response]
17
+ # @see Transport::Base#perform_request
18
+ #
19
+ def perform_request(method, path, params={}, body=nil)
20
+ super do |connection, url|
21
+ response = connection.connection.run_request \
22
+ method.downcase.to_sym,
23
+ url,
24
+ ( body ? __convert_to_json(body) : nil ),
25
+ {}
26
+ Response.new response.status, response.body, response.headers
27
+ end
28
+ end
29
+
30
+ # Builds and returns a collection of connections.
31
+ #
32
+ # @return [Connections::Collection]
33
+ #
34
+ def __build_connections
35
+ Connections::Collection.new \
36
+ :connections => hosts.map { |host|
37
+ host[:protocol] = host[:scheme] || DEFAULT_PROTOCOL
38
+ host[:port] ||= DEFAULT_PORT
39
+ url = __full_url(host)
40
+
41
+ Connections::Connection.new \
42
+ :host => host,
43
+ :connection => ::Faraday::Connection.new(url, (options[:transport_options] || {}), &@block )
44
+ },
45
+ :selector_class => options[:selector_class],
46
+ :selector => options[:selector]
47
+ end
48
+
49
+ # Returns an array of implementation specific connection errors.
50
+ #
51
+ # @return [Array]
52
+ #
53
+ def host_unreachable_exceptions
54
+ [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end