elasticsearch-transport 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. data/.gitignore +17 -0
  2. data/Gemfile +4 -0
  3. data/LICENSE.txt +13 -0
  4. data/README.md +276 -0
  5. data/Rakefile +67 -0
  6. data/elasticsearch-transport.gemspec +52 -0
  7. data/lib/elasticsearch-transport.rb +1 -0
  8. data/lib/elasticsearch/transport.rb +29 -0
  9. data/lib/elasticsearch/transport/client.rb +123 -0
  10. data/lib/elasticsearch/transport/extensions/test_cluster.rb +163 -0
  11. data/lib/elasticsearch/transport/transport/base.rb +236 -0
  12. data/lib/elasticsearch/transport/transport/connections/collection.rb +93 -0
  13. data/lib/elasticsearch/transport/transport/connections/connection.rb +117 -0
  14. data/lib/elasticsearch/transport/transport/connections/selector.rb +63 -0
  15. data/lib/elasticsearch/transport/transport/errors.rb +73 -0
  16. data/lib/elasticsearch/transport/transport/http/curb.rb +70 -0
  17. data/lib/elasticsearch/transport/transport/http/faraday.rb +59 -0
  18. data/lib/elasticsearch/transport/transport/response.rb +20 -0
  19. data/lib/elasticsearch/transport/transport/serializer/multi_json.rb +36 -0
  20. data/lib/elasticsearch/transport/transport/sniffer.rb +46 -0
  21. data/lib/elasticsearch/transport/version.rb +5 -0
  22. data/test/integration/client_test.rb +117 -0
  23. data/test/integration/transport_test.rb +37 -0
  24. data/test/profile/client_benchmark_test.rb +107 -0
  25. data/test/test_extensions.rb +139 -0
  26. data/test/test_helper.rb +58 -0
  27. data/test/unit/client_test.rb +109 -0
  28. data/test/unit/connection_collection_test.rb +83 -0
  29. data/test/unit/connection_selector_test.rb +64 -0
  30. data/test/unit/connection_test.rb +90 -0
  31. data/test/unit/serializer_test.rb +16 -0
  32. data/test/unit/sniffer_test.rb +146 -0
  33. data/test/unit/transport_base_test.rb +402 -0
  34. data/test/unit/transport_curb_test.rb +59 -0
  35. data/test/unit/transport_faraday_test.rb +73 -0
  36. metadata +342 -0
@@ -0,0 +1,93 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+
6
+ # Wraps the collection of connections for the transport object as an Enumerable object.
7
+ #
8
+ # @see Base#connections
9
+ # @see Selector::Base#select
10
+ # @see Connection
11
+ #
12
+ class Collection
13
+ include Enumerable
14
+
15
+ DEFAULT_SELECTOR = Selector::RoundRobin
16
+
17
+ attr_reader :selector
18
+
19
+ # @option arguments [Array] :connections An array of {Connection} objects.
20
+ # @option arguments [Constant] :selector_class The class to be used as a connection selector strategy.
21
+ # @option arguments [Object] :selector The selector strategy object.
22
+ #
23
+ def initialize(arguments={})
24
+ selector_class = arguments[:selector_class] || DEFAULT_SELECTOR
25
+ @connections = arguments[:connections] || []
26
+ @selector = arguments[:selector] || selector_class.new(arguments)
27
+ end
28
+
29
+ # Returns an Array of hosts information in this collection as Hashes.
30
+ #
31
+ # @return [Array]
32
+ #
33
+ def hosts
34
+ @connections.to_a.map { |c| c.host }
35
+ end
36
+
37
+ # Returns an Array of alive connections.
38
+ #
39
+ # @return [Array]
40
+ #
41
+ def connections
42
+ @connections.reject { |c| c.dead? }
43
+ end
44
+ alias :alive :connections
45
+
46
+ # Returns an Array of dead connections.
47
+ #
48
+ # @return [Array]
49
+ #
50
+ def dead
51
+ @connections.select { |c| c.dead? }
52
+ end
53
+
54
+ # Returns an Array of all connections, both dead and alive
55
+ #
56
+ # @return [Array]
57
+ #
58
+ def all
59
+ @connections
60
+ end
61
+
62
+ # Returns a connection.
63
+ #
64
+ # If there are no alive connections, resurrects a connection with least failures.
65
+ # Delegates to selector's `#select` method to get the connection.
66
+ #
67
+ # @return [Connection]
68
+ #
69
+ def get_connection(options={})
70
+ if connections.empty? && dead_connection = dead.sort { |a,b| a.failures <=> b.failures }.first
71
+ dead_connection.alive!
72
+ end
73
+ selector.select(options)
74
+ end
75
+
76
+ def each(&block)
77
+ connections.each(&block)
78
+ end
79
+
80
+ def slice(*args)
81
+ connections.slice(*args)
82
+ end
83
+ alias :[] :slice
84
+
85
+ def size
86
+ connections.size
87
+ end
88
+ end
89
+
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,117 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+
6
+ # Wraps the connection information and logic.
7
+ #
8
+ # The Connection instance wraps the host information (hostname, port, attributes, etc),
9
+ # as well as the "session" (a transport client object, such as a {HTTP::Faraday} instance).
10
+ #
11
+ # It provides methods to construct and properly encode the URLs and paths for passing them
12
+ # to the transport client object.
13
+ #
14
+ # It provides methods to handle connection livecycle (dead, alive, healthy).
15
+ #
16
+ class Connection
17
+ DEFAULT_RESURRECT_TIMEOUT = 60
18
+
19
+ attr_reader :host, :connection, :options, :failures, :dead_since
20
+
21
+ # @option arguments [Hash] :host Host information (example: `{host: 'localhost', port: 9200}`)
22
+ # @option arguments [Object] :connection The transport-specific physical connection or "session"
23
+ # @option arguments [Hash] :options Options (usually passed in from transport)
24
+ #
25
+ def initialize(arguments={})
26
+ @host = arguments[:host]
27
+ @connection = arguments[:connection]
28
+ @options = arguments[:options] || {}
29
+
30
+ @options[:resurrect_timeout] ||= DEFAULT_RESURRECT_TIMEOUT
31
+ @failures = 0
32
+ end
33
+
34
+ # Returns the complete endpoint URL with host, port, path and serialized parameters.
35
+ #
36
+ # @return [String]
37
+ #
38
+ def full_url(path, params={})
39
+ "#{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{full_path(path, params)}"
40
+ end
41
+
42
+ # Returns the complete endpoint path with serialized parameters.
43
+ #
44
+ # @return [String]
45
+ #
46
+ def full_path(path, params={})
47
+ path + (params.empty? ? '' : "?#{::Faraday::Utils::ParamsHash[params].to_query}")
48
+ end
49
+
50
+ # Returns true when this connection has been marked dead, false otherwise.
51
+ #
52
+ # @return [Boolean]
53
+ #
54
+ def dead?
55
+ @dead || false
56
+ end
57
+
58
+ # Marks this connection as dead, incrementing the `failures` counter and
59
+ # storing the current time as `dead_since`.
60
+ #
61
+ # @return [self]
62
+ #
63
+ def dead!
64
+ @dead = true
65
+ @failures += 1
66
+ @dead_since = Time.now
67
+ self
68
+ end
69
+
70
+ # Marks this connection as alive, ie. it is eligible to be returned from the pool by the selector.
71
+ #
72
+ # @return [self]
73
+ #
74
+ def alive!
75
+ @dead = false
76
+ self
77
+ end
78
+
79
+ # Marks this connection as healthy, ie. a request has been successfully performed with it.
80
+ #
81
+ # @return [self]
82
+ #
83
+ def healthy!
84
+ @dead = false
85
+ @failures = 0
86
+ self
87
+ end
88
+
89
+ # Marks this connection as alive, if the required timeout has passed.
90
+ #
91
+ # @return [self,nil]
92
+ # @see DEFAULT_RESURRECT_TIMEOUT
93
+ # @see #resurrectable?
94
+ #
95
+ def resurrect!
96
+ alive! if resurrectable?
97
+ end
98
+
99
+ # Returns true if the connection is eligible to be resurrected as alive, false otherwise.
100
+ #
101
+ # @return [Boolean]
102
+ #
103
+ def resurrectable?
104
+ Time.now > @dead_since + ( @options[:resurrect_timeout] * 2 ** (@failures-1) )
105
+ end
106
+
107
+ # @return [String]
108
+ #
109
+ def to_s
110
+ "<#{self.class.name} host: #{host} (#{dead? ? 'dead since ' + dead_since.to_s : 'alive'})>"
111
+ end
112
+ end
113
+
114
+ end
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,63 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module Connections
5
+ module Selector
6
+
7
+ # @abstract Common functionality for connection selector implementations.
8
+ #
9
+ module Base
10
+ attr_reader :connections
11
+
12
+ # @option arguments [Connections::Collection] :connections Collection with connections.
13
+ #
14
+ def initialize(arguments={})
15
+ @connections = arguments[:connections]
16
+ end
17
+
18
+ # @abstract Selector strategies implement this method to
19
+ # select and return a connection from the pool.
20
+ #
21
+ # @return [Connection]
22
+ #
23
+ def select(options={})
24
+ raise NoMethodError, "Implement this method in the selector implementation."
25
+ end
26
+ end
27
+
28
+ # "Random connection" selector strategy.
29
+ #
30
+ class Random
31
+ include Base
32
+
33
+ # Returns a random connection from the collection.
34
+ #
35
+ # @return [Connections::Connection]
36
+ #
37
+ def select(options={})
38
+ connections.to_a.send( defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' ? :sample : :choice)
39
+ end
40
+ end
41
+
42
+ # "Round-robin" selector strategy (default).
43
+ #
44
+ class RoundRobin
45
+ include Base
46
+
47
+ # Returns the next connection from the collection, rotating them in round-robin fashion.
48
+ #
49
+ # @return [Connections::Connection]
50
+ #
51
+ def select(options={})
52
+ # On Ruby 1.9, Array#rotate could be used instead
53
+ @current = @current.nil? ? 0 : @current+1
54
+ @current = 0 if @current >= connections.size
55
+ connections[@current]
56
+ end
57
+ end
58
+
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,73 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+
5
+ # Generic client error
6
+ #
7
+ class Error < StandardError; end
8
+
9
+ # Reloading connections timeout (1 sec by default)
10
+ #
11
+ class SnifferTimeoutError < Timeout::Error; end
12
+
13
+ # Elasticsearch server error (HTTP status 5xx)
14
+ #
15
+ class ServerError < StandardError; end
16
+
17
+ module Errors; end
18
+
19
+ HTTP_STATUSES = {
20
+ 300 => 'MultipleChoices',
21
+ 301 => 'MovedPermanently',
22
+ 302 => 'Found',
23
+ 303 => 'SeeOther',
24
+ 304 => 'NotModified',
25
+ 305 => 'UseProxy',
26
+ 307 => 'TemporaryRedirect',
27
+ 308 => 'PermanentRedirect',
28
+
29
+ 400 => 'BadRequest',
30
+ 401 => 'Unauthorized',
31
+ 402 => 'PaymentRequired',
32
+ 403 => 'Forbidden',
33
+ 404 => 'NotFound',
34
+ 405 => 'MethodNotAllowed',
35
+ 406 => 'NotAcceptable',
36
+ 407 => 'ProxyAuthenticationRequired',
37
+ 408 => 'RequestTimeout',
38
+ 409 => 'Conflict',
39
+ 410 => 'Gone',
40
+ 411 => 'LengthRequired',
41
+ 412 => 'PreconditionFailed',
42
+ 413 => 'RequestEntityTooLarge',
43
+ 414 => 'RequestURITooLong',
44
+ 415 => 'UnsupportedMediaType',
45
+ 416 => 'RequestedRangeNotSatisfiable',
46
+ 417 => 'ExpectationFailed',
47
+ 418 => 'ImATeapot',
48
+ 421 => 'TooManyConnectionsFromThisIP',
49
+ 426 => 'UpgradeRequired',
50
+ 450 => 'BlockedByWindowsParentalControls',
51
+ 494 => 'RequestHeaderTooLarge',
52
+ 497 => 'HTTPToHTTPS',
53
+ 499 => 'ClientClosedRequest',
54
+
55
+ 500 => 'InternalServerError',
56
+ 501 => 'NotImplemented',
57
+ 502 => 'BadGateway',
58
+ 503 => 'ServiceUnavailable',
59
+ 504 => 'GatewayTimeout',
60
+ 505 => 'HTTPVersionNotSupported',
61
+ 506 => 'VariantAlsoNegotiates',
62
+ 510 => 'NotExtended'
63
+ }
64
+
65
+ ERRORS = HTTP_STATUSES.inject({}) do |sum, error|
66
+ status, name = error
67
+ sum[status] = Errors.const_set name, Class.new(ServerError)
68
+ sum
69
+ end
70
+
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,70 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module HTTP
5
+
6
+ # Alternative HTTP transport implementation, using the [_Curb_](https://rubygems.org/gems/curb) client.
7
+ #
8
+ # @see Transport::Base
9
+ #
10
+ class Curb
11
+ include Base
12
+
13
+ # Performs the request by invoking {Transport::Base#perform_request} with a block.
14
+ #
15
+ # @return [Response]
16
+ # @see Transport::Base#perform_request
17
+ #
18
+ def perform_request(method, path, params={}, body=nil)
19
+ super do |connection,url|
20
+ connection.connection.url = url
21
+
22
+ case method
23
+ when 'HEAD', 'GET' then connection.connection.http method.downcase.to_sym
24
+ when 'PUT' then connection.connection.http_put serializer.dump(body)
25
+ when 'DELETE' then connection.connection.http_delete
26
+ when 'POST'
27
+ connection.connection.post_body = __convert_to_json(body) if body
28
+ connection.connection.http_post
29
+ else raise ArgumentError, "Unsupported HTTP method: #{method}"
30
+ end
31
+
32
+ Response.new connection.connection.response_code, connection.connection.body_str
33
+ end
34
+ end
35
+
36
+ # Builds and returns a collection of connections.
37
+ #
38
+ # @return [Connections::Collection]
39
+ #
40
+ def __build_connections
41
+ Connections::Collection.new \
42
+ :connections => hosts.map { |host|
43
+ host[:protocol] ||= DEFAULT_PROTOCOL
44
+ host[:port] ||= DEFAULT_PORT
45
+
46
+ client = ::Curl::Easy.new
47
+ client.resolve_mode = :ipv4
48
+ client.headers = {'Content-Type' => 'application/json'}
49
+ client.url = "#{host[:protocol]}://#{host[:host]}:#{host[:port]}"
50
+
51
+ client.instance_eval &@block if @block
52
+
53
+ Connections::Connection.new :host => host, :connection => client
54
+ },
55
+ :selector => options[:selector]
56
+ end
57
+
58
+ # Returns an array of implementation specific connection errors.
59
+ #
60
+ # @return [Array]
61
+ #
62
+ def host_unreachable_exceptions
63
+ [::Curl::Err::HostResolutionError, ::Curl::Err::ConnectionFailedError]
64
+ end
65
+ end
66
+
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,59 @@
1
+ module Elasticsearch
2
+ module Transport
3
+ module Transport
4
+ module HTTP
5
+
6
+ # The default transport implementation, using the [_Faraday_](https://rubygems.org/gems/faraday)
7
+ # library for abstracting the HTTP client.
8
+ #
9
+ # @see Transport::Base
10
+ #
11
+ class Faraday
12
+ include Base
13
+
14
+ # Performs the request by invoking {Transport::Base#perform_request} with a block.
15
+ #
16
+ # @return [Response]
17
+ # @see Transport::Base#perform_request
18
+ #
19
+ def perform_request(method, path, params={}, body=nil)
20
+ super do |connection, url|
21
+ connection.connection.run_request \
22
+ method.downcase.to_sym,
23
+ url,
24
+ ( body ? __convert_to_json(body) : nil ),
25
+ {'Content-Type' => 'application/json'}
26
+ end
27
+ end
28
+
29
+ # Builds and returns a collection of connections.
30
+ #
31
+ # @return [Connections::Collection]
32
+ #
33
+ def __build_connections
34
+ Connections::Collection.new \
35
+ :connections => hosts.map { |host|
36
+ host[:protocol] ||= DEFAULT_PROTOCOL
37
+ host[:port] ||= DEFAULT_PORT
38
+ url = "#{host[:protocol]}://#{host[:host]}:#{host[:port]}"
39
+
40
+ Connections::Connection.new \
41
+ :host => host,
42
+ :connection => ::Faraday::Connection.new( :url => url, &@block )
43
+ },
44
+ :selector_class => options[:selector_class],
45
+ :selector => options[:selector]
46
+ end
47
+
48
+ # Returns an array of implementation specific connection errors.
49
+ #
50
+ # @return [Array]
51
+ #
52
+ def host_unreachable_exceptions
53
+ [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end