elastomer-client 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.ruby-version +1 -0
  4. data/CHANGELOG.md +4 -0
  5. data/Gemfile +5 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +108 -0
  8. data/Rakefile +9 -0
  9. data/docs/notifications.md +71 -0
  10. data/elastomer-client.gemspec +30 -0
  11. data/lib/elastomer/client.rb +307 -0
  12. data/lib/elastomer/client/bulk.rb +257 -0
  13. data/lib/elastomer/client/cluster.rb +208 -0
  14. data/lib/elastomer/client/docs.rb +432 -0
  15. data/lib/elastomer/client/errors.rb +51 -0
  16. data/lib/elastomer/client/index.rb +407 -0
  17. data/lib/elastomer/client/multi_search.rb +115 -0
  18. data/lib/elastomer/client/nodes.rb +87 -0
  19. data/lib/elastomer/client/scan.rb +161 -0
  20. data/lib/elastomer/client/template.rb +85 -0
  21. data/lib/elastomer/client/warmer.rb +96 -0
  22. data/lib/elastomer/core_ext/time.rb +7 -0
  23. data/lib/elastomer/middleware/encode_json.rb +51 -0
  24. data/lib/elastomer/middleware/opaque_id.rb +69 -0
  25. data/lib/elastomer/middleware/parse_json.rb +39 -0
  26. data/lib/elastomer/notifications.rb +83 -0
  27. data/lib/elastomer/version.rb +7 -0
  28. data/script/bootstrap +16 -0
  29. data/script/cibuild +28 -0
  30. data/script/console +9 -0
  31. data/script/testsuite +10 -0
  32. data/test/assertions.rb +74 -0
  33. data/test/client/bulk_test.rb +226 -0
  34. data/test/client/cluster_test.rb +113 -0
  35. data/test/client/docs_test.rb +394 -0
  36. data/test/client/index_test.rb +244 -0
  37. data/test/client/multi_search_test.rb +129 -0
  38. data/test/client/nodes_test.rb +35 -0
  39. data/test/client/scan_test.rb +84 -0
  40. data/test/client/stubbed_client_tests.rb +40 -0
  41. data/test/client/template_test.rb +33 -0
  42. data/test/client/warmer_test.rb +56 -0
  43. data/test/client_test.rb +86 -0
  44. data/test/core_ext/time_test.rb +46 -0
  45. data/test/middleware/encode_json_test.rb +53 -0
  46. data/test/middleware/opaque_id_test.rb +39 -0
  47. data/test/middleware/parse_json_test.rb +54 -0
  48. data/test/test_helper.rb +94 -0
  49. metadata +210 -0
@@ -0,0 +1,69 @@
1
+ require 'securerandom'
2
+
3
+ module Elastomer
4
+ module Middleware
5
+
6
+ # This Faraday middleware implements the "X-Opaque-Id" request / response
7
+ # headers for ElasticSearch. The X-Opaque-Id header, when provided on the
8
+ # request header, will be returned as a header in the response. This is
9
+ # useful in environments which reuse connections to ensure that cross-talk
10
+ # does not occur between two requests.
11
+ #
12
+ # The SecureRandom lib is used to generate a UUID string for each request.
13
+ # This value is used as the content for the "X-Opaque-Id" header. If the
14
+ # value is different between the request and the response, then an
15
+ # `Elastomer::Client::OpaqueIdError` is raised. In this case no response
16
+ # will be returned.
17
+ #
18
+ # See [ElasticSearch "X-Opaque-Id"
19
+ # header](https://github.com/elasticsearch/elasticsearch/issues/1202)
20
+ # for more details.
21
+ class OpaqueId < ::Faraday::Middleware
22
+ X_OPAQUE_ID = 'X-Opaque-Id'.freeze
23
+ COUNTER_MAX = 2**32 - 1
24
+
25
+ # Faraday middleware implementation.
26
+ #
27
+ # env - Faraday environment Hash
28
+ #
29
+ # Returns the environment Hash
30
+ def call( env )
31
+ uuid = generate_uuid.freeze
32
+ env[:request_headers][X_OPAQUE_ID] = uuid
33
+
34
+ @app.call(env).on_complete do |renv|
35
+ if uuid != renv[:response_headers][X_OPAQUE_ID]
36
+ raise ::Elastomer::Client::OpaqueIdError, "conflicting 'X-Opaque-Id' headers"
37
+ end
38
+ end
39
+ end
40
+
41
+ # Generate a UUID using the built-in SecureRandom class. This can be a
42
+ # little slow at times, so we will reuse the same UUID and append an
43
+ # incrementing counter.
44
+ #
45
+ # Returns the UUID string.
46
+ def generate_uuid
47
+ t = Thread.current
48
+
49
+ unless t.key? :opaque_id_base
50
+ t[:opaque_id_base] = (SecureRandom.urlsafe_base64(12) + '%08x').freeze
51
+ t[:opaque_id_counter] = -1
52
+ end
53
+
54
+ t[:opaque_id_counter] += 1
55
+ t[:opaque_id_counter] = 0 if t[:opaque_id_counter] > COUNTER_MAX
56
+ t[:opaque_id_base] % t[:opaque_id_counter]
57
+ end
58
+
59
+ end # OpaqueId
60
+ end # Middleware
61
+
62
+ # Error raised when a conflict is detected between the UUID sent in the
63
+ # 'X-Opaque-Id' request header and the one received in the response header.
64
+ Client::OpaqueIdError = Class.new Client::Error
65
+
66
+ end # Elastomer
67
+
68
+ Faraday::Request.register_middleware \
69
+ :opaque_id => ::Elastomer::Middleware::OpaqueId
@@ -0,0 +1,39 @@
1
+ module Elastomer
2
+ module Middleware
3
+
4
+ # Parse response bodies as JSON.
5
+ class ParseJson < Faraday::Middleware
6
+ CONTENT_TYPE = 'Content-Type'.freeze
7
+ MIME_TYPE = 'application/json'.freeze
8
+
9
+ def call(environment)
10
+ @app.call(environment).on_complete do |env|
11
+ if process_response?(env)
12
+ env[:body] = parse env[:body]
13
+ end
14
+ end
15
+ end
16
+
17
+ # Parse the response body.
18
+ def parse(body)
19
+ MultiJson.load(body) if body.respond_to?(:to_str) and !body.strip.empty?
20
+ rescue StandardError, SyntaxError => e
21
+ raise Faraday::Error::ParsingError, e
22
+ end
23
+
24
+ def process_response?(env)
25
+ type = response_type(env)
26
+ type.empty? or type == MIME_TYPE
27
+ end
28
+
29
+ def response_type(env)
30
+ type = env[:response_headers][CONTENT_TYPE].to_s
31
+ type = type.split(';', 2).first if type.index(';')
32
+ type
33
+ end
34
+ end
35
+ end
36
+ end
37
+
38
+ Faraday::Response.register_middleware \
39
+ :parse_json => ::Elastomer::Middleware::ParseJson
@@ -0,0 +1,83 @@
1
+ require 'active_support/notifications'
2
+ require 'securerandom'
3
+ require 'elastomer/client'
4
+
5
+ module Elastomer
6
+
7
+ # So you want to get notifications from your ElasticSearch client? Well,
8
+ # you've come to the right place!
9
+ #
10
+ # require 'elastomer/notifications'
11
+ #
12
+ # Requiring this module will add ActiveSupport notifications to all
13
+ # ElasticSearch requests. To subscribe to those requests ...
14
+ #
15
+ # ActiveSupport::Notifications.subscribe('request.client.elastomer') do |name, start_time, end_time, _, payload|
16
+ # duration = end_time - start_time
17
+ # $stderr.puts '[%s] %s %s (%.3f)' % [payload[:status], payload[:index], payload[:action], duration]
18
+ # end
19
+ #
20
+ # The payload contains the following bits of information:
21
+ #
22
+ # * :index - index name (if any)
23
+ # * :type - documeht type (if any)
24
+ # * :action - the action being performed
25
+ # * :url - request URL
26
+ # * :method - request method (:head, :get, :put, :post, :delete)
27
+ # * :status - response status code
28
+ #
29
+ # If you want to use your own notifications service then you will need to
30
+ # let Elastomer know by setting the `service` here in the Notifications
31
+ # module. The service should adhere to the ActiveSupport::Notifications
32
+ # specification.
33
+ #
34
+ # Elastomer::Notifications.service = your_own_service
35
+ #
36
+ module Notifications
37
+
38
+ class << self
39
+ attr_accessor :service
40
+ end
41
+
42
+ # The name to subscribe to for notifications
43
+ NAME = 'request.client.elastomer'.freeze
44
+
45
+ # Internal: Execute the given block and provide instrumentaiton info to
46
+ # subscribers. The name we use for subscriptions is
47
+ # `request.client.elastomer` and a supplemental payload is provided with
48
+ # more information about the specific ElasticSearch request.
49
+ #
50
+ # path - The full request path as a String
51
+ # body - The request body as a String or `nil`
52
+ # params - The request params Hash
53
+ # block - The block that will be instrumented
54
+ #
55
+ # Returns the response from the block
56
+ def instrument( path, body, params )
57
+ payload = {
58
+ :index => params[:index],
59
+ :type => params[:type],
60
+ :action => params[:action],
61
+ :context => params[:context],
62
+ :body => body
63
+ }
64
+
65
+ ::Elastomer::Notifications.service.instrument(NAME, payload) do
66
+ response = yield
67
+ payload[:url] = response.env[:url]
68
+ payload[:method] = response.env[:method]
69
+ payload[:status] = response.status
70
+ response
71
+ end
72
+ end
73
+ end
74
+
75
+ # use ActiveSupport::Notifications as the default instrumentaiton service
76
+ Notifications.service = ActiveSupport::Notifications
77
+
78
+ # inject our instrument method into the Client class
79
+ class Client
80
+ remove_method :instrument
81
+ include ::Elastomer::Notifications
82
+ end
83
+ end
@@ -0,0 +1,7 @@
1
+ module Elastomer
2
+ VERSION = '0.3.1'
3
+
4
+ def self.version
5
+ VERSION
6
+ end
7
+ end
@@ -0,0 +1,16 @@
1
+ #!/bin/sh
2
+ set -e
3
+
4
+ # force use of gcc under mac due to hiredis issues under clang
5
+ if [ $(uname) = Darwin ]; then
6
+ export CC=/usr/bin/gcc
7
+ export CXX=/usr/bin/g++
8
+ export LD=/usr/bin/gcc
9
+ fi
10
+
11
+ cd "$(dirname "$0")/.."
12
+ if bundle check 1>/dev/null 2>&1; then
13
+ echo "Gem environment up-to-date"
14
+ else
15
+ exec bundle install --binstubs --path vendor/gems "$@"
16
+ fi
@@ -0,0 +1,28 @@
1
+ #!/bin/sh
2
+ # Usage: script/cibuild
3
+ # CI build script.
4
+ # This is tailored for the janky build machines.
5
+ set -e
6
+
7
+ # change into root dir and setup path
8
+ cd $(dirname "$0")/..
9
+ PATH="$(pwd)/bin:$(pwd)/script:/usr/share/rbenv/shims:$PATH"
10
+
11
+ # Write commit we're building at
12
+ git log -n 1 || true
13
+ echo
14
+
15
+ result=0
16
+
17
+ export RBENV_VERSION="$(cat .ruby-version)"
18
+ echo "Building under $RBENV_VERSION"
19
+ script/bootstrap
20
+ script/testsuite || result=$?
21
+
22
+ if [ $result -ne 0 ]; then
23
+ exit $result
24
+ fi
25
+
26
+ # echo
27
+ # echo "Running benchmarks"
28
+ # script/benchmark
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env ruby
2
+ require 'irb'
3
+ require 'rubygems'
4
+ require 'bundler/setup'
5
+
6
+ $LOAD_PATH.unshift 'lib'
7
+ require 'elastomer/client'
8
+
9
+ IRB.start
@@ -0,0 +1,10 @@
1
+ #!/bin/sh
2
+ set -e
3
+ cd "$(dirname "$0")/.."
4
+
5
+ # run entire test suite
6
+ ruby --version 1>&2
7
+ exec ruby -I. -rubygems \
8
+ -r "$(pwd)/test/test_helper" \
9
+ -e "ARGV.each { |f| require(f) }" \
10
+ -- $(find test -name '*_test.rb')
@@ -0,0 +1,74 @@
1
+ module Minitest::Assertions
2
+ #COMPATIBILITY
3
+ # ES 1.0 replaced the 'ok' attribute with a 'created' attribute
4
+ # in index responses. Check for either one so we are compatible
5
+ # with 0.90 and 1.0.
6
+ def assert_created(response)
7
+ assert response['created'] || response['ok'], 'document was not created'
8
+ end
9
+
10
+ #COMPATIBILITY
11
+ # ES 1.0 replaced the 'ok' attribute with an 'acknowledged' attribute
12
+ # in many responses. Check for either one so we are compatible
13
+ # with 0.90 and 1.0.
14
+ def assert_acknowledged(response)
15
+ assert response['acknowledged'] || response['ok'], 'document was not acknowledged'
16
+ end
17
+
18
+ #COMPATIBILITY
19
+ # ES 1.0 replaced the 'exists' attribute with a 'found' attribute in the
20
+ # get document response. Check for either one so we are compatible
21
+ # with 0.90 and 1.0.
22
+ def assert_found(response)
23
+ assert response['found'] || response['exists'], 'document was not found'
24
+ end
25
+
26
+ def refute_found(response)
27
+ refute response['found'] || response['exists'], 'document was unexpectedly found'
28
+ end
29
+
30
+ #COMPATIBILITY
31
+ # ES 1.0 replaced the 'ok' attribute in the bulk response item with a
32
+ # 'status' attribute. Here we check for either one for compatibility
33
+ # with 0.90 and 1.0.
34
+ def assert_bulk_index(item, message='bulk index did not succeed')
35
+ ok = item['index']['ok']
36
+ status = item['index']['status']
37
+ assert ok == true || status == 201, message
38
+ end
39
+
40
+ def assert_bulk_create(item, message='bulk create did not succeed')
41
+ ok = item['create']['ok']
42
+ status = item['create']['status']
43
+ assert ok == true || status == 201, message
44
+ end
45
+
46
+ def assert_bulk_delete(item, message='bulk delete did not succeed')
47
+ ok = item['delete']['ok']
48
+ status = item['delete']['status']
49
+ assert ok == true || status == 200, message
50
+ end
51
+
52
+ #COMPATIBILITY
53
+ # ES 1.0 nests mappings in a "mappings" element under the index name, e.g.
54
+ # mapping["test-index"]["mappings"]["doco"]
55
+ # ES 0.90 doesn't have the "mappings" element:
56
+ # mapping["test-index"]["doco"]
57
+ def assert_mapping_exists(response, type, message="mapping expected to exist, but doesn't")
58
+ mapping = if response.has_key?('mappings')
59
+ response['mappings'][type]
60
+ else
61
+ response[type]
62
+ end
63
+ refute_nil mapping, message
64
+ end
65
+
66
+ def assert_property_exists(response, type, property, message="property expected to exist, but doesn't")
67
+ mapping = if response.has_key?('mappings')
68
+ response['mappings'][type]
69
+ else
70
+ response[type]
71
+ end
72
+ assert mapping['properties'].has_key?(property), message
73
+ end
74
+ end
@@ -0,0 +1,226 @@
1
+ require File.expand_path('../../test_helper', __FILE__)
2
+
3
+ describe Elastomer::Client::Bulk do
4
+
5
+ before do
6
+ @name = 'elastomer-bulk-test'
7
+ @index = $client.index(@name)
8
+
9
+ unless @index.exists?
10
+ @index.create \
11
+ :settings => { 'index.number_of_shards' => 1, 'index.number_of_replicas' => 0 },
12
+ :mappings => {
13
+ :tweet => {
14
+ :_source => { :enabled => true }, :_all => { :enabled => false },
15
+ :properties => {
16
+ :message => { :type => 'string', :analyzer => 'standard' },
17
+ :author => { :type => 'string', :index => 'not_analyzed' }
18
+ }
19
+ },
20
+ :book => {
21
+ :_source => { :enabled => true }, :_all => { :enabled => false },
22
+ :properties => {
23
+ :title => { :type => 'string', :analyzer => 'standard' },
24
+ :author => { :type => 'string', :index => 'not_analyzed' }
25
+ }
26
+ }
27
+ }
28
+
29
+ wait_for_index(@name)
30
+ end
31
+ end
32
+
33
+ after do
34
+ @index.delete if @index.exists?
35
+ end
36
+
37
+ it 'performs bulk index actions' do
38
+ body = [
39
+ '{"index" : {"_id":"1", "_type":"tweet", "_index":"elastomer-bulk-test"}}',
40
+ '{"author":"pea53", "message":"just a test tweet"}',
41
+ '{"index" : {"_id":"1", "_type":"book", "_index":"elastomer-bulk-test"}}',
42
+ '{"author":"John Scalzi", "title":"Old Mans War"}',
43
+ nil
44
+ ]
45
+ body = body.join "\n"
46
+ h = $client.bulk body
47
+
48
+ assert_bulk_index(h['items'][0])
49
+ assert_bulk_index(h['items'][1])
50
+
51
+ @index.refresh
52
+
53
+ h = @index.docs('tweet').get :id => 1
54
+ assert_equal 'pea53', h['_source']['author']
55
+
56
+ h = @index.docs('book').get :id => 1
57
+ assert_equal 'John Scalzi', h['_source']['author']
58
+
59
+
60
+ body = [
61
+ '{"index" : {"_id":"2", "_type":"book"}}',
62
+ '{"author":"Tolkien", "title":"The Silmarillion"}',
63
+ '{"delete" : {"_id":"1", "_type":"book"}}',
64
+ nil
65
+ ]
66
+ body = body.join "\n"
67
+ h = $client.bulk body, :index => @name
68
+
69
+ assert_bulk_index h['items'].first, 'expected to index a book'
70
+ assert_bulk_delete h['items'].last, 'expected to delete a book'
71
+
72
+ @index.refresh
73
+
74
+ h = @index.docs('book').get :id => 1
75
+ assert !h['exists'], 'was not successfully deleted'
76
+
77
+ h = @index.docs('book').get :id => 2
78
+ assert_equal 'Tolkien', h['_source']['author']
79
+ end
80
+
81
+ it 'supports a nice block syntax' do
82
+ h = @index.bulk do |b|
83
+ b.index :_id => 1, :_type => 'tweet', :author => 'pea53', :message => 'just a test tweet'
84
+ b.index :_id => nil, :_type => 'book', :author => 'John Scalzi', :title => 'Old Mans War'
85
+ end
86
+ items = h['items']
87
+
88
+ assert_instance_of Fixnum, h['took']
89
+
90
+ assert_bulk_index h['items'].first
91
+ assert_bulk_create h['items'].last
92
+
93
+ book_id = items.last['create']['_id']
94
+ assert_match %r/^\S{22}$/, book_id
95
+
96
+ @index.refresh
97
+
98
+ h = @index.docs('tweet').get :id => 1
99
+ assert_equal 'pea53', h['_source']['author']
100
+
101
+ h = @index.docs('book').get :id => book_id
102
+ assert_equal 'John Scalzi', h['_source']['author']
103
+
104
+
105
+ h = @index.bulk do |b|
106
+ b.index :_id => '', :_type => 'book', :author => 'Tolkien', :title => 'The Silmarillion'
107
+ b.delete :_id => book_id, :_type => 'book'
108
+ end
109
+ items = h['items']
110
+
111
+ assert_bulk_create h['items'].first, 'expected to create a book'
112
+ assert_bulk_delete h['items'].last, 'expected to delete a book'
113
+
114
+ book_id2 = items.first['create']['_id']
115
+ assert_match %r/^\S{22}$/, book_id2
116
+
117
+ @index.refresh
118
+
119
+ h = @index.docs('book').get :id => book_id
120
+ assert !h['exists'], 'was not successfully deleted'
121
+
122
+ h = @index.docs('book').get :id => book_id2
123
+ assert_equal 'Tolkien', h['_source']['author']
124
+ end
125
+
126
+ it 'allows documents to be JSON strings' do
127
+ h = @index.bulk do |b|
128
+ b.index '{"author":"pea53", "message":"just a test tweet"}', :_id => 1, :_type => 'tweet'
129
+ b.create '{"author":"John Scalzi", "title":"Old Mans War"}', :_id => 1, :_type => 'book'
130
+ end
131
+ items = h['items']
132
+
133
+ assert_instance_of Fixnum, h['took']
134
+
135
+ assert_bulk_index h['items'].first
136
+ assert_bulk_create h['items'].last
137
+
138
+ @index.refresh
139
+
140
+ h = @index.docs('tweet').get :id => 1
141
+ assert_equal 'pea53', h['_source']['author']
142
+
143
+ h = @index.docs('book').get :id => 1
144
+ assert_equal 'John Scalzi', h['_source']['author']
145
+
146
+
147
+ h = @index.bulk do |b|
148
+ b.index '{"author":"Tolkien", "title":"The Silmarillion"}', :_id => 2, :_type => 'book'
149
+ b.delete :_id => 1, :_type => 'book'
150
+ end
151
+ items = h['items']
152
+
153
+ assert_bulk_index h['items'].first, 'expected to index a book'
154
+ assert_bulk_delete h['items'].last, 'expected to delete a book'
155
+
156
+ @index.refresh
157
+
158
+ h = @index.docs('book').get :id => 1
159
+ assert !h['exists'], 'was not successfully deleted'
160
+
161
+ h = @index.docs('book').get :id => 2
162
+ assert_equal 'Tolkien', h['_source']['author']
163
+ end
164
+
165
+ it 'executes a bulk API call when a request size is reached' do
166
+ ary = []
167
+ ary << @index.bulk(:request_size => 300) do |b|
168
+ 2.times { |num|
169
+ document = {:_id => num, :_type => 'tweet', :author => 'pea53', :message => "tweet #{num} is a 100 character request"}
170
+ ary << b.index(document)
171
+ }
172
+ ary.compact!
173
+ assert_equal 0, ary.length
174
+
175
+ 7.times { |num|
176
+ document = {:_id => num+2, :_type => 'tweet', :author => 'pea53', :message => "tweet #{num+2} is a 100 character request"}
177
+ ary << b.index(document)
178
+ }
179
+ ary.compact!
180
+ assert_equal 3, ary.length
181
+
182
+ document = {:_id => 10, :_type => 'tweet', :author => 'pea53', :message => "tweet 10 is a 102 character request"}
183
+ ary << b.index(document)
184
+ end
185
+ ary.compact!
186
+
187
+ assert_equal 4, ary.length
188
+ ary.each { |a| a['items'].each { |b| assert_bulk_index(b) } }
189
+
190
+ @index.refresh
191
+ h = @index.docs.search :q => '*:*', :search_type => 'count'
192
+
193
+ assert_equal 10, h['hits']['total']
194
+ end
195
+
196
+ it 'executes a bulk API call when an action count is reached' do
197
+ ary = []
198
+ ary << @index.bulk(:action_count => 3) do |b|
199
+ 2.times { |num|
200
+ document = {:_id => num, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number #{num}"}
201
+ ary << b.index(document)
202
+ }
203
+ ary.compact!
204
+ assert_equal 0, ary.length
205
+
206
+ 7.times { |num|
207
+ document = {:_id => num+2, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number #{num+2}"}
208
+ ary << b.index(document)
209
+ }
210
+ ary.compact!
211
+ assert_equal 3, ary.length
212
+
213
+ document = {:_id => 10, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number 10"}
214
+ ary << b.index(document)
215
+ end
216
+ ary.compact!
217
+
218
+ assert_equal 4, ary.length
219
+ ary.each { |a| a['items'].each { |b| assert_bulk_index(b) } }
220
+
221
+ @index.refresh
222
+ h = @index.docs.search :q => '*:*', :search_type => 'count'
223
+
224
+ assert_equal 10, h['hits']['total']
225
+ end
226
+ end