elastomer-client 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG.md +15 -0
  5. data/README.md +6 -7
  6. data/Rakefile +21 -0
  7. data/docs/README.md +44 -0
  8. data/docs/bulk_indexing.md +3 -0
  9. data/docs/client.md +240 -0
  10. data/docs/cluster.md +148 -0
  11. data/docs/docs.md +254 -0
  12. data/docs/index.md +161 -0
  13. data/docs/multi_search.md +3 -0
  14. data/docs/notifications.md +24 -11
  15. data/docs/scan_scroll.md +3 -0
  16. data/docs/snapshots.md +3 -0
  17. data/docs/templates.md +3 -0
  18. data/docs/warmers.md +3 -0
  19. data/elastomer-client.gemspec +2 -2
  20. data/lib/elastomer/client.rb +70 -43
  21. data/lib/elastomer/client/bulk.rb +2 -2
  22. data/lib/elastomer/client/cluster.rb +2 -2
  23. data/lib/elastomer/client/docs.rb +190 -54
  24. data/lib/elastomer/client/errors.rb +4 -2
  25. data/lib/elastomer/client/index.rb +111 -43
  26. data/lib/elastomer/client/multi_search.rb +1 -1
  27. data/lib/elastomer/client/nodes.rb +9 -4
  28. data/lib/elastomer/client/repository.rb +2 -2
  29. data/lib/elastomer/client/scroller.rb +235 -0
  30. data/lib/elastomer/client/snapshot.rb +1 -1
  31. data/lib/elastomer/client/template.rb +1 -1
  32. data/lib/elastomer/client/warmer.rb +1 -1
  33. data/lib/elastomer/notifications.rb +1 -1
  34. data/lib/elastomer/version.rb +1 -1
  35. data/script/bootstrap +0 -7
  36. data/script/cibuild +8 -3
  37. data/script/test +6 -0
  38. data/test/client/bulk_test.rb +2 -2
  39. data/test/client/cluster_test.rb +23 -2
  40. data/test/client/docs_test.rb +137 -6
  41. data/test/client/errors_test.rb +12 -8
  42. data/test/client/index_test.rb +88 -5
  43. data/test/client/multi_search_test.rb +29 -0
  44. data/test/client/repository_test.rb +36 -37
  45. data/test/client/{scan_test.rb → scroller_test.rb} +25 -6
  46. data/test/client/snapshot_test.rb +53 -43
  47. data/test/client/stubbed_client_test.rb +1 -1
  48. data/test/client_test.rb +60 -0
  49. data/test/notifications_test.rb +69 -0
  50. data/test/test_helper.rb +54 -11
  51. metadata +36 -23
  52. data/.ruby-version +0 -1
  53. data/lib/elastomer/client/scan.rb +0 -161
  54. data/script/testsuite +0 -10
@@ -30,7 +30,7 @@ describe 'stubbed client tests' do
30
30
  @stubs.post('/_cluster/nodes/_all/_shutdown') { [200, {'Content-Type' => 'application/json'}, '{"nodes":{"1":{"name":"Node1"}}}'] }
31
31
  @stubs.post('/_cluster/nodes/node2/_shutdown') { [200, {'Content-Type' => 'application/json'}, '{"nodes":{"2":{"name":"Node2"}}}'] }
32
32
 
33
- h = @client.nodes.shutdown
33
+ h = @client.nodes("_all").shutdown
34
34
  assert_equal "Node1", h['nodes']['1']['name']
35
35
 
36
36
  h = @client.nodes('node2').shutdown
data/test/client_test.rb CHANGED
@@ -55,6 +55,44 @@ describe Elastomer::Client do
55
55
  }
56
56
  end
57
57
 
58
+ describe 'when extracting and converting :body params' do
59
+ it 'deletes the :body from the params (or it gets the hose)' do
60
+ params = { :body => nil, :q => "what what?" }
61
+ body = $client.extract_body params
62
+
63
+ assert_nil body
64
+ assert_equal({:q => "what what?"}, params)
65
+ end
66
+
67
+ it 'leaves String values unchanged' do
68
+ body = $client.extract_body :body => '{"query":{"match_all":{}}}'
69
+ assert_equal '{"query":{"match_all":{}}}', body
70
+
71
+ body = $client.extract_body :body => 'not a JSON string, but who cares!'
72
+ assert_equal 'not a JSON string, but who cares!', body
73
+ end
74
+
75
+ it 'joins Array values' do
76
+ body = $client.extract_body :body => %w[foo bar baz]
77
+ assert_equal "foo\nbar\nbaz\n", body
78
+
79
+ body = $client.extract_body :body => [
80
+ 'the first entry',
81
+ 'the second entry',
82
+ nil
83
+ ]
84
+ assert_equal "the first entry\nthe second entry\n", body
85
+ end
86
+
87
+ it 'converts values to JSON' do
88
+ body = $client.extract_body :body => true
89
+ assert_equal "true", body
90
+
91
+ body = $client.extract_body :body => {:query => {:match_all => {}}}
92
+ assert_equal '{"query":{"match_all":{}}}', body
93
+ end
94
+ end
95
+
58
96
  describe 'when validating parameters' do
59
97
  it 'rejects nil values' do
60
98
  assert_raises(ArgumentError) { $client.assert_param_presence nil }
@@ -92,4 +130,26 @@ describe Elastomer::Client do
92
130
  assert_equal '9', $client.assert_param_presence(9)
93
131
  end
94
132
  end
133
+
134
+ describe 'top level actions' do
135
+ it 'pings the cluster' do
136
+ assert_equal true, $client.ping
137
+ assert_equal true, $client.available?
138
+ end
139
+
140
+ it 'gets cluster info' do
141
+ h = $client.info
142
+ assert h.key?('name'), 'expected cluster name to be returned'
143
+ assert h.key?('status'), 'expected cluster info status to be returned'
144
+ end
145
+
146
+ it 'gets cluster version' do
147
+ assert_match /[\d\.]+/, $client.version
148
+ end
149
+
150
+ it 'gets semantic version' do
151
+ version_string = $client.version
152
+ assert_equal Semantic::Version.new(version_string), $client.semantic_version
153
+ end
154
+ end
95
155
  end
@@ -0,0 +1,69 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+ require 'elastomer/notifications'
3
+
4
+ describe Elastomer::Notifications do
5
+ before do
6
+ @name = 'elastomer-notifications-test'
7
+ @index = $client.index @name
8
+ @index.delete if @index.exists?
9
+ @events = []
10
+ @subscriber = ActiveSupport::Notifications.subscribe do |*args|
11
+ @events << ActiveSupport::Notifications::Event.new(*args)
12
+ end
13
+ end
14
+
15
+ after do
16
+ ActiveSupport::Notifications.unsubscribe(@subscriber)
17
+ @index.delete if @index.exists?
18
+ end
19
+
20
+ it "instruments timeouts" do
21
+ $client.stub :connection, lambda { raise Faraday::Error::TimeoutError } do
22
+ assert_raises(Elastomer::Client::TimeoutError) { $client.info }
23
+ event = @events.detect { |e| e.payload[:action] == 'cluster.info' }
24
+ exception = event.payload[:exception]
25
+ assert_equal 'Elastomer::Client::TimeoutError', exception[0]
26
+ assert_match 'timeout', exception[1]
27
+ end
28
+ end
29
+
30
+ it 'instruments cluster actions' do
31
+ $client.ping; assert_action_event('cluster.ping')
32
+ $client.info; assert_action_event('cluster.info')
33
+ end
34
+
35
+ it 'instruments node actions' do
36
+ nodes = $client.nodes
37
+ nodes.info; assert_action_event('nodes.info')
38
+ nodes.stats; assert_action_event('nodes.stats')
39
+ nodes.hot_threads; assert_action_event('nodes.hot_threads')
40
+ end
41
+
42
+ it 'instruments node shutdown' do
43
+ client = stub_client(:post, '/_cluster/nodes/_shutdown')
44
+ client.nodes.shutdown; assert_action_event('nodes.shutdown')
45
+ end
46
+
47
+ it 'instruments index actions' do
48
+ @index.exists?; assert_action_event('index.exists')
49
+ @index.create('number_of_replicas' => 0)
50
+ assert_action_event('index.create')
51
+ @index.get_settings; assert_action_event('index.get_settings')
52
+ @index.update_settings('number_of_replicas' => 0)
53
+ assert_action_event('index.get_settings')
54
+ @index.close; assert_action_event('index.close')
55
+ @index.open; assert_action_event('index.open')
56
+ @index.delete; assert_action_event('index.delete')
57
+ end
58
+
59
+ def assert_action_event(action)
60
+ assert @events.detect { |e| e.payload[:action] == action }, "expected #{action} event"
61
+ end
62
+
63
+ def stub_client(method, url, status=200, body='{"acknowledged":true}')
64
+ stubs = Faraday::Adapter::Test::Stubs.new do |stub|
65
+ stub.send(method, url) { |env| [status, {}, body]}
66
+ end
67
+ Elastomer::Client.new($client_params.merge(:opaque_id => false, :adapter => [:test, stubs]))
68
+ end
69
+ end
data/test/test_helper.rb CHANGED
@@ -1,4 +1,4 @@
1
- require 'tmpdir'
1
+ require 'securerandom'
2
2
  require 'rubygems' unless defined? Gem
3
3
  require 'bundler'
4
4
  Bundler.require(:default, :development)
@@ -11,6 +11,8 @@ if ENV['COVERAGE'] == 'true'
11
11
  end
12
12
  end
13
13
 
14
+ ENV['SNAPSHOT_DIR'] ||= '/tmp/elastomer-client-snapshot-test'
15
+
14
16
  require 'minitest/spec'
15
17
  require 'minitest/autorun'
16
18
 
@@ -31,6 +33,8 @@ $client = Elastomer::Client.new $client_params
31
33
  # ensure we have an ElasticSearch server to test with
32
34
  raise "No server available at #{$client.url}" unless $client.available?
33
35
 
36
+ puts "Elasticsearch version is #{$client.version}"
37
+
34
38
  # remove any lingering test indices from the cluster
35
39
  MiniTest::Unit.after_tests do
36
40
  $client.cluster.indices.keys.each do |name|
@@ -87,6 +91,21 @@ def es_version_1_x?
87
91
  $client.semantic_version >= '1.0.0'
88
92
  end
89
93
 
94
+ # Elasticsearch 1.4 changed the response body for interacting with index
95
+ # alaises. If an index does not contain any aliases, then an "alises" key is no
96
+ # longer returned in the resopsne.
97
+ #
98
+ # Reeturns `true` if the response contains an "alises" key.
99
+ def es_version_always_returns_aliases?
100
+ $client.semantic_version <= '1.4.0' ||
101
+ $client.semantic_version >= '1.4.3'
102
+ end
103
+
104
+ # ElasticSearch 1.3 added the `search_shards` API endpoint.
105
+ def es_version_supports_search_shards?
106
+ $client.semantic_version >= '1.3.0'
107
+ end
108
+
90
109
  # Elasticsearch 1.2 removed support for gateway snapshots.
91
110
  #
92
111
  # Returns true if Elasticsearch version supports gateway snapshots.
@@ -94,19 +113,43 @@ def es_version_supports_gateway_snapshots?
94
113
  $client.semantic_version <= '1.2.0'
95
114
  end
96
115
 
116
+ def create_repo(name, settings = {})
117
+ default_settings = {:type => 'fs', :settings => {:location => ENV['SNAPSHOT_DIR']}}
118
+ $client.repository(name).create(default_settings.merge(settings))
119
+ end
97
120
 
98
- def with_tmp_repo(&block)
99
- Dir.mktmpdir do |dir|
100
- @repo.create({:type => 'fs', :settings => {:location => dir}})
101
- yield @repo
102
- @repo.delete if @repo.exists?
121
+ def delete_repo(name)
122
+ repo = $client.repository(name)
123
+ repo.delete if repo.exists?
124
+ end
125
+
126
+ def delete_repo_snapshots(name)
127
+ repo = $client.repository(name)
128
+ if repo.exists?
129
+ response = repo.snapshots.get
130
+ response["snapshots"].each do |snapshot_info|
131
+ repo.snapshot(snapshot_info["snapshot"]).delete
132
+ end
103
133
  end
104
134
  end
105
135
 
106
- def with_tmp_snapshot(&block)
107
- with_tmp_repo do
108
- @snapshot.create
109
- yield @snapshot
110
- @snapshot.delete if @snapshot.exists?
136
+ def with_tmp_repo(name = SecureRandom.uuid, &block)
137
+ begin
138
+ create_repo(name)
139
+ yield $client.repository(name)
140
+ ensure
141
+ delete_repo_snapshots(name)
142
+ delete_repo(name)
143
+ end
144
+ end
145
+
146
+ def create_snapshot(repo, name = SecureRandom.uuid)
147
+ repo.snapshot(name).create({}, :wait_for_completion => true)
148
+ end
149
+
150
+ def with_tmp_snapshot(name = SecureRandom.uuid, &block)
151
+ with_tmp_repo do |repo|
152
+ create_snapshot(repo, name)
153
+ yield repo.snapshot(name), repo
111
154
  end
112
155
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elastomer-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tim Pease
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-10-14 00:00:00.000000000 Z
12
+ date: 2015-01-21 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: addressable
@@ -81,6 +81,20 @@ dependencies:
81
81
  - - "~>"
82
82
  - !ruby/object:Gem::Version
83
83
  version: '1.5'
84
+ - !ruby/object:Gem::Dependency
85
+ name: activesupport
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '3.0'
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">="
96
+ - !ruby/object:Gem::Version
97
+ version: '3.0'
84
98
  - !ruby/object:Gem::Dependency
85
99
  name: minitest
86
100
  requirement: !ruby/object:Gem::Requirement
@@ -109,20 +123,6 @@ dependencies:
109
123
  - - ">="
110
124
  - !ruby/object:Gem::Version
111
125
  version: '0'
112
- - !ruby/object:Gem::Dependency
113
- name: debugger
114
- requirement: !ruby/object:Gem::Requirement
115
- requirements:
116
- - - "~>"
117
- - !ruby/object:Gem::Version
118
- version: 1.6.8
119
- type: :development
120
- prerelease: false
121
- version_requirements: !ruby/object:Gem::Requirement
122
- requirements:
123
- - - "~>"
124
- - !ruby/object:Gem::Version
125
- version: 1.6.8
126
126
  description: |-
127
127
  Elastomer is a low level API client for the
128
128
  Elasticsearch HTTP interface.
@@ -134,13 +134,24 @@ extensions: []
134
134
  extra_rdoc_files: []
135
135
  files:
136
136
  - ".gitignore"
137
- - ".ruby-version"
137
+ - ".travis.yml"
138
138
  - CHANGELOG.md
139
139
  - Gemfile
140
140
  - LICENSE.txt
141
141
  - README.md
142
142
  - Rakefile
143
+ - docs/README.md
144
+ - docs/bulk_indexing.md
145
+ - docs/client.md
146
+ - docs/cluster.md
147
+ - docs/docs.md
148
+ - docs/index.md
149
+ - docs/multi_search.md
143
150
  - docs/notifications.md
151
+ - docs/scan_scroll.md
152
+ - docs/snapshots.md
153
+ - docs/templates.md
154
+ - docs/warmers.md
144
155
  - elastomer-client.gemspec
145
156
  - lib/elastomer/client.rb
146
157
  - lib/elastomer/client/bulk.rb
@@ -151,7 +162,7 @@ files:
151
162
  - lib/elastomer/client/multi_search.rb
152
163
  - lib/elastomer/client/nodes.rb
153
164
  - lib/elastomer/client/repository.rb
154
- - lib/elastomer/client/scan.rb
165
+ - lib/elastomer/client/scroller.rb
155
166
  - lib/elastomer/client/snapshot.rb
156
167
  - lib/elastomer/client/template.rb
157
168
  - lib/elastomer/client/warmer.rb
@@ -164,7 +175,7 @@ files:
164
175
  - script/bootstrap
165
176
  - script/cibuild
166
177
  - script/console
167
- - script/testsuite
178
+ - script/test
168
179
  - test/assertions.rb
169
180
  - test/client/bulk_test.rb
170
181
  - test/client/cluster_test.rb
@@ -174,7 +185,7 @@ files:
174
185
  - test/client/multi_search_test.rb
175
186
  - test/client/nodes_test.rb
176
187
  - test/client/repository_test.rb
177
- - test/client/scan_test.rb
188
+ - test/client/scroller_test.rb
178
189
  - test/client/snapshot_test.rb
179
190
  - test/client/stubbed_client_test.rb
180
191
  - test/client/template_test.rb
@@ -184,6 +195,7 @@ files:
184
195
  - test/middleware/encode_json_test.rb
185
196
  - test/middleware/opaque_id_test.rb
186
197
  - test/middleware/parse_json_test.rb
198
+ - test/notifications_test.rb
187
199
  - test/test_helper.rb
188
200
  homepage: https://github.com/github/elastomer-client
189
201
  licenses:
@@ -205,10 +217,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
205
217
  version: '0'
206
218
  requirements: []
207
219
  rubyforge_project:
208
- rubygems_version: 2.2.0
220
+ rubygems_version: 2.2.2
209
221
  signing_key:
210
222
  specification_version: 4
211
- summary: A library for interacting with the GitHub Search infrastructure
223
+ summary: A library for interacting with Elasticsearch
212
224
  test_files:
213
225
  - test/assertions.rb
214
226
  - test/client/bulk_test.rb
@@ -219,7 +231,7 @@ test_files:
219
231
  - test/client/multi_search_test.rb
220
232
  - test/client/nodes_test.rb
221
233
  - test/client/repository_test.rb
222
- - test/client/scan_test.rb
234
+ - test/client/scroller_test.rb
223
235
  - test/client/snapshot_test.rb
224
236
  - test/client/stubbed_client_test.rb
225
237
  - test/client/template_test.rb
@@ -229,4 +241,5 @@ test_files:
229
241
  - test/middleware/encode_json_test.rb
230
242
  - test/middleware/opaque_id_test.rb
231
243
  - test/middleware/parse_json_test.rb
244
+ - test/notifications_test.rb
232
245
  - test/test_helper.rb
data/.ruby-version DELETED
@@ -1 +0,0 @@
1
- 2.1.0-github
@@ -1,161 +0,0 @@
1
- module Elastomer
2
- class Client
3
-
4
- # Create a new Scan instance for scrolling all results from a `query`.
5
- #
6
- # query - The query to scan as a Hash or a JSON encoded String
7
- # opts - Options Hash
8
- # :index - the name of the index to search
9
- # :type - the document type to search
10
- # :scroll - the keep alive time of the scrolling request (5 minutes by default)
11
- # :size - the number of documents per shard to fetch per scroll
12
- #
13
- # Examples
14
- #
15
- # scan = client.scan('{"query":{"match_all":{}}}', :index => 'test')
16
- # scan.each_document do |document|
17
- # document['_id']
18
- # document['_source']
19
- # end
20
- #
21
- # Returns a new Scan instance
22
- def scan( query, opts = {} )
23
- Scan.new self, query, opts
24
- end
25
-
26
- # Continue scrolling a scan query.
27
- # See http://www.elasticsearch.org/guide/reference/api/search/scroll/
28
- #
29
- # scroll_id - The current scroll ID as a String
30
- # scroll - The keep alive time of the scrolling request (5 minutes by default)
31
- #
32
- # Examples
33
- #
34
- # scroll_id = client.scan('{"query":{"match_all":{}}}', :index => 'test').scroll_id
35
- #
36
- # h = client.scroll scroll_id # scroll to get the next set of results
37
- # scroll_id = h['_scroll_id'] # and store the scroll_id to use later
38
- #
39
- # h = client.scroll scroll_id # scroll again to get the next set of results
40
- # scroll_id = h['_scroll_id'] # and store the scroll_id to use later
41
- #
42
- # # repeat until the results are empty
43
- #
44
- # Returns the response body as a Hash.
45
- def scroll( scroll_id, scroll = '5m' )
46
- response = get '/_search/scroll', :body => scroll_id, :scroll => scroll, :action => 'search.scroll'
47
- response.body
48
- end
49
-
50
-
51
- class Scan
52
- # Create a new scan client that can be used to iterate over all the
53
- # documents returned by the `query`.
54
- #
55
- # See http://www.elasticsearch.org/guide/reference/api/search/scroll/
56
- # and the "Scan" section of http://www.elasticsearch.org/guide/reference/api/search/search-type/
57
- #
58
- # client - Elastomer::Client used for HTTP requests to the server
59
- # query - The query to scan as a Hash or a JSON encoded String
60
- # opts - Options Hash
61
- # :index - the name of the index to search
62
- # :type - the document type to search
63
- # :scroll - the keep alive time of the scrolling request (5 minutes by default)
64
- # :size - the number of documents per shard to fetch per scroll
65
- #
66
- # Examples
67
- #
68
- # scan = Scan.new(client, {:query => {:match_all => {}}}, :index => 'test-1')
69
- # scan.each_document { |doc|
70
- # doc['_id']
71
- # doc['_source']
72
- # }
73
- #
74
- def initialize( client, query, opts = {} )
75
- @client = client
76
- @query = query
77
-
78
- @index = opts.fetch(:index, nil)
79
- @type = opts.fetch(:type, nil)
80
- @scroll = opts.fetch(:scroll, '5m')
81
- @size = opts.fetch(:size, 50)
82
-
83
- @offset = 0
84
- end
85
-
86
- attr_reader :client, :query, :index, :type, :scroll, :size
87
-
88
- # Iterate over all the search results from the scan query.
89
- #
90
- # block - The block will be called for each set of matching documents
91
- # returned from executing the scan query.
92
- #
93
- # Yields a hits Hash containing the 'total' number of hits, current
94
- # 'offset' into that total, and the Array of 'hits' document Hashes.
95
- #
96
- # Examples
97
- #
98
- # scan.each do |hits|
99
- # hits['total']
100
- # hits['offset']
101
- # hits['hits'].each { |document| ... }
102
- # end
103
- #
104
- # Returns this Scan instance.
105
- def each
106
- loop do
107
- body = client.scroll scroll_id, scroll
108
- @scroll_id = body['_scroll_id']
109
-
110
- hits = body['hits']
111
- break if hits['hits'].empty?
112
-
113
- hits['offset'] = @offset
114
- @offset += hits['hits'].length
115
-
116
- yield hits
117
- end
118
-
119
- self
120
- end
121
-
122
- # Iterate over each document from the scan query. This method is just a
123
- # convenience wrapper around the `each` method; it iterates the Array of
124
- # documents and passes them one by one to the block.
125
- #
126
- # block - The block will be called for each document returned from
127
- # executing the scan query.
128
- #
129
- # Yields a document Hash.
130
- #
131
- # Examples
132
- #
133
- # scan.each_document do |document|
134
- # document['_id']
135
- # document['_source']
136
- # end
137
- #
138
- # Returns this Scan instance.
139
- def each_document( &block )
140
- each { |hits| hits['hits'].each(&block) }
141
- end
142
-
143
- # Internal: Returns the current scroll ID as a String.
144
- def scroll_id
145
- return @scroll_id if defined? @scroll_id
146
-
147
- response = client.get '{/index}{/type}/_search',
148
- :action => 'search.scan',
149
- :search_type => 'scan',
150
- :scroll => scroll,
151
- :size => size,
152
- :index => index,
153
- :type => type,
154
- :body => query
155
-
156
- @scroll_id = response.body['_scroll_id']
157
- end
158
-
159
- end # Scan
160
- end # Client
161
- end # Elastomer