elastomer-client 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +12 -0
- data/CHANGELOG.md +15 -0
- data/README.md +6 -7
- data/Rakefile +21 -0
- data/docs/README.md +44 -0
- data/docs/bulk_indexing.md +3 -0
- data/docs/client.md +240 -0
- data/docs/cluster.md +148 -0
- data/docs/docs.md +254 -0
- data/docs/index.md +161 -0
- data/docs/multi_search.md +3 -0
- data/docs/notifications.md +24 -11
- data/docs/scan_scroll.md +3 -0
- data/docs/snapshots.md +3 -0
- data/docs/templates.md +3 -0
- data/docs/warmers.md +3 -0
- data/elastomer-client.gemspec +2 -2
- data/lib/elastomer/client.rb +70 -43
- data/lib/elastomer/client/bulk.rb +2 -2
- data/lib/elastomer/client/cluster.rb +2 -2
- data/lib/elastomer/client/docs.rb +190 -54
- data/lib/elastomer/client/errors.rb +4 -2
- data/lib/elastomer/client/index.rb +111 -43
- data/lib/elastomer/client/multi_search.rb +1 -1
- data/lib/elastomer/client/nodes.rb +9 -4
- data/lib/elastomer/client/repository.rb +2 -2
- data/lib/elastomer/client/scroller.rb +235 -0
- data/lib/elastomer/client/snapshot.rb +1 -1
- data/lib/elastomer/client/template.rb +1 -1
- data/lib/elastomer/client/warmer.rb +1 -1
- data/lib/elastomer/notifications.rb +1 -1
- data/lib/elastomer/version.rb +1 -1
- data/script/bootstrap +0 -7
- data/script/cibuild +8 -3
- data/script/test +6 -0
- data/test/client/bulk_test.rb +2 -2
- data/test/client/cluster_test.rb +23 -2
- data/test/client/docs_test.rb +137 -6
- data/test/client/errors_test.rb +12 -8
- data/test/client/index_test.rb +88 -5
- data/test/client/multi_search_test.rb +29 -0
- data/test/client/repository_test.rb +36 -37
- data/test/client/{scan_test.rb → scroller_test.rb} +25 -6
- data/test/client/snapshot_test.rb +53 -43
- data/test/client/stubbed_client_test.rb +1 -1
- data/test/client_test.rb +60 -0
- data/test/notifications_test.rb +69 -0
- data/test/test_helper.rb +54 -11
- metadata +36 -23
- data/.ruby-version +0 -1
- data/lib/elastomer/client/scan.rb +0 -161
- data/script/testsuite +0 -10
@@ -30,7 +30,7 @@ describe 'stubbed client tests' do
|
|
30
30
|
@stubs.post('/_cluster/nodes/_all/_shutdown') { [200, {'Content-Type' => 'application/json'}, '{"nodes":{"1":{"name":"Node1"}}}'] }
|
31
31
|
@stubs.post('/_cluster/nodes/node2/_shutdown') { [200, {'Content-Type' => 'application/json'}, '{"nodes":{"2":{"name":"Node2"}}}'] }
|
32
32
|
|
33
|
-
h = @client.nodes.shutdown
|
33
|
+
h = @client.nodes("_all").shutdown
|
34
34
|
assert_equal "Node1", h['nodes']['1']['name']
|
35
35
|
|
36
36
|
h = @client.nodes('node2').shutdown
|
data/test/client_test.rb
CHANGED
@@ -55,6 +55,44 @@ describe Elastomer::Client do
|
|
55
55
|
}
|
56
56
|
end
|
57
57
|
|
58
|
+
describe 'when extracting and converting :body params' do
|
59
|
+
it 'deletes the :body from the params (or it gets the hose)' do
|
60
|
+
params = { :body => nil, :q => "what what?" }
|
61
|
+
body = $client.extract_body params
|
62
|
+
|
63
|
+
assert_nil body
|
64
|
+
assert_equal({:q => "what what?"}, params)
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'leaves String values unchanged' do
|
68
|
+
body = $client.extract_body :body => '{"query":{"match_all":{}}}'
|
69
|
+
assert_equal '{"query":{"match_all":{}}}', body
|
70
|
+
|
71
|
+
body = $client.extract_body :body => 'not a JSON string, but who cares!'
|
72
|
+
assert_equal 'not a JSON string, but who cares!', body
|
73
|
+
end
|
74
|
+
|
75
|
+
it 'joins Array values' do
|
76
|
+
body = $client.extract_body :body => %w[foo bar baz]
|
77
|
+
assert_equal "foo\nbar\nbaz\n", body
|
78
|
+
|
79
|
+
body = $client.extract_body :body => [
|
80
|
+
'the first entry',
|
81
|
+
'the second entry',
|
82
|
+
nil
|
83
|
+
]
|
84
|
+
assert_equal "the first entry\nthe second entry\n", body
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'converts values to JSON' do
|
88
|
+
body = $client.extract_body :body => true
|
89
|
+
assert_equal "true", body
|
90
|
+
|
91
|
+
body = $client.extract_body :body => {:query => {:match_all => {}}}
|
92
|
+
assert_equal '{"query":{"match_all":{}}}', body
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
58
96
|
describe 'when validating parameters' do
|
59
97
|
it 'rejects nil values' do
|
60
98
|
assert_raises(ArgumentError) { $client.assert_param_presence nil }
|
@@ -92,4 +130,26 @@ describe Elastomer::Client do
|
|
92
130
|
assert_equal '9', $client.assert_param_presence(9)
|
93
131
|
end
|
94
132
|
end
|
133
|
+
|
134
|
+
describe 'top level actions' do
|
135
|
+
it 'pings the cluster' do
|
136
|
+
assert_equal true, $client.ping
|
137
|
+
assert_equal true, $client.available?
|
138
|
+
end
|
139
|
+
|
140
|
+
it 'gets cluster info' do
|
141
|
+
h = $client.info
|
142
|
+
assert h.key?('name'), 'expected cluster name to be returned'
|
143
|
+
assert h.key?('status'), 'expected cluster info status to be returned'
|
144
|
+
end
|
145
|
+
|
146
|
+
it 'gets cluster version' do
|
147
|
+
assert_match /[\d\.]+/, $client.version
|
148
|
+
end
|
149
|
+
|
150
|
+
it 'gets semantic version' do
|
151
|
+
version_string = $client.version
|
152
|
+
assert_equal Semantic::Version.new(version_string), $client.semantic_version
|
153
|
+
end
|
154
|
+
end
|
95
155
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
require 'elastomer/notifications'
|
3
|
+
|
4
|
+
describe Elastomer::Notifications do
|
5
|
+
before do
|
6
|
+
@name = 'elastomer-notifications-test'
|
7
|
+
@index = $client.index @name
|
8
|
+
@index.delete if @index.exists?
|
9
|
+
@events = []
|
10
|
+
@subscriber = ActiveSupport::Notifications.subscribe do |*args|
|
11
|
+
@events << ActiveSupport::Notifications::Event.new(*args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
after do
|
16
|
+
ActiveSupport::Notifications.unsubscribe(@subscriber)
|
17
|
+
@index.delete if @index.exists?
|
18
|
+
end
|
19
|
+
|
20
|
+
it "instruments timeouts" do
|
21
|
+
$client.stub :connection, lambda { raise Faraday::Error::TimeoutError } do
|
22
|
+
assert_raises(Elastomer::Client::TimeoutError) { $client.info }
|
23
|
+
event = @events.detect { |e| e.payload[:action] == 'cluster.info' }
|
24
|
+
exception = event.payload[:exception]
|
25
|
+
assert_equal 'Elastomer::Client::TimeoutError', exception[0]
|
26
|
+
assert_match 'timeout', exception[1]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'instruments cluster actions' do
|
31
|
+
$client.ping; assert_action_event('cluster.ping')
|
32
|
+
$client.info; assert_action_event('cluster.info')
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'instruments node actions' do
|
36
|
+
nodes = $client.nodes
|
37
|
+
nodes.info; assert_action_event('nodes.info')
|
38
|
+
nodes.stats; assert_action_event('nodes.stats')
|
39
|
+
nodes.hot_threads; assert_action_event('nodes.hot_threads')
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'instruments node shutdown' do
|
43
|
+
client = stub_client(:post, '/_cluster/nodes/_shutdown')
|
44
|
+
client.nodes.shutdown; assert_action_event('nodes.shutdown')
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'instruments index actions' do
|
48
|
+
@index.exists?; assert_action_event('index.exists')
|
49
|
+
@index.create('number_of_replicas' => 0)
|
50
|
+
assert_action_event('index.create')
|
51
|
+
@index.get_settings; assert_action_event('index.get_settings')
|
52
|
+
@index.update_settings('number_of_replicas' => 0)
|
53
|
+
assert_action_event('index.get_settings')
|
54
|
+
@index.close; assert_action_event('index.close')
|
55
|
+
@index.open; assert_action_event('index.open')
|
56
|
+
@index.delete; assert_action_event('index.delete')
|
57
|
+
end
|
58
|
+
|
59
|
+
def assert_action_event(action)
|
60
|
+
assert @events.detect { |e| e.payload[:action] == action }, "expected #{action} event"
|
61
|
+
end
|
62
|
+
|
63
|
+
def stub_client(method, url, status=200, body='{"acknowledged":true}')
|
64
|
+
stubs = Faraday::Adapter::Test::Stubs.new do |stub|
|
65
|
+
stub.send(method, url) { |env| [status, {}, body]}
|
66
|
+
end
|
67
|
+
Elastomer::Client.new($client_params.merge(:opaque_id => false, :adapter => [:test, stubs]))
|
68
|
+
end
|
69
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require '
|
1
|
+
require 'securerandom'
|
2
2
|
require 'rubygems' unless defined? Gem
|
3
3
|
require 'bundler'
|
4
4
|
Bundler.require(:default, :development)
|
@@ -11,6 +11,8 @@ if ENV['COVERAGE'] == 'true'
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
ENV['SNAPSHOT_DIR'] ||= '/tmp/elastomer-client-snapshot-test'
|
15
|
+
|
14
16
|
require 'minitest/spec'
|
15
17
|
require 'minitest/autorun'
|
16
18
|
|
@@ -31,6 +33,8 @@ $client = Elastomer::Client.new $client_params
|
|
31
33
|
# ensure we have an ElasticSearch server to test with
|
32
34
|
raise "No server available at #{$client.url}" unless $client.available?
|
33
35
|
|
36
|
+
puts "Elasticsearch version is #{$client.version}"
|
37
|
+
|
34
38
|
# remove any lingering test indices from the cluster
|
35
39
|
MiniTest::Unit.after_tests do
|
36
40
|
$client.cluster.indices.keys.each do |name|
|
@@ -87,6 +91,21 @@ def es_version_1_x?
|
|
87
91
|
$client.semantic_version >= '1.0.0'
|
88
92
|
end
|
89
93
|
|
94
|
+
# Elasticsearch 1.4 changed the response body for interacting with index
|
95
|
+
# alaises. If an index does not contain any aliases, then an "alises" key is no
|
96
|
+
# longer returned in the resopsne.
|
97
|
+
#
|
98
|
+
# Reeturns `true` if the response contains an "alises" key.
|
99
|
+
def es_version_always_returns_aliases?
|
100
|
+
$client.semantic_version <= '1.4.0' ||
|
101
|
+
$client.semantic_version >= '1.4.3'
|
102
|
+
end
|
103
|
+
|
104
|
+
# ElasticSearch 1.3 added the `search_shards` API endpoint.
|
105
|
+
def es_version_supports_search_shards?
|
106
|
+
$client.semantic_version >= '1.3.0'
|
107
|
+
end
|
108
|
+
|
90
109
|
# Elasticsearch 1.2 removed support for gateway snapshots.
|
91
110
|
#
|
92
111
|
# Returns true if Elasticsearch version supports gateway snapshots.
|
@@ -94,19 +113,43 @@ def es_version_supports_gateway_snapshots?
|
|
94
113
|
$client.semantic_version <= '1.2.0'
|
95
114
|
end
|
96
115
|
|
116
|
+
def create_repo(name, settings = {})
|
117
|
+
default_settings = {:type => 'fs', :settings => {:location => ENV['SNAPSHOT_DIR']}}
|
118
|
+
$client.repository(name).create(default_settings.merge(settings))
|
119
|
+
end
|
97
120
|
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
121
|
+
def delete_repo(name)
|
122
|
+
repo = $client.repository(name)
|
123
|
+
repo.delete if repo.exists?
|
124
|
+
end
|
125
|
+
|
126
|
+
def delete_repo_snapshots(name)
|
127
|
+
repo = $client.repository(name)
|
128
|
+
if repo.exists?
|
129
|
+
response = repo.snapshots.get
|
130
|
+
response["snapshots"].each do |snapshot_info|
|
131
|
+
repo.snapshot(snapshot_info["snapshot"]).delete
|
132
|
+
end
|
103
133
|
end
|
104
134
|
end
|
105
135
|
|
106
|
-
def
|
107
|
-
|
108
|
-
|
109
|
-
yield
|
110
|
-
|
136
|
+
def with_tmp_repo(name = SecureRandom.uuid, &block)
|
137
|
+
begin
|
138
|
+
create_repo(name)
|
139
|
+
yield $client.repository(name)
|
140
|
+
ensure
|
141
|
+
delete_repo_snapshots(name)
|
142
|
+
delete_repo(name)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def create_snapshot(repo, name = SecureRandom.uuid)
|
147
|
+
repo.snapshot(name).create({}, :wait_for_completion => true)
|
148
|
+
end
|
149
|
+
|
150
|
+
def with_tmp_snapshot(name = SecureRandom.uuid, &block)
|
151
|
+
with_tmp_repo do |repo|
|
152
|
+
create_snapshot(repo, name)
|
153
|
+
yield repo.snapshot(name), repo
|
111
154
|
end
|
112
155
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastomer-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tim Pease
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2015-01-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: addressable
|
@@ -81,6 +81,20 @@ dependencies:
|
|
81
81
|
- - "~>"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: '1.5'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: activesupport
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '3.0'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '3.0'
|
84
98
|
- !ruby/object:Gem::Dependency
|
85
99
|
name: minitest
|
86
100
|
requirement: !ruby/object:Gem::Requirement
|
@@ -109,20 +123,6 @@ dependencies:
|
|
109
123
|
- - ">="
|
110
124
|
- !ruby/object:Gem::Version
|
111
125
|
version: '0'
|
112
|
-
- !ruby/object:Gem::Dependency
|
113
|
-
name: debugger
|
114
|
-
requirement: !ruby/object:Gem::Requirement
|
115
|
-
requirements:
|
116
|
-
- - "~>"
|
117
|
-
- !ruby/object:Gem::Version
|
118
|
-
version: 1.6.8
|
119
|
-
type: :development
|
120
|
-
prerelease: false
|
121
|
-
version_requirements: !ruby/object:Gem::Requirement
|
122
|
-
requirements:
|
123
|
-
- - "~>"
|
124
|
-
- !ruby/object:Gem::Version
|
125
|
-
version: 1.6.8
|
126
126
|
description: |-
|
127
127
|
Elastomer is a low level API client for the
|
128
128
|
Elasticsearch HTTP interface.
|
@@ -134,13 +134,24 @@ extensions: []
|
|
134
134
|
extra_rdoc_files: []
|
135
135
|
files:
|
136
136
|
- ".gitignore"
|
137
|
-
- ".
|
137
|
+
- ".travis.yml"
|
138
138
|
- CHANGELOG.md
|
139
139
|
- Gemfile
|
140
140
|
- LICENSE.txt
|
141
141
|
- README.md
|
142
142
|
- Rakefile
|
143
|
+
- docs/README.md
|
144
|
+
- docs/bulk_indexing.md
|
145
|
+
- docs/client.md
|
146
|
+
- docs/cluster.md
|
147
|
+
- docs/docs.md
|
148
|
+
- docs/index.md
|
149
|
+
- docs/multi_search.md
|
143
150
|
- docs/notifications.md
|
151
|
+
- docs/scan_scroll.md
|
152
|
+
- docs/snapshots.md
|
153
|
+
- docs/templates.md
|
154
|
+
- docs/warmers.md
|
144
155
|
- elastomer-client.gemspec
|
145
156
|
- lib/elastomer/client.rb
|
146
157
|
- lib/elastomer/client/bulk.rb
|
@@ -151,7 +162,7 @@ files:
|
|
151
162
|
- lib/elastomer/client/multi_search.rb
|
152
163
|
- lib/elastomer/client/nodes.rb
|
153
164
|
- lib/elastomer/client/repository.rb
|
154
|
-
- lib/elastomer/client/
|
165
|
+
- lib/elastomer/client/scroller.rb
|
155
166
|
- lib/elastomer/client/snapshot.rb
|
156
167
|
- lib/elastomer/client/template.rb
|
157
168
|
- lib/elastomer/client/warmer.rb
|
@@ -164,7 +175,7 @@ files:
|
|
164
175
|
- script/bootstrap
|
165
176
|
- script/cibuild
|
166
177
|
- script/console
|
167
|
-
- script/
|
178
|
+
- script/test
|
168
179
|
- test/assertions.rb
|
169
180
|
- test/client/bulk_test.rb
|
170
181
|
- test/client/cluster_test.rb
|
@@ -174,7 +185,7 @@ files:
|
|
174
185
|
- test/client/multi_search_test.rb
|
175
186
|
- test/client/nodes_test.rb
|
176
187
|
- test/client/repository_test.rb
|
177
|
-
- test/client/
|
188
|
+
- test/client/scroller_test.rb
|
178
189
|
- test/client/snapshot_test.rb
|
179
190
|
- test/client/stubbed_client_test.rb
|
180
191
|
- test/client/template_test.rb
|
@@ -184,6 +195,7 @@ files:
|
|
184
195
|
- test/middleware/encode_json_test.rb
|
185
196
|
- test/middleware/opaque_id_test.rb
|
186
197
|
- test/middleware/parse_json_test.rb
|
198
|
+
- test/notifications_test.rb
|
187
199
|
- test/test_helper.rb
|
188
200
|
homepage: https://github.com/github/elastomer-client
|
189
201
|
licenses:
|
@@ -205,10 +217,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
205
217
|
version: '0'
|
206
218
|
requirements: []
|
207
219
|
rubyforge_project:
|
208
|
-
rubygems_version: 2.2.
|
220
|
+
rubygems_version: 2.2.2
|
209
221
|
signing_key:
|
210
222
|
specification_version: 4
|
211
|
-
summary: A library for interacting with
|
223
|
+
summary: A library for interacting with Elasticsearch
|
212
224
|
test_files:
|
213
225
|
- test/assertions.rb
|
214
226
|
- test/client/bulk_test.rb
|
@@ -219,7 +231,7 @@ test_files:
|
|
219
231
|
- test/client/multi_search_test.rb
|
220
232
|
- test/client/nodes_test.rb
|
221
233
|
- test/client/repository_test.rb
|
222
|
-
- test/client/
|
234
|
+
- test/client/scroller_test.rb
|
223
235
|
- test/client/snapshot_test.rb
|
224
236
|
- test/client/stubbed_client_test.rb
|
225
237
|
- test/client/template_test.rb
|
@@ -229,4 +241,5 @@ test_files:
|
|
229
241
|
- test/middleware/encode_json_test.rb
|
230
242
|
- test/middleware/opaque_id_test.rb
|
231
243
|
- test/middleware/parse_json_test.rb
|
244
|
+
- test/notifications_test.rb
|
232
245
|
- test/test_helper.rb
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.1.0-github
|
@@ -1,161 +0,0 @@
|
|
1
|
-
module Elastomer
|
2
|
-
class Client
|
3
|
-
|
4
|
-
# Create a new Scan instance for scrolling all results from a `query`.
|
5
|
-
#
|
6
|
-
# query - The query to scan as a Hash or a JSON encoded String
|
7
|
-
# opts - Options Hash
|
8
|
-
# :index - the name of the index to search
|
9
|
-
# :type - the document type to search
|
10
|
-
# :scroll - the keep alive time of the scrolling request (5 minutes by default)
|
11
|
-
# :size - the number of documents per shard to fetch per scroll
|
12
|
-
#
|
13
|
-
# Examples
|
14
|
-
#
|
15
|
-
# scan = client.scan('{"query":{"match_all":{}}}', :index => 'test')
|
16
|
-
# scan.each_document do |document|
|
17
|
-
# document['_id']
|
18
|
-
# document['_source']
|
19
|
-
# end
|
20
|
-
#
|
21
|
-
# Returns a new Scan instance
|
22
|
-
def scan( query, opts = {} )
|
23
|
-
Scan.new self, query, opts
|
24
|
-
end
|
25
|
-
|
26
|
-
# Continue scrolling a scan query.
|
27
|
-
# See http://www.elasticsearch.org/guide/reference/api/search/scroll/
|
28
|
-
#
|
29
|
-
# scroll_id - The current scroll ID as a String
|
30
|
-
# scroll - The keep alive time of the scrolling request (5 minutes by default)
|
31
|
-
#
|
32
|
-
# Examples
|
33
|
-
#
|
34
|
-
# scroll_id = client.scan('{"query":{"match_all":{}}}', :index => 'test').scroll_id
|
35
|
-
#
|
36
|
-
# h = client.scroll scroll_id # scroll to get the next set of results
|
37
|
-
# scroll_id = h['_scroll_id'] # and store the scroll_id to use later
|
38
|
-
#
|
39
|
-
# h = client.scroll scroll_id # scroll again to get the next set of results
|
40
|
-
# scroll_id = h['_scroll_id'] # and store the scroll_id to use later
|
41
|
-
#
|
42
|
-
# # repeat until the results are empty
|
43
|
-
#
|
44
|
-
# Returns the response body as a Hash.
|
45
|
-
def scroll( scroll_id, scroll = '5m' )
|
46
|
-
response = get '/_search/scroll', :body => scroll_id, :scroll => scroll, :action => 'search.scroll'
|
47
|
-
response.body
|
48
|
-
end
|
49
|
-
|
50
|
-
|
51
|
-
class Scan
|
52
|
-
# Create a new scan client that can be used to iterate over all the
|
53
|
-
# documents returned by the `query`.
|
54
|
-
#
|
55
|
-
# See http://www.elasticsearch.org/guide/reference/api/search/scroll/
|
56
|
-
# and the "Scan" section of http://www.elasticsearch.org/guide/reference/api/search/search-type/
|
57
|
-
#
|
58
|
-
# client - Elastomer::Client used for HTTP requests to the server
|
59
|
-
# query - The query to scan as a Hash or a JSON encoded String
|
60
|
-
# opts - Options Hash
|
61
|
-
# :index - the name of the index to search
|
62
|
-
# :type - the document type to search
|
63
|
-
# :scroll - the keep alive time of the scrolling request (5 minutes by default)
|
64
|
-
# :size - the number of documents per shard to fetch per scroll
|
65
|
-
#
|
66
|
-
# Examples
|
67
|
-
#
|
68
|
-
# scan = Scan.new(client, {:query => {:match_all => {}}}, :index => 'test-1')
|
69
|
-
# scan.each_document { |doc|
|
70
|
-
# doc['_id']
|
71
|
-
# doc['_source']
|
72
|
-
# }
|
73
|
-
#
|
74
|
-
def initialize( client, query, opts = {} )
|
75
|
-
@client = client
|
76
|
-
@query = query
|
77
|
-
|
78
|
-
@index = opts.fetch(:index, nil)
|
79
|
-
@type = opts.fetch(:type, nil)
|
80
|
-
@scroll = opts.fetch(:scroll, '5m')
|
81
|
-
@size = opts.fetch(:size, 50)
|
82
|
-
|
83
|
-
@offset = 0
|
84
|
-
end
|
85
|
-
|
86
|
-
attr_reader :client, :query, :index, :type, :scroll, :size
|
87
|
-
|
88
|
-
# Iterate over all the search results from the scan query.
|
89
|
-
#
|
90
|
-
# block - The block will be called for each set of matching documents
|
91
|
-
# returned from executing the scan query.
|
92
|
-
#
|
93
|
-
# Yields a hits Hash containing the 'total' number of hits, current
|
94
|
-
# 'offset' into that total, and the Array of 'hits' document Hashes.
|
95
|
-
#
|
96
|
-
# Examples
|
97
|
-
#
|
98
|
-
# scan.each do |hits|
|
99
|
-
# hits['total']
|
100
|
-
# hits['offset']
|
101
|
-
# hits['hits'].each { |document| ... }
|
102
|
-
# end
|
103
|
-
#
|
104
|
-
# Returns this Scan instance.
|
105
|
-
def each
|
106
|
-
loop do
|
107
|
-
body = client.scroll scroll_id, scroll
|
108
|
-
@scroll_id = body['_scroll_id']
|
109
|
-
|
110
|
-
hits = body['hits']
|
111
|
-
break if hits['hits'].empty?
|
112
|
-
|
113
|
-
hits['offset'] = @offset
|
114
|
-
@offset += hits['hits'].length
|
115
|
-
|
116
|
-
yield hits
|
117
|
-
end
|
118
|
-
|
119
|
-
self
|
120
|
-
end
|
121
|
-
|
122
|
-
# Iterate over each document from the scan query. This method is just a
|
123
|
-
# convenience wrapper around the `each` method; it iterates the Array of
|
124
|
-
# documents and passes them one by one to the block.
|
125
|
-
#
|
126
|
-
# block - The block will be called for each document returned from
|
127
|
-
# executing the scan query.
|
128
|
-
#
|
129
|
-
# Yields a document Hash.
|
130
|
-
#
|
131
|
-
# Examples
|
132
|
-
#
|
133
|
-
# scan.each_document do |document|
|
134
|
-
# document['_id']
|
135
|
-
# document['_source']
|
136
|
-
# end
|
137
|
-
#
|
138
|
-
# Returns this Scan instance.
|
139
|
-
def each_document( &block )
|
140
|
-
each { |hits| hits['hits'].each(&block) }
|
141
|
-
end
|
142
|
-
|
143
|
-
# Internal: Returns the current scroll ID as a String.
|
144
|
-
def scroll_id
|
145
|
-
return @scroll_id if defined? @scroll_id
|
146
|
-
|
147
|
-
response = client.get '{/index}{/type}/_search',
|
148
|
-
:action => 'search.scan',
|
149
|
-
:search_type => 'scan',
|
150
|
-
:scroll => scroll,
|
151
|
-
:size => size,
|
152
|
-
:index => index,
|
153
|
-
:type => type,
|
154
|
-
:body => query
|
155
|
-
|
156
|
-
@scroll_id = response.body['_scroll_id']
|
157
|
-
end
|
158
|
-
|
159
|
-
end # Scan
|
160
|
-
end # Client
|
161
|
-
end # Elastomer
|