es-elasticity 0.12.0 → 0.13.3.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.travis.yml DELETED
@@ -1,17 +0,0 @@
1
- language: ruby
2
- before_install:
3
- - gem update bundler
4
- - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-5.1.1.deb && sudo service elasticsearch restart
5
- rvm:
6
- - 2.2.2
7
- - 2.2.3
8
- addons:
9
- apt:
10
- packages:
11
- - oracle-java8-set-default
12
- before_script:
13
- - sleep 5
14
- env:
15
- global:
16
- - secure: "HZa3D2GGwC6Jl062LJulbWZLTCieeBFC3FOJrArC5ul7ACGR5CEtANe0/UTnIf/Ad40p7I5VhiTdNFTcHunTbNc7Ae7dE5fOkiBHtxo/zwgpvHZK0iPvIoxsSfdcHobHeaF7NvfcXUkYUKcdRUyplHdB56eHQqYPVsah66K/4XA="
17
- sudo: true
data/CHANGELOG DELETED
@@ -1,76 +0,0 @@
1
- v0.12.0
2
- - warn when configuring a index with subclasses if using ES version that does support them
3
- - raise exception when creating or adding document to an index configured with subclasses if using
4
- an ES version that does not support them
5
- v0.11.5
6
- - Give the option of retrying the deletion for certain exceptions during remap
7
- v0.11.4
8
- - Fully clean up if error occurs during remap (assign all aliases back to original index)
9
- v0.11.3
10
- - Adds support for preserving the order or normalized names of `highlight` through `highlighted_attrs`
11
-
12
- v0.11.2
13
- - Adds support for passing arguments to Search definition through `search(query, search_args)` in index searching and msearch
14
- - adds _explanation to hold value of returned explanations in the base document
15
-
16
- v0.11.1
17
- - support `action.destructive_requires_name` setting by being explict about which indices to delete
18
-
19
- v0.11.0
20
- - compatibilty with ES v6
21
- - change mappings for 'index' to boolean. "string" type was replaced with "text"
22
- - use "successful" from API response ('created' was removed)
23
- - stringify keys for :mappings so clients can use symbol keys
24
- v0.10.0
25
- - update remap to removing fields from the mapping that are not explicitly
26
- defined.
27
- v0.9.1
28
- - fix search enumerator, missing first result set
29
- v0.8.3
30
- - fix remap method to use the scan api properly.
31
- v0.8.2
32
- - fix scan api to work with more recent versions of elasticsearch ruby.
33
- v0.8.1
34
- - loosen support for elasticsearch-ruby versions to support more versions of
35
- elasticsearch
36
- v0.8.0
37
- - Make Elasticity::Strategies::AliasIndex the default
38
- - Use mapping instead of mappings, we wanna be consistent to ES not to elasticsearch-ruby
39
- - Better automatic index name and document type
40
- v0.7.1
41
- - add more response info to raised exceptions from reindexing
42
- v0.6.5
43
- - update search and multi search interfaces to allow passing of general
44
- search definition arguments found in https://github.com/elastic/elasticsearch-ruby/blob/bdf5e145e5acc21726dddcd34492debbbddde568/elasticsearch-api/lib/elasticsearch/api/actions/search.rb#L125-L162
45
- v0.6.4
46
- - update suggestions to pull from the proper key
47
- v0.6.3
48
- - add next_page and previous_page to be compatible with will_paginate
49
- interface
50
- v0.6.2
51
- - update multi search `[]` method to raise an exception with key name to
52
- make it easier to debug failed queries within a multi search hash.
53
- v0.6.0
54
- - Change documents to be able to define indexes with multiple doc types. A
55
- Document class can define subclasses which are of different doc_types and
56
- all live in the same index's mappings.
57
- - updated search queries to pass either a list of document types or a single
58
- document type.
59
- - Update documents to generate a default document_type from the class name
60
- so that Documents always have a document type. You'll still usually want to
61
- manually define the document type, but it's no longer necessary.
62
- v0.5.2
63
- - Add aggregations to multi_search
64
- v0.5.1
65
- - Add ability to reindex individual attributes of a document using the
66
- bulk_update API.
67
- v0.5.0
68
- - Refactor of multisearch and search facade
69
- - add Search::Results proxy object so pagination and meta data methods can
70
- be standardize across all responses
71
- - Searches no longer return a simple array, they now return the
72
- Search::Results object which proxies array and is enumerable.
73
- v0.4.5
74
- - Fix issue with hash strings and pagination
75
- v0.4.4
76
- - Added support for surfacing document _score on query results.
data/bin/rake DELETED
@@ -1,16 +0,0 @@
1
- #!/usr/bin/env ruby
2
- #
3
- # This file was generated by Bundler.
4
- #
5
- # The application 'rake' is installed as part of a gem, and
6
- # this file is here to facilitate running it.
7
- #
8
-
9
- require 'pathname'
10
- ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
11
- Pathname.new(__FILE__).realpath)
12
-
13
- require 'rubygems'
14
- require 'bundler/setup'
15
-
16
- load Gem.bin_path('rake', 'rake')
data/bin/rspec DELETED
@@ -1,16 +0,0 @@
1
- #!/usr/bin/env ruby
2
- #
3
- # This file was generated by Bundler.
4
- #
5
- # The application 'rspec' is installed as part of a gem, and
6
- # this file is here to facilitate running it.
7
- #
8
-
9
- require 'pathname'
10
- ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
11
- Pathname.new(__FILE__).realpath)
12
-
13
- require 'rubygems'
14
- require 'bundler/setup'
15
-
16
- load Gem.bin_path('rspec-core', 'rspec')
Binary file
@@ -1,439 +0,0 @@
1
- RSpec.describe "Persistence", elasticsearch: true do
2
- def random_birthdate
3
- Time.at(0.0 + rand * (Time.now.to_f - 0.0.to_f))
4
- end
5
-
6
- describe "single index strategy" do
7
- subject do
8
- Class.new(Elasticity::Document) do
9
- def self.name
10
- 'SomeClass'
11
- end
12
-
13
- configure do |c|
14
- c.index_base_name = "users"
15
- c.document_type = "user"
16
- c.strategy = Elasticity::Strategies::SingleIndex
17
-
18
- c.mapping = {
19
- "properties" => {
20
- name: { type: "text", index: false },
21
- birthdate: { type: "date" },
22
- },
23
- }
24
- end
25
-
26
- attr_accessor :name, :birthdate
27
-
28
- def to_document
29
- { name: name, birthdate: birthdate }
30
- end
31
- end
32
- end
33
-
34
- before do
35
- subject.recreate_index
36
- @elastic_search_client.cluster.health wait_for_status: 'yellow'
37
- end
38
-
39
- after do
40
- subject.delete_index
41
- end
42
-
43
- it "counts empty search" do
44
- count = subject.search({}).count
45
- expect(count).to eq 0
46
- end
47
-
48
- it "successfully index, update, search, count and delete" do
49
- john = subject.new(name: "John", birthdate: "1985-10-31", sort: ['john'])
50
- mari = subject.new(name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
51
-
52
- john.update
53
- mari.update
54
-
55
- subject.flush_index
56
-
57
- results = subject.search({})
58
- expect(results.total).to eq 2
59
-
60
- expect(subject.search({ query: { match_all: {} } }).count).to eq(2)
61
-
62
- john.update
63
- mari.delete
64
-
65
- subject.flush_index
66
-
67
- results = subject.search({})
68
- expect(results.total).to eq 1
69
-
70
- expect(results[0].name).to eq(john.name)
71
- end
72
- end
73
-
74
- describe 'multi mapping index' do
75
- class Animal < Elasticity::Document
76
- configure do |c|
77
- c.index_base_name = "cats_and_dogs"
78
- c.strategy = Elasticity::Strategies::SingleIndex
79
- c.subclasses = { cat: "Cat", dog: "Dog" }
80
- end
81
- end
82
-
83
- class Cat < Animal
84
- configure do |c|
85
- c.index_base_name = "cats_and_dogs"
86
- c.strategy = Elasticity::Strategies::SingleIndex
87
- c.document_type = "cat"
88
-
89
- c.mapping = { "properties" => {
90
- name: { type: "text", index: true },
91
- age: { type: "integer" }
92
- } }
93
- end
94
-
95
- attr_accessor :name, :age
96
-
97
- def to_document
98
- { name: name, age: age }
99
- end
100
- end
101
-
102
- class Dog < Animal
103
- configure do |c|
104
- c.index_base_name = "cats_and_dogs"
105
- c.strategy = Elasticity::Strategies::SingleIndex
106
- c.document_type = "dog"
107
- c.mapping = { "properties" => {
108
- name: { type: "text", index: true },
109
- age: { type: "integer" },
110
- hungry: { type: "boolean" }
111
- } }
112
- end
113
- attr_accessor :name, :age, :hungry
114
-
115
- def to_document
116
- { name: name, age: age, hungry: hungry }
117
- end
118
- end
119
-
120
- before do
121
- Animal.recreate_index
122
- @elastic_search_client.cluster.health wait_for_status: 'yellow'
123
- end
124
-
125
- it "successful index, update, search, count and deletes" do
126
- cat = Cat.new(name: "felix", age: 10)
127
- dog = Dog.new(name: "fido", age: 4, hungry: true)
128
-
129
- cat.update
130
- dog.update
131
-
132
- Animal.flush_index
133
-
134
- results = Animal.search({})
135
- expect(results.total).to eq 2
136
- expect(results.map(&:class)).to include(Cat, Dog)
137
-
138
- results = Cat.search({})
139
- expect(results.total).to eq 1
140
- expect(results.first.class).to eq Cat
141
-
142
- results = Dog.search({})
143
- expect(results.total).to eq 1
144
- expect(results.first.class).to eq Dog
145
-
146
- cat.delete
147
- Animal.flush_index
148
-
149
- results = Animal.search({})
150
- expect(results.total).to eq 1
151
- expect(results.map(&:class)).to include(Dog)
152
- expect(results.scan_documents.count).to eq(1)
153
- end
154
- end
155
-
156
- describe "alias index strategy" do
157
- subject do
158
- Class.new(Elasticity::Document) do
159
- def self.name
160
- "SomeClass"
161
- end
162
-
163
- configure do |c|
164
- c.index_base_name = "users"
165
- c.document_type = "user"
166
- c.strategy = Elasticity::Strategies::AliasIndex
167
-
168
- c.mapping = {
169
- "properties" => {
170
- id: { type: "integer" },
171
- name: { type: "text", index: true },
172
- birthdate: { type: "date" },
173
- },
174
- }
175
- end
176
-
177
- attr_accessor :id, :name, :birthdate
178
-
179
- def to_document
180
- { id: id, name: name, birthdate: birthdate }
181
- end
182
- end
183
- end
184
-
185
- before do
186
- subject.recreate_index
187
- end
188
-
189
- after do
190
- subject.delete_index
191
- end
192
-
193
- it "counts empty search" do
194
- count = subject.search({}).count
195
- expect(count).to eq 0
196
- end
197
-
198
- it "remaps to a different index transparently" do
199
- john = subject.new(_id: 1, id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
200
- mari = subject.new(_id: 2, id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
201
-
202
- john.update
203
- mari.update
204
-
205
- subject.flush_index
206
- results = subject.search({})
207
- expect(results.total).to eq 2
208
-
209
- subject.remap!
210
-
211
- john.update
212
- mari.delete
213
-
214
- subject.flush_index
215
-
216
- results = subject.search({})
217
- expect(results.total).to eq 1
218
-
219
- expect(results[0].name).to eq(john.name)
220
- end
221
-
222
- it "handles in between state while remapping" do
223
- number_of_docs = 2000
224
- docs = number_of_docs.times.map do |i|
225
- subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
226
- end
227
-
228
- t = Thread.new { subject.remap! }
229
-
230
- to_update = docs.sample(10)
231
- to_delete = (docs - to_update).sample(10)
232
-
233
- to_update.each(&:update)
234
- to_delete.each(&:delete)
235
-
236
- 20.times.map do |i|
237
- subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
238
- end
239
-
240
- t.join
241
-
242
- subject.flush_index
243
- results = subject.search({})
244
- expect(results.total).to eq(2010)
245
- end
246
-
247
- it "does not copy over fields not defined in the mapping" do
248
- john = subject.new(_id: 1, id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
249
- mari = subject.new(_id: 2, id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
250
-
251
- john.update
252
- mari.update
253
-
254
- subject.flush_index
255
- results = subject.search({})
256
- expect(results.first.birthdate).to be
257
-
258
- # no birthdate
259
- subject = Class.new(Elasticity::Document) do
260
- def self.name
261
- "SomeClass"
262
- end
263
-
264
- configure do |c|
265
- c.index_base_name = "users"
266
- c.document_type = "user"
267
- c.strategy = Elasticity::Strategies::AliasIndex
268
-
269
- c.mapping = {
270
- "properties" => {
271
- id: { type: "integer" },
272
- name: { type: "text", index: true },
273
- },
274
- }
275
- end
276
-
277
- attr_accessor :id, :name
278
-
279
- def to_document
280
- { id: id, name: name }
281
- end
282
- end
283
-
284
- subject.remap!
285
- subject.flush_index
286
-
287
- results = subject.search({})
288
- expect(results.first.respond_to?(:birthdate)).to be false
289
- end
290
-
291
- it "recover from remap interrupts" do
292
- number_of_docs = 2000
293
- docs = number_of_docs.times.map do |i|
294
- subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
295
- end
296
-
297
- t = Thread.new { subject.remap! }
298
-
299
- to_update = docs.sample(10)
300
- to_delete = (docs - to_update).sample(10)
301
-
302
- to_update.each(&:update)
303
- to_delete.each(&:delete)
304
-
305
- 20.times.map do |i|
306
- subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
307
- end
308
-
309
- t.raise("Test Interrupt")
310
- expect { t.join }.to raise_error("Test Interrupt")
311
-
312
- subject.flush_index
313
- results = subject.search({})
314
- expect(results.total).to eq(2010)
315
- end
316
-
317
- it "fully cleans up if error occurs deleting the old index during remap" do
318
- expected_aliases = %w[elasticity_test_users elasticity_test_users_update]
319
- original_aliases = all_aliases(subject)
320
- expect(original_aliases).to match_array(expected_aliases)
321
- number_of_docs = 20
322
- number_of_docs.times.map do |i|
323
- subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
324
- end
325
-
326
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise("KAPOW")
327
- expect do
328
- subject.remap!
329
- end.to raise_error("KAPOW")
330
- cleaned_up_aliases = all_aliases(subject)
331
- expect(cleaned_up_aliases).to match_array(expected_aliases)
332
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_call_original
333
- end
334
-
335
- context "recovering from remap errors" do
336
- let(:recoverable_message) do
337
- '[400] {"error":{"root_cause":[{"type":"remote_transport_exception","reason":"[your_cluster][cluster_id][indices:admin/delete]"}],"type":"illegal_argument_exception","reason":"Cannot delete indices that are being snapshotted: [[full_index_name_and_id]]. Try again after snapshot finishes or cancel the currently running snapshot."},"status":400}'
338
- end
339
- after do
340
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_call_original
341
- end
342
-
343
- it "waits for recoverable errors before deleting old index during remap" do
344
- original_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
345
-
346
- call_count = 0
347
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete) do
348
- call_count +=1
349
- if call_count < 3
350
- raise Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
351
- else
352
- []
353
- end
354
- end
355
- build_some_docs(subject)
356
-
357
- subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
358
- subject.flush_index
359
- results = subject.search({})
360
- expect(results.total).to eq(20)
361
- remapped_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
362
- expect(remapped_index_name).to_not eq(original_index_name)
363
- end
364
-
365
- it "will only retry for a set amount of time" do
366
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise(
367
- Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
368
- )
369
- build_some_docs(subject)
370
- expect {
371
- subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
372
- }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
373
- end
374
-
375
- it "will not only retry if the arguments say not to" do
376
- original_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
377
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).with(any_args).and_call_original
378
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).with(index: original_index_name).and_raise(
379
- Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
380
- )
381
- build_some_docs(subject)
382
- expect {
383
- subject.remap!(retry_delete_on_recoverable_errors: false)
384
- }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
385
- end
386
-
387
- it "will not retry for 'non-recoverable' errors" do
388
- exception_message = '[404] {"error":"alias [some_index_name] missing","status":404}'
389
- allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise(
390
- Elasticsearch::Transport::Transport::Errors::BadRequest.new(exception_message)
391
- )
392
- build_some_docs(subject)
393
- expect {
394
- subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
395
- }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
396
- end
397
- end
398
-
399
- it "bulk indexes, updates and delete" do
400
- docs = 2000.times.map do |i|
401
- subject.new(_id: i, id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
402
- end
403
-
404
- subject.bulk_index(docs)
405
- subject.flush_index
406
-
407
- results = subject.search(from: 0, size: 3000)
408
- expect(results.total).to eq 2000
409
-
410
- docs = 2000.times.map do |i|
411
- { _id: i, attr_name: "name", attr_value: "Updated" }
412
- end
413
-
414
- subject.bulk_update(docs)
415
- subject.flush_index
416
-
417
- results = subject.search(from: 0, size: 3000)
418
- expect(results.total).to eq 2000
419
- expect(subject.search({ query: { match: { name: "Updated" } } } ).count).to eq(2000)
420
-
421
- subject.bulk_delete(results.documents.map(&:_id))
422
- subject.flush_index
423
-
424
- results = subject.search(from: 0, size: 3000)
425
- expect(results.total).to eq 0
426
- end
427
- end
428
-
429
- def all_aliases(subj)
430
- base_name = subj.ref_index_name
431
- subj.config.client.index_get_alias(index: "#{base_name}-*", name: "#{base_name}*").values.first["aliases"].keys
432
- end
433
-
434
- def build_some_docs(subj, doc_count = 20)
435
- doc_count.times.map do |i|
436
- subj.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
437
- end
438
- end
439
- end