es-elasticity 0.8.0 → 0.8.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 720cd13dc5ba2ce7e66491cf089672f11d8a755d
4
- data.tar.gz: 203824852dd40864503b465430fa401fec2d758e
3
+ metadata.gz: 273883b20e8b274c5029f67bc93985b5f2405348
4
+ data.tar.gz: 5960c883e06d2bcfa72e501f3fc4dbd0cd941c46
5
5
  SHA512:
6
- metadata.gz: 1bfd03d5807b389ba5d9cd38105ce26d997c1ae2d028462b1f28a6f271d1220e375c32c48af8c700cd232a881648599524a56e0ccc912c0489739d55509820af
7
- data.tar.gz: 9246d84d925302de828eab261f940039b64bf28c6b72df57ae512c9854aeda14dd095a57cf62aa5da2f88ddd3769ba5ae226ec8ac60b70c8a0b05cb02da8e01b
6
+ metadata.gz: 1d9ecb7b3ede09bc2ea49b64cd3752fd8ca502b2381aee55e9dd3ce421f1a1c72f7d0dcb1a02fa8d8af5bdc66d9110f7cfc1025396d85fbb8a4126ab3e0b140b
7
+ data.tar.gz: d10e1f5da32facb6a8ecaead082f71add59a876de3e6220db95fb0984a50a09d2540d9cfd83e3fc3fa3186a70b745ba1c4b50dfedc17b2ba1c32f0f2ebd0ea85
data/.travis.yml CHANGED
@@ -1,13 +1,17 @@
1
1
  language: ruby
2
2
  before_install:
3
3
  - gem update bundler
4
+ - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-5.1.1.deb && sudo service elasticsearch restart
4
5
  rvm:
5
6
  - 2.2.2
6
7
  - 2.2.3
7
- services:
8
- - elasticsearch
8
+ addons:
9
+ apt:
10
+ packages:
11
+ - oracle-java8-set-default
9
12
  before_script:
10
13
  - sleep 5
11
14
  env:
12
15
  global:
13
16
  - secure: "HZa3D2GGwC6Jl062LJulbWZLTCieeBFC3FOJrArC5ul7ACGR5CEtANe0/UTnIf/Ad40p7I5VhiTdNFTcHunTbNc7Ae7dE5fOkiBHtxo/zwgpvHZK0iPvIoxsSfdcHobHeaF7NvfcXUkYUKcdRUyplHdB56eHQqYPVsah66K/4XA="
17
+ sudo: true
data/CHANGELOG CHANGED
@@ -1,3 +1,6 @@
1
+ v0.8.1
2
+ - loosen support for elasticsearch-ruby versions to support more versions of
3
+ elasticsearch
1
4
  v0.8.0
2
5
  - Make Elasticity::Strategies::AliasIndex the default
3
6
  - Use mapping instead of mappings, we wanna be consistent to ES not to elasticsearch-ruby
data/Gemfile CHANGED
@@ -2,3 +2,5 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in elasticity.gemspec
4
4
  gemspec
5
+
6
+ gem "elasticsearch", "5.0.3"
data/README.md CHANGED
@@ -28,6 +28,11 @@ Or install it yourself as:
28
28
 
29
29
  ## Usage
30
30
 
31
+ ### Version Support
32
+ This gem has [elasticsearch-ruby](https://github.com/elastic/elasticsearch-ruby) as a dependency. In order to use different versions of elasticsearch you will need to match your version of elasticsearch-ruby to the version of elasticsearch you want to use ([see here](https://github.com/elastic/elasticsearch-ruby#compatibility). Elasticity should work across all versions of elastisearch-ruby, although they have not all been tested so there are likely edge cases.
33
+
34
+ Currently tests are run on travis ci against elasticsearch 5.1.1 with elasticsearch-ruby 5.0.3.
35
+
31
36
  ### Configuration
32
37
 
33
38
  It is recommended you use Typhoeus for HTTP connections to Elasticsearch.
data/elasticity.gemspec CHANGED
@@ -25,10 +25,11 @@ Gem::Specification.new do |spec|
25
25
  spec.add_development_dependency "simplecov", "~> 0.7.1"
26
26
  spec.add_development_dependency "oj"
27
27
  spec.add_development_dependency "pry"
28
+ spec.add_development_dependency "byebug"
28
29
  spec.add_development_dependency "codeclimate-test-reporter"
29
30
  spec.add_development_dependency "redis"
30
31
 
31
32
  spec.add_dependency "activesupport", ">= 4.0.0", "< 6"
32
33
  spec.add_dependency "activemodel", ">= 4.0.0", "< 6"
33
- spec.add_dependency "elasticsearch", "~> 1.0.12"
34
+ spec.add_dependency "elasticsearch", ">= 1.0"
34
35
  end
@@ -0,0 +1,15 @@
1
+ module Elasticity
2
+ class ScrollableSearch
3
+ def self.search_type
4
+ if elasticsearch_gem_version < Gem::Version.create("2.0")
5
+ :scan
6
+ else
7
+ :query_then_fetch
8
+ end
9
+ end
10
+
11
+ def self.elasticsearch_gem_version
12
+ Gem.loaded_specs["elasticsearch"].version
13
+ end
14
+ end
15
+ end
@@ -177,7 +177,7 @@ module Elasticity
177
177
  def search
178
178
  return @search if defined?(@search)
179
179
  args = @search_definition.to_search_args
180
- args = args.merge(search_type: 'scan', size: @size, scroll: @scroll)
180
+ args = args.merge(search_type: ScrollableSearch.search_type, size: @size, scroll: @scroll)
181
181
  @search = @client.search(args)
182
182
  end
183
183
  end
@@ -52,9 +52,8 @@ module Elasticity
52
52
  })
53
53
 
54
54
  @client.index_flush(index: original_index)
55
- cursor = @client.search index: original_index, search_type: 'scan', scroll: '10m', _source: false, size: 100
55
+ cursor = @client.search index: original_index, search_type: ScrollableSearch.search_type, scroll: '10m', size: 100
56
56
  loop do
57
- cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
58
57
  hits = cursor['hits']['hits']
59
58
  break if hits.empty?
60
59
 
@@ -84,6 +83,7 @@ module Elasticity
84
83
  end
85
84
 
86
85
  @client.bulk(body: ops) unless ops.empty?
86
+ cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
87
87
  end
88
88
 
89
89
  # Update aliases to only point to the new index.
@@ -103,9 +103,8 @@ module Elasticity
103
103
  })
104
104
 
105
105
  @client.index_flush(index: new_index)
106
- cursor = @client.search index: new_index, search_type: 'scan', scroll: '1m', size: 100
106
+ cursor = @client.search index: new_index, search_type: ScrollableSearch.search_type, scroll: '1m', size: 100
107
107
  loop do
108
- cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
109
108
  hits = cursor['hits']['hits']
110
109
  break if hits.empty?
111
110
 
@@ -116,6 +115,7 @@ module Elasticity
116
115
  end
117
116
 
118
117
  @client.bulk(body: ops)
118
+ cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
119
119
  end
120
120
 
121
121
  @client.index_flush(index: original_index)
@@ -149,13 +149,13 @@ module Elasticity
149
149
  end
150
150
 
151
151
  def main_indexes
152
- @client.index_get_aliases(index: "#{@main_alias}-*", name: @main_alias).keys
152
+ @client.index_get_alias(index: "#{@main_alias}-*", name: @main_alias).keys
153
153
  rescue Elasticsearch::Transport::Transport::Errors::NotFound
154
154
  []
155
155
  end
156
156
 
157
157
  def update_indexes
158
- @client.index_get_aliases(index: "#{@main_alias}-*", name: @update_alias).keys
158
+ @client.index_get_alias(index: "#{@main_alias}-*", name: @update_alias).keys
159
159
  rescue Elasticsearch::Transport::Transport::Errors::NotFound
160
160
  []
161
161
  end
@@ -1,3 +1,3 @@
1
1
  module Elasticity
2
- VERSION = "0.8.0"
2
+ VERSION = "0.8.1"
3
3
  end
data/lib/elasticity.rb CHANGED
@@ -34,6 +34,7 @@ module Elasticity
34
34
  autoload :MultiSearch, "elasticity/multi_search"
35
35
  autoload :Search, "elasticity/search"
36
36
  autoload :Strategies, "elasticity/strategies"
37
+ autoload :ScrollableSearch, "elasticity/scrollable_search"
37
38
 
38
39
  def self.configure
39
40
  @config = Config.new
@@ -1,4 +1,8 @@
1
1
  RSpec.describe "Persistence", elasticsearch: true do
2
+ def random_birthdate
3
+ Time.at(0.0 + rand * (Time.now.to_f - 0.0.to_f))
4
+ end
5
+
2
6
  describe "single index strategy" do
3
7
  subject do
4
8
  Class.new(Elasticity::Document) do
@@ -13,7 +17,7 @@ RSpec.describe "Persistence", elasticsearch: true do
13
17
 
14
18
  c.mapping = {
15
19
  properties: {
16
- name: { type: "string" },
20
+ name: { type: "string", index: "not_analyzed" },
17
21
  birthdate: { type: "date" },
18
22
  },
19
23
  }
@@ -50,23 +54,20 @@ RSpec.describe "Persistence", elasticsearch: true do
50
54
 
51
55
  subject.flush_index
52
56
 
53
- results = subject.search(sort: :name)
57
+ results = subject.search({})
54
58
  expect(results.total).to eq 2
55
59
 
56
- expect(results[0]).to eq(john)
57
- expect(results[1]).to eq(mari)
58
-
59
- expect(subject.search({query: {filtered: { query: { match_all: {} } } } }).count).to eq(2)
60
+ expect(subject.search({ query: { match_all: {} } }).count).to eq(2)
60
61
 
61
62
  john.update
62
63
  mari.delete
63
64
 
64
65
  subject.flush_index
65
66
 
66
- results = subject.search(sort: :name)
67
+ results = subject.search({})
67
68
  expect(results.total).to eq 1
68
69
 
69
- expect(results[0]).to eq(john)
70
+ expect(results[0].name).to eq(john.name)
70
71
  end
71
72
  end
72
73
 
@@ -85,7 +86,7 @@ RSpec.describe "Persistence", elasticsearch: true do
85
86
  c.strategy = Elasticity::Strategies::SingleIndex
86
87
  c.document_type = "cat"
87
88
  c.mapping = { properties: {
88
- name: { type: "string" },
89
+ name: { type: "string", index: "not_analyzed" },
89
90
  age: { type: "integer" }
90
91
  } }
91
92
  end
@@ -103,7 +104,7 @@ RSpec.describe "Persistence", elasticsearch: true do
103
104
  c.strategy = Elasticity::Strategies::SingleIndex
104
105
  c.document_type = "dog"
105
106
  c.mapping = { properties: {
106
- name: { type: "string" },
107
+ name: { type: "string", index: "not_analyzed" },
107
108
  age: { type: "integer" },
108
109
  hungry: { type: "boolean" }
109
110
  } }
@@ -163,10 +164,9 @@ RSpec.describe "Persistence", elasticsearch: true do
163
164
  c.strategy = Elasticity::Strategies::AliasIndex
164
165
 
165
166
  c.mapping = {
166
- _id: { path: "id" },
167
167
  properties: {
168
168
  id: { type: "integer" },
169
- name: { type: "string" },
169
+ name: { type: "string", index: "not_analyzed" },
170
170
  birthdate: { type: "date" },
171
171
  },
172
172
  }
@@ -194,15 +194,14 @@ RSpec.describe "Persistence", elasticsearch: true do
194
194
  end
195
195
 
196
196
  it "remaps to a different index transparently" do
197
- john = subject.new(id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
198
- mari = subject.new(id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
197
+ john = subject.new(_id: 1, id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
198
+ mari = subject.new(_id: 2, id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
199
199
 
200
200
  john.update
201
201
  mari.update
202
202
 
203
203
  subject.flush_index
204
-
205
- results = subject.search(sort: :name)
204
+ results = subject.search({})
206
205
  expect(results.total).to eq 2
207
206
 
208
207
  subject.remap!
@@ -212,16 +211,16 @@ RSpec.describe "Persistence", elasticsearch: true do
212
211
 
213
212
  subject.flush_index
214
213
 
215
- results = subject.search(sort: :name)
214
+ results = subject.search({})
216
215
  expect(results.total).to eq 1
217
216
 
218
- expect(results[0]).to eq(john)
217
+ expect(results[0].name).to eq(john.name)
219
218
  end
220
219
 
221
220
  it "handles in between state while remapping" do
222
221
  number_of_docs = 2000
223
222
  docs = number_of_docs.times.map do |i|
224
- subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
223
+ subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
225
224
  end
226
225
 
227
226
  t = Thread.new { subject.remap! }
@@ -233,20 +232,20 @@ RSpec.describe "Persistence", elasticsearch: true do
233
232
  to_delete.each(&:delete)
234
233
 
235
234
  20.times.map do |i|
236
- subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
235
+ subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
237
236
  end
238
237
 
239
238
  t.join
240
239
 
241
240
  subject.flush_index
242
- results = subject.search(sort: :name)
241
+ results = subject.search({})
243
242
  expect(results.total).to eq(2010)
244
243
  end
245
244
 
246
245
  it "recover from remap interrupts" do
247
246
  number_of_docs = 2000
248
247
  docs = number_of_docs.times.map do |i|
249
- subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
248
+ subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
250
249
  end
251
250
 
252
251
  t = Thread.new { subject.remap! }
@@ -258,20 +257,20 @@ RSpec.describe "Persistence", elasticsearch: true do
258
257
  to_delete.each(&:delete)
259
258
 
260
259
  20.times.map do |i|
261
- subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
260
+ subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
262
261
  end
263
262
 
264
263
  t.raise("Test Interrupt")
265
264
  expect { t.join }.to raise_error("Test Interrupt")
266
265
 
267
266
  subject.flush_index
268
- results = subject.search(sort: :name)
267
+ results = subject.search({})
269
268
  expect(results.total).to eq(2010)
270
269
  end
271
270
 
272
271
  it "bulk indexes, updates and delete" do
273
272
  docs = 2000.times.map do |i|
274
- subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
273
+ subject.new(_id: i, id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
275
274
  end
276
275
 
277
276
  subject.bulk_index(docs)
@@ -289,7 +288,6 @@ RSpec.describe "Persistence", elasticsearch: true do
289
288
 
290
289
  results = subject.search(from: 0, size: 3000)
291
290
  expect(results.total).to eq 2000
292
-
293
291
  expect(subject.search({ query: { match: { name: "Updated" } } } ).count).to eq(2000)
294
292
 
295
293
  subject.bulk_delete(results.documents.map(&:_id))
data/spec/rspec_config.rb CHANGED
@@ -4,6 +4,8 @@ require "oj"
4
4
  SimpleCov.start
5
5
 
6
6
  require "elasticity"
7
+ require "pry"
8
+ require "byebug"
7
9
 
8
10
  def elastic_search_client
9
11
  return @elastic_search_client if defined?(@elastic_search_client)
@@ -108,7 +108,7 @@ RSpec.describe "Search" do
108
108
  end
109
109
 
110
110
  it "searches using scan&scroll" do
111
- expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: "scan", size: 100, scroll: "1m").and_return(scan_response)
111
+ expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: :query_then_fetch, size: 100, scroll: "1m").and_return(scan_response)
112
112
  expect(client).to receive(:scroll).with(scroll_id: "abc123", scroll: "1m").and_return(scroll_response)
113
113
  expect(client).to receive(:scroll).with(scroll_id: "abc456", scroll: "1m").and_return(empty_response)
114
114
 
@@ -8,7 +8,7 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
8
8
  "mappings" => {
9
9
  "document" => {
10
10
  "properties" => {
11
- "name" => { "type" => "string" }
11
+ "name" => { "type" => "text" }
12
12
  }
13
13
  }
14
14
  }
@@ -54,14 +54,14 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
54
54
  results_a = subject.bulk do |b|
55
55
  b.index "document", 1, name: "foo"
56
56
  end
57
- expect(results_a).to include("errors"=>false, "items"=>[{"index"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"1", "_version"=>1, "status"=>201}}])
57
+ expect(results_a["errors"]).to be_falsey
58
58
 
59
59
  results_b = subject.bulk do |b|
60
60
  b.index "document", 2, name: "bar"
61
61
  b.delete "document", 1
62
62
  end
63
63
 
64
- expect(results_b).to include("errors"=>false, "items"=>[{"index"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"2", "_version"=>1, "status"=>201}}, {"delete"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"1", "_version"=>2, "status"=>200, "found"=>true}}])
64
+ expect(results_b["errors"]).to be_falsey
65
65
 
66
66
  subject.flush
67
67
 
@@ -69,11 +69,12 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
69
69
  expect(subject.get_document("document", 2)).to eq({"_index"=>"test_index_name", "_type"=>"document", "_id"=>"2", "_version"=>1, "found"=>true, "_source"=>{"name"=>"bar"}})
70
70
  end
71
71
 
72
- it "allows deleting by queryu" do
72
+ it "allows deleting by query" do
73
73
  subject.index_document("document", 1, name: "foo")
74
74
  subject.index_document("document", 2, name: "bar")
75
75
 
76
- subject.delete_by_query("document", query: { term: { name: "foo" }})
76
+ subject.flush
77
+ subject.delete_by_query("document", query: { term: { name: "foo" } })
77
78
 
78
79
  expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
79
80
  expect { subject.get_document("document", 2) }.to_not raise_error
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: es-elasticity
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.8.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rodrigo Kochenburger
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-02-22 00:00:00.000000000 Z
11
+ date: 2017-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: byebug
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: codeclimate-test-reporter
99
113
  requirement: !ruby/object:Gem::Requirement
@@ -166,16 +180,16 @@ dependencies:
166
180
  name: elasticsearch
167
181
  requirement: !ruby/object:Gem::Requirement
168
182
  requirements:
169
- - - "~>"
183
+ - - ">="
170
184
  - !ruby/object:Gem::Version
171
- version: 1.0.12
185
+ version: '1.0'
172
186
  type: :runtime
173
187
  prerelease: false
174
188
  version_requirements: !ruby/object:Gem::Requirement
175
189
  requirements:
176
- - - "~>"
190
+ - - ">="
177
191
  - !ruby/object:Gem::Version
178
- version: 1.0.12
192
+ version: '1.0'
179
193
  description: Elasticity provides a higher level abstraction on top of [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby)
180
194
  gem
181
195
  email:
@@ -208,6 +222,7 @@ files:
208
222
  - lib/elasticity/log_subscriber.rb
209
223
  - lib/elasticity/multi_search.rb
210
224
  - lib/elasticity/railtie.rb
225
+ - lib/elasticity/scrollable_search.rb
211
226
  - lib/elasticity/search.rb
212
227
  - lib/elasticity/segmented_document.rb
213
228
  - lib/elasticity/strategies.rb
@@ -243,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
243
258
  version: '0'
244
259
  requirements: []
245
260
  rubyforge_project:
246
- rubygems_version: 2.2.3
261
+ rubygems_version: 2.4.5
247
262
  signing_key:
248
263
  specification_version: 4
249
264
  summary: ActiveModel-based library for working with Elasticsearch
@@ -257,4 +272,3 @@ test_files:
257
272
  - spec/units/multi_search_spec.rb
258
273
  - spec/units/search_spec.rb
259
274
  - spec/units/strategies/single_index_spec.rb
260
- has_rdoc: