es-elasticity 0.2.11 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,97 @@
1
+ module Elasticity
2
+ module Strategies
3
+ class SingleIndex
4
+ STATUSES = [:missing, :ok]
5
+
6
+ def initialize(client, index_name)
7
+ @client = client
8
+ @index_name = index_name
9
+ end
10
+
11
+ def remap!
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def missing?
16
+ not @client.index_exists(index: @index_name)
17
+ end
18
+
19
+ def create(index_def)
20
+ if missing?
21
+ @client.index_create(index: @index_name, body: index_def)
22
+ else
23
+ raise IndexError.new(@index_name, "index already exist")
24
+ end
25
+ end
26
+
27
+ def create_if_undefined(index_def)
28
+ create(index_def) if missing?
29
+ end
30
+
31
+ def delete
32
+ @client.index_delete(index: @index_name)
33
+ end
34
+
35
+ def delete_if_defined
36
+ delete unless missing?
37
+ end
38
+
39
+ def recreate(index_def)
40
+ delete_if_defined
41
+ create(index_def)
42
+ end
43
+
44
+ def index_document(type, id, attributes)
45
+ res = @client.index(index: @index_name, type: type, id: id, body: attributes)
46
+
47
+ if id = res["_id"]
48
+ [id, res["created"]]
49
+ else
50
+ raise IndexError.new(@update_alias, "failed to index document")
51
+ end
52
+ end
53
+
54
+ def delete_document(type, id)
55
+ @client.delete(index: @index_name, type: type, id: id)
56
+ end
57
+
58
+ def get_document(type, id)
59
+ @client.get(index: @index_name, type: type, id: id)
60
+ end
61
+
62
+ def search(type, body)
63
+ Search::Facade.new(@client, Search::Definition.new(@index_name, type, body))
64
+ end
65
+
66
+ def delete_by_query(type, body)
67
+ @client.delete_by_query(index: @index_name, type: type, body: body)
68
+ end
69
+
70
+ def bulk
71
+ b = Bulk::Index.new(@client, @index_name)
72
+ yield b
73
+ b.execute
74
+ end
75
+
76
+ def settings
77
+ args = { index: @index_name }
78
+ settings = @client.index_get_settings(index: @index_name)
79
+ settings[@index_name]["settings"]
80
+ rescue Elasticsearch::Transport::Transport::Errors::NotFound
81
+ nil
82
+ end
83
+
84
+ def mappings
85
+ args = { index: @index_name }
86
+ mapping = @client.index_get_mapping(index: @index_name)
87
+ mapping[@index_name]["mappings"]
88
+ rescue Elasticsearch::Transport::Transport::Errors::NotFound
89
+ nil
90
+ end
91
+
92
+ def flush
93
+ @client.index_flush(index: @index_name)
94
+ end
95
+ end
96
+ end
97
+ end
@@ -1,3 +1,3 @@
1
1
  module Elasticity
2
- VERSION = "0.2.11"
2
+ VERSION = "0.3.0"
3
3
  end
@@ -0,0 +1,167 @@
1
+ RSpec.describe "Persistence", elasticsearch: true do
2
+ describe "single index strategy" do
3
+ subject do
4
+ Class.new(Elasticity::Document) do
5
+ configure do |c|
6
+ c.index_base_name = "users"
7
+ c.document_type = "user"
8
+ c.strategy = Elasticity::Strategies::SingleIndex
9
+
10
+ c.mapping = {
11
+ properties: {
12
+ name: { type: "string" },
13
+ birthdate: { type: "date" },
14
+ },
15
+ }
16
+ end
17
+
18
+ attr_accessor :name, :birthdate
19
+
20
+ def to_document
21
+ { name: name, birthdate: birthdate }
22
+ end
23
+ end
24
+ end
25
+
26
+ before do
27
+ subject.recreate_index
28
+ end
29
+
30
+ after do
31
+ subject.delete_index
32
+ end
33
+
34
+ it "successfully index, update, search and delete" do
35
+ john = subject.new(name: "John", birthdate: "1985-10-31")
36
+ mari = subject.new(name: "Mari", birthdate: "1986-09-24")
37
+
38
+ john.update
39
+ mari.update
40
+
41
+ subject.flush_index
42
+
43
+ results = subject.search(sort: :name)
44
+ expect(results.total).to eq 2
45
+
46
+ expect(results[0]).to eq(john)
47
+ expect(results[1]).to eq(mari)
48
+
49
+ john.update
50
+ mari.delete
51
+
52
+ subject.flush_index
53
+
54
+ results = subject.search(sort: :name)
55
+ expect(results.total).to eq 1
56
+
57
+ expect(results[0]).to eq(john)
58
+ end
59
+ end
60
+
61
+ describe "alias index strategy" do
62
+ subject do
63
+ Class.new(Elasticity::Document) do
64
+ configure do |c|
65
+ c.index_base_name = "users"
66
+ c.document_type = "user"
67
+ c.strategy = Elasticity::Strategies::AliasIndex
68
+
69
+ c.mapping = {
70
+ properties: {
71
+ name: { type: "string" },
72
+ birthdate: { type: "date" },
73
+ },
74
+ }
75
+ end
76
+
77
+ attr_accessor :name, :birthdate
78
+
79
+ def to_document
80
+ { name: name, birthdate: birthdate }
81
+ end
82
+ end
83
+ end
84
+
85
+ before do
86
+ subject.recreate_index
87
+ end
88
+
89
+ after do
90
+ subject.delete_index
91
+ end
92
+
93
+ it "remaps to a different index transparently" do
94
+ john = subject.new(name: "John", birthdate: "1985-10-31")
95
+ mari = subject.new(name: "Mari", birthdate: "1986-09-24")
96
+
97
+ john.update
98
+ mari.update
99
+
100
+ subject.flush_index
101
+
102
+ results = subject.search(sort: :name)
103
+ expect(results.total).to eq 2
104
+
105
+ subject.remap!
106
+
107
+ john.update
108
+ mari.delete
109
+
110
+ subject.flush_index
111
+
112
+ results = subject.search(sort: :name)
113
+ expect(results.total).to eq 1
114
+
115
+ expect(results[0]).to eq(john)
116
+ end
117
+
118
+ it "handles in between state while remapping" do
119
+ docs = 2000.times.map do |i|
120
+ subject.new(name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
121
+ end
122
+
123
+ t = Thread.new { subject.remap! }
124
+
125
+ to_update = docs.sample(10)
126
+ to_delete = (docs - to_update).sample(10)
127
+
128
+ to_update.each(&:update)
129
+ to_delete.each(&:delete)
130
+
131
+ 20.times.map do |i|
132
+ subject.new(name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
133
+ end
134
+
135
+ t.join
136
+
137
+ subject.flush_index
138
+ results = subject.search(sort: :name)
139
+ expect(results.total).to eq(2010)
140
+ end
141
+
142
+ it "recover from remap interrupts" do
143
+ docs = 2000.times.map do |i|
144
+ subject.new(name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
145
+ end
146
+
147
+ t = Thread.new { subject.remap! }
148
+
149
+ to_update = docs.sample(10)
150
+ to_delete = (docs - to_update).sample(10)
151
+
152
+ to_update.each(&:update)
153
+ to_delete.each(&:delete)
154
+
155
+ 20.times.map do |i|
156
+ subject.new(name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
157
+ end
158
+
159
+ t.raise("Test Interrupt")
160
+ expect { t.join }.to raise_error("Test Interrupt")
161
+
162
+ subject.flush_index
163
+ results = subject.search(sort: :name)
164
+ expect(results.total).to eq(2010)
165
+ end
166
+ end
167
+ end
data/spec/rspec_config.rb CHANGED
@@ -1,4 +1,3 @@
1
- require "elasticity_base"
2
1
  require "codeclimate-test-reporter"
3
2
  require "simplecov"
4
3
  require "oj"
@@ -12,17 +11,21 @@ def elastic_search_client
12
11
  @elastic_search_client = Elasticsearch::Client.new host: "http://0.0.0.0:9200"
13
12
  end
14
13
 
15
- logger = Logger.new("spec/spec.log", File::WRONLY | File::APPEND | File::CREAT)
14
+ logger = Logger.new("spec/spec.log")
15
+ logger.level = Logger::DEBUG
16
+
17
+ ActiveSupport::LogSubscriber.logger = logger
18
+ Elasticity::LogSubscriber.attach_to(:elasticity)
16
19
 
17
20
  RSpec.configure do |c|
18
21
  c.disable_monkey_patching!
19
22
 
20
23
  c.before(:suite) do
21
- logger.info "rspec.init Starting test suite execution"
24
+ logger.info "init.rspec Starting test suite execution"
22
25
  end
23
26
 
24
27
  c.before(:each) do |example|
25
- logger.info "rspec.spec #{example.full_description}"
28
+ logger.info "spec.rspec #{example.full_description}"
26
29
 
27
30
  if example.metadata[:elasticsearch]
28
31
  client = elastic_search_client
@@ -31,7 +34,6 @@ RSpec.configure do |c|
31
34
  end
32
35
 
33
36
  Elasticity.configure do |e|
34
- e.logger = logger
35
37
  e.client = client
36
38
  e.namespace = "elasticity_test"
37
39
  end
@@ -16,13 +16,12 @@ RSpec.describe Elasticity::Document do
16
16
 
17
17
  let :klass do
18
18
  Class.new(described_class) do
19
- # Override the name since this is an anonymous class
20
- def self.name
21
- "ClassName"
19
+ configure do |c|
20
+ c.index_base_name = "class_names"
21
+ c.document_type = "class_name"
22
+ c.mapping = mappings
22
23
  end
23
24
 
24
- self.mappings = mappings
25
-
26
25
  attr_accessor :name, :items
27
26
 
28
27
  def to_document
@@ -31,12 +30,12 @@ RSpec.describe Elasticity::Document do
31
30
  end
32
31
  end
33
32
 
34
- let :index do
35
- double(:index, create_if_undefined: nil, name: "elasticity_test_class_names")
33
+ let :strategy do
34
+ double(:strategy)
36
35
  end
37
36
 
38
37
  before :each do
39
- allow(Elasticity::Index).to receive(:new).and_return(index)
38
+ allow(Elasticity::Strategies::SingleIndex).to receive(:new).and_return(strategy)
40
39
  end
41
40
 
42
41
  it "requires subclasses to define to_document method" do
@@ -46,53 +45,45 @@ RSpec.describe Elasticity::Document do
46
45
  context "class" do
47
46
  subject { klass }
48
47
 
49
- it "extracts index name and document type from the class name" do
50
- expect(subject.namespaced_index_name).to eq "elasticity_test_class_names"
51
- expect(subject.document_type).to eq "class_name"
52
- expect(subject.index.name).to eq "elasticity_test_class_names"
53
- end
54
-
55
- it "have an associated Index instance" do
56
- client = double(:client)
57
- settings = double(:settings)
58
-
59
- Elasticity.config.settings = settings
60
- Elasticity.config.client = client
61
-
62
- expect(Elasticity::Index).to receive(:new).with(client, "elasticity_test_class_names").and_return(index)
63
-
64
- expect(subject.index).to be index
48
+ it "properly instantiate from search hit" do
49
+ hit = { "_id" => 1, "_source" => { "name" => "foo", "items" => [{ name: "bar" }] }, "highlight" => { "name" => "<em>foo</em>" } }
50
+ doc = subject.from_hit(hit)
51
+ expect(doc.name).to eq "foo"
52
+ expect(doc.items).to eq [{ name: "bar" }]
53
+ expect(doc.highlighted.name).to eq "<em>foo</em>"
54
+ expect(doc.highlighted.items).to eq [{ name: "bar" }]
65
55
  end
66
56
 
67
57
  it "searches using DocumentSearch" do
68
58
  body = double(:body)
69
59
  search = double(:search)
70
- expect(Elasticity::Search).to receive(:new).with(index, "class_name", body).and_return(search)
60
+
61
+ expect(strategy).to receive(:search).with("class_name", body).and_return(search)
71
62
 
72
63
  doc_search = double(:doc_search)
73
- expect(Elasticity::DocumentSearchProxy).to receive(:new).with(search, subject).and_return(doc_search)
64
+ expect(Elasticity::Search::DocumentProxy).to receive(:new).with(search, subject).and_return(doc_search)
74
65
 
75
66
  expect(subject.search(body)).to be doc_search
76
67
  end
77
68
 
78
- it "gets specific document from the index" do
69
+ it "gets specific document from the strategy" do
79
70
  doc = { "_id" => 1, "_source" => { "name" => "Foo", "items" => [{ "name" => "Item1" }]}}
80
- expect(index).to receive(:get_document).with("class_name", 1).and_return(doc)
71
+ expect(strategy).to receive(:get_document).with("class_name", 1).and_return(doc)
81
72
  expect(subject.get(1)).to eq klass.new(_id: 1, name: "Foo", items: [{ "name" => "Item1" }])
82
73
  end
83
74
 
84
- it "deletes specific document from index" do
85
- index_ret = double(:index_return)
86
- expect(index).to receive(:delete_document).with("class_name", 1).and_return(index_ret)
87
- expect(subject.delete(1)).to eq index_ret
75
+ it "deletes specific document from strategy" do
76
+ strategy_ret = double(:strategy_return)
77
+ expect(strategy).to receive(:delete_document).with("class_name", 1).and_return(strategy_ret)
78
+ expect(subject.delete(1)).to eq strategy_ret
88
79
  end
89
80
  end
90
81
 
91
82
  context "instance" do
92
83
  subject { klass.new _id: 1, name: "Foo", items: [{ name: "Item1" }] }
93
84
 
94
- it "stores the document in the index" do
95
- expect(index).to receive(:index_document).with("class_name", 1, { name: "Foo", items: [{ name: "Item1" }] })
85
+ it "stores the document in the strategy" do
86
+ expect(strategy).to receive(:index_document).with("class_name", 1, { name: "Foo", items: [{ name: "Item1" }] }).and_return("_id" => "1", "created" => true)
96
87
  subject.update
97
88
  end
98
89
  end
@@ -2,10 +2,18 @@ require "elasticity/search"
2
2
  require "elasticity/multi_search"
3
3
 
4
4
  RSpec.describe Elasticity::MultiSearch do
5
+ let :client do
6
+ double(:client)
7
+ end
8
+
5
9
  let :klass do
6
10
  Class.new do
7
11
  include ActiveModel::Model
8
- attr_accessor :_id, :name, :highlighted
12
+ attr_accessor :_id, :name
13
+
14
+ def self.from_hit(hit)
15
+ new(_id: hit["_id"], name: hit["_source"]["name"])
16
+ end
9
17
 
10
18
  def ==(other)
11
19
  self._id == other._id && self.name == other.name
@@ -23,8 +31,8 @@ RSpec.describe Elasticity::MultiSearch do
23
31
  end
24
32
 
25
33
  it "performs multi search" do
26
- subject.add(:first, Elasticity::Search.new(double(:index, name: "index_first"), "document_first", { search: :first }), documents: klass)
27
- subject.add(:second, Elasticity::Search.new(double(:index, name: "index_second"), "document_second", { search: :second }), documents: klass)
34
+ subject.add(:first, Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new("index_first", "document_first", { search: :first })), documents: klass)
35
+ subject.add(:second, Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new("index_second", "document_second", { search: :second })), documents: klass)
28
36
 
29
37
  expect(Elasticity.config.client).to receive(:msearch).with(body: [
30
38
  { index: "index_first", type: "document_first", search: { search: :first } },
@@ -1,7 +1,8 @@
1
1
  require "elasticity/search"
2
2
 
3
3
  RSpec.describe "Search" do
4
- let(:index) { double(:index) }
4
+ let(:client) { double(:client) }
5
+ let(:index_name) { "index_name" }
5
6
  let(:document_type) { "document" }
6
7
  let(:body) { {} }
7
8
 
@@ -23,16 +24,25 @@ RSpec.describe "Search" do
23
24
  { "hits" => { "total" => 0, "hits" => [] }}
24
25
  end
25
26
 
26
- let :highlight_response do
27
- { "hits" => { "total" => 1, "hits" => [
28
- { "_id" => 1, "_source" => { "name" => "foo", "age" => 21 }, "highlight" => { "name" => "<em>foo</em>" } },
27
+ let :scan_response do
28
+ { "_scroll_id" => "abc123", "hits" => { "total" => 2 } }
29
+ end
30
+
31
+ let :scroll_response do
32
+ { "_scroll_id" => "abc456", "hits" => { "total" => 2, "hits" => [
33
+ { "_id" => 1, "_source" => { "name" => "foo" } },
34
+ { "_id" => 2, "_source" => { "name" => "bar" } },
29
35
  ]}}
30
36
  end
31
37
 
32
38
  let :klass do
33
39
  Class.new do
34
40
  include ActiveModel::Model
35
- attr_accessor :_id, :name, :age, :highlighted
41
+ attr_accessor :_id, :name, :age
42
+
43
+ def self.from_hit(hit)
44
+ new(_id: hit["_id"], name: hit["_source"]["name"], age: hit["_source"]["age"])
45
+ end
36
46
 
37
47
  def ==(other)
38
48
  self._id == other._id && self.name == other.name
@@ -40,13 +50,13 @@ RSpec.describe "Search" do
40
50
  end
41
51
  end
42
52
 
43
- describe Elasticity::Search do
53
+ describe Elasticity::Search::Facade do
44
54
  subject do
45
- described_class.new(index, document_type, body)
55
+ described_class.new(client, Elasticity::Search::Definition.new(index_name, document_type, body))
46
56
  end
47
57
 
48
58
  it "searches the index and return document models" do
49
- expect(index).to receive(:search).with(document_type, body).and_return(full_response)
59
+ expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response)
50
60
 
51
61
  docs = subject.documents(klass)
52
62
  expected = [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
@@ -64,43 +74,43 @@ RSpec.describe "Search" do
64
74
  expect(Array(docs)).to eq expected
65
75
  end
66
76
 
77
+ it "searches using scan&scroll" do
78
+ expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: "scan", size: 100, scroll: "1m").and_return(scan_response)
79
+ expect(client).to receive(:scroll).with(scroll_id: "abc123", scroll: "1m").and_return(scroll_response)
80
+ expect(client).to receive(:scroll).with(scroll_id: "abc456", scroll: "1m").and_return(empty_response)
81
+
82
+ docs = subject.scan_documents(klass)
83
+ expected = [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
84
+
85
+ expect(docs.total).to eq 2
86
+
87
+ expect(docs).to_not be_empty
88
+ expect(docs).to_not be_blank
89
+
90
+ expect(Array(docs)).to eq expected
91
+ end
92
+
67
93
  it "searches the index and return active record models" do
68
- expect(index).to receive(:search).with(document_type, body.merge(_source: false)).and_return(ids_response)
94
+ expect(client).to receive(:search).with(index: index_name, type: document_type, body: body.merge(_source: false)).and_return(ids_response)
69
95
 
70
96
  relation = double(:relation,
71
97
  connection: double(:connection),
72
98
  table_name: "table_name",
73
99
  klass: double(:klass, primary_key: "id"),
100
+ to_sql: "SELECT * FROM table_name WHERE id IN (1)"
74
101
  )
75
102
  allow(relation.connection).to receive(:quote_column_name) { |name| name }
76
103
 
77
- expect(relation).to receive(:where).with(id: [1,2]).and_return(relation)
104
+ expect(relation).to receive(:where).with("table_name.id IN (?)", [1, 2]).and_return(relation)
78
105
  expect(relation).to receive(:order).with("FIELD(table_name.id,1,2)").and_return(relation)
79
106
 
80
- expect(subject.active_records(relation).mapping).to be relation
81
- end
82
-
83
- it "return relation.none from activerecord relation with no matches" do
84
- expect(index).to receive(:search).with(document_type, body.merge(_source: false)).and_return(empty_response)
85
-
86
- relation = double(:relation)
87
- expect(relation).to receive(:none).and_return(relation)
88
-
89
- expect(subject.active_records(relation).mapping).to be relation
90
- end
91
-
92
- it "creates highlighted object for documents" do
93
- expect(index).to receive(:search).with(document_type, body).and_return(highlight_response)
94
- doc = subject.documents(klass).first
95
-
96
- expect(doc).to_not be_nil
97
- expect(doc.highlighted).to eq klass.new(_id: 1, name: "<em>foo</em>", age: 21)
107
+ expect(subject.active_records(relation).to_sql).to eq "SELECT * FROM table_name WHERE id IN (1)"
98
108
  end
99
109
  end
100
110
 
101
- describe Elasticity::DocumentSearchProxy do
111
+ describe Elasticity::Search::DocumentProxy do
102
112
  let :search do
103
- Elasticity::Search.new(index, "document", body)
113
+ Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new(index_name, "document", body))
104
114
  end
105
115
 
106
116
  subject do
@@ -108,7 +118,7 @@ RSpec.describe "Search" do
108
118
  end
109
119
 
110
120
  it "automatically maps the documents into the provided Document class" do
111
- expect(index).to receive(:search).with(document_type, body).and_return(full_response)
121
+ expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response)
112
122
  expect(Array(subject)).to eq [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
113
123
  end
114
124