elasticsearch-documents 0.1.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +5 -0
- data/elasticsearch-documents.gemspec +1 -0
- data/lib/elasticsearch/extensions/documents.rb +11 -1
- data/lib/elasticsearch/extensions/documents/aliased_index_store.rb +112 -0
- data/lib/elasticsearch/extensions/documents/direct_index_store.rb +65 -0
- data/lib/elasticsearch/extensions/documents/index.rb +15 -9
- data/lib/elasticsearch/extensions/documents/storage.rb +36 -0
- data/lib/elasticsearch/extensions/documents/version.rb +1 -1
- data/spec/elasticsearch/extensions/documents/aliased_index_store_spec.rb +158 -0
- data/spec/elasticsearch/extensions/documents/direct_index_store_spec.rb +78 -0
- data/spec/elasticsearch/extensions/documents/index_spec.rb +10 -15
- data/spec/elasticsearch/extensions/documents/storage_spec.rb +58 -0
- data/spec/test_env.rb +19 -0
- metadata +43 -21
- data/lib/elasticsearch/extensions/documents/indexer.rb +0 -63
- data/spec/elasticsearch/extensions/documents/indexer_spec.rb +0 -122
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 28646ee3ed70dd2e2522d287d9ce3e1589c60404
|
4
|
+
data.tar.gz: c1ddaf4c1eeed321de410083afb1bbea21797cf1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e520e5c94ef8bbd9b75ca79680a4356b8f6aaead55a2f953062fe9beb4d37a289b0c2623fcfb96706db7e575b02ad94f82252f688883428b72a08e9bc430456e
|
7
|
+
data.tar.gz: f18175a7a63a6b4f8a9d14e5efe300370eef15f525de8d0a5c160f3b586196294b9097c9c4a209af4596e8d6349376dbc91fbb7e5a89aca5523e1ef4c262c317
|
data/Rakefile
CHANGED
@@ -4,3 +4,8 @@ require "rspec/core/rake_task"
|
|
4
4
|
RSpec::Core::RakeTask.new(:spec)
|
5
5
|
task default: :spec
|
6
6
|
|
7
|
+
desc "Opens a pry session configured to use a local elasticsearch server"
|
8
|
+
task :console do
|
9
|
+
sh "bundle exec pry -I lib -I spec -r elasticsearch-documents -r test_env"
|
10
|
+
end
|
11
|
+
|
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency "bundler", "~> 1.3"
|
23
23
|
spec.add_development_dependency "rake"
|
24
24
|
spec.add_development_dependency "rspec", "~> 2.14"
|
25
|
+
spec.add_development_dependency "pry"
|
25
26
|
|
26
27
|
spec.add_runtime_dependency "elasticsearch"
|
27
28
|
spec.add_runtime_dependency "hashie"
|
@@ -4,8 +4,10 @@ require "ostruct"
|
|
4
4
|
require "elasticsearch/extensions/documents/version"
|
5
5
|
require "elasticsearch/extensions/documents/document"
|
6
6
|
require "elasticsearch/extensions/documents/index"
|
7
|
-
require "elasticsearch/extensions/documents/
|
7
|
+
require "elasticsearch/extensions/documents/aliased_index_store"
|
8
|
+
require "elasticsearch/extensions/documents/direct_index_store"
|
8
9
|
require "elasticsearch/extensions/documents/queryable"
|
10
|
+
require "elasticsearch/extensions/documents/storage"
|
9
11
|
require "elasticsearch/extensions/documents/utils"
|
10
12
|
|
11
13
|
module Elasticsearch
|
@@ -31,6 +33,14 @@ module Elasticsearch
|
|
31
33
|
def logger
|
32
34
|
self.configuration.client.logger ||= Logger.new(STDERR)
|
33
35
|
end
|
36
|
+
|
37
|
+
def index_adapter
|
38
|
+
case self.configuration.index_adapter
|
39
|
+
when :direct then DirectIndexStore.new
|
40
|
+
when :aliased then AliasedIndexStore.new
|
41
|
+
else DirectIndexStore.new end
|
42
|
+
end
|
43
|
+
|
34
44
|
end
|
35
45
|
|
36
46
|
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
module Elasticsearch
|
2
|
+
module Extensions
|
3
|
+
module Documents
|
4
|
+
class AliasedIndexStore
|
5
|
+
|
6
|
+
attr_reader :client, :storage, :read_alias, :write_alias
|
7
|
+
|
8
|
+
def initialize(options = {})
|
9
|
+
@client = options.fetch(:client) { Documents.client }
|
10
|
+
@storage = options.fetch(:storage) { Storage.new }
|
11
|
+
@write_alias = Documents.index_name + "_write"
|
12
|
+
@read_alias = Documents.index_name + "_read"
|
13
|
+
end
|
14
|
+
|
15
|
+
def index(payload)
|
16
|
+
client.index payload.merge(index: write_alias)
|
17
|
+
end
|
18
|
+
|
19
|
+
def delete(payload)
|
20
|
+
client.delete payload.merge(index: write_alias)
|
21
|
+
end
|
22
|
+
|
23
|
+
def search(payload)
|
24
|
+
client.search payload.merge(index: read_alias)
|
25
|
+
end
|
26
|
+
|
27
|
+
def refresh
|
28
|
+
client.indices.refresh index: read_alias
|
29
|
+
end
|
30
|
+
|
31
|
+
def reindex(options = {}, &block)
|
32
|
+
timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
|
33
|
+
new_index_name = Documents.index_name + "_#{timestamp}"
|
34
|
+
current_index_name = indices_for_alias(write_alias).first
|
35
|
+
|
36
|
+
storage.create_index new_index_name
|
37
|
+
swap_index_alias(alias: write_alias, old: current_index_name, new: new_index_name)
|
38
|
+
|
39
|
+
block.call(self) if block_given?
|
40
|
+
|
41
|
+
swap_index_alias(alias: read_alias, old: current_index_name, new: new_index_name)
|
42
|
+
storage.drop_index current_index_name
|
43
|
+
end
|
44
|
+
|
45
|
+
def indices_for_alias(alias_name)
|
46
|
+
client.indices.get_alias(name: alias_name).keys
|
47
|
+
end
|
48
|
+
|
49
|
+
def setup
|
50
|
+
reset_aliases
|
51
|
+
|
52
|
+
timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
|
53
|
+
new_index_name = Documents.index_name + "_#{timestamp}"
|
54
|
+
|
55
|
+
storage.create_index new_index_name
|
56
|
+
client.indices.put_alias index: new_index_name, name: read_alias
|
57
|
+
client.indices.put_alias index: new_index_name, name: write_alias
|
58
|
+
end
|
59
|
+
|
60
|
+
def swap_index_alias(options)
|
61
|
+
change_alias = options.fetch(:alias)
|
62
|
+
new_index = options.fetch(:new)
|
63
|
+
old_index = options.fetch(:old)
|
64
|
+
|
65
|
+
client.indices.update_aliases body: {
|
66
|
+
actions: [
|
67
|
+
{ remove: { index: old_index, alias: change_alias } },
|
68
|
+
{ add: { index: new_index, alias: change_alias } },
|
69
|
+
]
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
def reset_aliases
|
74
|
+
if client.indices.exists_alias(name: write_alias)
|
75
|
+
indices_for_alias(write_alias).each do |index|
|
76
|
+
client.indices.delete_alias index: index, name: write_alias
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
if client.indices.exists_alias(name: read_alias)
|
81
|
+
indices_for_alias(read_alias).each do |index|
|
82
|
+
client.indices.delete_alias index: index, name: read_alias
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def bulk_index(documents)
|
88
|
+
client.bulk body: bulk_index_operations(documents)
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def bulk_index_operations(documents)
|
94
|
+
documents.collect { |document| bulk_index_operation_hash(document) }
|
95
|
+
end
|
96
|
+
|
97
|
+
def bulk_index_operation_hash(document)
|
98
|
+
{
|
99
|
+
index: {
|
100
|
+
_index: write_alias,
|
101
|
+
_type: document.class.type,
|
102
|
+
_id: document.id,
|
103
|
+
data: document.as_hash,
|
104
|
+
}
|
105
|
+
}
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Elasticsearch
|
2
|
+
module Extensions
|
3
|
+
module Documents
|
4
|
+
class DirectIndexStore
|
5
|
+
attr_reader :client, :storage
|
6
|
+
|
7
|
+
def initialize(options = {})
|
8
|
+
@client = options.fetch(:client) { Documents.client }
|
9
|
+
@storage = options.fetch(:storage) { Storage.new }
|
10
|
+
end
|
11
|
+
|
12
|
+
def index(payload)
|
13
|
+
client.index payload.merge(index: Documents.index_name)
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete(payload)
|
17
|
+
client.delete payload.merge(index: Documents.index_name)
|
18
|
+
end
|
19
|
+
|
20
|
+
def search(payload)
|
21
|
+
client.search payload.merge(index: Documents.index_name)
|
22
|
+
end
|
23
|
+
|
24
|
+
def refresh
|
25
|
+
client.indices.refresh index: Documents.index_name
|
26
|
+
end
|
27
|
+
|
28
|
+
def reindex(options = {}, &block)
|
29
|
+
force_create = options.fetch(:force_create) { false }
|
30
|
+
|
31
|
+
storage.drop_index(Documents.index_name) if force_create
|
32
|
+
storage.create_index(Documents.index_name)
|
33
|
+
|
34
|
+
block.call(self) if block_given?
|
35
|
+
end
|
36
|
+
|
37
|
+
def setup
|
38
|
+
storage.drop_index(Documents.index_name)
|
39
|
+
storage.create_index(Documents.index_name)
|
40
|
+
end
|
41
|
+
|
42
|
+
def bulk_index(documents)
|
43
|
+
client.bulk body: bulk_index_operations(documents)
|
44
|
+
end
|
45
|
+
|
46
|
+
def bulk_index_operations(documents)
|
47
|
+
documents.collect { |document| bulk_index_operation_hash(document) }
|
48
|
+
end
|
49
|
+
|
50
|
+
def bulk_index_operation_hash(document)
|
51
|
+
{
|
52
|
+
index: {
|
53
|
+
_index: Documents.index_name,
|
54
|
+
_type: document.class.type,
|
55
|
+
_id: document.id,
|
56
|
+
data: document.as_hash,
|
57
|
+
}
|
58
|
+
}
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
@@ -4,40 +4,46 @@ module Elasticsearch
|
|
4
4
|
module Extensions
|
5
5
|
module Documents
|
6
6
|
class Index
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :adapter
|
8
8
|
|
9
|
-
def initialize(
|
10
|
-
@
|
9
|
+
def initialize(adapter = nil)
|
10
|
+
@adapter = adapter || Documents.index_adapter
|
11
11
|
end
|
12
12
|
|
13
13
|
def index(document)
|
14
14
|
payload = {
|
15
|
-
index: Documents.index_name,
|
16
15
|
type: document.class.type,
|
17
16
|
id: document.id,
|
18
17
|
body: document.as_hash,
|
19
18
|
}
|
20
|
-
|
19
|
+
adapter.index payload
|
21
20
|
end
|
22
21
|
|
23
22
|
def delete(document)
|
24
23
|
payload = {
|
25
|
-
index: Documents.index_name,
|
26
24
|
type: document.class.type,
|
27
25
|
id: document.id,
|
28
26
|
}
|
29
|
-
|
27
|
+
adapter.delete payload
|
30
28
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound => not_found
|
31
29
|
Documents.logger.info "[Documents] Attempted to delete missing document: #{not_found}"
|
32
30
|
end
|
33
31
|
|
34
32
|
def search(query)
|
35
|
-
response =
|
33
|
+
response = adapter.search(query.as_hash)
|
36
34
|
Hashie::Mash.new(response)
|
37
35
|
end
|
38
36
|
|
39
37
|
def refresh
|
40
|
-
|
38
|
+
adapter.refresh
|
39
|
+
end
|
40
|
+
|
41
|
+
def reindex(options = {}, &block)
|
42
|
+
adapter.reindex(options, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
def bulk_index(documents)
|
46
|
+
adapter.bulk_index(documents)
|
41
47
|
end
|
42
48
|
|
43
49
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Elasticsearch
|
2
|
+
module Extensions
|
3
|
+
module Documents
|
4
|
+
class Storage
|
5
|
+
|
6
|
+
attr_reader :client, :config
|
7
|
+
|
8
|
+
def initialize(options = {})
|
9
|
+
@client = options.fetch(:client) { Documents.client }
|
10
|
+
@config = options.fetch(:config) { Documents.configuration }
|
11
|
+
end
|
12
|
+
|
13
|
+
def drop_index(index_name)
|
14
|
+
client.indices.delete(index: index_name) if index_exists?(index_name)
|
15
|
+
end
|
16
|
+
|
17
|
+
def index_exists?(index_name)
|
18
|
+
client.indices.exists(index: index_name)
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_index(index_name)
|
22
|
+
client.indices.create(index: index_name, body: index_definition) unless index_exists?(index_name)
|
23
|
+
end
|
24
|
+
|
25
|
+
def index_definition
|
26
|
+
{}.tap do |body|
|
27
|
+
body[:settings] = config.settings if config.settings
|
28
|
+
body[:mappings] = config.mappings if config.mappings
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Elasticsearch
|
4
|
+
module Extensions
|
5
|
+
module Documents
|
6
|
+
describe AliasedIndexStore do
|
7
|
+
|
8
|
+
let(:client) { double(:client) }
|
9
|
+
let(:storage) { double(:storage) }
|
10
|
+
subject(:store) { described_class.new(client: client, storage: storage) }
|
11
|
+
|
12
|
+
describe "#index" do
|
13
|
+
it "tells the client to index using a write alias" do
|
14
|
+
expect(client).to receive(:index).with({ foo: :bar, index: "test_index_write" })
|
15
|
+
store.index(foo: :bar)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe "#delete" do
|
20
|
+
it "tells the client to delete a document using the write alias" do
|
21
|
+
expect(client).to receive(:delete).with({ id: 42, index: "test_index_write"})
|
22
|
+
store.delete(id: 42)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "#search" do
|
27
|
+
it "tells the client to search from the read alias" do
|
28
|
+
expect(client).to receive(:search).with({ foo: :bar, index: "test_index_read"})
|
29
|
+
store.search(foo: :bar)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "#refresh" do
|
34
|
+
it "tells the client indices to refresh the read alias" do
|
35
|
+
indices = double(:indices)
|
36
|
+
client.stub(:indices).and_return(indices)
|
37
|
+
expect(indices).to receive(:refresh).with(index: "test_index_read")
|
38
|
+
store.refresh
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe "#reindex" do
|
43
|
+
let(:indices) { double(:indices) }
|
44
|
+
before(:each) do
|
45
|
+
allow(store).to receive(:indices_for_alias).and_return(["test_index_old"])
|
46
|
+
allow(indices).to receive(:update_aliases)
|
47
|
+
allow(storage).to receive(:create_index)
|
48
|
+
allow(storage).to receive(:drop_index)
|
49
|
+
client.stub(:indices).and_return(indices)
|
50
|
+
time = double(:time, now: self, strftime: "timestamp")
|
51
|
+
Time.stub(:now).and_return(time)
|
52
|
+
end
|
53
|
+
|
54
|
+
it "creates a new index with a timestamp appended to the name" do
|
55
|
+
expect(storage).to receive(:create_index).with("test_index_timestamp")
|
56
|
+
store.reindex
|
57
|
+
end
|
58
|
+
|
59
|
+
it "points the write alias to the newly created index" do
|
60
|
+
swap_args = {
|
61
|
+
alias: "test_index_write",
|
62
|
+
old: "test_index_old",
|
63
|
+
new: "test_index_timestamp",
|
64
|
+
}
|
65
|
+
|
66
|
+
expect(store).to receive(:swap_index_alias).with(swap_args)
|
67
|
+
expect(store).to receive(:swap_index_alias)
|
68
|
+
|
69
|
+
store.reindex
|
70
|
+
end
|
71
|
+
|
72
|
+
it "points the read alias to the newly created index" do
|
73
|
+
swap_args = {
|
74
|
+
alias: "test_index_read",
|
75
|
+
old: "test_index_old",
|
76
|
+
new: "test_index_timestamp",
|
77
|
+
}
|
78
|
+
|
79
|
+
expect(store).to receive(:swap_index_alias).with(swap_args)
|
80
|
+
expect(store).to receive(:swap_index_alias)
|
81
|
+
|
82
|
+
store.reindex
|
83
|
+
end
|
84
|
+
|
85
|
+
it "calls a given block to index the documents" do
|
86
|
+
documents = double(:documents)
|
87
|
+
expect(store).to receive(:bulk_index).with(documents)
|
88
|
+
store.reindex { |store| store.bulk_index(documents) }
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe "#bulk_index" do
|
93
|
+
let(:models) { [double(:model, id: 1, a: 1, b: 2), double(:model, id: 2, a: 3, b: 4)] }
|
94
|
+
let(:documents) { models.map { |m| TestDocumentsDocument.new(m) } }
|
95
|
+
|
96
|
+
it 'passes operations to the client bulk action' do
|
97
|
+
expected_body = {
|
98
|
+
body: [
|
99
|
+
{
|
100
|
+
index: {
|
101
|
+
_index: 'test_index_write',
|
102
|
+
_type: 'documents_test',
|
103
|
+
_id: 1,
|
104
|
+
data: { a: 1, b: 2 },
|
105
|
+
}
|
106
|
+
},
|
107
|
+
{
|
108
|
+
index: {
|
109
|
+
_index: 'test_index_write',
|
110
|
+
_type: 'documents_test',
|
111
|
+
_id: 2,
|
112
|
+
data: { a: 3, b: 4 },
|
113
|
+
},
|
114
|
+
}
|
115
|
+
]
|
116
|
+
}
|
117
|
+
expect(client).to receive(:bulk).with(expected_body)
|
118
|
+
store.bulk_index(documents)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
describe "#setup" do
|
123
|
+
let(:indices) { double(:indices) }
|
124
|
+
before(:each) do
|
125
|
+
time = double(:time, now: self, strftime: "1234")
|
126
|
+
allow(Time).to receive(:now).and_return(time)
|
127
|
+
allow(client).to receive(:indices).and_return(indices)
|
128
|
+
allow(store).to receive(:indices_for_alias).and_return(["test_index_1234"])
|
129
|
+
allow(indices).to receive(:put_alias)
|
130
|
+
allow(indices).to receive(:delete_alias)
|
131
|
+
allow(indices).to receive(:exists_alias).and_return(true)
|
132
|
+
allow(storage).to receive(:create_index)
|
133
|
+
end
|
134
|
+
|
135
|
+
it "resets the aliases to start clean" do
|
136
|
+
expect(indices).to receive(:delete_alias).with(index: "test_index_1234", name: "test_index_write")
|
137
|
+
expect(indices).to receive(:delete_alias).with(index: "test_index_1234", name: "test_index_read")
|
138
|
+
store.setup
|
139
|
+
end
|
140
|
+
|
141
|
+
it "creates a new timestamped index" do
|
142
|
+
timestamped_index = "test_index_1234"
|
143
|
+
expect(storage).to receive(:create_index).with(timestamped_index)
|
144
|
+
store.setup
|
145
|
+
end
|
146
|
+
|
147
|
+
it "sets the read and write aliases to the new index" do
|
148
|
+
expect(indices).to receive(:put_alias).with(index: "test_index_1234", name: "test_index_write")
|
149
|
+
expect(indices).to receive(:put_alias).with(index: "test_index_1234", name: "test_index_read")
|
150
|
+
store.setup
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Elasticsearch
|
4
|
+
module Extensions
|
5
|
+
module Documents
|
6
|
+
describe DirectIndexStore do
|
7
|
+
|
8
|
+
let(:client) { double(:client) }
|
9
|
+
let(:storage) { double(:storage) }
|
10
|
+
subject(:store) { described_class.new(client: client, storage: storage) }
|
11
|
+
|
12
|
+
describe '#reindex' do
|
13
|
+
before(:each) do
|
14
|
+
storage.stub(:create_index)
|
15
|
+
storage.stub(:drop_index)
|
16
|
+
end
|
17
|
+
|
18
|
+
context 'with the :force_create option' do
|
19
|
+
it 'drops the index if exists' do
|
20
|
+
expect(storage).to receive(:drop_index)
|
21
|
+
store.reindex(force_create: true)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
context 'without the :force_create option' do
|
26
|
+
it 'does not drop the index if exists' do
|
27
|
+
expect(storage).not_to receive(:drop_index)
|
28
|
+
store.reindex
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'creates a new index' do
|
33
|
+
expect(storage).to receive(:create_index)
|
34
|
+
store.reindex
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'calls a given block to batch index the documents' do
|
38
|
+
documents = double(:documents)
|
39
|
+
expect(store).to receive(:bulk_index).with(documents)
|
40
|
+
store.reindex { |store| store.bulk_index(documents) }
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
describe "#bulk_index" do
|
46
|
+
let(:models) { [double(:model, id: 1, a: 1, b: 2), double(:model, id: 2, a: 3, b: 4)] }
|
47
|
+
let(:documents) { models.map { |m| TestDocumentsDocument.new(m) } }
|
48
|
+
|
49
|
+
it 'passes operations to the client bulk action' do
|
50
|
+
expected_body = {
|
51
|
+
body: [
|
52
|
+
{
|
53
|
+
index: {
|
54
|
+
_index: 'test_index',
|
55
|
+
_type: 'documents_test',
|
56
|
+
_id: 1,
|
57
|
+
data: { a: 1, b: 2 },
|
58
|
+
}
|
59
|
+
},
|
60
|
+
{
|
61
|
+
index: {
|
62
|
+
_index: 'test_index',
|
63
|
+
_type: 'documents_test',
|
64
|
+
_id: 2,
|
65
|
+
data: { a: 3, b: 4 },
|
66
|
+
},
|
67
|
+
}
|
68
|
+
]
|
69
|
+
}
|
70
|
+
expect(client).to receive(:bulk).with(expected_body)
|
71
|
+
store.bulk_index(documents)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
@@ -11,20 +11,19 @@ module Elasticsearch
|
|
11
11
|
end
|
12
12
|
|
13
13
|
describe Index do
|
14
|
-
let(:
|
14
|
+
let(:adapter) { double(:adapter) }
|
15
15
|
let(:model) { double(:model, id: 1) }
|
16
16
|
let(:document) { TestDocument.new(model) }
|
17
|
-
subject(:index) { Index.new(
|
17
|
+
subject(:index) { Index.new(adapter) }
|
18
18
|
|
19
19
|
describe '#index' do
|
20
20
|
it 'adds or replaces a document in the search index' do
|
21
21
|
payload = {
|
22
|
-
index: 'test_index',
|
23
22
|
type: 'test_doc',
|
24
23
|
id: 1,
|
25
24
|
body: {valueA: :a, valueB: :b}
|
26
25
|
}
|
27
|
-
expect(
|
26
|
+
expect(adapter).to receive(:index).with(payload)
|
28
27
|
index.index document
|
29
28
|
end
|
30
29
|
end
|
@@ -32,11 +31,10 @@ module Elasticsearch
|
|
32
31
|
describe '#delete' do
|
33
32
|
it 'removes a document from the search index' do
|
34
33
|
payload = {
|
35
|
-
index: 'test_index',
|
36
34
|
type: 'test_doc',
|
37
35
|
id: 1,
|
38
36
|
}
|
39
|
-
expect(
|
37
|
+
expect(adapter).to receive(:delete).with(payload)
|
40
38
|
index.delete document
|
41
39
|
end
|
42
40
|
end
|
@@ -44,7 +42,6 @@ module Elasticsearch
|
|
44
42
|
describe '#search' do
|
45
43
|
let(:query_params) do
|
46
44
|
{
|
47
|
-
index: 'test_index',
|
48
45
|
query: {
|
49
46
|
query_string: "search term",
|
50
47
|
analyzer: "snowball",
|
@@ -58,7 +55,7 @@ module Elasticsearch
|
|
58
55
|
"total" => 4000,
|
59
56
|
"max_score" => 4.222,
|
60
57
|
"hits" => [
|
61
|
-
{
|
58
|
+
{
|
62
59
|
"_type" => "user",
|
63
60
|
"_id" => 42,
|
64
61
|
"_score" => 4.222,
|
@@ -71,23 +68,21 @@ module Elasticsearch
|
|
71
68
|
|
72
69
|
let(:query) { double(:query, as_hash: query_params) }
|
73
70
|
|
74
|
-
it 'passes on the query request body to the
|
75
|
-
expect(
|
71
|
+
it 'passes on the query request body to the adapter' do
|
72
|
+
expect(adapter).to receive(:search).with(query_params)
|
76
73
|
index.search query
|
77
74
|
end
|
78
75
|
|
79
76
|
it 'returns a Hashie::Mash instance' do
|
80
|
-
|
77
|
+
adapter.stub(:search).and_return(response)
|
81
78
|
response = index.search(query)
|
82
79
|
response.should be_kind_of Hashie::Mash
|
83
80
|
end
|
84
81
|
end
|
85
82
|
|
86
83
|
describe '#refresh' do
|
87
|
-
it 'delegates to the
|
88
|
-
|
89
|
-
client.stub(:indices).and_return(indices)
|
90
|
-
expect(indices).to receive(:refresh).with(index: 'test_index')
|
84
|
+
it 'delegates to the adapter' do
|
85
|
+
expect(adapter).to receive(:refresh)
|
91
86
|
index.refresh
|
92
87
|
end
|
93
88
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Elasticsearch
|
4
|
+
module Extensions
|
5
|
+
module Documents
|
6
|
+
describe Storage do
|
7
|
+
|
8
|
+
let(:indices) { double(:indices) }
|
9
|
+
let(:client) { double(:client, indices: indices) }
|
10
|
+
subject(:storage) { described_class.new(client: client) }
|
11
|
+
|
12
|
+
describe '#create_index' do
|
13
|
+
it 'creates the index if it does not exist' do
|
14
|
+
expected_client_params = {
|
15
|
+
index: 'test_index',
|
16
|
+
body: {
|
17
|
+
settings: :fake_settings,
|
18
|
+
mappings: :fake_mappings,
|
19
|
+
}
|
20
|
+
}
|
21
|
+
indices.stub(:exists).and_return(false)
|
22
|
+
expect(indices).to receive(:create).with(expected_client_params)
|
23
|
+
storage.create_index('test_index')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'does not create the index if it exists' do
|
27
|
+
indices.stub(:exists).and_return(true)
|
28
|
+
expect(indices).not_to receive(:create)
|
29
|
+
storage.create_index('test_index')
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '#drop_index' do
|
34
|
+
it 'drops the index if it exists' do
|
35
|
+
indices.stub(:exists).and_return(true)
|
36
|
+
expect(indices).to receive(:delete)
|
37
|
+
storage.drop_index('test_index')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'does not drop the index if it does not exist' do
|
41
|
+
indices.stub(:exists).and_return(false)
|
42
|
+
expect(indices).not_to receive(:delete)
|
43
|
+
storage.drop_index('test_index')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#index_exists?' do
|
48
|
+
it 'delegates to the client indices' do
|
49
|
+
expect(indices).to receive(:exists).with(index: 'test_index')
|
50
|
+
storage.index_exists?('test_index')
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
data/spec/test_env.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Elasticsearch::Extensions::Documents.configure do |config|
|
2
|
+
config.index_name = 'documents_test'
|
3
|
+
config.settings = {}
|
4
|
+
config.mappings = {}
|
5
|
+
config.client.url = 'http://localhost:9200'
|
6
|
+
end
|
7
|
+
|
8
|
+
class TestDocumentsDocument < Elasticsearch::Extensions::Documents::Document
|
9
|
+
indexes_as_type :documents_test
|
10
|
+
|
11
|
+
def as_hash
|
12
|
+
{
|
13
|
+
a: object.a,
|
14
|
+
b: object.b,
|
15
|
+
}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
metadata
CHANGED
@@ -1,83 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticsearch-documents
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Houston
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - ~>
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '2.14'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - ~>
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '2.14'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pry
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: elasticsearch
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
|
-
- -
|
73
|
+
- - ">="
|
60
74
|
- !ruby/object:Gem::Version
|
61
75
|
version: '0'
|
62
76
|
type: :runtime
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
|
-
- -
|
80
|
+
- - ">="
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: hashie
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- -
|
87
|
+
- - ">="
|
74
88
|
- !ruby/object:Gem::Version
|
75
89
|
version: '0'
|
76
90
|
type: :runtime
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- -
|
94
|
+
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
83
97
|
description: Define mappings to turn model instances into indexable search documents
|
@@ -87,9 +101,9 @@ executables: []
|
|
87
101
|
extensions: []
|
88
102
|
extra_rdoc_files: []
|
89
103
|
files:
|
90
|
-
- .gitignore
|
91
|
-
- .rspec
|
92
|
-
- .travis.yml
|
104
|
+
- ".gitignore"
|
105
|
+
- ".rspec"
|
106
|
+
- ".travis.yml"
|
93
107
|
- Gemfile
|
94
108
|
- LICENSE.txt
|
95
109
|
- README.md
|
@@ -97,19 +111,24 @@ files:
|
|
97
111
|
- elasticsearch-documents.gemspec
|
98
112
|
- lib/elasticsearch-documents.rb
|
99
113
|
- lib/elasticsearch/extensions/documents.rb
|
114
|
+
- lib/elasticsearch/extensions/documents/aliased_index_store.rb
|
115
|
+
- lib/elasticsearch/extensions/documents/direct_index_store.rb
|
100
116
|
- lib/elasticsearch/extensions/documents/document.rb
|
101
117
|
- lib/elasticsearch/extensions/documents/index.rb
|
102
|
-
- lib/elasticsearch/extensions/documents/indexer.rb
|
103
118
|
- lib/elasticsearch/extensions/documents/queryable.rb
|
119
|
+
- lib/elasticsearch/extensions/documents/storage.rb
|
104
120
|
- lib/elasticsearch/extensions/documents/utils.rb
|
105
121
|
- lib/elasticsearch/extensions/documents/version.rb
|
122
|
+
- spec/elasticsearch/extensions/documents/aliased_index_store_spec.rb
|
123
|
+
- spec/elasticsearch/extensions/documents/direct_index_store_spec.rb
|
106
124
|
- spec/elasticsearch/extensions/documents/document_spec.rb
|
107
125
|
- spec/elasticsearch/extensions/documents/index_spec.rb
|
108
|
-
- spec/elasticsearch/extensions/documents/indexer_spec.rb
|
109
126
|
- spec/elasticsearch/extensions/documents/queryable_spec.rb
|
127
|
+
- spec/elasticsearch/extensions/documents/storage_spec.rb
|
110
128
|
- spec/elasticsearch/extensions/documents/utils_spec.rb
|
111
129
|
- spec/elasticsearch/extensions/documents_spec.rb
|
112
130
|
- spec/spec_helper.rb
|
131
|
+
- spec/test_env.rb
|
113
132
|
homepage: http://github.com/RyanHouston/elasticsearch-documents
|
114
133
|
licenses:
|
115
134
|
- MIT
|
@@ -120,25 +139,28 @@ require_paths:
|
|
120
139
|
- lib
|
121
140
|
required_ruby_version: !ruby/object:Gem::Requirement
|
122
141
|
requirements:
|
123
|
-
- -
|
142
|
+
- - ">="
|
124
143
|
- !ruby/object:Gem::Version
|
125
144
|
version: 1.9.3
|
126
145
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
127
146
|
requirements:
|
128
|
-
- -
|
147
|
+
- - ">="
|
129
148
|
- !ruby/object:Gem::Version
|
130
149
|
version: '0'
|
131
150
|
requirements: []
|
132
151
|
rubyforge_project:
|
133
|
-
rubygems_version: 2.
|
152
|
+
rubygems_version: 2.4.5
|
134
153
|
signing_key:
|
135
154
|
specification_version: 4
|
136
155
|
summary: A service wrapper to manage elasticsearch index documents
|
137
156
|
test_files:
|
157
|
+
- spec/elasticsearch/extensions/documents/aliased_index_store_spec.rb
|
158
|
+
- spec/elasticsearch/extensions/documents/direct_index_store_spec.rb
|
138
159
|
- spec/elasticsearch/extensions/documents/document_spec.rb
|
139
160
|
- spec/elasticsearch/extensions/documents/index_spec.rb
|
140
|
-
- spec/elasticsearch/extensions/documents/indexer_spec.rb
|
141
161
|
- spec/elasticsearch/extensions/documents/queryable_spec.rb
|
162
|
+
- spec/elasticsearch/extensions/documents/storage_spec.rb
|
142
163
|
- spec/elasticsearch/extensions/documents/utils_spec.rb
|
143
164
|
- spec/elasticsearch/extensions/documents_spec.rb
|
144
165
|
- spec/spec_helper.rb
|
166
|
+
- spec/test_env.rb
|
@@ -1,63 +0,0 @@
|
|
1
|
-
module Elasticsearch
|
2
|
-
module Extensions
|
3
|
-
module Documents
|
4
|
-
class Indexer
|
5
|
-
attr_reader :client, :config
|
6
|
-
|
7
|
-
def initialize(options = {})
|
8
|
-
@client = options.fetch(:client) { Documents.client }
|
9
|
-
@config = options.fetch(:config) { Documents.configuration }
|
10
|
-
end
|
11
|
-
|
12
|
-
def drop_index
|
13
|
-
client.indices.delete(index: config.index_name) if index_exists?
|
14
|
-
end
|
15
|
-
|
16
|
-
def index_exists?
|
17
|
-
client.indices.exists(index: config.index_name)
|
18
|
-
end
|
19
|
-
|
20
|
-
def create_index
|
21
|
-
client.indices.create(index: config.index_name, body: create_index_body) unless index_exists?
|
22
|
-
end
|
23
|
-
|
24
|
-
def create_index_body
|
25
|
-
{}.tap do |body|
|
26
|
-
body[:settings] = config.settings if config.settings
|
27
|
-
body[:mappings] = config.mappings if config.mappings
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def reindex(options = {}, &block)
|
32
|
-
force_create = options.fetch(:force_create) { false }
|
33
|
-
|
34
|
-
drop_index if force_create
|
35
|
-
create_index
|
36
|
-
|
37
|
-
block.call(self) if block_given?
|
38
|
-
end
|
39
|
-
|
40
|
-
def bulk_index(documents)
|
41
|
-
client.bulk body: bulk_index_operations(documents)
|
42
|
-
end
|
43
|
-
|
44
|
-
def bulk_index_operations(documents)
|
45
|
-
documents.collect { |document| bulk_index_operation_hash(document) }
|
46
|
-
end
|
47
|
-
|
48
|
-
def bulk_index_operation_hash(document)
|
49
|
-
{
|
50
|
-
index: {
|
51
|
-
_index: config.index_name,
|
52
|
-
_type: document.class.type,
|
53
|
-
_id: document.id,
|
54
|
-
data: document.as_hash,
|
55
|
-
}
|
56
|
-
}
|
57
|
-
end
|
58
|
-
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
@@ -1,122 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
module Elasticsearch
|
4
|
-
module Extensions
|
5
|
-
module Documents
|
6
|
-
describe Indexer do
|
7
|
-
|
8
|
-
let(:indices) { double(:indices) }
|
9
|
-
let(:client) { double(:client, indices: indices) }
|
10
|
-
subject(:indexer) { Indexer.new(client: client) }
|
11
|
-
|
12
|
-
describe '#create_index' do
|
13
|
-
it 'creates the index if it does not exist' do
|
14
|
-
expected_client_params = {
|
15
|
-
index: 'test_index',
|
16
|
-
body: {
|
17
|
-
settings: :fake_settings,
|
18
|
-
mappings: :fake_mappings,
|
19
|
-
}
|
20
|
-
}
|
21
|
-
indices.stub(:exists).and_return(false)
|
22
|
-
expect(indices).to receive(:create).with(expected_client_params)
|
23
|
-
indexer.create_index
|
24
|
-
end
|
25
|
-
|
26
|
-
it 'does not create the index if it exists' do
|
27
|
-
indices.stub(:exists).and_return(true)
|
28
|
-
expect(indices).not_to receive(:create)
|
29
|
-
indexer.create_index
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe '#drop_index' do
|
34
|
-
it 'drops the index if it exists' do
|
35
|
-
indices.stub(:exists).and_return(true)
|
36
|
-
expect(indices).to receive(:delete)
|
37
|
-
indexer.drop_index
|
38
|
-
end
|
39
|
-
|
40
|
-
it 'does not drop the index if it does not exist' do
|
41
|
-
indices.stub(:exists).and_return(false)
|
42
|
-
expect(indices).not_to receive(:delete)
|
43
|
-
indexer.drop_index
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe '#index_exists?' do
|
48
|
-
it 'delegates to the client indices' do
|
49
|
-
expect(indices).to receive(:exists).with(index: 'test_index')
|
50
|
-
indexer.index_exists?
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
describe '#bulk_index' do
|
55
|
-
let(:models) { [double(:model, id: 1, a: 1, b: 2), double(:model, id: 2, a: 3, b: 4)] }
|
56
|
-
let(:documents) { models.map { |m| TestDocumentsDocument.new(m) } }
|
57
|
-
|
58
|
-
it 'passes operations to the client bulk action' do
|
59
|
-
expected_body = {
|
60
|
-
body: [
|
61
|
-
{
|
62
|
-
index: {
|
63
|
-
_index: 'test_index',
|
64
|
-
_type: 'documents_test',
|
65
|
-
_id: 1,
|
66
|
-
data: { a: 1, b: 2 },
|
67
|
-
}
|
68
|
-
},
|
69
|
-
{
|
70
|
-
index: {
|
71
|
-
_index: 'test_index',
|
72
|
-
_type: 'documents_test',
|
73
|
-
_id: 2,
|
74
|
-
data: { a: 3, b: 4 },
|
75
|
-
},
|
76
|
-
}
|
77
|
-
]
|
78
|
-
}
|
79
|
-
expect(client).to receive(:bulk).with(expected_body)
|
80
|
-
indexer.bulk_index(documents)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
describe '#reindex' do
|
85
|
-
|
86
|
-
context 'with the :force_create option' do
|
87
|
-
it 'drops the index if exists' do
|
88
|
-
indices.stub(:exists).and_return(true)
|
89
|
-
expect(indexer).to receive(:drop_index)
|
90
|
-
indexer.reindex(force_create: true)
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
context 'without the :force_create option' do
|
95
|
-
it 'does not drop the index if exists' do
|
96
|
-
indices.stub(:exists).and_return(true)
|
97
|
-
expect(indexer).not_to receive(:drop_index)
|
98
|
-
indexer.reindex
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
it 'creates a new index' do
|
103
|
-
indices.stub(:exists).and_return(false)
|
104
|
-
expect(indexer).to receive(:create_index)
|
105
|
-
indexer.reindex
|
106
|
-
end
|
107
|
-
|
108
|
-
it 'calls a given block to batch index the documents' do
|
109
|
-
indexer.stub(:drop_index)
|
110
|
-
indexer.stub(:create_index)
|
111
|
-
documents = double(:documents)
|
112
|
-
expect(indexer).to receive(:bulk_index).with(documents)
|
113
|
-
indexer.reindex { |indexer| indexer.bulk_index(documents) }
|
114
|
-
end
|
115
|
-
|
116
|
-
end
|
117
|
-
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|