elastic_record 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/elastic_record.rb +2 -0
- data/lib/elastic_record/connection.rb +1 -2
- data/lib/elastic_record/index.rb +33 -1
- data/lib/elastic_record/index/documents.rb +34 -0
- data/lib/elastic_record/index/manage.rb +57 -9
- data/lib/elastic_record/railtie.rb +5 -0
- data/lib/elastic_record/relation/batches.rb +16 -3
- data/test/elastic_record/connection_test.rb +1 -1
- data/test/elastic_record/index/documents_test.rb +42 -0
- data/test/elastic_record/index/manage_test.rb +34 -0
- data/test/elastic_record/index_test.rb +2 -2
- data/test/elastic_record/relation/batches_test.rb +17 -7
- data/test/elastic_record/relation/search_methods_test.rb +2 -2
- data/test/support/widget.rb +11 -18
- metadata +5 -2
data/lib/elastic_record.rb
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
module ElasticRecord
|
2
2
|
module Connection
|
3
3
|
def elastic_connection
|
4
|
-
ElasticRecord::Config.connection_options
|
5
4
|
@elastic_connection ||= ElasticSearch.new(
|
6
5
|
ElasticRecord::Config.servers,
|
7
|
-
ElasticRecord::Config.connection_options.merge(index:
|
6
|
+
ElasticRecord::Config.connection_options.merge(index: elastic_index.alias_name, type: elastic_index.type)
|
8
7
|
)
|
9
8
|
end
|
10
9
|
|
data/lib/elastic_record/index.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
require 'elastic_record/index/documents'
|
1
2
|
require 'elastic_record/index/manage'
|
2
3
|
require 'elastic_record/index/mapping'
|
3
4
|
|
4
5
|
module ElasticRecord
|
5
6
|
class Index
|
7
|
+
include Documents
|
6
8
|
include Manage
|
7
9
|
include Mapping
|
8
10
|
|
@@ -12,12 +14,42 @@ module ElasticRecord
|
|
12
14
|
@model = model
|
13
15
|
end
|
14
16
|
|
15
|
-
def
|
17
|
+
def alias_name
|
16
18
|
model.model_name.collection
|
17
19
|
end
|
18
20
|
|
19
21
|
def type
|
20
22
|
model.model_name.element
|
21
23
|
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def new_index_name
|
27
|
+
"#{alias_name}_#{Time.now.to_i}"
|
28
|
+
end
|
29
|
+
|
30
|
+
def json_get(path)
|
31
|
+
ActiveSupport::JSON.decode http.get(path).body
|
32
|
+
end
|
33
|
+
|
34
|
+
def json_post(path, json)
|
35
|
+
http.post(path, ActiveSupport::JSON.encode(json))
|
36
|
+
end
|
37
|
+
|
38
|
+
def json_put(path, json)
|
39
|
+
http.put(path, ActiveSupport::JSON.encode(json))
|
40
|
+
end
|
41
|
+
|
42
|
+
def json_delete(path)
|
43
|
+
ActiveSupport::JSON.decode http.delete(path).body
|
44
|
+
end
|
45
|
+
|
46
|
+
def connection
|
47
|
+
@model.elastic_connection
|
48
|
+
end
|
49
|
+
|
50
|
+
def http
|
51
|
+
host, port = connection.current_server.split ':'
|
52
|
+
Net::HTTP.new(host, port)
|
53
|
+
end
|
22
54
|
end
|
23
55
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module ElasticRecord
|
2
|
+
class Index
|
3
|
+
module Documents
|
4
|
+
def index_record(record, index_name = nil)
|
5
|
+
index_name ||= alias_name
|
6
|
+
document = record.respond_to?(:as_search) ? record.as_search : record.default_as_search
|
7
|
+
|
8
|
+
connection.index(document, id: record.id, index: index_name)
|
9
|
+
# json_put "#{index_name}/#{type}/#{record.id}", document
|
10
|
+
end
|
11
|
+
|
12
|
+
def delete_record(record, index_name = nil)
|
13
|
+
index_name ||= alias_name
|
14
|
+
|
15
|
+
connection.delete(record.id, index: index_name)
|
16
|
+
# json_delete "#{index_name}/#{type}/#{record.id}"
|
17
|
+
end
|
18
|
+
|
19
|
+
def record_exists?(id)
|
20
|
+
!connection.get(id).nil?
|
21
|
+
end
|
22
|
+
|
23
|
+
def bulk_add(batch, index_name = nil)
|
24
|
+
index_name ||= alias_name
|
25
|
+
|
26
|
+
connection.bulk do
|
27
|
+
batch.each do |record|
|
28
|
+
index_record(record, index_name)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -1,21 +1,69 @@
|
|
1
1
|
module ElasticRecord
|
2
2
|
class Index
|
3
3
|
module Manage
|
4
|
-
def
|
5
|
-
|
4
|
+
def create_and_deploy(index_name = new_index_name)
|
5
|
+
create(index_name)
|
6
|
+
deploy(index_name)
|
7
|
+
index_name
|
8
|
+
end
|
9
|
+
|
10
|
+
def create(index_name = new_index_name)
|
11
|
+
http.put("/#{index_name}", '')
|
12
|
+
update_mapping(index_name)
|
13
|
+
index_name
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete(index_name)
|
17
|
+
http.delete("/#{index_name}")
|
18
|
+
end
|
6
19
|
|
7
|
-
|
8
|
-
|
20
|
+
def delete_all
|
21
|
+
all_names.each do |index_name|
|
22
|
+
delete index_name
|
9
23
|
end
|
10
24
|
end
|
11
25
|
|
12
|
-
def
|
13
|
-
|
26
|
+
def exists?(index_name)
|
27
|
+
http.head("/#{index_name}").code == '200'
|
14
28
|
end
|
15
29
|
|
16
|
-
def
|
17
|
-
|
18
|
-
|
30
|
+
def deploy(index_name)
|
31
|
+
actions = [
|
32
|
+
{
|
33
|
+
add: {
|
34
|
+
"index" => index_name,
|
35
|
+
"alias" => alias_name
|
36
|
+
}
|
37
|
+
}
|
38
|
+
]
|
39
|
+
|
40
|
+
aliased_indexes.each do |index_to_remove|
|
41
|
+
actions << {
|
42
|
+
remove: {
|
43
|
+
"index" => index_to_remove,
|
44
|
+
"alias" => alias_name
|
45
|
+
}
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
json_post '/_aliases', actions: actions
|
50
|
+
end
|
51
|
+
|
52
|
+
def update_mapping(index_name)
|
53
|
+
connection.update_mapping(mapping, index: index_name)
|
54
|
+
# json_put "/#{index_name}/#{type}/_mapping", type => mapping
|
55
|
+
end
|
56
|
+
|
57
|
+
def aliased_indexes
|
58
|
+
json = json_get '/_cluster/state'
|
59
|
+
json["metadata"]["indices"].select { |name, status| status["aliases"].include?(alias_name) }.map { |name, status| name }
|
60
|
+
end
|
61
|
+
|
62
|
+
def all_names
|
63
|
+
json = json_get '/_status'
|
64
|
+
|
65
|
+
regex = %r{^#{alias_name}_?}
|
66
|
+
json['indices'].keys.grep(regex)
|
19
67
|
end
|
20
68
|
end
|
21
69
|
end
|
@@ -1,13 +1,26 @@
|
|
1
1
|
module ElasticRecord
|
2
2
|
module Batches
|
3
3
|
def find_each
|
4
|
-
|
4
|
+
find_in_batches do |records|
|
5
|
+
records.each { |record| yield record }
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
def find_in_batches(options = {})
|
10
|
+
options[:size] ||= 100
|
11
|
+
options[:scroll] ||= '20m'
|
12
|
+
options[:search_type] = 'scan'
|
5
13
|
|
6
14
|
hits = klass.elastic_connection.search(as_elastic, options)
|
7
15
|
|
8
16
|
klass.elastic_connection.scroll(hits.scroll_id, scroll: options[:scroll], ids_only: true) do |hits|
|
9
|
-
|
10
|
-
|
17
|
+
yield klass.find(hits.to_a)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def reindex
|
22
|
+
relation.find_in_batches do |batch|
|
23
|
+
elastic_index.bulk_add(batch)
|
11
24
|
end
|
12
25
|
end
|
13
26
|
end
|
@@ -5,6 +5,6 @@ class ElasticRecord::ConnectionTest < MiniTest::Spec
|
|
5
5
|
connection = Widget.elastic_connection
|
6
6
|
|
7
7
|
assert_equal Widget.elastic_index.type, connection.default_type
|
8
|
-
assert_equal Widget.elastic_index.
|
8
|
+
assert_equal Widget.elastic_index.alias_name, connection.default_index
|
9
9
|
end
|
10
10
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
|
4
|
+
def setup
|
5
|
+
super
|
6
|
+
|
7
|
+
Widget.reset_index!
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_index_record
|
11
|
+
record = Widget.new(id: 'abc', color: 'red')
|
12
|
+
|
13
|
+
index.index_record(record)
|
14
|
+
|
15
|
+
assert index.record_exists?('abc')
|
16
|
+
refute index.record_exists?('xyz')
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_delete_record
|
20
|
+
record = Widget.new(id: 'abc', color: 'red')
|
21
|
+
|
22
|
+
index.index_record(record)
|
23
|
+
assert index.record_exists?('abc')
|
24
|
+
|
25
|
+
index.delete_record(record)
|
26
|
+
refute index.record_exists?('abc')
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_bulk
|
30
|
+
record = Widget.new(id: 'abc', color: 'red')
|
31
|
+
|
32
|
+
index.bulk_add [record]
|
33
|
+
|
34
|
+
assert index.record_exists?('abc')
|
35
|
+
refute index.record_exists?('xyz')
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
def index
|
40
|
+
@index ||= Widget.elastic_index
|
41
|
+
end
|
42
|
+
end
|
@@ -1,4 +1,38 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
3
|
class ElasticRecord::Index::ManageTest < MiniTest::Spec
|
4
|
+
def setup
|
5
|
+
super
|
6
|
+
|
7
|
+
index.delete_all
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_create
|
11
|
+
assert !index.exists?('widgets_foo')
|
12
|
+
|
13
|
+
index.create 'widgets_foo'
|
14
|
+
|
15
|
+
assert index.exists?('widgets_foo')
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_exists
|
19
|
+
index.create 'widgets_foo'
|
20
|
+
|
21
|
+
assert index.exists?('widgets_foo')
|
22
|
+
assert !index.exists?('widgets_bar')
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_deploy
|
26
|
+
index.create 'widgets_foo'
|
27
|
+
|
28
|
+
assert index.aliased_indexes.empty?
|
29
|
+
index.deploy 'widgets_foo'
|
30
|
+
|
31
|
+
assert_equal ['widgets_foo'], index.aliased_indexes
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
def index
|
36
|
+
@index ||= Widget.elastic_index
|
37
|
+
end
|
4
38
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
3
|
class ElasticRecord::IndexTest < MiniTest::Spec
|
4
|
-
def
|
4
|
+
def test_model_name
|
5
5
|
index = Widget.elastic_index
|
6
6
|
|
7
|
+
assert_equal 'widgets', index.alias_name
|
7
8
|
assert_equal 'widget', index.type
|
8
|
-
assert_equal 'widgets', index.name
|
9
9
|
end
|
10
10
|
end
|
@@ -14,19 +14,29 @@ class ElasticRecord::Relation::BatchesTest < MiniTest::Spec
|
|
14
14
|
assert_equal ['5', '10', '15'].to_set, results.to_set
|
15
15
|
end
|
16
16
|
|
17
|
-
def
|
17
|
+
def test_find_in_batches
|
18
18
|
results = []
|
19
|
-
Widget.elastic_relation.
|
20
|
-
results <<
|
19
|
+
Widget.elastic_relation.find_in_batches do |widgets|
|
20
|
+
results << widgets.map(&:id)
|
21
|
+
end
|
22
|
+
assert_equal [['5', '10', '15'].to_set], results.map(&:to_set)
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_find_in_batches_with_scope
|
26
|
+
results = []
|
27
|
+
Widget.elastic_relation.filter(color: %w(red blue)).find_in_batches do |widgets|
|
28
|
+
results << widgets.map(&:id)
|
21
29
|
end
|
22
|
-
assert_equal ['5', '10'].to_set, results.to_set
|
30
|
+
assert_equal [['5', '10'].to_set], results.map(&:to_set)
|
23
31
|
end
|
24
32
|
|
25
33
|
private
|
26
34
|
def create_widgets
|
27
|
-
Widget.
|
28
|
-
|
29
|
-
|
35
|
+
Widget.elastic_index.bulk_add [
|
36
|
+
Widget.new(id: 5, color: 'red'),
|
37
|
+
Widget.new(id: 10, color: 'blue'),
|
38
|
+
Widget.new(id: 15, color: 'green'),
|
39
|
+
]
|
30
40
|
|
31
41
|
Widget.elastic_connection.refresh
|
32
42
|
end
|
@@ -162,10 +162,10 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Spec
|
|
162
162
|
Widget.elastic_connection.index({'widget' => {'color' => 'red'}}, {index: 'widgets', type: 'widget', id: 5})
|
163
163
|
Widget.elastic_connection.index({'widget' => {'color' => 'blue'}}, {index: 'widgets', type: 'widget', id: 10})
|
164
164
|
|
165
|
-
records = relation.select { |record| record.id == '
|
165
|
+
records = relation.select { |record| record.id == '10' }
|
166
166
|
|
167
167
|
assert_equal 1, records.size
|
168
|
-
assert_equal '
|
168
|
+
assert_equal '10', records.first.id
|
169
169
|
end
|
170
170
|
|
171
171
|
def test_extending_with_block
|
data/test/support/widget.rb
CHANGED
@@ -2,6 +2,11 @@ class Widget
|
|
2
2
|
extend ActiveModel::Naming
|
3
3
|
include ElasticRecord::Model
|
4
4
|
|
5
|
+
self.elastic_index.mapping[:properties].update(color: {
|
6
|
+
type: 'string',
|
7
|
+
index: 'not_analyzed'
|
8
|
+
})
|
9
|
+
|
5
10
|
class << self
|
6
11
|
def find(ids)
|
7
12
|
ids.map { |id| new(id: id, color: 'red') }
|
@@ -18,24 +23,8 @@ class Widget
|
|
18
23
|
end
|
19
24
|
|
20
25
|
def reset_index!
|
21
|
-
elastic_index.
|
22
|
-
elastic_index.
|
23
|
-
elastic_connection.update_mapping(
|
24
|
-
{
|
25
|
-
properties: {
|
26
|
-
color: {
|
27
|
-
type: 'string',
|
28
|
-
index: 'not_analyzed'
|
29
|
-
}
|
30
|
-
},
|
31
|
-
_source: {
|
32
|
-
enabled: false
|
33
|
-
}
|
34
|
-
},
|
35
|
-
{
|
36
|
-
index: 'widgets'
|
37
|
-
}
|
38
|
-
)
|
26
|
+
elastic_index.delete_all
|
27
|
+
elastic_index.create_and_deploy
|
39
28
|
end
|
40
29
|
end
|
41
30
|
|
@@ -45,4 +34,8 @@ class Widget
|
|
45
34
|
send("#{key}=", val)
|
46
35
|
end
|
47
36
|
end
|
37
|
+
|
38
|
+
def as_search
|
39
|
+
{color: color}
|
40
|
+
end
|
48
41
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastic_record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-09-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: arelastic
|
@@ -68,11 +68,13 @@ extra_rdoc_files:
|
|
68
68
|
files:
|
69
69
|
- lib/elastic_record/config.rb
|
70
70
|
- lib/elastic_record/connection.rb
|
71
|
+
- lib/elastic_record/index/documents.rb
|
71
72
|
- lib/elastic_record/index/manage.rb
|
72
73
|
- lib/elastic_record/index/mapping.rb
|
73
74
|
- lib/elastic_record/index.rb
|
74
75
|
- lib/elastic_record/model.rb
|
75
76
|
- lib/elastic_record/orm/active_record.rb
|
77
|
+
- lib/elastic_record/railtie.rb
|
76
78
|
- lib/elastic_record/relation/batches.rb
|
77
79
|
- lib/elastic_record/relation/delegation.rb
|
78
80
|
- lib/elastic_record/relation/finder_methods.rb
|
@@ -84,6 +86,7 @@ files:
|
|
84
86
|
- lib/elastic_record.rb
|
85
87
|
- test/elastic_record/config_test.rb
|
86
88
|
- test/elastic_record/connection_test.rb
|
89
|
+
- test/elastic_record/index/documents_test.rb
|
87
90
|
- test/elastic_record/index/manage_test.rb
|
88
91
|
- test/elastic_record/index/mapping_test.rb
|
89
92
|
- test/elastic_record/index_test.rb
|