elastic_record 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -1
- data/Rakefile +2 -2
- data/elastic_record.gemspec +2 -2
- data/lib/elastic_record.rb +1 -0
- data/lib/elastic_record/callbacks.rb +13 -5
- data/lib/elastic_record/connection.rb +7 -8
- data/lib/elastic_record/errors.rb +10 -0
- data/lib/elastic_record/index/deferred.rb +7 -1
- data/lib/elastic_record/index/documents.rb +34 -17
- data/lib/elastic_record/index/manage.rb +8 -0
- data/lib/elastic_record/index/percolator.rb +16 -10
- data/lib/elastic_record/integration/active_record.rb +7 -0
- data/lib/elastic_record/integration/cassandra_object.rb +7 -0
- data/lib/elastic_record/model.rb +5 -10
- data/lib/elastic_record/railtie.rb +10 -0
- data/lib/elastic_record/relation.rb +26 -5
- data/lib/elastic_record/relation/batches.rb +4 -1
- data/lib/elastic_record/relation/finder_methods.rb +13 -3
- data/lib/elastic_record/relation/search_methods.rb +17 -0
- data/lib/elastic_record/relation/value_methods.rb +2 -2
- data/lib/elastic_record/searches_many.rb +1 -1
- data/lib/elastic_record/searches_many/association.rb +14 -10
- data/lib/elastic_record/searches_many/builder.rb +3 -3
- data/lib/elastic_record/searches_many/collection_proxy.rb +9 -3
- data/lib/elastic_record/searches_many/reflection.rb +4 -0
- data/test/elastic_record/callbacks_test.rb +65 -1
- data/test/elastic_record/config_test.rb +3 -3
- data/test/elastic_record/connection_test.rb +1 -1
- data/test/elastic_record/index/configurator_test.rb +1 -1
- data/test/elastic_record/index/documents_test.rb +23 -4
- data/test/elastic_record/index/manage_test.rb +1 -1
- data/test/elastic_record/index/mapping_test.rb +1 -1
- data/test/elastic_record/index/percolator_test.rb +1 -1
- data/test/elastic_record/index/settings_test.rb +1 -1
- data/test/elastic_record/index/warmer_test.rb +1 -1
- data/test/elastic_record/index_test.rb +1 -1
- data/test/elastic_record/integration/active_record_test.rb +39 -0
- data/test/elastic_record/log_subscriber_test.rb +11 -0
- data/test/elastic_record/lucene_test.rb +1 -1
- data/test/elastic_record/model_test.rb +1 -1
- data/test/elastic_record/railties/controller_runtime_test.rb +1 -1
- data/test/elastic_record/relation/admin_test.rb +13 -7
- data/test/elastic_record/relation/batches_test.rb +27 -1
- data/test/elastic_record/relation/delegation_test.rb +1 -1
- data/test/elastic_record/relation/finder_methods_test.rb +23 -8
- data/test/elastic_record/relation/merging_test.rb +1 -1
- data/test/elastic_record/relation/none_test.rb +1 -1
- data/test/elastic_record/relation/search_methods_test.rb +15 -2
- data/test/elastic_record/relation_test.rb +50 -3
- data/test/elastic_record/searches_many/association_test.rb +47 -0
- data/test/elastic_record/searches_many/autosave_test.rb +11 -10
- data/test/elastic_record/searches_many/collection_proxy_test.rb +1 -1
- data/test/elastic_record/searches_many/reflection_test.rb +7 -1
- data/test/elastic_record/searches_many_test.rb +11 -11
- data/test/elastic_record/searching_test.rb +1 -1
- data/test/helper.rb +19 -12
- data/test/support/models/option.rb +24 -0
- data/test/support/models/test_model.rb +22 -1
- data/test/support/models/warehouse.rb +2 -2
- data/test/support/models/widget.rb +4 -2
- data/test/support/query_counter.rb +56 -0
- metadata +10 -5
- data/lib/elastic_record/orm/active_record.rb +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0f00def1bf454a01cf7db164569a256fa3872f12
|
4
|
+
data.tar.gz: 3674835adb8a4d2b803f8b00b1f71e9239e3cf15
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cbe0b1f75dc593b47372e0b4d8da74398747dfd87c520e3286baf459e349f8665a6c9863111b3c3415da749c97f1a503a66356a58104cda647008dfd5351ccd2
|
7
|
+
data.tar.gz: a5d239eab2b96cab5aa461bd06671d6e46f7c22ff85ba688b7e0a5e22b4688c385446ba37af750e1b3476488f0028fd2d4da90b034afda842b35e8cfb053ebe4
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
data/elastic_record.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = 'elastic_record'
|
5
|
-
s.version = '1.1.
|
5
|
+
s.version = '1.1.5'
|
6
6
|
s.summary = 'Use Elastic Search with your objects'
|
7
7
|
s.description = 'Find your records with elastic search'
|
8
8
|
|
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.files = `git ls-files`.split("\n")
|
19
19
|
s.test_files = `git ls-files -- {test}/*`.split("\n")
|
20
20
|
|
21
|
-
s.add_dependency 'arelastic', '>= 0.
|
21
|
+
s.add_dependency 'arelastic', '>= 0.4.0'
|
22
22
|
s.add_dependency 'activemodel'
|
23
23
|
end
|
data/lib/elastic_record.rb
CHANGED
@@ -3,7 +3,7 @@ module ElasticRecord
|
|
3
3
|
def self.included(base)
|
4
4
|
base.class_eval do
|
5
5
|
after_save if: :changed? do
|
6
|
-
self.class.elastic_index.
|
6
|
+
self.class.elastic_index.index_record self
|
7
7
|
end
|
8
8
|
|
9
9
|
after_destroy do
|
@@ -12,15 +12,23 @@ module ElasticRecord
|
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
+
|
15
16
|
def as_search
|
16
17
|
json = {}
|
17
18
|
|
18
|
-
elastic_index.mapping[:properties].each do |
|
19
|
-
next if !respond_to?(
|
20
|
-
value = send(
|
19
|
+
elastic_index.mapping[:properties].each do |field, mapping|
|
20
|
+
next if !respond_to?(field)
|
21
|
+
value = send(field)
|
21
22
|
|
22
23
|
if value.present? || value == false
|
23
|
-
json[
|
24
|
+
json[field] = case mapping[:type]
|
25
|
+
when :object
|
26
|
+
value.as_search
|
27
|
+
when :nested
|
28
|
+
value.map(&:as_search)
|
29
|
+
else
|
30
|
+
value
|
31
|
+
end
|
24
32
|
end
|
25
33
|
end
|
26
34
|
|
@@ -1,13 +1,11 @@
|
|
1
1
|
require 'net/http'
|
2
2
|
|
3
3
|
module ElasticRecord
|
4
|
-
class ConnectionError < StandardError
|
5
|
-
end
|
6
|
-
|
7
4
|
class Connection
|
8
5
|
attr_accessor :servers, :options
|
9
6
|
attr_accessor :request_count, :current_server
|
10
7
|
attr_accessor :max_request_count
|
8
|
+
attr_accessor :bulk_stack
|
11
9
|
def initialize(servers, options = {})
|
12
10
|
if servers.is_a?(Array)
|
13
11
|
self.servers = servers
|
@@ -15,10 +13,11 @@ module ElasticRecord
|
|
15
13
|
self.servers = servers.split(',')
|
16
14
|
end
|
17
15
|
|
18
|
-
self.current_server
|
19
|
-
self.request_count
|
20
|
-
self.max_request_count
|
21
|
-
self.options
|
16
|
+
self.current_server = next_server
|
17
|
+
self.request_count = 0
|
18
|
+
self.max_request_count = 100
|
19
|
+
self.options = options
|
20
|
+
self.bulk_stack = []
|
22
21
|
end
|
23
22
|
|
24
23
|
def head(path)
|
@@ -78,7 +77,7 @@ module ElasticRecord
|
|
78
77
|
if @shuffled_servers.nil?
|
79
78
|
@shuffled_servers = servers.shuffle
|
80
79
|
else
|
81
|
-
@shuffled_servers.
|
80
|
+
@shuffled_servers.rotate!
|
82
81
|
end
|
83
82
|
|
84
83
|
@shuffled_servers.first
|
@@ -11,9 +11,11 @@ module ElasticRecord
|
|
11
11
|
attr_accessor :index
|
12
12
|
attr_accessor :deferred_actions
|
13
13
|
attr_accessor :writes_made
|
14
|
+
attr_accessor :bulk_stack
|
14
15
|
|
15
16
|
def initialize(index)
|
16
17
|
self.index = index
|
18
|
+
self.bulk_stack = []
|
17
19
|
reset!
|
18
20
|
end
|
19
21
|
|
@@ -45,12 +47,16 @@ module ElasticRecord
|
|
45
47
|
|
46
48
|
if READ_METHODS.include?(method)
|
47
49
|
flush!
|
48
|
-
index.real_connection.json_post
|
50
|
+
index.real_connection.json_post("/#{index.alias_name}/_refresh") if requires_refresh?(method, *args)
|
49
51
|
index.real_connection.send(method, *args, &block)
|
50
52
|
else
|
51
53
|
deferred_actions << DeferredAction.new(method, args, block)
|
52
54
|
end
|
53
55
|
end
|
56
|
+
|
57
|
+
def requires_refresh?(method, *args)
|
58
|
+
method == :json_get && args.first =~ /_search/
|
59
|
+
end
|
54
60
|
end
|
55
61
|
|
56
62
|
def enable_deferring!
|
@@ -3,14 +3,20 @@ require 'active_support/core_ext/object/to_query'
|
|
3
3
|
module ElasticRecord
|
4
4
|
class Index
|
5
5
|
module Documents
|
6
|
+
def index_record(record, index_name = nil)
|
7
|
+
return if disabled
|
8
|
+
|
9
|
+
index_document(record.send(record.class.primary_key), record.as_search, index_name)
|
10
|
+
end
|
11
|
+
|
6
12
|
def index_document(id, document, index_name = nil)
|
7
13
|
return if disabled
|
8
14
|
|
9
15
|
index_name ||= alias_name
|
10
16
|
|
11
|
-
if batch =
|
12
|
-
|
13
|
-
|
17
|
+
if batch = current_bulk_batch
|
18
|
+
batch << { index: { _index: index_name, _type: type, _id: id } }
|
19
|
+
batch << document
|
14
20
|
else
|
15
21
|
connection.json_put "/#{index_name}/#{type}/#{id}", document
|
16
22
|
end
|
@@ -19,7 +25,7 @@ module ElasticRecord
|
|
19
25
|
def delete_document(id, index_name = nil)
|
20
26
|
index_name ||= alias_name
|
21
27
|
|
22
|
-
if batch =
|
28
|
+
if batch = current_bulk_batch
|
23
29
|
batch << { delete: { _index: index_name, _type: type, _id: id } }
|
24
30
|
else
|
25
31
|
connection.json_delete "/#{index_name}/#{type}/#{id}"
|
@@ -52,15 +58,18 @@ module ElasticRecord
|
|
52
58
|
connection.json_get("/_search/scroll?#{options.to_query}")
|
53
59
|
end
|
54
60
|
|
55
|
-
def bulk
|
56
|
-
|
61
|
+
def bulk(options = {})
|
62
|
+
connection.bulk_stack.push []
|
63
|
+
|
57
64
|
yield
|
58
|
-
|
59
|
-
|
60
|
-
|
65
|
+
|
66
|
+
if current_bulk_batch.any?
|
67
|
+
body = current_bulk_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
|
68
|
+
results = connection.json_post("/_bulk?#{options.to_query}", body)
|
69
|
+
verify_bulk_results(results)
|
61
70
|
end
|
62
71
|
ensure
|
63
|
-
|
72
|
+
connection.bulk_stack.pop
|
64
73
|
end
|
65
74
|
|
66
75
|
def bulk_add(batch, index_name = nil)
|
@@ -68,18 +77,26 @@ module ElasticRecord
|
|
68
77
|
|
69
78
|
bulk do
|
70
79
|
batch.each do |record|
|
71
|
-
|
80
|
+
index_record(record, index_name)
|
72
81
|
end
|
73
82
|
end
|
74
83
|
end
|
75
84
|
|
76
|
-
def
|
77
|
-
|
78
|
-
@_batch
|
79
|
-
elsif model.superclass.respond_to?(:elastic_index)
|
80
|
-
model.superclass.elastic_index.current_batch
|
81
|
-
end
|
85
|
+
def current_bulk_batch
|
86
|
+
connection.bulk_stack.last
|
82
87
|
end
|
88
|
+
|
89
|
+
private
|
90
|
+
|
91
|
+
def verify_bulk_results(results)
|
92
|
+
return unless results.is_a?(Hash)
|
93
|
+
|
94
|
+
errors = results['items'].select do |item|
|
95
|
+
item.values.first['error']
|
96
|
+
end
|
97
|
+
|
98
|
+
raise ElasticRecord::BulkError.new(errors) unless errors.empty?
|
99
|
+
end
|
83
100
|
end
|
84
101
|
end
|
85
102
|
end
|
@@ -23,6 +23,14 @@ module ElasticRecord
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
+
def open(index_name)
|
27
|
+
connection.json_post("/#{index_name}/_open")
|
28
|
+
end
|
29
|
+
|
30
|
+
def close(index_name)
|
31
|
+
connection.json_post("/#{index_name}/_close")
|
32
|
+
end
|
33
|
+
|
26
34
|
def exists?(index_name)
|
27
35
|
connection.head("/#{index_name}") == '200'
|
28
36
|
end
|
@@ -2,13 +2,6 @@ module ElasticRecord
|
|
2
2
|
class Index
|
3
3
|
module Percolator
|
4
4
|
def create_percolator(name, elastic_query)
|
5
|
-
unless exists? percolator_index_name
|
6
|
-
create percolator_index_name
|
7
|
-
else
|
8
|
-
delete_mapping(percolator_index_name) if type_exists?(percolator_index_name)
|
9
|
-
update_mapping percolator_index_name
|
10
|
-
end
|
11
|
-
|
12
5
|
connection.json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
|
13
6
|
end
|
14
7
|
|
@@ -17,20 +10,33 @@ module ElasticRecord
|
|
17
10
|
end
|
18
11
|
|
19
12
|
def percolator_exists?(name)
|
20
|
-
|
13
|
+
!get_percolator(name).nil?
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_percolator(name)
|
17
|
+
json = connection.json_get("/_percolator/#{percolator_index_name}/#{name}")
|
18
|
+
json['_source'] if json['exists']
|
21
19
|
end
|
22
20
|
|
23
21
|
def percolate(document)
|
24
22
|
connection.json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
|
25
23
|
end
|
26
24
|
|
25
|
+
def all_percolators
|
26
|
+
if hits = connection.json_get("/_percolator/#{percolator_index_name}/_search?q=*&size=500")['hits']
|
27
|
+
hits['hits'].map { |hit| hit['_id'] }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
27
31
|
def reset_percolator
|
28
32
|
delete(percolator_index_name) if exists?(percolator_index_name)
|
33
|
+
create(percolator_index_name)
|
29
34
|
end
|
30
35
|
|
31
36
|
def percolator_index_name
|
32
|
-
|
37
|
+
alias_name
|
38
|
+
# @percolator_index_name ||= "percolate_#{alias_name}"
|
33
39
|
end
|
34
40
|
end
|
35
41
|
end
|
36
|
-
end
|
42
|
+
end
|
data/lib/elastic_record/model.rb
CHANGED
@@ -5,6 +5,9 @@ module ElasticRecord
|
|
5
5
|
extend Searching
|
6
6
|
extend ClassMethods
|
7
7
|
include SearchesMany
|
8
|
+
|
9
|
+
class_attribute :elastic_connection
|
10
|
+
self.elastic_connection = ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
|
8
11
|
end
|
9
12
|
end
|
10
13
|
|
@@ -17,20 +20,12 @@ module ElasticRecord
|
|
17
20
|
end
|
18
21
|
end
|
19
22
|
|
20
|
-
def elastic_connection
|
21
|
-
@elastic_connection ||= ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
|
22
|
-
end
|
23
|
-
|
24
|
-
def elastic_connection=(connection)
|
25
|
-
@elastic_connection = connection
|
26
|
-
end
|
27
|
-
|
28
23
|
def elastic_relation
|
29
|
-
ElasticRecord::Relation.new(self
|
24
|
+
ElasticRecord::Relation.new(self)
|
30
25
|
end
|
31
26
|
|
32
27
|
def arelastic
|
33
|
-
|
28
|
+
Arelastic::Builders::Search
|
34
29
|
end
|
35
30
|
|
36
31
|
def elastic_index
|
@@ -15,5 +15,15 @@ module ElasticRecord
|
|
15
15
|
include ElasticRecord::Railties::ControllerRuntime
|
16
16
|
end
|
17
17
|
end
|
18
|
+
|
19
|
+
initializer "elastic_record.orm" do |app|
|
20
|
+
ActiveSupport.on_load(:active_record) do
|
21
|
+
require 'elastic_record/integration/active_record'
|
22
|
+
end
|
23
|
+
|
24
|
+
ActiveSupport.on_load(:cassandra_object) do
|
25
|
+
require 'elastic_record/integration/cassandra_object'
|
26
|
+
end
|
27
|
+
end
|
18
28
|
end
|
19
29
|
end
|
@@ -11,11 +11,10 @@ module ElasticRecord
|
|
11
11
|
class Relation
|
12
12
|
include Admin, Batches, Delegation, FinderMethods, Merging, SearchMethods
|
13
13
|
|
14
|
-
attr_reader :klass, :
|
14
|
+
attr_reader :klass, :values
|
15
15
|
|
16
|
-
def initialize(klass
|
16
|
+
def initialize(klass)
|
17
17
|
@klass = klass
|
18
|
-
@arelastic = arelastic
|
19
18
|
@values = {}
|
20
19
|
end
|
21
20
|
|
@@ -36,10 +35,16 @@ module ElasticRecord
|
|
36
35
|
reset
|
37
36
|
end
|
38
37
|
|
38
|
+
def eager_loading?
|
39
|
+
@should_eager_load ||= eager_load_values.any?
|
40
|
+
end
|
41
|
+
|
39
42
|
def to_a
|
40
43
|
@records ||= begin
|
41
44
|
scope = select_values.any? ? klass.select(select_values) : klass
|
42
|
-
scope.
|
45
|
+
records = scope.load_elastic_record_hits(to_ids)
|
46
|
+
eager_load_associations(records) if eager_loading?
|
47
|
+
records
|
43
48
|
end
|
44
49
|
end
|
45
50
|
|
@@ -65,6 +70,7 @@ module ElasticRecord
|
|
65
70
|
private
|
66
71
|
def reset
|
67
72
|
@search_results = @records = nil
|
73
|
+
@should_eager_load = nil
|
68
74
|
end
|
69
75
|
|
70
76
|
def search_hits
|
@@ -74,5 +80,20 @@ module ElasticRecord
|
|
74
80
|
def search_results
|
75
81
|
@search_results ||= klass.elastic_index.search(as_elastic)
|
76
82
|
end
|
83
|
+
|
84
|
+
def eager_load_associations(records)
|
85
|
+
records = records.reject(&:marked_for_destruction?)
|
86
|
+
ids = records.map(&:id)
|
87
|
+
eager_load_values.each do |to_load|
|
88
|
+
reflection = klass.searches_many_reflections[to_load] ||(raise "searches_many #{to_load} does not exist on #{klass}")
|
89
|
+
foreign_key = reflection.foreign_key.to_sym
|
90
|
+
grouped_children = reflection.klass.elastic_search.filter(foreign_key => ids).limit(1000000).group_by(&foreign_key)
|
91
|
+
records.each do |record|
|
92
|
+
children = grouped_children.fetch(record.id, [])
|
93
|
+
record.send(to_load).eager_loaded(children)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
records
|
97
|
+
end
|
77
98
|
end
|
78
|
-
end
|
99
|
+
end
|