elastic_record 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +3 -1
  3. data/Rakefile +2 -2
  4. data/elastic_record.gemspec +2 -2
  5. data/lib/elastic_record.rb +1 -0
  6. data/lib/elastic_record/callbacks.rb +13 -5
  7. data/lib/elastic_record/connection.rb +7 -8
  8. data/lib/elastic_record/errors.rb +10 -0
  9. data/lib/elastic_record/index/deferred.rb +7 -1
  10. data/lib/elastic_record/index/documents.rb +34 -17
  11. data/lib/elastic_record/index/manage.rb +8 -0
  12. data/lib/elastic_record/index/percolator.rb +16 -10
  13. data/lib/elastic_record/integration/active_record.rb +7 -0
  14. data/lib/elastic_record/integration/cassandra_object.rb +7 -0
  15. data/lib/elastic_record/model.rb +5 -10
  16. data/lib/elastic_record/railtie.rb +10 -0
  17. data/lib/elastic_record/relation.rb +26 -5
  18. data/lib/elastic_record/relation/batches.rb +4 -1
  19. data/lib/elastic_record/relation/finder_methods.rb +13 -3
  20. data/lib/elastic_record/relation/search_methods.rb +17 -0
  21. data/lib/elastic_record/relation/value_methods.rb +2 -2
  22. data/lib/elastic_record/searches_many.rb +1 -1
  23. data/lib/elastic_record/searches_many/association.rb +14 -10
  24. data/lib/elastic_record/searches_many/builder.rb +3 -3
  25. data/lib/elastic_record/searches_many/collection_proxy.rb +9 -3
  26. data/lib/elastic_record/searches_many/reflection.rb +4 -0
  27. data/test/elastic_record/callbacks_test.rb +65 -1
  28. data/test/elastic_record/config_test.rb +3 -3
  29. data/test/elastic_record/connection_test.rb +1 -1
  30. data/test/elastic_record/index/configurator_test.rb +1 -1
  31. data/test/elastic_record/index/documents_test.rb +23 -4
  32. data/test/elastic_record/index/manage_test.rb +1 -1
  33. data/test/elastic_record/index/mapping_test.rb +1 -1
  34. data/test/elastic_record/index/percolator_test.rb +1 -1
  35. data/test/elastic_record/index/settings_test.rb +1 -1
  36. data/test/elastic_record/index/warmer_test.rb +1 -1
  37. data/test/elastic_record/index_test.rb +1 -1
  38. data/test/elastic_record/integration/active_record_test.rb +39 -0
  39. data/test/elastic_record/log_subscriber_test.rb +11 -0
  40. data/test/elastic_record/lucene_test.rb +1 -1
  41. data/test/elastic_record/model_test.rb +1 -1
  42. data/test/elastic_record/railties/controller_runtime_test.rb +1 -1
  43. data/test/elastic_record/relation/admin_test.rb +13 -7
  44. data/test/elastic_record/relation/batches_test.rb +27 -1
  45. data/test/elastic_record/relation/delegation_test.rb +1 -1
  46. data/test/elastic_record/relation/finder_methods_test.rb +23 -8
  47. data/test/elastic_record/relation/merging_test.rb +1 -1
  48. data/test/elastic_record/relation/none_test.rb +1 -1
  49. data/test/elastic_record/relation/search_methods_test.rb +15 -2
  50. data/test/elastic_record/relation_test.rb +50 -3
  51. data/test/elastic_record/searches_many/association_test.rb +47 -0
  52. data/test/elastic_record/searches_many/autosave_test.rb +11 -10
  53. data/test/elastic_record/searches_many/collection_proxy_test.rb +1 -1
  54. data/test/elastic_record/searches_many/reflection_test.rb +7 -1
  55. data/test/elastic_record/searches_many_test.rb +11 -11
  56. data/test/elastic_record/searching_test.rb +1 -1
  57. data/test/helper.rb +19 -12
  58. data/test/support/models/option.rb +24 -0
  59. data/test/support/models/test_model.rb +22 -1
  60. data/test/support/models/warehouse.rb +2 -2
  61. data/test/support/models/widget.rb +4 -2
  62. data/test/support/query_counter.rb +56 -0
  63. metadata +10 -5
  64. data/lib/elastic_record/orm/active_record.rb +0 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 43d163448ac2df0ed13d06df549e2f986f812491
4
- data.tar.gz: 99603b58b72f7d4c74d49cd99e0c9877ad9a32a9
3
+ metadata.gz: 0f00def1bf454a01cf7db164569a256fa3872f12
4
+ data.tar.gz: 3674835adb8a4d2b803f8b00b1f71e9239e3cf15
5
5
  SHA512:
6
- metadata.gz: 65ff3c68ddab4c46310bef51356c2f7c164f2aac953103c92bd7471854c5ee8feede0824f5e472db227404f0af560fc58247b474ec39d0484680a96158af73f1
7
- data.tar.gz: 97198a8a88d57eab1fc7b8ecc12ebfd368f172f5f3fd6548765e83d35bb8b53d4bc2ace3a2da001ac8392288421590c1cae5203f67c24e670260c34db87bb581
6
+ metadata.gz: cbe0b1f75dc593b47372e0b4d8da74398747dfd87c520e3286baf459e349f8665a6c9863111b3c3415da749c97f1a503a66356a58104cda647008dfd5351ccd2
7
+ data.tar.gz: a5d239eab2b96cab5aa461bd06671d6e46f7c22ff85ba688b7e0a5e22b4688c385446ba37af750e1b3476488f0028fd2d4da90b034afda842b35e8cfb053ebe4
data/Gemfile CHANGED
@@ -2,5 +2,7 @@ source 'https://rubygems.org'
2
2
 
3
3
  gemspec
4
4
 
5
- gem 'rake'
5
+ gem 'activerecord'
6
+ gem 'mysql2'
6
7
  gem 'fakeweb'
8
+ gem 'rake'
data/Rakefile CHANGED
@@ -8,6 +8,6 @@ desc 'Test.'
8
8
  Rake::TestTask.new(:test) do |t|
9
9
  t.libs << 'lib'
10
10
  t.libs << 'test'
11
- t.pattern = 'test/**/*_test.rb'
11
+ t.pattern = 'test/elastic_record/**/*_test.rb'
12
12
  t.verbose = true
13
- end
13
+ end
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = 'elastic_record'
5
- s.version = '1.1.4'
5
+ s.version = '1.1.5'
6
6
  s.summary = 'Use Elastic Search with your objects'
7
7
  s.description = 'Find your records with elastic search'
8
8
 
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
18
18
  s.files = `git ls-files`.split("\n")
19
19
  s.test_files = `git ls-files -- {test}/*`.split("\n")
20
20
 
21
- s.add_dependency 'arelastic', '>= 0.3.0'
21
+ s.add_dependency 'arelastic', '>= 0.4.0'
22
22
  s.add_dependency 'activemodel'
23
23
  end
@@ -21,4 +21,5 @@ module ElasticRecord
21
21
  end
22
22
  end
23
23
 
24
+ require 'elastic_record/errors'
24
25
  require 'elastic_record/railtie' if defined?(Rails)
@@ -3,7 +3,7 @@ module ElasticRecord
3
3
  def self.included(base)
4
4
  base.class_eval do
5
5
  after_save if: :changed? do
6
- self.class.elastic_index.index_document id, as_search
6
+ self.class.elastic_index.index_record self
7
7
  end
8
8
 
9
9
  after_destroy do
@@ -12,15 +12,23 @@ module ElasticRecord
12
12
  end
13
13
  end
14
14
 
15
+
15
16
  def as_search
16
17
  json = {}
17
18
 
18
- elastic_index.mapping[:properties].each do |key, value|
19
- next if !respond_to?(key) || value[:type] == 'object'
20
- value = send(key)
19
+ elastic_index.mapping[:properties].each do |field, mapping|
20
+ next if !respond_to?(field)
21
+ value = send(field)
21
22
 
22
23
  if value.present? || value == false
23
- json[key] = value
24
+ json[field] = case mapping[:type]
25
+ when :object
26
+ value.as_search
27
+ when :nested
28
+ value.map(&:as_search)
29
+ else
30
+ value
31
+ end
24
32
  end
25
33
  end
26
34
 
@@ -1,13 +1,11 @@
1
1
  require 'net/http'
2
2
 
3
3
  module ElasticRecord
4
- class ConnectionError < StandardError
5
- end
6
-
7
4
  class Connection
8
5
  attr_accessor :servers, :options
9
6
  attr_accessor :request_count, :current_server
10
7
  attr_accessor :max_request_count
8
+ attr_accessor :bulk_stack
11
9
  def initialize(servers, options = {})
12
10
  if servers.is_a?(Array)
13
11
  self.servers = servers
@@ -15,10 +13,11 @@ module ElasticRecord
15
13
  self.servers = servers.split(',')
16
14
  end
17
15
 
18
- self.current_server = next_server
19
- self.request_count = 0
20
- self.max_request_count = 100
21
- self.options = options
16
+ self.current_server = next_server
17
+ self.request_count = 0
18
+ self.max_request_count = 100
19
+ self.options = options
20
+ self.bulk_stack = []
22
21
  end
23
22
 
24
23
  def head(path)
@@ -78,7 +77,7 @@ module ElasticRecord
78
77
  if @shuffled_servers.nil?
79
78
  @shuffled_servers = servers.shuffle
80
79
  else
81
- @shuffled_servers.push(@shuffled_servers.shift)
80
+ @shuffled_servers.rotate!
82
81
  end
83
82
 
84
83
  @shuffled_servers.first
@@ -0,0 +1,10 @@
1
+ module ElasticRecord
2
+ class Error < StandardError
3
+ end
4
+
5
+ class ConnectionError < Error
6
+ end
7
+
8
+ class BulkError < Error
9
+ end
10
+ end
@@ -11,9 +11,11 @@ module ElasticRecord
11
11
  attr_accessor :index
12
12
  attr_accessor :deferred_actions
13
13
  attr_accessor :writes_made
14
+ attr_accessor :bulk_stack
14
15
 
15
16
  def initialize(index)
16
17
  self.index = index
18
+ self.bulk_stack = []
17
19
  reset!
18
20
  end
19
21
 
@@ -45,12 +47,16 @@ module ElasticRecord
45
47
 
46
48
  if READ_METHODS.include?(method)
47
49
  flush!
48
- index.real_connection.json_post "/#{index.alias_name}/_refresh"
50
+ index.real_connection.json_post("/#{index.alias_name}/_refresh") if requires_refresh?(method, *args)
49
51
  index.real_connection.send(method, *args, &block)
50
52
  else
51
53
  deferred_actions << DeferredAction.new(method, args, block)
52
54
  end
53
55
  end
56
+
57
+ def requires_refresh?(method, *args)
58
+ method == :json_get && args.first =~ /_search/
59
+ end
54
60
  end
55
61
 
56
62
  def enable_deferring!
@@ -3,14 +3,20 @@ require 'active_support/core_ext/object/to_query'
3
3
  module ElasticRecord
4
4
  class Index
5
5
  module Documents
6
+ def index_record(record, index_name = nil)
7
+ return if disabled
8
+
9
+ index_document(record.send(record.class.primary_key), record.as_search, index_name)
10
+ end
11
+
6
12
  def index_document(id, document, index_name = nil)
7
13
  return if disabled
8
14
 
9
15
  index_name ||= alias_name
10
16
 
11
- if batch = current_batch
12
- current_batch << { index: { _index: index_name, _type: type, _id: id } }
13
- current_batch << document
17
+ if batch = current_bulk_batch
18
+ batch << { index: { _index: index_name, _type: type, _id: id } }
19
+ batch << document
14
20
  else
15
21
  connection.json_put "/#{index_name}/#{type}/#{id}", document
16
22
  end
@@ -19,7 +25,7 @@ module ElasticRecord
19
25
  def delete_document(id, index_name = nil)
20
26
  index_name ||= alias_name
21
27
 
22
- if batch = current_batch
28
+ if batch = current_bulk_batch
23
29
  batch << { delete: { _index: index_name, _type: type, _id: id } }
24
30
  else
25
31
  connection.json_delete "/#{index_name}/#{type}/#{id}"
@@ -52,15 +58,18 @@ module ElasticRecord
52
58
  connection.json_get("/_search/scroll?#{options.to_query}")
53
59
  end
54
60
 
55
- def bulk
56
- @_batch = []
61
+ def bulk(options = {})
62
+ connection.bulk_stack.push []
63
+
57
64
  yield
58
- if @_batch.any?
59
- body = @_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
60
- connection.json_post "/_bulk", body
65
+
66
+ if current_bulk_batch.any?
67
+ body = current_bulk_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
68
+ results = connection.json_post("/_bulk?#{options.to_query}", body)
69
+ verify_bulk_results(results)
61
70
  end
62
71
  ensure
63
- @_batch = nil
72
+ connection.bulk_stack.pop
64
73
  end
65
74
 
66
75
  def bulk_add(batch, index_name = nil)
@@ -68,18 +77,26 @@ module ElasticRecord
68
77
 
69
78
  bulk do
70
79
  batch.each do |record|
71
- index_document(record.id, record.as_search, index_name)
80
+ index_record(record, index_name)
72
81
  end
73
82
  end
74
83
  end
75
84
 
76
- def current_batch
77
- if @_batch
78
- @_batch
79
- elsif model.superclass.respond_to?(:elastic_index)
80
- model.superclass.elastic_index.current_batch
81
- end
85
+ def current_bulk_batch
86
+ connection.bulk_stack.last
82
87
  end
88
+
89
+ private
90
+
91
+ def verify_bulk_results(results)
92
+ return unless results.is_a?(Hash)
93
+
94
+ errors = results['items'].select do |item|
95
+ item.values.first['error']
96
+ end
97
+
98
+ raise ElasticRecord::BulkError.new(errors) unless errors.empty?
99
+ end
83
100
  end
84
101
  end
85
102
  end
@@ -23,6 +23,14 @@ module ElasticRecord
23
23
  end
24
24
  end
25
25
 
26
+ def open(index_name)
27
+ connection.json_post("/#{index_name}/_open")
28
+ end
29
+
30
+ def close(index_name)
31
+ connection.json_post("/#{index_name}/_close")
32
+ end
33
+
26
34
  def exists?(index_name)
27
35
  connection.head("/#{index_name}") == '200'
28
36
  end
@@ -2,13 +2,6 @@ module ElasticRecord
2
2
  class Index
3
3
  module Percolator
4
4
  def create_percolator(name, elastic_query)
5
- unless exists? percolator_index_name
6
- create percolator_index_name
7
- else
8
- delete_mapping(percolator_index_name) if type_exists?(percolator_index_name)
9
- update_mapping percolator_index_name
10
- end
11
-
12
5
  connection.json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
13
6
  end
14
7
 
@@ -17,20 +10,33 @@ module ElasticRecord
17
10
  end
18
11
 
19
12
  def percolator_exists?(name)
20
- connection.json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
13
+ !get_percolator(name).nil?
14
+ end
15
+
16
+ def get_percolator(name)
17
+ json = connection.json_get("/_percolator/#{percolator_index_name}/#{name}")
18
+ json['_source'] if json['exists']
21
19
  end
22
20
 
23
21
  def percolate(document)
24
22
  connection.json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
25
23
  end
26
24
 
25
+ def all_percolators
26
+ if hits = connection.json_get("/_percolator/#{percolator_index_name}/_search?q=*&size=500")['hits']
27
+ hits['hits'].map { |hit| hit['_id'] }
28
+ end
29
+ end
30
+
27
31
  def reset_percolator
28
32
  delete(percolator_index_name) if exists?(percolator_index_name)
33
+ create(percolator_index_name)
29
34
  end
30
35
 
31
36
  def percolator_index_name
32
- @percolator_index_name ||= "percolate_#{alias_name}"
37
+ alias_name
38
+ # @percolator_index_name ||= "percolate_#{alias_name}"
33
39
  end
34
40
  end
35
41
  end
36
- end
42
+ end
@@ -0,0 +1,7 @@
1
+ module ActiveRecord
2
+ class Base
3
+ def self.load_elastic_record_hits(ids)
4
+ order("FIELD(#{connection.quote_column_name(primary_key)}, #{ids.join(',')})").find(ids)
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ module CassandraObject
2
+ class Base
3
+ def self.load_elastic_record_hits(ids)
4
+ find(ids)
5
+ end
6
+ end
7
+ end
@@ -5,6 +5,9 @@ module ElasticRecord
5
5
  extend Searching
6
6
  extend ClassMethods
7
7
  include SearchesMany
8
+
9
+ class_attribute :elastic_connection
10
+ self.elastic_connection = ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
8
11
  end
9
12
  end
10
13
 
@@ -17,20 +20,12 @@ module ElasticRecord
17
20
  end
18
21
  end
19
22
 
20
- def elastic_connection
21
- @elastic_connection ||= ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
22
- end
23
-
24
- def elastic_connection=(connection)
25
- @elastic_connection = connection
26
- end
27
-
28
23
  def elastic_relation
29
- ElasticRecord::Relation.new(self, arelastic)
24
+ ElasticRecord::Relation.new(self)
30
25
  end
31
26
 
32
27
  def arelastic
33
- @arelastic ||= Arelastic::Builders::Search.new
28
+ Arelastic::Builders::Search
34
29
  end
35
30
 
36
31
  def elastic_index
@@ -15,5 +15,15 @@ module ElasticRecord
15
15
  include ElasticRecord::Railties::ControllerRuntime
16
16
  end
17
17
  end
18
+
19
+ initializer "elastic_record.orm" do |app|
20
+ ActiveSupport.on_load(:active_record) do
21
+ require 'elastic_record/integration/active_record'
22
+ end
23
+
24
+ ActiveSupport.on_load(:cassandra_object) do
25
+ require 'elastic_record/integration/cassandra_object'
26
+ end
27
+ end
18
28
  end
19
29
  end
@@ -11,11 +11,10 @@ module ElasticRecord
11
11
  class Relation
12
12
  include Admin, Batches, Delegation, FinderMethods, Merging, SearchMethods
13
13
 
14
- attr_reader :klass, :arelastic, :values
14
+ attr_reader :klass, :values
15
15
 
16
- def initialize(klass, arelastic)
16
+ def initialize(klass)
17
17
  @klass = klass
18
- @arelastic = arelastic
19
18
  @values = {}
20
19
  end
21
20
 
@@ -36,10 +35,16 @@ module ElasticRecord
36
35
  reset
37
36
  end
38
37
 
38
+ def eager_loading?
39
+ @should_eager_load ||= eager_load_values.any?
40
+ end
41
+
39
42
  def to_a
40
43
  @records ||= begin
41
44
  scope = select_values.any? ? klass.select(select_values) : klass
42
- scope.find(to_ids)
45
+ records = scope.load_elastic_record_hits(to_ids)
46
+ eager_load_associations(records) if eager_loading?
47
+ records
43
48
  end
44
49
  end
45
50
 
@@ -65,6 +70,7 @@ module ElasticRecord
65
70
  private
66
71
  def reset
67
72
  @search_results = @records = nil
73
+ @should_eager_load = nil
68
74
  end
69
75
 
70
76
  def search_hits
@@ -74,5 +80,20 @@ module ElasticRecord
74
80
  def search_results
75
81
  @search_results ||= klass.elastic_index.search(as_elastic)
76
82
  end
83
+
84
+ def eager_load_associations(records)
85
+ records = records.reject(&:marked_for_destruction?)
86
+ ids = records.map(&:id)
87
+ eager_load_values.each do |to_load|
88
+ reflection = klass.searches_many_reflections[to_load] ||(raise "searches_many #{to_load} does not exist on #{klass}")
89
+ foreign_key = reflection.foreign_key.to_sym
90
+ grouped_children = reflection.klass.elastic_search.filter(foreign_key => ids).limit(1000000).group_by(&foreign_key)
91
+ records.each do |record|
92
+ children = grouped_children.fetch(record.id, [])
93
+ record.send(to_load).eager_loaded(children)
94
+ end
95
+ end
96
+ records
97
+ end
77
98
  end
78
- end
99
+ end