chewy 0.0.1 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +13 -5
  2. data/.gitignore +1 -0
  3. data/.travis.yml +5 -3
  4. data/CHANGELOG.md +75 -0
  5. data/README.md +487 -92
  6. data/Rakefile +3 -2
  7. data/chewy.gemspec +2 -2
  8. data/filters +76 -0
  9. data/lib/chewy.rb +5 -3
  10. data/lib/chewy/config.rb +36 -19
  11. data/lib/chewy/fields/base.rb +5 -1
  12. data/lib/chewy/index.rb +22 -10
  13. data/lib/chewy/index/actions.rb +13 -13
  14. data/lib/chewy/index/search.rb +7 -2
  15. data/lib/chewy/query.rb +382 -64
  16. data/lib/chewy/query/context.rb +174 -0
  17. data/lib/chewy/query/criteria.rb +127 -34
  18. data/lib/chewy/query/loading.rb +9 -9
  19. data/lib/chewy/query/nodes/and.rb +25 -0
  20. data/lib/chewy/query/nodes/base.rb +17 -0
  21. data/lib/chewy/query/nodes/bool.rb +32 -0
  22. data/lib/chewy/query/nodes/equal.rb +34 -0
  23. data/lib/chewy/query/nodes/exists.rb +20 -0
  24. data/lib/chewy/query/nodes/expr.rb +28 -0
  25. data/lib/chewy/query/nodes/field.rb +106 -0
  26. data/lib/chewy/query/nodes/missing.rb +20 -0
  27. data/lib/chewy/query/nodes/not.rb +25 -0
  28. data/lib/chewy/query/nodes/or.rb +25 -0
  29. data/lib/chewy/query/nodes/prefix.rb +18 -0
  30. data/lib/chewy/query/nodes/query.rb +20 -0
  31. data/lib/chewy/query/nodes/range.rb +63 -0
  32. data/lib/chewy/query/nodes/raw.rb +15 -0
  33. data/lib/chewy/query/nodes/regexp.rb +31 -0
  34. data/lib/chewy/query/nodes/script.rb +20 -0
  35. data/lib/chewy/query/pagination.rb +28 -22
  36. data/lib/chewy/railtie.rb +23 -0
  37. data/lib/chewy/rspec/update_index.rb +20 -3
  38. data/lib/chewy/type/adapter/active_record.rb +78 -5
  39. data/lib/chewy/type/adapter/base.rb +46 -0
  40. data/lib/chewy/type/adapter/object.rb +40 -8
  41. data/lib/chewy/type/base.rb +1 -1
  42. data/lib/chewy/type/import.rb +18 -44
  43. data/lib/chewy/type/observe.rb +24 -14
  44. data/lib/chewy/version.rb +1 -1
  45. data/lib/tasks/chewy.rake +27 -0
  46. data/spec/chewy/config_spec.rb +30 -12
  47. data/spec/chewy/fields/base_spec.rb +11 -5
  48. data/spec/chewy/index/actions_spec.rb +20 -20
  49. data/spec/chewy/index/search_spec.rb +5 -5
  50. data/spec/chewy/index_spec.rb +28 -8
  51. data/spec/chewy/query/context_spec.rb +173 -0
  52. data/spec/chewy/query/criteria_spec.rb +219 -12
  53. data/spec/chewy/query/loading_spec.rb +6 -4
  54. data/spec/chewy/query/nodes/and_spec.rb +16 -0
  55. data/spec/chewy/query/nodes/bool_spec.rb +22 -0
  56. data/spec/chewy/query/nodes/equal_spec.rb +32 -0
  57. data/spec/chewy/query/nodes/exists_spec.rb +18 -0
  58. data/spec/chewy/query/nodes/missing_spec.rb +15 -0
  59. data/spec/chewy/query/nodes/not_spec.rb +16 -0
  60. data/spec/chewy/query/nodes/or_spec.rb +16 -0
  61. data/spec/chewy/query/nodes/prefix_spec.rb +16 -0
  62. data/spec/chewy/query/nodes/query_spec.rb +12 -0
  63. data/spec/chewy/query/nodes/range_spec.rb +32 -0
  64. data/spec/chewy/query/nodes/raw_spec.rb +11 -0
  65. data/spec/chewy/query/nodes/regexp_spec.rb +31 -0
  66. data/spec/chewy/query/nodes/script_spec.rb +15 -0
  67. data/spec/chewy/query/pagination_spec.rb +3 -2
  68. data/spec/chewy/query_spec.rb +83 -26
  69. data/spec/chewy/rspec/update_index_spec.rb +20 -0
  70. data/spec/chewy/type/adapter/active_record_spec.rb +102 -0
  71. data/spec/chewy/type/adapter/object_spec.rb +82 -0
  72. data/spec/chewy/type/import_spec.rb +30 -1
  73. data/spec/chewy/type/mapping_spec.rb +1 -1
  74. data/spec/chewy/type/observe_spec.rb +46 -12
  75. data/spec/spec_helper.rb +7 -6
  76. data/spec/support/class_helpers.rb +2 -2
  77. metadata +98 -48
  78. data/.rvmrc +0 -1
  79. data/lib/chewy/index/client.rb +0 -13
  80. data/spec/chewy/index/client_spec.rb +0 -18
@@ -0,0 +1,20 @@
1
+ module Chewy
2
+ class Query
3
+ module Nodes
4
+ class Script < Expr
5
+ def initialize script, params = {}
6
+ @script = script
7
+ @params = params
8
+ @options = params.extract!(:cache)
9
+ end
10
+
11
+ def __render__
12
+ script = {script: @script}
13
+ script.merge!(params: @params) if @params.present?
14
+ script.merge!(_cache: !!@options[:cache]) if @options.key?(:cache)
15
+ {script: script}
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,38 +1,44 @@
1
- require 'kaminari'
2
-
3
1
  module Chewy
4
2
  class Query
5
3
  module Pagination
6
4
  extend ActiveSupport::Concern
7
5
 
8
6
  included do
9
- include Kaminari::PageScopeMethods
7
+ include Kaminari if defined?(::Kaminari)
8
+ end
10
9
 
11
- delegate :default_per_page, :max_per_page, :max_pages, to: :_kaminari_config
10
+ module Kaminari
11
+ extend ActiveSupport::Concern
12
12
 
13
- class_eval <<-RUBY, __FILE__, __LINE__ + 1
14
- def #{Kaminari.config.page_method_name}(num = 1)
15
- limit(limit_value).offset(limit_value * ([num.to_i, 1].max - 1))
16
- end
17
- RUBY
18
- end
13
+ included do
14
+ include ::Kaminari::PageScopeMethods
19
15
 
20
- def total_count
21
- _response['hits']['total']
22
- end
16
+ delegate :default_per_page, :max_per_page, :max_pages, to: :_kaminari_config
23
17
 
24
- def limit_value
25
- (criteria.search[:size].presence || default_per_page).to_i
26
- end
18
+ class_eval <<-RUBY, __FILE__, __LINE__ + 1
19
+ def #{::Kaminari.config.page_method_name}(num = 1)
20
+ limit(limit_value).offset(limit_value * ([num.to_i, 1].max - 1))
21
+ end
22
+ RUBY
23
+ end
27
24
 
28
- def offset_value
29
- criteria.search[:from].to_i
30
- end
25
+ def total_count
26
+ _response['hits']['total']
27
+ end
28
+
29
+ def limit_value
30
+ (criteria.options[:size].presence || default_per_page).to_i
31
+ end
32
+
33
+ def offset_value
34
+ criteria.options[:from].to_i
35
+ end
31
36
 
32
- private
37
+ private
33
38
 
34
- def _kaminari_config
35
- Kaminari.config
39
+ def _kaminari_config
40
+ ::Kaminari.config
41
+ end
36
42
  end
37
43
  end
38
44
  end
@@ -0,0 +1,23 @@
1
+ module Chewy
2
+ class Railtie < Rails::Railtie
3
+ rake_tasks do
4
+ load 'tasks/chewy.rake'
5
+ end
6
+
7
+ initializer 'chewy.add_app_chewy_path' do |app|
8
+ app.config.paths.add 'app/chewy'
9
+ end
10
+
11
+ initializer 'chewy.add_requests_logging' do |app|
12
+ ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
13
+ duration = ((finish - start).to_f * 10000).round / 10.0
14
+ Rails.logger.debug(" \e[1m\e[33m#{payload[:type]} Import (#{duration}ms)\e[0m #{payload[:import]}")
15
+ end
16
+
17
+ ActiveSupport::Notifications.subscribe('search_query.chewy') do |name, start, finish, id, payload|
18
+ duration = ((finish - start).to_f * 10000).round / 10.0
19
+ Rails.logger.debug(" \e[1m\e[33m#{payload[:index]} Search (#{duration}ms)\e[0m #{payload[:request]}")
20
+ end
21
+ end
22
+ end
23
+ end
@@ -1,4 +1,4 @@
1
- RSpec::Matchers.define :update_index do |type_name|
1
+ RSpec::Matchers.define :update_index do |type_name, options = {}|
2
2
  chain(:and_reindex) do |*args|
3
3
  @reindex ||= {}
4
4
  @reindex.merge!(extract_documents(*args))
@@ -9,9 +9,15 @@ RSpec::Matchers.define :update_index do |type_name|
9
9
  @delete.merge!(extract_documents(*args))
10
10
  end
11
11
 
12
+ chain(:only) do |*args|
13
+ @only = true
14
+ end
15
+
12
16
  match do |block|
13
17
  @reindex ||= {}
14
18
  @delete ||= {}
19
+ @missed_reindex = []
20
+ @missed_delete = []
15
21
 
16
22
  type = Chewy.derive_type(type_name)
17
23
  updated = []
@@ -24,7 +30,11 @@ RSpec::Matchers.define :update_index do |type_name|
24
30
  end
25
31
  end
26
32
 
27
- block.call
33
+ if options[:atomic] == false
34
+ block.call
35
+ else
36
+ Chewy.atomic { block.call }
37
+ end
28
38
 
29
39
  @updated = updated
30
40
  @updated.each do |updated_document|
@@ -32,10 +42,14 @@ RSpec::Matchers.define :update_index do |type_name|
32
42
  if document = @reindex[body[:_id].to_s]
33
43
  document[:real_count] += 1
34
44
  document[:real_attributes].merge!(body[:data])
45
+ else
46
+ @missed_reindex.push(body[:_id].to_s) if @only
35
47
  end
36
48
  elsif body = updated_document[:delete]
37
49
  if document = @delete[body[:_id].to_s]
38
50
  document[:real_count] += 1
51
+ else
52
+ @missed_delete.push(body[:_id].to_s) if @only
39
53
  end
40
54
  end
41
55
  end
@@ -51,7 +65,7 @@ RSpec::Matchers.define :update_index do |type_name|
51
65
  (document[:expected_count] && document[:expected_count] == document[:real_count])
52
66
  end
53
67
 
54
- @updated.any? &&
68
+ @updated.any? && @missed_reindex.none? && @missed_delete.none? &&
55
69
  @reindex.all? { |_, document| document[:match_count] && document[:match_attributes] } &&
56
70
  @delete.all? { |_, document| document[:match_count] }
57
71
  end
@@ -61,6 +75,9 @@ RSpec::Matchers.define :update_index do |type_name|
61
75
 
62
76
  if @updated.none?
63
77
  output << "Expected index `#{type_name}` to be updated, but it was not\n"
78
+ else
79
+ output << "Expected index `#{type_name}` to update documents #{@reindex.keys} only, but #{@missed_reindex} was updated also\n" if @missed_reindex.any?
80
+ output << "Expected index `#{type_name}` to delete documents #{@delete.keys} only, but #{@missed_delete} was deleted also\n" if @missed_delete.any?
64
81
  end
65
82
 
66
83
  output << @reindex.each.with_object('') do |(id, document), output|
@@ -1,11 +1,12 @@
1
+ require 'chewy/type/adapter/base'
2
+
1
3
  module Chewy
2
4
  module Type
3
5
  module Adapter
4
- class ActiveRecord
5
- attr_reader :model, :scope, :options
6
-
7
- def initialize subject, options = {}
8
- @options = options
6
+ class ActiveRecord < Base
7
+ def initialize *args
8
+ @options = args.extract_options!
9
+ subject = args.first
9
10
  if subject.is_a?(::ActiveRecord::Relation)
10
11
  @model = subject.klass
11
12
  @scope = subject
@@ -21,6 +22,78 @@ module Chewy
21
22
  def type_name
22
23
  @type_name ||= (options[:name].presence || model.model_name).to_s.underscore
23
24
  end
25
+
26
+ def import *args, &block
27
+ import_options = args.extract_options!
28
+ import_options[:batch_size] ||= BATCH_SIZE
29
+ collection = args.none? ? model_all :
30
+ (args.one? && args.first.is_a?(::ActiveRecord::Relation) ? args.first : args.flatten)
31
+ if collection.is_a?(::ActiveRecord::Relation)
32
+ result = false
33
+ merged_scope(collection).find_in_batches(import_options.slice(:batch_size)) do |group|
34
+ result = block.call grouped_objects(group)
35
+ end
36
+ result
37
+ else
38
+ if collection.all? { |object| object.respond_to?(:id) }
39
+ collection.in_groups_of(import_options[:batch_size], false).all? do |group|
40
+ block.call grouped_objects(group)
41
+ end
42
+ else
43
+ import_ids(collection, import_options, &block)
44
+ end
45
+ end
46
+ end
47
+
48
+ def load *args
49
+ load_options = args.extract_options!
50
+ objects = args.flatten
51
+
52
+ scope = model.where(id: objects.map(&:id))
53
+ loaded_objects = if load_options[:scope].is_a?(Proc)
54
+ scope.instance_eval(&load_options[:scope])
55
+ elsif load_options[:scope].is_a?(::ActiveRecord::Relation)
56
+ scope.merge(load_options[:scope])
57
+ else
58
+ scope
59
+ end.index_by { |object| object.id.to_s }
60
+
61
+ objects.map { |object| loaded_objects[object.id.to_s] }
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :model, :scope, :options
67
+
68
+ def import_ids(ids, import_options = {}, &block)
69
+ ids = ids.map(&:to_i).uniq
70
+
71
+ indexed = false
72
+ merged_scope(model.where(id: ids)).find_in_batches(import_options.slice(:batch_size)) do |objects|
73
+ ids -= objects.map(&:id)
74
+ indexed = block.call index: objects
75
+ end
76
+
77
+ deleted = ids.in_groups_of(import_options[:batch_size], false).all? do |group|
78
+ block.call(delete: group)
79
+ end
80
+
81
+ indexed && deleted
82
+ end
83
+
84
+ def grouped_objects(objects)
85
+ objects.group_by do |object|
86
+ object.destroyed? ? :delete : :index
87
+ end
88
+ end
89
+
90
+ def merged_scope(target)
91
+ scope ? scope.clone.merge(target) : target
92
+ end
93
+
94
+ def model_all
95
+ ::ActiveRecord::VERSION::MAJOR < 4 ? model.scoped : model.all
96
+ end
24
97
  end
25
98
  end
26
99
  end
@@ -0,0 +1,46 @@
1
+ module Chewy
2
+ module Type
3
+ module Adapter
4
+ # Basic adapter class. Contains interface, need to implement to add any classes support
5
+ class Base
6
+ BATCH_SIZE = 1000
7
+
8
+ # Camelcased name, used as type class constant name.
9
+ # For returned value 'Product' will be generated class name `ProductsIndex::Product`
10
+ #
11
+ def name
12
+ raise NotImplementedError
13
+ end
14
+
15
+ # Underscored type name, user for elasticsearch type creation
16
+ # and for type class access with ProductsIndex.type_hash hash or method.
17
+ # `ProductsIndex.type_hash['product']` or `ProductsIndex.product`
18
+ #
19
+ def type_name
20
+ raise NotImplementedError
21
+ end
22
+
23
+ # Splits passed objects to groups according to `:batch_size` options.
24
+ # For every group crates hash with action keys. Example:
25
+ #
26
+ # { delete: [object1, object2], index: [object3, object4, object5] }
27
+ #
28
+ # Returns true id all the block call returns true and false otherwise
29
+ #
30
+ def import *args, &block
31
+ raise NotImplementedError
32
+ end
33
+
34
+ # Returns array of loaded objects for passed objects array. If some object
35
+ # was not loaded, it returns `nil` in the place of this object
36
+ #
37
+ # load(double(id: 1), double(id: 2), double(id: 3)) #=>
38
+ # # [<Product id: 1>, nil, <Product id: 3>], assuming, #2 was not found
39
+ #
40
+ def load *args
41
+ raise NotImplementedError
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -1,20 +1,52 @@
1
+ require 'chewy/type/adapter/base'
2
+
1
3
  module Chewy
2
4
  module Type
3
5
  module Adapter
4
- class Object
5
- attr_reader :subject, :options
6
-
7
- def initialize subject, options = {}
8
- @options = options
9
- @subject = subject
6
+ class Object < Base
7
+ def initialize *args
8
+ @options = args.extract_options!
9
+ @target = args.first
10
10
  end
11
11
 
12
12
  def name
13
- @name ||= subject.to_s.camelize
13
+ @name ||= (options[:name] || target).to_s.camelize
14
14
  end
15
15
 
16
16
  def type_name
17
- @type_name ||= subject.to_s.underscore
17
+ @type_name ||= (options[:name] || target).to_s.underscore
18
+ end
19
+
20
+ def import *args, &block
21
+ import_options = args.extract_options!
22
+ batch_size = import_options.delete(:batch_size) || BATCH_SIZE
23
+ objects = args.flatten
24
+
25
+ objects.in_groups_of(batch_size, false).all? do |group|
26
+ action_groups = group.group_by do |object|
27
+ raise "Object is not a `#{target}`" if class_target? && !object.is_a?(target)
28
+ object.respond_to?(:destroyed?) && object.destroyed? ? :delete : :index
29
+ end
30
+ block.call action_groups
31
+ end
32
+ end
33
+
34
+ def load *args
35
+ load_options = args.extract_options!
36
+ objects = args.flatten
37
+ if class_target?
38
+ objects.map { |object| target.wrap(object) }
39
+ else
40
+ objects
41
+ end
42
+ end
43
+
44
+ private
45
+
46
+ attr_reader :target, :options
47
+
48
+ def class_target?
49
+ @class_target ||= @target.is_a?(Class)
18
50
  end
19
51
  end
20
52
  end
@@ -25,7 +25,7 @@ module Chewy
25
25
  raise NotImplementedError
26
26
  end
27
27
 
28
- def self.type_name(suggest = nil)
28
+ def self.type_name
29
29
  adapter.type_name
30
30
  end
31
31
 
@@ -3,62 +3,36 @@ module Chewy
3
3
  module Import
4
4
  extend ActiveSupport::Concern
5
5
 
6
- included do
7
- end
8
-
9
6
  module ClassMethods
10
7
  def bulk(options = {})
11
8
  client.bulk options.merge(index: index.index_name, type: type_name)
12
9
  end
13
10
 
14
11
  def import(*args)
15
- options = args.extract_options!
16
- collection = args.first || adapter.model.all
17
-
18
- if collection.is_a? ::ActiveRecord::Relation
19
- scoped_relation(collection)
20
- .find_in_batches(options.slice(:batch_size)) { |objects| import_objects objects }
21
- else
22
- collection = Array.wrap(collection)
23
- if collection.all? { |entity| entity.respond_to?(:id) }
24
- import_objects collection
25
- else
26
- import_ids collection, options
12
+ import_options = args.extract_options!
13
+ bulk_options = import_options.extract!(:refresh).reverse_merge!(refresh: true)
14
+ identify = {_index: index.index_name, _type: type_name}
15
+
16
+ adapter.import(*args, import_options) do |action_objects|
17
+ payload = {type: self}
18
+ payload.merge! import: Hash[action_objects.map { |action, objects| [action, objects.count] }]
19
+
20
+ ActiveSupport::Notifications.instrument 'import_objects.chewy', payload do
21
+ body = action_objects.each.with_object([]) do |(action, objects), result|
22
+ result.concat(if action == :delete
23
+ objects.map { |object| { action => identify.merge(_id: object.respond_to?(:id) ? object.id : object) } }
24
+ else
25
+ objects.map { |object| { action => identify.merge(_id: object.id, data: object_data(object)) } }
26
+ end)
27
+ end
28
+ body.any? ? !!bulk(bulk_options.merge(body: body)) : true
27
29
  end
28
30
  end
29
31
  end
30
32
 
31
33
  private
32
34
 
33
- def scoped_relation(relation)
34
- adapter.scope ? relation.merge(adapter.scope) : relation
35
- end
36
-
37
- def import_ids(ids, options = {})
38
- ids = ids.map(&:to_i).uniq
39
- scoped_relation(adapter.model.where(id: ids))
40
- .find_in_batches(options.slice(:batch_size)) do |objects|
41
- ids -= objects.map(&:id)
42
- import_objects objects
43
- end
44
-
45
- body = ids.map { |id| {delete: {_index: index.index_name, _type: type_name, _id: id}} }
46
- bulk refresh: true, body: body if body.any?
47
- end
48
-
49
- def import_objects(objects)
50
- body = objects.map do |object|
51
- identify = {_index: index.index_name, _type: type_name, _id: object.id}
52
- if object.respond_to?(:destroyed?) && object.destroyed?
53
- {delete: identify}
54
- else
55
- {index: identify.merge!(data: object_to_data(object))}
56
- end
57
- end
58
- bulk refresh: true, body: body if body.any?
59
- end
60
-
61
- def object_to_data(object)
35
+ def object_data(object)
62
36
  (self.root_object ||= build_root).compose(object)[type_name.to_sym]
63
37
  end
64
38
  end