chewy 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/lib/chewy/config.rb +2 -2
- data/lib/chewy/errors.rb +14 -0
- data/lib/chewy/index.rb +2 -2
- data/lib/chewy/index/actions.rb +16 -11
- data/lib/chewy/query/pagination.rb +2 -2
- data/lib/chewy/type/import.rb +54 -19
- data/lib/chewy/version.rb +1 -1
- data/spec/chewy/index/actions_spec.rb +48 -0
- data/spec/chewy/type/import_spec.rb +16 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37f0fefa72376c597d5ad48d1d6796371f15f809
|
4
|
+
data.tar.gz: 9f5400688fbdbd4e84022645c40aeeb9cafc665b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 076944d46b1082021ad1966e4be3543e1ac0a71a559e7013bd6f95879446ba5810fe7304af7b4b4c05775ebbfcbe2f9df5954d4a287411f3bceeb86621fac6fa
|
7
|
+
data.tar.gz: b5b8ee2719b6d19c478d5ec9e7551340f83e7aeb71de2757dbea33781995ddb82f54b9fab98e006200f8c7344f36fbd79aa556eda5ec11b84506d9a1283005d1
|
data/CHANGELOG.md
CHANGED
data/lib/chewy/config.rb
CHANGED
@@ -12,11 +12,11 @@ module Chewy
|
|
12
12
|
def self.repository name
|
13
13
|
plural_name = name.to_s.pluralize
|
14
14
|
|
15
|
-
class_eval <<-
|
15
|
+
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
16
16
|
def #{name}(name, options = nil)
|
17
17
|
options ? #{plural_name}[name.to_sym] = options : #{plural_name}[name.to_sym]
|
18
18
|
end
|
19
|
-
|
19
|
+
METHOD
|
20
20
|
end
|
21
21
|
|
22
22
|
def initialize
|
data/lib/chewy/errors.rb
CHANGED
@@ -10,4 +10,18 @@ module Chewy
|
|
10
10
|
|
11
11
|
class UnderivableType < Error
|
12
12
|
end
|
13
|
+
|
14
|
+
class FailedImport < Error
|
15
|
+
def initialize type, errors
|
16
|
+
output = "Import failed for `#{type}` with:\n"
|
17
|
+
errors.each do |action, errors|
|
18
|
+
output << " #{action.to_s.humanize} errors:\n"
|
19
|
+
errors.each do |error, documents|
|
20
|
+
output << " `#{error}`\n"
|
21
|
+
output << " on #{documents.count} documents: #{documents}\n"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
super output
|
25
|
+
end
|
26
|
+
end
|
13
27
|
end
|
data/lib/chewy/index.rb
CHANGED
@@ -80,11 +80,11 @@ module Chewy
|
|
80
80
|
type_class = Chewy::Type.new(self, target, options, &block)
|
81
81
|
self.type_hash = type_hash.merge(type_class.type_name => type_class)
|
82
82
|
|
83
|
-
class_eval <<-
|
83
|
+
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
84
84
|
def self.#{type_class.type_name}
|
85
85
|
type_hash['#{type_class.type_name}']
|
86
86
|
end
|
87
|
-
|
87
|
+
METHOD
|
88
88
|
end
|
89
89
|
|
90
90
|
# Types method has double usage.
|
data/lib/chewy/index/actions.rb
CHANGED
@@ -117,17 +117,22 @@ module Chewy
|
|
117
117
|
|
118
118
|
# Perform import operation for every defined type
|
119
119
|
#
|
120
|
-
# UsersIndex.import
|
121
|
-
# UsersIndex.import
|
122
|
-
# UsersIndex.import
|
123
|
-
# UsersIndex.import
|
124
|
-
#
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
120
|
+
# UsersIndex.import # imports default data for every index type
|
121
|
+
# UsersIndex.import user: User.active # imports specified objects for user type and default data for other types
|
122
|
+
# UsersIndex.import refresh: false # to disable index refreshing after import
|
123
|
+
# UsersIndex.import suffix: Time.now.to_i # imports data to index with specified suffix if such is exists
|
124
|
+
# UsersIndex.import batch_size: 300 # import batch size
|
125
|
+
#
|
126
|
+
[:import, :import!].each do |method|
|
127
|
+
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
128
|
+
def #{method} options = {}
|
129
|
+
objects = options.extract!(*type_names.map(&:to_sym))
|
130
|
+
types.map do |type|
|
131
|
+
args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
|
132
|
+
type.#{method} *args
|
133
|
+
end.all?
|
134
|
+
end
|
135
|
+
METHOD
|
131
136
|
end
|
132
137
|
|
133
138
|
# Deletes, creates and imports data to the index.
|
@@ -15,11 +15,11 @@ module Chewy
|
|
15
15
|
|
16
16
|
delegate :default_per_page, :max_per_page, :max_pages, to: :_kaminari_config
|
17
17
|
|
18
|
-
class_eval <<-
|
18
|
+
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
19
19
|
def #{::Kaminari.config.page_method_name}(num = 1)
|
20
20
|
limit(limit_value).offset(limit_value * ([num.to_i, 1].max - 1))
|
21
21
|
end
|
22
|
-
|
22
|
+
METHOD
|
23
23
|
end
|
24
24
|
|
25
25
|
def total_count
|
data/lib/chewy/type/import.rb
CHANGED
@@ -4,13 +4,17 @@ module Chewy
|
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
6
6
|
module ClassMethods
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
# Perform import operation for specified documents.
|
8
|
+
# Returns true or false depending on success.
|
9
|
+
#
|
10
|
+
# UsersIndex::User.import # imports default data set
|
11
|
+
# UsersIndex::User.import User.active # imports active users
|
12
|
+
# UsersIndex::User.import [1, 2, 3] # imports users with specified ids
|
13
|
+
# UsersIndex::User.import users # imports users collection
|
14
|
+
# UsersIndex::User.import refresh: false # to disable index refreshing after import
|
15
|
+
# UsersIndex::User.import suffix: Time.now.to_i # imports data to index with specified suffix if such is exists
|
16
|
+
# UsersIndex::User.import batch_size: 300 # import batch size
|
17
|
+
#
|
14
18
|
def import *args
|
15
19
|
import_options = args.extract_options!
|
16
20
|
bulk_options = import_options.extract!(:refresh, :suffix).reverse_merge!(refresh: true)
|
@@ -27,21 +31,39 @@ module Chewy
|
|
27
31
|
end
|
28
32
|
end
|
29
33
|
|
30
|
-
|
34
|
+
# Perform import operation for specified documents.
|
35
|
+
# Raises Chewy::FailedImport exception in case of import errors.
|
36
|
+
#
|
37
|
+
# UsersIndex::User.import! # imports default data set
|
38
|
+
# UsersIndex::User.import! User.active # imports active users
|
39
|
+
# UsersIndex::User.import! [1, 2, 3] # imports users with specified ids
|
40
|
+
# UsersIndex::User.import! users # imports users collection
|
41
|
+
# UsersIndex::User.import! refresh: false # to disable index refreshing after import
|
42
|
+
# UsersIndex::User.import! suffix: Time.now.to_i # imports data to index with specified suffix if such is exists
|
43
|
+
# UsersIndex::User.import! batch_size: 300 # import batch size
|
44
|
+
#
|
45
|
+
def import! *args
|
46
|
+
errors = nil
|
47
|
+
subscriber = ActiveSupport::Notifications.subscribe('import_objects.chewy') do |*args|
|
48
|
+
errors = args.last[:errors]
|
49
|
+
end
|
50
|
+
import *args
|
51
|
+
ActiveSupport::Notifications.unsubscribe(subscriber)
|
52
|
+
raise Chewy::FailedImport.new(self, errors) if errors.present?
|
53
|
+
true
|
54
|
+
end
|
31
55
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
{error => items.map { |item| item[:id] }}
|
40
|
-
end.reduce(&:merge)
|
41
|
-
{action => errors}
|
42
|
-
end.reduce(&:merge) || {}
|
56
|
+
# Wraps elasticsearch-ruby client indices bulk method.
|
57
|
+
# Adds `:suffix` option to bulk import to index with specified suffix.
|
58
|
+
def bulk options = {}
|
59
|
+
suffix = options.delete(:suffix)
|
60
|
+
result = client.bulk options.merge(index: index.build_index_name(suffix: suffix), type: type_name)
|
61
|
+
|
62
|
+
extract_errors result
|
43
63
|
end
|
44
64
|
|
65
|
+
private
|
66
|
+
|
45
67
|
def bulk_body action_objects
|
46
68
|
action_objects.each.with_object([]) do |(action, objects), result|
|
47
69
|
result.concat(if action == :delete
|
@@ -75,6 +97,19 @@ module Chewy
|
|
75
97
|
def object_data object
|
76
98
|
(self.root_object ||= build_root).compose(object)[type_name.to_sym]
|
77
99
|
end
|
100
|
+
|
101
|
+
def extract_errors result
|
102
|
+
result && result['items'].map do |item|
|
103
|
+
action = item.keys.first.to_sym
|
104
|
+
data = item.values.first
|
105
|
+
{action: action, id: data['_id'], error: data['error']} if data['error']
|
106
|
+
end.compact.group_by { |item| item[:action] }.map do |action, items|
|
107
|
+
errors = items.group_by { |item| item[:error] }.map do |error, items|
|
108
|
+
{error => items.map { |item| item[:id] }}
|
109
|
+
end.reduce(&:merge)
|
110
|
+
{action => errors}
|
111
|
+
end.reduce(&:merge) || {}
|
112
|
+
end
|
78
113
|
end
|
79
114
|
end
|
80
115
|
end
|
data/lib/chewy/version.rb
CHANGED
@@ -267,6 +267,54 @@ describe Chewy::Index::Actions do
|
|
267
267
|
end
|
268
268
|
end
|
269
269
|
|
270
|
+
describe '.import' do
|
271
|
+
before do
|
272
|
+
stub_model(:city)
|
273
|
+
stub_index(:cities) do
|
274
|
+
define_type City
|
275
|
+
end
|
276
|
+
end
|
277
|
+
let!(:dummy_cities) { 3.times.map { |i| City.create(name: "name#{i}") } }
|
278
|
+
|
279
|
+
specify { CitiesIndex.import.should == true }
|
280
|
+
|
281
|
+
context do
|
282
|
+
before do
|
283
|
+
stub_index(:cities) do
|
284
|
+
define_type City do
|
285
|
+
field :name, type: 'object'
|
286
|
+
end
|
287
|
+
end.tap(&:create!)
|
288
|
+
end
|
289
|
+
|
290
|
+
specify { CitiesIndex.import(city: dummy_cities).should == false }
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
describe '.import!' do
|
295
|
+
before do
|
296
|
+
stub_model(:city)
|
297
|
+
stub_index(:cities) do
|
298
|
+
define_type City
|
299
|
+
end
|
300
|
+
end
|
301
|
+
let!(:dummy_cities) { 3.times.map { |i| City.create(name: "name#{i}") } }
|
302
|
+
|
303
|
+
specify { CitiesIndex.import!.should == true }
|
304
|
+
|
305
|
+
context do
|
306
|
+
before do
|
307
|
+
stub_index(:cities) do
|
308
|
+
define_type City do
|
309
|
+
field :name, type: 'object'
|
310
|
+
end
|
311
|
+
end.tap(&:create!)
|
312
|
+
end
|
313
|
+
|
314
|
+
specify { expect { CitiesIndex.import!(city: dummy_cities) }.to raise_error Chewy::FailedImport }
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
270
318
|
describe '.reset!' do
|
271
319
|
before do
|
272
320
|
stub_model(:city)
|
@@ -20,10 +20,6 @@ describe Chewy::Type::Import do
|
|
20
20
|
let!(:dummy_cities) { 3.times.map { |i| City.create(name: "name#{i}") } }
|
21
21
|
let(:city) { CitiesIndex::City }
|
22
22
|
|
23
|
-
describe '.bulk' do
|
24
|
-
|
25
|
-
end
|
26
|
-
|
27
23
|
describe '.import' do
|
28
24
|
specify { city.import.should be_true }
|
29
25
|
specify { city.import([]).should be_true }
|
@@ -179,4 +175,20 @@ describe Chewy::Type::Import do
|
|
179
175
|
end
|
180
176
|
end
|
181
177
|
end
|
178
|
+
|
179
|
+
describe '.import!' do
|
180
|
+
specify { expect { city.import!.should }.not_to raise_error }
|
181
|
+
|
182
|
+
context do
|
183
|
+
before do
|
184
|
+
stub_index(:cities) do
|
185
|
+
define_type City do
|
186
|
+
field :name, type: 'object'
|
187
|
+
end
|
188
|
+
end.tap(&:create!)
|
189
|
+
end
|
190
|
+
|
191
|
+
specify { expect { city.import!(dummy_cities) }.to raise_error Chewy::FailedImport }
|
192
|
+
end
|
193
|
+
end
|
182
194
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chewy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- pyromaniac
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-02-
|
11
|
+
date: 2014-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|