es-reindex 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.markdown +13 -0
- data/lib/es-reindex.rb +23 -7
- data/lib/es-reindex/version.rb +1 -1
- data/spec/es-reindex_spec.rb +10 -0
- data/spec/integration/reindex_spec.rb +45 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 142c4bceb879849117f58dea9019a4ad38bebc58
|
4
|
+
data.tar.gz: 4676e75a520fe3eb18660000b72b1d4b639ffca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 896bbe493132b19ed83182fbd83464ec3dae02779ada820b08cac30b7801433ab0213cd6e7f71a56d1a5357d3f2df80bfff90f4c83055ef5883271535f938745
|
7
|
+
data.tar.gz: d220120b5a19c873eb0dfc7630a469ce37bc705e2510a8825588fae161a442e33c2ed3462bd3ec54a594930c4df87ea6fff827e933b6e59971b0b81cd5978f68
|
data/README.markdown
CHANGED
@@ -77,8 +77,21 @@ ESReindex.copy! 'http://my_server/index', 'http://my_server/index_copy',
|
|
77
77
|
after_copy: -> { finish_thing } # Runs after everything is copied over
|
78
78
|
```
|
79
79
|
|
80
|
+
#### Callbacks (guards)
|
81
|
+
|
82
|
+
You can also use the `:if` or `:unless` callbacks to prevent the copy/reindexing from occuring if conditions are (un)met. The source client and destination client are passed in:
|
83
|
+
|
84
|
+
```ruby
|
85
|
+
ESReindex.copy! 'http://my_server/index', 'http://my_server/index_copy',
|
86
|
+
if: ->(sclient,dclient) { Time.now.hour > 20 }, # Only copy the indexes if it's after 8pm
|
87
|
+
unless: ->(sclient,dclient) { Time.now.strftime("%A") == "Friday" } # Never copy on Fridays
|
88
|
+
```
|
89
|
+
|
90
|
+
For a more practical example, see the [reindex integration specs](spec/integration/reindex_spec.rb).
|
91
|
+
|
80
92
|
## Changelog
|
81
93
|
|
94
|
+
+ __0.3.0__: Add `:if` and `:unless` callbacks
|
82
95
|
+ __0.2.1__: [BUGFIX] Improve callback presence check
|
83
96
|
+ __0.2.0__: Lots of bugfixes, use elasticsearch client gem, add .reindex! method and callbacks
|
84
97
|
+ __0.1.0__: First gem release
|
data/lib/es-reindex.rb
CHANGED
@@ -12,14 +12,14 @@ class ESReindex
|
|
12
12
|
def self.copy!(src, dst, options = {})
|
13
13
|
self.new(src, dst, options).tap do |reindexer|
|
14
14
|
reindexer.setup_index_urls
|
15
|
-
reindexer.copy!
|
15
|
+
reindexer.copy! if reindexer.okay_to_proceed?
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
19
|
def self.reindex!(src, dst, options={})
|
20
20
|
self.new(src, dst, options.merge(copy_mappings: false)).tap do |reindexer|
|
21
21
|
reindexer.setup_index_urls
|
22
|
-
reindexer.copy!
|
22
|
+
reindexer.copy! if reindexer.okay_to_proceed?
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
@@ -36,6 +36,14 @@ class ESReindex
|
|
36
36
|
copy_mappings: true # Copy old mappings/settings
|
37
37
|
}.merge! options
|
38
38
|
|
39
|
+
%w{
|
40
|
+
if unless mappings settings before_create after_create before_each after_each after_copy
|
41
|
+
}.each do |callback|
|
42
|
+
if options[callback.to_sym].present? && !options[callback.to_sym].respond_to?(:call)
|
43
|
+
raise ArgumentError, "#{callback} must be a callable object"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
39
47
|
@done = 0
|
40
48
|
end
|
41
49
|
|
@@ -55,6 +63,14 @@ class ESReindex
|
|
55
63
|
@dclient = Elasticsearch::Client.new host: durl
|
56
64
|
end
|
57
65
|
|
66
|
+
def okay_to_proceed?
|
67
|
+
okay = true
|
68
|
+
okay = options[:if].call(sclient, dclient) if options[:if].present?
|
69
|
+
okay = (okay && !(options[:unless].call sclient, dclient)) if options[:unless].present?
|
70
|
+
log 'Skipping action due to guard callbacks' unless okay
|
71
|
+
okay
|
72
|
+
end
|
73
|
+
|
58
74
|
def copy!
|
59
75
|
log "Copying '#{surl}/#{sidx}' to '#{durl}/#{didx}'#{remove? ? ' with rewriting destination mapping!' : update? ? ' with updating existing documents!' : '.'}"
|
60
76
|
confirm if from_cli?
|
@@ -97,13 +113,13 @@ class ESReindex
|
|
97
113
|
create_msg = ""
|
98
114
|
end
|
99
115
|
|
100
|
-
options[:before_create].
|
116
|
+
options[:before_create].try(:call)
|
101
117
|
|
102
118
|
log "Creating '#{durl}/#{didx}' index#{create_msg}..."
|
103
119
|
dclient.indices.create index: didx, body: { settings: settings, mappings: mappings }
|
104
120
|
log "Succesfully created '#{durl}/#{didx}''#{create_msg}."
|
105
121
|
|
106
|
-
options[:after_create].
|
122
|
+
options[:after_create].try(:call)
|
107
123
|
end
|
108
124
|
|
109
125
|
true
|
@@ -141,13 +157,13 @@ class ESReindex
|
|
141
157
|
while scroll = sclient.scroll(scroll_id: scroll['_scroll_id'], scroll: '10m') and not scroll['hits']['hits'].empty? do
|
142
158
|
bulk = []
|
143
159
|
scroll['hits']['hits'].each do |doc|
|
144
|
-
options[:before_each].
|
160
|
+
options[:before_each].try(:call)
|
145
161
|
### === implement possible modifications to the document
|
146
162
|
### === end modifications to the document
|
147
163
|
base = {'_index' => didx, '_id' => doc['_id'], '_type' => doc['_type'], data: doc['_source']}
|
148
164
|
bulk << {action => base}
|
149
165
|
@done = done + 1
|
150
|
-
options[:after_each].
|
166
|
+
options[:after_each].try(:call)
|
151
167
|
end
|
152
168
|
unless bulk.empty?
|
153
169
|
dclient.bulk body: bulk
|
@@ -159,7 +175,7 @@ class ESReindex
|
|
159
175
|
|
160
176
|
log "Copy progress: %u/%u done in %s.\n" % [done, total, tm_len]
|
161
177
|
|
162
|
-
options[:after_copy].
|
178
|
+
options[:after_copy].try(:call)
|
163
179
|
|
164
180
|
true
|
165
181
|
end
|
data/lib/es-reindex/version.rb
CHANGED
data/spec/es-reindex_spec.rb
CHANGED
@@ -11,6 +11,16 @@ describe ESReindex do
|
|
11
11
|
expect(reindexer.options).to eq remove: false, update: true, frame: 1000, from_cli: false, copy_mappings: true
|
12
12
|
end
|
13
13
|
|
14
|
+
context "with invalid callback options" do
|
15
|
+
let(:options) { {if: :not_a_proc} }
|
16
|
+
|
17
|
+
it "fails" do
|
18
|
+
expect{
|
19
|
+
reindexer
|
20
|
+
}.to raise_error ArgumentError, "if must be a callable object"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
14
24
|
it "starts with 0 indexes done" do
|
15
25
|
expect(reindexer.done).to eq 0
|
16
26
|
end
|
@@ -8,7 +8,6 @@ describe "reindex!", type: :integration do
|
|
8
8
|
let!(:new_klass) { test_klass index_name: new_index_name, attributes: [:foo] }
|
9
9
|
|
10
10
|
let(:test_post) { original_klass.create id: 1, text: 'test_post' }
|
11
|
-
let(:test_post_2) { new_klass.create id: 2, text: 'other_post' }
|
12
11
|
|
13
12
|
# Create the index (test_index) on the test_klass:
|
14
13
|
before do
|
@@ -19,13 +18,55 @@ describe "reindex!", type: :integration do
|
|
19
18
|
|
20
19
|
let(:reindexed_post) { new_klass.find test_post.id }
|
21
20
|
|
21
|
+
let(:reindex) { ESReindex.reindex! "#{ES_HOST}/#{orig_index_name}", "#{ES_HOST}/#{new_index_name}", opts }
|
22
|
+
let(:mappings) { ->{ new_klass.mappings } }
|
23
|
+
let(:settings) { ->{ new_klass.settings } }
|
24
|
+
let(:other_opts) { {} }
|
25
|
+
let(:opts) { {mappings: mappings, settings: settings}.merge other_opts }
|
26
|
+
|
22
27
|
it "reindexes with the selected mappings" do
|
23
|
-
|
24
|
-
mappings: -> { new_klass.mappings },
|
25
|
-
settings: -> { new_klass.settings }
|
28
|
+
reindex
|
26
29
|
|
27
30
|
expect(reindexed_post.id).to eq test_post.id
|
28
31
|
expect(reindexed_post.text).to eq test_post.text
|
29
32
|
expect(reindexed_post).to respond_to :foo
|
30
33
|
end
|
34
|
+
|
35
|
+
context "with a :unless guard" do
|
36
|
+
let(:other_opts) do
|
37
|
+
{
|
38
|
+
unless: ->(sclient,dclient) {
|
39
|
+
existing_properties = sclient.indices.get_mapping(index: orig_index_name)[orig_index_name]['mappings']['test_klass']['properties']
|
40
|
+
existing_mappings = existing_properties.inject({}) do |result, (k,v)|
|
41
|
+
result[k.to_sym] = {type: v['type']}
|
42
|
+
result
|
43
|
+
end
|
44
|
+
new_mappings = klass_to_use_as_source.mappings.to_hash[:test_klass][:properties]
|
45
|
+
existing_mappings.sort == new_mappings.sort
|
46
|
+
}
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
context "when the unless call returns true" do
|
51
|
+
let(:klass_to_use_as_source) { original_klass }
|
52
|
+
|
53
|
+
it "does not reindex" do
|
54
|
+
reindex
|
55
|
+
original_klass.refresh_index!
|
56
|
+
expect(test_post.id).to be_present
|
57
|
+
expect { test_post.foo}.to raise_error NoMethodError
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context "when the unless call returns false" do
|
62
|
+
let(:klass_to_use_as_source) { new_klass }
|
63
|
+
|
64
|
+
it "reindexes" do
|
65
|
+
reindex
|
66
|
+
expect(reindexed_post.id).to eq test_post.id
|
67
|
+
expect(reindexed_post.text).to eq test_post.text
|
68
|
+
expect(reindexed_post).to respond_to :foo
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
31
72
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: es-reindex
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Aiken
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-02-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: elasticsearch
|