elastics-admin 1.0.7 → 1.0.8
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/bin/elastics-admin +6 -6
- data/lib/elastics/admin.rb +7 -6
- data/lib/elastics/admin_live_reindex.rb +39 -17
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.8
|
data/bin/elastics-admin
CHANGED
@@ -77,12 +77,12 @@ optparse = OptionParser.new do |opts|
|
|
77
77
|
opts.separator 'Other options:'
|
78
78
|
|
79
79
|
opts.on( '-v', '--version', 'Shows the version and exits' ) do
|
80
|
-
|
80
|
+
Prompter.say_ok version
|
81
81
|
exit
|
82
82
|
end
|
83
83
|
|
84
84
|
opts.on_tail( '-h', '--help', 'Displays this screen' ) do
|
85
|
-
|
85
|
+
Prompter.say_log copy, :style => [:blue, :bold]
|
86
86
|
puts opts
|
87
87
|
exit
|
88
88
|
end
|
@@ -92,7 +92,7 @@ end
|
|
92
92
|
optparse.parse!
|
93
93
|
command = ARGV.first
|
94
94
|
exec "#{$0} -h" if command.nil?
|
95
|
-
|
95
|
+
Prompter.say_log copy, :style => [:blue, :bold]
|
96
96
|
|
97
97
|
case command
|
98
98
|
|
@@ -103,10 +103,10 @@ when 'load'
|
|
103
103
|
Elastics::Admin::Tasks.new(options).load_from_file
|
104
104
|
|
105
105
|
when 'stats'
|
106
|
-
|
107
|
-
|
106
|
+
Prompter.say_notice '>> puts Elastics.index_stats.to_yaml'
|
107
|
+
Prompter.say_log Elastics.index_stats.to_yaml
|
108
108
|
|
109
109
|
else
|
110
|
-
|
110
|
+
Prompter.say_warning "unknown command: #{command.inspect}"
|
111
111
|
|
112
112
|
end
|
data/lib/elastics/admin.rb
CHANGED
@@ -22,8 +22,8 @@ module Elastics
|
|
22
22
|
:type => Conf.variables[:type],
|
23
23
|
:scroll => '5m',
|
24
24
|
:size => 50,
|
25
|
-
:timeout =>
|
26
|
-
:batch_size =>
|
25
|
+
:timeout => 60,
|
26
|
+
:batch_size => 500,
|
27
27
|
:verbose => true,
|
28
28
|
:index_map => nil }
|
29
29
|
end
|
@@ -31,6 +31,7 @@ module Elastics
|
|
31
31
|
def dump_to_file(cli=false)
|
32
32
|
vars = { :index => cli ? options[:index] : (options[:index] || Elastics::Tasks.new.config_hash.keys),
|
33
33
|
:type => options[:type] }
|
34
|
+
Prompter.say_title "Dumping indices: #{vars[:index].inspect}" if options[:verbose]
|
34
35
|
if options[:verbose]
|
35
36
|
total_hits = Elastics.count(vars)['count'].to_i
|
36
37
|
total_count = 0
|
@@ -63,10 +64,10 @@ module Elastics
|
|
63
64
|
if options[:verbose]
|
64
65
|
formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
|
65
66
|
pbar.pbar.finish unless pbar.pbar.finished?
|
66
|
-
|
67
|
+
Prompter.say_warning "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
|
67
68
|
unless total_hits == total_count
|
68
|
-
|
69
|
-
|
69
|
+
Prompter.say_notice "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
|
70
|
+
Prompter.say_log dump_stats.to_yaml
|
70
71
|
end
|
71
72
|
end
|
72
73
|
|
@@ -80,7 +81,7 @@ module Elastics
|
|
80
81
|
line_count = 0
|
81
82
|
file.lines { line_count += 1 }
|
82
83
|
file.rewind
|
83
|
-
|
84
|
+
Prompter.say_title "Loading: #{path}"
|
84
85
|
pbar = ProgBar.new(line_count / 2, options[:batch_size])
|
85
86
|
end
|
86
87
|
file.lines do |line|
|
@@ -62,6 +62,7 @@ module Elastics
|
|
62
62
|
|
63
63
|
def reindex(opts={})
|
64
64
|
yield self
|
65
|
+
opts[:verbose] = true unless opts.has_key?(:verbose)
|
65
66
|
perform(opts)
|
66
67
|
end
|
67
68
|
|
@@ -127,7 +128,11 @@ module Elastics
|
|
127
128
|
end
|
128
129
|
|
129
130
|
def perform(opts={})
|
130
|
-
|
131
|
+
Prompter.say_title 'Live-Reindex' if opts[:verbose]
|
132
|
+
if opts[:safe_reindex] == false
|
133
|
+
Conf.logger.warn 'Safe reindex is disabled!'
|
134
|
+
Prompter.say_warning 'WARNING: Safe reindex is disabled!' if opts[:verbose]
|
135
|
+
end
|
131
136
|
Redis.init
|
132
137
|
@indices = []
|
133
138
|
@timestamp = Time.now.strftime('%Y%m%d%H%M%S_')
|
@@ -153,30 +158,27 @@ module Elastics
|
|
153
158
|
|
154
159
|
@reindex.call
|
155
160
|
|
156
|
-
#
|
157
|
-
|
158
|
-
|
159
|
-
until (count = Redis.llen(:changes)) == 0 || tries > 9
|
160
|
-
count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
|
161
|
-
Elastics.post_bulk_string(:bulk_string => bulk_string)
|
162
|
-
bulk_string = ''
|
163
|
-
tries += 1
|
164
|
-
end
|
161
|
+
# try to empty the changes for 10 times before stopping the indexing
|
162
|
+
10.times{ index_changes(opts) }
|
163
|
+
|
165
164
|
# at this point the changes list should be empty or contain the minimum number of changes we could achieve live
|
166
165
|
# the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
166
|
+
if @stop_indexing
|
167
|
+
Prompter.say_notice 'Calling on_stop_indexing...' if opts[:verbose]
|
168
|
+
@stop_indexing.call
|
169
|
+
Prompter.say_notice 'Indexing stopped.' if opts[:verbose]
|
170
|
+
else
|
171
|
+
Prompter.say_warning 'No on_stop_indexing provided!' if opts[:verbose]
|
172
172
|
end
|
173
|
-
|
173
|
+
|
174
|
+
# if we have still changes, we can index them all, now that the indexing is stopped
|
175
|
+
index_changes(opts)
|
174
176
|
|
175
177
|
# deletes the old indices and create the aliases to the new
|
176
178
|
@indices.each do |index|
|
177
179
|
Elastics.delete_index :index => index
|
178
180
|
Elastics.put_index_alias :alias => index,
|
179
|
-
|
181
|
+
:index => @timestamp + index
|
180
182
|
end
|
181
183
|
# after the execution of this method the user should deploy the new code and then resume the regular app processing
|
182
184
|
|
@@ -201,8 +203,28 @@ module Elastics
|
|
201
203
|
Redis.reset_keys
|
202
204
|
end
|
203
205
|
|
206
|
+
def index_changes(opts)
|
207
|
+
left_changes_count = Redis.llen(:changes)
|
208
|
+
return if left_changes_count == 0
|
209
|
+
|
210
|
+
batch_size = opts[:batch_size] || 100
|
211
|
+
bulk_string = ''
|
212
|
+
Prompter.say_notice "Reindexing #{left_changes_count} live-changes..." if opts[:verbose]
|
213
|
+
|
214
|
+
until left_changes_count == 0
|
215
|
+
batch_count = left_changes_count > batch_size ? batch_size : left_changes_count
|
216
|
+
batch_count.times do
|
217
|
+
bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))
|
218
|
+
left_changes_count -= 1
|
219
|
+
end
|
220
|
+
Elastics.post_bulk_string(:bulk_string => bulk_string)
|
221
|
+
bulk_string = ''
|
222
|
+
end
|
223
|
+
end
|
204
224
|
|
205
225
|
def migrate_indices(opts)
|
226
|
+
Conf.http_client.options[:timeout] = opts[:timeout] || 60
|
227
|
+
|
206
228
|
opts[:verbose] = true unless opts.has_key?(:verbose)
|
207
229
|
pbar = ProgBar.new(Elastics.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
|
208
230
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastics-admin
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.8
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: elastics-client
|
@@ -18,7 +18,7 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - '='
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 1.0.
|
21
|
+
version: 1.0.8
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - '='
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 1.0.
|
29
|
+
version: 1.0.8
|
30
30
|
description: Provides binary and rake tasks to dump, load and optionally rename indices.
|
31
31
|
Implements live-reindex with hot-swap of old code/index with new code/index.
|
32
32
|
email: dd.nexus@gmail.com
|