elastics-admin 1.0.7 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/bin/elastics-admin +6 -6
- data/lib/elastics/admin.rb +7 -6
- data/lib/elastics/admin_live_reindex.rb +39 -17
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.8
|
data/bin/elastics-admin
CHANGED
@@ -77,12 +77,12 @@ optparse = OptionParser.new do |opts|
|
|
77
77
|
opts.separator 'Other options:'
|
78
78
|
|
79
79
|
opts.on( '-v', '--version', 'Shows the version and exits' ) do
|
80
|
-
|
80
|
+
Prompter.say_ok version
|
81
81
|
exit
|
82
82
|
end
|
83
83
|
|
84
84
|
opts.on_tail( '-h', '--help', 'Displays this screen' ) do
|
85
|
-
|
85
|
+
Prompter.say_log copy, :style => [:blue, :bold]
|
86
86
|
puts opts
|
87
87
|
exit
|
88
88
|
end
|
@@ -92,7 +92,7 @@ end
|
|
92
92
|
optparse.parse!
|
93
93
|
command = ARGV.first
|
94
94
|
exec "#{$0} -h" if command.nil?
|
95
|
-
|
95
|
+
Prompter.say_log copy, :style => [:blue, :bold]
|
96
96
|
|
97
97
|
case command
|
98
98
|
|
@@ -103,10 +103,10 @@ when 'load'
|
|
103
103
|
Elastics::Admin::Tasks.new(options).load_from_file
|
104
104
|
|
105
105
|
when 'stats'
|
106
|
-
|
107
|
-
|
106
|
+
Prompter.say_notice '>> puts Elastics.index_stats.to_yaml'
|
107
|
+
Prompter.say_log Elastics.index_stats.to_yaml
|
108
108
|
|
109
109
|
else
|
110
|
-
|
110
|
+
Prompter.say_warning "unknown command: #{command.inspect}"
|
111
111
|
|
112
112
|
end
|
data/lib/elastics/admin.rb
CHANGED
@@ -22,8 +22,8 @@ module Elastics
|
|
22
22
|
:type => Conf.variables[:type],
|
23
23
|
:scroll => '5m',
|
24
24
|
:size => 50,
|
25
|
-
:timeout =>
|
26
|
-
:batch_size =>
|
25
|
+
:timeout => 60,
|
26
|
+
:batch_size => 500,
|
27
27
|
:verbose => true,
|
28
28
|
:index_map => nil }
|
29
29
|
end
|
@@ -31,6 +31,7 @@ module Elastics
|
|
31
31
|
def dump_to_file(cli=false)
|
32
32
|
vars = { :index => cli ? options[:index] : (options[:index] || Elastics::Tasks.new.config_hash.keys),
|
33
33
|
:type => options[:type] }
|
34
|
+
Prompter.say_title "Dumping indices: #{vars[:index].inspect}" if options[:verbose]
|
34
35
|
if options[:verbose]
|
35
36
|
total_hits = Elastics.count(vars)['count'].to_i
|
36
37
|
total_count = 0
|
@@ -63,10 +64,10 @@ module Elastics
|
|
63
64
|
if options[:verbose]
|
64
65
|
formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
|
65
66
|
pbar.pbar.finish unless pbar.pbar.finished?
|
66
|
-
|
67
|
+
Prompter.say_warning "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
|
67
68
|
unless total_hits == total_count
|
68
|
-
|
69
|
-
|
69
|
+
Prompter.say_notice "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
|
70
|
+
Prompter.say_log dump_stats.to_yaml
|
70
71
|
end
|
71
72
|
end
|
72
73
|
|
@@ -80,7 +81,7 @@ module Elastics
|
|
80
81
|
line_count = 0
|
81
82
|
file.lines { line_count += 1 }
|
82
83
|
file.rewind
|
83
|
-
|
84
|
+
Prompter.say_title "Loading: #{path}"
|
84
85
|
pbar = ProgBar.new(line_count / 2, options[:batch_size])
|
85
86
|
end
|
86
87
|
file.lines do |line|
|
@@ -62,6 +62,7 @@ module Elastics
|
|
62
62
|
|
63
63
|
def reindex(opts={})
|
64
64
|
yield self
|
65
|
+
opts[:verbose] = true unless opts.has_key?(:verbose)
|
65
66
|
perform(opts)
|
66
67
|
end
|
67
68
|
|
@@ -127,7 +128,11 @@ module Elastics
|
|
127
128
|
end
|
128
129
|
|
129
130
|
def perform(opts={})
|
130
|
-
|
131
|
+
Prompter.say_title 'Live-Reindex' if opts[:verbose]
|
132
|
+
if opts[:safe_reindex] == false
|
133
|
+
Conf.logger.warn 'Safe reindex is disabled!'
|
134
|
+
Prompter.say_warning 'WARNING: Safe reindex is disabled!' if opts[:verbose]
|
135
|
+
end
|
131
136
|
Redis.init
|
132
137
|
@indices = []
|
133
138
|
@timestamp = Time.now.strftime('%Y%m%d%H%M%S_')
|
@@ -153,30 +158,27 @@ module Elastics
|
|
153
158
|
|
154
159
|
@reindex.call
|
155
160
|
|
156
|
-
#
|
157
|
-
|
158
|
-
|
159
|
-
until (count = Redis.llen(:changes)) == 0 || tries > 9
|
160
|
-
count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
|
161
|
-
Elastics.post_bulk_string(:bulk_string => bulk_string)
|
162
|
-
bulk_string = ''
|
163
|
-
tries += 1
|
164
|
-
end
|
161
|
+
# try to empty the changes for 10 times before stopping the indexing
|
162
|
+
10.times{ index_changes(opts) }
|
163
|
+
|
165
164
|
# at this point the changes list should be empty or contain the minimum number of changes we could achieve live
|
166
165
|
# the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
166
|
+
if @stop_indexing
|
167
|
+
Prompter.say_notice 'Calling on_stop_indexing...' if opts[:verbose]
|
168
|
+
@stop_indexing.call
|
169
|
+
Prompter.say_notice 'Indexing stopped.' if opts[:verbose]
|
170
|
+
else
|
171
|
+
Prompter.say_warning 'No on_stop_indexing provided!' if opts[:verbose]
|
172
172
|
end
|
173
|
-
|
173
|
+
|
174
|
+
# if we have still changes, we can index them all, now that the indexing is stopped
|
175
|
+
index_changes(opts)
|
174
176
|
|
175
177
|
# deletes the old indices and create the aliases to the new
|
176
178
|
@indices.each do |index|
|
177
179
|
Elastics.delete_index :index => index
|
178
180
|
Elastics.put_index_alias :alias => index,
|
179
|
-
|
181
|
+
:index => @timestamp + index
|
180
182
|
end
|
181
183
|
# after the execution of this method the user should deploy the new code and then resume the regular app processing
|
182
184
|
|
@@ -201,8 +203,28 @@ module Elastics
|
|
201
203
|
Redis.reset_keys
|
202
204
|
end
|
203
205
|
|
206
|
+
def index_changes(opts)
|
207
|
+
left_changes_count = Redis.llen(:changes)
|
208
|
+
return if left_changes_count == 0
|
209
|
+
|
210
|
+
batch_size = opts[:batch_size] || 100
|
211
|
+
bulk_string = ''
|
212
|
+
Prompter.say_notice "Reindexing #{left_changes_count} live-changes..." if opts[:verbose]
|
213
|
+
|
214
|
+
until left_changes_count == 0
|
215
|
+
batch_count = left_changes_count > batch_size ? batch_size : left_changes_count
|
216
|
+
batch_count.times do
|
217
|
+
bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))
|
218
|
+
left_changes_count -= 1
|
219
|
+
end
|
220
|
+
Elastics.post_bulk_string(:bulk_string => bulk_string)
|
221
|
+
bulk_string = ''
|
222
|
+
end
|
223
|
+
end
|
204
224
|
|
205
225
|
def migrate_indices(opts)
|
226
|
+
Conf.http_client.options[:timeout] = opts[:timeout] || 60
|
227
|
+
|
206
228
|
opts[:verbose] = true unless opts.has_key?(:verbose)
|
207
229
|
pbar = ProgBar.new(Elastics.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
|
208
230
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastics-admin
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.8
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: elastics-client
|
@@ -18,7 +18,7 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - '='
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 1.0.
|
21
|
+
version: 1.0.8
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - '='
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 1.0.
|
29
|
+
version: 1.0.8
|
30
30
|
description: Provides binary and rake tasks to dump, load and optionally rename indices.
|
31
31
|
Implements live-reindex with hot-swap of old code/index with new code/index.
|
32
32
|
email: dd.nexus@gmail.com
|