elastics-admin 1.0.7 → 1.0.8

Sign up to get free protection for your applications and to get access to all the features.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.0.7
1
+ 1.0.8
data/bin/elastics-admin CHANGED
@@ -77,12 +77,12 @@ optparse = OptionParser.new do |opts|
77
77
  opts.separator 'Other options:'
78
78
 
79
79
  opts.on( '-v', '--version', 'Shows the version and exits' ) do
80
- puts version
80
+ Prompter.say_ok version
81
81
  exit
82
82
  end
83
83
 
84
84
  opts.on_tail( '-h', '--help', 'Displays this screen' ) do
85
- puts copy
85
+ Prompter.say_log copy, :style => [:blue, :bold]
86
86
  puts opts
87
87
  exit
88
88
  end
@@ -92,7 +92,7 @@ end
92
92
  optparse.parse!
93
93
  command = ARGV.first
94
94
  exec "#{$0} -h" if command.nil?
95
- puts copy
95
+ Prompter.say_log copy, :style => [:blue, :bold]
96
96
 
97
97
  case command
98
98
 
@@ -103,10 +103,10 @@ when 'load'
103
103
  Elastics::Admin::Tasks.new(options).load_from_file
104
104
 
105
105
  when 'stats'
106
- puts '>> puts Elastics.index_stats.to_yaml'
107
- puts Elastics.index_stats.to_yaml
106
+ Prompter.say_notice '>> puts Elastics.index_stats.to_yaml'
107
+ Prompter.say_log Elastics.index_stats.to_yaml
108
108
 
109
109
  else
110
- puts "unknown command: #{command.inspect}"
110
+ Prompter.say_warning "unknown command: #{command.inspect}"
111
111
 
112
112
  end
@@ -22,8 +22,8 @@ module Elastics
22
22
  :type => Conf.variables[:type],
23
23
  :scroll => '5m',
24
24
  :size => 50,
25
- :timeout => 20,
26
- :batch_size => 1000,
25
+ :timeout => 60,
26
+ :batch_size => 500,
27
27
  :verbose => true,
28
28
  :index_map => nil }
29
29
  end
@@ -31,6 +31,7 @@ module Elastics
31
31
  def dump_to_file(cli=false)
32
32
  vars = { :index => cli ? options[:index] : (options[:index] || Elastics::Tasks.new.config_hash.keys),
33
33
  :type => options[:type] }
34
+ Prompter.say_title "Dumping indices: #{vars[:index].inspect}" if options[:verbose]
34
35
  if options[:verbose]
35
36
  total_hits = Elastics.count(vars)['count'].to_i
36
37
  total_count = 0
@@ -63,10 +64,10 @@ module Elastics
63
64
  if options[:verbose]
64
65
  formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
65
66
  pbar.pbar.finish unless pbar.pbar.finished?
66
- puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
67
+ Prompter.say_warning "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
67
68
  unless total_hits == total_count
68
- puts "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
69
- puts dump_stats.to_yaml
69
+ Prompter.say_notice "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
70
+ Prompter.say_log dump_stats.to_yaml
70
71
  end
71
72
  end
72
73
 
@@ -80,7 +81,7 @@ module Elastics
80
81
  line_count = 0
81
82
  file.lines { line_count += 1 }
82
83
  file.rewind
83
- puts "\nLoading from #{path}...\n"
84
+ Prompter.say_title "Loading: #{path}"
84
85
  pbar = ProgBar.new(line_count / 2, options[:batch_size])
85
86
  end
86
87
  file.lines do |line|
@@ -62,6 +62,7 @@ module Elastics
62
62
 
63
63
  def reindex(opts={})
64
64
  yield self
65
+ opts[:verbose] = true unless opts.has_key?(:verbose)
65
66
  perform(opts)
66
67
  end
67
68
 
@@ -127,7 +128,11 @@ module Elastics
127
128
  end
128
129
 
129
130
  def perform(opts={})
130
- Conf.logger.warn 'Safe reindex is disabled!' if opts[:safe_reindex] == false
131
+ Prompter.say_title 'Live-Reindex' if opts[:verbose]
132
+ if opts[:safe_reindex] == false
133
+ Conf.logger.warn 'Safe reindex is disabled!'
134
+ Prompter.say_warning 'WARNING: Safe reindex is disabled!' if opts[:verbose]
135
+ end
131
136
  Redis.init
132
137
  @indices = []
133
138
  @timestamp = Time.now.strftime('%Y%m%d%H%M%S_')
@@ -153,30 +158,27 @@ module Elastics
153
158
 
154
159
  @reindex.call
155
160
 
156
- # when the reindexing is finished we try to empty the changes list a few times
157
- tries = 0
158
- bulk_string = ''
159
- until (count = Redis.llen(:changes)) == 0 || tries > 9
160
- count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
161
- Elastics.post_bulk_string(:bulk_string => bulk_string)
162
- bulk_string = ''
163
- tries += 1
164
- end
161
+ # try to empty the changes for 10 times before stopping the indexing
162
+ 10.times{ index_changes(opts) }
163
+
165
164
  # at this point the changes list should be empty or contain the minimum number of changes we could achieve live
166
165
  # the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
167
- @stop_indexing.call if @stop_indexing
168
- # if we have still changes, we can index them (until the list will be empty)
169
- bulk_string = ''
170
- while (change = Redis.lpop(:changes))
171
- bulk_string << build_bulk_string_from_change(change)
166
+ if @stop_indexing
167
+ Prompter.say_notice 'Calling on_stop_indexing...' if opts[:verbose]
168
+ @stop_indexing.call
169
+ Prompter.say_notice 'Indexing stopped.' if opts[:verbose]
170
+ else
171
+ Prompter.say_warning 'No on_stop_indexing provided!' if opts[:verbose]
172
172
  end
173
- Elastics.post_bulk_string(:bulk_string => bulk_string)
173
+
174
+ # if we have still changes, we can index them all, now that the indexing is stopped
175
+ index_changes(opts)
174
176
 
175
177
  # deletes the old indices and create the aliases to the new
176
178
  @indices.each do |index|
177
179
  Elastics.delete_index :index => index
178
180
  Elastics.put_index_alias :alias => index,
179
- :index => @timestamp + index
181
+ :index => @timestamp + index
180
182
  end
181
183
  # after the execution of this method the user should deploy the new code and then resume the regular app processing
182
184
 
@@ -201,8 +203,28 @@ module Elastics
201
203
  Redis.reset_keys
202
204
  end
203
205
 
206
+ def index_changes(opts)
207
+ left_changes_count = Redis.llen(:changes)
208
+ return if left_changes_count == 0
209
+
210
+ batch_size = opts[:batch_size] || 100
211
+ bulk_string = ''
212
+ Prompter.say_notice "Reindexing #{left_changes_count} live-changes..." if opts[:verbose]
213
+
214
+ until left_changes_count == 0
215
+ batch_count = left_changes_count > batch_size ? batch_size : left_changes_count
216
+ batch_count.times do
217
+ bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))
218
+ left_changes_count -= 1
219
+ end
220
+ Elastics.post_bulk_string(:bulk_string => bulk_string)
221
+ bulk_string = ''
222
+ end
223
+ end
204
224
 
205
225
  def migrate_indices(opts)
226
+ Conf.http_client.options[:timeout] = opts[:timeout] || 60
227
+
206
228
  opts[:verbose] = true unless opts.has_key?(:verbose)
207
229
  pbar = ProgBar.new(Elastics.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
208
230
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elastics-admin
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.7
4
+ version: 1.0.8
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-08-22 00:00:00.000000000 Z
12
+ date: 2013-08-24 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: elastics-client
@@ -18,7 +18,7 @@ dependencies:
18
18
  requirements:
19
19
  - - '='
20
20
  - !ruby/object:Gem::Version
21
- version: 1.0.7
21
+ version: 1.0.8
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - '='
28
28
  - !ruby/object:Gem::Version
29
- version: 1.0.7
29
+ version: 1.0.8
30
30
  description: Provides binary and rake tasks to dump, load and optionally rename indices.
31
31
  Implements live-reindex with hot-swap of old code/index with new code/index.
32
32
  email: dd.nexus@gmail.com