elastics-admin 1.0.7 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.0.7
1
+ 1.0.8
data/bin/elastics-admin CHANGED
@@ -77,12 +77,12 @@ optparse = OptionParser.new do |opts|
77
77
  opts.separator 'Other options:'
78
78
 
79
79
  opts.on( '-v', '--version', 'Shows the version and exits' ) do
80
- puts version
80
+ Prompter.say_ok version
81
81
  exit
82
82
  end
83
83
 
84
84
  opts.on_tail( '-h', '--help', 'Displays this screen' ) do
85
- puts copy
85
+ Prompter.say_log copy, :style => [:blue, :bold]
86
86
  puts opts
87
87
  exit
88
88
  end
@@ -92,7 +92,7 @@ end
92
92
  optparse.parse!
93
93
  command = ARGV.first
94
94
  exec "#{$0} -h" if command.nil?
95
- puts copy
95
+ Prompter.say_log copy, :style => [:blue, :bold]
96
96
 
97
97
  case command
98
98
 
@@ -103,10 +103,10 @@ when 'load'
103
103
  Elastics::Admin::Tasks.new(options).load_from_file
104
104
 
105
105
  when 'stats'
106
- puts '>> puts Elastics.index_stats.to_yaml'
107
- puts Elastics.index_stats.to_yaml
106
+ Prompter.say_notice '>> puts Elastics.index_stats.to_yaml'
107
+ Prompter.say_log Elastics.index_stats.to_yaml
108
108
 
109
109
  else
110
- puts "unknown command: #{command.inspect}"
110
+ Prompter.say_warning "unknown command: #{command.inspect}"
111
111
 
112
112
  end
@@ -22,8 +22,8 @@ module Elastics
22
22
  :type => Conf.variables[:type],
23
23
  :scroll => '5m',
24
24
  :size => 50,
25
- :timeout => 20,
26
- :batch_size => 1000,
25
+ :timeout => 60,
26
+ :batch_size => 500,
27
27
  :verbose => true,
28
28
  :index_map => nil }
29
29
  end
@@ -31,6 +31,7 @@ module Elastics
31
31
  def dump_to_file(cli=false)
32
32
  vars = { :index => cli ? options[:index] : (options[:index] || Elastics::Tasks.new.config_hash.keys),
33
33
  :type => options[:type] }
34
+ Prompter.say_title "Dumping indices: #{vars[:index].inspect}" if options[:verbose]
34
35
  if options[:verbose]
35
36
  total_hits = Elastics.count(vars)['count'].to_i
36
37
  total_count = 0
@@ -63,10 +64,10 @@ module Elastics
63
64
  if options[:verbose]
64
65
  formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
65
66
  pbar.pbar.finish unless pbar.pbar.finished?
66
- puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
67
+ Prompter.say_warning "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
67
68
  unless total_hits == total_count
68
- puts "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
69
- puts dump_stats.to_yaml
69
+ Prompter.say_notice "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
70
+ Prompter.say_log dump_stats.to_yaml
70
71
  end
71
72
  end
72
73
 
@@ -80,7 +81,7 @@ module Elastics
80
81
  line_count = 0
81
82
  file.lines { line_count += 1 }
82
83
  file.rewind
83
- puts "\nLoading from #{path}...\n"
84
+ Prompter.say_title "Loading: #{path}"
84
85
  pbar = ProgBar.new(line_count / 2, options[:batch_size])
85
86
  end
86
87
  file.lines do |line|
@@ -62,6 +62,7 @@ module Elastics
62
62
 
63
63
  def reindex(opts={})
64
64
  yield self
65
+ opts[:verbose] = true unless opts.has_key?(:verbose)
65
66
  perform(opts)
66
67
  end
67
68
 
@@ -127,7 +128,11 @@ module Elastics
127
128
  end
128
129
 
129
130
  def perform(opts={})
130
- Conf.logger.warn 'Safe reindex is disabled!' if opts[:safe_reindex] == false
131
+ Prompter.say_title 'Live-Reindex' if opts[:verbose]
132
+ if opts[:safe_reindex] == false
133
+ Conf.logger.warn 'Safe reindex is disabled!'
134
+ Prompter.say_warning 'WARNING: Safe reindex is disabled!' if opts[:verbose]
135
+ end
131
136
  Redis.init
132
137
  @indices = []
133
138
  @timestamp = Time.now.strftime('%Y%m%d%H%M%S_')
@@ -153,30 +158,27 @@ module Elastics
153
158
 
154
159
  @reindex.call
155
160
 
156
- # when the reindexing is finished we try to empty the changes list a few times
157
- tries = 0
158
- bulk_string = ''
159
- until (count = Redis.llen(:changes)) == 0 || tries > 9
160
- count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
161
- Elastics.post_bulk_string(:bulk_string => bulk_string)
162
- bulk_string = ''
163
- tries += 1
164
- end
161
+ # try to empty the changes for 10 times before stopping the indexing
162
+ 10.times{ index_changes(opts) }
163
+
165
164
  # at this point the changes list should be empty or contain the minimum number of changes we could achieve live
166
165
  # the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
167
- @stop_indexing.call if @stop_indexing
168
- # if we have still changes, we can index them (until the list will be empty)
169
- bulk_string = ''
170
- while (change = Redis.lpop(:changes))
171
- bulk_string << build_bulk_string_from_change(change)
166
+ if @stop_indexing
167
+ Prompter.say_notice 'Calling on_stop_indexing...' if opts[:verbose]
168
+ @stop_indexing.call
169
+ Prompter.say_notice 'Indexing stopped.' if opts[:verbose]
170
+ else
171
+ Prompter.say_warning 'No on_stop_indexing provided!' if opts[:verbose]
172
172
  end
173
- Elastics.post_bulk_string(:bulk_string => bulk_string)
173
+
174
+ # if we have still changes, we can index them all, now that the indexing is stopped
175
+ index_changes(opts)
174
176
 
175
177
  # deletes the old indices and create the aliases to the new
176
178
  @indices.each do |index|
177
179
  Elastics.delete_index :index => index
178
180
  Elastics.put_index_alias :alias => index,
179
- :index => @timestamp + index
181
+ :index => @timestamp + index
180
182
  end
181
183
  # after the execution of this method the user should deploy the new code and then resume the regular app processing
182
184
 
@@ -201,8 +203,28 @@ module Elastics
201
203
  Redis.reset_keys
202
204
  end
203
205
 
206
+ def index_changes(opts)
207
+ left_changes_count = Redis.llen(:changes)
208
+ return if left_changes_count == 0
209
+
210
+ batch_size = opts[:batch_size] || 100
211
+ bulk_string = ''
212
+ Prompter.say_notice "Reindexing #{left_changes_count} live-changes..." if opts[:verbose]
213
+
214
+ until left_changes_count == 0
215
+ batch_count = left_changes_count > batch_size ? batch_size : left_changes_count
216
+ batch_count.times do
217
+ bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))
218
+ left_changes_count -= 1
219
+ end
220
+ Elastics.post_bulk_string(:bulk_string => bulk_string)
221
+ bulk_string = ''
222
+ end
223
+ end
204
224
 
205
225
  def migrate_indices(opts)
226
+ Conf.http_client.options[:timeout] = opts[:timeout] || 60
227
+
206
228
  opts[:verbose] = true unless opts.has_key?(:verbose)
207
229
  pbar = ProgBar.new(Elastics.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
208
230
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elastics-admin
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.7
4
+ version: 1.0.8
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-08-22 00:00:00.000000000 Z
12
+ date: 2013-08-24 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: elastics-client
@@ -18,7 +18,7 @@ dependencies:
18
18
  requirements:
19
19
  - - '='
20
20
  - !ruby/object:Gem::Version
21
- version: 1.0.7
21
+ version: 1.0.8
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - '='
28
28
  - !ruby/object:Gem::Version
29
- version: 1.0.7
29
+ version: 1.0.8
30
30
  description: Provides binary and rake tasks to dump, load and optionally rename indices.
31
31
  Implements live-reindex with hot-swap of old code/index with new code/index.
32
32
  email: dd.nexus@gmail.com