elastics-admin 1.0.7 → 1.0.8

data/VERSION

@@ -1 +1 @@
-1.0.7
+1.0.8

data/bin/elastics-admin

@@ -77,12 +77,12 @@ optparse = OptionParser.new do |opts|
   opts.separator 'Other options:'
 
   opts.on( '-v', '--version', 'Shows the version and exits' ) do
-    puts version
+    Prompter.say_ok version
     exit
   end
 
   opts.on_tail( '-h', '--help', 'Displays this screen' ) do
-    puts copy
+    Prompter.say_log copy, :style => [:blue, :bold]
     puts opts
     exit
   end
@@ -92,7 +92,7 @@ end
 optparse.parse!
 command = ARGV.first
 exec "#{$0} -h" if command.nil?
-puts copy
+Prompter.say_log copy, :style => [:blue, :bold]
 
 case command
 
@@ -103,10 +103,10 @@ when 'load'
   Elastics::Admin::Tasks.new(options).load_from_file
 
 when 'stats'
-  puts '>> puts Elastics.index_stats.to_yaml'
-  puts Elastics.index_stats.to_yaml
+  Prompter.say_notice '>> puts Elastics.index_stats.to_yaml'
+  Prompter.say_log Elastics.index_stats.to_yaml
 
 else
-  puts "unknown command: #{command.inspect}"
+  Prompter.say_warning "unknown command: #{command.inspect}"
 
 end

data/lib/elastics/admin.rb

@@ -22,8 +22,8 @@ module Elastics
                                :type       => Conf.variables[:type],
                                :scroll     => '5m',
                                :size       => 50,
-                               :timeout    => 20,
-                               :batch_size => 1000,
+                               :timeout    => 60,
+                               :batch_size => 500,
                                :verbose    => true,
                                :index_map  => nil }
       end
@@ -31,6 +31,7 @@ module Elastics
       def dump_to_file(cli=false)
         vars = { :index => cli ? options[:index] : (options[:index] || Elastics::Tasks.new.config_hash.keys),
                  :type  => options[:type] }
+        Prompter.say_title "Dumping indices: #{vars[:index].inspect}" if options[:verbose]
         if options[:verbose]
           total_hits  = Elastics.count(vars)['count'].to_i
           total_count = 0
@@ -63,10 +64,10 @@ module Elastics
         if options[:verbose]
           formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
           pbar.pbar.finish unless pbar.pbar.finished?
-          puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
+          Prompter.say_warning "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
                unless total_hits == total_count
-          puts "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
-          puts dump_stats.to_yaml
+          Prompter.say_notice "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
+          Prompter.say_log dump_stats.to_yaml
         end
       end
 
@@ -80,7 +81,7 @@ module Elastics
           line_count = 0
           file.lines { line_count += 1 }
           file.rewind
-          puts "\nLoading from #{path}...\n"
+          Prompter.say_title "Loading: #{path}"
           pbar = ProgBar.new(line_count / 2, options[:batch_size])
         end
         file.lines do |line|

data/lib/elastics/admin_live_reindex.rb

@@ -62,6 +62,7 @@ module Elastics
 
     def reindex(opts={})
       yield self
+      opts[:verbose] = true unless opts.has_key?(:verbose)
       perform(opts)
     end
 
@@ -127,7 +128,11 @@ module Elastics
     end
 
     def perform(opts={})
-      Conf.logger.warn 'Safe reindex is disabled!' if opts[:safe_reindex] == false
+      Prompter.say_title 'Live-Reindex' if opts[:verbose]
+      if opts[:safe_reindex] == false
+        Conf.logger.warn 'Safe reindex is disabled!'
+        Prompter.say_warning 'WARNING: Safe reindex is disabled!' if opts[:verbose]
+      end
       Redis.init
       @indices        = []
       @timestamp      = Time.now.strftime('%Y%m%d%H%M%S_')
@@ -153,30 +158,27 @@ module Elastics
 
       @reindex.call
 
-      # when the reindexing is finished we try to empty the changes list a few times
-      tries = 0
-      bulk_string = ''
-      until (count = Redis.llen(:changes)) == 0 || tries > 9
-        count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
-        Elastics.post_bulk_string(:bulk_string => bulk_string)
-        bulk_string = ''
-        tries += 1
-      end
+      # try to empty the changes for 10 times before stopping the indexing
+      10.times{ index_changes(opts) }
+
       # at this point the changes list should be empty or contain the minimum number of changes we could achieve live
       # the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
-      @stop_indexing.call if @stop_indexing
-      # if we have still changes, we can index them (until the list will be empty)
-      bulk_string = ''
-      while (change = Redis.lpop(:changes))
-        bulk_string << build_bulk_string_from_change(change)
+      if @stop_indexing
+        Prompter.say_notice 'Calling on_stop_indexing...' if opts[:verbose]
+        @stop_indexing.call
+        Prompter.say_notice 'Indexing stopped.' if opts[:verbose]
+      else
+        Prompter.say_warning 'No on_stop_indexing provided!' if opts[:verbose]
       end
-      Elastics.post_bulk_string(:bulk_string => bulk_string)
+
+      # if we have still changes, we can index them all, now that the indexing is stopped
+      index_changes(opts)
 
       # deletes the old indices and create the aliases to the new
       @indices.each do |index|
         Elastics.delete_index :index => index
         Elastics.put_index_alias :alias => index,
-                             :index => @timestamp + index
+                                 :index => @timestamp + index
       end
       # after the execution of this method the user should deploy the new code and then resume the regular app processing
 
@@ -201,8 +203,28 @@ module Elastics
       Redis.reset_keys
     end
 
+    def index_changes(opts)
+      left_changes_count = Redis.llen(:changes)
+      return if left_changes_count == 0
+
+      batch_size  = opts[:batch_size] || 100
+      bulk_string = ''
+      Prompter.say_notice "Reindexing #{left_changes_count} live-changes..." if opts[:verbose]
+
+      until left_changes_count == 0
+        batch_count = left_changes_count > batch_size ? batch_size : left_changes_count
+        batch_count.times do
+          bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))
+          left_changes_count -= 1
+        end
+        Elastics.post_bulk_string(:bulk_string => bulk_string)
+        bulk_string = ''
+      end
+    end
 
     def migrate_indices(opts)
+      Conf.http_client.options[:timeout] = opts[:timeout] || 60
+
       opts[:verbose] = true unless opts.has_key?(:verbose)
       pbar = ProgBar.new(Elastics.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastics-admin
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-22 00:00:00.000000000 Z
+date: 2013-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elastics-client
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.7
+        version: 1.0.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.7
+        version: 1.0.8
 description: Provides binary and rake tasks to dump, load and optionally rename indices.
   Implements live-reindex with hot-swap of old code/index with new code/index.
 email: dd.nexus@gmail.com