flex-admin 1.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012-2013 by Domizio Demichelis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+# Flex-admin
+
+Simple tool to dump/load one or more elasticsearch indices and types.
+
+## Links
+
+* [Flex Repository](https://github.com/ddnexus/flex)
+* [Flex Project (Global Documentation)](http://ddnexus.github.io/flex/doc/)
+* [flex-admin Gem (Specific Documentation)](http://ddnexus.github.io/flex/doc/6-flex-admin)
+* [Issues](https://github.com/ddnexus/flex-admin/issues)
+* [Pull Requests](https://github.com/ddnexus/flex-admin/pulls)
+
+## Branches
+
+The master branch reflects the last published gem. Then you may find a next-version branch (named after the version string), with the commits that will be merged in master just before publishing the next gem version. The next-version branch may get rebased or force pushed.
+
+## Credits
+
+Special thanks for their sponsorship to [Escalate Media](http://www.escalatemedia.com) and [Barquin International](http://www.barquin.com).
+
+## Copyright
+
+Copyright (c) 2012-2013 by [Domizio Demichelis](mailto://dd.nexus@gmail.com)<br>
+See [LICENSE](https://github.com/ddnexus/flex-admin/blob/master/LICENSE) for details.

data/VERSION

@@ -0,0 +1 @@
+1.0.1

data/bin/flex-admin

@@ -0,0 +1,112 @@
+#! /usr/bin/env ruby
+
+require 'optparse'
+require 'flex-admin'
+
+options  = Flex::Admin::Tasks.new.default_options
+
+version  = File.read(File.expand_path('../../VERSION', __FILE__)).strip
+copy     = "flex-admin #{version} (c) 2012-2013 by Domizio Demichelis"
+optparse = OptionParser.new do |opts|
+
+  opts.banner = <<-banner
+
+    flex-admin:
+        Generic binary tool to dump/load data from/to any elasticsearch index (no app needed).
+        If you need to migrate data, use the flex live-reindexing.
+
+    Usage:
+        flex-admin <command> [options]
+    <command>:
+        dump    dumps the data from one or more elasticsearch indices
+        load    loads a dumpfile
+        stats   prints the full elasticsearch stats
+
+    Notice: The load command will load the dump-file into elasticsearch without removing any pre-existent data.
+            If you need fresh indices, use the flex:index:delete and flex:index:create rake tasks from your
+            application, which will also recreate the mapping.
+  banner
+
+
+  opts.separator ''
+  opts.separator 'Common options:'
+
+  opts.on( '-f', '--file [FILE]', "The path of the dumpfile (default: '#{options[:file]}')" ) do |f|
+    options[:file] = f
+  end
+
+  opts.on( '-r', '--[no-]verbose', "Run verbosely (default: '#{options[:verbose]}')" ) do |q|
+    options[:verbose] = q
+  end
+
+  opts.separator ''
+  opts.separator 'Dump options:'
+
+  opts.on( '-i', '--index [INDEX_OR_INDICES]', Array, 'The index or comma separated indices to dump (default: all indices)' ) do |i|
+    options[:index] = i
+  end
+
+  opts.on( '-t', '--type [TYPE_OR_TYPES]', Array, 'The type or comma separated types to dump (default: all types)' ) do |t|
+    options[:type] = t
+  end
+
+  opts.on( '-s', '--scroll [TIME]', Integer, "The ElasticSearch scroll time (default: #{options[:scroll]})" ) do |s|
+    options[:scroll] = s
+  end
+
+  opts.on( '-z', '--size [SIZE]', Integer, "The chunk size to dump per shard (default: #{options[:size]} * number of shards)" ) do |z|
+    options[:size] = z
+  end
+
+  opts.separator ''
+  opts.separator 'Load options:'
+
+  opts.on( '-m', '--index-map [INDEX_MAP]', 'The index rename map (example: -m=dumped_index_name:loaded_index_name,a:b)') do |m|
+    options[:index_map] = Hash[m.split(',').map{|i|i.split(':')}]
+  end
+
+  opts.on( '-o', '--timeout [SECONDS]', Integer, "The http_client timeout for bulk loading (default: #{options[:timeout]} seconds)" ) do |o|
+    options[:timeout] = o
+  end
+
+  opts.on( '-b', '--batch-size [BATCH_SIZE]', Integer, "The batch size to load (default: #{options[:batch_size]})" ) do |z|
+    options[:batch_size] = z
+  end
+
+  opts.separator ''
+  opts.separator 'Other options:'
+
+  opts.on( '-v', '--version', 'Shows the version and exits' ) do
+    puts version
+    exit
+  end
+
+  opts.on_tail( '-h', '--help', 'Displays this screen' ) do
+    puts copy
+    puts opts
+    exit
+  end
+
+end
+
+optparse.parse!
+command = ARGV.first
+exec "#{$0} -h" if command.nil?
+puts copy
+
+case command
+
+when 'dump'
+  Flex::Admin::Tasks.new(options).dump_to_file(true)
+
+when 'load'
+  Flex::Admin::Tasks.new(options).load_from_file
+
+when 'stats'
+  puts '>> puts Flex.index_stats.to_yaml'
+  puts Flex.index_stats.to_yaml
+
+else
+  puts "unknown command: #{command.inspect}"
+
+end

data/flex-admin.gemspec

@@ -0,0 +1,20 @@
+require 'date'
+version = File.read(File.expand_path('../VERSION', __FILE__)).strip
+
+Gem::Specification.new do |s|
+  s.name                      = 'flex-admin'
+  s.summary                   = 'Dump/load/rename/live-redindex one or more elasticsearch indices and types.'
+  s.description               = 'Provides binary and rake tasks to dump, load and optionally rename indices. Implements live-reindex with hot-swap of old code/index with new code/index.'
+  s.homepage                  = 'http://github.com/ddnexus/flex-admin'
+  s.authors                   = ["Domizio Demichelis"]
+  s.email                     = 'dd.nexus@gmail.com'
+  s.executables               = %w[flex-admin]
+  s.extra_rdoc_files          = %w[README.md]
+  s.files                     = `git ls-files -z`.split("\0")
+  s.version                   = version
+  s.date                      = Date.today.to_s
+  s.required_rubygems_version = ">= 1.3.6"
+  s.rdoc_options              = %w[--charset=UTF-8]
+
+  s.add_runtime_dependency 'flex', version
+end

data/lib/flex-admin.rb

@@ -0,0 +1,7 @@
+require 'flex'
+require 'flex/admin'
+require 'flex/live_reindex'
+
+Flex::LIB_PATHS << File.dirname(__FILE__)
+
+Flex::Conf.redis = $redis || defined?(::Redis) && ::Redis.current

data/lib/flex/admin.rb

@@ -0,0 +1,114 @@
+module Flex
+  module Admin
+
+    class Tasks
+
+      attr_reader :options
+
+      def initialize(overrides={})
+        options = Flex::Utils.env2options *default_options.keys
+
+        options[:size]       = options[:size].to_i       if options[:size]
+        options[:timeout]    = options[:timeout].to_i    if options[:timeout]
+        options[:batch_size] = options[:batch_size].to_i if options[:batch_size]
+        options[:index_map]  = Hash[options[:index_map].split(',').map{|i|i.split(':')}] if options[:index_map] && options[:index_map].is_a?(String)
+
+        @options = default_options.merge(options).merge(overrides)
+      end
+
+      def default_options
+        @default_options ||= { :file       => './flex.dump',
+                               :index      => Conf.variables[:index],
+                               :type       => Conf.variables[:type],
+                               :scroll     => '5m',
+                               :size       => 50,
+                               :timeout    => 20,
+                               :batch_size => 1000,
+                               :verbose    => true,
+                               :index_map  => nil }
+      end
+
+      def dump_to_file(cli=false)
+        vars = { :index => cli ? options[:index] : (options[:index] || Flex::Tasks.new.config_hash.keys),
+                 :type  => options[:type] }
+        if options[:verbose]
+          total_hits  = Flex.count(vars)['count'].to_i
+          total_count = 0
+          pbar        = ProgBar.new(total_hits)
+          dump_stats  = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = 0 } }
+          file_size   = 0
+        end
+        vars.merge! :params => { :scroll => options[:scroll],
+                                 :size   => options[:size],
+                                 :fields => '_source,*' }
+
+        file = options[:file].is_a?(String) ? File.open(options[:file], 'wb') : options[:file]
+        path = file.path
+
+        Flex.dump_all(vars) do |batch|
+          bulk_string = ''
+          batch.each do |document|
+            dump_stats[document['_index']][document['_type']] += 1 if options[:verbose]
+            bulk_string << Flex.build_bulk_string(document)
+          end
+          file.puts bulk_string
+          if options[:verbose]
+            total_count += batch.size
+            pbar.pbar.inc(batch.size)
+          end
+        end
+        file_size = file.size if options[:verbose]
+        file.close
+
+        if options[:verbose]
+          formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
+          pbar.pbar.finish
+          puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
+               unless total_hits == total_count
+          puts "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
+          puts dump_stats.to_yaml
+        end
+      end
+
+      def load_from_file
+        Configuration.http_client.options[:timeout] = options[:timeout]
+        chunk_size  = options[:batch_size] * 2 # 2 lines per doc
+        bulk_string = ''
+        file        = options[:file].is_a?(String) ? File.open(options[:file]) : options[:file]
+        path        = file.path
+        if options[:verbose]
+          line_count = 0
+          file.lines { line_count += 1 }
+          file.rewind
+          puts "\nLoading from #{path}...\n"
+          pbar = ProgBar.new(line_count / 2, options[:batch_size])
+        end
+        file.lines do |line|
+          bulk_string << (options[:index_map] ? map_index(line) : line)
+          if (file.lineno % chunk_size) == 0
+            result = Flex.post_bulk_string :bulk_string => bulk_string
+            bulk_string  = ''
+            pbar.process_result(result, options[:batch_size]) if options[:verbose]
+          end
+        end
+        # last chunk
+        unless bulk_string == ''
+          result = Flex.post_bulk_string :bulk_string => bulk_string
+          pbar.process_result(result, (file.lineno % chunk_size) / 2) if options[:verbose]
+        end
+        file.close
+        pbar.finish if options[:verbose]
+      end
+
+    private
+
+      def map_index(line)
+        joined_keys = options[:index_map].keys.join('|')
+        line.sub(/"_index":"(#{joined_keys})"/) do |match_string|
+          options[:index_map].has_key?($1) ? %("_index":"#{options[:index_map][$1]}") : match_string
+        end
+      end
+
+    end
+  end
+end

data/lib/flex/live_reindex.rb

@@ -0,0 +1,248 @@
+module Flex
+  # private module
+  module LiveReindex
+
+    class MissingRedisError            < StandardError; end
+    class LiveReindexInProgressError   < StandardError; end
+    class MissingAppIdError            < StandardError; end
+    class MissingStopIndexingProcError < StandardError; end
+    class MissingEnsureIndicesError    < StandardError; end
+    class MissingOnReindexBlockError   < StandardError; end
+    class ExtraIndexError              < StandardError; end
+    class MultipleReindexError         < StandardError; end
+
+    # private module
+    module Redis
+
+      KEYS = { :pid     => 'flex-reindexing-pid',
+               :changes => 'flex-reindexing-changes' }
+
+      extend self
+
+      def method_missing(command, key, *args)
+        return unless Conf.redis
+        Conf.redis.send(command, "#{KEYS[key]}-#{Conf.app_id}", *args)
+      end
+
+      def reset_keys
+        KEYS.keys.each { |k| del k }
+      end
+
+      def init
+        begin
+          require 'redis'
+        rescue LoadError
+          raise MissingRedisError, 'The live-reindex feature rely on redis. Please, install redis and the "redis" gem.'
+        end
+        raise MissingAppIdError, 'You must set the Flex::Configuration.app_id, and be sure you deploy it before live-reindexing.' \
+              if Conf.app_id.nil? || Conf.app_id.empty?
+        raise LiveReindexInProgressError, %(It looks like a live-reindex is in progress (PID #{get(:pid)}). If you are sure that there is no live-reindex in progress, please run the "flex:reset_redis_keys" rake task and retry.) \
+              if get(:pid)
+        reset_keys # just in case
+        set(:pid, $$)
+      end
+
+    end
+
+    extend self
+
+    def on_reindex(&block)
+      @reindex = block
+    end
+
+    def on_each_change(&block)
+      @each_change = block
+    end
+    attr_reader :each_change
+
+    def on_stop_indexing(&block)
+      @stop_indexing = block
+    end
+
+    def reindex(opts={})
+      yield self
+      perform(opts)
+    end
+
+    def reindex_indices(opts={})
+      yield self if block_given?
+
+      opts[:verbose] = true unless opts.has_key?(:verbose)
+      opts[:index] ||= opts.delete(:indices) || config_hash.keys
+
+      # we override the on_reindex eventually set
+      on_reindex do
+        migrate_indices(opts)
+      end
+
+      perform(opts)
+    end
+
+    def should_prefix_index?
+      Redis.get(:pid) == $$.to_s
+    end
+
+    def should_track_change?
+      pid = Redis.get(:pid)
+      !!pid && !(pid == $$.to_s)
+    end
+
+    def track_change(action, document)
+      Redis.rpush(:changes, MultiJson.encode([action, document]))
+    end
+
+    # use this method when you are tracking the change of another app
+    # you must pass the app_id of the app being affected by the change
+    def track_external_change(app_id, action, document)
+      return unless Conf.redis
+      Conf.redis.rpush("#{KEYS[:changes]}-#{app_id}", MultiJson.encode([action, document]))
+    end
+
+    def prefix_index(index)
+      base = unprefix_index(index)
+      # raise if base is not included in @ensure_indices
+      raise ExtraIndexError, "The index #{base} is missing from the :ensure_indices option. Reindexing aborted." \
+            if @ensure_indices && !@ensure_indices.include?(base)
+      prefixed = @timestamp + base
+      unless @indices.include?(base)
+        unless Flex.exist?(:index => prefixed)
+          config_hash[base] = {} unless config_hash.has_key?(base)
+          Flex.POST "/#{prefixed}", config_hash[base]
+        end
+        @indices |= [base]
+      end
+      prefixed
+    end
+
+    # remove the (eventual) prefix
+    def unprefix_index(index)
+      index.sub(/^\d{14}_/, '')
+    end
+
+    private
+
+    def config_hash
+      @config_hash ||= ModelTasks.new.config_hash
+    end
+
+    def perform(opts={})
+      Conf.logger.warn 'Safe reindex is disabled!' if opts[:safe_reindex] == false
+      Redis.init
+      @indices        = []
+      @timestamp      = Time.now.strftime('%Y%m%d%H%M%S_')
+      @ensure_indices = nil
+
+      unless opts[:on_stop_indexing] == false || Conf.on_stop_indexing == false
+        @stop_indexing ||= Conf.on_stop_indexing || raise(MissingStopIndexingProcError, 'The on_stop_indexing block is not set.')
+      end
+
+      raise MissingOnReindexBlockError, 'You must configure an on_reindex block.' \
+            unless @reindex
+
+      raise MissingEnsureIndicesError, 'You must pass the :ensure_indices option when you pass the :models option.' \
+            if opts.has_key?(:models) && !opts.has_key?(:ensure_indices)
+      if opts[:ensure_indices]
+        @ensure_indices = opts.delete(:ensure_indices)
+        @ensure_indices = @ensure_indices.split(',') unless @ensure_indices.is_a?(Array)
+        each_change     = @each_change
+        @each_change    = nil
+        migrate_indices(:index => @ensure_indices)
+        @each_change    = each_change
+      end
+
+      @reindex.call
+
+      # when the reindexing is finished we try to empty the changes list a few times
+      tries = 0
+      bulk_string = ''
+      until (count = Redis.llen(:changes)) == 0 || tries > 9
+        count.times { bulk_string << build_bulk_string_from_change(Redis.lpop(:changes))}
+        Flex.post_bulk_string(:bulk_string => bulk_string)
+        bulk_string = ''
+        tries += 1
+      end
+      # at this point the changes list should be empty or contain the minimum number of changes we could achieve live
+      # the @stop_indexing should ensure to stop/suspend all the actions that would produce changes in the indices being reindexed
+      @stop_indexing.call if @stop_indexing
+      # if we have still changes, we can index them (until the list will be empty)
+      bulk_string = ''
+      while (change = Redis.lpop(:changes))
+        bulk_string << build_bulk_string_from_change(change)
+      end
+      Flex.post_bulk_string(:bulk_string => bulk_string)
+
+      # deletes the old indices and create the aliases to the new
+      @indices.each do |index|
+        Flex.delete_index :index => index
+        Flex.put_index_alias :alias => index,
+                             :index => @timestamp + index
+      end
+      # after the execution of this method the user should deploy the new code and then resume the regular app processing
+
+      # we redefine this method so it will raise an error if any new live-reindex is attempted during this session.
+      unless opts[:safe_reindex] == false
+        class_eval <<-ruby, __FILE__, __LINE__
+          def perform(*)
+            raise MultipleReindexError, "Multiple live-reindex attempted! You cannot use any reindexing method multiple times in the same session or you may corrupt your index/indices! The previous reindexing in this session successfully reindexed and swapped the new index/indices: #{@indices.map{|i| @timestamp + i}.join(', ')}. You must deploy now, and run the other reindexing in single successive deploys ASAP. Notice that if the code-changes that you are about to deploy rely on the successive reindexings that have been aborted, your app may fail. If you are working in development mode you must restart the session now. The next time you can silence this error by passing :safe_reindex => false"
+          end
+        ruby
+      end
+
+    rescue Exception
+      # delete all the created indices
+      @indices ||=[]
+      @indices.each do |index|
+        Flex.delete_index :index => @timestamp + index
+      end
+      raise
+
+    ensure
+      Redis.reset_keys
+    end
+
+
+    def migrate_indices(opts)
+      opts[:verbose] = true unless opts.has_key?(:verbose)
+      pbar = ProgBar.new(Flex.count(opts)['count'], nil, "index #{opts[:index].inspect}: ") if opts[:verbose]
+
+      Flex.dump_all(opts) do |batch|
+        result = process_and_post_batch(batch)
+        pbar.process_result(result, batch.size) if opts[:verbose]
+      end
+
+      pbar.finish if opts[:verbose]
+    end
+
+    def process_and_post_batch(batch)
+      bulk_string = ''
+      batch.each do |document|
+        bulk_string << build_bulk_string('index', document)
+      end
+      Flex.post_bulk_string(:bulk_string => bulk_string)
+    end
+
+    def build_bulk_string_from_change(change)
+      action, document = MultiJson.decode(change)
+      return '' unless @indices.include?(unprefix_index(document['_index']))
+      build_bulk_string(action, document)
+    end
+
+    def build_bulk_string(action, document)
+      result = if @each_change
+                 document.extend(Result::Document)
+                 document.extend(Result::DocumentLoader)
+                 @each_change.call(action, document)
+               else
+                 [{ action => document }]
+               end
+      result = [result] unless result.is_a?(Array)
+      bulk_string = ''
+      result.compact.each do |hash|
+        act, doc = hash.to_a.flatten
+        bulk_string << Flex.build_bulk_string(doc, :action => act)
+      end
+      bulk_string
+    end
+
+  end
+end

data/lib/tasks.rake

@@ -0,0 +1,17 @@
+require 'flex'
+require 'flex-admin'
+
+env = defined?(Rails) ? :environment : []
+
+namespace :flex do
+  namespace :admin do
+
+    desc 'Dumps the data from one or more ElasticSearch indices to a file'
+    task(:dump => env) { Flex::Admin::Tasks.new.dump_to_file }
+
+    desc 'Loads a dumpfile into ElasticSearch'
+    task(:load => env) { Flex::Admin::Tasks.new.load_from_file }
+
+  end
+
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: flex-admin
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Domizio Demichelis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: flex
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+description: Provides binary and rake tasks to dump, load and optionally rename indices.
+  Implements live-reindex with hot-swap of old code/index with new code/index.
+email: dd.nexus@gmail.com
+executables:
+- flex-admin
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- LICENSE
+- README.md
+- VERSION
+- bin/flex-admin
+- flex-admin.gemspec
+- lib/flex-admin.rb
+- lib/flex/admin.rb
+- lib/flex/live_reindex.rb
+- lib/tasks.rake
+homepage: http://github.com/ddnexus/flex-admin
+licenses: []
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Dump/load/rename/live-redindex one or more elasticsearch indices and types.
+test_files: []