es-reindex 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7620b0c56bc7c9531fd32c9c573bfef963a3a2eb
+  data.tar.gz: 86eb8a9c6f896e6697d3adb3de6af34dc99fd866
+SHA512:
+  metadata.gz: 853868790c7f1b8658995ec38b4d9b5fc87bdf5658b6fa9dcca32d5a64cb6c14e1af9b294e1256e36c18f38bf828e6a0daeadf5d48efb607e110020e7020d772
+  data.tar.gz: fd4f29b8dadb1cacf345161e2928f541ee4a366ad98349d24de49fac848bdb13dd602f3fbcb9698d52bb90d17a94bb2cee30d8c8779a1e6c00a1f068c0a2b91f

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+coverage/*

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format documentation

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, all_on_start: false, cmd: 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec/" }
+end

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2012 Jiri Nemecek
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.markdown

@@ -0,0 +1,73 @@
+# es-reindex - simple ruby gem for copying ElasticSearch index
+
+Simple ruby gem to copy and reindex ElasticSearch index,
+copying index settings and mapping(s).
+
+Progress and time estimation is displayed during the scrolling process.
+
+## Requirements
+
+Ruby 1.9.3 or newer is required, use [rvm](https://rvm.io/) for convenience.
+
+Following gems are required:
+
++ [rest-client](https://github.com/archiloque/rest-client)
++ [multi_json](https://github.com/intridea/multi_json)
+
+## Usage (command line)
+
+Refer to script's help:
+
+```bash
+$ ./es-reindex.rb -h
+
+Script to copy particular ES index including its (re)creation w/options set
+and mapping copied.
+
+Usage:
+
+  ./es-reindex.rb [-r] [-f <frame>] [source_url/]<index> [destination_url/]<index>
+
+    - -r - remove the index in the new location first
+    - -f - specify frame size to be obtained with one fetch during scrolling
+    - -u - update existing documents (default: only create non-existing)
+    - optional source/destination urls default to http://127.0.0.1:9200
+```
+
+## Usage (in project)
+
+You can also use it as a PORO:
+
+```ruby
+# Options:
+# remove: same as -r
+# frame: same as -f
+# update: same as -u cli option
+
+options = {
+  remove: true,
+  update: true
+}
+
+ESReindex.new('http://my_server/index', 'http://my_server/index_copy', options).go!
+```
+
+## Changelog
+
++ __0.1.0__: First gem release
++ __0.0.9__: Gemification, Oj -> MultiJSON
++ __0.0.8__: Optimization in string concat (@nara)
++ __0.0.7__: Document header arguments `_timestamp` and `_ttl` are copied as well
++ __0.0.6__: Document headers in bulks are now assembled and properly JSON dumped
++ __0.0.5__: Merge fix for trailing slash in urls (@ichinco), formatting cleanup
++ __0.0.4__: Force create only, update is optional (@pgaertig)
++ __0.0.3__: Yajl -> Oj
++ __0.0.2__: repated document count comparison
++ __0.0.1__: first revision
+
+## License
+es-reindex the gem is copyright (c)2014 Mojo Lingo, and released under the terms
+of the MIT license. See the LICENSE file for the gory details.
+
+es-reindex is copyright (c)2012 Jiri Nemecek, and released under the terms
+of the MIT license. See the LICENSE file for the gory details.

data/bin/es-reindex

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'es-reindex'
+require 'es-reindex/args_parser'
+
+STDOUT.sync = true
+
+if ARGV.size == 0 or ARGV[0] =~ /^-(?:h|-?help)$/
+  puts "
+  Script to copy particular ES index including its (re)creation w/options set
+  and mapping copied.
+
+  Usage:
+
+    #{__FILE__} [-r] [-f <frame>] [source_url/]<index> [destination_url/]<index>
+
+      - -r - remove the index in the new location first
+      - -f - specify frame size to be obtained with one fetch during scrolling
+      - -u - update existing documents (default: only create non-existing)
+      - optional source/destination urls default to http://127.0.0.1:9200
+  \n"
+
+  exit 1
+end
+
+src, dst, options = ESReindex::ArgsParser.parse(ARGV)
+ESReindex.new(src, dst, options).go!

data/es-reindex.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+require "es-reindex/version"
+
+Gem::Specification.new do |s|
+  s.name        = "es-reindex"
+  s.version     = ESReindex::VERSION
+  s.authors     = ["Justin Aiken"]
+  s.email       = ["jaiken@mojolingo.com"]
+  s.homepage    = "https://github.com/mojolingo/es-reindex"
+  s.summary     = %q{Ruby gem to copy ElasticSearch index (reindex).}
+  s.description = %q{Ruby gem to copy ElasticSearch index (reindex).}
+  s.license     = 'MIT'
+
+  s.rubyforge_project = "es-reindex"
+
+  s.required_ruby_version = '>= 1.9.3'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'rest-client', '>= 1.6.0'
+  s.add_runtime_dependency 'multi_json'
+
+  s.add_development_dependency %q<coveralls>, ['>= 0']
+  s.add_development_dependency %q<bundler>, ["~> 1.0"]
+  s.add_development_dependency %q<rspec>, ["~> 2.99"]
+  s.add_development_dependency %q<rake>, [">= 0"]
+  s.add_development_dependency %q<guard-rspec>, ['~> 4.5']
+ end

data/lib/es-reindex.rb

@@ -0,0 +1,190 @@
+require 'rest-client'
+require 'multi_json'
+
+class ESReindex
+
+  DEFAULT_URL = 'http://10.203.175.32:9200'
+
+  attr_accessor :src, :dst, :options, :start_time, :done
+
+  def initialize(src, dst, options = {})
+    @src     = src || ''
+    @dst     = dst || ''
+    @options = {
+      remove: false, # remove the index in the new location first
+      update: false, # update existing documents (default: only create non-existing)
+      frame:  1000   # specify frame size to be obtained with one fetch during scrolling
+    }.merge! options
+
+    @done = 0
+  end
+
+  def go!
+    MultiJson.load_options = {mode: :compat}
+    MultiJson.dump_options = {mode: :compat}
+
+    surl, durl, sidx, didx = '', '', '', ''
+    [[src, surl, sidx], [dst, durl, didx]].each do |param, url, idx|
+      if param =~ %r{^(.*)/(.*?)$}
+        url.replace $1
+        idx.replace $2
+      else
+        url.replace DEFAULT_URL
+        idx.replace param
+      end
+    end
+
+    printf "Copying '%s/%s' to '%s/%s'%s\n  Confirm or hit Ctrl-c to abort...\n",
+      surl, sidx, durl, didx,
+      remove? ?
+        ' with rewriting destination mapping!' :
+        update? ? ' with updating existing documents!' : '.'
+
+    $stdin.readline
+
+    # remove old index in case of remove=true
+    retried_request(:delete, "#{durl}/#{didx}") if remove? && retried_request(:get, "#{durl}/#{didx}/_status")
+
+    # (re)create destination index
+    unless retried_request :get, "#{durl}/#{didx}/_status"
+      # obtain the original index settings first
+      unless settings = retried_request(:get, "#{surl}/#{sidx}/_settings")
+        warn "Failed to obtain original index '#{surl}/#{sidx}' settings!"
+        exit 1
+      end
+      settings = MultiJson.load settings
+      sidx = settings.keys[0]
+      settings[sidx].delete 'index.version.created'
+      printf 'Creating \'%s/%s\' index with settings from \'%s/%s\'... ', durl, didx, surl, sidx
+      unless retried_request :post, "#{durl}/#{didx}", MultiJson.dump(settings[sidx])
+        puts 'FAILED!'
+        exit 1
+      else
+        puts 'OK.'
+      end
+      unless mappings = retried_request(:get, "#{surl}/#{sidx}/_mapping")
+        warn "Failed to obtain original index '#{surl}/#{sidx}' mappings!"
+        exit 1
+      end
+      mappings = MultiJson.load mappings
+      mappings = mappings[sidx]
+      mappings = mappings['mappings'] if mappings.is_a?(Hash) && mappings.has_key?('mappings')
+      mappings.each_pair do |type, mapping|
+        printf 'Copying mapping \'%s/%s/%s\'... ', durl, didx, type
+        unless retried_request(:put, "#{durl}/#{didx}/#{type}/_mapping", MultiJson.dump(type => mapping))
+          puts 'FAILED!'
+          exit 1
+        else
+          puts 'OK.'
+        end
+      end
+    end
+
+    printf "Copying '%s/%s' to '%s/%s'... \n", surl, sidx, durl, didx
+    @start_time = Time.now
+    shards = retried_request :get, "#{surl}/#{sidx}/_count?q=*"
+    shards = MultiJson.load(shards)['_shards']['total'].to_i
+    scan = retried_request :get, "#{surl}/#{sidx}/_search?search_type=scan&scroll=10m&size=#{frame / shards}"
+    scan = MultiJson.load scan
+    scroll_id = scan['_scroll_id']
+    total = scan['hits']['total']
+    printf "    %u/%u (%.1f%%) done.\r", done, total, 0
+
+    bulk_op = update? ? 'index' : 'create'
+
+    while true do
+      data = retried_request :get, "#{surl}/_search/scroll?scroll=10m&scroll_id=#{scroll_id}"
+      data = MultiJson.load data
+      break if data['hits']['hits'].empty?
+      scroll_id = data['_scroll_id']
+      bulk = ''
+      data['hits']['hits'].each do |doc|
+        ### === implement possible modifications to the document
+        ### === end modifications to the document
+        base = {'_index' => didx, '_id' => doc['_id'], '_type' => doc['_type']}
+        ['_timestamp', '_ttl'].each{|doc_arg|
+          base[doc_arg] = doc[doc_arg] if doc.key? doc_arg
+        }
+        bulk << MultiJson.dump(bulk_op => base) + "\n"
+        bulk << MultiJson.dump(doc['_source'])  + "\n"
+        @done = done + 1
+      end
+      unless bulk.empty?
+        bulk << "\n" # empty line in the end required
+        retried_request :post, "#{durl}/_bulk", bulk
+      end
+
+      eta = total * (Time.now - start_time) / done
+      printf "    %u/%u (%.1f%%) done in %s, E.T.A.: %s.\r",
+        done, total, 100.0 * done / total, tm_len, start_time + eta
+    end
+
+    printf "#{' ' * 80}\r    %u/%u done in %s.\n", done, total, tm_len
+
+    # no point for large reindexation with data still being stored in index
+    printf 'Checking document count... '
+    scount, dcount = 1, 0
+    begin
+      Timeout::timeout(60) do
+        while true
+          scount = retried_request :get, "#{surl}/#{sidx}/_count?q=*"
+          dcount = retried_request :get, "#{durl}/#{didx}/_count?q=*"
+          scount = MultiJson.load(scount)['count'].to_i
+          dcount = MultiJson.load(dcount)['count'].to_i
+          break if scount == dcount
+          sleep 1
+        end
+      end
+    rescue Timeout::Error
+    end
+    printf "%u == %u (%s\n", scount, dcount, scount == dcount ? 'equals).' : 'NOT EQUAL)!'
+
+    exit 0
+  end
+
+private
+
+  def remove?
+    @options[:remove]
+  end
+
+  def update?
+    @options[:update]
+  end
+
+  def frame
+    @options[:frame]
+  end
+
+  def tm_len
+    l = Time.now - @start_time
+    t = []
+    t.push l/86400; l %= 86400
+    t.push l/3600;  l %= 3600
+    t.push l/60;    l %= 60
+    t.push l
+    out = sprintf '%u', t.shift
+    out = out == '0' ? '' : out + ' days, '
+    out << sprintf('%u:%02u:%02u', *t)
+    out
+  end
+
+  def retried_request(method, url, data=nil)
+    while true
+      begin
+        return data ?
+          RestClient.send(method, url, data) :
+          RestClient.send(method, url)
+      rescue RestClient::ResourceNotFound # no point to retry
+        return nil
+      rescue RestClient::BadRequest => e # Something's wrong!
+        warn "\n#{method.to_s.upcase} #{url} :-> ERROR: #{e.class} - #{e.message}"
+        warn e.response
+        return nil
+      rescue => e
+        warn "\nRetrying #{method.to_s.upcase} ERROR: #{e.class} - #{e.message}"
+        warn e.response
+      end
+    end
+  end
+end

data/lib/es-reindex/args_parser.rb

@@ -0,0 +1,26 @@
+class ESReindex
+  class ArgsParser
+
+    def self.parse(args)
+      remove, update, frame, src, dst = false, false, 1000, nil, nil
+
+      while args[0]
+        case arg = args.shift
+        when '-r' then remove = true
+        when '-f' then frame  = args.shift.to_i
+        when '-u' then update = true
+        else
+          u = arg.chomp '/'
+          !src ? (src = u) : !dst ? (dst = u) :
+            raise("Unexpected parameter '#{arg}'. Use '-h' for help.")
+        end
+      end
+
+      return src, dst, {
+        remove: remove,
+        frame:  frame,
+        update: update
+      }
+    end
+  end
+end

data/lib/es-reindex/version.rb

@@ -0,0 +1,3 @@
+class ESReindex
+  VERSION = '0.1.0'
+end

data/spec/es-reindex/args_parser_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe ESReindex::ArgsParser do
+
+  let(:parsed_args)  { described_class.parse args }
+  let(:parsed_src)   { parsed_args[0] }
+  let(:parsed_dst)   { parsed_args[1] }
+  let(:parsed_opts)  { parsed_args[2] }
+
+  context "with no src or dst" do
+    let(:args) { ["-r", "-u"] }
+
+    it "src is nil" do
+      expect(parsed_src).to be_nil
+    end
+
+    it "dst is nil" do
+      expect(parsed_dst).to be_nil
+    end
+  end
+
+  context "with a src" do
+    let(:args) { ["-r", "-u", "http://foo/index"] }
+
+    it "src is set" do
+      expect(parsed_src).to eq 'http://foo/index'
+    end
+
+    it "dst is nil" do
+      expect(parsed_dst).to be_nil
+    end
+  end
+
+  context "with a src and dst" do
+    let(:args) { ["-r", "-u", "http://foo/index", "bar/index"] }
+
+    it "src is set" do
+      expect(parsed_src).to eq 'http://foo/index'
+    end
+
+    it "dst is set" do
+      expect(parsed_dst).to eq 'bar/index'
+    end
+  end
+
+  context "without -f" do
+    let(:args) { ["-u"] }
+
+    it "sets the frame to the default (1000)" do
+      expect(parsed_opts[:frame]).to eq 1000
+    end
+  end
+
+  context "with -f" do
+    let(:args) { ["-f", "1500"] }
+
+    it "sets the frame" do
+      expect(parsed_opts[:frame]).to eq 1500
+    end
+  end
+
+  context "without -u" do
+    let(:args) { ["-f", '1000', 'foosrc/index'] }
+
+    it "sets update to false" do
+      expect(parsed_opts[:update]).to be false
+    end
+  end
+
+  context "with -u" do
+    let(:args) { ["-u"] }
+
+    it "sets the frame" do
+      expect(parsed_opts[:update]).to be true
+    end
+  end
+end

data/spec/es-reindex_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe ESReindex do
+  let(:src)     { 'http://foo/bar_index' }
+  let(:dst)     { 'http://biz/baz_index' }
+  let(:options) { {update: true} }
+
+  let(:reindexer) { ESReindex.new src, dst, options }
+
+  it "can be freshly initialized with options" do
+    expect(reindexer.options).to eq remove: false, update: true, frame: 1000
+  end
+
+  it "starts with 0 indexes done" do
+    expect(reindexer.done).to eq 0
+  end
+
+  skip "it can actually do stuff"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'coveralls'
+Coveralls.wear!
+
+require 'es-reindex'
+require 'es-reindex/args_parser'
+
+RSpec.configure do |config|
+  config.tty = true
+
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+end
+

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: es-reindex
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Justin Aiken
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+description: Ruby gem to copy ElasticSearch index (reindex).
+email:
+- jaiken@mojolingo.com
+executables:
+- es-reindex
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Guardfile
+- LICENSE
+- README.markdown
+- bin/es-reindex
+- es-reindex.gemspec
+- lib/es-reindex.rb
+- lib/es-reindex/args_parser.rb
+- lib/es-reindex/version.rb
+- spec/es-reindex/args_parser_spec.rb
+- spec/es-reindex_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/mojolingo/es-reindex
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: es-reindex
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Ruby gem to copy ElasticSearch index (reindex).
+test_files:
+- spec/es-reindex/args_parser_spec.rb
+- spec/es-reindex_spec.rb
+- spec/spec_helper.rb