ej 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ba7f44c1f49100a84246846a99b87fa467ead32
-  data.tar.gz: 6223584d39aea4b09bfc2b87ecf8dd5f9e0a1127
+  metadata.gz: 71050072d6972560642c232f88d3f04c35618496
+  data.tar.gz: d84942d99527ff336b81029a9d13f5f90971e904
 SHA512:
-  metadata.gz: c167c8f5deea539ab58a5749f2bb6053603217fc5c7b227b7f34b5b77d52dba7211f0ef277bdbebb1343604435bbfca42a1ce76f5843e90951bfc4ef0814b536
-  data.tar.gz: 638512a42e5d2ddcdaedaa19085b2d17c054c338aacaecb8b92b1f7edddc610447e0ff3b80e42ecb0f80fe1bf09b88e8998eb63e8f3f4089bd7a3b47fdd7ec26
+  metadata.gz: dd9a4ab8ddb693fcfe77f10a2bf0146cf74c120d23aba43c428285c6ea1fbe2979edde28ca9cccfc573eb0d27cf6305b85586856a48ae32a84153182cb6aee18
+  data.tar.gz: bfb96e1a5258d6fa6714259216bfa7ed6f4fb4ad8264a271a2927b28e0a121b786cf343278f7ae5616e1642e846add75faee24b6b229de6518deb0fec15c5714

data/ej.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "yajl-ruby"
   spec.add_runtime_dependency "elasticsearch"
   spec.add_runtime_dependency "hashie"
+  spec.add_runtime_dependency "parallel"
 end

data/lib/ej/commands.rb

@@ -68,8 +68,9 @@ module Ej
     option :dest, type: :string, aliases: '--dest', required: true, desc: 'dest host'
     option :query, type: :string, aliases: '-q', default: nil, desc: 'query'
     option :per, type: :numeric, default: nil, desc: 'per'
+    option :proc_num, type: :numeric, default: 1, desc: 'proc num for multi thread'
     def copy
-      @core.copy(options['source'], options['dest'], options['query'], options['per'])
+      @core.copy(options['source'], options['dest'], options['query'], options['per'], options['proc_num'])
     end
 
     desc 'dump', 'dump index'

data/lib/ej/core.rb

@@ -5,12 +5,15 @@ require 'yajl'
 require 'elasticsearch'
 require 'hashie'
 require 'pp'
+require 'parallel'
 
 module Ej
   class Core
     DEFAULT_PER = 1000
     def initialize(host, index, debug)
-      @logger =  debug ? Logger.new($stderr) : nil
+      @logger =  Logger.new($stderr)
+      @logger.level = debug ? Logger::DEBUG : Logger::INFO
+
       @index = index
       @client = Elasticsearch::Client.new hosts: host, logger: @logger, index: @index
     end
@@ -39,20 +42,23 @@ module Ej
       @client.search index: @index, type: type, body: body
     end
 
-    def copy(source, dest, query, per_size)
+    def copy(source, dest, query, per_size, proc_num)
       per = per_size || DEFAULT_PER
       num = 0
       logger = Logger.new($stdout)
       source_client = Elasticsearch::Client.new hosts: source, index: @index
       dest_client = Elasticsearch::Client.new hosts: dest
-      while true
+      calculate_body = { size: 0 }
+      calculate_body[:query] = { query_string: { query: query } } unless query.nil?
+      calculate_data = Hashie::Mash.new(source_client.search index: @index, body: calculate_body)
+      total = calculate_data.hits.total
+      payloads = ((total/per) + 1).times.to_a
+      Parallel.map(payloads, in_processes: proc_num) do |num|
         from = num * per
         body = { size: per, from: from }
         body[:query] = { query_string: { query: query } } unless query.nil?
         data = Hashie::Mash.new(source_client.search index: @index, body: body)
         docs = data.hits.hits
-        total = data.hits.total
-        break if docs.empty?
         bulk_message = []
         docs.each do |doc|
           source = doc.delete('_source')
@@ -63,9 +69,9 @@ module Ej
           bulk_message << { index: doc.to_h }
           bulk_message << source
         end
-        dest_client.bulk body: bulk_message unless bulk_message.empty?
-        logger.info("copy complete #{from + docs.size}/#{total}")
-        num += 1
+        send_with_retry(dest_client, bulk_message)
+
+        logger.info("copy complete (#{from}-#{from + docs.size})/#{total}")
       end
     end
 
@@ -243,12 +249,27 @@ module Ej
         bulk_message << record
       end
       bulk_message.each_slice(10000).each do |block|
-        @client.bulk body: block
+        send_with_retry(@client, block)
       end
     end
 
     private
 
+    def send_with_retry(client, bulk_message, retry_on_failure = 5)
+      retries = 0
+      begin
+        client.bulk body: bulk_message unless bulk_message.empty?
+      rescue => e
+        if retries < retry_on_failure
+          retries += 1
+          @logger.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+          sleep 2**retries
+          retry
+        end
+        raise "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+      end
+    end
+
     def parse_json(buffer)
       begin
         data = Yajl::Parser.parse(buffer)

data/lib/ej/version.rb

@@ -1,3 +1,3 @@
 module Ej
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ej
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - toyama0919
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-02 00:00:00.000000000 Z
+date: 2017-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: elasticsearch command line utility. Returns all results in json.
 email:
 - toyama0919@gmail.com