ej 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e76e6b6fc5c6bebf55f97553afc843cbb9d9b730
-  data.tar.gz: '07830f36bd93af33e7686d264ef25dcf5b34d10b'
+  metadata.gz: ee1f79efd243a83e80d9b4ace0f58213ceee0c76
+  data.tar.gz: bc86ffef97ad3a387c332a735437c661f6825970
 SHA512:
-  metadata.gz: 96e69fe5ee1ba6ae3ecb8c53030b6710379bd312f8bef4ffcc5cea5d5e0281bfc3f11478b743d37122fcf01c5d06fe5476f4103805d18105a77f5a30c1926458
-  data.tar.gz: 9788d2673c94ac4f30084fc7951f660484268e440934a45495bfbe12ff1aeaa503708ff1af36c3b96d74b0539e75ed0df15816668d2602a9c36fde4b81aac3b5
+  metadata.gz: 4a1074107c6d568c830ee1153d5adf0b294bcaf7c408ee45117cd4a05b0a09504aa4d362b9f88d6b4bab4e916eb8d29687bbd327681ce8e258b94002cf144ee9
+  data.tar.gz: 5b627b9a60eb8cc01194795bb63aa99a9e28c03151b3964288a1bf18f763ae8f418f18706dbb8c95815e33598ca2b75bcd9731058d2a81bf2749fb29f814a2a6

data/ej.gemspec

@@ -24,7 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-doc"
   spec.add_development_dependency 'rspec', '~> 2.4'
   spec.add_runtime_dependency "thor"
-  spec.add_runtime_dependency "yajl-ruby"
   spec.add_runtime_dependency "elasticsearch"
   spec.add_runtime_dependency "hashie"
   spec.add_runtime_dependency "parallel"

data/lib/ej/commands.rb

@@ -1,6 +1,6 @@
 require 'ej/core'
 require 'thor'
-require 'yajl'
+require 'json'
 
 module Ej
   class Commands < Thor
@@ -71,6 +71,7 @@ module Ej
     option :per, type: :numeric, default: nil, desc: 'per'
     option :scroll, type: :string, default: "1m", desc: 'scroll'
     option :dest_index, type: :string, desc: 'dest index'
+    option :slice_max, type: :numeric, desc: 'slice max'
     def copy
       @core.copy(
         options[:source],
@@ -78,7 +79,8 @@ module Ej
         options[:query],
         options[:per],
         options[:scroll],
-        options[:dest_index]
+        options[:dest_index],
+        options[:slice_max]
       )
     end
 
@@ -171,7 +173,7 @@ module Ej
     desc 'not_analyzed', 'not analyzed'
     def not_analyzed
       json = File.read(File.expand_path('../../../template/not_analyze_template.json', __FILE__))
-      hash = Yajl::Parser.parse(json)
+      hash = JSON.parse(json)
       puts_with_format(@indices.put_template('ej_init', hash))
     end
 
@@ -186,7 +188,7 @@ module Ej
 
     desc 'put_template', 'put template'
     def put_template(name)
-      hash = Yajl::Parser.parse(STDIN.read)
+      hash = JSON.parse(STDIN.read)
       puts_with_format(@indices.put_template(name, hash))
     end
 
@@ -255,7 +257,7 @@ module Ej
     private
 
     def puts_with_format(object)
-      puts Yajl::Encoder.encode(object, pretty: true)
+      puts JSON.pretty_generate(object)
     end
 
   end

data/lib/ej/core.rb

@@ -1,4 +1,4 @@
-require 'yajl'
+require 'json'
 require 'elasticsearch'
 require 'hashie'
 require 'parallel'
@@ -54,8 +54,8 @@ module Ej
         docs.each do |doc|
           source = doc.delete('_source')
           doc.delete('_score')
-          bulk_message << Yajl::Encoder.encode({ 'index' => doc.to_h })
-          bulk_message << Yajl::Encoder.encode(source)
+          bulk_message << JSON.dump({ 'index' => doc.to_h })
+          bulk_message << JSON.dump(source)
         end
         num += 1
         puts bulk_message.join("\n")
@@ -147,37 +147,41 @@ module Ej
       connect_with_retry { @client.bulk body: bulk_message unless bulk_message.empty? }
     end
 
-    def copy(source, dest, query, per_size, scroll, dest_index)
+    def copy(source, dest, query, per_size, scroll, dest_index, slice_max)
       source_client = Elasticsearch::Client.new hosts: source
       dest_client = Elasticsearch::Client.new hosts: dest
 
-      scroll_option = get_scroll_option(@index, query, per_size, scroll)
-      r = connect_with_retry { source_client.search(scroll_option) }
-      total = r['hits']['total']
-      i = 0
-      i += bulk_results(r, dest_client, i, total, dest_index)
-
-      while r = connect_with_retry { source_client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) } and
-        (not r['hits']['hits'].empty?) do
-        i += bulk_results(r, dest_client, i, total, dest_index)
+      parallel_array = slice_max ? slice_max.times.to_a : [0]
+      Parallel.map(parallel_array, :in_processes=>parallel_array.size) do |slice_id|
+        scroll_option = get_scroll_option(@index, query, per_size, scroll, slice_id, slice_max)
+        r = connect_with_retry { source_client.search(scroll_option) }
+        total = r['hits']['total']
+        i = 0
+        i += bulk_results(r, dest_client, i, total, dest_index, slice_id)
+
+        while r = connect_with_retry { source_client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) } and
+          (not r['hits']['hits'].empty?) do
+          i += bulk_results(r, dest_client, i, total, dest_index, slice_id)
+        end
       end
     end
 
     private
 
-    def bulk_results(results, dest_client, before_size, total, dest_index)
+    def bulk_results(results, dest_client, before_size, total, dest_index, slice_id)
       bulk_message = convert_results(results, dest_index)
       connect_with_retry do
         dest_client.bulk body: bulk_message unless bulk_message.empty?
         to_size = before_size + (bulk_message.size/2)
-        @logger.info "copy complete (#{before_size}-#{to_size})/#{total}"
+        @logger.info "slice_id[#{slice_id}] copy complete (#{before_size}-#{to_size})/#{total}"
       end
       return (bulk_message.size/2)
     end
 
-    def get_scroll_option(index, query, size, scroll)
+    def get_scroll_option(index, query, size, scroll, slice_id, slice_max)
       body = {}
       body[:query] = { query_string: { query: query } } unless query.nil?
+      body[:slice] = { id: slice_id, max: slice_max } if slice_max
       search_option = { index: index, scroll: scroll, body: body, size: (size || DEFAULT_PER) }
       search_option
     end
@@ -207,7 +211,7 @@ module Ej
         if retries < retry_on_failure
           retries += 1
           @logger.warn "Could not connect to Elasticsearch, resetting connection and trying again. #{e.message}"
-          sleep 2**retries
+          sleep 10**retries
           retry
         end
         raise "Could not connect to Elasticsearch after #{retries} retries. #{e.message}"

data/lib/ej/util.rb

@@ -2,11 +2,11 @@ module Ej
   class Util
     def self.parse_json(buffer)
       begin
-        data = Yajl::Parser.parse(buffer)
+        data = JSON.parse(buffer)
       rescue => e
         data = []
         buffer.lines.each do |line|
-          data << Yajl::Parser.parse(line)
+          data << JSON.parse(line)
         end
       end
       data.class == Array ? data : [data]

data/lib/ej/version.rb

@@ -1,3 +1,3 @@
 module Ej
-  VERSION = '0.1.11'
+  VERSION = '0.1.12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ej
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - toyama0919
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-06 00:00:00.000000000 Z
+date: 2017-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: yajl-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement