embulk-input-elasticsearch 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e3e0bcff8c7de3a07cab81b1689f6f2f6b5dfe5
-  data.tar.gz: a2997eb7f98cdea800403211b75342a0ff360483
+  metadata.gz: e5dbc0a3f7f57fd192f65ba884bb66c6f3cc64e9
+  data.tar.gz: e2d7318385c5ef34c173a2a7e2a4f826ff526495
 SHA512:
-  metadata.gz: bb618f0f0c4af1cdce787ce20cc38b009a4b48eba7e678f0f198c0e0195038242d8609ef42e1744077b0a51e99a8c3d62662c42e9482c4e33802d7b7ac8adeab
-  data.tar.gz: 8b43fc2e6c9afa87bb0851e2a813c9a3633104df341db9b73c74b472e3c701ce1e04e5e287f11d94b7ea2b1e11a6a0b7e2ea9c25bc70978b627765bf87ba2abf
+  metadata.gz: 4c0ab03a6b629767ed2eaea29c349ed5541187667b5017806de60ed518dd4d34c840efc70bdc4f6fb5c7f126850985dc6d9c24cd0d92af6a118e63fac2fd58ff
+  data.tar.gz: 086e06172b34af6e8c88b7d8c02c03e8b2d2ee560ffab72cbe409b12fcf2aaad271029992d48d3443675b0239f5d4cd20670e5d624743b46f9bf3b8c7e54aae6

data/README.md

@@ -9,19 +9,19 @@
 
 ## Configuration
 - **nodes**: nodes (array, required)
-  - **host**: host (string, required, default: ``)
-  - **port**: port (integer, required, default: ``)
-- **queries**: query (array, required, default: ``)
-- **index**: index (string, required, default: ``)
-- **index_type**: index_type (string, default: ``)
-- **request_timeout**: request_timeout (string, default: ``)
+  - **host**: host (string, required)
+  - **port**: port (integer, required)
+- **queries**: query (array, required)
+- **index**: index (string, required)
+- **index_type**: index_type (string)
+- **request_timeout**: request_timeout (integer)
 - **per_size**: per_size (integer, required, default: `1000`)
 - **limit_size**: limit_size (integer, default: unlimit)
-- **fields**: fields (string, required, default: ``)
-  - **name**: name (string, required, default: ``)
-  - **type**: type (string, required, default: ``)
-  - **metadata**: metadata (boolean, required, default: ``)
-  - **time_format**: time_format (string, required, default: ``)
+- **fields**: fields (array, required)
+  - **name**: name (string, required)
+  - **type**: type (string, required)
+  - **metadata**: metadata (boolean, default: false)
+  - **time_format**: time_format (string, required)
 
 ## Example
 

data/embulk-input-elasticsearch.gemspec

@@ -1,10 +1,10 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-elasticsearch"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Elasticsearch input plugin for Embulk"
-  spec.description   = "Loads records from Elasticsearch."
+  spec.description   = "Loads records from Elasticsearch. parallel query support."
   spec.email         = ["toyama0919@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/toyama0919/embulk-input-elasticsearch"

data/lib/embulk/input/elasticsearch.rb

@@ -22,13 +22,28 @@ module Embulk
           "queries" => config.param("queries", :array),
           "sort" => config.param("sort", :hash, default: nil)
         }
+        # TODO: want max_threads
+        define_num_threads = config.param("num_threads", :integer, default: 1)
+        task['slice_queries'] = get_slice_from_num_threads(task['queries'], define_num_threads)
 
         columns = []
         task['fields'].each_with_index{ |field, i|
           columns << Column.new(i, field['name'], field['type'].to_sym)
         }
 
-        resume(task, columns, 1, &control)
+        resume(task, columns, task['slice_queries'].size, &control)
+      end
+
+      def self.get_slice_from_num_threads(array, define_num_threads)
+        num_threads = array.size < define_num_threads ? array.size : define_num_threads
+        per_queries = if (array.size % num_threads) == 0
+          (array.size / num_threads)
+        else
+          (array.size / num_threads) + 1
+        end
+        sliced = array.each_slice(per_queries).to_a
+        Embulk.logger.info("calculate num threads => #{sliced.size}")
+        return sliced
       end
 
       def self.resume(task, columns, count, &control)
@@ -57,10 +72,11 @@ module Embulk
       end
 
       def init
+        @queries = task['slice_queries'][@index]
+        Embulk.logger.info("this thread queries => #{@queries}")
         @client = self.class.create_client(task)
-        @index = task['index']
+        @index_name = task['index']
         @index_type = task['index_type']
-        @queries = task['queries']
         @per_size = task['per_size']
         @limit_size = task['limit_size']
         @fields = task['fields']
@@ -148,7 +164,7 @@ module Embulk
           body[:sort] = sorts
         end
         body[:query] = { query_string: { query: query } } unless query.nil?
-        search_option = { index: @index, type: type, body: body }
+        search_option = { index: @index_name, type: type, body: body }
         search_option[:routing] = routing unless routing.nil?
         search_option[:_source] = fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
         Embulk.logger.info(%Q{search_option => #{search_option}})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-13 00:00:00.000000000 Z
+date: 2016-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Loads records from Elasticsearch.
+description: Loads records from Elasticsearch. parallel query support.
 email:
 - toyama0919@gmail.com
 executables: []