embulk-input-elasticsearch 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0991bd544cf235c70290b6d307f164112280d1ea
-  data.tar.gz: 4e578ad476510b38eaa09ef9a20437f1548cc70c
+  metadata.gz: cc69f25bdb9b51f2eb62e6a794767e22e96b3ece
+  data.tar.gz: 91432ba12709c79ddcb470318253f53fa7f4aec3
 SHA512:
-  metadata.gz: b47b9034846fc515fda638e231b5c62f13b0c8fecf1fba8ef16ad8f084f967518b2089df5a95e7681bb6e9af2dd918da452b843d1072dd04e22f01f504af674e
-  data.tar.gz: ddc4879a8a6c0524499f1b10f3b1e871c1920024aefa81e1c4bf562c44533cc161ea6d8d202bf1dd89af30688f35790c3f2f9b0a9a637a9af9ebdd86e32b6e09
+  metadata.gz: 804a861e1eeffe08e1ba263484007fb61bd44299c648aab00227144cb92db11b1a4edeecd46d955b6c41673fc7881ce0712c8a16edbf50206fd4759f2ccc2d40
+  data.tar.gz: c763621544516ea30c1706820d87d9f2e9a450dfc0f4c72a290e9229ac66809364262dd79379fe20e37e6215ee4d8148e9b8aa971d7499ae9b6834b19d3bb89d

data/embulk-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-elasticsearch"
-  spec.version       = "0.3.0"
+  spec.version       = "0.3.1"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Elasticsearch input plugin for Embulk"
   spec.description   = "Loads records from Elasticsearch. parallel query support."

data/lib/embulk/input/elasticsearch.rb

@@ -24,7 +24,8 @@ module Embulk
           "fields" => config.param("fields", :array, default: nil),
           "queries" => config.param("queries", :array),
           "sort" => config.param("sort", :hash, default: nil),
-          "add_query_to_record" => config.param("add_query_to_record", :bool, default: false)
+          "add_query_to_record" => config.param("add_query_to_record", :bool, default: false),
+          "scroll" => config.param("scroll", :string, default: '1m')
         }
         # TODO: want max_threads
         define_num_threads = config.param("num_threads", :integer, default: 1)
@@ -59,6 +60,7 @@ module Embulk
         @fields = task['fields']
         @sort = task['sort']
         @add_query_to_record = task['add_query_to_record']
+        @scroll = task['scroll']
       end
 
       def run
@@ -87,7 +89,7 @@ module Embulk
           return if @limit_size == (i += 1)
         end
 
-        while r = @client.scroll(scroll_id: r['_scroll_id'], scroll: '1m') and (not r['hits']['hits'].empty?) do
+        while r = @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) and (not r['hits']['hits'].empty?) do
           get_sources(r, fields).each do |result|
             result_proc(result, query)
             return if @limit_size == (i += 1)
@@ -114,7 +116,7 @@ module Embulk
         else
           body[:sort] = ["_doc"]
         end
-        search_option = { index: @index_name, type: type, scroll: '1m', body: body, size: size }
+        search_option = { index: @index_name, type: type, scroll: @scroll, body: body, size: size }
         search_option[:_source] = fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
         search_option
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement