embulk-input-elasticsearch 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0991bd544cf235c70290b6d307f164112280d1ea
4
- data.tar.gz: 4e578ad476510b38eaa09ef9a20437f1548cc70c
3
+ metadata.gz: cc69f25bdb9b51f2eb62e6a794767e22e96b3ece
4
+ data.tar.gz: 91432ba12709c79ddcb470318253f53fa7f4aec3
5
5
  SHA512:
6
- metadata.gz: b47b9034846fc515fda638e231b5c62f13b0c8fecf1fba8ef16ad8f084f967518b2089df5a95e7681bb6e9af2dd918da452b843d1072dd04e22f01f504af674e
7
- data.tar.gz: ddc4879a8a6c0524499f1b10f3b1e871c1920024aefa81e1c4bf562c44533cc161ea6d8d202bf1dd89af30688f35790c3f2f9b0a9a637a9af9ebdd86e32b6e09
6
+ metadata.gz: 804a861e1eeffe08e1ba263484007fb61bd44299c648aab00227144cb92db11b1a4edeecd46d955b6c41673fc7881ce0712c8a16edbf50206fd4759f2ccc2d40
7
+ data.tar.gz: c763621544516ea30c1706820d87d9f2e9a450dfc0f4c72a290e9229ac66809364262dd79379fe20e37e6215ee4d8148e9b8aa971d7499ae9b6834b19d3bb89d
@@ -1,7 +1,7 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "embulk-input-elasticsearch"
4
- spec.version = "0.3.0"
4
+ spec.version = "0.3.1"
5
5
  spec.authors = ["toyama0919"]
6
6
  spec.summary = "Elasticsearch input plugin for Embulk"
7
7
  spec.description = "Loads records from Elasticsearch. parallel query support."
@@ -24,7 +24,8 @@ module Embulk
24
24
  "fields" => config.param("fields", :array, default: nil),
25
25
  "queries" => config.param("queries", :array),
26
26
  "sort" => config.param("sort", :hash, default: nil),
27
- "add_query_to_record" => config.param("add_query_to_record", :bool, default: false)
27
+ "add_query_to_record" => config.param("add_query_to_record", :bool, default: false),
28
+ "scroll" => config.param("scroll", :string, default: '1m')
28
29
  }
29
30
  # TODO: want max_threads
30
31
  define_num_threads = config.param("num_threads", :integer, default: 1)
@@ -59,6 +60,7 @@ module Embulk
59
60
  @fields = task['fields']
60
61
  @sort = task['sort']
61
62
  @add_query_to_record = task['add_query_to_record']
63
+ @scroll = task['scroll']
62
64
  end
63
65
 
64
66
  def run
@@ -87,7 +89,7 @@ module Embulk
87
89
  return if @limit_size == (i += 1)
88
90
  end
89
91
 
90
- while r = @client.scroll(scroll_id: r['_scroll_id'], scroll: '1m') and (not r['hits']['hits'].empty?) do
92
+ while r = @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) and (not r['hits']['hits'].empty?) do
91
93
  get_sources(r, fields).each do |result|
92
94
  result_proc(result, query)
93
95
  return if @limit_size == (i += 1)
@@ -114,7 +116,7 @@ module Embulk
114
116
  else
115
117
  body[:sort] = ["_doc"]
116
118
  end
117
- search_option = { index: @index_name, type: type, scroll: '1m', body: body, size: size }
119
+ search_option = { index: @index_name, type: type, scroll: @scroll, body: body, size: size }
118
120
  search_option[:_source] = fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
119
121
  search_option
120
122
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - toyama0919
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-24 00:00:00.000000000 Z
11
+ date: 2016-06-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement