embulk-input-elasticsearch 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/embulk-input-elasticsearch.gemspec +2 -2
- data/lib/embulk/input/elasticsearch.rb +19 -2
- metadata +5 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 209e87c44e92108a80a402616ef1d7ad7db9cbd7
|
|
4
|
+
data.tar.gz: ec7efbe789ee0e2b17d922750172ba0e9f745ef2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 337eac65409f489536ba0e4107cdca94115530b3781b2e2e8eb90b55e3df7e46e4f79530d9776b2ee341887bde394387fe3f08850b0aa2a1c0a50b9d9ba6c613
|
|
7
|
+
data.tar.gz: 612f003dccdab158bd094b34d2853c3cd368b955b860b5a58f71ceaa3cae416c492e556acf02550ddc28aec9920485ae15fac0a8b3b78419e4bc78f31f00cd43
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
jruby-9.
|
|
1
|
+
jruby-9.1.5.0
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
Gem::Specification.new do |spec|
|
|
3
3
|
spec.name = "embulk-input-elasticsearch"
|
|
4
|
-
spec.version = "0.3.
|
|
4
|
+
spec.version = "0.3.2"
|
|
5
5
|
spec.authors = ["toyama0919"]
|
|
6
6
|
spec.summary = "Elasticsearch input plugin for Embulk"
|
|
7
7
|
spec.description = "Loads records from Elasticsearch. parallel query support."
|
|
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
|
|
16
16
|
spec.add_dependency 'elasticsearch'
|
|
17
17
|
spec.add_dependency 'excon'
|
|
18
|
-
spec.add_development_dependency 'embulk', ['>= 0.8.
|
|
18
|
+
spec.add_development_dependency 'embulk', ['>= 0.8.18']
|
|
19
19
|
spec.add_development_dependency 'bundler', ['>= 1.10.6']
|
|
20
20
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
21
21
|
spec.add_development_dependency 'test-unit'
|
|
@@ -61,6 +61,7 @@ module Embulk
|
|
|
61
61
|
@sort = task['sort']
|
|
62
62
|
@add_query_to_record = task['add_query_to_record']
|
|
63
63
|
@scroll = task['scroll']
|
|
64
|
+
@retry_on_failure = task['retry_on_failure']
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
def run
|
|
@@ -82,14 +83,14 @@ module Embulk
|
|
|
82
83
|
def search_with_query(query, type, size, routing, fields, sort)
|
|
83
84
|
search_option = get_search_option(type, query, size, fields, sort)
|
|
84
85
|
Embulk.logger.info("#{search_option}")
|
|
85
|
-
r = @client.search(search_option)
|
|
86
|
+
r = search_with_retry { @client.search(search_option) }
|
|
86
87
|
i = 0
|
|
87
88
|
get_sources(r, fields).each do |result|
|
|
88
89
|
result_proc(result, query)
|
|
89
90
|
return if @limit_size == (i += 1)
|
|
90
91
|
end
|
|
91
92
|
|
|
92
|
-
while r = @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) and (not r['hits']['hits'].empty?) do
|
|
93
|
+
while r = (search_with_retry { @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) }) and (not r['hits']['hits'].empty?) do
|
|
93
94
|
get_sources(r, fields).each do |result|
|
|
94
95
|
result_proc(result, query)
|
|
95
96
|
return if @limit_size == (i += 1)
|
|
@@ -97,6 +98,22 @@ module Embulk
|
|
|
97
98
|
end
|
|
98
99
|
end
|
|
99
100
|
|
|
101
|
+
def search_with_retry
|
|
102
|
+
retries = 0
|
|
103
|
+
begin
|
|
104
|
+
yield if block_given?
|
|
105
|
+
rescue => e
|
|
106
|
+
if retries < @retry_on_failure
|
|
107
|
+
retries += 1
|
|
108
|
+
Embulk.logger.warn "Could not search to Elasticsearch, resetting connection and trying again. #{e.message}"
|
|
109
|
+
sleep 2**retries
|
|
110
|
+
retry
|
|
111
|
+
end
|
|
112
|
+
Embulk.logger.error "Could not search to Elasticsearch after #{retries} retries. #{e.message}"
|
|
113
|
+
raise
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
100
117
|
def result_proc(result, query)
|
|
101
118
|
if @add_query_to_record
|
|
102
119
|
result << query
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-elasticsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- toyama0919
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-03-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -43,7 +43,7 @@ dependencies:
|
|
|
43
43
|
requirements:
|
|
44
44
|
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: 0.8.
|
|
46
|
+
version: 0.8.18
|
|
47
47
|
name: embulk
|
|
48
48
|
prerelease: false
|
|
49
49
|
type: :development
|
|
@@ -51,7 +51,7 @@ dependencies:
|
|
|
51
51
|
requirements:
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 0.8.
|
|
54
|
+
version: 0.8.18
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
134
134
|
version: '0'
|
|
135
135
|
requirements: []
|
|
136
136
|
rubyforge_project:
|
|
137
|
-
rubygems_version: 2.
|
|
137
|
+
rubygems_version: 2.6.6
|
|
138
138
|
signing_key:
|
|
139
139
|
specification_version: 4
|
|
140
140
|
summary: Elasticsearch input plugin for Embulk
|