embulk-input-marketo 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/CHANGELOG.md +10 -0
- data/README.md +5 -0
- data/embulk-input-marketo.gemspec +1 -1
- data/lib/embulk/input/marketo/activity_log.rb +4 -0
- data/lib/embulk/input/marketo/lead.rb +36 -20
- data/lib/embulk/input/marketo_api/soap/activity_log.rb +8 -1
- data/lib/embulk/input/marketo_api/soap/base.rb +12 -7
- data/lib/embulk/input/marketo_api/soap/lead.rb +4 -4
- data/test/embulk/input/marketo/test_activity_log.rb +37 -0
- data/test/embulk/input/marketo/test_lead.rb +90 -7
- data/test/embulk/input/marketo_api/soap/test_activity_log.rb +1 -0
- data/test/embulk/input/marketo_api/soap/test_base.rb +59 -6
- data/test/embulk/input/marketo_api/soap/test_lead.rb +1 -1
- metadata +31 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dde3369b27304f507037e63efd59b2e4e7bdc449
|
4
|
+
data.tar.gz: fd6df82180b3d3fbf3ef0835b16ea6894f1e4161
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ec3f1ca521a50e53e6c62fe7c51e67c44c09bad74936ff4e34b52fe6e00f721848bef35a35d2030c75a641d405126ea7efe28871928fc9319041b5dcc200f13
|
7
|
+
data.tar.gz: 303b8b3f47e439d4e863f6f157f64de2cc7d46fd46b2487f206573e7e4d0a43e00e6edd6a902b10e10200204e2fd1149e0982352fda66efbbd2500909959363c
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
jruby-1.
|
1
|
+
jruby-9.0.1.0
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
## 0.4.0 - 2015-10-30
|
2
|
+
|
3
|
+
This version drops scheduled execution with marketo/lead.
|
4
|
+
|
5
|
+
* [enhancement] Append processed time column [#49](https://github.com/treasure-data/embulk-input-marketo/pull/49)
|
6
|
+
* [enhancement] Exponential backoff retry [#48](https://github.com/treasure-data/embulk-input-marketo/pull/48)
|
7
|
+
* [fixed] Fix preview didn't stop after fetched if multiple ranges have [#45](https://github.com/treasure-data/embulk-input-marketo/pull/45)
|
8
|
+
* [enhancement] activity_log: Use from..from+30m range for guess [#47](https://github.com/treasure-data/embulk-input-marketo/pull/47)
|
9
|
+
* [enhancement] Unsupport scheduled execution for lead [#46](https://github.com/treasure-data/embulk-input-marketo/pull/46) [#41](https://github.com/treasure-data/embulk-input-marketo/pull/41) [Reported by @muga. Thanks!]
|
10
|
+
|
1
11
|
## 0.3.2 - 2015-10-13
|
2
12
|
|
3
13
|
* [fixed] Prevent memoize in class [#44](https://github.com/treasure-data/embulk-input-marketo/pull/44)
|
data/README.md
CHANGED
@@ -43,6 +43,9 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
|
43
43
|
- **encryption_key** Your encryption key (string, reqiured)
|
44
44
|
- **from_datetime** Fetch leads since this time (string, required)
|
45
45
|
- **to_datetime** Fetch leads until this time (string, default: Time.now)
|
46
|
+
- **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
|
47
|
+
- **retry_limit**: Try to retry this times (integer, default: 5)
|
48
|
+
- **append_processed_time_column**: If you want the column for processed time (boolean, default: true)
|
46
49
|
|
47
50
|
### marketo/activity_log
|
48
51
|
|
@@ -52,6 +55,8 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
|
52
55
|
- **encryption_key** Your encryption key (string, reqiured)
|
53
56
|
- **from_datetime** Fetch activity_logs since this time (string, required)
|
54
57
|
- **to_datetime** Fetch activity_logs until this time (string, default: Time.now)
|
58
|
+
- **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
|
59
|
+
- **retry_limit**: Try to retry this times (integer, default: 5)
|
55
60
|
|
56
61
|
### Selecting plugin type
|
57
62
|
|
@@ -33,6 +33,8 @@ module Embulk
|
|
33
33
|
encryption_key: config.param(:encryption_key, :string),
|
34
34
|
from_datetime: range[:from],
|
35
35
|
to_datetime: range[:to],
|
36
|
+
retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
|
37
|
+
retry_limit: config.param(:retry_limit, :integer, default: 5),
|
36
38
|
columns: config.param(:columns, :array)
|
37
39
|
}
|
38
40
|
|
@@ -60,6 +62,8 @@ module Embulk
|
|
60
62
|
|
61
63
|
def run
|
62
64
|
options = {
|
65
|
+
retry_initial_wait_sec: task[:retry_initial_wait_sec],
|
66
|
+
retry_limit: task[:retry_limit],
|
63
67
|
to: task[:to_datetime],
|
64
68
|
batch_size: (preview? ? PREVIEW_COUNT : BATCH_SIZE_DEFAULT),
|
65
69
|
}
|
@@ -26,18 +26,17 @@ module Embulk
|
|
26
26
|
def self.resume(task, columns, count, &control)
|
27
27
|
task_reports = yield(task, columns, count)
|
28
28
|
|
29
|
-
|
30
|
-
return {} if task_reports.empty?
|
31
|
-
# all task returns same report as {from_datetime: to_datetime}
|
32
|
-
return task_reports.first
|
29
|
+
return {}
|
33
30
|
end
|
34
31
|
|
35
32
|
def self.transaction(config, &control)
|
36
33
|
endpoint_url = config.param(:endpoint, :string)
|
37
34
|
|
38
35
|
range = format_range(config)
|
39
|
-
|
40
36
|
ranges = timeslice(range[:from], range[:to], TIMESLICE_COUNT_PER_TASK)
|
37
|
+
|
38
|
+
append_processed_time_column = config.param(:append_processed_time_column, :bool, default: true)
|
39
|
+
|
41
40
|
task = {
|
42
41
|
endpoint_url: endpoint_url,
|
43
42
|
wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
|
@@ -46,10 +45,19 @@ module Embulk
|
|
46
45
|
from_datetime: range[:from],
|
47
46
|
to_datetime: range[:to],
|
48
47
|
ranges: ranges,
|
49
|
-
|
48
|
+
retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
|
49
|
+
retry_limit: config.param(:retry_limit, :integer, default: 5),
|
50
|
+
append_processed_time_column: append_processed_time_column,
|
51
|
+
columns: config.param(:columns, :array),
|
50
52
|
}
|
51
53
|
|
52
|
-
|
54
|
+
columns = embulk_columns(config)
|
55
|
+
if append_processed_time_column
|
56
|
+
processed_time_column = Column.new(nil, :processed_time, :timestamp, "%Y-%m-%dT%H:%M:%S%z")
|
57
|
+
columns << processed_time_column
|
58
|
+
end
|
59
|
+
|
60
|
+
resume(task, columns, ranges.size, &control)
|
53
61
|
end
|
54
62
|
|
55
63
|
def self.generate_columns(metadata)
|
@@ -85,31 +93,39 @@ module Embulk
|
|
85
93
|
@columns = task[:columns]
|
86
94
|
@ranges = task[:ranges][index]
|
87
95
|
@soap = MarketoApi.soap_client(task, target)
|
96
|
+
@append_processed_time_column = task[:append_processed_time_column]
|
88
97
|
end
|
89
98
|
|
90
99
|
def run
|
91
|
-
options = {
|
100
|
+
options = {
|
101
|
+
retry_initial_wait_sec: task[:retry_initial_wait_sec],
|
102
|
+
retry_limit: task[:retry_limit],
|
103
|
+
}
|
92
104
|
options[:batch_size] = PREVIEW_COUNT if preview?
|
93
105
|
|
94
106
|
counter = 0
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
107
|
+
catch(:finish) do
|
108
|
+
@ranges.each do |range|
|
109
|
+
soap.each(range, options) do |lead|
|
110
|
+
values = @columns.map do |column|
|
111
|
+
name = column["name"].to_s
|
112
|
+
value = (lead[name] || {})[:value]
|
113
|
+
cast_value(column, value)
|
114
|
+
end
|
115
|
+
|
116
|
+
if @append_processed_time_column
|
117
|
+
values << Time.parse(range["from"])
|
118
|
+
end
|
119
|
+
|
120
|
+
page_builder.add(values)
|
121
|
+
throw(:finish) if preview? && (counter += 1) >= PREVIEW_COUNT
|
101
122
|
end
|
102
|
-
|
103
|
-
page_builder.add(values)
|
104
|
-
break if preview? && (counter += 1) >= PREVIEW_COUNT
|
105
123
|
end
|
106
124
|
end
|
107
125
|
|
108
126
|
page_builder.finish
|
109
127
|
|
110
|
-
task_report = {
|
111
|
-
from_datetime: task[:to_datetime]
|
112
|
-
}
|
128
|
+
task_report = {}
|
113
129
|
return task_report
|
114
130
|
end
|
115
131
|
end
|
@@ -5,9 +5,15 @@ module Embulk
|
|
5
5
|
module MarketoApi
|
6
6
|
module Soap
|
7
7
|
class ActivityLog < Base
|
8
|
+
GUESS_DURATION = 60 * 30 # 30m
|
9
|
+
|
8
10
|
def metadata(from_datetime, options={})
|
9
11
|
activity_logs = []
|
10
12
|
|
13
|
+
from = Time.parse(from_datetime.to_s)
|
14
|
+
to = from + GUESS_DURATION
|
15
|
+
options[:to] = to.to_s
|
16
|
+
|
11
17
|
fetch_by_from_datetime(from_datetime, options) do |record|
|
12
18
|
activity_logs << record
|
13
19
|
end
|
@@ -45,11 +51,12 @@ module Embulk
|
|
45
51
|
}
|
46
52
|
request[:start_position][:offset] = options[:offset] if options[:offset]
|
47
53
|
|
54
|
+
Embulk.logger.info "Fetching from '#{from}' to '#{to}'..."
|
48
55
|
fetch(request, options, &block)
|
49
56
|
end
|
50
57
|
|
51
58
|
def fetch(request, options={}, &block)
|
52
|
-
response = savon_call(:get_lead_changes, message: request)
|
59
|
+
response = savon_call(:get_lead_changes, {message: request}, options)
|
53
60
|
remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
|
54
61
|
Embulk.logger.info "Remaining records: #{remaining}"
|
55
62
|
|
@@ -41,9 +41,9 @@ module Embulk
|
|
41
41
|
)
|
42
42
|
end
|
43
43
|
|
44
|
-
def savon_call(operation, locals={})
|
45
|
-
|
46
|
-
|
44
|
+
def savon_call(operation, locals={}, retry_options={})
|
45
|
+
with_retry(retry_options) do
|
46
|
+
catch_unretryable_error do
|
47
47
|
savon.call(operation, locals.merge(advanced_typecasting: false))
|
48
48
|
end
|
49
49
|
end
|
@@ -60,14 +60,19 @@ module Embulk
|
|
60
60
|
}
|
61
61
|
end
|
62
62
|
|
63
|
-
def with_retry(&block)
|
63
|
+
def with_retry(options, &block)
|
64
|
+
wait_sec = options[:retry_initial_wait_sec]
|
64
65
|
count = 0
|
65
66
|
begin
|
66
67
|
yield
|
67
|
-
rescue ::
|
68
|
+
rescue Embulk::ConfigError => e # TODO: Add Embulk::DataError for Embulk 0.7+
|
69
|
+
raise e
|
70
|
+
rescue ::Timeout::Error, StandardError => e
|
68
71
|
count += 1
|
69
|
-
raise e if count >
|
70
|
-
Embulk.logger.warn "
|
72
|
+
raise e if count > options[:retry_limit]
|
73
|
+
Embulk.logger.warn "Retrying after #{wait_sec} seconds [#{count}/#{RETRY_TIMEOUT_COUNT}] Error: #{e}"
|
74
|
+
sleep wait_sec
|
75
|
+
wait_sec *= 2
|
71
76
|
retry
|
72
77
|
end
|
73
78
|
end
|
@@ -35,18 +35,18 @@ module Embulk
|
|
35
35
|
}
|
36
36
|
Embulk.logger.info "Fetching from '#{from}' to '#{to}'..."
|
37
37
|
|
38
|
-
stream_position = fetch(request, &block)
|
38
|
+
stream_position = fetch(request, options, &block)
|
39
39
|
|
40
40
|
while stream_position
|
41
|
-
stream_position = fetch(request.merge(stream_position: stream_position), &block)
|
41
|
+
stream_position = fetch(request.merge(stream_position: stream_position), options, &block)
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
45
|
private
|
46
46
|
|
47
|
-
def fetch(request = {}, &block)
|
47
|
+
def fetch(request = {}, retry_options, &block)
|
48
48
|
start = Time.now
|
49
|
-
response = savon_call(:get_multiple_leads, message: request)
|
49
|
+
response = savon_call(:get_multiple_leads, {message: request}, retry_options)
|
50
50
|
Embulk.logger.info "Fetched in #{Time.now - start} seconds"
|
51
51
|
|
52
52
|
records = response.xpath('//leadRecordList/leadRecord')
|
@@ -42,6 +42,8 @@ module Embulk
|
|
42
42
|
encryption_key: "TOPSECRET",
|
43
43
|
from_datetime: from_datetime,
|
44
44
|
to_datetime: to_datetime,
|
45
|
+
retry_initial_wait_sec: 3,
|
46
|
+
retry_limit: 2,
|
45
47
|
columns: [
|
46
48
|
{"name" => :id, "type" => :long},
|
47
49
|
{"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
|
@@ -66,6 +68,8 @@ module Embulk
|
|
66
68
|
encryption_key: "TOPSECRET",
|
67
69
|
from_datetime: from_datetime,
|
68
70
|
to_datetime: to_datetime,
|
71
|
+
retry_initial_wait_sec: 3,
|
72
|
+
retry_limit: 2,
|
69
73
|
columns: [
|
70
74
|
{"name" => :id, "type" => :long},
|
71
75
|
{"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
|
@@ -142,6 +146,37 @@ module Embulk
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
149
|
+
class TestRetry < self
|
150
|
+
def setup
|
151
|
+
@soap = MarketoApi::Soap::ActivityLog.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
|
152
|
+
stub(ActivityLog).soap_client(task) { @soap }
|
153
|
+
|
154
|
+
@page_builder = Object.new
|
155
|
+
@plugin = ActivityLog.new(task, nil, nil, @page_builder)
|
156
|
+
end
|
157
|
+
|
158
|
+
def test_retry
|
159
|
+
any_instance_of(Savon::Client) do |klass|
|
160
|
+
stub(klass).call(:get_multiple_leads, anything) do
|
161
|
+
raise "foo"
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
|
166
|
+
task[:retry_limit].times do |n|
|
167
|
+
mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
|
172
|
+
stub(Embulk.logger).info {}
|
173
|
+
|
174
|
+
assert_raise do
|
175
|
+
@plugin.run
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
145
180
|
class RunTest < self
|
146
181
|
def setup_soap
|
147
182
|
@soap = MarketoApi::Soap::ActivityLog.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
|
@@ -290,6 +325,8 @@ module Embulk
|
|
290
325
|
encryption_key: "TOPSECRET",
|
291
326
|
from_datetime: from_datetime,
|
292
327
|
to_datetime: to_datetime,
|
328
|
+
retry_initial_wait_sec: 3,
|
329
|
+
retry_limit: 2,
|
293
330
|
columns: [
|
294
331
|
{"name" => :id, "type" => :long},
|
295
332
|
{"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
|
@@ -28,7 +28,6 @@ module Embulk
|
|
28
28
|
stub(klass).index { 0 }
|
29
29
|
end
|
30
30
|
@plugin = Lead.new(task, nil, nil, @page_builder)
|
31
|
-
mute_logger
|
32
31
|
end
|
33
32
|
|
34
33
|
def test_invalid_from_datetime_to_datetime
|
@@ -122,6 +121,33 @@ module Embulk
|
|
122
121
|
end
|
123
122
|
|
124
123
|
def test_run_through
|
124
|
+
mute_logger
|
125
|
+
stub(@plugin).preview? { false }
|
126
|
+
|
127
|
+
any_instance_of(Savon::Client) do |klass|
|
128
|
+
mock(klass).call(:get_multiple_leads, message: request) do
|
129
|
+
leads_response
|
130
|
+
end
|
131
|
+
|
132
|
+
mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
|
133
|
+
next_stream_leads_response
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
from = Time.parse(from_datetime)
|
138
|
+
mock(@page_builder).add(["manyo", from])
|
139
|
+
mock(@page_builder).add(["everyleaf", from])
|
140
|
+
mock(@page_builder).add(["ten-thousand-leaf", from])
|
141
|
+
mock(@page_builder).finish
|
142
|
+
|
143
|
+
@plugin.run
|
144
|
+
end
|
145
|
+
|
146
|
+
def test_run_no_processed_time_columns
|
147
|
+
mute_logger
|
148
|
+
|
149
|
+
no_processed__task = task.merge(append_processed_time_column: false)
|
150
|
+
@plugin = Lead.new(no_processed__task, nil, nil, @page_builder)
|
125
151
|
stub(@plugin).preview? { false }
|
126
152
|
|
127
153
|
any_instance_of(Savon::Client) do |klass|
|
@@ -142,16 +168,18 @@ module Embulk
|
|
142
168
|
@plugin.run
|
143
169
|
end
|
144
170
|
|
145
|
-
def
|
171
|
+
def test_run_task_report
|
172
|
+
mute_logger
|
146
173
|
# do not requests
|
147
174
|
stub(@page_builder).finish
|
148
175
|
stub(@plugin.soap).each { }
|
149
176
|
|
150
|
-
|
151
|
-
assert_equal
|
177
|
+
task_report = @plugin.run
|
178
|
+
assert_equal({}, task_report)
|
152
179
|
end
|
153
180
|
|
154
181
|
def test_preview_through
|
182
|
+
mute_logger
|
155
183
|
stub(@plugin).preview? { true }
|
156
184
|
|
157
185
|
any_instance_of(Savon::Client) do |klass|
|
@@ -160,14 +188,55 @@ module Embulk
|
|
160
188
|
end
|
161
189
|
end
|
162
190
|
|
191
|
+
from = Time.parse(from_datetime)
|
163
192
|
Lead::PREVIEW_COUNT.times do |count|
|
164
|
-
mock(@page_builder).add(["manyo#{count}"])
|
193
|
+
mock(@page_builder).add(["manyo#{count}", from])
|
194
|
+
end
|
195
|
+
mock(@page_builder).finish
|
196
|
+
|
197
|
+
@plugin.run
|
198
|
+
end
|
199
|
+
|
200
|
+
def test_preview_will_stop_fetching_when_defined_times_added
|
201
|
+
mute_logger
|
202
|
+
stub(@plugin).preview? { true }
|
203
|
+
@plugin.instance_variable_set(:@ranges, @plugin.task[:ranges].first * 3) # multiple ranges
|
204
|
+
|
205
|
+
any_instance_of(Savon::Client) do |klass|
|
206
|
+
mock(klass).call(:get_multiple_leads, anything) do
|
207
|
+
preview_leads_response
|
208
|
+
end
|
165
209
|
end
|
210
|
+
|
211
|
+
mock(@page_builder).add(anything).times(Lead::PREVIEW_COUNT)
|
166
212
|
mock(@page_builder).finish
|
167
213
|
|
168
214
|
@plugin.run
|
169
215
|
end
|
170
216
|
|
217
|
+
def test_retry
|
218
|
+
setup_plugin
|
219
|
+
|
220
|
+
any_instance_of(Savon::Client) do |klass|
|
221
|
+
stub(klass).call(:get_multiple_leads, anything) do
|
222
|
+
raise "foo"
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
|
227
|
+
task[:retry_limit].times do |n|
|
228
|
+
mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
|
233
|
+
stub(Embulk.logger).info {}
|
234
|
+
|
235
|
+
assert_raise do
|
236
|
+
@plugin.run
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
171
240
|
class SavonCallTest < self
|
172
241
|
def test_soap_error
|
173
242
|
assert_raise(Embulk::ConfigError) do
|
@@ -195,7 +264,8 @@ module Embulk
|
|
195
264
|
end
|
196
265
|
|
197
266
|
def test_socket_error
|
198
|
-
|
267
|
+
mute_logger
|
268
|
+
stub(@soap).endpoint { "http://foo.test/" }
|
199
269
|
|
200
270
|
assert_raise(Embulk::ConfigError) do
|
201
271
|
@plugin.run
|
@@ -368,6 +438,16 @@ module Embulk
|
|
368
438
|
end
|
369
439
|
|
370
440
|
def task
|
441
|
+
# Values in Lead#timeslice are converted String from Time in
|
442
|
+
# Embulk (.transaction -> #init),
|
443
|
+
# but below values are passed to #init directly, so convert them.
|
444
|
+
raw_timeslice = Lead.timeslice(from_datetime, to_datetime, Lead::TIMESLICE_COUNT_PER_TASK)
|
445
|
+
timeslice = raw_timeslice.map do |ranges|
|
446
|
+
ranges.map do |range|
|
447
|
+
{"from" => range["from"].to_s, "to" => range["to"].to_s}
|
448
|
+
end
|
449
|
+
end
|
450
|
+
|
371
451
|
{
|
372
452
|
endpoint_url: "https://marketo.example.com",
|
373
453
|
wsdl_url: "https://marketo.example.com/?wsdl",
|
@@ -375,7 +455,10 @@ module Embulk
|
|
375
455
|
encryption_key: "TOPSECRET",
|
376
456
|
from_datetime: from_datetime,
|
377
457
|
to_datetime: to_datetime,
|
378
|
-
|
458
|
+
retry_initial_wait_sec: 2,
|
459
|
+
retry_limit: 3,
|
460
|
+
append_processed_time_column: true,
|
461
|
+
ranges: timeslice,
|
379
462
|
columns: [
|
380
463
|
{"name" => "Name", "type" => "string"},
|
381
464
|
]
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "embulk/input/marketo_api/soap/base"
|
2
2
|
require "lead_fixtures"
|
3
|
+
require "override_assert_raise"
|
3
4
|
|
4
5
|
module Embulk
|
5
6
|
module Input
|
@@ -7,16 +8,68 @@ module Embulk
|
|
7
8
|
module Soap
|
8
9
|
class BaseTest < Test::Unit::TestCase
|
9
10
|
include LeadFixtures
|
11
|
+
include OverrideAssertRaise
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
stub(
|
13
|
+
class TestRetry < self
|
14
|
+
def setup
|
15
|
+
stub(Embulk.logger).warn {}
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_retry_timeout
|
19
|
+
any_instance_of(Savon::Client) do |klass|
|
20
|
+
stub(klass).call(:timeout_test, advanced_typecasting: false) { raise ::Timeout::Error }
|
21
|
+
end
|
22
|
+
|
23
|
+
any_instance_of(MarketoApi::Soap::Base) do |klass|
|
24
|
+
retry_options[:retry_limit].times do |n|
|
25
|
+
mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
|
30
|
+
|
31
|
+
assert_raise(::Timeout::Error) do
|
32
|
+
soap.send(:savon_call, :timeout_test, {}, retry_options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_retry_common_error
|
37
|
+
any_instance_of(Savon::Client) do |klass|
|
38
|
+
stub(klass).call(:timeout_test, advanced_typecasting: false) { raise "something error" }
|
39
|
+
end
|
40
|
+
|
41
|
+
any_instance_of(MarketoApi::Soap::Base) do |klass|
|
42
|
+
retry_options[:retry_limit].times do |n|
|
43
|
+
mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
|
48
|
+
|
49
|
+
assert_raise do
|
50
|
+
soap.send(:savon_call, :timeout_test, {}, retry_options)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_not_retry_config_error
|
55
|
+
any_instance_of(Savon::Client) do |klass|
|
56
|
+
stub(klass).call(:timeout_test, advanced_typecasting: false) { raise Embulk::ConfigError.new("config error") }
|
57
|
+
end
|
58
|
+
|
59
|
+
mock(Embulk.logger).warn(/Retrying/).never
|
60
|
+
|
61
|
+
assert_raise(Embulk::ConfigError) do
|
62
|
+
soap.send(:savon_call, :timeout_test, {}, retry_options)
|
63
|
+
end
|
14
64
|
end
|
15
65
|
|
16
|
-
|
66
|
+
private
|
17
67
|
|
18
|
-
|
19
|
-
|
68
|
+
def retry_options
|
69
|
+
{
|
70
|
+
retry_limit: 4,
|
71
|
+
retry_initial_wait_sec: 3,
|
72
|
+
}
|
20
73
|
end
|
21
74
|
end
|
22
75
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -9,12 +9,12 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-10-
|
12
|
+
date: 2015-10-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 2.11.1
|
20
20
|
name: savon
|
@@ -22,13 +22,13 @@ dependencies:
|
|
22
22
|
type: :runtime
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
|
-
- - ~>
|
25
|
+
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
version: 2.11.1
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
name: httpclient
|
@@ -36,16 +36,16 @@ dependencies:
|
|
36
36
|
type: :runtime
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
|
-
- -
|
39
|
+
- - ">="
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: '0'
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: 0.6.13
|
48
|
-
- - <
|
48
|
+
- - "<"
|
49
49
|
- !ruby/object:Gem::Version
|
50
50
|
version: '1.0'
|
51
51
|
name: embulk
|
@@ -53,16 +53,16 @@ dependencies:
|
|
53
53
|
type: :development
|
54
54
|
version_requirements: !ruby/object:Gem::Requirement
|
55
55
|
requirements:
|
56
|
-
- -
|
56
|
+
- - ">="
|
57
57
|
- !ruby/object:Gem::Version
|
58
58
|
version: 0.6.13
|
59
|
-
- - <
|
59
|
+
- - "<"
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '1.0'
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
|
-
- - ~>
|
65
|
+
- - "~>"
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: '1.0'
|
68
68
|
name: bundler
|
@@ -70,13 +70,13 @@ dependencies:
|
|
70
70
|
type: :development
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - ~>
|
73
|
+
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '1.0'
|
76
76
|
- !ruby/object:Gem::Dependency
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
|
-
- -
|
79
|
+
- - ">="
|
80
80
|
- !ruby/object:Gem::Version
|
81
81
|
version: '10.0'
|
82
82
|
name: rake
|
@@ -84,13 +84,13 @@ dependencies:
|
|
84
84
|
type: :development
|
85
85
|
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '10.0'
|
90
90
|
- !ruby/object:Gem::Dependency
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- -
|
93
|
+
- - ">="
|
94
94
|
- !ruby/object:Gem::Version
|
95
95
|
version: '0'
|
96
96
|
name: pry
|
@@ -98,13 +98,13 @@ dependencies:
|
|
98
98
|
type: :development
|
99
99
|
version_requirements: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
106
106
|
requirements:
|
107
|
-
- -
|
107
|
+
- - ">="
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: '0'
|
110
110
|
name: test-unit
|
@@ -112,13 +112,13 @@ dependencies:
|
|
112
112
|
type: :development
|
113
113
|
version_requirements: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- -
|
115
|
+
- - ">="
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
- !ruby/object:Gem::Dependency
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
|
-
- -
|
121
|
+
- - ">="
|
122
122
|
- !ruby/object:Gem::Version
|
123
123
|
version: '0'
|
124
124
|
name: test-unit-rr
|
@@ -126,13 +126,13 @@ dependencies:
|
|
126
126
|
type: :development
|
127
127
|
version_requirements: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- -
|
129
|
+
- - ">="
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
- !ruby/object:Gem::Dependency
|
133
133
|
requirement: !ruby/object:Gem::Requirement
|
134
134
|
requirements:
|
135
|
-
- -
|
135
|
+
- - ">="
|
136
136
|
- !ruby/object:Gem::Version
|
137
137
|
version: '0'
|
138
138
|
name: codeclimate-test-reporter
|
@@ -140,13 +140,13 @@ dependencies:
|
|
140
140
|
type: :development
|
141
141
|
version_requirements: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
|
-
- -
|
143
|
+
- - ">="
|
144
144
|
- !ruby/object:Gem::Version
|
145
145
|
version: '0'
|
146
146
|
- !ruby/object:Gem::Dependency
|
147
147
|
requirement: !ruby/object:Gem::Requirement
|
148
148
|
requirements:
|
149
|
-
- -
|
149
|
+
- - ">="
|
150
150
|
- !ruby/object:Gem::Version
|
151
151
|
version: '0'
|
152
152
|
name: everyleaf-embulk_helper
|
@@ -154,7 +154,7 @@ dependencies:
|
|
154
154
|
type: :development
|
155
155
|
version_requirements: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- -
|
157
|
+
- - ">="
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
description: Loads records from Marketo.
|
@@ -165,10 +165,10 @@ executables: []
|
|
165
165
|
extensions: []
|
166
166
|
extra_rdoc_files: []
|
167
167
|
files:
|
168
|
-
- .gitignore
|
169
|
-
- .ruby-version
|
170
|
-
- .travis.yml
|
171
|
-
- .travis.yml.erb
|
168
|
+
- ".gitignore"
|
169
|
+
- ".ruby-version"
|
170
|
+
- ".travis.yml"
|
171
|
+
- ".travis.yml.erb"
|
172
172
|
- CHANGELOG.md
|
173
173
|
- Gemfile
|
174
174
|
- LICENSE
|
@@ -230,17 +230,17 @@ require_paths:
|
|
230
230
|
- lib
|
231
231
|
required_ruby_version: !ruby/object:Gem::Requirement
|
232
232
|
requirements:
|
233
|
-
- -
|
233
|
+
- - ">="
|
234
234
|
- !ruby/object:Gem::Version
|
235
235
|
version: '0'
|
236
236
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
237
237
|
requirements:
|
238
|
-
- -
|
238
|
+
- - ">="
|
239
239
|
- !ruby/object:Gem::Version
|
240
240
|
version: '0'
|
241
241
|
requirements: []
|
242
242
|
rubyforge_project:
|
243
|
-
rubygems_version: 2.4.
|
243
|
+
rubygems_version: 2.4.8
|
244
244
|
signing_key:
|
245
245
|
specification_version: 4
|
246
246
|
summary: Marketo input plugin for Embulk
|