embulk-input-marketo 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +16 -62
- data/.travis.yml.erb +9 -14
- data/CHANGELOG.md +8 -0
- data/README.md +2 -4
- data/embulk-input-marketo.gemspec +2 -1
- data/gemfiles/embulk-latest +1 -1
- data/lib/embulk/input/marketo/activity_log.rb +13 -12
- data/lib/embulk/input/marketo/lead.rb +19 -17
- data/lib/embulk/input/marketo_api/soap/activity_log.rb +25 -23
- data/lib/embulk/input/marketo_api/soap/base.rb +14 -19
- data/lib/embulk/input/marketo_api/soap/lead.rb +22 -17
- data/test/activity_log_fixtures.rb +181 -150
- data/test/embulk/input/marketo/test_activity_log.rb +13 -15
- data/test/embulk/input/marketo/test_lead.rb +19 -22
- data/test/embulk/input/marketo_api/soap/test_activity_log.rb +33 -8
- data/test/embulk/input/marketo_api/soap/test_base.rb +2 -14
- data/test/embulk/input/marketo_api/soap/test_lead.rb +2 -2
- data/test/lead_fixtures.rb +11 -19
- data/test/savon_helper.rb +17 -0
- metadata +18 -25
- data/gemfiles/embulk-0.6.0-latest +0 -4
- data/gemfiles/embulk-0.6.13 +0 -4
- data/gemfiles/embulk-0.6.14 +0 -4
- data/gemfiles/embulk-0.6.15 +0 -4
- data/gemfiles/embulk-0.6.16 +0 -4
- data/gemfiles/embulk-0.6.17 +0 -4
- data/gemfiles/embulk-0.6.18 +0 -4
- data/gemfiles/embulk-0.6.19 +0 -4
- data/gemfiles/embulk-0.6.20 +0 -4
- data/gemfiles/embulk-0.6.21 +0 -4
- data/gemfiles/embulk-0.6.22 +0 -4
- data/gemfiles/embulk-0.6.23 +0 -4
- data/gemfiles/embulk-0.6.24 +0 -4
- data/gemfiles/embulk-0.6.25 +0 -4
- data/gemfiles/embulk-0.6.26 +0 -4
- data/gemfiles/embulk-0.6.27 +0 -4
- data/gemfiles/embulk-0.7.0 +0 -4
- data/gemfiles/embulk-0.7.0-latest +0 -4
- data/gemfiles/embulk-0.7.1 +0 -4
- data/gemfiles/embulk-0.7.2 +0 -4
- data/gemfiles/embulk-0.7.3 +0 -4
- data/gemfiles/embulk-0.7.4 +0 -4
- data/gemfiles/embulk-0.7.5 +0 -4
@@ -126,11 +126,11 @@ module Embulk
|
|
126
126
|
|
127
127
|
any_instance_of(Savon::Client) do |klass|
|
128
128
|
mock(klass).call(:get_multiple_leads, message: request) do
|
129
|
-
|
129
|
+
savon_response(xml_lead_response)
|
130
130
|
end
|
131
131
|
|
132
132
|
mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
|
133
|
-
|
133
|
+
savon_response(xml_lead_next)
|
134
134
|
end
|
135
135
|
end
|
136
136
|
|
@@ -140,6 +140,7 @@ module Embulk
|
|
140
140
|
mock(@page_builder).add(["ten-thousand-leaf", from])
|
141
141
|
mock(@page_builder).finish
|
142
142
|
|
143
|
+
@plugin.init
|
143
144
|
@plugin.run
|
144
145
|
end
|
145
146
|
|
@@ -152,11 +153,11 @@ module Embulk
|
|
152
153
|
|
153
154
|
any_instance_of(Savon::Client) do |klass|
|
154
155
|
mock(klass).call(:get_multiple_leads, message: request) do
|
155
|
-
|
156
|
+
savon_response(xml_lead_response)
|
156
157
|
end
|
157
158
|
|
158
159
|
mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
|
159
|
-
|
160
|
+
savon_response(xml_lead_next)
|
160
161
|
end
|
161
162
|
end
|
162
163
|
|
@@ -165,6 +166,7 @@ module Embulk
|
|
165
166
|
mock(@page_builder).add(["ten-thousand-leaf"])
|
166
167
|
mock(@page_builder).finish
|
167
168
|
|
169
|
+
@plugin.init
|
168
170
|
@plugin.run
|
169
171
|
end
|
170
172
|
|
@@ -184,7 +186,7 @@ module Embulk
|
|
184
186
|
|
185
187
|
any_instance_of(Savon::Client) do |klass|
|
186
188
|
mock(klass).call(:get_multiple_leads, message: request.merge(batch_size: Lead::PREVIEW_COUNT)) do
|
187
|
-
|
189
|
+
savon_response(xml_lead_preview)
|
188
190
|
end
|
189
191
|
end
|
190
192
|
|
@@ -194,6 +196,7 @@ module Embulk
|
|
194
196
|
end
|
195
197
|
mock(@page_builder).finish
|
196
198
|
|
199
|
+
@plugin.init
|
197
200
|
@plugin.run
|
198
201
|
end
|
199
202
|
|
@@ -204,13 +207,14 @@ module Embulk
|
|
204
207
|
|
205
208
|
any_instance_of(Savon::Client) do |klass|
|
206
209
|
mock(klass).call(:get_multiple_leads, anything) do
|
207
|
-
|
210
|
+
savon_response(xml_lead_preview)
|
208
211
|
end
|
209
212
|
end
|
210
213
|
|
211
214
|
mock(@page_builder).add(anything).times(Lead::PREVIEW_COUNT)
|
212
215
|
mock(@page_builder).finish
|
213
216
|
|
217
|
+
@plugin.init
|
214
218
|
@plugin.run
|
215
219
|
end
|
216
220
|
|
@@ -223,25 +227,25 @@ module Embulk
|
|
223
227
|
end
|
224
228
|
end
|
225
229
|
|
226
|
-
any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
|
227
|
-
task[:retry_limit].times do |n|
|
228
|
-
mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
|
229
|
-
end
|
230
|
-
end
|
231
|
-
|
232
230
|
mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
|
233
231
|
stub(Embulk.logger).info {}
|
234
232
|
|
235
233
|
assert_raise do
|
234
|
+
@plugin.init
|
236
235
|
@plugin.run
|
237
236
|
end
|
238
237
|
end
|
239
238
|
|
240
239
|
class SavonCallTest < self
|
241
240
|
def test_soap_error
|
241
|
+
nori_options = {
|
242
|
+
:strip_namespaces => true,
|
243
|
+
:convert_tags_to => lambda { |tag| tag.snakecase.to_sym},
|
244
|
+
:convert_attributes_to => lambda { |k,v| [k,v] },
|
245
|
+
}
|
242
246
|
assert_raise(Embulk::ConfigError) do
|
243
247
|
@soap.send(:catch_unretryable_error) do
|
244
|
-
raise Savon::SOAPFault.new(nil, Nori.new(
|
248
|
+
raise Savon::SOAPFault.new(nil, Nori.new(nori_options), xml)
|
245
249
|
end
|
246
250
|
end
|
247
251
|
end
|
@@ -268,6 +272,7 @@ module Embulk
|
|
268
272
|
stub(@soap).endpoint { "http://foo.test/" }
|
269
273
|
|
270
274
|
assert_raise(Embulk::ConfigError) do
|
275
|
+
@plugin.init
|
271
276
|
@plugin.run
|
272
277
|
end
|
273
278
|
end
|
@@ -287,14 +292,6 @@ module Embulk
|
|
287
292
|
XML
|
288
293
|
end
|
289
294
|
|
290
|
-
def default_nori_options
|
291
|
-
# https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
|
292
|
-
{
|
293
|
-
:strip_namespaces => true,
|
294
|
-
:convert_tags_to => lambda { |tag| tag.snakecase.to_sym},
|
295
|
-
:convert_attributes_to => lambda { |k,v| [k,v] },
|
296
|
-
}
|
297
|
-
end
|
298
295
|
end
|
299
296
|
|
300
297
|
class TestTimeslice < self
|
@@ -455,7 +452,7 @@ module Embulk
|
|
455
452
|
encryption_key: "TOPSECRET",
|
456
453
|
from_datetime: from_datetime,
|
457
454
|
to_datetime: to_datetime,
|
458
|
-
retry_initial_wait_sec:
|
455
|
+
retry_initial_wait_sec: 0,
|
459
456
|
retry_limit: 3,
|
460
457
|
append_processed_time_column: true,
|
461
458
|
ranges: timeslice,
|
@@ -22,18 +22,43 @@ module Embulk
|
|
22
22
|
|
23
23
|
any_instance_of(Savon::Client) do |klass|
|
24
24
|
mock(klass).call(:get_lead_changes, message: request) do
|
25
|
-
|
25
|
+
savon_response xml_ac_next_response
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
29
|
proc = proc{ "" }
|
30
|
-
activity_logs =
|
31
|
-
|
30
|
+
activity_logs = savon_response(xml_ac_next_response).xpath('//leadChangeRecord')
|
31
|
+
latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
|
32
32
|
|
33
33
|
mock(proc).call(anything).times(activity_logs.size)
|
34
|
-
assert_equal(
|
34
|
+
assert_equal(latest, soap.each(from_datetime, &proc))
|
35
35
|
end
|
36
36
|
|
37
|
+
def test_no_attributes
|
38
|
+
request = {
|
39
|
+
start_position: {
|
40
|
+
oldest_created_at: Time.parse(from_datetime).iso8601,
|
41
|
+
},
|
42
|
+
batch_size: 100
|
43
|
+
}
|
44
|
+
|
45
|
+
any_instance_of(Savon::Client) do |klass|
|
46
|
+
mock(klass).call(:get_lead_changes, message: request) do
|
47
|
+
savon_response xml_ac_response_no_attributes
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
proc = proc{ "" }
|
52
|
+
activity_logs = savon_response(xml_ac_response_no_attributes).xpath('//leadChangeRecord')
|
53
|
+
latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
|
54
|
+
|
55
|
+
mock(proc).call(anything).times(activity_logs.size)
|
56
|
+
assert_nothing_raised do
|
57
|
+
assert_equal(latest, soap.each(from_datetime, &proc))
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
|
37
62
|
def test_each_with_no_response
|
38
63
|
request = {
|
39
64
|
start_position: {
|
@@ -44,7 +69,7 @@ module Embulk
|
|
44
69
|
|
45
70
|
any_instance_of(Savon::Client) do |klass|
|
46
71
|
mock(klass).call(:get_lead_changes, message: request) do
|
47
|
-
|
72
|
+
savon_response xml_ac_none_response
|
48
73
|
end
|
49
74
|
end
|
50
75
|
|
@@ -63,14 +88,14 @@ module Embulk
|
|
63
88
|
|
64
89
|
def test_savon_call
|
65
90
|
mock(@savon).call(:get_lead_changes, message: request) {
|
66
|
-
|
91
|
+
savon_response xml_ac_next_response
|
67
92
|
}
|
68
93
|
soap.metadata(from_datetime)
|
69
94
|
end
|
70
95
|
|
71
96
|
def test_return_schema
|
72
97
|
stub(@savon).call(:get_lead_changes, message: request) {
|
73
|
-
|
98
|
+
savon_response xml_ac_next_response
|
74
99
|
}
|
75
100
|
assert_equal(schema, soap.metadata(from_datetime))
|
76
101
|
end
|
@@ -90,7 +115,7 @@ module Embulk
|
|
90
115
|
def schema
|
91
116
|
metadata = [
|
92
117
|
{index: 0, name: "id", type: :long},
|
93
|
-
{index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S
|
118
|
+
{index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%:z"}, # NOTE: `format` is the same as-is response (e.g. "2015-07-14T00:00:11+0000" to "%Y-%m-%dT%H:%M:%S%:z")
|
94
119
|
{index: 2, name: "activity_type", type: :string},
|
95
120
|
{index: 3, name: "mktg_asset_name", type: :string},
|
96
121
|
{index: 4, name: "mkt_person_id", type: :long},
|
@@ -20,15 +20,9 @@ module Embulk
|
|
20
20
|
stub(klass).call(:timeout_test, advanced_typecasting: false) { raise ::Timeout::Error }
|
21
21
|
end
|
22
22
|
|
23
|
-
any_instance_of(MarketoApi::Soap::Base) do |klass|
|
24
|
-
retry_options[:retry_limit].times do |n|
|
25
|
-
mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
23
|
mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
|
30
24
|
|
31
|
-
assert_raise(::
|
25
|
+
assert_raise(PerfectRetry::TooManyRetry) do
|
32
26
|
soap.send(:savon_call, :timeout_test, {}, retry_options)
|
33
27
|
end
|
34
28
|
end
|
@@ -38,12 +32,6 @@ module Embulk
|
|
38
32
|
stub(klass).call(:timeout_test, advanced_typecasting: false) { raise "something error" }
|
39
33
|
end
|
40
34
|
|
41
|
-
any_instance_of(MarketoApi::Soap::Base) do |klass|
|
42
|
-
retry_options[:retry_limit].times do |n|
|
43
|
-
mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
35
|
mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
|
48
36
|
|
49
37
|
assert_raise do
|
@@ -68,7 +56,7 @@ module Embulk
|
|
68
56
|
def retry_options
|
69
57
|
{
|
70
58
|
retry_limit: 4,
|
71
|
-
retry_initial_wait_sec:
|
59
|
+
retry_initial_wait_sec: 0,
|
72
60
|
}
|
73
61
|
end
|
74
62
|
end
|
@@ -47,12 +47,12 @@ module Embulk
|
|
47
47
|
|
48
48
|
any_instance_of(Savon::Client) do |klass|
|
49
49
|
mock(klass).call(:get_multiple_leads, anything) do
|
50
|
-
|
50
|
+
savon_response(xml_lead_next)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
54
|
proc = proc{ "" }
|
55
|
-
leads_count =
|
55
|
+
leads_count = savon_response(xml_lead_next).xpath('//leadRecord').length
|
56
56
|
mock(proc).call(anything).times(leads_count)
|
57
57
|
|
58
58
|
soap.each(timerange, {}, &proc)
|
data/test/lead_fixtures.rb
CHANGED
@@ -1,19 +1,11 @@
|
|
1
|
-
|
2
|
-
private
|
3
|
-
|
4
|
-
def leads_response
|
5
|
-
Nokogiri::XML(raw_response)
|
6
|
-
end
|
1
|
+
require "savon_helper"
|
7
2
|
|
8
|
-
|
9
|
-
|
10
|
-
end
|
3
|
+
module LeadFixtures
|
4
|
+
include SavonHelper
|
11
5
|
|
12
|
-
|
13
|
-
Nokogiri::XML(raw_preview_response)
|
14
|
-
end
|
6
|
+
private
|
15
7
|
|
16
|
-
def
|
8
|
+
def leads_xml(body)
|
17
9
|
<<XML
|
18
10
|
<?xml version="1.0" encoding="UTF-8"?>
|
19
11
|
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
|
@@ -28,8 +20,8 @@ module LeadFixtures
|
|
28
20
|
XML
|
29
21
|
end
|
30
22
|
|
31
|
-
def
|
32
|
-
|
23
|
+
def xml_lead_response
|
24
|
+
leads_xml(<<XML)
|
33
25
|
<remainingCount>1</remainingCount>
|
34
26
|
<newStreamPosition>#{stream_position}</newStreamPosition>
|
35
27
|
<leadRecordList>
|
@@ -67,8 +59,8 @@ XML
|
|
67
59
|
"next_steam_position"
|
68
60
|
end
|
69
61
|
|
70
|
-
def
|
71
|
-
|
62
|
+
def xml_lead_next
|
63
|
+
leads_xml(<<XML)
|
72
64
|
<returnCount>2</returnCount>
|
73
65
|
<remainingCount>0</remainingCount>
|
74
66
|
<newStreamPosition />
|
@@ -90,7 +82,7 @@ XML
|
|
90
82
|
XML
|
91
83
|
end
|
92
84
|
|
93
|
-
def
|
85
|
+
def xml_lead_preview
|
94
86
|
body = ""
|
95
87
|
15.times do |i|
|
96
88
|
body << <<XML
|
@@ -114,6 +106,6 @@ XML
|
|
114
106
|
</leadRecordList>
|
115
107
|
XML
|
116
108
|
end
|
117
|
-
|
109
|
+
leads_xml(body)
|
118
110
|
end
|
119
111
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module SavonHelper
|
2
|
+
private
|
3
|
+
|
4
|
+
def savon_response(body)
|
5
|
+
globals = {
|
6
|
+
namespace_identifier: :ns1,
|
7
|
+
env_namespace: 'SOAP-ENV',
|
8
|
+
|
9
|
+
# https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
|
10
|
+
strip_namespaces: true,
|
11
|
+
convert_response_tags_to: lambda { |tag| tag.snakecase.to_sym},
|
12
|
+
convert_attributes_to: lambda { |k,v| [k,v] },
|
13
|
+
}
|
14
|
+
httpi = HTTPI::Response.new(200, {}, body)
|
15
|
+
Savon::Response.new(httpi, globals, {advanced_typecasting: false})
|
16
|
+
end
|
17
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2016-04-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,6 +39,20 @@ dependencies:
|
|
39
39
|
- - ">="
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.3.2
|
48
|
+
name: perfect_retry
|
49
|
+
prerelease: false
|
50
|
+
type: :runtime
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: 0.3.2
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
@@ -175,29 +189,6 @@ files:
|
|
175
189
|
- README.md
|
176
190
|
- Rakefile
|
177
191
|
- embulk-input-marketo.gemspec
|
178
|
-
- gemfiles/embulk-0.6.0-latest
|
179
|
-
- gemfiles/embulk-0.6.13
|
180
|
-
- gemfiles/embulk-0.6.14
|
181
|
-
- gemfiles/embulk-0.6.15
|
182
|
-
- gemfiles/embulk-0.6.16
|
183
|
-
- gemfiles/embulk-0.6.17
|
184
|
-
- gemfiles/embulk-0.6.18
|
185
|
-
- gemfiles/embulk-0.6.19
|
186
|
-
- gemfiles/embulk-0.6.20
|
187
|
-
- gemfiles/embulk-0.6.21
|
188
|
-
- gemfiles/embulk-0.6.22
|
189
|
-
- gemfiles/embulk-0.6.23
|
190
|
-
- gemfiles/embulk-0.6.24
|
191
|
-
- gemfiles/embulk-0.6.25
|
192
|
-
- gemfiles/embulk-0.6.26
|
193
|
-
- gemfiles/embulk-0.6.27
|
194
|
-
- gemfiles/embulk-0.7.0
|
195
|
-
- gemfiles/embulk-0.7.0-latest
|
196
|
-
- gemfiles/embulk-0.7.1
|
197
|
-
- gemfiles/embulk-0.7.2
|
198
|
-
- gemfiles/embulk-0.7.3
|
199
|
-
- gemfiles/embulk-0.7.4
|
200
|
-
- gemfiles/embulk-0.7.5
|
201
192
|
- gemfiles/embulk-latest
|
202
193
|
- gemfiles/template.erb
|
203
194
|
- lib/embulk/input/marketo/activity_log.rb
|
@@ -220,6 +211,7 @@ files:
|
|
220
211
|
- test/override_assert_raise.rb
|
221
212
|
- test/prepare_embulk.rb
|
222
213
|
- test/run-test.rb
|
214
|
+
- test/savon_helper.rb
|
223
215
|
homepage: https://github.com/treasure-data/embulk-input-marketo
|
224
216
|
licenses:
|
225
217
|
- Apache2
|
@@ -258,3 +250,4 @@ test_files:
|
|
258
250
|
- test/override_assert_raise.rb
|
259
251
|
- test/prepare_embulk.rb
|
260
252
|
- test/run-test.rb
|
253
|
+
- test/savon_helper.rb
|
data/gemfiles/embulk-0.6.13
DELETED