embulk-input-marketo 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -62
  3. data/.travis.yml.erb +9 -14
  4. data/CHANGELOG.md +8 -0
  5. data/README.md +2 -4
  6. data/embulk-input-marketo.gemspec +2 -1
  7. data/gemfiles/embulk-latest +1 -1
  8. data/lib/embulk/input/marketo/activity_log.rb +13 -12
  9. data/lib/embulk/input/marketo/lead.rb +19 -17
  10. data/lib/embulk/input/marketo_api/soap/activity_log.rb +25 -23
  11. data/lib/embulk/input/marketo_api/soap/base.rb +14 -19
  12. data/lib/embulk/input/marketo_api/soap/lead.rb +22 -17
  13. data/test/activity_log_fixtures.rb +181 -150
  14. data/test/embulk/input/marketo/test_activity_log.rb +13 -15
  15. data/test/embulk/input/marketo/test_lead.rb +19 -22
  16. data/test/embulk/input/marketo_api/soap/test_activity_log.rb +33 -8
  17. data/test/embulk/input/marketo_api/soap/test_base.rb +2 -14
  18. data/test/embulk/input/marketo_api/soap/test_lead.rb +2 -2
  19. data/test/lead_fixtures.rb +11 -19
  20. data/test/savon_helper.rb +17 -0
  21. metadata +18 -25
  22. data/gemfiles/embulk-0.6.0-latest +0 -4
  23. data/gemfiles/embulk-0.6.13 +0 -4
  24. data/gemfiles/embulk-0.6.14 +0 -4
  25. data/gemfiles/embulk-0.6.15 +0 -4
  26. data/gemfiles/embulk-0.6.16 +0 -4
  27. data/gemfiles/embulk-0.6.17 +0 -4
  28. data/gemfiles/embulk-0.6.18 +0 -4
  29. data/gemfiles/embulk-0.6.19 +0 -4
  30. data/gemfiles/embulk-0.6.20 +0 -4
  31. data/gemfiles/embulk-0.6.21 +0 -4
  32. data/gemfiles/embulk-0.6.22 +0 -4
  33. data/gemfiles/embulk-0.6.23 +0 -4
  34. data/gemfiles/embulk-0.6.24 +0 -4
  35. data/gemfiles/embulk-0.6.25 +0 -4
  36. data/gemfiles/embulk-0.6.26 +0 -4
  37. data/gemfiles/embulk-0.6.27 +0 -4
  38. data/gemfiles/embulk-0.7.0 +0 -4
  39. data/gemfiles/embulk-0.7.0-latest +0 -4
  40. data/gemfiles/embulk-0.7.1 +0 -4
  41. data/gemfiles/embulk-0.7.2 +0 -4
  42. data/gemfiles/embulk-0.7.3 +0 -4
  43. data/gemfiles/embulk-0.7.4 +0 -4
  44. data/gemfiles/embulk-0.7.5 +0 -4
@@ -126,11 +126,11 @@ module Embulk
126
126
 
127
127
  any_instance_of(Savon::Client) do |klass|
128
128
  mock(klass).call(:get_multiple_leads, message: request) do
129
- leads_response
129
+ savon_response(xml_lead_response)
130
130
  end
131
131
 
132
132
  mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
133
- next_stream_leads_response
133
+ savon_response(xml_lead_next)
134
134
  end
135
135
  end
136
136
 
@@ -140,6 +140,7 @@ module Embulk
140
140
  mock(@page_builder).add(["ten-thousand-leaf", from])
141
141
  mock(@page_builder).finish
142
142
 
143
+ @plugin.init
143
144
  @plugin.run
144
145
  end
145
146
 
@@ -152,11 +153,11 @@ module Embulk
152
153
 
153
154
  any_instance_of(Savon::Client) do |klass|
154
155
  mock(klass).call(:get_multiple_leads, message: request) do
155
- leads_response
156
+ savon_response(xml_lead_response)
156
157
  end
157
158
 
158
159
  mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
159
- next_stream_leads_response
160
+ savon_response(xml_lead_next)
160
161
  end
161
162
  end
162
163
 
@@ -165,6 +166,7 @@ module Embulk
165
166
  mock(@page_builder).add(["ten-thousand-leaf"])
166
167
  mock(@page_builder).finish
167
168
 
169
+ @plugin.init
168
170
  @plugin.run
169
171
  end
170
172
 
@@ -184,7 +186,7 @@ module Embulk
184
186
 
185
187
  any_instance_of(Savon::Client) do |klass|
186
188
  mock(klass).call(:get_multiple_leads, message: request.merge(batch_size: Lead::PREVIEW_COUNT)) do
187
- preview_leads_response
189
+ savon_response(xml_lead_preview)
188
190
  end
189
191
  end
190
192
 
@@ -194,6 +196,7 @@ module Embulk
194
196
  end
195
197
  mock(@page_builder).finish
196
198
 
199
+ @plugin.init
197
200
  @plugin.run
198
201
  end
199
202
 
@@ -204,13 +207,14 @@ module Embulk
204
207
 
205
208
  any_instance_of(Savon::Client) do |klass|
206
209
  mock(klass).call(:get_multiple_leads, anything) do
207
- preview_leads_response
210
+ savon_response(xml_lead_preview)
208
211
  end
209
212
  end
210
213
 
211
214
  mock(@page_builder).add(anything).times(Lead::PREVIEW_COUNT)
212
215
  mock(@page_builder).finish
213
216
 
217
+ @plugin.init
214
218
  @plugin.run
215
219
  end
216
220
 
@@ -223,25 +227,25 @@ module Embulk
223
227
  end
224
228
  end
225
229
 
226
- any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
227
- task[:retry_limit].times do |n|
228
- mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
229
- end
230
- end
231
-
232
230
  mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
233
231
  stub(Embulk.logger).info {}
234
232
 
235
233
  assert_raise do
234
+ @plugin.init
236
235
  @plugin.run
237
236
  end
238
237
  end
239
238
 
240
239
  class SavonCallTest < self
241
240
  def test_soap_error
241
+ nori_options = {
242
+ :strip_namespaces => true,
243
+ :convert_tags_to => lambda { |tag| tag.snakecase.to_sym},
244
+ :convert_attributes_to => lambda { |k,v| [k,v] },
245
+ }
242
246
  assert_raise(Embulk::ConfigError) do
243
247
  @soap.send(:catch_unretryable_error) do
244
- raise Savon::SOAPFault.new(nil, Nori.new(default_nori_options), xml)
248
+ raise Savon::SOAPFault.new(nil, Nori.new(nori_options), xml)
245
249
  end
246
250
  end
247
251
  end
@@ -268,6 +272,7 @@ module Embulk
268
272
  stub(@soap).endpoint { "http://foo.test/" }
269
273
 
270
274
  assert_raise(Embulk::ConfigError) do
275
+ @plugin.init
271
276
  @plugin.run
272
277
  end
273
278
  end
@@ -287,14 +292,6 @@ module Embulk
287
292
  XML
288
293
  end
289
294
 
290
- def default_nori_options
291
- # https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
292
- {
293
- :strip_namespaces => true,
294
- :convert_tags_to => lambda { |tag| tag.snakecase.to_sym},
295
- :convert_attributes_to => lambda { |k,v| [k,v] },
296
- }
297
- end
298
295
  end
299
296
 
300
297
  class TestTimeslice < self
@@ -455,7 +452,7 @@ module Embulk
455
452
  encryption_key: "TOPSECRET",
456
453
  from_datetime: from_datetime,
457
454
  to_datetime: to_datetime,
458
- retry_initial_wait_sec: 2,
455
+ retry_initial_wait_sec: 0,
459
456
  retry_limit: 3,
460
457
  append_processed_time_column: true,
461
458
  ranges: timeslice,
@@ -22,18 +22,43 @@ module Embulk
22
22
 
23
23
  any_instance_of(Savon::Client) do |klass|
24
24
  mock(klass).call(:get_lead_changes, message: request) do
25
- next_stream_activity_logs_response
25
+ savon_response xml_ac_next_response
26
26
  end
27
27
  end
28
28
 
29
29
  proc = proc{ "" }
30
- activity_logs = next_stream_activity_logs_response[:body][:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
31
- last_activity_log = activity_logs.last
30
+ activity_logs = savon_response(xml_ac_next_response).xpath('//leadChangeRecord')
31
+ latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
32
32
 
33
33
  mock(proc).call(anything).times(activity_logs.size)
34
- assert_equal(last_activity_log[:activity_date_time], soap.each(from_datetime, &proc))
34
+ assert_equal(latest, soap.each(from_datetime, &proc))
35
35
  end
36
36
 
37
+ def test_no_attributes
38
+ request = {
39
+ start_position: {
40
+ oldest_created_at: Time.parse(from_datetime).iso8601,
41
+ },
42
+ batch_size: 100
43
+ }
44
+
45
+ any_instance_of(Savon::Client) do |klass|
46
+ mock(klass).call(:get_lead_changes, message: request) do
47
+ savon_response xml_ac_response_no_attributes
48
+ end
49
+ end
50
+
51
+ proc = proc{ "" }
52
+ activity_logs = savon_response(xml_ac_response_no_attributes).xpath('//leadChangeRecord')
53
+ latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
54
+
55
+ mock(proc).call(anything).times(activity_logs.size)
56
+ assert_nothing_raised do
57
+ assert_equal(latest, soap.each(from_datetime, &proc))
58
+ end
59
+ end
60
+
61
+
37
62
  def test_each_with_no_response
38
63
  request = {
39
64
  start_position: {
@@ -44,7 +69,7 @@ module Embulk
44
69
 
45
70
  any_instance_of(Savon::Client) do |klass|
46
71
  mock(klass).call(:get_lead_changes, message: request) do
47
- none_activity_log_response
72
+ savon_response xml_ac_none_response
48
73
  end
49
74
  end
50
75
 
@@ -63,14 +88,14 @@ module Embulk
63
88
 
64
89
  def test_savon_call
65
90
  mock(@savon).call(:get_lead_changes, message: request) {
66
- next_stream_activity_logs_response
91
+ savon_response xml_ac_next_response
67
92
  }
68
93
  soap.metadata(from_datetime)
69
94
  end
70
95
 
71
96
  def test_return_schema
72
97
  stub(@savon).call(:get_lead_changes, message: request) {
73
- next_stream_activity_logs_response
98
+ savon_response xml_ac_next_response
74
99
  }
75
100
  assert_equal(schema, soap.metadata(from_datetime))
76
101
  end
@@ -90,7 +115,7 @@ module Embulk
90
115
  def schema
91
116
  metadata = [
92
117
  {index: 0, name: "id", type: :long},
93
- {index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z"}, # NOTE: `format` is the same as-is response (e.g. "2015-07-14T00:00:11+0000" to "%Y-%m-%dT%H:%M:%S%z")
118
+ {index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%:z"}, # NOTE: `format` is the same as-is response (e.g. "2015-07-14T00:00:11+0000" to "%Y-%m-%dT%H:%M:%S%:z")
94
119
  {index: 2, name: "activity_type", type: :string},
95
120
  {index: 3, name: "mktg_asset_name", type: :string},
96
121
  {index: 4, name: "mkt_person_id", type: :long},
@@ -20,15 +20,9 @@ module Embulk
20
20
  stub(klass).call(:timeout_test, advanced_typecasting: false) { raise ::Timeout::Error }
21
21
  end
22
22
 
23
- any_instance_of(MarketoApi::Soap::Base) do |klass|
24
- retry_options[:retry_limit].times do |n|
25
- mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
26
- end
27
- end
28
-
29
23
  mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
30
24
 
31
- assert_raise(::Timeout::Error) do
25
+ assert_raise(PerfectRetry::TooManyRetry) do
32
26
  soap.send(:savon_call, :timeout_test, {}, retry_options)
33
27
  end
34
28
  end
@@ -38,12 +32,6 @@ module Embulk
38
32
  stub(klass).call(:timeout_test, advanced_typecasting: false) { raise "something error" }
39
33
  end
40
34
 
41
- any_instance_of(MarketoApi::Soap::Base) do |klass|
42
- retry_options[:retry_limit].times do |n|
43
- mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
44
- end
45
- end
46
-
47
35
  mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
48
36
 
49
37
  assert_raise do
@@ -68,7 +56,7 @@ module Embulk
68
56
  def retry_options
69
57
  {
70
58
  retry_limit: 4,
71
- retry_initial_wait_sec: 3,
59
+ retry_initial_wait_sec: 0,
72
60
  }
73
61
  end
74
62
  end
@@ -47,12 +47,12 @@ module Embulk
47
47
 
48
48
  any_instance_of(Savon::Client) do |klass|
49
49
  mock(klass).call(:get_multiple_leads, anything) do
50
- next_stream_leads_response
50
+ savon_response(xml_lead_next)
51
51
  end
52
52
  end
53
53
 
54
54
  proc = proc{ "" }
55
- leads_count = next_stream_leads_response.xpath('//leadRecord').length
55
+ leads_count = savon_response(xml_lead_next).xpath('//leadRecord').length
56
56
  mock(proc).call(anything).times(leads_count)
57
57
 
58
58
  soap.each(timerange, {}, &proc)
@@ -1,19 +1,11 @@
1
- module LeadFixtures
2
- private
3
-
4
- def leads_response
5
- Nokogiri::XML(raw_response)
6
- end
1
+ require "savon_helper"
7
2
 
8
- def next_stream_leads_response
9
- Nokogiri::XML(raw_next_stream_response)
10
- end
3
+ module LeadFixtures
4
+ include SavonHelper
11
5
 
12
- def preview_leads_response
13
- Nokogiri::XML(raw_preview_response)
14
- end
6
+ private
15
7
 
16
- def leads(body)
8
+ def leads_xml(body)
17
9
  <<XML
18
10
  <?xml version="1.0" encoding="UTF-8"?>
19
11
  <SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
@@ -28,8 +20,8 @@ module LeadFixtures
28
20
  XML
29
21
  end
30
22
 
31
- def raw_response
32
- leads(<<XML)
23
+ def xml_lead_response
24
+ leads_xml(<<XML)
33
25
  <remainingCount>1</remainingCount>
34
26
  <newStreamPosition>#{stream_position}</newStreamPosition>
35
27
  <leadRecordList>
@@ -67,8 +59,8 @@ XML
67
59
  "next_steam_position"
68
60
  end
69
61
 
70
- def raw_next_stream_response
71
- leads(<<XML)
62
+ def xml_lead_next
63
+ leads_xml(<<XML)
72
64
  <returnCount>2</returnCount>
73
65
  <remainingCount>0</remainingCount>
74
66
  <newStreamPosition />
@@ -90,7 +82,7 @@ XML
90
82
  XML
91
83
  end
92
84
 
93
- def raw_preview_response
85
+ def xml_lead_preview
94
86
  body = ""
95
87
  15.times do |i|
96
88
  body << <<XML
@@ -114,6 +106,6 @@ XML
114
106
  </leadRecordList>
115
107
  XML
116
108
  end
117
- leads(body)
109
+ leads_xml(body)
118
110
  end
119
111
  end
@@ -0,0 +1,17 @@
1
+ module SavonHelper
2
+ private
3
+
4
+ def savon_response(body)
5
+ globals = {
6
+ namespace_identifier: :ns1,
7
+ env_namespace: 'SOAP-ENV',
8
+
9
+ # https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
10
+ strip_namespaces: true,
11
+ convert_response_tags_to: lambda { |tag| tag.snakecase.to_sym},
12
+ convert_attributes_to: lambda { |k,v| [k,v] },
13
+ }
14
+ httpi = HTTPI::Response.new(200, {}, body)
15
+ Savon::Response.new(httpi, globals, {advanced_typecasting: false})
16
+ end
17
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-marketo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - uu59
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-10-30 00:00:00.000000000 Z
12
+ date: 2016-04-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -39,6 +39,20 @@ dependencies:
39
39
  - - ">="
40
40
  - !ruby/object:Gem::Version
41
41
  version: '0'
42
+ - !ruby/object:Gem::Dependency
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.3.2
48
+ name: perfect_retry
49
+ prerelease: false
50
+ type: :runtime
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - "~>"
54
+ - !ruby/object:Gem::Version
55
+ version: 0.3.2
42
56
  - !ruby/object:Gem::Dependency
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
@@ -175,29 +189,6 @@ files:
175
189
  - README.md
176
190
  - Rakefile
177
191
  - embulk-input-marketo.gemspec
178
- - gemfiles/embulk-0.6.0-latest
179
- - gemfiles/embulk-0.6.13
180
- - gemfiles/embulk-0.6.14
181
- - gemfiles/embulk-0.6.15
182
- - gemfiles/embulk-0.6.16
183
- - gemfiles/embulk-0.6.17
184
- - gemfiles/embulk-0.6.18
185
- - gemfiles/embulk-0.6.19
186
- - gemfiles/embulk-0.6.20
187
- - gemfiles/embulk-0.6.21
188
- - gemfiles/embulk-0.6.22
189
- - gemfiles/embulk-0.6.23
190
- - gemfiles/embulk-0.6.24
191
- - gemfiles/embulk-0.6.25
192
- - gemfiles/embulk-0.6.26
193
- - gemfiles/embulk-0.6.27
194
- - gemfiles/embulk-0.7.0
195
- - gemfiles/embulk-0.7.0-latest
196
- - gemfiles/embulk-0.7.1
197
- - gemfiles/embulk-0.7.2
198
- - gemfiles/embulk-0.7.3
199
- - gemfiles/embulk-0.7.4
200
- - gemfiles/embulk-0.7.5
201
192
  - gemfiles/embulk-latest
202
193
  - gemfiles/template.erb
203
194
  - lib/embulk/input/marketo/activity_log.rb
@@ -220,6 +211,7 @@ files:
220
211
  - test/override_assert_raise.rb
221
212
  - test/prepare_embulk.rb
222
213
  - test/run-test.rb
214
+ - test/savon_helper.rb
223
215
  homepage: https://github.com/treasure-data/embulk-input-marketo
224
216
  licenses:
225
217
  - Apache2
@@ -258,3 +250,4 @@ test_files:
258
250
  - test/override_assert_raise.rb
259
251
  - test/prepare_embulk.rb
260
252
  - test/run-test.rb
253
+ - test/savon_helper.rb
@@ -1,4 +0,0 @@
1
- source 'https://rubygems.org/'
2
- gemspec :path => '../'
3
-
4
- gem "embulk", "~> 0.6.0"
@@ -1,4 +0,0 @@
1
- source 'https://rubygems.org/'
2
- gemspec :path => '../'
3
-
4
- gem "embulk", "0.6.13"