embulk-input-marketo 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +16 -62
- data/.travis.yml.erb +9 -14
- data/CHANGELOG.md +8 -0
- data/README.md +2 -4
- data/embulk-input-marketo.gemspec +2 -1
- data/gemfiles/embulk-latest +1 -1
- data/lib/embulk/input/marketo/activity_log.rb +13 -12
- data/lib/embulk/input/marketo/lead.rb +19 -17
- data/lib/embulk/input/marketo_api/soap/activity_log.rb +25 -23
- data/lib/embulk/input/marketo_api/soap/base.rb +14 -19
- data/lib/embulk/input/marketo_api/soap/lead.rb +22 -17
- data/test/activity_log_fixtures.rb +181 -150
- data/test/embulk/input/marketo/test_activity_log.rb +13 -15
- data/test/embulk/input/marketo/test_lead.rb +19 -22
- data/test/embulk/input/marketo_api/soap/test_activity_log.rb +33 -8
- data/test/embulk/input/marketo_api/soap/test_base.rb +2 -14
- data/test/embulk/input/marketo_api/soap/test_lead.rb +2 -2
- data/test/lead_fixtures.rb +11 -19
- data/test/savon_helper.rb +17 -0
- metadata +18 -25
- data/gemfiles/embulk-0.6.0-latest +0 -4
- data/gemfiles/embulk-0.6.13 +0 -4
- data/gemfiles/embulk-0.6.14 +0 -4
- data/gemfiles/embulk-0.6.15 +0 -4
- data/gemfiles/embulk-0.6.16 +0 -4
- data/gemfiles/embulk-0.6.17 +0 -4
- data/gemfiles/embulk-0.6.18 +0 -4
- data/gemfiles/embulk-0.6.19 +0 -4
- data/gemfiles/embulk-0.6.20 +0 -4
- data/gemfiles/embulk-0.6.21 +0 -4
- data/gemfiles/embulk-0.6.22 +0 -4
- data/gemfiles/embulk-0.6.23 +0 -4
- data/gemfiles/embulk-0.6.24 +0 -4
- data/gemfiles/embulk-0.6.25 +0 -4
- data/gemfiles/embulk-0.6.26 +0 -4
- data/gemfiles/embulk-0.6.27 +0 -4
- data/gemfiles/embulk-0.7.0 +0 -4
- data/gemfiles/embulk-0.7.0-latest +0 -4
- data/gemfiles/embulk-0.7.1 +0 -4
- data/gemfiles/embulk-0.7.2 +0 -4
- data/gemfiles/embulk-0.7.3 +0 -4
- data/gemfiles/embulk-0.7.4 +0 -4
- data/gemfiles/embulk-0.7.5 +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8e89a971b0dca06b010f1081c37969c59c1ef04c
|
4
|
+
data.tar.gz: 8dda51b8e935c023e514c6db6e1e15b868fc59ff
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a4f0072db71cdbd8ea2b6cad7c1071c5fd5ca05efe827d27d0e6185142028464890f98b413343e98396330d037ced52e114bbeec9c229a2746f573bfda070b7a
|
7
|
+
data.tar.gz: e7e7f52566640fba0a3d5cb23087dc10f6e94dd7635e6a17e36018cd8aed257ced0608a50793596608e84b0f5cb31e2bbd6c36506185b1f98bb83bd7f4004f7f
|
data/.travis.yml
CHANGED
@@ -5,87 +5,41 @@ addons:
|
|
5
5
|
repo_token:
|
6
6
|
secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
|
7
7
|
|
8
|
-
notifications:
|
9
|
-
slack:
|
10
|
-
secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
|
11
|
-
|
12
8
|
before_install:
|
13
9
|
- |
|
14
10
|
# Currently, Travis can't treat jruby 9.0.1.0
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
11
|
+
ruby -v
|
12
|
+
rvm get head
|
13
|
+
rvm use jruby-9.0.5.0 --install
|
14
|
+
ruby -v
|
15
|
+
gem i bundler
|
16
|
+
bundle install
|
20
17
|
|
21
18
|
jdk: oraclejdk8
|
22
19
|
|
23
20
|
rvm:
|
24
|
-
- jruby-
|
25
|
-
|
21
|
+
- jruby-9.0.5.0
|
22
|
+
|
23
|
+
script: bundle exec rake test
|
26
24
|
|
27
25
|
gemfile:
|
28
|
-
- gemfiles/embulk-0.
|
29
|
-
- gemfiles/embulk-0.
|
30
|
-
- gemfiles/embulk-0.
|
31
|
-
- gemfiles/embulk-0.6.15
|
32
|
-
- gemfiles/embulk-0.6.16
|
33
|
-
- gemfiles/embulk-0.6.17
|
34
|
-
- gemfiles/embulk-0.6.18
|
35
|
-
- gemfiles/embulk-0.6.19
|
36
|
-
- gemfiles/embulk-0.6.20
|
37
|
-
- gemfiles/embulk-0.6.21
|
38
|
-
- gemfiles/embulk-0.6.22
|
39
|
-
- gemfiles/embulk-0.6.23
|
40
|
-
- gemfiles/embulk-0.6.24
|
41
|
-
- gemfiles/embulk-0.6.25
|
42
|
-
- gemfiles/embulk-0.6.26
|
43
|
-
- gemfiles/embulk-0.6.27
|
44
|
-
- gemfiles/embulk-0.7.0
|
45
|
-
- gemfiles/embulk-0.7.0-latest
|
46
|
-
- gemfiles/embulk-0.7.1
|
47
|
-
- gemfiles/embulk-0.7.2
|
48
|
-
- gemfiles/embulk-0.7.3
|
49
|
-
- gemfiles/embulk-0.7.4
|
50
|
-
- gemfiles/embulk-0.7.5
|
26
|
+
- gemfiles/embulk-0.8.0-latest
|
27
|
+
- gemfiles/embulk-0.8.7
|
28
|
+
- gemfiles/embulk-0.8.8
|
51
29
|
- gemfiles/embulk-latest
|
52
30
|
|
53
31
|
matrix:
|
54
32
|
exclude:
|
55
33
|
- jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
|
56
34
|
include:
|
57
|
-
- {rvm: jruby-
|
58
|
-
- {rvm: jruby-
|
59
|
-
- {rvm: jruby-
|
60
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.15}
|
61
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.16}
|
62
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.17}
|
63
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.18}
|
64
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.19}
|
65
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.20}
|
66
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.21}
|
67
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.22}
|
68
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.23}
|
69
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.24}
|
70
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.25}
|
71
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.26}
|
72
|
-
- {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.27}
|
73
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0}
|
74
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0-latest}
|
75
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.1}
|
76
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.2}
|
77
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.3}
|
78
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.4}
|
79
|
-
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.5}
|
35
|
+
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.0-latest}
|
36
|
+
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.7}
|
37
|
+
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.8}
|
80
38
|
- {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-latest}
|
81
39
|
|
82
40
|
|
83
41
|
allow_failures:
|
84
|
-
- gemfile: gemfiles/embulk-0.6.22
|
85
|
-
- gemfile: gemfiles/embulk-0.7.0
|
86
|
-
- gemfile: gemfiles/embulk-0.7.1
|
87
42
|
# Ignore failure for *-latest
|
88
|
-
- gemfile: embulk-0.
|
89
|
-
- gemfile: embulk-0.7.0-latest
|
43
|
+
- gemfile: embulk-0.8.0-latest
|
90
44
|
- gemfile: embulk-latest
|
91
45
|
|
data/.travis.yml.erb
CHANGED
@@ -5,24 +5,22 @@ addons:
|
|
5
5
|
repo_token:
|
6
6
|
secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
|
7
7
|
|
8
|
-
notifications:
|
9
|
-
slack:
|
10
|
-
secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
|
11
|
-
|
12
8
|
before_install:
|
13
9
|
- |
|
14
10
|
# Currently, Travis can't treat jruby 9.0.1.0
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
11
|
+
ruby -v
|
12
|
+
rvm get head
|
13
|
+
rvm use jruby-9.0.5.0 --install
|
14
|
+
ruby -v
|
15
|
+
gem i bundler
|
16
|
+
bundle install
|
20
17
|
|
21
18
|
jdk: oraclejdk8
|
22
19
|
|
23
20
|
rvm:
|
24
|
-
- jruby-
|
25
|
-
|
21
|
+
- jruby-9.0.5.0
|
22
|
+
|
23
|
+
script: bundle exec rake test
|
26
24
|
|
27
25
|
gemfile:
|
28
26
|
<% versions.each do |file| -%>
|
@@ -38,9 +36,6 @@ matrix:
|
|
38
36
|
<% end %>
|
39
37
|
|
40
38
|
allow_failures:
|
41
|
-
- gemfile: gemfiles/embulk-0.6.22
|
42
|
-
- gemfile: gemfiles/embulk-0.7.0
|
43
|
-
- gemfile: gemfiles/embulk-0.7.1
|
44
39
|
# Ignore failure for *-latest
|
45
40
|
<% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
|
46
41
|
- gemfile: <%= file %>
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
## 0.5.0 - 2016-04-06
|
2
|
+
|
3
|
+
This version drops old Embulk supports. Embulk 0.8 or later is required since this version.
|
4
|
+
|
5
|
+
* [enhancement] Add tests for Embulk 0.8 and drop support old Embulk [#52](https://github.com/treasure-data/embulk-input-marketo/pull/52)
|
6
|
+
* [maintenance] Refactor lead and activity [#51](https://github.com/treasure-data/embulk-input-marketo/pull/51)
|
7
|
+
* [maintenance] Refactor retry [#50](https://github.com/treasure-data/embulk-input-marketo/pull/50)
|
8
|
+
|
1
9
|
## 0.4.0 - 2015-10-30
|
2
10
|
|
3
11
|
This version drops scheduled execution with marketo/lead.
|
data/README.md
CHANGED
@@ -14,10 +14,10 @@ This plugin uses Marketo SOAP API.
|
|
14
14
|
|
15
15
|
## Overview
|
16
16
|
|
17
|
-
Required Embulk version >= 0.
|
17
|
+
Required Embulk version >= 0.8.7 (since 0.5.0).
|
18
18
|
|
19
19
|
* **Plugin type**: input
|
20
|
-
* **Resume supported**:
|
20
|
+
* **Resume supported**: no for `marketo/lead`, yes for `marketo/activity_log`
|
21
21
|
* **Cleanup supported**: no
|
22
22
|
* **Guess supported**: yes
|
23
23
|
|
@@ -35,8 +35,6 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
|
35
35
|
|
36
36
|
### marketo/lead
|
37
37
|
|
38
|
-
**NOTE: If you use feature of scheduled execution (resume) with marketo/lead, you should not specify `to_datetime` because this plugin can't place next to_datetime (can't know the date to run with new config).**
|
39
|
-
|
40
38
|
- **endpoint** SOAP endpoint URL for your account (string, required)
|
41
39
|
- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
|
42
40
|
- **user_id** Your user id (string, reqiured)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-input-marketo"
|
3
|
-
spec.version = "0.
|
3
|
+
spec.version = "0.5.0"
|
4
4
|
spec.authors = ["uu59", "yoshihara"]
|
5
5
|
spec.summary = "Marketo input plugin for Embulk"
|
6
6
|
spec.description = "Loads records from Marketo."
|
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
|
|
14
14
|
|
15
15
|
spec.add_dependency 'savon', ['~> 2.11.1']
|
16
16
|
spec.add_dependency 'httpclient'
|
17
|
+
spec.add_dependency 'perfect_retry', ["~> 0.3.2"]
|
17
18
|
spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
|
18
19
|
spec.add_development_dependency 'bundler', ['~> 1.0']
|
19
20
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
data/gemfiles/embulk-latest
CHANGED
@@ -58,25 +58,18 @@ module Embulk
|
|
58
58
|
def init
|
59
59
|
@columns = task[:columns]
|
60
60
|
@soap = MarketoApi.soap_client(task, target)
|
61
|
-
|
62
|
-
|
63
|
-
def run
|
64
|
-
options = {
|
61
|
+
@options = {
|
65
62
|
retry_initial_wait_sec: task[:retry_initial_wait_sec],
|
66
63
|
retry_limit: task[:retry_limit],
|
67
64
|
to: task[:to_datetime],
|
68
65
|
batch_size: (preview? ? PREVIEW_COUNT : BATCH_SIZE_DEFAULT),
|
69
66
|
}
|
67
|
+
end
|
70
68
|
|
69
|
+
def run
|
71
70
|
counter = 0
|
72
|
-
latest_updated_at = @soap.each(task[:from_datetime], options) do |activity_log|
|
73
|
-
|
74
|
-
name = column["name"].to_s
|
75
|
-
value = activity_log[name]
|
76
|
-
cast_value(column, value)
|
77
|
-
end
|
78
|
-
|
79
|
-
page_builder.add(values)
|
71
|
+
latest_updated_at = @soap.each(task[:from_datetime], @options) do |activity_log|
|
72
|
+
page_builder.add(format_record(activity_log))
|
80
73
|
break if preview? && (counter += 1) >= PREVIEW_COUNT
|
81
74
|
end
|
82
75
|
|
@@ -89,6 +82,14 @@ module Embulk
|
|
89
82
|
|
90
83
|
return task_report
|
91
84
|
end
|
85
|
+
|
86
|
+
def format_record(activity_log)
|
87
|
+
@columns.map do |column|
|
88
|
+
name = column["name"].to_s
|
89
|
+
value = activity_log[name]
|
90
|
+
cast_value(column, value)
|
91
|
+
end
|
92
|
+
end
|
92
93
|
end
|
93
94
|
end
|
94
95
|
end
|
@@ -94,30 +94,19 @@ module Embulk
|
|
94
94
|
@ranges = task[:ranges][index]
|
95
95
|
@soap = MarketoApi.soap_client(task, target)
|
96
96
|
@append_processed_time_column = task[:append_processed_time_column]
|
97
|
-
|
98
|
-
|
99
|
-
def run
|
100
|
-
options = {
|
97
|
+
@options = {
|
101
98
|
retry_initial_wait_sec: task[:retry_initial_wait_sec],
|
102
99
|
retry_limit: task[:retry_limit],
|
103
100
|
}
|
104
|
-
options[:batch_size] = PREVIEW_COUNT if preview?
|
101
|
+
@options[:batch_size] = PREVIEW_COUNT if preview?
|
102
|
+
end
|
105
103
|
|
104
|
+
def run
|
106
105
|
counter = 0
|
107
106
|
catch(:finish) do
|
108
107
|
@ranges.each do |range|
|
109
|
-
soap.each(range, options) do |lead|
|
110
|
-
|
111
|
-
name = column["name"].to_s
|
112
|
-
value = (lead[name] || {})[:value]
|
113
|
-
cast_value(column, value)
|
114
|
-
end
|
115
|
-
|
116
|
-
if @append_processed_time_column
|
117
|
-
values << Time.parse(range["from"])
|
118
|
-
end
|
119
|
-
|
120
|
-
page_builder.add(values)
|
108
|
+
soap.each(range, @options) do |lead|
|
109
|
+
page_builder.add(format_record(lead, range))
|
121
110
|
throw(:finish) if preview? && (counter += 1) >= PREVIEW_COUNT
|
122
111
|
end
|
123
112
|
end
|
@@ -128,6 +117,19 @@ module Embulk
|
|
128
117
|
task_report = {}
|
129
118
|
return task_report
|
130
119
|
end
|
120
|
+
|
121
|
+
def format_record(lead, range)
|
122
|
+
values = @columns.map do |column|
|
123
|
+
name = column["name"].to_s
|
124
|
+
value = (lead[name] || {})[:value]
|
125
|
+
cast_value(column, value)
|
126
|
+
end
|
127
|
+
|
128
|
+
if @append_processed_time_column
|
129
|
+
values << Time.parse(range["from"])
|
130
|
+
end
|
131
|
+
values
|
132
|
+
end
|
131
133
|
end
|
132
134
|
end
|
133
135
|
end
|
@@ -57,42 +57,44 @@ module Embulk
|
|
57
57
|
|
58
58
|
def fetch(request, options={}, &block)
|
59
59
|
response = savon_call(:get_lead_changes, {message: request}, options)
|
60
|
-
remaining = response.
|
60
|
+
remaining = response.xpath('//remainingCount').text.to_i
|
61
61
|
Embulk.logger.info "Remaining records: #{remaining}"
|
62
62
|
|
63
|
-
|
63
|
+
activities = response.xpath('//leadChangeRecord')
|
64
64
|
|
65
|
-
if
|
65
|
+
if activities.empty?
|
66
66
|
Embulk.logger.info "No record is fetched."
|
67
67
|
return {remaining_count: 0, offset: nil, from_datetime: nil}
|
68
68
|
end
|
69
69
|
|
70
|
-
activities = activities_list[:lead_change_record].sort_by { |activity| Time.parse(activity[:activity_date_time]) }
|
71
|
-
|
72
70
|
activities.each do |activity|
|
73
|
-
|
74
|
-
"id" => activity[:id],
|
75
|
-
"activity_date_time" => activity[:activity_date_time],
|
76
|
-
"activity_type" => activity[:activity_type],
|
77
|
-
"mktg_asset_name" => activity[:mktg_asset_name],
|
78
|
-
"mkt_person_id" => activity[:mkt_person_id],
|
79
|
-
}
|
80
|
-
|
81
|
-
activity[:activity_attributes][:attribute].each do |attributes|
|
82
|
-
name = attributes[:attr_name]
|
83
|
-
value = attributes[:attr_value]
|
84
|
-
|
85
|
-
record[name] = value
|
86
|
-
end
|
87
|
-
|
88
|
-
block.call(record)
|
71
|
+
process_record(activity, &block)
|
89
72
|
end
|
90
73
|
|
91
74
|
{
|
92
75
|
remaining_count: remaining,
|
93
|
-
offset: response.
|
94
|
-
from_datetime: activities.
|
76
|
+
offset: response.xpath('//newStartPosition/offset').text,
|
77
|
+
from_datetime: activities.map{|a| Time.parse(a.at('./activityDateTime').text) }.max,
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def process_record(activity, &block)
|
82
|
+
record = {
|
83
|
+
"id" => activity.at("./id").text,
|
84
|
+
"activity_date_time" => activity.at('./activityDateTime').text,
|
85
|
+
"activity_type" => activity.at('./activityType').text,
|
86
|
+
"mktg_asset_name" => activity.at('./mktgAssetName').text,
|
87
|
+
"mkt_person_id" => activity.at('./mktPersonId').text,
|
95
88
|
}
|
89
|
+
|
90
|
+
activity.xpath('./activityAttributes/attribute').each do |attr|
|
91
|
+
name = attr.xpath('attrName').text
|
92
|
+
value = attr.xpath('attrValue').text
|
93
|
+
|
94
|
+
record[name] = value
|
95
|
+
end
|
96
|
+
|
97
|
+
block.call(record)
|
96
98
|
end
|
97
99
|
end
|
98
100
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "savon"
|
2
2
|
require "httpclient" # net/http can't verify cert correctly
|
3
|
+
require "perfect_retry"
|
3
4
|
|
4
5
|
module Embulk
|
5
6
|
module Input
|
@@ -41,8 +42,19 @@ module Embulk
|
|
41
42
|
)
|
42
43
|
end
|
43
44
|
|
45
|
+
def retryer(retry_options)
|
46
|
+
PerfectRetry.new do |config|
|
47
|
+
config.sleep = proc{|n| retry_options[:retry_initial_wait_sec] * (2 ** (n - 1))}
|
48
|
+
config.limit = retry_options[:retry_limit]
|
49
|
+
config.dont_rescues = [Embulk::ConfigError]
|
50
|
+
config.rescues = [StandardError, Timeout::Error]
|
51
|
+
config.logger = Embulk.logger
|
52
|
+
config.log_level = nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
44
56
|
def savon_call(operation, locals={}, retry_options={})
|
45
|
-
|
57
|
+
retryer(retry_options).with_retry do
|
46
58
|
catch_unretryable_error do
|
47
59
|
savon.call(operation, locals.merge(advanced_typecasting: false))
|
48
60
|
end
|
@@ -60,23 +72,6 @@ module Embulk
|
|
60
72
|
}
|
61
73
|
end
|
62
74
|
|
63
|
-
def with_retry(options, &block)
|
64
|
-
wait_sec = options[:retry_initial_wait_sec]
|
65
|
-
count = 0
|
66
|
-
begin
|
67
|
-
yield
|
68
|
-
rescue Embulk::ConfigError => e # TODO: Add Embulk::DataError for Embulk 0.7+
|
69
|
-
raise e
|
70
|
-
rescue ::Timeout::Error, StandardError => e
|
71
|
-
count += 1
|
72
|
-
raise e if count > options[:retry_limit]
|
73
|
-
Embulk.logger.warn "Retrying after #{wait_sec} seconds [#{count}/#{RETRY_TIMEOUT_COUNT}] Error: #{e}"
|
74
|
-
sleep wait_sec
|
75
|
-
wait_sec *= 2
|
76
|
-
retry
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
75
|
def catch_unretryable_error(&block)
|
81
76
|
yield
|
82
77
|
rescue Savon::SOAPFault => e
|
@@ -101,7 +96,7 @@ module Embulk
|
|
101
96
|
# unretryable error such as Authentication Failed, Invalid Request, etc.
|
102
97
|
raise ConfigError.new soap_message
|
103
98
|
end
|
104
|
-
rescue SocketError, Errno::ECONNREFUSED => e
|
99
|
+
rescue SocketError, ::Java::JavaNet::UnknownHostException, Errno::ECONNREFUSED => e
|
105
100
|
# maybe endpoint/wsdl domain was wrong
|
106
101
|
Embulk.logger.debug "Connection error: endpoint=#{endpoint} wsdl=#{wsdl}"
|
107
102
|
raise ConfigError.new "Connection error: #{e.message} (endpoint is '#{endpoint}')"
|