embulk-input-marketo 0.5.3 → 0.5.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +1 -1
- data/embulk-input-marketo.gemspec +1 -1
- data/lib/embulk/input/marketo/activity_log.rb +2 -0
- data/lib/embulk/input/marketo/base.rb +8 -0
- data/lib/embulk/input/marketo/lead.rb +8 -1
- data/lib/embulk/input/marketo_api/soap/base.rb +5 -2
- data/test/embulk/input/marketo/test_activity_log.rb +41 -0
- data/test/embulk/input/marketo/test_lead.rb +41 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d4fb4473d0eb983aabb83a21397eb32dde08e678
|
4
|
+
data.tar.gz: 0c9fbac5a5cefcb755bc0a10be544d71a54eaba6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: de9235beec534f01d4d54d4051b4f22678a4519b3a0c8cd929be61c3a981499238dc41b0cf0c2b979687e24713f717f9217e1f6b77c13b6d1eb80085b68e7215
|
7
|
+
data.tar.gz: 9516e1e4796f5711bae5747e7d3e636bc80542fb76d7f85d6b938bad091d294c6adb75043e7b0ef99e0cabe4351393e4f9f662980bf398ac575691818124c77f
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.5.4 - 2016-10-26
|
2
|
+
* [enhancement] Validate wsdl_url and endpoint_url are the valid form [#61](https://github.com/treasure-data/embulk-input-marketo/pull/61)
|
3
|
+
* [enhancement] Minor readme change [#58](https://github.com/treasure-data/embulk-input-marketo/pull/58)
|
4
|
+
* [fixed] Fix error when retrying on guess/preview [#59](https://github.com/treasure-data/embulk-input-marketo/pull/59)
|
5
|
+
* [enhancement] Try newer date at first on preview to avoid miss hit [#60](https://github.com/treasure-data/embulk-input-marketo/pull/60)
|
6
|
+
|
1
7
|
## 0.5.3 - 2016-07-01
|
2
8
|
|
3
9
|
* [enhancement] make concurrent limit exceeded error retryable [#56](https://github.com/treasure-data/embulk-input-marketo/pull/56)
|
data/README.md
CHANGED
@@ -48,7 +48,7 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
|
48
48
|
### marketo/activity_log
|
49
49
|
|
50
50
|
- **endpoint** SOAP endpoint URL for your account (string, required)
|
51
|
-
- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
|
51
|
+
- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")(Note "?WSDL" needs to be in all caps)
|
52
52
|
- **user_id** Your user id (string, reqiured)
|
53
53
|
- **encryption_key** Your encryption key (string, reqiured)
|
54
54
|
- **from_datetime** Fetch activity_logs since this time (string, required)
|
@@ -37,6 +37,8 @@ module Embulk
|
|
37
37
|
retry_limit: config.param(:retry_limit, :integer, default: 5),
|
38
38
|
columns: config.param(:columns, :array)
|
39
39
|
}
|
40
|
+
validate_url(task[:endpoint_url], "endpoint")
|
41
|
+
validate_url(task[:wsdl_url], "wsdl")
|
40
42
|
|
41
43
|
resume(task, embulk_columns(config), 1, &control)
|
42
44
|
end
|
@@ -20,6 +20,8 @@ module Embulk
|
|
20
20
|
user_id: config.param(:user_id, :string),
|
21
21
|
encryption_key: config.param(:encryption_key, :string),
|
22
22
|
}
|
23
|
+
validate_url(soap_config[:endpoint_url], "endpoint")
|
24
|
+
validate_url(soap_config[:wsdl_url], "wsdl")
|
23
25
|
|
24
26
|
MarketoApi.soap_client(soap_config, target)
|
25
27
|
end
|
@@ -99,6 +101,12 @@ module Embulk
|
|
99
101
|
|
100
102
|
private
|
101
103
|
|
104
|
+
def self.validate_url(url, key)
|
105
|
+
URI.parse(url)
|
106
|
+
rescue URI::InvalidURIError
|
107
|
+
raise ConfigError.new("#{key}: '#{url}' is not a valid URL.")
|
108
|
+
end
|
109
|
+
|
102
110
|
def preview?
|
103
111
|
begin
|
104
112
|
org.embulk.spi.Exec.isPreview()
|
@@ -50,6 +50,8 @@ module Embulk
|
|
50
50
|
append_processed_time_column: append_processed_time_column,
|
51
51
|
columns: config.param(:columns, :array),
|
52
52
|
}
|
53
|
+
validate_url(task[:endpoint_url], "endpoint")
|
54
|
+
validate_url(task[:wsdl_url], "wsdl")
|
53
55
|
|
54
56
|
columns = embulk_columns(config)
|
55
57
|
if append_processed_time_column
|
@@ -91,7 +93,12 @@ module Embulk
|
|
91
93
|
|
92
94
|
def init
|
93
95
|
@columns = task[:columns]
|
94
|
-
|
96
|
+
if preview?
|
97
|
+
# Try newer date at first to reduce cache miss hit
|
98
|
+
@ranges = task[:ranges].flatten.sort_by{|range| Time.parse(range["to"])}.reverse
|
99
|
+
else
|
100
|
+
@ranges = task[:ranges][index]
|
101
|
+
end
|
95
102
|
@soap = MarketoApi.soap_client(task, target)
|
96
103
|
@append_processed_time_column = task[:append_processed_time_column]
|
97
104
|
@options = {
|
@@ -43,9 +43,12 @@ module Embulk
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def retryer(retry_options)
|
46
|
+
# guess/preview doesn't pass retry_options
|
47
|
+
initial_wait = retry_options[:retry_initial_wait_sec] || 1
|
48
|
+
retry_limit = retry_options[:retry_limit] || 5
|
46
49
|
PerfectRetry.new do |config|
|
47
|
-
config.sleep = proc{|n|
|
48
|
-
config.limit =
|
50
|
+
config.sleep = proc{|n| initial_wait * (2 ** (n - 1))}
|
51
|
+
config.limit = retry_limit
|
49
52
|
config.dont_rescues = [Embulk::ConfigError]
|
50
53
|
config.rescues = [StandardError, Timeout::Error]
|
51
54
|
config.logger = Embulk.logger
|
@@ -32,6 +32,34 @@ module Embulk
|
|
32
32
|
assert_equal(next_config_diff, actual)
|
33
33
|
end
|
34
34
|
|
35
|
+
data do
|
36
|
+
[
|
37
|
+
["endpoint", "endpoint"],
|
38
|
+
["wsdl", "wsdl"],
|
39
|
+
]
|
40
|
+
end
|
41
|
+
def test_invalid_url(key)
|
42
|
+
control = proc {} # dummy
|
43
|
+
|
44
|
+
settings = {
|
45
|
+
endpoint: "https://marketo.example.com",
|
46
|
+
wsdl: "https://marketo.example.com/?wsdl",
|
47
|
+
user_id: "user_id",
|
48
|
+
encryption_key: "TOPSECRET",
|
49
|
+
columns: [
|
50
|
+
{"name" => "Name", "type" => "string"},
|
51
|
+
],
|
52
|
+
from_datetime: Time.now,
|
53
|
+
to_datetime: Time.now + 3600,
|
54
|
+
}
|
55
|
+
settings[key.to_sym] = " invalid url "
|
56
|
+
config = DataSource[settings.to_a]
|
57
|
+
|
58
|
+
assert_raise(ConfigError) do
|
59
|
+
ActivityLog.transaction(config, &control)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
35
63
|
private
|
36
64
|
|
37
65
|
def settings
|
@@ -105,6 +133,19 @@ module Embulk
|
|
105
133
|
)
|
106
134
|
end
|
107
135
|
|
136
|
+
data do
|
137
|
+
[
|
138
|
+
["endpoint", "endpoint"],
|
139
|
+
["wsdl", "wsdl"],
|
140
|
+
]
|
141
|
+
end
|
142
|
+
def test_invalid_url(key)
|
143
|
+
config[key] = " invalid url "
|
144
|
+
assert_raise(ConfigError) do
|
145
|
+
Marketo::ActivityLog.guess(config)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
108
149
|
private
|
109
150
|
|
110
151
|
def records
|
@@ -51,6 +51,34 @@ module Embulk
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
+
data do
|
55
|
+
[
|
56
|
+
["endpoint", "endpoint"],
|
57
|
+
["wsdl", "wsdl"],
|
58
|
+
]
|
59
|
+
end
|
60
|
+
def test_invalid_url(key)
|
61
|
+
control = proc {} # dummy
|
62
|
+
|
63
|
+
settings = {
|
64
|
+
endpoint: "https://marketo.example.com",
|
65
|
+
wsdl: "https://marketo.example.com/?wsdl",
|
66
|
+
user_id: "user_id",
|
67
|
+
encryption_key: "TOPSECRET",
|
68
|
+
columns: [
|
69
|
+
{"name" => "Name", "type" => "string"},
|
70
|
+
],
|
71
|
+
from_datetime: Time.now,
|
72
|
+
to_datetime: Time.now + 3600,
|
73
|
+
}
|
74
|
+
settings[key.to_sym] = " invalid url "
|
75
|
+
config = DataSource[settings.to_a]
|
76
|
+
|
77
|
+
assert_raise(ConfigError) do
|
78
|
+
Lead.transaction(config, &control)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
54
82
|
def test_invalid_datetime_given
|
55
83
|
control = proc {} # dummy
|
56
84
|
|
@@ -396,6 +424,19 @@ module Embulk
|
|
396
424
|
Lead.guess(config)
|
397
425
|
)
|
398
426
|
end
|
427
|
+
|
428
|
+
data do
|
429
|
+
[
|
430
|
+
["endpoint", "endpoint"],
|
431
|
+
["wsdl", "wsdl"],
|
432
|
+
]
|
433
|
+
end
|
434
|
+
def test_invalid_url(key)
|
435
|
+
config[key] = " invalid url "
|
436
|
+
assert_raise(ConfigError) do
|
437
|
+
Lead.guess(config)
|
438
|
+
end
|
439
|
+
end
|
399
440
|
end
|
400
441
|
|
401
442
|
def test_generate_columns
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-
|
12
|
+
date: 2016-10-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|