embulk-input-marketo 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.ruby-version +1 -0
- data/.travis.yml +12 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile +3 -0
- data/LICENSE +13 -0
- data/README.md +48 -0
- data/Rakefile +8 -0
- data/embulk-input-marketo.gemspec +23 -0
- data/lib/embulk/input/marketo.rb +138 -0
- data/lib/embulk/input/marketo_api.rb +11 -0
- data/lib/embulk/input/marketo_api/soap.rb +112 -0
- data/test/embulk/input/marketo_api/test_soap.rb +123 -0
- data/test/embulk/input/test_marketo.rb +176 -0
- data/test/lead_fixtures.rb +119 -0
- data/test/prepare_embulk.rb +10 -0
- data/test/run-test.rb +18 -0
- metadata +186 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 297928aea65e91bd2295774e2c041b97a9c84906
|
4
|
+
data.tar.gz: 693b68594689b659b7e027d9cfacdf2c515b0f69
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4d1d412ee4f8c711a16d26b952eadc2599fb2b04e17ad12d1c9646477965fb6e1436518d9f0e568af28300f4f816008994ea90ea168ef2860defd51e4416ea31
|
7
|
+
data.tar.gz: 41f2a41de9ed400df4bdd2a3fe042afc75c6047483f6c7bcddaff96b380f99a83213306699810cdf1ec97b84ecc57fe2cba1b405b997ac0e8f781ab56fb56a1c
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
jruby-1.7.20
|
data/.travis.yml
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- jruby-19mode
|
4
|
+
addons:
|
5
|
+
code_climate:
|
6
|
+
repo_token:
|
7
|
+
secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
|
8
|
+
notifications:
|
9
|
+
slack:
|
10
|
+
secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
|
11
|
+
jdk:
|
12
|
+
- oraclejdk8
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2015 Everyleaf Corporation
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
|
2
|
+
|
3
|
+
[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
|
4
|
+
|
5
|
+
[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
|
6
|
+
|
7
|
+
# Marketo input plugin for Embulk
|
8
|
+
|
9
|
+
embulk-input-marketo is the Embulk input plugin for [Marketo](http://www.marketo.com/).
|
10
|
+
This plugin uses Marketo SOAP API.
|
11
|
+
|
12
|
+
## Overview
|
13
|
+
|
14
|
+
Required Embulk version >= 0.6.13.
|
15
|
+
|
16
|
+
* **Plugin type**: input
|
17
|
+
* **Resume supported**: no
|
18
|
+
* **Cleanup supported**: no
|
19
|
+
* **Guess supported**: yes
|
20
|
+
|
21
|
+
## Configuration
|
22
|
+
|
23
|
+
Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
24
|
+
|
25
|
+
- **endpoint** SOAP endpoint URL for your account (string, required)
|
26
|
+
- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
|
27
|
+
- **user_id** Your user id (string, reqiured)
|
28
|
+
- **encryption_key** Your encryption key (string, reqiured)
|
29
|
+
- **last_updated_at** Limit datetime that a lead has been updated (this plugin fetches leads updated after this datetime) (string, required)
|
30
|
+
|
31
|
+
## Example
|
32
|
+
|
33
|
+
```yaml
|
34
|
+
in:
|
35
|
+
type: marketo
|
36
|
+
endpoint: https://soap-end-point.mktoapi.com/
|
37
|
+
wsdl: https://wsdl-url.mktoapi.com/?WSDL
|
38
|
+
user_id: user_ABC123
|
39
|
+
encryption_key: TOPSECRET
|
40
|
+
last_updated_at: "2015-06-30"
|
41
|
+
```
|
42
|
+
|
43
|
+
|
44
|
+
## Build
|
45
|
+
|
46
|
+
```
|
47
|
+
$ rake
|
48
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
Gem::Specification.new do |spec|
|
2
|
+
spec.name = "embulk-input-marketo"
|
3
|
+
spec.version = "0.0.1"
|
4
|
+
spec.authors = ["uu59", "yoshihara"]
|
5
|
+
spec.summary = "Marketo input plugin for Embulk"
|
6
|
+
spec.description = "Loads records from Marketo."
|
7
|
+
spec.email = ["k@uu59.org", "h.yoshihara@everyleaf.com"]
|
8
|
+
spec.licenses = ["Apache2"]
|
9
|
+
spec.homepage = "https://github.com/treasure-data/embulk-input-marketo"
|
10
|
+
|
11
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
12
|
+
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
13
|
+
spec.require_paths = ["lib"]
|
14
|
+
|
15
|
+
spec.add_dependency 'savon', ['~> 2.11.1']
|
16
|
+
spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
|
17
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
18
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
19
|
+
spec.add_development_dependency 'pry'
|
20
|
+
spec.add_development_dependency 'test-unit'
|
21
|
+
spec.add_development_dependency 'test-unit-rr'
|
22
|
+
spec.add_development_dependency 'codeclimate-test-reporter'
|
23
|
+
end
|
@@ -0,0 +1,138 @@
|
|
1
|
+
require "embulk/input/marketo_api"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
class Marketo < InputPlugin
|
6
|
+
PREVIEW_COUNT = 15
|
7
|
+
|
8
|
+
Plugin.register_input("marketo", self)
|
9
|
+
|
10
|
+
def self.transaction(config, &control)
|
11
|
+
endpoint_url = config.param(:endpoint, :string)
|
12
|
+
|
13
|
+
task = {
|
14
|
+
endpoint_url: endpoint_url,
|
15
|
+
wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
|
16
|
+
user_id: config.param(:user_id, :string),
|
17
|
+
encryption_key: config.param(:encryption_key, :string),
|
18
|
+
last_updated_at: config.param(:last_updated_at, :string),
|
19
|
+
columns: config.param(:columns, :array)
|
20
|
+
}
|
21
|
+
|
22
|
+
columns = []
|
23
|
+
|
24
|
+
task[:columns].each do |column|
|
25
|
+
name = column["name"]
|
26
|
+
type = column["type"].to_sym
|
27
|
+
|
28
|
+
columns << Column.new(nil, name, type, column["format"])
|
29
|
+
end
|
30
|
+
|
31
|
+
resume(task, columns, 1, &control)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.resume(task, columns, count, &control)
|
35
|
+
commit_reports = yield(task, columns, count)
|
36
|
+
|
37
|
+
next_config_diff = {}
|
38
|
+
return next_config_diff
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.guess(config)
|
42
|
+
client = soap_client(config)
|
43
|
+
metadata = client.lead_metadata
|
44
|
+
|
45
|
+
return {"columns" => generate_columns(metadata)}
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.soap_client(config)
|
49
|
+
@soap ||=
|
50
|
+
begin
|
51
|
+
endpoint_url = config.param(:endpoint, :string),
|
52
|
+
soap_config = {
|
53
|
+
endpoint_url: endpoint_url,
|
54
|
+
wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
|
55
|
+
user_id: config.param(:user_id, :string),
|
56
|
+
encryption_key: config.param(:encryption_key, :string),
|
57
|
+
}
|
58
|
+
|
59
|
+
MarketoApi.soap_client(soap_config)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.generate_columns(metadata)
|
64
|
+
columns = [
|
65
|
+
{name: "id", type: "long"},
|
66
|
+
{name: "email", type: "string"},
|
67
|
+
]
|
68
|
+
|
69
|
+
metadata.each do |field|
|
70
|
+
type =
|
71
|
+
case field[:data_type]
|
72
|
+
when "integer"
|
73
|
+
"long"
|
74
|
+
when "dateTime", "date"
|
75
|
+
"timestamp"
|
76
|
+
when "string", "text", "phone", "currency"
|
77
|
+
"string"
|
78
|
+
when "boolean"
|
79
|
+
"boolean"
|
80
|
+
when "float"
|
81
|
+
"double"
|
82
|
+
else
|
83
|
+
"string"
|
84
|
+
end
|
85
|
+
|
86
|
+
columns << {name: field[:name], type: type}
|
87
|
+
end
|
88
|
+
|
89
|
+
columns
|
90
|
+
end
|
91
|
+
|
92
|
+
def init
|
93
|
+
@last_updated_at = task[:last_updated_at]
|
94
|
+
@columns = task[:columns]
|
95
|
+
@soap = MarketoApi.soap_client(task)
|
96
|
+
end
|
97
|
+
|
98
|
+
def run
|
99
|
+
# TODO: preview
|
100
|
+
count = 0
|
101
|
+
@soap.each_lead(@last_updated_at) do |lead|
|
102
|
+
values = @columns.map do |column|
|
103
|
+
name = column["name"].to_s
|
104
|
+
(lead[name] || {})[:value]
|
105
|
+
end
|
106
|
+
|
107
|
+
page_builder.add(values)
|
108
|
+
|
109
|
+
count += 1
|
110
|
+
break if preview? && count >= PREVIEW_COUNT
|
111
|
+
end
|
112
|
+
|
113
|
+
page_builder.finish
|
114
|
+
|
115
|
+
commit_report = {}
|
116
|
+
return commit_report
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.logger
|
120
|
+
Embulk.logger
|
121
|
+
end
|
122
|
+
|
123
|
+
def logger
|
124
|
+
self.class.logger
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def preview?
|
130
|
+
begin
|
131
|
+
org.embulk.spi.Exec.isPreview()
|
132
|
+
rescue java.lang.NullPointerException => e
|
133
|
+
false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require "savon"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module MarketoApi
|
6
|
+
class Soap
|
7
|
+
attr_reader :endpoint, :wsdl, :user_id, :encryption_key
|
8
|
+
|
9
|
+
def initialize(endpoint, wsdl, user_id, encryption_key)
|
10
|
+
@endpoint = endpoint
|
11
|
+
@wsdl = wsdl
|
12
|
+
@user_id = user_id
|
13
|
+
@encryption_key = encryption_key
|
14
|
+
end
|
15
|
+
|
16
|
+
def lead_metadata
|
17
|
+
# http://developers.marketo.com/documentation/soap/describemobject/
|
18
|
+
response = savon.call(:describe_m_object, message: {object_name: "LeadRecord"})
|
19
|
+
response.body[:success_describe_m_object][:result][:metadata][:field_list][:field]
|
20
|
+
end
|
21
|
+
|
22
|
+
def each_lead(last_updated_at, &block)
|
23
|
+
# http://developers.marketo.com/documentation/soap/getmultipleleads/
|
24
|
+
|
25
|
+
last_updated_at = Time.parse(last_updated_at).iso8601
|
26
|
+
request = {
|
27
|
+
lead_selector: {
|
28
|
+
oldest_updated_at: last_updated_at,
|
29
|
+
},
|
30
|
+
attributes!: {
|
31
|
+
lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
|
32
|
+
},
|
33
|
+
batch_size: 1000,
|
34
|
+
}
|
35
|
+
|
36
|
+
stream_position = fetch_leads(request, &block)
|
37
|
+
|
38
|
+
while stream_position
|
39
|
+
stream_position = fetch_leads(request.merge(stream_position: stream_position), &block)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def fetch_leads(request = {}, &block)
|
46
|
+
response = savon.call(:get_multiple_leads, message: request)
|
47
|
+
|
48
|
+
remaining = response.xpath('//remainingCount').text.to_i
|
49
|
+
Embulk.logger.info "Remaining records: #{remaining}"
|
50
|
+
response.xpath('//leadRecordList/leadRecord').each do |lead|
|
51
|
+
record = {
|
52
|
+
"id" => {type: :integer, value: lead.xpath('Id').text.to_i},
|
53
|
+
"email" => {type: :string, value: lead.xpath('Email').text}
|
54
|
+
}
|
55
|
+
lead.xpath('leadAttributeList/attribute').each do |attr|
|
56
|
+
name = attr.xpath('attrName').text
|
57
|
+
type = attr.xpath('attrType').text
|
58
|
+
value = attr.xpath('attrValue').text
|
59
|
+
record = record.merge(
|
60
|
+
name => {
|
61
|
+
type: type,
|
62
|
+
value: value
|
63
|
+
}
|
64
|
+
)
|
65
|
+
end
|
66
|
+
|
67
|
+
block.call(record)
|
68
|
+
end
|
69
|
+
|
70
|
+
if remaining > 0
|
71
|
+
response.xpath('//newStreamPosition').text
|
72
|
+
else
|
73
|
+
nil
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def savon
|
78
|
+
headers = {
|
79
|
+
'ns1:AuthenticationHeader' => {
|
80
|
+
"mktowsUserId" => user_id,
|
81
|
+
}.merge(signature)
|
82
|
+
}
|
83
|
+
# NOTE: Do not memoize this to use always fresh signature (avoid 20016 error)
|
84
|
+
# ref. https://jira.talendforge.org/secure/attachmentzip/unzip/167201/49761%5B1%5D/Marketo%20Enterprise%20API%202%200.pdf (41 page)
|
85
|
+
Savon.client(
|
86
|
+
log: true,
|
87
|
+
logger: Embulk.logger,
|
88
|
+
wsdl: wsdl,
|
89
|
+
soap_header: headers,
|
90
|
+
endpoint: endpoint,
|
91
|
+
open_timeout: 90,
|
92
|
+
read_timeout: 300,
|
93
|
+
raise_errors: true,
|
94
|
+
namespace_identifier: :ns1,
|
95
|
+
env_namespace: 'SOAP-ENV'
|
96
|
+
)
|
97
|
+
end
|
98
|
+
|
99
|
+
def signature
|
100
|
+
timestamp = Time.now.to_s
|
101
|
+
encryption_string = timestamp + user_id
|
102
|
+
digest = OpenSSL::Digest.new('sha1')
|
103
|
+
hashed_signature = OpenSSL::HMAC.hexdigest(digest, encryption_key, encryption_string)
|
104
|
+
{
|
105
|
+
'requestTimestamp' => timestamp,
|
106
|
+
'requestSignature' => hashed_signature.to_s
|
107
|
+
}
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require "embulk/input/marketo_api/soap"
|
2
|
+
require "lead_fixtures"
|
3
|
+
|
4
|
+
module Embulk
|
5
|
+
module Input
|
6
|
+
module MarketoApi
|
7
|
+
class SoapTest < Test::Unit::TestCase
|
8
|
+
include LeadFixtures
|
9
|
+
|
10
|
+
class TestSignature < self
|
11
|
+
def setup
|
12
|
+
@signature = soap.__send__(:signature)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_sigature_keys
|
16
|
+
assert_equal(%w(requestTimestamp requestSignature).sort, @signature.keys.sort)
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_is_hash
|
20
|
+
assert_equal(Hash, @signature.class)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_each_lead
|
25
|
+
stub(Embulk).logger { ::Logger.new(IO::NULL) }
|
26
|
+
last_updated_at = "2015-07-06"
|
27
|
+
|
28
|
+
request = {
|
29
|
+
lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
|
30
|
+
attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
|
31
|
+
batch_size: 1000
|
32
|
+
}
|
33
|
+
|
34
|
+
any_instance_of(Savon::Client) do |klass|
|
35
|
+
mock(klass).call(:get_multiple_leads, message: request) do
|
36
|
+
next_stream_leads_response
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
proc = proc{ "" }
|
41
|
+
leads_count = next_stream_leads_response.xpath('//leadRecord').length
|
42
|
+
mock(proc).call(anything).times(leads_count)
|
43
|
+
|
44
|
+
soap.each_lead(last_updated_at, &proc)
|
45
|
+
end
|
46
|
+
|
47
|
+
class TestLeadMetadata < self
|
48
|
+
def setup
|
49
|
+
@savon = soap.__send__(:savon)
|
50
|
+
stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_savon_call
|
54
|
+
mock(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
|
55
|
+
Struct.new(:body).new(body)
|
56
|
+
}
|
57
|
+
soap.lead_metadata
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_return_fields
|
61
|
+
stub(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
|
62
|
+
Struct.new(:body).new(body)
|
63
|
+
}
|
64
|
+
assert_equal(fields, soap.lead_metadata)
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def body
|
70
|
+
{
|
71
|
+
success_describe_m_object: {
|
72
|
+
result: {
|
73
|
+
metadata: {
|
74
|
+
field_list: {
|
75
|
+
field: fields
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
end
|
82
|
+
|
83
|
+
def fields
|
84
|
+
[
|
85
|
+
{
|
86
|
+
name: "FieldName",
|
87
|
+
description: nil,
|
88
|
+
display_name: "The Name of Field",
|
89
|
+
source_object: "Lead",
|
90
|
+
data_type: "datetime",
|
91
|
+
size: nil,
|
92
|
+
is_readonly: false,
|
93
|
+
is_update_blocked: false,
|
94
|
+
is_name: nil,
|
95
|
+
is_primary_key: false,
|
96
|
+
is_custom: true,
|
97
|
+
is_dynamic: true,
|
98
|
+
dynamic_field_ref: "leadAttributeList",
|
99
|
+
updated_at: DateTime.parse("2000-01-01 22:22:22")
|
100
|
+
}
|
101
|
+
]
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def soap
|
108
|
+
@soap ||= Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
|
109
|
+
end
|
110
|
+
|
111
|
+
def settings
|
112
|
+
{
|
113
|
+
endpoint: "https://marketo.example.com",
|
114
|
+
wsdl: "https://marketo.example.com/?wsdl",
|
115
|
+
user_id: "user_id",
|
116
|
+
encryption_key: "TOPSECRET",
|
117
|
+
}
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
@@ -0,0 +1,176 @@
|
|
1
|
+
require "prepare_embulk"
|
2
|
+
require "lead_fixtures"
|
3
|
+
require "embulk/input/marketo"
|
4
|
+
|
5
|
+
module Embulk
|
6
|
+
module Input
|
7
|
+
class MarketoTest < Test::Unit::TestCase
|
8
|
+
include LeadFixtures
|
9
|
+
|
10
|
+
def setup_soap
|
11
|
+
@soap = MarketoApi::Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
|
12
|
+
|
13
|
+
stub(MarketoApi).soap_client(task) { @soap }
|
14
|
+
end
|
15
|
+
|
16
|
+
def setup_plugin
|
17
|
+
@page_builder = Object.new
|
18
|
+
@plugin = Marketo.new(task, nil, nil, @page_builder)
|
19
|
+
stub(Embulk).logger { ::Logger.new(File::NULL) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_transaction
|
23
|
+
control = proc {} # dummy
|
24
|
+
columns = task[:columns].map do |col|
|
25
|
+
Column.new(nil, col["name"], col["type"].to_sym)
|
26
|
+
end
|
27
|
+
|
28
|
+
mock(Marketo).resume(task, columns, 1, &control)
|
29
|
+
Marketo.transaction(config, &control)
|
30
|
+
end
|
31
|
+
|
32
|
+
class RunTest < self
|
33
|
+
def setup
|
34
|
+
setup_soap
|
35
|
+
setup_plugin
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_run_through
|
39
|
+
stub(@plugin).preview? { false }
|
40
|
+
|
41
|
+
any_instance_of(Savon::Client) do |klass|
|
42
|
+
mock(klass).call(:get_multiple_leads, message: request) do
|
43
|
+
leads_response
|
44
|
+
end
|
45
|
+
|
46
|
+
mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
|
47
|
+
next_stream_leads_response
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
mock(@page_builder).add(["manyo"])
|
52
|
+
mock(@page_builder).add(["everyleaf"])
|
53
|
+
mock(@page_builder).add(["ten-thousand-leaf"])
|
54
|
+
mock(@page_builder).finish
|
55
|
+
|
56
|
+
@plugin.run
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_preview_through
|
60
|
+
stub(@plugin).preview? { true }
|
61
|
+
|
62
|
+
any_instance_of(Savon::Client) do |klass|
|
63
|
+
mock(klass).call(:get_multiple_leads, message: request) do
|
64
|
+
preview_leads_response
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
Marketo::PREVIEW_COUNT.times do |count|
|
69
|
+
mock(@page_builder).add(["manyo#{count}"])
|
70
|
+
end
|
71
|
+
mock(@page_builder).finish
|
72
|
+
|
73
|
+
@plugin.run
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def request
|
79
|
+
{
|
80
|
+
lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
|
81
|
+
attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
|
82
|
+
batch_size: 1000
|
83
|
+
}
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
class GuessTest < self
|
88
|
+
setup :setup_soap
|
89
|
+
|
90
|
+
def setup_soap
|
91
|
+
@soap = MarketoApi::Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
|
92
|
+
|
93
|
+
stub(Marketo).soap_client(config) { @soap }
|
94
|
+
end
|
95
|
+
|
96
|
+
def test_include_metadata
|
97
|
+
stub(@soap).lead_metadata { metadata }
|
98
|
+
|
99
|
+
assert_equal(
|
100
|
+
{"columns" => expected_guessed_columns},
|
101
|
+
Marketo.guess(config)
|
102
|
+
)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_generate_columns
|
107
|
+
assert_equal(expected_guessed_columns, Marketo.generate_columns(metadata))
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
def config
|
113
|
+
DataSource[settings.to_a]
|
114
|
+
end
|
115
|
+
|
116
|
+
def settings
|
117
|
+
{
|
118
|
+
endpoint: "https://marketo.example.com",
|
119
|
+
wsdl: "https://marketo.example.com/?wsdl",
|
120
|
+
user_id: "user_id",
|
121
|
+
encryption_key: "TOPSECRET",
|
122
|
+
last_updated_at: last_updated_at,
|
123
|
+
columns: [
|
124
|
+
{"name" => "Name", "type" => "string"},
|
125
|
+
]
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
def last_updated_at
|
130
|
+
"2015-07-01 00:00:00+00:00"
|
131
|
+
end
|
132
|
+
|
133
|
+
def task
|
134
|
+
{
|
135
|
+
endpoint_url: "https://marketo.example.com",
|
136
|
+
wsdl_url: "https://marketo.example.com/?wsdl",
|
137
|
+
user_id: "user_id",
|
138
|
+
encryption_key: "TOPSECRET",
|
139
|
+
last_updated_at: last_updated_at,
|
140
|
+
columns: [
|
141
|
+
{"name" => "Name", "type" => "string"},
|
142
|
+
]
|
143
|
+
}
|
144
|
+
end
|
145
|
+
|
146
|
+
def metadata
|
147
|
+
[
|
148
|
+
{
|
149
|
+
name: "FieldName",
|
150
|
+
description: nil,
|
151
|
+
display_name: "The Name of Field",
|
152
|
+
source_object: "Lead",
|
153
|
+
data_type: "datetime",
|
154
|
+
size: nil,
|
155
|
+
is_readonly: false,
|
156
|
+
is_update_blocked: false,
|
157
|
+
is_name: nil,
|
158
|
+
is_primary_key: false,
|
159
|
+
is_custom: true,
|
160
|
+
is_dynamic: true,
|
161
|
+
dynamic_field_ref: "leadAttributeList",
|
162
|
+
updated_at: DateTime.parse("2000-01-01 22:22:22")
|
163
|
+
}
|
164
|
+
]
|
165
|
+
end
|
166
|
+
|
167
|
+
def expected_guessed_columns
|
168
|
+
[
|
169
|
+
{name: "id", type: "long"},
|
170
|
+
{name: "email", type: "string"},
|
171
|
+
{name: "FieldName", type: "string"},
|
172
|
+
]
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module LeadFixtures
|
2
|
+
private
|
3
|
+
|
4
|
+
def leads_response
|
5
|
+
Nokogiri::XML(raw_response)
|
6
|
+
end
|
7
|
+
|
8
|
+
def next_stream_leads_response
|
9
|
+
Nokogiri::XML(raw_next_stream_response)
|
10
|
+
end
|
11
|
+
|
12
|
+
def preview_leads_response
|
13
|
+
Nokogiri::XML(raw_preview_response)
|
14
|
+
end
|
15
|
+
|
16
|
+
def leads(body)
|
17
|
+
<<XML
|
18
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
19
|
+
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
|
20
|
+
<SOAP-ENV:Body>
|
21
|
+
<ns1:successGetMultipleLeads>
|
22
|
+
<result>
|
23
|
+
#{body}
|
24
|
+
</result>
|
25
|
+
</ns1:successGetMultipleLeads>
|
26
|
+
</SOAP-ENV:Body>
|
27
|
+
</SOAP-ENV:Envelope>
|
28
|
+
XML
|
29
|
+
end
|
30
|
+
|
31
|
+
def raw_response
|
32
|
+
leads(<<XML)
|
33
|
+
<remainingCount>1</remainingCount>
|
34
|
+
<newStreamPosition>#{stream_position}</newStreamPosition>
|
35
|
+
<leadRecordList>
|
36
|
+
<leadRecord>
|
37
|
+
<Id>65835</Id>
|
38
|
+
<Email>manyo@marbketo.com</Email>
|
39
|
+
<ForeignSysPersonId xsi:nil="true" />
|
40
|
+
<ForeignSysType xsi:nil="true" />
|
41
|
+
<leadAttributeList>
|
42
|
+
<attribute>
|
43
|
+
<attrName>Name</attrName>
|
44
|
+
<attrType>string</attrType>
|
45
|
+
<attrValue>manyo</attrValue>
|
46
|
+
</attribute>
|
47
|
+
</leadAttributeList>
|
48
|
+
</leadRecord>
|
49
|
+
<leadRecord>
|
50
|
+
<Id>67508</Id>
|
51
|
+
<Email>everyleaf@marketo.com</Email>
|
52
|
+
<ForeignSysPersonId xsi:nil="true" />
|
53
|
+
<ForeignSysType xsi:nil="true" />
|
54
|
+
<leadAttributeList>
|
55
|
+
<attribute>
|
56
|
+
<attrName>Name</attrName>
|
57
|
+
<attrType>string</attrType>
|
58
|
+
<attrValue>everyleaf</attrValue>
|
59
|
+
</attribute>
|
60
|
+
</leadAttributeList>
|
61
|
+
</leadRecord>
|
62
|
+
</leadRecordList>
|
63
|
+
XML
|
64
|
+
end
|
65
|
+
|
66
|
+
def stream_position
|
67
|
+
"next_steam_position"
|
68
|
+
end
|
69
|
+
|
70
|
+
def raw_next_stream_response
|
71
|
+
leads(<<XML)
|
72
|
+
<returnCount>2</returnCount>
|
73
|
+
<remainingCount>0</remainingCount>
|
74
|
+
<newStreamPosition />
|
75
|
+
<leadRecordList>
|
76
|
+
<leadRecord>
|
77
|
+
<Id>65835</Id>
|
78
|
+
<Email>ten-thousand-leaf@marketo.com</Email>
|
79
|
+
<ForeignSysPersonId xsi:nil="true" />
|
80
|
+
<ForeignSysType xsi:nil="true" />
|
81
|
+
<leadAttributeList>
|
82
|
+
<attribute>
|
83
|
+
<attrName>Name</attrName>
|
84
|
+
<attrType>string</attrType>
|
85
|
+
<attrValue>ten-thousand-leaf</attrValue>
|
86
|
+
</attribute>
|
87
|
+
</leadAttributeList>
|
88
|
+
</leadRecord>
|
89
|
+
</leadRecordList>
|
90
|
+
XML
|
91
|
+
end
|
92
|
+
|
93
|
+
def raw_preview_response
|
94
|
+
body = ""
|
95
|
+
15.times do |i|
|
96
|
+
body << <<XML
|
97
|
+
<returnCount>2</returnCount>
|
98
|
+
<remainingCount>0</remainingCount>
|
99
|
+
<newStreamPosition />
|
100
|
+
<leadRecordList>
|
101
|
+
<leadRecord>
|
102
|
+
<Id>#{65835 + i}</Id>
|
103
|
+
<Email>manyo#{i}@marketo.com</Email>
|
104
|
+
<ForeignSysPersonId xsi:nil="true" />
|
105
|
+
<ForeignSysType xsi:nil="true" />
|
106
|
+
<leadAttributeList>
|
107
|
+
<attribute>
|
108
|
+
<attrName>Name</attrName>
|
109
|
+
<attrType>string</attrType>
|
110
|
+
<attrValue>manyo#{i}</attrValue>
|
111
|
+
</attribute>
|
112
|
+
</leadAttributeList>
|
113
|
+
</leadRecord>
|
114
|
+
</leadRecordList>
|
115
|
+
XML
|
116
|
+
end
|
117
|
+
leads(body)
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
require "embulk/command/embulk_run"
|
2
|
+
|
3
|
+
classpath_dir = Embulk.home("classpath")
|
4
|
+
jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
|
5
|
+
jars.each do |jar|
|
6
|
+
require File.join(classpath_dir, jar)
|
7
|
+
end
|
8
|
+
require "embulk/java/bootstrap"
|
9
|
+
|
10
|
+
require "embulk"
|
data/test/run-test.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
|
4
|
+
lib_dir = File.join(base_dir, "lib")
|
5
|
+
test_dir = File.join(base_dir, "test")
|
6
|
+
|
7
|
+
require "test-unit"
|
8
|
+
require "test/unit/rr"
|
9
|
+
require "codeclimate-test-reporter"
|
10
|
+
|
11
|
+
$LOAD_PATH.unshift(lib_dir)
|
12
|
+
$LOAD_PATH.unshift(test_dir)
|
13
|
+
|
14
|
+
ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
|
15
|
+
|
16
|
+
CodeClimate::TestReporter.start
|
17
|
+
|
18
|
+
exit Test::Unit::AutoRunner.run(true, test_dir)
|
metadata
ADDED
@@ -0,0 +1,186 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-input-marketo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- uu59
|
8
|
+
- yoshihara
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-07-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 2.11.1
|
20
|
+
name: savon
|
21
|
+
prerelease: false
|
22
|
+
type: :runtime
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ~>
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: 2.11.1
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.6.13
|
34
|
+
- - <
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '1.0'
|
37
|
+
name: embulk
|
38
|
+
prerelease: false
|
39
|
+
type: :development
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: 0.6.13
|
45
|
+
- - <
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.0'
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.0'
|
54
|
+
name: bundler
|
55
|
+
prerelease: false
|
56
|
+
type: :development
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - '>='
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '10.0'
|
68
|
+
name: rake
|
69
|
+
prerelease: false
|
70
|
+
type: :development
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '10.0'
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - '>='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
name: pry
|
83
|
+
prerelease: false
|
84
|
+
type: :development
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - '>='
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
name: test-unit
|
97
|
+
prerelease: false
|
98
|
+
type: :development
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
name: test-unit-rr
|
111
|
+
prerelease: false
|
112
|
+
type: :development
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
- !ruby/object:Gem::Dependency
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
name: codeclimate-test-reporter
|
125
|
+
prerelease: false
|
126
|
+
type: :development
|
127
|
+
version_requirements: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
description: Loads records from Marketo.
|
133
|
+
email:
|
134
|
+
- k@uu59.org
|
135
|
+
- h.yoshihara@everyleaf.com
|
136
|
+
executables: []
|
137
|
+
extensions: []
|
138
|
+
extra_rdoc_files: []
|
139
|
+
files:
|
140
|
+
- .gitignore
|
141
|
+
- .ruby-version
|
142
|
+
- .travis.yml
|
143
|
+
- CHANGELOG.md
|
144
|
+
- Gemfile
|
145
|
+
- LICENSE
|
146
|
+
- README.md
|
147
|
+
- Rakefile
|
148
|
+
- embulk-input-marketo.gemspec
|
149
|
+
- lib/embulk/input/marketo.rb
|
150
|
+
- lib/embulk/input/marketo_api.rb
|
151
|
+
- lib/embulk/input/marketo_api/soap.rb
|
152
|
+
- test/embulk/input/marketo_api/test_soap.rb
|
153
|
+
- test/embulk/input/test_marketo.rb
|
154
|
+
- test/lead_fixtures.rb
|
155
|
+
- test/prepare_embulk.rb
|
156
|
+
- test/run-test.rb
|
157
|
+
homepage: https://github.com/treasure-data/embulk-input-marketo
|
158
|
+
licenses:
|
159
|
+
- Apache2
|
160
|
+
metadata: {}
|
161
|
+
post_install_message:
|
162
|
+
rdoc_options: []
|
163
|
+
require_paths:
|
164
|
+
- lib
|
165
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - '>='
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '0'
|
170
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
171
|
+
requirements:
|
172
|
+
- - '>='
|
173
|
+
- !ruby/object:Gem::Version
|
174
|
+
version: '0'
|
175
|
+
requirements: []
|
176
|
+
rubyforge_project:
|
177
|
+
rubygems_version: 2.4.6
|
178
|
+
signing_key:
|
179
|
+
specification_version: 4
|
180
|
+
summary: Marketo input plugin for Embulk
|
181
|
+
test_files:
|
182
|
+
- test/embulk/input/marketo_api/test_soap.rb
|
183
|
+
- test/embulk/input/test_marketo.rb
|
184
|
+
- test/lead_fixtures.rb
|
185
|
+
- test/prepare_embulk.rb
|
186
|
+
- test/run-test.rb
|