embulk-input-marketo 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +25 -6
- data/Rakefile +53 -1
- data/embulk-input-marketo.gemspec +1 -1
- data/lib/embulk/input/marketo/activity_log.rb +55 -0
- data/lib/embulk/input/marketo/base.rb +89 -0
- data/lib/embulk/input/marketo/lead.rb +73 -0
- data/lib/embulk/input/marketo_api.rb +14 -3
- data/lib/embulk/input/marketo_api/soap/activity_log.rb +90 -0
- data/lib/embulk/input/marketo_api/soap/base.rb +55 -0
- data/lib/embulk/input/marketo_api/soap/lead.rb +75 -0
- data/test/activity_log_fixtures.rb +155 -0
- data/test/embulk/input/marketo/test_activity_log.rb +201 -0
- data/test/embulk/input/marketo/test_base.rb +63 -0
- data/test/embulk/input/marketo/test_lead.rb +172 -0
- data/test/embulk/input/marketo_api/soap/test_activity_log.rb +109 -0
- data/test/embulk/input/marketo_api/soap/test_base.rb +43 -0
- data/test/embulk/input/marketo_api/soap/test_lead.rb +110 -0
- data/test/embulk/input/test_marketo_api.rb +28 -0
- metadata +24 -8
- data/lib/embulk/input/marketo.rb +0 -138
- data/lib/embulk/input/marketo_api/soap.rb +0 -112
- data/test/embulk/input/marketo_api/test_soap.rb +0 -123
- data/test/embulk/input/test_marketo.rb +0 -176
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7de3f0f124846b84a0df2b1d4fd7824c1453b3ff
|
4
|
+
data.tar.gz: 7796e77550eb46c7f4208f373dc9c84b5b8b7e0d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e765f8a460352604ee6b77f49e03f39d25db1a1b4a77d288c55f7e946bd9e88e8bcccd9c680fb2f281eb08e148024af47c86cc8cf3e14f6c1e26ce2e1e7c6c4b
|
7
|
+
data.tar.gz: 56c092c2965c8c36f2866953e2a74e123d22573cb48ab23ffd7dcbc793b6ab167f22f29c0cfc83d81a6b36a9d1256b2d182b950a0f8cee93d076a1591528bb44
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.1.0 - 2015-07-15
|
2
|
+
|
3
|
+
We implemented activity_log plugin for marketo, so config generated from 0.0.1 should be modified. Please check README.md to do it.
|
4
|
+
|
5
|
+
* [enhancement] Implement activity_log plugin [#13](https://github.com/treasure-data/embulk-input-marketo/pull/13) [#14](https://github.com/treasure-data/embulk-input-marketo/pull/14) [#15](https://github.com/treasure-data/embulk-input-marketo/pull/15)
|
6
|
+
|
1
7
|
## 0.0.1 - 2015-07-06
|
2
8
|
|
3
9
|
The first release!!
|
data/README.md
CHANGED
@@ -6,7 +6,11 @@
|
|
6
6
|
|
7
7
|
# Marketo input plugin for Embulk
|
8
8
|
|
9
|
-
embulk-input-marketo is the Embulk input
|
9
|
+
embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
|
10
|
+
|
11
|
+
- Lead
|
12
|
+
- Activity log
|
13
|
+
|
10
14
|
This plugin uses Marketo SOAP API.
|
11
15
|
|
12
16
|
## Overview
|
@@ -18,8 +22,16 @@ Required Embulk version >= 0.6.13.
|
|
18
22
|
* **Cleanup supported**: no
|
19
23
|
* **Guess supported**: yes
|
20
24
|
|
25
|
+
## Install
|
26
|
+
|
27
|
+
```
|
28
|
+
$ embulk gem install embulk-input-marketo
|
29
|
+
```
|
30
|
+
|
21
31
|
## Configuration
|
22
32
|
|
33
|
+
### API
|
34
|
+
|
23
35
|
Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
24
36
|
|
25
37
|
- **endpoint** SOAP endpoint URL for your account (string, required)
|
@@ -28,21 +40,28 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
|
28
40
|
- **encryption_key** Your encryption key (string, reqiured)
|
29
41
|
- **last_updated_at** Limit datetime that a lead has been updated (this plugin fetches leads updated after this datetime) (string, required)
|
30
42
|
|
43
|
+
### Selecting plugin type
|
44
|
+
|
45
|
+
You should specify `type: marketo/lead` or `type: marketo/activity_log` on your demand.
|
46
|
+
|
47
|
+
|
31
48
|
## Example
|
32
49
|
|
50
|
+
For lead, you have `partial-config.yml` like below:
|
51
|
+
|
33
52
|
```yaml
|
34
53
|
in:
|
35
|
-
type: marketo
|
54
|
+
type: marketo/lead
|
36
55
|
endpoint: https://soap-end-point.mktoapi.com/
|
37
56
|
wsdl: https://wsdl-url.mktoapi.com/?WSDL
|
38
57
|
user_id: user_ABC123
|
39
58
|
encryption_key: TOPSECRET
|
40
59
|
last_updated_at: "2015-06-30"
|
60
|
+
out:
|
61
|
+
type: stdout
|
41
62
|
```
|
42
63
|
|
64
|
+
You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
|
43
65
|
|
44
|
-
|
66
|
+
Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
|
45
67
|
|
46
|
-
```
|
47
|
-
$ rake
|
48
|
-
```
|
data/Rakefile
CHANGED
@@ -1,8 +1,60 @@
|
|
1
1
|
require "bundler/gem_tasks"
|
2
|
+
require "json"
|
2
3
|
|
3
4
|
task default: :test
|
4
5
|
|
5
6
|
desc "Run tests"
|
6
7
|
task :test do
|
7
|
-
ruby("test/run-test.rb", "--use-color=yes")
|
8
|
+
ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
|
9
|
+
end
|
10
|
+
|
11
|
+
namespace :release do
|
12
|
+
desc "Add header of now version release to ChangeLog and bump up version"
|
13
|
+
task :prepare do
|
14
|
+
root_dir = Pathname.new(File.expand_path("../", __FILE__))
|
15
|
+
changelog_file = root_dir.join("CHANGELOG.md")
|
16
|
+
gemspec_file = root_dir.join("embulk-input-marketo.gemspec")
|
17
|
+
|
18
|
+
system("git fetch origin")
|
19
|
+
|
20
|
+
# detect merged PR
|
21
|
+
old_version = gemspec_file.read[/spec\.version += *"([0-9]+\.[0-9]+\.[0-9]+)"/, 1]
|
22
|
+
pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
|
23
|
+
|
24
|
+
if !$?.success? || pr_numbers.empty?
|
25
|
+
puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
|
26
|
+
exit(false)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Generate new version
|
30
|
+
major, minor, patch = old_version.split(".").map(&:to_i)
|
31
|
+
new_version = "#{major}.#{minor}.#{patch + 1}"
|
32
|
+
|
33
|
+
# Update ChangeLog
|
34
|
+
pr_descriptions = pr_numbers.map do |number|
|
35
|
+
body = open("https://api.github.com/repos/treasure-data/embulk-input-marketo/issues/#{number.gsub("#", "")}").read
|
36
|
+
payload = JSON.parse(body)
|
37
|
+
"* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-marketo/pull/#{number.gsub('#', '')})"
|
38
|
+
end.join("\n")
|
39
|
+
|
40
|
+
new_changelog = <<-HEADER
|
41
|
+
## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
|
42
|
+
#{pr_descriptions}
|
43
|
+
|
44
|
+
#{changelog_file.read.chomp}
|
45
|
+
HEADER
|
46
|
+
|
47
|
+
File.open(changelog_file, "w") {|f| f.write(new_changelog) }
|
48
|
+
|
49
|
+
# Update version.rb
|
50
|
+
old_content = gemspec_file.read
|
51
|
+
File.open(gemspec_file, "w") do |f|
|
52
|
+
f.write old_content.gsub(/(spec\.version += *)".*?"/, %Q!\\1"#{new_version}"!)
|
53
|
+
end
|
54
|
+
|
55
|
+
# Update Gemfile.lock
|
56
|
+
system("bundle install")
|
57
|
+
|
58
|
+
puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
|
59
|
+
end
|
8
60
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require "embulk/input/marketo/base"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module Marketo
|
6
|
+
class ActivityLog < Base
|
7
|
+
Plugin.register_input("marketo/activity_log", self)
|
8
|
+
|
9
|
+
def self.target
|
10
|
+
:activity_log
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.guess(config)
|
14
|
+
client = soap_client(config)
|
15
|
+
last_updated_at = config.param(:last_updated_at, :string)
|
16
|
+
|
17
|
+
schema = client.metadata(last_updated_at, batch_size: PREVIEW_COUNT)
|
18
|
+
columns = schema.map do |c|
|
19
|
+
column = {name: c.name, type: c.type}
|
20
|
+
column[:format] = c.format if c.format
|
21
|
+
column
|
22
|
+
end
|
23
|
+
|
24
|
+
return {"columns" => columns}
|
25
|
+
end
|
26
|
+
|
27
|
+
def run
|
28
|
+
if preview?
|
29
|
+
batch_size = PREVIEW_COUNT
|
30
|
+
else
|
31
|
+
batch_size = 100
|
32
|
+
end
|
33
|
+
|
34
|
+
count = 0
|
35
|
+
|
36
|
+
@soap.each(@last_updated_at, batch_size: batch_size) do |activity_log|
|
37
|
+
values = @columns.map do |column|
|
38
|
+
name = column["name"].to_s
|
39
|
+
activity_log[name]
|
40
|
+
end
|
41
|
+
|
42
|
+
page_builder.add(values)
|
43
|
+
count += 1
|
44
|
+
break if preview? && count >= PREVIEW_COUNT
|
45
|
+
end
|
46
|
+
|
47
|
+
page_builder.finish
|
48
|
+
|
49
|
+
commit_report = {}
|
50
|
+
return commit_report
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require "embulk/input/marketo_api"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module Marketo
|
6
|
+
class Base < InputPlugin
|
7
|
+
PREVIEW_COUNT = 15
|
8
|
+
|
9
|
+
def self.target
|
10
|
+
raise NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.transaction(config, &control)
|
14
|
+
endpoint_url = config.param(:endpoint, :string)
|
15
|
+
|
16
|
+
task = {
|
17
|
+
endpoint_url: endpoint_url,
|
18
|
+
wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
|
19
|
+
user_id: config.param(:user_id, :string),
|
20
|
+
encryption_key: config.param(:encryption_key, :string),
|
21
|
+
last_updated_at: config.param(:last_updated_at, :string),
|
22
|
+
columns: config.param(:columns, :array)
|
23
|
+
}
|
24
|
+
|
25
|
+
columns = []
|
26
|
+
|
27
|
+
task[:columns].each do |column|
|
28
|
+
name = column["name"]
|
29
|
+
type = column["type"].to_sym
|
30
|
+
|
31
|
+
columns << Column.new(nil, name, type, column["format"])
|
32
|
+
end
|
33
|
+
|
34
|
+
resume(task, columns, 1, &control)
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.resume(task, columns, count, &control)
|
38
|
+
commit_reports = yield(task, columns, count)
|
39
|
+
|
40
|
+
next_config_diff = {}
|
41
|
+
return next_config_diff
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.soap_client(config)
|
45
|
+
@soap ||=
|
46
|
+
begin
|
47
|
+
endpoint_url = config.param(:endpoint, :string),
|
48
|
+
soap_config = {
|
49
|
+
endpoint_url: endpoint_url,
|
50
|
+
wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
|
51
|
+
user_id: config.param(:user_id, :string),
|
52
|
+
encryption_key: config.param(:encryption_key, :string),
|
53
|
+
}
|
54
|
+
|
55
|
+
MarketoApi.soap_client(soap_config, target)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def init
|
60
|
+
@last_updated_at = task[:last_updated_at]
|
61
|
+
@columns = task[:columns]
|
62
|
+
@soap = MarketoApi.soap_client(task, target)
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.logger
|
66
|
+
Embulk.logger
|
67
|
+
end
|
68
|
+
|
69
|
+
def logger
|
70
|
+
self.class.logger
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
def preview?
|
76
|
+
begin
|
77
|
+
org.embulk.spi.Exec.isPreview()
|
78
|
+
rescue java.lang.NullPointerException => e
|
79
|
+
false
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def target
|
84
|
+
self.class.target
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require "embulk/input/marketo/base"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module Marketo
|
6
|
+
class Lead < Base
|
7
|
+
PREVIEW_COUNT = 15
|
8
|
+
|
9
|
+
Plugin.register_input("marketo/lead", self)
|
10
|
+
|
11
|
+
def self.target
|
12
|
+
:lead
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.guess(config)
|
16
|
+
client = soap_client(config)
|
17
|
+
metadata = client.metadata
|
18
|
+
|
19
|
+
return {"columns" => generate_columns(metadata)}
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.generate_columns(metadata)
|
23
|
+
columns = [
|
24
|
+
{name: "id", type: "long"},
|
25
|
+
{name: "email", type: "string"},
|
26
|
+
]
|
27
|
+
|
28
|
+
metadata.each do |field|
|
29
|
+
type =
|
30
|
+
case field[:data_type]
|
31
|
+
when "integer"
|
32
|
+
"long"
|
33
|
+
when "dateTime", "date"
|
34
|
+
"timestamp"
|
35
|
+
when "string", "text", "phone", "currency"
|
36
|
+
"string"
|
37
|
+
when "boolean"
|
38
|
+
"boolean"
|
39
|
+
when "float"
|
40
|
+
"double"
|
41
|
+
else
|
42
|
+
"string"
|
43
|
+
end
|
44
|
+
|
45
|
+
columns << {name: field[:name], type: type}
|
46
|
+
end
|
47
|
+
|
48
|
+
columns
|
49
|
+
end
|
50
|
+
|
51
|
+
def run
|
52
|
+
count = 0
|
53
|
+
@soap.each(@last_updated_at) do |lead|
|
54
|
+
values = @columns.map do |column|
|
55
|
+
name = column["name"].to_s
|
56
|
+
(lead[name] || {})[:value]
|
57
|
+
end
|
58
|
+
|
59
|
+
page_builder.add(values)
|
60
|
+
|
61
|
+
count += 1
|
62
|
+
break if preview? && count >= PREVIEW_COUNT
|
63
|
+
end
|
64
|
+
|
65
|
+
page_builder.finish
|
66
|
+
|
67
|
+
commit_report = {}
|
68
|
+
return commit_report
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -1,10 +1,21 @@
|
|
1
|
-
require "embulk/input/marketo_api/soap"
|
1
|
+
require "embulk/input/marketo_api/soap/base"
|
2
|
+
require "embulk/input/marketo_api/soap/lead"
|
3
|
+
require "embulk/input/marketo_api/soap/activity_log"
|
2
4
|
|
3
5
|
module Embulk
|
4
6
|
module Input
|
5
7
|
module MarketoApi
|
6
|
-
def self.soap_client(config)
|
7
|
-
|
8
|
+
def self.soap_client(config, target)
|
9
|
+
arguments = [config[:endpoint_url], config[:wsdl_url], config[:user_id], config[:encryption_key]]
|
10
|
+
|
11
|
+
case target
|
12
|
+
when :activity_log
|
13
|
+
MarketoApi::Soap::ActivityLog.new(*arguments)
|
14
|
+
when :lead
|
15
|
+
MarketoApi::Soap::Lead.new(*arguments)
|
16
|
+
else
|
17
|
+
raise "unknown target: #{target}"
|
18
|
+
end
|
8
19
|
end
|
9
20
|
end
|
10
21
|
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require "embulk/input/marketo_api/soap/base"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module MarketoApi
|
6
|
+
module Soap
|
7
|
+
class ActivityLog < Base
|
8
|
+
def metadata(last_updated_at, options={})
|
9
|
+
activity_logs = []
|
10
|
+
|
11
|
+
fetch_by_last_updated_at(last_updated_at, options) do |record|
|
12
|
+
activity_logs << record
|
13
|
+
end
|
14
|
+
|
15
|
+
Guess::SchemaGuess.from_hash_records(activity_logs)
|
16
|
+
end
|
17
|
+
|
18
|
+
def each(last_updated_at, options={}, &block)
|
19
|
+
offset = fetch_by_last_updated_at(last_updated_at, options, &block)
|
20
|
+
|
21
|
+
while offset
|
22
|
+
offset = fetch_by_offset(offset, options, &block)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def fetch_by_last_updated_at(last_updated_at, options={}, &block)
|
29
|
+
last_updated_at = last_updated_at.to_s
|
30
|
+
last_updated_at = Time.parse(last_updated_at).iso8601
|
31
|
+
|
32
|
+
request = {
|
33
|
+
start_position: {
|
34
|
+
oldest_created_at: last_updated_at,
|
35
|
+
},
|
36
|
+
}
|
37
|
+
|
38
|
+
fetch(request, options, &block)
|
39
|
+
end
|
40
|
+
|
41
|
+
def fetch_by_offset(offset, options={}, &block)
|
42
|
+
request = {
|
43
|
+
start_position: {
|
44
|
+
offset: offset,
|
45
|
+
},
|
46
|
+
}
|
47
|
+
|
48
|
+
fetch(request, options, &block)
|
49
|
+
end
|
50
|
+
|
51
|
+
def fetch(request, options={}, &block)
|
52
|
+
request[:batch_size] = options[:batch_size] || 100
|
53
|
+
|
54
|
+
response = savon.call(:get_lead_changes, message: request)
|
55
|
+
remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
|
56
|
+
Embulk.logger.info "Remaining records: #{remaining}"
|
57
|
+
|
58
|
+
activities = response.body[:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
|
59
|
+
activities.each do |activity|
|
60
|
+
record = {
|
61
|
+
"id" => activity[:id],
|
62
|
+
# embulk can't treat DateTime
|
63
|
+
"activity_date_time" => activity[:activity_date_time].to_time,
|
64
|
+
"activity_type" => activity[:activity_type],
|
65
|
+
"mktg_asset_name" => activity[:mktg_asset_name],
|
66
|
+
"mkt_person_id" => activity[:mkt_person_id],
|
67
|
+
}
|
68
|
+
|
69
|
+
activity[:activity_attributes][:attribute].each do |attributes|
|
70
|
+
name = attributes[:attr_name]
|
71
|
+
value = attributes[:attr_value]
|
72
|
+
|
73
|
+
record[name] = value
|
74
|
+
end
|
75
|
+
|
76
|
+
block.call(record)
|
77
|
+
end
|
78
|
+
|
79
|
+
if remaining > 0
|
80
|
+
response.body[:success_get_lead_changes][:result][:new_start_position][:offset]
|
81
|
+
else
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|