gooddata_marketo 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +131 -0
- data/README.md +207 -0
- data/bin/Gemfile +10 -0
- data/bin/auth.json +17 -0
- data/bin/main.rb +0 -0
- data/bin/process.rbx +541 -0
- data/examples/all_lead_changes.rb +119 -0
- data/examples/all_leads.rb +249 -0
- data/examples/lead_changes_to_ads.rb +63 -0
- data/gooddata_marketo.gemspec +24 -0
- data/gooddata_marketo_gem.zip +0 -0
- data/lib/gooddata_marketo.rb +24 -0
- data/lib/gooddata_marketo/adapters/rest.rb +287 -0
- data/lib/gooddata_marketo/client.rb +373 -0
- data/lib/gooddata_marketo/data/activity_types.rb +104 -0
- data/lib/gooddata_marketo/data/reserved_sql_keywords.rb +205 -0
- data/lib/gooddata_marketo/helpers/s3.rb +141 -0
- data/lib/gooddata_marketo/helpers/stringwizard.rb +32 -0
- data/lib/gooddata_marketo/helpers/table.rb +323 -0
- data/lib/gooddata_marketo/helpers/webdav.rb +118 -0
- data/lib/gooddata_marketo/loads.rb +235 -0
- data/lib/gooddata_marketo/models/campaigns.rb +57 -0
- data/lib/gooddata_marketo/models/channels.rb +30 -0
- data/lib/gooddata_marketo/models/child/activity.rb +104 -0
- data/lib/gooddata_marketo/models/child/criteria.rb +17 -0
- data/lib/gooddata_marketo/models/child/lead.rb +118 -0
- data/lib/gooddata_marketo/models/child/mobj.rb +68 -0
- data/lib/gooddata_marketo/models/etl.rb +75 -0
- data/lib/gooddata_marketo/models/leads.rb +493 -0
- data/lib/gooddata_marketo/models/load.rb +17 -0
- data/lib/gooddata_marketo/models/mobjects.rb +121 -0
- data/lib/gooddata_marketo/models/streams.rb +137 -0
- data/lib/gooddata_marketo/models/tags.rb +35 -0
- data/lib/gooddata_marketo/models/validate.rb +46 -0
- metadata +177 -0
@@ -0,0 +1,235 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# Jobs manages longer processes which may have failed or been terminated by the GoodData platform.
|
4
|
+
|
5
|
+
require 'gooddata_marketo/models/load'
|
6
|
+
require 'gooddata_marketo/helpers/webdav'
|
7
|
+
require 'digest/sha1'
|
8
|
+
|
9
|
+
class GoodDataMarketo::Loads
|
10
|
+
|
11
|
+
attr_accessor :available
|
12
|
+
|
13
|
+
def initialize config = {}
|
14
|
+
|
15
|
+
@user = config[:user]
|
16
|
+
@password = config[:pass] || config[:password]
|
17
|
+
@project = config[:project] || config[:pid]
|
18
|
+
@client = config[:marketo_client] || config[:marketo]
|
19
|
+
@webdav = WebDAV.new :user => @user, :password => @password, :project => @project
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
def instantiate config = {}
|
24
|
+
config[:webdav] = @webdav
|
25
|
+
config[:client] = @client
|
26
|
+
Load.new config
|
27
|
+
end
|
28
|
+
|
29
|
+
alias :create :instantiate
|
30
|
+
|
31
|
+
def open file
|
32
|
+
Load.new :file => file, :client => @client, :webdav => @webdav
|
33
|
+
end
|
34
|
+
|
35
|
+
def available?
|
36
|
+
loads = Dir.entries('.').select { |f| f.include? '_load.json' }
|
37
|
+
if loads.empty?
|
38
|
+
false
|
39
|
+
else
|
40
|
+
true
|
41
|
+
self.available = loads
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def upload_to_webdav file
|
46
|
+
raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
|
47
|
+
@webdav.upload file
|
48
|
+
end
|
49
|
+
|
50
|
+
def download_from_webdav file
|
51
|
+
raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
|
52
|
+
@webdav.download file
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
class Load
|
59
|
+
|
60
|
+
attr_accessor :client
|
61
|
+
attr_accessor :storage
|
62
|
+
attr_accessor :arguments
|
63
|
+
attr_reader :json
|
64
|
+
attr_reader :id
|
65
|
+
|
66
|
+
def initialize config = {}
|
67
|
+
|
68
|
+
@id = Digest::SHA1.hexdigest(Time.now.to_s)
|
69
|
+
|
70
|
+
def ensure_filetype_json string
|
71
|
+
s = string.to_s
|
72
|
+
if s.include? '.json'
|
73
|
+
s
|
74
|
+
else
|
75
|
+
s+'_load.json'
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
file = config[:file] || config[:name]
|
80
|
+
@file = ensure_filetype_json(file)
|
81
|
+
@webdav = config[:webdav]
|
82
|
+
@client = config[:client]
|
83
|
+
|
84
|
+
# So we don't make the JSON file a mess if we merge with config.
|
85
|
+
config.delete(:webdav)
|
86
|
+
config.delete(:client)
|
87
|
+
|
88
|
+
# Check to see if the file exists locally first.
|
89
|
+
if File.exists? @file
|
90
|
+
|
91
|
+
@json = JSON.parse(IO.read(@file), :symbolize_names => true)
|
92
|
+
|
93
|
+
# If not is the file on WebDAV, if so, download it.
|
94
|
+
elsif self.on_webdav?
|
95
|
+
|
96
|
+
self.refresh
|
97
|
+
|
98
|
+
# Otherwise create a new load file locally.
|
99
|
+
else
|
100
|
+
|
101
|
+
config[:id] = self.id
|
102
|
+
File.open(@file,'w'){ |f| JSON.dump(config, f) }
|
103
|
+
@json = config
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
@saved = true
|
108
|
+
@type = @json[:type] # "leads"
|
109
|
+
@method = @json[:method] # "get_changes"
|
110
|
+
@arguments = @json[:arguments] || @json[:parameters] # ARGS = ":oldest_created_at => 'March 26th 2014', :filters => ['Merge Leads']"
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
def execute config = {}
|
115
|
+
|
116
|
+
# Assign the current load to the client.
|
117
|
+
client = config[:client] || @client
|
118
|
+
client.set_load(self)
|
119
|
+
|
120
|
+
# EXAMPLE LOADS FOR TYPE "LEADS"
|
121
|
+
# get_lead_changes_from_march = {
|
122
|
+
# :name => 'get_lead_changes_from_march',
|
123
|
+
# :type => 'leads',
|
124
|
+
# :method => 'get_changes',
|
125
|
+
# :arguments => {
|
126
|
+
# :oldest_created_at => 'March 26th 2014',
|
127
|
+
# :latest_created_at => 'March 27th 2014',
|
128
|
+
# :filters => ['Merge Leads']
|
129
|
+
# }
|
130
|
+
# }
|
131
|
+
#
|
132
|
+
# get_multiple_from_january = {
|
133
|
+
# :name => 'get_all_leads_january',
|
134
|
+
# :type => 'leads',
|
135
|
+
# :method => 'get_multiple',
|
136
|
+
# :arguments => {
|
137
|
+
# :oldest_created_at => 'January 1st 2014',
|
138
|
+
# :latest_created_at => 'January 2nd 2014',
|
139
|
+
# :filters => ['']
|
140
|
+
# }
|
141
|
+
# }
|
142
|
+
|
143
|
+
case @type
|
144
|
+
|
145
|
+
when "leads"
|
146
|
+
self.log('REQUEST')
|
147
|
+
client.leads.send(@method, @arguments)
|
148
|
+
else
|
149
|
+
raise 'Unable to load JOB type (Ex. "leads").'
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
alias :start :execute
|
154
|
+
alias :run :execute
|
155
|
+
|
156
|
+
# Upload the updated log to WebDAV
|
157
|
+
def log type
|
158
|
+
|
159
|
+
file_name = 'marketo_load_log.txt'
|
160
|
+
log_file = File.open(file_name,'a')
|
161
|
+
log_file.puts("#{self.id} --TYPE #{self.json[:type]} --METHOD #{self.json[:method]} --TIMESTAMP #{Time.now} --STATE #{type}")
|
162
|
+
|
163
|
+
self.upload(file_name)
|
164
|
+
|
165
|
+
end
|
166
|
+
|
167
|
+
# Check if the current object "@file" is on WebDAV
|
168
|
+
def on_webdav? f = false
|
169
|
+
|
170
|
+
if f
|
171
|
+
file = f
|
172
|
+
else
|
173
|
+
file = @file
|
174
|
+
end
|
175
|
+
|
176
|
+
if @webdav.exists? file
|
177
|
+
true
|
178
|
+
else
|
179
|
+
false
|
180
|
+
end
|
181
|
+
|
182
|
+
end
|
183
|
+
|
184
|
+
def get_argument_value key
|
185
|
+
@arguments[key]
|
186
|
+
end
|
187
|
+
|
188
|
+
def set_argument_value key, value
|
189
|
+
@arguments[key] = value
|
190
|
+
end
|
191
|
+
|
192
|
+
alias :get_argument :get_argument_value
|
193
|
+
|
194
|
+
def merge_with_load load
|
195
|
+
@json = load
|
196
|
+
end
|
197
|
+
|
198
|
+
def upload file=nil
|
199
|
+
if file
|
200
|
+
@webdav.upload file
|
201
|
+
else
|
202
|
+
@webdav.upload @file
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def refresh
|
207
|
+
if File.exist? (@file)
|
208
|
+
File.delete(@file)
|
209
|
+
end
|
210
|
+
json = @webdav.download @file
|
211
|
+
File.open(@file,'w'){ |f| JSON.dump(json, f) }
|
212
|
+
end
|
213
|
+
|
214
|
+
def terminate
|
215
|
+
# Delete on local
|
216
|
+
File.delete(@file) if File.exists? @file
|
217
|
+
|
218
|
+
if @webdav.exists? @file
|
219
|
+
# Delete on remote
|
220
|
+
@webdav.delete(@file)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
alias :kill :terminate
|
225
|
+
|
226
|
+
def save
|
227
|
+
|
228
|
+
File.open(@file,'w'){ |f| JSON.dump(@json, f) }
|
229
|
+
@saved = true
|
230
|
+
|
231
|
+
self.upload
|
232
|
+
|
233
|
+
end
|
234
|
+
|
235
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# http://developers.marketo.com/documentation/soap/requestcampaign/
|
3
|
+
|
4
|
+
class GoodDataMarketo::Campaigns
|
5
|
+
|
6
|
+
attr_reader :client
|
7
|
+
|
8
|
+
def initialize config = {}
|
9
|
+
|
10
|
+
@client = config[:client]
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_campaign config = {}
|
15
|
+
|
16
|
+
request = {
|
17
|
+
:source => config[:source] || "MKTOWS",
|
18
|
+
:campaign_id => config[:id],
|
19
|
+
:lead_list => {
|
20
|
+
:lead_key => {
|
21
|
+
:key_type => config[:type] || "EMAIL",
|
22
|
+
:key_value => config[:lead] || config[:email] || config[:id] || config[:value]
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
client.call(:request_campaign, request)
|
28
|
+
end
|
29
|
+
|
30
|
+
alias :request_campaign :get_campaign
|
31
|
+
|
32
|
+
def get_campaigns_for_source config = {} # http://developers.marketo.com/documentation/soap/getcampaignsforsource/
|
33
|
+
|
34
|
+
# Ensure exact_name key is added to request if name key.
|
35
|
+
if config.has_key? :name
|
36
|
+
config[:exact_name] = "false" unless config.has_key? :exact_name
|
37
|
+
end
|
38
|
+
|
39
|
+
default = {
|
40
|
+
:source => "MKTOWS"
|
41
|
+
#:name => "Trigger", <-- Optional
|
42
|
+
#:exact_name => "false" <-- Optional
|
43
|
+
}
|
44
|
+
|
45
|
+
request = default.merge(config)
|
46
|
+
|
47
|
+
client.call(:get_campaigns_for_source, request)
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class GoodDataMarketo::Client
|
2
|
+
|
3
|
+
def channels config = {} # http://developers.marketo.com/documentation/soap/getchannels/
|
4
|
+
|
5
|
+
values = config[:values] || config[:channels]
|
6
|
+
if values.is_a? String
|
7
|
+
values = [values]
|
8
|
+
end
|
9
|
+
|
10
|
+
request = {
|
11
|
+
:tag => {
|
12
|
+
:values => {
|
13
|
+
:string_item => []
|
14
|
+
}
|
15
|
+
}
|
16
|
+
}
|
17
|
+
|
18
|
+
if values
|
19
|
+
request[:tag][:values][:string_item] = values
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
self.call(:get_campaigns_for_source, request)
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
class GoodDataMarketo::Activity
|
4
|
+
|
5
|
+
#attr_accessor :client
|
6
|
+
|
7
|
+
def initialize data, config = {}
|
8
|
+
|
9
|
+
data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
|
10
|
+
|
11
|
+
@activity = {
|
12
|
+
:id => data[:id],
|
13
|
+
:activity_date_time => data[:activity_date_time].to_s, # Force this to be ADS DATETIME
|
14
|
+
:activity_type => data[:activity_type],
|
15
|
+
:mktg_asset_name => data[:mktg_asset_name],
|
16
|
+
:raw => data
|
17
|
+
}
|
18
|
+
|
19
|
+
@headers = @activity.keys.map{|k| k.to_s.capitalize! }
|
20
|
+
@headers.pop()
|
21
|
+
|
22
|
+
attributes = data[:activity_attributes][:attribute]
|
23
|
+
attribute_map = Hash.new
|
24
|
+
attributes.map { |attr|
|
25
|
+
@headers << property = attr[:attr_name].gsub(" ","_").downcase
|
26
|
+
value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
|
27
|
+
attribute_map[property] = value
|
28
|
+
}
|
29
|
+
|
30
|
+
@attributes = attribute_map
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
def values
|
35
|
+
hash = Hash.new
|
36
|
+
hash['id'] = self.id
|
37
|
+
hash['activity_date_time'] = self.time
|
38
|
+
hash['activity_type'] = self.type
|
39
|
+
hash['mktg_asset_name'] = self.name
|
40
|
+
@attributes.each do |attr|
|
41
|
+
hash[attr[0].scan(/\w+/).join] = attr[1]
|
42
|
+
end
|
43
|
+
hash
|
44
|
+
end
|
45
|
+
|
46
|
+
def time
|
47
|
+
@activity[:activity_date_time]
|
48
|
+
end
|
49
|
+
|
50
|
+
def type
|
51
|
+
@activity[:activity_type]
|
52
|
+
end
|
53
|
+
|
54
|
+
def to_row
|
55
|
+
row = [self.id,self.time,self.type,self.name]
|
56
|
+
@attributes.each do |attr|
|
57
|
+
row << attr[1]
|
58
|
+
end
|
59
|
+
row.map! { |i| i.to_s }
|
60
|
+
end
|
61
|
+
|
62
|
+
alias :to_a :to_row
|
63
|
+
|
64
|
+
def headers
|
65
|
+
@headers.map { |h| h.scan(/\w+/).join.downcase }
|
66
|
+
end
|
67
|
+
|
68
|
+
alias :columns :headers
|
69
|
+
|
70
|
+
def date
|
71
|
+
@activity[:date]
|
72
|
+
end
|
73
|
+
|
74
|
+
def id
|
75
|
+
@activity[:id]
|
76
|
+
end
|
77
|
+
|
78
|
+
def name
|
79
|
+
@activity[:mktg_asset_name]
|
80
|
+
end
|
81
|
+
|
82
|
+
def attributes a = nil
|
83
|
+
|
84
|
+
if a
|
85
|
+
@attributes[a]
|
86
|
+
else
|
87
|
+
@attributes
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
def raw
|
93
|
+
@activity[:raw]
|
94
|
+
end
|
95
|
+
|
96
|
+
alias :to_a :to_row
|
97
|
+
|
98
|
+
alias :json :raw
|
99
|
+
|
100
|
+
alias :activity_date_time :time
|
101
|
+
|
102
|
+
alias :activity_type :type
|
103
|
+
|
104
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
class GoodDataMarketo::Criteria
|
3
|
+
def initialize name, value, comparison = nil
|
4
|
+
@name = name
|
5
|
+
@value = value
|
6
|
+
@comparison = comparison
|
7
|
+
end
|
8
|
+
def data
|
9
|
+
obj = {}
|
10
|
+
o[:m_obj_criteria] = {
|
11
|
+
:attr_name => @name,
|
12
|
+
:comparison => @comparison,
|
13
|
+
:attr_value => @value
|
14
|
+
}
|
15
|
+
obj
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
class GoodDataMarketo::Lead
|
4
|
+
|
5
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
6
|
+
# attr_accessor :client
|
7
|
+
|
8
|
+
def initialize data, config = {}
|
9
|
+
|
10
|
+
data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
|
11
|
+
|
12
|
+
@lead = {
|
13
|
+
:id => data[:id],
|
14
|
+
:email => data[:email],
|
15
|
+
:foreign_sys_person_id => data[:foreign_sys_person_id],
|
16
|
+
:foreign_sys_type => data[:foreign_sys_type],
|
17
|
+
:raw => data
|
18
|
+
}
|
19
|
+
|
20
|
+
@headers = @lead.keys.map {|k| k.to_s.capitalize! }
|
21
|
+
@headers.pop()
|
22
|
+
|
23
|
+
attributes = data[:lead_attribute_list][:attribute]
|
24
|
+
attribute_map = Hash.new
|
25
|
+
attributes.map { |attr|
|
26
|
+
@headers << property = attr[:attr_name].gsub(" ","_").downcase!
|
27
|
+
value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
|
28
|
+
attribute_map[property] = value
|
29
|
+
}
|
30
|
+
|
31
|
+
@attributes = attribute_map
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_row
|
36
|
+
row = [self.id,self.email,self.foreign_sys_person_id,self.foreign_sys_type]
|
37
|
+
@attributes.each do |attr|
|
38
|
+
row << attr[1]
|
39
|
+
end
|
40
|
+
row.map! { |i| i.to_s }
|
41
|
+
end
|
42
|
+
|
43
|
+
def headers
|
44
|
+
@headers.map { |h| h.downcase }
|
45
|
+
end
|
46
|
+
|
47
|
+
alias :columns :headers
|
48
|
+
|
49
|
+
alias :to_a :to_row
|
50
|
+
|
51
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
52
|
+
# def get_activities config = {}
|
53
|
+
# config[:type] = 'IDNUM'
|
54
|
+
# config[:value] = self.id
|
55
|
+
#
|
56
|
+
# client.leads.get_activities config
|
57
|
+
#
|
58
|
+
# end
|
59
|
+
|
60
|
+
#alias :get_activity :get_activities
|
61
|
+
#alias :activities :get_activities
|
62
|
+
|
63
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
64
|
+
# def get_changes config = {}
|
65
|
+
# config[:type] = 'IDNUM'
|
66
|
+
# config[:value] = self.id
|
67
|
+
#
|
68
|
+
# client.leads.get_changes config
|
69
|
+
#
|
70
|
+
# end
|
71
|
+
|
72
|
+
#alias :changes :get_changes
|
73
|
+
|
74
|
+
def values
|
75
|
+
hash = Hash.new
|
76
|
+
hash['id'] = self.id
|
77
|
+
hash['email'] = self.email
|
78
|
+
hash['foreign_sys_person_id'] = self.foreign_sys_person_id
|
79
|
+
hash['foreign_sys_type'] = self.foreign_sys_type
|
80
|
+
@attributes.each do |attr|
|
81
|
+
hash[attr[0]] = attr[1]
|
82
|
+
end
|
83
|
+
hash
|
84
|
+
end
|
85
|
+
|
86
|
+
def id
|
87
|
+
@lead[:id]
|
88
|
+
end
|
89
|
+
|
90
|
+
def email
|
91
|
+
@lead[:email]
|
92
|
+
end
|
93
|
+
|
94
|
+
def foreign_sys_person_id
|
95
|
+
@lead[:foreign_sys_person_id]
|
96
|
+
end
|
97
|
+
|
98
|
+
def foreign_sys_type
|
99
|
+
@lead[:foreign_sys_type]
|
100
|
+
end
|
101
|
+
|
102
|
+
def attributes a = nil
|
103
|
+
|
104
|
+
if a
|
105
|
+
@attributes[a]
|
106
|
+
else
|
107
|
+
@attributes
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
111
|
+
|
112
|
+
def raw
|
113
|
+
@lead[:raw]
|
114
|
+
end
|
115
|
+
|
116
|
+
alias :json :raw
|
117
|
+
|
118
|
+
end
|