gooddata_marketo 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +131 -0
- data/README.md +207 -0
- data/bin/Gemfile +10 -0
- data/bin/auth.json +17 -0
- data/bin/main.rb +0 -0
- data/bin/process.rbx +541 -0
- data/examples/all_lead_changes.rb +119 -0
- data/examples/all_leads.rb +249 -0
- data/examples/lead_changes_to_ads.rb +63 -0
- data/gooddata_marketo.gemspec +24 -0
- data/gooddata_marketo_gem.zip +0 -0
- data/lib/gooddata_marketo.rb +24 -0
- data/lib/gooddata_marketo/adapters/rest.rb +287 -0
- data/lib/gooddata_marketo/client.rb +373 -0
- data/lib/gooddata_marketo/data/activity_types.rb +104 -0
- data/lib/gooddata_marketo/data/reserved_sql_keywords.rb +205 -0
- data/lib/gooddata_marketo/helpers/s3.rb +141 -0
- data/lib/gooddata_marketo/helpers/stringwizard.rb +32 -0
- data/lib/gooddata_marketo/helpers/table.rb +323 -0
- data/lib/gooddata_marketo/helpers/webdav.rb +118 -0
- data/lib/gooddata_marketo/loads.rb +235 -0
- data/lib/gooddata_marketo/models/campaigns.rb +57 -0
- data/lib/gooddata_marketo/models/channels.rb +30 -0
- data/lib/gooddata_marketo/models/child/activity.rb +104 -0
- data/lib/gooddata_marketo/models/child/criteria.rb +17 -0
- data/lib/gooddata_marketo/models/child/lead.rb +118 -0
- data/lib/gooddata_marketo/models/child/mobj.rb +68 -0
- data/lib/gooddata_marketo/models/etl.rb +75 -0
- data/lib/gooddata_marketo/models/leads.rb +493 -0
- data/lib/gooddata_marketo/models/load.rb +17 -0
- data/lib/gooddata_marketo/models/mobjects.rb +121 -0
- data/lib/gooddata_marketo/models/streams.rb +137 -0
- data/lib/gooddata_marketo/models/tags.rb +35 -0
- data/lib/gooddata_marketo/models/validate.rb +46 -0
- metadata +177 -0
@@ -0,0 +1,235 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# Jobs manages longer processes which may have failed or been terminated by the GoodData platform.
|
4
|
+
|
5
|
+
require 'gooddata_marketo/models/load'
|
6
|
+
require 'gooddata_marketo/helpers/webdav'
|
7
|
+
require 'digest/sha1'
|
8
|
+
|
9
|
+
class GoodDataMarketo::Loads
|
10
|
+
|
11
|
+
attr_accessor :available
|
12
|
+
|
13
|
+
def initialize config = {}
|
14
|
+
|
15
|
+
@user = config[:user]
|
16
|
+
@password = config[:pass] || config[:password]
|
17
|
+
@project = config[:project] || config[:pid]
|
18
|
+
@client = config[:marketo_client] || config[:marketo]
|
19
|
+
@webdav = WebDAV.new :user => @user, :password => @password, :project => @project
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
def instantiate config = {}
|
24
|
+
config[:webdav] = @webdav
|
25
|
+
config[:client] = @client
|
26
|
+
Load.new config
|
27
|
+
end
|
28
|
+
|
29
|
+
alias :create :instantiate
|
30
|
+
|
31
|
+
def open file
|
32
|
+
Load.new :file => file, :client => @client, :webdav => @webdav
|
33
|
+
end
|
34
|
+
|
35
|
+
def available?
|
36
|
+
loads = Dir.entries('.').select { |f| f.include? '_load.json' }
|
37
|
+
if loads.empty?
|
38
|
+
false
|
39
|
+
else
|
40
|
+
true
|
41
|
+
self.available = loads
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def upload_to_webdav file
|
46
|
+
raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
|
47
|
+
@webdav.upload file
|
48
|
+
end
|
49
|
+
|
50
|
+
def download_from_webdav file
|
51
|
+
raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
|
52
|
+
@webdav.download file
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
class Load
|
59
|
+
|
60
|
+
attr_accessor :client
|
61
|
+
attr_accessor :storage
|
62
|
+
attr_accessor :arguments
|
63
|
+
attr_reader :json
|
64
|
+
attr_reader :id
|
65
|
+
|
66
|
+
def initialize config = {}
|
67
|
+
|
68
|
+
@id = Digest::SHA1.hexdigest(Time.now.to_s)
|
69
|
+
|
70
|
+
def ensure_filetype_json string
|
71
|
+
s = string.to_s
|
72
|
+
if s.include? '.json'
|
73
|
+
s
|
74
|
+
else
|
75
|
+
s+'_load.json'
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
file = config[:file] || config[:name]
|
80
|
+
@file = ensure_filetype_json(file)
|
81
|
+
@webdav = config[:webdav]
|
82
|
+
@client = config[:client]
|
83
|
+
|
84
|
+
# So we don't make the JSON file a mess if we merge with config.
|
85
|
+
config.delete(:webdav)
|
86
|
+
config.delete(:client)
|
87
|
+
|
88
|
+
# Check to see if the file exists locally first.
|
89
|
+
if File.exists? @file
|
90
|
+
|
91
|
+
@json = JSON.parse(IO.read(@file), :symbolize_names => true)
|
92
|
+
|
93
|
+
# If not is the file on WebDAV, if so, download it.
|
94
|
+
elsif self.on_webdav?
|
95
|
+
|
96
|
+
self.refresh
|
97
|
+
|
98
|
+
# Otherwise create a new load file locally.
|
99
|
+
else
|
100
|
+
|
101
|
+
config[:id] = self.id
|
102
|
+
File.open(@file,'w'){ |f| JSON.dump(config, f) }
|
103
|
+
@json = config
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
@saved = true
|
108
|
+
@type = @json[:type] # "leads"
|
109
|
+
@method = @json[:method] # "get_changes"
|
110
|
+
@arguments = @json[:arguments] || @json[:parameters] # ARGS = ":oldest_created_at => 'March 26th 2014', :filters => ['Merge Leads']"
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
def execute config = {}
|
115
|
+
|
116
|
+
# Assign the current load to the client.
|
117
|
+
client = config[:client] || @client
|
118
|
+
client.set_load(self)
|
119
|
+
|
120
|
+
# EXAMPLE LOADS FOR TYPE "LEADS"
|
121
|
+
# get_lead_changes_from_march = {
|
122
|
+
# :name => 'get_lead_changes_from_march',
|
123
|
+
# :type => 'leads',
|
124
|
+
# :method => 'get_changes',
|
125
|
+
# :arguments => {
|
126
|
+
# :oldest_created_at => 'March 26th 2014',
|
127
|
+
# :latest_created_at => 'March 27th 2014',
|
128
|
+
# :filters => ['Merge Leads']
|
129
|
+
# }
|
130
|
+
# }
|
131
|
+
#
|
132
|
+
# get_multiple_from_january = {
|
133
|
+
# :name => 'get_all_leads_january',
|
134
|
+
# :type => 'leads',
|
135
|
+
# :method => 'get_multiple',
|
136
|
+
# :arguments => {
|
137
|
+
# :oldest_created_at => 'January 1st 2014',
|
138
|
+
# :latest_created_at => 'January 2nd 2014',
|
139
|
+
# :filters => ['']
|
140
|
+
# }
|
141
|
+
# }
|
142
|
+
|
143
|
+
case @type
|
144
|
+
|
145
|
+
when "leads"
|
146
|
+
self.log('REQUEST')
|
147
|
+
client.leads.send(@method, @arguments)
|
148
|
+
else
|
149
|
+
raise 'Unable to load JOB type (Ex. "leads").'
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
alias :start :execute
|
154
|
+
alias :run :execute
|
155
|
+
|
156
|
+
# Upload the updated log to WebDAV
|
157
|
+
def log type
|
158
|
+
|
159
|
+
file_name = 'marketo_load_log.txt'
|
160
|
+
log_file = File.open(file_name,'a')
|
161
|
+
log_file.puts("#{self.id} --TYPE #{self.json[:type]} --METHOD #{self.json[:method]} --TIMESTAMP #{Time.now} --STATE #{type}")
|
162
|
+
|
163
|
+
self.upload(file_name)
|
164
|
+
|
165
|
+
end
|
166
|
+
|
167
|
+
# Check if the current object "@file" is on WebDAV
|
168
|
+
def on_webdav? f = false
|
169
|
+
|
170
|
+
if f
|
171
|
+
file = f
|
172
|
+
else
|
173
|
+
file = @file
|
174
|
+
end
|
175
|
+
|
176
|
+
if @webdav.exists? file
|
177
|
+
true
|
178
|
+
else
|
179
|
+
false
|
180
|
+
end
|
181
|
+
|
182
|
+
end
|
183
|
+
|
184
|
+
def get_argument_value key
|
185
|
+
@arguments[key]
|
186
|
+
end
|
187
|
+
|
188
|
+
def set_argument_value key, value
|
189
|
+
@arguments[key] = value
|
190
|
+
end
|
191
|
+
|
192
|
+
alias :get_argument :get_argument_value
|
193
|
+
|
194
|
+
def merge_with_load load
|
195
|
+
@json = load
|
196
|
+
end
|
197
|
+
|
198
|
+
def upload file=nil
|
199
|
+
if file
|
200
|
+
@webdav.upload file
|
201
|
+
else
|
202
|
+
@webdav.upload @file
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def refresh
|
207
|
+
if File.exist? (@file)
|
208
|
+
File.delete(@file)
|
209
|
+
end
|
210
|
+
json = @webdav.download @file
|
211
|
+
File.open(@file,'w'){ |f| JSON.dump(json, f) }
|
212
|
+
end
|
213
|
+
|
214
|
+
def terminate
|
215
|
+
# Delete on local
|
216
|
+
File.delete(@file) if File.exists? @file
|
217
|
+
|
218
|
+
if @webdav.exists? @file
|
219
|
+
# Delete on remote
|
220
|
+
@webdav.delete(@file)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
alias :kill :terminate
|
225
|
+
|
226
|
+
def save
|
227
|
+
|
228
|
+
File.open(@file,'w'){ |f| JSON.dump(@json, f) }
|
229
|
+
@saved = true
|
230
|
+
|
231
|
+
self.upload
|
232
|
+
|
233
|
+
end
|
234
|
+
|
235
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# http://developers.marketo.com/documentation/soap/requestcampaign/
|
3
|
+
|
4
|
+
class GoodDataMarketo::Campaigns
|
5
|
+
|
6
|
+
attr_reader :client
|
7
|
+
|
8
|
+
def initialize config = {}
|
9
|
+
|
10
|
+
@client = config[:client]
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_campaign config = {}
|
15
|
+
|
16
|
+
request = {
|
17
|
+
:source => config[:source] || "MKTOWS",
|
18
|
+
:campaign_id => config[:id],
|
19
|
+
:lead_list => {
|
20
|
+
:lead_key => {
|
21
|
+
:key_type => config[:type] || "EMAIL",
|
22
|
+
:key_value => config[:lead] || config[:email] || config[:id] || config[:value]
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
client.call(:request_campaign, request)
|
28
|
+
end
|
29
|
+
|
30
|
+
alias :request_campaign :get_campaign
|
31
|
+
|
32
|
+
def get_campaigns_for_source config = {} # http://developers.marketo.com/documentation/soap/getcampaignsforsource/
|
33
|
+
|
34
|
+
# Ensure exact_name key is added to request if name key.
|
35
|
+
if config.has_key? :name
|
36
|
+
config[:exact_name] = "false" unless config.has_key? :exact_name
|
37
|
+
end
|
38
|
+
|
39
|
+
default = {
|
40
|
+
:source => "MKTOWS"
|
41
|
+
#:name => "Trigger", <-- Optional
|
42
|
+
#:exact_name => "false" <-- Optional
|
43
|
+
}
|
44
|
+
|
45
|
+
request = default.merge(config)
|
46
|
+
|
47
|
+
client.call(:get_campaigns_for_source, request)
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class GoodDataMarketo::Client
|
2
|
+
|
3
|
+
def channels config = {} # http://developers.marketo.com/documentation/soap/getchannels/
|
4
|
+
|
5
|
+
values = config[:values] || config[:channels]
|
6
|
+
if values.is_a? String
|
7
|
+
values = [values]
|
8
|
+
end
|
9
|
+
|
10
|
+
request = {
|
11
|
+
:tag => {
|
12
|
+
:values => {
|
13
|
+
:string_item => []
|
14
|
+
}
|
15
|
+
}
|
16
|
+
}
|
17
|
+
|
18
|
+
if values
|
19
|
+
request[:tag][:values][:string_item] = values
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
self.call(:get_campaigns_for_source, request)
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
class GoodDataMarketo::Activity
|
4
|
+
|
5
|
+
#attr_accessor :client
|
6
|
+
|
7
|
+
def initialize data, config = {}
|
8
|
+
|
9
|
+
data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
|
10
|
+
|
11
|
+
@activity = {
|
12
|
+
:id => data[:id],
|
13
|
+
:activity_date_time => data[:activity_date_time].to_s, # Force this to be ADS DATETIME
|
14
|
+
:activity_type => data[:activity_type],
|
15
|
+
:mktg_asset_name => data[:mktg_asset_name],
|
16
|
+
:raw => data
|
17
|
+
}
|
18
|
+
|
19
|
+
@headers = @activity.keys.map{|k| k.to_s.capitalize! }
|
20
|
+
@headers.pop()
|
21
|
+
|
22
|
+
attributes = data[:activity_attributes][:attribute]
|
23
|
+
attribute_map = Hash.new
|
24
|
+
attributes.map { |attr|
|
25
|
+
@headers << property = attr[:attr_name].gsub(" ","_").downcase
|
26
|
+
value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
|
27
|
+
attribute_map[property] = value
|
28
|
+
}
|
29
|
+
|
30
|
+
@attributes = attribute_map
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
def values
|
35
|
+
hash = Hash.new
|
36
|
+
hash['id'] = self.id
|
37
|
+
hash['activity_date_time'] = self.time
|
38
|
+
hash['activity_type'] = self.type
|
39
|
+
hash['mktg_asset_name'] = self.name
|
40
|
+
@attributes.each do |attr|
|
41
|
+
hash[attr[0].scan(/\w+/).join] = attr[1]
|
42
|
+
end
|
43
|
+
hash
|
44
|
+
end
|
45
|
+
|
46
|
+
def time
|
47
|
+
@activity[:activity_date_time]
|
48
|
+
end
|
49
|
+
|
50
|
+
def type
|
51
|
+
@activity[:activity_type]
|
52
|
+
end
|
53
|
+
|
54
|
+
def to_row
|
55
|
+
row = [self.id,self.time,self.type,self.name]
|
56
|
+
@attributes.each do |attr|
|
57
|
+
row << attr[1]
|
58
|
+
end
|
59
|
+
row.map! { |i| i.to_s }
|
60
|
+
end
|
61
|
+
|
62
|
+
alias :to_a :to_row
|
63
|
+
|
64
|
+
def headers
|
65
|
+
@headers.map { |h| h.scan(/\w+/).join.downcase }
|
66
|
+
end
|
67
|
+
|
68
|
+
alias :columns :headers
|
69
|
+
|
70
|
+
def date
|
71
|
+
@activity[:date]
|
72
|
+
end
|
73
|
+
|
74
|
+
def id
|
75
|
+
@activity[:id]
|
76
|
+
end
|
77
|
+
|
78
|
+
def name
|
79
|
+
@activity[:mktg_asset_name]
|
80
|
+
end
|
81
|
+
|
82
|
+
def attributes a = nil
|
83
|
+
|
84
|
+
if a
|
85
|
+
@attributes[a]
|
86
|
+
else
|
87
|
+
@attributes
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
def raw
|
93
|
+
@activity[:raw]
|
94
|
+
end
|
95
|
+
|
96
|
+
alias :to_a :to_row
|
97
|
+
|
98
|
+
alias :json :raw
|
99
|
+
|
100
|
+
alias :activity_date_time :time
|
101
|
+
|
102
|
+
alias :activity_type :type
|
103
|
+
|
104
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
class GoodDataMarketo::Criteria
|
3
|
+
def initialize name, value, comparison = nil
|
4
|
+
@name = name
|
5
|
+
@value = value
|
6
|
+
@comparison = comparison
|
7
|
+
end
|
8
|
+
def data
|
9
|
+
obj = {}
|
10
|
+
o[:m_obj_criteria] = {
|
11
|
+
:attr_name => @name,
|
12
|
+
:comparison => @comparison,
|
13
|
+
:attr_value => @value
|
14
|
+
}
|
15
|
+
obj
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
class GoodDataMarketo::Lead
|
4
|
+
|
5
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
6
|
+
# attr_accessor :client
|
7
|
+
|
8
|
+
def initialize data, config = {}
|
9
|
+
|
10
|
+
data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
|
11
|
+
|
12
|
+
@lead = {
|
13
|
+
:id => data[:id],
|
14
|
+
:email => data[:email],
|
15
|
+
:foreign_sys_person_id => data[:foreign_sys_person_id],
|
16
|
+
:foreign_sys_type => data[:foreign_sys_type],
|
17
|
+
:raw => data
|
18
|
+
}
|
19
|
+
|
20
|
+
@headers = @lead.keys.map {|k| k.to_s.capitalize! }
|
21
|
+
@headers.pop()
|
22
|
+
|
23
|
+
attributes = data[:lead_attribute_list][:attribute]
|
24
|
+
attribute_map = Hash.new
|
25
|
+
attributes.map { |attr|
|
26
|
+
@headers << property = attr[:attr_name].gsub(" ","_").downcase!
|
27
|
+
value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
|
28
|
+
attribute_map[property] = value
|
29
|
+
}
|
30
|
+
|
31
|
+
@attributes = attribute_map
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_row
|
36
|
+
row = [self.id,self.email,self.foreign_sys_person_id,self.foreign_sys_type]
|
37
|
+
@attributes.each do |attr|
|
38
|
+
row << attr[1]
|
39
|
+
end
|
40
|
+
row.map! { |i| i.to_s }
|
41
|
+
end
|
42
|
+
|
43
|
+
def headers
|
44
|
+
@headers.map { |h| h.downcase }
|
45
|
+
end
|
46
|
+
|
47
|
+
alias :columns :headers
|
48
|
+
|
49
|
+
alias :to_a :to_row
|
50
|
+
|
51
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
52
|
+
# def get_activities config = {}
|
53
|
+
# config[:type] = 'IDNUM'
|
54
|
+
# config[:value] = self.id
|
55
|
+
#
|
56
|
+
# client.leads.get_activities config
|
57
|
+
#
|
58
|
+
# end
|
59
|
+
|
60
|
+
#alias :get_activity :get_activities
|
61
|
+
#alias :activities :get_activities
|
62
|
+
|
63
|
+
# CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
|
64
|
+
# def get_changes config = {}
|
65
|
+
# config[:type] = 'IDNUM'
|
66
|
+
# config[:value] = self.id
|
67
|
+
#
|
68
|
+
# client.leads.get_changes config
|
69
|
+
#
|
70
|
+
# end
|
71
|
+
|
72
|
+
#alias :changes :get_changes
|
73
|
+
|
74
|
+
def values
|
75
|
+
hash = Hash.new
|
76
|
+
hash['id'] = self.id
|
77
|
+
hash['email'] = self.email
|
78
|
+
hash['foreign_sys_person_id'] = self.foreign_sys_person_id
|
79
|
+
hash['foreign_sys_type'] = self.foreign_sys_type
|
80
|
+
@attributes.each do |attr|
|
81
|
+
hash[attr[0]] = attr[1]
|
82
|
+
end
|
83
|
+
hash
|
84
|
+
end
|
85
|
+
|
86
|
+
def id
|
87
|
+
@lead[:id]
|
88
|
+
end
|
89
|
+
|
90
|
+
def email
|
91
|
+
@lead[:email]
|
92
|
+
end
|
93
|
+
|
94
|
+
def foreign_sys_person_id
|
95
|
+
@lead[:foreign_sys_person_id]
|
96
|
+
end
|
97
|
+
|
98
|
+
def foreign_sys_type
|
99
|
+
@lead[:foreign_sys_type]
|
100
|
+
end
|
101
|
+
|
102
|
+
def attributes a = nil
|
103
|
+
|
104
|
+
if a
|
105
|
+
@attributes[a]
|
106
|
+
else
|
107
|
+
@attributes
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
111
|
+
|
112
|
+
def raw
|
113
|
+
@lead[:raw]
|
114
|
+
end
|
115
|
+
|
116
|
+
alias :json :raw
|
117
|
+
|
118
|
+
end
|